@dudousxd/nestjs-durable-core 0.85.11 → 0.86.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.cjs +192 -33
- package/dist/index.cjs.map +1 -1
- package/dist/index.d.cts +64 -3
- package/dist/index.d.ts +64 -3
- package/dist/index.js +191 -33
- package/dist/index.js.map +1 -1
- package/package.json +1 -1
package/dist/index.d.cts
CHANGED
|
@@ -902,7 +902,8 @@ type RunStatus =
|
|
|
902
902
|
* are actually published (a legacy/pre-handshake fleet skips the guard, design §7.7).
|
|
903
903
|
*/
|
|
904
904
|
| 'blocked' | 'completed' | 'failed' | 'cancelled'
|
|
905
|
-
/** Dead-letter: recovery gave up after `maxRecoveryAttempts` (a poison pill)
|
|
905
|
+
/** Dead-letter: recovery gave up after `maxRecoveryAttempts` (a poison pill), or no instance has
|
|
906
|
+
* the run's workflow version registered past `unregisteredWorkflowGraceMs`. Terminal; inspect it. */
|
|
906
907
|
| 'dead';
|
|
907
908
|
/** One execution of a workflow. The unit of durability and the unit shown in the dashboard. */
|
|
908
909
|
interface WorkflowRun {
|
|
@@ -2628,6 +2629,9 @@ type ControlMessage = {
|
|
|
2628
2629
|
} & ({
|
|
2629
2630
|
kind: 'event';
|
|
2630
2631
|
event: EngineEvent;
|
|
2632
|
+
} | {
|
|
2633
|
+
kind: 'events';
|
|
2634
|
+
events: EngineEvent[];
|
|
2631
2635
|
} | {
|
|
2632
2636
|
kind: 'cancel';
|
|
2633
2637
|
runId: string;
|
|
@@ -4475,6 +4479,17 @@ interface WorkflowEngineDeps {
|
|
|
4475
4479
|
* Omit for unlimited (the default — recovery always retries).
|
|
4476
4480
|
*/
|
|
4477
4481
|
maxRecoveryAttempts?: number | undefined;
|
|
4482
|
+
/**
|
|
4483
|
+
* How long (ms) a run may stay due while NO instance can resume it because its workflow version is
|
|
4484
|
+
* not registered anywhere, before it is moved to `dead` (code `workflow_not_registered`). A rolling
|
|
4485
|
+
* deploy skips such a run on purpose (skew protection): the pod still on the prior version picks it
|
|
4486
|
+
* up within seconds. A run that stays unresumable far past that — its workflow was renamed or
|
|
4487
|
+
* deleted — would otherwise be retried and logged on every tick forever. The clock starts when the
|
|
4488
|
+
* run became due (its `wakeAt` for a parked run, its last update otherwise), so a long sleep is not
|
|
4489
|
+
* counted. Default 24h; `0` disables (skip forever). A non-finite or negative value falls back to
|
|
4490
|
+
* the default.
|
|
4491
|
+
*/
|
|
4492
|
+
unregisteredWorkflowGraceMs?: number | undefined;
|
|
4478
4493
|
/**
|
|
4479
4494
|
* Opt-in liveness deadline (ms) for a remote workflow `advance`. If the worker neither returns a
|
|
4480
4495
|
* decision NOR sends a run-scoped {@link Heartbeat} within this window, the engine presumes it dead:
|
|
@@ -4626,6 +4641,13 @@ interface ListStalledRunsOptions {
|
|
|
4626
4641
|
/** Evaluate as of this clock time. Defaults to the engine clock. */
|
|
4627
4642
|
now?: number | undefined;
|
|
4628
4643
|
}
|
|
4644
|
+
/**
|
|
4645
|
+
* Thrown by {@link WorkflowEngine.resume} when nothing registered here (nor an inherited or
|
|
4646
|
+
* convention route) can run the run's pinned `workflow@version`.
|
|
4647
|
+
*/
|
|
4648
|
+
declare class WorkflowNotRegistered extends Error {
|
|
4649
|
+
constructor(workflow: string, version: string);
|
|
4650
|
+
}
|
|
4629
4651
|
/**
|
|
4630
4652
|
* The orchestrator. Owns workflow state and replays runs deterministically: each step's
|
|
4631
4653
|
* result is checkpointed, so on resume a completed step returns its saved output instead of
|
|
@@ -4648,6 +4670,7 @@ declare class WorkflowEngine {
|
|
|
4648
4670
|
*/
|
|
4649
4671
|
private readonly ownLeaseCutoffMs;
|
|
4650
4672
|
private readonly maxRecoveryAttempts?;
|
|
4673
|
+
private readonly unregisteredWorkflowGraceMs;
|
|
4651
4674
|
private readonly remoteAdvanceSilenceMs?;
|
|
4652
4675
|
private readonly reconcileMs;
|
|
4653
4676
|
private readonly timerBatchSize;
|
|
@@ -5089,6 +5112,16 @@ declare class WorkflowEngine {
|
|
|
5089
5112
|
/** Emit a locally-produced lifecycle event: deliver to subscribers AND broadcast it on the
|
|
5090
5113
|
* control plane so other instances (e.g. a dashboard pod) can live-tail this run. */
|
|
5091
5114
|
private emit;
|
|
5115
|
+
/** Events waiting for the control-plane publish in flight, in emit order (see {@link broadcastEvents}). */
|
|
5116
|
+
private readonly eventsToBroadcast;
|
|
5117
|
+
private broadcastingEvents;
|
|
5118
|
+
/**
|
|
5119
|
+
* Publish the queued events in the order they were emitted. A run's own events go out one per
|
|
5120
|
+
* message, as every instance expects them. Step events (a wide fan emits two per step) that queued
|
|
5121
|
+
* up while a publish was in flight go out together in one `events` message, so a fan of 40,000
|
|
5122
|
+
* steps costs the control plane a few hundred messages to every instance instead of 80,000.
|
|
5123
|
+
*/
|
|
5124
|
+
private broadcastEvents;
|
|
5092
5125
|
/** Deliver an event to local subscribers only (no re-broadcast) — used for both locally-produced
|
|
5093
5126
|
* events and ones received from the control plane, so an event shows up once on every instance. */
|
|
5094
5127
|
private deliver;
|
|
@@ -5359,6 +5392,33 @@ declare class WorkflowEngine {
|
|
|
5359
5392
|
* result to skip the resume, or `undefined` to proceed.
|
|
5360
5393
|
*/
|
|
5361
5394
|
private countRecovery;
|
|
5395
|
+
/** Tell everyone a run was just written `dead`: events, dead-letter routing, a waiting parent. */
|
|
5396
|
+
private announceDead;
|
|
5397
|
+
/** The synchronous half of {@link canResolveWorkflow}: callers check it first, so a registered run
|
|
5398
|
+
* pays no await (same rule as {@link resume}). */
|
|
5399
|
+
private isRegisteredHere;
|
|
5400
|
+
/**
|
|
5401
|
+
* Whether this instance can run `run`'s pinned `workflow@version`: registered here, inherited from a
|
|
5402
|
+
* remote ancestor, or routed by convention to a live worker group (the lookup {@link resume} does).
|
|
5403
|
+
*/
|
|
5404
|
+
private canResolveWorkflow;
|
|
5405
|
+
/**
|
|
5406
|
+
* A run whose lease this instance holds but whose `workflow@version` nothing here can run (see
|
|
5407
|
+
* {@link WorkflowNotRegistered}). Within {@link unregisteredWorkflowGraceMs} of becoming due it is a
|
|
5408
|
+
* rolling deploy's skew: the lease is released at once so the pod that still has the version takes
|
|
5409
|
+
* the run now, not after the lease expires. Past it, the workflow is gone from the whole fleet (a
|
|
5410
|
+
* capable pod would have taken the run long ago): dead-letter it.
|
|
5411
|
+
*
|
|
5412
|
+
* The age is read off `updatedAt` / `wakeAt`, so every path that would touch either (a recovery
|
|
5413
|
+
* count, a re-pend, a hand-off's lease-long `wakeAt`) must call this BEFORE writing — otherwise an
|
|
5414
|
+
* orphan never ages.
|
|
5415
|
+
*
|
|
5416
|
+
* Returns the `dead` result, `null` when the run left its status meanwhile (a lease-free
|
|
5417
|
+
* `cancel()`: settled, not ours to touch), or `undefined` when it was left for later.
|
|
5418
|
+
*/
|
|
5419
|
+
private settleUnregistered;
|
|
5420
|
+
/** Log a run skipped for an unregistered workflow — still there, still not progressing. */
|
|
5421
|
+
private logUnregisteredSkip;
|
|
5362
5422
|
/**
|
|
5363
5423
|
* Resume every suspended run whose durable timer is due. Call periodically (a poller) and on
|
|
5364
5424
|
* boot. A run still not due re-suspends cheaply without running new work.
|
|
@@ -5460,7 +5520,8 @@ declare class WorkflowEngine {
|
|
|
5460
5520
|
* Resume each run only if this instance can acquire its recovery lease — so when several
|
|
5461
5521
|
* replicas recover or poll at once, each run is picked up by exactly one of them.
|
|
5462
5522
|
*
|
|
5463
|
-
* A run this instance cannot resume is SKIPPED, not allowed to abort the batch
|
|
5523
|
+
* A run this instance cannot resume is SKIPPED, not allowed to abort the batch (a run whose workflow
|
|
5524
|
+
* stays registered nowhere past `unregisteredWorkflowGraceMs` is dead-lettered instead). The canonical
|
|
5464
5525
|
* case is skew protection: mid-rolling-deploy a pod leases a run whose workflow it does not have
|
|
5465
5526
|
* and `resume` throws by design — but the rest of the batch is runs it can drive perfectly well,
|
|
5466
5527
|
* and every caller here is a recovery sweep. Letting the first such run end the loop means one
|
|
@@ -7843,4 +7904,4 @@ declare function singletonAdmissionDenialPatch(runs: WorkflowRun[], runId: strin
|
|
|
7843
7904
|
updatedAt: Date;
|
|
7844
7905
|
} | undefined;
|
|
7845
7906
|
|
|
7846
|
-
export { type AccumulatorHost, type Admission, type AdmissionBackend, type AdmissionItem, type AllOptions, type AllSettledOptions, type AllStepsOptions, type AllStepsSettledOptions, type AnnouncedWorkflow, type AnnouncementEvidence, type AttributeFilter, type AttributeOp, type AttributePredicateOperand, type AttributeValue, type BackoffStrategy, type BlockedDispatch, type BlockedResolution, type BufferedSignalRow, CHILD_EDGE_PREFIXES, CONTROL_FLOW_SIGNAL, CORE_ORIGIN, CURRENT_PROTOCOL_VERSION, type CapabilityRequirement, type CheckpointAggregatingStore, type CheckpointNameKind, type CheckpointQuery, type CheckpointStat, type ChildCallOptions, type ChildWait, CodecStateStore, type CodecStateStoreOptions, type ConcurrencyConfig, ConcurrencyLimitError, type ConcurrencyQuota, ContinueAsNew, type ControlMessage, type ControlPlane, DEFAULT_CONCURRENCY_STATUSES, DEFAULT_PARTITION, DEFAULT_STALLED_AFTER_MS, DURABLE_OPTIONS, DURABLE_OPTIONS_CANONICAL, DURABLE_STEP_CONFIG, DURABLE_STEP_NAME, type Disagreement, type DispatchDiagnostics, type DispatchPlan, type DurableExecution, type DurableExecutionWrapper, type DurableTopology, type DurableWebhook, DurableWorkflow, type DynamicWorkflowCtx, ENGINE_MINTED_TAG_PREFIXES, ENTITY_WORKFLOW, EVENT_BUFFER_SCAN_LIMIT, EVENT_TOKEN_PREFIX, type EngineEvent, type EngineEventType, type EngineListener, Entities, type EntityConfig, type EntityHandler, type EntityHost, type EntityWorkflowHost, EventAccumulators, type EventBatchConfig, FAN_SUMMARY_FAILURES, type FanJoin, type FanPage, type FanPageQuery, type FanSummary, type FanWait, FatalError, type FindStalledRunsOptions, GatherError, type GatherItemFailure, type GroupHealth, HANDOFF_SEQ_BASE, type Heartbeat, type HeartbeatSighting, type HeartbeatStatus, type HistoryEvent, InMemoryAdmissionBackend, InMemoryRunQueue, InMemoryStateStore, InMemoryTransport, type InferSearchAttributes, LEGACY_V1_CAPABILITIES, LEGACY_V1_PROTOCOL, type ListStalledRunsOptions, type MetricsCollector, type MetricsSnapshot, type NamedTransport, type NegotiateOptions, type NegotiationOutcome, type NegotiationReason, type NegotiationResult, NestedWorkflowCallError, NonDeterminismError, type OnErrorInput, type OnErrorRoute, type OwnedLeaseReclaim, type PartitionSighting, type PayloadCodec, type QueueConfig, QueueController, REMOTE_EVENT, RUN_HISTORY_FIXTURE_VERSION, RUN_VALUE_FACET_LIMIT, RUN_VALUE_FACET_SCAN, type RawWorkerDescriptor, RemoteStepError, RemoteStepTimeout, type RemoteTask, RemoteWorkflowExecutor, RemoteWorkflowTimeout, type RetentionPolicy, type RoutableDispatch, type RoutableResolution, type RoutingResolution, type RunAttributeRow, type RunConsumerOptions, type RunDetail, type RunDetailOptions, type RunDispatch, type RunDispatcher, type RunFacetQuery, type RunFacetRow, RunGateway, type RunListItem, type RunQuery, type RunQueueOptions, type RunReply, type RunReplyResult, type RunRequest, type RunRequestKind, type RunResult, type RunSchedulesOptions, type RunScope, type RunStatus, type RunValueAxis, type RunValueFacetOptions, type RunValueFacetOrder, type RunValueFacetRow, type RunWaiting, SINGLETON_ACTIVE_STATUSES, SINGLETON_ADMITTED_TAG, SINGLETON_RESERVED_TAGS, SINGLETON_TRACKED_TAG, STATE_STORE, STATE_STORE_CANONICAL, SUMMARY_CHILD_LIMIT, type ScalarAttributeFilter, ScheduleAlreadyExistsError, ScheduleClient, type ScheduleClientOptions, type ScheduleDescription, type ScheduleHost, type ScheduleListQuery, ScheduleNotFoundError, type ScheduleOptions, type ScheduleQuery, type ScheduleRecord, type ScheduledWorkflow, type SchemaIssue, type SchemaValidationCode, SchemaValidationError, type SearchAttributes, type SearchAttributesSchema, type Settled, SignalTimeoutError, type SignalWaiter, type SingletonConfig, SingletonQueueFullError, type StalePendingStep, type StalledRunInfo, StandardSchemaV1, type StartOptions, type StartRunMessage, type StateStore, type StepCheckpoint, type StepConfig, type StepDef, type StepDispatchOpts, type StepError, type StepEvent, type StepHandler, type StepInterceptor, type StepInvocation, type StepKind, type StepLogger, type StepOptions, type StepRef, type StepResult, type StepScope, type StepSettlement, type StepUndo, type StoreTransaction, type SubProcessHandle, TERMINAL_RUN_STATUSES, TRANSPORT, TRANSPORT_CANONICAL, type TenantEvent, type TenantTokenSource, type TenantVerifier, type Transport, type UndoOf, type UpdateResult, type UpdateValidator, VERSION_UNDECLARED_TAG, type VerifiedTenant, WORKFLOW_NAME_KEY, type WireDates, type WorkerAdjust, type WorkerConcurrencyStatus, type WorkerDescriptor, type WorkerHeartbeat, type WorkerLifecycle, type WorkerStatus, type WorkflowAnnouncement, WorkflowBlocked, type WorkflowClass, type WorkflowCommand, type WorkflowCtx, type WorkflowDecision, type WorkflowDirectory, WorkflowEngine, type WorkflowEngineDeps, type WorkflowExecutor, type WorkflowHandler, type WorkflowInputOf, type WorkflowOutputOf, type WorkflowRef, type WorkflowRegistration, type WorkflowRun, type WorkflowStartClient, type WorkflowStepEvent, WorkflowSuspended, type WorkflowTask, aggregateAnnouncements, aggregatesCheckpoints, announcementsOf, applyAttributeQuery, assertConcurrency, assertRoutes, attributeColumnFor, attributeOperand, attributePredicateOperands, axisIsRunColumn, bindWorkflowClass, breakpointToken, buildWorkflowDirectory, canRoute, capableWorkers, checkpointNameKind, checkpointStatsOf, childIdsOf, classifyWaiterToken, clearDurableExecutionWrappers, clearTerminalSingletonAdmission, codeAndDetails, codeOf, collectMetrics, concurrencyTag, controlPlaneDescriptor, countRuns, coveredTokens, createStepLogger, currentStep, currentStepScope, currentWorkflowCtx, debug, describeWorker, descriptorHash, detailsOf, error, etagMatches, eventMatchOf, eventMatches, eventNameOf, eventPrefix, eventToken, facetOrigin, fanPageBounds, fanPageEtag, findStalledRuns, foldedCount, getRunsByIds, handoffHandlerId, handoffName, hasDurableExecutionWrappers, heartbeatStatus, hmacTenantVerifier, indexWaitersByRun, info, isEngineMintedTag, isEventToken, isFailedMember, isLegacyDescriptor, isNonEmptyRunScope, isRemoteEvent, isWorkflowControlFlowSignal, legacySingletonHolders, listSignalWaitersOfRuns, liveHandoff, log, markRemoteEvent, matchesAttributes, mergeRunFacetRows, mergeRunValueFacetRows, missingCapabilities, negotiate, nextCronFireMs, normalizeAttributeRows, normalizeDescriptor, observedAnnouncements, onErrorRunId, pageFan, pageFanFromStore, parentRunIdOf, parseDuration, parseWebhookToken, partitionOfToken, planDispatch, poolNamespace, prevCronFireMs, queryCheckpointsOf, readRunsByIds, readSearchAttributes, requiredCapabilities, resolveConcurrency, resolveRouting, resolveRunWaiting, resolveTenantToken, resolveWireTenant, retiredHandoffName, routeFor, runDetailEtag, runDurableExecution, runInStepLogger, runInStepScope, runInWorkflowCtx, runOutsideWorkflowCtx, runSchedules, runStepHandler, runValueFacetsFromRuns, sanitizeQueueToken, scheduleRunIdAt, scheduledRunId, signTenantToken, singletonAdmissionDenialPatch, singletonAdmissionTags, sqlComparator, stepConfigOf, stepId, stepNameOf, sub, subEvent, subProcess, summarizeFan, summarizeRunFromStore, summarizeTimeline, tenantGroup, useDurableExecution, warn, withSingletonTracked, withoutSingletonMarkers, workflowName };
|
|
7907
|
+
export { type AccumulatorHost, type Admission, type AdmissionBackend, type AdmissionItem, type AllOptions, type AllSettledOptions, type AllStepsOptions, type AllStepsSettledOptions, type AnnouncedWorkflow, type AnnouncementEvidence, type AttributeFilter, type AttributeOp, type AttributePredicateOperand, type AttributeValue, type BackoffStrategy, type BlockedDispatch, type BlockedResolution, type BufferedSignalRow, CHILD_EDGE_PREFIXES, CONTROL_FLOW_SIGNAL, CORE_ORIGIN, CURRENT_PROTOCOL_VERSION, type CapabilityRequirement, type CheckpointAggregatingStore, type CheckpointNameKind, type CheckpointQuery, type CheckpointStat, type ChildCallOptions, type ChildWait, CodecStateStore, type CodecStateStoreOptions, type ConcurrencyConfig, ConcurrencyLimitError, type ConcurrencyQuota, ContinueAsNew, type ControlMessage, type ControlPlane, DEFAULT_CONCURRENCY_STATUSES, DEFAULT_PARTITION, DEFAULT_STALLED_AFTER_MS, DURABLE_OPTIONS, DURABLE_OPTIONS_CANONICAL, DURABLE_STEP_CONFIG, DURABLE_STEP_NAME, type Disagreement, type DispatchDiagnostics, type DispatchPlan, type DurableExecution, type DurableExecutionWrapper, type DurableTopology, type DurableWebhook, DurableWorkflow, type DynamicWorkflowCtx, ENGINE_MINTED_TAG_PREFIXES, ENTITY_WORKFLOW, EVENT_BUFFER_SCAN_LIMIT, EVENT_TOKEN_PREFIX, type EngineEvent, type EngineEventType, type EngineListener, Entities, type EntityConfig, type EntityHandler, type EntityHost, type EntityWorkflowHost, EventAccumulators, type EventBatchConfig, FAN_SUMMARY_FAILURES, type FanJoin, type FanPage, type FanPageQuery, type FanSummary, type FanWait, FatalError, type FindStalledRunsOptions, GatherError, type GatherItemFailure, type GroupHealth, HANDOFF_SEQ_BASE, type Heartbeat, type HeartbeatSighting, type HeartbeatStatus, type HistoryEvent, InMemoryAdmissionBackend, InMemoryRunQueue, InMemoryStateStore, InMemoryTransport, type InferSearchAttributes, LEGACY_V1_CAPABILITIES, LEGACY_V1_PROTOCOL, type ListStalledRunsOptions, type MetricsCollector, type MetricsSnapshot, type NamedTransport, type NegotiateOptions, type NegotiationOutcome, type NegotiationReason, type NegotiationResult, NestedWorkflowCallError, NonDeterminismError, type OnErrorInput, type OnErrorRoute, type OwnedLeaseReclaim, type PartitionSighting, type PayloadCodec, type QueueConfig, QueueController, REMOTE_EVENT, RUN_HISTORY_FIXTURE_VERSION, RUN_VALUE_FACET_LIMIT, RUN_VALUE_FACET_SCAN, type RawWorkerDescriptor, RemoteStepError, RemoteStepTimeout, type RemoteTask, RemoteWorkflowExecutor, RemoteWorkflowTimeout, type RetentionPolicy, type RoutableDispatch, type RoutableResolution, type RoutingResolution, type RunAttributeRow, type RunConsumerOptions, type RunDetail, type RunDetailOptions, type RunDispatch, type RunDispatcher, type RunFacetQuery, type RunFacetRow, RunGateway, type RunListItem, type RunQuery, type RunQueueOptions, type RunReply, type RunReplyResult, type RunRequest, type RunRequestKind, type RunResult, type RunSchedulesOptions, type RunScope, type RunStatus, type RunValueAxis, type RunValueFacetOptions, type RunValueFacetOrder, type RunValueFacetRow, type RunWaiting, SINGLETON_ACTIVE_STATUSES, SINGLETON_ADMITTED_TAG, SINGLETON_RESERVED_TAGS, SINGLETON_TRACKED_TAG, STATE_STORE, STATE_STORE_CANONICAL, SUMMARY_CHILD_LIMIT, type ScalarAttributeFilter, ScheduleAlreadyExistsError, ScheduleClient, type ScheduleClientOptions, type ScheduleDescription, type ScheduleHost, type ScheduleListQuery, ScheduleNotFoundError, type ScheduleOptions, type ScheduleQuery, type ScheduleRecord, type ScheduledWorkflow, type SchemaIssue, type SchemaValidationCode, SchemaValidationError, type SearchAttributes, type SearchAttributesSchema, type Settled, SignalTimeoutError, type SignalWaiter, type SingletonConfig, SingletonQueueFullError, type StalePendingStep, type StalledRunInfo, StandardSchemaV1, type StartOptions, type StartRunMessage, type StateStore, type StepCheckpoint, type StepConfig, type StepDef, type StepDispatchOpts, type StepError, type StepEvent, type StepHandler, type StepInterceptor, type StepInvocation, type StepKind, type StepLogger, type StepOptions, type StepRef, type StepResult, type StepScope, type StepSettlement, type StepUndo, type StoreTransaction, type SubProcessHandle, TERMINAL_RUN_STATUSES, TRANSPORT, TRANSPORT_CANONICAL, type TenantEvent, type TenantTokenSource, type TenantVerifier, type Transport, type UndoOf, type UpdateResult, type UpdateValidator, VERSION_UNDECLARED_TAG, type VerifiedTenant, WORKFLOW_NAME_KEY, type WireDates, type WorkerAdjust, type WorkerConcurrencyStatus, type WorkerDescriptor, type WorkerHeartbeat, type WorkerLifecycle, type WorkerStatus, type WorkflowAnnouncement, WorkflowBlocked, type WorkflowClass, type WorkflowCommand, type WorkflowCtx, type WorkflowDecision, type WorkflowDirectory, WorkflowEngine, type WorkflowEngineDeps, type WorkflowExecutor, type WorkflowHandler, type WorkflowInputOf, WorkflowNotRegistered, type WorkflowOutputOf, type WorkflowRef, type WorkflowRegistration, type WorkflowRun, type WorkflowStartClient, type WorkflowStepEvent, WorkflowSuspended, type WorkflowTask, aggregateAnnouncements, aggregatesCheckpoints, announcementsOf, applyAttributeQuery, assertConcurrency, assertRoutes, attributeColumnFor, attributeOperand, attributePredicateOperands, axisIsRunColumn, bindWorkflowClass, breakpointToken, buildWorkflowDirectory, canRoute, capableWorkers, checkpointNameKind, checkpointStatsOf, childIdsOf, classifyWaiterToken, clearDurableExecutionWrappers, clearTerminalSingletonAdmission, codeAndDetails, codeOf, collectMetrics, concurrencyTag, controlPlaneDescriptor, countRuns, coveredTokens, createStepLogger, currentStep, currentStepScope, currentWorkflowCtx, debug, describeWorker, descriptorHash, detailsOf, error, etagMatches, eventMatchOf, eventMatches, eventNameOf, eventPrefix, eventToken, facetOrigin, fanPageBounds, fanPageEtag, findStalledRuns, foldedCount, getRunsByIds, handoffHandlerId, handoffName, hasDurableExecutionWrappers, heartbeatStatus, hmacTenantVerifier, indexWaitersByRun, info, isEngineMintedTag, isEventToken, isFailedMember, isLegacyDescriptor, isNonEmptyRunScope, isRemoteEvent, isWorkflowControlFlowSignal, legacySingletonHolders, listSignalWaitersOfRuns, liveHandoff, log, markRemoteEvent, matchesAttributes, mergeRunFacetRows, mergeRunValueFacetRows, missingCapabilities, negotiate, nextCronFireMs, normalizeAttributeRows, normalizeDescriptor, observedAnnouncements, onErrorRunId, pageFan, pageFanFromStore, parentRunIdOf, parseDuration, parseWebhookToken, partitionOfToken, planDispatch, poolNamespace, prevCronFireMs, queryCheckpointsOf, readRunsByIds, readSearchAttributes, requiredCapabilities, resolveConcurrency, resolveRouting, resolveRunWaiting, resolveTenantToken, resolveWireTenant, retiredHandoffName, routeFor, runDetailEtag, runDurableExecution, runInStepLogger, runInStepScope, runInWorkflowCtx, runOutsideWorkflowCtx, runSchedules, runStepHandler, runValueFacetsFromRuns, sanitizeQueueToken, scheduleRunIdAt, scheduledRunId, signTenantToken, singletonAdmissionDenialPatch, singletonAdmissionTags, sqlComparator, stepConfigOf, stepId, stepNameOf, sub, subEvent, subProcess, summarizeFan, summarizeRunFromStore, summarizeTimeline, tenantGroup, useDurableExecution, warn, withSingletonTracked, withoutSingletonMarkers, workflowName };
|
package/dist/index.d.ts
CHANGED
|
@@ -902,7 +902,8 @@ type RunStatus =
|
|
|
902
902
|
* are actually published (a legacy/pre-handshake fleet skips the guard, design §7.7).
|
|
903
903
|
*/
|
|
904
904
|
| 'blocked' | 'completed' | 'failed' | 'cancelled'
|
|
905
|
-
/** Dead-letter: recovery gave up after `maxRecoveryAttempts` (a poison pill)
|
|
905
|
+
/** Dead-letter: recovery gave up after `maxRecoveryAttempts` (a poison pill), or no instance has
|
|
906
|
+
* the run's workflow version registered past `unregisteredWorkflowGraceMs`. Terminal; inspect it. */
|
|
906
907
|
| 'dead';
|
|
907
908
|
/** One execution of a workflow. The unit of durability and the unit shown in the dashboard. */
|
|
908
909
|
interface WorkflowRun {
|
|
@@ -2628,6 +2629,9 @@ type ControlMessage = {
|
|
|
2628
2629
|
} & ({
|
|
2629
2630
|
kind: 'event';
|
|
2630
2631
|
event: EngineEvent;
|
|
2632
|
+
} | {
|
|
2633
|
+
kind: 'events';
|
|
2634
|
+
events: EngineEvent[];
|
|
2631
2635
|
} | {
|
|
2632
2636
|
kind: 'cancel';
|
|
2633
2637
|
runId: string;
|
|
@@ -4475,6 +4479,17 @@ interface WorkflowEngineDeps {
|
|
|
4475
4479
|
* Omit for unlimited (the default — recovery always retries).
|
|
4476
4480
|
*/
|
|
4477
4481
|
maxRecoveryAttempts?: number | undefined;
|
|
4482
|
+
/**
|
|
4483
|
+
* How long (ms) a run may stay due while NO instance can resume it because its workflow version is
|
|
4484
|
+
* not registered anywhere, before it is moved to `dead` (code `workflow_not_registered`). A rolling
|
|
4485
|
+
* deploy skips such a run on purpose (skew protection): the pod still on the prior version picks it
|
|
4486
|
+
* up within seconds. A run that stays unresumable far past that — its workflow was renamed or
|
|
4487
|
+
* deleted — would otherwise be retried and logged on every tick forever. The clock starts when the
|
|
4488
|
+
* run became due (its `wakeAt` for a parked run, its last update otherwise), so a long sleep is not
|
|
4489
|
+
* counted. Default 24h; `0` disables (skip forever). A non-finite or negative value falls back to
|
|
4490
|
+
* the default.
|
|
4491
|
+
*/
|
|
4492
|
+
unregisteredWorkflowGraceMs?: number | undefined;
|
|
4478
4493
|
/**
|
|
4479
4494
|
* Opt-in liveness deadline (ms) for a remote workflow `advance`. If the worker neither returns a
|
|
4480
4495
|
* decision NOR sends a run-scoped {@link Heartbeat} within this window, the engine presumes it dead:
|
|
@@ -4626,6 +4641,13 @@ interface ListStalledRunsOptions {
|
|
|
4626
4641
|
/** Evaluate as of this clock time. Defaults to the engine clock. */
|
|
4627
4642
|
now?: number | undefined;
|
|
4628
4643
|
}
|
|
4644
|
+
/**
|
|
4645
|
+
* Thrown by {@link WorkflowEngine.resume} when nothing registered here (nor an inherited or
|
|
4646
|
+
* convention route) can run the run's pinned `workflow@version`.
|
|
4647
|
+
*/
|
|
4648
|
+
declare class WorkflowNotRegistered extends Error {
|
|
4649
|
+
constructor(workflow: string, version: string);
|
|
4650
|
+
}
|
|
4629
4651
|
/**
|
|
4630
4652
|
* The orchestrator. Owns workflow state and replays runs deterministically: each step's
|
|
4631
4653
|
* result is checkpointed, so on resume a completed step returns its saved output instead of
|
|
@@ -4648,6 +4670,7 @@ declare class WorkflowEngine {
|
|
|
4648
4670
|
*/
|
|
4649
4671
|
private readonly ownLeaseCutoffMs;
|
|
4650
4672
|
private readonly maxRecoveryAttempts?;
|
|
4673
|
+
private readonly unregisteredWorkflowGraceMs;
|
|
4651
4674
|
private readonly remoteAdvanceSilenceMs?;
|
|
4652
4675
|
private readonly reconcileMs;
|
|
4653
4676
|
private readonly timerBatchSize;
|
|
@@ -5089,6 +5112,16 @@ declare class WorkflowEngine {
|
|
|
5089
5112
|
/** Emit a locally-produced lifecycle event: deliver to subscribers AND broadcast it on the
|
|
5090
5113
|
* control plane so other instances (e.g. a dashboard pod) can live-tail this run. */
|
|
5091
5114
|
private emit;
|
|
5115
|
+
/** Events waiting for the control-plane publish in flight, in emit order (see {@link broadcastEvents}). */
|
|
5116
|
+
private readonly eventsToBroadcast;
|
|
5117
|
+
private broadcastingEvents;
|
|
5118
|
+
/**
|
|
5119
|
+
* Publish the queued events in the order they were emitted. A run's own events go out one per
|
|
5120
|
+
* message, as every instance expects them. Step events (a wide fan emits two per step) that queued
|
|
5121
|
+
* up while a publish was in flight go out together in one `events` message, so a fan of 40,000
|
|
5122
|
+
* steps costs the control plane a few hundred messages to every instance instead of 80,000.
|
|
5123
|
+
*/
|
|
5124
|
+
private broadcastEvents;
|
|
5092
5125
|
/** Deliver an event to local subscribers only (no re-broadcast) — used for both locally-produced
|
|
5093
5126
|
* events and ones received from the control plane, so an event shows up once on every instance. */
|
|
5094
5127
|
private deliver;
|
|
@@ -5359,6 +5392,33 @@ declare class WorkflowEngine {
|
|
|
5359
5392
|
* result to skip the resume, or `undefined` to proceed.
|
|
5360
5393
|
*/
|
|
5361
5394
|
private countRecovery;
|
|
5395
|
+
/** Tell everyone a run was just written `dead`: events, dead-letter routing, a waiting parent. */
|
|
5396
|
+
private announceDead;
|
|
5397
|
+
/** The synchronous half of {@link canResolveWorkflow}: callers check it first, so a registered run
|
|
5398
|
+
* pays no await (same rule as {@link resume}). */
|
|
5399
|
+
private isRegisteredHere;
|
|
5400
|
+
/**
|
|
5401
|
+
* Whether this instance can run `run`'s pinned `workflow@version`: registered here, inherited from a
|
|
5402
|
+
* remote ancestor, or routed by convention to a live worker group (the lookup {@link resume} does).
|
|
5403
|
+
*/
|
|
5404
|
+
private canResolveWorkflow;
|
|
5405
|
+
/**
|
|
5406
|
+
* A run whose lease this instance holds but whose `workflow@version` nothing here can run (see
|
|
5407
|
+
* {@link WorkflowNotRegistered}). Within {@link unregisteredWorkflowGraceMs} of becoming due it is a
|
|
5408
|
+
* rolling deploy's skew: the lease is released at once so the pod that still has the version takes
|
|
5409
|
+
* the run now, not after the lease expires. Past it, the workflow is gone from the whole fleet (a
|
|
5410
|
+
* capable pod would have taken the run long ago): dead-letter it.
|
|
5411
|
+
*
|
|
5412
|
+
* The age is read off `updatedAt` / `wakeAt`, so every path that would touch either (a recovery
|
|
5413
|
+
* count, a re-pend, a hand-off's lease-long `wakeAt`) must call this BEFORE writing — otherwise an
|
|
5414
|
+
* orphan never ages.
|
|
5415
|
+
*
|
|
5416
|
+
* Returns the `dead` result, `null` when the run left its status meanwhile (a lease-free
|
|
5417
|
+
* `cancel()`: settled, not ours to touch), or `undefined` when it was left for later.
|
|
5418
|
+
*/
|
|
5419
|
+
private settleUnregistered;
|
|
5420
|
+
/** Log a run skipped for an unregistered workflow — still there, still not progressing. */
|
|
5421
|
+
private logUnregisteredSkip;
|
|
5362
5422
|
/**
|
|
5363
5423
|
* Resume every suspended run whose durable timer is due. Call periodically (a poller) and on
|
|
5364
5424
|
* boot. A run still not due re-suspends cheaply without running new work.
|
|
@@ -5460,7 +5520,8 @@ declare class WorkflowEngine {
|
|
|
5460
5520
|
* Resume each run only if this instance can acquire its recovery lease — so when several
|
|
5461
5521
|
* replicas recover or poll at once, each run is picked up by exactly one of them.
|
|
5462
5522
|
*
|
|
5463
|
-
* A run this instance cannot resume is SKIPPED, not allowed to abort the batch
|
|
5523
|
+
* A run this instance cannot resume is SKIPPED, not allowed to abort the batch (a run whose workflow
|
|
5524
|
+
* stays registered nowhere past `unregisteredWorkflowGraceMs` is dead-lettered instead). The canonical
|
|
5464
5525
|
* case is skew protection: mid-rolling-deploy a pod leases a run whose workflow it does not have
|
|
5465
5526
|
* and `resume` throws by design — but the rest of the batch is runs it can drive perfectly well,
|
|
5466
5527
|
* and every caller here is a recovery sweep. Letting the first such run end the loop means one
|
|
@@ -7843,4 +7904,4 @@ declare function singletonAdmissionDenialPatch(runs: WorkflowRun[], runId: strin
|
|
|
7843
7904
|
updatedAt: Date;
|
|
7844
7905
|
} | undefined;
|
|
7845
7906
|
|
|
7846
|
-
export { type AccumulatorHost, type Admission, type AdmissionBackend, type AdmissionItem, type AllOptions, type AllSettledOptions, type AllStepsOptions, type AllStepsSettledOptions, type AnnouncedWorkflow, type AnnouncementEvidence, type AttributeFilter, type AttributeOp, type AttributePredicateOperand, type AttributeValue, type BackoffStrategy, type BlockedDispatch, type BlockedResolution, type BufferedSignalRow, CHILD_EDGE_PREFIXES, CONTROL_FLOW_SIGNAL, CORE_ORIGIN, CURRENT_PROTOCOL_VERSION, type CapabilityRequirement, type CheckpointAggregatingStore, type CheckpointNameKind, type CheckpointQuery, type CheckpointStat, type ChildCallOptions, type ChildWait, CodecStateStore, type CodecStateStoreOptions, type ConcurrencyConfig, ConcurrencyLimitError, type ConcurrencyQuota, ContinueAsNew, type ControlMessage, type ControlPlane, DEFAULT_CONCURRENCY_STATUSES, DEFAULT_PARTITION, DEFAULT_STALLED_AFTER_MS, DURABLE_OPTIONS, DURABLE_OPTIONS_CANONICAL, DURABLE_STEP_CONFIG, DURABLE_STEP_NAME, type Disagreement, type DispatchDiagnostics, type DispatchPlan, type DurableExecution, type DurableExecutionWrapper, type DurableTopology, type DurableWebhook, DurableWorkflow, type DynamicWorkflowCtx, ENGINE_MINTED_TAG_PREFIXES, ENTITY_WORKFLOW, EVENT_BUFFER_SCAN_LIMIT, EVENT_TOKEN_PREFIX, type EngineEvent, type EngineEventType, type EngineListener, Entities, type EntityConfig, type EntityHandler, type EntityHost, type EntityWorkflowHost, EventAccumulators, type EventBatchConfig, FAN_SUMMARY_FAILURES, type FanJoin, type FanPage, type FanPageQuery, type FanSummary, type FanWait, FatalError, type FindStalledRunsOptions, GatherError, type GatherItemFailure, type GroupHealth, HANDOFF_SEQ_BASE, type Heartbeat, type HeartbeatSighting, type HeartbeatStatus, type HistoryEvent, InMemoryAdmissionBackend, InMemoryRunQueue, InMemoryStateStore, InMemoryTransport, type InferSearchAttributes, LEGACY_V1_CAPABILITIES, LEGACY_V1_PROTOCOL, type ListStalledRunsOptions, type MetricsCollector, type MetricsSnapshot, type NamedTransport, type NegotiateOptions, type NegotiationOutcome, type NegotiationReason, type NegotiationResult, NestedWorkflowCallError, NonDeterminismError, type OnErrorInput, type OnErrorRoute, type OwnedLeaseReclaim, type PartitionSighting, type PayloadCodec, type QueueConfig, QueueController, REMOTE_EVENT, RUN_HISTORY_FIXTURE_VERSION, RUN_VALUE_FACET_LIMIT, RUN_VALUE_FACET_SCAN, type RawWorkerDescriptor, RemoteStepError, RemoteStepTimeout, type RemoteTask, RemoteWorkflowExecutor, RemoteWorkflowTimeout, type RetentionPolicy, type RoutableDispatch, type RoutableResolution, type RoutingResolution, type RunAttributeRow, type RunConsumerOptions, type RunDetail, type RunDetailOptions, type RunDispatch, type RunDispatcher, type RunFacetQuery, type RunFacetRow, RunGateway, type RunListItem, type RunQuery, type RunQueueOptions, type RunReply, type RunReplyResult, type RunRequest, type RunRequestKind, type RunResult, type RunSchedulesOptions, type RunScope, type RunStatus, type RunValueAxis, type RunValueFacetOptions, type RunValueFacetOrder, type RunValueFacetRow, type RunWaiting, SINGLETON_ACTIVE_STATUSES, SINGLETON_ADMITTED_TAG, SINGLETON_RESERVED_TAGS, SINGLETON_TRACKED_TAG, STATE_STORE, STATE_STORE_CANONICAL, SUMMARY_CHILD_LIMIT, type ScalarAttributeFilter, ScheduleAlreadyExistsError, ScheduleClient, type ScheduleClientOptions, type ScheduleDescription, type ScheduleHost, type ScheduleListQuery, ScheduleNotFoundError, type ScheduleOptions, type ScheduleQuery, type ScheduleRecord, type ScheduledWorkflow, type SchemaIssue, type SchemaValidationCode, SchemaValidationError, type SearchAttributes, type SearchAttributesSchema, type Settled, SignalTimeoutError, type SignalWaiter, type SingletonConfig, SingletonQueueFullError, type StalePendingStep, type StalledRunInfo, StandardSchemaV1, type StartOptions, type StartRunMessage, type StateStore, type StepCheckpoint, type StepConfig, type StepDef, type StepDispatchOpts, type StepError, type StepEvent, type StepHandler, type StepInterceptor, type StepInvocation, type StepKind, type StepLogger, type StepOptions, type StepRef, type StepResult, type StepScope, type StepSettlement, type StepUndo, type StoreTransaction, type SubProcessHandle, TERMINAL_RUN_STATUSES, TRANSPORT, TRANSPORT_CANONICAL, type TenantEvent, type TenantTokenSource, type TenantVerifier, type Transport, type UndoOf, type UpdateResult, type UpdateValidator, VERSION_UNDECLARED_TAG, type VerifiedTenant, WORKFLOW_NAME_KEY, type WireDates, type WorkerAdjust, type WorkerConcurrencyStatus, type WorkerDescriptor, type WorkerHeartbeat, type WorkerLifecycle, type WorkerStatus, type WorkflowAnnouncement, WorkflowBlocked, type WorkflowClass, type WorkflowCommand, type WorkflowCtx, type WorkflowDecision, type WorkflowDirectory, WorkflowEngine, type WorkflowEngineDeps, type WorkflowExecutor, type WorkflowHandler, type WorkflowInputOf, type WorkflowOutputOf, type WorkflowRef, type WorkflowRegistration, type WorkflowRun, type WorkflowStartClient, type WorkflowStepEvent, WorkflowSuspended, type WorkflowTask, aggregateAnnouncements, aggregatesCheckpoints, announcementsOf, applyAttributeQuery, assertConcurrency, assertRoutes, attributeColumnFor, attributeOperand, attributePredicateOperands, axisIsRunColumn, bindWorkflowClass, breakpointToken, buildWorkflowDirectory, canRoute, capableWorkers, checkpointNameKind, checkpointStatsOf, childIdsOf, classifyWaiterToken, clearDurableExecutionWrappers, clearTerminalSingletonAdmission, codeAndDetails, codeOf, collectMetrics, concurrencyTag, controlPlaneDescriptor, countRuns, coveredTokens, createStepLogger, currentStep, currentStepScope, currentWorkflowCtx, debug, describeWorker, descriptorHash, detailsOf, error, etagMatches, eventMatchOf, eventMatches, eventNameOf, eventPrefix, eventToken, facetOrigin, fanPageBounds, fanPageEtag, findStalledRuns, foldedCount, getRunsByIds, handoffHandlerId, handoffName, hasDurableExecutionWrappers, heartbeatStatus, hmacTenantVerifier, indexWaitersByRun, info, isEngineMintedTag, isEventToken, isFailedMember, isLegacyDescriptor, isNonEmptyRunScope, isRemoteEvent, isWorkflowControlFlowSignal, legacySingletonHolders, listSignalWaitersOfRuns, liveHandoff, log, markRemoteEvent, matchesAttributes, mergeRunFacetRows, mergeRunValueFacetRows, missingCapabilities, negotiate, nextCronFireMs, normalizeAttributeRows, normalizeDescriptor, observedAnnouncements, onErrorRunId, pageFan, pageFanFromStore, parentRunIdOf, parseDuration, parseWebhookToken, partitionOfToken, planDispatch, poolNamespace, prevCronFireMs, queryCheckpointsOf, readRunsByIds, readSearchAttributes, requiredCapabilities, resolveConcurrency, resolveRouting, resolveRunWaiting, resolveTenantToken, resolveWireTenant, retiredHandoffName, routeFor, runDetailEtag, runDurableExecution, runInStepLogger, runInStepScope, runInWorkflowCtx, runOutsideWorkflowCtx, runSchedules, runStepHandler, runValueFacetsFromRuns, sanitizeQueueToken, scheduleRunIdAt, scheduledRunId, signTenantToken, singletonAdmissionDenialPatch, singletonAdmissionTags, sqlComparator, stepConfigOf, stepId, stepNameOf, sub, subEvent, subProcess, summarizeFan, summarizeRunFromStore, summarizeTimeline, tenantGroup, useDurableExecution, warn, withSingletonTracked, withoutSingletonMarkers, workflowName };
|
|
7907
|
+
export { type AccumulatorHost, type Admission, type AdmissionBackend, type AdmissionItem, type AllOptions, type AllSettledOptions, type AllStepsOptions, type AllStepsSettledOptions, type AnnouncedWorkflow, type AnnouncementEvidence, type AttributeFilter, type AttributeOp, type AttributePredicateOperand, type AttributeValue, type BackoffStrategy, type BlockedDispatch, type BlockedResolution, type BufferedSignalRow, CHILD_EDGE_PREFIXES, CONTROL_FLOW_SIGNAL, CORE_ORIGIN, CURRENT_PROTOCOL_VERSION, type CapabilityRequirement, type CheckpointAggregatingStore, type CheckpointNameKind, type CheckpointQuery, type CheckpointStat, type ChildCallOptions, type ChildWait, CodecStateStore, type CodecStateStoreOptions, type ConcurrencyConfig, ConcurrencyLimitError, type ConcurrencyQuota, ContinueAsNew, type ControlMessage, type ControlPlane, DEFAULT_CONCURRENCY_STATUSES, DEFAULT_PARTITION, DEFAULT_STALLED_AFTER_MS, DURABLE_OPTIONS, DURABLE_OPTIONS_CANONICAL, DURABLE_STEP_CONFIG, DURABLE_STEP_NAME, type Disagreement, type DispatchDiagnostics, type DispatchPlan, type DurableExecution, type DurableExecutionWrapper, type DurableTopology, type DurableWebhook, DurableWorkflow, type DynamicWorkflowCtx, ENGINE_MINTED_TAG_PREFIXES, ENTITY_WORKFLOW, EVENT_BUFFER_SCAN_LIMIT, EVENT_TOKEN_PREFIX, type EngineEvent, type EngineEventType, type EngineListener, Entities, type EntityConfig, type EntityHandler, type EntityHost, type EntityWorkflowHost, EventAccumulators, type EventBatchConfig, FAN_SUMMARY_FAILURES, type FanJoin, type FanPage, type FanPageQuery, type FanSummary, type FanWait, FatalError, type FindStalledRunsOptions, GatherError, type GatherItemFailure, type GroupHealth, HANDOFF_SEQ_BASE, type Heartbeat, type HeartbeatSighting, type HeartbeatStatus, type HistoryEvent, InMemoryAdmissionBackend, InMemoryRunQueue, InMemoryStateStore, InMemoryTransport, type InferSearchAttributes, LEGACY_V1_CAPABILITIES, LEGACY_V1_PROTOCOL, type ListStalledRunsOptions, type MetricsCollector, type MetricsSnapshot, type NamedTransport, type NegotiateOptions, type NegotiationOutcome, type NegotiationReason, type NegotiationResult, NestedWorkflowCallError, NonDeterminismError, type OnErrorInput, type OnErrorRoute, type OwnedLeaseReclaim, type PartitionSighting, type PayloadCodec, type QueueConfig, QueueController, REMOTE_EVENT, RUN_HISTORY_FIXTURE_VERSION, RUN_VALUE_FACET_LIMIT, RUN_VALUE_FACET_SCAN, type RawWorkerDescriptor, RemoteStepError, RemoteStepTimeout, type RemoteTask, RemoteWorkflowExecutor, RemoteWorkflowTimeout, type RetentionPolicy, type RoutableDispatch, type RoutableResolution, type RoutingResolution, type RunAttributeRow, type RunConsumerOptions, type RunDetail, type RunDetailOptions, type RunDispatch, type RunDispatcher, type RunFacetQuery, type RunFacetRow, RunGateway, type RunListItem, type RunQuery, type RunQueueOptions, type RunReply, type RunReplyResult, type RunRequest, type RunRequestKind, type RunResult, type RunSchedulesOptions, type RunScope, type RunStatus, type RunValueAxis, type RunValueFacetOptions, type RunValueFacetOrder, type RunValueFacetRow, type RunWaiting, SINGLETON_ACTIVE_STATUSES, SINGLETON_ADMITTED_TAG, SINGLETON_RESERVED_TAGS, SINGLETON_TRACKED_TAG, STATE_STORE, STATE_STORE_CANONICAL, SUMMARY_CHILD_LIMIT, type ScalarAttributeFilter, ScheduleAlreadyExistsError, ScheduleClient, type ScheduleClientOptions, type ScheduleDescription, type ScheduleHost, type ScheduleListQuery, ScheduleNotFoundError, type ScheduleOptions, type ScheduleQuery, type ScheduleRecord, type ScheduledWorkflow, type SchemaIssue, type SchemaValidationCode, SchemaValidationError, type SearchAttributes, type SearchAttributesSchema, type Settled, SignalTimeoutError, type SignalWaiter, type SingletonConfig, SingletonQueueFullError, type StalePendingStep, type StalledRunInfo, StandardSchemaV1, type StartOptions, type StartRunMessage, type StateStore, type StepCheckpoint, type StepConfig, type StepDef, type StepDispatchOpts, type StepError, type StepEvent, type StepHandler, type StepInterceptor, type StepInvocation, type StepKind, type StepLogger, type StepOptions, type StepRef, type StepResult, type StepScope, type StepSettlement, type StepUndo, type StoreTransaction, type SubProcessHandle, TERMINAL_RUN_STATUSES, TRANSPORT, TRANSPORT_CANONICAL, type TenantEvent, type TenantTokenSource, type TenantVerifier, type Transport, type UndoOf, type UpdateResult, type UpdateValidator, VERSION_UNDECLARED_TAG, type VerifiedTenant, WORKFLOW_NAME_KEY, type WireDates, type WorkerAdjust, type WorkerConcurrencyStatus, type WorkerDescriptor, type WorkerHeartbeat, type WorkerLifecycle, type WorkerStatus, type WorkflowAnnouncement, WorkflowBlocked, type WorkflowClass, type WorkflowCommand, type WorkflowCtx, type WorkflowDecision, type WorkflowDirectory, WorkflowEngine, type WorkflowEngineDeps, type WorkflowExecutor, type WorkflowHandler, type WorkflowInputOf, WorkflowNotRegistered, type WorkflowOutputOf, type WorkflowRef, type WorkflowRegistration, type WorkflowRun, type WorkflowStartClient, type WorkflowStepEvent, WorkflowSuspended, type WorkflowTask, aggregateAnnouncements, aggregatesCheckpoints, announcementsOf, applyAttributeQuery, assertConcurrency, assertRoutes, attributeColumnFor, attributeOperand, attributePredicateOperands, axisIsRunColumn, bindWorkflowClass, breakpointToken, buildWorkflowDirectory, canRoute, capableWorkers, checkpointNameKind, checkpointStatsOf, childIdsOf, classifyWaiterToken, clearDurableExecutionWrappers, clearTerminalSingletonAdmission, codeAndDetails, codeOf, collectMetrics, concurrencyTag, controlPlaneDescriptor, countRuns, coveredTokens, createStepLogger, currentStep, currentStepScope, currentWorkflowCtx, debug, describeWorker, descriptorHash, detailsOf, error, etagMatches, eventMatchOf, eventMatches, eventNameOf, eventPrefix, eventToken, facetOrigin, fanPageBounds, fanPageEtag, findStalledRuns, foldedCount, getRunsByIds, handoffHandlerId, handoffName, hasDurableExecutionWrappers, heartbeatStatus, hmacTenantVerifier, indexWaitersByRun, info, isEngineMintedTag, isEventToken, isFailedMember, isLegacyDescriptor, isNonEmptyRunScope, isRemoteEvent, isWorkflowControlFlowSignal, legacySingletonHolders, listSignalWaitersOfRuns, liveHandoff, log, markRemoteEvent, matchesAttributes, mergeRunFacetRows, mergeRunValueFacetRows, missingCapabilities, negotiate, nextCronFireMs, normalizeAttributeRows, normalizeDescriptor, observedAnnouncements, onErrorRunId, pageFan, pageFanFromStore, parentRunIdOf, parseDuration, parseWebhookToken, partitionOfToken, planDispatch, poolNamespace, prevCronFireMs, queryCheckpointsOf, readRunsByIds, readSearchAttributes, requiredCapabilities, resolveConcurrency, resolveRouting, resolveRunWaiting, resolveTenantToken, resolveWireTenant, retiredHandoffName, routeFor, runDetailEtag, runDurableExecution, runInStepLogger, runInStepScope, runInWorkflowCtx, runOutsideWorkflowCtx, runSchedules, runStepHandler, runValueFacetsFromRuns, sanitizeQueueToken, scheduleRunIdAt, scheduledRunId, signTenantToken, singletonAdmissionDenialPatch, singletonAdmissionTags, sqlComparator, stepConfigOf, stepId, stepNameOf, sub, subEvent, subProcess, summarizeFan, summarizeRunFromStore, summarizeTimeline, tenantGroup, useDurableExecution, warn, withSingletonTracked, withoutSingletonMarkers, workflowName };
|
package/dist/index.js
CHANGED
|
@@ -4814,6 +4814,8 @@ var ANNOUNCED_ROUTES_TTL_MS = 5e3;
|
|
|
4814
4814
|
var VERSION_UNDECLARED_TAG = "version:undeclared";
|
|
4815
4815
|
var versionKey = /* @__PURE__ */ __name((name, version) => `${name}@${version}`, "versionKey");
|
|
4816
4816
|
var STEP_FAN_CHUNK = 200;
|
|
4817
|
+
var STEP_EVENTS_PER_BROADCAST = 500;
|
|
4818
|
+
var isStepEvent = /* @__PURE__ */ __name((event) => event.type.startsWith("step."), "isStepEvent");
|
|
4817
4819
|
var FAN_REREAD_CHUNK = 1e3;
|
|
4818
4820
|
var STEP_FAN_SEND_ROUND = 2e3;
|
|
4819
4821
|
var PARENT_NOTIFY_ATTEMPTS = 4;
|
|
@@ -4947,6 +4949,22 @@ var NamespaceMismatch = class NamespaceMismatch2 extends Error {
|
|
|
4947
4949
|
super("namespace-mismatch");
|
|
4948
4950
|
}
|
|
4949
4951
|
};
|
|
4952
|
+
var WorkflowNotRegistered = class extends Error {
|
|
4953
|
+
static {
|
|
4954
|
+
__name(this, "WorkflowNotRegistered");
|
|
4955
|
+
}
|
|
4956
|
+
constructor(workflow, version) {
|
|
4957
|
+
super(`workflow ${workflow}@${version} is not registered \u2014 keep the prior version deployed so in-flight runs can drain (skew protection)`);
|
|
4958
|
+
this.name = "WorkflowNotRegistered";
|
|
4959
|
+
}
|
|
4960
|
+
};
|
|
4961
|
+
var UNREGISTERED_DEAD_LETTER_FROM = [
|
|
4962
|
+
"pending",
|
|
4963
|
+
"running",
|
|
4964
|
+
"suspended",
|
|
4965
|
+
"cancelling",
|
|
4966
|
+
"blocked"
|
|
4967
|
+
];
|
|
4950
4968
|
var WorkflowEngine = class {
|
|
4951
4969
|
static {
|
|
4952
4970
|
__name(this, "WorkflowEngine");
|
|
@@ -4966,6 +4984,7 @@ var WorkflowEngine = class {
|
|
|
4966
4984
|
*/
|
|
4967
4985
|
ownLeaseCutoffMs;
|
|
4968
4986
|
maxRecoveryAttempts;
|
|
4987
|
+
unregisteredWorkflowGraceMs;
|
|
4969
4988
|
remoteAdvanceSilenceMs;
|
|
4970
4989
|
reconcileMs;
|
|
4971
4990
|
timerBatchSize;
|
|
@@ -5152,6 +5171,8 @@ var WorkflowEngine = class {
|
|
|
5152
5171
|
this.leaseMs = deps.leaseMs ?? 3e4;
|
|
5153
5172
|
this.ownLeaseCutoffMs = this.clock() + this.leaseMs;
|
|
5154
5173
|
this.maxRecoveryAttempts = deps.maxRecoveryAttempts;
|
|
5174
|
+
const grace = deps.unregisteredWorkflowGraceMs;
|
|
5175
|
+
this.unregisteredWorkflowGraceMs = grace !== void 0 && Number.isFinite(grace) && grace >= 0 ? grace : 24 * 60 * 60 * 1e3;
|
|
5155
5176
|
this.remoteAdvanceSilenceMs = deps.remoteAdvanceSilenceMs;
|
|
5156
5177
|
this.reconcileMs = deps.reconcileMs === void 0 ? 3e5 : deps.reconcileMs || void 0;
|
|
5157
5178
|
const stalledAfter = deps.stalledAfterMs;
|
|
@@ -5284,6 +5305,13 @@ var WorkflowEngine = class {
|
|
|
5284
5305
|
...msg.event,
|
|
5285
5306
|
at: new Date(msg.event.at)
|
|
5286
5307
|
}));
|
|
5308
|
+
} else if (msg.kind === "events") {
|
|
5309
|
+
for (const event of msg.events) {
|
|
5310
|
+
this.deliver(markRemoteEvent({
|
|
5311
|
+
...event,
|
|
5312
|
+
at: new Date(event.at)
|
|
5313
|
+
}));
|
|
5314
|
+
}
|
|
5287
5315
|
} else if (msg.kind === "cancel") {
|
|
5288
5316
|
this.notifyCancelled(msg.runId);
|
|
5289
5317
|
} else if (msg.kind === "enqueued") {
|
|
@@ -5461,13 +5489,19 @@ var WorkflowEngine = class {
|
|
|
5461
5489
|
* that is lost brings it back due after that period. False when the transport refused the wakes,
|
|
5462
5490
|
* so the caller resumes them itself.
|
|
5463
5491
|
*/
|
|
5464
|
-
async handOffDue(runs, nowMs) {
|
|
5492
|
+
async handOffDue(runs, nowMs, settled) {
|
|
5465
5493
|
const claimed = [];
|
|
5466
5494
|
for (const run of runs) {
|
|
5467
5495
|
if (this.draining) break;
|
|
5468
5496
|
if (!await this.store.tryLockRun(run.id, this.instanceId, nowMs + this.leaseMs, nowMs)) {
|
|
5469
5497
|
continue;
|
|
5470
5498
|
}
|
|
5499
|
+
if (!this.isRegisteredHere(run) && !await this.canResolveWorkflow(run)) {
|
|
5500
|
+
const result = await this.settleUnregistered(run, nowMs);
|
|
5501
|
+
if (result) settled?.set(run.id, result);
|
|
5502
|
+
else if (result === void 0) this.logUnregisteredSkip(run);
|
|
5503
|
+
continue;
|
|
5504
|
+
}
|
|
5471
5505
|
try {
|
|
5472
5506
|
const patch = {
|
|
5473
5507
|
wakeAt: nowMs + this.leaseMs
|
|
@@ -5865,13 +5899,53 @@ var WorkflowEngine = class {
|
|
|
5865
5899
|
at: /* @__PURE__ */ new Date()
|
|
5866
5900
|
};
|
|
5867
5901
|
this.deliver(full);
|
|
5868
|
-
if (this.controlPlane)
|
|
5869
|
-
|
|
5870
|
-
|
|
5871
|
-
|
|
5872
|
-
|
|
5873
|
-
|
|
5874
|
-
|
|
5902
|
+
if (!this.controlPlane) return;
|
|
5903
|
+
this.eventsToBroadcast.push(full);
|
|
5904
|
+
if (!this.broadcastingEvents) void this.broadcastEvents();
|
|
5905
|
+
}
|
|
5906
|
+
/** Events waiting for the control-plane publish in flight, in emit order (see {@link broadcastEvents}). */
|
|
5907
|
+
eventsToBroadcast = [];
|
|
5908
|
+
broadcastingEvents = false;
|
|
5909
|
+
/**
|
|
5910
|
+
* Publish the queued events in the order they were emitted. A run's own events go out one per
|
|
5911
|
+
* message, as every instance expects them. Step events (a wide fan emits two per step) that queued
|
|
5912
|
+
* up while a publish was in flight go out together in one `events` message, so a fan of 40,000
|
|
5913
|
+
* steps costs the control plane a few hundred messages to every instance instead of 80,000.
|
|
5914
|
+
*/
|
|
5915
|
+
async broadcastEvents() {
|
|
5916
|
+
this.broadcastingEvents = true;
|
|
5917
|
+
try {
|
|
5918
|
+
while (this.eventsToBroadcast.length > 0 && this.controlPlane) {
|
|
5919
|
+
const first = this.eventsToBroadcast[0];
|
|
5920
|
+
let msg;
|
|
5921
|
+
if (isStepEvent(first)) {
|
|
5922
|
+
let n = 1;
|
|
5923
|
+
while (n < STEP_EVENTS_PER_BROADCAST && n < this.eventsToBroadcast.length && isStepEvent(this.eventsToBroadcast[n])) {
|
|
5924
|
+
n += 1;
|
|
5925
|
+
}
|
|
5926
|
+
const events = this.eventsToBroadcast.splice(0, n);
|
|
5927
|
+
msg = events.length === 1 ? {
|
|
5928
|
+
kind: "event",
|
|
5929
|
+
event: first,
|
|
5930
|
+
from: this.instanceId
|
|
5931
|
+
} : {
|
|
5932
|
+
kind: "events",
|
|
5933
|
+
events,
|
|
5934
|
+
from: this.instanceId
|
|
5935
|
+
};
|
|
5936
|
+
} else {
|
|
5937
|
+
this.eventsToBroadcast.shift();
|
|
5938
|
+
msg = {
|
|
5939
|
+
kind: "event",
|
|
5940
|
+
event: first,
|
|
5941
|
+
from: this.instanceId
|
|
5942
|
+
};
|
|
5943
|
+
}
|
|
5944
|
+
await this.controlPlane.publishControl(msg).catch(() => {
|
|
5945
|
+
});
|
|
5946
|
+
}
|
|
5947
|
+
} finally {
|
|
5948
|
+
this.broadcastingEvents = false;
|
|
5875
5949
|
}
|
|
5876
5950
|
}
|
|
5877
5951
|
/** Deliver an event to local subscribers only (no re-broadcast) — used for both locally-produced
|
|
@@ -6201,9 +6275,7 @@ var WorkflowEngine = class {
|
|
|
6201
6275
|
});
|
|
6202
6276
|
}
|
|
6203
6277
|
const registered = this.workflows.get(versionKey(run.workflow, run.workflowVersion)) ?? await this.findInheritedRegistration(run) ?? await this.resolveRemoteByConvention(run);
|
|
6204
|
-
if (!registered)
|
|
6205
|
-
throw new Error(`workflow ${run.workflow}@${run.workflowVersion} is not registered \u2014 keep the prior version deployed so in-flight runs can drain (skew protection)`);
|
|
6206
|
-
}
|
|
6278
|
+
if (!registered) throw new WorkflowNotRegistered(run.workflow, run.workflowVersion);
|
|
6207
6279
|
return this.track(this.execute(run, registered.fn));
|
|
6208
6280
|
}
|
|
6209
6281
|
/**
|
|
@@ -6618,6 +6690,15 @@ var WorkflowEngine = class {
|
|
|
6618
6690
|
};
|
|
6619
6691
|
}
|
|
6620
6692
|
}
|
|
6693
|
+
if (!this.isRegisteredHere(run) && !await this.canResolveWorkflow(run)) {
|
|
6694
|
+
const settled2 = await this.settleUnregistered(run, nowMs);
|
|
6695
|
+
if (settled2) return {
|
|
6696
|
+
result: settled2,
|
|
6697
|
+
dispatch: false
|
|
6698
|
+
};
|
|
6699
|
+
if (settled2 === void 0) this.logUnregisteredSkip(run);
|
|
6700
|
+
return void 0;
|
|
6701
|
+
}
|
|
6621
6702
|
const settled = await this.countRecovery(run);
|
|
6622
6703
|
if (settled) return {
|
|
6623
6704
|
result: settled,
|
|
@@ -6689,25 +6770,10 @@ var WorkflowEngine = class {
|
|
|
6689
6770
|
updatedAt: /* @__PURE__ */ new Date()
|
|
6690
6771
|
});
|
|
6691
6772
|
await this.store.releaseRunLock(run.id);
|
|
6692
|
-
this.
|
|
6693
|
-
type: "run.failed",
|
|
6694
|
-
runId: run.id,
|
|
6695
|
-
workflow: run.workflow,
|
|
6696
|
-
namespace: run.namespace,
|
|
6697
|
-
error: error2
|
|
6698
|
-
});
|
|
6699
|
-
this.notifyDead({
|
|
6773
|
+
return this.announceDead({
|
|
6700
6774
|
...run,
|
|
6701
|
-
status: "dead",
|
|
6702
|
-
error: error2,
|
|
6703
6775
|
recoveryAttempts: attempts
|
|
6704
|
-
});
|
|
6705
|
-
this.notifyParent(run.id, failedCompletion(error2));
|
|
6706
|
-
return {
|
|
6707
|
-
runId: run.id,
|
|
6708
|
-
status: "dead",
|
|
6709
|
-
error: error2
|
|
6710
|
-
};
|
|
6776
|
+
}, error2);
|
|
6711
6777
|
}
|
|
6712
6778
|
await this.store.updateRun(run.id, {
|
|
6713
6779
|
recoveryAttempts: attempts,
|
|
@@ -6715,6 +6781,85 @@ var WorkflowEngine = class {
|
|
|
6715
6781
|
});
|
|
6716
6782
|
return void 0;
|
|
6717
6783
|
}
|
|
6784
|
+
/** Tell everyone a run was just written `dead`: events, dead-letter routing, a waiting parent. */
|
|
6785
|
+
announceDead(run, error2) {
|
|
6786
|
+
this.emit({
|
|
6787
|
+
type: "run.failed",
|
|
6788
|
+
runId: run.id,
|
|
6789
|
+
workflow: run.workflow,
|
|
6790
|
+
namespace: run.namespace,
|
|
6791
|
+
error: error2
|
|
6792
|
+
});
|
|
6793
|
+
this.notifyDead({
|
|
6794
|
+
...run,
|
|
6795
|
+
status: "dead",
|
|
6796
|
+
error: error2
|
|
6797
|
+
});
|
|
6798
|
+
this.notifyParent(run.id, failedCompletion(error2));
|
|
6799
|
+
return {
|
|
6800
|
+
runId: run.id,
|
|
6801
|
+
status: "dead",
|
|
6802
|
+
error: error2
|
|
6803
|
+
};
|
|
6804
|
+
}
|
|
6805
|
+
/** The synchronous half of {@link canResolveWorkflow}: callers check it first, so a registered run
|
|
6806
|
+
* pays no await (same rule as {@link resume}). */
|
|
6807
|
+
isRegisteredHere(run) {
|
|
6808
|
+
return this.workflows.has(versionKey(run.workflow, run.workflowVersion));
|
|
6809
|
+
}
|
|
6810
|
+
/**
|
|
6811
|
+
* Whether this instance can run `run`'s pinned `workflow@version`: registered here, inherited from a
|
|
6812
|
+
* remote ancestor, or routed by convention to a live worker group (the lookup {@link resume} does).
|
|
6813
|
+
*/
|
|
6814
|
+
async canResolveWorkflow(run) {
|
|
6815
|
+
if (this.isRegisteredHere(run)) return true;
|
|
6816
|
+
return await this.findInheritedRegistration(run) !== void 0 || await this.resolveRemoteByConvention(run) !== void 0;
|
|
6817
|
+
}
|
|
6818
|
+
/**
|
|
6819
|
+
* A run whose lease this instance holds but whose `workflow@version` nothing here can run (see
|
|
6820
|
+
* {@link WorkflowNotRegistered}). Within {@link unregisteredWorkflowGraceMs} of becoming due it is a
|
|
6821
|
+
* rolling deploy's skew: the lease is released at once so the pod that still has the version takes
|
|
6822
|
+
* the run now, not after the lease expires. Past it, the workflow is gone from the whole fleet (a
|
|
6823
|
+
* capable pod would have taken the run long ago): dead-letter it.
|
|
6824
|
+
*
|
|
6825
|
+
* The age is read off `updatedAt` / `wakeAt`, so every path that would touch either (a recovery
|
|
6826
|
+
* count, a re-pend, a hand-off's lease-long `wakeAt`) must call this BEFORE writing — otherwise an
|
|
6827
|
+
* orphan never ages.
|
|
6828
|
+
*
|
|
6829
|
+
* Returns the `dead` result, `null` when the run left its status meanwhile (a lease-free
|
|
6830
|
+
* `cancel()`: settled, not ours to touch), or `undefined` when it was left for later.
|
|
6831
|
+
*/
|
|
6832
|
+
async settleUnregistered(listed, nowMs) {
|
|
6833
|
+
const run = await this.store.getRun(listed.id);
|
|
6834
|
+
if (!run || !UNREGISTERED_DEAD_LETTER_FROM.includes(run.status)) {
|
|
6835
|
+
await this.store.releaseRunLock(listed.id);
|
|
6836
|
+
return null;
|
|
6837
|
+
}
|
|
6838
|
+
const dueSince = Math.max(run.updatedAt.getTime(), run.wakeAt ?? 0);
|
|
6839
|
+
if (this.unregisteredWorkflowGraceMs === 0 || nowMs - dueSince < this.unregisteredWorkflowGraceMs) {
|
|
6840
|
+
await this.store.releaseRunLock(run.id);
|
|
6841
|
+
return void 0;
|
|
6842
|
+
}
|
|
6843
|
+
const error2 = {
|
|
6844
|
+
message: `workflow ${run.workflow}@${run.workflowVersion} is not registered on any live instance (unresumable for over ${this.unregisteredWorkflowGraceMs}ms) \u2014 moved to dead-letter`,
|
|
6845
|
+
code: "workflow_not_registered"
|
|
6846
|
+
};
|
|
6847
|
+
const applied = await (this.store.updateRunIf ?? this.updateRunIfFallback).call(this.store, run.id, [
|
|
6848
|
+
run.status
|
|
6849
|
+
], {
|
|
6850
|
+
status: "dead",
|
|
6851
|
+
error: error2,
|
|
6852
|
+
updatedAt: /* @__PURE__ */ new Date()
|
|
6853
|
+
});
|
|
6854
|
+
await this.store.releaseRunLock(run.id);
|
|
6855
|
+
if (!applied) return null;
|
|
6856
|
+
console.warn(`[nestjs-durable] run ${run.id}: ${error2.message}`);
|
|
6857
|
+
return this.announceDead(run, error2);
|
|
6858
|
+
}
|
|
6859
|
+
/** Log a run skipped for an unregistered workflow — still there, still not progressing. */
|
|
6860
|
+
logUnregisteredSkip(run) {
|
|
6861
|
+
console.error(`[nestjs-durable] could not resume run ${run.id} (${run.workflow}@${run.workflowVersion}); leaving it for another instance:`, new WorkflowNotRegistered(run.workflow, run.workflowVersion));
|
|
6862
|
+
}
|
|
6718
6863
|
/**
|
|
6719
6864
|
* Resume every suspended run whose durable timer is due. Call periodically (a poller) and on
|
|
6720
6865
|
* boot. A run still not due re-suspends cheaply without running new work.
|
|
@@ -6739,11 +6884,12 @@ var WorkflowEngine = class {
|
|
|
6739
6884
|
...due,
|
|
6740
6885
|
...blocked
|
|
6741
6886
|
];
|
|
6742
|
-
|
|
6743
|
-
|
|
6887
|
+
const settled = /* @__PURE__ */ new Map();
|
|
6888
|
+
if (await this.handOffDue(runs, nowMs, settled)) {
|
|
6889
|
+
return runs.map((run) => settled.get(run.id) ?? {
|
|
6744
6890
|
runId: run.id,
|
|
6745
6891
|
status: run.status
|
|
6746
|
-
})
|
|
6892
|
+
});
|
|
6747
6893
|
}
|
|
6748
6894
|
}
|
|
6749
6895
|
const waiterByRun = indexWaitersByRun(await listSignalWaitersOfRuns(this.store, [
|
|
@@ -6824,8 +6970,13 @@ var WorkflowEngine = class {
|
|
|
6824
6970
|
const nowMs = this.clock();
|
|
6825
6971
|
const acquired = await this.store.tryLockRun(runId, this.instanceId, nowMs + this.leaseMs, nowMs);
|
|
6826
6972
|
if (!acquired) return null;
|
|
6827
|
-
return this.resumeImplicit(runId).catch((err) => {
|
|
6973
|
+
return this.resumeImplicit(runId).catch(async (err) => {
|
|
6828
6974
|
if (err instanceof NamespaceMismatch) return null;
|
|
6975
|
+
if (err instanceof WorkflowNotRegistered) {
|
|
6976
|
+
const run = await this.store.getRun(runId);
|
|
6977
|
+
const settled = run ? await this.settleUnregistered(run, nowMs) : null;
|
|
6978
|
+
if (settled !== void 0) return settled;
|
|
6979
|
+
}
|
|
6829
6980
|
throw err;
|
|
6830
6981
|
});
|
|
6831
6982
|
}
|
|
@@ -7080,7 +7231,8 @@ var WorkflowEngine = class {
|
|
|
7080
7231
|
* Resume each run only if this instance can acquire its recovery lease — so when several
|
|
7081
7232
|
* replicas recover or poll at once, each run is picked up by exactly one of them.
|
|
7082
7233
|
*
|
|
7083
|
-
* A run this instance cannot resume is SKIPPED, not allowed to abort the batch
|
|
7234
|
+
* A run this instance cannot resume is SKIPPED, not allowed to abort the batch (a run whose workflow
|
|
7235
|
+
* stays registered nowhere past `unregisteredWorkflowGraceMs` is dead-lettered instead). The canonical
|
|
7084
7236
|
* case is skew protection: mid-rolling-deploy a pod leases a run whose workflow it does not have
|
|
7085
7237
|
* and `resume` throws by design — but the rest of the batch is runs it can drive perfectly well,
|
|
7086
7238
|
* and every caller here is a recovery sweep. Letting the first such run end the loop means one
|
|
@@ -7104,6 +7256,11 @@ var WorkflowEngine = class {
|
|
|
7104
7256
|
const settled = onLocked ? await onLocked(run) : void 0;
|
|
7105
7257
|
results.push(settled ?? await this.resumeImplicit(run.id));
|
|
7106
7258
|
} catch (error2) {
|
|
7259
|
+
if (error2 instanceof WorkflowNotRegistered) {
|
|
7260
|
+
const settled = await this.settleUnregistered(run, nowMs).catch(() => void 0);
|
|
7261
|
+
if (settled) results.push(settled);
|
|
7262
|
+
if (settled !== void 0) continue;
|
|
7263
|
+
}
|
|
7107
7264
|
console.error(`[nestjs-durable] could not resume run ${run.id} (${run.workflow}@${run.workflowVersion}); leaving it for another instance:`, error2);
|
|
7108
7265
|
}
|
|
7109
7266
|
}
|
|
@@ -12864,6 +13021,7 @@ export {
|
|
|
12864
13021
|
WORKFLOW_NAME_KEY,
|
|
12865
13022
|
WorkflowBlocked,
|
|
12866
13023
|
WorkflowEngine,
|
|
13024
|
+
WorkflowNotRegistered,
|
|
12867
13025
|
WorkflowSuspended,
|
|
12868
13026
|
aggregateAnnouncements,
|
|
12869
13027
|
aggregatesCheckpoints,
|