@dudousxd/nestjs-durable-core 0.85.11 → 0.86.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/index.d.cts CHANGED
@@ -902,7 +902,8 @@ type RunStatus =
902
902
  * are actually published (a legacy/pre-handshake fleet skips the guard, design §7.7).
903
903
  */
904
904
  | 'blocked' | 'completed' | 'failed' | 'cancelled'
905
- /** Dead-letter: recovery gave up after `maxRecoveryAttempts` (a poison pill). Terminal; inspect it. */
905
+ /** Dead-letter: recovery gave up after `maxRecoveryAttempts` (a poison pill), or no instance has
906
+ * the run's workflow version registered past `unregisteredWorkflowGraceMs`. Terminal; inspect it. */
906
907
  | 'dead';
907
908
  /** One execution of a workflow. The unit of durability and the unit shown in the dashboard. */
908
909
  interface WorkflowRun {
@@ -2628,6 +2629,9 @@ type ControlMessage = {
2628
2629
  } & ({
2629
2630
  kind: 'event';
2630
2631
  event: EngineEvent;
2632
+ } | {
2633
+ kind: 'events';
2634
+ events: EngineEvent[];
2631
2635
  } | {
2632
2636
  kind: 'cancel';
2633
2637
  runId: string;
@@ -4475,6 +4479,17 @@ interface WorkflowEngineDeps {
4475
4479
  * Omit for unlimited (the default — recovery always retries).
4476
4480
  */
4477
4481
  maxRecoveryAttempts?: number | undefined;
4482
+ /**
4483
+ * How long (ms) a run may stay due while NO instance can resume it because its workflow version is
4484
+ * not registered anywhere, before it is moved to `dead` (code `workflow_not_registered`). A rolling
4485
+ * deploy skips such a run on purpose (skew protection): the pod still on the prior version picks it
4486
+ * up within seconds. A run that stays unresumable far past that — its workflow was renamed or
4487
+ * deleted — would otherwise be retried and logged on every tick forever. The clock starts when the
4488
+ * run became due (its `wakeAt` for a parked run, its last update otherwise), so a long sleep is not
4489
+ * counted. Default 24h; `0` disables (skip forever). A non-finite or negative value falls back to
4490
+ * the default.
4491
+ */
4492
+ unregisteredWorkflowGraceMs?: number | undefined;
4478
4493
  /**
4479
4494
  * Opt-in liveness deadline (ms) for a remote workflow `advance`. If the worker neither returns a
4480
4495
  * decision NOR sends a run-scoped {@link Heartbeat} within this window, the engine presumes it dead:
@@ -4626,6 +4641,13 @@ interface ListStalledRunsOptions {
4626
4641
  /** Evaluate as of this clock time. Defaults to the engine clock. */
4627
4642
  now?: number | undefined;
4628
4643
  }
4644
+ /**
4645
+ * Thrown by {@link WorkflowEngine.resume} when nothing registered here (nor an inherited or
4646
+ * convention route) can run the run's pinned `workflow@version`.
4647
+ */
4648
+ declare class WorkflowNotRegistered extends Error {
4649
+ constructor(workflow: string, version: string);
4650
+ }
4629
4651
  /**
4630
4652
  * The orchestrator. Owns workflow state and replays runs deterministically: each step's
4631
4653
  * result is checkpointed, so on resume a completed step returns its saved output instead of
@@ -4648,6 +4670,7 @@ declare class WorkflowEngine {
4648
4670
  */
4649
4671
  private readonly ownLeaseCutoffMs;
4650
4672
  private readonly maxRecoveryAttempts?;
4673
+ private readonly unregisteredWorkflowGraceMs;
4651
4674
  private readonly remoteAdvanceSilenceMs?;
4652
4675
  private readonly reconcileMs;
4653
4676
  private readonly timerBatchSize;
@@ -5089,6 +5112,16 @@ declare class WorkflowEngine {
5089
5112
  /** Emit a locally-produced lifecycle event: deliver to subscribers AND broadcast it on the
5090
5113
  * control plane so other instances (e.g. a dashboard pod) can live-tail this run. */
5091
5114
  private emit;
5115
+ /** Events waiting for the control-plane publish in flight, in emit order (see {@link broadcastEvents}). */
5116
+ private readonly eventsToBroadcast;
5117
+ private broadcastingEvents;
5118
+ /**
5119
+ * Publish the queued events in the order they were emitted. A run's own events go out one per
5120
+ * message, as every instance expects them. Step events (a wide fan emits two per step) that queued
5121
+ * up while a publish was in flight go out together in one `events` message, so a fan of 40,000
5122
+ * steps costs the control plane a few hundred messages to every instance instead of 80,000.
5123
+ */
5124
+ private broadcastEvents;
5092
5125
  /** Deliver an event to local subscribers only (no re-broadcast) — used for both locally-produced
5093
5126
  * events and ones received from the control plane, so an event shows up once on every instance. */
5094
5127
  private deliver;
@@ -5359,6 +5392,33 @@ declare class WorkflowEngine {
5359
5392
  * result to skip the resume, or `undefined` to proceed.
5360
5393
  */
5361
5394
  private countRecovery;
5395
+ /** Tell everyone a run was just written `dead`: events, dead-letter routing, a waiting parent. */
5396
+ private announceDead;
5397
+ /** The synchronous half of {@link canResolveWorkflow}: callers check it first, so a registered run
5398
+ * pays no await (same rule as {@link resume}). */
5399
+ private isRegisteredHere;
5400
+ /**
5401
+ * Whether this instance can run `run`'s pinned `workflow@version`: registered here, inherited from a
5402
+ * remote ancestor, or routed by convention to a live worker group (the lookup {@link resume} does).
5403
+ */
5404
+ private canResolveWorkflow;
5405
+ /**
5406
+ * A run whose lease this instance holds but whose `workflow@version` nothing here can run (see
5407
+ * {@link WorkflowNotRegistered}). Within {@link unregisteredWorkflowGraceMs} of becoming due it is a
5408
+ * rolling deploy's skew: the lease is released at once so the pod that still has the version takes
5409
+ * the run now, not after the lease expires. Past it, the workflow is gone from the whole fleet (a
5410
+ * capable pod would have taken the run long ago): dead-letter it.
5411
+ *
5412
+ * The age is read off `updatedAt` / `wakeAt`, so every path that would touch either (a recovery
5413
+ * count, a re-pend, a hand-off's lease-long `wakeAt`) must call this BEFORE writing — otherwise an
5414
+ * orphan never ages.
5415
+ *
5416
+ * Returns the `dead` result, `null` when the run left its status meanwhile (a lease-free
5417
+ * `cancel()`: settled, not ours to touch), or `undefined` when it was left for later.
5418
+ */
5419
+ private settleUnregistered;
5420
+ /** Log a run skipped for an unregistered workflow — still there, still not progressing. */
5421
+ private logUnregisteredSkip;
5362
5422
  /**
5363
5423
  * Resume every suspended run whose durable timer is due. Call periodically (a poller) and on
5364
5424
  * boot. A run still not due re-suspends cheaply without running new work.
@@ -5460,7 +5520,8 @@ declare class WorkflowEngine {
5460
5520
  * Resume each run only if this instance can acquire its recovery lease — so when several
5461
5521
  * replicas recover or poll at once, each run is picked up by exactly one of them.
5462
5522
  *
5463
- * A run this instance cannot resume is SKIPPED, not allowed to abort the batch. The canonical
5523
+ * A run this instance cannot resume is SKIPPED, not allowed to abort the batch (a run whose workflow
5524
+ * stays registered nowhere past `unregisteredWorkflowGraceMs` is dead-lettered instead). The canonical
5464
5525
  * case is skew protection: mid-rolling-deploy a pod leases a run whose workflow it does not have
5465
5526
  * and `resume` throws by design — but the rest of the batch is runs it can drive perfectly well,
5466
5527
  * and every caller here is a recovery sweep. Letting the first such run end the loop means one
@@ -7843,4 +7904,4 @@ declare function singletonAdmissionDenialPatch(runs: WorkflowRun[], runId: strin
7843
7904
  updatedAt: Date;
7844
7905
  } | undefined;
7845
7906
 
7846
- export { type AccumulatorHost, type Admission, type AdmissionBackend, type AdmissionItem, type AllOptions, type AllSettledOptions, type AllStepsOptions, type AllStepsSettledOptions, type AnnouncedWorkflow, type AnnouncementEvidence, type AttributeFilter, type AttributeOp, type AttributePredicateOperand, type AttributeValue, type BackoffStrategy, type BlockedDispatch, type BlockedResolution, type BufferedSignalRow, CHILD_EDGE_PREFIXES, CONTROL_FLOW_SIGNAL, CORE_ORIGIN, CURRENT_PROTOCOL_VERSION, type CapabilityRequirement, type CheckpointAggregatingStore, type CheckpointNameKind, type CheckpointQuery, type CheckpointStat, type ChildCallOptions, type ChildWait, CodecStateStore, type CodecStateStoreOptions, type ConcurrencyConfig, ConcurrencyLimitError, type ConcurrencyQuota, ContinueAsNew, type ControlMessage, type ControlPlane, DEFAULT_CONCURRENCY_STATUSES, DEFAULT_PARTITION, DEFAULT_STALLED_AFTER_MS, DURABLE_OPTIONS, DURABLE_OPTIONS_CANONICAL, DURABLE_STEP_CONFIG, DURABLE_STEP_NAME, type Disagreement, type DispatchDiagnostics, type DispatchPlan, type DurableExecution, type DurableExecutionWrapper, type DurableTopology, type DurableWebhook, DurableWorkflow, type DynamicWorkflowCtx, ENGINE_MINTED_TAG_PREFIXES, ENTITY_WORKFLOW, EVENT_BUFFER_SCAN_LIMIT, EVENT_TOKEN_PREFIX, type EngineEvent, type EngineEventType, type EngineListener, Entities, type EntityConfig, type EntityHandler, type EntityHost, type EntityWorkflowHost, EventAccumulators, type EventBatchConfig, FAN_SUMMARY_FAILURES, type FanJoin, type FanPage, type FanPageQuery, type FanSummary, type FanWait, FatalError, type FindStalledRunsOptions, GatherError, type GatherItemFailure, type GroupHealth, HANDOFF_SEQ_BASE, type Heartbeat, type HeartbeatSighting, type HeartbeatStatus, type HistoryEvent, InMemoryAdmissionBackend, InMemoryRunQueue, InMemoryStateStore, InMemoryTransport, type InferSearchAttributes, LEGACY_V1_CAPABILITIES, LEGACY_V1_PROTOCOL, type ListStalledRunsOptions, type MetricsCollector, type MetricsSnapshot, type NamedTransport, type NegotiateOptions, type NegotiationOutcome, type NegotiationReason, type NegotiationResult, NestedWorkflowCallError, NonDeterminismError, type OnErrorInput, type OnErrorRoute, type OwnedLeaseReclaim, type PartitionSighting, type PayloadCodec, type QueueConfig, QueueController, REMOTE_EVENT, RUN_HISTORY_FIXTURE_VERSION, RUN_VALUE_FACET_LIMIT, RUN_VALUE_FACET_SCAN, type RawWorkerDescriptor, RemoteStepError, RemoteStepTimeout, type RemoteTask, RemoteWorkflowExecutor, RemoteWorkflowTimeout, type RetentionPolicy, type RoutableDispatch, type RoutableResolution, type RoutingResolution, type RunAttributeRow, type RunConsumerOptions, type RunDetail, type RunDetailOptions, type RunDispatch, type RunDispatcher, type RunFacetQuery, type RunFacetRow, RunGateway, type RunListItem, type RunQuery, type RunQueueOptions, type RunReply, type RunReplyResult, type RunRequest, type RunRequestKind, type RunResult, type RunSchedulesOptions, type RunScope, type RunStatus, type RunValueAxis, type RunValueFacetOptions, type RunValueFacetOrder, type RunValueFacetRow, type RunWaiting, SINGLETON_ACTIVE_STATUSES, SINGLETON_ADMITTED_TAG, SINGLETON_RESERVED_TAGS, SINGLETON_TRACKED_TAG, STATE_STORE, STATE_STORE_CANONICAL, SUMMARY_CHILD_LIMIT, type ScalarAttributeFilter, ScheduleAlreadyExistsError, ScheduleClient, type ScheduleClientOptions, type ScheduleDescription, type ScheduleHost, type ScheduleListQuery, ScheduleNotFoundError, type ScheduleOptions, type ScheduleQuery, type ScheduleRecord, type ScheduledWorkflow, type SchemaIssue, type SchemaValidationCode, SchemaValidationError, type SearchAttributes, type SearchAttributesSchema, type Settled, SignalTimeoutError, type SignalWaiter, type SingletonConfig, SingletonQueueFullError, type StalePendingStep, type StalledRunInfo, StandardSchemaV1, type StartOptions, type StartRunMessage, type StateStore, type StepCheckpoint, type StepConfig, type StepDef, type StepDispatchOpts, type StepError, type StepEvent, type StepHandler, type StepInterceptor, type StepInvocation, type StepKind, type StepLogger, type StepOptions, type StepRef, type StepResult, type StepScope, type StepSettlement, type StepUndo, type StoreTransaction, type SubProcessHandle, TERMINAL_RUN_STATUSES, TRANSPORT, TRANSPORT_CANONICAL, type TenantEvent, type TenantTokenSource, type TenantVerifier, type Transport, type UndoOf, type UpdateResult, type UpdateValidator, VERSION_UNDECLARED_TAG, type VerifiedTenant, WORKFLOW_NAME_KEY, type WireDates, type WorkerAdjust, type WorkerConcurrencyStatus, type WorkerDescriptor, type WorkerHeartbeat, type WorkerLifecycle, type WorkerStatus, type WorkflowAnnouncement, WorkflowBlocked, type WorkflowClass, type WorkflowCommand, type WorkflowCtx, type WorkflowDecision, type WorkflowDirectory, WorkflowEngine, type WorkflowEngineDeps, type WorkflowExecutor, type WorkflowHandler, type WorkflowInputOf, type WorkflowOutputOf, type WorkflowRef, type WorkflowRegistration, type WorkflowRun, type WorkflowStartClient, type WorkflowStepEvent, WorkflowSuspended, type WorkflowTask, aggregateAnnouncements, aggregatesCheckpoints, announcementsOf, applyAttributeQuery, assertConcurrency, assertRoutes, attributeColumnFor, attributeOperand, attributePredicateOperands, axisIsRunColumn, bindWorkflowClass, breakpointToken, buildWorkflowDirectory, canRoute, capableWorkers, checkpointNameKind, checkpointStatsOf, childIdsOf, classifyWaiterToken, clearDurableExecutionWrappers, clearTerminalSingletonAdmission, codeAndDetails, codeOf, collectMetrics, concurrencyTag, controlPlaneDescriptor, countRuns, coveredTokens, createStepLogger, currentStep, currentStepScope, currentWorkflowCtx, debug, describeWorker, descriptorHash, detailsOf, error, etagMatches, eventMatchOf, eventMatches, eventNameOf, eventPrefix, eventToken, facetOrigin, fanPageBounds, fanPageEtag, findStalledRuns, foldedCount, getRunsByIds, handoffHandlerId, handoffName, hasDurableExecutionWrappers, heartbeatStatus, hmacTenantVerifier, indexWaitersByRun, info, isEngineMintedTag, isEventToken, isFailedMember, isLegacyDescriptor, isNonEmptyRunScope, isRemoteEvent, isWorkflowControlFlowSignal, legacySingletonHolders, listSignalWaitersOfRuns, liveHandoff, log, markRemoteEvent, matchesAttributes, mergeRunFacetRows, mergeRunValueFacetRows, missingCapabilities, negotiate, nextCronFireMs, normalizeAttributeRows, normalizeDescriptor, observedAnnouncements, onErrorRunId, pageFan, pageFanFromStore, parentRunIdOf, parseDuration, parseWebhookToken, partitionOfToken, planDispatch, poolNamespace, prevCronFireMs, queryCheckpointsOf, readRunsByIds, readSearchAttributes, requiredCapabilities, resolveConcurrency, resolveRouting, resolveRunWaiting, resolveTenantToken, resolveWireTenant, retiredHandoffName, routeFor, runDetailEtag, runDurableExecution, runInStepLogger, runInStepScope, runInWorkflowCtx, runOutsideWorkflowCtx, runSchedules, runStepHandler, runValueFacetsFromRuns, sanitizeQueueToken, scheduleRunIdAt, scheduledRunId, signTenantToken, singletonAdmissionDenialPatch, singletonAdmissionTags, sqlComparator, stepConfigOf, stepId, stepNameOf, sub, subEvent, subProcess, summarizeFan, summarizeRunFromStore, summarizeTimeline, tenantGroup, useDurableExecution, warn, withSingletonTracked, withoutSingletonMarkers, workflowName };
7907
+ export { type AccumulatorHost, type Admission, type AdmissionBackend, type AdmissionItem, type AllOptions, type AllSettledOptions, type AllStepsOptions, type AllStepsSettledOptions, type AnnouncedWorkflow, type AnnouncementEvidence, type AttributeFilter, type AttributeOp, type AttributePredicateOperand, type AttributeValue, type BackoffStrategy, type BlockedDispatch, type BlockedResolution, type BufferedSignalRow, CHILD_EDGE_PREFIXES, CONTROL_FLOW_SIGNAL, CORE_ORIGIN, CURRENT_PROTOCOL_VERSION, type CapabilityRequirement, type CheckpointAggregatingStore, type CheckpointNameKind, type CheckpointQuery, type CheckpointStat, type ChildCallOptions, type ChildWait, CodecStateStore, type CodecStateStoreOptions, type ConcurrencyConfig, ConcurrencyLimitError, type ConcurrencyQuota, ContinueAsNew, type ControlMessage, type ControlPlane, DEFAULT_CONCURRENCY_STATUSES, DEFAULT_PARTITION, DEFAULT_STALLED_AFTER_MS, DURABLE_OPTIONS, DURABLE_OPTIONS_CANONICAL, DURABLE_STEP_CONFIG, DURABLE_STEP_NAME, type Disagreement, type DispatchDiagnostics, type DispatchPlan, type DurableExecution, type DurableExecutionWrapper, type DurableTopology, type DurableWebhook, DurableWorkflow, type DynamicWorkflowCtx, ENGINE_MINTED_TAG_PREFIXES, ENTITY_WORKFLOW, EVENT_BUFFER_SCAN_LIMIT, EVENT_TOKEN_PREFIX, type EngineEvent, type EngineEventType, type EngineListener, Entities, type EntityConfig, type EntityHandler, type EntityHost, type EntityWorkflowHost, EventAccumulators, type EventBatchConfig, FAN_SUMMARY_FAILURES, type FanJoin, type FanPage, type FanPageQuery, type FanSummary, type FanWait, FatalError, type FindStalledRunsOptions, GatherError, type GatherItemFailure, type GroupHealth, HANDOFF_SEQ_BASE, type Heartbeat, type HeartbeatSighting, type HeartbeatStatus, type HistoryEvent, InMemoryAdmissionBackend, InMemoryRunQueue, InMemoryStateStore, InMemoryTransport, type InferSearchAttributes, LEGACY_V1_CAPABILITIES, LEGACY_V1_PROTOCOL, type ListStalledRunsOptions, type MetricsCollector, type MetricsSnapshot, type NamedTransport, type NegotiateOptions, type NegotiationOutcome, type NegotiationReason, type NegotiationResult, NestedWorkflowCallError, NonDeterminismError, type OnErrorInput, type OnErrorRoute, type OwnedLeaseReclaim, type PartitionSighting, type PayloadCodec, type QueueConfig, QueueController, REMOTE_EVENT, RUN_HISTORY_FIXTURE_VERSION, RUN_VALUE_FACET_LIMIT, RUN_VALUE_FACET_SCAN, type RawWorkerDescriptor, RemoteStepError, RemoteStepTimeout, type RemoteTask, RemoteWorkflowExecutor, RemoteWorkflowTimeout, type RetentionPolicy, type RoutableDispatch, type RoutableResolution, type RoutingResolution, type RunAttributeRow, type RunConsumerOptions, type RunDetail, type RunDetailOptions, type RunDispatch, type RunDispatcher, type RunFacetQuery, type RunFacetRow, RunGateway, type RunListItem, type RunQuery, type RunQueueOptions, type RunReply, type RunReplyResult, type RunRequest, type RunRequestKind, type RunResult, type RunSchedulesOptions, type RunScope, type RunStatus, type RunValueAxis, type RunValueFacetOptions, type RunValueFacetOrder, type RunValueFacetRow, type RunWaiting, SINGLETON_ACTIVE_STATUSES, SINGLETON_ADMITTED_TAG, SINGLETON_RESERVED_TAGS, SINGLETON_TRACKED_TAG, STATE_STORE, STATE_STORE_CANONICAL, SUMMARY_CHILD_LIMIT, type ScalarAttributeFilter, ScheduleAlreadyExistsError, ScheduleClient, type ScheduleClientOptions, type ScheduleDescription, type ScheduleHost, type ScheduleListQuery, ScheduleNotFoundError, type ScheduleOptions, type ScheduleQuery, type ScheduleRecord, type ScheduledWorkflow, type SchemaIssue, type SchemaValidationCode, SchemaValidationError, type SearchAttributes, type SearchAttributesSchema, type Settled, SignalTimeoutError, type SignalWaiter, type SingletonConfig, SingletonQueueFullError, type StalePendingStep, type StalledRunInfo, StandardSchemaV1, type StartOptions, type StartRunMessage, type StateStore, type StepCheckpoint, type StepConfig, type StepDef, type StepDispatchOpts, type StepError, type StepEvent, type StepHandler, type StepInterceptor, type StepInvocation, type StepKind, type StepLogger, type StepOptions, type StepRef, type StepResult, type StepScope, type StepSettlement, type StepUndo, type StoreTransaction, type SubProcessHandle, TERMINAL_RUN_STATUSES, TRANSPORT, TRANSPORT_CANONICAL, type TenantEvent, type TenantTokenSource, type TenantVerifier, type Transport, type UndoOf, type UpdateResult, type UpdateValidator, VERSION_UNDECLARED_TAG, type VerifiedTenant, WORKFLOW_NAME_KEY, type WireDates, type WorkerAdjust, type WorkerConcurrencyStatus, type WorkerDescriptor, type WorkerHeartbeat, type WorkerLifecycle, type WorkerStatus, type WorkflowAnnouncement, WorkflowBlocked, type WorkflowClass, type WorkflowCommand, type WorkflowCtx, type WorkflowDecision, type WorkflowDirectory, WorkflowEngine, type WorkflowEngineDeps, type WorkflowExecutor, type WorkflowHandler, type WorkflowInputOf, WorkflowNotRegistered, type WorkflowOutputOf, type WorkflowRef, type WorkflowRegistration, type WorkflowRun, type WorkflowStartClient, type WorkflowStepEvent, WorkflowSuspended, type WorkflowTask, aggregateAnnouncements, aggregatesCheckpoints, announcementsOf, applyAttributeQuery, assertConcurrency, assertRoutes, attributeColumnFor, attributeOperand, attributePredicateOperands, axisIsRunColumn, bindWorkflowClass, breakpointToken, buildWorkflowDirectory, canRoute, capableWorkers, checkpointNameKind, checkpointStatsOf, childIdsOf, classifyWaiterToken, clearDurableExecutionWrappers, clearTerminalSingletonAdmission, codeAndDetails, codeOf, collectMetrics, concurrencyTag, controlPlaneDescriptor, countRuns, coveredTokens, createStepLogger, currentStep, currentStepScope, currentWorkflowCtx, debug, describeWorker, descriptorHash, detailsOf, error, etagMatches, eventMatchOf, eventMatches, eventNameOf, eventPrefix, eventToken, facetOrigin, fanPageBounds, fanPageEtag, findStalledRuns, foldedCount, getRunsByIds, handoffHandlerId, handoffName, hasDurableExecutionWrappers, heartbeatStatus, hmacTenantVerifier, indexWaitersByRun, info, isEngineMintedTag, isEventToken, isFailedMember, isLegacyDescriptor, isNonEmptyRunScope, isRemoteEvent, isWorkflowControlFlowSignal, legacySingletonHolders, listSignalWaitersOfRuns, liveHandoff, log, markRemoteEvent, matchesAttributes, mergeRunFacetRows, mergeRunValueFacetRows, missingCapabilities, negotiate, nextCronFireMs, normalizeAttributeRows, normalizeDescriptor, observedAnnouncements, onErrorRunId, pageFan, pageFanFromStore, parentRunIdOf, parseDuration, parseWebhookToken, partitionOfToken, planDispatch, poolNamespace, prevCronFireMs, queryCheckpointsOf, readRunsByIds, readSearchAttributes, requiredCapabilities, resolveConcurrency, resolveRouting, resolveRunWaiting, resolveTenantToken, resolveWireTenant, retiredHandoffName, routeFor, runDetailEtag, runDurableExecution, runInStepLogger, runInStepScope, runInWorkflowCtx, runOutsideWorkflowCtx, runSchedules, runStepHandler, runValueFacetsFromRuns, sanitizeQueueToken, scheduleRunIdAt, scheduledRunId, signTenantToken, singletonAdmissionDenialPatch, singletonAdmissionTags, sqlComparator, stepConfigOf, stepId, stepNameOf, sub, subEvent, subProcess, summarizeFan, summarizeRunFromStore, summarizeTimeline, tenantGroup, useDurableExecution, warn, withSingletonTracked, withoutSingletonMarkers, workflowName };
package/dist/index.d.ts CHANGED
@@ -902,7 +902,8 @@ type RunStatus =
902
902
  * are actually published (a legacy/pre-handshake fleet skips the guard, design §7.7).
903
903
  */
904
904
  | 'blocked' | 'completed' | 'failed' | 'cancelled'
905
- /** Dead-letter: recovery gave up after `maxRecoveryAttempts` (a poison pill). Terminal; inspect it. */
905
+ /** Dead-letter: recovery gave up after `maxRecoveryAttempts` (a poison pill), or no instance has
906
+ * the run's workflow version registered past `unregisteredWorkflowGraceMs`. Terminal; inspect it. */
906
907
  | 'dead';
907
908
  /** One execution of a workflow. The unit of durability and the unit shown in the dashboard. */
908
909
  interface WorkflowRun {
@@ -2628,6 +2629,9 @@ type ControlMessage = {
2628
2629
  } & ({
2629
2630
  kind: 'event';
2630
2631
  event: EngineEvent;
2632
+ } | {
2633
+ kind: 'events';
2634
+ events: EngineEvent[];
2631
2635
  } | {
2632
2636
  kind: 'cancel';
2633
2637
  runId: string;
@@ -4475,6 +4479,17 @@ interface WorkflowEngineDeps {
4475
4479
  * Omit for unlimited (the default — recovery always retries).
4476
4480
  */
4477
4481
  maxRecoveryAttempts?: number | undefined;
4482
+ /**
4483
+ * How long (ms) a run may stay due while NO instance can resume it because its workflow version is
4484
+ * not registered anywhere, before it is moved to `dead` (code `workflow_not_registered`). A rolling
4485
+ * deploy skips such a run on purpose (skew protection): the pod still on the prior version picks it
4486
+ * up within seconds. A run that stays unresumable far past that — its workflow was renamed or
4487
+ * deleted — would otherwise be retried and logged on every tick forever. The clock starts when the
4488
+ * run became due (its `wakeAt` for a parked run, its last update otherwise), so a long sleep is not
4489
+ * counted. Default 24h; `0` disables (skip forever). A non-finite or negative value falls back to
4490
+ * the default.
4491
+ */
4492
+ unregisteredWorkflowGraceMs?: number | undefined;
4478
4493
  /**
4479
4494
  * Opt-in liveness deadline (ms) for a remote workflow `advance`. If the worker neither returns a
4480
4495
  * decision NOR sends a run-scoped {@link Heartbeat} within this window, the engine presumes it dead:
@@ -4626,6 +4641,13 @@ interface ListStalledRunsOptions {
4626
4641
  /** Evaluate as of this clock time. Defaults to the engine clock. */
4627
4642
  now?: number | undefined;
4628
4643
  }
4644
+ /**
4645
+ * Thrown by {@link WorkflowEngine.resume} when nothing registered here (nor an inherited or
4646
+ * convention route) can run the run's pinned `workflow@version`.
4647
+ */
4648
+ declare class WorkflowNotRegistered extends Error {
4649
+ constructor(workflow: string, version: string);
4650
+ }
4629
4651
  /**
4630
4652
  * The orchestrator. Owns workflow state and replays runs deterministically: each step's
4631
4653
  * result is checkpointed, so on resume a completed step returns its saved output instead of
@@ -4648,6 +4670,7 @@ declare class WorkflowEngine {
4648
4670
  */
4649
4671
  private readonly ownLeaseCutoffMs;
4650
4672
  private readonly maxRecoveryAttempts?;
4673
+ private readonly unregisteredWorkflowGraceMs;
4651
4674
  private readonly remoteAdvanceSilenceMs?;
4652
4675
  private readonly reconcileMs;
4653
4676
  private readonly timerBatchSize;
@@ -5089,6 +5112,16 @@ declare class WorkflowEngine {
5089
5112
  /** Emit a locally-produced lifecycle event: deliver to subscribers AND broadcast it on the
5090
5113
  * control plane so other instances (e.g. a dashboard pod) can live-tail this run. */
5091
5114
  private emit;
5115
+ /** Events waiting for the control-plane publish in flight, in emit order (see {@link broadcastEvents}). */
5116
+ private readonly eventsToBroadcast;
5117
+ private broadcastingEvents;
5118
+ /**
5119
+ * Publish the queued events in the order they were emitted. A run's own events go out one per
5120
+ * message, as every instance expects them. Step events (a wide fan emits two per step) that queued
5121
+ * up while a publish was in flight go out together in one `events` message, so a fan of 40,000
5122
+ * steps costs the control plane a few hundred messages to every instance instead of 80,000.
5123
+ */
5124
+ private broadcastEvents;
5092
5125
  /** Deliver an event to local subscribers only (no re-broadcast) — used for both locally-produced
5093
5126
  * events and ones received from the control plane, so an event shows up once on every instance. */
5094
5127
  private deliver;
@@ -5359,6 +5392,33 @@ declare class WorkflowEngine {
5359
5392
  * result to skip the resume, or `undefined` to proceed.
5360
5393
  */
5361
5394
  private countRecovery;
5395
+ /** Tell everyone a run was just written `dead`: events, dead-letter routing, a waiting parent. */
5396
+ private announceDead;
5397
+ /** The synchronous half of {@link canResolveWorkflow}: callers check it first, so a registered run
5398
+ * pays no await (same rule as {@link resume}). */
5399
+ private isRegisteredHere;
5400
+ /**
5401
+ * Whether this instance can run `run`'s pinned `workflow@version`: registered here, inherited from a
5402
+ * remote ancestor, or routed by convention to a live worker group (the lookup {@link resume} does).
5403
+ */
5404
+ private canResolveWorkflow;
5405
+ /**
5406
+ * A run whose lease this instance holds but whose `workflow@version` nothing here can run (see
5407
+ * {@link WorkflowNotRegistered}). Within {@link unregisteredWorkflowGraceMs} of becoming due it is a
5408
+ * rolling deploy's skew: the lease is released at once so the pod that still has the version takes
5409
+ * the run now, not after the lease expires. Past it, the workflow is gone from the whole fleet (a
5410
+ * capable pod would have taken the run long ago): dead-letter it.
5411
+ *
5412
+ * The age is read off `updatedAt` / `wakeAt`, so every path that would touch either (a recovery
5413
+ * count, a re-pend, a hand-off's lease-long `wakeAt`) must call this BEFORE writing — otherwise an
5414
+ * orphan never ages.
5415
+ *
5416
+ * Returns the `dead` result, `null` when the run left its status meanwhile (a lease-free
5417
+ * `cancel()`: settled, not ours to touch), or `undefined` when it was left for later.
5418
+ */
5419
+ private settleUnregistered;
5420
+ /** Log a run skipped for an unregistered workflow — still there, still not progressing. */
5421
+ private logUnregisteredSkip;
5362
5422
  /**
5363
5423
  * Resume every suspended run whose durable timer is due. Call periodically (a poller) and on
5364
5424
  * boot. A run still not due re-suspends cheaply without running new work.
@@ -5460,7 +5520,8 @@ declare class WorkflowEngine {
5460
5520
  * Resume each run only if this instance can acquire its recovery lease — so when several
5461
5521
  * replicas recover or poll at once, each run is picked up by exactly one of them.
5462
5522
  *
5463
- * A run this instance cannot resume is SKIPPED, not allowed to abort the batch. The canonical
5523
+ * A run this instance cannot resume is SKIPPED, not allowed to abort the batch (a run whose workflow
5524
+ * stays registered nowhere past `unregisteredWorkflowGraceMs` is dead-lettered instead). The canonical
5464
5525
  * case is skew protection: mid-rolling-deploy a pod leases a run whose workflow it does not have
5465
5526
  * and `resume` throws by design — but the rest of the batch is runs it can drive perfectly well,
5466
5527
  * and every caller here is a recovery sweep. Letting the first such run end the loop means one
@@ -7843,4 +7904,4 @@ declare function singletonAdmissionDenialPatch(runs: WorkflowRun[], runId: strin
7843
7904
  updatedAt: Date;
7844
7905
  } | undefined;
7845
7906
 
7846
- export { type AccumulatorHost, type Admission, type AdmissionBackend, type AdmissionItem, type AllOptions, type AllSettledOptions, type AllStepsOptions, type AllStepsSettledOptions, type AnnouncedWorkflow, type AnnouncementEvidence, type AttributeFilter, type AttributeOp, type AttributePredicateOperand, type AttributeValue, type BackoffStrategy, type BlockedDispatch, type BlockedResolution, type BufferedSignalRow, CHILD_EDGE_PREFIXES, CONTROL_FLOW_SIGNAL, CORE_ORIGIN, CURRENT_PROTOCOL_VERSION, type CapabilityRequirement, type CheckpointAggregatingStore, type CheckpointNameKind, type CheckpointQuery, type CheckpointStat, type ChildCallOptions, type ChildWait, CodecStateStore, type CodecStateStoreOptions, type ConcurrencyConfig, ConcurrencyLimitError, type ConcurrencyQuota, ContinueAsNew, type ControlMessage, type ControlPlane, DEFAULT_CONCURRENCY_STATUSES, DEFAULT_PARTITION, DEFAULT_STALLED_AFTER_MS, DURABLE_OPTIONS, DURABLE_OPTIONS_CANONICAL, DURABLE_STEP_CONFIG, DURABLE_STEP_NAME, type Disagreement, type DispatchDiagnostics, type DispatchPlan, type DurableExecution, type DurableExecutionWrapper, type DurableTopology, type DurableWebhook, DurableWorkflow, type DynamicWorkflowCtx, ENGINE_MINTED_TAG_PREFIXES, ENTITY_WORKFLOW, EVENT_BUFFER_SCAN_LIMIT, EVENT_TOKEN_PREFIX, type EngineEvent, type EngineEventType, type EngineListener, Entities, type EntityConfig, type EntityHandler, type EntityHost, type EntityWorkflowHost, EventAccumulators, type EventBatchConfig, FAN_SUMMARY_FAILURES, type FanJoin, type FanPage, type FanPageQuery, type FanSummary, type FanWait, FatalError, type FindStalledRunsOptions, GatherError, type GatherItemFailure, type GroupHealth, HANDOFF_SEQ_BASE, type Heartbeat, type HeartbeatSighting, type HeartbeatStatus, type HistoryEvent, InMemoryAdmissionBackend, InMemoryRunQueue, InMemoryStateStore, InMemoryTransport, type InferSearchAttributes, LEGACY_V1_CAPABILITIES, LEGACY_V1_PROTOCOL, type ListStalledRunsOptions, type MetricsCollector, type MetricsSnapshot, type NamedTransport, type NegotiateOptions, type NegotiationOutcome, type NegotiationReason, type NegotiationResult, NestedWorkflowCallError, NonDeterminismError, type OnErrorInput, type OnErrorRoute, type OwnedLeaseReclaim, type PartitionSighting, type PayloadCodec, type QueueConfig, QueueController, REMOTE_EVENT, RUN_HISTORY_FIXTURE_VERSION, RUN_VALUE_FACET_LIMIT, RUN_VALUE_FACET_SCAN, type RawWorkerDescriptor, RemoteStepError, RemoteStepTimeout, type RemoteTask, RemoteWorkflowExecutor, RemoteWorkflowTimeout, type RetentionPolicy, type RoutableDispatch, type RoutableResolution, type RoutingResolution, type RunAttributeRow, type RunConsumerOptions, type RunDetail, type RunDetailOptions, type RunDispatch, type RunDispatcher, type RunFacetQuery, type RunFacetRow, RunGateway, type RunListItem, type RunQuery, type RunQueueOptions, type RunReply, type RunReplyResult, type RunRequest, type RunRequestKind, type RunResult, type RunSchedulesOptions, type RunScope, type RunStatus, type RunValueAxis, type RunValueFacetOptions, type RunValueFacetOrder, type RunValueFacetRow, type RunWaiting, SINGLETON_ACTIVE_STATUSES, SINGLETON_ADMITTED_TAG, SINGLETON_RESERVED_TAGS, SINGLETON_TRACKED_TAG, STATE_STORE, STATE_STORE_CANONICAL, SUMMARY_CHILD_LIMIT, type ScalarAttributeFilter, ScheduleAlreadyExistsError, ScheduleClient, type ScheduleClientOptions, type ScheduleDescription, type ScheduleHost, type ScheduleListQuery, ScheduleNotFoundError, type ScheduleOptions, type ScheduleQuery, type ScheduleRecord, type ScheduledWorkflow, type SchemaIssue, type SchemaValidationCode, SchemaValidationError, type SearchAttributes, type SearchAttributesSchema, type Settled, SignalTimeoutError, type SignalWaiter, type SingletonConfig, SingletonQueueFullError, type StalePendingStep, type StalledRunInfo, StandardSchemaV1, type StartOptions, type StartRunMessage, type StateStore, type StepCheckpoint, type StepConfig, type StepDef, type StepDispatchOpts, type StepError, type StepEvent, type StepHandler, type StepInterceptor, type StepInvocation, type StepKind, type StepLogger, type StepOptions, type StepRef, type StepResult, type StepScope, type StepSettlement, type StepUndo, type StoreTransaction, type SubProcessHandle, TERMINAL_RUN_STATUSES, TRANSPORT, TRANSPORT_CANONICAL, type TenantEvent, type TenantTokenSource, type TenantVerifier, type Transport, type UndoOf, type UpdateResult, type UpdateValidator, VERSION_UNDECLARED_TAG, type VerifiedTenant, WORKFLOW_NAME_KEY, type WireDates, type WorkerAdjust, type WorkerConcurrencyStatus, type WorkerDescriptor, type WorkerHeartbeat, type WorkerLifecycle, type WorkerStatus, type WorkflowAnnouncement, WorkflowBlocked, type WorkflowClass, type WorkflowCommand, type WorkflowCtx, type WorkflowDecision, type WorkflowDirectory, WorkflowEngine, type WorkflowEngineDeps, type WorkflowExecutor, type WorkflowHandler, type WorkflowInputOf, type WorkflowOutputOf, type WorkflowRef, type WorkflowRegistration, type WorkflowRun, type WorkflowStartClient, type WorkflowStepEvent, WorkflowSuspended, type WorkflowTask, aggregateAnnouncements, aggregatesCheckpoints, announcementsOf, applyAttributeQuery, assertConcurrency, assertRoutes, attributeColumnFor, attributeOperand, attributePredicateOperands, axisIsRunColumn, bindWorkflowClass, breakpointToken, buildWorkflowDirectory, canRoute, capableWorkers, checkpointNameKind, checkpointStatsOf, childIdsOf, classifyWaiterToken, clearDurableExecutionWrappers, clearTerminalSingletonAdmission, codeAndDetails, codeOf, collectMetrics, concurrencyTag, controlPlaneDescriptor, countRuns, coveredTokens, createStepLogger, currentStep, currentStepScope, currentWorkflowCtx, debug, describeWorker, descriptorHash, detailsOf, error, etagMatches, eventMatchOf, eventMatches, eventNameOf, eventPrefix, eventToken, facetOrigin, fanPageBounds, fanPageEtag, findStalledRuns, foldedCount, getRunsByIds, handoffHandlerId, handoffName, hasDurableExecutionWrappers, heartbeatStatus, hmacTenantVerifier, indexWaitersByRun, info, isEngineMintedTag, isEventToken, isFailedMember, isLegacyDescriptor, isNonEmptyRunScope, isRemoteEvent, isWorkflowControlFlowSignal, legacySingletonHolders, listSignalWaitersOfRuns, liveHandoff, log, markRemoteEvent, matchesAttributes, mergeRunFacetRows, mergeRunValueFacetRows, missingCapabilities, negotiate, nextCronFireMs, normalizeAttributeRows, normalizeDescriptor, observedAnnouncements, onErrorRunId, pageFan, pageFanFromStore, parentRunIdOf, parseDuration, parseWebhookToken, partitionOfToken, planDispatch, poolNamespace, prevCronFireMs, queryCheckpointsOf, readRunsByIds, readSearchAttributes, requiredCapabilities, resolveConcurrency, resolveRouting, resolveRunWaiting, resolveTenantToken, resolveWireTenant, retiredHandoffName, routeFor, runDetailEtag, runDurableExecution, runInStepLogger, runInStepScope, runInWorkflowCtx, runOutsideWorkflowCtx, runSchedules, runStepHandler, runValueFacetsFromRuns, sanitizeQueueToken, scheduleRunIdAt, scheduledRunId, signTenantToken, singletonAdmissionDenialPatch, singletonAdmissionTags, sqlComparator, stepConfigOf, stepId, stepNameOf, sub, subEvent, subProcess, summarizeFan, summarizeRunFromStore, summarizeTimeline, tenantGroup, useDurableExecution, warn, withSingletonTracked, withoutSingletonMarkers, workflowName };
7907
+ export { type AccumulatorHost, type Admission, type AdmissionBackend, type AdmissionItem, type AllOptions, type AllSettledOptions, type AllStepsOptions, type AllStepsSettledOptions, type AnnouncedWorkflow, type AnnouncementEvidence, type AttributeFilter, type AttributeOp, type AttributePredicateOperand, type AttributeValue, type BackoffStrategy, type BlockedDispatch, type BlockedResolution, type BufferedSignalRow, CHILD_EDGE_PREFIXES, CONTROL_FLOW_SIGNAL, CORE_ORIGIN, CURRENT_PROTOCOL_VERSION, type CapabilityRequirement, type CheckpointAggregatingStore, type CheckpointNameKind, type CheckpointQuery, type CheckpointStat, type ChildCallOptions, type ChildWait, CodecStateStore, type CodecStateStoreOptions, type ConcurrencyConfig, ConcurrencyLimitError, type ConcurrencyQuota, ContinueAsNew, type ControlMessage, type ControlPlane, DEFAULT_CONCURRENCY_STATUSES, DEFAULT_PARTITION, DEFAULT_STALLED_AFTER_MS, DURABLE_OPTIONS, DURABLE_OPTIONS_CANONICAL, DURABLE_STEP_CONFIG, DURABLE_STEP_NAME, type Disagreement, type DispatchDiagnostics, type DispatchPlan, type DurableExecution, type DurableExecutionWrapper, type DurableTopology, type DurableWebhook, DurableWorkflow, type DynamicWorkflowCtx, ENGINE_MINTED_TAG_PREFIXES, ENTITY_WORKFLOW, EVENT_BUFFER_SCAN_LIMIT, EVENT_TOKEN_PREFIX, type EngineEvent, type EngineEventType, type EngineListener, Entities, type EntityConfig, type EntityHandler, type EntityHost, type EntityWorkflowHost, EventAccumulators, type EventBatchConfig, FAN_SUMMARY_FAILURES, type FanJoin, type FanPage, type FanPageQuery, type FanSummary, type FanWait, FatalError, type FindStalledRunsOptions, GatherError, type GatherItemFailure, type GroupHealth, HANDOFF_SEQ_BASE, type Heartbeat, type HeartbeatSighting, type HeartbeatStatus, type HistoryEvent, InMemoryAdmissionBackend, InMemoryRunQueue, InMemoryStateStore, InMemoryTransport, type InferSearchAttributes, LEGACY_V1_CAPABILITIES, LEGACY_V1_PROTOCOL, type ListStalledRunsOptions, type MetricsCollector, type MetricsSnapshot, type NamedTransport, type NegotiateOptions, type NegotiationOutcome, type NegotiationReason, type NegotiationResult, NestedWorkflowCallError, NonDeterminismError, type OnErrorInput, type OnErrorRoute, type OwnedLeaseReclaim, type PartitionSighting, type PayloadCodec, type QueueConfig, QueueController, REMOTE_EVENT, RUN_HISTORY_FIXTURE_VERSION, RUN_VALUE_FACET_LIMIT, RUN_VALUE_FACET_SCAN, type RawWorkerDescriptor, RemoteStepError, RemoteStepTimeout, type RemoteTask, RemoteWorkflowExecutor, RemoteWorkflowTimeout, type RetentionPolicy, type RoutableDispatch, type RoutableResolution, type RoutingResolution, type RunAttributeRow, type RunConsumerOptions, type RunDetail, type RunDetailOptions, type RunDispatch, type RunDispatcher, type RunFacetQuery, type RunFacetRow, RunGateway, type RunListItem, type RunQuery, type RunQueueOptions, type RunReply, type RunReplyResult, type RunRequest, type RunRequestKind, type RunResult, type RunSchedulesOptions, type RunScope, type RunStatus, type RunValueAxis, type RunValueFacetOptions, type RunValueFacetOrder, type RunValueFacetRow, type RunWaiting, SINGLETON_ACTIVE_STATUSES, SINGLETON_ADMITTED_TAG, SINGLETON_RESERVED_TAGS, SINGLETON_TRACKED_TAG, STATE_STORE, STATE_STORE_CANONICAL, SUMMARY_CHILD_LIMIT, type ScalarAttributeFilter, ScheduleAlreadyExistsError, ScheduleClient, type ScheduleClientOptions, type ScheduleDescription, type ScheduleHost, type ScheduleListQuery, ScheduleNotFoundError, type ScheduleOptions, type ScheduleQuery, type ScheduleRecord, type ScheduledWorkflow, type SchemaIssue, type SchemaValidationCode, SchemaValidationError, type SearchAttributes, type SearchAttributesSchema, type Settled, SignalTimeoutError, type SignalWaiter, type SingletonConfig, SingletonQueueFullError, type StalePendingStep, type StalledRunInfo, StandardSchemaV1, type StartOptions, type StartRunMessage, type StateStore, type StepCheckpoint, type StepConfig, type StepDef, type StepDispatchOpts, type StepError, type StepEvent, type StepHandler, type StepInterceptor, type StepInvocation, type StepKind, type StepLogger, type StepOptions, type StepRef, type StepResult, type StepScope, type StepSettlement, type StepUndo, type StoreTransaction, type SubProcessHandle, TERMINAL_RUN_STATUSES, TRANSPORT, TRANSPORT_CANONICAL, type TenantEvent, type TenantTokenSource, type TenantVerifier, type Transport, type UndoOf, type UpdateResult, type UpdateValidator, VERSION_UNDECLARED_TAG, type VerifiedTenant, WORKFLOW_NAME_KEY, type WireDates, type WorkerAdjust, type WorkerConcurrencyStatus, type WorkerDescriptor, type WorkerHeartbeat, type WorkerLifecycle, type WorkerStatus, type WorkflowAnnouncement, WorkflowBlocked, type WorkflowClass, type WorkflowCommand, type WorkflowCtx, type WorkflowDecision, type WorkflowDirectory, WorkflowEngine, type WorkflowEngineDeps, type WorkflowExecutor, type WorkflowHandler, type WorkflowInputOf, WorkflowNotRegistered, type WorkflowOutputOf, type WorkflowRef, type WorkflowRegistration, type WorkflowRun, type WorkflowStartClient, type WorkflowStepEvent, WorkflowSuspended, type WorkflowTask, aggregateAnnouncements, aggregatesCheckpoints, announcementsOf, applyAttributeQuery, assertConcurrency, assertRoutes, attributeColumnFor, attributeOperand, attributePredicateOperands, axisIsRunColumn, bindWorkflowClass, breakpointToken, buildWorkflowDirectory, canRoute, capableWorkers, checkpointNameKind, checkpointStatsOf, childIdsOf, classifyWaiterToken, clearDurableExecutionWrappers, clearTerminalSingletonAdmission, codeAndDetails, codeOf, collectMetrics, concurrencyTag, controlPlaneDescriptor, countRuns, coveredTokens, createStepLogger, currentStep, currentStepScope, currentWorkflowCtx, debug, describeWorker, descriptorHash, detailsOf, error, etagMatches, eventMatchOf, eventMatches, eventNameOf, eventPrefix, eventToken, facetOrigin, fanPageBounds, fanPageEtag, findStalledRuns, foldedCount, getRunsByIds, handoffHandlerId, handoffName, hasDurableExecutionWrappers, heartbeatStatus, hmacTenantVerifier, indexWaitersByRun, info, isEngineMintedTag, isEventToken, isFailedMember, isLegacyDescriptor, isNonEmptyRunScope, isRemoteEvent, isWorkflowControlFlowSignal, legacySingletonHolders, listSignalWaitersOfRuns, liveHandoff, log, markRemoteEvent, matchesAttributes, mergeRunFacetRows, mergeRunValueFacetRows, missingCapabilities, negotiate, nextCronFireMs, normalizeAttributeRows, normalizeDescriptor, observedAnnouncements, onErrorRunId, pageFan, pageFanFromStore, parentRunIdOf, parseDuration, parseWebhookToken, partitionOfToken, planDispatch, poolNamespace, prevCronFireMs, queryCheckpointsOf, readRunsByIds, readSearchAttributes, requiredCapabilities, resolveConcurrency, resolveRouting, resolveRunWaiting, resolveTenantToken, resolveWireTenant, retiredHandoffName, routeFor, runDetailEtag, runDurableExecution, runInStepLogger, runInStepScope, runInWorkflowCtx, runOutsideWorkflowCtx, runSchedules, runStepHandler, runValueFacetsFromRuns, sanitizeQueueToken, scheduleRunIdAt, scheduledRunId, signTenantToken, singletonAdmissionDenialPatch, singletonAdmissionTags, sqlComparator, stepConfigOf, stepId, stepNameOf, sub, subEvent, subProcess, summarizeFan, summarizeRunFromStore, summarizeTimeline, tenantGroup, useDurableExecution, warn, withSingletonTracked, withoutSingletonMarkers, workflowName };
package/dist/index.js CHANGED
@@ -4814,6 +4814,8 @@ var ANNOUNCED_ROUTES_TTL_MS = 5e3;
4814
4814
  var VERSION_UNDECLARED_TAG = "version:undeclared";
4815
4815
  var versionKey = /* @__PURE__ */ __name((name, version) => `${name}@${version}`, "versionKey");
4816
4816
  var STEP_FAN_CHUNK = 200;
4817
+ var STEP_EVENTS_PER_BROADCAST = 500;
4818
+ var isStepEvent = /* @__PURE__ */ __name((event) => event.type.startsWith("step."), "isStepEvent");
4817
4819
  var FAN_REREAD_CHUNK = 1e3;
4818
4820
  var STEP_FAN_SEND_ROUND = 2e3;
4819
4821
  var PARENT_NOTIFY_ATTEMPTS = 4;
@@ -4947,6 +4949,22 @@ var NamespaceMismatch = class NamespaceMismatch2 extends Error {
4947
4949
  super("namespace-mismatch");
4948
4950
  }
4949
4951
  };
4952
+ var WorkflowNotRegistered = class extends Error {
4953
+ static {
4954
+ __name(this, "WorkflowNotRegistered");
4955
+ }
4956
+ constructor(workflow, version) {
4957
+ super(`workflow ${workflow}@${version} is not registered \u2014 keep the prior version deployed so in-flight runs can drain (skew protection)`);
4958
+ this.name = "WorkflowNotRegistered";
4959
+ }
4960
+ };
4961
+ var UNREGISTERED_DEAD_LETTER_FROM = [
4962
+ "pending",
4963
+ "running",
4964
+ "suspended",
4965
+ "cancelling",
4966
+ "blocked"
4967
+ ];
4950
4968
  var WorkflowEngine = class {
4951
4969
  static {
4952
4970
  __name(this, "WorkflowEngine");
@@ -4966,6 +4984,7 @@ var WorkflowEngine = class {
4966
4984
  */
4967
4985
  ownLeaseCutoffMs;
4968
4986
  maxRecoveryAttempts;
4987
+ unregisteredWorkflowGraceMs;
4969
4988
  remoteAdvanceSilenceMs;
4970
4989
  reconcileMs;
4971
4990
  timerBatchSize;
@@ -5152,6 +5171,8 @@ var WorkflowEngine = class {
5152
5171
  this.leaseMs = deps.leaseMs ?? 3e4;
5153
5172
  this.ownLeaseCutoffMs = this.clock() + this.leaseMs;
5154
5173
  this.maxRecoveryAttempts = deps.maxRecoveryAttempts;
5174
+ const grace = deps.unregisteredWorkflowGraceMs;
5175
+ this.unregisteredWorkflowGraceMs = grace !== void 0 && Number.isFinite(grace) && grace >= 0 ? grace : 24 * 60 * 60 * 1e3;
5155
5176
  this.remoteAdvanceSilenceMs = deps.remoteAdvanceSilenceMs;
5156
5177
  this.reconcileMs = deps.reconcileMs === void 0 ? 3e5 : deps.reconcileMs || void 0;
5157
5178
  const stalledAfter = deps.stalledAfterMs;
@@ -5284,6 +5305,13 @@ var WorkflowEngine = class {
5284
5305
  ...msg.event,
5285
5306
  at: new Date(msg.event.at)
5286
5307
  }));
5308
+ } else if (msg.kind === "events") {
5309
+ for (const event of msg.events) {
5310
+ this.deliver(markRemoteEvent({
5311
+ ...event,
5312
+ at: new Date(event.at)
5313
+ }));
5314
+ }
5287
5315
  } else if (msg.kind === "cancel") {
5288
5316
  this.notifyCancelled(msg.runId);
5289
5317
  } else if (msg.kind === "enqueued") {
@@ -5461,13 +5489,19 @@ var WorkflowEngine = class {
5461
5489
  * that is lost brings it back due after that period. False when the transport refused the wakes,
5462
5490
  * so the caller resumes them itself.
5463
5491
  */
5464
- async handOffDue(runs, nowMs) {
5492
+ async handOffDue(runs, nowMs, settled) {
5465
5493
  const claimed = [];
5466
5494
  for (const run of runs) {
5467
5495
  if (this.draining) break;
5468
5496
  if (!await this.store.tryLockRun(run.id, this.instanceId, nowMs + this.leaseMs, nowMs)) {
5469
5497
  continue;
5470
5498
  }
5499
+ if (!this.isRegisteredHere(run) && !await this.canResolveWorkflow(run)) {
5500
+ const result = await this.settleUnregistered(run, nowMs);
5501
+ if (result) settled?.set(run.id, result);
5502
+ else if (result === void 0) this.logUnregisteredSkip(run);
5503
+ continue;
5504
+ }
5471
5505
  try {
5472
5506
  const patch = {
5473
5507
  wakeAt: nowMs + this.leaseMs
@@ -5865,13 +5899,53 @@ var WorkflowEngine = class {
5865
5899
  at: /* @__PURE__ */ new Date()
5866
5900
  };
5867
5901
  this.deliver(full);
5868
- if (this.controlPlane) {
5869
- void this.controlPlane.publishControl({
5870
- kind: "event",
5871
- event: full,
5872
- from: this.instanceId
5873
- }).catch(() => {
5874
- });
5902
+ if (!this.controlPlane) return;
5903
+ this.eventsToBroadcast.push(full);
5904
+ if (!this.broadcastingEvents) void this.broadcastEvents();
5905
+ }
5906
+ /** Events waiting for the control-plane publish in flight, in emit order (see {@link broadcastEvents}). */
5907
+ eventsToBroadcast = [];
5908
+ broadcastingEvents = false;
5909
+ /**
5910
+ * Publish the queued events in the order they were emitted. A run's own events go out one per
5911
+ * message, as every instance expects them. Step events (a wide fan emits two per step) that queued
5912
+ * up while a publish was in flight go out together in one `events` message, so a fan of 40,000
5913
+ * steps costs the control plane a few hundred messages to every instance instead of 80,000.
5914
+ */
5915
+ async broadcastEvents() {
5916
+ this.broadcastingEvents = true;
5917
+ try {
5918
+ while (this.eventsToBroadcast.length > 0 && this.controlPlane) {
5919
+ const first = this.eventsToBroadcast[0];
5920
+ let msg;
5921
+ if (isStepEvent(first)) {
5922
+ let n = 1;
5923
+ while (n < STEP_EVENTS_PER_BROADCAST && n < this.eventsToBroadcast.length && isStepEvent(this.eventsToBroadcast[n])) {
5924
+ n += 1;
5925
+ }
5926
+ const events = this.eventsToBroadcast.splice(0, n);
5927
+ msg = events.length === 1 ? {
5928
+ kind: "event",
5929
+ event: first,
5930
+ from: this.instanceId
5931
+ } : {
5932
+ kind: "events",
5933
+ events,
5934
+ from: this.instanceId
5935
+ };
5936
+ } else {
5937
+ this.eventsToBroadcast.shift();
5938
+ msg = {
5939
+ kind: "event",
5940
+ event: first,
5941
+ from: this.instanceId
5942
+ };
5943
+ }
5944
+ await this.controlPlane.publishControl(msg).catch(() => {
5945
+ });
5946
+ }
5947
+ } finally {
5948
+ this.broadcastingEvents = false;
5875
5949
  }
5876
5950
  }
5877
5951
  /** Deliver an event to local subscribers only (no re-broadcast) — used for both locally-produced
@@ -6201,9 +6275,7 @@ var WorkflowEngine = class {
6201
6275
  });
6202
6276
  }
6203
6277
  const registered = this.workflows.get(versionKey(run.workflow, run.workflowVersion)) ?? await this.findInheritedRegistration(run) ?? await this.resolveRemoteByConvention(run);
6204
- if (!registered) {
6205
- throw new Error(`workflow ${run.workflow}@${run.workflowVersion} is not registered \u2014 keep the prior version deployed so in-flight runs can drain (skew protection)`);
6206
- }
6278
+ if (!registered) throw new WorkflowNotRegistered(run.workflow, run.workflowVersion);
6207
6279
  return this.track(this.execute(run, registered.fn));
6208
6280
  }
6209
6281
  /**
@@ -6618,6 +6690,15 @@ var WorkflowEngine = class {
6618
6690
  };
6619
6691
  }
6620
6692
  }
6693
+ if (!this.isRegisteredHere(run) && !await this.canResolveWorkflow(run)) {
6694
+ const settled2 = await this.settleUnregistered(run, nowMs);
6695
+ if (settled2) return {
6696
+ result: settled2,
6697
+ dispatch: false
6698
+ };
6699
+ if (settled2 === void 0) this.logUnregisteredSkip(run);
6700
+ return void 0;
6701
+ }
6621
6702
  const settled = await this.countRecovery(run);
6622
6703
  if (settled) return {
6623
6704
  result: settled,
@@ -6689,25 +6770,10 @@ var WorkflowEngine = class {
6689
6770
  updatedAt: /* @__PURE__ */ new Date()
6690
6771
  });
6691
6772
  await this.store.releaseRunLock(run.id);
6692
- this.emit({
6693
- type: "run.failed",
6694
- runId: run.id,
6695
- workflow: run.workflow,
6696
- namespace: run.namespace,
6697
- error: error2
6698
- });
6699
- this.notifyDead({
6773
+ return this.announceDead({
6700
6774
  ...run,
6701
- status: "dead",
6702
- error: error2,
6703
6775
  recoveryAttempts: attempts
6704
- });
6705
- this.notifyParent(run.id, failedCompletion(error2));
6706
- return {
6707
- runId: run.id,
6708
- status: "dead",
6709
- error: error2
6710
- };
6776
+ }, error2);
6711
6777
  }
6712
6778
  await this.store.updateRun(run.id, {
6713
6779
  recoveryAttempts: attempts,
@@ -6715,6 +6781,85 @@ var WorkflowEngine = class {
6715
6781
  });
6716
6782
  return void 0;
6717
6783
  }
6784
+ /** Tell everyone a run was just written `dead`: events, dead-letter routing, a waiting parent. */
6785
+ announceDead(run, error2) {
6786
+ this.emit({
6787
+ type: "run.failed",
6788
+ runId: run.id,
6789
+ workflow: run.workflow,
6790
+ namespace: run.namespace,
6791
+ error: error2
6792
+ });
6793
+ this.notifyDead({
6794
+ ...run,
6795
+ status: "dead",
6796
+ error: error2
6797
+ });
6798
+ this.notifyParent(run.id, failedCompletion(error2));
6799
+ return {
6800
+ runId: run.id,
6801
+ status: "dead",
6802
+ error: error2
6803
+ };
6804
+ }
6805
+ /** The synchronous half of {@link canResolveWorkflow}: callers check it first, so a registered run
6806
+ * pays no await (same rule as {@link resume}). */
6807
+ isRegisteredHere(run) {
6808
+ return this.workflows.has(versionKey(run.workflow, run.workflowVersion));
6809
+ }
6810
+ /**
6811
+ * Whether this instance can run `run`'s pinned `workflow@version`: registered here, inherited from a
6812
+ * remote ancestor, or routed by convention to a live worker group (the lookup {@link resume} does).
6813
+ */
6814
+ async canResolveWorkflow(run) {
6815
+ if (this.isRegisteredHere(run)) return true;
6816
+ return await this.findInheritedRegistration(run) !== void 0 || await this.resolveRemoteByConvention(run) !== void 0;
6817
+ }
6818
+ /**
6819
+ * A run whose lease this instance holds but whose `workflow@version` nothing here can run (see
6820
+ * {@link WorkflowNotRegistered}). Within {@link unregisteredWorkflowGraceMs} of becoming due it is a
6821
+ * rolling deploy's skew: the lease is released at once so the pod that still has the version takes
6822
+ * the run now, not after the lease expires. Past it, the workflow is gone from the whole fleet (a
6823
+ * capable pod would have taken the run long ago): dead-letter it.
6824
+ *
6825
+ * The age is read off `updatedAt` / `wakeAt`, so every path that would touch either (a recovery
6826
+ * count, a re-pend, a hand-off's lease-long `wakeAt`) must call this BEFORE writing — otherwise an
6827
+ * orphan never ages.
6828
+ *
6829
+ * Returns the `dead` result, `null` when the run left its status meanwhile (a lease-free
6830
+ * `cancel()`: settled, not ours to touch), or `undefined` when it was left for later.
6831
+ */
6832
+ async settleUnregistered(listed, nowMs) {
6833
+ const run = await this.store.getRun(listed.id);
6834
+ if (!run || !UNREGISTERED_DEAD_LETTER_FROM.includes(run.status)) {
6835
+ await this.store.releaseRunLock(listed.id);
6836
+ return null;
6837
+ }
6838
+ const dueSince = Math.max(run.updatedAt.getTime(), run.wakeAt ?? 0);
6839
+ if (this.unregisteredWorkflowGraceMs === 0 || nowMs - dueSince < this.unregisteredWorkflowGraceMs) {
6840
+ await this.store.releaseRunLock(run.id);
6841
+ return void 0;
6842
+ }
6843
+ const error2 = {
6844
+ message: `workflow ${run.workflow}@${run.workflowVersion} is not registered on any live instance (unresumable for over ${this.unregisteredWorkflowGraceMs}ms) \u2014 moved to dead-letter`,
6845
+ code: "workflow_not_registered"
6846
+ };
6847
+ const applied = await (this.store.updateRunIf ?? this.updateRunIfFallback).call(this.store, run.id, [
6848
+ run.status
6849
+ ], {
6850
+ status: "dead",
6851
+ error: error2,
6852
+ updatedAt: /* @__PURE__ */ new Date()
6853
+ });
6854
+ await this.store.releaseRunLock(run.id);
6855
+ if (!applied) return null;
6856
+ console.warn(`[nestjs-durable] run ${run.id}: ${error2.message}`);
6857
+ return this.announceDead(run, error2);
6858
+ }
6859
+ /** Log a run skipped for an unregistered workflow — still there, still not progressing. */
6860
+ logUnregisteredSkip(run) {
6861
+ console.error(`[nestjs-durable] could not resume run ${run.id} (${run.workflow}@${run.workflowVersion}); leaving it for another instance:`, new WorkflowNotRegistered(run.workflow, run.workflowVersion));
6862
+ }
6718
6863
  /**
6719
6864
  * Resume every suspended run whose durable timer is due. Call periodically (a poller) and on
6720
6865
  * boot. A run still not due re-suspends cheaply without running new work.
@@ -6739,11 +6884,12 @@ var WorkflowEngine = class {
6739
6884
  ...due,
6740
6885
  ...blocked
6741
6886
  ];
6742
- if (await this.handOffDue(runs, nowMs)) {
6743
- return runs.map((run) => ({
6887
+ const settled = /* @__PURE__ */ new Map();
6888
+ if (await this.handOffDue(runs, nowMs, settled)) {
6889
+ return runs.map((run) => settled.get(run.id) ?? {
6744
6890
  runId: run.id,
6745
6891
  status: run.status
6746
- }));
6892
+ });
6747
6893
  }
6748
6894
  }
6749
6895
  const waiterByRun = indexWaitersByRun(await listSignalWaitersOfRuns(this.store, [
@@ -6824,8 +6970,13 @@ var WorkflowEngine = class {
6824
6970
  const nowMs = this.clock();
6825
6971
  const acquired = await this.store.tryLockRun(runId, this.instanceId, nowMs + this.leaseMs, nowMs);
6826
6972
  if (!acquired) return null;
6827
- return this.resumeImplicit(runId).catch((err) => {
6973
+ return this.resumeImplicit(runId).catch(async (err) => {
6828
6974
  if (err instanceof NamespaceMismatch) return null;
6975
+ if (err instanceof WorkflowNotRegistered) {
6976
+ const run = await this.store.getRun(runId);
6977
+ const settled = run ? await this.settleUnregistered(run, nowMs) : null;
6978
+ if (settled !== void 0) return settled;
6979
+ }
6829
6980
  throw err;
6830
6981
  });
6831
6982
  }
@@ -7080,7 +7231,8 @@ var WorkflowEngine = class {
7080
7231
  * Resume each run only if this instance can acquire its recovery lease — so when several
7081
7232
  * replicas recover or poll at once, each run is picked up by exactly one of them.
7082
7233
  *
7083
- * A run this instance cannot resume is SKIPPED, not allowed to abort the batch. The canonical
7234
+ * A run this instance cannot resume is SKIPPED, not allowed to abort the batch (a run whose workflow
7235
+ * stays registered nowhere past `unregisteredWorkflowGraceMs` is dead-lettered instead). The canonical
7084
7236
  * case is skew protection: mid-rolling-deploy a pod leases a run whose workflow it does not have
7085
7237
  * and `resume` throws by design — but the rest of the batch is runs it can drive perfectly well,
7086
7238
  * and every caller here is a recovery sweep. Letting the first such run end the loop means one
@@ -7104,6 +7256,11 @@ var WorkflowEngine = class {
7104
7256
  const settled = onLocked ? await onLocked(run) : void 0;
7105
7257
  results.push(settled ?? await this.resumeImplicit(run.id));
7106
7258
  } catch (error2) {
7259
+ if (error2 instanceof WorkflowNotRegistered) {
7260
+ const settled = await this.settleUnregistered(run, nowMs).catch(() => void 0);
7261
+ if (settled) results.push(settled);
7262
+ if (settled !== void 0) continue;
7263
+ }
7107
7264
  console.error(`[nestjs-durable] could not resume run ${run.id} (${run.workflow}@${run.workflowVersion}); leaving it for another instance:`, error2);
7108
7265
  }
7109
7266
  }
@@ -12864,6 +13021,7 @@ export {
12864
13021
  WORKFLOW_NAME_KEY,
12865
13022
  WorkflowBlocked,
12866
13023
  WorkflowEngine,
13024
+ WorkflowNotRegistered,
12867
13025
  WorkflowSuspended,
12868
13026
  aggregateAnnouncements,
12869
13027
  aggregatesCheckpoints,