@lostgradient/weft 0.8.0 → 0.10.0

package/dist/core/engine/index.js

@@ -1,3 +1,4 @@
+import { AlertManager } from "../../alerting/alert-manager.js";
 import {
   KEYS,
   requireStorageCapability
@@ -19,9 +20,9 @@ import {
 } from "./aggregate.js";
 import {
   completeAsyncActivity as completeAsyncActivityFromInternals,
-  failAsyncActivity as failAsyncActivityFromInternals,
-  recoverPendingAsyncActivities
+  failAsyncActivity as failAsyncActivityFromInternals
 } from "./async-activity-completion.js";
+import { recoverPendingAsyncActivities } from "./async-activity-records.js";
 import { broadcast as broadcastFromInternals } from "./broadcast.js";
 import {
   cancelAll as cancelAllWorkflows,
@@ -52,7 +53,6 @@ import {
 import {
   copyWorkflowDefinition,
   createActivityWorkerDispatcher,
-  createAlertManagerForEngine,
   createExecutionStrategyBundle,
   definitionEntries,
   resolveEngineInterceptors,
@@ -70,7 +70,9 @@ import {
   createQueuedInlineWorkflowStartHandler,
   createSecondInstanceDetectorResolver,
   drainQueuedInlineWorkflowStartsForEngine,
-  isActivityDefinition
+  isActivityDefinition,
+  shouldStartEngineScheduler,
+  validateEngineCreateBackgroundTaskOptions
 } from "./engine-runtime-helpers.js";
 import { EngineCreateNameMismatchError, EngineDisposedError } from "./errors.js";
 import { assertLeaseHeldForEngineWork, commitFencedEngineWrite } from "./fenced-write.js";
@@ -209,9 +211,20 @@ export { assertCompatiblePersistedDataVersion };
 export const ENGINE_PARKED_WORKFLOW_COUNT_FOR_TESTING = Symbol("engineParkedWorkflowCountForTesting"), ENGINE_SIGNAL_WAITER_COUNT_FOR_TESTING = Symbol("engineSignalWaiterCountForTesting"), ENGINE_SLEEP_RESOLVER_COUNT_FOR_TESTING = Symbol("engineSleepResolverCountForTesting"), ENGINE_LEASE_SYNCHRONOUS_DISPOSE_WARNING_NAME = "WeftEngineLeaseSynchronousDisposeWarning";
 
 export { ENGINE_LEASE_LOST_WARNING_NAME };
+function scheduleDefinitionFromInternals(internals, definition) {
+  const workflowType = typeof definition.workflow === "string" ? definition.workflow : definition.workflow.name, definitionSpec = definition.every !== void 0 ? { every: definition.every } : { cron: definition.cron ?? "" };
+  return scheduleFromInternals(internals, workflowType, definition.input, definitionSpec, {
+    ...definition.id !== void 0 && { id: definition.id },
+    ...definition.description !== void 0 && { description: definition.description },
+    ...definition.overlapPolicy !== void 0 && { overlap: definition.overlapPolicy },
+    ...definition.backfill !== void 0 && { backfill: definition.backfill },
+    ...definition.jitter !== void 0 && { jitter: definition.jitter }
+  });
+}
 
 export class Engine extends EventTarget {
   static async create(options) {
+    validateEngineCreateBackgroundTaskOptions(options);
     const engine = new Engine(options);
     try {
       await assertCompatiblePersistedDataVersion(getInternals(engine).storage);
@@ -228,7 +241,7 @@ export class Engine extends EventTarget {
       await engine.#acquireLeaseIfConfigured();
       if (options.recover !== !1)
         await engine.recoverAll(options.acknowledgeUnknownWorkflowTypes !== void 0 ? { acknowledgeUnknownWorkflowTypes: options.acknowledgeUnknownWorkflowTypes } : {});
-      if (options.startScheduler ?? options.recover !== !1)
+      if (shouldStartEngineScheduler(options, getInternals(engine).options.backgroundTaskMode))
         getInternals(engine).scheduler.start();
     } catch (error) {
       await engine[Symbol.asyncDispose]();
@@ -267,6 +280,7 @@ export class Engine extends EventTarget {
     getInternals(this).updateWaitersByWorkflow = new Map;
     getInternals(this).sleepResolvers = new Map;
     getInternals(this).sleepResolversByWorkflow = new Map;
+    getInternals(this).sleepTimersFiredWithoutResolver = new Map;
     getInternals(this).interceptors = resolveEngineInterceptors(options);
     getInternals(this).composedWorkflowInterceptor = void 0;
     getInternals(this).composedActivityInterceptor = void 0;
@@ -330,7 +344,7 @@ export class Engine extends EventTarget {
       cleanupInterval: null,
       secondInstanceDetectionInterval: null,
       testToken: consumeNextEngineLeakWarningTokenForTesting()
-    }, cleanupInterval = setInterval(createCleanupIntervalTick(weakEngine, cleanupIntervalDisposalTracker), 60000);
+    }, cleanupInterval = resolvedOptions.backgroundTaskMode === "automatic" ? setInterval(createCleanupIntervalTick(weakEngine, cleanupIntervalDisposalTracker), 60000) : null;
     cleanupIntervalDisposalTracker.cleanupInterval = cleanupInterval;
     getInternals(this).cleanupInterval = cleanupInterval;
     getInternals(this).cleanupIntervalDisposalTracker = cleanupIntervalDisposalTracker;
@@ -353,7 +367,7 @@ export class Engine extends EventTarget {
     getInternals(this).workflowVisibilityWatermarkExpiresAt = void 0;
     getInternals(this).activityWorkerDispatcher = createActivityWorkerDispatcher(options?.activityExecution);
     getInternals(this).strategy.onMessage(this.#handleStrategyMessage.bind(this));
-    getInternals(this).alertManager = createAlertManagerForEngine(this, options?.alerts, getNow);
+    getInternals(this).alertManager = options?.alerts ? new AlertManager(this, options.alerts, getNow, resolvedOptions.backgroundTaskMode === "automatic") : null;
     this.#ensureRetentionSweepInterval();
     this.#startSecondInstanceDetection();
   }
@@ -575,6 +589,18 @@ export class Engine extends EventTarget {
   getRetentionOverview() {
     return getRetentionOverviewSnapshot(getInternals(this), (type) => resolveWorkflowTypeRetention(getInternals(this), type));
   }
+  async runMaintenance(now = getInternals(this).options.getNow()) {
+    const internals = getInternals(this);
+    await internals.scheduler.tick(now);
+    try {
+      await internals.updateCoordinator.cleanupExpiredResponses();
+    } catch (error) {
+      this.#createTerminationCallbacks().handleCleanupError("cleanupExpiredResponses", error);
+    }
+    if (this.#hasConfiguredRetention())
+      await this.#runRetentionSweep();
+    internals.alertManager?.tick();
+  }
   async purge(filter) {
     return purgeWorkflows(getInternals(this), filter, (workflowId) => cleanupWaitersFromTermination(getInternals(this), workflowId, this.#createTerminationCallbacks()));
   }
@@ -599,15 +625,8 @@ export class Engine extends EventTarget {
     return untagAllWorkflows(getInternals(this), filter, tags, options);
   }
   async schedule(typeOrDefinition, input, spec, options) {
-    if (typeof typeOrDefinition === "object") {
-      const definition = typeOrDefinition, workflowType = typeof definition.workflow === "string" ? definition.workflow : definition.workflow.name, definitionSpec = definition.every !== void 0 ? { every: definition.every } : { cron: definition.cron ?? "" };
-      return scheduleFromInternals(getInternals(this), workflowType, definition.input, definitionSpec, {
-        ...definition.id !== void 0 && { id: definition.id },
-        ...definition.overlapPolicy !== void 0 && { overlap: definition.overlapPolicy },
-        ...definition.backfill !== void 0 && { backfill: definition.backfill },
-        ...definition.jitter !== void 0 && { jitter: definition.jitter }
-      });
-    }
+    if (typeof typeOrDefinition === "object")
+      return scheduleDefinitionFromInternals(getInternals(this), typeOrDefinition);
     if (spec === void 0)
       throw Error("A cron string or schedule spec must be provided when scheduling by workflow type.");
     return scheduleFromInternals(getInternals(this), typeOrDefinition, input, spec, options);

package/dist/core/engine/inline-launch-queue.js

@@ -58,7 +58,7 @@ async function startQueuedInlineWorkflowExecution(internals, start, callbacks) {
       return;
     internals.queuedInlineWorkflowStartIds.delete(start.workflowId);
     internals.engine.dispatchEvent(new WorkflowStartedEvent(start.workflowId, start.workflowType, start.input));
-    startWorkflowExecution(internals, start.workflowId, start.workflowType, start.input, start.checkpoint, start.nestingDepth, start.executionDeadline, start.executionStateOwnerId);
+    startWorkflowExecution(internals, start.workflowId, state.workflowExecutionToken, start.workflowType, start.input, start.checkpoint, start.nestingDepth, start.executionDeadline, start.executionStateOwnerId);
     await callbacks.processPendingUpdatesAfterInlineAdvance(start.workflowId);
   } finally {
     internals.queuedInlineWorkflowStartIds.delete(start.workflowId);

package/dist/core/engine/internals.d.ts

@@ -90,8 +90,30 @@ export interface EngineInternals {
     conditionWaiters: Map<string, () => void>;
     updateWaiters: Map<string, (payload: unknown) => void>;
     updateWaitersByWorkflow: Map<string, TrackedWaiterKeys>;
-    sleepResolvers: Map<string, () => void>;
+    /**
+     * In-process sleep resolvers keyed by `${workflowId}:${operationId}`. Each
+     * carries the run's expected `fireAt` so a fired timer only settles the sleep
+     * whose deadline it represents — a stale timer from a terminated run that
+     * reused the same deterministic operationId (its durable timer outlives
+     * terminal cleanup) is ignored rather than resolving a replacement run early.
+     */
+    sleepResolvers: Map<string, {
+        resolve: () => void;
+        fireAt: number;
+    }>;
     sleepResolversByWorkflow: Map<string, Set<string>>;
+    /**
+     * Per-workflow maps of `operationId` → fired timer `fireAt` for sleep timers
+     * the scheduler tick fired before the resolver was registered. Keyed by
+     * `workflowId` so per-workflow cleanup (`cleanupSleepResolvers`,
+     * `evictSleepResolversWithoutResolving`) can call `.delete(workflowId)` and
+     * sweep all markers in O(1) — including orphaned ones where the timer fired
+     * while the workflow was suspended (no `sleepResolversByWorkflow` entry) or
+     * during recovery of an already-elapsed sleep that takes the early-return
+     * path. The `fireAt` lets `processSleepOperation` ignore a marker left by a
+     * stale earlier-run timer. Cleared entirely at engine disposal.
+     */
+    sleepTimersFiredWithoutResolver: Map<string, Map<string, number>>;
     interceptors: Interceptor[];
     composedWorkflowInterceptor: ComposedWorkflowInterceptor | null | undefined;
     composedActivityInterceptor: ComposedActivityInterceptor | null | undefined;
@@ -164,13 +186,13 @@ export interface EngineInternals {
      * by their durable task token. Mirrored to storage (`KEYS.asyncActivity`) and
      * reloaded by `recoverAll()`. See `async-activity-completion.ts`.
      */
-    pendingAsyncActivities: Map<string, import('./async-activity-completion.ts').PendingAsyncActivity>;
+    pendingAsyncActivities: Map<string, import('./async-activity-records.ts').PendingAsyncActivity>;
     /**
      * Completed or failed async-activity tokens that were consumed before inline
      * recovery adopted the workflow generator. Replay drains these by workflow id
      * when it reaches the same deterministic async-activity token again.
      */
-    pendingAsyncActivityResolutions: Map<string, import('./async-activity-completion.ts').PendingAsyncActivityResolution[]>;
+    pendingAsyncActivityResolutions: Map<string, import('./async-activity-records.ts').PendingAsyncActivityResolution[]>;
     pendingStarts: Set<string>;
     pendingScheduleCreations: Set<string>;
     workflowsNeedingTerminalCleanup: Set<string>;

package/dist/core/engine/lifecycle/fork-helpers.js

@@ -22,6 +22,7 @@ export function createForkedWorkflowState(_internals, workflowId, sourceState, v
     type: sourceState.type,
     status: "running",
     input: sourceState.input,
+    workflowExecutionToken: crypto.randomUUID(),
     versionTuple,
     executionStateOwnerId: workflowId,
     createdAt: forkedAt,

package/dist/core/engine/lifecycle/recovered-services.d.ts

@@ -1,4 +1,4 @@
-import type { WorkflowState } from '../../types.ts';
+import type { WorkflowServicesResolverInfo, WorkflowState } from '../../types.ts';
 import type { EngineInternals } from '../internals.ts';
 /**
  * Re-provide a recovered inline workflow's non-serialized `services` before its
@@ -36,7 +36,9 @@ import type { EngineInternals } from '../internals.ts';
  * @param onCommitError - Records a fail-warn when `failRun` itself throws, so the
  *   swallowed terminal-commit fault is still observable.
  */
-export declare function reprovideRecoveredServices(internals: EngineInternals, state: WorkflowState, failRun: (workflowId: string, error: Error) => Promise<void>, onCommitError: (source: string, error: unknown, workflowId: string) => void, dispatchDiagnostic?: (event: Event) => void): Promise<boolean>;
+export declare function reprovideRecoveredServices(internals: EngineInternals, state: WorkflowState, failRun: (workflowId: string, error: Error) => Promise<void>, onCommitError: (source: string, error: unknown, workflowId: string) => void, dispatchDiagnostic?: (event: Event) => void, resolverInfo?: WorkflowServicesResolverInfo): Promise<boolean>;
+/** Build the durable recovery context shared by the services resolver and recovery hook. */
+export declare function workflowServicesResolverInfoFromState(internals: EngineInternals, state: WorkflowState): Promise<WorkflowServicesResolverInfo>;
 /**
  * The canonical terminal error for any run whose services could not be provided —
  * both recovery re-provision paths and a scheduled-occurrence launch. Shared so

package/dist/core/engine/lifecycle/recovered-services.js

@@ -1,7 +1,9 @@
 import { KEYS } from "../../../storage/interface.js";
 import { DevelopmentWarningEvent } from "../../events.js";
+import { decodeScheduleRunMetadata } from "../schedule-run-metadata.js";
+import { loadScheduleState } from "../storage-io.js";
 const RESOLVE_WORKFLOW_SERVICES_OPTION_PATH = "EngineOptions.resolveWorkflowServices";
-export async function reprovideRecoveredServices(internals, state, failRun, onCommitError, dispatchDiagnostic = () => {}) {
+export async function reprovideRecoveredServices(internals, state, failRun, onCommitError, dispatchDiagnostic = () => {}, resolverInfo) {
   const resolver = internals.options.resolveWorkflowServices;
   if (internals.inlineStrategy === null)
     return !1;
@@ -20,13 +22,9 @@ export async function reprovideRecoveredServices(internals, state, failRun, onCo
     return !0;
   }
   let reason;
+  const info = resolverInfo ?? await workflowServicesResolverInfoFromState(internals, state);
   try {
-    const resolution = await resolver({
-      workflowId: state.id,
-      workflowType: state.type,
-      input: state.input,
-      launchOptions: launchOptionsFromWorkflowState(state)
-    });
+    const resolution = await resolver(info);
     if (resolution.status === "available") {
       internals.workflowServices.set(state.id, resolution.services);
       return !1;
@@ -42,6 +40,16 @@ export async function reprovideRecoveredServices(internals, state, failRun, onCo
   }
   return !0;
 }
+export async function workflowServicesResolverInfoFromState(internals, state) {
+  const schedule = await scheduleFromWorkflowState(internals, state);
+  return {
+    workflowId: state.id,
+    workflowType: state.type,
+    input: state.input,
+    launchOptions: launchOptionsFromWorkflowState(state),
+    ...schedule !== null ? { schedule } : {}
+  };
+}
 export function unavailableServicesError(workflowId, reason) {
   return Error(`Workflow "${workflowId}" services unavailable: ${reason}`);
 }
@@ -54,3 +62,25 @@ function launchOptionsFromWorkflowState(state) {
     ...state.tags !== void 0 && state.tags.length > 0 ? { tags: [...state.tags] } : {}
   };
 }
+async function scheduleFromWorkflowState(internals, state) {
+  const bytes = await internals.storage.get(KEYS.scheduleRun(state.id));
+  if (bytes === null)
+    return null;
+  const metadata = decodeScheduleRunMetadata(bytes);
+  if (metadata === null)
+    return null;
+  const scheduleState = await loadScheduleState(internals, metadata.id);
+  if (scheduleState === null)
+    return null;
+  if (scheduleState.workflowType !== state.type)
+    return null;
+  if (scheduleState.overlap === "allow")
+    return metadata.occurrence !== void 0 ? metadata : null;
+  if (scheduleState.currentWorkflowId === state.id)
+    return metadata;
+  if (metadata.occurrence !== void 0 && scheduleState.nextFireAt === metadata.occurrence)
+    return metadata;
+  if (metadata.occurrence === void 0 && scheduleState.overlap === "queue" && scheduleState.queuedRuns > 0)
+    return metadata;
+  return null;
+}

package/dist/core/engine/lifecycle/resume.d.ts

@@ -1,4 +1,4 @@
 import { type WorkflowHandle } from '../handles.ts';
 import type { EngineInternals } from '../internals.ts';
-import { type LifecycleCallbacks } from './shared.ts';
-export declare function resumeWorkflowFromStorage(internals: EngineInternals, workflowId: string, dispatchResumedEvent: boolean, callbacks: LifecycleCallbacks): Promise<WorkflowHandle>;
+import { type LifecycleCallbacks, type RecoverAllOptions } from './shared.ts';
+export declare function resumeWorkflowFromStorage(internals: EngineInternals, workflowId: string, dispatchResumedEvent: boolean, callbacks: LifecycleCallbacks, onRecoveredWorkflow?: RecoverAllOptions['onRecoveredWorkflow']): Promise<WorkflowHandle>;

package/dist/core/engine/lifecycle/resume.js

@@ -15,15 +15,47 @@ import { getComposedWorkflowInterceptor } from "../strategy-helpers.js";
 import { decodeWorkflowState } from "../validation.js";
 import { buildWorkflowVisibilityIndexTransition } from "../workflow-indexes.js";
 import { prepareResumeState } from "./persist.js";
-import { reprovideRecoveredServices } from "./recovered-services.js";
+import {
+  reprovideRecoveredServices,
+  workflowServicesResolverInfoFromState
+} from "./recovered-services.js";
 import {
   enforceHistoryPolicyBeforeReplay,
   loadTerminalCleanupTrackedState,
   loadWorkflowStartHeaders,
   setWorkflowStartHeaders
 } from "./shared.js";
-async function prepareRecoveredServicesOrFail(internals, state, callbacks) {
-  return reprovideRecoveredServices(internals, state, callbacks.failWorkflowForUnavailableServices, callbacks.handleCleanupError, callbacks.dispatchEvent);
+async function prepareRecoveredServicesOrFail(internals, state, callbacks, resolverInfo) {
+  return reprovideRecoveredServices(internals, state, callbacks.failWorkflowForUnavailableServices, callbacks.handleCleanupError, callbacks.dispatchEvent, resolverInfo);
+}
+async function runRecoveredWorkflowHookOrFail(internals, state, handle, resolverInfo, onRecoveredWorkflow, callbacks) {
+  const info = {
+    ...resolverInfo,
+    handle,
+    launchOptions: resolverInfo.launchOptions ?? { id: state.id },
+    services: internals.workflowServices.get(state.id)
+  };
+  try {
+    await onRecoveredWorkflow(info);
+    return !1;
+  } catch (error) {
+    const reason = error instanceof Error ? error.message : String(error), hookError = Error(`Recovery hook failed for workflow "${state.id}": ${reason}`);
+    try {
+      await callbacks.failWorkflowForRecoveryHook(state.id, hookError);
+    } catch (commitError) {
+      callbacks.handleCleanupError("onRecoveredWorkflow", commitError, state.id);
+    }
+    return !0;
+  }
+}
+async function prepareRecoveredWorkflowOrFail(internals, state, callbacks, onRecoveredWorkflow) {
+  const resolverInfo = onRecoveredWorkflow === void 0 ? void 0 : await workflowServicesResolverInfoFromState(internals, state), handle = callbacks.getHandle(state.id);
+  if (await prepareRecoveredServicesOrFail(internals, state, callbacks, resolverInfo))
+    return { handle, shouldStop: !0 };
+  if (onRecoveredWorkflow === void 0 || resolverInfo === void 0)
+    return { handle, shouldStop: !1 };
+  const shouldStop = await runRecoveredWorkflowHookOrFail(internals, state, handle, resolverInfo, onRecoveredWorkflow, callbacks);
+  return { handle, shouldStop };
 }
 function assertResumeNotTerminating(internals, workflowId) {
   if (internals.terminalizingWorkflows.has(workflowId))
@@ -51,6 +83,9 @@ async function relaunchInlineWorkflowAfterResume(internals, latestState, args) {
   await rehydrateChildCancellationHandlers(internals, workflowId, callbacks);
   const context = new Context({
     workflowId,
+    ...latestState.workflowExecutionToken !== void 0 && {
+      workflowExecutionToken: latestState.workflowExecutionToken
+    },
     workflowType: latestState.type,
     startedAt: getWorkflowExecutionStartedAt(latestState),
     abortController: workflowAbort,
@@ -85,6 +120,9 @@ async function relaunchWorkerWorkflowAfterResume(internals, latestState, args) {
   const serialized = serializeCheckpoint(resumeCheckpoint);
   internals.strategy.startWorkflow({
     workflowId,
+    ...latestState.workflowExecutionToken !== void 0 && {
+      workflowExecutionToken: latestState.workflowExecutionToken
+    },
     workflowType: latestState.type,
     input: latestState.input,
     checkpoint: serialized,
@@ -132,7 +170,7 @@ async function reactivateSuspendedWorkflowState(internals, state) {
     }) : []
   ], [], () => Error(`Resume of workflow "${state.id}" lost its CAS race.`));
 }
-export async function resumeWorkflowFromStorage(internals, workflowId, dispatchResumedEvent, callbacks) {
+export async function resumeWorkflowFromStorage(internals, workflowId, dispatchResumedEvent, callbacks, onRecoveredWorkflow) {
   const stateBytes = await internals.storage.get(KEYS.workflow(workflowId));
   if (!stateBytes)
     throw Error(`Workflow "${workflowId}" not found in storage`);
@@ -150,9 +188,9 @@ export async function resumeWorkflowFromStorage(internals, workflowId, dispatchR
     return callbacks.getHandle(workflowId);
   const workflowStartHeaders = await loadWorkflowStartHeaders(internals, workflowId, callbacks);
   await loadTerminalCleanupTrackedState(internals, workflowId, callbacks);
-  if (await prepareRecoveredServicesOrFail(internals, state, callbacks))
-    return callbacks.getHandle(workflowId);
-  const handle = callbacks.getHandle(workflowId);
+  const { handle, shouldStop } = await prepareRecoveredWorkflowOrFail(internals, state, callbacks, onRecoveredWorkflow);
+  if (shouldStop)
+    return handle;
   await callbacks.runSerializedWorkflowStateWrite(workflowId, () => performSerializedResume(internals, {
     workflowId,
     resumeCheckpoint,

package/dist/core/engine/lifecycle/shared.d.ts

@@ -1,10 +1,35 @@
 import type { ComposedWorkflowInterceptor } from '../../interceptor.ts';
+import type { WorkflowServicesResolverLaunchOptions, WorkflowServicesResolverScheduleInfo } from '../../types.ts';
 import type { QueuedInlineWorkflowExecutionStart } from '../engine-internal-types.ts';
 import { type WorkflowHandle } from '../handles.ts';
 import type { EngineInternals } from '../internals.ts';
 export type RegistrationEntry = EngineInternals['registrations'] extends Map<string, infer Entry> ? Entry : never;
 export declare const FORK_LINEAGE_ATTRIBUTE = "weft:forkedFrom";
 export declare const EMPTY_STORAGE_VALUE: Uint8Array<ArrayBuffer>;
+/**
+ * Durable run context passed to {@link RecoverAllOptions.onRecoveredWorkflow}
+ * before recovered user code advances.
+ *
+ * @example
+ * ```ts
+ * import type { RecoveredWorkflowInfo } from '@lostgradient/weft';
+ *
+ * function registerSurface(info: RecoveredWorkflowInfo): void {
+ *   console.log(`Recovered ${info.workflowType}/${info.workflowId}`);
+ * }
+ * ```
+ */
+export interface RecoveredWorkflowInfo {
+    /** Handle for the recovered run. */
+    handle: WorkflowHandle;
+    workflowId: string;
+    workflowType: string;
+    input: unknown;
+    launchOptions: WorkflowServicesResolverLaunchOptions;
+    schedule?: WorkflowServicesResolverScheduleInfo;
+    /** Re-provided host services, or `undefined` when the run did not use services. */
+    services: unknown;
+}
 /**
  * Options for {@link Engine.recoverAll}. The acknowledgement flag is an
  * explicit escape hatch for deployments that intentionally skip stored
@@ -25,6 +50,13 @@ export type RecoverAllOptions = {
      * rolling deploys or explicit operator storage repair.
      */
     acknowledgeUnknownWorkflowTypes?: boolean;
+    /**
+     * Register consumer-owned live state for each recovered run after services
+     * are re-provided but before the workflow generator advances. The callback is
+     * awaited. If it throws, only that run fails with a `system` failure and
+     * recovery continues with its siblings.
+     */
+    onRecoveredWorkflow?: (info: RecoveredWorkflowInfo) => void | Promise<void>;
 };
 export type LifecycleCallbacks = {
     dispatchEvent: (event: Event) => void;
@@ -56,6 +88,8 @@ export type LifecycleCallbacks = {
      * `error` is the canonical {@link unavailableServicesError}.
      */
     failWorkflowForUnavailableServices: (workflowId: string, error: Error) => Promise<void>;
+    /** Fail one recovered workflow whose pre-resume consumer hook threw. */
+    failWorkflowForRecoveryHook: (workflowId: string, error: Error) => Promise<void>;
     /**
      * Force a recovered workflow to a terminal `failed` state because its
      * persisted checkpoint cannot be decoded on this runtime. The failure is

package/dist/core/engine/lifecycle/start-exec.d.ts

@@ -2,8 +2,8 @@ import type { Checkpoint, StartOptions, WorkflowState } from '../../types.ts';
 import type { EngineInternals } from '../internals.ts';
 import { type LifecycleCallbacks, type RegistrationEntry } from './shared.ts';
 export declare function runWorkflowStartInterceptor(_internals: EngineInternals, workflowId: string, workflowType: string, input: unknown, parentHeaders: Map<string, string> | undefined, callbacks: LifecycleCallbacks): Map<string, string> | undefined;
-export declare function startWorkflowExecution(internals: EngineInternals, workflowId: string, workflowType: string, input: unknown, checkpoint: Checkpoint, nestingDepth: number, executionDeadline: number | undefined, executionStateOwnerId: string, _callbacks?: LifecycleCallbacks): void;
-export declare function beginWorkflowExecution(internals: EngineInternals, workflowId: string, workflowType: string, input: unknown, checkpoint: Checkpoint, executionDeadline: number | undefined, executionStateOwnerId: string, _registration: RegistrationEntry, callbacks: LifecycleCallbacks, onStarted?: () => void): void;
+export declare function startWorkflowExecution(internals: EngineInternals, workflowId: string, workflowExecutionToken: string | undefined, workflowType: string, input: unknown, checkpoint: Checkpoint, nestingDepth: number, executionDeadline: number | undefined, executionStateOwnerId: string, _callbacks?: LifecycleCallbacks): void;
+export declare function beginWorkflowExecution(internals: EngineInternals, workflowId: string, workflowExecutionToken: string | undefined, workflowType: string, input: unknown, checkpoint: Checkpoint, executionDeadline: number | undefined, executionStateOwnerId: string, _registration: RegistrationEntry, callbacks: LifecycleCallbacks, onStarted?: () => void): void;
 /**
  * `defer: false` is an inline-only liveness gate: it awaits the moment the
  * generator is first driven. A worker-mode start queues to the Worker transport

package/dist/core/engine/lifecycle/start-exec.js

@@ -20,12 +20,13 @@ export function runWorkflowStartInterceptor(_internals, workflowId, workflowType
   });
   return capturedHeaders;
 }
-export function startWorkflowExecution(internals, workflowId, workflowType, input, checkpoint, nestingDepth, executionDeadline, executionStateOwnerId, _callbacks) {
+export function startWorkflowExecution(internals, workflowId, workflowExecutionToken, workflowType, input, checkpoint, nestingDepth, executionDeadline, executionStateOwnerId, _callbacks) {
   if (nestingDepth !== 0)
     internals.workflowNestingDepths.set(workflowId, nestingDepth);
   internals.workflowTypeByWorkflowId.set(workflowId, workflowType);
   internals.strategy.startWorkflow({
     workflowId,
+    ...workflowExecutionToken !== void 0 && { workflowExecutionToken },
     workflowType,
     input,
     checkpoint: serializeCheckpoint(checkpoint),
@@ -39,12 +40,13 @@ export function startWorkflowExecution(internals, workflowId, workflowType, inpu
     }
   });
 }
-export function beginWorkflowExecution(internals, workflowId, workflowType, input, checkpoint, executionDeadline, executionStateOwnerId, _registration, callbacks, onStarted) {
+export function beginWorkflowExecution(internals, workflowId, workflowExecutionToken, workflowType, input, checkpoint, executionDeadline, executionStateOwnerId, _registration, callbacks, onStarted) {
   const nestingDepth = internals.pendingNestingDepth ?? 0;
   internals.pendingNestingDepth = void 0;
   if (internals.inlineStrategy !== null) {
     callbacks.queueInlineWorkflowExecutionStart({
       workflowId,
+      ...workflowExecutionToken !== void 0 && { workflowExecutionToken },
       workflowType,
       input,
       checkpoint,
@@ -56,7 +58,7 @@ export function beginWorkflowExecution(internals, workflowId, workflowType, inpu
     return;
   }
   callbacks.dispatchEvent(new WorkflowStartedEvent(workflowId, workflowType, input));
-  startWorkflowExecution(internals, workflowId, workflowType, input, checkpoint, nestingDepth, executionDeadline, executionStateOwnerId, callbacks);
+  startWorkflowExecution(internals, workflowId, workflowExecutionToken, workflowType, input, checkpoint, nestingDepth, executionDeadline, executionStateOwnerId, callbacks);
   onStarted?.();
 }
 export function assertDeferSupported(internals, options, isDelayedStart) {
@@ -71,7 +73,7 @@ export async function beginExecutionAwaitingLiveness(internals, params, workflow
   if (params.isDelayed)
     return;
   const liveness = params.options?.defer === !1 ? Promise.withResolvers() : void 0;
-  beginWorkflowExecution(internals, workflowId, params.type, params.input, params.checkpoint, params.state.executionDeadline, params.state.executionStateOwnerId ?? workflowId, params.registration, callbacks, liveness ? () => liveness.resolve() : void 0);
+  beginWorkflowExecution(internals, workflowId, params.state.workflowExecutionToken, params.type, params.input, params.checkpoint, params.state.executionDeadline, params.state.executionStateOwnerId ?? workflowId, params.registration, callbacks, liveness ? () => liveness.resolve() : void 0);
   if (liveness)
     await liveness.promise;
 }

package/dist/core/engine/lifecycle/start.js

@@ -145,6 +145,7 @@ function buildInitialIdentitySlice(workflowId, type, input, versionTuple, execut
     status: delayedStartTimer ? "pending" : "running",
     input,
     versionTuple,
+    workflowExecutionToken: crypto.randomUUID(),
     ...executionStateOwnerId !== void 0 && { executionStateOwnerId },
     createdAt: now,
     ...!delayedStartTimer && { startedAt: now },

package/dist/core/engine/lifecycle/transition.d.ts

@@ -3,6 +3,6 @@ import { type WorkflowHandle } from '../handles.ts';
 import type { EngineInternals } from '../internals.ts';
 import { type LifecycleCallbacks, type RecoverAllOptions, type RegistrationEntry } from './shared.ts';
 export declare function recoverAll(internals: EngineInternals, callbacks: LifecycleCallbacks, options?: RecoverAllOptions): Promise<WorkflowHandle[]>;
-export declare function resume(internals: EngineInternals, workflowId: string, callbacks: LifecycleCallbacks): Promise<WorkflowHandle>;
+export declare function resume(internals: EngineInternals, workflowId: string, callbacks: LifecycleCallbacks, onRecoveredWorkflow?: RecoverAllOptions['onRecoveredWorkflow']): Promise<WorkflowHandle>;
 export declare function fork(internals: EngineInternals, sourceWorkflowId: string, options: ForkOptions | undefined, callbacks: LifecycleCallbacks): Promise<WorkflowHandle>;
 export declare function launchWorkflowFromCheckpoint(internals: EngineInternals, workflowId: string, state: WorkflowState, checkpoint: Checkpoint, registration: RegistrationEntry, callbacks: LifecycleCallbacks): WorkflowHandle;

package/dist/core/engine/lifecycle/transition.js

@@ -77,7 +77,7 @@ export async function recoverAll(internals, callbacks, options) {
       continue;
     }
     try {
-      handles.push(await resume(internals, entry.workflowId, callbacks));
+      handles.push(await resume(internals, entry.workflowId, callbacks, options?.onRecoveredWorkflow));
     } catch (error) {
       if (error instanceof RegExpExtensionDecodeError) {
         await callbacks.failWorkflowForCheckpointDecodeError(entry.workflowId, error);
@@ -88,7 +88,7 @@ export async function recoverAll(internals, callbacks, options) {
   }
   return handles;
 }
-export async function resume(internals, workflowId, callbacks) {
+export async function resume(internals, workflowId, callbacks, onRecoveredWorkflow) {
   const workflowState = await loadWorkflowState(internals, workflowId);
   if (workflowState !== null) {
     if (callbacks.isInlineWorkflowLocallyOwned(workflowId, workflowState.status) || callbacks.hasLocalCheckpointOwnership(workflowId, workflowState.status)) {
@@ -96,7 +96,7 @@ export async function resume(internals, workflowId, callbacks) {
       return callbacks.getHandle(workflowId);
     }
   }
-  return resumeWorkflowFromStorage(internals, workflowId, !0, callbacks);
+  return resumeWorkflowFromStorage(internals, workflowId, !0, callbacks, onRecoveredWorkflow);
 }
 export async function fork(internals, sourceWorkflowId, options, callbacks) {
   const sourceState = await loadWorkflowState(internals, sourceWorkflowId);
@@ -144,6 +144,9 @@ function launchInlineWorkflowFromCheckpoint(internals, workflowId, state, checkp
   resetCancelHandlers(internals, workflowId);
   const context = new Context({
     workflowId,
+    ...state.workflowExecutionToken !== void 0 && {
+      workflowExecutionToken: state.workflowExecutionToken
+    },
     workflowType: state.type,
     startedAt: getWorkflowExecutionStartedAt(state),
     abortController: workflowAbort,
@@ -172,6 +175,9 @@ function launchWorkerWorkflowFromCheckpoint(internals, workflowId, state, checkp
   const serialized = serializeCheckpoint(checkpoint);
   internals.strategy.startWorkflow({
     workflowId,
+    ...state.workflowExecutionToken !== void 0 && {
+      workflowExecutionToken: state.workflowExecutionToken
+    },
     workflowType: state.type,
     input: state.input,
     checkpoint: serialized,

package/dist/core/engine/lifecycle.d.ts

@@ -1,4 +1,4 @@
-export { EMPTY_STORAGE_VALUE, createWorkflowHandle, loadTerminalCleanupTrackedState, loadWorkflowStartHeaders, normalizeStartWorkflowTags, processPendingUpdatesAfterReplay, setWorkflowStartHeaders, type LifecycleCallbacks, type RecoverAllOptions, } from './lifecycle/shared.ts';
+export { EMPTY_STORAGE_VALUE, createWorkflowHandle, loadTerminalCleanupTrackedState, loadWorkflowStartHeaders, normalizeStartWorkflowTags, processPendingUpdatesAfterReplay, setWorkflowStartHeaders, type LifecycleCallbacks, type RecoverAllOptions, type RecoveredWorkflowInfo, } from './lifecycle/shared.ts';
 export { createWorkflowVersionTuple, derivePreparedExecutionState, prepareResumeState, throwVersionMismatch, workflowStateWithVersionTuple, workflowVersionTupleFromState, } from './lifecycle/persist.ts';
 export { createInitialCheckpoint, createInitialWorkflowState, parseStartOptionDuration, resolveScheduledStartAt, start, startWorkflow, } from './lifecycle/start.ts';
 export { startOrSignal, startWithIdempotency, type StartOrSignalCallbacks, type StartOrSignalResult, } from './lifecycle/start-or-signal.ts';

package/dist/core/engine/memo-durable-activity.js

@@ -149,7 +149,7 @@ class MemoDurableActivityScope {
       finishTimerWrite();
     }
     this.#throwIfClosed();
-    registerSleepResolver(this.#internals, this.#workflowId, operation.operationId, resolve);
+    registerSleepResolver(this.#internals, this.#workflowId, operation.operationId, resolve, operation.scheduledFireAt);
     await promise;
   }
   #activityOperationId(ordinal, attempt) {
@@ -198,15 +198,13 @@ class MemoDurableActivityScope {
         onAbort();
         return;
       }
-      operation.then((value) => {
-        settle(() => {
-          if (this.#closed || signal.aborted) {
-            reject(this.#abortError());
-            return;
-          }
-          resolve(value);
-        });
-      }).catch((error) => {
+      operation.then((value) => settle(() => {
+        if (this.#closed || signal.aborted) {
+          reject(this.#abortError());
+          return;
+        }
+        resolve(value);
+      })).catch((error) => {
         settle(() => reject(error));
       });
     });

package/dist/core/engine/operations-activity.d.ts

@@ -25,7 +25,7 @@ export type ActivityOperationCallbacks = {
 };
 export declare function buildActivityVerification(_internals: EngineInternals, activityName: string, verify: ActivityFunctionWithMetadata['verify'], context: ReturnType<typeof buildActivityVerificationContext>, result: unknown): Promise<void>;
 export declare function buildActivityCompensation(internals: EngineInternals, workflowId: string, operation: ActivityOperation, result: unknown): (() => Promise<void>) | undefined;
-export declare function invokeWorkerActivity(internals: EngineInternals, operationId: string, activityName: string, input: unknown, attempt: number): Promise<unknown>;
+export declare function invokeWorkerActivity(internals: EngineInternals, operationId: string, activityName: string, input: unknown, attempt: number, workflowExecutionToken: string | undefined, activityAttemptToken: string | undefined): Promise<unknown>;
 export declare function invokeInlineActivity(internals: EngineInternals, workflowId: string, operation: ActivityOperation, activityContext: ActivityContext, _activityName: string, input: unknown): unknown;
 /**
  * Execute an activity function, dispatching to a Web Worker pool when