@lostgradient/weft 0.3.0 → 0.4.0

package/dist/core/engine/lease-errors.d.ts

@@ -0,0 +1,99 @@
+import { WeftError } from '../weft-error.ts';
+/**
+ * Thrown when `ownership: 'lease'` is configured and the engine cannot acquire the
+ * storage lease within the configured wait window — another live instance still
+ * holds it. Raised from the lease-acquire step, which runs both inside
+ * {@link Engine.create} and at the top of {@link Engine.recoverAll} (so a
+ * `new Engine({ ownership: 'lease' })` + `recoverAll()` boot must handle it too).
+ * In a rolling deploy this means the
+ * outgoing instance has not released the lease (or its lease has not yet expired)
+ * within `leaseWaitTimeout`; size that window above the outgoing instance's drain
+ * time and above the lease TTL so a crash (no clean release) still resolves once
+ * the held lease expires. Inspect `waitedMs` for how long this instance waited
+ * and `heldBy` for the holder id observed when it gave up (`null` if unknown).
+ *
+ * @example
+ * ```ts
+ * import { EngineLeaseAcquisitionTimeoutError } from '@lostgradient/weft';
+ *
+ * function isLeaseHandoffStuck(error: unknown): boolean {
+ *   return error instanceof EngineLeaseAcquisitionTimeoutError;
+ * }
+ * ```
+ */
+export declare class EngineLeaseAcquisitionTimeoutError extends WeftError<'EngineLeaseAcquisitionTimeoutError'> {
+    readonly waitedMs: number;
+    readonly heldBy: string | null;
+    constructor(waitedMs: number, heldBy: string | null);
+}
+/**
+ * Thrown when `ownership: 'lease'` is configured but the lease keys in storage are
+ * corrupt. Raised from the lease-acquire step, which runs both inside
+ * {@link Engine.create} and at the top of {@link Engine.recoverAll} (so a
+ * `new Engine({ ownership: 'lease' })` + `recoverAll()` boot must handle it too) —
+ * the `lease:epoch` high-water mark is present
+ * but does not decode to a valid epoch, or a holder record exists with no epoch
+ * key (which the lease protocol never produces, since `release()` deletes only the
+ * holder and never the epoch). The lease epoch is the sole source of truth for the
+ * monotonic fencing generation, so a booting instance fails closed rather than
+ * re-minting a generation at or below the true high-water mark and risking a
+ * split-brain. Resolve by operator repair: delete both `lease:` keys only if you
+ * are certain no other instance is running, or reset `lease:epoch` above its last
+ * known value.
+ *
+ * @example
+ * ```ts
+ * import { EngineLeaseCorruptedError } from '@lostgradient/weft';
+ *
+ * function isLeaseCorrupted(error: unknown): boolean {
+ *   return error instanceof EngineLeaseCorruptedError;
+ * }
+ * ```
+ */
+export declare class EngineLeaseCorruptedError extends WeftError<'EngineLeaseCorruptedError'> {
+    constructor(detail: string);
+}
+/**
+ * Thrown by {@link Engine.start} and {@link Engine.startOrSignal} when
+ * `ownership: 'lease'` is configured but the engine does not yet hold the lease —
+ * it was constructed with `new Engine({ ownership: 'lease' })` and used to start
+ * work before `recoverAll()` ran, or created with `recover: false`. In lease mode
+ * an engine must hold the single-writer lease (and have recovered existing runs)
+ * before it durably writes new workflow state, or a fresh start could race an
+ * instance that legitimately owns the store. {@link Engine.create} acquires the
+ * lease and recovers for you; a direct constructor must `await engine.recoverAll()`
+ * first.
+ *
+ * @example
+ * ```ts
+ * import { Engine, EngineLeaseNotHeldError } from '@lostgradient/weft';
+ *
+ * const engine = new Engine({ ownership: 'lease' });
+ * try {
+ *   await engine.start('ping', null);
+ * } catch (error) {
+ *   if (error instanceof EngineLeaseNotHeldError) {
+ *     await engine.recoverAll(); // acquires the lease, then retry the start
+ *   }
+ * }
+ * ```
+ */
+export declare class EngineLeaseNotHeldError extends WeftError<'EngineLeaseNotHeldError'> {
+    constructor();
+}
+/**
+ * Thrown internally on the durable-write path when `ownership: 'lease'` is
+ * configured and a fenced write's CAS fails because the held lease epoch is
+ * stale — a successor instance has taken the lease and bumped the epoch, so this
+ * instance has been deposed. It is the local unwind that stops the commit from
+ * advancing in-memory state past a durable write that did not land; the actual
+ * "this engine is deposed" reaction (set the `deposed` flag, warn the operator,
+ * tear the engine down) is driven by {@link handleDeposition} at the detection
+ * site, not by this error propagating — the inline strategy swallows turn
+ * rejections, so the throw never reaches a handler. Internal-only: it is never
+ * surfaced to user code (deposition reaches operators through the
+ * `WeftEngineLeaseLostWarning` process warning).
+ */
+export declare class EngineDeposedError extends WeftError<'EngineDeposedError'> {
+    constructor();
+}

package/dist/core/engine/lease-errors.js

@@ -0,0 +1,30 @@
+import { WeftError } from "../weft-error.js";
+
+export class EngineLeaseAcquisitionTimeoutError extends WeftError {
+  waitedMs;
+  heldBy;
+  constructor(waitedMs, heldBy) {
+    const heldClause = heldBy === null ? "" : ` (currently held by "${heldBy}")`;
+    super("EngineLeaseAcquisitionTimeoutError", `Could not acquire the storage ownership lease within ${waitedMs}ms${heldClause}. Another engine instance still holds it. Increase leaseWaitTimeout above the outgoing instance's drain time and the lease TTL, or ensure the previous instance releases the lease on shutdown.`);
+    this.waitedMs = waitedMs;
+    this.heldBy = heldBy;
+  }
+}
+
+export class EngineLeaseCorruptedError extends WeftError {
+  constructor(detail) {
+    super("EngineLeaseCorruptedError", `The ownership lease state in storage is corrupt and cannot be used safely: ${detail}. Resolve by operator repair (delete both "lease:" keys only if no other instance is running, or reset "lease:epoch" above its last known value).`);
+  }
+}
+
+export class EngineLeaseNotHeldError extends WeftError {
+  constructor() {
+    super("EngineLeaseNotHeldError", "ownership: 'lease' requires the engine to hold the lease before starting work. Use `Engine.create({ ownership: \"lease\" })` (which acquires the lease and recovers), or call `await engine.recoverAll()` on a directly-constructed engine before `start()`.");
+  }
+}
+
+export class EngineDeposedError extends WeftError {
+  constructor() {
+    super("EngineDeposedError", "Engine was deposed: a fenced durable write lost its CAS race because another instance now holds the ownership lease at a newer epoch. This engine is halting.");
+  }
+}

package/dist/core/engine/lease-manager.d.ts

@@ -0,0 +1,114 @@
+/**
+ * Lease-fenced single-writer ownership over a shared durable store. This is the
+ * opt-in `Engine.create({ ownership: 'lease' })` mechanism — a CORRECTNESS-PATH
+ * coordinator, NOT a best-effort smoke alarm like the second-instance detector.
+ * Lease outcomes are acted on, never silently dropped, but the channel differs by
+ * phase: a failed acquire THROWS and blocks recovery (corruption fails closed, a
+ * timed-out handoff raises a typed error); a renewal reports loss through
+ * `onLeaseLost` rather than throwing — a CAS-failed renewal means a successor took
+ * the lease ('deposed'), and a transient renewal storage error is tolerated until
+ * the lease is too close to lapsing to prove ownership ('renewal-unconfirmable').
+ * In Step 1 that loss is observability only (the engine warns); Step 2 will make it
+ * enforceable by fencing every durable write on the lease epoch.
+ *
+ * **Why a lease.** Weft's supported model is one engine process per durable
+ * store. Without a lease, a rolling deploy briefly runs two engines (old draining
+ * + new booting), both recovering and both issuing at-least-once activities. The
+ * lease turns that into a clean handoff: the booting instance ACQUIRES the lease
+ * before recovering, the draining instance RELEASES it on dispose, and the new
+ * instance only recovers once it owns the lease.
+ *
+ * **Two-key schema (this is load-bearing, not an optimization).** Ownership is
+ * tracked by two keys, never one:
+ *
+ * - `lease:epoch` — an 8-byte big-endian uint64 fencing token. It changes ONLY on
+ *   ownership transfer (cold acquire, or a steal/re-acquire after the prior
+ *   holder's lease lapsed), NEVER on a renewal. It is a monotonic high-water mark
+ *   that SURVIVES release.
+ * - `lease:holder` — a JSON `{ holderId, expiresAt, epoch }` record renewed on
+ *   every heartbeat (its `expiresAt` advances), so its bytes churn constantly.
+ *
+ * They are split because `conditionalBatch` compares the WHOLE stored value as
+ * bytes. If the epoch lived inside the churning holder record, the fencing token
+ * (Step 2) would change on every renewal, spuriously failing the fence and
+ * self-terminating healthy holders. Keeping the epoch in its own stable key lets
+ * a holder cache its epoch once ({@link LeaseManager.currentEpochBytes}) and use
+ * it unchanged as a fencing condition across many renewals.
+ *
+ * **Epoch monotonicity is the anti-split-brain invariant.** Every transfer
+ * conditions on BOTH the holder AND the current epoch and bumps the epoch by one;
+ * `release()` deletes ONLY the holder and never the epoch. This guarantees a
+ * deposed zombie from generation N can never see its epoch re-minted under it —
+ * so once Step 2 fences durable writes on the epoch, the zombie always loses.
+ *
+ * @module core/engine/lease-manager
+ */
+import { type Storage } from '../../storage/interface.ts';
+/** Why a holder lost its lease — surfaced to {@link LeaseManagerOptions.onLeaseLost}. */
+export type LeaseLostReason = 'deposed' | 'renewal-unconfirmable';
+/** Options for {@link createLeaseManager}. */
+export type LeaseManagerOptions = {
+    storage: Storage;
+    /** This engine's unique instance id (the lease holder id). */
+    holderId: string;
+    /** Wall-clock source (ms), injected so tests can advance time deterministically. */
+    getNow: () => number;
+    /** Lease time-to-live (ms). A stolen lease becomes available `ttlMs` after its last renewal. */
+    ttlMs: number;
+    /** Renewal interval (ms). The holder re-asserts the lease this often; must be `< ttlMs`. */
+    renewIntervalMs: number;
+    /** Boot-time wait window (ms) before {@link LeaseManager.acquire} throws. */
+    waitTimeoutMs: number;
+    /** Poll interval (ms) for the acquire wait loop. */
+    acquirePollIntervalMs?: number;
+    /**
+     * Delay primitive for the acquire poll loop. Defaults to a real `setTimeout`.
+     * Injected so tests drive the loop deterministically: a test `delay` advances
+     * the same injected clock by the poll interval and resolves immediately, so the
+     * wait deadline (read from `getNow`) trips without real waiting. Mirrors the
+     * `getNow` injection — not test-only scaffolding.
+     */
+    delay?: (ms: number) => Promise<void>;
+    /**
+     * Called once when this holder loses the lease while running — either CAS-false
+     * on renewal (`'deposed'`: a successor stole it) or storage failures that make
+     * the holder unable to prove it still holds before the lease lapses
+     * (`'renewal-unconfirmable'`). In Step 1 the engine reacts by warning; Step 2
+     * uses this to halt fenced writes. The lease manager never throws into the
+     * renewal timer — it reports through this seam instead.
+     */
+    onLeaseLost?: (reason: LeaseLostReason) => void;
+};
+/**
+ * A running lease manager.
+ *
+ * - `acquire()` blocks until this instance owns the lease, then resolves; throws
+ *   {@link EngineLeaseAcquisitionTimeoutError} on timeout. Call before recovery.
+ *   One exception to "resolved ⇒ owned": if the manager was `stop()`ped (disposal)
+ *   while acquire was waiting, it resolves WITHOUT owning — the engine gates
+ *   recovery on `disposed` so a stopped-then-resolved acquire never proceeds.
+ * - `startRenewal()` begins the heartbeat that keeps the lease held (drives
+ *   {@link LeaseManager.renewOnce} on an interval).
+ * - `renewOnce()` runs a single renewal round; exposed for tests so renewal can be
+ *   driven deterministically with an injected clock, mirroring the detector's
+ *   `tick()`.
+ * - `currentEpochBytes()` returns the held epoch as bytes for Step-2 fencing
+ *   conditions; `null` before a successful acquire. Synchronous and stable across
+ *   renewals.
+ * - `release()` best-effort relinquishes the lease (holder key only) on dispose.
+ */
+export type LeaseManager = {
+    acquire(): Promise<void>;
+    startRenewal(): void;
+    renewOnce(): Promise<void>;
+    currentEpochBytes(): Uint8Array | null;
+    release(): Promise<void>;
+    stop(): void;
+};
+/**
+ * Create a lease manager. Does not start any timer or acquire anything itself —
+ * the engine drives `acquire()` at the boot gate (before recovery) and
+ * `startRenewal()` afterward, and clears the renewal timer through its own
+ * disposal path (mirroring how it owns the second-instance detector's interval).
+ */
+export declare function createLeaseManager(options: LeaseManagerOptions): LeaseManager;

package/dist/core/engine/lease-manager.js

@@ -0,0 +1,135 @@
+import { KEYS, storageConditionalBatch } from "../../storage/interface.js";
+import {
+  decodeEpoch,
+  decodeHolder,
+  encodeEpoch,
+  encodeHolder
+} from "./lease-codec.js";
+import { EngineLeaseAcquisitionTimeoutError, EngineLeaseCorruptedError } from "./lease-errors.js";
+const DEFAULT_ACQUIRE_POLL_INTERVAL_MS = 1000;
+export function createLeaseManager(options) {
+  const { storage, holderId, getNow, ttlMs, renewIntervalMs, waitTimeoutMs } = options, acquirePollIntervalMs = options.acquirePollIntervalMs ?? DEFAULT_ACQUIRE_POLL_INTERVAL_MS, unconfirmableMarginMs = renewIntervalMs;
+  let stopped = !1, heldEpoch = null, heldEpochBytes = null, lastHolderBytes = null, renewalInterval = null, leaseLost = !1, inFlightRenewal = null;
+  const delay = options.delay ?? ((ms) => new Promise((resolve) => setTimeout(resolve, ms)));
+  function reportLeaseLost(reason) {
+    if (leaseLost)
+      return;
+    leaseLost = !0;
+    options.onLeaseLost?.(reason);
+  }
+  async function readState() {
+    const holderRaw = await storage.get(KEYS.leaseHolder()), epochRaw = await storage.get(KEYS.leaseEpoch());
+    return {
+      epochRaw,
+      holderRaw,
+      epoch: epochRaw === null ? null : decodeEpoch(epochRaw),
+      holder: holderRaw === null ? null : decodeHolder(holderRaw)
+    };
+  }
+  async function takeOwnership(expectedEpochBytes, expectedHolderBytes, nextEpoch) {
+    const expiresAt = getNow() + ttlMs, holderBytes = encodeHolder({ holderId, expiresAt, epoch: nextEpoch }), committed = await storageConditionalBatch(storage, [
+      { key: KEYS.leaseEpoch(), expectedValue: expectedEpochBytes },
+      { key: KEYS.leaseHolder(), expectedValue: expectedHolderBytes }
+    ], [
+      { type: "put", key: KEYS.leaseEpoch(), value: encodeEpoch(nextEpoch) },
+      { type: "put", key: KEYS.leaseHolder(), value: holderBytes }
+    ]);
+    if (committed) {
+      heldEpoch = nextEpoch;
+      heldEpochBytes = encodeEpoch(nextEpoch);
+      lastHolderBytes = holderBytes;
+    }
+    return committed;
+  }
+  async function tryAcquireOnce() {
+    const { epochRaw, holderRaw, epoch, holder } = await readState();
+    if (epochRaw !== null && epoch === null)
+      throw new EngineLeaseCorruptedError('the "lease:epoch" high-water mark is present but does not decode to a valid epoch');
+    if (epochRaw === null && holderRaw !== null)
+      throw new EngineLeaseCorruptedError('a "lease:holder" record exists with no "lease:epoch" key (the lease protocol never deletes the epoch)');
+    if (holder !== null && getNow() < holder.expiresAt)
+      return !1;
+    const nextEpoch = (epoch ?? 0) + 1;
+    if (nextEpoch >= Number.MAX_SAFE_INTEGER)
+      throw new EngineLeaseCorruptedError(`the next ownership epoch (${nextEpoch}) is at or above the safe-integer ceiling and cannot be minted without bricking future boots`);
+    return takeOwnership(epochRaw, holderRaw, nextEpoch);
+  }
+  async function acquire() {
+    const startedAt = getNow(), deadline = startedAt + waitTimeoutMs;
+    let lastHolderId = null, firstAttempt = !0;
+    for (;; ) {
+      if (stopped)
+        return;
+      if (!firstAttempt && getNow() >= deadline)
+        throw new EngineLeaseAcquisitionTimeoutError(getNow() - startedAt, lastHolderId);
+      firstAttempt = !1;
+      if (await tryAcquireOnce())
+        return;
+      const holderRaw = await storage.get(KEYS.leaseHolder());
+      lastHolderId = holderRaw === null ? null : decodeHolder(holderRaw)?.holderId ?? null;
+      if (getNow() >= deadline)
+        throw new EngineLeaseAcquisitionTimeoutError(getNow() - startedAt, lastHolderId);
+      await delay(acquirePollIntervalMs);
+    }
+  }
+  async function renewOnce() {
+    if (stopped || leaseLost || heldEpoch === null || lastHolderBytes === null)
+      return;
+    const expiresAt = getNow() + ttlMs, holderBytes = encodeHolder({ holderId, expiresAt, epoch: heldEpoch });
+    let committed;
+    try {
+      committed = await storageConditionalBatch(storage, [{ key: KEYS.leaseHolder(), expectedValue: lastHolderBytes }], [{ type: "put", key: KEYS.leaseHolder(), value: holderBytes }]);
+    } catch {
+      const priorExpiry = decodeHolder(lastHolderBytes)?.expiresAt ?? 0;
+      if (getNow() >= priorExpiry - unconfirmableMarginMs)
+        reportLeaseLost("renewal-unconfirmable");
+      return;
+    }
+    if (committed) {
+      lastHolderBytes = holderBytes;
+      return;
+    }
+    reportLeaseLost("deposed");
+  }
+  function renewUnderGuard() {
+    if (inFlightRenewal !== null)
+      return;
+    inFlightRenewal = renewOnce().catch(() => {}).finally(() => {
+      inFlightRenewal = null;
+    });
+  }
+  function startRenewal() {
+    if (stopped || renewalInterval !== null)
+      return;
+    renewalInterval = setInterval(() => {
+      renewUnderGuard();
+    }, renewIntervalMs);
+    renewalInterval.unref?.();
+  }
+  function currentEpochBytes() {
+    return heldEpochBytes === null ? null : heldEpochBytes.slice();
+  }
+  function clearRenewal() {
+    if (renewalInterval !== null) {
+      clearInterval(renewalInterval);
+      renewalInterval = null;
+    }
+  }
+  async function release() {
+    stopped = !0;
+    clearRenewal();
+    if (inFlightRenewal !== null)
+      await inFlightRenewal;
+    if (lastHolderBytes === null)
+      return;
+    try {
+      if (await storageConditionalBatch(storage, [{ key: KEYS.leaseHolder(), expectedValue: lastHolderBytes }], [{ type: "delete", key: KEYS.leaseHolder() }]))
+        lastHolderBytes = null;
+    } catch {}
+  }
+  function stop() {
+    stopped = !0;
+    clearRenewal();
+  }
+  return { acquire, startRenewal, renewOnce, currentEpochBytes, release, stop };
+}

package/dist/core/engine/lifecycle/persist.js

@@ -1,15 +1,8 @@
-import { serializeCheckpoint } from "../../checkpoint.js";
-import { encode } from "../../codec.js";
-import {
-  VersionMismatchError,
-  buildVersionUpdateOperations,
-  checkVersionCompatibility,
-  migrateCheckpoint
-} from "../../versioning.js";
+import { VersionMismatchError, checkVersionCompatibility } from "../../versioning.js";
 import {
   diffWorkflowVersionTuples
 } from "../../workflow-version-tuple.js";
-import { buildWorkflowVisibilityIndexTransition } from "../workflow-indexes.js";
+import { hydrateCheckpointReplayState } from "../checkpoint-replay.js";
 export function createWorkflowVersionTuple(_internals, registration, _callbacks) {
   return {
     workflowVersion: registration.version
@@ -26,37 +19,22 @@ export function workflowStateWithVersionTuple(internals, state, versionTuple, _c
   };
 }
 export function derivePreparedExecutionState(internals, workflowId, state, checkpoint, registration, callbacks) {
-  const compatibility = checkVersionCompatibility(checkpoint.version, registration.version, !!registration.migrate), registeredVersionTuple = createWorkflowVersionTuple(internals, registration, callbacks), versionDiff = diffWorkflowVersionTuples(workflowVersionTupleFromState(internals, state, callbacks), registeredVersionTuple), hasVersionTupleDrift = versionDiff.workflowVersion !== void 0 || versionDiff.agentVersion !== void 0 || versionDiff.toolVersions !== void 0;
-  if (compatibility === "incompatible" || hasVersionTupleDrift && !registration.migrate)
+  const compatibility = checkVersionCompatibility(checkpoint.version, registration.version), registeredVersionTuple = createWorkflowVersionTuple(internals, registration, callbacks), versionDiff = diffWorkflowVersionTuples(workflowVersionTupleFromState(internals, state, callbacks), registeredVersionTuple), hasVersionTupleDrift = versionDiff.workflowVersion !== void 0 || versionDiff.agentVersion !== void 0 || versionDiff.toolVersions !== void 0;
+  if (compatibility === "incompatible" || hasVersionTupleDrift)
     throwVersionMismatch(internals, workflowId, state, registration, versionDiff, callbacks);
-  let preparedState = state, preparedCheckpoint = checkpoint, shouldPersistPreparedState = !1;
-  if ((compatibility === "needs-migration" || hasVersionTupleDrift) && registration.migrate) {
-    const migrated = migrateCheckpoint(checkpoint, checkpoint.version, registration.version, registration.migrate);
-    migrated.version = registeredVersionTuple.workflowVersion;
-    preparedCheckpoint = migrated;
-    preparedState = workflowStateWithVersionTuple(internals, state, registeredVersionTuple, callbacks);
-    shouldPersistPreparedState = !0;
-  }
   return {
-    state: preparedState,
-    checkpoint: preparedCheckpoint,
+    state,
+    checkpoint,
     versionTuple: registeredVersionTuple,
-    shouldPersistPreparedState
+    shouldPersistPreparedState: !1
   };
 }
 export async function prepareResumeState(internals, workflowId, state, checkpoint, checkpointBytes, registration, callbacks) {
-  const preparedExecutionState = derivePreparedExecutionState(internals, workflowId, state, checkpoint, registration, callbacks);
-  if (preparedExecutionState.shouldPersistPreparedState) {
-    const versionVisibilityOperations = buildWorkflowVisibilityIndexTransition(workflowId, state, preparedExecutionState.state).batchOps;
-    await internals.storage.batch([
-      ...buildVersionUpdateOperations(workflowId, serializeCheckpoint(preparedExecutionState.checkpoint), preparedExecutionState.versionTuple.workflowVersion, encode(preparedExecutionState.state)),
-      ...versionVisibilityOperations
-    ]);
-  }
+  const preparedExecutionState = derivePreparedExecutionState(internals, workflowId, state, checkpoint, registration, callbacks), hydratedCheckpoint = await hydrateCheckpointReplayState(internals.storage, workflowId, preparedExecutionState.checkpoint);
   return {
     state: preparedExecutionState.state,
-    checkpoint: preparedExecutionState.checkpoint,
-    serializedCheckpoint: preparedExecutionState.shouldPersistPreparedState ? serializeCheckpoint(preparedExecutionState.checkpoint) : checkpointBytes,
+    checkpoint: hydratedCheckpoint,
+    serializedCheckpoint: checkpointBytes,
     versionTuple: preparedExecutionState.versionTuple
   };
 }

package/dist/core/engine/lifecycle/recovered-services.d.ts

@@ -16,8 +16,8 @@ import type { EngineInternals } from '../internals.ts';
  * has no marker and proceeds without touching the resolver — so a fail-closed
  * resolver does not fail healthy no-services runs.
  *
- * Returns `false` to proceed (services available, none expected, or no resolver
- * is configured). Returns `true` to STOP — the run was terminally
+ * Returns `false` to proceed (services available or none expected). Returns
+ * `true` to STOP — the run was terminally
  * failed (services unavailable), or the terminal commit faulted and the run was
  * left for a later boot to retry. Either way the generator must not advance:
  * driving it without services would crash the body and that throw would escape
@@ -36,10 +36,12 @@ import type { EngineInternals } from '../internals.ts';
  * @param onCommitError - Records a fail-warn when `failRun` itself throws, so the
  *   swallowed terminal-commit fault is still observable.
  */
-export declare function reprovideRecoveredServices(internals: EngineInternals, state: WorkflowState, failRun: (workflowId: string, error: Error) => Promise<void>, onCommitError: (source: string, error: unknown, workflowId: string) => void): Promise<boolean>;
+export declare function reprovideRecoveredServices(internals: EngineInternals, state: WorkflowState, failRun: (workflowId: string, error: Error) => Promise<void>, onCommitError: (source: string, error: unknown, workflowId: string) => void, dispatchDiagnostic?: (event: Event) => void): Promise<boolean>;
 /**
- * The canonical terminal error for a recovered run whose services could not be
- * re-provided. Shared by both recovery paths so they fail with an identical
- * message (the failure category is `system`, the default for `failWorkflow`).
+ * The canonical terminal error for any run whose services could not be provided —
+ * both recovery re-provision paths and a scheduled-occurrence launch. Shared so
+ * every caller fails with an identical message (the failure category is `system`,
+ * the default for `failWorkflow`). The message is recovery-agnostic because a
+ * freshly launched scheduled occurrence has never been recovered.
  */
 export declare function unavailableServicesError(workflowId: string, reason: string): Error;

package/dist/core/engine/lifecycle/recovered-services.js

@@ -1,12 +1,24 @@
 import { KEYS } from "../../../storage/interface.js";
-export async function reprovideRecoveredServices(internals, state, failRun, onCommitError) {
+import { DevelopmentWarningEvent } from "../../events.js";
+const RESOLVE_WORKFLOW_SERVICES_OPTION_PATH = "EngineOptions.resolveWorkflowServices";
+export async function reprovideRecoveredServices(internals, state, failRun, onCommitError, dispatchDiagnostic = () => {}) {
   const resolver = internals.options.resolveWorkflowServices;
-  if (internals.inlineStrategy === null || !resolver)
+  if (internals.inlineStrategy === null)
     return !1;
   if (internals.workflowServices.has(state.id))
     return !1;
   if (await internals.storage.get(KEYS.workflowHasServices(state.id)) === null)
     return !1;
+  if (!resolver) {
+    const reason = missingServicesResolverReason(state.id);
+    dispatchDiagnostic(new DevelopmentWarningEvent(state.id, reason, [RESOLVE_WORKFLOW_SERVICES_OPTION_PATH]));
+    try {
+      await failRun(state.id, unavailableServicesError(state.id, reason));
+    } catch (error) {
+      onCommitError("reprovideRecoveredServices", error, state.id);
+    }
+    return !0;
+  }
   let reason;
   try {
     const resolution = await resolver({
@@ -30,5 +42,8 @@ export async function reprovideRecoveredServices(internals, state, failRun, onCo
   return !0;
 }
 export function unavailableServicesError(workflowId, reason) {
-  return Error(`Recovered workflow "${workflowId}" services unavailable: ${reason}`);
+  return Error(`Workflow "${workflowId}" services unavailable: ${reason}`);
+}
+function missingServicesResolverReason(workflowId) {
+  return `Workflow "${workflowId}" was started with services, but this engine has no resolveWorkflowServices option. Configure EngineOptions.resolveWorkflowServices before recovery so ctx.services can be re-provided.`;
 }

package/dist/core/engine/lifecycle/resume.js

@@ -7,6 +7,8 @@ import { WorkflowResumedEvent } from "../../events.js";
 import { buildTimerBatchOperations } from "../../scheduler.js";
 import { createCancelHandlerRegistration, resetCancelHandlers } from "../cancel-handlers.js";
 import { rememberCommittedCheckpointBytes } from "../checkpoint-commit-snapshots.js";
+import { rehydrateChildCancellationHandlers } from "../child-workflow-cancellation.js";
+import { commitFencedEngineWrite } from "../fenced-write.js";
 import { getWorkflowExecutionStartedAt } from "../handles.js";
 import { loadWorkflowState } from "../storage-io.js";
 import { getComposedWorkflowInterceptor } from "../strategy-helpers.js";
@@ -21,7 +23,7 @@ import {
   setWorkflowStartHeaders
 } from "./shared.js";
 async function prepareRecoveredServicesOrFail(internals, state, callbacks) {
-  return reprovideRecoveredServices(internals, state, callbacks.failWorkflowForUnavailableServices, callbacks.handleCleanupError);
+  return reprovideRecoveredServices(internals, state, callbacks.failWorkflowForUnavailableServices, callbacks.handleCleanupError, callbacks.dispatchEvent);
 }
 function assertResumeNotTerminating(internals, workflowId) {
   if (internals.terminalizingWorkflows.has(workflowId))
@@ -43,9 +45,10 @@ function commitSerializedResumeState(internals, args) {
   setWorkflowStartHeaders(internals, workflowId, args.workflowStartHeaders, callbacks);
   internals.parkedInlineWorkflows.delete(workflowId);
 }
-function relaunchInlineWorkflowAfterResume(internals, latestState, args) {
+async function relaunchInlineWorkflowAfterResume(internals, latestState, args) {
   const { workflowId, resumeCheckpoint, registration, callbacks } = args, accumulatedResults = new Map(resumeCheckpoint.accumulatedResults), workflowAbort = new AbortController;
   resetCancelHandlers(internals, workflowId);
+  await rehydrateChildCancellationHandlers(internals, workflowId, callbacks);
   const context = new Context({
     workflowId,
     workflowType: latestState.type,
@@ -65,7 +68,8 @@ function relaunchInlineWorkflowAfterResume(internals, latestState, args) {
     ...latestState.executionDeadline !== void 0 && {
       deadline: latestState.executionDeadline
     },
-    services: internals.workflowServices.get(workflowId)
+    services: internals.workflowServices.get(workflowId),
+    ...internals.options.onLog != null && { logSink: internals.options.onLog }
   });
   setContextWorkflowInterceptor(context, getComposedWorkflowInterceptor(internals));
   if (internals.options.development)
@@ -74,8 +78,11 @@ function relaunchInlineWorkflowAfterResume(internals, latestState, args) {
   inlineStrategy.adoptWorkflow(workflowId, generator, context, workflowAbort);
   inlineStrategy.continueWorkflow(workflowId, void 0);
 }
-function relaunchWorkerWorkflowAfterResume(internals, latestState, args) {
-  const { workflowId, resumeCheckpoint, workflowStartHeaders } = args, serialized = serializeCheckpoint(resumeCheckpoint);
+async function relaunchWorkerWorkflowAfterResume(internals, latestState, args) {
+  const { workflowId, resumeCheckpoint, workflowStartHeaders, callbacks } = args;
+  resetCancelHandlers(internals, workflowId);
+  await rehydrateChildCancellationHandlers(internals, workflowId, callbacks);
+  const serialized = serializeCheckpoint(resumeCheckpoint);
   internals.strategy.startWorkflow({
     workflowId,
     workflowType: latestState.type,
@@ -103,10 +110,10 @@ async function performSerializedResume(internals, args) {
   await reactivateSuspendedWorkflowState(internals, latestState);
   commitSerializedResumeState(internals, args);
   if (internals.inlineStrategy) {
-    relaunchInlineWorkflowAfterResume(internals, latestState, args);
+    await relaunchInlineWorkflowAfterResume(internals, latestState, args);
     return;
   }
-  relaunchWorkerWorkflowAfterResume(internals, latestState, args);
+  await relaunchWorkerWorkflowAfterResume(internals, latestState, args);
 }
 async function reactivateSuspendedWorkflowState(internals, state) {
   if (state.status !== "suspended")
@@ -114,7 +121,7 @@ async function reactivateSuspendedWorkflowState(internals, state) {
   const previousState = { ...state };
   state.status = "running";
   state.updatedAt = internals.options.getNow();
-  await internals.storage.batch([
+  await commitFencedEngineWrite(internals, [
     { type: "put", key: KEYS.workflow(state.id), value: encode(state) },
     ...buildWorkflowVisibilityIndexTransition(state.id, previousState, state).batchOps,
     ...state.executionDeadline !== void 0 ? buildTimerBatchOperations({
@@ -123,7 +130,7 @@ async function reactivateSuspendedWorkflowState(internals, state) {
       fireAt: state.executionDeadline,
       kind: "execution-deadline"
     }) : []
-  ]);
+  ], [], () => Error(`Resume of workflow "${state.id}" lost its CAS race.`));
 }
 export async function resumeWorkflowFromStorage(internals, workflowId, dispatchResumedEvent, callbacks) {
   const stateBytes = await internals.storage.get(KEYS.workflow(workflowId));

package/dist/core/engine/lifecycle/shared.d.ts

@@ -22,7 +22,7 @@ export declare const EMPTY_STORAGE_VALUE: Uint8Array<ArrayBuffer>;
 export type RecoverAllOptions = {
     /**
      * Skip stored running workflows whose type is not registered. Use only for
-     * rolling deploys or explicit storage migrations.
+     * rolling deploys or explicit operator storage repair.
      */
     acknowledgeUnknownWorkflowTypes?: boolean;
 };
@@ -56,6 +56,13 @@ export type LifecycleCallbacks = {
      * `error` is the canonical {@link unavailableServicesError}.
      */
     failWorkflowForUnavailableServices: (workflowId: string, error: Error) => Promise<void>;
+    /**
+     * Force a recovered workflow to a terminal `failed` state because its
+     * persisted checkpoint cannot be decoded on this runtime. The failure is
+     * isolated to the affected run so `recoverAll()` can continue recovering
+     * other workflows from the same storage backend.
+     */
+    failWorkflowForCheckpointDecodeError: (workflowId: string, error: Error) => Promise<void>;
 };
 /**
  * Pre-replay history circuit breaker. Called at every restore-from-checkpoint

package/dist/core/engine/lifecycle/start-batch.d.ts

@@ -2,6 +2,6 @@ import type { BatchOperation } from '../../../storage/interface.ts';
 import type { Checkpoint, SearchAttributeValue, StartOptions, TimerEntry, WorkflowState } from '../../types.ts';
 import type { EngineInternals } from '../internals.ts';
 import { type LifecycleCallbacks, type RegistrationEntry } from './shared.ts';
-export declare function buildStartBatchOperations(_internals: EngineInternals, workflowId: string, state: WorkflowState, checkpoint: Checkpoint, registration: RegistrationEntry, options: StartOptions | undefined, executionDeadline: number | undefined, delayedStartTimer: TimerEntry | undefined, workflowStartHeaders: Map<string, string> | undefined, additionalOperations: BatchOperation[] | undefined, callbacks: LifecycleCallbacks): BatchOperation[];
+export declare function buildStartBatchOperations(_internals: EngineInternals, workflowId: string, state: WorkflowState, checkpoint: Checkpoint, registration: RegistrationEntry, options: StartOptions | undefined, executionDeadline: number | undefined, delayedStartTimer: TimerEntry | undefined, workflowStartHeaders: Map<string, string> | undefined, additionalOperations: BatchOperation[] | undefined, callbacks: LifecycleCallbacks, purgeDeleteOperations: BatchOperation[] | undefined): BatchOperation[];
 export declare function buildInitialSearchAttributeOperations(_internals: EngineInternals, workflowId: string, registration: RegistrationEntry, searchAttributes: StartOptions['searchAttributes'], callbacks: LifecycleCallbacks): BatchOperation[];
 export declare function validateSearchAttributes(_internals: EngineInternals, registration: RegistrationEntry, searchAttributes: Record<string, SearchAttributeValue>, _callbacks: LifecycleCallbacks): void;

package/dist/core/engine/lifecycle/start-batch.js

@@ -8,8 +8,9 @@ import { validateAttributeValueSizes } from "../attributes-tags.js";
 import { encodeWorkflowStartHeaders } from "../state-utilities.js";
 import { buildWorkflowVisibilityIndexOperations } from "../workflow-indexes.js";
 import { EMPTY_STORAGE_VALUE } from "./shared.js";
-export function buildStartBatchOperations(_internals, workflowId, state, checkpoint, registration, options, executionDeadline, delayedStartTimer, workflowStartHeaders, additionalOperations, callbacks) {
+export function buildStartBatchOperations(_internals, workflowId, state, checkpoint, registration, options, executionDeadline, delayedStartTimer, workflowStartHeaders, additionalOperations, callbacks, purgeDeleteOperations) {
   const visibilityIndexOperations = buildWorkflowVisibilityIndexOperations(workflowId, null, state).batchOps, operations = [
+    ...purgeDeleteOperations ?? [],
     { type: "put", key: KEYS.workflow(workflowId), value: encode(state) },
     {
       type: "put",