@lostgradient/weft 0.3.0 → 0.5.0

package/dist/core/engine/index.js

@@ -1,4 +1,7 @@
-import { KEYS } from "../../storage/interface.js";
+import {
+  KEYS,
+  requireStorageCapability
+} from "../../storage/interface.js";
 import {
   ActivityRegistry
 } from "../activity-registry.js";
@@ -24,6 +27,7 @@ import {
   cancelAll as cancelAllWorkflows,
   deleteAll as deleteAllWorkflows,
   purge as purgeWorkflows,
+  retryFailedAll as retryFailedAllWorkflows,
   signalAll as signalAllWorkflows,
   tagAll as tagAllWorkflows,
   untagAll as untagAllWorkflows
@@ -68,7 +72,9 @@ import {
   drainQueuedInlineWorkflowStartsForEngine,
   isActivityDefinition
 } from "./engine-runtime-helpers.js";
-import { EngineCreateNameMismatchError } from "./errors.js";
+import { EngineCreateNameMismatchError, EngineDisposedError } from "./errors.js";
+import { assertLeaseHeldForEngineWork, commitFencedEngineWrite } from "./fenced-write.js";
+import { recordFinalizerState } from "./finalizer-state.js";
 import {
   createWorkflowHandleWithResultPromise as createWorkflowHandleWithResultPromiseFromInternals,
   getWorkflowResultPromise as getWorkflowResultPromiseFromInternals
@@ -80,6 +86,8 @@ import {
   resumeParkedInlineWorkflow as resumeParkedInlineWorkflowFromInternals
 } from "./inline-parking.js";
 import { getInternals, initializeInternals } from "./internals.js";
+import { ENGINE_LEASE_LOST_WARNING_NAME, handleDeposition } from "./lease-deposition.js";
+import { createLeaseManager } from "./lease-manager.js";
 import {
   fork as forkFromLifecycle,
   recoverAll as recoverAllFromLifecycle,
@@ -124,6 +132,7 @@ import {
   cancelSchedule as cancelScheduleFromInternals,
   listSchedules as listSchedulesFromInternals,
   pauseSchedule as pauseScheduleFromInternals,
+  recoverOrphanedScheduleTimers,
   resumeSchedule as resumeScheduleFromInternals,
   schedule as scheduleFromInternals,
   toScheduleSummary,
@@ -174,12 +183,19 @@ export {
   PersistedDataIncompatibleError,
   StartOrSignalConflictError,
   WorkflowAlreadyExistsError,
+  WorkflowConcurrencyLimitExceededError,
   WorkflowNotFoundError,
   WorkflowNotRegisteredError,
   WorkflowSuspendNotSupportedError,
+  WorkflowTeardownPendingError,
   WorkflowTypeNotRegisteredForRecoveryError
 } from "./errors.js";
 export { HANDLE_RESULT_PROMISE, WorkflowHandle } from "./handles.js";
+export {
+  EngineLeaseAcquisitionTimeoutError,
+  EngineLeaseCorruptedError,
+  EngineLeaseNotHeldError
+} from "./lease-errors.js";
 export { ScheduleHandle } from "./schedule-handle.js";
 export {
   clearEngineLeakWarningTokenForTesting,
@@ -192,6 +208,8 @@ export {
 export { assertCompatiblePersistedDataVersion };
 export const ENGINE_PARKED_WORKFLOW_COUNT_FOR_TESTING = Symbol("engineParkedWorkflowCountForTesting"), ENGINE_SIGNAL_WAITER_COUNT_FOR_TESTING = Symbol("engineSignalWaiterCountForTesting"), ENGINE_SLEEP_RESOLVER_COUNT_FOR_TESTING = Symbol("engineSleepResolverCountForTesting");
 
+export { ENGINE_LEASE_LOST_WARNING_NAME };
+
 export class Engine extends EventTarget {
   static async create(options) {
     const engine = new Engine(options);
@@ -207,8 +225,11 @@ export class Engine extends EventTarget {
           throw new EngineCreateNameMismatchError("workflow", name, definition.name);
         engine.register(definition);
       }
-      if (options.recover !== !1)
+      await engine.#acquireLeaseIfConfigured();
+      if (options.recover !== !1) {
         await engine.recoverAll(options.acknowledgeUnknownWorkflowTypes !== void 0 ? { acknowledgeUnknownWorkflowTypes: options.acknowledgeUnknownWorkflowTypes } : {});
+        getInternals(engine).scheduler.start();
+      }
     } catch (error) {
       await engine[Symbol.asyncDispose]();
       throw error;
@@ -230,7 +251,9 @@ export class Engine extends EventTarget {
       getComposedWorkflowInterceptor: () => getComposedWorkflowInterceptor(getInternals(this)),
       resolveWorkflowType: this.#resolveWorkflowTypeTarget.bind(this),
       registerCancelHandler: (workflowId, handler) => registerCancelHandler(getInternals(this), workflowId, handler),
-      getWorkflowServices: (workflowId) => getInternals(this).workflowServices.get(workflowId)
+      recordFinalizerState: (workflowId, value) => recordFinalizerState(getInternals(this), workflowId, value),
+      getWorkflowServices: (workflowId) => getInternals(this).workflowServices.get(workflowId),
+      getLogSink: () => resolvedOptions.onLog ?? void 0
     });
     getInternals(this).storage = storage;
     getInternals(this).abortController = new AbortController;
@@ -239,6 +262,7 @@ export class Engine extends EventTarget {
     getInternals(this).resultResolvers = new Map;
     getInternals(this).signalWaiters = new Map;
     getInternals(this).signalWaitersByWorkflow = new Map;
+    getInternals(this).conditionWaiters = new Map;
     getInternals(this).updateWaiters = new Map;
     getInternals(this).updateWaitersByWorkflow = new Map;
     getInternals(this).sleepResolvers = new Map;
@@ -265,7 +289,8 @@ export class Engine extends EventTarget {
     getInternals(this).scheduler = new Scheduler({
       storage,
       onTimerFired: (entry) => handleTimerFiredFromInternals(getInternals(this), entry, this.#createTimeOperationCallbacks()),
-      getNow
+      getNow,
+      commitTimerCleanup: (operations) => commitFencedEngineWrite(getInternals(this), operations, [], () => Error("Timer cleanup lost its fenced CAS race"))
     });
     getInternals(this).strategy = strategyBundle.strategy;
     getInternals(this).inlineStrategy = strategyBundle.inlineStrategy;
@@ -278,8 +303,10 @@ export class Engine extends EventTarget {
     if (queuedInlineWorkflowStartChannel !== null)
       queuedInlineWorkflowStartChannel.port1.onmessage = createQueuedInlineWorkflowStartHandler(weakEngine, queuedInlineWorkflowStartChannel);
     getInternals(this).heartbeatDetails = new Map;
+    getInternals(this).lastHeartbeatDetailsByStep = new Map;
     getInternals(this).workflowServices = new Map;
     getInternals(this).pendingAsyncActivities = new Map;
+    getInternals(this).pendingAsyncActivityResolutions = new Map;
     getInternals(this).pendingStarts = new Set;
     getInternals(this).pendingScheduleCreations = new Set;
     getInternals(this).workflowsNeedingTerminalCleanup = new Set;
@@ -290,10 +317,12 @@ export class Engine extends EventTarget {
     getInternals(this).workflowReviewIds = new Map;
     getInternals(this).parkedInlineWorkflows = new Set;
     getInternals(this).terminalizingWorkflows = new Set;
+    getInternals(this).deliveredPendingUpdateIds = new Map;
     getInternals(this).cancelHandlersByWorkflow = new Map;
     getInternals(this).reviewTimerIds = new Map;
     getInternals(this).pendingWebhooks = new Set;
     getInternals(this).pendingTimelineEntries = new Map;
+    getInternals(this).pendingAtomicWorkflowCommitSideEffects = new Map;
     getInternals(this).cleanupIntervalDisposalTracker = null;
     const cleanupIntervalDisposalTracker = {
       disposed: !1,
@@ -310,15 +339,90 @@ export class Engine extends EventTarget {
     getInternals(this).nextRetentionSweepAt = null;
     getInternals(this).secondInstanceDetectionInterval = null;
     getInternals(this).secondInstanceDetector = null;
+    getInternals(this).leaseManager = null;
+    getInternals(this).inFlightLeaseAcquire = null;
+    getInternals(this).deposed = !1;
+    getInternals(this).tearDownAfterDeposition = () => {
+      this.#disposeAfterDeposition();
+    };
     getInternals(this).eventLogHeads = new Map;
     getInternals(this).workflowFeedListeners = new Map;
     getInternals(this).workflowVersionTuples = new Map;
+    getInternals(this).workflowVisibilityWatermark = void 0;
+    getInternals(this).workflowVisibilityWatermarkExpiresAt = void 0;
     getInternals(this).activityWorkerDispatcher = createActivityWorkerDispatcher(options?.activityExecution);
     getInternals(this).strategy.onMessage(this.#handleStrategyMessage.bind(this));
     getInternals(this).alertManager = createAlertManagerForEngine(this, options?.alerts, getNow);
     this.#ensureRetentionSweepInterval();
     this.#startSecondInstanceDetection();
   }
+  addEventListener(type, listener, options) {
+    super.addEventListener(type, listener, options);
+  }
+  removeEventListener(type, listener, options) {
+    super.removeEventListener(type, listener, options);
+  }
+  async#acquireLeaseIfConfigured() {
+    const internals = getInternals(this);
+    if (internals.options.ownershipMode !== "lease")
+      return;
+    if (internals.disposed)
+      throw new EngineDisposedError;
+    if (internals.inFlightLeaseAcquire !== null) {
+      await internals.inFlightLeaseAcquire;
+      return;
+    }
+    if (internals.leaseManager !== null)
+      return;
+    requireStorageCapability(internals.storage, "conditionalBatch", "ownership: 'lease'");
+    const manager = createLeaseManager({
+      storage: internals.storage,
+      holderId: crypto.randomUUID(),
+      getNow: internals.options.getNow,
+      ttlMs: internals.options.leaseTtlMs,
+      renewIntervalMs: internals.options.leaseRenewIntervalMs,
+      waitTimeoutMs: internals.options.leaseWaitTimeoutMs,
+      onLeaseLost: (reason) => {
+        if (reason === "deposed") {
+          handleDeposition(internals);
+          return;
+        }
+        process.emitWarning("engine ownership lease renewal could not be confirmed (transient storage error); another instance may take over if this persists. Weft supports one engine process " + "per store \u2014 keep infrastructure-level single-instance enforcement in place.", ENGINE_LEASE_LOST_WARNING_NAME);
+      }
+    });
+    internals.leaseManager = manager;
+    const acquisition = (async () => {
+      try {
+        await manager.acquire();
+      } catch (error) {
+        if (internals.leaseManager === manager)
+          internals.leaseManager = null;
+        throw error;
+      }
+      if (internals.disposed) {
+        await manager.release();
+        if (internals.leaseManager === manager)
+          internals.leaseManager = null;
+        throw new EngineDisposedError;
+      }
+      manager.startRenewal();
+    })();
+    internals.inFlightLeaseAcquire = acquisition;
+    try {
+      await acquisition;
+    } finally {
+      if (internals.inFlightLeaseAcquire === acquisition)
+        internals.inFlightLeaseAcquire = null;
+    }
+  }
+  #disposeAfterDeposition() {
+    const internals = getInternals(this);
+    if (internals.disposed)
+      return;
+    const leaseManager = internals.leaseManager;
+    disposeEngine(internals);
+    leaseManager?.release().catch(() => {});
+  }
   #startSecondInstanceDetection() {
     const internals = getInternals(this);
     if (!internals.options.secondInstanceDetectionEnabled)
@@ -437,11 +541,13 @@ export class Engine extends EventTarget {
     return getInternals(this).activityRegistry.listDefinitions();
   }
   async start(type, input, options) {
+    assertLeaseHeldForEngineWork(getInternals(this));
     if (options?.idempotencyKey !== void 0)
       return startWithIdempotencyFromLifecycle(getInternals(this), type, input, options, this.#createLifecycleCallbacks());
     return startWorkflowFromLifecycle(getInternals(this), type, input, options, void 0, this.#createLifecycleCallbacks());
   }
   async startOrSignal(type, input, signal, options) {
+    assertLeaseHeldForEngineWork(getInternals(this));
     return startOrSignalFromLifecycle(getInternals(this), type, input, signal, options, this.#createStartOrSignalCallbacks());
   }
   #createStartOrSignalCallbacks() {
@@ -474,6 +580,9 @@ export class Engine extends EventTarget {
   async cancelAll(filter, options) {
     return cancelAllWorkflows(getInternals(this), filter, options);
   }
+  async retryFailedAll(filter, options) {
+    return retryFailedAllWorkflows(getInternals(this), filter, options);
+  }
   async signalAll(filter, name, payloadOrOptions, options) {
     if (options === void 0)
       return signalAllWorkflows(getInternals(this), filter, name, payloadOrOptions);
@@ -494,7 +603,8 @@ export class Engine extends EventTarget {
       return scheduleFromInternals(getInternals(this), workflowType, definition.input, definitionSpec, {
         ...definition.id !== void 0 && { id: definition.id },
         ...definition.overlapPolicy !== void 0 && { overlap: definition.overlapPolicy },
-        ...definition.backfill !== void 0 && { backfill: definition.backfill }
+        ...definition.backfill !== void 0 && { backfill: definition.backfill },
+        ...definition.jitter !== void 0 && { jitter: definition.jitter }
       });
     }
     if (spec === void 0)
@@ -554,20 +664,32 @@ export class Engine extends EventTarget {
     return getOffloadFromInternals(getInternals(this), workflowId, key);
   }
   async fork(sourceWorkflowId, options) {
+    assertLeaseHeldForEngineWork(getInternals(this));
     return forkFromLifecycle(getInternals(this), sourceWorkflowId, options, this.#createLifecycleCallbacks());
   }
   async resume(workflowId) {
+    assertLeaseHeldForEngineWork(getInternals(this));
     return resumeFromLifecycle(getInternals(this), workflowId, this.#createLifecycleCallbacks());
   }
   async recoverAll(options) {
+    await this.#acquireLeaseIfConfigured();
+    if (getInternals(this).disposed)
+      throw new EngineDisposedError;
     await recoverPendingAsyncActivities(getInternals(this));
+    await recoverOrphanedScheduleTimers(getInternals(this));
     return recoverAllFromLifecycle(getInternals(this), this.#createLifecycleCallbacks(), options);
   }
   async completeAsyncActivity(token, result) {
-    await completeAsyncActivityFromInternals(getInternals(this), token, result, (workflowId, outcome) => feedOperationResult(getInternals(this), workflowId, outcome), (workflowId, status, output) => finalizePendingTimelineEntry(getInternals(this), workflowId, status, output));
+    await completeAsyncActivityFromInternals(getInternals(this), token, result, {
+      feedOperationResult: (workflowId, outcome) => feedOperationResult(getInternals(this), workflowId, outcome),
+      finalizeTimeline: (workflowId, status, output) => finalizePendingTimelineEntry(getInternals(this), workflowId, status, output)
+    });
   }
   async failAsyncActivity(token, error) {
-    await failAsyncActivityFromInternals(getInternals(this), token, error, (workflowId, outcome, originalReason) => feedOperationResult(getInternals(this), workflowId, outcome, originalReason), (workflowId, status, output) => finalizePendingTimelineEntry(getInternals(this), workflowId, status, output));
+    await failAsyncActivityFromInternals(getInternals(this), token, error, {
+      feedOperationResult: (workflowId, outcome, originalReason) => feedOperationResult(getInternals(this), workflowId, outcome, originalReason),
+      finalizeTimeline: (workflowId, status, output) => finalizePendingTimelineEntry(getInternals(this), workflowId, status, output)
+    });
   }
   async cancel(workflowId) {
     await cancelWorkflowFromTermination(getInternals(this), workflowId, this.#createTerminationCallbacks());
@@ -647,14 +769,19 @@ export class Engine extends EventTarget {
     return submitCoordinatedUpdateFromInternals(getInternals(this), workflowId, name, payload, options, this.#createUpdateCallbacks());
   }
   [Symbol.dispose]() {
+    const leaseManager = getInternals(this).leaseManager;
     disposeEngine(getInternals(this));
+    leaseManager?.release();
   }
   async[Symbol.asyncDispose]() {
     if (!getInternals(this).disposed) {
+      const leaseManager = getInternals(this).leaseManager;
       try {
         await drainQueuedInlineWorkflowStartsForEngine(this);
       } finally {
-        this[Symbol.dispose]();
+        disposeEngine(getInternals(this));
+        await getInternals(this).inFlightLeaseAcquire?.catch(() => {});
+        await leaseManager?.release();
       }
       return;
     }

package/dist/core/engine/inline-parking.js

@@ -1,3 +1,4 @@
+import { hasExposedAccessors, hasUpdateHandlers } from "../context/context-presence.js";
 import {
   resumeWorkflowFromStorage as resumeWorkflowFromStorageFromLifecycle
 } from "./lifecycle.js";
@@ -9,7 +10,7 @@ export async function parkInlineWorkflowAfterCheckpoint(internals, workflowId, o
   if (operation.type !== "wait-signal" || internals.inlineStrategy === null)
     return !1;
   const inlineStrategy = internals.inlineStrategy, context = inlineStrategy.getContext(workflowId);
-  if (context?.hasUpdateHandlers || context?.hasExposedAccessors)
+  if (context && (hasUpdateHandlers(context) || hasExposedAccessors(context)))
     return !1;
   if (await callbacks.hasBufferedSignal(workflowId, operation.signalName))
     return !1;
@@ -17,7 +18,7 @@ export async function parkInlineWorkflowAfterCheckpoint(internals, workflowId, o
     const latestState = await callbacks.loadWorkflowState(workflowId);
     if (internals.terminalizingWorkflows.has(workflowId) || !latestState || latestState.status !== "running")
       return !1;
-    inlineStrategy.parkWorkflow(workflowId);
+    inlineStrategy.parkWorkflow(workflowId, { retainContext: !0 });
     internals.parkedInlineWorkflows.add(workflowId);
     return !0;
   }))

package/dist/core/engine/internals.d.ts

@@ -28,16 +28,19 @@ import type { InlineExecutionStrategy } from '../inline-execution-strategy.ts';
 import type { ComposedActivityInterceptor, ComposedWorkflowInterceptor, Interceptor } from '../interceptor.ts';
 import type { HumanReviewResult, ReviewCoordinator } from '../review/index.ts';
 import type { Scheduler } from '../scheduler.ts';
-import type { Checkpoint } from '../types.ts';
+import type { Checkpoint, StartWorkflowOptions } from '../types.ts';
 import type { UpdateCoordinator } from '../updates.ts';
 import type { WorkflowVersionTuple } from '../workflow-version-tuple.ts';
 import type { PendingTimelineEntry, QueuedInlineWorkflowExecutionStart, RegistrationEntry, ResolvedOptions, TrackedWaiterKeys, WorkflowResultWaiter } from './engine-internal-types.ts';
 import type { EngineCleanupIntervalDisposalTracker } from './engine-leak-warnings.ts';
 import type { WorkflowHandle, WorkflowHandleEngine } from './handles.ts';
 import type { WorkflowFeedListener } from './index.ts';
+import type { LeaseManager } from './lease-manager.ts';
 import type { ScheduleHandleEngine } from './schedule-handle.ts';
 import type { SecondInstanceDetector } from './second-instance-detector.ts';
-type EngineRuntime = WorkflowHandleEngine & ScheduleHandleEngine;
+type EngineRuntime = WorkflowHandleEngine & ScheduleHandleEngine & {
+    start(type: string, input: unknown, options?: StartWorkflowOptions): Promise<WorkflowHandle>;
+};
 export interface EngineInternals {
     engine: EngineRuntime;
     storage: WeftStorage;
@@ -67,6 +70,23 @@ export interface EngineInternals {
     resultResolvers: Map<string, WorkflowResultWaiter>;
     signalWaiters: Map<string, () => void>;
     signalWaitersByWorkflow: Map<string, TrackedWaiterKeys>;
+    /**
+     * In-process resolvers for inline `ctx.waitUntil` waits, keyed by `workflowId`
+     * ALONE — a workflow has at most one active wait-condition because its inline
+     * generator is suspended at exactly one yield, and the top-level-only guard in
+     * `executeSubOperation` rejects `waitUntil` as a `race`/`all`/`speculate`
+     * branch. So unlike `signalWaiters` (which need a per-workflow string-or-Set
+     * index because they CAN be concurrent sub-operations), this is a flat map with
+     * no secondary index. Calling the resolver wakes `processWaitConditionOperation`
+     * to re-evaluate its predicate. Never checkpointed — engine-memory state cleared
+     * on terminal cleanup.
+     *
+     * Note: the deadline TIMER is still keyed `cond:${workflowId}:${step}` (step is
+     * stable across replay, so recovery does not double-arm). Only this waiter map
+     * keys by `workflowId`. If `waitUntil`-in-`race`/`all`/`speculate` is ever
+     * supported, this keying must revert to `${workflowId}:${step}` + a Set index.
+     */
+    conditionWaiters: Map<string, () => void>;
     updateWaiters: Map<string, (payload: unknown) => void>;
     updateWaitersByWorkflow: Map<string, TrackedWaiterKeys>;
     sleepResolvers: Map<string, () => void>;
@@ -110,11 +130,25 @@ export interface EngineInternals {
     broadcastChannel: BroadcastChannel | null;
     pendingNestingDepth: number | undefined;
     pendingParentHeaders: Map<string, string> | undefined;
-    pendingExecutionStateOwnerId: string | undefined;
+    pendingExecutionStateOwnerId: string | null | undefined;
     workflowNestingDepths: Map<string, number>;
     workflowHeaders: Map<string, Map<string, string>>;
     workflowStateWriteChains: Map<string, Promise<void>>;
     heartbeatDetails: Map<string, unknown>;
+    /**
+     * Last heartbeat payload PER ACTIVITY STEP, so a retry of a step can read the
+     * heartbeat its previous attempt recorded (the resumable-batch pattern). Keyed
+     * `workflowId -> step -> details`. This is separate from {@link heartbeatDetails}
+     * (which is keyed by `workflowId` alone and powers the `activityProgress` query):
+     * concurrent activities inside one `ctx.all` would clobber a workflow-keyed map,
+     * so a retry could read a sibling's heartbeat. The step is stable across attempts
+     * (assigned once at `stepIndex++`), so a retry reads its OWN prior heartbeat.
+     * Never checkpointed — held only in engine memory and cleared (by workflowId, the
+     * outer key) on terminal cleanup and purge, the same lifecycle as
+     * {@link heartbeatDetails}. Inline-execution only; worker-executed activities run
+     * their function out of process and never observe this.
+     */
+    lastHeartbeatDetailsByStep: Map<string, Map<number, unknown>>;
     /**
      * Per-run, non-serialized `services` value exposed to the workflow body as
      * `ctx.services`. Set at `engine.start({ services })` and re-provided on
@@ -130,6 +164,12 @@ export interface EngineInternals {
      * reloaded by `recoverAll()`. See `async-activity-completion.ts`.
      */
     pendingAsyncActivities: Map<string, import('./async-activity-completion.ts').PendingAsyncActivity>;
+    /**
+     * Completed or failed async-activity tokens that were consumed before inline
+     * recovery adopted the workflow generator. Replay drains these by workflow id
+     * when it reaches the same deterministic async-activity token again.
+     */
+    pendingAsyncActivityResolutions: Map<string, import('./async-activity-completion.ts').PendingAsyncActivityResolution[]>;
     pendingStarts: Set<string>;
     pendingScheduleCreations: Set<string>;
     workflowsNeedingTerminalCleanup: Set<string>;
@@ -142,6 +182,45 @@ export interface EngineInternals {
     secondInstanceDetectionInterval: ReturnType<typeof setInterval> | null;
     /** The active second-instance detector; `null` when detection is disabled. */
     secondInstanceDetector: SecondInstanceDetector | null;
+    /**
+     * The active lease manager for `ownership: 'lease'`; `null` when ownership is
+     * `'none'`. Acquired before recovery, renewed on a heartbeat, released on
+     * dispose. Unlike the second-instance detector its acquire/renew errors are real
+     * (a failed acquire blocks recovery). Renewal loss is surfaced as a warning in
+     * Step 1; epoch fencing of durable writes (Step 2) is what makes it enforceable.
+     */
+    leaseManager: LeaseManager | null;
+    /**
+     * The in-flight lease acquisition, or `null` when none is running. Set while
+     * `#acquireLeaseIfConfigured` awaits `acquire()` (which can park for the whole
+     * `leaseWaitTimeout` waiting for a handoff) and cleared when it settles. Disposal
+     * awaits this so a dispose that races a parked acquire is a clean handoff — the
+     * holder this engine may take is released before `asyncDispose` resolves, rather
+     * than leaking until TTL on an already-disposed engine.
+     */
+    inFlightLeaseAcquire: Promise<void> | null;
+    /**
+     * Set to `true` the instant this engine is detected as deposed under
+     * `ownership: 'lease'` — either a fenced durable write's CAS failed against a
+     * newer epoch ({@link commitFencedEngineWrite}) or the lease manager reported
+     * `onLeaseLost('deposed')`. It is the synchronous half of the deposition halt:
+     * {@link commitFencedEngineWrite} short-circuits at its top on this flag, so a
+     * write that *starts* after detection is rejected before reaching storage,
+     * while writes already in flight are caught by the epoch CAS itself. Set once
+     * by {@link handleDeposition}, which also warns the operator and schedules a
+     * deferred engine teardown. Always `false` under `ownership: 'none'`.
+     */
+    deposed: boolean;
+    /**
+     * Teardown callback invoked (deferred to a later tick) when this engine is
+     * deposed. Set once during engine construction to the engine's own
+     * `disposeAfterDeposition` method. It exists as an injected field rather than a
+     * direct import so {@link handleDeposition} (reached from the durable-write
+     * helper) does not statically import `disposal.ts`, which would close an import
+     * cycle (`storage-io → fenced-write → lease-deposition → disposal → … →
+     * storage-io`). `null` until the constructor wires it.
+     */
+    tearDownAfterDeposition: (() => void) | null;
     reviewCoordinator: ReviewCoordinator;
     reviewWaiters: Map<string, (decision: HumanReviewResult) => void>;
     reviewWaitersByWorkflow: Map<string, TrackedWaiterKeys>;
@@ -152,6 +231,22 @@ export interface EngineInternals {
     workflowReviewIds: Map<string, Set<string>>;
     parkedInlineWorkflows: Set<string>;
     terminalizingWorkflows: Set<string>;
+    /**
+     * Coordinated update IDs already claimed for delivery by a pending-update
+     * drain, keyed by workflow. Several drain triggers (each `update()` schedules a
+     * `setTimeout(0)` drain; the post-advance path drains too) can fire
+     * near-simultaneously, and the durable consume-delete (`buildResponseOperations`
+     * deletes the pending key via async `storage.batch`) lags the in-memory
+     * `getPendingUpdates` scan — so overlapping drains would re-read and re-deliver
+     * the same buffered update. Each drain claims an update's id SYNCHRONOUSLY (no
+     * `await` between the membership check and the add) before delivering it, so a
+     * racing drain that scans the same id skips it. The claim persists across
+     * drains (unlike a per-drain guard), which is what makes delivery idempotent
+     * against the cross-drain race. Cleared per workflow on terminal cleanup; empty
+     * after a crash, which matches durable state (recovery re-delivers exactly the
+     * updates whose delete never committed).
+     */
+    deliveredPendingUpdateIds: Map<string, Set<string>>;
     cancelHandlersByWorkflow: Map<string, Array<() => Promise<void> | void>>;
     reviewTimerIds: Map<string, string[]>;
     pendingWebhooks: Set<AbortController>;
@@ -159,7 +254,16 @@ export interface EngineInternals {
     eventLogHeads: Map<string, Readonly<EventHeadRecord>>;
     workflowFeedListeners: Map<string, Set<WorkflowFeedListener>>;
     workflowVersionTuples: Map<string, WorkflowVersionTuple>;
+    /**
+     * Cached visibility-index watermark for query planning. `undefined` means the
+     * watermark has not been read yet or an in-process caller invalidated it.
+     * `workflowVisibilityWatermarkExpiresAt` bounds how long external maintenance
+     * commands can leave this process planning against a stale watermark.
+     */
+    workflowVisibilityWatermark: 'current' | 'stale' | undefined;
+    workflowVisibilityWatermarkExpiresAt: number | undefined;
     pendingTimelineEntries: Map<string, PendingTimelineEntry>;
+    pendingAtomicWorkflowCommitSideEffects: Map<string, import('./checkpoint-side-effects.ts').AtomicWorkflowCommitSideEffects>;
 }
 /**
  * Set up an empty `EngineInternals` skeleton in the WeakMap. The Engine

package/dist/core/engine/lease-codec.d.ts

@@ -0,0 +1,35 @@
+/**
+ * Pure codecs for the two lease ownership keys (`lease:epoch` and `lease:holder`).
+ * Extracted from `lease-manager.ts` to keep that file under the `max-lines` ceiling;
+ * these functions have no engine state, so they live and test cleanly on their own.
+ *
+ * @module core/engine/lease-codec
+ */
+/** The decoded `lease:holder` record. */
+export type LeaseHolderRecord = {
+    holderId: string;
+    /** Wall-clock ms (from the engine's `getNow`) after which the lease may be stolen. */
+    expiresAt: number;
+    /** The ownership epoch this holder acquired. Mirrors `lease:epoch`. */
+    epoch: number;
+};
+/**
+ * Encode an 8-byte big-endian uint64 epoch. `DataView.setBigUint64` is the
+ * standard, overflow-safe encoder — preferred over a hand-rolled byte loop.
+ */
+export declare function encodeEpoch(epoch: number): Uint8Array;
+/**
+ * Decode an 8-byte big-endian uint64 epoch, or `null` when the stored value is
+ * not a usable epoch — not exactly 8 bytes, or outside `[1, MAX_SAFE_INTEGER)`.
+ * The epoch is the monotonic fencing high-water mark, so it must stay exactly
+ * comparable AND have room to increment: acquisition always mints `epoch + 1`, so
+ * `Number.MAX_SAFE_INTEGER` itself is rejected too — minting `2^53` would fail
+ * `Number.isSafeInteger` on the next boot and brick the lease. A value at or above
+ * the safe-integer ceiling (or below `1`) routes to the corruption path
+ * (fail-closed at boot) rather than silently re-minting an imprecise generation.
+ */
+export declare function decodeEpoch(raw: Uint8Array): number | null;
+/** Encode a holder record to its stored JSON bytes. */
+export declare function encodeHolder(record: LeaseHolderRecord): Uint8Array;
+/** Decode a stored holder record, tolerating any malformed/foreign value as `null`. */
+export declare function decodeHolder(raw: Uint8Array): LeaseHolderRecord | null;

package/dist/core/engine/lease-codec.js

@@ -0,0 +1,32 @@
+const textEncoder = new TextEncoder, textDecoder = new TextDecoder;
+export function encodeEpoch(epoch) {
+  const bytes = new Uint8Array(8);
+  new DataView(bytes.buffer).setBigUint64(0, BigInt(epoch), !1);
+  return bytes;
+}
+export function decodeEpoch(raw) {
+  if (raw.byteLength !== 8)
+    return null;
+  const value = new DataView(raw.buffer, raw.byteOffset, raw.byteLength).getBigUint64(0, !1), numberValue = Number(value);
+  return Number.isSafeInteger(numberValue) && numberValue >= 1 && numberValue < Number.MAX_SAFE_INTEGER ? numberValue : null;
+}
+export function encodeHolder(record) {
+  return textEncoder.encode(JSON.stringify(record));
+}
+function isRecord(value) {
+  return typeof value === "object" && value !== null;
+}
+export function decodeHolder(raw) {
+  let parsed;
+  try {
+    parsed = JSON.parse(textDecoder.decode(raw));
+  } catch {
+    return null;
+  }
+  if (!isRecord(parsed))
+    return null;
+  const { holderId, expiresAt, epoch } = parsed;
+  if (typeof holderId !== "string" || typeof expiresAt !== "number" || !Number.isSafeInteger(expiresAt) || expiresAt < 0 || typeof epoch !== "number" || !Number.isSafeInteger(epoch) || epoch < 1)
+    return null;
+  return { holderId, expiresAt, epoch };
+}

package/dist/core/engine/lease-deposition.d.ts

@@ -0,0 +1,66 @@
+/**
+ * Deposition handling for `ownership: 'lease'` Step-2 fencing (issue #470).
+ *
+ * "Deposed" means another instance now holds the ownership lease at a strictly
+ * newer epoch, so this instance has lost the store. Deposition is detected at two
+ * sites and both funnel here:
+ *
+ * - A fenced durable write's CAS fails against a newer epoch
+ *   ({@link commitFencedEngineWrite} re-reads `lease:epoch` to disambiguate).
+ * - The lease manager's renewal reports `onLeaseLost('deposed')` (a successor
+ *   stole the holder out from under an otherwise-idle engine).
+ *
+ * The renewal-loss reason `'renewal-unconfirmable'` is deliberately NOT routed
+ * here: it is a transient storage blip (the holder could not prove it still
+ * holds), and halting a healthy holder on a momentary storage hiccup would be a
+ * self-inflicted outage. Only a confirmed `'deposed'` triggers the halt.
+ *
+ * {@link handleDeposition} is idempotent (guarded on {@link EngineInternals.deposed})
+ * and does the minimum synchronously — set the flag, warn the operator — then
+ * schedules engine teardown on a later tick. Teardown is never run inline:
+ * detection happens mid-commit while a workflow generator is advancing, and
+ * synchronous {@link disposeEngine} clears maps that the unwinding commit and
+ * other in-flight workflows still touch. The flag (plus the
+ * {@link commitFencedEngineWrite} top short-circuit) carries the halt guarantee;
+ * deferring the actual dispose by a tick is safe because every write in that
+ * window is still fenced on the now-stale epoch and loses its CAS.
+ *
+ * This module is allow-listed for import only from `src/core/engine/**`.
+ */
+import type { EngineInternals } from './internals.ts';
+/**
+ * The `name` of the `process` warning emitted when an engine configured with
+ * `ownership: 'lease'` can no longer be sure it owns the store. The same name
+ * covers all three signals, so a single `process.on('warning')` handler catches
+ * every "this engine may no longer own the store" event:
+ *
+ * - **deposed (confirmed)** — a renewal's CAS failed (a successor stole the lease),
+ *   or a fenced durable write was rejected because a newer epoch holds the store
+ *   (Step 2). The engine halts.
+ * - **renewal-unconfirmable (transient)** — a storage error left the holder unable
+ *   to prove it still holds within the lease window. This is NOT a confirmed
+ *   deposition; the engine keeps running, and a later renewal or fenced write
+ *   resolves the truth. The warning is informational so operators can investigate.
+ *
+ * Filter on this name to react to ownership-loss signals (alert, drain, restart).
+ *
+ * @example
+ * ```ts
+ * import { ENGINE_LEASE_LOST_WARNING_NAME } from '@lostgradient/weft';
+ *
+ * process.on('warning', (warning) => {
+ *   if (warning.name === ENGINE_LEASE_LOST_WARNING_NAME) {
+ *     // This engine may no longer own the store — investigate / drain / restart.
+ *   }
+ * });
+ * ```
+ */
+export declare const ENGINE_LEASE_LOST_WARNING_NAME = "WeftEngineLeaseLostWarning";
+/**
+ * React to a confirmed deposition. Idempotent: the first call sets the deposed
+ * flag, emits the operator warning, and schedules a deferred engine teardown;
+ * subsequent calls are no-ops. Returns immediately — the caller is expected to
+ * then throw {@link EngineDeposedError} to unwind the current commit so it does
+ * not advance in-memory state past a durable write that did not land.
+ */
+export declare function handleDeposition(internals: EngineInternals): void;

package/dist/core/engine/lease-deposition.js

@@ -0,0 +1,10 @@
+export const ENGINE_LEASE_LOST_WARNING_NAME = "WeftEngineLeaseLostWarning";
+export function handleDeposition(internals) {
+  if (internals.deposed)
+    return;
+  internals.deposed = !0;
+  process.emitWarning("engine deposed: a durable write was fenced out because another instance now holds this store's ownership lease at a newer epoch. This engine is halting to avoid split-brain writes. Weft supports one engine process per store; ensure your deploy hands off cleanly (graceful shutdown releases the lease) and keep infrastructure-level single-instance enforcement in place.", ENGINE_LEASE_LOST_WARNING_NAME);
+  const tearDown = internals.tearDownAfterDeposition;
+  if (tearDown !== null)
+    Promise.resolve().then(tearDown).catch(() => {});
+}