@lostgradient/weft 0.2.1 → 0.4.0

package/dist/core/engine/termination/cleanup.js

@@ -1,14 +1,19 @@
-import { KEYS, encodeStorageKeyComponent } from "../../../storage/interface.js";
+import { KEYS, encodeStorageKeyComponent, storageHas } from "../../../storage/interface.js";
 import { CleanupWarningEvent } from "../../events.js";
 import { asyncActivityWorkflowPrefix } from "../async-activity-completion.js";
 import { forgetCommittedCheckpointBytes } from "../checkpoint-commit-snapshots.js";
 import { parseTerminalCleanupTimerId, workflowFeedListenerKey } from "../state-utilities.js";
+import { releaseWorkflowConcurrencySlot } from "../workflow-concurrency.js";
 export const TERMINAL_WORKFLOW_STATUSES = new Set([
   "completed",
   "failed",
   "cancelled",
   "timed-out"
-]);
+]), FORCIBLY_TERMINABLE_STATUSES = [
+  "running",
+  "pending",
+  "suspended"
+];
 function selectTrackedWaiterMaps(internals, kind) {
   return {
     signal: () => ({
@@ -48,6 +53,24 @@ function cleanupSleepResolvers(internals, workflowId) {
   }
   internals.sleepResolversByWorkflow.delete(workflowId);
 }
+function evictSleepResolversWithoutResolving(internals, workflowId) {
+  const sleepOps = internals.sleepResolversByWorkflow.get(workflowId);
+  if (!sleepOps)
+    return;
+  for (const operationId of sleepOps)
+    internals.sleepResolvers.delete(`${workflowId}:${operationId}`);
+  internals.sleepResolversByWorkflow.delete(workflowId);
+}
+function cleanupConditionWaiters(internals, workflowId) {
+  const resolver = internals.conditionWaiters.get(workflowId);
+  if (!resolver)
+    return;
+  resolver();
+  internals.conditionWaiters.delete(workflowId);
+}
+function evictConditionWaitersWithoutResolving(internals, workflowId) {
+  internals.conditionWaiters.delete(workflowId);
+}
 function cleanupReviewEscalations(internals, workflowId, callbacks) {
   const reviewIds = internals.workflowReviewIds.get(workflowId);
   if (!reviewIds)
@@ -68,15 +91,25 @@ export function cleanupWaiters(internals, workflowId, callbacks) {
   for (const kind of TRACKED_WAITER_KINDS)
     cleanupTrackedWaiter(internals, workflowId, kind);
   cleanupSleepResolvers(internals, workflowId);
+  cleanupConditionWaiters(internals, workflowId);
   cleanupReviewEscalations(internals, workflowId, callbacks);
+  internals.deliveredPendingUpdateIds.delete(workflowId);
   internals.workflowNestingDepths.delete(workflowId);
   internals.workflowHeaders.delete(workflowId);
   internals.workflowServices.delete(workflowId);
   internals.workflowTypeByWorkflowId.delete(workflowId);
 }
+export function evictSuspendedWorkflowWaiters(internals, workflowId, callbacks) {
+  for (const kind of TRACKED_WAITER_KINDS)
+    cleanupTrackedWaiter(internals, workflowId, kind);
+  evictSleepResolversWithoutResolving(internals, workflowId);
+  evictConditionWaitersWithoutResolving(internals, workflowId);
+  cleanupReviewEscalations(internals, workflowId, callbacks);
+}
 export async function cleanupWorkflowStorage(internals, workflowId, includeOutputArtifacts) {
   const encodedWorkflowId = encodeStorageKeyComponent(workflowId), prefixes = [
     KEYS.activityReconciliationPrefix(workflowId),
+    KEYS.childCancellationPrefix(workflowId),
     asyncActivityWorkflowPrefix(workflowId),
     KEYS.signalAcceptedResponsePrefix(workflowId),
     `sig:${encodedWorkflowId}:`,
@@ -87,7 +120,10 @@ export async function cleanupWorkflowStorage(internals, workflowId, includeOutpu
     prefixes.push(`offload:${encodedWorkflowId}:`, `blob:${encodedWorkflowId}:`);
   await internals.storage.delete(KEYS.workflowHeaders(workflowId));
   await internals.storage.delete(KEYS.signalSequence(workflowId));
+  await releaseWorkflowConcurrencySlot(internals, workflowId);
   await internals.storage.delete(KEYS.workflowHasServices(workflowId));
+  if (!await storageHas(internals.storage, KEYS.teardownOwed(workflowId)))
+    await internals.storage.delete(KEYS.finalizerState(workflowId));
   if (internals.storage.deletePrefix) {
     for (const prefix of prefixes)
       await internals.storage.deletePrefix(prefix);
@@ -114,12 +150,18 @@ export function cleanupTerminalWorkflowMemory(internals, workflowId, callbacks)
   forgetCommittedCheckpointBytes(internals, workflowId);
   internals.checkpoints.delete(workflowId);
   internals.heartbeatDetails.delete(workflowId);
+  internals.lastHeartbeatDetailsByStep.delete(workflowId);
   for (const [token, pending] of internals.pendingAsyncActivities)
     if (pending.workflowId === workflowId)
       internals.pendingAsyncActivities.delete(token);
   internals.eventLogHeads.delete(workflowId);
   internals.pendingTimelineEntries.delete(workflowId);
+  internals.pendingAsyncActivityResolutions ??= new Map;
+  internals.pendingAsyncActivityResolutions.delete(workflowId);
+  internals.pendingAtomicWorkflowCommitSideEffects ??= new Map;
+  internals.pendingAtomicWorkflowCommitSideEffects.delete(workflowId);
   internals.parkedInlineWorkflows.delete(workflowId);
+  internals.inlineStrategy?.clearParkedContext(workflowId);
   internals.workflowVersionTuples.delete(workflowId);
   internals.cancelHandlersByWorkflow.delete(workflowId);
   internals.workflowFeedListeners.delete(workflowFeedListenerKey(workflowId, "events"));

package/dist/core/engine/termination/complete.js

@@ -18,15 +18,22 @@ import {
 } from "../attributes-tags.js";
 import { TERMINAL_CLEANUP_DELAY_MS } from "../bulk-operations.js";
 import { takeCancelHandlers } from "../cancel-handlers.js";
+import { pendingAtomicWorkflowCommitSideEffectsStagePut } from "../checkpoint-side-effects.js";
 import { getWorkflowExecutionStartedAt } from "../handles.js";
 import { dropQueuedInlineWorkflowStart } from "../inline-launch-queue.js";
 import { EMPTY_STORAGE_VALUE } from "../lifecycle.js";
-import { createTerminalCleanupTimerId, summarizeTimelineValue } from "../state-utilities.js";
+import {
+  createTeardownTimerId,
+  createTerminalCleanupTimerId,
+  summarizeTimelineValue
+} from "../state-utilities.js";
+import { releaseWorkflowConcurrencySlot } from "../workflow-concurrency.js";
 import { buildWorkflowVisibilityIndexTransition } from "../workflow-indexes.js";
 import {
   cleanupTerminalWorkflowImmediately,
   cleanupTerminalWorkflowSynchronously,
-  finalizeScheduledWorkflowTerminal
+  finalizeScheduledWorkflowTerminal,
+  FORCIBLY_TERMINABLE_STATUSES
 } from "./cleanup.js";
 async function runCancelHandlers(handlers, callbacks, workflowId) {
   for (const handler of handlers)
@@ -53,17 +60,26 @@ export async function terminateWorkflow(internals, workflowId, status, callbacks
   dropQueuedInlineWorkflowStart(internals, workflowId);
   internals.strategy.cancelWorkflow(workflowId);
   try {
-    const attributeBytes = await internals.storage.get(KEYS.attribute(workflowId)), attributes = attributeBytes ? decode(attributeBytes) : {}, retainedAttributes = buildRetainedTerminalSearchAttributes(attributes), terminationMessage = status === "timed-out" ? "Workflow timed out" : "Workflow cancelled", terminationResult = await updateWorkflowState(internals, workflowId, { status, ...reason !== void 0 ? { terminationReason: reason } : {} }, {
-      allowedStatuses: ["running", "pending"],
+    const terminalCleanupToken = internals.workflowsNeedingTerminalCleanup.has(workflowId) ? crypto.randomUUID() : void 0, attributeBytes = await internals.storage.get(KEYS.attribute(workflowId)), attributes = attributeBytes ? decode(attributeBytes) : {}, finalizerStatePresent = await internals.storage.get(KEYS.finalizerState(workflowId)) !== null || pendingAtomicWorkflowCommitSideEffectsStagePut(internals, workflowId, KEYS.finalizerState(workflowId)), retainedAttributes = buildRetainedTerminalSearchAttributes(attributes), terminationMessage = status === "timed-out" ? "Workflow timed out" : "Workflow cancelled", terminationResult = await updateWorkflowState(internals, workflowId, {
+      status,
+      ...reason !== void 0 ? { terminationReason: reason } : {},
+      ...terminalCleanupToken !== void 0 ? { terminalCleanupToken } : {}
+    }, {
+      allowedStatuses: FORCIBLY_TERMINABLE_STATUSES,
       buildAdditionalOperations: (_previousState, updatedAt) => {
         finalizePendingTimelineEntry(internals, workflowId, status, terminationMessage, updatedAt);
         const pendingTimelineOperation = buildPendingTimelineOperation(internals, workflowId);
-        return pendingTimelineOperation ? [pendingTimelineOperation] : [];
+        return [
+          ...pendingTimelineOperation ? [pendingTimelineOperation] : [],
+          ...terminalCleanupToken !== void 0 ? buildTerminalCleanupTimerOperations(internals, workflowId, !0, updatedAt, terminalCleanupToken) : [],
+          ...buildTeardownOperations(workflowId, finalizerStatePresent, updatedAt)
+        ];
       }
     });
     if (!terminationResult)
       return;
     await runCancellationHandlersForStatus(internals, workflowId, status, callbacks);
+    await releaseWorkflowConcurrencySlot(internals, workflowId);
     const { previousState, updatedAt } = terminationResult, elapsed = updatedAt - getWorkflowExecutionStartedAt(previousState);
     await cleanupAttributeIndex(internals, workflowId, attributes);
     await writeRetainedTerminalSearchAttributes(internals, workflowId, retainedAttributes);
@@ -164,11 +180,14 @@ export async function completeWorkflow(internals, workflowId, result, callbacks)
     appendSearchAttributeOperations(completionOperations, workflowId, currentAttributes);
     if (terminalCleanupToken !== void 0)
       completionOperations.push(...buildTerminalCleanupTimerOperations(internals, workflowId, !1, now, terminalCleanupToken));
-    await callbacks.commitWorkflowStateOperations(state, completionOperations);
+    await callbacks.commitWorkflowStateOperations(state, completionOperations, {
+      includePendingAtomicSideEffects: !0
+    });
     return { duration };
   });
   if (!completionMetadata)
     return;
+  await releaseWorkflowConcurrencySlot(internals, workflowId);
   notifyCompletionWaiters(internals, workflowId, result, completionMetadata.duration, callbacks);
 }
 export async function failWorkflow(internals, workflowId, error, callbacks, failureCategory = "system") {
@@ -178,18 +197,24 @@ export async function failWorkflow(internals, workflowId, error, callbacks, fail
     status: "failed",
     error: error.message,
     failureCategory
-  };
+  }, terminalCleanupToken = internals.workflowsNeedingTerminalCleanup.has(workflowId) ? crypto.randomUUID() : void 0;
   if (error.stack !== void 0)
     stateUpdate.errorStack = error.stack;
+  if (terminalCleanupToken !== void 0)
+    stateUpdate.terminalCleanupToken = terminalCleanupToken;
   if (!await updateWorkflowState(internals, workflowId, stateUpdate, {
-    allowedStatuses: ["running", "pending"],
+    allowedStatuses: FORCIBLY_TERMINABLE_STATUSES,
     buildAdditionalOperations: (_previousState, updatedAt) => {
       finalizePendingTimelineEntry(internals, workflowId, "failed", error.message, updatedAt);
       const pendingTimelineOperation = buildPendingTimelineOperation(internals, workflowId);
-      return pendingTimelineOperation ? [pendingTimelineOperation] : [];
+      return [
+        ...pendingTimelineOperation ? [pendingTimelineOperation] : [],
+        ...terminalCleanupToken !== void 0 ? buildTerminalCleanupTimerOperations(internals, workflowId, !1, updatedAt, terminalCleanupToken) : []
+      ];
     }
   }))
     return;
+  await releaseWorkflowConcurrencySlot(internals, workflowId);
   await cleanupAttributeIndex(internals, workflowId, attributes);
   callbacks.swallowPromiseRejection(internals.scheduler.cancel(`deadline:${workflowId}`, workflowId));
   await writeRetainedTerminalSearchAttributes(internals, workflowId, retainedAttributes);
@@ -218,6 +243,20 @@ export function buildTerminalCleanupTimerOperations(_internals, workflowId, incl
     kind: "terminal-cleanup"
   });
 }
+function buildTeardownOperations(workflowId, finalizerStatePresent, terminalizedAt) {
+  if (!finalizerStatePresent)
+    return [];
+  const token = crypto.randomUUID(), claim = { status: "owed", attempts: 0, token };
+  return [
+    { type: "put", key: KEYS.teardownOwed(workflowId), value: encode(claim) },
+    ...buildTimerBatchOperations({
+      id: createTeardownTimerId(token),
+      workflowId,
+      fireAt: terminalizedAt,
+      kind: "teardown"
+    })
+  ];
+}
 export async function ensureTerminalCleanupTracked(internals, workflowId) {
   if (internals.workflowsNeedingTerminalCleanup.has(workflowId))
     return;

package/dist/core/engine/termination/finalizer-activity.d.ts

@@ -0,0 +1,53 @@
+/**
+ * The `runFinalizerActivity` primitive (#446 Phase 2). A workflow's definition-level
+ * `finalizer` is an ordinary activity, but it runs POST-terminal — after the engine
+ * has evicted the workflow's generator and abort controller — so it cannot go through
+ * the live-workflow `executeActivity` path: that path reads
+ * `inlineStrategy.getAbortController(workflowId)` (gone after cancel) and the
+ * per-workflow type/heartbeat bookkeeping (swept by terminal cleanup). This primitive
+ * therefore owns its OWN `AbortController`, resolves the implementation from the
+ * engine-lifetime registration entry, and never touches the operation-result feedback
+ * channel.
+ *
+ * Outer retry/backoff is the teardown timer reschedule in `runWorkflowFinalizer`, NOT
+ * the activity's own `retry` policy — so this primitive runs exactly one attempt and
+ * surfaces success or a thrown error to the caller. The activity's `timeout` is honored
+ * as a per-attempt cap.
+ *
+ * @module core/engine/termination/finalizer-activity
+ */
+import type { ActivityContext, Duration } from '../../types.ts';
+/**
+ * The minimal finalizer shape this primitive uses. A registered finalizer is stored
+ * as `AnyActivityDefinition`, whose `execute` is typed `ActivityFunction<never>` (the
+ * input is contravariantly `never`) and which erases `timeout` at the type level —
+ * neither shape is directly invokable with `unknown` input. The drive narrows the
+ * registry entry to this structural type, which is what the engine actually relies on
+ * at teardown: a named, callable activity with an optional per-attempt `timeout`.
+ */
+export interface RunnableFinalizer {
+    readonly name: string;
+    readonly timeout?: Duration;
+    execute(input: unknown, context?: ActivityContext): unknown;
+}
+/** Outcome of a single finalizer attempt. */
+export type FinalizerAttemptResult = {
+    ok: true;
+} | {
+    ok: false;
+    error: unknown;
+    abortedByShutdown: boolean;
+};
+/**
+ * Run one attempt of a workflow's finalizer activity against the recorded finalizer
+ * state, under a caller-supplied abort signal (the engine shutdown signal) composed
+ * with the activity's own per-attempt timeout. Resolves to a structured result rather
+ * than throwing, so the drive can branch on success / retryable-failure / dead-letter
+ * without a try/catch around it.
+ *
+ * @param finalizer - the resolved finalizer activity definition (from the engine-lifetime registry)
+ * @param input - the decoded `ctx.setFinalizerState` payload passed as the activity input
+ * @param attempt - 1-based attempt number, for the per-attempt timeout error message
+ * @param shutdownSignal - the engine's dispose/shutdown abort signal; aborting it stops a cooperating finalizer
+ */
+export declare function runFinalizerActivity(finalizer: RunnableFinalizer, input: unknown, attempt: number, shutdownSignal: AbortSignal): Promise<FinalizerAttemptResult>;

package/dist/core/engine/termination/finalizer-activity.js

@@ -0,0 +1,30 @@
+import { parseDuration } from "../../scheduler.js";
+import { withPerAttemptTimeout } from "../activity-per-attempt-timeout.js";
+export async function runFinalizerActivity(finalizer, input, attempt, shutdownSignal) {
+  const perAttemptTimeoutMs = finalizer.timeout === void 0 ? void 0 : parseDuration(finalizer.timeout), attemptController = new AbortController, relayShutdownAbort = () => attemptController.abort(shutdownSignal.reason);
+  if (shutdownSignal.aborted)
+    attemptController.abort(shutdownSignal.reason);
+  else
+    shutdownSignal.addEventListener("abort", relayShutdownAbort, { once: !0 });
+  const activityContext = {
+    signal: attemptController.signal,
+    heartbeat: () => {},
+    completeAsync: () => {
+      throw Error(`Finalizer "${finalizer.name}" called ctx.completeAsync(), which is not supported in a workflow finalizer: the finalizer runs after the workflow is already terminal, so there is no durable step to complete out of band. Run the teardown synchronously instead.`);
+    }
+  };
+  try {
+    await withPerAttemptTimeout(finalizer.execute(input, activityContext), perAttemptTimeoutMs, finalizer.name, attempt, attemptController);
+    if (shutdownSignal.aborted)
+      return {
+        ok: !1,
+        error: Error(`Finalizer "${finalizer.name}" resolved while the engine was shutting down; treating attempt ${attempt} as shutdown-aborted so the teardown re-runs after restart.`, { cause: shutdownSignal.reason }),
+        abortedByShutdown: !0
+      };
+    return { ok: !0 };
+  } catch (error) {
+    return { ok: !1, error, abortedByShutdown: shutdownSignal.aborted };
+  } finally {
+    shutdownSignal.removeEventListener("abort", relayShutdownAbort);
+  }
+}

package/dist/core/engine/termination/finalizer-claim.d.ts

@@ -0,0 +1,110 @@
+/**
+ * Durable-claim mechanics for the workflow finalizer drive (issue #446 Phase 2).
+ * This module owns the `teardownOwed` marker's lifecycle — the stale-claim reclaim
+ * horizon, the retry backoff schedule, the dead-letter record shape, and every fenced
+ * write that mutates the marker (claim CAS, settle CAS, re-arm, clear, dead-letter). The
+ * orchestration that decides WHEN to call these lives in `./finalizer.ts`; keeping the
+ * byte-level claim transitions here keeps each module under the size budget and isolates
+ * the part where CAS correctness matters most.
+ *
+ * Concurrency contract (see `./finalizer.ts` for the full model): a holder fenced-CAS's
+ * `owed → running` (stamping `claimedAt`) before running the finalizer, then settle-CAS's
+ * the EXACT `running` bytes it wrote when clearing, rescheduling, or dead-lettering — so a
+ * concurrent reclaimer can never clobber a fresher claim. Liveness is decided purely by
+ * the clock via {@link teardownStaleThresholdMs}.
+ *
+ * @module core/engine/termination/finalizer-claim
+ */
+import type { BatchOperation } from '../../../storage/interface.ts';
+import type { EngineInternals } from '../internals.ts';
+import { type TeardownClaim } from '../state-utilities.ts';
+import type { RunnableFinalizer } from './finalizer-activity.ts';
+/** Maximum finalizer attempts before the teardown is dead-lettered (the leak horizon). */
+export declare const MAX_TEARDOWN_ATTEMPTS = 8;
+/** Short re-arm delay for the self-heal timer written on a non-settling drive exit. */
+export declare const TEARDOWN_SELF_HEAL_DELAY_MS = 30000;
+/** Backoff for the timer rescheduled after a failed finalizer attempt (1-based attempt). */
+export declare function teardownBackoffMs(failedAttempt: number): number;
+/**
+ * The reclaim horizon for a `running` claim: a holder that has not settled within this
+ * window of `claimedAt` is presumed crashed/deposed and its claim is reclaimable. Equal
+ * to the finalizer's per-attempt budget (its declared `timeout`, or
+ * {@link DEFAULT_FINALIZER_STALE_BUDGET_MS}) plus {@link TEARDOWN_STALE_MARGIN_MS}.
+ */
+export declare function teardownStaleThresholdMs(finalizer: RunnableFinalizer): number;
+/** Whether a `running` claim is reclaimable: its holder has not settled within the stale window. */
+export declare function runningClaimIsStale(internals: EngineInternals, claim: TeardownClaim, finalizer: RunnableFinalizer): boolean;
+/**
+ * Durable audit record written to {@link KEYS.teardownDeadLetter} when a workflow's
+ * finalizer permanently fails — the retry horizon was reached, or the recorded resource
+ * state vanished so the finalizer can never run. It is the supported operator surface
+ * for a leaked external resource: raw storage inspection of the `wf-teardown-deadletter:`
+ * key is the contract (in-process teardown events are best-effort and not durable, and
+ * the record is excluded from the workflow purge delete-set so it survives after the
+ * workflow record is gone).
+ */
+export interface TeardownDeadLetterRecord {
+    /** The workflow type whose finalizer leaked. */
+    type: string;
+    /** The last finalizer error message (or the reason teardown was abandoned). */
+    lastError: string;
+    /** The attempt count reached before dead-lettering. */
+    attempts: number;
+    /** Engine clock when the record was written. */
+    deadLetteredAt: number;
+    /** The decoded `ctx.setFinalizerState` payload, when it was still recoverable. */
+    finalizerInput?: unknown;
+}
+/** Build the operations that arm a fresh `wf-teardown:` timer at `fireAt` (same token). */
+export declare function teardownTimerOperations(token: string, workflowId: string, fireAt: number): BatchOperation[];
+/** Encode an `owed` claim (attempts as given, no `claimedAt` while owed). */
+export declare function encodeOwedClaim(attempts: number, token: string): Uint8Array;
+/** Encode the `running` claim a holder writes to atomically claim the marker. */
+export declare function encodeRunningClaim(attempts: number, token: string, claimedAt: number): Uint8Array;
+/**
+ * Re-arm a future `wf-teardown:` timer for a non-settling drive exit (lost CAS, a
+ * presumed-live `running` claim, a shutdown abort, or a leave-the-marker case), so the
+ * claim is not stranded after the scheduler deletes the fired timer. The marker bytes
+ * are left untouched — only the timer is (re)written. Fenced, best-effort: a deposed
+ * engine that loses the fence simply yields to the new owner, whose own timer drives it.
+ */
+export declare function rearmTeardownTimer(internals: EngineInternals, workflowId: string, token: string, delayMs: number): Promise<void>;
+/**
+ * Clear the teardown marker for a workflow that turned out not to owe a finalizer run
+ * after all — a stale timer or a vanished/ineligible workflow. Conditioned on the marker
+ * still being byte-for-byte `expectedBytes` (the bytes this drive read), so a concurrent
+ * drive that already re-claimed or re-armed the marker — e.g. a same-id rerun whose fresh
+ * cancellation wrote a NEW claim — is never clobbered. A lost CAS (someone changed it
+ * first) is a benign no-op. Swallows a lost-fence error: a deposed engine simply leaves
+ * the marker for the new owner. No timer is re-armed: clearing the marker IS the settle.
+ */
+export declare function clearTeardownMarker(internals: EngineInternals, workflowId: string, expectedBytes: Uint8Array): Promise<boolean>;
+/**
+ * Atomically claim the marker: CAS `owed → running` only if it is byte-for-byte the
+ * `expectedBytes` we read (so concurrent reclaimers can't both win), fenced on the lease
+ * epoch. Returns the `running` bytes we wrote on success (needed as the settle CAS
+ * precondition), or `null` on a lost CAS.
+ */
+export declare function claimTeardownMarker(internals: EngineInternals, workflowId: string, expectedBytes: Uint8Array, attempts: number, token: string): Promise<Uint8Array | null>;
+/**
+ * Commit a settle batch conditioned on the `teardownOwed` marker still equalling the
+ * exact `running` bytes this drive wrote (Codex MF2). If a reclaimer overwrote the
+ * marker first, the CAS fails and this returns `false` WITHOUT committing — the caller
+ * must then skip dispatching any teardown event. A deposition still hard-halts (the
+ * fenced helper throws), which the drive's outer try/catch routes to a cleanup error.
+ */
+export declare function settleOnRunningClaim(internals: EngineInternals, workflowId: string, runningBytes: Uint8Array, operations: BatchOperation[]): Promise<boolean>;
+/**
+ * Write the durable dead-letter record and clear the teardown + finalizer-state keys,
+ * conditioned on the marker still being byte-for-byte `expectedBytes`. Used both at the
+ * retry horizon (`expectedBytes` = the `running` bytes this drive wrote) and when the
+ * recorded resource state vanished before any claim (`expectedBytes` = the `owed` bytes
+ * this drive read). Conditioning on `expectedBytes` in BOTH cases prevents a stale drive
+ * from dead-lettering after a concurrent drive already settled the marker — which would
+ * falsely report a leak after a successful teardown. Returns whether the durable write
+ * committed; the caller dispatches the dead-lettered event only when it did.
+ */
+export declare function deadLetterTeardown(internals: EngineInternals, workflowId: string, workflowType: string, attempts: number, expectedBytes: Uint8Array, details: {
+    lastError: string;
+    finalizerInput: unknown;
+}): Promise<boolean>;

package/dist/core/engine/termination/finalizer-claim.js

@@ -0,0 +1,80 @@
+import { KEYS } from "../../../storage/interface.js";
+import { encode } from "../../codec.js";
+import { buildTimerBatchOperations, parseDuration } from "../../scheduler.js";
+import {
+  commitFencedEngineWrite,
+  commitFencedEngineWriteAllowingPreconditionFailure
+} from "../fenced-write.js";
+import { createTeardownTimerId } from "../state-utilities.js";
+export const MAX_TEARDOWN_ATTEMPTS = 8;
+const TEARDOWN_BACKOFF_SCHEDULE_MS = [
+  60000,
+  300000,
+  900000,
+  3600000
+], DEFAULT_FINALIZER_STALE_BUDGET_MS = 300000, TEARDOWN_STALE_MARGIN_MS = 30000;
+export const TEARDOWN_SELF_HEAL_DELAY_MS = 30000;
+export function teardownBackoffMs(failedAttempt) {
+  const index = failedAttempt - 1;
+  return TEARDOWN_BACKOFF_SCHEDULE_MS[index] ?? TEARDOWN_BACKOFF_SCHEDULE_MS[TEARDOWN_BACKOFF_SCHEDULE_MS.length - 1];
+}
+export function teardownStaleThresholdMs(finalizer) {
+  return (finalizer.timeout === void 0 ? DEFAULT_FINALIZER_STALE_BUDGET_MS : parseDuration(finalizer.timeout)) + TEARDOWN_STALE_MARGIN_MS;
+}
+export function runningClaimIsStale(internals, claim, finalizer) {
+  if (claim.status !== "running")
+    return !0;
+  if (claim.claimedAt === void 0)
+    return !0;
+  return internals.options.getNow() - claim.claimedAt >= teardownStaleThresholdMs(finalizer);
+}
+export function teardownTimerOperations(token, workflowId, fireAt) {
+  return buildTimerBatchOperations({
+    id: createTeardownTimerId(token),
+    workflowId,
+    fireAt,
+    kind: "teardown"
+  });
+}
+export function encodeOwedClaim(attempts, token) {
+  return encode({ status: "owed", attempts, token });
+}
+export function encodeRunningClaim(attempts, token, claimedAt) {
+  return encode({ status: "running", attempts, token, claimedAt });
+}
+export async function rearmTeardownTimer(internals, workflowId, token, delayMs) {
+  const fireAt = internals.options.getNow() + delayMs;
+  try {
+    await commitFencedEngineWrite(internals, teardownTimerOperations(token, workflowId, fireAt), [], () => Error("teardown self-heal re-arm lost the lease fence"));
+  } catch {}
+}
+export async function clearTeardownMarker(internals, workflowId, expectedBytes) {
+  try {
+    return await commitFencedEngineWriteAllowingPreconditionFailure(internals, [{ type: "delete", key: KEYS.teardownOwed(workflowId) }], [{ key: KEYS.teardownOwed(workflowId), expectedValue: expectedBytes }]);
+  } catch {
+    return !1;
+  }
+}
+export async function claimTeardownMarker(internals, workflowId, expectedBytes, attempts, token) {
+  const runningBytes = encodeRunningClaim(attempts, token, internals.options.getNow());
+  return await commitFencedEngineWriteAllowingPreconditionFailure(internals, [{ type: "put", key: KEYS.teardownOwed(workflowId), value: runningBytes }], [{ key: KEYS.teardownOwed(workflowId), expectedValue: expectedBytes }]) ? runningBytes : null;
+}
+export async function settleOnRunningClaim(internals, workflowId, runningBytes, operations) {
+  return commitFencedEngineWriteAllowingPreconditionFailure(internals, operations, [
+    { key: KEYS.teardownOwed(workflowId), expectedValue: runningBytes }
+  ]);
+}
+export async function deadLetterTeardown(internals, workflowId, workflowType, attempts, expectedBytes, details) {
+  const deadLetter = {
+    type: workflowType,
+    lastError: details.lastError,
+    attempts,
+    deadLetteredAt: internals.options.getNow(),
+    ...details.finalizerInput === void 0 ? {} : { finalizerInput: details.finalizerInput }
+  };
+  return settleOnRunningClaim(internals, workflowId, expectedBytes, [
+    { type: "delete", key: KEYS.teardownOwed(workflowId) },
+    { type: "delete", key: KEYS.finalizerState(workflowId) },
+    { type: "put", key: KEYS.teardownDeadLetter(workflowId), value: encode(deadLetter) }
+  ]);
+}

package/dist/core/engine/termination/finalizer.d.ts

@@ -0,0 +1,48 @@
+/**
+ * The engine-side finalizer drive (`runWorkflowFinalizer`) for issue #446 Phase 2.
+ * Dispatched by the scheduler when a `wf-teardown:` timer fires, it drives a
+ * workflow's definition-level `finalizer` to durable completion after a
+ * `cancelled`/`timed-out` terminal — claiming the durable teardown marker, running
+ * the finalizer activity (via {@link runFinalizerActivity}), and then clearing,
+ * backing off, or dead-lettering based on the outcome. The byte-level claim mechanics
+ * (CAS, settle, re-arm, dead-letter, the stale horizon and backoff) live in
+ * `./finalizer-claim.ts`; this module is the orchestration that decides which to call.
+ *
+ * Concurrency model — a single durable, TIME-based claim:
+ * the durable `teardownOwed` marker carries a `{ status, attempts, token, claimedAt }`
+ * claim ({@link TeardownClaim}). A holder fenced-CAS's `owed → running` (stamping
+ * `claimedAt`) before running, and settle-CAS's the exact `running` bytes it wrote when
+ * clearing or rescheduling — so a concurrent reclaimer can never clobber a fresher
+ * claim. Liveness is decided purely by the clock: a `running` claim is reclaimable once
+ * `claimedAt` is older than {@link teardownStaleThresholdMs} (the finalizer's per-attempt
+ * timeout plus a margin). There is NO in-memory liveness set and NO epoch in the record;
+ * crash recovery is an ordinary stale-claim retry driven by the timer that survived the
+ * terminal batch. The tradeoff is that a finalizer running past the stale threshold may
+ * be re-driven concurrently, which is why workflow finalizers must be idempotent (a
+ * contract of the #446 design).
+ *
+ * Self-heal invariant: every exit that does NOT settle the claim (a lost claim CAS, a
+ * presumed-live `running` claim, a shutdown-aborted attempt, or a missing registration)
+ * re-arms a future `wf-teardown:` timer before returning, because the scheduler deletes
+ * the fired timer once this returns without throwing. A non-settling exit that forgot to
+ * re-arm would strand the marker with no timer to re-drive it.
+ *
+ * @module core/engine/termination/finalizer
+ */
+import type { WorkflowState } from '../../types.ts';
+import type { EngineInternals } from '../internals.ts';
+export { teardownStaleThresholdMs, type TeardownDeadLetterRecord } from './finalizer-claim.ts';
+/** The subset of termination callbacks the finalizer drive needs. */
+export interface FinalizerDriveCallbacks {
+    loadWorkflowState: (workflowId: string) => Promise<WorkflowState | null>;
+    dispatchEvent: (event: Event) => void;
+    handleCleanupError: (source: string, error: unknown, workflowId?: string) => void;
+}
+/**
+ * Drive one teardown attempt for a workflow whose `wf-teardown:` timer just fired.
+ * Never throws: the scheduler treats a thrown timer callback as "retry on the next
+ * tick", which would defeat the backoff schedule, so every failure path is handled
+ * internally and the function returns normally (letting the scheduler delete the
+ * fired timer; a backoff/self-heal reschedule writes a fresh timer entry).
+ */
+export declare function runWorkflowFinalizer(internals: EngineInternals, workflowId: string, timerId: string, callbacks: FinalizerDriveCallbacks): Promise<void>;