@lostgradient/weft 0.17.0 → 0.19.0

package/dist/server/runtime/authentication-bridge.js

@@ -1,7 +1,5 @@
-import { decode } from "../../core/codec.js";
 import { WorkerDisconnectedEvent } from "../../core/events.js";
 import { handleMcpHttpRequest } from "../../mcp/http.js";
-import { KEYS } from "../../storage/interface.js";
 import { authContextToPrincipal, handleRequest } from "../handler.js";
 import { handleJsonRpcHttpRequestSafely } from "../json-rpc-transport-helpers.js";
 import {
@@ -10,6 +8,7 @@ import {
   openJsonRpcWebSocketSession
 } from "../json-rpc-websocket-runtime.js";
 import { API_PREFIX } from "../route-model.js";
+import { decodeRemoteTaskRecord, taskLedgerKey } from "../task-ledger.js";
 import { buildPreflightResponse, decorateResponseWithCors, isPreflightRequest } from "./cors.js";
 import { gateRequest } from "./request-gate.js";
 import { handleTaskPollRequest, handleTaskResultRequest } from "./task-polling.js";
@@ -25,7 +24,7 @@ import {
   replayWatchEvents
 } from "./websocket-stream.js";
 import { handleWebSocketUpgrade } from "./websocket-upgrade.js";
-import { handleWorkerWebSocketMessage, isInflightRecord } from "./websocket-worker.js";
+import { handleWorkerWebSocketMessage } from "./websocket-worker.js";
 export function deriveSupportedOpenApiSecuritySchemes(auth) {
   const schemes = new Set;
   if (auth?.jwt !== void 0)
@@ -206,18 +205,11 @@ function runWorkerDisconnectRequeue(context, options, workerId, _ws, cleanupWork
   for (const task of inFlightTasks)
     (async () => {
       try {
-        const inflightKey = KEYS.operationInflight(task.operationId), existing = await options.engine.storage.get(inflightKey);
-        if (existing) {
-          const record = decode(existing);
-          if (!isInflightRecord(record)) {
-            console.error(`[weft] Corrupt inflight record for task "${task.operationId}" \u2014 skipping reassignment`);
-            return;
-          }
+        const record = decodeRemoteTaskRecord(await options.engine.storage.get(taskLedgerKey(task.operationId)));
+        if (record !== null && record.state === "leased")
           await reassignOrExpireTask(context, options, task.operationId, record, "worker-disconnect");
-        } else {
-          console.warn(`[weft] No inflight record found in storage for task "${task.operationId}" \u2014 skipping reassignment`);
-          await options.engine.storage.delete(inflightKey);
-        }
+        else
+          console.warn(`[weft] No leased ledger record found for task "${task.operationId}" \u2014 skipping reassignment`);
       } catch (error) {
         console.error(`[weft] Failed to reassign task "${task.operationId}" from worker "${workerId}":`, error);
       }

package/dist/server/runtime/context.d.ts

@@ -13,6 +13,19 @@ import type { createLiveOperationRegistry, createLiveRestBindings } from '../res
 import type { TaskQueue } from '../task-queue.ts';
 import type { WorkflowEventFeed } from '../workflow-event-feed.ts';
 import type { ResolvedCorsPolicy } from './cors.ts';
+/**
+ * Startup task-ledger recovery gate (WFT-23). `ready` resolves once every
+ * non-terminal ledger record has been reconstructed into the in-memory
+ * registry, deadline tracker, and task queue indexes; it rejects
+ * (permanently — a settled promise replays the same outcome to every future
+ * awaiter) if the recovery scan itself failed. Task-plane entry points
+ * (`dispatchTaskImpl`, long-poll claim/result, and worker registration) await
+ * this before touching the ledger, so a scan failure blocks new claims with
+ * an actionable error instead of silently continuing with partial indexes.
+ */
+export type TaskLedgerRecoveryGate = Readonly<{
+    ready: Promise<void>;
+}>;
 /**
  * Internal closure state for a single `serve()` invocation.
  *
@@ -77,4 +90,14 @@ export interface ServerContext {
     readonly processingOperations: Set<string>;
     /** Mutex: prevents concurrent reconciliation scans from running simultaneously. */
     reconciliationRunning: boolean;
+    /** Startup task-ledger recovery gate — see {@link TaskLedgerRecoveryGate}. */
+    readonly taskLedgerRecovery: TaskLedgerRecoveryGate;
+    /**
+     * Set once `server.stop()`'s timer-clearing disposer has run. Startup
+     * recovery's scan loop and `scheduleDelayedDispatch` both check this before
+     * doing further work so a still-running recovery scan (or a reconciliation
+     * pass racing shutdown) cannot arm a new timer or issue a durable write
+     * after `pendingTimers` has already been cleared.
+     */
+    stopping: boolean;
 }

package/dist/server/runtime/task-dispatch.d.ts

@@ -1,6 +1,12 @@
 import type { ServeOptions, TaskDispatch } from '../index.ts';
 import type { ServerContext } from './context.ts';
-/** Schedule a delayed dispatch, tracking the timer for cleanup on shutdown. */
+/**
+ * Schedule a delayed dispatch, tracking the timer for cleanup on shutdown.
+ *
+ * No-ops once `context.stopping` is set — `server.stop()`'s timer-clearing
+ * disposer has already run by then, so arming a new timer here would leak a
+ * callback that fires against a stopped server instead of being cleared.
+ */
 export declare function scheduleDelayedDispatch(context: ServerContext, options: ServeOptions, task: TaskDispatch, delay: number): void;
 export declare function resolveTaskPriority(_context: ServerContext, _options: ServeOptions, task: TaskDispatch): number | undefined;
 export declare function dispatchTaskImpl(context: ServerContext, options: ServeOptions, task: TaskDispatch): Promise<boolean>;

package/dist/server/runtime/task-dispatch.js

@@ -1,5 +1,16 @@
+import { isJSONValue } from "../../core/json.js";
+import { buildWorkerExecutionIdentity } from "../../worker/manifest/execution-identity.js";
 import { evictOldestAffinityEntries } from "../runtime-helpers.js";
-import { markQueued, readQueuedRecord, transitionQueuedToInflight } from "../task-state.js";
+import {
+  claimQueued,
+  createQueued
+} from "../task-ledger-transitions.js";
+import {
+  decodeRemoteTaskRecord,
+  REMOTE_TASK_RECORD_VERSION,
+  taskLedgerKey
+} from "../task-ledger.js";
+import { commitTaskLedgerTransition } from "./task-ledger-runtime.js";
 import {
   recordTaskBacklogMetric,
   recordTaskQueueLatencyMetric,
@@ -12,6 +23,8 @@ function clampVisibilityTimeout(value) {
   return Math.min(Math.max(value, MIN_VISIBILITY_TIMEOUT), MAX_VISIBILITY_TIMEOUT);
 }
 export function scheduleDelayedDispatch(context, options, task, delay) {
+  if (context.stopping)
+    return;
   const timer = setTimeout(() => {
     context.pendingTimers.delete(timer);
     dispatchTaskImpl(context, options, task).catch((err) => console.error(`[weft] Delayed redispatch failed for "${task.operationId}":`, err));
@@ -52,6 +65,55 @@ function recordDispatchOutcome(context, task, workerId) {
   operationIds.add(task.operationId);
   context.operationToWorkflow.set(task.operationId, task.workflowId);
 }
+function buildOptionalCreateQueuedFields(task) {
+  return {
+    ...task.workflowId !== void 0 ? { workflowId: task.workflowId } : {},
+    ...task.workflowExecutionToken !== void 0 ? { workflowExecutionToken: task.workflowExecutionToken } : {},
+    ...task.priority !== void 0 ? { priority: task.priority } : {},
+    ...task.fairShareKey !== void 0 ? { fairShareKey: task.fairShareKey } : {},
+    ...task.sticky && task.workflowId !== void 0 ? { stickyWorkflowId: task.workflowId } : {},
+    ...task.retryPolicy !== void 0 ? { retryPolicy: task.retryPolicy } : {}
+  };
+}
+function buildCreateQueuedInput(task, queue, visibilityTimeout) {
+  const input = task.input === void 0 ? null : task.input;
+  if (!isJSONValue(input))
+    throw Error(`TaskDispatch for operation "${task.operationId}" has a non-JSON-serializable "input" \u2014 the durable task ledger requires JSON-safe input.`);
+  return {
+    recordVersion: REMOTE_TASK_RECORD_VERSION,
+    operationId: task.operationId,
+    workflowType: task.workflowType,
+    activityName: task.activityName,
+    queue,
+    input,
+    headers: task.headers ?? {},
+    visibilityTimeoutMilliseconds: visibilityTimeout,
+    createdAt: Date.now(),
+    ...buildOptionalCreateQueuedFields(task)
+  };
+}
+function buildCreateThenClaimTransition(createInput, claimInput) {
+  return (current, now) => {
+    let queuedRecord;
+    if (current === null) {
+      const created = createQueued(null, createInput, now);
+      if (!created.ok)
+        return created;
+      queuedRecord = created.nextRecord;
+    } else if (current.state === "queued")
+      queuedRecord = current;
+    else
+      return {
+        ok: !1,
+        reason: `operation "${createInput.operationId}" is not claimable from state "${current.state}"`
+      };
+    return claimQueued(queuedRecord, { ...claimInput, expectedGeneration: queuedRecord.generation }, now);
+  };
+}
+function bareActivityName(qualifiedActivityName) {
+  const dotIndex = qualifiedActivityName.indexOf(".");
+  return dotIndex === -1 ? qualifiedActivityName : qualifiedActivityName.slice(dotIndex + 1);
+}
 async function selectAndReserveWorker(context, options, task, queue, visibilityTimeout) {
   const routingOptions = buildRoutingOptions(context, task, queue), worker = context.registry.findWorker(task.activityName, routingOptions);
   if (!worker)
@@ -59,86 +121,85 @@ async function selectAndReserveWorker(context, options, task, queue, visibilityT
   const ws = context.workerSockets.get(worker.id);
   if (!ws)
     return !1;
-  const now = Date.now(), existingQueuedRecord = await readQueuedRecord(options.engine.storage, task.operationId), attemptToken = crypto.randomUUID();
+  const createInput = buildCreateQueuedInput(task, queue, visibilityTimeout), attemptToken = crypto.randomUUID(), executionIdentity = buildWorkerExecutionIdentity({
+    manifest: worker.manifest,
+    manifestDigest: worker.acceptedManifestDigest,
+    workerId: worker.id,
+    workflowType: task.workflowType,
+    activityName: bareActivityName(task.activityName)
+  });
   context.registry.assignTask(worker.id, task.operationId, visibilityTimeout, task.fairShareKey, attemptToken);
-  const deadline = now + visibilityTimeout;
-  context.deadlineTracker.add({ operationId: task.operationId, deadline });
-  const inflightRecord = {
+  let result;
+  try {
+    result = await commitTaskLedgerTransition(options.engine.storage, task.operationId, buildCreateThenClaimTransition(createInput, {
+      attemptToken,
+      workerSessionId: worker.id,
+      ...executionIdentity !== void 0 ? { executionIdentity } : {},
+      leaseDurationMilliseconds: visibilityTimeout
+    }), 1);
+  } catch (error) {
+    context.registry.releaseReservation(task.operationId);
+    throw error;
+  }
+  if (!result.ok) {
+    context.registry.releaseReservation(task.operationId);
+    return !1;
+  }
+  context.deadlineTracker.add({
     operationId: task.operationId,
-    workerId: worker.id,
-    deadline,
-    activityName: task.activityName,
-    queue,
-    input: task.input,
-    attempt: task.attempt ?? 1,
-    visibilityTimeout,
-    retryPolicy: task.retryPolicy,
-    workflowId: task.workflowId,
-    workflowExecutionToken: task.workflowExecutionToken,
-    attemptToken
-  }, normalizedInflightRecord = await transitionQueuedToInflight(options.engine.storage, task.operationId, inflightRecord, {
-    queuedRecord: existingQueuedRecord,
-    now
+    deadline: result.record.leaseDeadline
   });
   ws.send(JSON.stringify({
     type: "task",
     operationId: task.operationId,
     activityName: task.activityName,
     input: task.input === void 0 ? null : task.input,
-    attempt: task.attempt ?? 1,
+    attempt: result.record.attempt,
     attemptToken,
     ...task.workflowExecutionToken !== void 0 && {
       workflowExecutionToken: task.workflowExecutionToken
     },
     ...task.headers ? { headers: task.headers } : {}
   }));
-  recordTaskQueueLatencyMetric(context.metricsCollector, normalizedInflightRecord);
+  recordTaskQueueLatencyMetric(context.metricsCollector, {
+    lastQueuedAt: result.record.lastQueuedAt,
+    lastDispatchedAt: Date.now()
+  });
   recordWorkerCapacitySaturationMetric(context.metricsCollector, context.registry);
   recordDispatchOutcome(context, task, worker.id);
   return !0;
 }
-function mergeQueuedRecordLifecycle(fresh, existing) {
-  return {
-    ...fresh,
-    firstQueuedAt: existing?.firstQueuedAt ?? fresh.queuedAt,
-    lastQueuedAt: fresh.queuedAt,
-    lastDispatchedAt: existing?.lastDispatchedAt,
-    startedAt: existing?.startedAt,
-    retryCount: existing?.retryCount ?? fresh.retryCount,
-    requeueCount: existing?.requeueCount ?? fresh.requeueCount,
-    lastRequeueReason: existing?.lastRequeueReason
-  };
-}
 async function enqueueTaskForLongPoll(context, options, task, queue, visibilityTimeout, resolvedPriority) {
-  const attempt = task.attempt ?? 1, freshRecord = {
-    operationId: task.operationId,
-    activityName: task.activityName,
-    input: task.input,
-    queue,
-    attempt,
-    visibilityTimeout,
-    retryPolicy: task.retryPolicy,
-    queuedAt: Date.now(),
-    workflowId: task.workflowId,
-    workflowExecutionToken: task.workflowExecutionToken,
-    retryCount: Math.max(0, attempt - 1),
-    requeueCount: 0
-  }, existingQueuedRecord = await readQueuedRecord(options.engine.storage, task.operationId), normalizedQueuedRecord = await markQueued(options.engine.storage, mergeQueuedRecordLifecycle(freshRecord, existingQueuedRecord)), enqueued = context.taskQueue.enqueue(queue, {
+  const storage = options.engine.storage, createInput = buildCreateQueuedInput(task, queue, visibilityTimeout), rawExisting = await storage.get(taskLedgerKey(task.operationId)), current = decodeRemoteTaskRecord(rawExisting);
+  let queuedRecord;
+  if (current === null) {
+    const result = await commitTaskLedgerTransition(storage, task.operationId, (freshCurrent, now) => createQueued(freshCurrent, createInput, now), 1);
+    if (!result.ok) {
+      const raced = decodeRemoteTaskRecord(await storage.get(taskLedgerKey(task.operationId)));
+      if (raced === null || raced.state !== "queued")
+        return !1;
+      queuedRecord = raced;
+    } else
+      queuedRecord = result.record;
+  } else if (current.state === "queued")
+    queuedRecord = current;
+  else
+    return !1;
+  const enqueued = context.taskQueue.enqueue(queue, {
     operationId: task.operationId,
     activityName: task.activityName,
     input: task.input,
-    attempt,
+    attempt: queuedRecord.attempt,
     retryPolicy: task.retryPolicy,
     visibilityTimeout,
     workflowId: task.workflowId,
     workflowExecutionToken: task.workflowExecutionToken,
-    firstQueuedAt: normalizedQueuedRecord.firstQueuedAt,
-    lastQueuedAt: normalizedQueuedRecord.lastQueuedAt,
-    lastDispatchedAt: normalizedQueuedRecord.lastDispatchedAt,
-    startedAt: normalizedQueuedRecord.startedAt,
-    retryCount: normalizedQueuedRecord.retryCount,
-    requeueCount: normalizedQueuedRecord.requeueCount,
-    lastRequeueReason: normalizedQueuedRecord.lastRequeueReason,
+    firstQueuedAt: queuedRecord.firstQueuedAt,
+    lastQueuedAt: queuedRecord.lastQueuedAt,
+    lastDispatchedAt: queuedRecord.lastDispatchedAt,
+    startedAt: queuedRecord.startedAt,
+    retryCount: queuedRecord.retryCount,
+    requeueCount: queuedRecord.requeueCount,
     ...task.headers ? { headers: task.headers } : {},
     ...resolvedPriority !== void 0 ? { priority: resolvedPriority } : {}
   });
@@ -146,6 +207,16 @@ async function enqueueTaskForLongPoll(context, options, task, queue, visibilityT
   return enqueued;
 }
 export async function dispatchTaskImpl(context, options, task) {
+  try {
+    await context.taskLedgerRecovery.ready;
+  } catch (error) {
+    throw Error(`Cannot dispatch task "${task.operationId}" \u2014 startup task-ledger recovery failed: ${error instanceof Error ? error.message : String(error)}`, { cause: error });
+  }
+  if (!task.workflowType)
+    throw Error(`TaskDispatch for operation "${task.operationId}" is missing required field "workflowType".`);
+  const dotIndex = task.activityName.indexOf(".");
+  if (dotIndex !== -1 && task.activityName.slice(0, dotIndex) !== task.workflowType)
+    throw Error(`TaskDispatch for operation "${task.operationId}" has activityName "${task.activityName}" whose qualifier does not match workflowType "${task.workflowType}".`);
   const queue = task.queue ?? "default", visibilityTimeout = clampVisibilityTimeout(task.visibilityTimeout), resolvedPriority = resolveTaskPriority(context, options, task);
   if (!validateTaskEnvelope(context, task))
     return !1;

package/dist/server/runtime/task-ledger-completion.d.ts

@@ -0,0 +1,68 @@
+/**
+ * Shared two-step commit for applying a worker's task result to the durable
+ * remote task ledger (WFT-22) — used by both the WebSocket (`onTaskResultMessage`)
+ * and long-poll (`handleTaskResultRequest`) completion paths.
+ *
+ * `Leased --> Completing --> Terminal` is committed as two separate durable
+ * writes, not one, even though the caller already holds the full result: the
+ * project brief models `Completing` as an observable intermediate state
+ * specifically so a crash between the two writes leaves a durable, recoverable
+ * marker ("Completing prevents visibility scanning or disconnect handling from
+ * requeueing an attempt while its terminal result is being applied") that
+ * WFT-23's recovery is designed to resume. Collapsing the two transitions into
+ * one write would make that state unobservable and defeat the point of having
+ * it.
+ *
+ * A submitter can itself resume that crash window: if the record is already
+ * `completing` with the same `attemptToken` AND the same `pendingResultDigest`
+ * — the worker retrying the identical result after a server crash between the
+ * two writes, or a benign duplicate submission — this skips straight to
+ * `commitTerminalResult` instead of rejecting a legitimate resubmission with
+ * "expected task state leased". A `completing` record whose digest or token
+ * differs is a genuinely different result and is rejected normally.
+ *
+ * On exhausted retries — specifically when the *second* write
+ * (`commitTerminalFromCompleting`) exhausts its CAS retry budget after the
+ * record is already durably `completing` — this attempts one best-effort
+ * `Completing --> DeadLettered` write (WFT-24's `commitDeadLetter`,
+ * `../task-ledger-transitions.ts`) so a sustained, operation-specific
+ * storage write failure becomes an operator-visible dead letter instead of a
+ * silently stuck record. If the *first* write (`beginCompletion`) exhausts
+ * instead, the record never left `leased` and no dead letter is attempted —
+ * the visibility scanner's ordinary expiry path already covers that case.
+ * If the dead-letter write itself also fails, this falls back to the plain
+ * `ok: false` result exactly as before WFT-24: the record stays `completing`,
+ * a worker resubmitting the identical result can still resume through the
+ * `resuming` branch above, and no data is lost.
+ *
+ * @module server/runtime/task-ledger-completion
+ */
+import type { Storage } from '../../storage/interface.ts';
+import type { ServeOptions } from '../index.ts';
+import { type RemoteTaskCompleting, type RemoteTaskDeadLettered, type RemoteTaskTerminal } from '../task-ledger.ts';
+export type TaskLedgerCompletionInput = Readonly<{
+    operationId: string;
+    attemptToken: string;
+    status: 'completed' | 'failed';
+    value?: unknown;
+    error?: string;
+}>;
+export type TaskLedgerCompletionResult = Readonly<{
+    ok: true;
+    completing: RemoteTaskCompleting;
+    terminal: RemoteTaskTerminal;
+}> | Readonly<{
+    ok: false;
+    reason: string;
+    deadLettered?: RemoteTaskDeadLettered;
+}>;
+/**
+ * Dispatch {@link TaskResultDeadLetteredEvent} for a record `commitTaskLedgerCompletion`
+ * dead-lettered. Shared by the WebSocket and long-poll completion paths so
+ * both build the event from the same fields — `workflowId`/`activityName`/`queue`
+ * come from the dead-lettered record itself (present on every `RemoteTaskBase`),
+ * not from the caller's original request, so the event is accurate even if the
+ * caller only had partial information.
+ */
+export declare function dispatchTaskDeadLetteredEvent(options: ServeOptions, operationId: string, deadLettered: RemoteTaskDeadLettered, workerId: string | undefined): void;
+export declare function commitTaskLedgerCompletion(storage: Storage, input: TaskLedgerCompletionInput): Promise<TaskLedgerCompletionResult>;

package/dist/server/runtime/task-ledger-completion.js

@@ -0,0 +1,72 @@
+import { TaskResultDeadLetteredEvent } from "../../core/events.js";
+import { isJSONValue } from "../../core/json.js";
+import { sha256Hex } from "../../worker/manifest/content-digest.js";
+import {
+  beginCompletion,
+  commitDeadLetter,
+  commitTerminalResult
+} from "../task-ledger-transitions.js";
+import {
+  decodeRemoteTaskRecord,
+  taskLedgerKey
+} from "../task-ledger.js";
+import { commitTaskLedgerTransition } from "./task-ledger-runtime.js";
+const COMPLETION_MAX_ATTEMPTS = 3, DEAD_LETTER_MAX_ATTEMPTS = 1;
+async function pendingResultDigest(input) {
+  return sha256Hex(JSON.stringify({
+    status: input.status,
+    value: input.value ?? null,
+    error: input.error ?? null
+  }));
+}
+function commitTerminalFromCompleting(storage, input, resultDigest) {
+  return commitTaskLedgerTransition(storage, input.operationId, (current, now) => commitTerminalResult(current, {
+    attemptToken: input.attemptToken,
+    resultDigest,
+    ...input.status === "failed" && input.error !== void 0 ? { error: input.error } : {}
+  }, now), COMPLETION_MAX_ATTEMPTS);
+}
+async function attemptDeadLetter(storage, input, resultDigest, persistenceFailureReason) {
+  const deadLettered = await commitTaskLedgerTransition(storage, input.operationId, (current, now) => commitDeadLetter(current, {
+    attemptToken: input.attemptToken,
+    resultDigest,
+    ...isJSONValue(input.value) ? { value: input.value } : {},
+    ...input.error !== void 0 ? { error: input.error } : {},
+    persistenceFailureReason
+  }, now), DEAD_LETTER_MAX_ATTEMPTS);
+  return deadLettered.ok ? deadLettered.record : void 0;
+}
+export function dispatchTaskDeadLetteredEvent(options, operationId, deadLettered, workerId) {
+  options.engine.dispatchEvent(new TaskResultDeadLetteredEvent({
+    operationId,
+    workflowId: deadLettered.workflowId,
+    activityName: deadLettered.activityName,
+    queue: deadLettered.queue,
+    workerId,
+    errorMessage: deadLettered.persistenceFailureReason
+  }));
+}
+export async function commitTaskLedgerCompletion(storage, input) {
+  const resultDigest = await pendingResultDigest(input), existing = decodeRemoteTaskRecord(await storage.get(taskLedgerKey(input.operationId)));
+  if (existing !== null && existing.state === "completing" && existing.attemptToken === input.attemptToken && existing.pendingResultDigest === resultDigest) {
+    const committed = await commitTerminalFromCompleting(storage, input, resultDigest);
+    if (!committed.ok) {
+      const deadLettered = await attemptDeadLetter(storage, input, resultDigest, committed.reason);
+      return { ...committed, ...deadLettered !== void 0 ? { deadLettered } : {} };
+    }
+    return { ok: !0, completing: existing, terminal: committed.record };
+  }
+  const begun = await commitTaskLedgerTransition(storage, input.operationId, (current) => beginCompletion(current, {
+    attemptToken: input.attemptToken,
+    pendingStatus: input.status,
+    pendingResultDigest: resultDigest
+  }), COMPLETION_MAX_ATTEMPTS);
+  if (!begun.ok)
+    return begun;
+  const committed = await commitTerminalFromCompleting(storage, input, resultDigest);
+  if (!committed.ok) {
+    const deadLettered = await attemptDeadLetter(storage, input, resultDigest, committed.reason);
+    return { ...committed, ...deadLettered !== void 0 ? { deadLettered } : {} };
+  }
+  return { ok: !0, completing: begun.record, terminal: committed.record };
+}

package/dist/server/runtime/task-ledger-recovery.d.ts

@@ -0,0 +1,55 @@
+/**
+ * Startup task-ledger recovery (WFT-23).
+ *
+ * A one-shot full scan of the durable `task-ledger:` keyspace, run once per
+ * `serve()` call before the recovery gate on `ServerContext.taskLedgerRecovery`
+ * opens (see `buildServerContext` in `serve-internals.ts`). Reconstructs the
+ * in-memory registry, deadline tracker, and task-queue indexes that
+ * `restoreInflightTasks` used to rebuild from the retired `op:inflight:`
+ * keyspace — those indexes are process-local caches; the ledger is the
+ * durable authority, so nothing here writes state, only rehydrates memory to
+ * match what storage already says.
+ *
+ * One branch per non-terminal state:
+ *  - `queued`   — redispatch through the normal dispatch path so a lost
+ *                 `scheduleDelayedDispatch` timer (or a record this process
+ *                 never dispatched at all) still becomes runnable.
+ *  - `leased`, deadline still in the future — rehydrate registry ownership
+ *                 and the deadline tracker so heartbeats and completions
+ *                 from the original worker are recognized after reconnect.
+ *  - `leased`, deadline already passed — requeue or exhaust through the same
+ *                 `reassignOrExpireTask` CAS path `scanExpiredTasks` and
+ *                 `reconcileOrphanedRecords` use; deletion is never correct
+ *                 here (see `RequeueExpiredAttemptInput`'s contract).
+ *  - `completing` — rehydrate registry ownership only. The pending result
+ *                 value isn't persisted (only its digest), so recovery
+ *                 cannot resolve this state itself; only the worker's
+ *                 redelivered `taskResult`, landing on
+ *                 `commitTaskLedgerCompletion`'s crash-resumption branch,
+ *                 can. Recovery's job is to make sure that redelivery still
+ *                 passes the WebSocket ownership guard after a restart.
+ *  - `cancelling` — rehydrate registry ownership only, for the same reason:
+ *                 nothing in the current runtime calls `commitCancellation`
+ *                 yet (a future project's scope), so there is no action to
+ *                 resume, only ownership to preserve.
+ *  - `terminal`, `deadLettered` — already resolved; skipped.
+ *
+ * A single corrupt or unrecognized record is logged and skipped — ordinary
+ * data hygiene, not a recovery failure. A failure in the scan itself (the
+ * storage iterator throwing) propagates and rejects the whole gate: per the
+ * project brief, "a scan failure leaves the remote worker plane unhealthy
+ * and prevents new claims rather than logging and continuing with partial
+ * indexes."
+ *
+ * @module server/runtime/task-ledger-recovery
+ */
+import type { ServeOptions } from '../index.ts';
+import type { ServerContext } from './context.ts';
+/**
+ * Run startup task-ledger recovery to completion. Resolves once every
+ * non-terminal record has been reconstructed; rejects if the scan itself
+ * failed. Callers gate task-plane entry points on the resulting promise
+ * rather than awaiting this directly from within another recovery-time
+ * scan — see `ServerContext.taskLedgerRecovery`'s doc comment.
+ */
+export declare function runTaskLedgerRecovery(context: ServerContext, options: ServeOptions): Promise<void>;

package/dist/server/runtime/task-ledger-recovery.js

@@ -0,0 +1,73 @@
+import {
+  decodeRemoteTaskRecord
+} from "../task-ledger.js";
+import { scheduleDelayedDispatch } from "./task-dispatch.js";
+import { reassignOrExpireTask, taskDispatchFromLedgerRecord } from "./task-reconciliation.js";
+function rebuildWorkflowIndex(context, operationId, workflowId) {
+  if (!workflowId)
+    return;
+  let operationIds = context.workflowOperations.get(workflowId);
+  if (!operationIds) {
+    operationIds = new Set;
+    context.workflowOperations.set(workflowId, operationIds);
+  }
+  operationIds.add(operationId);
+  context.operationToWorkflow.set(operationId, workflowId);
+}
+function rehydrateWorkerOwnership(context, record, now) {
+  const remaining = Math.max(0, record.leaseDeadline - now);
+  context.registry.assignTask(record.workerSessionId, record.operationId, remaining, record.fairShareKey, record.attemptToken);
+  const tracked = context.registry.getWorkerTasks(record.workerSessionId).find((task) => task.operationId === record.operationId);
+  if (tracked)
+    tracked.visibilityTimeout = record.visibilityTimeoutMilliseconds;
+  rebuildWorkflowIndex(context, record.operationId, record.workflowId);
+}
+async function recoverTaskLedgerRecord(context, options, decoded, now) {
+  switch (decoded.state) {
+    case "queued": {
+      const delay = Math.max(0, decoded.availableAt - now);
+      scheduleDelayedDispatch(context, options, taskDispatchFromLedgerRecord(decoded), delay);
+      return;
+    }
+    case "leased": {
+      if (decoded.leaseDeadline > now) {
+        rehydrateWorkerOwnership(context, decoded, now);
+        context.deadlineTracker.add({
+          operationId: decoded.operationId,
+          deadline: decoded.leaseDeadline
+        });
+        return;
+      }
+      await reassignOrExpireTask(context, options, decoded.operationId, decoded);
+      return;
+    }
+    case "completing":
+    case "cancelling": {
+      rehydrateWorkerOwnership(context, decoded, now);
+      return;
+    }
+    case "terminal":
+    case "deadLettered":
+      return;
+    default: {
+      const exhaustive = decoded;
+    }
+  }
+}
+export async function runTaskLedgerRecovery(context, options) {
+  const now = Date.now();
+  for await (const [key, value] of options.engine.storage.scan("task-ledger:")) {
+    if (context.stopping)
+      return;
+    try {
+      const decoded = decodeRemoteTaskRecord(value);
+      if (decoded === null) {
+        console.error(`[weft] Unrecognized task ledger record at "${key}" during startup recovery \u2014 skipping`);
+        continue;
+      }
+      await recoverTaskLedgerRecord(context, options, decoded, now);
+    } catch (error) {
+      console.error(`[weft] Failed to recover task ledger record at "${key}" \u2014 skipping:`, error);
+    }
+  }
+}