@lostgradient/weft 0.17.0 → 0.19.0

package/dist/server/runtime/websocket-worker-registration.js

@@ -0,0 +1,134 @@
+import { WorkerConnectedEvent } from "../../core/events.js";
+import { digestCanonicalWorkerManifest, parseWorkerManifest } from "../../worker/manifest/index.js";
+import {
+  REMOTE_WORKER_PROTOCOL_VERSION
+} from "../../worker/protocol.js";
+import { isAuthenticated } from "../principal.js";
+import { rejectRegistration, sendWorkerProtocolMessage } from "./websocket-worker-messaging.js";
+const MAX_WORKER_CONCURRENCY = 1000, DEFAULT_WORKER_CONCURRENCY = 10;
+function rejectAndRecordRegistration(context, ws, code, message, workerId, requestedProtocolVersion, identity) {
+  context.registry.recordRejection({
+    code,
+    ...workerId !== void 0 ? { workerId } : {},
+    rejectedAt: Date.now(),
+    queue: ws.data.queue ?? "default",
+    ...identity?.deploymentName !== void 0 ? { deploymentName: identity.deploymentName } : {},
+    ...identity?.buildId !== void 0 ? { buildId: identity.buildId } : {}
+  });
+  rejectRegistration(ws, code, message, requestedProtocolVersion);
+}
+function principalMayRegisterWorker(principal) {
+  if (principal === void 0)
+    return !0;
+  return isAuthenticated(principal) && principal.hasScope("workers:write");
+}
+function deriveActivitiesFromManifest(manifest) {
+  const activities = [];
+  for (const [workflowType, workflow] of Object.entries(manifest.workflows))
+    for (const activityName of Object.keys(workflow.activities))
+      activities.push(`${workflowType}.${activityName}`);
+  return activities;
+}
+function buildWorkerRegistrationInfo(message, manifest, acceptedManifestDigest, queue, concurrency) {
+  const runtimeVersion = manifest.runtime.version === "" ? void 0 : manifest.runtime.version;
+  return {
+    id: message.workerId,
+    queue,
+    activities: deriveActivitiesFromManifest(manifest),
+    concurrency,
+    deploymentName: manifest.deployment.name,
+    buildId: manifest.deployment.buildId,
+    ...runtimeVersion !== void 0 ? { runtimeVersion } : {},
+    manifest,
+    acceptedManifestDigest,
+    ...message.startedAt !== void 0 ? { startedAt: message.startedAt } : {},
+    capabilities: manifest.capabilities
+  };
+}
+function admitReconnectingSocket(context, ws, message, manifest) {
+  const pendingRequeue = context.pendingWorkerRequeues.get(message.workerId), isGracePeriodReconnect = pendingRequeue !== void 0;
+  if (isGracePeriodReconnect) {
+    clearTimeout(pendingRequeue);
+    context.pendingWorkerRequeues.delete(message.workerId);
+  }
+  const existingSocket = context.workerSockets.get(message.workerId);
+  if (!isGracePeriodReconnect && existingSocket !== void 0 && existingSocket !== ws) {
+    rejectAndRecordRegistration(context, ws, "invalid_registration", "workerId is already registered to an active connection", message.workerId, message.protocolVersion, { deploymentName: manifest.deployment.name, buildId: manifest.deployment.buildId });
+    return !1;
+  }
+  return !0;
+}
+function evaluateWorkerAdmissionPolicy(policy, request) {
+  try {
+    return policy(request);
+  } catch (error) {
+    return {
+      status: "rejected",
+      reason: `workerAdmissionPolicy threw: ${error instanceof Error ? error.message : String(error)}`
+    };
+  }
+}
+function commitWorkerRegistration(context, options, ws, message, manifest, queue, concurrency, acceptedManifestDigest) {
+  if (ws.readyState !== WebSocket.OPEN)
+    return;
+  if (!context.registry.checkDeploymentConsistency(manifest.deployment.name, manifest.deployment.buildId, manifest.deployment.artifactDigest).ok) {
+    rejectAndRecordRegistration(context, ws, "deployment_conflict", `Deployment "${manifest.deployment.name}" build "${manifest.deployment.buildId}" is already registered with a different artifact digest`, message.workerId, message.protocolVersion, { deploymentName: manifest.deployment.name, buildId: manifest.deployment.buildId });
+    return;
+  }
+  if (!admitReconnectingSocket(context, ws, message, manifest))
+    return;
+  context.registry.recordDeploymentConsistency(manifest.deployment.name, manifest.deployment.buildId, manifest.deployment.artifactDigest);
+  const registrationInfo = buildWorkerRegistrationInfo(message, manifest, acceptedManifestDigest, queue, concurrency);
+  ws.data.workerId = message.workerId;
+  ws.data.workerRegistered = !0;
+  ws.data.workerProtocolVersion = message.protocolVersion;
+  context.registry.register(registrationInfo);
+  context.workerSockets.set(message.workerId, ws);
+  sendWorkerProtocolMessage(ws, {
+    type: "registerAck",
+    protocolVersion: REMOTE_WORKER_PROTOCOL_VERSION,
+    workerId: message.workerId,
+    queue,
+    concurrency,
+    acceptedManifestDigest,
+    serverCapabilities: []
+  });
+  options.engine.dispatchEvent(new WorkerConnectedEvent(message.workerId, queue, registrationInfo.activities, concurrency));
+}
+export async function registerWorker(context, options, ws, message) {
+  try {
+    await context.taskLedgerRecovery.ready;
+  } catch (error) {
+    rejectAndRecordRegistration(context, ws, "registration_rejected", `Server task-ledger recovery failed \u2014 cannot admit new workers: ${error instanceof Error ? error.message : String(error)}`, message.workerId, message.protocolVersion);
+    return;
+  }
+  if (!principalMayRegisterWorker(ws.data.principal)) {
+    rejectAndRecordRegistration(context, ws, "invalid_registration", "Worker registration requires the workers:write scope", message.workerId, message.protocolVersion);
+    return;
+  }
+  const parsedManifest = parseWorkerManifest(message.manifest);
+  if (!parsedManifest.ok) {
+    rejectAndRecordRegistration(context, ws, "invalid_registration", parsedManifest.message, message.workerId, message.protocolVersion);
+    return;
+  }
+  const { manifest, canonicalJson } = parsedManifest;
+  if (manifest.protocolVersion !== message.protocolVersion) {
+    rejectAndRecordRegistration(context, ws, "invalid_registration", `Manifest declares protocolVersion ${String(manifest.protocolVersion)}, which does not match the register message's protocolVersion ${String(message.protocolVersion)}`, message.workerId, message.protocolVersion, { deploymentName: manifest.deployment.name, buildId: manifest.deployment.buildId });
+    return;
+  }
+  const queue = ws.data.queue ?? "default";
+  if (options.workerAdmissionPolicy !== void 0) {
+    const decision = evaluateWorkerAdmissionPolicy(options.workerAdmissionPolicy, {
+      principal: ws.data.principal,
+      workerId: message.workerId,
+      queue,
+      manifest
+    });
+    if (decision.status === "rejected") {
+      rejectAndRecordRegistration(context, ws, "registration_rejected", decision.reason, message.workerId, message.protocolVersion, { deploymentName: manifest.deployment.name, buildId: manifest.deployment.buildId });
+      return;
+    }
+  }
+  const rawConcurrency = message.concurrency ?? DEFAULT_WORKER_CONCURRENCY, clampedConcurrency = Math.min(Math.max(1, Math.floor(rawConcurrency)), MAX_WORKER_CONCURRENCY), acceptedManifestDigest = await digestCanonicalWorkerManifest(canonicalJson);
+  commitWorkerRegistration(context, options, ws, message, manifest, queue, clampedConcurrency, acceptedManifestDigest);
+}

package/dist/server/runtime/websocket-worker.d.ts

@@ -2,6 +2,5 @@ import type { ServerWebSocket } from 'bun';
 import type { ServeOptions } from '../index.ts';
 import type { WebSocketData } from '../json-rpc-websocket-runtime.ts';
 import type { ServerContext } from './context.ts';
-export { isInflightRecord } from '../task-state.ts';
 export { withRetry } from './retry.ts';
 export declare function handleWorkerWebSocketMessage(context: ServerContext, options: ServeOptions, ws: ServerWebSocket<WebSocketData>, rawMessage: string | Buffer, cleanupWorkflowIndex: (operationId: string) => void): void;

package/dist/server/runtime/websocket-worker.js

@@ -1,104 +1,31 @@
-import { decode, encode } from "../../core/codec.js";
-import { WorkerConnectedEvent } from "../../core/events.js";
-import { KEYS } from "../../storage/interface.js";
 import {
   REMOTE_WORKER_PROTOCOL_VERSION,
-  REMOTE_WORKER_SUPPORTED_PROTOCOL_VERSIONS,
   parseWorkerToServerMessage
 } from "../../worker/protocol.js";
 import { workerProtocolIncompatibleMessage } from "../../worker/worker-protocol-incompatible-error.js";
-import { isAuthenticated } from "../principal.js";
-import { isInflightRecord, readInflightRecord } from "../task-state.js";
+import { renewAttemptLease } from "../task-ledger-transitions.js";
 import { withRetry } from "./retry.js";
-import { recordWorkerCapacitySaturationMetric } from "./task-metrics.js";
 import {
-  taskResultPayloadSizeError,
-  transitionTaskResultToResolvedWithRetry
-} from "./task-result-resolution.js";
+  commitTaskLedgerCompletion,
+  dispatchTaskDeadLetteredEvent
+} from "./task-ledger-completion.js";
+import { commitTaskLedgerTransition } from "./task-ledger-runtime.js";
+import {
+  recordTaskExecutionLatencyMetric,
+  recordWorkerCapacitySaturationMetric
+} from "./task-metrics.js";
+import { taskResultPayloadSizeError } from "./task-result-resolution.js";
 import { WORKER_STREAM_RE } from "./websocket-upgrade.js";
-const MAX_WORKER_CONCURRENCY = 1000, DEFAULT_WORKER_CONCURRENCY = 10, WORKER_PROTOCOL_CLOSE_CODE = 1002, WORKER_REGISTRATION_CLOSE_CODE = 1008;
+import {
+  rejectProtocolMessage,
+  rejectRegistration,
+  sendWorkerProtocolMessage
+} from "./websocket-worker-messaging.js";
+import { registerWorker } from "./websocket-worker-registration.js";
 function isWorkerConnection(pathname) {
   return WORKER_STREAM_RE.test(pathname);
 }
-export { isInflightRecord } from "../task-state.js";
 export { withRetry } from "./retry.js";
-function sendWorkerProtocolMessage(ws, message) {
-  ws.send(JSON.stringify(message));
-}
-function closeWorkerSocket(ws, code, reason) {
-  try {
-    ws.unsubscribe(ws.data.pathname);
-  } catch {}
-  ws.close(code, reason);
-  setTimeout(() => {
-    try {
-      ws.terminate();
-    } catch {}
-  }, 10);
-}
-function rejectRegistration(ws, code, message, requestedProtocolVersion) {
-  sendWorkerProtocolMessage(ws, {
-    type: "registerError",
-    code,
-    message,
-    supportedProtocolVersions: REMOTE_WORKER_SUPPORTED_PROTOCOL_VERSIONS,
-    ...requestedProtocolVersion !== void 0 ? { requestedProtocolVersion } : {}
-  });
-  closeWorkerSocket(ws, WORKER_REGISTRATION_CLOSE_CODE, code);
-}
-function rejectProtocolMessage(ws, code, message) {
-  sendWorkerProtocolMessage(ws, { type: "protocolError", code, message });
-  closeWorkerSocket(ws, WORKER_PROTOCOL_CLOSE_CODE, code);
-}
-function principalMayRegisterWorker(principal) {
-  if (principal === void 0)
-    return !0;
-  return isAuthenticated(principal) && principal.hasScope("workers:write");
-}
-function buildWorkerRegistrationInfo(message, queue, concurrency) {
-  return {
-    id: message.workerId,
-    queue,
-    activities: [...message.activities],
-    concurrency,
-    ...message.deploymentName !== void 0 ? { deploymentName: message.deploymentName } : {},
-    ...message.buildId !== void 0 ? { buildId: message.buildId } : {},
-    ...message.runtimeVersion !== void 0 ? { runtimeVersion: message.runtimeVersion } : {},
-    ...message.gitSha !== void 0 ? { gitSha: message.gitSha } : {},
-    ...message.startedAt !== void 0 ? { startedAt: message.startedAt } : {},
-    ...message.capabilities !== void 0 ? { capabilities: message.capabilities } : {}
-  };
-}
-function registerWorker(context, options, ws, message) {
-  if (!principalMayRegisterWorker(ws.data.principal)) {
-    rejectRegistration(ws, "invalid_registration", "Worker registration requires the workers:write scope", message.protocolVersion);
-    return;
-  }
-  const rawConcurrency = message.concurrency ?? DEFAULT_WORKER_CONCURRENCY, clampedConcurrency = Math.min(Math.max(1, Math.floor(rawConcurrency)), MAX_WORKER_CONCURRENCY), queue = ws.data.queue ?? "default", pendingRequeue = context.pendingWorkerRequeues.get(message.workerId), isGracePeriodReconnect = pendingRequeue !== void 0;
-  if (isGracePeriodReconnect) {
-    clearTimeout(pendingRequeue);
-    context.pendingWorkerRequeues.delete(message.workerId);
-  }
-  const existingSocket = context.workerSockets.get(message.workerId);
-  if (!isGracePeriodReconnect && existingSocket !== void 0 && existingSocket !== ws) {
-    rejectRegistration(ws, "invalid_registration", "workerId is already registered to an active connection", message.protocolVersion);
-    return;
-  }
-  ws.data.workerId = message.workerId;
-  ws.data.workerRegistered = !0;
-  ws.data.workerProtocolVersion = message.protocolVersion;
-  context.registry.register(buildWorkerRegistrationInfo(message, queue, clampedConcurrency));
-  context.workerSockets.set(message.workerId, ws);
-  sendWorkerProtocolMessage(ws, {
-    type: "registerAck",
-    protocolVersion: REMOTE_WORKER_PROTOCOL_VERSION,
-    workerId: message.workerId,
-    queue,
-    activities: [...message.activities],
-    concurrency: clampedConcurrency
-  });
-  options.engine.dispatchEvent(new WorkerConnectedEvent(message.workerId, queue, [...message.activities], clampedConcurrency));
-}
 function resolveTaskResultStatus(message) {
   return message.status === "completed" ? "completed" : "failed";
 }
@@ -121,9 +48,7 @@ function onTaskResultMessage(context, options, ws, message, cleanupWorkflowIndex
     return;
   }
   const resolvedStatus = resolveTaskResultStatus(message), payloadError = taskResultPayloadSizeError({
-    operationId,
     status: resolvedStatus,
-    resolutionReason: resolvedStatus,
     ...message.status === "completed" ? { value: message.value } : { error: message.error }
   }, context.payloadSizeMaxBytes);
   context.registry.completeTask(operationId);
@@ -131,30 +56,40 @@ function onTaskResultMessage(context, options, ws, message, cleanupWorkflowIndex
   cleanupWorkflowIndex(operationId);
   recordWorkerCapacitySaturationMetric(context.metricsCollector, context.registry);
   (async () => {
-    const inflightRecord = await readInflightRecord(options.engine.storage, operationId);
     if (payloadError !== null) {
       sendWorkerProtocolMessage(ws, {
         type: "protocolError",
         code: "invalid_message",
         message: payloadError.message
       });
-      await transitionTaskResultToResolvedWithRetry(context, options, {
+      const rejected = await commitTaskLedgerCompletion(options.engine.storage, {
         operationId,
+        attemptToken: message.attemptToken,
         status: "failed",
-        resolutionReason: "failed",
-        inflightRecord,
-        error: payloadError.message,
-        skipPayloadSizeCheck: !0
+        error: payloadError.message
       });
+      if (rejected.ok)
+        recordTaskExecutionLatencyMetric(context.metricsCollector, { startedAt: rejected.completing.startedAt }, Date.now());
+      else {
+        console.error(`[weft] Failed to persist oversized task result rejection for task "${operationId}":`, rejected.reason);
+        if (rejected.deadLettered !== void 0)
+          dispatchTaskDeadLetteredEvent(options, operationId, rejected.deadLettered, workerId);
+      }
       return;
     }
-    await transitionTaskResultToResolvedWithRetry(context, options, {
+    const committed = await commitTaskLedgerCompletion(options.engine.storage, {
       operationId,
+      attemptToken: message.attemptToken,
       status: resolvedStatus,
-      resolutionReason: resolvedStatus,
-      inflightRecord,
       ...message.status === "completed" ? { value: message.value } : { error: message.error }
     });
+    if (committed.ok)
+      recordTaskExecutionLatencyMetric(context.metricsCollector, { startedAt: committed.completing.startedAt }, Date.now());
+    else {
+      console.error(`[weft] Failed to commit task result for "${operationId}" through the durable ledger:`, committed.reason);
+      if (committed.deadLettered !== void 0)
+        dispatchTaskDeadLetteredEvent(options, operationId, committed.deadLettered, workerId);
+    }
   })().catch((error) => {
     console.error(`[weft] Failed to transition task "${operationId}" to resolved \u2014 inflight record may leak:`, error);
   });
@@ -169,29 +104,40 @@ function onHeartbeatMessage(context, options, ws, _message) {
     if (newDeadline !== void 0) {
       context.deadlineTracker.remove(task.operationId);
       context.deadlineTracker.add({ operationId: task.operationId, deadline: newDeadline });
-      const opId = task.operationId, heartbeatWorkerId = ws.data.workerId;
+      const opId = task.operationId, heartbeatWorkerId = ws.data.workerId, attemptToken = task.attemptToken;
       withRetry(async () => {
         if (!context.registry.isAssigned(opId))
           return;
         if (!context.registry.getWorkerTasks(heartbeatWorkerId ?? "").find((trackedTask) => trackedTask.operationId === opId))
           return;
-        const inflightKey = KEYS.operationInflight(opId), existing = await options.engine.storage.get(inflightKey);
-        if (existing) {
-          const decoded = decode(existing);
-          if (!isInflightRecord(decoded)) {
-            console.error(`[weft] Corrupt inflight record for task "${opId}" during heartbeat \u2014 skipping visibility extension`);
-            return;
-          }
-          const updated = { ...decoded, deadline: newDeadline, lastHeartbeatAt: Date.now() };
-          await options.engine.storage.put(inflightKey, encode(updated));
-        }
+        await commitTaskLedgerTransition(options.engine.storage, opId, (current, now) => renewAttemptLease(current, {
+          attemptToken,
+          workerSessionId: heartbeatWorkerId ?? "",
+          leaseDurationMilliseconds: task.visibilityTimeout
+        }, now), 1);
       }, `extend visibility for task "${opId}"`).catch((error) => {
         console.error(`[weft] Failed to extend visibility for task "${opId}":`, error);
       });
     }
   }
 }
-function parseAndValidateWorkerFrame(ws, rawMessage) {
+function extractOptionalWorkerId(parsed) {
+  if (typeof parsed !== "object" || parsed === null)
+    return;
+  const workerId = parsed.workerId;
+  return typeof workerId === "string" && workerId.length > 0 ? workerId : void 0;
+}
+function recordAndRejectRegistrationFrame(context, ws, parsed, code, message, requestedProtocolVersion) {
+  const rejectedWorkerId = extractOptionalWorkerId(parsed);
+  context.registry.recordRejection({
+    code,
+    ...rejectedWorkerId !== void 0 ? { workerId: rejectedWorkerId } : {},
+    rejectedAt: Date.now(),
+    queue: ws.data.queue ?? "default"
+  });
+  rejectRegistration(ws, code, message, requestedProtocolVersion);
+}
+function parseAndValidateWorkerFrame(context, ws, rawMessage) {
   const text = typeof rawMessage === "string" ? rawMessage : new TextDecoder().decode(rawMessage);
   let parsed;
   try {
@@ -202,12 +148,12 @@ function parseAndValidateWorkerFrame(ws, rawMessage) {
   }
   const result = parseWorkerToServerMessage(parsed);
   if (!result.ok) {
-    if (result.error.code === "invalid_registration" || result.error.code === "unsupported_protocol_version") {
+    if (result.error.code === "invalid_registration" || result.error.code === "unsupported_protocol_version" || result.error.code === "deployment_conflict" || result.error.code === "registration_rejected") {
       const message = result.error.code === "unsupported_protocol_version" ? workerProtocolIncompatibleMessage({
         expected: REMOTE_WORKER_PROTOCOL_VERSION,
         received: result.error.requestedProtocolVersion
       }) : result.error.message;
-      rejectRegistration(ws, result.error.code, message, result.error.requestedProtocolVersion);
+      recordAndRejectRegistrationFrame(context, ws, parsed, result.error.code, message, result.error.requestedProtocolVersion);
       return { ok: !1 };
     }
     rejectProtocolMessage(ws, result.error.code, result.error.message);
@@ -218,7 +164,7 @@ function parseAndValidateWorkerFrame(ws, rawMessage) {
 export function handleWorkerWebSocketMessage(context, options, ws, rawMessage, cleanupWorkflowIndex) {
   if (!isWorkerConnection(ws.data.pathname))
     return;
-  const parsed = parseAndValidateWorkerFrame(ws, rawMessage);
+  const parsed = parseAndValidateWorkerFrame(context, ws, rawMessage);
   if (!parsed.ok)
     return;
   const { message } = parsed;
@@ -228,7 +174,9 @@ export function handleWorkerWebSocketMessage(context, options, ws, rawMessage, c
   }
   switch (message.type) {
     case "register": {
-      registerWorker(context, options, ws, message);
+      registerWorker(context, options, ws, message).catch((error) => {
+        console.error(`[weft] Failed to register worker "${message.workerId}":`, error);
+      });
       break;
     }
     case "taskResult": {

package/dist/server/serve-internals.d.ts

@@ -81,6 +81,15 @@ export declare function resolveNetworkConfig(options: ServeOptions): ResolvedNet
 /**
  * Builds the initial `ServerContext` from resolved options. All mutable state
  * maps and the metrics collector are allocated here so `serve()` stays linear.
+ *
+ * Also starts startup task-ledger recovery (WFT-23) against the fully
+ * constructed context before returning it. Recovery runs asynchronously —
+ * this function itself stays synchronous, same as `serve()` — but its
+ * outcome is captured into `context.taskLedgerRecovery.ready` immediately,
+ * before any request can reach a gated entry point, rather than in a
+ * separate call `serve()` makes later (as the retired `restoreInflightTasks`
+ * required): starting it here minimizes the window between context
+ * construction and recovery beginning.
  */
 export declare function buildServerContext(options: ResolvedServeOptions, serverMetricsCollector: MetricsCollector): ServerContext;
 /**
@@ -113,9 +122,3 @@ export declare function buildBunServeConfig(port: number, hostname: string, deve
  * the most-recently-registered item is torn down first.
  */
 export declare function registerStackDisposers(stack: AsyncDisposableStack, context: ServerContext, options: ServeOptions, broadcastingHandle: EventBroadcastingHandle, onOperationCleanup: (operationId: string) => void): void;
-/**
- * Restores persisted in-flight task records from storage into the in-memory
- * context. Records whose deadline has already elapsed are removed from storage;
- * the engine will retry them on the next dispatch cycle.
- */
-export declare function restoreInflightTasks(context: ServerContext, options: ServeOptions): void;

package/dist/server/serve-internals.js

@@ -1,4 +1,3 @@
-import { decode } from "../core/codec.js";
 import { getEnginePayloadSizeMaxBytes } from "../core/engine/payload-size-policy.js";
 import { createMcpSessionManager } from "../mcp/session.js";
 import { createMetricsCollectorExporter, MetricsCollector } from "../observability/metrics.js";
@@ -25,13 +24,13 @@ import {
   registerWorkflowEventLifecycle
 } from "./runtime/event-broadcasting.js";
 import { shutdownAllWorkers } from "./runtime/shutdown.js";
+import { runTaskLedgerRecovery } from "./runtime/task-ledger-recovery.js";
 import {
   consumeManualTaskReconciliationForTesting as consumeManual,
   reconcileOrphanedRecords,
   scanExpiredTasks
 } from "./runtime/task-reconciliation.js";
 import { DEFAULT_MAX_STREAM_CONNECTIONS_PER_WORKFLOW } from "./runtime/websocket-stream.js";
-import { isInflightRecord, withRetry } from "./runtime/websocket-worker.js";
 import { TaskQueue } from "./task-queue.js";
 import { createWorkflowEventFeed } from "./workflow-event-feed.js";
 const RECONCILIATION_MULTIPLIER = 12, DEFAULT_WORKER_RECONNECT_GRACE_PERIOD_MS = 2000, MAX_WORKER_RECONNECT_GRACE_PERIOD_MS = 5000;
@@ -90,7 +89,13 @@ export function resolveNetworkConfig(options) {
 }
 export function buildServerContext(options, serverMetricsCollector) {
   const workerRegistry = new WorkerRegistry(options.routingPolicy !== void 0 ? { policy: options.routingPolicy } : void 0), taskQueue = new TaskQueue(options.schedulingPolicy !== void 0 ? { schedulingPolicy: options.schedulingPolicy } : void 0), engine = options.engine, eventFeedBackend = createEngineEventFeedBackend(engine);
-  return {
+  let resolveTaskLedgerRecovery, rejectTaskLedgerRecovery;
+  const taskLedgerRecoveryReady = new Promise((resolve, reject) => {
+    resolveTaskLedgerRecovery = resolve;
+    rejectTaskLedgerRecovery = reject;
+  });
+  taskLedgerRecoveryReady.catch(() => {});
+  const context = {
     registry: workerRegistry,
     taskQueue,
     workerSockets: new Map,
@@ -125,8 +130,12 @@ export function buildServerContext(options, serverMetricsCollector) {
     pendingWorkerRequeues: new Map,
     scanRunning: !1,
     processingOperations: new Set,
-    reconciliationRunning: !1
+    reconciliationRunning: !1,
+    taskLedgerRecovery: { ready: taskLedgerRecoveryReady },
+    stopping: !1
   };
+  runTaskLedgerRecovery(context, options).then(resolveTaskLedgerRecovery, rejectTaskLedgerRecovery);
+  return context;
 }
 export function buildFetchHandler(serverHolder, context, options) {
   return (request) => {
@@ -176,6 +185,7 @@ export function registerStackDisposers(stack, context, options, broadcastingHand
     reconcileOrphanedRecords(context, options, onOperationCleanup);
   }, reconciliationIntervalMs) : void 0;
   stack.defer(() => {
+    context.stopping = !0;
     if (visibilityPollHandle !== void 0)
       clearInterval(visibilityPollHandle);
     if (reconciliationHandle !== void 0)
@@ -191,39 +201,3 @@ export function registerStackDisposers(stack, context, options, broadcastingHand
     context.rateLimiter?.dispose();
   });
 }
-export function restoreInflightTasks(context, options) {
-  withRetry(async () => {
-    for await (const [key, value] of options.engine.storage.scan("op:inflight:")) {
-      const decoded = decode(value);
-      if (!isInflightRecord(decoded)) {
-        console.error(`[weft] Corrupt inflight record at "${key}" during restore \u2014 skipping`);
-        continue;
-      }
-      const record = decoded, now = Date.now();
-      if (record.deadline <= now) {
-        options.engine.storage.delete(key);
-        continue;
-      }
-      const remaining = record.deadline - now;
-      context.registry.assignTask(record.workerId, record.operationId, remaining, void 0, record.attemptToken);
-      context.deadlineTracker.add({ operationId: record.operationId, deadline: record.deadline });
-      const tracked = context.registry.getWorkerTasks(record.workerId).find((t) => t.operationId === record.operationId);
-      if (tracked)
-        tracked.visibilityTimeout = record.visibilityTimeout;
-      rebuildWorkflowIndex(context, record.operationId, record.workflowId);
-    }
-  }, "restore in-flight tasks from storage").catch((error) => {
-    console.error("[weft] Failed to restore in-flight tasks from storage:", error);
-  });
-}
-function rebuildWorkflowIndex(context, operationId, workflowId) {
-  if (!workflowId)
-    return;
-  let opIds = context.workflowOperations.get(workflowId);
-  if (!opIds) {
-    opIds = new Set;
-    context.workflowOperations.set(workflowId, opIds);
-  }
-  opIds.add(operationId);
-  context.operationToWorkflow.set(operationId, workflowId);
-}

package/dist/server/task-ledger-codec.d.ts

@@ -0,0 +1,40 @@
+/**
+ * Type guards and encode/decode for the durable remote task ledger (WFT-25).
+ *
+ * Every stored or wire-derived field is proven from `unknown` — the project
+ * brief's Security section requires operation IDs, workflow IDs, names,
+ * queues, header counts, header bytes, retry fields, and payload sizes to be
+ * bounded before a storage write, and `input`/`headers` are treated as
+ * hostile the same way `worker/manifest/parse.ts` treats manifest content.
+ *
+ * Guards here intentionally check only the fields that are load-bearing for
+ * correctness and the fields the brief calls out for bounding — matching the
+ * existing `isQueuedRecord`/`isInflightRecord` style in `task-state.ts`
+ * rather than re-deriving every field's shape from the type system.
+ *
+ * @module server/task-ledger-codec
+ */
+import type { RemoteTaskCancelling, RemoteTaskCompleting, RemoteTaskDeadLettered, RemoteTaskLeased, RemoteTaskQueued, RemoteTaskRecord, RemoteTaskTerminal, RemoteTaskTerminalCancelled, RemoteTaskTerminalResolved, RemoteTaskTerminalRetryExhausted } from './task-ledger-types.ts';
+/** Bounded header map: at most {@link MAX_TASK_HEADER_COUNT} entries, each key/value at most {@link MAX_TASK_HEADER_VALUE_BYTES}. */
+export declare function isValidTaskHeaders(value: unknown): value is Readonly<Record<string, string>>;
+export declare function isRemoteTaskQueued(value: unknown): value is RemoteTaskQueued;
+export declare function isRemoteTaskLeased(value: unknown): value is RemoteTaskLeased;
+export declare function isRemoteTaskCompleting(value: unknown): value is RemoteTaskCompleting;
+export declare function isRemoteTaskCancelling(value: unknown): value is RemoteTaskCancelling;
+export declare function isRemoteTaskTerminalResolved(value: unknown): value is RemoteTaskTerminalResolved;
+export declare function isRemoteTaskTerminalCancelled(value: unknown): value is RemoteTaskTerminalCancelled;
+export declare function isRemoteTaskTerminalRetryExhausted(value: unknown): value is RemoteTaskTerminalRetryExhausted;
+export declare function isRemoteTaskTerminal(value: unknown): value is RemoteTaskTerminal;
+export declare function isRemoteTaskDeadLettered(value: unknown): value is RemoteTaskDeadLettered;
+/** Discriminate and validate a decoded value as any {@link RemoteTaskRecord} state. */
+export declare function isRemoteTaskRecord(value: unknown): value is RemoteTaskRecord;
+/** Canonical encoding for a validated {@link RemoteTaskRecord}. */
+export declare function encodeRemoteTaskRecord(record: RemoteTaskRecord): Uint8Array;
+/**
+ * Decode and validate a task ledger record. Returns `null` when the bytes are
+ * absent or decode to a value that fails bounds validation. This does not
+ * catch `decode()` throwing on bytes that are not valid MessagePack; storage
+ * bytes are always ones this codebase wrote, so byte-level corruption is
+ * treated as a storage integrity failure worth surfacing, not swallowing.
+ */
+export declare function decodeRemoteTaskRecord(bytes: Uint8Array | null): RemoteTaskRecord | null;