@lostgradient/weft 0.18.0 → 0.19.0

package/dist/server/runtime/task-ledger-completion.js

@@ -0,0 +1,72 @@
+import { TaskResultDeadLetteredEvent } from "../../core/events.js";
+import { isJSONValue } from "../../core/json.js";
+import { sha256Hex } from "../../worker/manifest/content-digest.js";
+import {
+  beginCompletion,
+  commitDeadLetter,
+  commitTerminalResult
+} from "../task-ledger-transitions.js";
+import {
+  decodeRemoteTaskRecord,
+  taskLedgerKey
+} from "../task-ledger.js";
+import { commitTaskLedgerTransition } from "./task-ledger-runtime.js";
+const COMPLETION_MAX_ATTEMPTS = 3, DEAD_LETTER_MAX_ATTEMPTS = 1;
+async function pendingResultDigest(input) {
+  return sha256Hex(JSON.stringify({
+    status: input.status,
+    value: input.value ?? null,
+    error: input.error ?? null
+  }));
+}
+function commitTerminalFromCompleting(storage, input, resultDigest) {
+  return commitTaskLedgerTransition(storage, input.operationId, (current, now) => commitTerminalResult(current, {
+    attemptToken: input.attemptToken,
+    resultDigest,
+    ...input.status === "failed" && input.error !== void 0 ? { error: input.error } : {}
+  }, now), COMPLETION_MAX_ATTEMPTS);
+}
+async function attemptDeadLetter(storage, input, resultDigest, persistenceFailureReason) {
+  const deadLettered = await commitTaskLedgerTransition(storage, input.operationId, (current, now) => commitDeadLetter(current, {
+    attemptToken: input.attemptToken,
+    resultDigest,
+    ...isJSONValue(input.value) ? { value: input.value } : {},
+    ...input.error !== void 0 ? { error: input.error } : {},
+    persistenceFailureReason
+  }, now), DEAD_LETTER_MAX_ATTEMPTS);
+  return deadLettered.ok ? deadLettered.record : void 0;
+}
+export function dispatchTaskDeadLetteredEvent(options, operationId, deadLettered, workerId) {
+  options.engine.dispatchEvent(new TaskResultDeadLetteredEvent({
+    operationId,
+    workflowId: deadLettered.workflowId,
+    activityName: deadLettered.activityName,
+    queue: deadLettered.queue,
+    workerId,
+    errorMessage: deadLettered.persistenceFailureReason
+  }));
+}
+export async function commitTaskLedgerCompletion(storage, input) {
+  const resultDigest = await pendingResultDigest(input), existing = decodeRemoteTaskRecord(await storage.get(taskLedgerKey(input.operationId)));
+  if (existing !== null && existing.state === "completing" && existing.attemptToken === input.attemptToken && existing.pendingResultDigest === resultDigest) {
+    const committed = await commitTerminalFromCompleting(storage, input, resultDigest);
+    if (!committed.ok) {
+      const deadLettered = await attemptDeadLetter(storage, input, resultDigest, committed.reason);
+      return { ...committed, ...deadLettered !== void 0 ? { deadLettered } : {} };
+    }
+    return { ok: !0, completing: existing, terminal: committed.record };
+  }
+  const begun = await commitTaskLedgerTransition(storage, input.operationId, (current) => beginCompletion(current, {
+    attemptToken: input.attemptToken,
+    pendingStatus: input.status,
+    pendingResultDigest: resultDigest
+  }), COMPLETION_MAX_ATTEMPTS);
+  if (!begun.ok)
+    return begun;
+  const committed = await commitTerminalFromCompleting(storage, input, resultDigest);
+  if (!committed.ok) {
+    const deadLettered = await attemptDeadLetter(storage, input, resultDigest, committed.reason);
+    return { ...committed, ...deadLettered !== void 0 ? { deadLettered } : {} };
+  }
+  return { ok: !0, completing: begun.record, terminal: committed.record };
+}

package/dist/server/runtime/task-ledger-recovery.d.ts

@@ -0,0 +1,55 @@
+/**
+ * Startup task-ledger recovery (WFT-23).
+ *
+ * A one-shot full scan of the durable `task-ledger:` keyspace, run once per
+ * `serve()` call before the recovery gate on `ServerContext.taskLedgerRecovery`
+ * opens (see `buildServerContext` in `serve-internals.ts`). Reconstructs the
+ * in-memory registry, deadline tracker, and task-queue indexes that
+ * `restoreInflightTasks` used to rebuild from the retired `op:inflight:`
+ * keyspace — those indexes are process-local caches; the ledger is the
+ * durable authority, so nothing here writes state, only rehydrates memory to
+ * match what storage already says.
+ *
+ * One branch per non-terminal state:
+ *  - `queued`   — redispatch through the normal dispatch path so a lost
+ *                 `scheduleDelayedDispatch` timer (or a record this process
+ *                 never dispatched at all) still becomes runnable.
+ *  - `leased`, deadline still in the future — rehydrate registry ownership
+ *                 and the deadline tracker so heartbeats and completions
+ *                 from the original worker are recognized after reconnect.
+ *  - `leased`, deadline already passed — requeue or exhaust through the same
+ *                 `reassignOrExpireTask` CAS path `scanExpiredTasks` and
+ *                 `reconcileOrphanedRecords` use; deletion is never correct
+ *                 here (see `RequeueExpiredAttemptInput`'s contract).
+ *  - `completing` — rehydrate registry ownership only. The pending result
+ *                 value isn't persisted (only its digest), so recovery
+ *                 cannot resolve this state itself; only the worker's
+ *                 redelivered `taskResult`, landing on
+ *                 `commitTaskLedgerCompletion`'s crash-resumption branch,
+ *                 can. Recovery's job is to make sure that redelivery still
+ *                 passes the WebSocket ownership guard after a restart.
+ *  - `cancelling` — rehydrate registry ownership only, for the same reason:
+ *                 nothing in the current runtime calls `commitCancellation`
+ *                 yet (a future project's scope), so there is no action to
+ *                 resume, only ownership to preserve.
+ *  - `terminal`, `deadLettered` — already resolved; skipped.
+ *
+ * A single corrupt or unrecognized record is logged and skipped — ordinary
+ * data hygiene, not a recovery failure. A failure in the scan itself (the
+ * storage iterator throwing) propagates and rejects the whole gate: per the
+ * project brief, "a scan failure leaves the remote worker plane unhealthy
+ * and prevents new claims rather than logging and continuing with partial
+ * indexes."
+ *
+ * @module server/runtime/task-ledger-recovery
+ */
+import type { ServeOptions } from '../index.ts';
+import type { ServerContext } from './context.ts';
+/**
+ * Run startup task-ledger recovery to completion. Resolves once every
+ * non-terminal record has been reconstructed; rejects if the scan itself
+ * failed. Callers gate task-plane entry points on the resulting promise
+ * rather than awaiting this directly from within another recovery-time
+ * scan — see `ServerContext.taskLedgerRecovery`'s doc comment.
+ */
+export declare function runTaskLedgerRecovery(context: ServerContext, options: ServeOptions): Promise<void>;

package/dist/server/runtime/task-ledger-recovery.js

@@ -0,0 +1,73 @@
+import {
+  decodeRemoteTaskRecord
+} from "../task-ledger.js";
+import { scheduleDelayedDispatch } from "./task-dispatch.js";
+import { reassignOrExpireTask, taskDispatchFromLedgerRecord } from "./task-reconciliation.js";
+function rebuildWorkflowIndex(context, operationId, workflowId) {
+  if (!workflowId)
+    return;
+  let operationIds = context.workflowOperations.get(workflowId);
+  if (!operationIds) {
+    operationIds = new Set;
+    context.workflowOperations.set(workflowId, operationIds);
+  }
+  operationIds.add(operationId);
+  context.operationToWorkflow.set(operationId, workflowId);
+}
+function rehydrateWorkerOwnership(context, record, now) {
+  const remaining = Math.max(0, record.leaseDeadline - now);
+  context.registry.assignTask(record.workerSessionId, record.operationId, remaining, record.fairShareKey, record.attemptToken);
+  const tracked = context.registry.getWorkerTasks(record.workerSessionId).find((task) => task.operationId === record.operationId);
+  if (tracked)
+    tracked.visibilityTimeout = record.visibilityTimeoutMilliseconds;
+  rebuildWorkflowIndex(context, record.operationId, record.workflowId);
+}
+async function recoverTaskLedgerRecord(context, options, decoded, now) {
+  switch (decoded.state) {
+    case "queued": {
+      const delay = Math.max(0, decoded.availableAt - now);
+      scheduleDelayedDispatch(context, options, taskDispatchFromLedgerRecord(decoded), delay);
+      return;
+    }
+    case "leased": {
+      if (decoded.leaseDeadline > now) {
+        rehydrateWorkerOwnership(context, decoded, now);
+        context.deadlineTracker.add({
+          operationId: decoded.operationId,
+          deadline: decoded.leaseDeadline
+        });
+        return;
+      }
+      await reassignOrExpireTask(context, options, decoded.operationId, decoded);
+      return;
+    }
+    case "completing":
+    case "cancelling": {
+      rehydrateWorkerOwnership(context, decoded, now);
+      return;
+    }
+    case "terminal":
+    case "deadLettered":
+      return;
+    default: {
+      const exhaustive = decoded;
+    }
+  }
+}
+export async function runTaskLedgerRecovery(context, options) {
+  const now = Date.now();
+  for await (const [key, value] of options.engine.storage.scan("task-ledger:")) {
+    if (context.stopping)
+      return;
+    try {
+      const decoded = decodeRemoteTaskRecord(value);
+      if (decoded === null) {
+        console.error(`[weft] Unrecognized task ledger record at "${key}" during startup recovery \u2014 skipping`);
+        continue;
+      }
+      await recoverTaskLedgerRecord(context, options, decoded, now);
+    } catch (error) {
+      console.error(`[weft] Failed to recover task ledger record at "${key}" \u2014 skipping:`, error);
+    }
+  }
+}

package/dist/server/runtime/task-ledger-runtime.d.ts

@@ -0,0 +1,64 @@
+/**
+ * Shared commit loop for the durable remote task ledger's live dispatch path
+ * (WFT-22) — claim, heartbeat, completion, and requeue all attempt a single
+ * pure transition against the current ledger record through
+ * `storage.conditionalBatch`, retrying only when the compare-and-swap itself
+ * is lost (a concurrent writer changed the record between read and write),
+ * never when the transition's own precondition rejects the freshly re-read
+ * record. A precondition rejection means the requested transition genuinely
+ * does not apply — for example the task is already terminal — and retrying
+ * the same rejected transition would just reject again.
+ *
+ * `commitTaskLedgerDelete` (WFT-24) is the delete counterpart, used by
+ * terminal-task retention: a conditional delete gated on a precondition
+ * function rather than a transition function, since deletion has no next
+ * record to write.
+ *
+ * @module server/runtime/task-ledger-runtime
+ */
+import { type Storage } from '../../storage/interface.ts';
+import type { TaskLedgerPreconditionResult, TaskLedgerTransitionResult } from '../task-ledger-transitions.ts';
+import { type RemoteTaskRecord } from '../task-ledger.ts';
+export type TaskLedgerCommitResult<T extends RemoteTaskRecord> = Readonly<{
+    ok: true;
+    record: T;
+}> | Readonly<{
+    ok: false;
+    reason: string;
+}>;
+export type TaskLedgerDeleteResult = Readonly<{
+    ok: true;
+}> | Readonly<{
+    ok: false;
+    reason: string;
+}>;
+/**
+ * Read the current ledger record, apply `transitionFn`, and commit the
+ * result with `expectedValue` set to the exact bytes just read — CAS by raw
+ * byte equality, not `encode(decode(bytes))` re-serialization (see
+ * `task-ledger-transitions.ts`'s module doc comment).
+ *
+ * Retries only a lost CAS (up to `maxAttempts`), re-reading and
+ * re-evaluating `transitionFn` against the fresh record each time — a
+ * concurrent heartbeat renewal, for instance, is a benign generation bump
+ * worth retrying past. A `transitionFn` precondition rejection returns
+ * immediately without consuming a retry, since the fresh record already
+ * reflects reality and re-attempting the identical transition cannot change
+ * the outcome.
+ */
+export declare function commitTaskLedgerTransition<T extends RemoteTaskRecord>(storage: Storage, operationId: string, transitionFn: (current: RemoteTaskRecord | null, now: number) => TaskLedgerTransitionResult<T>, maxAttempts?: number): Promise<TaskLedgerCommitResult<T>>;
+/**
+ * Read the current ledger record, check a delete-only precondition (WFT-24
+ * retention: {@link canDeleteRetainedTerminalTask}), and conditionally delete
+ * with `expectedValue` set to the exact bytes just read — the delete
+ * counterpart to {@link commitTaskLedgerTransition}, needed because
+ * `canDeleteRetainedTerminalTask` has no next record to write, only a
+ * precondition to satisfy before issuing a delete.
+ *
+ * Retries only a lost CAS, same as `commitTaskLedgerTransition` — a
+ * concurrent writer that changed the record between read and delete (for
+ * example a later adoption call bumping `retentionGeneration`) means the
+ * fresh record deserves a fresh precondition check, not a delete of state
+ * the caller never actually observed.
+ */
+export declare function commitTaskLedgerDelete(storage: Storage, operationId: string, preconditionFn: (current: RemoteTaskRecord | null) => TaskLedgerPreconditionResult, maxAttempts?: number): Promise<TaskLedgerDeleteResult>;

package/dist/server/runtime/task-ledger-runtime.js

@@ -0,0 +1,34 @@
+import { storageConditionalBatch } from "../../storage/interface.js";
+import {
+  decodeRemoteTaskRecord,
+  encodeRemoteTaskRecord,
+  taskLedgerKey
+} from "../task-ledger.js";
+export async function commitTaskLedgerTransition(storage, operationId, transitionFn, maxAttempts = 1) {
+  const key = taskLedgerKey(operationId);
+  for (let attempt = 1;attempt <= maxAttempts; attempt++) {
+    const rawExisting = await storage.get(key), current = decodeRemoteTaskRecord(rawExisting), result = transitionFn(current, Date.now());
+    if (!result.ok)
+      return result;
+    if (await storageConditionalBatch(storage, [{ key, expectedValue: rawExisting }], [{ type: "put", key, value: encodeRemoteTaskRecord(result.nextRecord) }]))
+      return { ok: !0, record: result.nextRecord };
+  }
+  return {
+    ok: !1,
+    reason: `lost the compare-and-swap race on operation "${operationId}" after ${String(maxAttempts)} attempt(s)`
+  };
+}
+export async function commitTaskLedgerDelete(storage, operationId, preconditionFn, maxAttempts = 1) {
+  const key = taskLedgerKey(operationId);
+  for (let attempt = 1;attempt <= maxAttempts; attempt++) {
+    const rawExisting = await storage.get(key), current = decodeRemoteTaskRecord(rawExisting), result = preconditionFn(current);
+    if (!result.ok)
+      return result;
+    if (await storageConditionalBatch(storage, [{ key, expectedValue: rawExisting }], [{ type: "delete", key }]))
+      return { ok: !0 };
+  }
+  return {
+    ok: !1,
+    reason: `lost the compare-and-swap race deleting operation "${operationId}" after ${String(maxAttempts)} attempt(s)`
+  };
+}

package/dist/server/runtime/task-metrics.d.ts

@@ -1,12 +1,14 @@
 import { type MetricsCollector } from '../../observability/metrics.ts';
 import type { WorkerRegistry } from '../../worker/registry.ts';
 import type { TaskQueue } from '../task-queue.ts';
-import { type InflightRecord, type ResolvedRecord } from '../task-state.ts';
+import { type TaskTimingFields } from '../task-state.ts';
 export declare function recordTaskBacklogMetric(metricsCollector: MetricsCollector | undefined, taskQueue: TaskQueue): void;
-export declare function recordTaskQueueLatencyMetric(metricsCollector: MetricsCollector | undefined, record: InflightRecord | ResolvedRecord): void;
-export declare function recordTaskExecutionLatencyMetric(metricsCollector: MetricsCollector | undefined, record: InflightRecord | ResolvedRecord, completedAt: number): void;
+export declare function recordTaskQueueLatencyMetric(metricsCollector: MetricsCollector | undefined, record: TaskTimingFields): void;
+export declare function recordTaskExecutionLatencyMetric(metricsCollector: MetricsCollector | undefined, record: TaskTimingFields, completedAt: number): void;
 export declare function recordTaskRetryMetric(metricsCollector: MetricsCollector | undefined, count?: number): void;
 export declare function recordTaskRequeueMetric(metricsCollector: MetricsCollector | undefined, count?: number): void;
-export declare function isTaskHeartbeatStaleForMetrics(record: InflightRecord, currentTime: number, staleAfterMs?: number): boolean;
+export declare function isTaskHeartbeatStaleForMetrics(record: TaskTimingFields & {
+    deadline?: number | undefined;
+}, currentTime: number, staleAfterMs?: number): boolean;
 export declare function recordTaskStaleHeartbeatMetric(metricsCollector: MetricsCollector | undefined, count: number): void;
 export declare function recordWorkerCapacitySaturationMetric(metricsCollector: MetricsCollector | undefined, registry: WorkerRegistry): void;

package/dist/server/runtime/task-polling.d.ts

@@ -1,23 +1,25 @@
 import type { ServeOptions } from '../index.ts';
 import { type Principal } from '../principal.ts';
 import type { PendingTask } from '../task-queue-types.ts';
-import type { InflightRecord } from '../task-state.ts';
 import type { ServerContext } from './context.ts';
-/**
- * A freshly created long-poll inflight record always carries an `attemptToken`.
- * The intersection makes that invariant a compile-time fact so the claim can read
- * the token directly without an unreachable empty-string fallback.
- */
-type TokenedInflightRecord = InflightRecord & {
-    attemptToken: string;
-};
-export declare function createLongPollInflightRecord(queue: string, task: PendingTask): TokenedInflightRecord;
 /** The worker-facing identity of a long-poll claim: the synthetic worker id and its per-claim token. */
 export interface LongPollClaim {
     workerId: string;
     attemptToken: string;
 }
-export declare function markTaskClaimedByLongPollWorker(context: ServerContext, options: ServeOptions, queue: string, task: PendingTask): Promise<LongPollClaim>;
+/**
+ * Conditionally claim the durable `queued` ledger record for a task the
+ * in-memory `TaskQueue` matched to a long-poll waiter. Returns `null` when
+ * the claim loses — the ledger disagrees with the in-memory match hint,
+ * meaning another actor already claimed or cancelled this operationId — in
+ * which case the caller treats the poll as if nothing matched, per "index
+ * disagreement never authorizes a state transition".
+ *
+ * Long-poll workers never call `WorkerRegistry.register()`, so there is no
+ * manifest to build a `WorkerExecutionIdentity` from; the claim always omits
+ * `executionIdentity` (see `RemoteTaskLeased.executionIdentity`'s doc
+ * comment) rather than fabricate one.
+ */
+export declare function markTaskClaimedByLongPollWorker(context: ServerContext, options: ServeOptions, task: PendingTask): Promise<LongPollClaim | null>;
 export declare function handleTaskPollRequest(context: ServerContext, options: ServeOptions, request: Request, url: URL, principal?: Principal): Promise<Response | null>;
 export declare function handleTaskResultRequest(context: ServerContext, options: ServeOptions, request: Request, url: URL, principal?: Principal): Promise<Response | null>;
-export {};

package/dist/server/runtime/task-polling.js

@@ -1,11 +1,18 @@
 import { isAuthenticated } from "../principal.js";
 import { readRestJsonBody } from "../rest-body.js";
-import { readInflightRecord, transitionQueuedToInflight } from "../task-state.js";
-import { recordTaskBacklogMetric, recordTaskQueueLatencyMetric } from "./task-metrics.js";
+import { claimQueued } from "../task-ledger-transitions.js";
+import { decodeRemoteTaskRecord, taskLedgerKey } from "../task-ledger.js";
 import {
-  taskResultPayloadSizeError,
-  transitionTaskResultToResolvedWithRetry
-} from "./task-result-resolution.js";
+  commitTaskLedgerCompletion,
+  dispatchTaskDeadLetteredEvent
+} from "./task-ledger-completion.js";
+import { commitTaskLedgerTransition } from "./task-ledger-runtime.js";
+import {
+  recordTaskBacklogMetric,
+  recordTaskExecutionLatencyMetric,
+  recordTaskQueueLatencyMetric
+} from "./task-metrics.js";
+import { taskResultPayloadSizeError } from "./task-result-resolution.js";
 const TASK_POLL_RE = /^\/v1\/tasks\/([\w-]+)$/, TASK_RESULT_RE = /^\/v1\/tasks\/([\w-]+)\/result$/, TASK_DIAGNOSTICS_PATH = "/v1/tasks/diagnostics", MAX_POLL_TIMEOUT = 60000, DEFAULT_POLL_TIMEOUT = 30000, DEFAULT_VISIBILITY_TIMEOUT = 30000;
 async function parseTaskResultBody(request, options) {
   try {
@@ -26,6 +33,16 @@ function authorizeWorkerPrincipal(principal) {
     return null;
   return Response.json({ error: "Forbidden" }, { status: 403 });
 }
+async function awaitTaskLedgerRecovery(context) {
+  try {
+    await context.taskLedgerRecovery.ready;
+    return null;
+  } catch (error) {
+    return Response.json({
+      error: `Startup task-ledger recovery failed \u2014 cannot admit new task claims: ${error instanceof Error ? error.message : String(error)}`
+    }, { status: 503 });
+  }
+}
 function validateTaskResultBody(body) {
   const { operationId, status } = body;
   if (typeof operationId !== "string" || typeof status !== "string")
@@ -44,37 +61,35 @@ function validateTaskResultBody(body) {
     attemptToken: rawAttemptToken
   };
 }
-async function applyTaskResult(context, options, result, inflightRecord) {
-  const { operationId, status, value, error } = result, resolvedStatus = status === "failed" ? "failed" : "completed";
-  try {
-    context.taskQueue.complete({ operationId, status, value, error });
-    context.deadlineTracker.remove(operationId);
-    await transitionTaskResultToResolvedWithRetry(context, options, {
-      operationId,
-      status: resolvedStatus,
-      resolutionReason: resolvedStatus,
-      inflightRecord,
-      ...status === "completed" ? { value } : { error }
-    });
-  } catch (storageError) {
-    console.error(`[weft] Failed to transition task "${operationId}" to resolved \u2014 inflight record may leak:`, storageError);
-  }
+function isLongPollCompletionAuthorized(record, validated) {
+  if (record === null)
+    return !1;
+  if (record.state !== "leased" && record.state !== "completing")
+    return !1;
+  if (validated.workerId === void 0 || record.workerSessionId !== validated.workerId)
+    return !1;
+  return validated.attemptToken === record.attemptToken;
 }
-async function applyPayloadRejectedTaskResult(context, options, result, inflightRecord, error) {
-  context.taskQueue.complete({
-    operationId: result.operationId,
-    status: "failed",
-    error: error.message
-  });
-  context.deadlineTracker.remove(result.operationId);
-  await transitionTaskResultToResolvedWithRetry(context, options, {
-    operationId: result.operationId,
-    status: "failed",
-    resolutionReason: "failed",
-    inflightRecord,
-    error: error.message,
-    skipPayloadSizeCheck: !0
+async function applyTaskResult(context, options, result) {
+  const { operationId, status, value, error } = result, committed = await commitTaskLedgerCompletion(options.engine.storage, {
+    operationId,
+    attemptToken: result.attemptToken,
+    status,
+    ...status === "completed" ? { value } : {},
+    ...status === "failed" && error !== void 0 ? { error } : {}
   });
+  if (!committed.ok) {
+    if (committed.deadLettered !== void 0) {
+      context.deadlineTracker.remove(operationId);
+      dispatchTaskDeadLetteredEvent(options, operationId, committed.deadLettered, result.workerId);
+    }
+    return { ok: !1, reason: committed.reason };
+  }
+  recordTaskExecutionLatencyMetric(context.metricsCollector, { startedAt: committed.completing.startedAt }, Date.now());
+  context.taskQueue.complete({ operationId, status, value, error });
+  context.deadlineTracker.remove(operationId);
+  recordTaskBacklogMetric(context.metricsCollector, context.taskQueue);
+  return { ok: !0 };
 }
 function payloadSizeExceededResponse(error) {
   return Response.json({
@@ -85,43 +100,32 @@ function payloadSizeExceededResponse(error) {
     payloadKind: error.payloadKind
   }, { status: 413 });
 }
-export function createLongPollInflightRecord(queue, task) {
-  const now = Date.now(), visibilityTimeout = task.visibilityTimeout ?? DEFAULT_VISIBILITY_TIMEOUT, deadline = now + visibilityTimeout;
-  return {
-    operationId: task.operationId,
-    workerId: `longpoll-${crypto.randomUUID().slice(0, 8)}`,
-    deadline,
-    activityName: task.activityName,
-    queue,
-    input: task.input,
-    attempt: task.attempt ?? 1,
-    visibilityTimeout,
-    retryPolicy: task.retryPolicy,
-    workflowId: task.workflowId,
-    workflowExecutionToken: task.workflowExecutionToken,
-    attemptToken: crypto.randomUUID(),
-    firstQueuedAt: task.firstQueuedAt ?? task.enqueuedAt ?? now,
-    lastQueuedAt: task.lastQueuedAt ?? task.enqueuedAt ?? now,
-    lastDispatchedAt: now,
-    startedAt: now,
-    retryCount: task.retryCount ?? Math.max(0, (task.attempt ?? 1) - 1),
-    requeueCount: task.requeueCount ?? 0,
-    lastRequeueReason: task.lastRequeueReason
-  };
-}
-export async function markTaskClaimedByLongPollWorker(context, options, queue, task) {
-  const inflightRecord = createLongPollInflightRecord(queue, task);
+export async function markTaskClaimedByLongPollWorker(context, options, task) {
+  const workerSessionId = `longpoll-${crypto.randomUUID().slice(0, 8)}`, attemptToken = crypto.randomUUID(), visibilityTimeout = task.visibilityTimeout ?? DEFAULT_VISIBILITY_TIMEOUT, result = await commitTaskLedgerTransition(options.engine.storage, task.operationId, (current, now) => {
+    if (current === null || current.state !== "queued")
+      return {
+        ok: !1,
+        reason: `operation "${task.operationId}" is not claimable from its current ledger state`
+      };
+    return claimQueued(current, {
+      expectedGeneration: current.generation,
+      attemptToken,
+      workerSessionId,
+      leaseDurationMilliseconds: visibilityTimeout
+    }, now);
+  }, 1);
+  if (!result.ok)
+    return null;
   context.deadlineTracker.add({
     operationId: task.operationId,
-    deadline: inflightRecord.deadline
+    deadline: result.record.leaseDeadline
+  });
+  recordTaskQueueLatencyMetric(context.metricsCollector, {
+    lastQueuedAt: result.record.lastQueuedAt,
+    lastDispatchedAt: Date.now()
   });
-  const normalizedInflightRecord = await transitionQueuedToInflight(options.engine.storage, task.operationId, inflightRecord);
-  recordTaskQueueLatencyMetric(context.metricsCollector, normalizedInflightRecord);
   recordTaskBacklogMetric(context.metricsCollector, context.taskQueue);
-  return {
-    workerId: normalizedInflightRecord.workerId,
-    attemptToken: inflightRecord.attemptToken
-  };
+  return { workerId: workerSessionId, attemptToken };
 }
 export async function handleTaskPollRequest(context, options, request, url, principal) {
   if (request.method !== "GET")
@@ -134,12 +138,17 @@ export async function handleTaskPollRequest(context, options, request, url, prin
   const authorizationResponse = authorizeWorkerPrincipal(principal);
   if (authorizationResponse !== null)
     return authorizationResponse;
+  const recoveryResponse = await awaitTaskLedgerRecovery(context);
+  if (recoveryResponse !== null)
+    return recoveryResponse;
   const queue = decodeURIComponent(pollMatch[1]), activities = url.searchParams.getAll("activity");
   if (activities.length === 0)
     return Response.json({ error: 'At least one "activity" query parameter is required' }, { status: 400 });
   const rawTimeout = url.searchParams.get("timeout"), timeout = rawTimeout !== null ? Math.min(Math.max(0, Number(rawTimeout)), MAX_POLL_TIMEOUT) : DEFAULT_POLL_TIMEOUT, task = await context.taskQueue.poll(queue, activities, timeout, request.signal);
   if (task !== null) {
-    const claim = await markTaskClaimedByLongPollWorker(context, options, queue, task);
+    const claim = await markTaskClaimedByLongPollWorker(context, options, task);
+    if (claim === null)
+      return new Response(null, { status: 204 });
     return Response.json({
       ...task,
       workerId: claim.workerId,
@@ -159,6 +168,9 @@ export async function handleTaskResultRequest(context, options, request, url, pr
   const authorizationResponse = authorizeWorkerPrincipal(principal);
   if (authorizationResponse !== null)
     return authorizationResponse;
+  const recoveryResponse = await awaitTaskLedgerRecovery(context);
+  if (recoveryResponse !== null)
+    return recoveryResponse;
   const body = await parseTaskResultBody(request, options.maxRequestBodyBytes !== void 0 ? { maxBodyBytes: options.maxRequestBodyBytes } : {});
   if (body instanceof Response)
     return body;
@@ -167,28 +179,40 @@ export async function handleTaskResultRequest(context, options, request, url, pr
   const validated = validateTaskResultBody(body);
   if (validated instanceof Response)
     return validated;
-  const inflightRecord = await readInflightRecord(options.engine.storage, validated.operationId);
-  if (inflightRecord !== null && !isLongPollCompletionAuthorized(inflightRecord, validated))
+  const record = decodeRemoteTaskRecord(await options.engine.storage.get(taskLedgerKey(validated.operationId)));
+  if (!isLongPollCompletionAuthorized(record, validated))
     return Response.json({ error: "Forbidden" }, { status: 403 });
   const payloadError = taskResultPayloadSizeError({
-    operationId: validated.operationId,
     status: validated.status,
-    resolutionReason: validated.status,
     ...validated.status === "completed" ? { value: validated.value } : { error: validated.error }
   }, context.payloadSizeMaxBytes);
   if (payloadError !== null) {
-    try {
-      await applyPayloadRejectedTaskResult(context, options, validated, inflightRecord, payloadError);
-    } catch (error) {
-      console.error(`[weft] Failed to persist oversized task result rejection for task "${validated.operationId}":`, error);
+    const rejected = await commitTaskLedgerCompletion(options.engine.storage, {
+      operationId: validated.operationId,
+      attemptToken: validated.attemptToken,
+      status: "failed",
+      error: payloadError.message
+    });
+    if (rejected.ok) {
+      recordTaskExecutionLatencyMetric(context.metricsCollector, { startedAt: rejected.completing.startedAt }, Date.now());
+      context.taskQueue.complete({
+        operationId: validated.operationId,
+        status: "failed",
+        error: payloadError.message
+      });
+      context.deadlineTracker.remove(validated.operationId);
+      recordTaskBacklogMetric(context.metricsCollector, context.taskQueue);
+    } else {
+      console.error(`[weft] Failed to persist oversized task result rejection for task "${validated.operationId}":`, rejected.reason);
+      if (rejected.deadLettered !== void 0)
+        dispatchTaskDeadLetteredEvent(options, validated.operationId, rejected.deadLettered, validated.workerId);
     }
     return payloadSizeExceededResponse(payloadError);
   }
-  await applyTaskResult(context, options, validated, inflightRecord);
+  const applied = await applyTaskResult(context, options, validated);
+  if (!applied.ok) {
+    console.error(`[weft] Failed to commit task result for "${validated.operationId}" through the durable ledger:`, applied.reason);
+    return Response.json({ error: "Forbidden" }, { status: 403 });
+  }
   return Response.json({ ok: !0 });
 }
-function isLongPollCompletionAuthorized(inflightRecord, validated) {
-  if (validated.workerId === void 0 || inflightRecord.workerId !== validated.workerId)
-    return !1;
-  return validated.attemptToken === inflightRecord.attemptToken;
-}