@lostgradient/weft 0.18.0 → 0.20.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (171) hide show
  1. package/README.md +3 -1
  2. package/dist/cli/conformance.js +14 -18
  3. package/dist/cli/generated/operation-catalog.snapshot.json +1597 -21
  4. package/dist/cli/generated/operation-client.generated.d.ts +37 -6
  5. package/dist/cli/generated/operation-client.generated.js +6 -0
  6. package/dist/core/context/child-workflow-pipe.d.ts +1 -1
  7. package/dist/core/context/child-workflow-pipe.js +1 -1
  8. package/dist/core/engine/construction.d.ts +2 -0
  9. package/dist/core/engine/construction.js +3 -0
  10. package/dist/core/engine/index.js +1 -0
  11. package/dist/core/engine/listing.d.ts +1 -0
  12. package/dist/core/engine/listing.js +6 -4
  13. package/dist/core/weft-error.d.ts +1 -1
  14. package/dist/core/weft-error.js +2 -1
  15. package/dist/core/worker-execution-dispatcher.d.ts +8 -0
  16. package/dist/core/worker-execution-dispatcher.js +4 -0
  17. package/dist/core/worker-execution-strategy-options.d.ts +16 -0
  18. package/dist/core/worker-execution-strategy.js +71 -89
  19. package/dist/core/worker-fault-handling.d.ts +45 -0
  20. package/dist/core/worker-fault-handling.js +88 -0
  21. package/dist/core/worker-protocol.d.ts +18 -1
  22. package/dist/core/worker-protocol.js +1 -1
  23. package/dist/core/worker-realm-readiness.d.ts +78 -0
  24. package/dist/core/worker-realm-readiness.js +127 -0
  25. package/dist/index.d.ts +3 -1
  26. package/dist/index.js +28 -1
  27. package/dist/json-schema.js +1 -1
  28. package/dist/runtime/portable.d.ts +14 -0
  29. package/dist/runtime/portable.js +10 -0
  30. package/dist/server/index.d.ts +71 -8
  31. package/dist/server/index.js +7 -3
  32. package/dist/server/operations/get-task-detail-schema.d.ts +341 -0
  33. package/dist/server/operations/get-task-detail-schema.js +118 -0
  34. package/dist/server/operations/get-task-detail.d.ts +346 -0
  35. package/dist/server/operations/get-task-detail.js +215 -0
  36. package/dist/server/operations/get-task-diagnostics.d.ts +43 -18
  37. package/dist/server/operations/get-task-diagnostics.js +87 -126
  38. package/dist/server/operations/get-worker-diagnostics.d.ts +177 -0
  39. package/dist/server/operations/get-worker-diagnostics.js +117 -0
  40. package/dist/server/operations/list-worker-registration-rejections.d.ts +79 -0
  41. package/dist/server/operations/list-worker-registration-rejections.js +60 -0
  42. package/dist/server/operations/list-workers.js +1 -3
  43. package/dist/server/operations/static-registrations.js +8 -20
  44. package/dist/server/rest-bindings.js +31 -2
  45. package/dist/server/runtime/authentication-bridge.js +6 -14
  46. package/dist/server/runtime/context.d.ts +23 -0
  47. package/dist/server/runtime/task-dispatch.d.ts +7 -1
  48. package/dist/server/runtime/task-dispatch.js +126 -55
  49. package/dist/server/runtime/task-ledger-completion.d.ts +68 -0
  50. package/dist/server/runtime/task-ledger-completion.js +72 -0
  51. package/dist/server/runtime/task-ledger-recovery.d.ts +55 -0
  52. package/dist/server/runtime/task-ledger-recovery.js +73 -0
  53. package/dist/server/runtime/task-ledger-runtime.d.ts +64 -0
  54. package/dist/server/runtime/task-ledger-runtime.js +34 -0
  55. package/dist/server/runtime/task-metrics.d.ts +6 -4
  56. package/dist/server/runtime/task-polling.d.ts +14 -12
  57. package/dist/server/runtime/task-polling.js +106 -82
  58. package/dist/server/runtime/task-reconciliation.d.ts +35 -8
  59. package/dist/server/runtime/task-reconciliation.js +75 -74
  60. package/dist/server/runtime/task-result-resolution.d.ts +2 -10
  61. package/dist/server/runtime/task-result-resolution.js +0 -67
  62. package/dist/server/runtime/task-result-view.d.ts +82 -0
  63. package/dist/server/runtime/task-result-view.js +48 -0
  64. package/dist/server/runtime/websocket-worker-messaging.d.ts +16 -0
  65. package/dist/server/runtime/websocket-worker-messaging.js +32 -0
  66. package/dist/server/runtime/websocket-worker-registration.d.ts +41 -0
  67. package/dist/server/runtime/websocket-worker-registration.js +134 -0
  68. package/dist/server/runtime/websocket-worker.d.ts +0 -1
  69. package/dist/server/runtime/websocket-worker.js +64 -116
  70. package/dist/server/serve-internals.d.ts +9 -6
  71. package/dist/server/serve-internals.js +14 -40
  72. package/dist/server/task-ledger-codec.d.ts +40 -0
  73. package/dist/server/task-ledger-codec.js +195 -0
  74. package/dist/server/task-ledger-keys.d.ts +13 -0
  75. package/dist/server/task-ledger-keys.js +5 -0
  76. package/dist/server/task-ledger-limits.d.ts +34 -0
  77. package/dist/server/task-ledger-limits.js +5 -0
  78. package/dist/server/task-ledger-transition-helpers.d.ts +38 -0
  79. package/dist/server/task-ledger-transition-helpers.js +42 -0
  80. package/dist/server/task-ledger-transitions-cancellation.d.ts +31 -0
  81. package/dist/server/task-ledger-transitions-cancellation.js +73 -0
  82. package/dist/server/task-ledger-transitions.d.ts +129 -0
  83. package/dist/server/task-ledger-transitions.js +162 -0
  84. package/dist/server/task-ledger-types.d.ts +220 -0
  85. package/dist/server/task-ledger-types.js +1 -0
  86. package/dist/server/task-ledger.d.ts +17 -0
  87. package/dist/server/task-ledger.js +24 -0
  88. package/dist/server/task-state.d.ts +25 -152
  89. package/dist/server/task-state.js +0 -188
  90. package/dist/server/worker-admission-policy.d.ts +70 -0
  91. package/dist/server/worker-admission-policy.js +0 -0
  92. package/dist/storage/compressed-storage.js +1 -1
  93. package/dist/storage/index.d.ts +0 -2
  94. package/dist/storage/interface.d.ts +0 -2
  95. package/dist/storage/interface.js +1 -1
  96. package/dist/storage/lmdb.js +1 -1
  97. package/dist/storage/memory.js +1 -1
  98. package/dist/storage/neon.js +1 -1
  99. package/dist/storage/postgres.js +1 -1
  100. package/dist/storage/resolve.js +1 -1
  101. package/dist/storage/scoped-storage.js +1 -1
  102. package/dist/storage/testing.js +1 -1
  103. package/dist/storage/turso.js +1 -1
  104. package/dist/storage/web-extension.js +2 -2
  105. package/dist/version.d.ts +1 -1
  106. package/dist/version.js +1 -1
  107. package/dist/web-extension.js +1 -1
  108. package/dist/worker/index.d.ts +2 -0
  109. package/dist/worker/index.js +30 -4
  110. package/dist/worker/manifest/canonical-json.d.ts +20 -0
  111. package/dist/worker/manifest/canonical-json.js +13 -0
  112. package/dist/worker/manifest/capabilities.d.ts +24 -0
  113. package/dist/worker/manifest/capabilities.js +74 -0
  114. package/dist/worker/manifest/content-digest.d.ts +20 -0
  115. package/dist/worker/manifest/content-digest.js +13 -0
  116. package/dist/worker/manifest/declared-shape-digest.d.ts +14 -0
  117. package/dist/worker/manifest/declared-shape-digest.js +4 -0
  118. package/dist/worker/manifest/digest.d.ts +82 -0
  119. package/dist/worker/manifest/digest.js +9 -0
  120. package/dist/worker/manifest/execution-identity.d.ts +92 -0
  121. package/dist/worker/manifest/execution-identity.js +36 -0
  122. package/dist/worker/manifest/failure.d.ts +59 -0
  123. package/dist/worker/manifest/failure.js +4 -0
  124. package/dist/worker/manifest/index.d.ts +22 -0
  125. package/dist/worker/manifest/index.js +23 -0
  126. package/dist/worker/manifest/internal-realm.d.ts +50 -0
  127. package/dist/worker/manifest/internal-realm.js +39 -0
  128. package/dist/worker/manifest/is-record.d.ts +12 -0
  129. package/dist/worker/manifest/is-record.js +6 -0
  130. package/dist/worker/manifest/json-scan.d.ts +26 -0
  131. package/dist/worker/manifest/json-scan.js +88 -0
  132. package/dist/worker/manifest/limits.d.ts +138 -0
  133. package/dist/worker/manifest/limits.js +1 -0
  134. package/dist/worker/manifest/normalize.d.ts +63 -0
  135. package/dist/worker/manifest/normalize.js +73 -0
  136. package/dist/worker/manifest/parse-json.d.ts +24 -0
  137. package/dist/worker/manifest/parse-json.js +15 -0
  138. package/dist/worker/manifest/parse.d.ts +86 -0
  139. package/dist/worker/manifest/parse.js +188 -0
  140. package/dist/worker/manifest/registry-contract-builder.d.ts +120 -0
  141. package/dist/worker/manifest/registry-contract-builder.js +65 -0
  142. package/dist/worker/manifest/types.d.ts +263 -0
  143. package/dist/worker/manifest/types.js +1 -0
  144. package/dist/worker/manifest/utf8.d.ts +13 -0
  145. package/dist/worker/manifest/utf8.js +7 -0
  146. package/dist/worker/options.d.ts +55 -8
  147. package/dist/worker/options.js +65 -9
  148. package/dist/worker/protocol-internals.d.ts +1 -2
  149. package/dist/worker/protocol-internals.js +0 -5
  150. package/dist/worker/protocol-messages.d.ts +33 -17
  151. package/dist/worker/protocol-schemas.d.ts +36 -74
  152. package/dist/worker/protocol-schemas.js +25 -18
  153. package/dist/worker/protocol-version.d.ts +18 -14
  154. package/dist/worker/protocol-version.js +1 -1
  155. package/dist/worker/protocol.js +1 -1
  156. package/dist/worker/registry/deployment-consistency.d.ts +78 -0
  157. package/dist/worker/registry/deployment-consistency.js +27 -0
  158. package/dist/worker/registry/drain.d.ts +14 -0
  159. package/dist/worker/registry/drain.js +24 -0
  160. package/dist/worker/registry/rejections.d.ts +31 -0
  161. package/dist/worker/registry/rejections.js +10 -0
  162. package/dist/worker/registry/summary.d.ts +0 -3
  163. package/dist/worker/registry/summary.js +2 -5
  164. package/dist/worker/registry/types.d.ts +10 -3
  165. package/dist/worker/registry.d.ts +47 -2
  166. package/dist/worker/registry.js +45 -32
  167. package/dist/workers/workflow-worker-entry.d.ts +12 -7
  168. package/dist/workers/workflow-worker-entry.js +9 -2
  169. package/package.json +3 -3
  170. package/dist/server/task-resolved-record.d.ts +0 -12
  171. package/dist/server/task-resolved-record.js +0 -35
@@ -1,5 +1,5 @@
1
- import type { ServeOptions } from '../index.ts';
2
- import type { InflightRecord, TaskRequeueReason } from '../task-state.ts';
1
+ import type { ServeOptions, TaskDispatch } from '../index.ts';
2
+ import { type RemoteTaskBase, type RemoteTaskLeased } from '../task-ledger.ts';
3
3
  import type { ServerContext } from './context.ts';
4
4
  interface ManualTaskReconciliationForTesting {
5
5
  readonly options: ServeOptions;
@@ -10,17 +10,44 @@ export declare function useManualTaskReconciliationForTesting(options: ServeOpti
10
10
  /** @internal Installs the narrow one-shot test controller before a server starts. */
11
11
  export declare function consumeManualTaskReconciliationForTesting(options: ServeOptions, context: ServerContext, cleanupWorkflowIndex: (operationId: string) => void): boolean;
12
12
  /**
13
- * Given a persisted inflight record, either permanently fail the task (if
14
- * retry attempts are exhausted) or transition it back to queued and
15
- * re-dispatch with backoff. Both the worker-disconnect handler and the
16
- * visibility-timeout scanner share this logic.
13
+ * Given a leased ledger record whose visibility deadline has passed, either
14
+ * permanently fail the task (retry attempts exhausted) or requeue it with
15
+ * backoff for redispatch. Both the worker-disconnect handler and the
16
+ * visibility-timeout scanner share this logic. A single attempt, no retry —
17
+ * a lost compare-and-swap here means a concurrent heartbeat renewed the
18
+ * lease or another actor already resolved this attempt, and retrying the
19
+ * identical transition cannot change that outcome.
17
20
  */
18
- export declare function reassignOrExpireTask(context: ServerContext, options: ServeOptions, operationId: string, record: InflightRecord, reason?: TaskRequeueReason): Promise<void>;
21
+ export declare function reassignOrExpireTask(context: ServerContext, options: ServeOptions, operationId: string, record: RemoteTaskLeased, reason?: string): Promise<void>;
22
+ /**
23
+ * Reconstruct the `TaskDispatch` a fresh dispatch of this operation would
24
+ * have used, from the durable envelope alone. Shared by expired-lease
25
+ * redispatch ({@link reassignOrExpireTask}) and startup recovery's
26
+ * `queued`-record redispatch (`task-ledger-recovery.ts`) — both cases hand a
27
+ * record back to `dispatchTaskImpl` with no original caller-supplied
28
+ * `TaskDispatch` still in memory. Every `RemoteTaskBase` field
29
+ * `buildRoutingOptions`/`resolveTaskPriority` read at dispatch time
30
+ * (`priority`, `fairShareKey`, sticky affinity) must round-trip here too, or
31
+ * a requeued or recovered task silently loses its original routing intent.
32
+ */
33
+ export declare function taskDispatchFromLedgerRecord(record: RemoteTaskBase): TaskDispatch;
19
34
  /**
20
35
  * Drain expired entries from the in-memory deadline heap and reassign
21
36
  * their tasks. Only touches storage for the specific operations whose
22
- * deadlines have actually passed — no full `op:inflight:*` scan.
37
+ * deadlines have actually passed — no full ledger scan.
23
38
  */
24
39
  export declare function scanExpiredTasks(context: ServerContext, options: ServeOptions, cleanupWorkflowIndex: (operationId: string) => void, now?: number): Promise<void>;
40
+ /**
41
+ * Periodic full-ledger safety net (WFT-23), independent of the in-memory
42
+ * deadline heap `scanExpiredTasks` drains: catches `leased` records whose
43
+ * expiry was never tracked in the heap (written by a peer, or lost to a
44
+ * restart), `queued` records whose durable `availableAt` has elapsed but
45
+ * whose `scheduleDelayedDispatch` timer never fired (same causes), and
46
+ * (WFT-24) adopted `terminal` records old enough to reap under
47
+ * {@link ServeOptions.taskRetentionWindowMs}. `completing`, `cancelling`,
48
+ * and `deadLettered` are left untouched — resolving those is either the
49
+ * worker's redelivered result (`completing`) or explicitly out of this
50
+ * slice's scope.
51
+ */
25
52
  export declare function reconcileOrphanedRecords(context: ServerContext, options: ServeOptions, cleanupWorkflowIndex: (operationId: string) => void): Promise<void>;
26
53
  export {};
@@ -1,14 +1,15 @@
1
- import { decode } from "../../core/codec.js";
2
1
  import { ActivityFailedEvent } from "../../core/events.js";
3
- import { calculateBackoff } from "../../core/scheduler.js";
4
- import { KEYS } from "../../storage/interface.js";
5
2
  import { restoreExtendedDeadlineIfStillActive } from "../runtime-helpers.js";
6
3
  import {
7
- isTaskDeadLettered,
8
- transitionInflightToQueued,
9
- transitionInflightToResolved
10
- } from "../task-state.js";
11
- import { dispatchTaskImpl, scheduleDelayedDispatch } from "./task-dispatch.js";
4
+ canDeleteRetainedTerminalTask,
5
+ requeueExpiredAttempt
6
+ } from "../task-ledger-transitions.js";
7
+ import {
8
+ decodeRemoteTaskRecord,
9
+ taskLedgerKey
10
+ } from "../task-ledger.js";
11
+ import { scheduleDelayedDispatch } from "./task-dispatch.js";
12
+ import { commitTaskLedgerDelete, commitTaskLedgerTransition } from "./task-ledger-runtime.js";
12
13
  import {
13
14
  isTaskHeartbeatStaleForMetrics,
14
15
  recordTaskRequeueMetric,
@@ -16,7 +17,6 @@ import {
16
17
  recordTaskStaleHeartbeatMetric,
17
18
  recordWorkerCapacitySaturationMetric
18
19
  } from "./task-metrics.js";
19
- import { isInflightRecord } from "./websocket-worker.js";
20
20
  const manualTaskReconciliationRegistrations = new WeakMap;
21
21
  export function useManualTaskReconciliationForTesting(options) {
22
22
  const registration = {};
@@ -42,69 +42,65 @@ export function consumeManualTaskReconciliationForTesting(options, context, clea
42
42
  return !0;
43
43
  }
44
44
  export async function reassignOrExpireTask(context, options, operationId, record, reason = "visibility-timeout") {
45
- const nextAttempt = (record.attempt ?? 1) + 1, policy = record.retryPolicy, nextRetryCount = Math.max(record.retryCount ?? 0, nextAttempt - 1);
46
- if (hasExceededMaxAttempts(policy, nextAttempt)) {
47
- await expireTaskAfterMaxAttempts(context, options, operationId, record, policy);
45
+ const skipDeadlineCheck = reason === "worker-disconnect", result = await commitTaskLedgerTransition(options.engine.storage, operationId, (current, now) => requeueExpiredAttempt(current, { attemptToken: record.attemptToken, requeueReason: reason, skipDeadlineCheck }, now), 1);
46
+ if (!result.ok) {
47
+ console.error(`[weft] Failed to requeue/expire task "${operationId}": ${result.reason}`);
48
+ return;
49
+ }
50
+ recordWorkerCapacitySaturationMetric(context.metricsCollector, context.registry);
51
+ if (result.record.state === "terminal") {
52
+ options.engine.dispatchEvent(new ActivityFailedEvent(operationId, record.workflowId ?? "", record.activityName, Error(result.record.error), record.attempt));
48
53
  return;
49
54
  }
50
- const queuedRecord = createRequeuedRecord(record, nextAttempt, nextRetryCount, reason);
51
- await transitionInflightToQueued(options.engine.storage, operationId, queuedRecord);
52
55
  recordTaskRetryMetric(context.metricsCollector);
53
56
  recordTaskRequeueMetric(context.metricsCollector);
54
- recordWorkerCapacitySaturationMetric(context.metricsCollector, context.registry);
55
- scheduleTaskRedispatch(context, options, createRequeuedTaskDispatch(record, nextAttempt), policy);
56
- }
57
- function hasExceededMaxAttempts(policy, nextAttempt) {
58
- return policy !== void 0 && nextAttempt > policy.maxAttempts;
59
- }
60
- async function expireTaskAfterMaxAttempts(context, options, operationId, record, policy) {
61
- await transitionInflightToResolved(options.engine.storage, operationId, "failed", {
62
- record,
63
- resolutionReason: "max-attempts-exceeded"
64
- });
65
- recordWorkerCapacitySaturationMetric(context.metricsCollector, context.registry);
66
- options.engine.dispatchEvent(new ActivityFailedEvent(record.operationId, record.workflowId ?? "", record.activityName, Error(`Activity "${record.activityName}" exhausted all ${policy.maxAttempts} retry attempts`), record.attempt ?? 1));
57
+ const delay = Math.max(0, result.record.availableAt - Date.now());
58
+ scheduleDelayedDispatch(context, options, taskDispatchFromLedgerRecord(record), delay);
67
59
  }
68
- function createRequeuedRecord(record, nextAttempt, nextRetryCount, reason) {
69
- return {
70
- operationId: record.operationId,
71
- activityName: record.activityName,
72
- input: record.input,
73
- queue: record.queue,
74
- attempt: nextAttempt,
75
- visibilityTimeout: record.visibilityTimeout,
76
- retryPolicy: record.retryPolicy,
77
- queuedAt: Date.now(),
78
- workflowId: record.workflowId,
79
- firstQueuedAt: record.firstQueuedAt,
80
- lastDispatchedAt: record.lastDispatchedAt,
81
- startedAt: record.startedAt,
82
- retryCount: nextRetryCount,
83
- requeueCount: (record.requeueCount ?? 0) + 1,
84
- lastRequeueReason: reason
85
- };
86
- }
87
- function createRequeuedTaskDispatch(record, nextAttempt) {
60
+ export function taskDispatchFromLedgerRecord(record) {
88
61
  const taskDispatch = {
89
62
  operationId: record.operationId,
90
63
  activityName: record.activityName,
64
+ workflowType: record.workflowType,
91
65
  input: record.input,
92
66
  queue: record.queue,
93
- attempt: nextAttempt,
94
- visibilityTimeout: record.visibilityTimeout,
95
- workflowId: record.workflowId,
67
+ visibilityTimeout: record.visibilityTimeoutMilliseconds,
96
68
  workflowExecutionToken: record.workflowExecutionToken
97
69
  };
70
+ if (record.workflowId !== void 0)
71
+ taskDispatch.workflowId = record.workflowId;
98
72
  if (record.retryPolicy !== void 0)
99
73
  taskDispatch.retryPolicy = record.retryPolicy;
74
+ if (Object.keys(record.headers).length > 0)
75
+ taskDispatch.headers = record.headers;
76
+ if (record.priority !== void 0)
77
+ taskDispatch.priority = record.priority;
78
+ if (record.fairShareKey !== void 0)
79
+ taskDispatch.fairShareKey = record.fairShareKey;
80
+ if (record.stickyWorkflowId !== void 0)
81
+ taskDispatch.sticky = !0;
100
82
  return taskDispatch;
101
83
  }
102
- function scheduleTaskRedispatch(context, options, taskDispatch, policy) {
103
- if (policy) {
104
- const delay = calculateBackoff((taskDispatch.attempt ?? 1) - 1, policy);
105
- scheduleDelayedDispatch(context, options, taskDispatch, delay);
106
- } else
107
- dispatchTaskImpl(context, options, taskDispatch).catch((err) => console.error(`[weft] Redispatch failed for "${taskDispatch.operationId}":`, err));
84
+ function redispatchAvailableQueuedRecord(context, options, decoded) {
85
+ if (context.registry.isAssigned(decoded.operationId))
86
+ return;
87
+ if (context.taskQueue.isTracked(decoded.operationId))
88
+ return;
89
+ scheduleDelayedDispatch(context, options, taskDispatchFromLedgerRecord(decoded), 0);
90
+ }
91
+ async function reapRetainedTerminalRecord(options, decoded, now) {
92
+ if (options.taskRetentionWindowMs === void 0)
93
+ return;
94
+ if (!decoded.adopted)
95
+ return;
96
+ const retainedSince = decoded.adoptedAt ?? decoded.terminalAt;
97
+ if (now - retainedSince < options.taskRetentionWindowMs)
98
+ return;
99
+ const deleted = await commitTaskLedgerDelete(options.engine.storage, decoded.operationId, (current) => canDeleteRetainedTerminalTask(current, {
100
+ expectedRetentionGeneration: decoded.retentionGeneration
101
+ }), 1);
102
+ if (!deleted.ok)
103
+ console.error(`[weft] Failed to reap retained terminal task "${decoded.operationId}": ${deleted.reason}`);
108
104
  }
109
105
  export async function scanExpiredTasks(context, options, cleanupWorkflowIndex, now = Date.now()) {
110
106
  if (context.scanRunning)
@@ -119,17 +115,10 @@ export async function scanExpiredTasks(context, options, cleanupWorkflowIndex, n
119
115
  }
120
116
  context.processingOperations.add(operationId);
121
117
  try {
122
- const inflightKey = KEYS.operationInflight(operationId), existing = await options.engine.storage.get(inflightKey);
123
- if (!existing)
124
- continue;
125
- const decoded = decode(existing);
126
- if (!isInflightRecord(decoded)) {
127
- console.error(`[weft] Corrupt inflight record for task "${operationId}" \u2014 skipping`);
128
- continue;
129
- }
130
- if (await isTaskDeadLettered(options.engine.storage, operationId))
118
+ const decoded = decodeRemoteTaskRecord(await options.engine.storage.get(taskLedgerKey(operationId)));
119
+ if (decoded === null || decoded.state !== "leased")
131
120
  continue;
132
- if (restoreExtendedDeadlineIfStillActive(context.deadlineTracker, operationId, decoded.deadline, now))
121
+ if (restoreExtendedDeadlineIfStillActive(context.deadlineTracker, operationId, decoded.leaseDeadline, now))
133
122
  continue;
134
123
  context.registry.completeTask(decoded.operationId);
135
124
  cleanupWorkflowIndex(decoded.operationId);
@@ -154,22 +143,33 @@ export async function reconcileOrphanedRecords(context, options, cleanupWorkflow
154
143
  try {
155
144
  const now = Date.now();
156
145
  let staleHeartbeatCount = 0;
157
- for await (const [, value] of options.engine.storage.scan("op:inflight:"))
146
+ for await (const [, value] of options.engine.storage.scan("task-ledger:")) {
147
+ if (context.stopping)
148
+ return;
158
149
  try {
159
- const decoded = decode(value);
160
- if (!isInflightRecord(decoded))
150
+ const decoded = decodeRemoteTaskRecord(value);
151
+ if (decoded === null)
152
+ continue;
153
+ if (decoded.state === "queued") {
154
+ if (decoded.availableAt <= now)
155
+ redispatchAvailableQueuedRecord(context, options, decoded);
156
+ continue;
157
+ }
158
+ if (decoded.state === "terminal") {
159
+ await reapRetainedTerminalRecord(options, decoded, now);
160
+ continue;
161
+ }
162
+ if (decoded.state !== "leased")
161
163
  continue;
162
164
  if (isTaskHeartbeatStaleForMetrics(decoded, now))
163
165
  staleHeartbeatCount += 1;
164
- if (await isTaskDeadLettered(options.engine.storage, decoded.operationId))
165
- continue;
166
- if (decoded.deadline > now) {
166
+ if (decoded.leaseDeadline > now) {
167
167
  if (context.processingOperations.has(decoded.operationId))
168
168
  continue;
169
169
  context.deadlineTracker.remove(decoded.operationId);
170
170
  context.deadlineTracker.add({
171
171
  operationId: decoded.operationId,
172
- deadline: decoded.deadline
172
+ deadline: decoded.leaseDeadline
173
173
  });
174
174
  continue;
175
175
  }
@@ -185,8 +185,9 @@ export async function reconcileOrphanedRecords(context, options, cleanupWorkflow
185
185
  context.processingOperations.delete(decoded.operationId);
186
186
  }
187
187
  } catch (error) {
188
- console.error("[weft] Failed to reconcile inflight record \u2014 skipping:", error);
188
+ console.error("[weft] Failed to reconcile leased ledger record \u2014 skipping:", error);
189
189
  }
190
+ }
190
191
  recordTaskStaleHeartbeatMetric(context.metricsCollector, staleHeartbeatCount);
191
192
  } catch (error) {
192
193
  console.error("[weft] Reconciliation scanner error:", error);
@@ -1,15 +1,7 @@
1
1
  import { PayloadSizeExceededError } from '../../core/payload-size.ts';
2
- import type { ServeOptions } from '../index.ts';
3
- import { type InflightRecord, type TaskResolutionReason } from '../task-state.ts';
4
- import type { ServerContext } from './context.ts';
5
- export type TaskResultResolutionInput = {
6
- readonly operationId: string;
2
+ export type TaskResultPayloadSizeInput = {
7
3
  readonly status: 'completed' | 'failed';
8
- readonly resolutionReason: TaskResolutionReason;
9
4
  readonly value?: unknown;
10
5
  readonly error?: string | undefined;
11
- readonly inflightRecord?: InflightRecord | null | undefined;
12
- readonly skipPayloadSizeCheck?: boolean | undefined;
13
6
  };
14
- export declare function taskResultPayloadSizeError(input: TaskResultResolutionInput, maxBytes: number | null): PayloadSizeExceededError | null;
15
- export declare function transitionTaskResultToResolvedWithRetry(context: ServerContext, options: ServeOptions, input: TaskResultResolutionInput): Promise<void>;
7
+ export declare function taskResultPayloadSizeError(input: TaskResultPayloadSizeInput, maxBytes: number | null): PayloadSizeExceededError | null;
@@ -1,13 +1,4 @@
1
- import { TaskResultDeadLetteredEvent } from "../../core/events.js";
2
1
  import { PayloadSizeExceededError, assertPayloadWithinLimit } from "../../core/payload-size.js";
3
- import {
4
- readInflightRecord,
5
- transitionInflightToResolved,
6
- writeDeadLetteredTaskRecord
7
- } from "../task-state.js";
8
- import { withRetry } from "./retry.js";
9
- import { recordTaskExecutionLatencyMetric } from "./task-metrics.js";
10
- const TASK_RESULT_RESOLUTION_RETRY_ATTEMPTS = 2;
11
2
  function taskResultPayloadForSizeCheck(input) {
12
3
  return input.status === "completed" ? input.value : input.error ?? "";
13
4
  }
@@ -21,61 +12,3 @@ export function taskResultPayloadSizeError(input, maxBytes) {
21
12
  throw error;
22
13
  }
23
14
  }
24
- export async function transitionTaskResultToResolvedWithRetry(context, options, input) {
25
- if (input.skipPayloadSizeCheck !== !0) {
26
- const payloadError = taskResultPayloadSizeError(input, context.payloadSizeMaxBytes);
27
- if (payloadError !== null)
28
- throw payloadError;
29
- }
30
- const storage = options.engine.storage, resolvedAt = Date.now();
31
- let latestInflightRecord = input.inflightRecord;
32
- try {
33
- await withRetry(async () => {
34
- if (latestInflightRecord === void 0)
35
- latestInflightRecord = await readInflightRecord(storage, input.operationId);
36
- await transitionInflightToResolved(storage, input.operationId, input.status, {
37
- ...latestInflightRecord === null ? {} : { record: latestInflightRecord },
38
- resolvedAt,
39
- resolutionReason: input.resolutionReason,
40
- ...input.status === "completed" ? { value: input.value } : { error: input.error }
41
- });
42
- }, `transition task "${input.operationId}" to resolved`, TASK_RESULT_RESOLUTION_RETRY_ATTEMPTS);
43
- } catch (error) {
44
- const fallbackInflightRecord = latestInflightRecord ?? await readInflightRecord(storage, input.operationId).catch(() => null);
45
- await withRetry(async () => {
46
- await writeTaskResultDeadLetter(options, input, fallbackInflightRecord);
47
- }, `dead-letter task "${input.operationId}" after result-resolution retries`, TASK_RESULT_RESOLUTION_RETRY_ATTEMPTS);
48
- console.error(`[weft] Failed to transition task "${input.operationId}" to resolved after retries \u2014 dead-lettered:`, error);
49
- return;
50
- }
51
- if (latestInflightRecord !== null && latestInflightRecord !== void 0)
52
- recordTaskExecutionLatencyMetric(context.metricsCollector, latestInflightRecord, resolvedAt);
53
- }
54
- async function writeTaskResultDeadLetter(options, input, inflightRecord) {
55
- const storageErrorCode = "result-resolution-storage-exhausted", record = {
56
- operationId: input.operationId,
57
- reason: storageErrorCode,
58
- deadLetteredAt: Date.now(),
59
- errorMessage: storageErrorCode,
60
- retryAttempts: TASK_RESULT_RESOLUTION_RETRY_ATTEMPTS,
61
- status: input.status,
62
- ...inflightRecord?.activityName === void 0 ? {} : { activityName: inflightRecord.activityName },
63
- ...inflightRecord?.queue === void 0 ? {} : { queue: inflightRecord.queue },
64
- ...inflightRecord?.workerId === void 0 ? {} : { workerId: inflightRecord.workerId },
65
- ...inflightRecord?.attempt === void 0 ? {} : { attempt: inflightRecord.attempt },
66
- ...inflightRecord?.visibilityTimeout === void 0 ? {} : { visibilityTimeout: inflightRecord.visibilityTimeout },
67
- ...inflightRecord?.workflowId === void 0 ? {} : { workflowId: inflightRecord.workflowId },
68
- ...inflightRecord?.retryCount === void 0 ? {} : { retryCount: inflightRecord.retryCount },
69
- ...inflightRecord?.requeueCount === void 0 ? {} : { requeueCount: inflightRecord.requeueCount },
70
- ...inflightRecord?.lastRequeueReason === void 0 ? {} : { lastRequeueReason: inflightRecord.lastRequeueReason }
71
- };
72
- await writeDeadLetteredTaskRecord(options.engine.storage, record);
73
- options.engine.dispatchEvent(new TaskResultDeadLetteredEvent({
74
- operationId: record.operationId,
75
- workflowId: record.workflowId,
76
- activityName: record.activityName,
77
- queue: record.queue,
78
- workerId: record.workerId,
79
- errorMessage: record.errorMessage
80
- }));
81
- }
@@ -0,0 +1,82 @@
1
+ /**
2
+ * Public read/adopt surface over a single durable task-ledger record
3
+ * (WFT-24) — backs `WeftServer.getTaskResult` and `WeftServer.adoptTaskResult`.
4
+ *
5
+ * `TaskResultView` is a deliberately narrow projection of
6
+ * `RemoteTaskRecord`, not a re-export of it: the ledger types are
7
+ * server-internal (`task-ledger.ts`'s own doc comment — "not re-exported
8
+ * from `src/index.ts`") and carry fields with no business being public,
9
+ * such as `workerSessionId`, `attemptToken`, and `executionIdentity`. A
10
+ * resolved terminal record's actual result *value* is deliberately absent
11
+ * from this view for the same reason it is absent from the ledger itself:
12
+ * `RemoteTaskTerminalResolved` only ever stores `resultDigest` (a content
13
+ * hash), never the value — the durable ledger proves which attempt won,
14
+ * it does not re-deliver the payload. `resultDigest` exists so a caller
15
+ * that already has the value through whatever channel actually delivered
16
+ * it can verify it matches before adopting.
17
+ *
18
+ * "Adoption" (`adoptTaskResultImpl`) is an explicit caller assertion, not
19
+ * something this module infers: nothing in the engine today automatically
20
+ * links a workflow's own checkpoint to a remote task's terminal ledger
21
+ * record (that bridge is out of this project's scope — see the `blocks`
22
+ * relations on the WFT-24 Linear issue). A terminal record an adopter never
23
+ * calls this for is retained forever, by design; `reapRetainedTerminalRecord`
24
+ * (`task-reconciliation.ts`) only reaps *adopted* records.
25
+ *
26
+ * @module server/runtime/task-result-view
27
+ */
28
+ import type { Storage } from '../../storage/interface.ts';
29
+ /**
30
+ * Public, storage-agnostic view of a single task-ledger record.
31
+ *
32
+ * @example
33
+ * ```ts
34
+ * import { serve, type TaskResultView } from '@lostgradient/weft/server';
35
+ * import { Engine, MemoryStorage } from '@lostgradient/weft';
36
+ *
37
+ * await using engine = new Engine({ storage: new MemoryStorage() });
38
+ * await using server = serve({ engine, port: 0 });
39
+ *
40
+ * const view: TaskResultView | null = await server.getTaskResult('op-1');
41
+ * if (view?.status === 'terminal' && !view.adopted) {
42
+ * await server.adoptTaskResult('op-1', view.resultDigest);
43
+ * }
44
+ * ```
45
+ */
46
+ export type TaskResultView = Readonly<{
47
+ status: 'pending';
48
+ state: 'queued' | 'leased' | 'completing' | 'cancelling';
49
+ }> | Readonly<{
50
+ status: 'terminal';
51
+ disposition: 'resolved' | 'cancelled' | 'retryExhausted';
52
+ resultDigest: string;
53
+ terminalAt: number;
54
+ adopted: boolean;
55
+ adoptedAt?: number;
56
+ /** Only present for `disposition: 'resolved'` — whether the worker reported success or failure. */
57
+ resultStatus?: 'completed' | 'failed';
58
+ error?: string;
59
+ }> | Readonly<{
60
+ status: 'deadLettered';
61
+ persistenceFailureReason: string;
62
+ pendingStatus: 'completed' | 'failed';
63
+ deadLetteredAt: number;
64
+ error?: string;
65
+ }>;
66
+ /**
67
+ * Read the current public view of a task-ledger record. Returns `null` if
68
+ * no record exists for `operationId` — either it was never dispatched, or a
69
+ * retained terminal record has already been reaped.
70
+ */
71
+ export declare function getTaskResultViewImpl(storage: Storage, operationId: string): Promise<TaskResultView | null>;
72
+ /**
73
+ * Mark a terminal task's result as adopted by the caller — the durable
74
+ * assertion that "the workflow checkpoint (or whatever consumed this
75
+ * result) has incorporated it," per the project brief's adoption/retention
76
+ * split. Returns `true` once adopted, `false` if the record is not
77
+ * currently `terminal` or `resultDigest` does not match (including a
78
+ * record that no longer exists, e.g. already reaped). Idempotent: adopting
79
+ * an already-adopted record with the same digest succeeds again, refreshing
80
+ * `adoptedAt`.
81
+ */
82
+ export declare function adoptTaskResultImpl(storage: Storage, operationId: string, resultDigest: string): Promise<boolean>;
@@ -0,0 +1,48 @@
1
+ import { markWorkflowResultAdopted } from "../task-ledger-transitions.js";
2
+ import {
3
+ decodeRemoteTaskRecord,
4
+ taskLedgerKey
5
+ } from "../task-ledger.js";
6
+ import { commitTaskLedgerTransition } from "./task-ledger-runtime.js";
7
+ function terminalTaskResultView(decoded) {
8
+ return {
9
+ status: "terminal",
10
+ disposition: decoded.disposition,
11
+ resultDigest: decoded.resultDigest,
12
+ terminalAt: decoded.terminalAt,
13
+ adopted: decoded.adopted,
14
+ ...decoded.adoptedAt !== void 0 ? { adoptedAt: decoded.adoptedAt } : {},
15
+ ...decoded.disposition === "resolved" ? { resultStatus: decoded.status } : {},
16
+ ..."error" in decoded && decoded.error !== void 0 ? { error: decoded.error } : {}
17
+ };
18
+ }
19
+ function deadLetteredTaskResultView(decoded) {
20
+ return {
21
+ status: "deadLettered",
22
+ persistenceFailureReason: decoded.persistenceFailureReason,
23
+ pendingStatus: decoded.pendingStatus,
24
+ deadLetteredAt: decoded.deadLetteredAt,
25
+ ...decoded.error !== void 0 ? { error: decoded.error } : {}
26
+ };
27
+ }
28
+ export async function getTaskResultViewImpl(storage, operationId) {
29
+ const decoded = decodeRemoteTaskRecord(await storage.get(taskLedgerKey(operationId)));
30
+ if (decoded === null)
31
+ return null;
32
+ switch (decoded.state) {
33
+ case "queued":
34
+ case "leased":
35
+ case "completing":
36
+ case "cancelling":
37
+ return { status: "pending", state: decoded.state };
38
+ case "terminal":
39
+ return terminalTaskResultView(decoded);
40
+ case "deadLettered":
41
+ return deadLetteredTaskResultView(decoded);
42
+ default:
43
+ return decoded;
44
+ }
45
+ }
46
+ export async function adoptTaskResultImpl(storage, operationId, resultDigest) {
47
+ return (await commitTaskLedgerTransition(storage, operationId, (current, now) => markWorkflowResultAdopted(current, { expectedResultDigest: resultDigest }, now), 1)).ok;
48
+ }
@@ -0,0 +1,16 @@
1
+ /**
2
+ * Low-level send/close primitives for the worker WebSocket protocol.
3
+ *
4
+ * Split out as a leaf module so both `websocket-worker.ts` (task/heartbeat
5
+ * handling) and `websocket-worker-registration.ts` (registration handling)
6
+ * can depend on it without forming an import cycle between those two.
7
+ *
8
+ * @module server/runtime/websocket-worker-messaging
9
+ */
10
+ import type { ServerWebSocket } from 'bun';
11
+ import { type ProtocolErrorMessage, type RegisterErrorMessage } from '../../worker/protocol.ts';
12
+ import type { WebSocketData } from '../json-rpc-websocket-runtime.ts';
13
+ export declare function sendWorkerProtocolMessage(ws: ServerWebSocket<WebSocketData>, message: ProtocolErrorMessage | RegisterErrorMessage | Record<string, unknown>): void;
14
+ export declare function closeWorkerSocket(ws: ServerWebSocket<WebSocketData>, code: number, reason: string): void;
15
+ export declare function rejectRegistration(ws: ServerWebSocket<WebSocketData>, code: RegisterErrorMessage['code'], message: string, requestedProtocolVersion?: number): void;
16
+ export declare function rejectProtocolMessage(ws: ServerWebSocket<WebSocketData>, code: ProtocolErrorMessage['code'], message: string): void;
@@ -0,0 +1,32 @@
1
+ import {
2
+ REMOTE_WORKER_SUPPORTED_PROTOCOL_VERSIONS
3
+ } from "../../worker/protocol.js";
4
+ const WORKER_PROTOCOL_CLOSE_CODE = 1002, WORKER_REGISTRATION_CLOSE_CODE = 1008;
5
+ export function sendWorkerProtocolMessage(ws, message) {
6
+ ws.send(JSON.stringify(message));
7
+ }
8
+ export function closeWorkerSocket(ws, code, reason) {
9
+ try {
10
+ ws.unsubscribe(ws.data.pathname);
11
+ } catch {}
12
+ ws.close(code, reason);
13
+ setTimeout(() => {
14
+ try {
15
+ ws.terminate();
16
+ } catch {}
17
+ }, 10);
18
+ }
19
+ export function rejectRegistration(ws, code, message, requestedProtocolVersion) {
20
+ sendWorkerProtocolMessage(ws, {
21
+ type: "registerError",
22
+ code,
23
+ message,
24
+ supportedProtocolVersions: REMOTE_WORKER_SUPPORTED_PROTOCOL_VERSIONS,
25
+ ...requestedProtocolVersion !== void 0 ? { requestedProtocolVersion } : {}
26
+ });
27
+ closeWorkerSocket(ws, WORKER_REGISTRATION_CLOSE_CODE, code);
28
+ }
29
+ export function rejectProtocolMessage(ws, code, message) {
30
+ sendWorkerProtocolMessage(ws, { type: "protocolError", code, message });
31
+ closeWorkerSocket(ws, WORKER_PROTOCOL_CLOSE_CODE, code);
32
+ }
@@ -0,0 +1,41 @@
1
+ /**
2
+ * Worker registration handling: manifest validation, deployment consistency,
3
+ * admission policy, and the workerId hijack/reconnect guard.
4
+ *
5
+ * Split out of `websocket-worker.ts` to keep that module under the
6
+ * repository's per-file line ceiling — this is the registration slice of the
7
+ * same WebSocket message handler.
8
+ *
9
+ * @module server/runtime/websocket-worker-registration
10
+ */
11
+ import type { ServerWebSocket } from 'bun';
12
+ import { type RegisterMessage } from '../../worker/protocol.ts';
13
+ import type { ServeOptions } from '../index.ts';
14
+ import type { WebSocketData } from '../json-rpc-websocket-runtime.ts';
15
+ import type { ServerContext } from './context.ts';
16
+ /**
17
+ * Validate, admit, and register a worker from its `register` message.
18
+ *
19
+ * Layering, in order: principal scope check, manifest parse
20
+ * (`parseWorkerManifest`), embedded/wire protocol-version agreement,
21
+ * admission policy, manifest digest, then — synchronously, with no further
22
+ * `await` between any of them — a live-socket check, a read-only deployment-
23
+ * consistency check, the workerId hijack/reconnect guard, and only once both
24
+ * of those gates have passed: the deployment-consistency digest record,
25
+ * registry insertion, and `registerAck`.
26
+ *
27
+ * That whole final block must commit atomically with respect to the event
28
+ * loop for three independent reasons: two concurrent registrations for the
29
+ * same not-yet-seen workerId could otherwise both pass the hijack guard
30
+ * before either commits; a peer that closes while the digest above is still
31
+ * pending leaves `ws.data.workerId` unset, so the close handler runs no
32
+ * cleanup, and completing registration afterward would create a ghost worker
33
+ * on a dead socket; and recording the deployment-consistency digest before
34
+ * every rejection gate — including the hijack guard, not just the admission
35
+ * policy — has passed would let a worker this function ultimately declines
36
+ * still permanently poison that deployment/build slot for future legitimate
37
+ * workers, since the guard never evicts. See {@link commitWorkerRegistration}
38
+ * for why the digest record specifically runs after, not alongside, the
39
+ * hijack guard.
40
+ */
41
+ export declare function registerWorker(context: ServerContext, options: ServeOptions, ws: ServerWebSocket<WebSocketData>, message: RegisterMessage): Promise<void>;