@lostgradient/weft 0.18.0 → 0.20.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (171) hide show
  1. package/README.md +3 -1
  2. package/dist/cli/conformance.js +14 -18
  3. package/dist/cli/generated/operation-catalog.snapshot.json +1597 -21
  4. package/dist/cli/generated/operation-client.generated.d.ts +37 -6
  5. package/dist/cli/generated/operation-client.generated.js +6 -0
  6. package/dist/core/context/child-workflow-pipe.d.ts +1 -1
  7. package/dist/core/context/child-workflow-pipe.js +1 -1
  8. package/dist/core/engine/construction.d.ts +2 -0
  9. package/dist/core/engine/construction.js +3 -0
  10. package/dist/core/engine/index.js +1 -0
  11. package/dist/core/engine/listing.d.ts +1 -0
  12. package/dist/core/engine/listing.js +6 -4
  13. package/dist/core/weft-error.d.ts +1 -1
  14. package/dist/core/weft-error.js +2 -1
  15. package/dist/core/worker-execution-dispatcher.d.ts +8 -0
  16. package/dist/core/worker-execution-dispatcher.js +4 -0
  17. package/dist/core/worker-execution-strategy-options.d.ts +16 -0
  18. package/dist/core/worker-execution-strategy.js +71 -89
  19. package/dist/core/worker-fault-handling.d.ts +45 -0
  20. package/dist/core/worker-fault-handling.js +88 -0
  21. package/dist/core/worker-protocol.d.ts +18 -1
  22. package/dist/core/worker-protocol.js +1 -1
  23. package/dist/core/worker-realm-readiness.d.ts +78 -0
  24. package/dist/core/worker-realm-readiness.js +127 -0
  25. package/dist/index.d.ts +3 -1
  26. package/dist/index.js +28 -1
  27. package/dist/json-schema.js +1 -1
  28. package/dist/runtime/portable.d.ts +14 -0
  29. package/dist/runtime/portable.js +10 -0
  30. package/dist/server/index.d.ts +71 -8
  31. package/dist/server/index.js +7 -3
  32. package/dist/server/operations/get-task-detail-schema.d.ts +341 -0
  33. package/dist/server/operations/get-task-detail-schema.js +118 -0
  34. package/dist/server/operations/get-task-detail.d.ts +346 -0
  35. package/dist/server/operations/get-task-detail.js +215 -0
  36. package/dist/server/operations/get-task-diagnostics.d.ts +43 -18
  37. package/dist/server/operations/get-task-diagnostics.js +87 -126
  38. package/dist/server/operations/get-worker-diagnostics.d.ts +177 -0
  39. package/dist/server/operations/get-worker-diagnostics.js +117 -0
  40. package/dist/server/operations/list-worker-registration-rejections.d.ts +79 -0
  41. package/dist/server/operations/list-worker-registration-rejections.js +60 -0
  42. package/dist/server/operations/list-workers.js +1 -3
  43. package/dist/server/operations/static-registrations.js +8 -20
  44. package/dist/server/rest-bindings.js +31 -2
  45. package/dist/server/runtime/authentication-bridge.js +6 -14
  46. package/dist/server/runtime/context.d.ts +23 -0
  47. package/dist/server/runtime/task-dispatch.d.ts +7 -1
  48. package/dist/server/runtime/task-dispatch.js +126 -55
  49. package/dist/server/runtime/task-ledger-completion.d.ts +68 -0
  50. package/dist/server/runtime/task-ledger-completion.js +72 -0
  51. package/dist/server/runtime/task-ledger-recovery.d.ts +55 -0
  52. package/dist/server/runtime/task-ledger-recovery.js +73 -0
  53. package/dist/server/runtime/task-ledger-runtime.d.ts +64 -0
  54. package/dist/server/runtime/task-ledger-runtime.js +34 -0
  55. package/dist/server/runtime/task-metrics.d.ts +6 -4
  56. package/dist/server/runtime/task-polling.d.ts +14 -12
  57. package/dist/server/runtime/task-polling.js +106 -82
  58. package/dist/server/runtime/task-reconciliation.d.ts +35 -8
  59. package/dist/server/runtime/task-reconciliation.js +75 -74
  60. package/dist/server/runtime/task-result-resolution.d.ts +2 -10
  61. package/dist/server/runtime/task-result-resolution.js +0 -67
  62. package/dist/server/runtime/task-result-view.d.ts +82 -0
  63. package/dist/server/runtime/task-result-view.js +48 -0
  64. package/dist/server/runtime/websocket-worker-messaging.d.ts +16 -0
  65. package/dist/server/runtime/websocket-worker-messaging.js +32 -0
  66. package/dist/server/runtime/websocket-worker-registration.d.ts +41 -0
  67. package/dist/server/runtime/websocket-worker-registration.js +134 -0
  68. package/dist/server/runtime/websocket-worker.d.ts +0 -1
  69. package/dist/server/runtime/websocket-worker.js +64 -116
  70. package/dist/server/serve-internals.d.ts +9 -6
  71. package/dist/server/serve-internals.js +14 -40
  72. package/dist/server/task-ledger-codec.d.ts +40 -0
  73. package/dist/server/task-ledger-codec.js +195 -0
  74. package/dist/server/task-ledger-keys.d.ts +13 -0
  75. package/dist/server/task-ledger-keys.js +5 -0
  76. package/dist/server/task-ledger-limits.d.ts +34 -0
  77. package/dist/server/task-ledger-limits.js +5 -0
  78. package/dist/server/task-ledger-transition-helpers.d.ts +38 -0
  79. package/dist/server/task-ledger-transition-helpers.js +42 -0
  80. package/dist/server/task-ledger-transitions-cancellation.d.ts +31 -0
  81. package/dist/server/task-ledger-transitions-cancellation.js +73 -0
  82. package/dist/server/task-ledger-transitions.d.ts +129 -0
  83. package/dist/server/task-ledger-transitions.js +162 -0
  84. package/dist/server/task-ledger-types.d.ts +220 -0
  85. package/dist/server/task-ledger-types.js +1 -0
  86. package/dist/server/task-ledger.d.ts +17 -0
  87. package/dist/server/task-ledger.js +24 -0
  88. package/dist/server/task-state.d.ts +25 -152
  89. package/dist/server/task-state.js +0 -188
  90. package/dist/server/worker-admission-policy.d.ts +70 -0
  91. package/dist/server/worker-admission-policy.js +0 -0
  92. package/dist/storage/compressed-storage.js +1 -1
  93. package/dist/storage/index.d.ts +0 -2
  94. package/dist/storage/interface.d.ts +0 -2
  95. package/dist/storage/interface.js +1 -1
  96. package/dist/storage/lmdb.js +1 -1
  97. package/dist/storage/memory.js +1 -1
  98. package/dist/storage/neon.js +1 -1
  99. package/dist/storage/postgres.js +1 -1
  100. package/dist/storage/resolve.js +1 -1
  101. package/dist/storage/scoped-storage.js +1 -1
  102. package/dist/storage/testing.js +1 -1
  103. package/dist/storage/turso.js +1 -1
  104. package/dist/storage/web-extension.js +2 -2
  105. package/dist/version.d.ts +1 -1
  106. package/dist/version.js +1 -1
  107. package/dist/web-extension.js +1 -1
  108. package/dist/worker/index.d.ts +2 -0
  109. package/dist/worker/index.js +30 -4
  110. package/dist/worker/manifest/canonical-json.d.ts +20 -0
  111. package/dist/worker/manifest/canonical-json.js +13 -0
  112. package/dist/worker/manifest/capabilities.d.ts +24 -0
  113. package/dist/worker/manifest/capabilities.js +74 -0
  114. package/dist/worker/manifest/content-digest.d.ts +20 -0
  115. package/dist/worker/manifest/content-digest.js +13 -0
  116. package/dist/worker/manifest/declared-shape-digest.d.ts +14 -0
  117. package/dist/worker/manifest/declared-shape-digest.js +4 -0
  118. package/dist/worker/manifest/digest.d.ts +82 -0
  119. package/dist/worker/manifest/digest.js +9 -0
  120. package/dist/worker/manifest/execution-identity.d.ts +92 -0
  121. package/dist/worker/manifest/execution-identity.js +36 -0
  122. package/dist/worker/manifest/failure.d.ts +59 -0
  123. package/dist/worker/manifest/failure.js +4 -0
  124. package/dist/worker/manifest/index.d.ts +22 -0
  125. package/dist/worker/manifest/index.js +23 -0
  126. package/dist/worker/manifest/internal-realm.d.ts +50 -0
  127. package/dist/worker/manifest/internal-realm.js +39 -0
  128. package/dist/worker/manifest/is-record.d.ts +12 -0
  129. package/dist/worker/manifest/is-record.js +6 -0
  130. package/dist/worker/manifest/json-scan.d.ts +26 -0
  131. package/dist/worker/manifest/json-scan.js +88 -0
  132. package/dist/worker/manifest/limits.d.ts +138 -0
  133. package/dist/worker/manifest/limits.js +1 -0
  134. package/dist/worker/manifest/normalize.d.ts +63 -0
  135. package/dist/worker/manifest/normalize.js +73 -0
  136. package/dist/worker/manifest/parse-json.d.ts +24 -0
  137. package/dist/worker/manifest/parse-json.js +15 -0
  138. package/dist/worker/manifest/parse.d.ts +86 -0
  139. package/dist/worker/manifest/parse.js +188 -0
  140. package/dist/worker/manifest/registry-contract-builder.d.ts +120 -0
  141. package/dist/worker/manifest/registry-contract-builder.js +65 -0
  142. package/dist/worker/manifest/types.d.ts +263 -0
  143. package/dist/worker/manifest/types.js +1 -0
  144. package/dist/worker/manifest/utf8.d.ts +13 -0
  145. package/dist/worker/manifest/utf8.js +7 -0
  146. package/dist/worker/options.d.ts +55 -8
  147. package/dist/worker/options.js +65 -9
  148. package/dist/worker/protocol-internals.d.ts +1 -2
  149. package/dist/worker/protocol-internals.js +0 -5
  150. package/dist/worker/protocol-messages.d.ts +33 -17
  151. package/dist/worker/protocol-schemas.d.ts +36 -74
  152. package/dist/worker/protocol-schemas.js +25 -18
  153. package/dist/worker/protocol-version.d.ts +18 -14
  154. package/dist/worker/protocol-version.js +1 -1
  155. package/dist/worker/protocol.js +1 -1
  156. package/dist/worker/registry/deployment-consistency.d.ts +78 -0
  157. package/dist/worker/registry/deployment-consistency.js +27 -0
  158. package/dist/worker/registry/drain.d.ts +14 -0
  159. package/dist/worker/registry/drain.js +24 -0
  160. package/dist/worker/registry/rejections.d.ts +31 -0
  161. package/dist/worker/registry/rejections.js +10 -0
  162. package/dist/worker/registry/summary.d.ts +0 -3
  163. package/dist/worker/registry/summary.js +2 -5
  164. package/dist/worker/registry/types.d.ts +10 -3
  165. package/dist/worker/registry.d.ts +47 -2
  166. package/dist/worker/registry.js +45 -32
  167. package/dist/workers/workflow-worker-entry.d.ts +12 -7
  168. package/dist/workers/workflow-worker-entry.js +9 -2
  169. package/package.json +3 -3
  170. package/dist/server/task-resolved-record.d.ts +0 -12
  171. package/dist/server/task-resolved-record.js +0 -35
@@ -0,0 +1,60 @@
1
+ import { z } from "zod";
2
+ import { defineOperation } from "../operation-registry.js";
3
+ const DEFAULT_LIMIT = 20, MAX_LIMIT = 100, registrationRejectionCodeSchema = z.enum([
4
+ "invalid_registration",
5
+ "unsupported_protocol_version",
6
+ "deployment_conflict",
7
+ "registration_rejected"
8
+ ]), registrationRejectionEntrySchema = z.object({
9
+ code: registrationRejectionCodeSchema,
10
+ workerId: z.string().optional(),
11
+ rejectedAt: z.number(),
12
+ queue: z.string().optional(),
13
+ deploymentName: z.string().optional(),
14
+ buildId: z.string().optional()
15
+ }).strict(), listWorkerRegistrationRejectionsInput = z.object({
16
+ limit: z.number().int().min(1).max(MAX_LIMIT).default(DEFAULT_LIMIT)
17
+ }), listWorkerRegistrationRejectionsOutput = z.object({
18
+ items: z.array(registrationRejectionEntrySchema),
19
+ limit: z.number().int().min(1).max(MAX_LIMIT)
20
+ });
21
+ export function createListWorkerRegistrationRejectionsOperation(options = {}) {
22
+ const registry = options.workerRegistry;
23
+ return defineOperation({
24
+ name: "weft.workers.rejections",
25
+ mcpExposable: !1,
26
+ summary: "List recently declined worker registration attempts",
27
+ description: "Bounded, most-recent-first log of declined `register` attempts: rejection code, attempted workerId when known, and whatever deployment/queue identity had already been parsed by the time the rejecting gate ran. No free-text message and no manifest content.",
28
+ destructive: !1,
29
+ tags: ["Observability"],
30
+ inputSchema: listWorkerRegistrationRejectionsInput,
31
+ outputSchema: listWorkerRegistrationRejectionsOutput,
32
+ access: { kind: "scoped", scopes: { kind: "anyOf", scopes: ["system:read"] } },
33
+ producibleFaults: [],
34
+ discoverable: !0,
35
+ transports: { http: !0, jsonRpcHttp: !0, jsonRpcWebSocket: !0, jsonRpcStdio: !0 },
36
+ unknownKeyPolicy: { http: "strip", jsonRpc: "reject" },
37
+ invoke: async ({ input }) => {
38
+ if (registry === void 0)
39
+ throw Error("weft.workers.rejections invoked from a discovery-only operation registry; no WorkerRegistry was wired in");
40
+ return { items: registry.getRecentRejections(input.limit), limit: input.limit };
41
+ }
42
+ });
43
+ }
44
+ export const listWorkerRegistrationRejectionsOperation = createListWorkerRegistrationRejectionsOperation();
45
+ export function createListWorkerRegistrationRejectionsRestBinding() {
46
+ return {
47
+ method: "GET",
48
+ path: "/v1/workers/registration-rejections",
49
+ pathParamNames: [],
50
+ operationName: "weft.workers.rejections",
51
+ inputSources: {
52
+ limit: { kind: "query", queryParam: "limit" }
53
+ },
54
+ extractInput: async (request) => {
55
+ const limit = new URL(request.url).searchParams.get("limit");
56
+ return { limit: limit === null || limit.length === 0 ? void 0 : Number(limit) };
57
+ },
58
+ success: { kind: "json", status: 200 }
59
+ };
60
+ }
@@ -27,13 +27,11 @@ const listWorkersInput = z.object({}), routingPolicySchema = z.enum([
27
27
  health: workerHealthSchema,
28
28
  deploymentName: z.string().optional(),
29
29
  buildId: z.string().optional(),
30
- runtimeVersion: z.string().optional(),
31
- gitSha: z.string().optional()
30
+ runtimeVersion: z.string().optional()
32
31
  }), workerDeploymentSummarySchema = z.object({
33
32
  deploymentName: z.string().nullable(),
34
33
  buildId: z.string().nullable(),
35
34
  runtimeVersion: z.string().nullable(),
36
- gitSha: z.string().nullable(),
37
35
  health: workerHealthSchema,
38
36
  workers: z.number(),
39
37
  activeWorkers: z.number(),
@@ -1,7 +1,4 @@
1
- import {
2
- addWorkflowTagsOperation,
3
- addWorkflowTagsRestBinding
4
- } from "./add-workflow-tags.js";
1
+ import { addWorkflowTagsOperation, addWorkflowTagsRestBinding } from "./add-workflow-tags.js";
5
2
  import {
6
3
  aggregateWorkflowsOperation,
7
4
  aggregateWorkflowsRestBinding
@@ -44,6 +41,7 @@ import { getReviewOperation, getReviewRestBinding } from "./get-review.js";
44
41
  import { getScheduleOperation, getScheduleRestBinding } from "./get-schedule.js";
45
42
  import { getStreamChunksOperation, getStreamChunksRestBinding } from "./get-stream-chunks.js";
46
43
  import { getSystemLeaseOperation, getSystemLeaseRestBinding } from "./get-system-lease.js";
44
+ import { getTaskDetailOperation, getTaskDetailRestBinding } from "./get-task-detail.js";
47
45
  import {
48
46
  clearTaskDeadLetterOperation,
49
47
  clearTaskDeadLetterRestBinding,
@@ -54,15 +52,9 @@ import {
54
52
  getWorkflowAttributesOperation,
55
53
  getWorkflowAttributesRestBinding
56
54
  } from "./get-workflow-attributes.js";
57
- import {
58
- getWorkflowEventsOperation,
59
- getWorkflowEventsRestBinding
60
- } from "./get-workflow-events.js";
55
+ import { getWorkflowEventsOperation, getWorkflowEventsRestBinding } from "./get-workflow-events.js";
61
56
  import * as workflowObservability from "./get-workflow-observability.js";
62
- import {
63
- getWorkflowResultOperation,
64
- getWorkflowResultRestBinding
65
- } from "./get-workflow-result.js";
57
+ import { getWorkflowResultOperation, getWorkflowResultRestBinding } from "./get-workflow-result.js";
66
58
  import {
67
59
  getWorkflowTimelineOperation,
68
60
  getWorkflowTimelineRestBinding
@@ -113,10 +105,7 @@ import {
113
105
  storageScanOperation,
114
106
  storageScanRestBinding
115
107
  } from "./storage.js";
116
- import {
117
- streamWorkflowSseOperation,
118
- streamWorkflowSseRestBinding
119
- } from "./stream-workflow-sse.js";
108
+ import { streamWorkflowSseOperation, streamWorkflowSseRestBinding } from "./stream-workflow-sse.js";
120
109
  import {
121
110
  submitReviewDecisionOperation,
122
111
  submitReviewDecisionRestBinding
@@ -125,10 +114,7 @@ import { suspendWorkflowOperation, suspendWorkflowRestBinding } from "./suspend-
125
114
  import { timeoutWorkflowOperation, timeoutWorkflowRestBinding } from "./timeout-workflow.js";
126
115
  import { updateScheduleOperation, updateScheduleRestBinding } from "./update-schedule.js";
127
116
  import { updateWorkflowOperation, updateWorkflowRestBinding } from "./update-workflow.js";
128
- import {
129
- workflowEventsSseOperation,
130
- workflowEventsSseRestBinding
131
- } from "./workflow-events-sse.js";
117
+ import { workflowEventsSseOperation, workflowEventsSseRestBinding } from "./workflow-events-sse.js";
132
118
  import { workflowEventsSubscriptionOperation } from "./workflow-events-subscription.js";
133
119
  export const STATIC_REST_BINDINGS = [
134
120
  listAlertsRestBinding,
@@ -185,6 +171,7 @@ export const STATIC_REST_BINDINGS = [
185
171
  fleetEventsSseRestBinding,
186
172
  getTaskDiagnosticsRestBinding,
187
173
  clearTaskDeadLetterRestBinding,
174
+ getTaskDetailRestBinding,
188
175
  listSchedulesRestBinding,
189
176
  getScheduleRestBinding,
190
177
  replayWorkflowRestBinding,
@@ -248,6 +235,7 @@ export const STATIC_REST_BINDINGS = [
248
235
  workflowEventsSubscriptionOperation,
249
236
  fleetEventsSubscriptionOperation,
250
237
  clearTaskDeadLetterOperation,
238
+ getTaskDetailOperation,
251
239
  listSchedulesOperation,
252
240
  listAlertsOperation,
253
241
  getPrincipalOperation,
@@ -7,11 +7,21 @@ import {
7
7
  createGetTaskDiagnosticsOperation,
8
8
  getTaskDiagnosticsOperation
9
9
  } from "./operations/get-task-diagnostics.js";
10
+ import {
11
+ createGetWorkerDiagnosticsOperation,
12
+ createGetWorkerDiagnosticsRestBinding,
13
+ getWorkerDiagnosticsOperation
14
+ } from "./operations/get-worker-diagnostics.js";
10
15
  import {
11
16
  createListTaskQueuesOperation,
12
17
  createListTaskQueuesRestBinding,
13
18
  listTaskQueuesOperation
14
19
  } from "./operations/list-task-queues.js";
20
+ import {
21
+ createListWorkerRegistrationRejectionsOperation,
22
+ createListWorkerRegistrationRejectionsRestBinding,
23
+ listWorkerRegistrationRejectionsOperation
24
+ } from "./operations/list-worker-registration-rejections.js";
15
25
  import {
16
26
  createListWorkersOperation,
17
27
  createListWorkersRestBinding,
@@ -42,7 +52,9 @@ export function createLiveRestBindings() {
42
52
  createClearWorkerDrainRestBinding(),
43
53
  createDrainDeploymentRestBinding(),
44
54
  createClearDeploymentDrainRestBinding(),
45
- createListTaskQueuesRestBinding()
55
+ createListTaskQueuesRestBinding(),
56
+ createGetWorkerDiagnosticsRestBinding(),
57
+ createListWorkerRegistrationRejectionsRestBinding()
46
58
  ];
47
59
  }
48
60
  function buildSystemMetricsOperation(options) {
@@ -100,6 +112,21 @@ function buildTaskDiagnosticsOperationForRegistry(options) {
100
112
  ...options.clock !== void 0 ? { now: options.clock } : {}
101
113
  });
102
114
  }
115
+ function buildWorkerDiagnosticsOperationForRegistry(options) {
116
+ if (options.workerRegistry === void 0)
117
+ return getWorkerDiagnosticsOperation;
118
+ return createGetWorkerDiagnosticsOperation({
119
+ workerRegistry: options.workerRegistry,
120
+ ...options.clock !== void 0 ? { clock: options.clock } : {}
121
+ });
122
+ }
123
+ function buildWorkerRegistrationRejectionsOperationForRegistry(options) {
124
+ if (options.workerRegistry === void 0)
125
+ return listWorkerRegistrationRejectionsOperation;
126
+ return createListWorkerRegistrationRejectionsOperation({
127
+ workerRegistry: options.workerRegistry
128
+ });
129
+ }
103
130
  export function createLiveOperationRegistry(options) {
104
131
  const resolved = options ?? {};
105
132
  return createOperationRegistry([
@@ -111,6 +138,8 @@ export function createLiveOperationRegistry(options) {
111
138
  buildClearWorkerDrainOperationForRegistry(resolved),
112
139
  buildDrainDeploymentOperationForRegistry(resolved),
113
140
  buildClearDeploymentDrainOperationForRegistry(resolved),
114
- buildListTaskQueuesOperationForRegistry(resolved)
141
+ buildListTaskQueuesOperationForRegistry(resolved),
142
+ buildWorkerDiagnosticsOperationForRegistry(resolved),
143
+ buildWorkerRegistrationRejectionsOperationForRegistry(resolved)
115
144
  ]);
116
145
  }
@@ -1,7 +1,5 @@
1
- import { decode } from "../../core/codec.js";
2
1
  import { WorkerDisconnectedEvent } from "../../core/events.js";
3
2
  import { handleMcpHttpRequest } from "../../mcp/http.js";
4
- import { KEYS } from "../../storage/interface.js";
5
3
  import { authContextToPrincipal, handleRequest } from "../handler.js";
6
4
  import { handleJsonRpcHttpRequestSafely } from "../json-rpc-transport-helpers.js";
7
5
  import {
@@ -10,6 +8,7 @@ import {
10
8
  openJsonRpcWebSocketSession
11
9
  } from "../json-rpc-websocket-runtime.js";
12
10
  import { API_PREFIX } from "../route-model.js";
11
+ import { decodeRemoteTaskRecord, taskLedgerKey } from "../task-ledger.js";
13
12
  import { buildPreflightResponse, decorateResponseWithCors, isPreflightRequest } from "./cors.js";
14
13
  import { gateRequest } from "./request-gate.js";
15
14
  import { handleTaskPollRequest, handleTaskResultRequest } from "./task-polling.js";
@@ -25,7 +24,7 @@ import {
25
24
  replayWatchEvents
26
25
  } from "./websocket-stream.js";
27
26
  import { handleWebSocketUpgrade } from "./websocket-upgrade.js";
28
- import { handleWorkerWebSocketMessage, isInflightRecord } from "./websocket-worker.js";
27
+ import { handleWorkerWebSocketMessage } from "./websocket-worker.js";
29
28
  export function deriveSupportedOpenApiSecuritySchemes(auth) {
30
29
  const schemes = new Set;
31
30
  if (auth?.jwt !== void 0)
@@ -206,18 +205,11 @@ function runWorkerDisconnectRequeue(context, options, workerId, _ws, cleanupWork
206
205
  for (const task of inFlightTasks)
207
206
  (async () => {
208
207
  try {
209
- const inflightKey = KEYS.operationInflight(task.operationId), existing = await options.engine.storage.get(inflightKey);
210
- if (existing) {
211
- const record = decode(existing);
212
- if (!isInflightRecord(record)) {
213
- console.error(`[weft] Corrupt inflight record for task "${task.operationId}" \u2014 skipping reassignment`);
214
- return;
215
- }
208
+ const record = decodeRemoteTaskRecord(await options.engine.storage.get(taskLedgerKey(task.operationId)));
209
+ if (record !== null && record.state === "leased")
216
210
  await reassignOrExpireTask(context, options, task.operationId, record, "worker-disconnect");
217
- } else {
218
- console.warn(`[weft] No inflight record found in storage for task "${task.operationId}" \u2014 skipping reassignment`);
219
- await options.engine.storage.delete(inflightKey);
220
- }
211
+ else
212
+ console.warn(`[weft] No leased ledger record found for task "${task.operationId}" \u2014 skipping reassignment`);
221
213
  } catch (error) {
222
214
  console.error(`[weft] Failed to reassign task "${task.operationId}" from worker "${workerId}":`, error);
223
215
  }
@@ -13,6 +13,19 @@ import type { createLiveOperationRegistry, createLiveRestBindings } from '../res
13
13
  import type { TaskQueue } from '../task-queue.ts';
14
14
  import type { WorkflowEventFeed } from '../workflow-event-feed.ts';
15
15
  import type { ResolvedCorsPolicy } from './cors.ts';
16
+ /**
17
+ * Startup task-ledger recovery gate (WFT-23). `ready` resolves once every
18
+ * non-terminal ledger record has been reconstructed into the in-memory
19
+ * registry, deadline tracker, and task queue indexes; it rejects
20
+ * (permanently — a settled promise replays the same outcome to every future
21
+ * awaiter) if the recovery scan itself failed. Task-plane entry points
22
+ * (`dispatchTaskImpl`, long-poll claim/result, and worker registration) await
23
+ * this before touching the ledger, so a scan failure blocks new claims with
24
+ * an actionable error instead of silently continuing with partial indexes.
25
+ */
26
+ export type TaskLedgerRecoveryGate = Readonly<{
27
+ ready: Promise<void>;
28
+ }>;
16
29
  /**
17
30
  * Internal closure state for a single `serve()` invocation.
18
31
  *
@@ -77,4 +90,14 @@ export interface ServerContext {
77
90
  readonly processingOperations: Set<string>;
78
91
  /** Mutex: prevents concurrent reconciliation scans from running simultaneously. */
79
92
  reconciliationRunning: boolean;
93
+ /** Startup task-ledger recovery gate — see {@link TaskLedgerRecoveryGate}. */
94
+ readonly taskLedgerRecovery: TaskLedgerRecoveryGate;
95
+ /**
96
+ * Set once `server.stop()`'s timer-clearing disposer has run. Startup
97
+ * recovery's scan loop and `scheduleDelayedDispatch` both check this before
98
+ * doing further work so a still-running recovery scan (or a reconciliation
99
+ * pass racing shutdown) cannot arm a new timer or issue a durable write
100
+ * after `pendingTimers` has already been cleared.
101
+ */
102
+ stopping: boolean;
80
103
  }
@@ -1,6 +1,12 @@
1
1
  import type { ServeOptions, TaskDispatch } from '../index.ts';
2
2
  import type { ServerContext } from './context.ts';
3
- /** Schedule a delayed dispatch, tracking the timer for cleanup on shutdown. */
3
+ /**
4
+ * Schedule a delayed dispatch, tracking the timer for cleanup on shutdown.
5
+ *
6
+ * No-ops once `context.stopping` is set — `server.stop()`'s timer-clearing
7
+ * disposer has already run by then, so arming a new timer here would leak a
8
+ * callback that fires against a stopped server instead of being cleared.
9
+ */
4
10
  export declare function scheduleDelayedDispatch(context: ServerContext, options: ServeOptions, task: TaskDispatch, delay: number): void;
5
11
  export declare function resolveTaskPriority(_context: ServerContext, _options: ServeOptions, task: TaskDispatch): number | undefined;
6
12
  export declare function dispatchTaskImpl(context: ServerContext, options: ServeOptions, task: TaskDispatch): Promise<boolean>;
@@ -1,5 +1,16 @@
1
+ import { isJSONValue } from "../../core/json.js";
2
+ import { buildWorkerExecutionIdentity } from "../../worker/manifest/execution-identity.js";
1
3
  import { evictOldestAffinityEntries } from "../runtime-helpers.js";
2
- import { markQueued, readQueuedRecord, transitionQueuedToInflight } from "../task-state.js";
4
+ import {
5
+ claimQueued,
6
+ createQueued
7
+ } from "../task-ledger-transitions.js";
8
+ import {
9
+ decodeRemoteTaskRecord,
10
+ REMOTE_TASK_RECORD_VERSION,
11
+ taskLedgerKey
12
+ } from "../task-ledger.js";
13
+ import { commitTaskLedgerTransition } from "./task-ledger-runtime.js";
3
14
  import {
4
15
  recordTaskBacklogMetric,
5
16
  recordTaskQueueLatencyMetric,
@@ -12,6 +23,8 @@ function clampVisibilityTimeout(value) {
12
23
  return Math.min(Math.max(value, MIN_VISIBILITY_TIMEOUT), MAX_VISIBILITY_TIMEOUT);
13
24
  }
14
25
  export function scheduleDelayedDispatch(context, options, task, delay) {
26
+ if (context.stopping)
27
+ return;
15
28
  const timer = setTimeout(() => {
16
29
  context.pendingTimers.delete(timer);
17
30
  dispatchTaskImpl(context, options, task).catch((err) => console.error(`[weft] Delayed redispatch failed for "${task.operationId}":`, err));
@@ -52,6 +65,55 @@ function recordDispatchOutcome(context, task, workerId) {
52
65
  operationIds.add(task.operationId);
53
66
  context.operationToWorkflow.set(task.operationId, task.workflowId);
54
67
  }
68
+ function buildOptionalCreateQueuedFields(task) {
69
+ return {
70
+ ...task.workflowId !== void 0 ? { workflowId: task.workflowId } : {},
71
+ ...task.workflowExecutionToken !== void 0 ? { workflowExecutionToken: task.workflowExecutionToken } : {},
72
+ ...task.priority !== void 0 ? { priority: task.priority } : {},
73
+ ...task.fairShareKey !== void 0 ? { fairShareKey: task.fairShareKey } : {},
74
+ ...task.sticky && task.workflowId !== void 0 ? { stickyWorkflowId: task.workflowId } : {},
75
+ ...task.retryPolicy !== void 0 ? { retryPolicy: task.retryPolicy } : {}
76
+ };
77
+ }
78
+ function buildCreateQueuedInput(task, queue, visibilityTimeout) {
79
+ const input = task.input === void 0 ? null : task.input;
80
+ if (!isJSONValue(input))
81
+ throw Error(`TaskDispatch for operation "${task.operationId}" has a non-JSON-serializable "input" \u2014 the durable task ledger requires JSON-safe input.`);
82
+ return {
83
+ recordVersion: REMOTE_TASK_RECORD_VERSION,
84
+ operationId: task.operationId,
85
+ workflowType: task.workflowType,
86
+ activityName: task.activityName,
87
+ queue,
88
+ input,
89
+ headers: task.headers ?? {},
90
+ visibilityTimeoutMilliseconds: visibilityTimeout,
91
+ createdAt: Date.now(),
92
+ ...buildOptionalCreateQueuedFields(task)
93
+ };
94
+ }
95
+ function buildCreateThenClaimTransition(createInput, claimInput) {
96
+ return (current, now) => {
97
+ let queuedRecord;
98
+ if (current === null) {
99
+ const created = createQueued(null, createInput, now);
100
+ if (!created.ok)
101
+ return created;
102
+ queuedRecord = created.nextRecord;
103
+ } else if (current.state === "queued")
104
+ queuedRecord = current;
105
+ else
106
+ return {
107
+ ok: !1,
108
+ reason: `operation "${createInput.operationId}" is not claimable from state "${current.state}"`
109
+ };
110
+ return claimQueued(queuedRecord, { ...claimInput, expectedGeneration: queuedRecord.generation }, now);
111
+ };
112
+ }
113
+ function bareActivityName(qualifiedActivityName) {
114
+ const dotIndex = qualifiedActivityName.indexOf(".");
115
+ return dotIndex === -1 ? qualifiedActivityName : qualifiedActivityName.slice(dotIndex + 1);
116
+ }
55
117
  async function selectAndReserveWorker(context, options, task, queue, visibilityTimeout) {
56
118
  const routingOptions = buildRoutingOptions(context, task, queue), worker = context.registry.findWorker(task.activityName, routingOptions);
57
119
  if (!worker)
@@ -59,86 +121,85 @@ async function selectAndReserveWorker(context, options, task, queue, visibilityT
59
121
  const ws = context.workerSockets.get(worker.id);
60
122
  if (!ws)
61
123
  return !1;
62
- const now = Date.now(), existingQueuedRecord = await readQueuedRecord(options.engine.storage, task.operationId), attemptToken = crypto.randomUUID();
124
+ const createInput = buildCreateQueuedInput(task, queue, visibilityTimeout), attemptToken = crypto.randomUUID(), executionIdentity = buildWorkerExecutionIdentity({
125
+ manifest: worker.manifest,
126
+ manifestDigest: worker.acceptedManifestDigest,
127
+ workerId: worker.id,
128
+ workflowType: task.workflowType,
129
+ activityName: bareActivityName(task.activityName)
130
+ });
63
131
  context.registry.assignTask(worker.id, task.operationId, visibilityTimeout, task.fairShareKey, attemptToken);
64
- const deadline = now + visibilityTimeout;
65
- context.deadlineTracker.add({ operationId: task.operationId, deadline });
66
- const inflightRecord = {
132
+ let result;
133
+ try {
134
+ result = await commitTaskLedgerTransition(options.engine.storage, task.operationId, buildCreateThenClaimTransition(createInput, {
135
+ attemptToken,
136
+ workerSessionId: worker.id,
137
+ ...executionIdentity !== void 0 ? { executionIdentity } : {},
138
+ leaseDurationMilliseconds: visibilityTimeout
139
+ }), 1);
140
+ } catch (error) {
141
+ context.registry.releaseReservation(task.operationId);
142
+ throw error;
143
+ }
144
+ if (!result.ok) {
145
+ context.registry.releaseReservation(task.operationId);
146
+ return !1;
147
+ }
148
+ context.deadlineTracker.add({
67
149
  operationId: task.operationId,
68
- workerId: worker.id,
69
- deadline,
70
- activityName: task.activityName,
71
- queue,
72
- input: task.input,
73
- attempt: task.attempt ?? 1,
74
- visibilityTimeout,
75
- retryPolicy: task.retryPolicy,
76
- workflowId: task.workflowId,
77
- workflowExecutionToken: task.workflowExecutionToken,
78
- attemptToken
79
- }, normalizedInflightRecord = await transitionQueuedToInflight(options.engine.storage, task.operationId, inflightRecord, {
80
- queuedRecord: existingQueuedRecord,
81
- now
150
+ deadline: result.record.leaseDeadline
82
151
  });
83
152
  ws.send(JSON.stringify({
84
153
  type: "task",
85
154
  operationId: task.operationId,
86
155
  activityName: task.activityName,
87
156
  input: task.input === void 0 ? null : task.input,
88
- attempt: task.attempt ?? 1,
157
+ attempt: result.record.attempt,
89
158
  attemptToken,
90
159
  ...task.workflowExecutionToken !== void 0 && {
91
160
  workflowExecutionToken: task.workflowExecutionToken
92
161
  },
93
162
  ...task.headers ? { headers: task.headers } : {}
94
163
  }));
95
- recordTaskQueueLatencyMetric(context.metricsCollector, normalizedInflightRecord);
164
+ recordTaskQueueLatencyMetric(context.metricsCollector, {
165
+ lastQueuedAt: result.record.lastQueuedAt,
166
+ lastDispatchedAt: Date.now()
167
+ });
96
168
  recordWorkerCapacitySaturationMetric(context.metricsCollector, context.registry);
97
169
  recordDispatchOutcome(context, task, worker.id);
98
170
  return !0;
99
171
  }
100
- function mergeQueuedRecordLifecycle(fresh, existing) {
101
- return {
102
- ...fresh,
103
- firstQueuedAt: existing?.firstQueuedAt ?? fresh.queuedAt,
104
- lastQueuedAt: fresh.queuedAt,
105
- lastDispatchedAt: existing?.lastDispatchedAt,
106
- startedAt: existing?.startedAt,
107
- retryCount: existing?.retryCount ?? fresh.retryCount,
108
- requeueCount: existing?.requeueCount ?? fresh.requeueCount,
109
- lastRequeueReason: existing?.lastRequeueReason
110
- };
111
- }
112
172
  async function enqueueTaskForLongPoll(context, options, task, queue, visibilityTimeout, resolvedPriority) {
113
- const attempt = task.attempt ?? 1, freshRecord = {
114
- operationId: task.operationId,
115
- activityName: task.activityName,
116
- input: task.input,
117
- queue,
118
- attempt,
119
- visibilityTimeout,
120
- retryPolicy: task.retryPolicy,
121
- queuedAt: Date.now(),
122
- workflowId: task.workflowId,
123
- workflowExecutionToken: task.workflowExecutionToken,
124
- retryCount: Math.max(0, attempt - 1),
125
- requeueCount: 0
126
- }, existingQueuedRecord = await readQueuedRecord(options.engine.storage, task.operationId), normalizedQueuedRecord = await markQueued(options.engine.storage, mergeQueuedRecordLifecycle(freshRecord, existingQueuedRecord)), enqueued = context.taskQueue.enqueue(queue, {
173
+ const storage = options.engine.storage, createInput = buildCreateQueuedInput(task, queue, visibilityTimeout), rawExisting = await storage.get(taskLedgerKey(task.operationId)), current = decodeRemoteTaskRecord(rawExisting);
174
+ let queuedRecord;
175
+ if (current === null) {
176
+ const result = await commitTaskLedgerTransition(storage, task.operationId, (freshCurrent, now) => createQueued(freshCurrent, createInput, now), 1);
177
+ if (!result.ok) {
178
+ const raced = decodeRemoteTaskRecord(await storage.get(taskLedgerKey(task.operationId)));
179
+ if (raced === null || raced.state !== "queued")
180
+ return !1;
181
+ queuedRecord = raced;
182
+ } else
183
+ queuedRecord = result.record;
184
+ } else if (current.state === "queued")
185
+ queuedRecord = current;
186
+ else
187
+ return !1;
188
+ const enqueued = context.taskQueue.enqueue(queue, {
127
189
  operationId: task.operationId,
128
190
  activityName: task.activityName,
129
191
  input: task.input,
130
- attempt,
192
+ attempt: queuedRecord.attempt,
131
193
  retryPolicy: task.retryPolicy,
132
194
  visibilityTimeout,
133
195
  workflowId: task.workflowId,
134
196
  workflowExecutionToken: task.workflowExecutionToken,
135
- firstQueuedAt: normalizedQueuedRecord.firstQueuedAt,
136
- lastQueuedAt: normalizedQueuedRecord.lastQueuedAt,
137
- lastDispatchedAt: normalizedQueuedRecord.lastDispatchedAt,
138
- startedAt: normalizedQueuedRecord.startedAt,
139
- retryCount: normalizedQueuedRecord.retryCount,
140
- requeueCount: normalizedQueuedRecord.requeueCount,
141
- lastRequeueReason: normalizedQueuedRecord.lastRequeueReason,
197
+ firstQueuedAt: queuedRecord.firstQueuedAt,
198
+ lastQueuedAt: queuedRecord.lastQueuedAt,
199
+ lastDispatchedAt: queuedRecord.lastDispatchedAt,
200
+ startedAt: queuedRecord.startedAt,
201
+ retryCount: queuedRecord.retryCount,
202
+ requeueCount: queuedRecord.requeueCount,
142
203
  ...task.headers ? { headers: task.headers } : {},
143
204
  ...resolvedPriority !== void 0 ? { priority: resolvedPriority } : {}
144
205
  });
@@ -146,6 +207,16 @@ async function enqueueTaskForLongPoll(context, options, task, queue, visibilityT
146
207
  return enqueued;
147
208
  }
148
209
  export async function dispatchTaskImpl(context, options, task) {
210
+ try {
211
+ await context.taskLedgerRecovery.ready;
212
+ } catch (error) {
213
+ throw Error(`Cannot dispatch task "${task.operationId}" \u2014 startup task-ledger recovery failed: ${error instanceof Error ? error.message : String(error)}`, { cause: error });
214
+ }
215
+ if (!task.workflowType)
216
+ throw Error(`TaskDispatch for operation "${task.operationId}" is missing required field "workflowType".`);
217
+ const dotIndex = task.activityName.indexOf(".");
218
+ if (dotIndex !== -1 && task.activityName.slice(0, dotIndex) !== task.workflowType)
219
+ throw Error(`TaskDispatch for operation "${task.operationId}" has activityName "${task.activityName}" whose qualifier does not match workflowType "${task.workflowType}".`);
149
220
  const queue = task.queue ?? "default", visibilityTimeout = clampVisibilityTimeout(task.visibilityTimeout), resolvedPriority = resolveTaskPriority(context, options, task);
150
221
  if (!validateTaskEnvelope(context, task))
151
222
  return !1;
@@ -0,0 +1,68 @@
1
+ /**
2
+ * Shared two-step commit for applying a worker's task result to the durable
3
+ * remote task ledger (WFT-22) — used by both the WebSocket (`onTaskResultMessage`)
4
+ * and long-poll (`handleTaskResultRequest`) completion paths.
5
+ *
6
+ * `Leased --> Completing --> Terminal` is committed as two separate durable
7
+ * writes, not one, even though the caller already holds the full result: the
8
+ * project brief models `Completing` as an observable intermediate state
9
+ * specifically so a crash between the two writes leaves a durable, recoverable
10
+ * marker ("Completing prevents visibility scanning or disconnect handling from
11
+ * requeueing an attempt while its terminal result is being applied") that
12
+ * WFT-23's recovery is designed to resume. Collapsing the two transitions into
13
+ * one write would make that state unobservable and defeat the point of having
14
+ * it.
15
+ *
16
+ * A submitter can itself resume that crash window: if the record is already
17
+ * `completing` with the same `attemptToken` AND the same `pendingResultDigest`
18
+ * — the worker retrying the identical result after a server crash between the
19
+ * two writes, or a benign duplicate submission — this skips straight to
20
+ * `commitTerminalResult` instead of rejecting a legitimate resubmission with
21
+ * "expected task state leased". A `completing` record whose digest or token
22
+ * differs is a genuinely different result and is rejected normally.
23
+ *
24
+ * On exhausted retries — specifically when the *second* write
25
+ * (`commitTerminalFromCompleting`) exhausts its CAS retry budget after the
26
+ * record is already durably `completing` — this attempts one best-effort
27
+ * `Completing --> DeadLettered` write (WFT-24's `commitDeadLetter`,
28
+ * `../task-ledger-transitions.ts`) so a sustained, operation-specific
29
+ * storage write failure becomes an operator-visible dead letter instead of a
30
+ * silently stuck record. If the *first* write (`beginCompletion`) exhausts
31
+ * instead, the record never left `leased` and no dead letter is attempted —
32
+ * the visibility scanner's ordinary expiry path already covers that case.
33
+ * If the dead-letter write itself also fails, this falls back to the plain
34
+ * `ok: false` result exactly as before WFT-24: the record stays `completing`,
35
+ * a worker resubmitting the identical result can still resume through the
36
+ * `resuming` branch above, and no data is lost.
37
+ *
38
+ * @module server/runtime/task-ledger-completion
39
+ */
40
+ import type { Storage } from '../../storage/interface.ts';
41
+ import type { ServeOptions } from '../index.ts';
42
+ import { type RemoteTaskCompleting, type RemoteTaskDeadLettered, type RemoteTaskTerminal } from '../task-ledger.ts';
43
+ export type TaskLedgerCompletionInput = Readonly<{
44
+ operationId: string;
45
+ attemptToken: string;
46
+ status: 'completed' | 'failed';
47
+ value?: unknown;
48
+ error?: string;
49
+ }>;
50
+ export type TaskLedgerCompletionResult = Readonly<{
51
+ ok: true;
52
+ completing: RemoteTaskCompleting;
53
+ terminal: RemoteTaskTerminal;
54
+ }> | Readonly<{
55
+ ok: false;
56
+ reason: string;
57
+ deadLettered?: RemoteTaskDeadLettered;
58
+ }>;
59
+ /**
60
+ * Dispatch {@link TaskResultDeadLetteredEvent} for a record `commitTaskLedgerCompletion`
61
+ * dead-lettered. Shared by the WebSocket and long-poll completion paths so
62
+ * both build the event from the same fields — `workflowId`/`activityName`/`queue`
63
+ * come from the dead-lettered record itself (present on every `RemoteTaskBase`),
64
+ * not from the caller's original request, so the event is accurate even if the
65
+ * caller only had partial information.
66
+ */
67
+ export declare function dispatchTaskDeadLetteredEvent(options: ServeOptions, operationId: string, deadLettered: RemoteTaskDeadLettered, workerId: string | undefined): void;
68
+ export declare function commitTaskLedgerCompletion(storage: Storage, input: TaskLedgerCompletionInput): Promise<TaskLedgerCompletionResult>;