@lostgradient/weft 0.18.0 → 0.19.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (167) hide show
  1. package/README.md +3 -1
  2. package/dist/cli/conformance.js +14 -18
  3. package/dist/cli/generated/operation-catalog.snapshot.json +258 -16
  4. package/dist/cli/generated/operation-client.generated.d.ts +30 -6
  5. package/dist/cli/generated/operation-client.generated.js +4 -0
  6. package/dist/core/context/child-workflow-pipe.d.ts +1 -1
  7. package/dist/core/context/child-workflow-pipe.js +1 -1
  8. package/dist/core/engine/construction.d.ts +2 -0
  9. package/dist/core/engine/construction.js +3 -0
  10. package/dist/core/engine/index.js +1 -0
  11. package/dist/core/engine/listing.d.ts +1 -0
  12. package/dist/core/engine/listing.js +6 -4
  13. package/dist/core/weft-error.d.ts +1 -1
  14. package/dist/core/weft-error.js +2 -1
  15. package/dist/core/worker-execution-dispatcher.d.ts +8 -0
  16. package/dist/core/worker-execution-dispatcher.js +4 -0
  17. package/dist/core/worker-execution-strategy-options.d.ts +16 -0
  18. package/dist/core/worker-execution-strategy.js +71 -89
  19. package/dist/core/worker-fault-handling.d.ts +45 -0
  20. package/dist/core/worker-fault-handling.js +88 -0
  21. package/dist/core/worker-protocol.d.ts +18 -1
  22. package/dist/core/worker-protocol.js +1 -1
  23. package/dist/core/worker-realm-readiness.d.ts +78 -0
  24. package/dist/core/worker-realm-readiness.js +127 -0
  25. package/dist/index.d.ts +3 -1
  26. package/dist/index.js +28 -1
  27. package/dist/json-schema.js +1 -1
  28. package/dist/runtime/portable.d.ts +14 -0
  29. package/dist/runtime/portable.js +10 -0
  30. package/dist/server/index.d.ts +71 -8
  31. package/dist/server/index.js +7 -3
  32. package/dist/server/operations/get-task-diagnostics.d.ts +43 -18
  33. package/dist/server/operations/get-task-diagnostics.js +87 -126
  34. package/dist/server/operations/get-worker-diagnostics.d.ts +177 -0
  35. package/dist/server/operations/get-worker-diagnostics.js +117 -0
  36. package/dist/server/operations/list-worker-registration-rejections.d.ts +79 -0
  37. package/dist/server/operations/list-worker-registration-rejections.js +60 -0
  38. package/dist/server/operations/list-workers.js +1 -3
  39. package/dist/server/operations/static-registrations.js +5 -20
  40. package/dist/server/rest-bindings.js +31 -2
  41. package/dist/server/runtime/authentication-bridge.js +6 -14
  42. package/dist/server/runtime/context.d.ts +23 -0
  43. package/dist/server/runtime/task-dispatch.d.ts +7 -1
  44. package/dist/server/runtime/task-dispatch.js +126 -55
  45. package/dist/server/runtime/task-ledger-completion.d.ts +68 -0
  46. package/dist/server/runtime/task-ledger-completion.js +72 -0
  47. package/dist/server/runtime/task-ledger-recovery.d.ts +55 -0
  48. package/dist/server/runtime/task-ledger-recovery.js +73 -0
  49. package/dist/server/runtime/task-ledger-runtime.d.ts +64 -0
  50. package/dist/server/runtime/task-ledger-runtime.js +34 -0
  51. package/dist/server/runtime/task-metrics.d.ts +6 -4
  52. package/dist/server/runtime/task-polling.d.ts +14 -12
  53. package/dist/server/runtime/task-polling.js +106 -82
  54. package/dist/server/runtime/task-reconciliation.d.ts +35 -8
  55. package/dist/server/runtime/task-reconciliation.js +75 -74
  56. package/dist/server/runtime/task-result-resolution.d.ts +2 -10
  57. package/dist/server/runtime/task-result-resolution.js +0 -67
  58. package/dist/server/runtime/task-result-view.d.ts +82 -0
  59. package/dist/server/runtime/task-result-view.js +48 -0
  60. package/dist/server/runtime/websocket-worker-messaging.d.ts +16 -0
  61. package/dist/server/runtime/websocket-worker-messaging.js +32 -0
  62. package/dist/server/runtime/websocket-worker-registration.d.ts +41 -0
  63. package/dist/server/runtime/websocket-worker-registration.js +134 -0
  64. package/dist/server/runtime/websocket-worker.d.ts +0 -1
  65. package/dist/server/runtime/websocket-worker.js +64 -116
  66. package/dist/server/serve-internals.d.ts +9 -6
  67. package/dist/server/serve-internals.js +14 -40
  68. package/dist/server/task-ledger-codec.d.ts +40 -0
  69. package/dist/server/task-ledger-codec.js +195 -0
  70. package/dist/server/task-ledger-keys.d.ts +13 -0
  71. package/dist/server/task-ledger-keys.js +5 -0
  72. package/dist/server/task-ledger-limits.d.ts +34 -0
  73. package/dist/server/task-ledger-limits.js +5 -0
  74. package/dist/server/task-ledger-transition-helpers.d.ts +38 -0
  75. package/dist/server/task-ledger-transition-helpers.js +42 -0
  76. package/dist/server/task-ledger-transitions-cancellation.d.ts +31 -0
  77. package/dist/server/task-ledger-transitions-cancellation.js +73 -0
  78. package/dist/server/task-ledger-transitions.d.ts +129 -0
  79. package/dist/server/task-ledger-transitions.js +162 -0
  80. package/dist/server/task-ledger-types.d.ts +220 -0
  81. package/dist/server/task-ledger-types.js +1 -0
  82. package/dist/server/task-ledger.d.ts +17 -0
  83. package/dist/server/task-ledger.js +24 -0
  84. package/dist/server/task-state.d.ts +25 -152
  85. package/dist/server/task-state.js +0 -188
  86. package/dist/server/worker-admission-policy.d.ts +70 -0
  87. package/dist/server/worker-admission-policy.js +0 -0
  88. package/dist/storage/compressed-storage.js +1 -1
  89. package/dist/storage/index.d.ts +0 -2
  90. package/dist/storage/interface.d.ts +0 -2
  91. package/dist/storage/interface.js +1 -1
  92. package/dist/storage/lmdb.js +1 -1
  93. package/dist/storage/memory.js +1 -1
  94. package/dist/storage/neon.js +1 -1
  95. package/dist/storage/postgres.js +1 -1
  96. package/dist/storage/resolve.js +1 -1
  97. package/dist/storage/scoped-storage.js +1 -1
  98. package/dist/storage/testing.js +1 -1
  99. package/dist/storage/turso.js +1 -1
  100. package/dist/storage/web-extension.js +2 -2
  101. package/dist/version.d.ts +1 -1
  102. package/dist/version.js +1 -1
  103. package/dist/web-extension.js +1 -1
  104. package/dist/worker/index.d.ts +2 -0
  105. package/dist/worker/index.js +30 -4
  106. package/dist/worker/manifest/canonical-json.d.ts +20 -0
  107. package/dist/worker/manifest/canonical-json.js +13 -0
  108. package/dist/worker/manifest/capabilities.d.ts +24 -0
  109. package/dist/worker/manifest/capabilities.js +74 -0
  110. package/dist/worker/manifest/content-digest.d.ts +20 -0
  111. package/dist/worker/manifest/content-digest.js +13 -0
  112. package/dist/worker/manifest/declared-shape-digest.d.ts +14 -0
  113. package/dist/worker/manifest/declared-shape-digest.js +4 -0
  114. package/dist/worker/manifest/digest.d.ts +82 -0
  115. package/dist/worker/manifest/digest.js +9 -0
  116. package/dist/worker/manifest/execution-identity.d.ts +92 -0
  117. package/dist/worker/manifest/execution-identity.js +36 -0
  118. package/dist/worker/manifest/failure.d.ts +59 -0
  119. package/dist/worker/manifest/failure.js +4 -0
  120. package/dist/worker/manifest/index.d.ts +22 -0
  121. package/dist/worker/manifest/index.js +23 -0
  122. package/dist/worker/manifest/internal-realm.d.ts +50 -0
  123. package/dist/worker/manifest/internal-realm.js +39 -0
  124. package/dist/worker/manifest/is-record.d.ts +12 -0
  125. package/dist/worker/manifest/is-record.js +6 -0
  126. package/dist/worker/manifest/json-scan.d.ts +26 -0
  127. package/dist/worker/manifest/json-scan.js +88 -0
  128. package/dist/worker/manifest/limits.d.ts +138 -0
  129. package/dist/worker/manifest/limits.js +1 -0
  130. package/dist/worker/manifest/normalize.d.ts +63 -0
  131. package/dist/worker/manifest/normalize.js +73 -0
  132. package/dist/worker/manifest/parse-json.d.ts +24 -0
  133. package/dist/worker/manifest/parse-json.js +15 -0
  134. package/dist/worker/manifest/parse.d.ts +86 -0
  135. package/dist/worker/manifest/parse.js +188 -0
  136. package/dist/worker/manifest/registry-contract-builder.d.ts +120 -0
  137. package/dist/worker/manifest/registry-contract-builder.js +65 -0
  138. package/dist/worker/manifest/types.d.ts +263 -0
  139. package/dist/worker/manifest/types.js +1 -0
  140. package/dist/worker/manifest/utf8.d.ts +13 -0
  141. package/dist/worker/manifest/utf8.js +7 -0
  142. package/dist/worker/options.d.ts +55 -8
  143. package/dist/worker/options.js +65 -9
  144. package/dist/worker/protocol-internals.d.ts +1 -2
  145. package/dist/worker/protocol-internals.js +0 -5
  146. package/dist/worker/protocol-messages.d.ts +33 -17
  147. package/dist/worker/protocol-schemas.d.ts +36 -74
  148. package/dist/worker/protocol-schemas.js +25 -18
  149. package/dist/worker/protocol-version.d.ts +18 -14
  150. package/dist/worker/protocol-version.js +1 -1
  151. package/dist/worker/protocol.js +1 -1
  152. package/dist/worker/registry/deployment-consistency.d.ts +78 -0
  153. package/dist/worker/registry/deployment-consistency.js +27 -0
  154. package/dist/worker/registry/drain.d.ts +14 -0
  155. package/dist/worker/registry/drain.js +24 -0
  156. package/dist/worker/registry/rejections.d.ts +31 -0
  157. package/dist/worker/registry/rejections.js +10 -0
  158. package/dist/worker/registry/summary.d.ts +0 -3
  159. package/dist/worker/registry/summary.js +2 -5
  160. package/dist/worker/registry/types.d.ts +10 -3
  161. package/dist/worker/registry.d.ts +47 -2
  162. package/dist/worker/registry.js +45 -32
  163. package/dist/workers/workflow-worker-entry.d.ts +12 -7
  164. package/dist/workers/workflow-worker-entry.js +9 -2
  165. package/package.json +3 -3
  166. package/dist/server/task-resolved-record.d.ts +0 -12
  167. package/dist/server/task-resolved-record.js +0 -35
@@ -1,18 +1,15 @@
1
1
  import { z } from "zod";
2
- import { decode } from "../../core/codec.js";
3
- import { KEYS } from "../../storage/interface.js";
2
+ import { raiseFault } from "../operation-catalog/raise-fault.js";
4
3
  import { defineOperation } from "../operation-registry.js";
4
+ import { commitTaskLedgerDelete } from "../runtime/task-ledger-runtime.js";
5
+ import { canClearDeadLetteredTask } from "../task-ledger-transitions.js";
6
+ import { decodeRemoteTaskRecord } from "../task-ledger.js";
5
7
  import {
6
8
  calculateExecutionLatencyMs,
7
9
  calculateHeartbeatAgeMs,
8
- calculateQueueLatencyMs,
9
- clearDeadLetteredTaskRecord,
10
- isDeadLetteredTaskRecord,
11
- isInflightRecord,
12
- isQueuedRecord,
13
- isResolvedRecord
10
+ calculateQueueLatencyMs
14
11
  } from "../task-state.js";
15
- const DEFAULT_STALE_QUEUED_AFTER_MS = 60000, DEFAULT_STALE_HEARTBEAT_AFTER_MS = 60000, DEFAULT_RETRY_STORM_MINIMUM_ATTEMPTS = 3, DEFAULT_LIMIT = 50, MAX_LIMIT = 200, RESOLVED_HISTORY_SCAN_LIMIT = 1000, taskDiagnosticKindSchema = z.enum([
12
+ const DEFAULT_STALE_QUEUED_AFTER_MS = 60000, DEFAULT_STALE_HEARTBEAT_AFTER_MS = 60000, DEFAULT_RETRY_STORM_MINIMUM_ATTEMPTS = 3, DEFAULT_LIMIT = 50, MAX_LIMIT = 200, taskDiagnosticKindSchema = z.enum([
16
13
  "stuck-queued",
17
14
  "stale-inflight",
18
15
  "retry-storm",
@@ -31,7 +28,7 @@ const DEFAULT_STALE_QUEUED_AFTER_MS = 60000, DEFAULT_STALE_HEARTBEAT_AFTER_MS =
31
28
  queueLatencyMs: z.number().nonnegative().optional(),
32
29
  executionLatencyMs: z.number().nonnegative().optional(),
33
30
  heartbeatAgeMs: z.number().nonnegative().optional(),
34
- lastRequeueReason: z.enum(["visibility-timeout", "worker-disconnect"]).optional(),
31
+ lastRequeueReason: z.string().optional(),
35
32
  resolutionReason: z.string().optional(),
36
33
  deadLetteredAt: z.number().nonnegative().optional(),
37
34
  deadLetterReason: z.literal("result-resolution-storage-exhausted").optional(),
@@ -105,12 +102,17 @@ export const getTaskDiagnosticsOperation = createGetTaskDiagnosticsOperation(),
105
102
  kind: "scoped",
106
103
  scopes: { kind: "anyOf", scopes: ["system:admin"] }
107
104
  },
108
- producibleFaults: [],
105
+ producibleFaults: ["NotFound"],
109
106
  discoverable: !0,
110
107
  transports: restOnlyTaskDiagnosticsTransports,
111
108
  unknownKeyPolicy: { http: "reject", jsonRpc: "reject" },
112
109
  invoke: async ({ input, engine }) => {
113
- await clearDeadLetteredTaskRecord(engine.storage, input.operationId);
110
+ if (!(await commitTaskLedgerDelete(engine.storage, input.operationId, canClearDeadLetteredTask, 1)).ok)
111
+ raiseFault(clearTaskDeadLetterOperation, {
112
+ code: "NotFound",
113
+ message: `No dead-lettered task found for operation "${input.operationId}"`,
114
+ data: { resource: "task", identifier: input.operationId }
115
+ });
114
116
  return { ok: !0 };
115
117
  }
116
118
  });
@@ -127,39 +129,19 @@ async function collectTaskDiagnostics({
127
129
  retryStorms: 0,
128
130
  allWorkersAtCapacity: 0,
129
131
  deadLettered: 0
130
- }, relevantQueues = new Set, deadLetteredOperationIds = new Set, addItem = (item) => {
132
+ }, relevantQueues = new Set, addItem = (item) => {
131
133
  incrementSummary(summary, item.kind);
132
134
  if (items.length < input.limit)
133
135
  items.push(item);
134
136
  };
135
- for await (const record of scanDeadLetterRecords(engine)) {
136
- if (!matchesTaskRecordFilter(record, input))
137
- continue;
138
- deadLetteredOperationIds.add(record.operationId);
139
- if (record.queue !== void 0)
140
- relevantQueues.add(record.queue);
141
- addDeadLetterDiagnostics(record, addItem);
142
- }
143
- for await (const record of scanQueuedRecords(engine)) {
144
- if (!matchesTaskRecordFilter(record, input))
137
+ for await (const [, value] of engine.storage.scan("task-ledger:")) {
138
+ const decoded = decodeRemoteTaskRecord(value);
139
+ if (decoded === null)
145
140
  continue;
146
- relevantQueues.add(record.queue);
147
- addQueuedDiagnostics(record, input, currentTime, addItem);
148
- }
149
- for await (const record of scanInflightRecords(engine)) {
150
- if (!matchesTaskRecordFilter(record, input))
151
- continue;
152
- if (deadLetteredOperationIds.has(record.operationId))
141
+ if (!matchesTaskRecordFilter(decoded, input))
153
142
  continue;
154
- relevantQueues.add(record.queue);
155
- addInflightDiagnostics(record, input, currentTime, addItem);
156
- }
157
- for await (const record of scanResolvedRecords(engine, input)) {
158
- if (!matchesTaskRecordFilter(record, input))
159
- continue;
160
- if (record.queue !== void 0)
161
- relevantQueues.add(record.queue);
162
- addRetryStormDiagnostic(record, "resolved", input, addItem);
143
+ relevantQueues.add(decoded.queue);
144
+ addRecordDiagnostics(decoded, input, currentTime, addItem);
163
145
  }
164
146
  addCapacityDiagnostics({
165
147
  registry,
@@ -170,88 +152,71 @@ async function collectTaskDiagnostics({
170
152
  });
171
153
  return { items, summary, limit: input.limit };
172
154
  }
173
- async function* scanDeadLetterRecords(engine) {
174
- for await (const [, value] of engine.storage.scan(KEYS.operationDeadLetterPrefix())) {
175
- const decoded = decode(value);
176
- if (isDeadLetteredTaskRecord(decoded))
177
- yield decoded;
178
- }
179
- }
180
- async function* scanQueuedRecords(engine) {
181
- for await (const [, value] of engine.storage.scan("op:queued:")) {
182
- const decoded = decode(value);
183
- if (isQueuedRecord(decoded))
184
- yield decoded;
185
- }
186
- }
187
- async function* scanInflightRecords(engine) {
188
- for await (const [, value] of engine.storage.scan("op:inflight:")) {
189
- const decoded = decode(value);
190
- if (isInflightRecord(decoded))
191
- yield decoded;
192
- }
193
- }
194
- async function* scanResolvedRecords(engine, input) {
195
- if (input.operationId !== void 0) {
196
- const value = await engine.storage.get(KEYS.operationResolved(input.operationId));
197
- if (value === null)
155
+ function addRecordDiagnostics(decoded, input, currentTime, addItem) {
156
+ switch (decoded.state) {
157
+ case "queued":
158
+ if (decoded.availableAt <= currentTime)
159
+ addQueuedDiagnostics(decoded, input, currentTime, addItem);
160
+ addRetryStormDiagnostic(decoded, "queued", input, addItem);
198
161
  return;
199
- const decoded = decode(value);
200
- if (isResolvedRecord(decoded))
201
- yield decoded;
202
- return;
203
- }
204
- for await (const [, value] of engine.storage.scan(KEYS.operationResolvedByTimePrefix(), {
205
- limit: RESOLVED_HISTORY_SCAN_LIMIT,
206
- reverse: !0
207
- })) {
208
- const decoded = decode(value);
209
- if (isResolvedRecord(decoded))
210
- yield decoded;
162
+ case "leased":
163
+ case "completing":
164
+ case "cancelling":
165
+ addInflightDiagnostics(decoded, input, currentTime, addItem);
166
+ addRetryStormDiagnostic(decoded, "inflight", input, addItem);
167
+ return;
168
+ case "terminal":
169
+ return;
170
+ case "deadLettered":
171
+ addDeadLetterDiagnostics(decoded, addItem);
172
+ return;
173
+ default: {
174
+ const exhaustive = decoded;
175
+ }
211
176
  }
212
177
  }
213
178
  function addQueuedDiagnostics(record, input, currentTime, addItem) {
214
- const queuedAt = record.lastQueuedAt ?? record.queuedAt, queueLatencyMs = Math.max(0, currentTime - queuedAt);
215
- if (queueLatencyMs >= input.staleQueuedAfterMs)
216
- addItem({
217
- kind: "stuck-queued",
218
- state: "queued",
219
- operationId: record.operationId,
220
- workflowId: record.workflowId,
221
- activityName: record.activityName,
222
- queue: record.queue,
223
- retryCount: record.retryCount ?? Math.max(0, (record.attempt ?? 1) - 1),
224
- requeueCount: record.requeueCount ?? 0,
225
- queueLatencyMs,
226
- lastRequeueReason: record.lastRequeueReason,
227
- evidence: [
228
- `Task has waited ${queueLatencyMs}ms in queue "${record.queue}" without a worker claim`
229
- ]
230
- });
231
- addRetryStormDiagnostic(record, "queued", input, addItem);
179
+ const queueLatencyMs = Math.max(0, currentTime - record.lastQueuedAt);
180
+ if (queueLatencyMs < input.staleQueuedAfterMs)
181
+ return;
182
+ addItem({
183
+ kind: "stuck-queued",
184
+ state: "queued",
185
+ operationId: record.operationId,
186
+ workflowId: record.workflowId,
187
+ activityName: record.activityName,
188
+ queue: record.queue,
189
+ retryCount: record.retryCount,
190
+ requeueCount: record.requeueCount,
191
+ queueLatencyMs,
192
+ lastRequeueReason: record.lastRequeueReason,
193
+ evidence: [
194
+ `Task has waited ${queueLatencyMs}ms in queue "${record.queue}" without a worker claim`
195
+ ]
196
+ });
232
197
  }
233
198
  function addInflightDiagnostics(record, input, currentTime, addItem) {
234
199
  const heartbeatAgeMs = calculateHeartbeatAgeMs(record, currentTime) ?? 0;
235
- if (heartbeatAgeMs >= input.staleHeartbeatAfterMs)
236
- addItem({
237
- kind: "stale-inflight",
238
- state: "inflight",
239
- operationId: record.operationId,
240
- workflowId: record.workflowId,
241
- activityName: record.activityName,
242
- queue: record.queue,
243
- workerId: record.workerId,
244
- retryCount: record.retryCount ?? Math.max(0, (record.attempt ?? 1) - 1),
245
- requeueCount: record.requeueCount ?? 0,
246
- queueLatencyMs: calculateQueueLatencyMs(record),
247
- executionLatencyMs: calculateExecutionLatencyMs(record, currentTime),
248
- heartbeatAgeMs,
249
- lastRequeueReason: record.lastRequeueReason,
250
- evidence: [
251
- `Worker "${record.workerId}" has not sent a heartbeat for ${heartbeatAgeMs}ms on queue "${record.queue}"`
252
- ]
253
- });
254
- addRetryStormDiagnostic(record, "inflight", input, addItem);
200
+ if (heartbeatAgeMs < input.staleHeartbeatAfterMs)
201
+ return;
202
+ addItem({
203
+ kind: "stale-inflight",
204
+ state: "inflight",
205
+ operationId: record.operationId,
206
+ workflowId: record.workflowId,
207
+ activityName: record.activityName,
208
+ queue: record.queue,
209
+ workerId: record.workerSessionId,
210
+ retryCount: record.retryCount,
211
+ requeueCount: record.requeueCount,
212
+ queueLatencyMs: calculateQueueLatencyMs(record),
213
+ executionLatencyMs: calculateExecutionLatencyMs(record, currentTime),
214
+ heartbeatAgeMs,
215
+ lastRequeueReason: record.lastRequeueReason,
216
+ evidence: [
217
+ `Worker "${record.workerSessionId}" has not sent a heartbeat for ${heartbeatAgeMs}ms on queue "${record.queue}"`
218
+ ]
219
+ });
255
220
  }
256
221
  function addDeadLetterDiagnostics(record, addItem) {
257
222
  addItem({
@@ -261,21 +226,19 @@ function addDeadLetterDiagnostics(record, addItem) {
261
226
  workflowId: record.workflowId,
262
227
  activityName: record.activityName,
263
228
  queue: record.queue,
264
- workerId: record.workerId,
265
- retryCount: record.retryCount ?? Math.max(0, (record.attempt ?? 1) - 1),
266
- requeueCount: record.requeueCount ?? 0,
229
+ retryCount: record.retryCount,
230
+ requeueCount: record.requeueCount,
267
231
  lastRequeueReason: record.lastRequeueReason,
268
232
  deadLetteredAt: record.deadLetteredAt,
269
- deadLetterReason: record.reason,
270
- storageError: record.errorMessage,
271
- retryAttempts: record.retryAttempts,
233
+ deadLetterReason: "result-resolution-storage-exhausted",
234
+ storageError: record.persistenceFailureReason,
272
235
  evidence: [
273
- `Task result transition exhausted ${record.retryAttempts} storage attempts; reconciliation will not re-dispatch operation "${record.operationId}" until the dead-letter entry is cleared`
236
+ `Task result could not be durably persisted (${record.persistenceFailureReason}); reconciliation will not re-dispatch operation "${record.operationId}" until the dead-letter entry is cleared`
274
237
  ]
275
238
  });
276
239
  }
277
240
  function addRetryStormDiagnostic(record, state, input, addItem) {
278
- const retryCount = record.retryCount ?? Math.max(0, (record.attempt ?? 1) - 1), requeueCount = record.requeueCount ?? 0;
241
+ const { retryCount, requeueCount } = record;
279
242
  if (retryCount < input.retryStormMinimumAttempts && requeueCount < input.retryStormMinimumAttempts)
280
243
  return;
281
244
  addItem({
@@ -283,15 +246,13 @@ function addRetryStormDiagnostic(record, state, input, addItem) {
283
246
  state,
284
247
  operationId: record.operationId,
285
248
  workflowId: record.workflowId,
286
- activityName: "activityName" in record ? record.activityName : void 0,
249
+ activityName: record.activityName,
287
250
  queue: record.queue,
288
- workerId: "workerId" in record ? record.workerId : void 0,
251
+ workerId: "workerSessionId" in record ? record.workerSessionId : void 0,
289
252
  retryCount,
290
253
  requeueCount,
291
254
  queueLatencyMs: calculateQueueLatencyMs(record),
292
- executionLatencyMs: "resolvedAt" in record ? calculateExecutionLatencyMs(record, record.resolvedAt) : void 0,
293
255
  lastRequeueReason: record.lastRequeueReason,
294
- resolutionReason: "resolutionReason" in record ? record.resolutionReason : void 0,
295
256
  evidence: [
296
257
  `Task has ${retryCount} retries and ${requeueCount} requeues, meeting retry storm threshold ${input.retryStormMinimumAttempts}`
297
258
  ]
@@ -0,0 +1,177 @@
1
+ /**
2
+ * `weft.workers.diagnostics` operation + REST binding.
3
+ *
4
+ * Bounded diagnostic detail for one connected worker (WFT-29), split into
5
+ * two structurally distinct sub-objects: `instance` (this live socket —
6
+ * queue, health, connection timestamps) and `deploymentVersion` (what
7
+ * artifact it loaded — SDK/runtime identity, manifest digest, and every
8
+ * advertised workflow's version/revision/contract hash). Keeping these
9
+ * separate mirrors the manifest's own split between "which process" and
10
+ * "which build" questions.
11
+ *
12
+ * Deliberately excludes `capabilities` and the raw payload schemas backing
13
+ * each `contractHash` — this surfaces identity for drift detection, not a
14
+ * schema dump.
15
+ *
16
+ * @module server/operations/get-worker-diagnostics
17
+ */
18
+ import { z } from 'zod';
19
+ import type { WorkerHealth, WorkerRegistry } from '../../worker/registry.ts';
20
+ import type { UnknownRestBinding } from '../rest-bindings.ts';
21
+ declare const workerDiagnosticsEntrySchema: z.ZodObject<{
22
+ instance: z.ZodObject<{
23
+ workerId: z.ZodString;
24
+ queue: z.ZodString;
25
+ health: z.ZodType<WorkerHealth, unknown, z.core.$ZodTypeInternals<WorkerHealth, unknown>>;
26
+ connectedAt: z.ZodNumber;
27
+ startedAt: z.ZodNumber;
28
+ lastHeartbeatAt: z.ZodNumber;
29
+ heartbeatAgeMs: z.ZodNumber;
30
+ }, z.core.$strict>;
31
+ deploymentVersion: z.ZodObject<{
32
+ deploymentName: z.ZodString;
33
+ buildId: z.ZodString;
34
+ artifactDigest: z.ZodString;
35
+ runtimeName: z.ZodString;
36
+ runtimeVersion: z.ZodString;
37
+ sdkVersion: z.ZodString;
38
+ manifestVersion: z.ZodNumber;
39
+ protocolVersion: z.ZodNumber;
40
+ manifestDigest: z.ZodString;
41
+ workflows: z.ZodRecord<z.ZodString, z.ZodObject<{
42
+ workflowVersion: z.ZodString;
43
+ workflowRevision: z.ZodString;
44
+ contractHash: z.ZodString;
45
+ activities: z.ZodRecord<z.ZodString, z.ZodObject<{
46
+ contractHash: z.ZodString;
47
+ implementationRevision: z.ZodString;
48
+ }, z.core.$strict>>;
49
+ }, z.core.$strict>>;
50
+ }, z.core.$strict>;
51
+ }, z.core.$strict>;
52
+ declare const getWorkerDiagnosticsInput: z.ZodObject<{
53
+ workerId: z.ZodString;
54
+ }, z.core.$strip>;
55
+ declare const getWorkerDiagnosticsOutput: z.ZodObject<{
56
+ worker: z.ZodNullable<z.ZodObject<{
57
+ instance: z.ZodObject<{
58
+ workerId: z.ZodString;
59
+ queue: z.ZodString;
60
+ health: z.ZodType<WorkerHealth, unknown, z.core.$ZodTypeInternals<WorkerHealth, unknown>>;
61
+ connectedAt: z.ZodNumber;
62
+ startedAt: z.ZodNumber;
63
+ lastHeartbeatAt: z.ZodNumber;
64
+ heartbeatAgeMs: z.ZodNumber;
65
+ }, z.core.$strict>;
66
+ deploymentVersion: z.ZodObject<{
67
+ deploymentName: z.ZodString;
68
+ buildId: z.ZodString;
69
+ artifactDigest: z.ZodString;
70
+ runtimeName: z.ZodString;
71
+ runtimeVersion: z.ZodString;
72
+ sdkVersion: z.ZodString;
73
+ manifestVersion: z.ZodNumber;
74
+ protocolVersion: z.ZodNumber;
75
+ manifestDigest: z.ZodString;
76
+ workflows: z.ZodRecord<z.ZodString, z.ZodObject<{
77
+ workflowVersion: z.ZodString;
78
+ workflowRevision: z.ZodString;
79
+ contractHash: z.ZodString;
80
+ activities: z.ZodRecord<z.ZodString, z.ZodObject<{
81
+ contractHash: z.ZodString;
82
+ implementationRevision: z.ZodString;
83
+ }, z.core.$strict>>;
84
+ }, z.core.$strict>>;
85
+ }, z.core.$strict>;
86
+ }, z.core.$strict>>;
87
+ }, z.core.$strip>;
88
+ export type GetWorkerDiagnosticsInput = z.infer<typeof getWorkerDiagnosticsInput>;
89
+ export type WorkerDiagnosticsEntry = z.infer<typeof workerDiagnosticsEntrySchema>;
90
+ export type GetWorkerDiagnosticsOutput = z.infer<typeof getWorkerDiagnosticsOutput>;
91
+ type GetWorkerDiagnosticsOptions = {
92
+ workerRegistry?: WorkerRegistry;
93
+ clock?: () => number;
94
+ };
95
+ /**
96
+ * Build the `weft.workers.diagnostics` operation, optionally bound to a live
97
+ * `WorkerRegistry` and clock. Mirrors `createListWorkersOperation`'s
98
+ * discovery-only fallback: when `workerRegistry` is omitted, `invoke` throws
99
+ * if reached — reserved for discovery-only registries (OpenAPI/AsyncAPI).
100
+ */
101
+ export declare function createGetWorkerDiagnosticsOperation(options?: GetWorkerDiagnosticsOptions): import("../operation-catalog.ts").OperationDefinition<{
102
+ workerId: string;
103
+ }, {
104
+ worker: {
105
+ instance: {
106
+ workerId: string;
107
+ queue: string;
108
+ health: WorkerHealth;
109
+ connectedAt: number;
110
+ startedAt: number;
111
+ lastHeartbeatAt: number;
112
+ heartbeatAgeMs: number;
113
+ };
114
+ deploymentVersion: {
115
+ deploymentName: string;
116
+ buildId: string;
117
+ artifactDigest: string;
118
+ runtimeName: string;
119
+ runtimeVersion: string;
120
+ sdkVersion: string;
121
+ manifestVersion: number;
122
+ protocolVersion: number;
123
+ manifestDigest: string;
124
+ workflows: Record<string, {
125
+ workflowVersion: string;
126
+ workflowRevision: string;
127
+ contractHash: string;
128
+ activities: Record<string, {
129
+ contractHash: string;
130
+ implementationRevision: string;
131
+ }>;
132
+ }>;
133
+ };
134
+ } | null;
135
+ }, unknown>;
136
+ /** Default discovery-only operation; live servers use `createGetWorkerDiagnosticsOperation(...)`. */
137
+ export declare const getWorkerDiagnosticsOperation: import("../operation-catalog.ts").OperationDefinition<{
138
+ workerId: string;
139
+ }, {
140
+ worker: {
141
+ instance: {
142
+ workerId: string;
143
+ queue: string;
144
+ health: WorkerHealth;
145
+ connectedAt: number;
146
+ startedAt: number;
147
+ lastHeartbeatAt: number;
148
+ heartbeatAgeMs: number;
149
+ };
150
+ deploymentVersion: {
151
+ deploymentName: string;
152
+ buildId: string;
153
+ artifactDigest: string;
154
+ runtimeName: string;
155
+ runtimeVersion: string;
156
+ sdkVersion: string;
157
+ manifestVersion: number;
158
+ protocolVersion: number;
159
+ manifestDigest: string;
160
+ workflows: Record<string, {
161
+ workflowVersion: string;
162
+ workflowRevision: string;
163
+ contractHash: string;
164
+ activities: Record<string, {
165
+ contractHash: string;
166
+ implementationRevision: string;
167
+ }>;
168
+ }>;
169
+ };
170
+ } | null;
171
+ }, unknown>;
172
+ /**
173
+ * Build the REST binding for `weft.workers.diagnostics`. The binding is
174
+ * metadata only; the live `WorkerRegistry` is wired into the operation.
175
+ */
176
+ export declare function createGetWorkerDiagnosticsRestBinding(): UnknownRestBinding;
177
+ export {};
@@ -0,0 +1,117 @@
1
+ import { z } from "zod";
2
+ import { shapeOperationFaultAsJson } from "../operation-fault.js";
3
+ import { defineOperation } from "../operation-registry.js";
4
+ const workerHealthSchema = z.enum(["active", "draining", "drained"]), workerActivityContractSchema = z.object({
5
+ contractHash: z.string(),
6
+ implementationRevision: z.string()
7
+ }).strict(), workerWorkflowContractSummarySchema = z.object({
8
+ workflowVersion: z.string(),
9
+ workflowRevision: z.string(),
10
+ contractHash: z.string(),
11
+ activities: z.record(z.string(), workerActivityContractSchema)
12
+ }).strict(), workerInstanceIdentitySchema = z.object({
13
+ workerId: z.string(),
14
+ queue: z.string(),
15
+ health: workerHealthSchema,
16
+ connectedAt: z.number(),
17
+ startedAt: z.number(),
18
+ lastHeartbeatAt: z.number(),
19
+ heartbeatAgeMs: z.number()
20
+ }).strict(), workerDeploymentVersionIdentitySchema = z.object({
21
+ deploymentName: z.string(),
22
+ buildId: z.string(),
23
+ artifactDigest: z.string(),
24
+ runtimeName: z.string(),
25
+ runtimeVersion: z.string(),
26
+ sdkVersion: z.string(),
27
+ manifestVersion: z.number(),
28
+ protocolVersion: z.number(),
29
+ manifestDigest: z.string(),
30
+ workflows: z.record(z.string(), workerWorkflowContractSummarySchema)
31
+ }).strict(), workerDiagnosticsEntrySchema = z.object({
32
+ instance: workerInstanceIdentitySchema,
33
+ deploymentVersion: workerDeploymentVersionIdentitySchema
34
+ }).strict(), getWorkerDiagnosticsInput = z.object({
35
+ workerId: z.string().min(1)
36
+ }), getWorkerDiagnosticsOutput = z.object({
37
+ worker: workerDiagnosticsEntrySchema.nullable()
38
+ });
39
+ function projectWorkerDiagnostics(registry, workerId, now) {
40
+ const worker = registry.getWorker(workerId);
41
+ if (worker === void 0)
42
+ return null;
43
+ const summary = registry.getWorkerSummaries(now).find((entry) => entry.id === workerId);
44
+ if (summary === void 0)
45
+ return null;
46
+ const workflows = {};
47
+ for (const [workflowType, contract] of Object.entries(worker.manifest.workflows))
48
+ workflows[workflowType] = {
49
+ workflowVersion: contract.workflowVersion,
50
+ workflowRevision: contract.workflowRevision,
51
+ contractHash: contract.contractHash,
52
+ activities: { ...contract.activities }
53
+ };
54
+ return {
55
+ instance: {
56
+ workerId: worker.id,
57
+ queue: worker.queue,
58
+ health: summary.health,
59
+ connectedAt: worker.connectedAt,
60
+ startedAt: worker.startedAt,
61
+ lastHeartbeatAt: worker.lastHeartbeat,
62
+ heartbeatAgeMs: summary.heartbeatAgeMs
63
+ },
64
+ deploymentVersion: {
65
+ deploymentName: worker.manifest.deployment.name,
66
+ buildId: worker.manifest.deployment.buildId,
67
+ artifactDigest: worker.manifest.deployment.artifactDigest,
68
+ runtimeName: worker.manifest.runtime.name,
69
+ runtimeVersion: worker.manifest.runtime.version,
70
+ sdkVersion: worker.manifest.sdkVersion,
71
+ manifestVersion: worker.manifest.manifestVersion,
72
+ protocolVersion: worker.manifest.protocolVersion,
73
+ manifestDigest: worker.acceptedManifestDigest,
74
+ workflows
75
+ }
76
+ };
77
+ }
78
+ export function createGetWorkerDiagnosticsOperation(options = {}) {
79
+ const registry = options.workerRegistry, clock = options.clock ?? Date.now;
80
+ return defineOperation({
81
+ name: "weft.workers.diagnostics",
82
+ mcpExposable: !1,
83
+ summary: "Get bounded instance and deployment-version diagnostics for one connected worker",
84
+ description: "Report one connected worker's identity split into two parts: `instance` (this live " + "connection \u2014 queue, health, connection timestamps) and `deploymentVersion` (the " + "artifact it loaded \u2014 SDK/runtime identity, manifest digest, and every advertised " + "workflow's version, revision, and contract hash). Returns `worker: null` when the workerId is not currently connected. Excludes capabilities and raw payload schemas.",
85
+ destructive: !1,
86
+ tags: ["Observability"],
87
+ inputSchema: getWorkerDiagnosticsInput,
88
+ outputSchema: getWorkerDiagnosticsOutput,
89
+ access: { kind: "scoped", scopes: { kind: "anyOf", scopes: ["system:read"] } },
90
+ producibleFaults: [],
91
+ discoverable: !0,
92
+ transports: { http: !0, jsonRpcHttp: !0, jsonRpcWebSocket: !0, jsonRpcStdio: !0 },
93
+ unknownKeyPolicy: { http: "strip", jsonRpc: "reject" },
94
+ invoke: async ({ input }) => {
95
+ if (registry === void 0)
96
+ throw Error("weft.workers.diagnostics invoked from a discovery-only operation registry; no WorkerRegistry was wired in");
97
+ return { worker: projectWorkerDiagnostics(registry, input.workerId, clock()) };
98
+ }
99
+ });
100
+ }
101
+ export const getWorkerDiagnosticsOperation = createGetWorkerDiagnosticsOperation();
102
+ export function createGetWorkerDiagnosticsRestBinding() {
103
+ return {
104
+ method: "GET",
105
+ path: "/v1/workers/:workerId/diagnostics",
106
+ pathParamNames: ["workerId"],
107
+ operationName: "weft.workers.diagnostics",
108
+ inputSources: {
109
+ workerId: { kind: "path", pathParam: "workerId" }
110
+ },
111
+ extractInput: async (_request, pathParams) => ({
112
+ workerId: pathParams.workerId ?? ""
113
+ }),
114
+ success: { kind: "json", status: 200 },
115
+ shapeFault: shapeOperationFaultAsJson
116
+ };
117
+ }
@@ -0,0 +1,79 @@
1
+ /**
2
+ * `weft.workers.rejections` operation + REST binding.
3
+ *
4
+ * Bounded, auditable log of recently declined `register` attempts (WFT-29):
5
+ * the rejection code, the attempted workerId when known, and whatever
6
+ * deployment/queue identity had already been parsed by the time the gate
7
+ * that rejected it ran. No free-text rejection message and no manifest
8
+ * content — this surfaces "who got rejected, when, and why (by code)", not
9
+ * a diagnostic dump.
10
+ *
11
+ * @module server/operations/list-worker-registration-rejections
12
+ */
13
+ import { z } from 'zod';
14
+ import type { WorkerRegistry } from '../../worker/registry.ts';
15
+ import type { UnknownRestBinding } from '../rest-bindings.ts';
16
+ declare const listWorkerRegistrationRejectionsInput: z.ZodObject<{
17
+ limit: z.ZodDefault<z.ZodNumber>;
18
+ }, z.core.$strip>;
19
+ declare const listWorkerRegistrationRejectionsOutput: z.ZodObject<{
20
+ items: z.ZodArray<z.ZodObject<{
21
+ code: z.ZodEnum<{
22
+ invalid_registration: "invalid_registration";
23
+ unsupported_protocol_version: "unsupported_protocol_version";
24
+ deployment_conflict: "deployment_conflict";
25
+ registration_rejected: "registration_rejected";
26
+ }>;
27
+ workerId: z.ZodOptional<z.ZodString>;
28
+ rejectedAt: z.ZodNumber;
29
+ queue: z.ZodOptional<z.ZodString>;
30
+ deploymentName: z.ZodOptional<z.ZodString>;
31
+ buildId: z.ZodOptional<z.ZodString>;
32
+ }, z.core.$strict>>;
33
+ limit: z.ZodNumber;
34
+ }, z.core.$strip>;
35
+ export type ListWorkerRegistrationRejectionsInput = z.infer<typeof listWorkerRegistrationRejectionsInput>;
36
+ export type ListWorkerRegistrationRejectionsOutput = z.infer<typeof listWorkerRegistrationRejectionsOutput>;
37
+ type ListWorkerRegistrationRejectionsOptions = {
38
+ workerRegistry?: WorkerRegistry;
39
+ };
40
+ /**
41
+ * Build the `weft.workers.rejections` operation, optionally bound to a live
42
+ * `WorkerRegistry`. Mirrors `createListWorkersOperation`'s discovery-only
43
+ * fallback: when `workerRegistry` is omitted, `invoke` throws if reached —
44
+ * reserved for discovery-only registries (OpenAPI/AsyncAPI).
45
+ */
46
+ export declare function createListWorkerRegistrationRejectionsOperation(options?: ListWorkerRegistrationRejectionsOptions): import("../operation-catalog.ts").OperationDefinition<{
47
+ limit: number;
48
+ }, {
49
+ items: {
50
+ code: "invalid_registration" | "unsupported_protocol_version" | "deployment_conflict" | "registration_rejected";
51
+ rejectedAt: number;
52
+ workerId?: string | undefined;
53
+ queue?: string | undefined;
54
+ deploymentName?: string | undefined;
55
+ buildId?: string | undefined;
56
+ }[];
57
+ limit: number;
58
+ }, unknown>;
59
+ /** Default discovery-only operation; live servers use `createListWorkerRegistrationRejectionsOperation(...)`. */
60
+ export declare const listWorkerRegistrationRejectionsOperation: import("../operation-catalog.ts").OperationDefinition<{
61
+ limit: number;
62
+ }, {
63
+ items: {
64
+ code: "invalid_registration" | "unsupported_protocol_version" | "deployment_conflict" | "registration_rejected";
65
+ rejectedAt: number;
66
+ workerId?: string | undefined;
67
+ queue?: string | undefined;
68
+ deploymentName?: string | undefined;
69
+ buildId?: string | undefined;
70
+ }[];
71
+ limit: number;
72
+ }, unknown>;
73
+ /**
74
+ * Build the REST binding for `weft.workers.rejections`. The
75
+ * binding is metadata only; the live `WorkerRegistry` is wired into the
76
+ * operation, not the binding.
77
+ */
78
+ export declare function createListWorkerRegistrationRejectionsRestBinding(): UnknownRestBinding;
79
+ export {};