@lostgradient/weft 0.17.0 → 0.19.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (172) hide show
  1. package/README.md +3 -1
  2. package/dist/cli/conformance.js +14 -18
  3. package/dist/cli/generated/operation-catalog.snapshot.json +331 -16
  4. package/dist/cli/generated/operation-client.generated.d.ts +39 -6
  5. package/dist/cli/generated/operation-client.generated.js +6 -0
  6. package/dist/core/context/child-workflow-pipe.d.ts +1 -1
  7. package/dist/core/context/child-workflow-pipe.js +1 -1
  8. package/dist/core/engine/construction.d.ts +2 -0
  9. package/dist/core/engine/construction.js +3 -0
  10. package/dist/core/engine/index.js +1 -0
  11. package/dist/core/engine/listing.d.ts +1 -0
  12. package/dist/core/engine/listing.js +6 -4
  13. package/dist/core/weft-error.d.ts +1 -1
  14. package/dist/core/weft-error.js +2 -1
  15. package/dist/core/worker-execution-dispatcher.d.ts +8 -0
  16. package/dist/core/worker-execution-dispatcher.js +4 -0
  17. package/dist/core/worker-execution-strategy-options.d.ts +16 -0
  18. package/dist/core/worker-execution-strategy.js +71 -89
  19. package/dist/core/worker-fault-handling.d.ts +45 -0
  20. package/dist/core/worker-fault-handling.js +88 -0
  21. package/dist/core/worker-protocol.d.ts +18 -1
  22. package/dist/core/worker-protocol.js +1 -1
  23. package/dist/core/worker-realm-readiness.d.ts +78 -0
  24. package/dist/core/worker-realm-readiness.js +127 -0
  25. package/dist/index.d.ts +3 -1
  26. package/dist/index.js +28 -1
  27. package/dist/json-schema.js +1 -1
  28. package/dist/runtime/portable.d.ts +14 -0
  29. package/dist/runtime/portable.js +10 -0
  30. package/dist/server/authorization-scope.d.ts +32 -2
  31. package/dist/server/index.d.ts +73 -8
  32. package/dist/server/index.js +11 -3
  33. package/dist/server/operations/get-principal.d.ts +75 -0
  34. package/dist/server/operations/get-principal.js +41 -0
  35. package/dist/server/operations/get-task-diagnostics.d.ts +43 -18
  36. package/dist/server/operations/get-task-diagnostics.js +87 -126
  37. package/dist/server/operations/get-worker-diagnostics.d.ts +177 -0
  38. package/dist/server/operations/get-worker-diagnostics.js +117 -0
  39. package/dist/server/operations/list-worker-registration-rejections.d.ts +79 -0
  40. package/dist/server/operations/list-worker-registration-rejections.js +60 -0
  41. package/dist/server/operations/list-workers.js +1 -3
  42. package/dist/server/operations/static-registrations.d.ts +28 -0
  43. package/dist/server/operations/static-registrations.js +248 -0
  44. package/dist/server/rest-bindings.d.ts +5 -0
  45. package/dist/server/rest-bindings.js +34 -313
  46. package/dist/server/runtime/authentication-bridge.js +6 -14
  47. package/dist/server/runtime/context.d.ts +23 -0
  48. package/dist/server/runtime/task-dispatch.d.ts +7 -1
  49. package/dist/server/runtime/task-dispatch.js +126 -55
  50. package/dist/server/runtime/task-ledger-completion.d.ts +68 -0
  51. package/dist/server/runtime/task-ledger-completion.js +72 -0
  52. package/dist/server/runtime/task-ledger-recovery.d.ts +55 -0
  53. package/dist/server/runtime/task-ledger-recovery.js +73 -0
  54. package/dist/server/runtime/task-ledger-runtime.d.ts +64 -0
  55. package/dist/server/runtime/task-ledger-runtime.js +34 -0
  56. package/dist/server/runtime/task-metrics.d.ts +6 -4
  57. package/dist/server/runtime/task-polling.d.ts +14 -12
  58. package/dist/server/runtime/task-polling.js +106 -82
  59. package/dist/server/runtime/task-reconciliation.d.ts +35 -8
  60. package/dist/server/runtime/task-reconciliation.js +75 -74
  61. package/dist/server/runtime/task-result-resolution.d.ts +2 -10
  62. package/dist/server/runtime/task-result-resolution.js +0 -67
  63. package/dist/server/runtime/task-result-view.d.ts +82 -0
  64. package/dist/server/runtime/task-result-view.js +48 -0
  65. package/dist/server/runtime/websocket-worker-messaging.d.ts +16 -0
  66. package/dist/server/runtime/websocket-worker-messaging.js +32 -0
  67. package/dist/server/runtime/websocket-worker-registration.d.ts +41 -0
  68. package/dist/server/runtime/websocket-worker-registration.js +134 -0
  69. package/dist/server/runtime/websocket-worker.d.ts +0 -1
  70. package/dist/server/runtime/websocket-worker.js +64 -116
  71. package/dist/server/serve-internals.d.ts +9 -6
  72. package/dist/server/serve-internals.js +14 -40
  73. package/dist/server/task-ledger-codec.d.ts +40 -0
  74. package/dist/server/task-ledger-codec.js +195 -0
  75. package/dist/server/task-ledger-keys.d.ts +13 -0
  76. package/dist/server/task-ledger-keys.js +5 -0
  77. package/dist/server/task-ledger-limits.d.ts +34 -0
  78. package/dist/server/task-ledger-limits.js +5 -0
  79. package/dist/server/task-ledger-transition-helpers.d.ts +38 -0
  80. package/dist/server/task-ledger-transition-helpers.js +42 -0
  81. package/dist/server/task-ledger-transitions-cancellation.d.ts +31 -0
  82. package/dist/server/task-ledger-transitions-cancellation.js +73 -0
  83. package/dist/server/task-ledger-transitions.d.ts +129 -0
  84. package/dist/server/task-ledger-transitions.js +162 -0
  85. package/dist/server/task-ledger-types.d.ts +220 -0
  86. package/dist/server/task-ledger-types.js +1 -0
  87. package/dist/server/task-ledger.d.ts +17 -0
  88. package/dist/server/task-ledger.js +24 -0
  89. package/dist/server/task-state.d.ts +25 -152
  90. package/dist/server/task-state.js +0 -188
  91. package/dist/server/worker-admission-policy.d.ts +70 -0
  92. package/dist/server/worker-admission-policy.js +0 -0
  93. package/dist/storage/compressed-storage.js +1 -1
  94. package/dist/storage/index.d.ts +0 -2
  95. package/dist/storage/interface.d.ts +0 -2
  96. package/dist/storage/interface.js +1 -1
  97. package/dist/storage/lmdb.js +1 -1
  98. package/dist/storage/memory.js +1 -1
  99. package/dist/storage/neon.js +1 -1
  100. package/dist/storage/postgres.js +1 -1
  101. package/dist/storage/resolve.js +1 -1
  102. package/dist/storage/scoped-storage.js +1 -1
  103. package/dist/storage/testing.js +1 -1
  104. package/dist/storage/turso.js +1 -1
  105. package/dist/storage/web-extension.js +2 -2
  106. package/dist/version.d.ts +1 -1
  107. package/dist/version.js +1 -1
  108. package/dist/web-extension.js +1 -1
  109. package/dist/worker/index.d.ts +2 -0
  110. package/dist/worker/index.js +30 -4
  111. package/dist/worker/manifest/canonical-json.d.ts +20 -0
  112. package/dist/worker/manifest/canonical-json.js +13 -0
  113. package/dist/worker/manifest/capabilities.d.ts +24 -0
  114. package/dist/worker/manifest/capabilities.js +74 -0
  115. package/dist/worker/manifest/content-digest.d.ts +20 -0
  116. package/dist/worker/manifest/content-digest.js +13 -0
  117. package/dist/worker/manifest/declared-shape-digest.d.ts +14 -0
  118. package/dist/worker/manifest/declared-shape-digest.js +4 -0
  119. package/dist/worker/manifest/digest.d.ts +82 -0
  120. package/dist/worker/manifest/digest.js +9 -0
  121. package/dist/worker/manifest/execution-identity.d.ts +92 -0
  122. package/dist/worker/manifest/execution-identity.js +36 -0
  123. package/dist/worker/manifest/failure.d.ts +59 -0
  124. package/dist/worker/manifest/failure.js +4 -0
  125. package/dist/worker/manifest/index.d.ts +22 -0
  126. package/dist/worker/manifest/index.js +23 -0
  127. package/dist/worker/manifest/internal-realm.d.ts +50 -0
  128. package/dist/worker/manifest/internal-realm.js +39 -0
  129. package/dist/worker/manifest/is-record.d.ts +12 -0
  130. package/dist/worker/manifest/is-record.js +6 -0
  131. package/dist/worker/manifest/json-scan.d.ts +26 -0
  132. package/dist/worker/manifest/json-scan.js +88 -0
  133. package/dist/worker/manifest/limits.d.ts +138 -0
  134. package/dist/worker/manifest/limits.js +1 -0
  135. package/dist/worker/manifest/normalize.d.ts +63 -0
  136. package/dist/worker/manifest/normalize.js +73 -0
  137. package/dist/worker/manifest/parse-json.d.ts +24 -0
  138. package/dist/worker/manifest/parse-json.js +15 -0
  139. package/dist/worker/manifest/parse.d.ts +86 -0
  140. package/dist/worker/manifest/parse.js +188 -0
  141. package/dist/worker/manifest/registry-contract-builder.d.ts +120 -0
  142. package/dist/worker/manifest/registry-contract-builder.js +65 -0
  143. package/dist/worker/manifest/types.d.ts +263 -0
  144. package/dist/worker/manifest/types.js +1 -0
  145. package/dist/worker/manifest/utf8.d.ts +13 -0
  146. package/dist/worker/manifest/utf8.js +7 -0
  147. package/dist/worker/options.d.ts +55 -8
  148. package/dist/worker/options.js +65 -9
  149. package/dist/worker/protocol-internals.d.ts +1 -2
  150. package/dist/worker/protocol-internals.js +0 -5
  151. package/dist/worker/protocol-messages.d.ts +33 -17
  152. package/dist/worker/protocol-schemas.d.ts +36 -74
  153. package/dist/worker/protocol-schemas.js +25 -18
  154. package/dist/worker/protocol-version.d.ts +18 -14
  155. package/dist/worker/protocol-version.js +1 -1
  156. package/dist/worker/protocol.js +1 -1
  157. package/dist/worker/registry/deployment-consistency.d.ts +78 -0
  158. package/dist/worker/registry/deployment-consistency.js +27 -0
  159. package/dist/worker/registry/drain.d.ts +14 -0
  160. package/dist/worker/registry/drain.js +24 -0
  161. package/dist/worker/registry/rejections.d.ts +31 -0
  162. package/dist/worker/registry/rejections.js +10 -0
  163. package/dist/worker/registry/summary.d.ts +0 -3
  164. package/dist/worker/registry/summary.js +2 -5
  165. package/dist/worker/registry/types.d.ts +10 -3
  166. package/dist/worker/registry.d.ts +47 -2
  167. package/dist/worker/registry.js +45 -32
  168. package/dist/workers/workflow-worker-entry.d.ts +12 -7
  169. package/dist/workers/workflow-worker-entry.js +9 -2
  170. package/package.json +3 -3
  171. package/dist/server/task-resolved-record.d.ts +0 -12
  172. package/dist/server/task-resolved-record.js +0 -35
@@ -0,0 +1,75 @@
1
+ /**
2
+ * `weft.system.principal` operation + REST binding.
3
+ *
4
+ * Reports the caller's own resolved principal: the authentication method,
5
+ * the normalized subject, and the granted scope set. Public access by
6
+ * design — introspection has to work for every credential state to be
7
+ * useful (a dashboard probes its principal before it knows anything), and
8
+ * an unauthenticated caller learns only what it already knows: that it is
9
+ * anonymous with no scopes. The response is strictly principal-shaped —
10
+ * nothing about the server's auth configuration (key inventory,
11
+ * `unauthenticatedAccess` posture, JWT issuers) is exposed.
12
+ */
13
+ import { z } from 'zod';
14
+ import type { UnknownRestBinding } from '../rest-bindings.ts';
15
+ declare const getPrincipalInput: z.ZodObject<{}, z.core.$strip>;
16
+ declare const getPrincipalOutput: z.ZodObject<{
17
+ method: z.ZodEnum<{
18
+ jwt: "jwt";
19
+ "api-key": "api-key";
20
+ mtls: "mtls";
21
+ "stdio-local": "stdio-local";
22
+ unauthenticated: "unauthenticated";
23
+ }>;
24
+ subject: z.ZodNullable<z.ZodString>;
25
+ scopes: z.ZodArray<z.ZodEnum<{
26
+ "workflows:read": "workflows:read";
27
+ "workflows:write": "workflows:write";
28
+ "workflows:admin": "workflows:admin";
29
+ "schedules:read": "schedules:read";
30
+ "schedules:write": "schedules:write";
31
+ "signals:write": "signals:write";
32
+ "updates:write": "updates:write";
33
+ "queries:read": "queries:read";
34
+ "reviews:read": "reviews:read";
35
+ "reviews:write": "reviews:write";
36
+ "attributes:read": "attributes:read";
37
+ "attributes:write": "attributes:write";
38
+ "tags:write": "tags:write";
39
+ "streams:read": "streams:read";
40
+ "events:read": "events:read";
41
+ "storage:read": "storage:read";
42
+ "storage:write": "storage:write";
43
+ "storage:admin": "storage:admin";
44
+ "workers:write": "workers:write";
45
+ "system:read": "system:read";
46
+ "system:admin": "system:admin";
47
+ }>>;
48
+ }, z.core.$strip>;
49
+ export type GetPrincipalInput = z.infer<typeof getPrincipalInput>;
50
+ /**
51
+ * Output of `weft.system.principal`: the caller's authentication method,
52
+ * normalized subject (`null` when the credential carries none), and granted
53
+ * scopes, sorted. Anonymous callers receive `method: 'unauthenticated'`
54
+ * with an empty scope list.
55
+ *
56
+ * @example
57
+ * ```ts
58
+ * import { type GetPrincipalOutput } from '@lostgradient/weft/server';
59
+ *
60
+ * const anonymous: GetPrincipalOutput = {
61
+ * method: 'unauthenticated',
62
+ * subject: null,
63
+ * scopes: [],
64
+ * };
65
+ * console.log(anonymous.method);
66
+ * ```
67
+ */
68
+ export type GetPrincipalOutput = z.infer<typeof getPrincipalOutput>;
69
+ export declare const getPrincipalOperation: import("../operation-catalog.ts").OperationDefinition<Record<string, never>, {
70
+ method: "jwt" | "api-key" | "mtls" | "stdio-local" | "unauthenticated";
71
+ subject: string | null;
72
+ scopes: ("workflows:read" | "workflows:write" | "workflows:admin" | "schedules:read" | "schedules:write" | "signals:write" | "updates:write" | "queries:read" | "reviews:read" | "reviews:write" | "attributes:read" | "attributes:write" | "tags:write" | "streams:read" | "events:read" | "storage:read" | "storage:write" | "storage:admin" | "workers:write" | "system:read" | "system:admin")[];
73
+ }, unknown>;
74
+ export declare const getPrincipalRestBinding: UnknownRestBinding;
75
+ export {};
@@ -0,0 +1,41 @@
1
+ import { z } from "zod";
2
+ import { AUTHORIZATION_SCOPES } from "../authorization-scope.js";
3
+ import { shapeOperationFaultAsJson } from "../operation-fault.js";
4
+ import { defineOperation } from "../operation-registry.js";
5
+ const principalMethodSchema = z.enum(["jwt", "api-key", "mtls", "stdio-local", "unauthenticated"]), getPrincipalInput = z.object({}), getPrincipalOutput = z.object({
6
+ method: principalMethodSchema,
7
+ subject: z.string().nullable(),
8
+ scopes: z.array(z.enum(AUTHORIZATION_SCOPES))
9
+ });
10
+ export const getPrincipalOperation = defineOperation({
11
+ name: "weft.system.principal",
12
+ mcpExposable: !1,
13
+ summary: "Report the caller's resolved principal and granted scopes",
14
+ description: `Report the caller's own resolved principal: the authentication method ('jwt', 'api-key', 'mtls', 'stdio-local', or 'unauthenticated'), the normalized subject when one exists, and the granted authorization scopes, sorted. Public access: an unauthenticated caller receives method 'unauthenticated' with an empty scope list rather than an error, so clients can distinguish "no credential" from "credential with few scopes" without probing ` + "other operations. Reflects only the request credential \u2014 never server auth configuration.",
15
+ destructive: !1,
16
+ tags: ["System"],
17
+ inputSchema: getPrincipalInput,
18
+ outputSchema: getPrincipalOutput,
19
+ access: { kind: "public" },
20
+ discoverable: !0,
21
+ transports: { http: !0, jsonRpcHttp: !0, jsonRpcWebSocket: !0, jsonRpcStdio: !0 },
22
+ unknownKeyPolicy: { http: "strip", jsonRpc: "reject" },
23
+ invoke: async ({ principal }) => {
24
+ if (principal.method === "unauthenticated")
25
+ return { method: "unauthenticated", subject: null, scopes: [] };
26
+ return {
27
+ method: principal.method,
28
+ subject: principal.subject ?? null,
29
+ scopes: [...principal.scopes].toSorted()
30
+ };
31
+ }
32
+ }), getPrincipalRestBinding = {
33
+ method: "GET",
34
+ path: "/v1/principal",
35
+ pathParamNames: [],
36
+ operationName: "weft.system.principal",
37
+ inputSources: {},
38
+ extractInput: async () => ({}),
39
+ success: { kind: "json", status: 200 },
40
+ shapeFault: shapeOperationFaultAsJson
41
+ };
@@ -1,10 +1,41 @@
1
1
  /**
2
2
  * `weft.tasks.diagnostics` operation + REST binding.
3
3
  *
4
- * Scans durable task records and live worker state to identify queue pressure,
5
- * stale in-flight work, retry storms, and worker capacity saturation. Results
6
- * are intentionally bounded and low-cardinality so operators can use them in a
7
- * dashboard without turning workflow or worker identifiers into metrics labels.
4
+ * Scans the durable task ledger (WFT-24 previously the retired
5
+ * `op:queued:`/`op:inflight:`/`op:resolved:`/`op:dead-letter:` keys, which
6
+ * nothing has written since the WFT-22 ledger cutover) and live worker state
7
+ * to identify queue pressure, stale in-flight work, retry storms, dead
8
+ * letters, and worker capacity saturation. Results are intentionally bounded
9
+ * and low-cardinality so operators can use them in a dashboard without
10
+ * turning workflow or worker identifiers into metrics labels.
11
+ *
12
+ * The output schema's field names and `kind`/`state` vocabulary are
13
+ * unchanged from the pre-ledger version — only the internal scan and
14
+ * classification logic changed — so existing dashboards built against this
15
+ * endpoint's shape keep working. `leased`, `completing`, and `cancelling`
16
+ * ledger states are all reported as diagnostic `state: 'inflight'`: they
17
+ * share the same lease-holder/heartbeat shape and the same operator
18
+ * question ("is a worker still making progress on this attempt?"), so
19
+ * exposing the ledger's more granular internal states here would add
20
+ * distinctions operators cannot act on differently.
21
+ *
22
+ * Retry-storm detection (`kind: 'retry-storm'`) no longer covers `terminal`
23
+ * records: `RemoteTaskTerminal` carries no `retryCount`/`requeueCount` (WFT-25
24
+ * deliberately dropped attempt-count history once a task resolves), so
25
+ * there is nothing left to detect a storm from once an attempt reaches a
26
+ * disposition. The `resolved` value in the `state` enum is kept in the
27
+ * output schema for backward compatibility but nothing produces it anymore.
28
+ * `dead-lettered` diagnostics are unaffected — that is a distinct, separate
29
+ * `kind`.
30
+ *
31
+ * Unlike the pre-ledger scan, the full ledger scan this operation performs
32
+ * is one combined keyspace across every state (queued through terminal),
33
+ * not separate per-state prefixes — there is no time-bounded history index
34
+ * to limit how many terminal records get walked past on the way to
35
+ * classifying the ones that still matter. Operators who dispatch high
36
+ * volumes of tasks and want this scan to stay cheap should set
37
+ * {@link ServeOptions.taskRetentionWindowMs} so adopted terminal records
38
+ * are reaped rather than accumulating indefinitely.
8
39
  *
9
40
  * @module server/operations/get-task-diagnostics
10
41
  */
@@ -30,8 +61,8 @@ declare const taskDiagnosticItemSchema: z.ZodObject<{
30
61
  state: z.ZodEnum<{
31
62
  "dead-lettered": "dead-lettered";
32
63
  queued: "queued";
33
- inflight: "inflight";
34
64
  resolved: "resolved";
65
+ inflight: "inflight";
35
66
  capacity: "capacity";
36
67
  }>;
37
68
  operationId: z.ZodOptional<z.ZodString>;
@@ -44,10 +75,7 @@ declare const taskDiagnosticItemSchema: z.ZodObject<{
44
75
  queueLatencyMs: z.ZodOptional<z.ZodNumber>;
45
76
  executionLatencyMs: z.ZodOptional<z.ZodNumber>;
46
77
  heartbeatAgeMs: z.ZodOptional<z.ZodNumber>;
47
- lastRequeueReason: z.ZodOptional<z.ZodEnum<{
48
- "visibility-timeout": "visibility-timeout";
49
- "worker-disconnect": "worker-disconnect";
50
- }>>;
78
+ lastRequeueReason: z.ZodOptional<z.ZodString>;
51
79
  resolutionReason: z.ZodOptional<z.ZodString>;
52
80
  deadLetteredAt: z.ZodOptional<z.ZodNumber>;
53
81
  deadLetterReason: z.ZodOptional<z.ZodLiteral<"result-resolution-storage-exhausted">>;
@@ -74,8 +102,8 @@ declare const getTaskDiagnosticsOutput: z.ZodObject<{
74
102
  state: z.ZodEnum<{
75
103
  "dead-lettered": "dead-lettered";
76
104
  queued: "queued";
77
- inflight: "inflight";
78
105
  resolved: "resolved";
106
+ inflight: "inflight";
79
107
  capacity: "capacity";
80
108
  }>;
81
109
  operationId: z.ZodOptional<z.ZodString>;
@@ -88,10 +116,7 @@ declare const getTaskDiagnosticsOutput: z.ZodObject<{
88
116
  queueLatencyMs: z.ZodOptional<z.ZodNumber>;
89
117
  executionLatencyMs: z.ZodOptional<z.ZodNumber>;
90
118
  heartbeatAgeMs: z.ZodOptional<z.ZodNumber>;
91
- lastRequeueReason: z.ZodOptional<z.ZodEnum<{
92
- "visibility-timeout": "visibility-timeout";
93
- "worker-disconnect": "worker-disconnect";
94
- }>>;
119
+ lastRequeueReason: z.ZodOptional<z.ZodString>;
95
120
  resolutionReason: z.ZodOptional<z.ZodString>;
96
121
  deadLetteredAt: z.ZodOptional<z.ZodNumber>;
97
122
  deadLetterReason: z.ZodOptional<z.ZodLiteral<"result-resolution-storage-exhausted">>;
@@ -146,7 +171,7 @@ export declare function createGetTaskDiagnosticsOperation(options?: GetTaskDiagn
146
171
  }, {
147
172
  items: {
148
173
  kind: "dead-lettered" | "stuck-queued" | "stale-inflight" | "retry-storm" | "all-workers-at-capacity";
149
- state: "dead-lettered" | "queued" | "inflight" | "resolved" | "capacity";
174
+ state: "dead-lettered" | "queued" | "resolved" | "inflight" | "capacity";
150
175
  retryCount: number;
151
176
  requeueCount: number;
152
177
  evidence: string[];
@@ -158,7 +183,7 @@ export declare function createGetTaskDiagnosticsOperation(options?: GetTaskDiagn
158
183
  queueLatencyMs?: number | undefined;
159
184
  executionLatencyMs?: number | undefined;
160
185
  heartbeatAgeMs?: number | undefined;
161
- lastRequeueReason?: "visibility-timeout" | "worker-disconnect" | undefined;
186
+ lastRequeueReason?: string | undefined;
162
187
  resolutionReason?: string | undefined;
163
188
  deadLetteredAt?: number | undefined;
164
189
  deadLetterReason?: "result-resolution-storage-exhausted" | undefined;
@@ -185,7 +210,7 @@ export declare const getTaskDiagnosticsOperation: import("../operation-catalog.t
185
210
  }, {
186
211
  items: {
187
212
  kind: "dead-lettered" | "stuck-queued" | "stale-inflight" | "retry-storm" | "all-workers-at-capacity";
188
- state: "dead-lettered" | "queued" | "inflight" | "resolved" | "capacity";
213
+ state: "dead-lettered" | "queued" | "resolved" | "inflight" | "capacity";
189
214
  retryCount: number;
190
215
  requeueCount: number;
191
216
  evidence: string[];
@@ -197,7 +222,7 @@ export declare const getTaskDiagnosticsOperation: import("../operation-catalog.t
197
222
  queueLatencyMs?: number | undefined;
198
223
  executionLatencyMs?: number | undefined;
199
224
  heartbeatAgeMs?: number | undefined;
200
- lastRequeueReason?: "visibility-timeout" | "worker-disconnect" | undefined;
225
+ lastRequeueReason?: string | undefined;
201
226
  resolutionReason?: string | undefined;
202
227
  deadLetteredAt?: number | undefined;
203
228
  deadLetterReason?: "result-resolution-storage-exhausted" | undefined;
@@ -1,18 +1,15 @@
1
1
  import { z } from "zod";
2
- import { decode } from "../../core/codec.js";
3
- import { KEYS } from "../../storage/interface.js";
2
+ import { raiseFault } from "../operation-catalog/raise-fault.js";
4
3
  import { defineOperation } from "../operation-registry.js";
4
+ import { commitTaskLedgerDelete } from "../runtime/task-ledger-runtime.js";
5
+ import { canClearDeadLetteredTask } from "../task-ledger-transitions.js";
6
+ import { decodeRemoteTaskRecord } from "../task-ledger.js";
5
7
  import {
6
8
  calculateExecutionLatencyMs,
7
9
  calculateHeartbeatAgeMs,
8
- calculateQueueLatencyMs,
9
- clearDeadLetteredTaskRecord,
10
- isDeadLetteredTaskRecord,
11
- isInflightRecord,
12
- isQueuedRecord,
13
- isResolvedRecord
10
+ calculateQueueLatencyMs
14
11
  } from "../task-state.js";
15
- const DEFAULT_STALE_QUEUED_AFTER_MS = 60000, DEFAULT_STALE_HEARTBEAT_AFTER_MS = 60000, DEFAULT_RETRY_STORM_MINIMUM_ATTEMPTS = 3, DEFAULT_LIMIT = 50, MAX_LIMIT = 200, RESOLVED_HISTORY_SCAN_LIMIT = 1000, taskDiagnosticKindSchema = z.enum([
12
+ const DEFAULT_STALE_QUEUED_AFTER_MS = 60000, DEFAULT_STALE_HEARTBEAT_AFTER_MS = 60000, DEFAULT_RETRY_STORM_MINIMUM_ATTEMPTS = 3, DEFAULT_LIMIT = 50, MAX_LIMIT = 200, taskDiagnosticKindSchema = z.enum([
16
13
  "stuck-queued",
17
14
  "stale-inflight",
18
15
  "retry-storm",
@@ -31,7 +28,7 @@ const DEFAULT_STALE_QUEUED_AFTER_MS = 60000, DEFAULT_STALE_HEARTBEAT_AFTER_MS =
31
28
  queueLatencyMs: z.number().nonnegative().optional(),
32
29
  executionLatencyMs: z.number().nonnegative().optional(),
33
30
  heartbeatAgeMs: z.number().nonnegative().optional(),
34
- lastRequeueReason: z.enum(["visibility-timeout", "worker-disconnect"]).optional(),
31
+ lastRequeueReason: z.string().optional(),
35
32
  resolutionReason: z.string().optional(),
36
33
  deadLetteredAt: z.number().nonnegative().optional(),
37
34
  deadLetterReason: z.literal("result-resolution-storage-exhausted").optional(),
@@ -105,12 +102,17 @@ export const getTaskDiagnosticsOperation = createGetTaskDiagnosticsOperation(),
105
102
  kind: "scoped",
106
103
  scopes: { kind: "anyOf", scopes: ["system:admin"] }
107
104
  },
108
- producibleFaults: [],
105
+ producibleFaults: ["NotFound"],
109
106
  discoverable: !0,
110
107
  transports: restOnlyTaskDiagnosticsTransports,
111
108
  unknownKeyPolicy: { http: "reject", jsonRpc: "reject" },
112
109
  invoke: async ({ input, engine }) => {
113
- await clearDeadLetteredTaskRecord(engine.storage, input.operationId);
110
+ if (!(await commitTaskLedgerDelete(engine.storage, input.operationId, canClearDeadLetteredTask, 1)).ok)
111
+ raiseFault(clearTaskDeadLetterOperation, {
112
+ code: "NotFound",
113
+ message: `No dead-lettered task found for operation "${input.operationId}"`,
114
+ data: { resource: "task", identifier: input.operationId }
115
+ });
114
116
  return { ok: !0 };
115
117
  }
116
118
  });
@@ -127,39 +129,19 @@ async function collectTaskDiagnostics({
127
129
  retryStorms: 0,
128
130
  allWorkersAtCapacity: 0,
129
131
  deadLettered: 0
130
- }, relevantQueues = new Set, deadLetteredOperationIds = new Set, addItem = (item) => {
132
+ }, relevantQueues = new Set, addItem = (item) => {
131
133
  incrementSummary(summary, item.kind);
132
134
  if (items.length < input.limit)
133
135
  items.push(item);
134
136
  };
135
- for await (const record of scanDeadLetterRecords(engine)) {
136
- if (!matchesTaskRecordFilter(record, input))
137
- continue;
138
- deadLetteredOperationIds.add(record.operationId);
139
- if (record.queue !== void 0)
140
- relevantQueues.add(record.queue);
141
- addDeadLetterDiagnostics(record, addItem);
142
- }
143
- for await (const record of scanQueuedRecords(engine)) {
144
- if (!matchesTaskRecordFilter(record, input))
137
+ for await (const [, value] of engine.storage.scan("task-ledger:")) {
138
+ const decoded = decodeRemoteTaskRecord(value);
139
+ if (decoded === null)
145
140
  continue;
146
- relevantQueues.add(record.queue);
147
- addQueuedDiagnostics(record, input, currentTime, addItem);
148
- }
149
- for await (const record of scanInflightRecords(engine)) {
150
- if (!matchesTaskRecordFilter(record, input))
151
- continue;
152
- if (deadLetteredOperationIds.has(record.operationId))
141
+ if (!matchesTaskRecordFilter(decoded, input))
153
142
  continue;
154
- relevantQueues.add(record.queue);
155
- addInflightDiagnostics(record, input, currentTime, addItem);
156
- }
157
- for await (const record of scanResolvedRecords(engine, input)) {
158
- if (!matchesTaskRecordFilter(record, input))
159
- continue;
160
- if (record.queue !== void 0)
161
- relevantQueues.add(record.queue);
162
- addRetryStormDiagnostic(record, "resolved", input, addItem);
143
+ relevantQueues.add(decoded.queue);
144
+ addRecordDiagnostics(decoded, input, currentTime, addItem);
163
145
  }
164
146
  addCapacityDiagnostics({
165
147
  registry,
@@ -170,88 +152,71 @@ async function collectTaskDiagnostics({
170
152
  });
171
153
  return { items, summary, limit: input.limit };
172
154
  }
173
- async function* scanDeadLetterRecords(engine) {
174
- for await (const [, value] of engine.storage.scan(KEYS.operationDeadLetterPrefix())) {
175
- const decoded = decode(value);
176
- if (isDeadLetteredTaskRecord(decoded))
177
- yield decoded;
178
- }
179
- }
180
- async function* scanQueuedRecords(engine) {
181
- for await (const [, value] of engine.storage.scan("op:queued:")) {
182
- const decoded = decode(value);
183
- if (isQueuedRecord(decoded))
184
- yield decoded;
185
- }
186
- }
187
- async function* scanInflightRecords(engine) {
188
- for await (const [, value] of engine.storage.scan("op:inflight:")) {
189
- const decoded = decode(value);
190
- if (isInflightRecord(decoded))
191
- yield decoded;
192
- }
193
- }
194
- async function* scanResolvedRecords(engine, input) {
195
- if (input.operationId !== void 0) {
196
- const value = await engine.storage.get(KEYS.operationResolved(input.operationId));
197
- if (value === null)
155
+ function addRecordDiagnostics(decoded, input, currentTime, addItem) {
156
+ switch (decoded.state) {
157
+ case "queued":
158
+ if (decoded.availableAt <= currentTime)
159
+ addQueuedDiagnostics(decoded, input, currentTime, addItem);
160
+ addRetryStormDiagnostic(decoded, "queued", input, addItem);
198
161
  return;
199
- const decoded = decode(value);
200
- if (isResolvedRecord(decoded))
201
- yield decoded;
202
- return;
203
- }
204
- for await (const [, value] of engine.storage.scan(KEYS.operationResolvedByTimePrefix(), {
205
- limit: RESOLVED_HISTORY_SCAN_LIMIT,
206
- reverse: !0
207
- })) {
208
- const decoded = decode(value);
209
- if (isResolvedRecord(decoded))
210
- yield decoded;
162
+ case "leased":
163
+ case "completing":
164
+ case "cancelling":
165
+ addInflightDiagnostics(decoded, input, currentTime, addItem);
166
+ addRetryStormDiagnostic(decoded, "inflight", input, addItem);
167
+ return;
168
+ case "terminal":
169
+ return;
170
+ case "deadLettered":
171
+ addDeadLetterDiagnostics(decoded, addItem);
172
+ return;
173
+ default: {
174
+ const exhaustive = decoded;
175
+ }
211
176
  }
212
177
  }
213
178
  function addQueuedDiagnostics(record, input, currentTime, addItem) {
214
- const queuedAt = record.lastQueuedAt ?? record.queuedAt, queueLatencyMs = Math.max(0, currentTime - queuedAt);
215
- if (queueLatencyMs >= input.staleQueuedAfterMs)
216
- addItem({
217
- kind: "stuck-queued",
218
- state: "queued",
219
- operationId: record.operationId,
220
- workflowId: record.workflowId,
221
- activityName: record.activityName,
222
- queue: record.queue,
223
- retryCount: record.retryCount ?? Math.max(0, (record.attempt ?? 1) - 1),
224
- requeueCount: record.requeueCount ?? 0,
225
- queueLatencyMs,
226
- lastRequeueReason: record.lastRequeueReason,
227
- evidence: [
228
- `Task has waited ${queueLatencyMs}ms in queue "${record.queue}" without a worker claim`
229
- ]
230
- });
231
- addRetryStormDiagnostic(record, "queued", input, addItem);
179
+ const queueLatencyMs = Math.max(0, currentTime - record.lastQueuedAt);
180
+ if (queueLatencyMs < input.staleQueuedAfterMs)
181
+ return;
182
+ addItem({
183
+ kind: "stuck-queued",
184
+ state: "queued",
185
+ operationId: record.operationId,
186
+ workflowId: record.workflowId,
187
+ activityName: record.activityName,
188
+ queue: record.queue,
189
+ retryCount: record.retryCount,
190
+ requeueCount: record.requeueCount,
191
+ queueLatencyMs,
192
+ lastRequeueReason: record.lastRequeueReason,
193
+ evidence: [
194
+ `Task has waited ${queueLatencyMs}ms in queue "${record.queue}" without a worker claim`
195
+ ]
196
+ });
232
197
  }
233
198
  function addInflightDiagnostics(record, input, currentTime, addItem) {
234
199
  const heartbeatAgeMs = calculateHeartbeatAgeMs(record, currentTime) ?? 0;
235
- if (heartbeatAgeMs >= input.staleHeartbeatAfterMs)
236
- addItem({
237
- kind: "stale-inflight",
238
- state: "inflight",
239
- operationId: record.operationId,
240
- workflowId: record.workflowId,
241
- activityName: record.activityName,
242
- queue: record.queue,
243
- workerId: record.workerId,
244
- retryCount: record.retryCount ?? Math.max(0, (record.attempt ?? 1) - 1),
245
- requeueCount: record.requeueCount ?? 0,
246
- queueLatencyMs: calculateQueueLatencyMs(record),
247
- executionLatencyMs: calculateExecutionLatencyMs(record, currentTime),
248
- heartbeatAgeMs,
249
- lastRequeueReason: record.lastRequeueReason,
250
- evidence: [
251
- `Worker "${record.workerId}" has not sent a heartbeat for ${heartbeatAgeMs}ms on queue "${record.queue}"`
252
- ]
253
- });
254
- addRetryStormDiagnostic(record, "inflight", input, addItem);
200
+ if (heartbeatAgeMs < input.staleHeartbeatAfterMs)
201
+ return;
202
+ addItem({
203
+ kind: "stale-inflight",
204
+ state: "inflight",
205
+ operationId: record.operationId,
206
+ workflowId: record.workflowId,
207
+ activityName: record.activityName,
208
+ queue: record.queue,
209
+ workerId: record.workerSessionId,
210
+ retryCount: record.retryCount,
211
+ requeueCount: record.requeueCount,
212
+ queueLatencyMs: calculateQueueLatencyMs(record),
213
+ executionLatencyMs: calculateExecutionLatencyMs(record, currentTime),
214
+ heartbeatAgeMs,
215
+ lastRequeueReason: record.lastRequeueReason,
216
+ evidence: [
217
+ `Worker "${record.workerSessionId}" has not sent a heartbeat for ${heartbeatAgeMs}ms on queue "${record.queue}"`
218
+ ]
219
+ });
255
220
  }
256
221
  function addDeadLetterDiagnostics(record, addItem) {
257
222
  addItem({
@@ -261,21 +226,19 @@ function addDeadLetterDiagnostics(record, addItem) {
261
226
  workflowId: record.workflowId,
262
227
  activityName: record.activityName,
263
228
  queue: record.queue,
264
- workerId: record.workerId,
265
- retryCount: record.retryCount ?? Math.max(0, (record.attempt ?? 1) - 1),
266
- requeueCount: record.requeueCount ?? 0,
229
+ retryCount: record.retryCount,
230
+ requeueCount: record.requeueCount,
267
231
  lastRequeueReason: record.lastRequeueReason,
268
232
  deadLetteredAt: record.deadLetteredAt,
269
- deadLetterReason: record.reason,
270
- storageError: record.errorMessage,
271
- retryAttempts: record.retryAttempts,
233
+ deadLetterReason: "result-resolution-storage-exhausted",
234
+ storageError: record.persistenceFailureReason,
272
235
  evidence: [
273
- `Task result transition exhausted ${record.retryAttempts} storage attempts; reconciliation will not re-dispatch operation "${record.operationId}" until the dead-letter entry is cleared`
236
+ `Task result could not be durably persisted (${record.persistenceFailureReason}); reconciliation will not re-dispatch operation "${record.operationId}" until the dead-letter entry is cleared`
274
237
  ]
275
238
  });
276
239
  }
277
240
  function addRetryStormDiagnostic(record, state, input, addItem) {
278
- const retryCount = record.retryCount ?? Math.max(0, (record.attempt ?? 1) - 1), requeueCount = record.requeueCount ?? 0;
241
+ const { retryCount, requeueCount } = record;
279
242
  if (retryCount < input.retryStormMinimumAttempts && requeueCount < input.retryStormMinimumAttempts)
280
243
  return;
281
244
  addItem({
@@ -283,15 +246,13 @@ function addRetryStormDiagnostic(record, state, input, addItem) {
283
246
  state,
284
247
  operationId: record.operationId,
285
248
  workflowId: record.workflowId,
286
- activityName: "activityName" in record ? record.activityName : void 0,
249
+ activityName: record.activityName,
287
250
  queue: record.queue,
288
- workerId: "workerId" in record ? record.workerId : void 0,
251
+ workerId: "workerSessionId" in record ? record.workerSessionId : void 0,
289
252
  retryCount,
290
253
  requeueCount,
291
254
  queueLatencyMs: calculateQueueLatencyMs(record),
292
- executionLatencyMs: "resolvedAt" in record ? calculateExecutionLatencyMs(record, record.resolvedAt) : void 0,
293
255
  lastRequeueReason: record.lastRequeueReason,
294
- resolutionReason: "resolutionReason" in record ? record.resolutionReason : void 0,
295
256
  evidence: [
296
257
  `Task has ${retryCount} retries and ${requeueCount} requeues, meeting retry storm threshold ${input.retryStormMinimumAttempts}`
297
258
  ]