@lostgradient/weft 0.17.0 → 0.19.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +3 -1
- package/dist/cli/conformance.js +14 -18
- package/dist/cli/generated/operation-catalog.snapshot.json +331 -16
- package/dist/cli/generated/operation-client.generated.d.ts +39 -6
- package/dist/cli/generated/operation-client.generated.js +6 -0
- package/dist/core/context/child-workflow-pipe.d.ts +1 -1
- package/dist/core/context/child-workflow-pipe.js +1 -1
- package/dist/core/engine/construction.d.ts +2 -0
- package/dist/core/engine/construction.js +3 -0
- package/dist/core/engine/index.js +1 -0
- package/dist/core/engine/listing.d.ts +1 -0
- package/dist/core/engine/listing.js +6 -4
- package/dist/core/weft-error.d.ts +1 -1
- package/dist/core/weft-error.js +2 -1
- package/dist/core/worker-execution-dispatcher.d.ts +8 -0
- package/dist/core/worker-execution-dispatcher.js +4 -0
- package/dist/core/worker-execution-strategy-options.d.ts +16 -0
- package/dist/core/worker-execution-strategy.js +71 -89
- package/dist/core/worker-fault-handling.d.ts +45 -0
- package/dist/core/worker-fault-handling.js +88 -0
- package/dist/core/worker-protocol.d.ts +18 -1
- package/dist/core/worker-protocol.js +1 -1
- package/dist/core/worker-realm-readiness.d.ts +78 -0
- package/dist/core/worker-realm-readiness.js +127 -0
- package/dist/index.d.ts +3 -1
- package/dist/index.js +28 -1
- package/dist/json-schema.js +1 -1
- package/dist/runtime/portable.d.ts +14 -0
- package/dist/runtime/portable.js +10 -0
- package/dist/server/authorization-scope.d.ts +32 -2
- package/dist/server/index.d.ts +73 -8
- package/dist/server/index.js +11 -3
- package/dist/server/operations/get-principal.d.ts +75 -0
- package/dist/server/operations/get-principal.js +41 -0
- package/dist/server/operations/get-task-diagnostics.d.ts +43 -18
- package/dist/server/operations/get-task-diagnostics.js +87 -126
- package/dist/server/operations/get-worker-diagnostics.d.ts +177 -0
- package/dist/server/operations/get-worker-diagnostics.js +117 -0
- package/dist/server/operations/list-worker-registration-rejections.d.ts +79 -0
- package/dist/server/operations/list-worker-registration-rejections.js +60 -0
- package/dist/server/operations/list-workers.js +1 -3
- package/dist/server/operations/static-registrations.d.ts +28 -0
- package/dist/server/operations/static-registrations.js +248 -0
- package/dist/server/rest-bindings.d.ts +5 -0
- package/dist/server/rest-bindings.js +34 -313
- package/dist/server/runtime/authentication-bridge.js +6 -14
- package/dist/server/runtime/context.d.ts +23 -0
- package/dist/server/runtime/task-dispatch.d.ts +7 -1
- package/dist/server/runtime/task-dispatch.js +126 -55
- package/dist/server/runtime/task-ledger-completion.d.ts +68 -0
- package/dist/server/runtime/task-ledger-completion.js +72 -0
- package/dist/server/runtime/task-ledger-recovery.d.ts +55 -0
- package/dist/server/runtime/task-ledger-recovery.js +73 -0
- package/dist/server/runtime/task-ledger-runtime.d.ts +64 -0
- package/dist/server/runtime/task-ledger-runtime.js +34 -0
- package/dist/server/runtime/task-metrics.d.ts +6 -4
- package/dist/server/runtime/task-polling.d.ts +14 -12
- package/dist/server/runtime/task-polling.js +106 -82
- package/dist/server/runtime/task-reconciliation.d.ts +35 -8
- package/dist/server/runtime/task-reconciliation.js +75 -74
- package/dist/server/runtime/task-result-resolution.d.ts +2 -10
- package/dist/server/runtime/task-result-resolution.js +0 -67
- package/dist/server/runtime/task-result-view.d.ts +82 -0
- package/dist/server/runtime/task-result-view.js +48 -0
- package/dist/server/runtime/websocket-worker-messaging.d.ts +16 -0
- package/dist/server/runtime/websocket-worker-messaging.js +32 -0
- package/dist/server/runtime/websocket-worker-registration.d.ts +41 -0
- package/dist/server/runtime/websocket-worker-registration.js +134 -0
- package/dist/server/runtime/websocket-worker.d.ts +0 -1
- package/dist/server/runtime/websocket-worker.js +64 -116
- package/dist/server/serve-internals.d.ts +9 -6
- package/dist/server/serve-internals.js +14 -40
- package/dist/server/task-ledger-codec.d.ts +40 -0
- package/dist/server/task-ledger-codec.js +195 -0
- package/dist/server/task-ledger-keys.d.ts +13 -0
- package/dist/server/task-ledger-keys.js +5 -0
- package/dist/server/task-ledger-limits.d.ts +34 -0
- package/dist/server/task-ledger-limits.js +5 -0
- package/dist/server/task-ledger-transition-helpers.d.ts +38 -0
- package/dist/server/task-ledger-transition-helpers.js +42 -0
- package/dist/server/task-ledger-transitions-cancellation.d.ts +31 -0
- package/dist/server/task-ledger-transitions-cancellation.js +73 -0
- package/dist/server/task-ledger-transitions.d.ts +129 -0
- package/dist/server/task-ledger-transitions.js +162 -0
- package/dist/server/task-ledger-types.d.ts +220 -0
- package/dist/server/task-ledger-types.js +1 -0
- package/dist/server/task-ledger.d.ts +17 -0
- package/dist/server/task-ledger.js +24 -0
- package/dist/server/task-state.d.ts +25 -152
- package/dist/server/task-state.js +0 -188
- package/dist/server/worker-admission-policy.d.ts +70 -0
- package/dist/server/worker-admission-policy.js +0 -0
- package/dist/storage/compressed-storage.js +1 -1
- package/dist/storage/index.d.ts +0 -2
- package/dist/storage/interface.d.ts +0 -2
- package/dist/storage/interface.js +1 -1
- package/dist/storage/lmdb.js +1 -1
- package/dist/storage/memory.js +1 -1
- package/dist/storage/neon.js +1 -1
- package/dist/storage/postgres.js +1 -1
- package/dist/storage/resolve.js +1 -1
- package/dist/storage/scoped-storage.js +1 -1
- package/dist/storage/testing.js +1 -1
- package/dist/storage/turso.js +1 -1
- package/dist/storage/web-extension.js +2 -2
- package/dist/version.d.ts +1 -1
- package/dist/version.js +1 -1
- package/dist/web-extension.js +1 -1
- package/dist/worker/index.d.ts +2 -0
- package/dist/worker/index.js +30 -4
- package/dist/worker/manifest/canonical-json.d.ts +20 -0
- package/dist/worker/manifest/canonical-json.js +13 -0
- package/dist/worker/manifest/capabilities.d.ts +24 -0
- package/dist/worker/manifest/capabilities.js +74 -0
- package/dist/worker/manifest/content-digest.d.ts +20 -0
- package/dist/worker/manifest/content-digest.js +13 -0
- package/dist/worker/manifest/declared-shape-digest.d.ts +14 -0
- package/dist/worker/manifest/declared-shape-digest.js +4 -0
- package/dist/worker/manifest/digest.d.ts +82 -0
- package/dist/worker/manifest/digest.js +9 -0
- package/dist/worker/manifest/execution-identity.d.ts +92 -0
- package/dist/worker/manifest/execution-identity.js +36 -0
- package/dist/worker/manifest/failure.d.ts +59 -0
- package/dist/worker/manifest/failure.js +4 -0
- package/dist/worker/manifest/index.d.ts +22 -0
- package/dist/worker/manifest/index.js +23 -0
- package/dist/worker/manifest/internal-realm.d.ts +50 -0
- package/dist/worker/manifest/internal-realm.js +39 -0
- package/dist/worker/manifest/is-record.d.ts +12 -0
- package/dist/worker/manifest/is-record.js +6 -0
- package/dist/worker/manifest/json-scan.d.ts +26 -0
- package/dist/worker/manifest/json-scan.js +88 -0
- package/dist/worker/manifest/limits.d.ts +138 -0
- package/dist/worker/manifest/limits.js +1 -0
- package/dist/worker/manifest/normalize.d.ts +63 -0
- package/dist/worker/manifest/normalize.js +73 -0
- package/dist/worker/manifest/parse-json.d.ts +24 -0
- package/dist/worker/manifest/parse-json.js +15 -0
- package/dist/worker/manifest/parse.d.ts +86 -0
- package/dist/worker/manifest/parse.js +188 -0
- package/dist/worker/manifest/registry-contract-builder.d.ts +120 -0
- package/dist/worker/manifest/registry-contract-builder.js +65 -0
- package/dist/worker/manifest/types.d.ts +263 -0
- package/dist/worker/manifest/types.js +1 -0
- package/dist/worker/manifest/utf8.d.ts +13 -0
- package/dist/worker/manifest/utf8.js +7 -0
- package/dist/worker/options.d.ts +55 -8
- package/dist/worker/options.js +65 -9
- package/dist/worker/protocol-internals.d.ts +1 -2
- package/dist/worker/protocol-internals.js +0 -5
- package/dist/worker/protocol-messages.d.ts +33 -17
- package/dist/worker/protocol-schemas.d.ts +36 -74
- package/dist/worker/protocol-schemas.js +25 -18
- package/dist/worker/protocol-version.d.ts +18 -14
- package/dist/worker/protocol-version.js +1 -1
- package/dist/worker/protocol.js +1 -1
- package/dist/worker/registry/deployment-consistency.d.ts +78 -0
- package/dist/worker/registry/deployment-consistency.js +27 -0
- package/dist/worker/registry/drain.d.ts +14 -0
- package/dist/worker/registry/drain.js +24 -0
- package/dist/worker/registry/rejections.d.ts +31 -0
- package/dist/worker/registry/rejections.js +10 -0
- package/dist/worker/registry/summary.d.ts +0 -3
- package/dist/worker/registry/summary.js +2 -5
- package/dist/worker/registry/types.d.ts +10 -3
- package/dist/worker/registry.d.ts +47 -2
- package/dist/worker/registry.js +45 -32
- package/dist/workers/workflow-worker-entry.d.ts +12 -7
- package/dist/workers/workflow-worker-entry.js +9 -2
- package/package.json +3 -3
- package/dist/server/task-resolved-record.d.ts +0 -12
- package/dist/server/task-resolved-record.js +0 -35
|
@@ -0,0 +1,75 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* `weft.system.principal` operation + REST binding.
|
|
3
|
+
*
|
|
4
|
+
* Reports the caller's own resolved principal: the authentication method,
|
|
5
|
+
* the normalized subject, and the granted scope set. Public access by
|
|
6
|
+
* design — introspection has to work for every credential state to be
|
|
7
|
+
* useful (a dashboard probes its principal before it knows anything), and
|
|
8
|
+
* an unauthenticated caller learns only what it already knows: that it is
|
|
9
|
+
* anonymous with no scopes. The response is strictly principal-shaped —
|
|
10
|
+
* nothing about the server's auth configuration (key inventory,
|
|
11
|
+
* `unauthenticatedAccess` posture, JWT issuers) is exposed.
|
|
12
|
+
*/
|
|
13
|
+
import { z } from 'zod';
|
|
14
|
+
import type { UnknownRestBinding } from '../rest-bindings.ts';
|
|
15
|
+
declare const getPrincipalInput: z.ZodObject<{}, z.core.$strip>;
|
|
16
|
+
declare const getPrincipalOutput: z.ZodObject<{
|
|
17
|
+
method: z.ZodEnum<{
|
|
18
|
+
jwt: "jwt";
|
|
19
|
+
"api-key": "api-key";
|
|
20
|
+
mtls: "mtls";
|
|
21
|
+
"stdio-local": "stdio-local";
|
|
22
|
+
unauthenticated: "unauthenticated";
|
|
23
|
+
}>;
|
|
24
|
+
subject: z.ZodNullable<z.ZodString>;
|
|
25
|
+
scopes: z.ZodArray<z.ZodEnum<{
|
|
26
|
+
"workflows:read": "workflows:read";
|
|
27
|
+
"workflows:write": "workflows:write";
|
|
28
|
+
"workflows:admin": "workflows:admin";
|
|
29
|
+
"schedules:read": "schedules:read";
|
|
30
|
+
"schedules:write": "schedules:write";
|
|
31
|
+
"signals:write": "signals:write";
|
|
32
|
+
"updates:write": "updates:write";
|
|
33
|
+
"queries:read": "queries:read";
|
|
34
|
+
"reviews:read": "reviews:read";
|
|
35
|
+
"reviews:write": "reviews:write";
|
|
36
|
+
"attributes:read": "attributes:read";
|
|
37
|
+
"attributes:write": "attributes:write";
|
|
38
|
+
"tags:write": "tags:write";
|
|
39
|
+
"streams:read": "streams:read";
|
|
40
|
+
"events:read": "events:read";
|
|
41
|
+
"storage:read": "storage:read";
|
|
42
|
+
"storage:write": "storage:write";
|
|
43
|
+
"storage:admin": "storage:admin";
|
|
44
|
+
"workers:write": "workers:write";
|
|
45
|
+
"system:read": "system:read";
|
|
46
|
+
"system:admin": "system:admin";
|
|
47
|
+
}>>;
|
|
48
|
+
}, z.core.$strip>;
|
|
49
|
+
export type GetPrincipalInput = z.infer<typeof getPrincipalInput>;
|
|
50
|
+
/**
|
|
51
|
+
* Output of `weft.system.principal`: the caller's authentication method,
|
|
52
|
+
* normalized subject (`null` when the credential carries none), and granted
|
|
53
|
+
* scopes, sorted. Anonymous callers receive `method: 'unauthenticated'`
|
|
54
|
+
* with an empty scope list.
|
|
55
|
+
*
|
|
56
|
+
* @example
|
|
57
|
+
* ```ts
|
|
58
|
+
* import { type GetPrincipalOutput } from '@lostgradient/weft/server';
|
|
59
|
+
*
|
|
60
|
+
* const anonymous: GetPrincipalOutput = {
|
|
61
|
+
* method: 'unauthenticated',
|
|
62
|
+
* subject: null,
|
|
63
|
+
* scopes: [],
|
|
64
|
+
* };
|
|
65
|
+
* console.log(anonymous.method);
|
|
66
|
+
* ```
|
|
67
|
+
*/
|
|
68
|
+
export type GetPrincipalOutput = z.infer<typeof getPrincipalOutput>;
|
|
69
|
+
export declare const getPrincipalOperation: import("../operation-catalog.ts").OperationDefinition<Record<string, never>, {
|
|
70
|
+
method: "jwt" | "api-key" | "mtls" | "stdio-local" | "unauthenticated";
|
|
71
|
+
subject: string | null;
|
|
72
|
+
scopes: ("workflows:read" | "workflows:write" | "workflows:admin" | "schedules:read" | "schedules:write" | "signals:write" | "updates:write" | "queries:read" | "reviews:read" | "reviews:write" | "attributes:read" | "attributes:write" | "tags:write" | "streams:read" | "events:read" | "storage:read" | "storage:write" | "storage:admin" | "workers:write" | "system:read" | "system:admin")[];
|
|
73
|
+
}, unknown>;
|
|
74
|
+
export declare const getPrincipalRestBinding: UnknownRestBinding;
|
|
75
|
+
export {};
|
|
@@ -0,0 +1,41 @@
|
|
|
1
|
+
import { z } from "zod";
|
|
2
|
+
import { AUTHORIZATION_SCOPES } from "../authorization-scope.js";
|
|
3
|
+
import { shapeOperationFaultAsJson } from "../operation-fault.js";
|
|
4
|
+
import { defineOperation } from "../operation-registry.js";
|
|
5
|
+
const principalMethodSchema = z.enum(["jwt", "api-key", "mtls", "stdio-local", "unauthenticated"]), getPrincipalInput = z.object({}), getPrincipalOutput = z.object({
|
|
6
|
+
method: principalMethodSchema,
|
|
7
|
+
subject: z.string().nullable(),
|
|
8
|
+
scopes: z.array(z.enum(AUTHORIZATION_SCOPES))
|
|
9
|
+
});
|
|
10
|
+
export const getPrincipalOperation = defineOperation({
|
|
11
|
+
name: "weft.system.principal",
|
|
12
|
+
mcpExposable: !1,
|
|
13
|
+
summary: "Report the caller's resolved principal and granted scopes",
|
|
14
|
+
description: `Report the caller's own resolved principal: the authentication method ('jwt', 'api-key', 'mtls', 'stdio-local', or 'unauthenticated'), the normalized subject when one exists, and the granted authorization scopes, sorted. Public access: an unauthenticated caller receives method 'unauthenticated' with an empty scope list rather than an error, so clients can distinguish "no credential" from "credential with few scopes" without probing ` + "other operations. Reflects only the request credential \u2014 never server auth configuration.",
|
|
15
|
+
destructive: !1,
|
|
16
|
+
tags: ["System"],
|
|
17
|
+
inputSchema: getPrincipalInput,
|
|
18
|
+
outputSchema: getPrincipalOutput,
|
|
19
|
+
access: { kind: "public" },
|
|
20
|
+
discoverable: !0,
|
|
21
|
+
transports: { http: !0, jsonRpcHttp: !0, jsonRpcWebSocket: !0, jsonRpcStdio: !0 },
|
|
22
|
+
unknownKeyPolicy: { http: "strip", jsonRpc: "reject" },
|
|
23
|
+
invoke: async ({ principal }) => {
|
|
24
|
+
if (principal.method === "unauthenticated")
|
|
25
|
+
return { method: "unauthenticated", subject: null, scopes: [] };
|
|
26
|
+
return {
|
|
27
|
+
method: principal.method,
|
|
28
|
+
subject: principal.subject ?? null,
|
|
29
|
+
scopes: [...principal.scopes].toSorted()
|
|
30
|
+
};
|
|
31
|
+
}
|
|
32
|
+
}), getPrincipalRestBinding = {
|
|
33
|
+
method: "GET",
|
|
34
|
+
path: "/v1/principal",
|
|
35
|
+
pathParamNames: [],
|
|
36
|
+
operationName: "weft.system.principal",
|
|
37
|
+
inputSources: {},
|
|
38
|
+
extractInput: async () => ({}),
|
|
39
|
+
success: { kind: "json", status: 200 },
|
|
40
|
+
shapeFault: shapeOperationFaultAsJson
|
|
41
|
+
};
|
|
@@ -1,10 +1,41 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* `weft.tasks.diagnostics` operation + REST binding.
|
|
3
3
|
*
|
|
4
|
-
* Scans durable task
|
|
5
|
-
*
|
|
6
|
-
*
|
|
7
|
-
*
|
|
4
|
+
* Scans the durable task ledger (WFT-24 — previously the retired
|
|
5
|
+
* `op:queued:`/`op:inflight:`/`op:resolved:`/`op:dead-letter:` keys, which
|
|
6
|
+
* nothing has written since the WFT-22 ledger cutover) and live worker state
|
|
7
|
+
* to identify queue pressure, stale in-flight work, retry storms, dead
|
|
8
|
+
* letters, and worker capacity saturation. Results are intentionally bounded
|
|
9
|
+
* and low-cardinality so operators can use them in a dashboard without
|
|
10
|
+
* turning workflow or worker identifiers into metrics labels.
|
|
11
|
+
*
|
|
12
|
+
* The output schema's field names and `kind`/`state` vocabulary are
|
|
13
|
+
* unchanged from the pre-ledger version — only the internal scan and
|
|
14
|
+
* classification logic changed — so existing dashboards built against this
|
|
15
|
+
* endpoint's shape keep working. `leased`, `completing`, and `cancelling`
|
|
16
|
+
* ledger states are all reported as diagnostic `state: 'inflight'`: they
|
|
17
|
+
* share the same lease-holder/heartbeat shape and the same operator
|
|
18
|
+
* question ("is a worker still making progress on this attempt?"), so
|
|
19
|
+
* exposing the ledger's more granular internal states here would add
|
|
20
|
+
* distinctions operators cannot act on differently.
|
|
21
|
+
*
|
|
22
|
+
* Retry-storm detection (`kind: 'retry-storm'`) no longer covers `terminal`
|
|
23
|
+
* records: `RemoteTaskTerminal` carries no `retryCount`/`requeueCount` (WFT-25
|
|
24
|
+
* deliberately dropped attempt-count history once a task resolves), so
|
|
25
|
+
* there is nothing left to detect a storm from once an attempt reaches a
|
|
26
|
+
* disposition. The `resolved` value in the `state` enum is kept in the
|
|
27
|
+
* output schema for backward compatibility but nothing produces it anymore.
|
|
28
|
+
* `dead-lettered` diagnostics are unaffected — that is a distinct, separate
|
|
29
|
+
* `kind`.
|
|
30
|
+
*
|
|
31
|
+
* Unlike the pre-ledger scan, the full ledger scan this operation performs
|
|
32
|
+
* is one combined keyspace across every state (queued through terminal),
|
|
33
|
+
* not separate per-state prefixes — there is no time-bounded history index
|
|
34
|
+
* to limit how many terminal records get walked past on the way to
|
|
35
|
+
* classifying the ones that still matter. Operators who dispatch high
|
|
36
|
+
* volumes of tasks and want this scan to stay cheap should set
|
|
37
|
+
* {@link ServeOptions.taskRetentionWindowMs} so adopted terminal records
|
|
38
|
+
* are reaped rather than accumulating indefinitely.
|
|
8
39
|
*
|
|
9
40
|
* @module server/operations/get-task-diagnostics
|
|
10
41
|
*/
|
|
@@ -30,8 +61,8 @@ declare const taskDiagnosticItemSchema: z.ZodObject<{
|
|
|
30
61
|
state: z.ZodEnum<{
|
|
31
62
|
"dead-lettered": "dead-lettered";
|
|
32
63
|
queued: "queued";
|
|
33
|
-
inflight: "inflight";
|
|
34
64
|
resolved: "resolved";
|
|
65
|
+
inflight: "inflight";
|
|
35
66
|
capacity: "capacity";
|
|
36
67
|
}>;
|
|
37
68
|
operationId: z.ZodOptional<z.ZodString>;
|
|
@@ -44,10 +75,7 @@ declare const taskDiagnosticItemSchema: z.ZodObject<{
|
|
|
44
75
|
queueLatencyMs: z.ZodOptional<z.ZodNumber>;
|
|
45
76
|
executionLatencyMs: z.ZodOptional<z.ZodNumber>;
|
|
46
77
|
heartbeatAgeMs: z.ZodOptional<z.ZodNumber>;
|
|
47
|
-
lastRequeueReason: z.ZodOptional<z.
|
|
48
|
-
"visibility-timeout": "visibility-timeout";
|
|
49
|
-
"worker-disconnect": "worker-disconnect";
|
|
50
|
-
}>>;
|
|
78
|
+
lastRequeueReason: z.ZodOptional<z.ZodString>;
|
|
51
79
|
resolutionReason: z.ZodOptional<z.ZodString>;
|
|
52
80
|
deadLetteredAt: z.ZodOptional<z.ZodNumber>;
|
|
53
81
|
deadLetterReason: z.ZodOptional<z.ZodLiteral<"result-resolution-storage-exhausted">>;
|
|
@@ -74,8 +102,8 @@ declare const getTaskDiagnosticsOutput: z.ZodObject<{
|
|
|
74
102
|
state: z.ZodEnum<{
|
|
75
103
|
"dead-lettered": "dead-lettered";
|
|
76
104
|
queued: "queued";
|
|
77
|
-
inflight: "inflight";
|
|
78
105
|
resolved: "resolved";
|
|
106
|
+
inflight: "inflight";
|
|
79
107
|
capacity: "capacity";
|
|
80
108
|
}>;
|
|
81
109
|
operationId: z.ZodOptional<z.ZodString>;
|
|
@@ -88,10 +116,7 @@ declare const getTaskDiagnosticsOutput: z.ZodObject<{
|
|
|
88
116
|
queueLatencyMs: z.ZodOptional<z.ZodNumber>;
|
|
89
117
|
executionLatencyMs: z.ZodOptional<z.ZodNumber>;
|
|
90
118
|
heartbeatAgeMs: z.ZodOptional<z.ZodNumber>;
|
|
91
|
-
lastRequeueReason: z.ZodOptional<z.
|
|
92
|
-
"visibility-timeout": "visibility-timeout";
|
|
93
|
-
"worker-disconnect": "worker-disconnect";
|
|
94
|
-
}>>;
|
|
119
|
+
lastRequeueReason: z.ZodOptional<z.ZodString>;
|
|
95
120
|
resolutionReason: z.ZodOptional<z.ZodString>;
|
|
96
121
|
deadLetteredAt: z.ZodOptional<z.ZodNumber>;
|
|
97
122
|
deadLetterReason: z.ZodOptional<z.ZodLiteral<"result-resolution-storage-exhausted">>;
|
|
@@ -146,7 +171,7 @@ export declare function createGetTaskDiagnosticsOperation(options?: GetTaskDiagn
|
|
|
146
171
|
}, {
|
|
147
172
|
items: {
|
|
148
173
|
kind: "dead-lettered" | "stuck-queued" | "stale-inflight" | "retry-storm" | "all-workers-at-capacity";
|
|
149
|
-
state: "dead-lettered" | "queued" | "
|
|
174
|
+
state: "dead-lettered" | "queued" | "resolved" | "inflight" | "capacity";
|
|
150
175
|
retryCount: number;
|
|
151
176
|
requeueCount: number;
|
|
152
177
|
evidence: string[];
|
|
@@ -158,7 +183,7 @@ export declare function createGetTaskDiagnosticsOperation(options?: GetTaskDiagn
|
|
|
158
183
|
queueLatencyMs?: number | undefined;
|
|
159
184
|
executionLatencyMs?: number | undefined;
|
|
160
185
|
heartbeatAgeMs?: number | undefined;
|
|
161
|
-
lastRequeueReason?:
|
|
186
|
+
lastRequeueReason?: string | undefined;
|
|
162
187
|
resolutionReason?: string | undefined;
|
|
163
188
|
deadLetteredAt?: number | undefined;
|
|
164
189
|
deadLetterReason?: "result-resolution-storage-exhausted" | undefined;
|
|
@@ -185,7 +210,7 @@ export declare const getTaskDiagnosticsOperation: import("../operation-catalog.t
|
|
|
185
210
|
}, {
|
|
186
211
|
items: {
|
|
187
212
|
kind: "dead-lettered" | "stuck-queued" | "stale-inflight" | "retry-storm" | "all-workers-at-capacity";
|
|
188
|
-
state: "dead-lettered" | "queued" | "
|
|
213
|
+
state: "dead-lettered" | "queued" | "resolved" | "inflight" | "capacity";
|
|
189
214
|
retryCount: number;
|
|
190
215
|
requeueCount: number;
|
|
191
216
|
evidence: string[];
|
|
@@ -197,7 +222,7 @@ export declare const getTaskDiagnosticsOperation: import("../operation-catalog.t
|
|
|
197
222
|
queueLatencyMs?: number | undefined;
|
|
198
223
|
executionLatencyMs?: number | undefined;
|
|
199
224
|
heartbeatAgeMs?: number | undefined;
|
|
200
|
-
lastRequeueReason?:
|
|
225
|
+
lastRequeueReason?: string | undefined;
|
|
201
226
|
resolutionReason?: string | undefined;
|
|
202
227
|
deadLetteredAt?: number | undefined;
|
|
203
228
|
deadLetterReason?: "result-resolution-storage-exhausted" | undefined;
|
|
@@ -1,18 +1,15 @@
|
|
|
1
1
|
import { z } from "zod";
|
|
2
|
-
import {
|
|
3
|
-
import { KEYS } from "../../storage/interface.js";
|
|
2
|
+
import { raiseFault } from "../operation-catalog/raise-fault.js";
|
|
4
3
|
import { defineOperation } from "../operation-registry.js";
|
|
4
|
+
import { commitTaskLedgerDelete } from "../runtime/task-ledger-runtime.js";
|
|
5
|
+
import { canClearDeadLetteredTask } from "../task-ledger-transitions.js";
|
|
6
|
+
import { decodeRemoteTaskRecord } from "../task-ledger.js";
|
|
5
7
|
import {
|
|
6
8
|
calculateExecutionLatencyMs,
|
|
7
9
|
calculateHeartbeatAgeMs,
|
|
8
|
-
calculateQueueLatencyMs
|
|
9
|
-
clearDeadLetteredTaskRecord,
|
|
10
|
-
isDeadLetteredTaskRecord,
|
|
11
|
-
isInflightRecord,
|
|
12
|
-
isQueuedRecord,
|
|
13
|
-
isResolvedRecord
|
|
10
|
+
calculateQueueLatencyMs
|
|
14
11
|
} from "../task-state.js";
|
|
15
|
-
const DEFAULT_STALE_QUEUED_AFTER_MS = 60000, DEFAULT_STALE_HEARTBEAT_AFTER_MS = 60000, DEFAULT_RETRY_STORM_MINIMUM_ATTEMPTS = 3, DEFAULT_LIMIT = 50, MAX_LIMIT = 200,
|
|
12
|
+
const DEFAULT_STALE_QUEUED_AFTER_MS = 60000, DEFAULT_STALE_HEARTBEAT_AFTER_MS = 60000, DEFAULT_RETRY_STORM_MINIMUM_ATTEMPTS = 3, DEFAULT_LIMIT = 50, MAX_LIMIT = 200, taskDiagnosticKindSchema = z.enum([
|
|
16
13
|
"stuck-queued",
|
|
17
14
|
"stale-inflight",
|
|
18
15
|
"retry-storm",
|
|
@@ -31,7 +28,7 @@ const DEFAULT_STALE_QUEUED_AFTER_MS = 60000, DEFAULT_STALE_HEARTBEAT_AFTER_MS =
|
|
|
31
28
|
queueLatencyMs: z.number().nonnegative().optional(),
|
|
32
29
|
executionLatencyMs: z.number().nonnegative().optional(),
|
|
33
30
|
heartbeatAgeMs: z.number().nonnegative().optional(),
|
|
34
|
-
lastRequeueReason: z.
|
|
31
|
+
lastRequeueReason: z.string().optional(),
|
|
35
32
|
resolutionReason: z.string().optional(),
|
|
36
33
|
deadLetteredAt: z.number().nonnegative().optional(),
|
|
37
34
|
deadLetterReason: z.literal("result-resolution-storage-exhausted").optional(),
|
|
@@ -105,12 +102,17 @@ export const getTaskDiagnosticsOperation = createGetTaskDiagnosticsOperation(),
|
|
|
105
102
|
kind: "scoped",
|
|
106
103
|
scopes: { kind: "anyOf", scopes: ["system:admin"] }
|
|
107
104
|
},
|
|
108
|
-
producibleFaults: [],
|
|
105
|
+
producibleFaults: ["NotFound"],
|
|
109
106
|
discoverable: !0,
|
|
110
107
|
transports: restOnlyTaskDiagnosticsTransports,
|
|
111
108
|
unknownKeyPolicy: { http: "reject", jsonRpc: "reject" },
|
|
112
109
|
invoke: async ({ input, engine }) => {
|
|
113
|
-
await
|
|
110
|
+
if (!(await commitTaskLedgerDelete(engine.storage, input.operationId, canClearDeadLetteredTask, 1)).ok)
|
|
111
|
+
raiseFault(clearTaskDeadLetterOperation, {
|
|
112
|
+
code: "NotFound",
|
|
113
|
+
message: `No dead-lettered task found for operation "${input.operationId}"`,
|
|
114
|
+
data: { resource: "task", identifier: input.operationId }
|
|
115
|
+
});
|
|
114
116
|
return { ok: !0 };
|
|
115
117
|
}
|
|
116
118
|
});
|
|
@@ -127,39 +129,19 @@ async function collectTaskDiagnostics({
|
|
|
127
129
|
retryStorms: 0,
|
|
128
130
|
allWorkersAtCapacity: 0,
|
|
129
131
|
deadLettered: 0
|
|
130
|
-
}, relevantQueues = new Set,
|
|
132
|
+
}, relevantQueues = new Set, addItem = (item) => {
|
|
131
133
|
incrementSummary(summary, item.kind);
|
|
132
134
|
if (items.length < input.limit)
|
|
133
135
|
items.push(item);
|
|
134
136
|
};
|
|
135
|
-
for await (const
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
deadLetteredOperationIds.add(record.operationId);
|
|
139
|
-
if (record.queue !== void 0)
|
|
140
|
-
relevantQueues.add(record.queue);
|
|
141
|
-
addDeadLetterDiagnostics(record, addItem);
|
|
142
|
-
}
|
|
143
|
-
for await (const record of scanQueuedRecords(engine)) {
|
|
144
|
-
if (!matchesTaskRecordFilter(record, input))
|
|
137
|
+
for await (const [, value] of engine.storage.scan("task-ledger:")) {
|
|
138
|
+
const decoded = decodeRemoteTaskRecord(value);
|
|
139
|
+
if (decoded === null)
|
|
145
140
|
continue;
|
|
146
|
-
|
|
147
|
-
addQueuedDiagnostics(record, input, currentTime, addItem);
|
|
148
|
-
}
|
|
149
|
-
for await (const record of scanInflightRecords(engine)) {
|
|
150
|
-
if (!matchesTaskRecordFilter(record, input))
|
|
151
|
-
continue;
|
|
152
|
-
if (deadLetteredOperationIds.has(record.operationId))
|
|
141
|
+
if (!matchesTaskRecordFilter(decoded, input))
|
|
153
142
|
continue;
|
|
154
|
-
relevantQueues.add(
|
|
155
|
-
|
|
156
|
-
}
|
|
157
|
-
for await (const record of scanResolvedRecords(engine, input)) {
|
|
158
|
-
if (!matchesTaskRecordFilter(record, input))
|
|
159
|
-
continue;
|
|
160
|
-
if (record.queue !== void 0)
|
|
161
|
-
relevantQueues.add(record.queue);
|
|
162
|
-
addRetryStormDiagnostic(record, "resolved", input, addItem);
|
|
143
|
+
relevantQueues.add(decoded.queue);
|
|
144
|
+
addRecordDiagnostics(decoded, input, currentTime, addItem);
|
|
163
145
|
}
|
|
164
146
|
addCapacityDiagnostics({
|
|
165
147
|
registry,
|
|
@@ -170,88 +152,71 @@ async function collectTaskDiagnostics({
|
|
|
170
152
|
});
|
|
171
153
|
return { items, summary, limit: input.limit };
|
|
172
154
|
}
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
}
|
|
180
|
-
async function* scanQueuedRecords(engine) {
|
|
181
|
-
for await (const [, value] of engine.storage.scan("op:queued:")) {
|
|
182
|
-
const decoded = decode(value);
|
|
183
|
-
if (isQueuedRecord(decoded))
|
|
184
|
-
yield decoded;
|
|
185
|
-
}
|
|
186
|
-
}
|
|
187
|
-
async function* scanInflightRecords(engine) {
|
|
188
|
-
for await (const [, value] of engine.storage.scan("op:inflight:")) {
|
|
189
|
-
const decoded = decode(value);
|
|
190
|
-
if (isInflightRecord(decoded))
|
|
191
|
-
yield decoded;
|
|
192
|
-
}
|
|
193
|
-
}
|
|
194
|
-
async function* scanResolvedRecords(engine, input) {
|
|
195
|
-
if (input.operationId !== void 0) {
|
|
196
|
-
const value = await engine.storage.get(KEYS.operationResolved(input.operationId));
|
|
197
|
-
if (value === null)
|
|
155
|
+
function addRecordDiagnostics(decoded, input, currentTime, addItem) {
|
|
156
|
+
switch (decoded.state) {
|
|
157
|
+
case "queued":
|
|
158
|
+
if (decoded.availableAt <= currentTime)
|
|
159
|
+
addQueuedDiagnostics(decoded, input, currentTime, addItem);
|
|
160
|
+
addRetryStormDiagnostic(decoded, "queued", input, addItem);
|
|
198
161
|
return;
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
|
|
205
|
-
|
|
206
|
-
|
|
207
|
-
|
|
208
|
-
|
|
209
|
-
|
|
210
|
-
|
|
162
|
+
case "leased":
|
|
163
|
+
case "completing":
|
|
164
|
+
case "cancelling":
|
|
165
|
+
addInflightDiagnostics(decoded, input, currentTime, addItem);
|
|
166
|
+
addRetryStormDiagnostic(decoded, "inflight", input, addItem);
|
|
167
|
+
return;
|
|
168
|
+
case "terminal":
|
|
169
|
+
return;
|
|
170
|
+
case "deadLettered":
|
|
171
|
+
addDeadLetterDiagnostics(decoded, addItem);
|
|
172
|
+
return;
|
|
173
|
+
default: {
|
|
174
|
+
const exhaustive = decoded;
|
|
175
|
+
}
|
|
211
176
|
}
|
|
212
177
|
}
|
|
213
178
|
function addQueuedDiagnostics(record, input, currentTime, addItem) {
|
|
214
|
-
const
|
|
215
|
-
if (queueLatencyMs
|
|
216
|
-
|
|
217
|
-
|
|
218
|
-
|
|
219
|
-
|
|
220
|
-
|
|
221
|
-
|
|
222
|
-
|
|
223
|
-
|
|
224
|
-
|
|
225
|
-
|
|
226
|
-
|
|
227
|
-
|
|
228
|
-
|
|
229
|
-
|
|
230
|
-
|
|
231
|
-
|
|
179
|
+
const queueLatencyMs = Math.max(0, currentTime - record.lastQueuedAt);
|
|
180
|
+
if (queueLatencyMs < input.staleQueuedAfterMs)
|
|
181
|
+
return;
|
|
182
|
+
addItem({
|
|
183
|
+
kind: "stuck-queued",
|
|
184
|
+
state: "queued",
|
|
185
|
+
operationId: record.operationId,
|
|
186
|
+
workflowId: record.workflowId,
|
|
187
|
+
activityName: record.activityName,
|
|
188
|
+
queue: record.queue,
|
|
189
|
+
retryCount: record.retryCount,
|
|
190
|
+
requeueCount: record.requeueCount,
|
|
191
|
+
queueLatencyMs,
|
|
192
|
+
lastRequeueReason: record.lastRequeueReason,
|
|
193
|
+
evidence: [
|
|
194
|
+
`Task has waited ${queueLatencyMs}ms in queue "${record.queue}" without a worker claim`
|
|
195
|
+
]
|
|
196
|
+
});
|
|
232
197
|
}
|
|
233
198
|
function addInflightDiagnostics(record, input, currentTime, addItem) {
|
|
234
199
|
const heartbeatAgeMs = calculateHeartbeatAgeMs(record, currentTime) ?? 0;
|
|
235
|
-
if (heartbeatAgeMs
|
|
236
|
-
|
|
237
|
-
|
|
238
|
-
|
|
239
|
-
|
|
240
|
-
|
|
241
|
-
|
|
242
|
-
|
|
243
|
-
|
|
244
|
-
|
|
245
|
-
|
|
246
|
-
|
|
247
|
-
|
|
248
|
-
|
|
249
|
-
|
|
250
|
-
|
|
251
|
-
|
|
252
|
-
|
|
253
|
-
|
|
254
|
-
|
|
200
|
+
if (heartbeatAgeMs < input.staleHeartbeatAfterMs)
|
|
201
|
+
return;
|
|
202
|
+
addItem({
|
|
203
|
+
kind: "stale-inflight",
|
|
204
|
+
state: "inflight",
|
|
205
|
+
operationId: record.operationId,
|
|
206
|
+
workflowId: record.workflowId,
|
|
207
|
+
activityName: record.activityName,
|
|
208
|
+
queue: record.queue,
|
|
209
|
+
workerId: record.workerSessionId,
|
|
210
|
+
retryCount: record.retryCount,
|
|
211
|
+
requeueCount: record.requeueCount,
|
|
212
|
+
queueLatencyMs: calculateQueueLatencyMs(record),
|
|
213
|
+
executionLatencyMs: calculateExecutionLatencyMs(record, currentTime),
|
|
214
|
+
heartbeatAgeMs,
|
|
215
|
+
lastRequeueReason: record.lastRequeueReason,
|
|
216
|
+
evidence: [
|
|
217
|
+
`Worker "${record.workerSessionId}" has not sent a heartbeat for ${heartbeatAgeMs}ms on queue "${record.queue}"`
|
|
218
|
+
]
|
|
219
|
+
});
|
|
255
220
|
}
|
|
256
221
|
function addDeadLetterDiagnostics(record, addItem) {
|
|
257
222
|
addItem({
|
|
@@ -261,21 +226,19 @@ function addDeadLetterDiagnostics(record, addItem) {
|
|
|
261
226
|
workflowId: record.workflowId,
|
|
262
227
|
activityName: record.activityName,
|
|
263
228
|
queue: record.queue,
|
|
264
|
-
|
|
265
|
-
|
|
266
|
-
requeueCount: record.requeueCount ?? 0,
|
|
229
|
+
retryCount: record.retryCount,
|
|
230
|
+
requeueCount: record.requeueCount,
|
|
267
231
|
lastRequeueReason: record.lastRequeueReason,
|
|
268
232
|
deadLetteredAt: record.deadLetteredAt,
|
|
269
|
-
deadLetterReason:
|
|
270
|
-
storageError: record.
|
|
271
|
-
retryAttempts: record.retryAttempts,
|
|
233
|
+
deadLetterReason: "result-resolution-storage-exhausted",
|
|
234
|
+
storageError: record.persistenceFailureReason,
|
|
272
235
|
evidence: [
|
|
273
|
-
`Task result
|
|
236
|
+
`Task result could not be durably persisted (${record.persistenceFailureReason}); reconciliation will not re-dispatch operation "${record.operationId}" until the dead-letter entry is cleared`
|
|
274
237
|
]
|
|
275
238
|
});
|
|
276
239
|
}
|
|
277
240
|
function addRetryStormDiagnostic(record, state, input, addItem) {
|
|
278
|
-
const
|
|
241
|
+
const { retryCount, requeueCount } = record;
|
|
279
242
|
if (retryCount < input.retryStormMinimumAttempts && requeueCount < input.retryStormMinimumAttempts)
|
|
280
243
|
return;
|
|
281
244
|
addItem({
|
|
@@ -283,15 +246,13 @@ function addRetryStormDiagnostic(record, state, input, addItem) {
|
|
|
283
246
|
state,
|
|
284
247
|
operationId: record.operationId,
|
|
285
248
|
workflowId: record.workflowId,
|
|
286
|
-
activityName:
|
|
249
|
+
activityName: record.activityName,
|
|
287
250
|
queue: record.queue,
|
|
288
|
-
workerId: "
|
|
251
|
+
workerId: "workerSessionId" in record ? record.workerSessionId : void 0,
|
|
289
252
|
retryCount,
|
|
290
253
|
requeueCount,
|
|
291
254
|
queueLatencyMs: calculateQueueLatencyMs(record),
|
|
292
|
-
executionLatencyMs: "resolvedAt" in record ? calculateExecutionLatencyMs(record, record.resolvedAt) : void 0,
|
|
293
255
|
lastRequeueReason: record.lastRequeueReason,
|
|
294
|
-
resolutionReason: "resolutionReason" in record ? record.resolutionReason : void 0,
|
|
295
256
|
evidence: [
|
|
296
257
|
`Task has ${retryCount} retries and ${requeueCount} requeues, meeting retry storm threshold ${input.retryStormMinimumAttempts}`
|
|
297
258
|
]
|