@lostgradient/weft 0.17.0 → 0.19.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +3 -1
- package/dist/cli/conformance.js +14 -18
- package/dist/cli/generated/operation-catalog.snapshot.json +331 -16
- package/dist/cli/generated/operation-client.generated.d.ts +39 -6
- package/dist/cli/generated/operation-client.generated.js +6 -0
- package/dist/core/context/child-workflow-pipe.d.ts +1 -1
- package/dist/core/context/child-workflow-pipe.js +1 -1
- package/dist/core/engine/construction.d.ts +2 -0
- package/dist/core/engine/construction.js +3 -0
- package/dist/core/engine/index.js +1 -0
- package/dist/core/engine/listing.d.ts +1 -0
- package/dist/core/engine/listing.js +6 -4
- package/dist/core/weft-error.d.ts +1 -1
- package/dist/core/weft-error.js +2 -1
- package/dist/core/worker-execution-dispatcher.d.ts +8 -0
- package/dist/core/worker-execution-dispatcher.js +4 -0
- package/dist/core/worker-execution-strategy-options.d.ts +16 -0
- package/dist/core/worker-execution-strategy.js +71 -89
- package/dist/core/worker-fault-handling.d.ts +45 -0
- package/dist/core/worker-fault-handling.js +88 -0
- package/dist/core/worker-protocol.d.ts +18 -1
- package/dist/core/worker-protocol.js +1 -1
- package/dist/core/worker-realm-readiness.d.ts +78 -0
- package/dist/core/worker-realm-readiness.js +127 -0
- package/dist/index.d.ts +3 -1
- package/dist/index.js +28 -1
- package/dist/json-schema.js +1 -1
- package/dist/runtime/portable.d.ts +14 -0
- package/dist/runtime/portable.js +10 -0
- package/dist/server/authorization-scope.d.ts +32 -2
- package/dist/server/index.d.ts +73 -8
- package/dist/server/index.js +11 -3
- package/dist/server/operations/get-principal.d.ts +75 -0
- package/dist/server/operations/get-principal.js +41 -0
- package/dist/server/operations/get-task-diagnostics.d.ts +43 -18
- package/dist/server/operations/get-task-diagnostics.js +87 -126
- package/dist/server/operations/get-worker-diagnostics.d.ts +177 -0
- package/dist/server/operations/get-worker-diagnostics.js +117 -0
- package/dist/server/operations/list-worker-registration-rejections.d.ts +79 -0
- package/dist/server/operations/list-worker-registration-rejections.js +60 -0
- package/dist/server/operations/list-workers.js +1 -3
- package/dist/server/operations/static-registrations.d.ts +28 -0
- package/dist/server/operations/static-registrations.js +248 -0
- package/dist/server/rest-bindings.d.ts +5 -0
- package/dist/server/rest-bindings.js +34 -313
- package/dist/server/runtime/authentication-bridge.js +6 -14
- package/dist/server/runtime/context.d.ts +23 -0
- package/dist/server/runtime/task-dispatch.d.ts +7 -1
- package/dist/server/runtime/task-dispatch.js +126 -55
- package/dist/server/runtime/task-ledger-completion.d.ts +68 -0
- package/dist/server/runtime/task-ledger-completion.js +72 -0
- package/dist/server/runtime/task-ledger-recovery.d.ts +55 -0
- package/dist/server/runtime/task-ledger-recovery.js +73 -0
- package/dist/server/runtime/task-ledger-runtime.d.ts +64 -0
- package/dist/server/runtime/task-ledger-runtime.js +34 -0
- package/dist/server/runtime/task-metrics.d.ts +6 -4
- package/dist/server/runtime/task-polling.d.ts +14 -12
- package/dist/server/runtime/task-polling.js +106 -82
- package/dist/server/runtime/task-reconciliation.d.ts +35 -8
- package/dist/server/runtime/task-reconciliation.js +75 -74
- package/dist/server/runtime/task-result-resolution.d.ts +2 -10
- package/dist/server/runtime/task-result-resolution.js +0 -67
- package/dist/server/runtime/task-result-view.d.ts +82 -0
- package/dist/server/runtime/task-result-view.js +48 -0
- package/dist/server/runtime/websocket-worker-messaging.d.ts +16 -0
- package/dist/server/runtime/websocket-worker-messaging.js +32 -0
- package/dist/server/runtime/websocket-worker-registration.d.ts +41 -0
- package/dist/server/runtime/websocket-worker-registration.js +134 -0
- package/dist/server/runtime/websocket-worker.d.ts +0 -1
- package/dist/server/runtime/websocket-worker.js +64 -116
- package/dist/server/serve-internals.d.ts +9 -6
- package/dist/server/serve-internals.js +14 -40
- package/dist/server/task-ledger-codec.d.ts +40 -0
- package/dist/server/task-ledger-codec.js +195 -0
- package/dist/server/task-ledger-keys.d.ts +13 -0
- package/dist/server/task-ledger-keys.js +5 -0
- package/dist/server/task-ledger-limits.d.ts +34 -0
- package/dist/server/task-ledger-limits.js +5 -0
- package/dist/server/task-ledger-transition-helpers.d.ts +38 -0
- package/dist/server/task-ledger-transition-helpers.js +42 -0
- package/dist/server/task-ledger-transitions-cancellation.d.ts +31 -0
- package/dist/server/task-ledger-transitions-cancellation.js +73 -0
- package/dist/server/task-ledger-transitions.d.ts +129 -0
- package/dist/server/task-ledger-transitions.js +162 -0
- package/dist/server/task-ledger-types.d.ts +220 -0
- package/dist/server/task-ledger-types.js +1 -0
- package/dist/server/task-ledger.d.ts +17 -0
- package/dist/server/task-ledger.js +24 -0
- package/dist/server/task-state.d.ts +25 -152
- package/dist/server/task-state.js +0 -188
- package/dist/server/worker-admission-policy.d.ts +70 -0
- package/dist/server/worker-admission-policy.js +0 -0
- package/dist/storage/compressed-storage.js +1 -1
- package/dist/storage/index.d.ts +0 -2
- package/dist/storage/interface.d.ts +0 -2
- package/dist/storage/interface.js +1 -1
- package/dist/storage/lmdb.js +1 -1
- package/dist/storage/memory.js +1 -1
- package/dist/storage/neon.js +1 -1
- package/dist/storage/postgres.js +1 -1
- package/dist/storage/resolve.js +1 -1
- package/dist/storage/scoped-storage.js +1 -1
- package/dist/storage/testing.js +1 -1
- package/dist/storage/turso.js +1 -1
- package/dist/storage/web-extension.js +2 -2
- package/dist/version.d.ts +1 -1
- package/dist/version.js +1 -1
- package/dist/web-extension.js +1 -1
- package/dist/worker/index.d.ts +2 -0
- package/dist/worker/index.js +30 -4
- package/dist/worker/manifest/canonical-json.d.ts +20 -0
- package/dist/worker/manifest/canonical-json.js +13 -0
- package/dist/worker/manifest/capabilities.d.ts +24 -0
- package/dist/worker/manifest/capabilities.js +74 -0
- package/dist/worker/manifest/content-digest.d.ts +20 -0
- package/dist/worker/manifest/content-digest.js +13 -0
- package/dist/worker/manifest/declared-shape-digest.d.ts +14 -0
- package/dist/worker/manifest/declared-shape-digest.js +4 -0
- package/dist/worker/manifest/digest.d.ts +82 -0
- package/dist/worker/manifest/digest.js +9 -0
- package/dist/worker/manifest/execution-identity.d.ts +92 -0
- package/dist/worker/manifest/execution-identity.js +36 -0
- package/dist/worker/manifest/failure.d.ts +59 -0
- package/dist/worker/manifest/failure.js +4 -0
- package/dist/worker/manifest/index.d.ts +22 -0
- package/dist/worker/manifest/index.js +23 -0
- package/dist/worker/manifest/internal-realm.d.ts +50 -0
- package/dist/worker/manifest/internal-realm.js +39 -0
- package/dist/worker/manifest/is-record.d.ts +12 -0
- package/dist/worker/manifest/is-record.js +6 -0
- package/dist/worker/manifest/json-scan.d.ts +26 -0
- package/dist/worker/manifest/json-scan.js +88 -0
- package/dist/worker/manifest/limits.d.ts +138 -0
- package/dist/worker/manifest/limits.js +1 -0
- package/dist/worker/manifest/normalize.d.ts +63 -0
- package/dist/worker/manifest/normalize.js +73 -0
- package/dist/worker/manifest/parse-json.d.ts +24 -0
- package/dist/worker/manifest/parse-json.js +15 -0
- package/dist/worker/manifest/parse.d.ts +86 -0
- package/dist/worker/manifest/parse.js +188 -0
- package/dist/worker/manifest/registry-contract-builder.d.ts +120 -0
- package/dist/worker/manifest/registry-contract-builder.js +65 -0
- package/dist/worker/manifest/types.d.ts +263 -0
- package/dist/worker/manifest/types.js +1 -0
- package/dist/worker/manifest/utf8.d.ts +13 -0
- package/dist/worker/manifest/utf8.js +7 -0
- package/dist/worker/options.d.ts +55 -8
- package/dist/worker/options.js +65 -9
- package/dist/worker/protocol-internals.d.ts +1 -2
- package/dist/worker/protocol-internals.js +0 -5
- package/dist/worker/protocol-messages.d.ts +33 -17
- package/dist/worker/protocol-schemas.d.ts +36 -74
- package/dist/worker/protocol-schemas.js +25 -18
- package/dist/worker/protocol-version.d.ts +18 -14
- package/dist/worker/protocol-version.js +1 -1
- package/dist/worker/protocol.js +1 -1
- package/dist/worker/registry/deployment-consistency.d.ts +78 -0
- package/dist/worker/registry/deployment-consistency.js +27 -0
- package/dist/worker/registry/drain.d.ts +14 -0
- package/dist/worker/registry/drain.js +24 -0
- package/dist/worker/registry/rejections.d.ts +31 -0
- package/dist/worker/registry/rejections.js +10 -0
- package/dist/worker/registry/summary.d.ts +0 -3
- package/dist/worker/registry/summary.js +2 -5
- package/dist/worker/registry/types.d.ts +10 -3
- package/dist/worker/registry.d.ts +47 -2
- package/dist/worker/registry.js +45 -32
- package/dist/workers/workflow-worker-entry.d.ts +12 -7
- package/dist/workers/workflow-worker-entry.js +9 -2
- package/package.json +3 -3
- package/dist/server/task-resolved-record.d.ts +0 -12
- package/dist/server/task-resolved-record.js +0 -35
|
@@ -0,0 +1,64 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Shared commit loop for the durable remote task ledger's live dispatch path
|
|
3
|
+
* (WFT-22) — claim, heartbeat, completion, and requeue all attempt a single
|
|
4
|
+
* pure transition against the current ledger record through
|
|
5
|
+
* `storage.conditionalBatch`, retrying only when the compare-and-swap itself
|
|
6
|
+
* is lost (a concurrent writer changed the record between read and write),
|
|
7
|
+
* never when the transition's own precondition rejects the freshly re-read
|
|
8
|
+
* record. A precondition rejection means the requested transition genuinely
|
|
9
|
+
* does not apply — for example the task is already terminal — and retrying
|
|
10
|
+
* the same rejected transition would just reject again.
|
|
11
|
+
*
|
|
12
|
+
* `commitTaskLedgerDelete` (WFT-24) is the delete counterpart, used by
|
|
13
|
+
* terminal-task retention: a conditional delete gated on a precondition
|
|
14
|
+
* function rather than a transition function, since deletion has no next
|
|
15
|
+
* record to write.
|
|
16
|
+
*
|
|
17
|
+
* @module server/runtime/task-ledger-runtime
|
|
18
|
+
*/
|
|
19
|
+
import { type Storage } from '../../storage/interface.ts';
|
|
20
|
+
import type { TaskLedgerPreconditionResult, TaskLedgerTransitionResult } from '../task-ledger-transitions.ts';
|
|
21
|
+
import { type RemoteTaskRecord } from '../task-ledger.ts';
|
|
22
|
+
export type TaskLedgerCommitResult<T extends RemoteTaskRecord> = Readonly<{
|
|
23
|
+
ok: true;
|
|
24
|
+
record: T;
|
|
25
|
+
}> | Readonly<{
|
|
26
|
+
ok: false;
|
|
27
|
+
reason: string;
|
|
28
|
+
}>;
|
|
29
|
+
export type TaskLedgerDeleteResult = Readonly<{
|
|
30
|
+
ok: true;
|
|
31
|
+
}> | Readonly<{
|
|
32
|
+
ok: false;
|
|
33
|
+
reason: string;
|
|
34
|
+
}>;
|
|
35
|
+
/**
|
|
36
|
+
* Read the current ledger record, apply `transitionFn`, and commit the
|
|
37
|
+
* result with `expectedValue` set to the exact bytes just read — CAS by raw
|
|
38
|
+
* byte equality, not `encode(decode(bytes))` re-serialization (see
|
|
39
|
+
* `task-ledger-transitions.ts`'s module doc comment).
|
|
40
|
+
*
|
|
41
|
+
* Retries only a lost CAS (up to `maxAttempts`), re-reading and
|
|
42
|
+
* re-evaluating `transitionFn` against the fresh record each time — a
|
|
43
|
+
* concurrent heartbeat renewal, for instance, is a benign generation bump
|
|
44
|
+
* worth retrying past. A `transitionFn` precondition rejection returns
|
|
45
|
+
* immediately without consuming a retry, since the fresh record already
|
|
46
|
+
* reflects reality and re-attempting the identical transition cannot change
|
|
47
|
+
* the outcome.
|
|
48
|
+
*/
|
|
49
|
+
export declare function commitTaskLedgerTransition<T extends RemoteTaskRecord>(storage: Storage, operationId: string, transitionFn: (current: RemoteTaskRecord | null, now: number) => TaskLedgerTransitionResult<T>, maxAttempts?: number): Promise<TaskLedgerCommitResult<T>>;
|
|
50
|
+
/**
|
|
51
|
+
* Read the current ledger record, check a delete-only precondition (WFT-24
|
|
52
|
+
* retention: {@link canDeleteRetainedTerminalTask}), and conditionally delete
|
|
53
|
+
* with `expectedValue` set to the exact bytes just read — the delete
|
|
54
|
+
* counterpart to {@link commitTaskLedgerTransition}, needed because
|
|
55
|
+
* `canDeleteRetainedTerminalTask` has no next record to write, only a
|
|
56
|
+
* precondition to satisfy before issuing a delete.
|
|
57
|
+
*
|
|
58
|
+
* Retries only a lost CAS, same as `commitTaskLedgerTransition` — a
|
|
59
|
+
* concurrent writer that changed the record between read and delete (for
|
|
60
|
+
* example a later adoption call bumping `retentionGeneration`) means the
|
|
61
|
+
* fresh record deserves a fresh precondition check, not a delete of state
|
|
62
|
+
* the caller never actually observed.
|
|
63
|
+
*/
|
|
64
|
+
export declare function commitTaskLedgerDelete(storage: Storage, operationId: string, preconditionFn: (current: RemoteTaskRecord | null) => TaskLedgerPreconditionResult, maxAttempts?: number): Promise<TaskLedgerDeleteResult>;
|
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
import { storageConditionalBatch } from "../../storage/interface.js";
|
|
2
|
+
import {
|
|
3
|
+
decodeRemoteTaskRecord,
|
|
4
|
+
encodeRemoteTaskRecord,
|
|
5
|
+
taskLedgerKey
|
|
6
|
+
} from "../task-ledger.js";
|
|
7
|
+
export async function commitTaskLedgerTransition(storage, operationId, transitionFn, maxAttempts = 1) {
|
|
8
|
+
const key = taskLedgerKey(operationId);
|
|
9
|
+
for (let attempt = 1;attempt <= maxAttempts; attempt++) {
|
|
10
|
+
const rawExisting = await storage.get(key), current = decodeRemoteTaskRecord(rawExisting), result = transitionFn(current, Date.now());
|
|
11
|
+
if (!result.ok)
|
|
12
|
+
return result;
|
|
13
|
+
if (await storageConditionalBatch(storage, [{ key, expectedValue: rawExisting }], [{ type: "put", key, value: encodeRemoteTaskRecord(result.nextRecord) }]))
|
|
14
|
+
return { ok: !0, record: result.nextRecord };
|
|
15
|
+
}
|
|
16
|
+
return {
|
|
17
|
+
ok: !1,
|
|
18
|
+
reason: `lost the compare-and-swap race on operation "${operationId}" after ${String(maxAttempts)} attempt(s)`
|
|
19
|
+
};
|
|
20
|
+
}
|
|
21
|
+
export async function commitTaskLedgerDelete(storage, operationId, preconditionFn, maxAttempts = 1) {
|
|
22
|
+
const key = taskLedgerKey(operationId);
|
|
23
|
+
for (let attempt = 1;attempt <= maxAttempts; attempt++) {
|
|
24
|
+
const rawExisting = await storage.get(key), current = decodeRemoteTaskRecord(rawExisting), result = preconditionFn(current);
|
|
25
|
+
if (!result.ok)
|
|
26
|
+
return result;
|
|
27
|
+
if (await storageConditionalBatch(storage, [{ key, expectedValue: rawExisting }], [{ type: "delete", key }]))
|
|
28
|
+
return { ok: !0 };
|
|
29
|
+
}
|
|
30
|
+
return {
|
|
31
|
+
ok: !1,
|
|
32
|
+
reason: `lost the compare-and-swap race deleting operation "${operationId}" after ${String(maxAttempts)} attempt(s)`
|
|
33
|
+
};
|
|
34
|
+
}
|
|
@@ -1,12 +1,14 @@
|
|
|
1
1
|
import { type MetricsCollector } from '../../observability/metrics.ts';
|
|
2
2
|
import type { WorkerRegistry } from '../../worker/registry.ts';
|
|
3
3
|
import type { TaskQueue } from '../task-queue.ts';
|
|
4
|
-
import { type
|
|
4
|
+
import { type TaskTimingFields } from '../task-state.ts';
|
|
5
5
|
export declare function recordTaskBacklogMetric(metricsCollector: MetricsCollector | undefined, taskQueue: TaskQueue): void;
|
|
6
|
-
export declare function recordTaskQueueLatencyMetric(metricsCollector: MetricsCollector | undefined, record:
|
|
7
|
-
export declare function recordTaskExecutionLatencyMetric(metricsCollector: MetricsCollector | undefined, record:
|
|
6
|
+
export declare function recordTaskQueueLatencyMetric(metricsCollector: MetricsCollector | undefined, record: TaskTimingFields): void;
|
|
7
|
+
export declare function recordTaskExecutionLatencyMetric(metricsCollector: MetricsCollector | undefined, record: TaskTimingFields, completedAt: number): void;
|
|
8
8
|
export declare function recordTaskRetryMetric(metricsCollector: MetricsCollector | undefined, count?: number): void;
|
|
9
9
|
export declare function recordTaskRequeueMetric(metricsCollector: MetricsCollector | undefined, count?: number): void;
|
|
10
|
-
export declare function isTaskHeartbeatStaleForMetrics(record:
|
|
10
|
+
export declare function isTaskHeartbeatStaleForMetrics(record: TaskTimingFields & {
|
|
11
|
+
deadline?: number | undefined;
|
|
12
|
+
}, currentTime: number, staleAfterMs?: number): boolean;
|
|
11
13
|
export declare function recordTaskStaleHeartbeatMetric(metricsCollector: MetricsCollector | undefined, count: number): void;
|
|
12
14
|
export declare function recordWorkerCapacitySaturationMetric(metricsCollector: MetricsCollector | undefined, registry: WorkerRegistry): void;
|
|
@@ -1,23 +1,25 @@
|
|
|
1
1
|
import type { ServeOptions } from '../index.ts';
|
|
2
2
|
import { type Principal } from '../principal.ts';
|
|
3
3
|
import type { PendingTask } from '../task-queue-types.ts';
|
|
4
|
-
import type { InflightRecord } from '../task-state.ts';
|
|
5
4
|
import type { ServerContext } from './context.ts';
|
|
6
|
-
/**
|
|
7
|
-
* A freshly created long-poll inflight record always carries an `attemptToken`.
|
|
8
|
-
* The intersection makes that invariant a compile-time fact so the claim can read
|
|
9
|
-
* the token directly without an unreachable empty-string fallback.
|
|
10
|
-
*/
|
|
11
|
-
type TokenedInflightRecord = InflightRecord & {
|
|
12
|
-
attemptToken: string;
|
|
13
|
-
};
|
|
14
|
-
export declare function createLongPollInflightRecord(queue: string, task: PendingTask): TokenedInflightRecord;
|
|
15
5
|
/** The worker-facing identity of a long-poll claim: the synthetic worker id and its per-claim token. */
|
|
16
6
|
export interface LongPollClaim {
|
|
17
7
|
workerId: string;
|
|
18
8
|
attemptToken: string;
|
|
19
9
|
}
|
|
20
|
-
|
|
10
|
+
/**
|
|
11
|
+
* Conditionally claim the durable `queued` ledger record for a task the
|
|
12
|
+
* in-memory `TaskQueue` matched to a long-poll waiter. Returns `null` when
|
|
13
|
+
* the claim loses — the ledger disagrees with the in-memory match hint,
|
|
14
|
+
* meaning another actor already claimed or cancelled this operationId — in
|
|
15
|
+
* which case the caller treats the poll as if nothing matched, per "index
|
|
16
|
+
* disagreement never authorizes a state transition".
|
|
17
|
+
*
|
|
18
|
+
* Long-poll workers never call `WorkerRegistry.register()`, so there is no
|
|
19
|
+
* manifest to build a `WorkerExecutionIdentity` from; the claim always omits
|
|
20
|
+
* `executionIdentity` (see `RemoteTaskLeased.executionIdentity`'s doc
|
|
21
|
+
* comment) rather than fabricate one.
|
|
22
|
+
*/
|
|
23
|
+
export declare function markTaskClaimedByLongPollWorker(context: ServerContext, options: ServeOptions, task: PendingTask): Promise<LongPollClaim | null>;
|
|
21
24
|
export declare function handleTaskPollRequest(context: ServerContext, options: ServeOptions, request: Request, url: URL, principal?: Principal): Promise<Response | null>;
|
|
22
25
|
export declare function handleTaskResultRequest(context: ServerContext, options: ServeOptions, request: Request, url: URL, principal?: Principal): Promise<Response | null>;
|
|
23
|
-
export {};
|
|
@@ -1,11 +1,18 @@
|
|
|
1
1
|
import { isAuthenticated } from "../principal.js";
|
|
2
2
|
import { readRestJsonBody } from "../rest-body.js";
|
|
3
|
-
import {
|
|
4
|
-
import {
|
|
3
|
+
import { claimQueued } from "../task-ledger-transitions.js";
|
|
4
|
+
import { decodeRemoteTaskRecord, taskLedgerKey } from "../task-ledger.js";
|
|
5
5
|
import {
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
} from "./task-
|
|
6
|
+
commitTaskLedgerCompletion,
|
|
7
|
+
dispatchTaskDeadLetteredEvent
|
|
8
|
+
} from "./task-ledger-completion.js";
|
|
9
|
+
import { commitTaskLedgerTransition } from "./task-ledger-runtime.js";
|
|
10
|
+
import {
|
|
11
|
+
recordTaskBacklogMetric,
|
|
12
|
+
recordTaskExecutionLatencyMetric,
|
|
13
|
+
recordTaskQueueLatencyMetric
|
|
14
|
+
} from "./task-metrics.js";
|
|
15
|
+
import { taskResultPayloadSizeError } from "./task-result-resolution.js";
|
|
9
16
|
const TASK_POLL_RE = /^\/v1\/tasks\/([\w-]+)$/, TASK_RESULT_RE = /^\/v1\/tasks\/([\w-]+)\/result$/, TASK_DIAGNOSTICS_PATH = "/v1/tasks/diagnostics", MAX_POLL_TIMEOUT = 60000, DEFAULT_POLL_TIMEOUT = 30000, DEFAULT_VISIBILITY_TIMEOUT = 30000;
|
|
10
17
|
async function parseTaskResultBody(request, options) {
|
|
11
18
|
try {
|
|
@@ -26,6 +33,16 @@ function authorizeWorkerPrincipal(principal) {
|
|
|
26
33
|
return null;
|
|
27
34
|
return Response.json({ error: "Forbidden" }, { status: 403 });
|
|
28
35
|
}
|
|
36
|
+
async function awaitTaskLedgerRecovery(context) {
|
|
37
|
+
try {
|
|
38
|
+
await context.taskLedgerRecovery.ready;
|
|
39
|
+
return null;
|
|
40
|
+
} catch (error) {
|
|
41
|
+
return Response.json({
|
|
42
|
+
error: `Startup task-ledger recovery failed \u2014 cannot admit new task claims: ${error instanceof Error ? error.message : String(error)}`
|
|
43
|
+
}, { status: 503 });
|
|
44
|
+
}
|
|
45
|
+
}
|
|
29
46
|
function validateTaskResultBody(body) {
|
|
30
47
|
const { operationId, status } = body;
|
|
31
48
|
if (typeof operationId !== "string" || typeof status !== "string")
|
|
@@ -44,37 +61,35 @@ function validateTaskResultBody(body) {
|
|
|
44
61
|
attemptToken: rawAttemptToken
|
|
45
62
|
};
|
|
46
63
|
}
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
resolutionReason: resolvedStatus,
|
|
56
|
-
inflightRecord,
|
|
57
|
-
...status === "completed" ? { value } : { error }
|
|
58
|
-
});
|
|
59
|
-
} catch (storageError) {
|
|
60
|
-
console.error(`[weft] Failed to transition task "${operationId}" to resolved \u2014 inflight record may leak:`, storageError);
|
|
61
|
-
}
|
|
64
|
+
function isLongPollCompletionAuthorized(record, validated) {
|
|
65
|
+
if (record === null)
|
|
66
|
+
return !1;
|
|
67
|
+
if (record.state !== "leased" && record.state !== "completing")
|
|
68
|
+
return !1;
|
|
69
|
+
if (validated.workerId === void 0 || record.workerSessionId !== validated.workerId)
|
|
70
|
+
return !1;
|
|
71
|
+
return validated.attemptToken === record.attemptToken;
|
|
62
72
|
}
|
|
63
|
-
async function
|
|
64
|
-
|
|
65
|
-
operationId
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
await transitionTaskResultToResolvedWithRetry(context, options, {
|
|
71
|
-
operationId: result.operationId,
|
|
72
|
-
status: "failed",
|
|
73
|
-
resolutionReason: "failed",
|
|
74
|
-
inflightRecord,
|
|
75
|
-
error: error.message,
|
|
76
|
-
skipPayloadSizeCheck: !0
|
|
73
|
+
async function applyTaskResult(context, options, result) {
|
|
74
|
+
const { operationId, status, value, error } = result, committed = await commitTaskLedgerCompletion(options.engine.storage, {
|
|
75
|
+
operationId,
|
|
76
|
+
attemptToken: result.attemptToken,
|
|
77
|
+
status,
|
|
78
|
+
...status === "completed" ? { value } : {},
|
|
79
|
+
...status === "failed" && error !== void 0 ? { error } : {}
|
|
77
80
|
});
|
|
81
|
+
if (!committed.ok) {
|
|
82
|
+
if (committed.deadLettered !== void 0) {
|
|
83
|
+
context.deadlineTracker.remove(operationId);
|
|
84
|
+
dispatchTaskDeadLetteredEvent(options, operationId, committed.deadLettered, result.workerId);
|
|
85
|
+
}
|
|
86
|
+
return { ok: !1, reason: committed.reason };
|
|
87
|
+
}
|
|
88
|
+
recordTaskExecutionLatencyMetric(context.metricsCollector, { startedAt: committed.completing.startedAt }, Date.now());
|
|
89
|
+
context.taskQueue.complete({ operationId, status, value, error });
|
|
90
|
+
context.deadlineTracker.remove(operationId);
|
|
91
|
+
recordTaskBacklogMetric(context.metricsCollector, context.taskQueue);
|
|
92
|
+
return { ok: !0 };
|
|
78
93
|
}
|
|
79
94
|
function payloadSizeExceededResponse(error) {
|
|
80
95
|
return Response.json({
|
|
@@ -85,43 +100,32 @@ function payloadSizeExceededResponse(error) {
|
|
|
85
100
|
payloadKind: error.payloadKind
|
|
86
101
|
}, { status: 413 });
|
|
87
102
|
}
|
|
88
|
-
export function
|
|
89
|
-
const
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
lastQueuedAt: task.lastQueuedAt ?? task.enqueuedAt ?? now,
|
|
105
|
-
lastDispatchedAt: now,
|
|
106
|
-
startedAt: now,
|
|
107
|
-
retryCount: task.retryCount ?? Math.max(0, (task.attempt ?? 1) - 1),
|
|
108
|
-
requeueCount: task.requeueCount ?? 0,
|
|
109
|
-
lastRequeueReason: task.lastRequeueReason
|
|
110
|
-
};
|
|
111
|
-
}
|
|
112
|
-
export async function markTaskClaimedByLongPollWorker(context, options, queue, task) {
|
|
113
|
-
const inflightRecord = createLongPollInflightRecord(queue, task);
|
|
103
|
+
export async function markTaskClaimedByLongPollWorker(context, options, task) {
|
|
104
|
+
const workerSessionId = `longpoll-${crypto.randomUUID().slice(0, 8)}`, attemptToken = crypto.randomUUID(), visibilityTimeout = task.visibilityTimeout ?? DEFAULT_VISIBILITY_TIMEOUT, result = await commitTaskLedgerTransition(options.engine.storage, task.operationId, (current, now) => {
|
|
105
|
+
if (current === null || current.state !== "queued")
|
|
106
|
+
return {
|
|
107
|
+
ok: !1,
|
|
108
|
+
reason: `operation "${task.operationId}" is not claimable from its current ledger state`
|
|
109
|
+
};
|
|
110
|
+
return claimQueued(current, {
|
|
111
|
+
expectedGeneration: current.generation,
|
|
112
|
+
attemptToken,
|
|
113
|
+
workerSessionId,
|
|
114
|
+
leaseDurationMilliseconds: visibilityTimeout
|
|
115
|
+
}, now);
|
|
116
|
+
}, 1);
|
|
117
|
+
if (!result.ok)
|
|
118
|
+
return null;
|
|
114
119
|
context.deadlineTracker.add({
|
|
115
120
|
operationId: task.operationId,
|
|
116
|
-
deadline:
|
|
121
|
+
deadline: result.record.leaseDeadline
|
|
122
|
+
});
|
|
123
|
+
recordTaskQueueLatencyMetric(context.metricsCollector, {
|
|
124
|
+
lastQueuedAt: result.record.lastQueuedAt,
|
|
125
|
+
lastDispatchedAt: Date.now()
|
|
117
126
|
});
|
|
118
|
-
const normalizedInflightRecord = await transitionQueuedToInflight(options.engine.storage, task.operationId, inflightRecord);
|
|
119
|
-
recordTaskQueueLatencyMetric(context.metricsCollector, normalizedInflightRecord);
|
|
120
127
|
recordTaskBacklogMetric(context.metricsCollector, context.taskQueue);
|
|
121
|
-
return {
|
|
122
|
-
workerId: normalizedInflightRecord.workerId,
|
|
123
|
-
attemptToken: inflightRecord.attemptToken
|
|
124
|
-
};
|
|
128
|
+
return { workerId: workerSessionId, attemptToken };
|
|
125
129
|
}
|
|
126
130
|
export async function handleTaskPollRequest(context, options, request, url, principal) {
|
|
127
131
|
if (request.method !== "GET")
|
|
@@ -134,12 +138,17 @@ export async function handleTaskPollRequest(context, options, request, url, prin
|
|
|
134
138
|
const authorizationResponse = authorizeWorkerPrincipal(principal);
|
|
135
139
|
if (authorizationResponse !== null)
|
|
136
140
|
return authorizationResponse;
|
|
141
|
+
const recoveryResponse = await awaitTaskLedgerRecovery(context);
|
|
142
|
+
if (recoveryResponse !== null)
|
|
143
|
+
return recoveryResponse;
|
|
137
144
|
const queue = decodeURIComponent(pollMatch[1]), activities = url.searchParams.getAll("activity");
|
|
138
145
|
if (activities.length === 0)
|
|
139
146
|
return Response.json({ error: 'At least one "activity" query parameter is required' }, { status: 400 });
|
|
140
147
|
const rawTimeout = url.searchParams.get("timeout"), timeout = rawTimeout !== null ? Math.min(Math.max(0, Number(rawTimeout)), MAX_POLL_TIMEOUT) : DEFAULT_POLL_TIMEOUT, task = await context.taskQueue.poll(queue, activities, timeout, request.signal);
|
|
141
148
|
if (task !== null) {
|
|
142
|
-
const claim = await markTaskClaimedByLongPollWorker(context, options,
|
|
149
|
+
const claim = await markTaskClaimedByLongPollWorker(context, options, task);
|
|
150
|
+
if (claim === null)
|
|
151
|
+
return new Response(null, { status: 204 });
|
|
143
152
|
return Response.json({
|
|
144
153
|
...task,
|
|
145
154
|
workerId: claim.workerId,
|
|
@@ -159,6 +168,9 @@ export async function handleTaskResultRequest(context, options, request, url, pr
|
|
|
159
168
|
const authorizationResponse = authorizeWorkerPrincipal(principal);
|
|
160
169
|
if (authorizationResponse !== null)
|
|
161
170
|
return authorizationResponse;
|
|
171
|
+
const recoveryResponse = await awaitTaskLedgerRecovery(context);
|
|
172
|
+
if (recoveryResponse !== null)
|
|
173
|
+
return recoveryResponse;
|
|
162
174
|
const body = await parseTaskResultBody(request, options.maxRequestBodyBytes !== void 0 ? { maxBodyBytes: options.maxRequestBodyBytes } : {});
|
|
163
175
|
if (body instanceof Response)
|
|
164
176
|
return body;
|
|
@@ -167,28 +179,40 @@ export async function handleTaskResultRequest(context, options, request, url, pr
|
|
|
167
179
|
const validated = validateTaskResultBody(body);
|
|
168
180
|
if (validated instanceof Response)
|
|
169
181
|
return validated;
|
|
170
|
-
const
|
|
171
|
-
if (
|
|
182
|
+
const record = decodeRemoteTaskRecord(await options.engine.storage.get(taskLedgerKey(validated.operationId)));
|
|
183
|
+
if (!isLongPollCompletionAuthorized(record, validated))
|
|
172
184
|
return Response.json({ error: "Forbidden" }, { status: 403 });
|
|
173
185
|
const payloadError = taskResultPayloadSizeError({
|
|
174
|
-
operationId: validated.operationId,
|
|
175
186
|
status: validated.status,
|
|
176
|
-
resolutionReason: validated.status,
|
|
177
187
|
...validated.status === "completed" ? { value: validated.value } : { error: validated.error }
|
|
178
188
|
}, context.payloadSizeMaxBytes);
|
|
179
189
|
if (payloadError !== null) {
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
|
|
190
|
+
const rejected = await commitTaskLedgerCompletion(options.engine.storage, {
|
|
191
|
+
operationId: validated.operationId,
|
|
192
|
+
attemptToken: validated.attemptToken,
|
|
193
|
+
status: "failed",
|
|
194
|
+
error: payloadError.message
|
|
195
|
+
});
|
|
196
|
+
if (rejected.ok) {
|
|
197
|
+
recordTaskExecutionLatencyMetric(context.metricsCollector, { startedAt: rejected.completing.startedAt }, Date.now());
|
|
198
|
+
context.taskQueue.complete({
|
|
199
|
+
operationId: validated.operationId,
|
|
200
|
+
status: "failed",
|
|
201
|
+
error: payloadError.message
|
|
202
|
+
});
|
|
203
|
+
context.deadlineTracker.remove(validated.operationId);
|
|
204
|
+
recordTaskBacklogMetric(context.metricsCollector, context.taskQueue);
|
|
205
|
+
} else {
|
|
206
|
+
console.error(`[weft] Failed to persist oversized task result rejection for task "${validated.operationId}":`, rejected.reason);
|
|
207
|
+
if (rejected.deadLettered !== void 0)
|
|
208
|
+
dispatchTaskDeadLetteredEvent(options, validated.operationId, rejected.deadLettered, validated.workerId);
|
|
184
209
|
}
|
|
185
210
|
return payloadSizeExceededResponse(payloadError);
|
|
186
211
|
}
|
|
187
|
-
await applyTaskResult(context, options, validated
|
|
212
|
+
const applied = await applyTaskResult(context, options, validated);
|
|
213
|
+
if (!applied.ok) {
|
|
214
|
+
console.error(`[weft] Failed to commit task result for "${validated.operationId}" through the durable ledger:`, applied.reason);
|
|
215
|
+
return Response.json({ error: "Forbidden" }, { status: 403 });
|
|
216
|
+
}
|
|
188
217
|
return Response.json({ ok: !0 });
|
|
189
218
|
}
|
|
190
|
-
function isLongPollCompletionAuthorized(inflightRecord, validated) {
|
|
191
|
-
if (validated.workerId === void 0 || inflightRecord.workerId !== validated.workerId)
|
|
192
|
-
return !1;
|
|
193
|
-
return validated.attemptToken === inflightRecord.attemptToken;
|
|
194
|
-
}
|
|
@@ -1,5 +1,5 @@
|
|
|
1
|
-
import type { ServeOptions } from '../index.ts';
|
|
2
|
-
import type
|
|
1
|
+
import type { ServeOptions, TaskDispatch } from '../index.ts';
|
|
2
|
+
import { type RemoteTaskBase, type RemoteTaskLeased } from '../task-ledger.ts';
|
|
3
3
|
import type { ServerContext } from './context.ts';
|
|
4
4
|
interface ManualTaskReconciliationForTesting {
|
|
5
5
|
readonly options: ServeOptions;
|
|
@@ -10,17 +10,44 @@ export declare function useManualTaskReconciliationForTesting(options: ServeOpti
|
|
|
10
10
|
/** @internal Installs the narrow one-shot test controller before a server starts. */
|
|
11
11
|
export declare function consumeManualTaskReconciliationForTesting(options: ServeOptions, context: ServerContext, cleanupWorkflowIndex: (operationId: string) => void): boolean;
|
|
12
12
|
/**
|
|
13
|
-
* Given a
|
|
14
|
-
* retry attempts
|
|
15
|
-
*
|
|
16
|
-
* visibility-timeout scanner share this logic.
|
|
13
|
+
* Given a leased ledger record whose visibility deadline has passed, either
|
|
14
|
+
* permanently fail the task (retry attempts exhausted) or requeue it with
|
|
15
|
+
* backoff for redispatch. Both the worker-disconnect handler and the
|
|
16
|
+
* visibility-timeout scanner share this logic. A single attempt, no retry —
|
|
17
|
+
* a lost compare-and-swap here means a concurrent heartbeat renewed the
|
|
18
|
+
* lease or another actor already resolved this attempt, and retrying the
|
|
19
|
+
* identical transition cannot change that outcome.
|
|
17
20
|
*/
|
|
18
|
-
export declare function reassignOrExpireTask(context: ServerContext, options: ServeOptions, operationId: string, record:
|
|
21
|
+
export declare function reassignOrExpireTask(context: ServerContext, options: ServeOptions, operationId: string, record: RemoteTaskLeased, reason?: string): Promise<void>;
|
|
22
|
+
/**
|
|
23
|
+
* Reconstruct the `TaskDispatch` a fresh dispatch of this operation would
|
|
24
|
+
* have used, from the durable envelope alone. Shared by expired-lease
|
|
25
|
+
* redispatch ({@link reassignOrExpireTask}) and startup recovery's
|
|
26
|
+
* `queued`-record redispatch (`task-ledger-recovery.ts`) — both cases hand a
|
|
27
|
+
* record back to `dispatchTaskImpl` with no original caller-supplied
|
|
28
|
+
* `TaskDispatch` still in memory. Every `RemoteTaskBase` field
|
|
29
|
+
* `buildRoutingOptions`/`resolveTaskPriority` read at dispatch time
|
|
30
|
+
* (`priority`, `fairShareKey`, sticky affinity) must round-trip here too, or
|
|
31
|
+
* a requeued or recovered task silently loses its original routing intent.
|
|
32
|
+
*/
|
|
33
|
+
export declare function taskDispatchFromLedgerRecord(record: RemoteTaskBase): TaskDispatch;
|
|
19
34
|
/**
|
|
20
35
|
* Drain expired entries from the in-memory deadline heap and reassign
|
|
21
36
|
* their tasks. Only touches storage for the specific operations whose
|
|
22
|
-
* deadlines have actually passed — no full
|
|
37
|
+
* deadlines have actually passed — no full ledger scan.
|
|
23
38
|
*/
|
|
24
39
|
export declare function scanExpiredTasks(context: ServerContext, options: ServeOptions, cleanupWorkflowIndex: (operationId: string) => void, now?: number): Promise<void>;
|
|
40
|
+
/**
|
|
41
|
+
* Periodic full-ledger safety net (WFT-23), independent of the in-memory
|
|
42
|
+
* deadline heap `scanExpiredTasks` drains: catches `leased` records whose
|
|
43
|
+
* expiry was never tracked in the heap (written by a peer, or lost to a
|
|
44
|
+
* restart), `queued` records whose durable `availableAt` has elapsed but
|
|
45
|
+
* whose `scheduleDelayedDispatch` timer never fired (same causes), and
|
|
46
|
+
* (WFT-24) adopted `terminal` records old enough to reap under
|
|
47
|
+
* {@link ServeOptions.taskRetentionWindowMs}. `completing`, `cancelling`,
|
|
48
|
+
* and `deadLettered` are left untouched — resolving those is either the
|
|
49
|
+
* worker's redelivered result (`completing`) or explicitly out of this
|
|
50
|
+
* slice's scope.
|
|
51
|
+
*/
|
|
25
52
|
export declare function reconcileOrphanedRecords(context: ServerContext, options: ServeOptions, cleanupWorkflowIndex: (operationId: string) => void): Promise<void>;
|
|
26
53
|
export {};
|