@lostgradient/weft 0.18.0 → 0.19.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +3 -1
- package/dist/cli/conformance.js +14 -18
- package/dist/cli/generated/operation-catalog.snapshot.json +258 -16
- package/dist/cli/generated/operation-client.generated.d.ts +30 -6
- package/dist/cli/generated/operation-client.generated.js +4 -0
- package/dist/core/context/child-workflow-pipe.d.ts +1 -1
- package/dist/core/context/child-workflow-pipe.js +1 -1
- package/dist/core/engine/construction.d.ts +2 -0
- package/dist/core/engine/construction.js +3 -0
- package/dist/core/engine/index.js +1 -0
- package/dist/core/engine/listing.d.ts +1 -0
- package/dist/core/engine/listing.js +6 -4
- package/dist/core/weft-error.d.ts +1 -1
- package/dist/core/weft-error.js +2 -1
- package/dist/core/worker-execution-dispatcher.d.ts +8 -0
- package/dist/core/worker-execution-dispatcher.js +4 -0
- package/dist/core/worker-execution-strategy-options.d.ts +16 -0
- package/dist/core/worker-execution-strategy.js +71 -89
- package/dist/core/worker-fault-handling.d.ts +45 -0
- package/dist/core/worker-fault-handling.js +88 -0
- package/dist/core/worker-protocol.d.ts +18 -1
- package/dist/core/worker-protocol.js +1 -1
- package/dist/core/worker-realm-readiness.d.ts +78 -0
- package/dist/core/worker-realm-readiness.js +127 -0
- package/dist/index.d.ts +3 -1
- package/dist/index.js +28 -1
- package/dist/json-schema.js +1 -1
- package/dist/runtime/portable.d.ts +14 -0
- package/dist/runtime/portable.js +10 -0
- package/dist/server/index.d.ts +71 -8
- package/dist/server/index.js +7 -3
- package/dist/server/operations/get-task-diagnostics.d.ts +43 -18
- package/dist/server/operations/get-task-diagnostics.js +87 -126
- package/dist/server/operations/get-worker-diagnostics.d.ts +177 -0
- package/dist/server/operations/get-worker-diagnostics.js +117 -0
- package/dist/server/operations/list-worker-registration-rejections.d.ts +79 -0
- package/dist/server/operations/list-worker-registration-rejections.js +60 -0
- package/dist/server/operations/list-workers.js +1 -3
- package/dist/server/operations/static-registrations.js +5 -20
- package/dist/server/rest-bindings.js +31 -2
- package/dist/server/runtime/authentication-bridge.js +6 -14
- package/dist/server/runtime/context.d.ts +23 -0
- package/dist/server/runtime/task-dispatch.d.ts +7 -1
- package/dist/server/runtime/task-dispatch.js +126 -55
- package/dist/server/runtime/task-ledger-completion.d.ts +68 -0
- package/dist/server/runtime/task-ledger-completion.js +72 -0
- package/dist/server/runtime/task-ledger-recovery.d.ts +55 -0
- package/dist/server/runtime/task-ledger-recovery.js +73 -0
- package/dist/server/runtime/task-ledger-runtime.d.ts +64 -0
- package/dist/server/runtime/task-ledger-runtime.js +34 -0
- package/dist/server/runtime/task-metrics.d.ts +6 -4
- package/dist/server/runtime/task-polling.d.ts +14 -12
- package/dist/server/runtime/task-polling.js +106 -82
- package/dist/server/runtime/task-reconciliation.d.ts +35 -8
- package/dist/server/runtime/task-reconciliation.js +75 -74
- package/dist/server/runtime/task-result-resolution.d.ts +2 -10
- package/dist/server/runtime/task-result-resolution.js +0 -67
- package/dist/server/runtime/task-result-view.d.ts +82 -0
- package/dist/server/runtime/task-result-view.js +48 -0
- package/dist/server/runtime/websocket-worker-messaging.d.ts +16 -0
- package/dist/server/runtime/websocket-worker-messaging.js +32 -0
- package/dist/server/runtime/websocket-worker-registration.d.ts +41 -0
- package/dist/server/runtime/websocket-worker-registration.js +134 -0
- package/dist/server/runtime/websocket-worker.d.ts +0 -1
- package/dist/server/runtime/websocket-worker.js +64 -116
- package/dist/server/serve-internals.d.ts +9 -6
- package/dist/server/serve-internals.js +14 -40
- package/dist/server/task-ledger-codec.d.ts +40 -0
- package/dist/server/task-ledger-codec.js +195 -0
- package/dist/server/task-ledger-keys.d.ts +13 -0
- package/dist/server/task-ledger-keys.js +5 -0
- package/dist/server/task-ledger-limits.d.ts +34 -0
- package/dist/server/task-ledger-limits.js +5 -0
- package/dist/server/task-ledger-transition-helpers.d.ts +38 -0
- package/dist/server/task-ledger-transition-helpers.js +42 -0
- package/dist/server/task-ledger-transitions-cancellation.d.ts +31 -0
- package/dist/server/task-ledger-transitions-cancellation.js +73 -0
- package/dist/server/task-ledger-transitions.d.ts +129 -0
- package/dist/server/task-ledger-transitions.js +162 -0
- package/dist/server/task-ledger-types.d.ts +220 -0
- package/dist/server/task-ledger-types.js +1 -0
- package/dist/server/task-ledger.d.ts +17 -0
- package/dist/server/task-ledger.js +24 -0
- package/dist/server/task-state.d.ts +25 -152
- package/dist/server/task-state.js +0 -188
- package/dist/server/worker-admission-policy.d.ts +70 -0
- package/dist/server/worker-admission-policy.js +0 -0
- package/dist/storage/compressed-storage.js +1 -1
- package/dist/storage/index.d.ts +0 -2
- package/dist/storage/interface.d.ts +0 -2
- package/dist/storage/interface.js +1 -1
- package/dist/storage/lmdb.js +1 -1
- package/dist/storage/memory.js +1 -1
- package/dist/storage/neon.js +1 -1
- package/dist/storage/postgres.js +1 -1
- package/dist/storage/resolve.js +1 -1
- package/dist/storage/scoped-storage.js +1 -1
- package/dist/storage/testing.js +1 -1
- package/dist/storage/turso.js +1 -1
- package/dist/storage/web-extension.js +2 -2
- package/dist/version.d.ts +1 -1
- package/dist/version.js +1 -1
- package/dist/web-extension.js +1 -1
- package/dist/worker/index.d.ts +2 -0
- package/dist/worker/index.js +30 -4
- package/dist/worker/manifest/canonical-json.d.ts +20 -0
- package/dist/worker/manifest/canonical-json.js +13 -0
- package/dist/worker/manifest/capabilities.d.ts +24 -0
- package/dist/worker/manifest/capabilities.js +74 -0
- package/dist/worker/manifest/content-digest.d.ts +20 -0
- package/dist/worker/manifest/content-digest.js +13 -0
- package/dist/worker/manifest/declared-shape-digest.d.ts +14 -0
- package/dist/worker/manifest/declared-shape-digest.js +4 -0
- package/dist/worker/manifest/digest.d.ts +82 -0
- package/dist/worker/manifest/digest.js +9 -0
- package/dist/worker/manifest/execution-identity.d.ts +92 -0
- package/dist/worker/manifest/execution-identity.js +36 -0
- package/dist/worker/manifest/failure.d.ts +59 -0
- package/dist/worker/manifest/failure.js +4 -0
- package/dist/worker/manifest/index.d.ts +22 -0
- package/dist/worker/manifest/index.js +23 -0
- package/dist/worker/manifest/internal-realm.d.ts +50 -0
- package/dist/worker/manifest/internal-realm.js +39 -0
- package/dist/worker/manifest/is-record.d.ts +12 -0
- package/dist/worker/manifest/is-record.js +6 -0
- package/dist/worker/manifest/json-scan.d.ts +26 -0
- package/dist/worker/manifest/json-scan.js +88 -0
- package/dist/worker/manifest/limits.d.ts +138 -0
- package/dist/worker/manifest/limits.js +1 -0
- package/dist/worker/manifest/normalize.d.ts +63 -0
- package/dist/worker/manifest/normalize.js +73 -0
- package/dist/worker/manifest/parse-json.d.ts +24 -0
- package/dist/worker/manifest/parse-json.js +15 -0
- package/dist/worker/manifest/parse.d.ts +86 -0
- package/dist/worker/manifest/parse.js +188 -0
- package/dist/worker/manifest/registry-contract-builder.d.ts +120 -0
- package/dist/worker/manifest/registry-contract-builder.js +65 -0
- package/dist/worker/manifest/types.d.ts +263 -0
- package/dist/worker/manifest/types.js +1 -0
- package/dist/worker/manifest/utf8.d.ts +13 -0
- package/dist/worker/manifest/utf8.js +7 -0
- package/dist/worker/options.d.ts +55 -8
- package/dist/worker/options.js +65 -9
- package/dist/worker/protocol-internals.d.ts +1 -2
- package/dist/worker/protocol-internals.js +0 -5
- package/dist/worker/protocol-messages.d.ts +33 -17
- package/dist/worker/protocol-schemas.d.ts +36 -74
- package/dist/worker/protocol-schemas.js +25 -18
- package/dist/worker/protocol-version.d.ts +18 -14
- package/dist/worker/protocol-version.js +1 -1
- package/dist/worker/protocol.js +1 -1
- package/dist/worker/registry/deployment-consistency.d.ts +78 -0
- package/dist/worker/registry/deployment-consistency.js +27 -0
- package/dist/worker/registry/drain.d.ts +14 -0
- package/dist/worker/registry/drain.js +24 -0
- package/dist/worker/registry/rejections.d.ts +31 -0
- package/dist/worker/registry/rejections.js +10 -0
- package/dist/worker/registry/summary.d.ts +0 -3
- package/dist/worker/registry/summary.js +2 -5
- package/dist/worker/registry/types.d.ts +10 -3
- package/dist/worker/registry.d.ts +47 -2
- package/dist/worker/registry.js +45 -32
- package/dist/workers/workflow-worker-entry.d.ts +12 -7
- package/dist/workers/workflow-worker-entry.js +9 -2
- package/package.json +3 -3
- package/dist/server/task-resolved-record.d.ts +0 -12
- package/dist/server/task-resolved-record.js +0 -35
|
@@ -25,7 +25,7 @@
|
|
|
25
25
|
* }
|
|
26
26
|
* ```
|
|
27
27
|
*/
|
|
28
|
-
export type WeftErrorCode = 'WorkflowAlreadyExistsError' | 'BulkDeleteRequiresTerminalWorkflowsError' | 'BulkOperationConfirmationError' | 'WorkflowTypeNotRegisteredForRecoveryError' | 'EngineCreateNameMismatchError' | 'EngineDisposedError' | 'EngineDisposalError' | 'WorkflowNotFoundError' | 'WorkflowNotRegisteredError' | 'WorkflowConcurrencyLimitExceededError' | 'WorkflowSuspendNotSupportedError' | 'ActivityResolutionError' | 'BranchTopologyChangedError' | 'PersistedDataIncompatibleError' | 'WorkflowTimeoutError' | 'HttpClientError' | 'WorkerProtocolIncompatibleError' | 'UpdateTimeoutError' | 'UpdateValidationError' | 'WorkflowTerminalError' | 'WorkflowBuilderError' | 'VersionMismatchError' | 'EffectReplayConflictError' | 'ReviewTimeoutError' | 'AtomicStateConflictError' | 'StandardSchemaValidationError' | 'ActivityReconciliationCapabilityError' | 'ActivityReconciliationConflictError' | 'ActivityReconciliationIndeterminateError' | 'DurableActivityScopeError' | 'DurableActivityUnsupportedError' | 'AsyncActivityTokenNotFoundError' | 'ActivityScheduleToCloseTimeoutError' | 'ActivityPerAttemptTimeoutError' | 'PayloadSizeExceededError' | 'StartOrSignalConflictError' | 'WorkflowTeardownPendingError' | 'IdempotencyKeyPurgedError';
|
|
28
|
+
export type WeftErrorCode = 'WorkflowAlreadyExistsError' | 'BulkDeleteRequiresTerminalWorkflowsError' | 'BulkOperationConfirmationError' | 'WorkflowTypeNotRegisteredForRecoveryError' | 'EngineCreateNameMismatchError' | 'EngineDisposedError' | 'EngineDisposalError' | 'WorkflowNotFoundError' | 'WorkflowNotRegisteredError' | 'WorkflowConcurrencyLimitExceededError' | 'WorkflowSuspendNotSupportedError' | 'ActivityResolutionError' | 'BranchTopologyChangedError' | 'PersistedDataIncompatibleError' | 'WorkflowTimeoutError' | 'HttpClientError' | 'WorkerProtocolIncompatibleError' | 'UpdateTimeoutError' | 'UpdateValidationError' | 'WorkflowTerminalError' | 'WorkflowBuilderError' | 'VersionMismatchError' | 'EffectReplayConflictError' | 'ReviewTimeoutError' | 'AtomicStateConflictError' | 'StandardSchemaValidationError' | 'ActivityReconciliationCapabilityError' | 'ActivityReconciliationConflictError' | 'ActivityReconciliationIndeterminateError' | 'DurableActivityScopeError' | 'DurableActivityUnsupportedError' | 'AsyncActivityTokenNotFoundError' | 'ActivityScheduleToCloseTimeoutError' | 'ActivityPerAttemptTimeoutError' | 'PayloadSizeExceededError' | 'StartOrSignalConflictError' | 'WorkflowTeardownPendingError' | 'IdempotencyKeyPurgedError' | 'WorkerManifestBuildError';
|
|
29
29
|
/**
|
|
30
30
|
* Generic abstract base for all Weft library errors. The `TCode` parameter
|
|
31
31
|
* makes each subclass's `code` its own literal type; the exported base surface
|
package/dist/core/weft-error.js
CHANGED
|
@@ -44,7 +44,8 @@ const publicWeftErrorCodeMap = {
|
|
|
44
44
|
PayloadSizeExceededError: !0,
|
|
45
45
|
StartOrSignalConflictError: !0,
|
|
46
46
|
WorkflowTeardownPendingError: !0,
|
|
47
|
-
IdempotencyKeyPurgedError: !0
|
|
47
|
+
IdempotencyKeyPurgedError: !0,
|
|
48
|
+
WorkerManifestBuildError: !0
|
|
48
49
|
}, PUBLIC_WEFT_ERROR_CODES = new Set(Object.keys(publicWeftErrorCodeMap));
|
|
49
50
|
export function isWeftError(value) {
|
|
50
51
|
return value instanceof WeftError;
|
|
@@ -9,6 +9,14 @@ export interface WorkerExecutionDispatcherDependencies {
|
|
|
9
9
|
validateHostToWorkerMessage: (workflowId: string, message: WorkerInboundMessage, worker?: Worker) => boolean;
|
|
10
10
|
attachWorkerListeners: (worker: Worker) => void;
|
|
11
11
|
detachWorkerListenersIfIdle: (worker: Worker) => void;
|
|
12
|
+
/**
|
|
13
|
+
* Resolve once `worker`'s realm-ready handshake has settled (or
|
|
14
|
+
* immediately, if realm-readiness isn't required). `false` means the
|
|
15
|
+
* handshake failed or timed out; the implementation is responsible for
|
|
16
|
+
* discarding the worker and emitting the workflow failure itself, the same
|
|
17
|
+
* contract `validateHostToWorkerMessage` already uses.
|
|
18
|
+
*/
|
|
19
|
+
ensureRealmReady: (worker: Worker, workflowId: string) => Promise<boolean>;
|
|
12
20
|
beginTurn: (worker: Worker, workflowId: string, turnId: number, kind: 'run' | 'resume') => void;
|
|
13
21
|
clearTurn: (worker: Worker) => void;
|
|
14
22
|
discardWorkerAndFailWorkflows: (worker: Worker, options: {
|
|
@@ -11,6 +11,10 @@ export class WorkerExecutionDispatcher {
|
|
|
11
11
|
const worker = await this.#dependencies.pool.acquire();
|
|
12
12
|
this.#dependencies.ownership.setActive(workflowId, worker);
|
|
13
13
|
this.#dependencies.attachWorkerListeners(worker);
|
|
14
|
+
if (!await this.#dependencies.ensureRealmReady(worker, workflowId))
|
|
15
|
+
return;
|
|
16
|
+
if (this.#dependencies.isDisposed() || this.#dependencies.ownership.getActiveWorker(workflowId) !== worker)
|
|
17
|
+
return;
|
|
14
18
|
this.#sendActiveMessage(worker, workflowId, message, "run");
|
|
15
19
|
} catch (error) {
|
|
16
20
|
this.#dependencies.emit({
|
|
@@ -5,6 +5,22 @@ export interface WorkerExecutionStrategyOptions {
|
|
|
5
5
|
maxProtocolMessageBytes?: number;
|
|
6
6
|
requireProtocolVersion?: boolean;
|
|
7
7
|
discardOnCancel?: boolean;
|
|
8
|
+
/**
|
|
9
|
+
* Require every freshly acquired worker to complete the realm-ready
|
|
10
|
+
* handshake (WFT-28) — send a valid `ready` message whose manifest
|
|
11
|
+
* contains a matching contract for every type in
|
|
12
|
+
* {@link getExpectedWorkflowTypes} (a subset check, not exact-manifest
|
|
13
|
+
* equality) — before it receives its first `run` turn. Defaults to
|
|
14
|
+
* `false` so direct/test construction of this class is unaffected;
|
|
15
|
+
* `createExecutionStrategyBundle` hardcodes this to `true` for every
|
|
16
|
+
* engine-constructed worker-mode strategy, mirroring how
|
|
17
|
+
* `requireProtocolVersion` is already hardcoded there.
|
|
18
|
+
*/
|
|
19
|
+
requireRealmReady?: boolean;
|
|
20
|
+
/** Required when {@link requireRealmReady} is `true`. Called fresh on every handshake. */
|
|
21
|
+
getExpectedWorkflowTypes?: () => readonly string[];
|
|
22
|
+
/** Bound (ms) on the realm-ready wait. Defaults to `DEFAULT_WORKER_REALM_READY_TIMEOUT_MS`. */
|
|
23
|
+
realmReadyTimeoutMs?: number;
|
|
8
24
|
/**
|
|
9
25
|
* The engine host's `EngineOptions.onLog` sink (#529). When present, the strategy
|
|
10
26
|
* tells each worker (`hostHasLogSink: true` on `run`/`resume`) to forward `ctx.log`
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import { WorkerCheckpointResumeState } from "./worker-checkpoint-resume-state.js";
|
|
2
2
|
import { WorkerExecutionDispatcher } from "./worker-execution-dispatcher.js";
|
|
3
3
|
import { WorkerExecutionOwnership } from "./worker-execution-ownership.js";
|
|
4
|
+
import { WorkerFaultHandler } from "./worker-fault-handling.js";
|
|
4
5
|
import {
|
|
5
6
|
buildResumeMessage,
|
|
6
7
|
buildRunMessage
|
|
@@ -15,7 +16,11 @@ import {
|
|
|
15
16
|
} from "./worker-message-helpers.js";
|
|
16
17
|
import { WorkerProtocolGuard } from "./worker-protocol-guard.js";
|
|
17
18
|
import { isWorkerLogMessage } from "./worker-protocol-log.js";
|
|
18
|
-
import {
|
|
19
|
+
import {
|
|
20
|
+
DEFAULT_WORKER_REALM_READY_TIMEOUT_MS,
|
|
21
|
+
WORKER_PROTOCOL_VERSION
|
|
22
|
+
} from "./worker-protocol.js";
|
|
23
|
+
import { isWorkerRealmReadyMessage, WorkerRealmReadiness } from "./worker-realm-readiness.js";
|
|
19
24
|
import {
|
|
20
25
|
WorkerTurnWatchdog
|
|
21
26
|
} from "./worker-turn-watchdog.js";
|
|
@@ -34,6 +39,8 @@ export class WorkerExecutionStrategy {
|
|
|
34
39
|
#forwardedLogGate;
|
|
35
40
|
#turnWatchdog;
|
|
36
41
|
#protocolGuard;
|
|
42
|
+
#realmReadiness;
|
|
43
|
+
#faultHandler;
|
|
37
44
|
#dispatcher;
|
|
38
45
|
#messageHandler;
|
|
39
46
|
#disposed;
|
|
@@ -44,7 +51,10 @@ export class WorkerExecutionStrategy {
|
|
|
44
51
|
maxProtocolMessageBytes,
|
|
45
52
|
requireProtocolVersion = !1,
|
|
46
53
|
discardOnCancel = !1,
|
|
47
|
-
broadcastEvents = !1
|
|
54
|
+
broadcastEvents = !1,
|
|
55
|
+
requireRealmReady = !1,
|
|
56
|
+
getExpectedWorkflowTypes,
|
|
57
|
+
realmReadyTimeoutMs = DEFAULT_WORKER_REALM_READY_TIMEOUT_MS
|
|
48
58
|
} = options ?? {};
|
|
49
59
|
this.#pool = pool;
|
|
50
60
|
this.#ownership = new WorkerExecutionOwnership;
|
|
@@ -56,21 +66,36 @@ export class WorkerExecutionStrategy {
|
|
|
56
66
|
this.#discardOnCancel = discardOnCancel;
|
|
57
67
|
this.#forwardedLogGate = forwardedLogGateFromStrategyOptions(options, maxProtocolMessageBytes);
|
|
58
68
|
this.#turnWatchdog = new WorkerTurnWatchdog(this.#workflowTurnTimeoutMs, (turn) => {
|
|
59
|
-
this.#handleTurnTimeout(turn);
|
|
69
|
+
this.#faultHandler.handleTurnTimeout(turn);
|
|
60
70
|
});
|
|
61
71
|
this.#protocolGuard = new WorkerProtocolGuard(this.#maxProtocolMessageBytes, this.#requireProtocolVersion, this.#turnWatchdog);
|
|
72
|
+
this.#realmReadiness = WorkerExecutionStrategy.#buildRealmReadiness(requireRealmReady, getExpectedWorkflowTypes, realmReadyTimeoutMs, this.#maxProtocolMessageBytes);
|
|
73
|
+
this.#faultHandler = new WorkerFaultHandler({
|
|
74
|
+
ownership: this.#ownership,
|
|
75
|
+
checkpointResumeState: this.#checkpointResumeState,
|
|
76
|
+
turnWatchdog: this.#turnWatchdog,
|
|
77
|
+
forwardedLogGate: this.#forwardedLogGate,
|
|
78
|
+
realmReadiness: this.#realmReadiness,
|
|
79
|
+
workerListeners: this.#workerListeners,
|
|
80
|
+
protocolGuard: this.#protocolGuard,
|
|
81
|
+
pool: this.#pool,
|
|
82
|
+
emit: (message) => {
|
|
83
|
+
this.#emit(message);
|
|
84
|
+
}
|
|
85
|
+
});
|
|
62
86
|
this.#dispatcher = new WorkerExecutionDispatcher({
|
|
63
87
|
pool: this.#pool,
|
|
64
88
|
ownership: this.#ownership,
|
|
65
89
|
isDisposed: () => this.#disposed,
|
|
66
90
|
requireProtocolVersion: () => this.#requireProtocolVersion,
|
|
67
|
-
validateHostToWorkerMessage: (workflowId, message, worker) => this.#assertHostToWorkerMessageWithinLimit(workflowId, message, worker),
|
|
91
|
+
validateHostToWorkerMessage: (workflowId, message, worker) => this.#faultHandler.assertHostToWorkerMessageWithinLimit(workflowId, message, worker),
|
|
68
92
|
attachWorkerListeners: (worker) => {
|
|
69
93
|
this.#attachWorkerListeners(worker);
|
|
70
94
|
},
|
|
71
95
|
detachWorkerListenersIfIdle: (worker) => {
|
|
72
96
|
this.#detachWorkerListenersIfIdle(worker);
|
|
73
97
|
},
|
|
98
|
+
ensureRealmReady: (worker, workflowId) => this.#ensureRealmReady(worker, workflowId),
|
|
74
99
|
beginTurn: (worker, workflowId, turnId, kind) => {
|
|
75
100
|
this.#turnWatchdog.begin(worker, workflowId, turnId, kind);
|
|
76
101
|
},
|
|
@@ -78,7 +103,7 @@ export class WorkerExecutionStrategy {
|
|
|
78
103
|
this.#turnWatchdog.clear(worker);
|
|
79
104
|
},
|
|
80
105
|
discardWorkerAndFailWorkflows: (worker, discardOptions) => {
|
|
81
|
-
this.#discardWorkerAndFailWorkflows(worker, discardOptions);
|
|
106
|
+
this.#faultHandler.discardWorkerAndFailWorkflows(worker, discardOptions);
|
|
82
107
|
},
|
|
83
108
|
emit: (message) => {
|
|
84
109
|
this.#emit(message);
|
|
@@ -108,7 +133,7 @@ export class WorkerExecutionStrategy {
|
|
|
108
133
|
this.#ownership.resetWorkflow(parameters.workflowId);
|
|
109
134
|
this.#checkpointResumeState.resetWorkflow(parameters.workflowId);
|
|
110
135
|
const message = buildRunMessage(parameters, this.#inboundMessageContext());
|
|
111
|
-
if (!this.#assertHostToWorkerMessageWithinLimit(parameters.workflowId, message))
|
|
136
|
+
if (!this.#faultHandler.assertHostToWorkerMessageWithinLimit(parameters.workflowId, message))
|
|
112
137
|
return;
|
|
113
138
|
this.#dispatcher.acquireAndSend(parameters.workflowId, message);
|
|
114
139
|
}
|
|
@@ -135,6 +160,23 @@ export class WorkerExecutionStrategy {
|
|
|
135
160
|
error: `No worker assigned for workflow: ${parameters.workflowId}`
|
|
136
161
|
});
|
|
137
162
|
}
|
|
163
|
+
async#ensureRealmReady(worker, workflowId) {
|
|
164
|
+
if (!this.#realmReadiness)
|
|
165
|
+
return !0;
|
|
166
|
+
if (this.#realmReadiness.isReady(worker))
|
|
167
|
+
return !0;
|
|
168
|
+
const outcome = await this.#realmReadiness.waitForReady(worker);
|
|
169
|
+
if (outcome.ok)
|
|
170
|
+
return !0;
|
|
171
|
+
this.#faultHandler.discardWorkerAndFailWorkflows(worker, {
|
|
172
|
+
targetWorkflowId: workflowId,
|
|
173
|
+
targetCategory: outcome.failureCategory,
|
|
174
|
+
targetError: outcome.error,
|
|
175
|
+
otherCategory: "system",
|
|
176
|
+
otherError: `Worker discarded after realm-ready handshake failed: ${outcome.error}`
|
|
177
|
+
});
|
|
178
|
+
return !1;
|
|
179
|
+
}
|
|
138
180
|
#inboundMessageContext() {
|
|
139
181
|
return {
|
|
140
182
|
turnId: this.#nextTurnId++,
|
|
@@ -147,7 +189,7 @@ export class WorkerExecutionStrategy {
|
|
|
147
189
|
if (worker) {
|
|
148
190
|
this.#ownership.markCancelled(workflowId);
|
|
149
191
|
if (this.#discardOnCancel) {
|
|
150
|
-
this.#discardWorkerAndFailWorkflows(worker, {
|
|
192
|
+
this.#faultHandler.discardWorkerAndFailWorkflows(worker, {
|
|
151
193
|
targetWorkflowId: workflowId,
|
|
152
194
|
skipTarget: !0,
|
|
153
195
|
otherCategory: "system",
|
|
@@ -171,7 +213,7 @@ export class WorkerExecutionStrategy {
|
|
|
171
213
|
this.#ownership.markCancelled(workflowId);
|
|
172
214
|
this.#ownership.deleteParked(workflowId);
|
|
173
215
|
if (this.#discardOnCancel) {
|
|
174
|
-
this.#discardWorkerAndFailWorkflows(parkedWorker, {
|
|
216
|
+
this.#faultHandler.discardWorkerAndFailWorkflows(parkedWorker, {
|
|
175
217
|
targetWorkflowId: workflowId,
|
|
176
218
|
skipTarget: !0,
|
|
177
219
|
otherCategory: "system",
|
|
@@ -197,6 +239,7 @@ export class WorkerExecutionStrategy {
|
|
|
197
239
|
this.#broadcastChannel.close();
|
|
198
240
|
}
|
|
199
241
|
this.#turnWatchdog.clearAll();
|
|
242
|
+
this.#realmReadiness?.clear();
|
|
200
243
|
const activeWorkflowIds = this.#ownership.activeWorkflowIds();
|
|
201
244
|
for (const workflowId of activeWorkflowIds)
|
|
202
245
|
this.#releaseActiveWorker(workflowId);
|
|
@@ -206,13 +249,18 @@ export class WorkerExecutionStrategy {
|
|
|
206
249
|
this.#messageHandler = null;
|
|
207
250
|
}
|
|
208
251
|
async#handleWorkerMessage(worker, message) {
|
|
252
|
+
if (isWorkerRealmReadyMessage(message)) {
|
|
253
|
+
if (this.#realmReadiness)
|
|
254
|
+
await this.#realmReadiness.noteReadyMessage(worker, message);
|
|
255
|
+
return;
|
|
256
|
+
}
|
|
209
257
|
if (isWorkerLogMessage(message)) {
|
|
210
258
|
const owns = (id) => this.#ownership.getTargetWorker(id) === worker, abuseDiscard = this.#forwardedLogGate.handle(worker, message, owns);
|
|
211
259
|
if (abuseDiscard)
|
|
212
|
-
this.#discardWorkerAndFailWorkflows(worker, abuseDiscard);
|
|
260
|
+
this.#faultHandler.discardWorkerAndFailWorkflows(worker, abuseDiscard);
|
|
213
261
|
return;
|
|
214
262
|
}
|
|
215
|
-
if (!this.#acceptWorkerMessage(worker, message))
|
|
263
|
+
if (!this.#faultHandler.acceptWorkerMessage(worker, message))
|
|
216
264
|
return;
|
|
217
265
|
this.#turnWatchdog.clear(worker);
|
|
218
266
|
const resumeVersionBeforeCheckpointHandling = this.#checkpointResumeState.beginCheckpointHandling(message), emitResult = emitWorkerMessageToEngine(this.#messageHandler, message), handlerFailed = emitResult instanceof Promise ? await emitResult : emitResult;
|
|
@@ -244,14 +292,6 @@ export class WorkerExecutionStrategy {
|
|
|
244
292
|
if (this.#ownership.parkActive(message.workflowId, worker))
|
|
245
293
|
this.#pool.release(worker);
|
|
246
294
|
}
|
|
247
|
-
#handleWorkerError(worker, errorEvent) {
|
|
248
|
-
this.#discardWorkerAndFailWorkflows(worker, {
|
|
249
|
-
targetCategory: "system",
|
|
250
|
-
targetError: `Worker crashed: ${errorEvent.message ?? "unknown error"}`,
|
|
251
|
-
otherCategory: "system",
|
|
252
|
-
otherError: `Worker crashed: ${errorEvent.message ?? "unknown error"}`
|
|
253
|
-
});
|
|
254
|
-
}
|
|
255
295
|
#releaseActiveWorker(workflowId) {
|
|
256
296
|
const worker = this.#ownership.releaseActive(workflowId);
|
|
257
297
|
if (worker) {
|
|
@@ -266,10 +306,10 @@ export class WorkerExecutionStrategy {
|
|
|
266
306
|
this.#handleWorkerMessage(worker, message).catch(() => {});
|
|
267
307
|
},
|
|
268
308
|
error: (errorEvent) => {
|
|
269
|
-
this.#handleWorkerError(worker, errorEvent);
|
|
309
|
+
this.#faultHandler.handleWorkerError(worker, errorEvent);
|
|
270
310
|
},
|
|
271
311
|
messageerror: () => {
|
|
272
|
-
this.#discardWorkerAndFailWorkflows(worker, {
|
|
312
|
+
this.#faultHandler.discardWorkerAndFailWorkflows(worker, {
|
|
273
313
|
targetCategory: "system",
|
|
274
314
|
targetError: "Worker messageerror event",
|
|
275
315
|
otherCategory: "system",
|
|
@@ -281,75 +321,6 @@ export class WorkerExecutionStrategy {
|
|
|
281
321
|
#detachWorkerListenersIfIdle(worker) {
|
|
282
322
|
this.#workerListeners.detachIfIdle(worker, (candidate) => this.#ownership.workerIsIdle(candidate));
|
|
283
323
|
}
|
|
284
|
-
#assertHostToWorkerMessageWithinLimit(workflowId, message, worker) {
|
|
285
|
-
const failure = this.#protocolGuard.validateHostToWorkerMessage(message);
|
|
286
|
-
if (!failure)
|
|
287
|
-
return !0;
|
|
288
|
-
if (worker) {
|
|
289
|
-
this.#discardWorkerAndFailWorkflows(worker, {
|
|
290
|
-
targetWorkflowId: workflowId,
|
|
291
|
-
targetCategory: failure.failureCategory,
|
|
292
|
-
targetError: failure.error,
|
|
293
|
-
otherCategory: "system",
|
|
294
|
-
otherError: `Worker discarded after protocol send failure for workflow: ${workflowId}`
|
|
295
|
-
});
|
|
296
|
-
return !1;
|
|
297
|
-
}
|
|
298
|
-
this.#emit({
|
|
299
|
-
type: "failed",
|
|
300
|
-
workflowId,
|
|
301
|
-
error: failure.error,
|
|
302
|
-
failureCategory: failure.failureCategory
|
|
303
|
-
});
|
|
304
|
-
return !1;
|
|
305
|
-
}
|
|
306
|
-
#acceptWorkerMessage(worker, message) {
|
|
307
|
-
const result = this.#protocolGuard.acceptWorkerMessage(worker, message);
|
|
308
|
-
if (result.accepted)
|
|
309
|
-
return !0;
|
|
310
|
-
this.#discardWorkerAndFailWorkflows(worker, {
|
|
311
|
-
...result.failure.targetWorkflowId === void 0 ? {} : { targetWorkflowId: result.failure.targetWorkflowId },
|
|
312
|
-
targetCategory: result.failure.failureCategory,
|
|
313
|
-
targetError: result.failure.error,
|
|
314
|
-
otherCategory: "system",
|
|
315
|
-
otherError: result.failure.otherError
|
|
316
|
-
});
|
|
317
|
-
return !1;
|
|
318
|
-
}
|
|
319
|
-
#handleTurnTimeout(turn) {
|
|
320
|
-
this.#discardWorkerAndFailWorkflows(turn.worker, {
|
|
321
|
-
targetWorkflowId: turn.workflowId,
|
|
322
|
-
targetCategory: "timeout",
|
|
323
|
-
targetError: `Worker workflow turn timed out after ${turn.timeoutMs}ms`,
|
|
324
|
-
otherCategory: "timeout",
|
|
325
|
-
otherError: `Worker discarded after workflow turn timed out: ${turn.workflowId}`
|
|
326
|
-
});
|
|
327
|
-
}
|
|
328
|
-
#discardWorkerAndFailWorkflows(worker, options) {
|
|
329
|
-
const workflowIds = this.#ownership.workflowIdsForWorker(worker);
|
|
330
|
-
if (workflowIds.length === 0) {
|
|
331
|
-
this.#turnWatchdog.clear(worker);
|
|
332
|
-
this.#forwardedLogGate.forget(worker);
|
|
333
|
-
return;
|
|
334
|
-
}
|
|
335
|
-
for (const workflowId of workflowIds) {
|
|
336
|
-
const isTarget = workflowId === options.targetWorkflowId;
|
|
337
|
-
this.#ownership.forgetWorkflow(workflowId);
|
|
338
|
-
this.#checkpointResumeState.forgetWorkflowIfClosed(workflowId, !0);
|
|
339
|
-
if (isTarget && options.skipTarget)
|
|
340
|
-
continue;
|
|
341
|
-
this.#emit({
|
|
342
|
-
type: "failed",
|
|
343
|
-
workflowId,
|
|
344
|
-
error: isTarget ? options.targetError ?? options.otherError : options.otherError,
|
|
345
|
-
failureCategory: isTarget ? options.targetCategory ?? options.otherCategory : options.otherCategory
|
|
346
|
-
});
|
|
347
|
-
}
|
|
348
|
-
this.#turnWatchdog.clear(worker);
|
|
349
|
-
this.#forwardedLogGate.forget(worker);
|
|
350
|
-
this.#workerListeners.detach(worker);
|
|
351
|
-
this.#pool.discard(worker);
|
|
352
|
-
}
|
|
353
324
|
#handleBroadcastMessage(data) {
|
|
354
325
|
if (data.type === "signal:received" && typeof data.workflowId === "string") {
|
|
355
326
|
const targetWorker = this.#ownership.getTargetWorker(data.workflowId);
|
|
@@ -362,4 +333,15 @@ export class WorkerExecutionStrategy {
|
|
|
362
333
|
if (result instanceof Promise)
|
|
363
334
|
result.catch(() => {});
|
|
364
335
|
}
|
|
336
|
+
static #buildRealmReadiness(requireRealmReady, getExpectedWorkflowTypes, timeoutMs, maxProtocolMessageBytes) {
|
|
337
|
+
if (!requireRealmReady)
|
|
338
|
+
return null;
|
|
339
|
+
if (!getExpectedWorkflowTypes)
|
|
340
|
+
throw Error("WorkerExecutionStrategyOptions.getExpectedWorkflowTypes is required when requireRealmReady is true");
|
|
341
|
+
return new WorkerRealmReadiness({
|
|
342
|
+
getExpectedWorkflowTypes,
|
|
343
|
+
timeoutMs,
|
|
344
|
+
maxProtocolMessageBytes
|
|
345
|
+
});
|
|
346
|
+
}
|
|
365
347
|
}
|
|
@@ -0,0 +1,45 @@
|
|
|
1
|
+
import type { WorkerPool } from '../workers/pool.ts';
|
|
2
|
+
import type { FailureCategory, WorkerInboundMessage, WorkerOutboundMessage } from './types.ts';
|
|
3
|
+
import type { WorkerCheckpointResumeState } from './worker-checkpoint-resume-state.ts';
|
|
4
|
+
import type { WorkerExecutionOwnership } from './worker-execution-ownership.ts';
|
|
5
|
+
import type { WorkerListenerRegistry } from './worker-listener-registry.ts';
|
|
6
|
+
import type { ForwardedLogGate } from './worker-log-abuse-counter.ts';
|
|
7
|
+
import type { WorkerProtocolGuard } from './worker-protocol-guard.ts';
|
|
8
|
+
import type { WorkerRealmReadiness } from './worker-realm-readiness.ts';
|
|
9
|
+
import type { WorkerTurnState, WorkerTurnWatchdog } from './worker-turn-watchdog.ts';
|
|
10
|
+
export interface WorkerFaultHandlerDependencies {
|
|
11
|
+
ownership: WorkerExecutionOwnership;
|
|
12
|
+
checkpointResumeState: WorkerCheckpointResumeState;
|
|
13
|
+
turnWatchdog: WorkerTurnWatchdog;
|
|
14
|
+
forwardedLogGate: ForwardedLogGate;
|
|
15
|
+
realmReadiness: WorkerRealmReadiness | null;
|
|
16
|
+
workerListeners: WorkerListenerRegistry;
|
|
17
|
+
protocolGuard: WorkerProtocolGuard;
|
|
18
|
+
pool: WorkerPool;
|
|
19
|
+
emit: (message: WorkerOutboundMessage) => void;
|
|
20
|
+
}
|
|
21
|
+
export interface WorkerDiscardOptions {
|
|
22
|
+
targetWorkflowId?: string;
|
|
23
|
+
targetCategory?: FailureCategory;
|
|
24
|
+
targetError?: string;
|
|
25
|
+
skipTarget?: boolean;
|
|
26
|
+
otherCategory: FailureCategory;
|
|
27
|
+
otherError: string;
|
|
28
|
+
}
|
|
29
|
+
/**
|
|
30
|
+
* Owns every path that ends in a worker being discarded and its owned
|
|
31
|
+
* workflows failed: protocol violations, turn timeouts, worker crashes, and
|
|
32
|
+
* (WFT-28) realm-ready handshake failures. Extracted from
|
|
33
|
+
* {@link WorkerExecutionStrategy} to keep that file under the repository's
|
|
34
|
+
* file-size ceiling; every method here previously lived as a private method
|
|
35
|
+
* on that class and is unchanged in behavior.
|
|
36
|
+
*/
|
|
37
|
+
export declare class WorkerFaultHandler {
|
|
38
|
+
#private;
|
|
39
|
+
constructor(dependencies: WorkerFaultHandlerDependencies);
|
|
40
|
+
assertHostToWorkerMessageWithinLimit(workflowId: string, message: WorkerInboundMessage, worker?: Worker): boolean;
|
|
41
|
+
acceptWorkerMessage(worker: Worker, message: unknown): message is WorkerOutboundMessage;
|
|
42
|
+
handleTurnTimeout(turn: WorkerTurnState): void;
|
|
43
|
+
handleWorkerError(worker: Worker, errorEvent: ErrorEvent): void;
|
|
44
|
+
discardWorkerAndFailWorkflows(worker: Worker, options: WorkerDiscardOptions): void;
|
|
45
|
+
}
|
|
@@ -0,0 +1,88 @@
|
|
|
1
|
+
export class WorkerFaultHandler {
|
|
2
|
+
#dependencies;
|
|
3
|
+
constructor(dependencies) {
|
|
4
|
+
this.#dependencies = dependencies;
|
|
5
|
+
}
|
|
6
|
+
assertHostToWorkerMessageWithinLimit(workflowId, message, worker) {
|
|
7
|
+
const failure = this.#dependencies.protocolGuard.validateHostToWorkerMessage(message);
|
|
8
|
+
if (!failure)
|
|
9
|
+
return !0;
|
|
10
|
+
if (worker) {
|
|
11
|
+
this.discardWorkerAndFailWorkflows(worker, {
|
|
12
|
+
targetWorkflowId: workflowId,
|
|
13
|
+
targetCategory: failure.failureCategory,
|
|
14
|
+
targetError: failure.error,
|
|
15
|
+
otherCategory: "system",
|
|
16
|
+
otherError: `Worker discarded after protocol send failure for workflow: ${workflowId}`
|
|
17
|
+
});
|
|
18
|
+
return !1;
|
|
19
|
+
}
|
|
20
|
+
this.#dependencies.emit({
|
|
21
|
+
type: "failed",
|
|
22
|
+
workflowId,
|
|
23
|
+
error: failure.error,
|
|
24
|
+
failureCategory: failure.failureCategory
|
|
25
|
+
});
|
|
26
|
+
return !1;
|
|
27
|
+
}
|
|
28
|
+
acceptWorkerMessage(worker, message) {
|
|
29
|
+
const result = this.#dependencies.protocolGuard.acceptWorkerMessage(worker, message);
|
|
30
|
+
if (result.accepted)
|
|
31
|
+
return !0;
|
|
32
|
+
this.discardWorkerAndFailWorkflows(worker, {
|
|
33
|
+
...result.failure.targetWorkflowId === void 0 ? {} : { targetWorkflowId: result.failure.targetWorkflowId },
|
|
34
|
+
targetCategory: result.failure.failureCategory,
|
|
35
|
+
targetError: result.failure.error,
|
|
36
|
+
otherCategory: "system",
|
|
37
|
+
otherError: result.failure.otherError
|
|
38
|
+
});
|
|
39
|
+
return !1;
|
|
40
|
+
}
|
|
41
|
+
handleTurnTimeout(turn) {
|
|
42
|
+
this.discardWorkerAndFailWorkflows(turn.worker, {
|
|
43
|
+
targetWorkflowId: turn.workflowId,
|
|
44
|
+
targetCategory: "timeout",
|
|
45
|
+
targetError: `Worker workflow turn timed out after ${turn.timeoutMs}ms`,
|
|
46
|
+
otherCategory: "timeout",
|
|
47
|
+
otherError: `Worker discarded after workflow turn timed out: ${turn.workflowId}`
|
|
48
|
+
});
|
|
49
|
+
}
|
|
50
|
+
handleWorkerError(worker, errorEvent) {
|
|
51
|
+
this.discardWorkerAndFailWorkflows(worker, {
|
|
52
|
+
targetCategory: "system",
|
|
53
|
+
targetError: `Worker crashed: ${errorEvent.message ?? "unknown error"}`,
|
|
54
|
+
otherCategory: "system",
|
|
55
|
+
otherError: `Worker crashed: ${errorEvent.message ?? "unknown error"}`
|
|
56
|
+
});
|
|
57
|
+
}
|
|
58
|
+
discardWorkerAndFailWorkflows(worker, options) {
|
|
59
|
+
const { ownership } = this.#dependencies, workflowIds = ownership.workflowIdsForWorker(worker);
|
|
60
|
+
if (workflowIds.length === 0) {
|
|
61
|
+
this.#clearWorkerState(worker);
|
|
62
|
+
return;
|
|
63
|
+
}
|
|
64
|
+
for (const workflowId of workflowIds)
|
|
65
|
+
this.#failOwnedWorkflow(workflowId, workflowId === options.targetWorkflowId, options);
|
|
66
|
+
this.#clearWorkerState(worker);
|
|
67
|
+
this.#dependencies.workerListeners.detach(worker);
|
|
68
|
+
this.#dependencies.pool.discard(worker);
|
|
69
|
+
}
|
|
70
|
+
#failOwnedWorkflow(workflowId, isTarget, options) {
|
|
71
|
+
const { ownership, checkpointResumeState } = this.#dependencies;
|
|
72
|
+
ownership.forgetWorkflow(workflowId);
|
|
73
|
+
checkpointResumeState.forgetWorkflowIfClosed(workflowId, !0);
|
|
74
|
+
if (isTarget && options.skipTarget)
|
|
75
|
+
return;
|
|
76
|
+
this.#dependencies.emit({
|
|
77
|
+
type: "failed",
|
|
78
|
+
workflowId,
|
|
79
|
+
error: isTarget ? options.targetError ?? options.otherError : options.otherError,
|
|
80
|
+
failureCategory: isTarget ? options.targetCategory ?? options.otherCategory : options.otherCategory
|
|
81
|
+
});
|
|
82
|
+
}
|
|
83
|
+
#clearWorkerState(worker) {
|
|
84
|
+
this.#dependencies.turnWatchdog.clear(worker);
|
|
85
|
+
this.#dependencies.forwardedLogGate.forget(worker);
|
|
86
|
+
this.#dependencies.realmReadiness?.forget(worker);
|
|
87
|
+
}
|
|
88
|
+
}
|
|
@@ -2,10 +2,27 @@ import type { ContextOperationRequest } from './context.ts';
|
|
|
2
2
|
import type { FailureCategory, OperationRequest, WorkerOutboundMessage } from './types.ts';
|
|
3
3
|
import { type WorkerReplayOperationSignature } from './types/checkpoint.ts';
|
|
4
4
|
export { WORKER_REPLAY_SIGNATURE_FORMAT, type WorkerReplayOperationSignature, } from './types/checkpoint.ts';
|
|
5
|
-
|
|
5
|
+
/**
|
|
6
|
+
* Bumped 1 -> 2 for the realm-ready handshake (WFT-28): a worker realm must
|
|
7
|
+
* now send a `ready` message before its first turn, a required pre-turn
|
|
8
|
+
* message the wire semantics didn't previously have. Host and worker are
|
|
9
|
+
* always built from the same package version — there is no independent peer
|
|
10
|
+
* to stay backward compatible with — so this is a clean-break bump, the same
|
|
11
|
+
* reasoning the RemoteWorker protocol v2 -> v3 bump used.
|
|
12
|
+
*/
|
|
13
|
+
export declare const WORKER_PROTOCOL_VERSION = 2;
|
|
6
14
|
export declare const DEFAULT_WORKER_TURN_TIMEOUT_MS = 1000;
|
|
7
15
|
export declare const DEFAULT_WORKER_PROTOCOL_MESSAGE_BYTES = 1048576;
|
|
8
16
|
export declare const MIN_WORKER_PROTOCOL_MESSAGE_BYTES = 4096;
|
|
17
|
+
/**
|
|
18
|
+
* Default bound (ms) on how long the host waits for a freshly acquired
|
|
19
|
+
* worker's `ready` handshake message before discarding it (WFT-28). Worker boot
|
|
20
|
+
* is module load plus evaluation with no I/O, but a cold Bun Worker spinning
|
|
21
|
+
* up a large bundled module graph can take meaningfully longer than a single
|
|
22
|
+
* workflow turn, so this is deliberately more generous than
|
|
23
|
+
* {@link DEFAULT_WORKER_TURN_TIMEOUT_MS}.
|
|
24
|
+
*/
|
|
25
|
+
export declare const DEFAULT_WORKER_REALM_READY_TIMEOUT_MS = 5000;
|
|
9
26
|
/**
|
|
10
27
|
* Default fixed-window length (ms) for the forwarded-log flood budget (#545). Paired
|
|
11
28
|
* with {@link DEFAULT_FORWARDED_LOG_FLOOD_THRESHOLD} over this window.
|
|
@@ -7,7 +7,7 @@ import { isValidWorkerLogRecord } from "./worker-protocol-log.js";
|
|
|
7
7
|
export {
|
|
8
8
|
WORKER_REPLAY_SIGNATURE_FORMAT
|
|
9
9
|
} from "./types/checkpoint.js";
|
|
10
|
-
export const WORKER_PROTOCOL_VERSION =
|
|
10
|
+
export const WORKER_PROTOCOL_VERSION = 2, DEFAULT_WORKER_TURN_TIMEOUT_MS = 1000, DEFAULT_WORKER_PROTOCOL_MESSAGE_BYTES = 1048576, MIN_WORKER_PROTOCOL_MESSAGE_BYTES = 4096, DEFAULT_WORKER_REALM_READY_TIMEOUT_MS = 5000, DEFAULT_FORWARDED_LOG_FLOOD_WINDOW_MS = 60000, DEFAULT_FORWARDED_LOG_FLOOD_THRESHOLD = 5000, FORWARDED_LOG_STRIKE_THRESHOLD = 5;
|
|
11
11
|
const BINARY_MARKER = "__weftBinaryBytes", DATE_MARKER = "__weftDate", ERROR_MARKER = "__weftError", MAP_MARKER = "__weftMap", SET_MARKER = "__weftSet", MAX_BOUNDED_ERROR_LENGTH = 512, WORKER_SIGNATURE_EXCLUDED_FIELDS = new Set([
|
|
12
12
|
"attempt",
|
|
13
13
|
"callerStack",
|
|
@@ -0,0 +1,78 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Realm-ready handshake for internal `workflowExecutionMode: 'worker'`
|
|
3
|
+
* Workers (WFT-28).
|
|
4
|
+
*
|
|
5
|
+
* An internal Worker realm sends a `ready` message — its own manifest, built
|
|
6
|
+
* from exactly the workflow types its bootstrap script registered — before
|
|
7
|
+
* it can receive a `run` turn. The host checks that every workflow type it
|
|
8
|
+
* has registered appears in that manifest with a matching contract (a
|
|
9
|
+
* subset check, not exact-set equality — a realm may legitimately advertise
|
|
10
|
+
* more types than any one host dispatches to it, e.g. a shared worker pool
|
|
11
|
+
* serving several engines). A missing or mismatched type means the realm's
|
|
12
|
+
* bundle disagrees with the host about a workflow it needs (a stale build, a
|
|
13
|
+
* bootstrap script that fell out of sync), and the realm is discarded before
|
|
14
|
+
* it can ever execute a turn, rather than failing opaquely mid-dispatch.
|
|
15
|
+
*
|
|
16
|
+
* `ready` has no `workflowId` — it is a per-worker-lifetime handshake, not a
|
|
17
|
+
* per-workflow-turn message — so it cannot fit {@link WorkerOutboundMessage}'s
|
|
18
|
+
* shape (every variant there requires one) and is validated on its own path
|
|
19
|
+
* instead of through {@link assertWorkerOutboundMessageShape}, the same way
|
|
20
|
+
* `log` is handled outside the strict turn gate.
|
|
21
|
+
*
|
|
22
|
+
* @module core/worker-realm-readiness
|
|
23
|
+
*/
|
|
24
|
+
import type { WorkerManifest } from '../worker/manifest/types.ts';
|
|
25
|
+
import type { FailureCategory } from './types.ts';
|
|
26
|
+
/** Sent once by a Worker realm, before its first `run` turn. See {@link WorkerRealmReadiness}. */
|
|
27
|
+
export type WorkerRealmReadyMessage = Readonly<{
|
|
28
|
+
type: 'ready';
|
|
29
|
+
protocolVersion: number;
|
|
30
|
+
realmGeneration: string;
|
|
31
|
+
manifest: WorkerManifest;
|
|
32
|
+
}>;
|
|
33
|
+
/** Shallow type guard — deep field validation happens in {@link WorkerRealmReadiness.noteReadyMessage}. */
|
|
34
|
+
export declare function isWorkerRealmReadyMessage(message: unknown): message is WorkerRealmReadyMessage;
|
|
35
|
+
export type RealmReadyOutcome = {
|
|
36
|
+
ok: true;
|
|
37
|
+
realmGeneration: string;
|
|
38
|
+
manifestDigest: string;
|
|
39
|
+
} | {
|
|
40
|
+
ok: false;
|
|
41
|
+
error: string;
|
|
42
|
+
failureCategory: FailureCategory;
|
|
43
|
+
};
|
|
44
|
+
export interface WorkerRealmReadinessDependencies {
|
|
45
|
+
/**
|
|
46
|
+
* Live accessor for the host's registered workflow types, called fresh on
|
|
47
|
+
* every handshake rather than snapshotted at construction — the engine's
|
|
48
|
+
* registrations map is still empty when the strategy is constructed (the
|
|
49
|
+
* registration loop runs later in `Engine.create()`).
|
|
50
|
+
*/
|
|
51
|
+
getExpectedWorkflowTypes: () => readonly string[];
|
|
52
|
+
timeoutMs: number;
|
|
53
|
+
maxProtocolMessageBytes: number | undefined;
|
|
54
|
+
}
|
|
55
|
+
/**
|
|
56
|
+
* Tracks, per pooled `Worker` instance, whether its one-time ready handshake
|
|
57
|
+
* has completed. A worker is validated at most once per lifetime — recycled
|
|
58
|
+
* workers skip straight to {@link isReady}.
|
|
59
|
+
*/
|
|
60
|
+
export declare class WorkerRealmReadiness {
|
|
61
|
+
#private;
|
|
62
|
+
constructor(dependencies: WorkerRealmReadinessDependencies);
|
|
63
|
+
isReady(worker: Worker): boolean;
|
|
64
|
+
/**
|
|
65
|
+
* Wait for `worker`'s ready handshake. Resolves immediately if the worker
|
|
66
|
+
* already completed it. The caller must attach its message listener (which
|
|
67
|
+
* routes into {@link noteReadyMessage}) before calling this, in the same
|
|
68
|
+
* synchronous continuation — `acquireAndSend` does exactly that, so no
|
|
69
|
+
* `ready` message can arrive before this registers its pending waiter.
|
|
70
|
+
*/
|
|
71
|
+
waitForReady(worker: Worker): Promise<RealmReadyOutcome>;
|
|
72
|
+
/** Validate an inbound `ready` message and settle any pending {@link waitForReady} call. */
|
|
73
|
+
noteReadyMessage(worker: Worker, message: unknown): Promise<void>;
|
|
74
|
+
/** Drop a discarded worker's state, settling any pending waiter with a failure. */
|
|
75
|
+
forget(worker: Worker): void;
|
|
76
|
+
/** Settle every pending waiter on strategy disposal so no wait hangs past teardown. */
|
|
77
|
+
clear(): void;
|
|
78
|
+
}
|