@lostgradient/weft 0.17.0 → 0.19.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +3 -1
- package/dist/cli/conformance.js +14 -18
- package/dist/cli/generated/operation-catalog.snapshot.json +331 -16
- package/dist/cli/generated/operation-client.generated.d.ts +39 -6
- package/dist/cli/generated/operation-client.generated.js +6 -0
- package/dist/core/context/child-workflow-pipe.d.ts +1 -1
- package/dist/core/context/child-workflow-pipe.js +1 -1
- package/dist/core/engine/construction.d.ts +2 -0
- package/dist/core/engine/construction.js +3 -0
- package/dist/core/engine/index.js +1 -0
- package/dist/core/engine/listing.d.ts +1 -0
- package/dist/core/engine/listing.js +6 -4
- package/dist/core/weft-error.d.ts +1 -1
- package/dist/core/weft-error.js +2 -1
- package/dist/core/worker-execution-dispatcher.d.ts +8 -0
- package/dist/core/worker-execution-dispatcher.js +4 -0
- package/dist/core/worker-execution-strategy-options.d.ts +16 -0
- package/dist/core/worker-execution-strategy.js +71 -89
- package/dist/core/worker-fault-handling.d.ts +45 -0
- package/dist/core/worker-fault-handling.js +88 -0
- package/dist/core/worker-protocol.d.ts +18 -1
- package/dist/core/worker-protocol.js +1 -1
- package/dist/core/worker-realm-readiness.d.ts +78 -0
- package/dist/core/worker-realm-readiness.js +127 -0
- package/dist/index.d.ts +3 -1
- package/dist/index.js +28 -1
- package/dist/json-schema.js +1 -1
- package/dist/runtime/portable.d.ts +14 -0
- package/dist/runtime/portable.js +10 -0
- package/dist/server/authorization-scope.d.ts +32 -2
- package/dist/server/index.d.ts +73 -8
- package/dist/server/index.js +11 -3
- package/dist/server/operations/get-principal.d.ts +75 -0
- package/dist/server/operations/get-principal.js +41 -0
- package/dist/server/operations/get-task-diagnostics.d.ts +43 -18
- package/dist/server/operations/get-task-diagnostics.js +87 -126
- package/dist/server/operations/get-worker-diagnostics.d.ts +177 -0
- package/dist/server/operations/get-worker-diagnostics.js +117 -0
- package/dist/server/operations/list-worker-registration-rejections.d.ts +79 -0
- package/dist/server/operations/list-worker-registration-rejections.js +60 -0
- package/dist/server/operations/list-workers.js +1 -3
- package/dist/server/operations/static-registrations.d.ts +28 -0
- package/dist/server/operations/static-registrations.js +248 -0
- package/dist/server/rest-bindings.d.ts +5 -0
- package/dist/server/rest-bindings.js +34 -313
- package/dist/server/runtime/authentication-bridge.js +6 -14
- package/dist/server/runtime/context.d.ts +23 -0
- package/dist/server/runtime/task-dispatch.d.ts +7 -1
- package/dist/server/runtime/task-dispatch.js +126 -55
- package/dist/server/runtime/task-ledger-completion.d.ts +68 -0
- package/dist/server/runtime/task-ledger-completion.js +72 -0
- package/dist/server/runtime/task-ledger-recovery.d.ts +55 -0
- package/dist/server/runtime/task-ledger-recovery.js +73 -0
- package/dist/server/runtime/task-ledger-runtime.d.ts +64 -0
- package/dist/server/runtime/task-ledger-runtime.js +34 -0
- package/dist/server/runtime/task-metrics.d.ts +6 -4
- package/dist/server/runtime/task-polling.d.ts +14 -12
- package/dist/server/runtime/task-polling.js +106 -82
- package/dist/server/runtime/task-reconciliation.d.ts +35 -8
- package/dist/server/runtime/task-reconciliation.js +75 -74
- package/dist/server/runtime/task-result-resolution.d.ts +2 -10
- package/dist/server/runtime/task-result-resolution.js +0 -67
- package/dist/server/runtime/task-result-view.d.ts +82 -0
- package/dist/server/runtime/task-result-view.js +48 -0
- package/dist/server/runtime/websocket-worker-messaging.d.ts +16 -0
- package/dist/server/runtime/websocket-worker-messaging.js +32 -0
- package/dist/server/runtime/websocket-worker-registration.d.ts +41 -0
- package/dist/server/runtime/websocket-worker-registration.js +134 -0
- package/dist/server/runtime/websocket-worker.d.ts +0 -1
- package/dist/server/runtime/websocket-worker.js +64 -116
- package/dist/server/serve-internals.d.ts +9 -6
- package/dist/server/serve-internals.js +14 -40
- package/dist/server/task-ledger-codec.d.ts +40 -0
- package/dist/server/task-ledger-codec.js +195 -0
- package/dist/server/task-ledger-keys.d.ts +13 -0
- package/dist/server/task-ledger-keys.js +5 -0
- package/dist/server/task-ledger-limits.d.ts +34 -0
- package/dist/server/task-ledger-limits.js +5 -0
- package/dist/server/task-ledger-transition-helpers.d.ts +38 -0
- package/dist/server/task-ledger-transition-helpers.js +42 -0
- package/dist/server/task-ledger-transitions-cancellation.d.ts +31 -0
- package/dist/server/task-ledger-transitions-cancellation.js +73 -0
- package/dist/server/task-ledger-transitions.d.ts +129 -0
- package/dist/server/task-ledger-transitions.js +162 -0
- package/dist/server/task-ledger-types.d.ts +220 -0
- package/dist/server/task-ledger-types.js +1 -0
- package/dist/server/task-ledger.d.ts +17 -0
- package/dist/server/task-ledger.js +24 -0
- package/dist/server/task-state.d.ts +25 -152
- package/dist/server/task-state.js +0 -188
- package/dist/server/worker-admission-policy.d.ts +70 -0
- package/dist/server/worker-admission-policy.js +0 -0
- package/dist/storage/compressed-storage.js +1 -1
- package/dist/storage/index.d.ts +0 -2
- package/dist/storage/interface.d.ts +0 -2
- package/dist/storage/interface.js +1 -1
- package/dist/storage/lmdb.js +1 -1
- package/dist/storage/memory.js +1 -1
- package/dist/storage/neon.js +1 -1
- package/dist/storage/postgres.js +1 -1
- package/dist/storage/resolve.js +1 -1
- package/dist/storage/scoped-storage.js +1 -1
- package/dist/storage/testing.js +1 -1
- package/dist/storage/turso.js +1 -1
- package/dist/storage/web-extension.js +2 -2
- package/dist/version.d.ts +1 -1
- package/dist/version.js +1 -1
- package/dist/web-extension.js +1 -1
- package/dist/worker/index.d.ts +2 -0
- package/dist/worker/index.js +30 -4
- package/dist/worker/manifest/canonical-json.d.ts +20 -0
- package/dist/worker/manifest/canonical-json.js +13 -0
- package/dist/worker/manifest/capabilities.d.ts +24 -0
- package/dist/worker/manifest/capabilities.js +74 -0
- package/dist/worker/manifest/content-digest.d.ts +20 -0
- package/dist/worker/manifest/content-digest.js +13 -0
- package/dist/worker/manifest/declared-shape-digest.d.ts +14 -0
- package/dist/worker/manifest/declared-shape-digest.js +4 -0
- package/dist/worker/manifest/digest.d.ts +82 -0
- package/dist/worker/manifest/digest.js +9 -0
- package/dist/worker/manifest/execution-identity.d.ts +92 -0
- package/dist/worker/manifest/execution-identity.js +36 -0
- package/dist/worker/manifest/failure.d.ts +59 -0
- package/dist/worker/manifest/failure.js +4 -0
- package/dist/worker/manifest/index.d.ts +22 -0
- package/dist/worker/manifest/index.js +23 -0
- package/dist/worker/manifest/internal-realm.d.ts +50 -0
- package/dist/worker/manifest/internal-realm.js +39 -0
- package/dist/worker/manifest/is-record.d.ts +12 -0
- package/dist/worker/manifest/is-record.js +6 -0
- package/dist/worker/manifest/json-scan.d.ts +26 -0
- package/dist/worker/manifest/json-scan.js +88 -0
- package/dist/worker/manifest/limits.d.ts +138 -0
- package/dist/worker/manifest/limits.js +1 -0
- package/dist/worker/manifest/normalize.d.ts +63 -0
- package/dist/worker/manifest/normalize.js +73 -0
- package/dist/worker/manifest/parse-json.d.ts +24 -0
- package/dist/worker/manifest/parse-json.js +15 -0
- package/dist/worker/manifest/parse.d.ts +86 -0
- package/dist/worker/manifest/parse.js +188 -0
- package/dist/worker/manifest/registry-contract-builder.d.ts +120 -0
- package/dist/worker/manifest/registry-contract-builder.js +65 -0
- package/dist/worker/manifest/types.d.ts +263 -0
- package/dist/worker/manifest/types.js +1 -0
- package/dist/worker/manifest/utf8.d.ts +13 -0
- package/dist/worker/manifest/utf8.js +7 -0
- package/dist/worker/options.d.ts +55 -8
- package/dist/worker/options.js +65 -9
- package/dist/worker/protocol-internals.d.ts +1 -2
- package/dist/worker/protocol-internals.js +0 -5
- package/dist/worker/protocol-messages.d.ts +33 -17
- package/dist/worker/protocol-schemas.d.ts +36 -74
- package/dist/worker/protocol-schemas.js +25 -18
- package/dist/worker/protocol-version.d.ts +18 -14
- package/dist/worker/protocol-version.js +1 -1
- package/dist/worker/protocol.js +1 -1
- package/dist/worker/registry/deployment-consistency.d.ts +78 -0
- package/dist/worker/registry/deployment-consistency.js +27 -0
- package/dist/worker/registry/drain.d.ts +14 -0
- package/dist/worker/registry/drain.js +24 -0
- package/dist/worker/registry/rejections.d.ts +31 -0
- package/dist/worker/registry/rejections.js +10 -0
- package/dist/worker/registry/summary.d.ts +0 -3
- package/dist/worker/registry/summary.js +2 -5
- package/dist/worker/registry/types.d.ts +10 -3
- package/dist/worker/registry.d.ts +47 -2
- package/dist/worker/registry.js +45 -32
- package/dist/workers/workflow-worker-entry.d.ts +12 -7
- package/dist/workers/workflow-worker-entry.js +9 -2
- package/package.json +3 -3
- package/dist/server/task-resolved-record.d.ts +0 -12
- package/dist/server/task-resolved-record.js +0 -35
|
@@ -27,7 +27,7 @@ export function resolveMapConcurrency(totalItems, requestedConcurrency) {
|
|
|
27
27
|
export function createCompositionToken(internals, kind) {
|
|
28
28
|
return `${kind}:${internals.stepIndex}`;
|
|
29
29
|
}
|
|
30
|
-
export function createCompositionChildWorkflowOptions(internals, token, index, options
|
|
30
|
+
export function createCompositionChildWorkflowOptions(internals, token, index, options) {
|
|
31
31
|
assertCompositionAwaitsChildWorkflow(options);
|
|
32
32
|
if (options?.id !== void 0)
|
|
33
33
|
return options;
|
|
@@ -51,6 +51,8 @@ export declare function createExecutionStrategyBundle(parameters: {
|
|
|
51
51
|
development: boolean;
|
|
52
52
|
broadcastEvents: boolean;
|
|
53
53
|
getRegistration: (workflowType: string) => RegistrationEntry | undefined;
|
|
54
|
+
/** Live accessor for registered workflow type names (WFT-28); registrations are still empty here. */
|
|
55
|
+
listRegisteredWorkflowTypes: () => Iterable<string>;
|
|
54
56
|
getComposedWorkflowInterceptor?: () => ComposedWorkflowInterceptor | null;
|
|
55
57
|
resolveWorkflowType: (target: string | Function) => string;
|
|
56
58
|
registerCancelHandler?: (workflowId: string, handler: () => Promise<void> | void) => () => void;
|
|
@@ -187,6 +187,7 @@ export function createExecutionStrategyBundle(parameters) {
|
|
|
187
187
|
development,
|
|
188
188
|
broadcastEvents,
|
|
189
189
|
getRegistration,
|
|
190
|
+
listRegisteredWorkflowTypes,
|
|
190
191
|
getComposedWorkflowInterceptor,
|
|
191
192
|
resolveWorkflowType,
|
|
192
193
|
registerCancelHandler,
|
|
@@ -203,6 +204,8 @@ export function createExecutionStrategyBundle(parameters) {
|
|
|
203
204
|
broadcastEvents,
|
|
204
205
|
requireProtocolVersion: workerExecutionConfiguration.requireProtocolVersion,
|
|
205
206
|
discardOnCancel: workerExecutionConfiguration.discardOnCancel,
|
|
207
|
+
requireRealmReady: !0,
|
|
208
|
+
getExpectedWorkflowTypes: () => [...listRegisteredWorkflowTypes()],
|
|
206
209
|
workflowTurnTimeoutMs: workerExecutionConfiguration.workflowTurnTimeoutMs,
|
|
207
210
|
maxProtocolMessageBytes: workerExecutionConfiguration.maxProtocolMessageBytes,
|
|
208
211
|
getNow,
|
|
@@ -280,6 +280,7 @@ export class Engine extends EventTarget {
|
|
|
280
280
|
development: resolvedOptions.development,
|
|
281
281
|
broadcastEvents: resolvedOptions.broadcastEvents,
|
|
282
282
|
getRegistration: getInternals(this).registrations.get.bind(getInternals(this).registrations),
|
|
283
|
+
listRegisteredWorkflowTypes: () => getInternals(this).registrations.keys(),
|
|
283
284
|
getComposedWorkflowInterceptor: () => getComposedWorkflowInterceptor(getInternals(this)),
|
|
284
285
|
resolveWorkflowType: this.#resolveWorkflowTypeTarget.bind(this),
|
|
285
286
|
registerCancelHandler: (workflowId, handler) => registerCancelHandler(getInternals(this), workflowId, handler),
|
|
@@ -3,6 +3,7 @@ import type { EngineInternals } from './internals.ts';
|
|
|
3
3
|
export declare const BULK_OPERATION_BATCH_SIZE = 1000;
|
|
4
4
|
/** List workflow summaries that match a filter, using indexes when available. */
|
|
5
5
|
export declare function list(internals: EngineInternals, filter?: ListFilter, options?: ListOptions): Promise<PaginatedResult<WorkflowSummary>>;
|
|
6
|
+
export declare function assertWorkflowListScanWithinCap(scannedRows: number): void;
|
|
6
7
|
/** Stream decoded workflow states that match a list filter. */
|
|
7
8
|
export declare function streamWorkflowStates(internals: EngineInternals, filter?: ListFilter): AsyncGenerator<WorkflowState>;
|
|
8
9
|
/** Stream decoded workflow states in fixed-size batches for bulk operations. */
|
|
@@ -25,8 +25,7 @@ export async function list(internals, filter, options) {
|
|
|
25
25
|
}
|
|
26
26
|
async function collectSummariesFromConstrainedIds(internals, constrainedIds, normalizedFilter, normalizedTagFilters, options) {
|
|
27
27
|
const orderedIds = [...constrainedIds];
|
|
28
|
-
|
|
29
|
-
throw new WorkflowListScanCapExceededError(MAX_LIST_SCAN_ROWS);
|
|
28
|
+
assertWorkflowListScanWithinCap(orderedIds.length);
|
|
30
29
|
const items = [];
|
|
31
30
|
for (let start = 0;start < orderedIds.length; start += CONSTRAINED_ID_CHUNK_SIZE) {
|
|
32
31
|
const chunkIds = orderedIds.slice(start, start + CONSTRAINED_ID_CHUNK_SIZE), chunkBytes = await Promise.all(chunkIds.map((workflowId) => internals.storage.get(KEYS.workflow(workflowId)))), searchAttributesByWorkflowId = await readSearchAttributesForStates(internals, chunkBytes, normalizedFilter), matchingStates = [];
|
|
@@ -49,8 +48,7 @@ async function collectSummariesFromFullScan(internals, normalizedFilter, normali
|
|
|
49
48
|
if (!isTopLevelWorkflowStateKey(key))
|
|
50
49
|
continue;
|
|
51
50
|
scanned += 1;
|
|
52
|
-
|
|
53
|
-
throw new WorkflowListScanCapExceededError(MAX_LIST_SCAN_ROWS);
|
|
51
|
+
assertWorkflowListScanWithinCap(scanned);
|
|
54
52
|
const state = decodeWorkflowState(value), searchAttributes = await readSearchAttributesForFilter(internals, state.id, normalizedFilter);
|
|
55
53
|
if (!matchesListFilter(state, normalizedFilter, null, normalizedTagFilters, searchAttributes))
|
|
56
54
|
continue;
|
|
@@ -59,6 +57,10 @@ async function collectSummariesFromFullScan(internals, normalizedFilter, normali
|
|
|
59
57
|
}
|
|
60
58
|
return items;
|
|
61
59
|
}
|
|
60
|
+
export function assertWorkflowListScanWithinCap(scannedRows) {
|
|
61
|
+
if (scannedRows > MAX_LIST_SCAN_ROWS)
|
|
62
|
+
throw new WorkflowListScanCapExceededError(MAX_LIST_SCAN_ROWS);
|
|
63
|
+
}
|
|
62
64
|
async function summariesFromStates(internals, states, options) {
|
|
63
65
|
const attributeBytesByWorkflowId = new Map;
|
|
64
66
|
await Promise.all(states.map(async (state) => {
|
|
@@ -25,7 +25,7 @@
|
|
|
25
25
|
* }
|
|
26
26
|
* ```
|
|
27
27
|
*/
|
|
28
|
-
export type WeftErrorCode = 'WorkflowAlreadyExistsError' | 'BulkDeleteRequiresTerminalWorkflowsError' | 'BulkOperationConfirmationError' | 'WorkflowTypeNotRegisteredForRecoveryError' | 'EngineCreateNameMismatchError' | 'EngineDisposedError' | 'EngineDisposalError' | 'WorkflowNotFoundError' | 'WorkflowNotRegisteredError' | 'WorkflowConcurrencyLimitExceededError' | 'WorkflowSuspendNotSupportedError' | 'ActivityResolutionError' | 'BranchTopologyChangedError' | 'PersistedDataIncompatibleError' | 'WorkflowTimeoutError' | 'HttpClientError' | 'WorkerProtocolIncompatibleError' | 'UpdateTimeoutError' | 'UpdateValidationError' | 'WorkflowTerminalError' | 'WorkflowBuilderError' | 'VersionMismatchError' | 'EffectReplayConflictError' | 'ReviewTimeoutError' | 'AtomicStateConflictError' | 'StandardSchemaValidationError' | 'ActivityReconciliationCapabilityError' | 'ActivityReconciliationConflictError' | 'ActivityReconciliationIndeterminateError' | 'DurableActivityScopeError' | 'DurableActivityUnsupportedError' | 'AsyncActivityTokenNotFoundError' | 'ActivityScheduleToCloseTimeoutError' | 'ActivityPerAttemptTimeoutError' | 'PayloadSizeExceededError' | 'StartOrSignalConflictError' | 'WorkflowTeardownPendingError' | 'IdempotencyKeyPurgedError';
|
|
28
|
+
export type WeftErrorCode = 'WorkflowAlreadyExistsError' | 'BulkDeleteRequiresTerminalWorkflowsError' | 'BulkOperationConfirmationError' | 'WorkflowTypeNotRegisteredForRecoveryError' | 'EngineCreateNameMismatchError' | 'EngineDisposedError' | 'EngineDisposalError' | 'WorkflowNotFoundError' | 'WorkflowNotRegisteredError' | 'WorkflowConcurrencyLimitExceededError' | 'WorkflowSuspendNotSupportedError' | 'ActivityResolutionError' | 'BranchTopologyChangedError' | 'PersistedDataIncompatibleError' | 'WorkflowTimeoutError' | 'HttpClientError' | 'WorkerProtocolIncompatibleError' | 'UpdateTimeoutError' | 'UpdateValidationError' | 'WorkflowTerminalError' | 'WorkflowBuilderError' | 'VersionMismatchError' | 'EffectReplayConflictError' | 'ReviewTimeoutError' | 'AtomicStateConflictError' | 'StandardSchemaValidationError' | 'ActivityReconciliationCapabilityError' | 'ActivityReconciliationConflictError' | 'ActivityReconciliationIndeterminateError' | 'DurableActivityScopeError' | 'DurableActivityUnsupportedError' | 'AsyncActivityTokenNotFoundError' | 'ActivityScheduleToCloseTimeoutError' | 'ActivityPerAttemptTimeoutError' | 'PayloadSizeExceededError' | 'StartOrSignalConflictError' | 'WorkflowTeardownPendingError' | 'IdempotencyKeyPurgedError' | 'WorkerManifestBuildError';
|
|
29
29
|
/**
|
|
30
30
|
* Generic abstract base for all Weft library errors. The `TCode` parameter
|
|
31
31
|
* makes each subclass's `code` its own literal type; the exported base surface
|
package/dist/core/weft-error.js
CHANGED
|
@@ -44,7 +44,8 @@ const publicWeftErrorCodeMap = {
|
|
|
44
44
|
PayloadSizeExceededError: !0,
|
|
45
45
|
StartOrSignalConflictError: !0,
|
|
46
46
|
WorkflowTeardownPendingError: !0,
|
|
47
|
-
IdempotencyKeyPurgedError: !0
|
|
47
|
+
IdempotencyKeyPurgedError: !0,
|
|
48
|
+
WorkerManifestBuildError: !0
|
|
48
49
|
}, PUBLIC_WEFT_ERROR_CODES = new Set(Object.keys(publicWeftErrorCodeMap));
|
|
49
50
|
export function isWeftError(value) {
|
|
50
51
|
return value instanceof WeftError;
|
|
@@ -9,6 +9,14 @@ export interface WorkerExecutionDispatcherDependencies {
|
|
|
9
9
|
validateHostToWorkerMessage: (workflowId: string, message: WorkerInboundMessage, worker?: Worker) => boolean;
|
|
10
10
|
attachWorkerListeners: (worker: Worker) => void;
|
|
11
11
|
detachWorkerListenersIfIdle: (worker: Worker) => void;
|
|
12
|
+
/**
|
|
13
|
+
* Resolve once `worker`'s realm-ready handshake has settled (or
|
|
14
|
+
* immediately, if realm-readiness isn't required). `false` means the
|
|
15
|
+
* handshake failed or timed out; the implementation is responsible for
|
|
16
|
+
* discarding the worker and emitting the workflow failure itself, the same
|
|
17
|
+
* contract `validateHostToWorkerMessage` already uses.
|
|
18
|
+
*/
|
|
19
|
+
ensureRealmReady: (worker: Worker, workflowId: string) => Promise<boolean>;
|
|
12
20
|
beginTurn: (worker: Worker, workflowId: string, turnId: number, kind: 'run' | 'resume') => void;
|
|
13
21
|
clearTurn: (worker: Worker) => void;
|
|
14
22
|
discardWorkerAndFailWorkflows: (worker: Worker, options: {
|
|
@@ -11,6 +11,10 @@ export class WorkerExecutionDispatcher {
|
|
|
11
11
|
const worker = await this.#dependencies.pool.acquire();
|
|
12
12
|
this.#dependencies.ownership.setActive(workflowId, worker);
|
|
13
13
|
this.#dependencies.attachWorkerListeners(worker);
|
|
14
|
+
if (!await this.#dependencies.ensureRealmReady(worker, workflowId))
|
|
15
|
+
return;
|
|
16
|
+
if (this.#dependencies.isDisposed() || this.#dependencies.ownership.getActiveWorker(workflowId) !== worker)
|
|
17
|
+
return;
|
|
14
18
|
this.#sendActiveMessage(worker, workflowId, message, "run");
|
|
15
19
|
} catch (error) {
|
|
16
20
|
this.#dependencies.emit({
|
|
@@ -5,6 +5,22 @@ export interface WorkerExecutionStrategyOptions {
|
|
|
5
5
|
maxProtocolMessageBytes?: number;
|
|
6
6
|
requireProtocolVersion?: boolean;
|
|
7
7
|
discardOnCancel?: boolean;
|
|
8
|
+
/**
|
|
9
|
+
* Require every freshly acquired worker to complete the realm-ready
|
|
10
|
+
* handshake (WFT-28) — send a valid `ready` message whose manifest
|
|
11
|
+
* contains a matching contract for every type in
|
|
12
|
+
* {@link getExpectedWorkflowTypes} (a subset check, not exact-manifest
|
|
13
|
+
* equality) — before it receives its first `run` turn. Defaults to
|
|
14
|
+
* `false` so direct/test construction of this class is unaffected;
|
|
15
|
+
* `createExecutionStrategyBundle` hardcodes this to `true` for every
|
|
16
|
+
* engine-constructed worker-mode strategy, mirroring how
|
|
17
|
+
* `requireProtocolVersion` is already hardcoded there.
|
|
18
|
+
*/
|
|
19
|
+
requireRealmReady?: boolean;
|
|
20
|
+
/** Required when {@link requireRealmReady} is `true`. Called fresh on every handshake. */
|
|
21
|
+
getExpectedWorkflowTypes?: () => readonly string[];
|
|
22
|
+
/** Bound (ms) on the realm-ready wait. Defaults to `DEFAULT_WORKER_REALM_READY_TIMEOUT_MS`. */
|
|
23
|
+
realmReadyTimeoutMs?: number;
|
|
8
24
|
/**
|
|
9
25
|
* The engine host's `EngineOptions.onLog` sink (#529). When present, the strategy
|
|
10
26
|
* tells each worker (`hostHasLogSink: true` on `run`/`resume`) to forward `ctx.log`
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import { WorkerCheckpointResumeState } from "./worker-checkpoint-resume-state.js";
|
|
2
2
|
import { WorkerExecutionDispatcher } from "./worker-execution-dispatcher.js";
|
|
3
3
|
import { WorkerExecutionOwnership } from "./worker-execution-ownership.js";
|
|
4
|
+
import { WorkerFaultHandler } from "./worker-fault-handling.js";
|
|
4
5
|
import {
|
|
5
6
|
buildResumeMessage,
|
|
6
7
|
buildRunMessage
|
|
@@ -15,7 +16,11 @@ import {
|
|
|
15
16
|
} from "./worker-message-helpers.js";
|
|
16
17
|
import { WorkerProtocolGuard } from "./worker-protocol-guard.js";
|
|
17
18
|
import { isWorkerLogMessage } from "./worker-protocol-log.js";
|
|
18
|
-
import {
|
|
19
|
+
import {
|
|
20
|
+
DEFAULT_WORKER_REALM_READY_TIMEOUT_MS,
|
|
21
|
+
WORKER_PROTOCOL_VERSION
|
|
22
|
+
} from "./worker-protocol.js";
|
|
23
|
+
import { isWorkerRealmReadyMessage, WorkerRealmReadiness } from "./worker-realm-readiness.js";
|
|
19
24
|
import {
|
|
20
25
|
WorkerTurnWatchdog
|
|
21
26
|
} from "./worker-turn-watchdog.js";
|
|
@@ -34,6 +39,8 @@ export class WorkerExecutionStrategy {
|
|
|
34
39
|
#forwardedLogGate;
|
|
35
40
|
#turnWatchdog;
|
|
36
41
|
#protocolGuard;
|
|
42
|
+
#realmReadiness;
|
|
43
|
+
#faultHandler;
|
|
37
44
|
#dispatcher;
|
|
38
45
|
#messageHandler;
|
|
39
46
|
#disposed;
|
|
@@ -44,7 +51,10 @@ export class WorkerExecutionStrategy {
|
|
|
44
51
|
maxProtocolMessageBytes,
|
|
45
52
|
requireProtocolVersion = !1,
|
|
46
53
|
discardOnCancel = !1,
|
|
47
|
-
broadcastEvents = !1
|
|
54
|
+
broadcastEvents = !1,
|
|
55
|
+
requireRealmReady = !1,
|
|
56
|
+
getExpectedWorkflowTypes,
|
|
57
|
+
realmReadyTimeoutMs = DEFAULT_WORKER_REALM_READY_TIMEOUT_MS
|
|
48
58
|
} = options ?? {};
|
|
49
59
|
this.#pool = pool;
|
|
50
60
|
this.#ownership = new WorkerExecutionOwnership;
|
|
@@ -56,21 +66,36 @@ export class WorkerExecutionStrategy {
|
|
|
56
66
|
this.#discardOnCancel = discardOnCancel;
|
|
57
67
|
this.#forwardedLogGate = forwardedLogGateFromStrategyOptions(options, maxProtocolMessageBytes);
|
|
58
68
|
this.#turnWatchdog = new WorkerTurnWatchdog(this.#workflowTurnTimeoutMs, (turn) => {
|
|
59
|
-
this.#handleTurnTimeout(turn);
|
|
69
|
+
this.#faultHandler.handleTurnTimeout(turn);
|
|
60
70
|
});
|
|
61
71
|
this.#protocolGuard = new WorkerProtocolGuard(this.#maxProtocolMessageBytes, this.#requireProtocolVersion, this.#turnWatchdog);
|
|
72
|
+
this.#realmReadiness = WorkerExecutionStrategy.#buildRealmReadiness(requireRealmReady, getExpectedWorkflowTypes, realmReadyTimeoutMs, this.#maxProtocolMessageBytes);
|
|
73
|
+
this.#faultHandler = new WorkerFaultHandler({
|
|
74
|
+
ownership: this.#ownership,
|
|
75
|
+
checkpointResumeState: this.#checkpointResumeState,
|
|
76
|
+
turnWatchdog: this.#turnWatchdog,
|
|
77
|
+
forwardedLogGate: this.#forwardedLogGate,
|
|
78
|
+
realmReadiness: this.#realmReadiness,
|
|
79
|
+
workerListeners: this.#workerListeners,
|
|
80
|
+
protocolGuard: this.#protocolGuard,
|
|
81
|
+
pool: this.#pool,
|
|
82
|
+
emit: (message) => {
|
|
83
|
+
this.#emit(message);
|
|
84
|
+
}
|
|
85
|
+
});
|
|
62
86
|
this.#dispatcher = new WorkerExecutionDispatcher({
|
|
63
87
|
pool: this.#pool,
|
|
64
88
|
ownership: this.#ownership,
|
|
65
89
|
isDisposed: () => this.#disposed,
|
|
66
90
|
requireProtocolVersion: () => this.#requireProtocolVersion,
|
|
67
|
-
validateHostToWorkerMessage: (workflowId, message, worker) => this.#assertHostToWorkerMessageWithinLimit(workflowId, message, worker),
|
|
91
|
+
validateHostToWorkerMessage: (workflowId, message, worker) => this.#faultHandler.assertHostToWorkerMessageWithinLimit(workflowId, message, worker),
|
|
68
92
|
attachWorkerListeners: (worker) => {
|
|
69
93
|
this.#attachWorkerListeners(worker);
|
|
70
94
|
},
|
|
71
95
|
detachWorkerListenersIfIdle: (worker) => {
|
|
72
96
|
this.#detachWorkerListenersIfIdle(worker);
|
|
73
97
|
},
|
|
98
|
+
ensureRealmReady: (worker, workflowId) => this.#ensureRealmReady(worker, workflowId),
|
|
74
99
|
beginTurn: (worker, workflowId, turnId, kind) => {
|
|
75
100
|
this.#turnWatchdog.begin(worker, workflowId, turnId, kind);
|
|
76
101
|
},
|
|
@@ -78,7 +103,7 @@ export class WorkerExecutionStrategy {
|
|
|
78
103
|
this.#turnWatchdog.clear(worker);
|
|
79
104
|
},
|
|
80
105
|
discardWorkerAndFailWorkflows: (worker, discardOptions) => {
|
|
81
|
-
this.#discardWorkerAndFailWorkflows(worker, discardOptions);
|
|
106
|
+
this.#faultHandler.discardWorkerAndFailWorkflows(worker, discardOptions);
|
|
82
107
|
},
|
|
83
108
|
emit: (message) => {
|
|
84
109
|
this.#emit(message);
|
|
@@ -108,7 +133,7 @@ export class WorkerExecutionStrategy {
|
|
|
108
133
|
this.#ownership.resetWorkflow(parameters.workflowId);
|
|
109
134
|
this.#checkpointResumeState.resetWorkflow(parameters.workflowId);
|
|
110
135
|
const message = buildRunMessage(parameters, this.#inboundMessageContext());
|
|
111
|
-
if (!this.#assertHostToWorkerMessageWithinLimit(parameters.workflowId, message))
|
|
136
|
+
if (!this.#faultHandler.assertHostToWorkerMessageWithinLimit(parameters.workflowId, message))
|
|
112
137
|
return;
|
|
113
138
|
this.#dispatcher.acquireAndSend(parameters.workflowId, message);
|
|
114
139
|
}
|
|
@@ -135,6 +160,23 @@ export class WorkerExecutionStrategy {
|
|
|
135
160
|
error: `No worker assigned for workflow: ${parameters.workflowId}`
|
|
136
161
|
});
|
|
137
162
|
}
|
|
163
|
+
async#ensureRealmReady(worker, workflowId) {
|
|
164
|
+
if (!this.#realmReadiness)
|
|
165
|
+
return !0;
|
|
166
|
+
if (this.#realmReadiness.isReady(worker))
|
|
167
|
+
return !0;
|
|
168
|
+
const outcome = await this.#realmReadiness.waitForReady(worker);
|
|
169
|
+
if (outcome.ok)
|
|
170
|
+
return !0;
|
|
171
|
+
this.#faultHandler.discardWorkerAndFailWorkflows(worker, {
|
|
172
|
+
targetWorkflowId: workflowId,
|
|
173
|
+
targetCategory: outcome.failureCategory,
|
|
174
|
+
targetError: outcome.error,
|
|
175
|
+
otherCategory: "system",
|
|
176
|
+
otherError: `Worker discarded after realm-ready handshake failed: ${outcome.error}`
|
|
177
|
+
});
|
|
178
|
+
return !1;
|
|
179
|
+
}
|
|
138
180
|
#inboundMessageContext() {
|
|
139
181
|
return {
|
|
140
182
|
turnId: this.#nextTurnId++,
|
|
@@ -147,7 +189,7 @@ export class WorkerExecutionStrategy {
|
|
|
147
189
|
if (worker) {
|
|
148
190
|
this.#ownership.markCancelled(workflowId);
|
|
149
191
|
if (this.#discardOnCancel) {
|
|
150
|
-
this.#discardWorkerAndFailWorkflows(worker, {
|
|
192
|
+
this.#faultHandler.discardWorkerAndFailWorkflows(worker, {
|
|
151
193
|
targetWorkflowId: workflowId,
|
|
152
194
|
skipTarget: !0,
|
|
153
195
|
otherCategory: "system",
|
|
@@ -171,7 +213,7 @@ export class WorkerExecutionStrategy {
|
|
|
171
213
|
this.#ownership.markCancelled(workflowId);
|
|
172
214
|
this.#ownership.deleteParked(workflowId);
|
|
173
215
|
if (this.#discardOnCancel) {
|
|
174
|
-
this.#discardWorkerAndFailWorkflows(parkedWorker, {
|
|
216
|
+
this.#faultHandler.discardWorkerAndFailWorkflows(parkedWorker, {
|
|
175
217
|
targetWorkflowId: workflowId,
|
|
176
218
|
skipTarget: !0,
|
|
177
219
|
otherCategory: "system",
|
|
@@ -197,6 +239,7 @@ export class WorkerExecutionStrategy {
|
|
|
197
239
|
this.#broadcastChannel.close();
|
|
198
240
|
}
|
|
199
241
|
this.#turnWatchdog.clearAll();
|
|
242
|
+
this.#realmReadiness?.clear();
|
|
200
243
|
const activeWorkflowIds = this.#ownership.activeWorkflowIds();
|
|
201
244
|
for (const workflowId of activeWorkflowIds)
|
|
202
245
|
this.#releaseActiveWorker(workflowId);
|
|
@@ -206,13 +249,18 @@ export class WorkerExecutionStrategy {
|
|
|
206
249
|
this.#messageHandler = null;
|
|
207
250
|
}
|
|
208
251
|
async#handleWorkerMessage(worker, message) {
|
|
252
|
+
if (isWorkerRealmReadyMessage(message)) {
|
|
253
|
+
if (this.#realmReadiness)
|
|
254
|
+
await this.#realmReadiness.noteReadyMessage(worker, message);
|
|
255
|
+
return;
|
|
256
|
+
}
|
|
209
257
|
if (isWorkerLogMessage(message)) {
|
|
210
258
|
const owns = (id) => this.#ownership.getTargetWorker(id) === worker, abuseDiscard = this.#forwardedLogGate.handle(worker, message, owns);
|
|
211
259
|
if (abuseDiscard)
|
|
212
|
-
this.#discardWorkerAndFailWorkflows(worker, abuseDiscard);
|
|
260
|
+
this.#faultHandler.discardWorkerAndFailWorkflows(worker, abuseDiscard);
|
|
213
261
|
return;
|
|
214
262
|
}
|
|
215
|
-
if (!this.#acceptWorkerMessage(worker, message))
|
|
263
|
+
if (!this.#faultHandler.acceptWorkerMessage(worker, message))
|
|
216
264
|
return;
|
|
217
265
|
this.#turnWatchdog.clear(worker);
|
|
218
266
|
const resumeVersionBeforeCheckpointHandling = this.#checkpointResumeState.beginCheckpointHandling(message), emitResult = emitWorkerMessageToEngine(this.#messageHandler, message), handlerFailed = emitResult instanceof Promise ? await emitResult : emitResult;
|
|
@@ -244,14 +292,6 @@ export class WorkerExecutionStrategy {
|
|
|
244
292
|
if (this.#ownership.parkActive(message.workflowId, worker))
|
|
245
293
|
this.#pool.release(worker);
|
|
246
294
|
}
|
|
247
|
-
#handleWorkerError(worker, errorEvent) {
|
|
248
|
-
this.#discardWorkerAndFailWorkflows(worker, {
|
|
249
|
-
targetCategory: "system",
|
|
250
|
-
targetError: `Worker crashed: ${errorEvent.message ?? "unknown error"}`,
|
|
251
|
-
otherCategory: "system",
|
|
252
|
-
otherError: `Worker crashed: ${errorEvent.message ?? "unknown error"}`
|
|
253
|
-
});
|
|
254
|
-
}
|
|
255
295
|
#releaseActiveWorker(workflowId) {
|
|
256
296
|
const worker = this.#ownership.releaseActive(workflowId);
|
|
257
297
|
if (worker) {
|
|
@@ -266,10 +306,10 @@ export class WorkerExecutionStrategy {
|
|
|
266
306
|
this.#handleWorkerMessage(worker, message).catch(() => {});
|
|
267
307
|
},
|
|
268
308
|
error: (errorEvent) => {
|
|
269
|
-
this.#handleWorkerError(worker, errorEvent);
|
|
309
|
+
this.#faultHandler.handleWorkerError(worker, errorEvent);
|
|
270
310
|
},
|
|
271
311
|
messageerror: () => {
|
|
272
|
-
this.#discardWorkerAndFailWorkflows(worker, {
|
|
312
|
+
this.#faultHandler.discardWorkerAndFailWorkflows(worker, {
|
|
273
313
|
targetCategory: "system",
|
|
274
314
|
targetError: "Worker messageerror event",
|
|
275
315
|
otherCategory: "system",
|
|
@@ -281,75 +321,6 @@ export class WorkerExecutionStrategy {
|
|
|
281
321
|
#detachWorkerListenersIfIdle(worker) {
|
|
282
322
|
this.#workerListeners.detachIfIdle(worker, (candidate) => this.#ownership.workerIsIdle(candidate));
|
|
283
323
|
}
|
|
284
|
-
#assertHostToWorkerMessageWithinLimit(workflowId, message, worker) {
|
|
285
|
-
const failure = this.#protocolGuard.validateHostToWorkerMessage(message);
|
|
286
|
-
if (!failure)
|
|
287
|
-
return !0;
|
|
288
|
-
if (worker) {
|
|
289
|
-
this.#discardWorkerAndFailWorkflows(worker, {
|
|
290
|
-
targetWorkflowId: workflowId,
|
|
291
|
-
targetCategory: failure.failureCategory,
|
|
292
|
-
targetError: failure.error,
|
|
293
|
-
otherCategory: "system",
|
|
294
|
-
otherError: `Worker discarded after protocol send failure for workflow: ${workflowId}`
|
|
295
|
-
});
|
|
296
|
-
return !1;
|
|
297
|
-
}
|
|
298
|
-
this.#emit({
|
|
299
|
-
type: "failed",
|
|
300
|
-
workflowId,
|
|
301
|
-
error: failure.error,
|
|
302
|
-
failureCategory: failure.failureCategory
|
|
303
|
-
});
|
|
304
|
-
return !1;
|
|
305
|
-
}
|
|
306
|
-
#acceptWorkerMessage(worker, message) {
|
|
307
|
-
const result = this.#protocolGuard.acceptWorkerMessage(worker, message);
|
|
308
|
-
if (result.accepted)
|
|
309
|
-
return !0;
|
|
310
|
-
this.#discardWorkerAndFailWorkflows(worker, {
|
|
311
|
-
...result.failure.targetWorkflowId === void 0 ? {} : { targetWorkflowId: result.failure.targetWorkflowId },
|
|
312
|
-
targetCategory: result.failure.failureCategory,
|
|
313
|
-
targetError: result.failure.error,
|
|
314
|
-
otherCategory: "system",
|
|
315
|
-
otherError: result.failure.otherError
|
|
316
|
-
});
|
|
317
|
-
return !1;
|
|
318
|
-
}
|
|
319
|
-
#handleTurnTimeout(turn) {
|
|
320
|
-
this.#discardWorkerAndFailWorkflows(turn.worker, {
|
|
321
|
-
targetWorkflowId: turn.workflowId,
|
|
322
|
-
targetCategory: "timeout",
|
|
323
|
-
targetError: `Worker workflow turn timed out after ${turn.timeoutMs}ms`,
|
|
324
|
-
otherCategory: "timeout",
|
|
325
|
-
otherError: `Worker discarded after workflow turn timed out: ${turn.workflowId}`
|
|
326
|
-
});
|
|
327
|
-
}
|
|
328
|
-
#discardWorkerAndFailWorkflows(worker, options) {
|
|
329
|
-
const workflowIds = this.#ownership.workflowIdsForWorker(worker);
|
|
330
|
-
if (workflowIds.length === 0) {
|
|
331
|
-
this.#turnWatchdog.clear(worker);
|
|
332
|
-
this.#forwardedLogGate.forget(worker);
|
|
333
|
-
return;
|
|
334
|
-
}
|
|
335
|
-
for (const workflowId of workflowIds) {
|
|
336
|
-
const isTarget = workflowId === options.targetWorkflowId;
|
|
337
|
-
this.#ownership.forgetWorkflow(workflowId);
|
|
338
|
-
this.#checkpointResumeState.forgetWorkflowIfClosed(workflowId, !0);
|
|
339
|
-
if (isTarget && options.skipTarget)
|
|
340
|
-
continue;
|
|
341
|
-
this.#emit({
|
|
342
|
-
type: "failed",
|
|
343
|
-
workflowId,
|
|
344
|
-
error: isTarget ? options.targetError ?? options.otherError : options.otherError,
|
|
345
|
-
failureCategory: isTarget ? options.targetCategory ?? options.otherCategory : options.otherCategory
|
|
346
|
-
});
|
|
347
|
-
}
|
|
348
|
-
this.#turnWatchdog.clear(worker);
|
|
349
|
-
this.#forwardedLogGate.forget(worker);
|
|
350
|
-
this.#workerListeners.detach(worker);
|
|
351
|
-
this.#pool.discard(worker);
|
|
352
|
-
}
|
|
353
324
|
#handleBroadcastMessage(data) {
|
|
354
325
|
if (data.type === "signal:received" && typeof data.workflowId === "string") {
|
|
355
326
|
const targetWorker = this.#ownership.getTargetWorker(data.workflowId);
|
|
@@ -362,4 +333,15 @@ export class WorkerExecutionStrategy {
|
|
|
362
333
|
if (result instanceof Promise)
|
|
363
334
|
result.catch(() => {});
|
|
364
335
|
}
|
|
336
|
+
static #buildRealmReadiness(requireRealmReady, getExpectedWorkflowTypes, timeoutMs, maxProtocolMessageBytes) {
|
|
337
|
+
if (!requireRealmReady)
|
|
338
|
+
return null;
|
|
339
|
+
if (!getExpectedWorkflowTypes)
|
|
340
|
+
throw Error("WorkerExecutionStrategyOptions.getExpectedWorkflowTypes is required when requireRealmReady is true");
|
|
341
|
+
return new WorkerRealmReadiness({
|
|
342
|
+
getExpectedWorkflowTypes,
|
|
343
|
+
timeoutMs,
|
|
344
|
+
maxProtocolMessageBytes
|
|
345
|
+
});
|
|
346
|
+
}
|
|
365
347
|
}
|
|
@@ -0,0 +1,45 @@
|
|
|
1
|
+
import type { WorkerPool } from '../workers/pool.ts';
|
|
2
|
+
import type { FailureCategory, WorkerInboundMessage, WorkerOutboundMessage } from './types.ts';
|
|
3
|
+
import type { WorkerCheckpointResumeState } from './worker-checkpoint-resume-state.ts';
|
|
4
|
+
import type { WorkerExecutionOwnership } from './worker-execution-ownership.ts';
|
|
5
|
+
import type { WorkerListenerRegistry } from './worker-listener-registry.ts';
|
|
6
|
+
import type { ForwardedLogGate } from './worker-log-abuse-counter.ts';
|
|
7
|
+
import type { WorkerProtocolGuard } from './worker-protocol-guard.ts';
|
|
8
|
+
import type { WorkerRealmReadiness } from './worker-realm-readiness.ts';
|
|
9
|
+
import type { WorkerTurnState, WorkerTurnWatchdog } from './worker-turn-watchdog.ts';
|
|
10
|
+
export interface WorkerFaultHandlerDependencies {
|
|
11
|
+
ownership: WorkerExecutionOwnership;
|
|
12
|
+
checkpointResumeState: WorkerCheckpointResumeState;
|
|
13
|
+
turnWatchdog: WorkerTurnWatchdog;
|
|
14
|
+
forwardedLogGate: ForwardedLogGate;
|
|
15
|
+
realmReadiness: WorkerRealmReadiness | null;
|
|
16
|
+
workerListeners: WorkerListenerRegistry;
|
|
17
|
+
protocolGuard: WorkerProtocolGuard;
|
|
18
|
+
pool: WorkerPool;
|
|
19
|
+
emit: (message: WorkerOutboundMessage) => void;
|
|
20
|
+
}
|
|
21
|
+
export interface WorkerDiscardOptions {
|
|
22
|
+
targetWorkflowId?: string;
|
|
23
|
+
targetCategory?: FailureCategory;
|
|
24
|
+
targetError?: string;
|
|
25
|
+
skipTarget?: boolean;
|
|
26
|
+
otherCategory: FailureCategory;
|
|
27
|
+
otherError: string;
|
|
28
|
+
}
|
|
29
|
+
/**
|
|
30
|
+
* Owns every path that ends in a worker being discarded and its owned
|
|
31
|
+
* workflows failed: protocol violations, turn timeouts, worker crashes, and
|
|
32
|
+
* (WFT-28) realm-ready handshake failures. Extracted from
|
|
33
|
+
* {@link WorkerExecutionStrategy} to keep that file under the repository's
|
|
34
|
+
* file-size ceiling; every method here previously lived as a private method
|
|
35
|
+
* on that class and is unchanged in behavior.
|
|
36
|
+
*/
|
|
37
|
+
export declare class WorkerFaultHandler {
|
|
38
|
+
#private;
|
|
39
|
+
constructor(dependencies: WorkerFaultHandlerDependencies);
|
|
40
|
+
assertHostToWorkerMessageWithinLimit(workflowId: string, message: WorkerInboundMessage, worker?: Worker): boolean;
|
|
41
|
+
acceptWorkerMessage(worker: Worker, message: unknown): message is WorkerOutboundMessage;
|
|
42
|
+
handleTurnTimeout(turn: WorkerTurnState): void;
|
|
43
|
+
handleWorkerError(worker: Worker, errorEvent: ErrorEvent): void;
|
|
44
|
+
discardWorkerAndFailWorkflows(worker: Worker, options: WorkerDiscardOptions): void;
|
|
45
|
+
}
|
|
@@ -0,0 +1,88 @@
|
|
|
1
|
+
export class WorkerFaultHandler {
|
|
2
|
+
#dependencies;
|
|
3
|
+
constructor(dependencies) {
|
|
4
|
+
this.#dependencies = dependencies;
|
|
5
|
+
}
|
|
6
|
+
assertHostToWorkerMessageWithinLimit(workflowId, message, worker) {
|
|
7
|
+
const failure = this.#dependencies.protocolGuard.validateHostToWorkerMessage(message);
|
|
8
|
+
if (!failure)
|
|
9
|
+
return !0;
|
|
10
|
+
if (worker) {
|
|
11
|
+
this.discardWorkerAndFailWorkflows(worker, {
|
|
12
|
+
targetWorkflowId: workflowId,
|
|
13
|
+
targetCategory: failure.failureCategory,
|
|
14
|
+
targetError: failure.error,
|
|
15
|
+
otherCategory: "system",
|
|
16
|
+
otherError: `Worker discarded after protocol send failure for workflow: ${workflowId}`
|
|
17
|
+
});
|
|
18
|
+
return !1;
|
|
19
|
+
}
|
|
20
|
+
this.#dependencies.emit({
|
|
21
|
+
type: "failed",
|
|
22
|
+
workflowId,
|
|
23
|
+
error: failure.error,
|
|
24
|
+
failureCategory: failure.failureCategory
|
|
25
|
+
});
|
|
26
|
+
return !1;
|
|
27
|
+
}
|
|
28
|
+
acceptWorkerMessage(worker, message) {
|
|
29
|
+
const result = this.#dependencies.protocolGuard.acceptWorkerMessage(worker, message);
|
|
30
|
+
if (result.accepted)
|
|
31
|
+
return !0;
|
|
32
|
+
this.discardWorkerAndFailWorkflows(worker, {
|
|
33
|
+
...result.failure.targetWorkflowId === void 0 ? {} : { targetWorkflowId: result.failure.targetWorkflowId },
|
|
34
|
+
targetCategory: result.failure.failureCategory,
|
|
35
|
+
targetError: result.failure.error,
|
|
36
|
+
otherCategory: "system",
|
|
37
|
+
otherError: result.failure.otherError
|
|
38
|
+
});
|
|
39
|
+
return !1;
|
|
40
|
+
}
|
|
41
|
+
handleTurnTimeout(turn) {
|
|
42
|
+
this.discardWorkerAndFailWorkflows(turn.worker, {
|
|
43
|
+
targetWorkflowId: turn.workflowId,
|
|
44
|
+
targetCategory: "timeout",
|
|
45
|
+
targetError: `Worker workflow turn timed out after ${turn.timeoutMs}ms`,
|
|
46
|
+
otherCategory: "timeout",
|
|
47
|
+
otherError: `Worker discarded after workflow turn timed out: ${turn.workflowId}`
|
|
48
|
+
});
|
|
49
|
+
}
|
|
50
|
+
handleWorkerError(worker, errorEvent) {
|
|
51
|
+
this.discardWorkerAndFailWorkflows(worker, {
|
|
52
|
+
targetCategory: "system",
|
|
53
|
+
targetError: `Worker crashed: ${errorEvent.message ?? "unknown error"}`,
|
|
54
|
+
otherCategory: "system",
|
|
55
|
+
otherError: `Worker crashed: ${errorEvent.message ?? "unknown error"}`
|
|
56
|
+
});
|
|
57
|
+
}
|
|
58
|
+
discardWorkerAndFailWorkflows(worker, options) {
|
|
59
|
+
const { ownership } = this.#dependencies, workflowIds = ownership.workflowIdsForWorker(worker);
|
|
60
|
+
if (workflowIds.length === 0) {
|
|
61
|
+
this.#clearWorkerState(worker);
|
|
62
|
+
return;
|
|
63
|
+
}
|
|
64
|
+
for (const workflowId of workflowIds)
|
|
65
|
+
this.#failOwnedWorkflow(workflowId, workflowId === options.targetWorkflowId, options);
|
|
66
|
+
this.#clearWorkerState(worker);
|
|
67
|
+
this.#dependencies.workerListeners.detach(worker);
|
|
68
|
+
this.#dependencies.pool.discard(worker);
|
|
69
|
+
}
|
|
70
|
+
#failOwnedWorkflow(workflowId, isTarget, options) {
|
|
71
|
+
const { ownership, checkpointResumeState } = this.#dependencies;
|
|
72
|
+
ownership.forgetWorkflow(workflowId);
|
|
73
|
+
checkpointResumeState.forgetWorkflowIfClosed(workflowId, !0);
|
|
74
|
+
if (isTarget && options.skipTarget)
|
|
75
|
+
return;
|
|
76
|
+
this.#dependencies.emit({
|
|
77
|
+
type: "failed",
|
|
78
|
+
workflowId,
|
|
79
|
+
error: isTarget ? options.targetError ?? options.otherError : options.otherError,
|
|
80
|
+
failureCategory: isTarget ? options.targetCategory ?? options.otherCategory : options.otherCategory
|
|
81
|
+
});
|
|
82
|
+
}
|
|
83
|
+
#clearWorkerState(worker) {
|
|
84
|
+
this.#dependencies.turnWatchdog.clear(worker);
|
|
85
|
+
this.#dependencies.forwardedLogGate.forget(worker);
|
|
86
|
+
this.#dependencies.realmReadiness?.forget(worker);
|
|
87
|
+
}
|
|
88
|
+
}
|
|
@@ -2,10 +2,27 @@ import type { ContextOperationRequest } from './context.ts';
|
|
|
2
2
|
import type { FailureCategory, OperationRequest, WorkerOutboundMessage } from './types.ts';
|
|
3
3
|
import { type WorkerReplayOperationSignature } from './types/checkpoint.ts';
|
|
4
4
|
export { WORKER_REPLAY_SIGNATURE_FORMAT, type WorkerReplayOperationSignature, } from './types/checkpoint.ts';
|
|
5
|
-
|
|
5
|
+
/**
|
|
6
|
+
* Bumped 1 -> 2 for the realm-ready handshake (WFT-28): a worker realm must
|
|
7
|
+
* now send a `ready` message before its first turn, a required pre-turn
|
|
8
|
+
* message the wire semantics didn't previously have. Host and worker are
|
|
9
|
+
* always built from the same package version — there is no independent peer
|
|
10
|
+
* to stay backward compatible with — so this is a clean-break bump, the same
|
|
11
|
+
* reasoning the RemoteWorker protocol v2 -> v3 bump used.
|
|
12
|
+
*/
|
|
13
|
+
export declare const WORKER_PROTOCOL_VERSION = 2;
|
|
6
14
|
export declare const DEFAULT_WORKER_TURN_TIMEOUT_MS = 1000;
|
|
7
15
|
export declare const DEFAULT_WORKER_PROTOCOL_MESSAGE_BYTES = 1048576;
|
|
8
16
|
export declare const MIN_WORKER_PROTOCOL_MESSAGE_BYTES = 4096;
|
|
17
|
+
/**
|
|
18
|
+
* Default bound (ms) on how long the host waits for a freshly acquired
|
|
19
|
+
* worker's `ready` handshake message before discarding it (WFT-28). Worker boot
|
|
20
|
+
* is module load plus evaluation with no I/O, but a cold Bun Worker spinning
|
|
21
|
+
* up a large bundled module graph can take meaningfully longer than a single
|
|
22
|
+
* workflow turn, so this is deliberately more generous than
|
|
23
|
+
* {@link DEFAULT_WORKER_TURN_TIMEOUT_MS}.
|
|
24
|
+
*/
|
|
25
|
+
export declare const DEFAULT_WORKER_REALM_READY_TIMEOUT_MS = 5000;
|
|
9
26
|
/**
|
|
10
27
|
* Default fixed-window length (ms) for the forwarded-log flood budget (#545). Paired
|
|
11
28
|
* with {@link DEFAULT_FORWARDED_LOG_FLOOD_THRESHOLD} over this window.
|
|
@@ -7,7 +7,7 @@ import { isValidWorkerLogRecord } from "./worker-protocol-log.js";
|
|
|
7
7
|
export {
|
|
8
8
|
WORKER_REPLAY_SIGNATURE_FORMAT
|
|
9
9
|
} from "./types/checkpoint.js";
|
|
10
|
-
export const WORKER_PROTOCOL_VERSION =
|
|
10
|
+
export const WORKER_PROTOCOL_VERSION = 2, DEFAULT_WORKER_TURN_TIMEOUT_MS = 1000, DEFAULT_WORKER_PROTOCOL_MESSAGE_BYTES = 1048576, MIN_WORKER_PROTOCOL_MESSAGE_BYTES = 4096, DEFAULT_WORKER_REALM_READY_TIMEOUT_MS = 5000, DEFAULT_FORWARDED_LOG_FLOOD_WINDOW_MS = 60000, DEFAULT_FORWARDED_LOG_FLOOD_THRESHOLD = 5000, FORWARDED_LOG_STRIKE_THRESHOLD = 5;
|
|
11
11
|
const BINARY_MARKER = "__weftBinaryBytes", DATE_MARKER = "__weftDate", ERROR_MARKER = "__weftError", MAP_MARKER = "__weftMap", SET_MARKER = "__weftSet", MAX_BOUNDED_ERROR_LENGTH = 512, WORKER_SIGNATURE_EXCLUDED_FIELDS = new Set([
|
|
12
12
|
"attempt",
|
|
13
13
|
"callerStack",
|