@lostgradient/weft 0.18.0 → 0.19.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (167) hide show
  1. package/README.md +3 -1
  2. package/dist/cli/conformance.js +14 -18
  3. package/dist/cli/generated/operation-catalog.snapshot.json +258 -16
  4. package/dist/cli/generated/operation-client.generated.d.ts +30 -6
  5. package/dist/cli/generated/operation-client.generated.js +4 -0
  6. package/dist/core/context/child-workflow-pipe.d.ts +1 -1
  7. package/dist/core/context/child-workflow-pipe.js +1 -1
  8. package/dist/core/engine/construction.d.ts +2 -0
  9. package/dist/core/engine/construction.js +3 -0
  10. package/dist/core/engine/index.js +1 -0
  11. package/dist/core/engine/listing.d.ts +1 -0
  12. package/dist/core/engine/listing.js +6 -4
  13. package/dist/core/weft-error.d.ts +1 -1
  14. package/dist/core/weft-error.js +2 -1
  15. package/dist/core/worker-execution-dispatcher.d.ts +8 -0
  16. package/dist/core/worker-execution-dispatcher.js +4 -0
  17. package/dist/core/worker-execution-strategy-options.d.ts +16 -0
  18. package/dist/core/worker-execution-strategy.js +71 -89
  19. package/dist/core/worker-fault-handling.d.ts +45 -0
  20. package/dist/core/worker-fault-handling.js +88 -0
  21. package/dist/core/worker-protocol.d.ts +18 -1
  22. package/dist/core/worker-protocol.js +1 -1
  23. package/dist/core/worker-realm-readiness.d.ts +78 -0
  24. package/dist/core/worker-realm-readiness.js +127 -0
  25. package/dist/index.d.ts +3 -1
  26. package/dist/index.js +28 -1
  27. package/dist/json-schema.js +1 -1
  28. package/dist/runtime/portable.d.ts +14 -0
  29. package/dist/runtime/portable.js +10 -0
  30. package/dist/server/index.d.ts +71 -8
  31. package/dist/server/index.js +7 -3
  32. package/dist/server/operations/get-task-diagnostics.d.ts +43 -18
  33. package/dist/server/operations/get-task-diagnostics.js +87 -126
  34. package/dist/server/operations/get-worker-diagnostics.d.ts +177 -0
  35. package/dist/server/operations/get-worker-diagnostics.js +117 -0
  36. package/dist/server/operations/list-worker-registration-rejections.d.ts +79 -0
  37. package/dist/server/operations/list-worker-registration-rejections.js +60 -0
  38. package/dist/server/operations/list-workers.js +1 -3
  39. package/dist/server/operations/static-registrations.js +5 -20
  40. package/dist/server/rest-bindings.js +31 -2
  41. package/dist/server/runtime/authentication-bridge.js +6 -14
  42. package/dist/server/runtime/context.d.ts +23 -0
  43. package/dist/server/runtime/task-dispatch.d.ts +7 -1
  44. package/dist/server/runtime/task-dispatch.js +126 -55
  45. package/dist/server/runtime/task-ledger-completion.d.ts +68 -0
  46. package/dist/server/runtime/task-ledger-completion.js +72 -0
  47. package/dist/server/runtime/task-ledger-recovery.d.ts +55 -0
  48. package/dist/server/runtime/task-ledger-recovery.js +73 -0
  49. package/dist/server/runtime/task-ledger-runtime.d.ts +64 -0
  50. package/dist/server/runtime/task-ledger-runtime.js +34 -0
  51. package/dist/server/runtime/task-metrics.d.ts +6 -4
  52. package/dist/server/runtime/task-polling.d.ts +14 -12
  53. package/dist/server/runtime/task-polling.js +106 -82
  54. package/dist/server/runtime/task-reconciliation.d.ts +35 -8
  55. package/dist/server/runtime/task-reconciliation.js +75 -74
  56. package/dist/server/runtime/task-result-resolution.d.ts +2 -10
  57. package/dist/server/runtime/task-result-resolution.js +0 -67
  58. package/dist/server/runtime/task-result-view.d.ts +82 -0
  59. package/dist/server/runtime/task-result-view.js +48 -0
  60. package/dist/server/runtime/websocket-worker-messaging.d.ts +16 -0
  61. package/dist/server/runtime/websocket-worker-messaging.js +32 -0
  62. package/dist/server/runtime/websocket-worker-registration.d.ts +41 -0
  63. package/dist/server/runtime/websocket-worker-registration.js +134 -0
  64. package/dist/server/runtime/websocket-worker.d.ts +0 -1
  65. package/dist/server/runtime/websocket-worker.js +64 -116
  66. package/dist/server/serve-internals.d.ts +9 -6
  67. package/dist/server/serve-internals.js +14 -40
  68. package/dist/server/task-ledger-codec.d.ts +40 -0
  69. package/dist/server/task-ledger-codec.js +195 -0
  70. package/dist/server/task-ledger-keys.d.ts +13 -0
  71. package/dist/server/task-ledger-keys.js +5 -0
  72. package/dist/server/task-ledger-limits.d.ts +34 -0
  73. package/dist/server/task-ledger-limits.js +5 -0
  74. package/dist/server/task-ledger-transition-helpers.d.ts +38 -0
  75. package/dist/server/task-ledger-transition-helpers.js +42 -0
  76. package/dist/server/task-ledger-transitions-cancellation.d.ts +31 -0
  77. package/dist/server/task-ledger-transitions-cancellation.js +73 -0
  78. package/dist/server/task-ledger-transitions.d.ts +129 -0
  79. package/dist/server/task-ledger-transitions.js +162 -0
  80. package/dist/server/task-ledger-types.d.ts +220 -0
  81. package/dist/server/task-ledger-types.js +1 -0
  82. package/dist/server/task-ledger.d.ts +17 -0
  83. package/dist/server/task-ledger.js +24 -0
  84. package/dist/server/task-state.d.ts +25 -152
  85. package/dist/server/task-state.js +0 -188
  86. package/dist/server/worker-admission-policy.d.ts +70 -0
  87. package/dist/server/worker-admission-policy.js +0 -0
  88. package/dist/storage/compressed-storage.js +1 -1
  89. package/dist/storage/index.d.ts +0 -2
  90. package/dist/storage/interface.d.ts +0 -2
  91. package/dist/storage/interface.js +1 -1
  92. package/dist/storage/lmdb.js +1 -1
  93. package/dist/storage/memory.js +1 -1
  94. package/dist/storage/neon.js +1 -1
  95. package/dist/storage/postgres.js +1 -1
  96. package/dist/storage/resolve.js +1 -1
  97. package/dist/storage/scoped-storage.js +1 -1
  98. package/dist/storage/testing.js +1 -1
  99. package/dist/storage/turso.js +1 -1
  100. package/dist/storage/web-extension.js +2 -2
  101. package/dist/version.d.ts +1 -1
  102. package/dist/version.js +1 -1
  103. package/dist/web-extension.js +1 -1
  104. package/dist/worker/index.d.ts +2 -0
  105. package/dist/worker/index.js +30 -4
  106. package/dist/worker/manifest/canonical-json.d.ts +20 -0
  107. package/dist/worker/manifest/canonical-json.js +13 -0
  108. package/dist/worker/manifest/capabilities.d.ts +24 -0
  109. package/dist/worker/manifest/capabilities.js +74 -0
  110. package/dist/worker/manifest/content-digest.d.ts +20 -0
  111. package/dist/worker/manifest/content-digest.js +13 -0
  112. package/dist/worker/manifest/declared-shape-digest.d.ts +14 -0
  113. package/dist/worker/manifest/declared-shape-digest.js +4 -0
  114. package/dist/worker/manifest/digest.d.ts +82 -0
  115. package/dist/worker/manifest/digest.js +9 -0
  116. package/dist/worker/manifest/execution-identity.d.ts +92 -0
  117. package/dist/worker/manifest/execution-identity.js +36 -0
  118. package/dist/worker/manifest/failure.d.ts +59 -0
  119. package/dist/worker/manifest/failure.js +4 -0
  120. package/dist/worker/manifest/index.d.ts +22 -0
  121. package/dist/worker/manifest/index.js +23 -0
  122. package/dist/worker/manifest/internal-realm.d.ts +50 -0
  123. package/dist/worker/manifest/internal-realm.js +39 -0
  124. package/dist/worker/manifest/is-record.d.ts +12 -0
  125. package/dist/worker/manifest/is-record.js +6 -0
  126. package/dist/worker/manifest/json-scan.d.ts +26 -0
  127. package/dist/worker/manifest/json-scan.js +88 -0
  128. package/dist/worker/manifest/limits.d.ts +138 -0
  129. package/dist/worker/manifest/limits.js +1 -0
  130. package/dist/worker/manifest/normalize.d.ts +63 -0
  131. package/dist/worker/manifest/normalize.js +73 -0
  132. package/dist/worker/manifest/parse-json.d.ts +24 -0
  133. package/dist/worker/manifest/parse-json.js +15 -0
  134. package/dist/worker/manifest/parse.d.ts +86 -0
  135. package/dist/worker/manifest/parse.js +188 -0
  136. package/dist/worker/manifest/registry-contract-builder.d.ts +120 -0
  137. package/dist/worker/manifest/registry-contract-builder.js +65 -0
  138. package/dist/worker/manifest/types.d.ts +263 -0
  139. package/dist/worker/manifest/types.js +1 -0
  140. package/dist/worker/manifest/utf8.d.ts +13 -0
  141. package/dist/worker/manifest/utf8.js +7 -0
  142. package/dist/worker/options.d.ts +55 -8
  143. package/dist/worker/options.js +65 -9
  144. package/dist/worker/protocol-internals.d.ts +1 -2
  145. package/dist/worker/protocol-internals.js +0 -5
  146. package/dist/worker/protocol-messages.d.ts +33 -17
  147. package/dist/worker/protocol-schemas.d.ts +36 -74
  148. package/dist/worker/protocol-schemas.js +25 -18
  149. package/dist/worker/protocol-version.d.ts +18 -14
  150. package/dist/worker/protocol-version.js +1 -1
  151. package/dist/worker/protocol.js +1 -1
  152. package/dist/worker/registry/deployment-consistency.d.ts +78 -0
  153. package/dist/worker/registry/deployment-consistency.js +27 -0
  154. package/dist/worker/registry/drain.d.ts +14 -0
  155. package/dist/worker/registry/drain.js +24 -0
  156. package/dist/worker/registry/rejections.d.ts +31 -0
  157. package/dist/worker/registry/rejections.js +10 -0
  158. package/dist/worker/registry/summary.d.ts +0 -3
  159. package/dist/worker/registry/summary.js +2 -5
  160. package/dist/worker/registry/types.d.ts +10 -3
  161. package/dist/worker/registry.d.ts +47 -2
  162. package/dist/worker/registry.js +45 -32
  163. package/dist/workers/workflow-worker-entry.d.ts +12 -7
  164. package/dist/workers/workflow-worker-entry.js +9 -2
  165. package/package.json +3 -3
  166. package/dist/server/task-resolved-record.d.ts +0 -12
  167. package/dist/server/task-resolved-record.js +0 -35
@@ -25,7 +25,7 @@
25
25
  * }
26
26
  * ```
27
27
  */
28
- export type WeftErrorCode = 'WorkflowAlreadyExistsError' | 'BulkDeleteRequiresTerminalWorkflowsError' | 'BulkOperationConfirmationError' | 'WorkflowTypeNotRegisteredForRecoveryError' | 'EngineCreateNameMismatchError' | 'EngineDisposedError' | 'EngineDisposalError' | 'WorkflowNotFoundError' | 'WorkflowNotRegisteredError' | 'WorkflowConcurrencyLimitExceededError' | 'WorkflowSuspendNotSupportedError' | 'ActivityResolutionError' | 'BranchTopologyChangedError' | 'PersistedDataIncompatibleError' | 'WorkflowTimeoutError' | 'HttpClientError' | 'WorkerProtocolIncompatibleError' | 'UpdateTimeoutError' | 'UpdateValidationError' | 'WorkflowTerminalError' | 'WorkflowBuilderError' | 'VersionMismatchError' | 'EffectReplayConflictError' | 'ReviewTimeoutError' | 'AtomicStateConflictError' | 'StandardSchemaValidationError' | 'ActivityReconciliationCapabilityError' | 'ActivityReconciliationConflictError' | 'ActivityReconciliationIndeterminateError' | 'DurableActivityScopeError' | 'DurableActivityUnsupportedError' | 'AsyncActivityTokenNotFoundError' | 'ActivityScheduleToCloseTimeoutError' | 'ActivityPerAttemptTimeoutError' | 'PayloadSizeExceededError' | 'StartOrSignalConflictError' | 'WorkflowTeardownPendingError' | 'IdempotencyKeyPurgedError';
28
+ export type WeftErrorCode = 'WorkflowAlreadyExistsError' | 'BulkDeleteRequiresTerminalWorkflowsError' | 'BulkOperationConfirmationError' | 'WorkflowTypeNotRegisteredForRecoveryError' | 'EngineCreateNameMismatchError' | 'EngineDisposedError' | 'EngineDisposalError' | 'WorkflowNotFoundError' | 'WorkflowNotRegisteredError' | 'WorkflowConcurrencyLimitExceededError' | 'WorkflowSuspendNotSupportedError' | 'ActivityResolutionError' | 'BranchTopologyChangedError' | 'PersistedDataIncompatibleError' | 'WorkflowTimeoutError' | 'HttpClientError' | 'WorkerProtocolIncompatibleError' | 'UpdateTimeoutError' | 'UpdateValidationError' | 'WorkflowTerminalError' | 'WorkflowBuilderError' | 'VersionMismatchError' | 'EffectReplayConflictError' | 'ReviewTimeoutError' | 'AtomicStateConflictError' | 'StandardSchemaValidationError' | 'ActivityReconciliationCapabilityError' | 'ActivityReconciliationConflictError' | 'ActivityReconciliationIndeterminateError' | 'DurableActivityScopeError' | 'DurableActivityUnsupportedError' | 'AsyncActivityTokenNotFoundError' | 'ActivityScheduleToCloseTimeoutError' | 'ActivityPerAttemptTimeoutError' | 'PayloadSizeExceededError' | 'StartOrSignalConflictError' | 'WorkflowTeardownPendingError' | 'IdempotencyKeyPurgedError' | 'WorkerManifestBuildError';
29
29
  /**
30
30
  * Generic abstract base for all Weft library errors. The `TCode` parameter
31
31
  * makes each subclass's `code` its own literal type; the exported base surface
@@ -44,7 +44,8 @@ const publicWeftErrorCodeMap = {
44
44
  PayloadSizeExceededError: !0,
45
45
  StartOrSignalConflictError: !0,
46
46
  WorkflowTeardownPendingError: !0,
47
- IdempotencyKeyPurgedError: !0
47
+ IdempotencyKeyPurgedError: !0,
48
+ WorkerManifestBuildError: !0
48
49
  }, PUBLIC_WEFT_ERROR_CODES = new Set(Object.keys(publicWeftErrorCodeMap));
49
50
  export function isWeftError(value) {
50
51
  return value instanceof WeftError;
@@ -9,6 +9,14 @@ export interface WorkerExecutionDispatcherDependencies {
9
9
  validateHostToWorkerMessage: (workflowId: string, message: WorkerInboundMessage, worker?: Worker) => boolean;
10
10
  attachWorkerListeners: (worker: Worker) => void;
11
11
  detachWorkerListenersIfIdle: (worker: Worker) => void;
12
+ /**
13
+ * Resolve once `worker`'s realm-ready handshake has settled (or
14
+ * immediately, if realm-readiness isn't required). `false` means the
15
+ * handshake failed or timed out; the implementation is responsible for
16
+ * discarding the worker and emitting the workflow failure itself, the same
17
+ * contract `validateHostToWorkerMessage` already uses.
18
+ */
19
+ ensureRealmReady: (worker: Worker, workflowId: string) => Promise<boolean>;
12
20
  beginTurn: (worker: Worker, workflowId: string, turnId: number, kind: 'run' | 'resume') => void;
13
21
  clearTurn: (worker: Worker) => void;
14
22
  discardWorkerAndFailWorkflows: (worker: Worker, options: {
@@ -11,6 +11,10 @@ export class WorkerExecutionDispatcher {
11
11
  const worker = await this.#dependencies.pool.acquire();
12
12
  this.#dependencies.ownership.setActive(workflowId, worker);
13
13
  this.#dependencies.attachWorkerListeners(worker);
14
+ if (!await this.#dependencies.ensureRealmReady(worker, workflowId))
15
+ return;
16
+ if (this.#dependencies.isDisposed() || this.#dependencies.ownership.getActiveWorker(workflowId) !== worker)
17
+ return;
14
18
  this.#sendActiveMessage(worker, workflowId, message, "run");
15
19
  } catch (error) {
16
20
  this.#dependencies.emit({
@@ -5,6 +5,22 @@ export interface WorkerExecutionStrategyOptions {
5
5
  maxProtocolMessageBytes?: number;
6
6
  requireProtocolVersion?: boolean;
7
7
  discardOnCancel?: boolean;
8
+ /**
9
+ * Require every freshly acquired worker to complete the realm-ready
10
+ * handshake (WFT-28) — send a valid `ready` message whose manifest
11
+ * contains a matching contract for every type in
12
+ * {@link getExpectedWorkflowTypes} (a subset check, not exact-manifest
13
+ * equality) — before it receives its first `run` turn. Defaults to
14
+ * `false` so direct/test construction of this class is unaffected;
15
+ * `createExecutionStrategyBundle` hardcodes this to `true` for every
16
+ * engine-constructed worker-mode strategy, mirroring how
17
+ * `requireProtocolVersion` is already hardcoded there.
18
+ */
19
+ requireRealmReady?: boolean;
20
+ /** Required when {@link requireRealmReady} is `true`. Called fresh on every handshake. */
21
+ getExpectedWorkflowTypes?: () => readonly string[];
22
+ /** Bound (ms) on the realm-ready wait. Defaults to `DEFAULT_WORKER_REALM_READY_TIMEOUT_MS`. */
23
+ realmReadyTimeoutMs?: number;
8
24
  /**
9
25
  * The engine host's `EngineOptions.onLog` sink (#529). When present, the strategy
10
26
  * tells each worker (`hostHasLogSink: true` on `run`/`resume`) to forward `ctx.log`
@@ -1,6 +1,7 @@
1
1
  import { WorkerCheckpointResumeState } from "./worker-checkpoint-resume-state.js";
2
2
  import { WorkerExecutionDispatcher } from "./worker-execution-dispatcher.js";
3
3
  import { WorkerExecutionOwnership } from "./worker-execution-ownership.js";
4
+ import { WorkerFaultHandler } from "./worker-fault-handling.js";
4
5
  import {
5
6
  buildResumeMessage,
6
7
  buildRunMessage
@@ -15,7 +16,11 @@ import {
15
16
  } from "./worker-message-helpers.js";
16
17
  import { WorkerProtocolGuard } from "./worker-protocol-guard.js";
17
18
  import { isWorkerLogMessage } from "./worker-protocol-log.js";
18
- import { WORKER_PROTOCOL_VERSION } from "./worker-protocol.js";
19
+ import {
20
+ DEFAULT_WORKER_REALM_READY_TIMEOUT_MS,
21
+ WORKER_PROTOCOL_VERSION
22
+ } from "./worker-protocol.js";
23
+ import { isWorkerRealmReadyMessage, WorkerRealmReadiness } from "./worker-realm-readiness.js";
19
24
  import {
20
25
  WorkerTurnWatchdog
21
26
  } from "./worker-turn-watchdog.js";
@@ -34,6 +39,8 @@ export class WorkerExecutionStrategy {
34
39
  #forwardedLogGate;
35
40
  #turnWatchdog;
36
41
  #protocolGuard;
42
+ #realmReadiness;
43
+ #faultHandler;
37
44
  #dispatcher;
38
45
  #messageHandler;
39
46
  #disposed;
@@ -44,7 +51,10 @@ export class WorkerExecutionStrategy {
44
51
  maxProtocolMessageBytes,
45
52
  requireProtocolVersion = !1,
46
53
  discardOnCancel = !1,
47
- broadcastEvents = !1
54
+ broadcastEvents = !1,
55
+ requireRealmReady = !1,
56
+ getExpectedWorkflowTypes,
57
+ realmReadyTimeoutMs = DEFAULT_WORKER_REALM_READY_TIMEOUT_MS
48
58
  } = options ?? {};
49
59
  this.#pool = pool;
50
60
  this.#ownership = new WorkerExecutionOwnership;
@@ -56,21 +66,36 @@ export class WorkerExecutionStrategy {
56
66
  this.#discardOnCancel = discardOnCancel;
57
67
  this.#forwardedLogGate = forwardedLogGateFromStrategyOptions(options, maxProtocolMessageBytes);
58
68
  this.#turnWatchdog = new WorkerTurnWatchdog(this.#workflowTurnTimeoutMs, (turn) => {
59
- this.#handleTurnTimeout(turn);
69
+ this.#faultHandler.handleTurnTimeout(turn);
60
70
  });
61
71
  this.#protocolGuard = new WorkerProtocolGuard(this.#maxProtocolMessageBytes, this.#requireProtocolVersion, this.#turnWatchdog);
72
+ this.#realmReadiness = WorkerExecutionStrategy.#buildRealmReadiness(requireRealmReady, getExpectedWorkflowTypes, realmReadyTimeoutMs, this.#maxProtocolMessageBytes);
73
+ this.#faultHandler = new WorkerFaultHandler({
74
+ ownership: this.#ownership,
75
+ checkpointResumeState: this.#checkpointResumeState,
76
+ turnWatchdog: this.#turnWatchdog,
77
+ forwardedLogGate: this.#forwardedLogGate,
78
+ realmReadiness: this.#realmReadiness,
79
+ workerListeners: this.#workerListeners,
80
+ protocolGuard: this.#protocolGuard,
81
+ pool: this.#pool,
82
+ emit: (message) => {
83
+ this.#emit(message);
84
+ }
85
+ });
62
86
  this.#dispatcher = new WorkerExecutionDispatcher({
63
87
  pool: this.#pool,
64
88
  ownership: this.#ownership,
65
89
  isDisposed: () => this.#disposed,
66
90
  requireProtocolVersion: () => this.#requireProtocolVersion,
67
- validateHostToWorkerMessage: (workflowId, message, worker) => this.#assertHostToWorkerMessageWithinLimit(workflowId, message, worker),
91
+ validateHostToWorkerMessage: (workflowId, message, worker) => this.#faultHandler.assertHostToWorkerMessageWithinLimit(workflowId, message, worker),
68
92
  attachWorkerListeners: (worker) => {
69
93
  this.#attachWorkerListeners(worker);
70
94
  },
71
95
  detachWorkerListenersIfIdle: (worker) => {
72
96
  this.#detachWorkerListenersIfIdle(worker);
73
97
  },
98
+ ensureRealmReady: (worker, workflowId) => this.#ensureRealmReady(worker, workflowId),
74
99
  beginTurn: (worker, workflowId, turnId, kind) => {
75
100
  this.#turnWatchdog.begin(worker, workflowId, turnId, kind);
76
101
  },
@@ -78,7 +103,7 @@ export class WorkerExecutionStrategy {
78
103
  this.#turnWatchdog.clear(worker);
79
104
  },
80
105
  discardWorkerAndFailWorkflows: (worker, discardOptions) => {
81
- this.#discardWorkerAndFailWorkflows(worker, discardOptions);
106
+ this.#faultHandler.discardWorkerAndFailWorkflows(worker, discardOptions);
82
107
  },
83
108
  emit: (message) => {
84
109
  this.#emit(message);
@@ -108,7 +133,7 @@ export class WorkerExecutionStrategy {
108
133
  this.#ownership.resetWorkflow(parameters.workflowId);
109
134
  this.#checkpointResumeState.resetWorkflow(parameters.workflowId);
110
135
  const message = buildRunMessage(parameters, this.#inboundMessageContext());
111
- if (!this.#assertHostToWorkerMessageWithinLimit(parameters.workflowId, message))
136
+ if (!this.#faultHandler.assertHostToWorkerMessageWithinLimit(parameters.workflowId, message))
112
137
  return;
113
138
  this.#dispatcher.acquireAndSend(parameters.workflowId, message);
114
139
  }
@@ -135,6 +160,23 @@ export class WorkerExecutionStrategy {
135
160
  error: `No worker assigned for workflow: ${parameters.workflowId}`
136
161
  });
137
162
  }
163
+ async#ensureRealmReady(worker, workflowId) {
164
+ if (!this.#realmReadiness)
165
+ return !0;
166
+ if (this.#realmReadiness.isReady(worker))
167
+ return !0;
168
+ const outcome = await this.#realmReadiness.waitForReady(worker);
169
+ if (outcome.ok)
170
+ return !0;
171
+ this.#faultHandler.discardWorkerAndFailWorkflows(worker, {
172
+ targetWorkflowId: workflowId,
173
+ targetCategory: outcome.failureCategory,
174
+ targetError: outcome.error,
175
+ otherCategory: "system",
176
+ otherError: `Worker discarded after realm-ready handshake failed: ${outcome.error}`
177
+ });
178
+ return !1;
179
+ }
138
180
  #inboundMessageContext() {
139
181
  return {
140
182
  turnId: this.#nextTurnId++,
@@ -147,7 +189,7 @@ export class WorkerExecutionStrategy {
147
189
  if (worker) {
148
190
  this.#ownership.markCancelled(workflowId);
149
191
  if (this.#discardOnCancel) {
150
- this.#discardWorkerAndFailWorkflows(worker, {
192
+ this.#faultHandler.discardWorkerAndFailWorkflows(worker, {
151
193
  targetWorkflowId: workflowId,
152
194
  skipTarget: !0,
153
195
  otherCategory: "system",
@@ -171,7 +213,7 @@ export class WorkerExecutionStrategy {
171
213
  this.#ownership.markCancelled(workflowId);
172
214
  this.#ownership.deleteParked(workflowId);
173
215
  if (this.#discardOnCancel) {
174
- this.#discardWorkerAndFailWorkflows(parkedWorker, {
216
+ this.#faultHandler.discardWorkerAndFailWorkflows(parkedWorker, {
175
217
  targetWorkflowId: workflowId,
176
218
  skipTarget: !0,
177
219
  otherCategory: "system",
@@ -197,6 +239,7 @@ export class WorkerExecutionStrategy {
197
239
  this.#broadcastChannel.close();
198
240
  }
199
241
  this.#turnWatchdog.clearAll();
242
+ this.#realmReadiness?.clear();
200
243
  const activeWorkflowIds = this.#ownership.activeWorkflowIds();
201
244
  for (const workflowId of activeWorkflowIds)
202
245
  this.#releaseActiveWorker(workflowId);
@@ -206,13 +249,18 @@ export class WorkerExecutionStrategy {
206
249
  this.#messageHandler = null;
207
250
  }
208
251
  async#handleWorkerMessage(worker, message) {
252
+ if (isWorkerRealmReadyMessage(message)) {
253
+ if (this.#realmReadiness)
254
+ await this.#realmReadiness.noteReadyMessage(worker, message);
255
+ return;
256
+ }
209
257
  if (isWorkerLogMessage(message)) {
210
258
  const owns = (id) => this.#ownership.getTargetWorker(id) === worker, abuseDiscard = this.#forwardedLogGate.handle(worker, message, owns);
211
259
  if (abuseDiscard)
212
- this.#discardWorkerAndFailWorkflows(worker, abuseDiscard);
260
+ this.#faultHandler.discardWorkerAndFailWorkflows(worker, abuseDiscard);
213
261
  return;
214
262
  }
215
- if (!this.#acceptWorkerMessage(worker, message))
263
+ if (!this.#faultHandler.acceptWorkerMessage(worker, message))
216
264
  return;
217
265
  this.#turnWatchdog.clear(worker);
218
266
  const resumeVersionBeforeCheckpointHandling = this.#checkpointResumeState.beginCheckpointHandling(message), emitResult = emitWorkerMessageToEngine(this.#messageHandler, message), handlerFailed = emitResult instanceof Promise ? await emitResult : emitResult;
@@ -244,14 +292,6 @@ export class WorkerExecutionStrategy {
244
292
  if (this.#ownership.parkActive(message.workflowId, worker))
245
293
  this.#pool.release(worker);
246
294
  }
247
- #handleWorkerError(worker, errorEvent) {
248
- this.#discardWorkerAndFailWorkflows(worker, {
249
- targetCategory: "system",
250
- targetError: `Worker crashed: ${errorEvent.message ?? "unknown error"}`,
251
- otherCategory: "system",
252
- otherError: `Worker crashed: ${errorEvent.message ?? "unknown error"}`
253
- });
254
- }
255
295
  #releaseActiveWorker(workflowId) {
256
296
  const worker = this.#ownership.releaseActive(workflowId);
257
297
  if (worker) {
@@ -266,10 +306,10 @@ export class WorkerExecutionStrategy {
266
306
  this.#handleWorkerMessage(worker, message).catch(() => {});
267
307
  },
268
308
  error: (errorEvent) => {
269
- this.#handleWorkerError(worker, errorEvent);
309
+ this.#faultHandler.handleWorkerError(worker, errorEvent);
270
310
  },
271
311
  messageerror: () => {
272
- this.#discardWorkerAndFailWorkflows(worker, {
312
+ this.#faultHandler.discardWorkerAndFailWorkflows(worker, {
273
313
  targetCategory: "system",
274
314
  targetError: "Worker messageerror event",
275
315
  otherCategory: "system",
@@ -281,75 +321,6 @@ export class WorkerExecutionStrategy {
281
321
  #detachWorkerListenersIfIdle(worker) {
282
322
  this.#workerListeners.detachIfIdle(worker, (candidate) => this.#ownership.workerIsIdle(candidate));
283
323
  }
284
- #assertHostToWorkerMessageWithinLimit(workflowId, message, worker) {
285
- const failure = this.#protocolGuard.validateHostToWorkerMessage(message);
286
- if (!failure)
287
- return !0;
288
- if (worker) {
289
- this.#discardWorkerAndFailWorkflows(worker, {
290
- targetWorkflowId: workflowId,
291
- targetCategory: failure.failureCategory,
292
- targetError: failure.error,
293
- otherCategory: "system",
294
- otherError: `Worker discarded after protocol send failure for workflow: ${workflowId}`
295
- });
296
- return !1;
297
- }
298
- this.#emit({
299
- type: "failed",
300
- workflowId,
301
- error: failure.error,
302
- failureCategory: failure.failureCategory
303
- });
304
- return !1;
305
- }
306
- #acceptWorkerMessage(worker, message) {
307
- const result = this.#protocolGuard.acceptWorkerMessage(worker, message);
308
- if (result.accepted)
309
- return !0;
310
- this.#discardWorkerAndFailWorkflows(worker, {
311
- ...result.failure.targetWorkflowId === void 0 ? {} : { targetWorkflowId: result.failure.targetWorkflowId },
312
- targetCategory: result.failure.failureCategory,
313
- targetError: result.failure.error,
314
- otherCategory: "system",
315
- otherError: result.failure.otherError
316
- });
317
- return !1;
318
- }
319
- #handleTurnTimeout(turn) {
320
- this.#discardWorkerAndFailWorkflows(turn.worker, {
321
- targetWorkflowId: turn.workflowId,
322
- targetCategory: "timeout",
323
- targetError: `Worker workflow turn timed out after ${turn.timeoutMs}ms`,
324
- otherCategory: "timeout",
325
- otherError: `Worker discarded after workflow turn timed out: ${turn.workflowId}`
326
- });
327
- }
328
- #discardWorkerAndFailWorkflows(worker, options) {
329
- const workflowIds = this.#ownership.workflowIdsForWorker(worker);
330
- if (workflowIds.length === 0) {
331
- this.#turnWatchdog.clear(worker);
332
- this.#forwardedLogGate.forget(worker);
333
- return;
334
- }
335
- for (const workflowId of workflowIds) {
336
- const isTarget = workflowId === options.targetWorkflowId;
337
- this.#ownership.forgetWorkflow(workflowId);
338
- this.#checkpointResumeState.forgetWorkflowIfClosed(workflowId, !0);
339
- if (isTarget && options.skipTarget)
340
- continue;
341
- this.#emit({
342
- type: "failed",
343
- workflowId,
344
- error: isTarget ? options.targetError ?? options.otherError : options.otherError,
345
- failureCategory: isTarget ? options.targetCategory ?? options.otherCategory : options.otherCategory
346
- });
347
- }
348
- this.#turnWatchdog.clear(worker);
349
- this.#forwardedLogGate.forget(worker);
350
- this.#workerListeners.detach(worker);
351
- this.#pool.discard(worker);
352
- }
353
324
  #handleBroadcastMessage(data) {
354
325
  if (data.type === "signal:received" && typeof data.workflowId === "string") {
355
326
  const targetWorker = this.#ownership.getTargetWorker(data.workflowId);
@@ -362,4 +333,15 @@ export class WorkerExecutionStrategy {
362
333
  if (result instanceof Promise)
363
334
  result.catch(() => {});
364
335
  }
336
+ static #buildRealmReadiness(requireRealmReady, getExpectedWorkflowTypes, timeoutMs, maxProtocolMessageBytes) {
337
+ if (!requireRealmReady)
338
+ return null;
339
+ if (!getExpectedWorkflowTypes)
340
+ throw Error("WorkerExecutionStrategyOptions.getExpectedWorkflowTypes is required when requireRealmReady is true");
341
+ return new WorkerRealmReadiness({
342
+ getExpectedWorkflowTypes,
343
+ timeoutMs,
344
+ maxProtocolMessageBytes
345
+ });
346
+ }
365
347
  }
@@ -0,0 +1,45 @@
1
+ import type { WorkerPool } from '../workers/pool.ts';
2
+ import type { FailureCategory, WorkerInboundMessage, WorkerOutboundMessage } from './types.ts';
3
+ import type { WorkerCheckpointResumeState } from './worker-checkpoint-resume-state.ts';
4
+ import type { WorkerExecutionOwnership } from './worker-execution-ownership.ts';
5
+ import type { WorkerListenerRegistry } from './worker-listener-registry.ts';
6
+ import type { ForwardedLogGate } from './worker-log-abuse-counter.ts';
7
+ import type { WorkerProtocolGuard } from './worker-protocol-guard.ts';
8
+ import type { WorkerRealmReadiness } from './worker-realm-readiness.ts';
9
+ import type { WorkerTurnState, WorkerTurnWatchdog } from './worker-turn-watchdog.ts';
10
+ export interface WorkerFaultHandlerDependencies {
11
+ ownership: WorkerExecutionOwnership;
12
+ checkpointResumeState: WorkerCheckpointResumeState;
13
+ turnWatchdog: WorkerTurnWatchdog;
14
+ forwardedLogGate: ForwardedLogGate;
15
+ realmReadiness: WorkerRealmReadiness | null;
16
+ workerListeners: WorkerListenerRegistry;
17
+ protocolGuard: WorkerProtocolGuard;
18
+ pool: WorkerPool;
19
+ emit: (message: WorkerOutboundMessage) => void;
20
+ }
21
+ export interface WorkerDiscardOptions {
22
+ targetWorkflowId?: string;
23
+ targetCategory?: FailureCategory;
24
+ targetError?: string;
25
+ skipTarget?: boolean;
26
+ otherCategory: FailureCategory;
27
+ otherError: string;
28
+ }
29
+ /**
30
+ * Owns every path that ends in a worker being discarded and its owned
31
+ * workflows failed: protocol violations, turn timeouts, worker crashes, and
32
+ * (WFT-28) realm-ready handshake failures. Extracted from
33
+ * {@link WorkerExecutionStrategy} to keep that file under the repository's
34
+ * file-size ceiling; every method here previously lived as a private method
35
+ * on that class and is unchanged in behavior.
36
+ */
37
+ export declare class WorkerFaultHandler {
38
+ #private;
39
+ constructor(dependencies: WorkerFaultHandlerDependencies);
40
+ assertHostToWorkerMessageWithinLimit(workflowId: string, message: WorkerInboundMessage, worker?: Worker): boolean;
41
+ acceptWorkerMessage(worker: Worker, message: unknown): message is WorkerOutboundMessage;
42
+ handleTurnTimeout(turn: WorkerTurnState): void;
43
+ handleWorkerError(worker: Worker, errorEvent: ErrorEvent): void;
44
+ discardWorkerAndFailWorkflows(worker: Worker, options: WorkerDiscardOptions): void;
45
+ }
@@ -0,0 +1,88 @@
1
+ export class WorkerFaultHandler {
2
+ #dependencies;
3
+ constructor(dependencies) {
4
+ this.#dependencies = dependencies;
5
+ }
6
+ assertHostToWorkerMessageWithinLimit(workflowId, message, worker) {
7
+ const failure = this.#dependencies.protocolGuard.validateHostToWorkerMessage(message);
8
+ if (!failure)
9
+ return !0;
10
+ if (worker) {
11
+ this.discardWorkerAndFailWorkflows(worker, {
12
+ targetWorkflowId: workflowId,
13
+ targetCategory: failure.failureCategory,
14
+ targetError: failure.error,
15
+ otherCategory: "system",
16
+ otherError: `Worker discarded after protocol send failure for workflow: ${workflowId}`
17
+ });
18
+ return !1;
19
+ }
20
+ this.#dependencies.emit({
21
+ type: "failed",
22
+ workflowId,
23
+ error: failure.error,
24
+ failureCategory: failure.failureCategory
25
+ });
26
+ return !1;
27
+ }
28
+ acceptWorkerMessage(worker, message) {
29
+ const result = this.#dependencies.protocolGuard.acceptWorkerMessage(worker, message);
30
+ if (result.accepted)
31
+ return !0;
32
+ this.discardWorkerAndFailWorkflows(worker, {
33
+ ...result.failure.targetWorkflowId === void 0 ? {} : { targetWorkflowId: result.failure.targetWorkflowId },
34
+ targetCategory: result.failure.failureCategory,
35
+ targetError: result.failure.error,
36
+ otherCategory: "system",
37
+ otherError: result.failure.otherError
38
+ });
39
+ return !1;
40
+ }
41
+ handleTurnTimeout(turn) {
42
+ this.discardWorkerAndFailWorkflows(turn.worker, {
43
+ targetWorkflowId: turn.workflowId,
44
+ targetCategory: "timeout",
45
+ targetError: `Worker workflow turn timed out after ${turn.timeoutMs}ms`,
46
+ otherCategory: "timeout",
47
+ otherError: `Worker discarded after workflow turn timed out: ${turn.workflowId}`
48
+ });
49
+ }
50
+ handleWorkerError(worker, errorEvent) {
51
+ this.discardWorkerAndFailWorkflows(worker, {
52
+ targetCategory: "system",
53
+ targetError: `Worker crashed: ${errorEvent.message ?? "unknown error"}`,
54
+ otherCategory: "system",
55
+ otherError: `Worker crashed: ${errorEvent.message ?? "unknown error"}`
56
+ });
57
+ }
58
+ discardWorkerAndFailWorkflows(worker, options) {
59
+ const { ownership } = this.#dependencies, workflowIds = ownership.workflowIdsForWorker(worker);
60
+ if (workflowIds.length === 0) {
61
+ this.#clearWorkerState(worker);
62
+ return;
63
+ }
64
+ for (const workflowId of workflowIds)
65
+ this.#failOwnedWorkflow(workflowId, workflowId === options.targetWorkflowId, options);
66
+ this.#clearWorkerState(worker);
67
+ this.#dependencies.workerListeners.detach(worker);
68
+ this.#dependencies.pool.discard(worker);
69
+ }
70
+ #failOwnedWorkflow(workflowId, isTarget, options) {
71
+ const { ownership, checkpointResumeState } = this.#dependencies;
72
+ ownership.forgetWorkflow(workflowId);
73
+ checkpointResumeState.forgetWorkflowIfClosed(workflowId, !0);
74
+ if (isTarget && options.skipTarget)
75
+ return;
76
+ this.#dependencies.emit({
77
+ type: "failed",
78
+ workflowId,
79
+ error: isTarget ? options.targetError ?? options.otherError : options.otherError,
80
+ failureCategory: isTarget ? options.targetCategory ?? options.otherCategory : options.otherCategory
81
+ });
82
+ }
83
+ #clearWorkerState(worker) {
84
+ this.#dependencies.turnWatchdog.clear(worker);
85
+ this.#dependencies.forwardedLogGate.forget(worker);
86
+ this.#dependencies.realmReadiness?.forget(worker);
87
+ }
88
+ }
@@ -2,10 +2,27 @@ import type { ContextOperationRequest } from './context.ts';
2
2
  import type { FailureCategory, OperationRequest, WorkerOutboundMessage } from './types.ts';
3
3
  import { type WorkerReplayOperationSignature } from './types/checkpoint.ts';
4
4
  export { WORKER_REPLAY_SIGNATURE_FORMAT, type WorkerReplayOperationSignature, } from './types/checkpoint.ts';
5
- export declare const WORKER_PROTOCOL_VERSION = 1;
5
+ /**
6
+ * Bumped 1 -> 2 for the realm-ready handshake (WFT-28): a worker realm must
7
+ * now send a `ready` message before its first turn, a required pre-turn
8
+ * message the wire semantics didn't previously have. Host and worker are
9
+ * always built from the same package version — there is no independent peer
10
+ * to stay backward compatible with — so this is a clean-break bump, the same
11
+ * reasoning the RemoteWorker protocol v2 -> v3 bump used.
12
+ */
13
+ export declare const WORKER_PROTOCOL_VERSION = 2;
6
14
  export declare const DEFAULT_WORKER_TURN_TIMEOUT_MS = 1000;
7
15
  export declare const DEFAULT_WORKER_PROTOCOL_MESSAGE_BYTES = 1048576;
8
16
  export declare const MIN_WORKER_PROTOCOL_MESSAGE_BYTES = 4096;
17
+ /**
18
+ * Default bound (ms) on how long the host waits for a freshly acquired
19
+ * worker's `ready` handshake message before discarding it (WFT-28). Worker boot
20
+ * is module load plus evaluation with no I/O, but a cold Bun Worker spinning
21
+ * up a large bundled module graph can take meaningfully longer than a single
22
+ * workflow turn, so this is deliberately more generous than
23
+ * {@link DEFAULT_WORKER_TURN_TIMEOUT_MS}.
24
+ */
25
+ export declare const DEFAULT_WORKER_REALM_READY_TIMEOUT_MS = 5000;
9
26
  /**
10
27
  * Default fixed-window length (ms) for the forwarded-log flood budget (#545). Paired
11
28
  * with {@link DEFAULT_FORWARDED_LOG_FLOOD_THRESHOLD} over this window.
@@ -7,7 +7,7 @@ import { isValidWorkerLogRecord } from "./worker-protocol-log.js";
7
7
  export {
8
8
  WORKER_REPLAY_SIGNATURE_FORMAT
9
9
  } from "./types/checkpoint.js";
10
- export const WORKER_PROTOCOL_VERSION = 1, DEFAULT_WORKER_TURN_TIMEOUT_MS = 1000, DEFAULT_WORKER_PROTOCOL_MESSAGE_BYTES = 1048576, MIN_WORKER_PROTOCOL_MESSAGE_BYTES = 4096, DEFAULT_FORWARDED_LOG_FLOOD_WINDOW_MS = 60000, DEFAULT_FORWARDED_LOG_FLOOD_THRESHOLD = 5000, FORWARDED_LOG_STRIKE_THRESHOLD = 5;
10
+ export const WORKER_PROTOCOL_VERSION = 2, DEFAULT_WORKER_TURN_TIMEOUT_MS = 1000, DEFAULT_WORKER_PROTOCOL_MESSAGE_BYTES = 1048576, MIN_WORKER_PROTOCOL_MESSAGE_BYTES = 4096, DEFAULT_WORKER_REALM_READY_TIMEOUT_MS = 5000, DEFAULT_FORWARDED_LOG_FLOOD_WINDOW_MS = 60000, DEFAULT_FORWARDED_LOG_FLOOD_THRESHOLD = 5000, FORWARDED_LOG_STRIKE_THRESHOLD = 5;
11
11
  const BINARY_MARKER = "__weftBinaryBytes", DATE_MARKER = "__weftDate", ERROR_MARKER = "__weftError", MAP_MARKER = "__weftMap", SET_MARKER = "__weftSet", MAX_BOUNDED_ERROR_LENGTH = 512, WORKER_SIGNATURE_EXCLUDED_FIELDS = new Set([
12
12
  "attempt",
13
13
  "callerStack",
@@ -0,0 +1,78 @@
1
+ /**
2
+ * Realm-ready handshake for internal `workflowExecutionMode: 'worker'`
3
+ * Workers (WFT-28).
4
+ *
5
+ * An internal Worker realm sends a `ready` message — its own manifest, built
6
+ * from exactly the workflow types its bootstrap script registered — before
7
+ * it can receive a `run` turn. The host checks that every workflow type it
8
+ * has registered appears in that manifest with a matching contract (a
9
+ * subset check, not exact-set equality — a realm may legitimately advertise
10
+ * more types than any one host dispatches to it, e.g. a shared worker pool
11
+ * serving several engines). A missing or mismatched type means the realm's
12
+ * bundle disagrees with the host about a workflow it needs (a stale build, a
13
+ * bootstrap script that fell out of sync), and the realm is discarded before
14
+ * it can ever execute a turn, rather than failing opaquely mid-dispatch.
15
+ *
16
+ * `ready` has no `workflowId` — it is a per-worker-lifetime handshake, not a
17
+ * per-workflow-turn message — so it cannot fit {@link WorkerOutboundMessage}'s
18
+ * shape (every variant there requires one) and is validated on its own path
19
+ * instead of through {@link assertWorkerOutboundMessageShape}, the same way
20
+ * `log` is handled outside the strict turn gate.
21
+ *
22
+ * @module core/worker-realm-readiness
23
+ */
24
+ import type { WorkerManifest } from '../worker/manifest/types.ts';
25
+ import type { FailureCategory } from './types.ts';
26
+ /** Sent once by a Worker realm, before its first `run` turn. See {@link WorkerRealmReadiness}. */
27
+ export type WorkerRealmReadyMessage = Readonly<{
28
+ type: 'ready';
29
+ protocolVersion: number;
30
+ realmGeneration: string;
31
+ manifest: WorkerManifest;
32
+ }>;
33
+ /** Shallow type guard — deep field validation happens in {@link WorkerRealmReadiness.noteReadyMessage}. */
34
+ export declare function isWorkerRealmReadyMessage(message: unknown): message is WorkerRealmReadyMessage;
35
+ export type RealmReadyOutcome = {
36
+ ok: true;
37
+ realmGeneration: string;
38
+ manifestDigest: string;
39
+ } | {
40
+ ok: false;
41
+ error: string;
42
+ failureCategory: FailureCategory;
43
+ };
44
+ export interface WorkerRealmReadinessDependencies {
45
+ /**
46
+ * Live accessor for the host's registered workflow types, called fresh on
47
+ * every handshake rather than snapshotted at construction — the engine's
48
+ * registrations map is still empty when the strategy is constructed (the
49
+ * registration loop runs later in `Engine.create()`).
50
+ */
51
+ getExpectedWorkflowTypes: () => readonly string[];
52
+ timeoutMs: number;
53
+ maxProtocolMessageBytes: number | undefined;
54
+ }
55
+ /**
56
+ * Tracks, per pooled `Worker` instance, whether its one-time ready handshake
57
+ * has completed. A worker is validated at most once per lifetime — recycled
58
+ * workers skip straight to {@link isReady}.
59
+ */
60
+ export declare class WorkerRealmReadiness {
61
+ #private;
62
+ constructor(dependencies: WorkerRealmReadinessDependencies);
63
+ isReady(worker: Worker): boolean;
64
+ /**
65
+ * Wait for `worker`'s ready handshake. Resolves immediately if the worker
66
+ * already completed it. The caller must attach its message listener (which
67
+ * routes into {@link noteReadyMessage}) before calling this, in the same
68
+ * synchronous continuation — `acquireAndSend` does exactly that, so no
69
+ * `ready` message can arrive before this registers its pending waiter.
70
+ */
71
+ waitForReady(worker: Worker): Promise<RealmReadyOutcome>;
72
+ /** Validate an inbound `ready` message and settle any pending {@link waitForReady} call. */
73
+ noteReadyMessage(worker: Worker, message: unknown): Promise<void>;
74
+ /** Drop a discarded worker's state, settling any pending waiter with a failure. */
75
+ forget(worker: Worker): void;
76
+ /** Settle every pending waiter on strategy disposal so no wait hangs past teardown. */
77
+ clear(): void;
78
+ }