@lostgradient/weft 0.18.0 → 0.20.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +3 -1
- package/dist/cli/conformance.js +14 -18
- package/dist/cli/generated/operation-catalog.snapshot.json +1597 -21
- package/dist/cli/generated/operation-client.generated.d.ts +37 -6
- package/dist/cli/generated/operation-client.generated.js +6 -0
- package/dist/core/context/child-workflow-pipe.d.ts +1 -1
- package/dist/core/context/child-workflow-pipe.js +1 -1
- package/dist/core/engine/construction.d.ts +2 -0
- package/dist/core/engine/construction.js +3 -0
- package/dist/core/engine/index.js +1 -0
- package/dist/core/engine/listing.d.ts +1 -0
- package/dist/core/engine/listing.js +6 -4
- package/dist/core/weft-error.d.ts +1 -1
- package/dist/core/weft-error.js +2 -1
- package/dist/core/worker-execution-dispatcher.d.ts +8 -0
- package/dist/core/worker-execution-dispatcher.js +4 -0
- package/dist/core/worker-execution-strategy-options.d.ts +16 -0
- package/dist/core/worker-execution-strategy.js +71 -89
- package/dist/core/worker-fault-handling.d.ts +45 -0
- package/dist/core/worker-fault-handling.js +88 -0
- package/dist/core/worker-protocol.d.ts +18 -1
- package/dist/core/worker-protocol.js +1 -1
- package/dist/core/worker-realm-readiness.d.ts +78 -0
- package/dist/core/worker-realm-readiness.js +127 -0
- package/dist/index.d.ts +3 -1
- package/dist/index.js +28 -1
- package/dist/json-schema.js +1 -1
- package/dist/runtime/portable.d.ts +14 -0
- package/dist/runtime/portable.js +10 -0
- package/dist/server/index.d.ts +71 -8
- package/dist/server/index.js +7 -3
- package/dist/server/operations/get-task-detail-schema.d.ts +341 -0
- package/dist/server/operations/get-task-detail-schema.js +118 -0
- package/dist/server/operations/get-task-detail.d.ts +346 -0
- package/dist/server/operations/get-task-detail.js +215 -0
- package/dist/server/operations/get-task-diagnostics.d.ts +43 -18
- package/dist/server/operations/get-task-diagnostics.js +87 -126
- package/dist/server/operations/get-worker-diagnostics.d.ts +177 -0
- package/dist/server/operations/get-worker-diagnostics.js +117 -0
- package/dist/server/operations/list-worker-registration-rejections.d.ts +79 -0
- package/dist/server/operations/list-worker-registration-rejections.js +60 -0
- package/dist/server/operations/list-workers.js +1 -3
- package/dist/server/operations/static-registrations.js +8 -20
- package/dist/server/rest-bindings.js +31 -2
- package/dist/server/runtime/authentication-bridge.js +6 -14
- package/dist/server/runtime/context.d.ts +23 -0
- package/dist/server/runtime/task-dispatch.d.ts +7 -1
- package/dist/server/runtime/task-dispatch.js +126 -55
- package/dist/server/runtime/task-ledger-completion.d.ts +68 -0
- package/dist/server/runtime/task-ledger-completion.js +72 -0
- package/dist/server/runtime/task-ledger-recovery.d.ts +55 -0
- package/dist/server/runtime/task-ledger-recovery.js +73 -0
- package/dist/server/runtime/task-ledger-runtime.d.ts +64 -0
- package/dist/server/runtime/task-ledger-runtime.js +34 -0
- package/dist/server/runtime/task-metrics.d.ts +6 -4
- package/dist/server/runtime/task-polling.d.ts +14 -12
- package/dist/server/runtime/task-polling.js +106 -82
- package/dist/server/runtime/task-reconciliation.d.ts +35 -8
- package/dist/server/runtime/task-reconciliation.js +75 -74
- package/dist/server/runtime/task-result-resolution.d.ts +2 -10
- package/dist/server/runtime/task-result-resolution.js +0 -67
- package/dist/server/runtime/task-result-view.d.ts +82 -0
- package/dist/server/runtime/task-result-view.js +48 -0
- package/dist/server/runtime/websocket-worker-messaging.d.ts +16 -0
- package/dist/server/runtime/websocket-worker-messaging.js +32 -0
- package/dist/server/runtime/websocket-worker-registration.d.ts +41 -0
- package/dist/server/runtime/websocket-worker-registration.js +134 -0
- package/dist/server/runtime/websocket-worker.d.ts +0 -1
- package/dist/server/runtime/websocket-worker.js +64 -116
- package/dist/server/serve-internals.d.ts +9 -6
- package/dist/server/serve-internals.js +14 -40
- package/dist/server/task-ledger-codec.d.ts +40 -0
- package/dist/server/task-ledger-codec.js +195 -0
- package/dist/server/task-ledger-keys.d.ts +13 -0
- package/dist/server/task-ledger-keys.js +5 -0
- package/dist/server/task-ledger-limits.d.ts +34 -0
- package/dist/server/task-ledger-limits.js +5 -0
- package/dist/server/task-ledger-transition-helpers.d.ts +38 -0
- package/dist/server/task-ledger-transition-helpers.js +42 -0
- package/dist/server/task-ledger-transitions-cancellation.d.ts +31 -0
- package/dist/server/task-ledger-transitions-cancellation.js +73 -0
- package/dist/server/task-ledger-transitions.d.ts +129 -0
- package/dist/server/task-ledger-transitions.js +162 -0
- package/dist/server/task-ledger-types.d.ts +220 -0
- package/dist/server/task-ledger-types.js +1 -0
- package/dist/server/task-ledger.d.ts +17 -0
- package/dist/server/task-ledger.js +24 -0
- package/dist/server/task-state.d.ts +25 -152
- package/dist/server/task-state.js +0 -188
- package/dist/server/worker-admission-policy.d.ts +70 -0
- package/dist/server/worker-admission-policy.js +0 -0
- package/dist/storage/compressed-storage.js +1 -1
- package/dist/storage/index.d.ts +0 -2
- package/dist/storage/interface.d.ts +0 -2
- package/dist/storage/interface.js +1 -1
- package/dist/storage/lmdb.js +1 -1
- package/dist/storage/memory.js +1 -1
- package/dist/storage/neon.js +1 -1
- package/dist/storage/postgres.js +1 -1
- package/dist/storage/resolve.js +1 -1
- package/dist/storage/scoped-storage.js +1 -1
- package/dist/storage/testing.js +1 -1
- package/dist/storage/turso.js +1 -1
- package/dist/storage/web-extension.js +2 -2
- package/dist/version.d.ts +1 -1
- package/dist/version.js +1 -1
- package/dist/web-extension.js +1 -1
- package/dist/worker/index.d.ts +2 -0
- package/dist/worker/index.js +30 -4
- package/dist/worker/manifest/canonical-json.d.ts +20 -0
- package/dist/worker/manifest/canonical-json.js +13 -0
- package/dist/worker/manifest/capabilities.d.ts +24 -0
- package/dist/worker/manifest/capabilities.js +74 -0
- package/dist/worker/manifest/content-digest.d.ts +20 -0
- package/dist/worker/manifest/content-digest.js +13 -0
- package/dist/worker/manifest/declared-shape-digest.d.ts +14 -0
- package/dist/worker/manifest/declared-shape-digest.js +4 -0
- package/dist/worker/manifest/digest.d.ts +82 -0
- package/dist/worker/manifest/digest.js +9 -0
- package/dist/worker/manifest/execution-identity.d.ts +92 -0
- package/dist/worker/manifest/execution-identity.js +36 -0
- package/dist/worker/manifest/failure.d.ts +59 -0
- package/dist/worker/manifest/failure.js +4 -0
- package/dist/worker/manifest/index.d.ts +22 -0
- package/dist/worker/manifest/index.js +23 -0
- package/dist/worker/manifest/internal-realm.d.ts +50 -0
- package/dist/worker/manifest/internal-realm.js +39 -0
- package/dist/worker/manifest/is-record.d.ts +12 -0
- package/dist/worker/manifest/is-record.js +6 -0
- package/dist/worker/manifest/json-scan.d.ts +26 -0
- package/dist/worker/manifest/json-scan.js +88 -0
- package/dist/worker/manifest/limits.d.ts +138 -0
- package/dist/worker/manifest/limits.js +1 -0
- package/dist/worker/manifest/normalize.d.ts +63 -0
- package/dist/worker/manifest/normalize.js +73 -0
- package/dist/worker/manifest/parse-json.d.ts +24 -0
- package/dist/worker/manifest/parse-json.js +15 -0
- package/dist/worker/manifest/parse.d.ts +86 -0
- package/dist/worker/manifest/parse.js +188 -0
- package/dist/worker/manifest/registry-contract-builder.d.ts +120 -0
- package/dist/worker/manifest/registry-contract-builder.js +65 -0
- package/dist/worker/manifest/types.d.ts +263 -0
- package/dist/worker/manifest/types.js +1 -0
- package/dist/worker/manifest/utf8.d.ts +13 -0
- package/dist/worker/manifest/utf8.js +7 -0
- package/dist/worker/options.d.ts +55 -8
- package/dist/worker/options.js +65 -9
- package/dist/worker/protocol-internals.d.ts +1 -2
- package/dist/worker/protocol-internals.js +0 -5
- package/dist/worker/protocol-messages.d.ts +33 -17
- package/dist/worker/protocol-schemas.d.ts +36 -74
- package/dist/worker/protocol-schemas.js +25 -18
- package/dist/worker/protocol-version.d.ts +18 -14
- package/dist/worker/protocol-version.js +1 -1
- package/dist/worker/protocol.js +1 -1
- package/dist/worker/registry/deployment-consistency.d.ts +78 -0
- package/dist/worker/registry/deployment-consistency.js +27 -0
- package/dist/worker/registry/drain.d.ts +14 -0
- package/dist/worker/registry/drain.js +24 -0
- package/dist/worker/registry/rejections.d.ts +31 -0
- package/dist/worker/registry/rejections.js +10 -0
- package/dist/worker/registry/summary.d.ts +0 -3
- package/dist/worker/registry/summary.js +2 -5
- package/dist/worker/registry/types.d.ts +10 -3
- package/dist/worker/registry.d.ts +47 -2
- package/dist/worker/registry.js +45 -32
- package/dist/workers/workflow-worker-entry.d.ts +12 -7
- package/dist/workers/workflow-worker-entry.js +9 -2
- package/package.json +3 -3
- package/dist/server/task-resolved-record.d.ts +0 -12
- package/dist/server/task-resolved-record.js +0 -35
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import { type CatalogWeftClient, type WeftClientConnection } from '../operation-client-runtime.ts';
|
|
2
|
-
export declare const CATALOG_OPERATION_NAMES: readonly ["weft.activities.complete", "weft.activities.fail", "weft.alerts.list", "weft.recover.all", "weft.retention.get", "weft.reviews.decision.submit", "weft.reviews.get", "weft.reviews.list", "weft.schedules.cancel", "weft.schedules.create", "weft.schedules.get", "weft.schedules.list", "weft.schedules.pause", "weft.schedules.resume", "weft.schedules.update", "weft.storage.capabilities", "weft.system.lease", "weft.system.metrics", "weft.system.principal", "weft.system.registry", "weft.task.queues.list", "weft.tasks.diagnostics", "weft.updates.result.get", "weft.worker.deployments.drain", "weft.worker.deployments.resume", "weft.workers.drain", "weft.workers.list", "weft.workers.resume", "weft.workflows.activities.pending.list", "weft.workflows.aggregate", "weft.workflows.attributes.get", "weft.workflows.attributes.set", "weft.workflows.bulk.cancel", "weft.workflows.bulk.delete", "weft.workflows.bulk.retryfailed", "weft.workflows.bulk.signal", "weft.workflows.bulk.tags", "weft.workflows.cancel", "weft.workflows.checkpoints.get", "weft.workflows.checkpoints.list", "weft.workflows.events.list", "weft.workflows.finalizer.get", "weft.workflows.fork", "weft.workflows.get", "weft.workflows.list", "weft.workflows.purge", "weft.workflows.query", "weft.workflows.replay", "weft.workflows.result.get", "weft.workflows.resume", "weft.workflows.scheduleprovenance.get", "weft.workflows.signal", "weft.workflows.start", "weft.workflows.startorsignal", "weft.workflows.streams.chunks", "weft.workflows.suspend", "weft.workflows.tags.add", "weft.workflows.tags.remove", "weft.workflows.timeline.get", "weft.workflows.timeout", "weft.workflows.update"];
|
|
2
|
+
export declare const CATALOG_OPERATION_NAMES: readonly ["weft.activities.complete", "weft.activities.fail", "weft.alerts.list", "weft.recover.all", "weft.retention.get", "weft.reviews.decision.submit", "weft.reviews.get", "weft.reviews.list", "weft.schedules.cancel", "weft.schedules.create", "weft.schedules.get", "weft.schedules.list", "weft.schedules.pause", "weft.schedules.resume", "weft.schedules.update", "weft.storage.capabilities", "weft.system.lease", "weft.system.metrics", "weft.system.principal", "weft.system.registry", "weft.task.queues.list", "weft.tasks.diagnostics", "weft.tasks.get", "weft.updates.result.get", "weft.worker.deployments.drain", "weft.worker.deployments.resume", "weft.workers.diagnostics", "weft.workers.drain", "weft.workers.list", "weft.workers.rejections", "weft.workers.resume", "weft.workflows.activities.pending.list", "weft.workflows.aggregate", "weft.workflows.attributes.get", "weft.workflows.attributes.set", "weft.workflows.bulk.cancel", "weft.workflows.bulk.delete", "weft.workflows.bulk.retryfailed", "weft.workflows.bulk.signal", "weft.workflows.bulk.tags", "weft.workflows.cancel", "weft.workflows.checkpoints.get", "weft.workflows.checkpoints.list", "weft.workflows.events.list", "weft.workflows.finalizer.get", "weft.workflows.fork", "weft.workflows.get", "weft.workflows.list", "weft.workflows.purge", "weft.workflows.query", "weft.workflows.replay", "weft.workflows.result.get", "weft.workflows.resume", "weft.workflows.scheduleprovenance.get", "weft.workflows.signal", "weft.workflows.start", "weft.workflows.startorsignal", "weft.workflows.streams.chunks", "weft.workflows.suspend", "weft.workflows.tags.add", "weft.workflows.tags.remove", "weft.workflows.timeline.get", "weft.workflows.timeout", "weft.workflows.update"];
|
|
3
3
|
export type CatalogOperationName = (typeof CATALOG_OPERATION_NAMES)[number];
|
|
4
|
-
export declare const CLIENT_OPERATION_NAMES: readonly ["weft.activities.complete", "weft.activities.fail", "weft.alerts.list", "weft.recover.all", "weft.retention.get", "weft.reviews.decision.submit", "weft.reviews.get", "weft.reviews.list", "weft.schedules.cancel", "weft.schedules.create", "weft.schedules.get", "weft.schedules.list", "weft.schedules.pause", "weft.schedules.resume", "weft.schedules.update", "weft.storage.capabilities", "weft.system.lease", "weft.system.metrics", "weft.system.principal", "weft.system.registry", "weft.task.queues.list", "weft.tasks.diagnostics", "weft.tasks.diagnostics.deadletters.clear", "weft.updates.result.get", "weft.worker.deployments.drain", "weft.worker.deployments.resume", "weft.workers.drain", "weft.workers.list", "weft.workers.resume", "weft.workflows.activities.pending.list", "weft.workflows.aggregate", "weft.workflows.attributes.get", "weft.workflows.attributes.set", "weft.workflows.bulk.cancel", "weft.workflows.bulk.delete", "weft.workflows.bulk.retryfailed", "weft.workflows.bulk.signal", "weft.workflows.bulk.tags", "weft.workflows.cancel", "weft.workflows.checkpoints.get", "weft.workflows.checkpoints.list", "weft.workflows.events.list", "weft.workflows.finalizer.get", "weft.workflows.fork", "weft.workflows.get", "weft.workflows.list", "weft.workflows.purge", "weft.workflows.query", "weft.workflows.replay", "weft.workflows.result.get", "weft.workflows.resume", "weft.workflows.scheduleprovenance.get", "weft.workflows.signal", "weft.workflows.start", "weft.workflows.startorsignal", "weft.workflows.streams.chunks", "weft.workflows.suspend", "weft.workflows.tags.add", "weft.workflows.tags.remove", "weft.workflows.timeline.get", "weft.workflows.timeout", "weft.workflows.update"];
|
|
4
|
+
export declare const CLIENT_OPERATION_NAMES: readonly ["weft.activities.complete", "weft.activities.fail", "weft.alerts.list", "weft.recover.all", "weft.retention.get", "weft.reviews.decision.submit", "weft.reviews.get", "weft.reviews.list", "weft.schedules.cancel", "weft.schedules.create", "weft.schedules.get", "weft.schedules.list", "weft.schedules.pause", "weft.schedules.resume", "weft.schedules.update", "weft.storage.capabilities", "weft.system.lease", "weft.system.metrics", "weft.system.principal", "weft.system.registry", "weft.task.queues.list", "weft.tasks.diagnostics", "weft.tasks.diagnostics.deadletters.clear", "weft.tasks.get", "weft.updates.result.get", "weft.worker.deployments.drain", "weft.worker.deployments.resume", "weft.workers.diagnostics", "weft.workers.drain", "weft.workers.list", "weft.workers.rejections", "weft.workers.resume", "weft.workflows.activities.pending.list", "weft.workflows.aggregate", "weft.workflows.attributes.get", "weft.workflows.attributes.set", "weft.workflows.bulk.cancel", "weft.workflows.bulk.delete", "weft.workflows.bulk.retryfailed", "weft.workflows.bulk.signal", "weft.workflows.bulk.tags", "weft.workflows.cancel", "weft.workflows.checkpoints.get", "weft.workflows.checkpoints.list", "weft.workflows.events.list", "weft.workflows.finalizer.get", "weft.workflows.fork", "weft.workflows.get", "weft.workflows.list", "weft.workflows.purge", "weft.workflows.query", "weft.workflows.replay", "weft.workflows.result.get", "weft.workflows.resume", "weft.workflows.scheduleprovenance.get", "weft.workflows.signal", "weft.workflows.start", "weft.workflows.startorsignal", "weft.workflows.streams.chunks", "weft.workflows.suspend", "weft.workflows.tags.add", "weft.workflows.tags.remove", "weft.workflows.timeline.get", "weft.workflows.timeout", "weft.workflows.update"];
|
|
5
5
|
export type ClientOperationName = (typeof CLIENT_OPERATION_NAMES)[number];
|
|
6
6
|
export declare const CLIENT_REST_OPERATION_BINDINGS: {
|
|
7
7
|
readonly 'weft.tasks.diagnostics.deadletters.clear': {
|
|
@@ -304,7 +304,7 @@ export type ClientOperationTypes = {
|
|
|
304
304
|
readonly executionLatencyMs?: number;
|
|
305
305
|
readonly heartbeatAgeMs?: number;
|
|
306
306
|
readonly kind: 'stuck-queued' | 'stale-inflight' | 'retry-storm' | 'all-workers-at-capacity' | 'dead-lettered';
|
|
307
|
-
readonly lastRequeueReason?:
|
|
307
|
+
readonly lastRequeueReason?: string;
|
|
308
308
|
readonly operationId?: string;
|
|
309
309
|
readonly queue?: string;
|
|
310
310
|
readonly queueLatencyMs?: number;
|
|
@@ -335,7 +335,14 @@ export type ClientOperationTypes = {
|
|
|
335
335
|
readonly output: {
|
|
336
336
|
readonly ok: boolean;
|
|
337
337
|
};
|
|
338
|
-
readonly faults:
|
|
338
|
+
readonly faults: 'NotFound';
|
|
339
|
+
};
|
|
340
|
+
'weft.tasks.get': {
|
|
341
|
+
readonly input: {
|
|
342
|
+
readonly operationId: string;
|
|
343
|
+
};
|
|
344
|
+
readonly output: unknown;
|
|
345
|
+
readonly faults: 'NotFound';
|
|
339
346
|
};
|
|
340
347
|
'weft.updates.result.get': {
|
|
341
348
|
readonly input: {
|
|
@@ -359,6 +366,15 @@ export type ClientOperationTypes = {
|
|
|
359
366
|
readonly output: unknown;
|
|
360
367
|
readonly faults: never;
|
|
361
368
|
};
|
|
369
|
+
'weft.workers.diagnostics': {
|
|
370
|
+
readonly input: {
|
|
371
|
+
readonly workerId: string;
|
|
372
|
+
};
|
|
373
|
+
readonly output: {
|
|
374
|
+
readonly worker: unknown;
|
|
375
|
+
};
|
|
376
|
+
readonly faults: never;
|
|
377
|
+
};
|
|
362
378
|
'weft.workers.drain': {
|
|
363
379
|
readonly input: {
|
|
364
380
|
readonly reason?: string;
|
|
@@ -376,7 +392,6 @@ export type ClientOperationTypes = {
|
|
|
376
392
|
readonly deploymentName: unknown;
|
|
377
393
|
readonly drainedWorkers: number;
|
|
378
394
|
readonly drainingWorkers: number;
|
|
379
|
-
readonly gitSha: unknown;
|
|
380
395
|
readonly health: 'active' | 'draining' | 'drained';
|
|
381
396
|
readonly inFlight: number;
|
|
382
397
|
readonly oldestStartedAt: unknown;
|
|
@@ -391,7 +406,6 @@ export type ClientOperationTypes = {
|
|
|
391
406
|
readonly concurrency: number;
|
|
392
407
|
readonly connectedAt: number;
|
|
393
408
|
readonly deploymentName?: string;
|
|
394
|
-
readonly gitSha?: string;
|
|
395
409
|
readonly health: 'active' | 'draining' | 'drained';
|
|
396
410
|
readonly heartbeatAgeMs: number;
|
|
397
411
|
readonly id: string;
|
|
@@ -405,6 +419,23 @@ export type ClientOperationTypes = {
|
|
|
405
419
|
};
|
|
406
420
|
readonly faults: never;
|
|
407
421
|
};
|
|
422
|
+
'weft.workers.rejections': {
|
|
423
|
+
readonly input: {
|
|
424
|
+
readonly limit: number;
|
|
425
|
+
};
|
|
426
|
+
readonly output: {
|
|
427
|
+
readonly items: ReadonlyArray<{
|
|
428
|
+
readonly buildId?: string;
|
|
429
|
+
readonly code: 'invalid_registration' | 'unsupported_protocol_version' | 'deployment_conflict' | 'registration_rejected';
|
|
430
|
+
readonly deploymentName?: string;
|
|
431
|
+
readonly queue?: string;
|
|
432
|
+
readonly rejectedAt: number;
|
|
433
|
+
readonly workerId?: string;
|
|
434
|
+
}>;
|
|
435
|
+
readonly limit: number;
|
|
436
|
+
};
|
|
437
|
+
readonly faults: never;
|
|
438
|
+
};
|
|
408
439
|
'weft.workers.resume': {
|
|
409
440
|
readonly input: {
|
|
410
441
|
readonly workerId: string;
|
|
@@ -25,11 +25,14 @@ export const CATALOG_OPERATION_NAMES = [
|
|
|
25
25
|
"weft.system.registry",
|
|
26
26
|
"weft.task.queues.list",
|
|
27
27
|
"weft.tasks.diagnostics",
|
|
28
|
+
"weft.tasks.get",
|
|
28
29
|
"weft.updates.result.get",
|
|
29
30
|
"weft.worker.deployments.drain",
|
|
30
31
|
"weft.worker.deployments.resume",
|
|
32
|
+
"weft.workers.diagnostics",
|
|
31
33
|
"weft.workers.drain",
|
|
32
34
|
"weft.workers.list",
|
|
35
|
+
"weft.workers.rejections",
|
|
33
36
|
"weft.workers.resume",
|
|
34
37
|
"weft.workflows.activities.pending.list",
|
|
35
38
|
"weft.workflows.aggregate",
|
|
@@ -88,11 +91,14 @@ export const CATALOG_OPERATION_NAMES = [
|
|
|
88
91
|
"weft.task.queues.list",
|
|
89
92
|
"weft.tasks.diagnostics",
|
|
90
93
|
"weft.tasks.diagnostics.deadletters.clear",
|
|
94
|
+
"weft.tasks.get",
|
|
91
95
|
"weft.updates.result.get",
|
|
92
96
|
"weft.worker.deployments.drain",
|
|
93
97
|
"weft.worker.deployments.resume",
|
|
98
|
+
"weft.workers.diagnostics",
|
|
94
99
|
"weft.workers.drain",
|
|
95
100
|
"weft.workers.list",
|
|
101
|
+
"weft.workers.rejections",
|
|
96
102
|
"weft.workers.resume",
|
|
97
103
|
"weft.workflows.activities.pending.list",
|
|
98
104
|
"weft.workflows.aggregate",
|
|
@@ -9,7 +9,7 @@ export declare function normalizePipeStage(internals: ContextInternals, stage: W
|
|
|
9
9
|
export declare function resolveChildWorkflowTarget<TInput = unknown, TOutput = unknown>(internals: ContextInternals, target: ChildWorkflowTarget<TInput, TOutput>): string;
|
|
10
10
|
export declare function resolveMapConcurrency(totalItems: number, requestedConcurrency: number | undefined): number;
|
|
11
11
|
export declare function createCompositionToken(internals: ContextInternals, kind: 'map' | 'pipe' | 'reduce'): string;
|
|
12
|
-
export declare function createCompositionChildWorkflowOptions(internals: ContextInternals, token: string, index: number, options?: AwaitChildWorkflowOptions
|
|
12
|
+
export declare function createCompositionChildWorkflowOptions(internals: ContextInternals, token: string, index: number, options?: AwaitChildWorkflowOptions): AwaitChildWorkflowOptions;
|
|
13
13
|
export declare function createReduceChildWorkflowOptions(internals: ContextInternals, token: string, index: number, options: WorkflowReduceOptions | undefined): AwaitChildWorkflowOptions;
|
|
14
14
|
export declare function primeParallelOperations(operations: Generator<ContextOperationRequest, unknown, unknown>[]): ContextOperationRequest[];
|
|
15
15
|
export declare function startChild<TResult = unknown>(context: Context, internals: ContextInternals, workflowType: string, input: unknown, options?: ChildWorkflowOptions): Generator<ContextOperationRequest, TResult | ChildWorkflowHandle<TResult>, unknown>;
|
|
@@ -27,7 +27,7 @@ export function resolveMapConcurrency(totalItems, requestedConcurrency) {
|
|
|
27
27
|
export function createCompositionToken(internals, kind) {
|
|
28
28
|
return `${kind}:${internals.stepIndex}`;
|
|
29
29
|
}
|
|
30
|
-
export function createCompositionChildWorkflowOptions(internals, token, index, options
|
|
30
|
+
export function createCompositionChildWorkflowOptions(internals, token, index, options) {
|
|
31
31
|
assertCompositionAwaitsChildWorkflow(options);
|
|
32
32
|
if (options?.id !== void 0)
|
|
33
33
|
return options;
|
|
@@ -51,6 +51,8 @@ export declare function createExecutionStrategyBundle(parameters: {
|
|
|
51
51
|
development: boolean;
|
|
52
52
|
broadcastEvents: boolean;
|
|
53
53
|
getRegistration: (workflowType: string) => RegistrationEntry | undefined;
|
|
54
|
+
/** Live accessor for registered workflow type names (WFT-28); registrations are still empty here. */
|
|
55
|
+
listRegisteredWorkflowTypes: () => Iterable<string>;
|
|
54
56
|
getComposedWorkflowInterceptor?: () => ComposedWorkflowInterceptor | null;
|
|
55
57
|
resolveWorkflowType: (target: string | Function) => string;
|
|
56
58
|
registerCancelHandler?: (workflowId: string, handler: () => Promise<void> | void) => () => void;
|
|
@@ -187,6 +187,7 @@ export function createExecutionStrategyBundle(parameters) {
|
|
|
187
187
|
development,
|
|
188
188
|
broadcastEvents,
|
|
189
189
|
getRegistration,
|
|
190
|
+
listRegisteredWorkflowTypes,
|
|
190
191
|
getComposedWorkflowInterceptor,
|
|
191
192
|
resolveWorkflowType,
|
|
192
193
|
registerCancelHandler,
|
|
@@ -203,6 +204,8 @@ export function createExecutionStrategyBundle(parameters) {
|
|
|
203
204
|
broadcastEvents,
|
|
204
205
|
requireProtocolVersion: workerExecutionConfiguration.requireProtocolVersion,
|
|
205
206
|
discardOnCancel: workerExecutionConfiguration.discardOnCancel,
|
|
207
|
+
requireRealmReady: !0,
|
|
208
|
+
getExpectedWorkflowTypes: () => [...listRegisteredWorkflowTypes()],
|
|
206
209
|
workflowTurnTimeoutMs: workerExecutionConfiguration.workflowTurnTimeoutMs,
|
|
207
210
|
maxProtocolMessageBytes: workerExecutionConfiguration.maxProtocolMessageBytes,
|
|
208
211
|
getNow,
|
|
@@ -280,6 +280,7 @@ export class Engine extends EventTarget {
|
|
|
280
280
|
development: resolvedOptions.development,
|
|
281
281
|
broadcastEvents: resolvedOptions.broadcastEvents,
|
|
282
282
|
getRegistration: getInternals(this).registrations.get.bind(getInternals(this).registrations),
|
|
283
|
+
listRegisteredWorkflowTypes: () => getInternals(this).registrations.keys(),
|
|
283
284
|
getComposedWorkflowInterceptor: () => getComposedWorkflowInterceptor(getInternals(this)),
|
|
284
285
|
resolveWorkflowType: this.#resolveWorkflowTypeTarget.bind(this),
|
|
285
286
|
registerCancelHandler: (workflowId, handler) => registerCancelHandler(getInternals(this), workflowId, handler),
|
|
@@ -3,6 +3,7 @@ import type { EngineInternals } from './internals.ts';
|
|
|
3
3
|
export declare const BULK_OPERATION_BATCH_SIZE = 1000;
|
|
4
4
|
/** List workflow summaries that match a filter, using indexes when available. */
|
|
5
5
|
export declare function list(internals: EngineInternals, filter?: ListFilter, options?: ListOptions): Promise<PaginatedResult<WorkflowSummary>>;
|
|
6
|
+
export declare function assertWorkflowListScanWithinCap(scannedRows: number): void;
|
|
6
7
|
/** Stream decoded workflow states that match a list filter. */
|
|
7
8
|
export declare function streamWorkflowStates(internals: EngineInternals, filter?: ListFilter): AsyncGenerator<WorkflowState>;
|
|
8
9
|
/** Stream decoded workflow states in fixed-size batches for bulk operations. */
|
|
@@ -25,8 +25,7 @@ export async function list(internals, filter, options) {
|
|
|
25
25
|
}
|
|
26
26
|
async function collectSummariesFromConstrainedIds(internals, constrainedIds, normalizedFilter, normalizedTagFilters, options) {
|
|
27
27
|
const orderedIds = [...constrainedIds];
|
|
28
|
-
|
|
29
|
-
throw new WorkflowListScanCapExceededError(MAX_LIST_SCAN_ROWS);
|
|
28
|
+
assertWorkflowListScanWithinCap(orderedIds.length);
|
|
30
29
|
const items = [];
|
|
31
30
|
for (let start = 0;start < orderedIds.length; start += CONSTRAINED_ID_CHUNK_SIZE) {
|
|
32
31
|
const chunkIds = orderedIds.slice(start, start + CONSTRAINED_ID_CHUNK_SIZE), chunkBytes = await Promise.all(chunkIds.map((workflowId) => internals.storage.get(KEYS.workflow(workflowId)))), searchAttributesByWorkflowId = await readSearchAttributesForStates(internals, chunkBytes, normalizedFilter), matchingStates = [];
|
|
@@ -49,8 +48,7 @@ async function collectSummariesFromFullScan(internals, normalizedFilter, normali
|
|
|
49
48
|
if (!isTopLevelWorkflowStateKey(key))
|
|
50
49
|
continue;
|
|
51
50
|
scanned += 1;
|
|
52
|
-
|
|
53
|
-
throw new WorkflowListScanCapExceededError(MAX_LIST_SCAN_ROWS);
|
|
51
|
+
assertWorkflowListScanWithinCap(scanned);
|
|
54
52
|
const state = decodeWorkflowState(value), searchAttributes = await readSearchAttributesForFilter(internals, state.id, normalizedFilter);
|
|
55
53
|
if (!matchesListFilter(state, normalizedFilter, null, normalizedTagFilters, searchAttributes))
|
|
56
54
|
continue;
|
|
@@ -59,6 +57,10 @@ async function collectSummariesFromFullScan(internals, normalizedFilter, normali
|
|
|
59
57
|
}
|
|
60
58
|
return items;
|
|
61
59
|
}
|
|
60
|
+
export function assertWorkflowListScanWithinCap(scannedRows) {
|
|
61
|
+
if (scannedRows > MAX_LIST_SCAN_ROWS)
|
|
62
|
+
throw new WorkflowListScanCapExceededError(MAX_LIST_SCAN_ROWS);
|
|
63
|
+
}
|
|
62
64
|
async function summariesFromStates(internals, states, options) {
|
|
63
65
|
const attributeBytesByWorkflowId = new Map;
|
|
64
66
|
await Promise.all(states.map(async (state) => {
|
|
@@ -25,7 +25,7 @@
|
|
|
25
25
|
* }
|
|
26
26
|
* ```
|
|
27
27
|
*/
|
|
28
|
-
export type WeftErrorCode = 'WorkflowAlreadyExistsError' | 'BulkDeleteRequiresTerminalWorkflowsError' | 'BulkOperationConfirmationError' | 'WorkflowTypeNotRegisteredForRecoveryError' | 'EngineCreateNameMismatchError' | 'EngineDisposedError' | 'EngineDisposalError' | 'WorkflowNotFoundError' | 'WorkflowNotRegisteredError' | 'WorkflowConcurrencyLimitExceededError' | 'WorkflowSuspendNotSupportedError' | 'ActivityResolutionError' | 'BranchTopologyChangedError' | 'PersistedDataIncompatibleError' | 'WorkflowTimeoutError' | 'HttpClientError' | 'WorkerProtocolIncompatibleError' | 'UpdateTimeoutError' | 'UpdateValidationError' | 'WorkflowTerminalError' | 'WorkflowBuilderError' | 'VersionMismatchError' | 'EffectReplayConflictError' | 'ReviewTimeoutError' | 'AtomicStateConflictError' | 'StandardSchemaValidationError' | 'ActivityReconciliationCapabilityError' | 'ActivityReconciliationConflictError' | 'ActivityReconciliationIndeterminateError' | 'DurableActivityScopeError' | 'DurableActivityUnsupportedError' | 'AsyncActivityTokenNotFoundError' | 'ActivityScheduleToCloseTimeoutError' | 'ActivityPerAttemptTimeoutError' | 'PayloadSizeExceededError' | 'StartOrSignalConflictError' | 'WorkflowTeardownPendingError' | 'IdempotencyKeyPurgedError';
|
|
28
|
+
export type WeftErrorCode = 'WorkflowAlreadyExistsError' | 'BulkDeleteRequiresTerminalWorkflowsError' | 'BulkOperationConfirmationError' | 'WorkflowTypeNotRegisteredForRecoveryError' | 'EngineCreateNameMismatchError' | 'EngineDisposedError' | 'EngineDisposalError' | 'WorkflowNotFoundError' | 'WorkflowNotRegisteredError' | 'WorkflowConcurrencyLimitExceededError' | 'WorkflowSuspendNotSupportedError' | 'ActivityResolutionError' | 'BranchTopologyChangedError' | 'PersistedDataIncompatibleError' | 'WorkflowTimeoutError' | 'HttpClientError' | 'WorkerProtocolIncompatibleError' | 'UpdateTimeoutError' | 'UpdateValidationError' | 'WorkflowTerminalError' | 'WorkflowBuilderError' | 'VersionMismatchError' | 'EffectReplayConflictError' | 'ReviewTimeoutError' | 'AtomicStateConflictError' | 'StandardSchemaValidationError' | 'ActivityReconciliationCapabilityError' | 'ActivityReconciliationConflictError' | 'ActivityReconciliationIndeterminateError' | 'DurableActivityScopeError' | 'DurableActivityUnsupportedError' | 'AsyncActivityTokenNotFoundError' | 'ActivityScheduleToCloseTimeoutError' | 'ActivityPerAttemptTimeoutError' | 'PayloadSizeExceededError' | 'StartOrSignalConflictError' | 'WorkflowTeardownPendingError' | 'IdempotencyKeyPurgedError' | 'WorkerManifestBuildError';
|
|
29
29
|
/**
|
|
30
30
|
* Generic abstract base for all Weft library errors. The `TCode` parameter
|
|
31
31
|
* makes each subclass's `code` its own literal type; the exported base surface
|
package/dist/core/weft-error.js
CHANGED
|
@@ -44,7 +44,8 @@ const publicWeftErrorCodeMap = {
|
|
|
44
44
|
PayloadSizeExceededError: !0,
|
|
45
45
|
StartOrSignalConflictError: !0,
|
|
46
46
|
WorkflowTeardownPendingError: !0,
|
|
47
|
-
IdempotencyKeyPurgedError: !0
|
|
47
|
+
IdempotencyKeyPurgedError: !0,
|
|
48
|
+
WorkerManifestBuildError: !0
|
|
48
49
|
}, PUBLIC_WEFT_ERROR_CODES = new Set(Object.keys(publicWeftErrorCodeMap));
|
|
49
50
|
export function isWeftError(value) {
|
|
50
51
|
return value instanceof WeftError;
|
|
@@ -9,6 +9,14 @@ export interface WorkerExecutionDispatcherDependencies {
|
|
|
9
9
|
validateHostToWorkerMessage: (workflowId: string, message: WorkerInboundMessage, worker?: Worker) => boolean;
|
|
10
10
|
attachWorkerListeners: (worker: Worker) => void;
|
|
11
11
|
detachWorkerListenersIfIdle: (worker: Worker) => void;
|
|
12
|
+
/**
|
|
13
|
+
* Resolve once `worker`'s realm-ready handshake has settled (or
|
|
14
|
+
* immediately, if realm-readiness isn't required). `false` means the
|
|
15
|
+
* handshake failed or timed out; the implementation is responsible for
|
|
16
|
+
* discarding the worker and emitting the workflow failure itself, the same
|
|
17
|
+
* contract `validateHostToWorkerMessage` already uses.
|
|
18
|
+
*/
|
|
19
|
+
ensureRealmReady: (worker: Worker, workflowId: string) => Promise<boolean>;
|
|
12
20
|
beginTurn: (worker: Worker, workflowId: string, turnId: number, kind: 'run' | 'resume') => void;
|
|
13
21
|
clearTurn: (worker: Worker) => void;
|
|
14
22
|
discardWorkerAndFailWorkflows: (worker: Worker, options: {
|
|
@@ -11,6 +11,10 @@ export class WorkerExecutionDispatcher {
|
|
|
11
11
|
const worker = await this.#dependencies.pool.acquire();
|
|
12
12
|
this.#dependencies.ownership.setActive(workflowId, worker);
|
|
13
13
|
this.#dependencies.attachWorkerListeners(worker);
|
|
14
|
+
if (!await this.#dependencies.ensureRealmReady(worker, workflowId))
|
|
15
|
+
return;
|
|
16
|
+
if (this.#dependencies.isDisposed() || this.#dependencies.ownership.getActiveWorker(workflowId) !== worker)
|
|
17
|
+
return;
|
|
14
18
|
this.#sendActiveMessage(worker, workflowId, message, "run");
|
|
15
19
|
} catch (error) {
|
|
16
20
|
this.#dependencies.emit({
|
|
@@ -5,6 +5,22 @@ export interface WorkerExecutionStrategyOptions {
|
|
|
5
5
|
maxProtocolMessageBytes?: number;
|
|
6
6
|
requireProtocolVersion?: boolean;
|
|
7
7
|
discardOnCancel?: boolean;
|
|
8
|
+
/**
|
|
9
|
+
* Require every freshly acquired worker to complete the realm-ready
|
|
10
|
+
* handshake (WFT-28) — send a valid `ready` message whose manifest
|
|
11
|
+
* contains a matching contract for every type in
|
|
12
|
+
* {@link getExpectedWorkflowTypes} (a subset check, not exact-manifest
|
|
13
|
+
* equality) — before it receives its first `run` turn. Defaults to
|
|
14
|
+
* `false` so direct/test construction of this class is unaffected;
|
|
15
|
+
* `createExecutionStrategyBundle` hardcodes this to `true` for every
|
|
16
|
+
* engine-constructed worker-mode strategy, mirroring how
|
|
17
|
+
* `requireProtocolVersion` is already hardcoded there.
|
|
18
|
+
*/
|
|
19
|
+
requireRealmReady?: boolean;
|
|
20
|
+
/** Required when {@link requireRealmReady} is `true`. Called fresh on every handshake. */
|
|
21
|
+
getExpectedWorkflowTypes?: () => readonly string[];
|
|
22
|
+
/** Bound (ms) on the realm-ready wait. Defaults to `DEFAULT_WORKER_REALM_READY_TIMEOUT_MS`. */
|
|
23
|
+
realmReadyTimeoutMs?: number;
|
|
8
24
|
/**
|
|
9
25
|
* The engine host's `EngineOptions.onLog` sink (#529). When present, the strategy
|
|
10
26
|
* tells each worker (`hostHasLogSink: true` on `run`/`resume`) to forward `ctx.log`
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import { WorkerCheckpointResumeState } from "./worker-checkpoint-resume-state.js";
|
|
2
2
|
import { WorkerExecutionDispatcher } from "./worker-execution-dispatcher.js";
|
|
3
3
|
import { WorkerExecutionOwnership } from "./worker-execution-ownership.js";
|
|
4
|
+
import { WorkerFaultHandler } from "./worker-fault-handling.js";
|
|
4
5
|
import {
|
|
5
6
|
buildResumeMessage,
|
|
6
7
|
buildRunMessage
|
|
@@ -15,7 +16,11 @@ import {
|
|
|
15
16
|
} from "./worker-message-helpers.js";
|
|
16
17
|
import { WorkerProtocolGuard } from "./worker-protocol-guard.js";
|
|
17
18
|
import { isWorkerLogMessage } from "./worker-protocol-log.js";
|
|
18
|
-
import {
|
|
19
|
+
import {
|
|
20
|
+
DEFAULT_WORKER_REALM_READY_TIMEOUT_MS,
|
|
21
|
+
WORKER_PROTOCOL_VERSION
|
|
22
|
+
} from "./worker-protocol.js";
|
|
23
|
+
import { isWorkerRealmReadyMessage, WorkerRealmReadiness } from "./worker-realm-readiness.js";
|
|
19
24
|
import {
|
|
20
25
|
WorkerTurnWatchdog
|
|
21
26
|
} from "./worker-turn-watchdog.js";
|
|
@@ -34,6 +39,8 @@ export class WorkerExecutionStrategy {
|
|
|
34
39
|
#forwardedLogGate;
|
|
35
40
|
#turnWatchdog;
|
|
36
41
|
#protocolGuard;
|
|
42
|
+
#realmReadiness;
|
|
43
|
+
#faultHandler;
|
|
37
44
|
#dispatcher;
|
|
38
45
|
#messageHandler;
|
|
39
46
|
#disposed;
|
|
@@ -44,7 +51,10 @@ export class WorkerExecutionStrategy {
|
|
|
44
51
|
maxProtocolMessageBytes,
|
|
45
52
|
requireProtocolVersion = !1,
|
|
46
53
|
discardOnCancel = !1,
|
|
47
|
-
broadcastEvents = !1
|
|
54
|
+
broadcastEvents = !1,
|
|
55
|
+
requireRealmReady = !1,
|
|
56
|
+
getExpectedWorkflowTypes,
|
|
57
|
+
realmReadyTimeoutMs = DEFAULT_WORKER_REALM_READY_TIMEOUT_MS
|
|
48
58
|
} = options ?? {};
|
|
49
59
|
this.#pool = pool;
|
|
50
60
|
this.#ownership = new WorkerExecutionOwnership;
|
|
@@ -56,21 +66,36 @@ export class WorkerExecutionStrategy {
|
|
|
56
66
|
this.#discardOnCancel = discardOnCancel;
|
|
57
67
|
this.#forwardedLogGate = forwardedLogGateFromStrategyOptions(options, maxProtocolMessageBytes);
|
|
58
68
|
this.#turnWatchdog = new WorkerTurnWatchdog(this.#workflowTurnTimeoutMs, (turn) => {
|
|
59
|
-
this.#handleTurnTimeout(turn);
|
|
69
|
+
this.#faultHandler.handleTurnTimeout(turn);
|
|
60
70
|
});
|
|
61
71
|
this.#protocolGuard = new WorkerProtocolGuard(this.#maxProtocolMessageBytes, this.#requireProtocolVersion, this.#turnWatchdog);
|
|
72
|
+
this.#realmReadiness = WorkerExecutionStrategy.#buildRealmReadiness(requireRealmReady, getExpectedWorkflowTypes, realmReadyTimeoutMs, this.#maxProtocolMessageBytes);
|
|
73
|
+
this.#faultHandler = new WorkerFaultHandler({
|
|
74
|
+
ownership: this.#ownership,
|
|
75
|
+
checkpointResumeState: this.#checkpointResumeState,
|
|
76
|
+
turnWatchdog: this.#turnWatchdog,
|
|
77
|
+
forwardedLogGate: this.#forwardedLogGate,
|
|
78
|
+
realmReadiness: this.#realmReadiness,
|
|
79
|
+
workerListeners: this.#workerListeners,
|
|
80
|
+
protocolGuard: this.#protocolGuard,
|
|
81
|
+
pool: this.#pool,
|
|
82
|
+
emit: (message) => {
|
|
83
|
+
this.#emit(message);
|
|
84
|
+
}
|
|
85
|
+
});
|
|
62
86
|
this.#dispatcher = new WorkerExecutionDispatcher({
|
|
63
87
|
pool: this.#pool,
|
|
64
88
|
ownership: this.#ownership,
|
|
65
89
|
isDisposed: () => this.#disposed,
|
|
66
90
|
requireProtocolVersion: () => this.#requireProtocolVersion,
|
|
67
|
-
validateHostToWorkerMessage: (workflowId, message, worker) => this.#assertHostToWorkerMessageWithinLimit(workflowId, message, worker),
|
|
91
|
+
validateHostToWorkerMessage: (workflowId, message, worker) => this.#faultHandler.assertHostToWorkerMessageWithinLimit(workflowId, message, worker),
|
|
68
92
|
attachWorkerListeners: (worker) => {
|
|
69
93
|
this.#attachWorkerListeners(worker);
|
|
70
94
|
},
|
|
71
95
|
detachWorkerListenersIfIdle: (worker) => {
|
|
72
96
|
this.#detachWorkerListenersIfIdle(worker);
|
|
73
97
|
},
|
|
98
|
+
ensureRealmReady: (worker, workflowId) => this.#ensureRealmReady(worker, workflowId),
|
|
74
99
|
beginTurn: (worker, workflowId, turnId, kind) => {
|
|
75
100
|
this.#turnWatchdog.begin(worker, workflowId, turnId, kind);
|
|
76
101
|
},
|
|
@@ -78,7 +103,7 @@ export class WorkerExecutionStrategy {
|
|
|
78
103
|
this.#turnWatchdog.clear(worker);
|
|
79
104
|
},
|
|
80
105
|
discardWorkerAndFailWorkflows: (worker, discardOptions) => {
|
|
81
|
-
this.#discardWorkerAndFailWorkflows(worker, discardOptions);
|
|
106
|
+
this.#faultHandler.discardWorkerAndFailWorkflows(worker, discardOptions);
|
|
82
107
|
},
|
|
83
108
|
emit: (message) => {
|
|
84
109
|
this.#emit(message);
|
|
@@ -108,7 +133,7 @@ export class WorkerExecutionStrategy {
|
|
|
108
133
|
this.#ownership.resetWorkflow(parameters.workflowId);
|
|
109
134
|
this.#checkpointResumeState.resetWorkflow(parameters.workflowId);
|
|
110
135
|
const message = buildRunMessage(parameters, this.#inboundMessageContext());
|
|
111
|
-
if (!this.#assertHostToWorkerMessageWithinLimit(parameters.workflowId, message))
|
|
136
|
+
if (!this.#faultHandler.assertHostToWorkerMessageWithinLimit(parameters.workflowId, message))
|
|
112
137
|
return;
|
|
113
138
|
this.#dispatcher.acquireAndSend(parameters.workflowId, message);
|
|
114
139
|
}
|
|
@@ -135,6 +160,23 @@ export class WorkerExecutionStrategy {
|
|
|
135
160
|
error: `No worker assigned for workflow: ${parameters.workflowId}`
|
|
136
161
|
});
|
|
137
162
|
}
|
|
163
|
+
async#ensureRealmReady(worker, workflowId) {
|
|
164
|
+
if (!this.#realmReadiness)
|
|
165
|
+
return !0;
|
|
166
|
+
if (this.#realmReadiness.isReady(worker))
|
|
167
|
+
return !0;
|
|
168
|
+
const outcome = await this.#realmReadiness.waitForReady(worker);
|
|
169
|
+
if (outcome.ok)
|
|
170
|
+
return !0;
|
|
171
|
+
this.#faultHandler.discardWorkerAndFailWorkflows(worker, {
|
|
172
|
+
targetWorkflowId: workflowId,
|
|
173
|
+
targetCategory: outcome.failureCategory,
|
|
174
|
+
targetError: outcome.error,
|
|
175
|
+
otherCategory: "system",
|
|
176
|
+
otherError: `Worker discarded after realm-ready handshake failed: ${outcome.error}`
|
|
177
|
+
});
|
|
178
|
+
return !1;
|
|
179
|
+
}
|
|
138
180
|
#inboundMessageContext() {
|
|
139
181
|
return {
|
|
140
182
|
turnId: this.#nextTurnId++,
|
|
@@ -147,7 +189,7 @@ export class WorkerExecutionStrategy {
|
|
|
147
189
|
if (worker) {
|
|
148
190
|
this.#ownership.markCancelled(workflowId);
|
|
149
191
|
if (this.#discardOnCancel) {
|
|
150
|
-
this.#discardWorkerAndFailWorkflows(worker, {
|
|
192
|
+
this.#faultHandler.discardWorkerAndFailWorkflows(worker, {
|
|
151
193
|
targetWorkflowId: workflowId,
|
|
152
194
|
skipTarget: !0,
|
|
153
195
|
otherCategory: "system",
|
|
@@ -171,7 +213,7 @@ export class WorkerExecutionStrategy {
|
|
|
171
213
|
this.#ownership.markCancelled(workflowId);
|
|
172
214
|
this.#ownership.deleteParked(workflowId);
|
|
173
215
|
if (this.#discardOnCancel) {
|
|
174
|
-
this.#discardWorkerAndFailWorkflows(parkedWorker, {
|
|
216
|
+
this.#faultHandler.discardWorkerAndFailWorkflows(parkedWorker, {
|
|
175
217
|
targetWorkflowId: workflowId,
|
|
176
218
|
skipTarget: !0,
|
|
177
219
|
otherCategory: "system",
|
|
@@ -197,6 +239,7 @@ export class WorkerExecutionStrategy {
|
|
|
197
239
|
this.#broadcastChannel.close();
|
|
198
240
|
}
|
|
199
241
|
this.#turnWatchdog.clearAll();
|
|
242
|
+
this.#realmReadiness?.clear();
|
|
200
243
|
const activeWorkflowIds = this.#ownership.activeWorkflowIds();
|
|
201
244
|
for (const workflowId of activeWorkflowIds)
|
|
202
245
|
this.#releaseActiveWorker(workflowId);
|
|
@@ -206,13 +249,18 @@ export class WorkerExecutionStrategy {
|
|
|
206
249
|
this.#messageHandler = null;
|
|
207
250
|
}
|
|
208
251
|
async#handleWorkerMessage(worker, message) {
|
|
252
|
+
if (isWorkerRealmReadyMessage(message)) {
|
|
253
|
+
if (this.#realmReadiness)
|
|
254
|
+
await this.#realmReadiness.noteReadyMessage(worker, message);
|
|
255
|
+
return;
|
|
256
|
+
}
|
|
209
257
|
if (isWorkerLogMessage(message)) {
|
|
210
258
|
const owns = (id) => this.#ownership.getTargetWorker(id) === worker, abuseDiscard = this.#forwardedLogGate.handle(worker, message, owns);
|
|
211
259
|
if (abuseDiscard)
|
|
212
|
-
this.#discardWorkerAndFailWorkflows(worker, abuseDiscard);
|
|
260
|
+
this.#faultHandler.discardWorkerAndFailWorkflows(worker, abuseDiscard);
|
|
213
261
|
return;
|
|
214
262
|
}
|
|
215
|
-
if (!this.#acceptWorkerMessage(worker, message))
|
|
263
|
+
if (!this.#faultHandler.acceptWorkerMessage(worker, message))
|
|
216
264
|
return;
|
|
217
265
|
this.#turnWatchdog.clear(worker);
|
|
218
266
|
const resumeVersionBeforeCheckpointHandling = this.#checkpointResumeState.beginCheckpointHandling(message), emitResult = emitWorkerMessageToEngine(this.#messageHandler, message), handlerFailed = emitResult instanceof Promise ? await emitResult : emitResult;
|
|
@@ -244,14 +292,6 @@ export class WorkerExecutionStrategy {
|
|
|
244
292
|
if (this.#ownership.parkActive(message.workflowId, worker))
|
|
245
293
|
this.#pool.release(worker);
|
|
246
294
|
}
|
|
247
|
-
#handleWorkerError(worker, errorEvent) {
|
|
248
|
-
this.#discardWorkerAndFailWorkflows(worker, {
|
|
249
|
-
targetCategory: "system",
|
|
250
|
-
targetError: `Worker crashed: ${errorEvent.message ?? "unknown error"}`,
|
|
251
|
-
otherCategory: "system",
|
|
252
|
-
otherError: `Worker crashed: ${errorEvent.message ?? "unknown error"}`
|
|
253
|
-
});
|
|
254
|
-
}
|
|
255
295
|
#releaseActiveWorker(workflowId) {
|
|
256
296
|
const worker = this.#ownership.releaseActive(workflowId);
|
|
257
297
|
if (worker) {
|
|
@@ -266,10 +306,10 @@ export class WorkerExecutionStrategy {
|
|
|
266
306
|
this.#handleWorkerMessage(worker, message).catch(() => {});
|
|
267
307
|
},
|
|
268
308
|
error: (errorEvent) => {
|
|
269
|
-
this.#handleWorkerError(worker, errorEvent);
|
|
309
|
+
this.#faultHandler.handleWorkerError(worker, errorEvent);
|
|
270
310
|
},
|
|
271
311
|
messageerror: () => {
|
|
272
|
-
this.#discardWorkerAndFailWorkflows(worker, {
|
|
312
|
+
this.#faultHandler.discardWorkerAndFailWorkflows(worker, {
|
|
273
313
|
targetCategory: "system",
|
|
274
314
|
targetError: "Worker messageerror event",
|
|
275
315
|
otherCategory: "system",
|
|
@@ -281,75 +321,6 @@ export class WorkerExecutionStrategy {
|
|
|
281
321
|
#detachWorkerListenersIfIdle(worker) {
|
|
282
322
|
this.#workerListeners.detachIfIdle(worker, (candidate) => this.#ownership.workerIsIdle(candidate));
|
|
283
323
|
}
|
|
284
|
-
#assertHostToWorkerMessageWithinLimit(workflowId, message, worker) {
|
|
285
|
-
const failure = this.#protocolGuard.validateHostToWorkerMessage(message);
|
|
286
|
-
if (!failure)
|
|
287
|
-
return !0;
|
|
288
|
-
if (worker) {
|
|
289
|
-
this.#discardWorkerAndFailWorkflows(worker, {
|
|
290
|
-
targetWorkflowId: workflowId,
|
|
291
|
-
targetCategory: failure.failureCategory,
|
|
292
|
-
targetError: failure.error,
|
|
293
|
-
otherCategory: "system",
|
|
294
|
-
otherError: `Worker discarded after protocol send failure for workflow: ${workflowId}`
|
|
295
|
-
});
|
|
296
|
-
return !1;
|
|
297
|
-
}
|
|
298
|
-
this.#emit({
|
|
299
|
-
type: "failed",
|
|
300
|
-
workflowId,
|
|
301
|
-
error: failure.error,
|
|
302
|
-
failureCategory: failure.failureCategory
|
|
303
|
-
});
|
|
304
|
-
return !1;
|
|
305
|
-
}
|
|
306
|
-
#acceptWorkerMessage(worker, message) {
|
|
307
|
-
const result = this.#protocolGuard.acceptWorkerMessage(worker, message);
|
|
308
|
-
if (result.accepted)
|
|
309
|
-
return !0;
|
|
310
|
-
this.#discardWorkerAndFailWorkflows(worker, {
|
|
311
|
-
...result.failure.targetWorkflowId === void 0 ? {} : { targetWorkflowId: result.failure.targetWorkflowId },
|
|
312
|
-
targetCategory: result.failure.failureCategory,
|
|
313
|
-
targetError: result.failure.error,
|
|
314
|
-
otherCategory: "system",
|
|
315
|
-
otherError: result.failure.otherError
|
|
316
|
-
});
|
|
317
|
-
return !1;
|
|
318
|
-
}
|
|
319
|
-
#handleTurnTimeout(turn) {
|
|
320
|
-
this.#discardWorkerAndFailWorkflows(turn.worker, {
|
|
321
|
-
targetWorkflowId: turn.workflowId,
|
|
322
|
-
targetCategory: "timeout",
|
|
323
|
-
targetError: `Worker workflow turn timed out after ${turn.timeoutMs}ms`,
|
|
324
|
-
otherCategory: "timeout",
|
|
325
|
-
otherError: `Worker discarded after workflow turn timed out: ${turn.workflowId}`
|
|
326
|
-
});
|
|
327
|
-
}
|
|
328
|
-
#discardWorkerAndFailWorkflows(worker, options) {
|
|
329
|
-
const workflowIds = this.#ownership.workflowIdsForWorker(worker);
|
|
330
|
-
if (workflowIds.length === 0) {
|
|
331
|
-
this.#turnWatchdog.clear(worker);
|
|
332
|
-
this.#forwardedLogGate.forget(worker);
|
|
333
|
-
return;
|
|
334
|
-
}
|
|
335
|
-
for (const workflowId of workflowIds) {
|
|
336
|
-
const isTarget = workflowId === options.targetWorkflowId;
|
|
337
|
-
this.#ownership.forgetWorkflow(workflowId);
|
|
338
|
-
this.#checkpointResumeState.forgetWorkflowIfClosed(workflowId, !0);
|
|
339
|
-
if (isTarget && options.skipTarget)
|
|
340
|
-
continue;
|
|
341
|
-
this.#emit({
|
|
342
|
-
type: "failed",
|
|
343
|
-
workflowId,
|
|
344
|
-
error: isTarget ? options.targetError ?? options.otherError : options.otherError,
|
|
345
|
-
failureCategory: isTarget ? options.targetCategory ?? options.otherCategory : options.otherCategory
|
|
346
|
-
});
|
|
347
|
-
}
|
|
348
|
-
this.#turnWatchdog.clear(worker);
|
|
349
|
-
this.#forwardedLogGate.forget(worker);
|
|
350
|
-
this.#workerListeners.detach(worker);
|
|
351
|
-
this.#pool.discard(worker);
|
|
352
|
-
}
|
|
353
324
|
#handleBroadcastMessage(data) {
|
|
354
325
|
if (data.type === "signal:received" && typeof data.workflowId === "string") {
|
|
355
326
|
const targetWorker = this.#ownership.getTargetWorker(data.workflowId);
|
|
@@ -362,4 +333,15 @@ export class WorkerExecutionStrategy {
|
|
|
362
333
|
if (result instanceof Promise)
|
|
363
334
|
result.catch(() => {});
|
|
364
335
|
}
|
|
336
|
+
static #buildRealmReadiness(requireRealmReady, getExpectedWorkflowTypes, timeoutMs, maxProtocolMessageBytes) {
|
|
337
|
+
if (!requireRealmReady)
|
|
338
|
+
return null;
|
|
339
|
+
if (!getExpectedWorkflowTypes)
|
|
340
|
+
throw Error("WorkerExecutionStrategyOptions.getExpectedWorkflowTypes is required when requireRealmReady is true");
|
|
341
|
+
return new WorkerRealmReadiness({
|
|
342
|
+
getExpectedWorkflowTypes,
|
|
343
|
+
timeoutMs,
|
|
344
|
+
maxProtocolMessageBytes
|
|
345
|
+
});
|
|
346
|
+
}
|
|
365
347
|
}
|
|
@@ -0,0 +1,45 @@
|
|
|
1
|
+
import type { WorkerPool } from '../workers/pool.ts';
|
|
2
|
+
import type { FailureCategory, WorkerInboundMessage, WorkerOutboundMessage } from './types.ts';
|
|
3
|
+
import type { WorkerCheckpointResumeState } from './worker-checkpoint-resume-state.ts';
|
|
4
|
+
import type { WorkerExecutionOwnership } from './worker-execution-ownership.ts';
|
|
5
|
+
import type { WorkerListenerRegistry } from './worker-listener-registry.ts';
|
|
6
|
+
import type { ForwardedLogGate } from './worker-log-abuse-counter.ts';
|
|
7
|
+
import type { WorkerProtocolGuard } from './worker-protocol-guard.ts';
|
|
8
|
+
import type { WorkerRealmReadiness } from './worker-realm-readiness.ts';
|
|
9
|
+
import type { WorkerTurnState, WorkerTurnWatchdog } from './worker-turn-watchdog.ts';
|
|
10
|
+
export interface WorkerFaultHandlerDependencies {
|
|
11
|
+
ownership: WorkerExecutionOwnership;
|
|
12
|
+
checkpointResumeState: WorkerCheckpointResumeState;
|
|
13
|
+
turnWatchdog: WorkerTurnWatchdog;
|
|
14
|
+
forwardedLogGate: ForwardedLogGate;
|
|
15
|
+
realmReadiness: WorkerRealmReadiness | null;
|
|
16
|
+
workerListeners: WorkerListenerRegistry;
|
|
17
|
+
protocolGuard: WorkerProtocolGuard;
|
|
18
|
+
pool: WorkerPool;
|
|
19
|
+
emit: (message: WorkerOutboundMessage) => void;
|
|
20
|
+
}
|
|
21
|
+
export interface WorkerDiscardOptions {
|
|
22
|
+
targetWorkflowId?: string;
|
|
23
|
+
targetCategory?: FailureCategory;
|
|
24
|
+
targetError?: string;
|
|
25
|
+
skipTarget?: boolean;
|
|
26
|
+
otherCategory: FailureCategory;
|
|
27
|
+
otherError: string;
|
|
28
|
+
}
|
|
29
|
+
/**
|
|
30
|
+
* Owns every path that ends in a worker being discarded and its owned
|
|
31
|
+
* workflows failed: protocol violations, turn timeouts, worker crashes, and
|
|
32
|
+
* (WFT-28) realm-ready handshake failures. Extracted from
|
|
33
|
+
* {@link WorkerExecutionStrategy} to keep that file under the repository's
|
|
34
|
+
* file-size ceiling; every method here previously lived as a private method
|
|
35
|
+
* on that class and is unchanged in behavior.
|
|
36
|
+
*/
|
|
37
|
+
export declare class WorkerFaultHandler {
|
|
38
|
+
#private;
|
|
39
|
+
constructor(dependencies: WorkerFaultHandlerDependencies);
|
|
40
|
+
assertHostToWorkerMessageWithinLimit(workflowId: string, message: WorkerInboundMessage, worker?: Worker): boolean;
|
|
41
|
+
acceptWorkerMessage(worker: Worker, message: unknown): message is WorkerOutboundMessage;
|
|
42
|
+
handleTurnTimeout(turn: WorkerTurnState): void;
|
|
43
|
+
handleWorkerError(worker: Worker, errorEvent: ErrorEvent): void;
|
|
44
|
+
discardWorkerAndFailWorkflows(worker: Worker, options: WorkerDiscardOptions): void;
|
|
45
|
+
}
|