@bastani/atomic 0.9.43-alpha.1 → 0.9.44-alpha.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +45 -0
- package/dist/builtin/intercom/CHANGELOG.md +18 -0
- package/dist/builtin/intercom/broker/broker.ts +31 -7
- package/dist/builtin/intercom/broker/client.ts +65 -0
- package/dist/builtin/intercom/index.bundle.mjs +227 -95
- package/dist/builtin/intercom/live-route-refusal.ts +20 -0
- package/dist/builtin/intercom/package.json +1 -1
- package/dist/builtin/intercom/skills/intercom/SKILL.md +10 -6
- package/dist/builtin/intercom/types.ts +20 -1
- package/dist/builtin/mcp/CHANGELOG.md +7 -0
- package/dist/builtin/mcp/package.json +1 -1
- package/dist/builtin/subagents/CHANGELOG.md +9 -0
- package/dist/builtin/subagents/README.md +8 -8
- package/dist/builtin/subagents/agents/worker.md +1 -1
- package/dist/builtin/subagents/package.json +1 -1
- package/dist/builtin/subagents/skills/agent-browser/SKILL.md +1 -1
- package/dist/builtin/subagents/skills/cua-driver/EMBEDDING.md +21 -2
- package/dist/builtin/subagents/skills/cua-driver/LINUX.md +2 -2
- package/dist/builtin/subagents/skills/cua-driver/MACOS.md +342 -10
- package/dist/builtin/subagents/skills/cua-driver/README.md +4 -3
- package/dist/builtin/subagents/skills/cua-driver/RUNTIME.md +3 -1
- package/dist/builtin/subagents/skills/cua-driver/SKILL.md +7 -4
- package/dist/builtin/subagents/skills/cua-driver/WINDOWS.md +6 -5
- package/dist/builtin/subagents/skills/cua-driver/WORKFLOW.md +15 -0
- package/dist/builtin/subagents/skills/herdr/SKILL.md +22 -10
- package/dist/builtin/subagents/skills/subagent/SKILL.md +8 -5
- package/dist/builtin/subagents/src/extension/index.bundle.mjs +6 -6
- package/dist/builtin/web-access/package.json +1 -1
- package/dist/builtin/workflows/CHANGELOG.md +33 -0
- package/dist/builtin/workflows/README.md +1 -1
- package/dist/builtin/workflows/builtin/{chunk-frqxd4d1.js → chunk-k8d7ehjx.js} +22 -0
- package/dist/builtin/workflows/builtin/{chunk-3s95f7ry.js → chunk-qxpymnaf.js} +1 -1
- package/dist/builtin/workflows/builtin/{chunk-edhf30ef.js → chunk-zjecnjxx.js} +1 -1
- package/dist/builtin/workflows/builtin/goal.js +2 -2
- package/dist/builtin/workflows/builtin/index.js +3 -3
- package/dist/builtin/workflows/builtin/ralph.js +2 -2
- package/dist/builtin/workflows/package.json +1 -1
- package/dist/builtin/workflows/src/extension/index.bundle.mjs +1750 -598
- package/dist/builtin/workflows/src/extension/local-worker-entry.mjs +1786 -600
- package/dist/builtin/workflows/src/index.js +1802 -650
- package/dist/builtin/workflows/src/local-worker-entry.mjs +1786 -600
- package/dist/cli/args.js +1 -1
- package/dist/cli/args.js.map +1 -1
- package/dist/core/exec.d.ts.map +1 -1
- package/dist/core/exec.js +11 -2
- package/dist/core/exec.js.map +1 -1
- package/dist/core/package-manager-resolver.d.ts.map +1 -1
- package/dist/core/package-manager-resolver.js +25 -14
- package/dist/core/package-manager-resolver.js.map +1 -1
- package/dist/core/package-manager-types.d.ts +8 -0
- package/dist/core/package-manager-types.d.ts.map +1 -1
- package/dist/core/package-manager-types.js.map +1 -1
- package/dist/core/package-manager.d.ts +1 -1
- package/dist/core/package-manager.d.ts.map +1 -1
- package/dist/core/package-manager.js.map +1 -1
- package/dist/core/resource-loader-extension-warnings.d.ts +4 -1
- package/dist/core/resource-loader-extension-warnings.d.ts.map +1 -1
- package/dist/core/resource-loader-extension-warnings.js +15 -0
- package/dist/core/resource-loader-extension-warnings.js.map +1 -1
- package/dist/core/resource-loader-package-resources.d.ts.map +1 -1
- package/dist/core/resource-loader-package-resources.js +1 -0
- package/dist/core/resource-loader-package-resources.js.map +1 -1
- package/dist/core/resource-loader-reload.d.ts.map +1 -1
- package/dist/core/resource-loader-reload.js +5 -3
- package/dist/core/resource-loader-reload.js.map +1 -1
- package/dist/extensions/mcp/client/oauth/flow.d.ts.map +1 -1
- package/dist/extensions/mcp/client/oauth/flow.js +4 -1
- package/dist/extensions/mcp/client/oauth/flow.js.map +1 -1
- package/dist/extensions/mcp/client/oauth/types.d.ts +1 -0
- package/dist/extensions/mcp/client/oauth/types.d.ts.map +1 -1
- package/dist/extensions/mcp/client/oauth/types.js +8 -4
- package/dist/extensions/mcp/client/oauth/types.js.map +1 -1
- package/dist/extensions/mcp/client/transports/streamable-http.d.ts.map +1 -1
- package/dist/extensions/mcp/client/transports/streamable-http.js +2 -1
- package/dist/extensions/mcp/client/transports/streamable-http.js.map +1 -1
- package/dist/index.d.ts +1 -1
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js.map +1 -1
- package/dist/main.d.ts.map +1 -1
- package/dist/main.js +20 -10
- package/dist/main.js.map +1 -1
- package/dist/modes/interactive/components/model-selector.d.ts.map +1 -1
- package/dist/modes/interactive/components/model-selector.js +4 -2
- package/dist/modes/interactive/components/model-selector.js.map +1 -1
- package/dist/modes/interactive/components/scoped-models-selector.d.ts.map +1 -1
- package/dist/modes/interactive/components/scoped-models-selector.js +4 -2
- package/dist/modes/interactive/components/scoped-models-selector.js.map +1 -1
- package/dist/modes/interactive/components/tool-execution.d.ts +6 -0
- package/dist/modes/interactive/components/tool-execution.d.ts.map +1 -1
- package/dist/modes/interactive/components/tool-execution.js +11 -1
- package/dist/modes/interactive/components/tool-execution.js.map +1 -1
- package/dist/modes/interactive/model-search.d.ts +10 -0
- package/dist/modes/interactive/model-search.d.ts.map +1 -1
- package/dist/modes/interactive/model-search.js +21 -0
- package/dist/modes/interactive/model-search.js.map +1 -1
- package/dist/package-manager-cli.d.ts.map +1 -1
- package/dist/package-manager-cli.js +5 -0
- package/dist/package-manager-cli.js.map +1 -1
- package/dist/utils/tools-manager.d.ts.map +1 -1
- package/dist/utils/tools-manager.js +30 -35
- package/dist/utils/tools-manager.js.map +1 -1
- package/docs/computer-use.md +5 -5
- package/docs/custom-provider/streaming.md +9 -14
- package/docs/environment-variables.md +1 -0
- package/docs/intercom.md +5 -1
- package/docs/packages.md +2 -0
- package/docs/settings.md +2 -2
- package/docs/skills.md +2 -2
- package/docs/subagents.md +38 -9
- package/docs/workflows/api-reference.md +1 -1
- package/docs/workflows/authoring.md +1 -1
- package/docs/workflows/operations.md +8 -6
- package/docs/workflows/reliable-design.md +23 -5
- package/examples/extensions/custom-provider-anthropic/index.ts +31 -31
- package/examples/extensions/todo.ts +7 -0
- package/npm-shrinkwrap.json +32 -32
- package/package.json +3 -3
|
@@ -8030,7 +8030,13 @@ function createStoreContext(state = createStoreState()) {
|
|
|
8030
8030
|
}
|
|
8031
8031
|
return cachedGraphSnapshot;
|
|
8032
8032
|
}
|
|
8033
|
+
let batchDepth = 0;
|
|
8034
|
+
let batchedNotification = false;
|
|
8033
8035
|
function notify() {
|
|
8036
|
+
if (batchDepth > 0) {
|
|
8037
|
+
batchedNotification = true;
|
|
8038
|
+
return;
|
|
8039
|
+
}
|
|
8034
8040
|
observation.capture();
|
|
8035
8041
|
for (const fn of state.invalidationListeners) {
|
|
8036
8042
|
try {
|
|
@@ -8050,6 +8056,18 @@ function createStoreContext(state = createStoreState()) {
|
|
|
8050
8056
|
state.version++;
|
|
8051
8057
|
notify();
|
|
8052
8058
|
}
|
|
8059
|
+
function batch(update) {
|
|
8060
|
+
batchDepth++;
|
|
8061
|
+
try {
|
|
8062
|
+
return update();
|
|
8063
|
+
} finally {
|
|
8064
|
+
batchDepth--;
|
|
8065
|
+
if (batchDepth === 0 && batchedNotification) {
|
|
8066
|
+
batchedNotification = false;
|
|
8067
|
+
notify();
|
|
8068
|
+
}
|
|
8069
|
+
}
|
|
8070
|
+
}
|
|
8053
8071
|
function findRun(runId) {
|
|
8054
8072
|
return state.runs.find((r) => r.id === runId);
|
|
8055
8073
|
}
|
|
@@ -8103,6 +8121,7 @@ function createStoreContext(state = createStoreState()) {
|
|
|
8103
8121
|
snapshot,
|
|
8104
8122
|
graphSnapshot,
|
|
8105
8123
|
notify,
|
|
8124
|
+
batch,
|
|
8106
8125
|
bumpAndNotify,
|
|
8107
8126
|
findRun,
|
|
8108
8127
|
findStage,
|
|
@@ -8918,6 +8937,9 @@ function createRunStoreMethods(context) {
|
|
|
8918
8937
|
subscribeInvalidation(fn) {
|
|
8919
8938
|
state.invalidationListeners.add(fn);
|
|
8920
8939
|
return () => state.invalidationListeners.delete(fn);
|
|
8940
|
+
},
|
|
8941
|
+
batch(update) {
|
|
8942
|
+
return context.batch(update);
|
|
8921
8943
|
}
|
|
8922
8944
|
};
|
|
8923
8945
|
}
|
|
@@ -13403,7 +13425,7 @@ class ConcurrencyLimiter {
|
|
|
13403
13425
|
}
|
|
13404
13426
|
}
|
|
13405
13427
|
function createRunLimiter(defaultConcurrency) {
|
|
13406
|
-
return new ConcurrencyLimiter(defaultConcurrency ??
|
|
13428
|
+
return new ConcurrencyLimiter(defaultConcurrency ?? 1000);
|
|
13407
13429
|
}
|
|
13408
13430
|
|
|
13409
13431
|
// dist/builtin/workflows/src/shared/abort.ts
|
|
@@ -57173,6 +57195,44 @@ var init_executor = __esm(() => {
|
|
|
57173
57195
|
init_executor_run();
|
|
57174
57196
|
});
|
|
57175
57197
|
|
|
57198
|
+
// dist/builtin/workflows/src/shared/external-run-routes.ts
|
|
57199
|
+
function claimExternalRunRoute(store, runId, owner) {
|
|
57200
|
+
let owners = registry.owners.get(store);
|
|
57201
|
+
if (owners === undefined) {
|
|
57202
|
+
owners = new Map;
|
|
57203
|
+
registry.owners.set(store, owners);
|
|
57204
|
+
}
|
|
57205
|
+
owners.set(runId, owner);
|
|
57206
|
+
return () => {
|
|
57207
|
+
const current = registry.owners.get(store);
|
|
57208
|
+
if (current?.get(runId) !== owner)
|
|
57209
|
+
return;
|
|
57210
|
+
current.delete(runId);
|
|
57211
|
+
if (current.size === 0)
|
|
57212
|
+
registry.owners.delete(store);
|
|
57213
|
+
};
|
|
57214
|
+
}
|
|
57215
|
+
function externalRunRouteOwner(store, runId) {
|
|
57216
|
+
return registry.owners.get(store)?.get(runId);
|
|
57217
|
+
}
|
|
57218
|
+
function registerPendingStageMessageRouter(store, router) {
|
|
57219
|
+
registry.routers.set(store, router);
|
|
57220
|
+
return () => {
|
|
57221
|
+
if (registry.routers.get(store) === router)
|
|
57222
|
+
registry.routers.delete(store);
|
|
57223
|
+
};
|
|
57224
|
+
}
|
|
57225
|
+
function pendingStageMessageRouter(store) {
|
|
57226
|
+
return registry.routers.get(store);
|
|
57227
|
+
}
|
|
57228
|
+
var registryKey, processState, registry;
|
|
57229
|
+
var init_external_run_routes = __esm(() => {
|
|
57230
|
+
registryKey = Symbol.for("atomic-workflows/external-run-routes@1");
|
|
57231
|
+
processState = globalThis;
|
|
57232
|
+
registry = processState[registryKey] ?? { owners: new WeakMap, routers: new WeakMap };
|
|
57233
|
+
processState[registryKey] = registry;
|
|
57234
|
+
});
|
|
57235
|
+
|
|
57176
57236
|
// dist/builtin/workflows/src/shared/pending-stage-route-capability.ts
|
|
57177
57237
|
import { randomUUID as randomUUID10 } from "node:crypto";
|
|
57178
57238
|
function capabilitiesFor(store) {
|
|
@@ -57197,12 +57257,12 @@ function rotateWorkflowPendingStageRouteCapability(store, runId) {
|
|
|
57197
57257
|
capabilitiesFor(store).set(runId, capability);
|
|
57198
57258
|
return capability;
|
|
57199
57259
|
}
|
|
57200
|
-
var routeCapabilitiesKey,
|
|
57260
|
+
var routeCapabilitiesKey, processState2, routeCapabilities;
|
|
57201
57261
|
var init_pending_stage_route_capability = __esm(() => {
|
|
57202
57262
|
routeCapabilitiesKey = Symbol.for("atomic-workflows/pending-stage-route-capabilities@1");
|
|
57203
|
-
|
|
57204
|
-
routeCapabilities =
|
|
57205
|
-
|
|
57263
|
+
processState2 = globalThis;
|
|
57264
|
+
routeCapabilities = processState2[routeCapabilitiesKey] ?? new WeakMap;
|
|
57265
|
+
processState2[routeCapabilitiesKey] = routeCapabilities;
|
|
57206
57266
|
});
|
|
57207
57267
|
|
|
57208
57268
|
// dist/builtin/workflows/src/shared/pending-stage-route-readiness.ts
|
|
@@ -57234,12 +57294,12 @@ function hasWorkflowPendingStageRouteOwner(store) {
|
|
|
57234
57294
|
function workflowPendingStageRouteReady(store, runId) {
|
|
57235
57295
|
return owners.get(store)?.ready(runId);
|
|
57236
57296
|
}
|
|
57237
|
-
var routeOwnersKey,
|
|
57297
|
+
var routeOwnersKey, processState3, owners;
|
|
57238
57298
|
var init_pending_stage_route_readiness = __esm(() => {
|
|
57239
57299
|
routeOwnersKey = Symbol.for("atomic-workflows/pending-stage-route-owners@1");
|
|
57240
|
-
|
|
57241
|
-
owners =
|
|
57242
|
-
|
|
57300
|
+
processState3 = globalThis;
|
|
57301
|
+
owners = processState3[routeOwnersKey] ?? new WeakMap;
|
|
57302
|
+
processState3[routeOwnersKey] = owners;
|
|
57243
57303
|
});
|
|
57244
57304
|
|
|
57245
57305
|
// dist/builtin/workflows/src/shared/workflow-stage-path-matching.ts
|
|
@@ -57540,6 +57600,9 @@ function registerPendingStageIntercomBridge(pi, activeStore, ownsRun = () => tru
|
|
|
57540
57600
|
notificationId,
|
|
57541
57601
|
reason
|
|
57542
57602
|
};
|
|
57603
|
+
const external = externalRunRouteOwner(activeStore, entry.runId);
|
|
57604
|
+
if (external !== undefined)
|
|
57605
|
+
return external.notifyUndeliverable(payload);
|
|
57543
57606
|
pi.events?.emit?.(PENDING_STAGE_UNDELIVERABLE_EVENT, payload);
|
|
57544
57607
|
return payload.handled && await payload.completion === true;
|
|
57545
57608
|
};
|
|
@@ -57551,19 +57614,29 @@ function registerPendingStageIntercomBridge(pi, activeStore, ownsRun = () => tru
|
|
|
57551
57614
|
return;
|
|
57552
57615
|
const ownedRunIds = new Set;
|
|
57553
57616
|
const runs = activeStore.runs();
|
|
57617
|
+
const runById = new Map(runs.map((run) => [run.id, run]));
|
|
57618
|
+
const childrenByBoundary = new Map;
|
|
57619
|
+
for (const run of runs) {
|
|
57620
|
+
if (run.parentRunId === undefined || run.parentStageId === undefined)
|
|
57621
|
+
continue;
|
|
57622
|
+
const key = JSON.stringify([run.parentRunId, run.parentStageId]);
|
|
57623
|
+
const children = childrenByBoundary.get(key) ?? [];
|
|
57624
|
+
children.push(run);
|
|
57625
|
+
childrenByBoundary.set(key, children);
|
|
57626
|
+
}
|
|
57554
57627
|
const resolveBoundarySegments = createWorkflowBoundarySegmentsResolver(runs);
|
|
57555
57628
|
for (const run of runs) {
|
|
57556
|
-
if (!ownsRun(run.id))
|
|
57629
|
+
if (externalRunRouteOwner(activeStore, run.id) !== undefined || !ownsRun(run.id))
|
|
57557
57630
|
continue;
|
|
57558
|
-
const rootRunId =
|
|
57631
|
+
const rootRunId = reciprocalWorkflowRootRunId(runById, run.id);
|
|
57559
57632
|
if (rootRunId === undefined)
|
|
57560
57633
|
continue;
|
|
57561
57634
|
ownedRunIds.add(run.id);
|
|
57562
|
-
const parentRun =
|
|
57635
|
+
const parentRun = run.parentRunId === undefined ? undefined : runById.get(run.parentRunId);
|
|
57563
57636
|
const boundary = parentRun?.stages.find((stage) => stage.id === run.parentStageId);
|
|
57564
57637
|
const parent = parentRun === undefined || boundary === undefined ? undefined : {
|
|
57565
57638
|
runId: parentRun.id,
|
|
57566
|
-
stageKeys: [boundary.id, boundary.name].filter((key) =>
|
|
57639
|
+
stageKeys: [boundary.id, boundary.name].filter((key) => resolveIndexedChildRun(childrenByBoundary, parentRun, key)?.id === run.id)
|
|
57567
57640
|
};
|
|
57568
57641
|
const announcement = {
|
|
57569
57642
|
runId: run.id,
|
|
@@ -57704,16 +57777,22 @@ function registerPendingStageIntercomBridge(pi, activeStore, ownsRun = () => tru
|
|
|
57704
57777
|
payload.handled = true;
|
|
57705
57778
|
payload.completion = queueAndPersist(activeStore, resolvedEvent, workflowInvocationIntercomGroup(rootRunId), stage.pendingStageDeliveryAvailable === true);
|
|
57706
57779
|
};
|
|
57707
|
-
const
|
|
57708
|
-
const stickySubscription = pi.events?.on?.(STICKY_LIVE_DELIVERED_EVENT, (payload) => {
|
|
57780
|
+
const routeStickyLiveDelivered = (payload) => {
|
|
57709
57781
|
if (disposed || !isStickyLiveDeliveredEvent(payload) || payload.handled)
|
|
57710
57782
|
return;
|
|
57711
57783
|
payload.handled = true;
|
|
57712
57784
|
payload.completion = outsideWorkflowExecutions(() => recordConfirmedStickyDeliveries(activeStore, payload));
|
|
57785
|
+
};
|
|
57786
|
+
const subscription = pi.events?.on?.(PENDING_STAGE_MESSAGE_EVENT, (payload) => outsideWorkflowExecutions(() => routePendingStageMessage(payload)));
|
|
57787
|
+
const stickySubscription = pi.events?.on?.(STICKY_LIVE_DELIVERED_EVENT, routeStickyLiveDelivered);
|
|
57788
|
+
const disposeRouter = registerPendingStageMessageRouter(activeStore, {
|
|
57789
|
+
route: (payload) => outsideWorkflowExecutions(() => routePendingStageMessage(payload)),
|
|
57790
|
+
sticky: routeStickyLiveDelivered
|
|
57713
57791
|
});
|
|
57714
57792
|
const dispose = () => {
|
|
57715
57793
|
disposed = true;
|
|
57716
57794
|
disposeReadiness();
|
|
57795
|
+
disposeRouter();
|
|
57717
57796
|
routeCompletions.clear();
|
|
57718
57797
|
unsubscribeStore();
|
|
57719
57798
|
if (typeof subscription === "function")
|
|
@@ -57839,6 +57918,12 @@ function isMaterializedNonAgentTarget(runs, target) {
|
|
|
57839
57918
|
return false;
|
|
57840
57919
|
return nonAgentNodes(run).some((node) => node.id === stageKey || node.name === stageKey);
|
|
57841
57920
|
}
|
|
57921
|
+
function resolveIndexedChildRun(childrenByBoundary, parent, segment) {
|
|
57922
|
+
const boundariesById = parent.stages.filter((stage) => stage.id === segment);
|
|
57923
|
+
const boundaries = boundariesById.length > 0 ? boundariesById : parent.stages.filter((stage) => stage.name === segment);
|
|
57924
|
+
const children = boundaries.flatMap((boundary) => childrenByBoundary.get(JSON.stringify([parent.id, boundary.id])) ?? []);
|
|
57925
|
+
return children.length === 1 ? children[0] : undefined;
|
|
57926
|
+
}
|
|
57842
57927
|
function resolveChildRun(runs, parent, segment) {
|
|
57843
57928
|
const boundariesById = parent.stages.filter((stage) => stage.id === segment);
|
|
57844
57929
|
const boundaries = boundariesById.length > 0 ? boundariesById : parent.stages.filter((stage) => stage.name === segment);
|
|
@@ -58178,6 +58263,7 @@ var init_pending_stage_intercom = __esm(() => {
|
|
|
58178
58263
|
init_execution_context_boundary();
|
|
58179
58264
|
init_run_owner_backend();
|
|
58180
58265
|
init_store_backend();
|
|
58266
|
+
init_external_run_routes();
|
|
58181
58267
|
init_intercom_group();
|
|
58182
58268
|
init_pending_stage_route_capability();
|
|
58183
58269
|
init_pending_stage_route_readiness();
|
|
@@ -58636,7 +58722,7 @@ function isWorkflowFutureStageRosterEntries(value2) {
|
|
|
58636
58722
|
function isWorkflowStageRosterEntries(value2) {
|
|
58637
58723
|
return Array.isArray(value2) && value2.every((entry) => typeof entry === "object" && entry !== null && entry.kind === "workflow-stage" && typeof entry.runId === "string" && typeof entry.stageId === "string" && typeof entry.stageName === "string" && typeof entry.target === "string" && (entry.lifecycle === "pending" || entry.lifecycle === "running") && typeof entry.group === "string" && (entry.sessionId === undefined || typeof entry.sessionId === "string"));
|
|
58638
58724
|
}
|
|
58639
|
-
var BROKER_SOCKET, BROKER_CONNECT_TIMEOUT_MS = 1e4, PENDING_STAGE_ROUTE_ACK_TIMEOUT_MS = 1e4, GROUP_REQUEST_TIMEOUT_MS = 5000, PRESENCE_ACK_TIMEOUT_MS = 5000, RESTORING_RECIPIENT_REQUEST_TIMEOUT_MS = 5000, HELD_SEND_GRACE_MS = 5000, IntercomClient;
|
|
58725
|
+
var BROKER_SOCKET, BROKER_CONNECT_TIMEOUT_MS = 1e4, PENDING_STAGE_ROUTE_ACK_TIMEOUT_MS = 1e4, GROUP_REQUEST_TIMEOUT_MS = 5000, PRESENCE_ACK_TIMEOUT_MS = 5000, RESTORING_RECIPIENT_REQUEST_TIMEOUT_MS = 5000, HELD_SEND_GRACE_MS = 5000, IntercomPendingStageRouteRefusedError, PENDING_STAGE_ROUTE_REFUSAL_CODES, IntercomClient;
|
|
58640
58726
|
var init_client = __esm(() => {
|
|
58641
58727
|
init_paths();
|
|
58642
58728
|
init_pending_send_registry();
|
|
@@ -58645,6 +58731,21 @@ var init_client = __esm(() => {
|
|
|
58645
58731
|
init_live_route_refusal();
|
|
58646
58732
|
init_retry_policy();
|
|
58647
58733
|
BROKER_SOCKET = getBrokerSocketPath();
|
|
58734
|
+
IntercomPendingStageRouteRefusedError = class IntercomPendingStageRouteRefusedError extends Error {
|
|
58735
|
+
connectionKept;
|
|
58736
|
+
code;
|
|
58737
|
+
constructor(reason, refusal = { connectionKept: false }) {
|
|
58738
|
+
super(reason);
|
|
58739
|
+
this.name = "IntercomPendingStageRouteRefusedError";
|
|
58740
|
+
this.connectionKept = refusal.connectionKept;
|
|
58741
|
+
this.code = refusal.code;
|
|
58742
|
+
}
|
|
58743
|
+
};
|
|
58744
|
+
PENDING_STAGE_ROUTE_REFUSAL_CODES = new Set([
|
|
58745
|
+
"not_authorized",
|
|
58746
|
+
"invalid_stage_roster",
|
|
58747
|
+
"invalid_possible_stage_roster"
|
|
58748
|
+
]);
|
|
58648
58749
|
IntercomClient = class IntercomClient extends EventEmitter {
|
|
58649
58750
|
returnAddress;
|
|
58650
58751
|
preferredSessionId;
|
|
@@ -58652,6 +58753,8 @@ var init_client = __esm(() => {
|
|
|
58652
58753
|
connectTimeoutMs;
|
|
58653
58754
|
brokerAcknowledgesReceipt = false;
|
|
58654
58755
|
brokerAcknowledgesPendingStageRoutes = false;
|
|
58756
|
+
multiplexesPendingStageRoutes;
|
|
58757
|
+
brokerRefusesPendingStageRoutesInPlace = false;
|
|
58655
58758
|
brokerAnswersStatus = false;
|
|
58656
58759
|
socket = null;
|
|
58657
58760
|
_sessionId = null;
|
|
@@ -58717,6 +58820,7 @@ var init_client = __esm(() => {
|
|
|
58717
58820
|
this.preferredSessionId = options.preferredSessionId;
|
|
58718
58821
|
this.acknowledgesReceipt = options.acknowledgesReceipt === true;
|
|
58719
58822
|
this.connectTimeoutMs = options.connectTimeoutMs ?? BROKER_CONNECT_TIMEOUT_MS;
|
|
58823
|
+
this.multiplexesPendingStageRoutes = options.multiplexesPendingStageRoutes === true;
|
|
58720
58824
|
}
|
|
58721
58825
|
failPending(error) {
|
|
58722
58826
|
this.pendingSends.rejectAll(error);
|
|
@@ -58829,6 +58933,7 @@ var init_client = __esm(() => {
|
|
|
58829
58933
|
this._supervisorSessionId = null;
|
|
58830
58934
|
this.brokerAcknowledgesReceipt = false;
|
|
58831
58935
|
this.brokerAcknowledgesPendingStageRoutes = false;
|
|
58936
|
+
this.brokerRefusesPendingStageRoutesInPlace = false;
|
|
58832
58937
|
this.brokerAnswersStatus = false;
|
|
58833
58938
|
this._messageSource = undefined;
|
|
58834
58939
|
this.disconnectError = null;
|
|
@@ -58907,7 +59012,7 @@ var init_client = __esm(() => {
|
|
|
58907
59012
|
}
|
|
58908
59013
|
switch (brokerMessage.type) {
|
|
58909
59014
|
case "registered": {
|
|
58910
|
-
if (typeof brokerMessage.sessionId !== "string" || brokerMessage.supervisorSessionId !== undefined && typeof brokerMessage.supervisorSessionId !== "string" || brokerMessage.acknowledgesReceipt !== undefined && brokerMessage.acknowledgesReceipt !== true || brokerMessage.acknowledgesPendingStageRoutes !== undefined && brokerMessage.acknowledgesPendingStageRoutes !== true || brokerMessage.brokerStatus !== undefined && brokerMessage.brokerStatus !== true) {
|
|
59015
|
+
if (typeof brokerMessage.sessionId !== "string" || brokerMessage.supervisorSessionId !== undefined && typeof brokerMessage.supervisorSessionId !== "string" || brokerMessage.acknowledgesReceipt !== undefined && brokerMessage.acknowledgesReceipt !== true || brokerMessage.acknowledgesPendingStageRoutes !== undefined && brokerMessage.acknowledgesPendingStageRoutes !== true || brokerMessage.refusesPendingStageRoutesInPlace !== undefined && brokerMessage.refusesPendingStageRoutesInPlace !== true || brokerMessage.brokerStatus !== undefined && brokerMessage.brokerStatus !== true) {
|
|
58911
59016
|
throw new Error("Invalid registered message");
|
|
58912
59017
|
}
|
|
58913
59018
|
if (this._sessionId !== null) {
|
|
@@ -58917,6 +59022,7 @@ var init_client = __esm(() => {
|
|
|
58917
59022
|
this._supervisorSessionId = typeof brokerMessage.supervisorSessionId === "string" ? brokerMessage.supervisorSessionId : null;
|
|
58918
59023
|
this.brokerAcknowledgesReceipt = this.acknowledgesReceipt && brokerMessage.acknowledgesReceipt === true;
|
|
58919
59024
|
this.brokerAcknowledgesPendingStageRoutes = brokerMessage.acknowledgesPendingStageRoutes === true;
|
|
59025
|
+
this.brokerRefusesPendingStageRoutesInPlace = brokerMessage.refusesPendingStageRoutesInPlace === true;
|
|
58920
59026
|
this.brokerAnswersStatus = brokerMessage.brokerStatus === true;
|
|
58921
59027
|
this.emit("_registered", { type: "registered", sessionId: brokerMessage.sessionId });
|
|
58922
59028
|
break;
|
|
@@ -58929,6 +59035,12 @@ var init_client = __esm(() => {
|
|
|
58929
59035
|
this.emit("_registration_failed", refusal);
|
|
58930
59036
|
break;
|
|
58931
59037
|
}
|
|
59038
|
+
const refusedRouteId = typeof brokerMessage.requestId === "string" ? brokerMessage.requestId : "";
|
|
59039
|
+
const refusedRoute = this.pendingStageRouteRegistrations.get(refusedRouteId);
|
|
59040
|
+
if (refusedRoute !== undefined) {
|
|
59041
|
+
this.pendingStageRouteRegistrations.delete(refusedRouteId);
|
|
59042
|
+
refusedRoute.reject(new IntercomPendingStageRouteRefusedError(brokerMessage.reason));
|
|
59043
|
+
}
|
|
58932
59044
|
this.disconnectError ??= refusal;
|
|
58933
59045
|
this.failPending(this.disconnectError);
|
|
58934
59046
|
this.socket?.destroy();
|
|
@@ -59054,6 +59166,21 @@ var init_client = __esm(() => {
|
|
|
59054
59166
|
this.emit("pending_stage_route_revoked", { runId });
|
|
59055
59167
|
break;
|
|
59056
59168
|
}
|
|
59169
|
+
case "pending_stage_route_refused": {
|
|
59170
|
+
const { requestId, runId, code, reason } = brokerMessage;
|
|
59171
|
+
if (typeof requestId !== "string" || typeof runId !== "string" || typeof reason !== "string" || typeof code !== "string") {
|
|
59172
|
+
throw new Error("Invalid pending-stage route refusal");
|
|
59173
|
+
}
|
|
59174
|
+
const pending = this.pendingStageRouteRegistrations.get(requestId);
|
|
59175
|
+
if (!pending)
|
|
59176
|
+
return;
|
|
59177
|
+
this.pendingStageRouteRegistrations.delete(requestId);
|
|
59178
|
+
pending.reject(new IntercomPendingStageRouteRefusedError(reason, {
|
|
59179
|
+
connectionKept: true,
|
|
59180
|
+
...PENDING_STAGE_ROUTE_REFUSAL_CODES.has(code) ? { code } : {}
|
|
59181
|
+
}));
|
|
59182
|
+
break;
|
|
59183
|
+
}
|
|
59057
59184
|
case "pending_stage_route_registered": {
|
|
59058
59185
|
const { requestId } = brokerMessage;
|
|
59059
59186
|
if (typeof requestId !== "string")
|
|
@@ -59557,7 +59684,8 @@ var init_client = __esm(() => {
|
|
|
59557
59684
|
stages,
|
|
59558
59685
|
possibleStages,
|
|
59559
59686
|
...parent === undefined ? {} : { parent },
|
|
59560
|
-
...dormant === true ? { dormant: true } : {}
|
|
59687
|
+
...dormant === true ? { dormant: true } : {},
|
|
59688
|
+
...this.multiplexesPendingStageRoutes && this.brokerRefusesPendingStageRoutesInPlace ? { refusalKeepsConnection: true } : {}
|
|
59561
59689
|
});
|
|
59562
59690
|
} catch (error) {
|
|
59563
59691
|
clearTimeout(timeout);
|
|
@@ -60291,120 +60419,165 @@ var init_reconnect_backoff = __esm(() => {
|
|
|
60291
60419
|
BACKOFF_MS = [1000, 2000, 5000, 1e4, 30000];
|
|
60292
60420
|
});
|
|
60293
60421
|
|
|
60294
|
-
// dist/builtin/intercom/route-connect-gate.ts
|
|
60295
|
-
function createConcurrencyGate(limit) {
|
|
60296
|
-
let active = 0;
|
|
60297
|
-
const waiting = [];
|
|
60298
|
-
const release = () => {
|
|
60299
|
-
const next = waiting.shift();
|
|
60300
|
-
if (next === undefined)
|
|
60301
|
-
active -= 1;
|
|
60302
|
-
else
|
|
60303
|
-
next();
|
|
60304
|
-
};
|
|
60305
|
-
return {
|
|
60306
|
-
async run(operation) {
|
|
60307
|
-
if (active < limit)
|
|
60308
|
-
active += 1;
|
|
60309
|
-
else
|
|
60310
|
-
await new Promise((resolve) => waiting.push(resolve));
|
|
60311
|
-
try {
|
|
60312
|
-
return await operation();
|
|
60313
|
-
} finally {
|
|
60314
|
-
release();
|
|
60315
|
-
}
|
|
60316
|
-
}
|
|
60317
|
-
};
|
|
60318
|
-
}
|
|
60319
|
-
var ROUTE_CLIENT_CONNECT_CONCURRENCY = 4;
|
|
60320
|
-
|
|
60321
60422
|
// dist/builtin/intercom/workflow-route-owner.ts
|
|
60322
60423
|
import { randomUUID as randomUUID13 } from "node:crypto";
|
|
60323
60424
|
import { createEventBus, getBuiltinPackagePaths } from "@bastani/atomic";
|
|
60425
|
+
function closedError() {
|
|
60426
|
+
return new Error("Workflow Intercom route owner closed");
|
|
60427
|
+
}
|
|
60324
60428
|
function createWorkflowRouteOwner(cwd) {
|
|
60325
60429
|
const events = createEventBus();
|
|
60326
60430
|
const config = loadConfig();
|
|
60327
60431
|
const id = randomUUID13();
|
|
60328
60432
|
const routes = new Map;
|
|
60329
|
-
const
|
|
60433
|
+
const connections = new Map;
|
|
60434
|
+
const isCurrent = (state) => routes.get(state.route.runId) === state;
|
|
60435
|
+
const connectionKey = (state) => state.isolated ? state.route.runId : state.route.group;
|
|
60436
|
+
function connectionOf(state) {
|
|
60437
|
+
const key = connectionKey(state);
|
|
60438
|
+
let connection = connections.get(key);
|
|
60439
|
+
if (connection === undefined) {
|
|
60440
|
+
connection = { key, group: state.route.group, acknowledged: false, routes: new Map };
|
|
60441
|
+
connections.set(key, connection);
|
|
60442
|
+
}
|
|
60443
|
+
connection.routes.set(state.route.runId, state);
|
|
60444
|
+
return connection;
|
|
60445
|
+
}
|
|
60446
|
+
function detach(state) {
|
|
60447
|
+
const key = connectionKey(state);
|
|
60448
|
+
const connection = connections.get(key);
|
|
60449
|
+
connection?.routes.delete(state.route.runId);
|
|
60450
|
+
if (!state.isolated || connection === undefined)
|
|
60451
|
+
return;
|
|
60452
|
+
connections.delete(key);
|
|
60453
|
+
connection.client?.disconnect().catch(() => {});
|
|
60454
|
+
}
|
|
60455
|
+
function settle(state, registered) {
|
|
60456
|
+
state.registered = registered;
|
|
60457
|
+
if (!registered && isCurrent(state))
|
|
60458
|
+
return;
|
|
60459
|
+
for (const waiter of state.waiters)
|
|
60460
|
+
waiter(registered);
|
|
60461
|
+
state.waiters.clear();
|
|
60462
|
+
}
|
|
60330
60463
|
function scheduleRetry(state) {
|
|
60331
|
-
if (state
|
|
60464
|
+
if (!isCurrent(state) || state.retry !== undefined)
|
|
60332
60465
|
return;
|
|
60333
60466
|
state.retry = setTimeout(() => {
|
|
60334
60467
|
state.retry = undefined;
|
|
60335
|
-
publish(state, state.route).catch(() =>
|
|
60468
|
+
publish(state, state.route).catch(() => {});
|
|
60336
60469
|
}, jitteredReconnectDelayMs(state.attempt++));
|
|
60337
60470
|
state.retry.unref();
|
|
60338
60471
|
}
|
|
60339
|
-
|
|
60340
|
-
if (
|
|
60341
|
-
|
|
60342
|
-
|
|
60343
|
-
|
|
60344
|
-
|
|
60345
|
-
|
|
60346
|
-
|
|
60347
|
-
}
|
|
60348
|
-
|
|
60349
|
-
|
|
60350
|
-
|
|
60351
|
-
if (
|
|
60352
|
-
|
|
60353
|
-
|
|
60354
|
-
|
|
60355
|
-
|
|
60356
|
-
|
|
60357
|
-
|
|
60358
|
-
|
|
60359
|
-
|
|
60360
|
-
|
|
60361
|
-
|
|
60362
|
-
|
|
60363
|
-
if (
|
|
60364
|
-
|
|
60365
|
-
const
|
|
60366
|
-
|
|
60367
|
-
|
|
60368
|
-
|
|
60369
|
-
|
|
60370
|
-
|
|
60371
|
-
|
|
60372
|
-
|
|
60373
|
-
|
|
60374
|
-
|
|
60375
|
-
|
|
60376
|
-
|
|
60377
|
-
|
|
60378
|
-
|
|
60379
|
-
|
|
60380
|
-
|
|
60381
|
-
|
|
60382
|
-
|
|
60383
|
-
|
|
60472
|
+
function drop(connection, client) {
|
|
60473
|
+
if (connection.client === client) {
|
|
60474
|
+
connection.client = undefined;
|
|
60475
|
+
connection.acknowledged = false;
|
|
60476
|
+
for (const state of connection.routes.values()) {
|
|
60477
|
+
settle(state, false);
|
|
60478
|
+
scheduleRetry(state);
|
|
60479
|
+
}
|
|
60480
|
+
}
|
|
60481
|
+
client.disconnect().catch(() => {});
|
|
60482
|
+
}
|
|
60483
|
+
function connect(connection) {
|
|
60484
|
+
if (connection.client?.isConnected())
|
|
60485
|
+
return Promise.resolve(connection.client);
|
|
60486
|
+
connection.connecting ??= (async () => {
|
|
60487
|
+
const [extensionDir] = getBuiltinPackagePaths({
|
|
60488
|
+
workflows: false,
|
|
60489
|
+
subagents: false,
|
|
60490
|
+
mcp: false,
|
|
60491
|
+
"web-access": false
|
|
60492
|
+
});
|
|
60493
|
+
if (extensionDir === undefined)
|
|
60494
|
+
throw new Error("Intercom builtin package is unavailable");
|
|
60495
|
+
await spawnBrokerIfNeeded(config.brokerCommand, config.brokerArgs, undefined, extensionDir);
|
|
60496
|
+
if (connections.get(connection.key) !== connection)
|
|
60497
|
+
throw closedError();
|
|
60498
|
+
const client = new IntercomClient(`${id}:pending-stage-route:${connection.key}`, {
|
|
60499
|
+
multiplexesPendingStageRoutes: true
|
|
60500
|
+
});
|
|
60501
|
+
connection.client = client;
|
|
60502
|
+
connection.acknowledged = false;
|
|
60503
|
+
client.on("error", () => {});
|
|
60504
|
+
client.on("disconnected", () => drop(connection, client));
|
|
60505
|
+
client.on("pending_stage_message", (request) => {
|
|
60506
|
+
if (connection.client !== client)
|
|
60507
|
+
return;
|
|
60508
|
+
const event = { ...request, handled: false };
|
|
60509
|
+
events.emit("atomic:workflow-pending-stage-message", event);
|
|
60510
|
+
const completion = event.handled && event.completion !== undefined ? event.completion : Promise.resolve({ outcome: "refused", reason: "Session not found" });
|
|
60511
|
+
completion.catch((error) => ({ outcome: "refused", reason: error.message })).then((result) => client.respondPendingStageMessage(request.requestId, result)).catch(() => {});
|
|
60512
|
+
});
|
|
60513
|
+
client.on("sticky_live_delivered", (notice) => {
|
|
60514
|
+
if (connection.client !== client)
|
|
60515
|
+
return;
|
|
60516
|
+
events.emit("atomic:workflow-sticky-live-delivered", { ...notice, handled: false });
|
|
60517
|
+
});
|
|
60518
|
+
try {
|
|
60519
|
+
await client.connect({
|
|
60520
|
+
cwd,
|
|
60521
|
+
model: "workflow",
|
|
60522
|
+
pid: process.pid,
|
|
60523
|
+
startedAt: Date.now(),
|
|
60524
|
+
lastActivity: Date.now(),
|
|
60525
|
+
group: connection.group,
|
|
60526
|
+
groups: [connection.group],
|
|
60527
|
+
recipientPurpose: "control"
|
|
60528
|
+
});
|
|
60529
|
+
if (connections.get(connection.key) !== connection)
|
|
60530
|
+
throw closedError();
|
|
60531
|
+
} catch (error) {
|
|
60532
|
+
drop(connection, client);
|
|
60533
|
+
throw error;
|
|
60534
|
+
}
|
|
60535
|
+
for (const state of connection.routes.values()) {
|
|
60536
|
+
if (state.registered || state.retry === undefined)
|
|
60537
|
+
continue;
|
|
60538
|
+
clearTimeout(state.retry);
|
|
60539
|
+
state.retry = undefined;
|
|
60540
|
+
publish(state, state.route).catch(() => {});
|
|
60541
|
+
}
|
|
60542
|
+
return client;
|
|
60543
|
+
})().finally(() => {
|
|
60544
|
+
connection.connecting = undefined;
|
|
60384
60545
|
});
|
|
60385
|
-
return
|
|
60546
|
+
return connection.connecting;
|
|
60386
60547
|
}
|
|
60387
60548
|
function publish(state, route) {
|
|
60388
|
-
const completion = state.tail.catch(() => {}).then(
|
|
60389
|
-
if (state
|
|
60390
|
-
throw
|
|
60391
|
-
const
|
|
60549
|
+
const completion = state.tail.catch(() => {}).then(async () => {
|
|
60550
|
+
if (!isCurrent(state))
|
|
60551
|
+
throw closedError();
|
|
60552
|
+
const connection = connectionOf(state);
|
|
60553
|
+
const established = connection.acknowledged && connection.client?.isConnected() ? connection.client : undefined;
|
|
60554
|
+
let client;
|
|
60392
60555
|
try {
|
|
60393
|
-
|
|
60556
|
+
client = await connect(connection);
|
|
60394
60557
|
await client.registerPendingStageRouteAcknowledged(route.runId, route.group, route.capability, route.stages, route.possibleStages, route.parent);
|
|
60558
|
+
if (connection.client !== client || !isCurrent(state))
|
|
60559
|
+
return;
|
|
60560
|
+
connection.acknowledged = true;
|
|
60395
60561
|
state.attempt = 0;
|
|
60396
60562
|
clearTimeout(state.retry);
|
|
60397
60563
|
state.retry = undefined;
|
|
60564
|
+
settle(state, true);
|
|
60398
60565
|
} catch (error) {
|
|
60399
|
-
|
|
60400
|
-
|
|
60401
|
-
|
|
60402
|
-
|
|
60566
|
+
if (error instanceof IntercomPendingStageRouteRefusedError && error.connectionKept) {
|
|
60567
|
+
scheduleRetry(state);
|
|
60568
|
+
throw error;
|
|
60569
|
+
}
|
|
60570
|
+
if (error instanceof IntercomPendingStageRouteRefusedError && !state.isolated && isCurrent(state)) {
|
|
60571
|
+
detach(state);
|
|
60572
|
+
state.isolated = true;
|
|
60573
|
+
connectionOf(state);
|
|
60403
60574
|
}
|
|
60575
|
+
if (client !== undefined && (client !== established || !client.isConnected()))
|
|
60576
|
+
drop(connection, client);
|
|
60404
60577
|
scheduleRetry(state);
|
|
60405
60578
|
throw error;
|
|
60406
60579
|
}
|
|
60407
|
-
})
|
|
60580
|
+
});
|
|
60408
60581
|
state.tail = completion;
|
|
60409
60582
|
completion.catch(() => {});
|
|
60410
60583
|
return completion;
|
|
@@ -60413,10 +60586,15 @@ function createWorkflowRouteOwner(cwd) {
|
|
|
60413
60586
|
const route = payload;
|
|
60414
60587
|
let state = routes.get(route.runId);
|
|
60415
60588
|
if (state === undefined) {
|
|
60416
|
-
state = { route, tail: Promise.resolve(), attempt: 0,
|
|
60589
|
+
state = { route, tail: Promise.resolve(), attempt: 0, registered: false, isolated: false, waiters: new Set };
|
|
60417
60590
|
routes.set(route.runId, state);
|
|
60591
|
+
} else if (state.route.group !== route.group) {
|
|
60592
|
+
detach(state);
|
|
60593
|
+
state.isolated = false;
|
|
60594
|
+
settle(state, false);
|
|
60418
60595
|
}
|
|
60419
60596
|
state.route = route;
|
|
60597
|
+
connectionOf(state);
|
|
60420
60598
|
route.completion = publish(state, route);
|
|
60421
60599
|
});
|
|
60422
60600
|
events.on("atomic:workflow-pending-stage-undeliverable", (payload) => {
|
|
@@ -60427,7 +60605,7 @@ function createWorkflowRouteOwner(cwd) {
|
|
|
60427
60605
|
event.handled = true;
|
|
60428
60606
|
event.completion = (async () => {
|
|
60429
60607
|
await publish(state, state.route);
|
|
60430
|
-
const client = state
|
|
60608
|
+
const client = connections.get(connectionKey(state))?.client;
|
|
60431
60609
|
if (client === undefined || !client.isConnected())
|
|
60432
60610
|
return false;
|
|
60433
60611
|
const text = `Pending workflow stage could not receive intercom message: ${event.reason}`;
|
|
@@ -60437,18 +60615,33 @@ function createWorkflowRouteOwner(cwd) {
|
|
|
60437
60615
|
});
|
|
60438
60616
|
return {
|
|
60439
60617
|
events,
|
|
60618
|
+
registered(runId) {
|
|
60619
|
+
const state = routes.get(runId);
|
|
60620
|
+
if (state === undefined)
|
|
60621
|
+
return Promise.resolve(false);
|
|
60622
|
+
if (state.registered)
|
|
60623
|
+
return Promise.resolve(true);
|
|
60624
|
+
return new Promise((resolve) => state.waiters.add(resolve));
|
|
60625
|
+
},
|
|
60440
60626
|
async closeGroup(group) {
|
|
60441
60627
|
const closing = [...routes.values()].filter((state) => state.route.group === group);
|
|
60628
|
+
const closingConnections = [...connections.values()].filter((connection) => connection.group === group);
|
|
60629
|
+
for (const connection of closingConnections) {
|
|
60630
|
+
connections.delete(connection.key);
|
|
60631
|
+
connection.routes.clear();
|
|
60632
|
+
}
|
|
60442
60633
|
for (const state of closing) {
|
|
60443
|
-
state.closed = true;
|
|
60444
|
-
clearTimeout(state.retry);
|
|
60445
60634
|
routes.delete(state.route.runId);
|
|
60635
|
+
clearTimeout(state.retry);
|
|
60636
|
+
state.retry = undefined;
|
|
60637
|
+
settle(state, false);
|
|
60446
60638
|
}
|
|
60447
|
-
await Promise.all(
|
|
60448
|
-
await
|
|
60449
|
-
await
|
|
60450
|
-
await state.client?.disconnect();
|
|
60639
|
+
await Promise.all(closingConnections.map(async (connection) => {
|
|
60640
|
+
await connection.client?.disconnect();
|
|
60641
|
+
await connection.connecting?.catch(() => {});
|
|
60451
60642
|
}));
|
|
60643
|
+
await Promise.all(closing.map((state) => state.tail.catch(() => {})));
|
|
60644
|
+
await Promise.all(closingConnections.map((connection) => connection.client?.disconnect()));
|
|
60452
60645
|
}
|
|
60453
60646
|
};
|
|
60454
60647
|
}
|
|
@@ -60631,7 +60824,7 @@ var init_workflow_infrastructure_error = __esm(() => {
|
|
|
60631
60824
|
[/Local worker (?:process )?cleanup (?:was not confirmed|unconfirmed)/, "worker_cleanup_unconfirmed"],
|
|
60632
60825
|
[/Local worker retry limit/, "worker_retry_limit"],
|
|
60633
60826
|
[/Local worker (?:startup timed out|exited before readiness)/, "worker_startup_failed"],
|
|
60634
|
-
[/Worker exited before delivery acknowledgement/, "worker_lost"],
|
|
60827
|
+
[/Worker exited before delivery acknowledgement|Local worker pool coordinator exited/, "worker_lost"],
|
|
60635
60828
|
[/Stale work lease/, "stale_work_lease"],
|
|
60636
60829
|
[/Intercom runtime no longer active/, "intercom_unavailable"],
|
|
60637
60830
|
[/Storage unavailable/, "storage_unavailable"]
|
|
@@ -60889,7 +61082,7 @@ var init_definitions = __esm(() => {
|
|
|
60889
61082
|
// dist/builtin/workflows/src/durable/journal/pool-mutations.ts
|
|
60890
61083
|
import { StorageProvider as StorageProvider2 } from "@bastani/atomic/storage";
|
|
60891
61084
|
async function validatePoolMutation(tx, journal, root, work, mutation) {
|
|
60892
|
-
if (!["checkpoint", "session-create", "session-
|
|
61085
|
+
if (!["checkpoint", "session-create", "session-replace"].includes(mutation.kind))
|
|
60893
61086
|
throw new Error("Invalid pool mutation");
|
|
60894
61087
|
const checkpoint = mutation.checkpoint;
|
|
60895
61088
|
const prefix = `${work.scope}:`;
|
|
@@ -60957,10 +61150,18 @@ async function sessionStore(tx, prefix) {
|
|
|
60957
61150
|
return new StorageProvider2().openSessionStore({ kind: "database", database, prefix }, { agentDir: "", domain: "sessions" });
|
|
60958
61151
|
}
|
|
60959
61152
|
async function applyPoolSession(tx, prefix, workId, mutation) {
|
|
61153
|
+
const owners = `${prefix}session_owners`;
|
|
61154
|
+
if (mutation.kind === "session-append") {
|
|
61155
|
+
if (typeof mutation.sessionId !== "string" || mutation.sessionId === "")
|
|
61156
|
+
throw new Error("Pool sessions require database identity");
|
|
61157
|
+
const owner = await tx.get(`SELECT work FROM ${owners} WHERE session = ?`, mutation.sessionId);
|
|
61158
|
+
if (owner?.work !== workId)
|
|
61159
|
+
throw new Error("Pool session scope mismatch");
|
|
61160
|
+
return withSessionStore(tx, prefix, (store) => store.append(mutation.sessionId, mutation.entries, { expectSeq: mutation.expectSeq }));
|
|
61161
|
+
}
|
|
60960
61162
|
const { sessionId, replayKey } = mutation.checkpoint;
|
|
60961
61163
|
if (!sessionId || mutation.checkpoint.sessionFile !== undefined)
|
|
60962
61164
|
throw new Error("Pool sessions require database identity");
|
|
60963
|
-
const owners = `${prefix}session_owners`;
|
|
60964
61165
|
if (mutation.kind === "session-create") {
|
|
60965
61166
|
if (mutation.header.id !== sessionId)
|
|
60966
61167
|
throw new Error("Pool session identity mismatch");
|
|
@@ -60970,10 +61171,12 @@ async function applyPoolSession(tx, prefix, workId, mutation) {
|
|
|
60970
61171
|
if (owner?.work !== workId || owner.replay_key !== replayKey)
|
|
60971
61172
|
throw new Error("Pool session scope mismatch");
|
|
60972
61173
|
}
|
|
61174
|
+
return withSessionStore(tx, prefix, (store) => mutation.kind === "session-create" ? store.create(mutation.header, mutation.entries) : store.replace(sessionId, mutation.header, mutation.entries, { expectSeq: mutation.expectSeq }));
|
|
61175
|
+
}
|
|
61176
|
+
async function withSessionStore(tx, prefix, write) {
|
|
60973
61177
|
const store = await sessionStore(tx, prefix);
|
|
60974
61178
|
try {
|
|
60975
|
-
|
|
60976
|
-
return result.lastSeq;
|
|
61179
|
+
return (await write(store)).lastSeq;
|
|
60977
61180
|
} finally {
|
|
60978
61181
|
await store.close();
|
|
60979
61182
|
}
|
|
@@ -61021,8 +61224,11 @@ var init_pool_stage_usage = __esm(() => {
|
|
|
61021
61224
|
});
|
|
61022
61225
|
|
|
61023
61226
|
// dist/builtin/workflows/src/durable/journal/sql-pool-queue.ts
|
|
61024
|
-
import { randomUUID as randomUUID15 } from "node:crypto";
|
|
61227
|
+
import { createHash as createHash4, randomUUID as randomUUID15 } from "node:crypto";
|
|
61025
61228
|
import { assertStorageSchemaVersion as assertStorageSchemaVersion2 } from "@bastani/atomic/storage";
|
|
61229
|
+
function requestIdentity(request) {
|
|
61230
|
+
return createHash4("sha256").update(request).digest("hex");
|
|
61231
|
+
}
|
|
61026
61232
|
function fromRow(row) {
|
|
61027
61233
|
return {
|
|
61028
61234
|
...JSON.parse(row.descriptor),
|
|
@@ -61078,10 +61284,11 @@ class SqlPoolQueue {
|
|
|
61078
61284
|
const versions = await tx.all(`SELECT version FROM ${pool.meta}`);
|
|
61079
61285
|
if (versions.length > 1)
|
|
61080
61286
|
throw new Error("Corrupt pool schema metadata");
|
|
61081
|
-
|
|
61082
|
-
|
|
61083
|
-
|
|
61084
|
-
|
|
61287
|
+
const version = versions[0]?.version;
|
|
61288
|
+
if (version === undefined)
|
|
61289
|
+
await tx.run(`INSERT INTO ${pool.meta} VALUES (?)`, POOL_SCHEMA_VERSION);
|
|
61290
|
+
else if (version !== 1)
|
|
61291
|
+
assertStorageSchemaVersion2("journal", version, POOL_SCHEMA_VERSION);
|
|
61085
61292
|
await tx.exec(`
|
|
61086
61293
|
CREATE TABLE IF NOT EXISTS ${pool.roots}(root TEXT PRIMARY KEY REFERENCES ${journal.runs}(run_id) ON DELETE CASCADE, timeout INTEGER NOT NULL);
|
|
61087
61294
|
CREATE TABLE IF NOT EXISTS ${pool.work}(
|
|
@@ -61112,9 +61319,23 @@ class SqlPoolQueue {
|
|
|
61112
61319
|
state TEXT NOT NULL DEFAULT 'running', revision INTEGER NOT NULL DEFAULT 0
|
|
61113
61320
|
)`);
|
|
61114
61321
|
await tx.exec(poolUsageSchema(journal.runs));
|
|
61322
|
+
if (version === 1)
|
|
61323
|
+
await pool.migrateFromVersion1(tx);
|
|
61115
61324
|
});
|
|
61116
61325
|
return pool;
|
|
61117
61326
|
}
|
|
61327
|
+
async migrateFromVersion1(tx) {
|
|
61328
|
+
await tx.run(`DELETE FROM ${this.operations} WHERE substr(request, 1, 1) = '['`);
|
|
61329
|
+
for (let after = 0;; ) {
|
|
61330
|
+
const rows = await tx.all(`SELECT rowid, request FROM ${this.operations} WHERE rowid > ? ORDER BY rowid LIMIT 256`, after);
|
|
61331
|
+
if (rows.length === 0)
|
|
61332
|
+
break;
|
|
61333
|
+
for (const row of rows)
|
|
61334
|
+
await tx.run(`UPDATE ${this.operations} SET request = ? WHERE rowid = ?`, requestIdentity(String(row.request)), row.rowid);
|
|
61335
|
+
after = Number(rows.at(-1).rowid);
|
|
61336
|
+
}
|
|
61337
|
+
await tx.run(`UPDATE ${this.meta} SET version = ?`, POOL_SCHEMA_VERSION);
|
|
61338
|
+
}
|
|
61118
61339
|
async authority(tx, lease, timeout) {
|
|
61119
61340
|
const row = await tx.get(`SELECT l.*, r.status, p.timeout, ${STORE_NOW2} AS now FROM ${this.journal.leases} l JOIN ${this.journal.runs} r ON r.run_id = l.run_id LEFT JOIN ${this.roots} p ON p.root = l.run_id WHERE l.run_id = ?`, lease.runId);
|
|
61120
61341
|
if (row?.timeout === null && timeout === undefined)
|
|
@@ -61255,31 +61476,18 @@ class SqlPoolQueue {
|
|
|
61255
61476
|
throw new Error(STALE_WORK_LEASE);
|
|
61256
61477
|
return row;
|
|
61257
61478
|
}
|
|
61258
|
-
async renew(lease, token,
|
|
61479
|
+
async renew(lease, token, durationMs) {
|
|
61259
61480
|
positive(durationMs);
|
|
61260
61481
|
lease = snapshotLease(lease);
|
|
61261
61482
|
token = { ...token };
|
|
61262
|
-
validateText("key", operationId);
|
|
61263
|
-
const request = JSON.stringify([token.epoch, token.generation, token.worker, durationMs]);
|
|
61264
61483
|
return this.transaction(async (tx) => {
|
|
61265
61484
|
const { now } = await this.authority(tx, lease);
|
|
61266
61485
|
const row = await this.current(tx, lease, token);
|
|
61267
61486
|
if (row.epoch !== lease.epoch || row.expires_at < now)
|
|
61268
61487
|
throw new Error(STALE_WORK_LEASE);
|
|
61269
|
-
const
|
|
61270
|
-
|
|
61271
|
-
|
|
61272
|
-
throw new Error("Pool operation identity conflict");
|
|
61273
|
-
return JSON.parse(previous.receipt);
|
|
61274
|
-
}
|
|
61275
|
-
const receipt = {
|
|
61276
|
-
...fromRow(row),
|
|
61277
|
-
expiresAt: Math.max(Number(row.expires_at), expiry(now, durationMs)),
|
|
61278
|
-
renewedAt: now
|
|
61279
|
-
};
|
|
61280
|
-
await tx.run(`UPDATE ${this.work} SET expires_at = ?, renewed_at = ? WHERE id = ?`, receipt.expiresAt, now, row.id);
|
|
61281
|
-
await tx.run(`INSERT INTO ${this.operations} VALUES (?, ?, ?, ?, ?)`, row.id, token.generation, operationId, request, JSON.stringify(receipt));
|
|
61282
|
-
return receipt;
|
|
61488
|
+
const expiresAt = Math.max(Number(row.expires_at), expiry(now, durationMs));
|
|
61489
|
+
await tx.run(`UPDATE ${this.work} SET expires_at = ?, renewed_at = ? WHERE id = ?`, expiresAt, now, row.id);
|
|
61490
|
+
return { ...fromRow(row), expiresAt, renewedAt: now };
|
|
61283
61491
|
});
|
|
61284
61492
|
}
|
|
61285
61493
|
async commit(lease, token, operationId, mutation) {
|
|
@@ -61288,6 +61496,7 @@ class SqlPoolQueue {
|
|
|
61288
61496
|
validateText("key", operationId);
|
|
61289
61497
|
assertWorkflowSerializableObject(mutation, "Pool mutation");
|
|
61290
61498
|
const request = serializeJson({ epoch: token.epoch, worker: token.worker, mutation });
|
|
61499
|
+
const identity = requestIdentity(request);
|
|
61291
61500
|
mutation = JSON.parse(request).mutation;
|
|
61292
61501
|
return this.transaction(async (tx) => {
|
|
61293
61502
|
const { now, status } = await this.authority(tx, lease);
|
|
@@ -61296,7 +61505,7 @@ class SqlPoolQueue {
|
|
|
61296
61505
|
throw new Error(STALE_WORK_LEASE);
|
|
61297
61506
|
const previous = await tx.get(`SELECT * FROM ${this.operations} WHERE work = ? AND generation = ? AND operation = ?`, token.id, token.generation, operationId);
|
|
61298
61507
|
if (previous) {
|
|
61299
|
-
if (previous.request !==
|
|
61508
|
+
if (previous.request !== identity)
|
|
61300
61509
|
throw new Error("Pool operation identity conflict");
|
|
61301
61510
|
return JSON.parse(previous.receipt);
|
|
61302
61511
|
}
|
|
@@ -61304,34 +61513,41 @@ class SqlPoolQueue {
|
|
|
61304
61513
|
throw new Error(STALE_WORK_LEASE);
|
|
61305
61514
|
if (!await this.admitsAttemptWrites(tx, lease, status))
|
|
61306
61515
|
throw new Error("Pool root is not running");
|
|
61307
|
-
const
|
|
61308
|
-
await
|
|
61309
|
-
if (checkpoint.completedAt > Math.max(now, Date.now()) + MAX_WORKER_CLOCK_SKEW_MS)
|
|
61310
|
-
throw new Error("Pool checkpoint timestamp is in the future");
|
|
61311
|
-
assertWorkflowSerializableObject(checkpoint, "Pool checkpoint");
|
|
61312
|
-
const key = JSON.stringify([RUN_EVENT_TYPES.checkpointRecorded, checkpoint.kind, checkpoint.checkpointId]);
|
|
61313
|
-
const payload = serializeJson(checkpoint);
|
|
61314
|
-
const existing = await tx.get(`SELECT seq, at, payload FROM ${this.journal.events} WHERE run_id = ? AND key = ?`, lease.runId, key);
|
|
61315
|
-
if (existing && existing.payload !== payload)
|
|
61316
|
-
throw new Error("Pool checkpoint identity conflict");
|
|
61317
|
-
if (existing && mutation.kind !== "checkpoint")
|
|
61318
|
-
throw new Error("Pool session checkpoint identity conflict");
|
|
61319
|
-
const lastSeq = mutation.kind === "checkpoint" ? undefined : await applyPoolSession(tx, `${this.journal.runs}_pool_`, row.id, mutation);
|
|
61320
|
-
const latest = await tx.get(`SELECT COALESCE(MAX(seq), 0) AS seq FROM ${this.journal.events} WHERE run_id = ?`, lease.runId);
|
|
61321
|
-
const receipt = existing ? { seq: Number(existing.seq), at: Number(existing.at) } : {
|
|
61322
|
-
seq: journalCounter(lease.runId, "seq", Number(latest.seq) + 1, "overflow"),
|
|
61323
|
-
at: now,
|
|
61324
|
-
...lastSeq === undefined ? {} : { lastSeq }
|
|
61325
|
-
};
|
|
61326
|
-
if (!existing)
|
|
61327
|
-
await tx.run(`INSERT INTO ${this.journal.events} VALUES (?, ?, ?, ?, ?, ?, ?)`, lease.runId, receipt.seq, lease.epoch, key, RUN_EVENT_TYPES.checkpointRecorded, payload, now);
|
|
61328
|
-
const billed = poolCheckpointUsage(checkpoint);
|
|
61329
|
-
if (billed && !existing)
|
|
61330
|
-
await upsertPoolUsage(tx, this.journal.runs, row.id, token.generation, billed.key, billed.usage);
|
|
61331
|
-
await tx.run(`INSERT INTO ${this.operations} VALUES (?, ?, ?, ?, ?)`, token.id, token.generation, operationId, request, JSON.stringify(receipt));
|
|
61516
|
+
const receipt = mutation.kind === "session-append" ? await this.appendSession(tx, row, mutation, now) : await this.recordCheckpoint(tx, lease, token, row, mutation, now);
|
|
61517
|
+
await tx.run(`INSERT INTO ${this.operations} VALUES (?, ?, ?, ?, ?)`, token.id, token.generation, operationId, identity, JSON.stringify(receipt));
|
|
61332
61518
|
return receipt;
|
|
61333
61519
|
});
|
|
61334
61520
|
}
|
|
61521
|
+
async appendSession(tx, row, mutation, now) {
|
|
61522
|
+
return { at: now, lastSeq: await applyPoolSession(tx, `${this.journal.runs}_pool_`, row.id, mutation) };
|
|
61523
|
+
}
|
|
61524
|
+
async recordCheckpoint(tx, lease, token, row, mutation, now) {
|
|
61525
|
+
const checkpoint = mutation.checkpoint;
|
|
61526
|
+
await validatePoolMutation(tx, this.journal, lease.runId, fromRow(row), mutation);
|
|
61527
|
+
if (checkpoint.completedAt > Math.max(now, Date.now()) + MAX_WORKER_CLOCK_SKEW_MS)
|
|
61528
|
+
throw new Error("Pool checkpoint timestamp is in the future");
|
|
61529
|
+
assertWorkflowSerializableObject(checkpoint, "Pool checkpoint");
|
|
61530
|
+
const key = JSON.stringify([RUN_EVENT_TYPES.checkpointRecorded, checkpoint.kind, checkpoint.checkpointId]);
|
|
61531
|
+
const payload = serializeJson(checkpoint);
|
|
61532
|
+
const existing = await tx.get(`SELECT seq, at, payload FROM ${this.journal.events} WHERE run_id = ? AND key = ?`, lease.runId, key);
|
|
61533
|
+
if (existing && existing.payload !== payload)
|
|
61534
|
+
throw new Error("Pool checkpoint identity conflict");
|
|
61535
|
+
if (existing && mutation.kind !== "checkpoint")
|
|
61536
|
+
throw new Error("Pool session checkpoint identity conflict");
|
|
61537
|
+
const lastSeq = mutation.kind === "checkpoint" ? undefined : await applyPoolSession(tx, `${this.journal.runs}_pool_`, row.id, mutation);
|
|
61538
|
+
const latest = await tx.get(`SELECT COALESCE(MAX(seq), 0) AS seq FROM ${this.journal.events} WHERE run_id = ?`, lease.runId);
|
|
61539
|
+
const receipt = existing ? { seq: Number(existing.seq), at: Number(existing.at) } : {
|
|
61540
|
+
seq: journalCounter(lease.runId, "seq", Number(latest.seq) + 1, "overflow"),
|
|
61541
|
+
at: now,
|
|
61542
|
+
...lastSeq === undefined ? {} : { lastSeq }
|
|
61543
|
+
};
|
|
61544
|
+
if (!existing)
|
|
61545
|
+
await tx.run(`INSERT INTO ${this.journal.events} VALUES (?, ?, ?, ?, ?, ?, ?)`, lease.runId, receipt.seq, lease.epoch, key, RUN_EVENT_TYPES.checkpointRecorded, payload, now);
|
|
61546
|
+
const billed = poolCheckpointUsage(checkpoint);
|
|
61547
|
+
if (billed && !existing)
|
|
61548
|
+
await upsertPoolUsage(tx, this.journal.runs, row.id, token.generation, billed.key, billed.usage);
|
|
61549
|
+
return receipt;
|
|
61550
|
+
}
|
|
61335
61551
|
async complete(lease, token, result) {
|
|
61336
61552
|
lease = snapshotLease(lease);
|
|
61337
61553
|
token = { ...token };
|
|
@@ -61418,7 +61634,7 @@ function openSqlPoolQueue(journal) {
|
|
|
61418
61634
|
return Promise.reject(new Error("Worker pool requires a SQL run journal"));
|
|
61419
61635
|
return SqlPoolQueue.open(storage);
|
|
61420
61636
|
}
|
|
61421
|
-
var STORE_NOW2 = "CAST(unixepoch('subsec') * 1000 AS INTEGER)", MAX_WORKER_CLOCK_SKEW_MS = 1000, STALE_WORK_LEASE = "Stale work lease";
|
|
61637
|
+
var STORE_NOW2 = "CAST(unixepoch('subsec') * 1000 AS INTEGER)", MAX_WORKER_CLOCK_SKEW_MS = 1000, POOL_SCHEMA_VERSION = 2, STALE_WORK_LEASE = "Stale work lease";
|
|
61422
61638
|
var init_sql_pool_queue = __esm(() => {
|
|
61423
61639
|
init_serializable();
|
|
61424
61640
|
init_boundary_topology();
|
|
@@ -61430,6 +61646,15 @@ var init_sql_pool_queue = __esm(() => {
|
|
|
61430
61646
|
init_values();
|
|
61431
61647
|
});
|
|
61432
61648
|
|
|
61649
|
+
// dist/builtin/workflows/src/workers/coordinator-protocol.ts
|
|
61650
|
+
var POOL_COORDINATOR_ROLE = "pool-coordinator", POOL_SUMMARY_INTERVAL_MS = 250;
|
|
61651
|
+
|
|
61652
|
+
// dist/builtin/workflows/src/workers/host-options.ts
|
|
61653
|
+
var hostWorkerBindings;
|
|
61654
|
+
var init_host_options = __esm(() => {
|
|
61655
|
+
hostWorkerBindings = Symbol("host-worker-bindings");
|
|
61656
|
+
});
|
|
61657
|
+
|
|
61433
61658
|
// dist/builtin/workflows/src/durable/journal/sql-pool-usage.ts
|
|
61434
61659
|
function validate(report) {
|
|
61435
61660
|
validateText("key", report.key);
|
|
@@ -61500,10 +61725,870 @@ var init_sql_pool_usage = __esm(() => {
|
|
|
61500
61725
|
init_values();
|
|
61501
61726
|
});
|
|
61502
61727
|
|
|
61503
|
-
// dist/builtin/workflows/src/workers/
|
|
61504
|
-
|
|
61505
|
-
|
|
61506
|
-
|
|
61728
|
+
// dist/builtin/workflows/src/workers/pool-coordinator.ts
|
|
61729
|
+
import { fork } from "node:child_process";
|
|
61730
|
+
import { randomUUID as randomUUID16 } from "node:crypto";
|
|
61731
|
+
import { createWriteStream } from "node:fs";
|
|
61732
|
+
import { mkdir as mkdir6, rm as rm3, rmdir } from "node:fs/promises";
|
|
61733
|
+
import { dirname as dirname12, join as join28, resolve as resolve11 } from "node:path";
|
|
61734
|
+
import { setTimeout as delay2 } from "node:timers/promises";
|
|
61735
|
+
function workerLogDirectory(journalPath, root) {
|
|
61736
|
+
return join28(dirname12(resolve11(journalPath)), "worker-logs", encodeURIComponent(root).replaceAll(".", "%2E"));
|
|
61737
|
+
}
|
|
61738
|
+
function token(work) {
|
|
61739
|
+
return {
|
|
61740
|
+
id: work.id,
|
|
61741
|
+
generation: work.generation,
|
|
61742
|
+
worker: work.worker,
|
|
61743
|
+
epoch: work.epoch,
|
|
61744
|
+
renewedAt: work.renewedAt,
|
|
61745
|
+
expiresAt: work.expiresAt
|
|
61746
|
+
};
|
|
61747
|
+
}
|
|
61748
|
+
function outside(runId) {
|
|
61749
|
+
return new Error(`Pool stage publication is outside the work subtree: ${runId}`);
|
|
61750
|
+
}
|
|
61751
|
+
function isLiveStage(stage) {
|
|
61752
|
+
return (stage.sessionId !== undefined || stage.sessionFile !== undefined) && stage.status !== "completed" && stage.status !== "failed" && stage.status !== "skipped";
|
|
61753
|
+
}
|
|
61754
|
+
function projectRootRun(run) {
|
|
61755
|
+
return {
|
|
61756
|
+
id: run.id,
|
|
61757
|
+
name: run.name,
|
|
61758
|
+
status: run.status,
|
|
61759
|
+
startedAt: run.startedAt,
|
|
61760
|
+
stages: run.stages.map((stage) => ({
|
|
61761
|
+
id: stage.id,
|
|
61762
|
+
name: stage.name,
|
|
61763
|
+
status: stage.status,
|
|
61764
|
+
parentIds: [],
|
|
61765
|
+
toolEvents: [],
|
|
61766
|
+
...stage.replayKey === undefined ? {} : { replayKey: stage.replayKey },
|
|
61767
|
+
...stage.intercomGroup === undefined ? {} : { intercomGroup: stage.intercomGroup },
|
|
61768
|
+
...stage.workflowChildRun === undefined ? {} : { workflowChildRun: stage.workflowChildRun },
|
|
61769
|
+
...stage.workflowChild === undefined ? {} : { workflowChild: { ...stage.workflowChild, outputs: {} } }
|
|
61770
|
+
}))
|
|
61771
|
+
};
|
|
61772
|
+
}
|
|
61773
|
+
|
|
61774
|
+
class PooledRoutes {
|
|
61775
|
+
root;
|
|
61776
|
+
owner;
|
|
61777
|
+
mirror = createStore();
|
|
61778
|
+
owners = new Map;
|
|
61779
|
+
attempts = new Map;
|
|
61780
|
+
activated = false;
|
|
61781
|
+
bridge;
|
|
61782
|
+
constructor(root, owner) {
|
|
61783
|
+
this.root = root;
|
|
61784
|
+
this.owner = owner;
|
|
61785
|
+
}
|
|
61786
|
+
static async open(root, cwd, rootRun, forward) {
|
|
61787
|
+
await Promise.resolve().then(() => init_workflow_route_owner());
|
|
61788
|
+
const owner = createWorkflowRouteOwner(cwd);
|
|
61789
|
+
const routes = new PooledRoutes(root, owner);
|
|
61790
|
+
if (rootRun !== undefined)
|
|
61791
|
+
routes.updateRoot(rootRun);
|
|
61792
|
+
routes.bridge = registerPendingStageIntercomBridge({
|
|
61793
|
+
events: {
|
|
61794
|
+
emit: (event, payload) => owner.events.emit(event, payload),
|
|
61795
|
+
on: (event, listener) => event === PENDING_STAGE_MESSAGE_EVENT2 || event === STICKY_LIVE_DELIVERED_EVENT2 ? undefined : owner.events.on(event, listener)
|
|
61796
|
+
}
|
|
61797
|
+
}, routes.mirror, (runId) => routes.activated && routes.owners.has(runId));
|
|
61798
|
+
for (const [event, kind] of [
|
|
61799
|
+
[PENDING_STAGE_MESSAGE_EVENT2, "pending-stage-message"],
|
|
61800
|
+
[STICKY_LIVE_DELIVERED_EVENT2, "sticky-live-delivered"]
|
|
61801
|
+
])
|
|
61802
|
+
owner.events.on(event, (payload) => {
|
|
61803
|
+
const claimed = payload;
|
|
61804
|
+
if (claimed.handled !== false)
|
|
61805
|
+
return;
|
|
61806
|
+
claimed.handled = true;
|
|
61807
|
+
const { handled: _handled, completion: _completion, ...forwarded } = claimed;
|
|
61808
|
+
claimed.completion = forward({ kind, payload: forwarded });
|
|
61809
|
+
});
|
|
61810
|
+
return routes;
|
|
61811
|
+
}
|
|
61812
|
+
updateRoot(rootRun) {
|
|
61813
|
+
this.mirror.recordRunStart({ ...rootRun, inputs: {}, toolNodes: [], rootRunId: this.root });
|
|
61814
|
+
}
|
|
61815
|
+
accept(entries) {
|
|
61816
|
+
this.mirror.batch(() => {
|
|
61817
|
+
for (const { work, runs } of entries) {
|
|
61818
|
+
let attempt = this.attempts.get(work.id);
|
|
61819
|
+
if (attempt !== undefined && work.generation < attempt.generation)
|
|
61820
|
+
continue;
|
|
61821
|
+
if (attempt === undefined || attempt.generation !== work.generation) {
|
|
61822
|
+
if (attempt !== undefined)
|
|
61823
|
+
this.release(attempt);
|
|
61824
|
+
attempt = { generation: work.generation, runs: attempt?.runs ?? new Set, fenced: false };
|
|
61825
|
+
this.attempts.set(work.id, attempt);
|
|
61826
|
+
}
|
|
61827
|
+
if (attempt.fenced)
|
|
61828
|
+
continue;
|
|
61829
|
+
for (const run of runs) {
|
|
61830
|
+
attempt.runs.add(run.id);
|
|
61831
|
+
this.owners.set(run.id, work.id);
|
|
61832
|
+
const { pendingStageMessages: _messages, ...published } = run;
|
|
61833
|
+
this.mirror.recordRunStart({ ...published, rootRunId: this.root });
|
|
61834
|
+
if (run.stages.some((stage) => stage.pendingStageDeliveryAvailable === true))
|
|
61835
|
+
this.activated = true;
|
|
61836
|
+
}
|
|
61837
|
+
}
|
|
61838
|
+
});
|
|
61839
|
+
}
|
|
61840
|
+
fence(work) {
|
|
61841
|
+
const attempt = this.attempts.get(work.id);
|
|
61842
|
+
if (attempt === undefined || attempt.generation > work.generation)
|
|
61843
|
+
return;
|
|
61844
|
+
if (attempt.generation < work.generation) {
|
|
61845
|
+
this.release(attempt);
|
|
61846
|
+
this.attempts.set(work.id, { generation: work.generation, runs: attempt.runs, fenced: true });
|
|
61847
|
+
return;
|
|
61848
|
+
}
|
|
61849
|
+
this.release(attempt);
|
|
61850
|
+
attempt.fenced = true;
|
|
61851
|
+
}
|
|
61852
|
+
async route(work, runId) {
|
|
61853
|
+
this.requireOwned(work, runId);
|
|
61854
|
+
const readiness = workflowPendingStageRouteReady(this.mirror, runId);
|
|
61855
|
+
await readiness?.completion.catch(() => {});
|
|
61856
|
+
readiness?.assertCurrent();
|
|
61857
|
+
const waiting = new AbortController;
|
|
61858
|
+
await Promise.race([
|
|
61859
|
+
this.owner.registered(runId),
|
|
61860
|
+
delay2(ROUTE_REGISTRATION_WAIT_MS, undefined, { signal: waiting.signal, ref: false }).catch(() => {})
|
|
61861
|
+
]);
|
|
61862
|
+
waiting.abort();
|
|
61863
|
+
this.requireOwned(work, runId);
|
|
61864
|
+
return workflowPendingStageRouteCapability(this.mirror, runId);
|
|
61865
|
+
}
|
|
61866
|
+
async undeliverable(payload) {
|
|
61867
|
+
const event = { ...payload, handled: false };
|
|
61868
|
+
this.owner.events.emit(PENDING_STAGE_UNDELIVERABLE_EVENT2, event);
|
|
61869
|
+
return event.handled && await event.completion === true;
|
|
61870
|
+
}
|
|
61871
|
+
async close() {
|
|
61872
|
+
const bridge = this.bridge;
|
|
61873
|
+
if (bridge === undefined)
|
|
61874
|
+
return;
|
|
61875
|
+
this.bridge = undefined;
|
|
61876
|
+
try {
|
|
61877
|
+
await bridge.flush();
|
|
61878
|
+
} finally {
|
|
61879
|
+
bridge.refresh();
|
|
61880
|
+
bridge();
|
|
61881
|
+
await this.owner.closeGroup(workflowInvocationIntercomGroup(this.root));
|
|
61882
|
+
}
|
|
61883
|
+
}
|
|
61884
|
+
requireOwned(work, runId) {
|
|
61885
|
+
const attempt = this.attempts.get(work.id);
|
|
61886
|
+
if (attempt === undefined || attempt.generation !== work.generation || attempt.fenced)
|
|
61887
|
+
throw new Error(STALE_WORK_LEASE);
|
|
61888
|
+
if (!attempt.runs.has(runId))
|
|
61889
|
+
throw outside(runId);
|
|
61890
|
+
}
|
|
61891
|
+
release(attempt) {
|
|
61892
|
+
this.mirror.batch(() => {
|
|
61893
|
+
for (const id of attempt.runs) {
|
|
61894
|
+
rotateWorkflowPendingStageRouteCapability(this.mirror, id);
|
|
61895
|
+
const run = this.mirror.runs().find((candidate) => candidate.id === id);
|
|
61896
|
+
if (run?.stages.some(isLiveStage) !== true)
|
|
61897
|
+
continue;
|
|
61898
|
+
this.mirror.recordRunStart({
|
|
61899
|
+
...run,
|
|
61900
|
+
stages: run.stages.map((stage) => {
|
|
61901
|
+
if (!isLiveStage(stage))
|
|
61902
|
+
return stage;
|
|
61903
|
+
const { sessionId: _session, sessionFile: _file, ...rest } = stage;
|
|
61904
|
+
return { ...rest, attachable: false, attached: false };
|
|
61905
|
+
})
|
|
61906
|
+
});
|
|
61907
|
+
}
|
|
61908
|
+
});
|
|
61909
|
+
}
|
|
61910
|
+
}
|
|
61911
|
+
|
|
61912
|
+
class PoolCoordinator {
|
|
61913
|
+
port;
|
|
61914
|
+
children = new Set;
|
|
61915
|
+
incarnations = new Map;
|
|
61916
|
+
readyWorkers = new Set;
|
|
61917
|
+
logs = new Map;
|
|
61918
|
+
cleanups = new Set;
|
|
61919
|
+
controlAcks = new Map;
|
|
61920
|
+
lastHeard = new Map;
|
|
61921
|
+
pendingActivity = new Map;
|
|
61922
|
+
batches = new Map;
|
|
61923
|
+
publicationFailures = new Map;
|
|
61924
|
+
requests = new Map;
|
|
61925
|
+
deliveryRequests = new Map;
|
|
61926
|
+
inflight = new Set;
|
|
61927
|
+
lastTick = performance.now();
|
|
61928
|
+
stalledMs = 0;
|
|
61929
|
+
ticker;
|
|
61930
|
+
recoveryTimer;
|
|
61931
|
+
flushTimer;
|
|
61932
|
+
lastFlushAt = 0;
|
|
61933
|
+
batchSequence = 0;
|
|
61934
|
+
requestSequence = 0;
|
|
61935
|
+
usageChanged = false;
|
|
61936
|
+
budgetBlocked = false;
|
|
61937
|
+
desiredControl;
|
|
61938
|
+
root;
|
|
61939
|
+
lease;
|
|
61940
|
+
options;
|
|
61941
|
+
journal;
|
|
61942
|
+
queue;
|
|
61943
|
+
usageLedger;
|
|
61944
|
+
usageTotal;
|
|
61945
|
+
admission;
|
|
61946
|
+
routes;
|
|
61947
|
+
publishing = false;
|
|
61948
|
+
sources = [];
|
|
61949
|
+
logDirectory;
|
|
61950
|
+
starting;
|
|
61951
|
+
recovering;
|
|
61952
|
+
nextRecoveryAt = 0;
|
|
61953
|
+
cleanupFailure;
|
|
61954
|
+
closed = false;
|
|
61955
|
+
closing;
|
|
61956
|
+
replacements = 0;
|
|
61957
|
+
constructor(port) {
|
|
61958
|
+
this.port = port;
|
|
61959
|
+
port.onMessage((message) => this.receive(message));
|
|
61960
|
+
this.ticker = setInterval(() => this.observeResponsiveness(), TICK_INTERVAL_MS).unref();
|
|
61961
|
+
}
|
|
61962
|
+
async shutdown() {
|
|
61963
|
+
await this.closing?.catch(() => {});
|
|
61964
|
+
while (this.inflight.size > 0)
|
|
61965
|
+
await Promise.allSettled([...this.inflight]);
|
|
61966
|
+
await this.journal?.close().catch(() => {});
|
|
61967
|
+
}
|
|
61968
|
+
receive(message) {
|
|
61969
|
+
if (message.kind === "request-reply") {
|
|
61970
|
+
const pending = this.requests.get(message.id);
|
|
61971
|
+
this.requests.delete(message.id);
|
|
61972
|
+
if (message.error !== undefined)
|
|
61973
|
+
pending?.reject(new Error(message.error));
|
|
61974
|
+
else
|
|
61975
|
+
pending?.resolve(message.value);
|
|
61976
|
+
return;
|
|
61977
|
+
}
|
|
61978
|
+
if ("id" in message) {
|
|
61979
|
+
const command = message;
|
|
61980
|
+
const handled = this.execute(command).then((value2) => this.port.send({ kind: "reply", id: command.id, value: value2 }), (error) => this.port.send({ kind: "reply", id: command.id, error: error.message }));
|
|
61981
|
+
this.inflight.add(handled);
|
|
61982
|
+
handled.finally(() => this.inflight.delete(handled));
|
|
61983
|
+
return;
|
|
61984
|
+
}
|
|
61985
|
+
this.notice(message);
|
|
61986
|
+
}
|
|
61987
|
+
notice(notice) {
|
|
61988
|
+
if (notice.kind === "budget")
|
|
61989
|
+
this.budgetBlocked = notice.blocked;
|
|
61990
|
+
else if (notice.kind === "root")
|
|
61991
|
+
this.routes?.updateRoot(notice.rootRun);
|
|
61992
|
+
else {
|
|
61993
|
+
const batch = this.batches.get(notice.batch);
|
|
61994
|
+
this.batches.delete(notice.batch);
|
|
61995
|
+
const rejected = new Map(notice.rejected.map((entry) => [entry.workId, entry]));
|
|
61996
|
+
for (const entry of rejected.values())
|
|
61997
|
+
this.publicationFailures.set(entry.workId, { generation: entry.generation, error: entry.error });
|
|
61998
|
+
this.routes?.accept((batch?.entries ?? []).filter((entry) => rejected.get(entry.work.id)?.generation !== entry.work.generation));
|
|
61999
|
+
batch?.settle();
|
|
62000
|
+
}
|
|
62001
|
+
}
|
|
62002
|
+
async execute(command) {
|
|
62003
|
+
switch (command.kind) {
|
|
62004
|
+
case "prepare":
|
|
62005
|
+
return this.prepare(command.root, command.lease, command.options, command.control);
|
|
62006
|
+
case "control":
|
|
62007
|
+
return this.control(command.state);
|
|
62008
|
+
case "start":
|
|
62009
|
+
this.starting ??= this.start(command.sources, command.cwd, command.publish, command.rootRun);
|
|
62010
|
+
return this.starting;
|
|
62011
|
+
case "enqueue":
|
|
62012
|
+
return this.enqueue(command);
|
|
62013
|
+
case "transport":
|
|
62014
|
+
return this.transport(command.request, command);
|
|
62015
|
+
case "undeliverable":
|
|
62016
|
+
return await this.routes?.undeliverable(command.payload) ?? false;
|
|
62017
|
+
case "close":
|
|
62018
|
+
this.closing ??= this.close();
|
|
62019
|
+
return this.closing;
|
|
62020
|
+
}
|
|
62021
|
+
}
|
|
62022
|
+
async prepare(root, lease, options, control) {
|
|
62023
|
+
if (this.closed)
|
|
62024
|
+
throw new Error("Local worker pool is closed");
|
|
62025
|
+
if (this.root !== undefined && this.root !== root)
|
|
62026
|
+
throw new Error("Local worker pool belongs to another root");
|
|
62027
|
+
this.root = root;
|
|
62028
|
+
this.options = options;
|
|
62029
|
+
this.desiredControl ??= control;
|
|
62030
|
+
this.journal ??= await resolveRunJournal({ journal: { path: options.journalPath } });
|
|
62031
|
+
this.usageLedger ??= await openSqlPoolUsage(this.journal);
|
|
62032
|
+
this.usageTotal = await this.usageLedger.total(root);
|
|
62033
|
+
this.lease = lease;
|
|
62034
|
+
const queue = await openSqlPoolQueue(this.journal);
|
|
62035
|
+
await queue.initialize(lease, COORDINATOR_LEASE_MS);
|
|
62036
|
+
await queue.releasePreviousIncarnations(lease);
|
|
62037
|
+
for (;; ) {
|
|
62038
|
+
const intended = this.desiredControl ?? "running";
|
|
62039
|
+
await queue.setControl(lease, intended);
|
|
62040
|
+
if (intended === (this.desiredControl ?? "running"))
|
|
62041
|
+
break;
|
|
62042
|
+
}
|
|
62043
|
+
this.queue = queue;
|
|
62044
|
+
return this.usageTotal;
|
|
62045
|
+
}
|
|
62046
|
+
async control(state) {
|
|
62047
|
+
this.desiredControl = state;
|
|
62048
|
+
if (!this.queue || !this.lease)
|
|
62049
|
+
return;
|
|
62050
|
+
const control = await this.queue.setControl(this.lease, state);
|
|
62051
|
+
const deadline = Date.now() + CONTROL_ACKNOWLEDGEMENT_TIMEOUT_MS;
|
|
62052
|
+
while ([...this.readyWorkers].some((worker) => (this.controlAcks.get(worker) ?? -1) < control.revision)) {
|
|
62053
|
+
if (this.desiredControl !== state)
|
|
62054
|
+
return;
|
|
62055
|
+
if (Date.now() >= deadline) {
|
|
62056
|
+
if (state === "quit")
|
|
62057
|
+
break;
|
|
62058
|
+
throw new Error("Worker control acknowledgement unresolved");
|
|
62059
|
+
}
|
|
62060
|
+
await delay2(25);
|
|
62061
|
+
}
|
|
62062
|
+
this.flush();
|
|
62063
|
+
}
|
|
62064
|
+
async start(sources, cwd, publish, rootRun) {
|
|
62065
|
+
if (this.closed)
|
|
62066
|
+
throw new Error("Local worker pool is closed");
|
|
62067
|
+
const root = this.root;
|
|
62068
|
+
if (!this.queue || !this.lease || !this.options || root === undefined || this.lease.runId !== root)
|
|
62069
|
+
throw new Error("Local worker pool requires root preparation");
|
|
62070
|
+
this.sources = sources;
|
|
62071
|
+
this.publishing = publish;
|
|
62072
|
+
if (publish) {
|
|
62073
|
+
this.routes = await PooledRoutes.open(root, cwd, rootRun, (request) => this.request(request));
|
|
62074
|
+
if (this.closed) {
|
|
62075
|
+
await this.routes.close();
|
|
62076
|
+
throw new Error("Local worker pool is closed");
|
|
62077
|
+
}
|
|
62078
|
+
}
|
|
62079
|
+
if (this.options.admission)
|
|
62080
|
+
this.admission = await SqlResourceAdmission.open(this.journal, {
|
|
62081
|
+
...this.options.admission,
|
|
62082
|
+
root,
|
|
62083
|
+
holder: "coordinator"
|
|
62084
|
+
});
|
|
62085
|
+
this.logDirectory = workerLogDirectory(this.options.journalPath, root);
|
|
62086
|
+
await mkdir6(this.logDirectory, { recursive: true }).catch(() => {
|
|
62087
|
+
return;
|
|
62088
|
+
});
|
|
62089
|
+
this.recoveryTimer = setInterval(() => {
|
|
62090
|
+
this.recoverExpired().catch((error) => this.fail(error));
|
|
62091
|
+
}, RECOVERY_INTERVAL_MS).unref();
|
|
62092
|
+
const workers = Array.from({ length: this.options.workers }, () => this.spawn());
|
|
62093
|
+
try {
|
|
62094
|
+
await Promise.any(workers);
|
|
62095
|
+
} catch (error) {
|
|
62096
|
+
throw error.errors[0];
|
|
62097
|
+
}
|
|
62098
|
+
}
|
|
62099
|
+
async enqueue(command) {
|
|
62100
|
+
if (!this.queue || !this.lease || this.root === undefined)
|
|
62101
|
+
throw new Error("Local worker pool requires root preparation");
|
|
62102
|
+
if (command.rootRun !== undefined)
|
|
62103
|
+
this.routes?.updateRoot(command.rootRun);
|
|
62104
|
+
const work = await this.queue.enqueue(this.lease, command.descriptor, command.boundary);
|
|
62105
|
+
const result = await this.queue.result(this.root, work.id);
|
|
62106
|
+
return { work, ...result === undefined ? {} : { result } };
|
|
62107
|
+
}
|
|
62108
|
+
async transport(request, payload) {
|
|
62109
|
+
const target = this.deliveryRequests.get(request);
|
|
62110
|
+
if (target === undefined || !this.incarnations.has(target.slot.worker))
|
|
62111
|
+
throw new Error("Worker exited before delivery acknowledgement");
|
|
62112
|
+
const { slot, workerRequest } = target;
|
|
62113
|
+
await new Promise((resolve, reject) => {
|
|
62114
|
+
const id = randomUUID16();
|
|
62115
|
+
slot.deliveries.set(id, { resolve, reject });
|
|
62116
|
+
slot.child.send({ id: workerRequest, delivery: { id, from: payload.from, message: payload.message } }, (error) => {
|
|
62117
|
+
if (error) {
|
|
62118
|
+
slot.deliveries.delete(id);
|
|
62119
|
+
reject(error);
|
|
62120
|
+
}
|
|
62121
|
+
});
|
|
62122
|
+
});
|
|
62123
|
+
}
|
|
62124
|
+
request(body) {
|
|
62125
|
+
const id = ++this.requestSequence;
|
|
62126
|
+
return new Promise((resolve, reject) => {
|
|
62127
|
+
this.requests.set(id, { resolve, reject });
|
|
62128
|
+
this.port.send({ kind: "request", id, request: body });
|
|
62129
|
+
});
|
|
62130
|
+
}
|
|
62131
|
+
fail(error) {
|
|
62132
|
+
this.port.send({ kind: "failure", error: error.message });
|
|
62133
|
+
}
|
|
62134
|
+
publish(work, runs) {
|
|
62135
|
+
const failure = this.publicationFailures.get(work.id);
|
|
62136
|
+
if (failure?.generation === work.generation)
|
|
62137
|
+
throw new Error(failure.error);
|
|
62138
|
+
if (!this.publishing)
|
|
62139
|
+
return;
|
|
62140
|
+
this.pendingActivity.set(work.id, { work: token(work), runs });
|
|
62141
|
+
this.scheduleFlush();
|
|
62142
|
+
}
|
|
62143
|
+
scheduleFlush() {
|
|
62144
|
+
if (this.flushTimer !== undefined || this.closed)
|
|
62145
|
+
return;
|
|
62146
|
+
const wait = Math.max(0, this.lastFlushAt + POOL_SUMMARY_INTERVAL_MS - performance.now());
|
|
62147
|
+
this.flushTimer = setTimeout(() => {
|
|
62148
|
+
this.flushTimer = undefined;
|
|
62149
|
+
if (this.batches.size > 0) {
|
|
62150
|
+
this.lastFlushAt = performance.now();
|
|
62151
|
+
this.scheduleFlush();
|
|
62152
|
+
return;
|
|
62153
|
+
}
|
|
62154
|
+
this.flush();
|
|
62155
|
+
}, wait);
|
|
62156
|
+
}
|
|
62157
|
+
flush() {
|
|
62158
|
+
clearTimeout(this.flushTimer);
|
|
62159
|
+
this.flushTimer = undefined;
|
|
62160
|
+
if (this.pendingActivity.size === 0 && !this.usageChanged) {
|
|
62161
|
+
const last = [...this.batches.values()].at(-1);
|
|
62162
|
+
return last === undefined ? Promise.resolve() : new Promise((resolve) => this.chainSettle(last, resolve));
|
|
62163
|
+
}
|
|
62164
|
+
this.lastFlushAt = performance.now();
|
|
62165
|
+
const batch = ++this.batchSequence;
|
|
62166
|
+
const entries = [...this.pendingActivity.values()];
|
|
62167
|
+
this.pendingActivity.clear();
|
|
62168
|
+
const usage = this.usageChanged ? this.usageTotal : undefined;
|
|
62169
|
+
this.usageChanged = false;
|
|
62170
|
+
const settled = new Promise((resolve) => this.batches.set(batch, { entries, settle: resolve }));
|
|
62171
|
+
this.port.send({ kind: "activity", batch, entries, ...usage === undefined ? {} : { usage } });
|
|
62172
|
+
return settled;
|
|
62173
|
+
}
|
|
62174
|
+
chainSettle(batch, resolve) {
|
|
62175
|
+
const settle = batch.settle;
|
|
62176
|
+
batch.settle = () => {
|
|
62177
|
+
settle();
|
|
62178
|
+
resolve();
|
|
62179
|
+
};
|
|
62180
|
+
}
|
|
62181
|
+
noteUsage(usage) {
|
|
62182
|
+
this.usageTotal = usage;
|
|
62183
|
+
this.usageChanged = true;
|
|
62184
|
+
this.scheduleFlush();
|
|
62185
|
+
}
|
|
62186
|
+
async sendResult(workId) {
|
|
62187
|
+
const result = await this.queue.result(this.root, workId);
|
|
62188
|
+
if (result === undefined)
|
|
62189
|
+
return false;
|
|
62190
|
+
this.flush();
|
|
62191
|
+
this.port.send({
|
|
62192
|
+
kind: "result",
|
|
62193
|
+
workId,
|
|
62194
|
+
result,
|
|
62195
|
+
...this.usageTotal === undefined ? {} : { usage: this.usageTotal }
|
|
62196
|
+
});
|
|
62197
|
+
return true;
|
|
62198
|
+
}
|
|
62199
|
+
fence(work) {
|
|
62200
|
+
this.pendingActivity.delete(work.id);
|
|
62201
|
+
this.routes?.fence(work);
|
|
62202
|
+
this.port.send({ kind: "fence", work: { id: work.id, generation: work.generation } });
|
|
62203
|
+
}
|
|
62204
|
+
spawn() {
|
|
62205
|
+
if (this.closed)
|
|
62206
|
+
return Promise.reject(new Error("Local worker pool is closed"));
|
|
62207
|
+
const root = this.root;
|
|
62208
|
+
const options = this.options;
|
|
62209
|
+
const child = fork(options.workerEntry ?? process.argv[1], [
|
|
62210
|
+
JSON.stringify({
|
|
62211
|
+
journalPath: options.journalPath,
|
|
62212
|
+
root,
|
|
62213
|
+
definitions: this.sources,
|
|
62214
|
+
admission: options.admission
|
|
62215
|
+
})
|
|
62216
|
+
], { stdio: ["ignore", "ignore", "pipe", "ipc"], execArgv: [] });
|
|
62217
|
+
const disconnected = new Promise((resolve) => child.once("disconnect", () => resolve()));
|
|
62218
|
+
this.children.add(child);
|
|
62219
|
+
if (child.pid !== undefined)
|
|
62220
|
+
this.port.send({ kind: "spawned", pid: child.pid });
|
|
62221
|
+
const worker = randomUUID16();
|
|
62222
|
+
this.incarnations.set(worker, child);
|
|
62223
|
+
const slot = { worker, child, deliveries: new Map };
|
|
62224
|
+
let ready = false;
|
|
62225
|
+
let serial = Promise.resolve();
|
|
62226
|
+
const logPath = join28(this.logDirectory, `${worker}.log`);
|
|
62227
|
+
this.logs.set(child, logPath);
|
|
62228
|
+
let logFailure;
|
|
62229
|
+
const log = createWriteStream(logPath, { flags: "a" });
|
|
62230
|
+
log.on("error", (error) => {
|
|
62231
|
+
logFailure = error.message;
|
|
62232
|
+
});
|
|
62233
|
+
let stderr = "";
|
|
62234
|
+
child.stderr?.on("data", (chunk) => {
|
|
62235
|
+
stderr = (stderr + chunk.toString()).slice(-STDERR_TAIL_CHARACTERS);
|
|
62236
|
+
});
|
|
62237
|
+
child.stderr?.on("error", (error) => {
|
|
62238
|
+
logFailure ??= error.message;
|
|
62239
|
+
log.end();
|
|
62240
|
+
});
|
|
62241
|
+
child.stderr?.pipe(log);
|
|
62242
|
+
const diagnosed = (summary) => new Error(`${summary} (worker stderr log: ${logFailure === undefined ? logPath : `unavailable, ${logFailure}`})${stderr ? `: ${stderr}` : ""}`);
|
|
62243
|
+
const startupTimeoutMs = options.startupTimeoutMs;
|
|
62244
|
+
let timedOut = false;
|
|
62245
|
+
let terminated = false;
|
|
62246
|
+
const terminate = () => {
|
|
62247
|
+
if (terminated)
|
|
62248
|
+
return;
|
|
62249
|
+
terminated = true;
|
|
62250
|
+
child.kill("SIGKILL");
|
|
62251
|
+
};
|
|
62252
|
+
const replied = (error) => {
|
|
62253
|
+
if (error)
|
|
62254
|
+
terminate();
|
|
62255
|
+
};
|
|
62256
|
+
return new Promise((resolve, reject) => {
|
|
62257
|
+
const startup = setTimeout(() => {
|
|
62258
|
+
timedOut = true;
|
|
62259
|
+
terminate();
|
|
62260
|
+
}, startupTimeoutMs);
|
|
62261
|
+
child.on("error", (error) => {
|
|
62262
|
+
clearTimeout(startup);
|
|
62263
|
+
reject(error);
|
|
62264
|
+
terminate();
|
|
62265
|
+
});
|
|
62266
|
+
const closed = new Promise((resolve) => child.once("close", () => resolve()));
|
|
62267
|
+
child.once("exit", () => {
|
|
62268
|
+
Promise.race([closed, delay2(STDERR_DRAIN_GRACE_MS)]).then(() => reject(diagnosed(timedOut ? `Local worker startup timed out after ${startupTimeoutMs} ms` : "Local worker exited before readiness")));
|
|
62269
|
+
this.children.delete(child);
|
|
62270
|
+
this.logs.delete(child);
|
|
62271
|
+
if (child.pid !== undefined)
|
|
62272
|
+
this.port.send({ kind: "exited", pid: child.pid });
|
|
62273
|
+
clearTimeout(startup);
|
|
62274
|
+
this.incarnations.delete(worker);
|
|
62275
|
+
this.readyWorkers.delete(worker);
|
|
62276
|
+
this.controlAcks.delete(worker);
|
|
62277
|
+
this.lastHeard.delete(worker);
|
|
62278
|
+
if (slot.work)
|
|
62279
|
+
this.fence(slot.work);
|
|
62280
|
+
for (const delivery of slot.deliveries.values())
|
|
62281
|
+
delivery.reject(new Error("Worker exited before delivery acknowledgement"));
|
|
62282
|
+
slot.deliveries.clear();
|
|
62283
|
+
const cleanup = disconnected.then(() => serial).then(async () => {
|
|
62284
|
+
try {
|
|
62285
|
+
if (await this.admission?.releaseStoppedSessions(worker))
|
|
62286
|
+
throw new Error("Local worker process cleanup unconfirmed; permits quarantined");
|
|
62287
|
+
} catch (error) {
|
|
62288
|
+
this.cleanupFailure = error instanceof Error ? error : new Error(String(error));
|
|
62289
|
+
throw this.cleanupFailure;
|
|
62290
|
+
}
|
|
62291
|
+
}).then(async () => {
|
|
62292
|
+
if (this.closed || !ready || this.desiredControl === "quit")
|
|
62293
|
+
return;
|
|
62294
|
+
let work = slot.work;
|
|
62295
|
+
if (work && await this.sendResult(work.id)) {
|
|
62296
|
+
work = undefined;
|
|
62297
|
+
slot.work = undefined;
|
|
62298
|
+
this.replacements = 0;
|
|
62299
|
+
}
|
|
62300
|
+
if (this.closed)
|
|
62301
|
+
return;
|
|
62302
|
+
if (work)
|
|
62303
|
+
await this.queue.releaseExited(this.lease, work);
|
|
62304
|
+
const exhausted = work ? work.generation >= 3 : ++this.replacements > 3;
|
|
62305
|
+
if (exhausted)
|
|
62306
|
+
this.fail(diagnosed("Local worker retry limit"));
|
|
62307
|
+
else
|
|
62308
|
+
await this.spawn();
|
|
62309
|
+
}).catch((error) => {
|
|
62310
|
+
this.fail(error);
|
|
62311
|
+
});
|
|
62312
|
+
this.cleanups.add(cleanup);
|
|
62313
|
+
cleanup.finally(() => this.cleanups.delete(cleanup));
|
|
62314
|
+
});
|
|
62315
|
+
child.on("message", (message) => {
|
|
62316
|
+
this.observeResponsiveness();
|
|
62317
|
+
this.lastHeard.set(worker, { at: performance.now(), stalled: this.stalledMs });
|
|
62318
|
+
if (message.kind === "delivery-ack") {
|
|
62319
|
+
const pending = slot.deliveries.get(message.deliveryId);
|
|
62320
|
+
slot.deliveries.delete(message.deliveryId);
|
|
62321
|
+
if (message.error)
|
|
62322
|
+
pending?.reject(new Error(message.error));
|
|
62323
|
+
else
|
|
62324
|
+
pending?.resolve();
|
|
62325
|
+
return;
|
|
62326
|
+
}
|
|
62327
|
+
let routeReady;
|
|
62328
|
+
const handled = serial.then(async () => {
|
|
62329
|
+
if (message.kind === "ready") {
|
|
62330
|
+
if (terminated)
|
|
62331
|
+
return;
|
|
62332
|
+
ready = true;
|
|
62333
|
+
this.readyWorkers.add(worker);
|
|
62334
|
+
clearTimeout(startup);
|
|
62335
|
+
resolve();
|
|
62336
|
+
return;
|
|
62337
|
+
}
|
|
62338
|
+
await this.starting;
|
|
62339
|
+
const queue = this.queue;
|
|
62340
|
+
const lease = this.lease;
|
|
62341
|
+
if (message.kind === "control")
|
|
62342
|
+
return queue.control(lease);
|
|
62343
|
+
if (message.kind === "controlled") {
|
|
62344
|
+
const control = await queue.control(lease);
|
|
62345
|
+
if (message.revision > control.revision)
|
|
62346
|
+
throw new Error("Unknown control revision");
|
|
62347
|
+
this.controlAcks.set(worker, Math.max(this.controlAcks.get(worker) ?? -1, message.revision));
|
|
62348
|
+
return;
|
|
62349
|
+
}
|
|
62350
|
+
if (message.kind === "claim") {
|
|
62351
|
+
if (slot.work)
|
|
62352
|
+
throw new Error("Worker already holds an attempt");
|
|
62353
|
+
if (this.budgetBlocked)
|
|
62354
|
+
return;
|
|
62355
|
+
await this.recoverExpired();
|
|
62356
|
+
slot.work = await queue.claim(lease, worker, this.sources.map((source) => ({
|
|
62357
|
+
definition: `${source.path}#${source.exportName}`,
|
|
62358
|
+
digest: source.digest
|
|
62359
|
+
})), options.leaseMs);
|
|
62360
|
+
return slot.work;
|
|
62361
|
+
}
|
|
62362
|
+
let work = slot.work;
|
|
62363
|
+
if (!work)
|
|
62364
|
+
throw new Error("Worker has no active attempt");
|
|
62365
|
+
if (message.kind === "renew" || work.expiresAt - Date.now() < options.leaseMs / 2) {
|
|
62366
|
+
work = await queue.extend(lease, work, options.leaseMs);
|
|
62367
|
+
slot.work = work;
|
|
62368
|
+
}
|
|
62369
|
+
if (message.kind === "activity")
|
|
62370
|
+
return this.publish(work, message.runs);
|
|
62371
|
+
if (message.kind === "route") {
|
|
62372
|
+
if (this.routes === undefined)
|
|
62373
|
+
return;
|
|
62374
|
+
await this.flush();
|
|
62375
|
+
this.assertPublished(work);
|
|
62376
|
+
routeReady = this.routes.route(work, message.runId);
|
|
62377
|
+
return;
|
|
62378
|
+
}
|
|
62379
|
+
if (message.kind === "delivery-ready") {
|
|
62380
|
+
if (!this.publishing)
|
|
62381
|
+
return;
|
|
62382
|
+
await this.flush();
|
|
62383
|
+
this.assertPublished(work);
|
|
62384
|
+
return this.request({
|
|
62385
|
+
kind: "ready",
|
|
62386
|
+
work: token(work),
|
|
62387
|
+
runId: message.runId,
|
|
62388
|
+
stageId: message.stageId,
|
|
62389
|
+
stageName: message.stageName,
|
|
62390
|
+
...message.sessionId === undefined ? {} : { sessionId: message.sessionId }
|
|
62391
|
+
});
|
|
62392
|
+
}
|
|
62393
|
+
if (message.kind === "delivery") {
|
|
62394
|
+
if (!message.recipient)
|
|
62395
|
+
throw new Error("Worker delivery requires a session identity");
|
|
62396
|
+
if (!this.publishing)
|
|
62397
|
+
return;
|
|
62398
|
+
await this.flush();
|
|
62399
|
+
this.assertPublished(work);
|
|
62400
|
+
const id = ++this.requestSequence;
|
|
62401
|
+
this.deliveryRequests.set(id, { slot, workerRequest: message.id });
|
|
62402
|
+
try {
|
|
62403
|
+
return await new Promise((resolve, reject) => {
|
|
62404
|
+
this.requests.set(id, { resolve, reject });
|
|
62405
|
+
this.port.send({
|
|
62406
|
+
kind: "request",
|
|
62407
|
+
id,
|
|
62408
|
+
request: {
|
|
62409
|
+
kind: "deliver",
|
|
62410
|
+
work: token(work),
|
|
62411
|
+
runId: message.runId,
|
|
62412
|
+
stageId: message.stageId,
|
|
62413
|
+
stageName: message.stageName,
|
|
62414
|
+
recipient: message.recipient
|
|
62415
|
+
}
|
|
62416
|
+
});
|
|
62417
|
+
});
|
|
62418
|
+
} finally {
|
|
62419
|
+
this.deliveryRequests.delete(id);
|
|
62420
|
+
}
|
|
62421
|
+
}
|
|
62422
|
+
if (message.kind === "renew")
|
|
62423
|
+
return;
|
|
62424
|
+
if (message.kind === "commit") {
|
|
62425
|
+
const receipt = await queue.commit(lease, work, message.operation, message.mutation);
|
|
62426
|
+
this.noteUsage(await this.usageLedger.total(root));
|
|
62427
|
+
return receipt;
|
|
62428
|
+
}
|
|
62429
|
+
if (message.kind === "usage") {
|
|
62430
|
+
this.noteUsage(await this.usageLedger.report(lease, work, message.reports));
|
|
62431
|
+
return;
|
|
62432
|
+
}
|
|
62433
|
+
if (message.kind === "budget") {
|
|
62434
|
+
await queue.assertCurrent(lease, work);
|
|
62435
|
+
if (this.budgetBlocked)
|
|
62436
|
+
throw new Error("Root workflow budget exhausted");
|
|
62437
|
+
return queue.control(lease);
|
|
62438
|
+
}
|
|
62439
|
+
if (message.kind !== "complete")
|
|
62440
|
+
throw new Error("Invalid worker operation");
|
|
62441
|
+
await queue.complete(lease, work, message.result);
|
|
62442
|
+
this.replacements = 0;
|
|
62443
|
+
slot.work = undefined;
|
|
62444
|
+
await this.sendResult(work.id);
|
|
62445
|
+
});
|
|
62446
|
+
const response = handled.then((value2) => routeReady ?? value2);
|
|
62447
|
+
if (message.kind !== "delivery" && message.kind !== "delivery-ready")
|
|
62448
|
+
serial = handled.then(() => {}, () => {});
|
|
62449
|
+
response.then((value2) => child.send({ id: message.id, value: value2 }, replied), (error) => child.send({ id: message.id, error: error.message }, replied));
|
|
62450
|
+
});
|
|
62451
|
+
});
|
|
62452
|
+
}
|
|
62453
|
+
assertPublished(work) {
|
|
62454
|
+
const failure = this.publicationFailures.get(work.id);
|
|
62455
|
+
if (failure?.generation === work.generation)
|
|
62456
|
+
throw new Error(failure.error);
|
|
62457
|
+
}
|
|
62458
|
+
recoverExpired() {
|
|
62459
|
+
if (this.recovering)
|
|
62460
|
+
return this.recovering;
|
|
62461
|
+
const root = this.root;
|
|
62462
|
+
if (this.closed || root === undefined || !this.queue || performance.now() < this.nextRecoveryAt)
|
|
62463
|
+
return Promise.resolve();
|
|
62464
|
+
const queue = this.queue;
|
|
62465
|
+
const lease = this.lease;
|
|
62466
|
+
const leaseMs = this.options.leaseMs;
|
|
62467
|
+
this.recovering = (async () => {
|
|
62468
|
+
await new Promise((resolve) => setImmediate(resolve));
|
|
62469
|
+
await this.admission?.reconcile();
|
|
62470
|
+
const expiredBefore = Date.now();
|
|
62471
|
+
for (const work of await queue.inspect(root)) {
|
|
62472
|
+
if (work.state !== "leased" || work.expiresAt >= expiredBefore)
|
|
62473
|
+
continue;
|
|
62474
|
+
const child = work.worker === null ? undefined : this.incarnations.get(work.worker);
|
|
62475
|
+
if (child && work.worker !== null && this.isLive(work.worker, child)) {
|
|
62476
|
+
await queue.extend(lease, work, leaseMs).catch((error) => {
|
|
62477
|
+
if (error.message !== STALE_WORK_LEASE)
|
|
62478
|
+
throw error;
|
|
62479
|
+
});
|
|
62480
|
+
continue;
|
|
62481
|
+
}
|
|
62482
|
+
if (child && !await this.stop(child).then(() => true, () => false))
|
|
62483
|
+
continue;
|
|
62484
|
+
await queue.expire(lease, work);
|
|
62485
|
+
}
|
|
62486
|
+
})().finally(() => {
|
|
62487
|
+
this.recovering = undefined;
|
|
62488
|
+
this.nextRecoveryAt = performance.now() + RECOVERY_INTERVAL_MS;
|
|
62489
|
+
});
|
|
62490
|
+
return this.recovering;
|
|
62491
|
+
}
|
|
62492
|
+
observeResponsiveness() {
|
|
62493
|
+
const now = performance.now();
|
|
62494
|
+
const gap = now - this.lastTick;
|
|
62495
|
+
if (gap > STALL_THRESHOLD_MS)
|
|
62496
|
+
this.stalledMs += gap - TICK_INTERVAL_MS;
|
|
62497
|
+
this.lastTick = now;
|
|
62498
|
+
}
|
|
62499
|
+
isLive(worker, child) {
|
|
62500
|
+
this.observeResponsiveness();
|
|
62501
|
+
const heard = this.lastHeard.get(worker);
|
|
62502
|
+
return child.exitCode === null && child.signalCode === null && child.connected && heard !== undefined && performance.now() - heard.at - (this.stalledMs - heard.stalled) < this.options.leaseMs;
|
|
62503
|
+
}
|
|
62504
|
+
stop(child) {
|
|
62505
|
+
if (child.pid === undefined || child.exitCode !== null || child.signalCode !== null)
|
|
62506
|
+
return Promise.resolve();
|
|
62507
|
+
return new Promise((resolve, reject) => {
|
|
62508
|
+
const timeout = setTimeout(() => reject(new Error("Local worker cleanup was not confirmed")), 5000);
|
|
62509
|
+
child.once("exit", () => {
|
|
62510
|
+
clearTimeout(timeout);
|
|
62511
|
+
resolve();
|
|
62512
|
+
});
|
|
62513
|
+
child.kill("SIGKILL");
|
|
62514
|
+
});
|
|
62515
|
+
}
|
|
62516
|
+
async close() {
|
|
62517
|
+
this.closed = true;
|
|
62518
|
+
clearInterval(this.ticker);
|
|
62519
|
+
clearInterval(this.recoveryTimer);
|
|
62520
|
+
let failure;
|
|
62521
|
+
let stopping = [];
|
|
62522
|
+
try {
|
|
62523
|
+
await this.recovering?.catch(() => {});
|
|
62524
|
+
stopping = [...this.children].map((child) => ({ child, log: this.logs.get(child) }));
|
|
62525
|
+
await Promise.all(stopping.map(({ child }) => this.stop(child)));
|
|
62526
|
+
await Promise.all([...this.cleanups]);
|
|
62527
|
+
if (this.cleanupFailure)
|
|
62528
|
+
throw this.cleanupFailure;
|
|
62529
|
+
} catch (error) {
|
|
62530
|
+
failure = error;
|
|
62531
|
+
}
|
|
62532
|
+
if (failure === undefined) {
|
|
62533
|
+
const stopped = stopping.flatMap(({ child, log }) => log !== undefined && (child.exitCode !== null || child.signalCode !== null) ? [log] : []);
|
|
62534
|
+
await Promise.all(stopped.map((log) => rm3(log, { force: true }).catch(() => {})));
|
|
62535
|
+
if (this.logDirectory !== undefined)
|
|
62536
|
+
await rmdir(this.logDirectory).catch(() => {});
|
|
62537
|
+
}
|
|
62538
|
+
this.flush();
|
|
62539
|
+
try {
|
|
62540
|
+
await this.routes?.close();
|
|
62541
|
+
} catch (error) {
|
|
62542
|
+
failure ??= error;
|
|
62543
|
+
}
|
|
62544
|
+
for (const batch of this.batches.values())
|
|
62545
|
+
batch.settle();
|
|
62546
|
+
this.batches.clear();
|
|
62547
|
+
for (const pending of this.requests.values())
|
|
62548
|
+
pending.reject(new Error("Local worker pool is closed"));
|
|
62549
|
+
this.requests.clear();
|
|
62550
|
+
if (failure !== undefined)
|
|
62551
|
+
throw failure;
|
|
62552
|
+
}
|
|
62553
|
+
}
|
|
62554
|
+
function processPort() {
|
|
62555
|
+
return {
|
|
62556
|
+
send(message) {
|
|
62557
|
+
if (process.connected)
|
|
62558
|
+
process.send?.(message, () => {});
|
|
62559
|
+
},
|
|
62560
|
+
onMessage(listener) {
|
|
62561
|
+
process.on("message", (message) => listener(message));
|
|
62562
|
+
},
|
|
62563
|
+
onClose(listener) {
|
|
62564
|
+
process.once("disconnect", () => listener(new Error("Root host disconnected")));
|
|
62565
|
+
},
|
|
62566
|
+
async close() {
|
|
62567
|
+
if (process.connected)
|
|
62568
|
+
process.disconnect();
|
|
62569
|
+
}
|
|
62570
|
+
};
|
|
62571
|
+
}
|
|
62572
|
+
function runPoolCoordinator() {
|
|
62573
|
+
if (typeof process.send !== "function")
|
|
62574
|
+
throw new Error("Local worker pool coordinator requires an IPC channel");
|
|
62575
|
+
const port = processPort();
|
|
62576
|
+
const coordinator = new PoolCoordinator(port);
|
|
62577
|
+
port.onClose(() => process.exit(0));
|
|
62578
|
+
return coordinator;
|
|
62579
|
+
}
|
|
62580
|
+
var CONTROL_ACKNOWLEDGEMENT_TIMEOUT_MS = 5000, RECOVERY_INTERVAL_MS = 500, TICK_INTERVAL_MS = 100, STALL_THRESHOLD_MS = 500, COORDINATOR_LEASE_MS, STDERR_TAIL_CHARACTERS = 8192, STDERR_DRAIN_GRACE_MS = 1000, ROUTE_REGISTRATION_WAIT_MS = 120000, PENDING_STAGE_MESSAGE_EVENT2 = "atomic:workflow-pending-stage-message", STICKY_LIVE_DELIVERED_EVENT2 = "atomic:workflow-sticky-live-delivered", PENDING_STAGE_UNDELIVERABLE_EVENT2 = "atomic:workflow-pending-stage-undeliverable";
|
|
62581
|
+
var init_pool_coordinator = __esm(() => {
|
|
62582
|
+
init_resolution();
|
|
62583
|
+
init_sql_pool_queue();
|
|
62584
|
+
init_sql_pool_usage();
|
|
62585
|
+
init_pending_stage_intercom();
|
|
62586
|
+
init_intercom_group();
|
|
62587
|
+
init_pending_stage_route_capability();
|
|
62588
|
+
init_pending_stage_route_readiness();
|
|
62589
|
+
init_store();
|
|
62590
|
+
init_resource_admission();
|
|
62591
|
+
COORDINATOR_LEASE_MS = 3 * STAGE_SESSION_HEARTBEAT_INTERVAL_MS;
|
|
61507
62592
|
});
|
|
61508
62593
|
|
|
61509
62594
|
// dist/builtin/workflows/src/workers/protocol.ts
|
|
@@ -61753,21 +62838,26 @@ class PoolStageHandoff {
|
|
|
61753
62838
|
store;
|
|
61754
62839
|
queue;
|
|
61755
62840
|
lease;
|
|
62841
|
+
claimRun;
|
|
61756
62842
|
attempts = new Map;
|
|
61757
62843
|
owners = new Map;
|
|
61758
62844
|
publications = new Map;
|
|
61759
62845
|
drains = new Map;
|
|
61760
|
-
constructor(backend, store, queue, lease) {
|
|
62846
|
+
constructor(backend, store, queue, lease, claimRun) {
|
|
61761
62847
|
this.backend = backend;
|
|
61762
62848
|
this.store = store;
|
|
61763
62849
|
this.queue = queue;
|
|
61764
62850
|
this.lease = lease;
|
|
62851
|
+
this.claimRun = claimRun;
|
|
61765
62852
|
}
|
|
61766
62853
|
publish(work, runs) {
|
|
61767
62854
|
const previous = this.publications.get(work.id) ?? Promise.resolve();
|
|
61768
62855
|
const publication = previous.catch(() => {
|
|
61769
62856
|
return;
|
|
61770
|
-
}).then(() =>
|
|
62857
|
+
}).then(async () => {
|
|
62858
|
+
const commit = await this.prepare(work, runs);
|
|
62859
|
+
this.store.batch(commit);
|
|
62860
|
+
});
|
|
61771
62861
|
const settled = publication.catch(() => {
|
|
61772
62862
|
return;
|
|
61773
62863
|
});
|
|
@@ -61778,6 +62868,21 @@ class PoolStageHandoff {
|
|
|
61778
62868
|
});
|
|
61779
62869
|
return publication;
|
|
61780
62870
|
}
|
|
62871
|
+
async publishBatch(publications) {
|
|
62872
|
+
const prepared = [];
|
|
62873
|
+
for (const { work, runs } of publications)
|
|
62874
|
+
prepared.push(await this.prepare(work, runs).catch((error) => error));
|
|
62875
|
+
return this.store.batch(() => prepared.map((commit) => {
|
|
62876
|
+
if (commit instanceof Error)
|
|
62877
|
+
return commit;
|
|
62878
|
+
try {
|
|
62879
|
+
commit();
|
|
62880
|
+
return;
|
|
62881
|
+
} catch (error) {
|
|
62882
|
+
return error instanceof Error ? error : new Error(String(error));
|
|
62883
|
+
}
|
|
62884
|
+
}));
|
|
62885
|
+
}
|
|
61781
62886
|
async route(work, runId) {
|
|
61782
62887
|
this.requireOwned(await this.begin(work), runId);
|
|
61783
62888
|
const readiness = workflowPendingStageRouteReady(this.store, runId);
|
|
@@ -61828,7 +62933,7 @@ class PoolStageHandoff {
|
|
|
61828
62933
|
const result = await this.queue.result(root, work.id);
|
|
61829
62934
|
if (result === undefined)
|
|
61830
62935
|
return;
|
|
61831
|
-
await this.backend.
|
|
62936
|
+
await this.backend.catchUpWorkflow(root);
|
|
61832
62937
|
const terminals = new Map;
|
|
61833
62938
|
for (const checkpoint of this.backend.listCheckpoints(root)) {
|
|
61834
62939
|
const boundary = checkpoint.kind === "stage" ? checkpoint.topology?.boundary : undefined;
|
|
@@ -61838,6 +62943,9 @@ class PoolStageHandoff {
|
|
|
61838
62943
|
if (child?.runId === boundary.child.runId)
|
|
61839
62944
|
terminals.set(child.runId, { status: child.status, outputs: child.outputs });
|
|
61840
62945
|
}
|
|
62946
|
+
this.store.batch(() => this.settle(work, result, terminals));
|
|
62947
|
+
}
|
|
62948
|
+
settle(work, result, terminals) {
|
|
61841
62949
|
for (const run of this.store.runs()) {
|
|
61842
62950
|
if (run.endedAt !== undefined || this.owners.get(run.id) !== work.id)
|
|
61843
62951
|
continue;
|
|
@@ -61861,22 +62969,31 @@ class PoolStageHandoff {
|
|
|
61861
62969
|
if (current !== undefined && current.generation > work.generation)
|
|
61862
62970
|
return;
|
|
61863
62971
|
const attempt = current?.generation === work.generation ? current : this.start(work, current);
|
|
61864
|
-
this.release(attempt);
|
|
62972
|
+
this.store.batch(() => this.release(attempt));
|
|
61865
62973
|
attempt.fenced = true;
|
|
61866
62974
|
}
|
|
61867
|
-
async
|
|
62975
|
+
async prepare(work, runs) {
|
|
61868
62976
|
const subtree = this.subtree(work, runs);
|
|
61869
62977
|
const root = this.lease.runId;
|
|
61870
|
-
const
|
|
61871
|
-
|
|
61872
|
-
|
|
61873
|
-
|
|
61874
|
-
|
|
61875
|
-
|
|
61876
|
-
|
|
61877
|
-
|
|
61878
|
-
|
|
62978
|
+
const owned = (known) => {
|
|
62979
|
+
for (const run of subtree) {
|
|
62980
|
+
const existing = known.get(run.id);
|
|
62981
|
+
const owner = this.owners.get(run.id);
|
|
62982
|
+
if (owner !== undefined && owner !== work.id || owner === undefined && existing !== undefined && (existing.parentRunId !== run.parentRunId || existing.parentStageId !== run.parentStageId))
|
|
62983
|
+
throw outside2(run.id);
|
|
62984
|
+
}
|
|
62985
|
+
};
|
|
62986
|
+
const before = new Map(this.store.runs().map((run) => [run.id, run]));
|
|
62987
|
+
owned(before);
|
|
62988
|
+
if (subtree.some((run) => before.get(run.id) === undefined))
|
|
62989
|
+
await this.backend.catchUpWorkflow(root);
|
|
61879
62990
|
await this.queue.assertCurrent(this.lease, work);
|
|
62991
|
+
return () => this.commit(work, subtree, owned);
|
|
62992
|
+
}
|
|
62993
|
+
commit(work, subtree, owned) {
|
|
62994
|
+
const root = this.lease.runId;
|
|
62995
|
+
const known = new Map(this.store.runs().map((run) => [run.id, run]));
|
|
62996
|
+
owned(known);
|
|
61880
62997
|
const attempt = this.enter(work);
|
|
61881
62998
|
const execution = {
|
|
61882
62999
|
worker: work.worker ?? "",
|
|
@@ -61888,11 +63005,13 @@ class PoolStageHandoff {
|
|
|
61888
63005
|
...run,
|
|
61889
63006
|
rootRunId: root,
|
|
61890
63007
|
pendingStageMessages: [
|
|
61891
|
-
...
|
|
63008
|
+
...known.get(run.id)?.pendingStageMessages ?? pendingStageMessagesForDurableRun(this.backend, run.id, root)
|
|
61892
63009
|
],
|
|
61893
63010
|
stages: run.stages.map((stage) => ({ ...stage, workerExecution: execution }))
|
|
61894
63011
|
}));
|
|
61895
63012
|
const changed = published.some((run) => activity(known.get(run.id)) !== activity(run));
|
|
63013
|
+
for (const run of published)
|
|
63014
|
+
this.claimRun?.(run.id);
|
|
61896
63015
|
for (const run of published) {
|
|
61897
63016
|
attempt.runs.add(run.id);
|
|
61898
63017
|
this.owners.set(run.id, work.id);
|
|
@@ -61917,7 +63036,7 @@ class PoolStageHandoff {
|
|
|
61917
63036
|
const byId = new Map;
|
|
61918
63037
|
for (const run of runs) {
|
|
61919
63038
|
if (run.id === root || byId.has(run.id))
|
|
61920
|
-
throw
|
|
63039
|
+
throw outside2(run.id);
|
|
61921
63040
|
byId.set(run.id, run);
|
|
61922
63041
|
}
|
|
61923
63042
|
if (runs.length === 0)
|
|
@@ -61930,25 +63049,25 @@ class PoolStageHandoff {
|
|
|
61930
63049
|
const top = byId.get(work.childRunId);
|
|
61931
63050
|
if (top !== undefined) {
|
|
61932
63051
|
if (top.parentRunId !== root || top.parentStageId !== work.boundaryId)
|
|
61933
|
-
throw
|
|
63052
|
+
throw outside2(top.id);
|
|
61934
63053
|
const pending = [top];
|
|
61935
63054
|
for (let run = pending.shift();run !== undefined; run = pending.shift()) {
|
|
61936
63055
|
if (run.rootRunId !== undefined && run.rootRunId !== root)
|
|
61937
|
-
throw
|
|
63056
|
+
throw outside2(run.id);
|
|
61938
63057
|
ordered.push(run);
|
|
61939
63058
|
for (const child of runs) {
|
|
61940
63059
|
if (child.parentRunId !== run.id)
|
|
61941
63060
|
continue;
|
|
61942
63061
|
const parentStage = run.stages.find((stage) => stage.id === child.parentStageId);
|
|
61943
63062
|
if (authoritativeWorkflowChildRunId(parentStage) !== child.id)
|
|
61944
|
-
throw
|
|
63063
|
+
throw outside2(child.id);
|
|
61945
63064
|
pending.push(child);
|
|
61946
63065
|
}
|
|
61947
63066
|
}
|
|
61948
63067
|
}
|
|
61949
63068
|
const stray = runs.find((run) => !ordered.includes(run));
|
|
61950
63069
|
if (stray !== undefined)
|
|
61951
|
-
throw
|
|
63070
|
+
throw outside2(stray.id);
|
|
61952
63071
|
return ordered;
|
|
61953
63072
|
}
|
|
61954
63073
|
async begin(work) {
|
|
@@ -62003,7 +63122,7 @@ class PoolStageHandoff {
|
|
|
62003
63122
|
}
|
|
62004
63123
|
requireOwned(attempt, runId) {
|
|
62005
63124
|
if (!attempt.runs.has(runId))
|
|
62006
|
-
throw
|
|
63125
|
+
throw outside2(runId);
|
|
62007
63126
|
}
|
|
62008
63127
|
delivery(attempt, work, runId, stageId, stageName) {
|
|
62009
63128
|
const key = JSON.stringify([runId, stageId]);
|
|
@@ -62021,7 +63140,7 @@ class PoolStageHandoff {
|
|
|
62021
63140
|
return delivery;
|
|
62022
63141
|
}
|
|
62023
63142
|
}
|
|
62024
|
-
function
|
|
63143
|
+
function outside2(runId) {
|
|
62025
63144
|
return new Error(`Pool stage publication is outside the work subtree: ${runId}`);
|
|
62026
63145
|
}
|
|
62027
63146
|
function isLive(stage) {
|
|
@@ -62046,33 +63165,122 @@ var init_stage_handoff = __esm(() => {
|
|
|
62046
63165
|
});
|
|
62047
63166
|
|
|
62048
63167
|
// dist/builtin/workflows/src/workers/local-pool.ts
|
|
62049
|
-
import { fork } from "node:child_process";
|
|
62050
|
-
import {
|
|
62051
|
-
import {
|
|
62052
|
-
import { mkdir as mkdir6, rm as rm3, rmdir } from "node:fs/promises";
|
|
62053
|
-
import { dirname as dirname12, join as join28, resolve as resolve11 } from "node:path";
|
|
62054
|
-
import { setTimeout as delay2 } from "node:timers/promises";
|
|
63168
|
+
import { fork as fork2 } from "node:child_process";
|
|
63169
|
+
import { resolve as resolve12 } from "node:path";
|
|
63170
|
+
import { setTimeout as delay3 } from "node:timers/promises";
|
|
62055
63171
|
import { fileURLToPath as fileURLToPath5 } from "node:url";
|
|
62056
63172
|
import { isBunBinary as isBunBinary2 } from "@bastani/atomic";
|
|
62057
|
-
function
|
|
62058
|
-
return
|
|
63173
|
+
function defaultWorkerEntry() {
|
|
63174
|
+
return isBunBinary2 ? INTERNAL_WORKFLOW_WORKER_ARG : fileURLToPath5(new URL("./local-worker-entry.mjs", import.meta.url));
|
|
63175
|
+
}
|
|
63176
|
+
function launchCoordinatorProcess() {
|
|
63177
|
+
const child = fork2(defaultWorkerEntry(), [JSON.stringify({ role: POOL_COORDINATOR_ROLE })], {
|
|
63178
|
+
stdio: ["ignore", "ignore", "pipe", "ipc"],
|
|
63179
|
+
execArgv: []
|
|
63180
|
+
});
|
|
63181
|
+
let stderr = "";
|
|
63182
|
+
child.stderr?.on("data", (chunk) => {
|
|
63183
|
+
stderr = (stderr + chunk.toString()).slice(-COORDINATOR_STDERR_TAIL_CHARACTERS);
|
|
63184
|
+
});
|
|
63185
|
+
const listeners = [];
|
|
63186
|
+
let lostReason;
|
|
63187
|
+
const lose = (reason) => {
|
|
63188
|
+
if (lostReason !== undefined)
|
|
63189
|
+
return;
|
|
63190
|
+
lostReason = reason;
|
|
63191
|
+
for (const listener of listeners)
|
|
63192
|
+
listener(reason);
|
|
63193
|
+
};
|
|
63194
|
+
const exited = new Promise((resolve) => child.once("exit", () => resolve()));
|
|
63195
|
+
child.on("error", lose);
|
|
63196
|
+
child.once("exit", (code, signal) => lose(new Error(`Local worker pool coordinator exited (${signal ?? `code ${code}`})${stderr ? `: ${stderr.trim()}` : ""}`)));
|
|
63197
|
+
return {
|
|
63198
|
+
pid: child.pid,
|
|
63199
|
+
port: {
|
|
63200
|
+
send(message) {
|
|
63201
|
+
if (lostReason !== undefined || !child.connected)
|
|
63202
|
+
return;
|
|
63203
|
+
child.send(message, (error) => {
|
|
63204
|
+
if (error)
|
|
63205
|
+
lose(error);
|
|
63206
|
+
});
|
|
63207
|
+
},
|
|
63208
|
+
onMessage(listener) {
|
|
63209
|
+
child.on("message", (message) => listener(message));
|
|
63210
|
+
},
|
|
63211
|
+
onClose(listener) {
|
|
63212
|
+
if (lostReason !== undefined)
|
|
63213
|
+
listener(lostReason);
|
|
63214
|
+
else
|
|
63215
|
+
listeners.push(listener);
|
|
63216
|
+
},
|
|
63217
|
+
async close() {
|
|
63218
|
+
if (child.connected)
|
|
63219
|
+
child.disconnect();
|
|
63220
|
+
if (child.exitCode !== null || child.signalCode !== null)
|
|
63221
|
+
return;
|
|
63222
|
+
const timedOut = await Promise.race([exited.then(() => false), delay3(COORDINATOR_EXIT_GRACE_MS, true)]);
|
|
63223
|
+
if (timedOut) {
|
|
63224
|
+
child.kill("SIGKILL");
|
|
63225
|
+
await exited;
|
|
63226
|
+
}
|
|
63227
|
+
}
|
|
63228
|
+
}
|
|
63229
|
+
};
|
|
63230
|
+
}
|
|
63231
|
+
function launchCoordinatorInProcess() {
|
|
63232
|
+
const mainListeners = [];
|
|
63233
|
+
const coordinatorListeners = [];
|
|
63234
|
+
const copy = (message) => JSON.parse(JSON.stringify(message));
|
|
63235
|
+
const coordinator = new PoolCoordinator({
|
|
63236
|
+
send(message) {
|
|
63237
|
+
const copied = copy(message);
|
|
63238
|
+
setImmediate(() => {
|
|
63239
|
+
for (const listener of mainListeners)
|
|
63240
|
+
listener(copied);
|
|
63241
|
+
});
|
|
63242
|
+
},
|
|
63243
|
+
onMessage(listener) {
|
|
63244
|
+
coordinatorListeners.push(listener);
|
|
63245
|
+
},
|
|
63246
|
+
onClose() {},
|
|
63247
|
+
async close() {}
|
|
63248
|
+
});
|
|
63249
|
+
return {
|
|
63250
|
+
pid: process.pid,
|
|
63251
|
+
port: {
|
|
63252
|
+
send(message) {
|
|
63253
|
+
const copied = copy(message);
|
|
63254
|
+
setImmediate(() => {
|
|
63255
|
+
for (const listener of coordinatorListeners)
|
|
63256
|
+
listener(copied);
|
|
63257
|
+
});
|
|
63258
|
+
},
|
|
63259
|
+
onMessage(listener) {
|
|
63260
|
+
mainListeners.push(listener);
|
|
63261
|
+
},
|
|
63262
|
+
onClose() {},
|
|
63263
|
+
async close() {
|
|
63264
|
+
coordinator.shutdown();
|
|
63265
|
+
}
|
|
63266
|
+
}
|
|
63267
|
+
};
|
|
62059
63268
|
}
|
|
62060
63269
|
|
|
62061
63270
|
class LocalWorkerPool {
|
|
62062
63271
|
backend;
|
|
62063
63272
|
options;
|
|
62064
|
-
|
|
62065
|
-
|
|
62066
|
-
|
|
62067
|
-
|
|
62068
|
-
|
|
62069
|
-
|
|
62070
|
-
|
|
62071
|
-
|
|
62072
|
-
|
|
62073
|
-
|
|
62074
|
-
|
|
62075
|
-
rootStalledMs = 0;
|
|
63273
|
+
pids = new Set;
|
|
63274
|
+
coordinator;
|
|
63275
|
+
coordinatorLost;
|
|
63276
|
+
commands = new Map;
|
|
63277
|
+
commandSequence = 0;
|
|
63278
|
+
inbound = Promise.resolve();
|
|
63279
|
+
works = new Map;
|
|
63280
|
+
attempts = new Map;
|
|
63281
|
+
results = new Map;
|
|
63282
|
+
changed = Promise.withResolvers();
|
|
63283
|
+
waiting = 0;
|
|
62076
63284
|
ticker;
|
|
62077
63285
|
leaseMs;
|
|
62078
63286
|
desiredControl;
|
|
@@ -62080,18 +63288,17 @@ class LocalWorkerPool {
|
|
|
62080
63288
|
store;
|
|
62081
63289
|
handoff;
|
|
62082
63290
|
releaseRoutes;
|
|
63291
|
+
routeClaims = new Map;
|
|
62083
63292
|
sources = [];
|
|
62084
63293
|
queue;
|
|
62085
63294
|
lease;
|
|
62086
|
-
admission;
|
|
62087
|
-
usageLedger;
|
|
62088
63295
|
usageTotal;
|
|
62089
63296
|
rootBudget;
|
|
63297
|
+
budgetBlocked = false;
|
|
63298
|
+
rootSignature;
|
|
62090
63299
|
starting;
|
|
62091
|
-
cleanupFailure;
|
|
62092
63300
|
failure;
|
|
62093
63301
|
closed = false;
|
|
62094
|
-
replacements = 0;
|
|
62095
63302
|
releaseBinding;
|
|
62096
63303
|
constructor(backend, options) {
|
|
62097
63304
|
this.backend = backend;
|
|
@@ -62106,10 +63313,12 @@ class LocalWorkerPool {
|
|
|
62106
63313
|
this.releaseBinding = bindChildWorkerDispatcher(backend, this, options.rootId);
|
|
62107
63314
|
if (options.admission && options.rootId === undefined)
|
|
62108
63315
|
workflowResourceAdmission.set(backend, options.admission);
|
|
62109
|
-
this.ticker = setInterval(() => this.observeResponsiveness(), ROOT_TICK_INTERVAL_MS).unref();
|
|
62110
63316
|
}
|
|
62111
63317
|
get workerPids() {
|
|
62112
|
-
return [...this.
|
|
63318
|
+
return [...this.pids];
|
|
63319
|
+
}
|
|
63320
|
+
get coordinatorPid() {
|
|
63321
|
+
return this.coordinator?.pid;
|
|
62113
63322
|
}
|
|
62114
63323
|
usage() {
|
|
62115
63324
|
return this.usageTotal;
|
|
@@ -62117,47 +63326,39 @@ class LocalWorkerPool {
|
|
|
62117
63326
|
async prepare(root, store) {
|
|
62118
63327
|
if (this.closed)
|
|
62119
63328
|
throw new Error("Local worker pool is closed");
|
|
62120
|
-
if (!(this.backend.journal instanceof SqlRunJournal) ||
|
|
63329
|
+
if (!(this.backend.journal instanceof SqlRunJournal) || resolve12(this.options.journalPath) !== resolve12(this.backend.journal.path) || sqlPoolStorage(this.backend.journal)?.tables.runs !== "wf_runs")
|
|
62121
63330
|
throw new Error("Local workers require the same default-prefix SQLite journal");
|
|
62122
63331
|
if (this.root !== undefined && this.root !== root)
|
|
62123
63332
|
throw new Error("Local worker pool belongs to another root");
|
|
62124
63333
|
this.root = root;
|
|
62125
63334
|
if (store)
|
|
62126
63335
|
this.store = store;
|
|
62127
|
-
this.usageLedger ??= await openSqlPoolUsage(this.backend.journal);
|
|
62128
|
-
this.usageTotal = await this.usageLedger.total(root);
|
|
62129
63336
|
const state = await this.backend.journal.leaseState(root);
|
|
62130
63337
|
if (!state || state.holderId !== this.backend.holderId)
|
|
62131
63338
|
throw new Error("Local worker pool requires the root owner");
|
|
62132
63339
|
this.lease = { runId: root, holderId: state.holderId, epoch: state.epoch };
|
|
62133
|
-
const
|
|
62134
|
-
await
|
|
62135
|
-
|
|
62136
|
-
|
|
62137
|
-
|
|
62138
|
-
|
|
62139
|
-
|
|
62140
|
-
|
|
62141
|
-
|
|
62142
|
-
|
|
63340
|
+
const workerEntry = this.options.workerEntry ?? (isBunBinary2 || this.options.coordinator === "in-process" ? defaultWorkerEntry() : undefined);
|
|
63341
|
+
this.usageTotal = await this.command({
|
|
63342
|
+
kind: "prepare",
|
|
63343
|
+
root,
|
|
63344
|
+
lease: this.lease,
|
|
63345
|
+
options: {
|
|
63346
|
+
journalPath: this.options.journalPath,
|
|
63347
|
+
workers: this.options.workers,
|
|
63348
|
+
leaseMs: this.leaseMs,
|
|
63349
|
+
startupTimeoutMs: this.options.startupTimeoutMs ?? DEFAULT_STARTUP_TIMEOUT_MS,
|
|
63350
|
+
...this.options.admission === undefined ? {} : { admission: this.options.admission },
|
|
63351
|
+
...workerEntry === undefined ? {} : { workerEntry }
|
|
63352
|
+
},
|
|
63353
|
+
...this.desiredControl === undefined ? {} : { control: this.desiredControl }
|
|
63354
|
+
});
|
|
63355
|
+
this.queue ??= await openSqlPoolQueue(this.backend.journal);
|
|
62143
63356
|
}
|
|
62144
63357
|
async control(state) {
|
|
62145
63358
|
this.desiredControl = state;
|
|
62146
63359
|
try {
|
|
62147
|
-
if (
|
|
62148
|
-
|
|
62149
|
-
const control = await this.queue.setControl(this.lease, state);
|
|
62150
|
-
const deadline = Date.now() + CONTROL_ACKNOWLEDGEMENT_TIMEOUT_MS;
|
|
62151
|
-
while ([...this.readyWorkers].some((worker) => (this.controlAcks.get(worker) ?? -1) < control.revision)) {
|
|
62152
|
-
if (this.desiredControl !== state)
|
|
62153
|
-
return;
|
|
62154
|
-
if (Date.now() >= deadline) {
|
|
62155
|
-
if (state === "quit")
|
|
62156
|
-
break;
|
|
62157
|
-
throw new Error("Worker control acknowledgement unresolved");
|
|
62158
|
-
}
|
|
62159
|
-
await delay2(25);
|
|
62160
|
-
}
|
|
63360
|
+
if (this.coordinator !== undefined)
|
|
63361
|
+
await this.command({ kind: "control", state });
|
|
62161
63362
|
} finally {
|
|
62162
63363
|
if (state === "quit")
|
|
62163
63364
|
await this.close();
|
|
@@ -62203,290 +63404,61 @@ class LocalWorkerPool {
|
|
|
62203
63404
|
throw new Error(`Worker definition is not importable: ${binding.exportName}`);
|
|
62204
63405
|
return source;
|
|
62205
63406
|
}));
|
|
62206
|
-
|
|
62207
|
-
|
|
63407
|
+
const cwd = String(this.backend.getWorkflow(root)?.invocationCwd ?? process.cwd());
|
|
63408
|
+
const store = this.store;
|
|
63409
|
+
if (store) {
|
|
63410
|
+
this.releaseRoutes = await acquireStandaloneIntercom(store, root, cwd);
|
|
62208
63411
|
if (this.closed) {
|
|
62209
63412
|
await this.releaseRoutes();
|
|
62210
63413
|
this.releaseRoutes = undefined;
|
|
62211
63414
|
throw new Error("Local worker pool is closed");
|
|
62212
63415
|
}
|
|
62213
|
-
|
|
62214
|
-
|
|
62215
|
-
|
|
62216
|
-
this.
|
|
62217
|
-
|
|
62218
|
-
root,
|
|
62219
|
-
holder: "coordinator"
|
|
63416
|
+
const routeOwner = {
|
|
63417
|
+
notifyUndeliverable: async (payload) => !this.closed && await this.command({ kind: "undeliverable", payload }) === true
|
|
63418
|
+
};
|
|
63419
|
+
this.handoff = new PoolStageHandoff(this.backend, store, this.queueView(), this.lease, (runId) => {
|
|
63420
|
+
this.routeClaims.set(runId, claimExternalRunRoute(store, runId, routeOwner));
|
|
62220
63421
|
});
|
|
62221
|
-
this.logDirectory = workerLogDirectory(this.options.journalPath, root);
|
|
62222
|
-
await mkdir6(this.logDirectory, { recursive: true }).catch(() => {
|
|
62223
|
-
return;
|
|
62224
|
-
});
|
|
62225
|
-
const workers = Array.from({ length: this.options.workers }, () => this.spawn(root));
|
|
62226
|
-
try {
|
|
62227
|
-
await Promise.any(workers);
|
|
62228
|
-
} catch (error) {
|
|
62229
|
-
throw error.errors[0];
|
|
62230
63422
|
}
|
|
62231
|
-
|
|
62232
|
-
|
|
62233
|
-
|
|
62234
|
-
|
|
62235
|
-
|
|
62236
|
-
|
|
62237
|
-
journalPath: this.options.journalPath,
|
|
62238
|
-
root,
|
|
62239
|
-
definitions: this.sources,
|
|
62240
|
-
admission: this.options.admission
|
|
62241
|
-
})
|
|
62242
|
-
], { stdio: ["ignore", "ignore", "pipe", "ipc"], execArgv: [] });
|
|
62243
|
-
const disconnected = new Promise((resolve) => child.once("disconnect", () => resolve()));
|
|
62244
|
-
this.children.add(child);
|
|
62245
|
-
const worker = randomUUID16();
|
|
62246
|
-
this.incarnations.set(worker, child);
|
|
62247
|
-
let ready = false;
|
|
62248
|
-
let serial = Promise.resolve();
|
|
62249
|
-
let work;
|
|
62250
|
-
const deliveries = new Map;
|
|
62251
|
-
const logPath = join28(this.logDirectory, `${worker}.log`);
|
|
62252
|
-
this.logs.set(child, logPath);
|
|
62253
|
-
let logFailure;
|
|
62254
|
-
const log = createWriteStream(logPath, { flags: "a" });
|
|
62255
|
-
log.on("error", (error) => {
|
|
62256
|
-
logFailure = error.message;
|
|
62257
|
-
});
|
|
62258
|
-
let stderr = "";
|
|
62259
|
-
child.stderr?.on("data", (chunk) => {
|
|
62260
|
-
stderr = (stderr + chunk.toString()).slice(-STDERR_TAIL_CHARACTERS);
|
|
62261
|
-
});
|
|
62262
|
-
child.stderr?.on("error", (error) => {
|
|
62263
|
-
logFailure ??= error.message;
|
|
62264
|
-
log.end();
|
|
62265
|
-
});
|
|
62266
|
-
child.stderr?.pipe(log);
|
|
62267
|
-
const diagnosed = (summary) => new Error(`${summary} (worker stderr log: ${logFailure === undefined ? logPath : `unavailable, ${logFailure}`})${stderr ? `: ${stderr}` : ""}`);
|
|
62268
|
-
const startupTimeoutMs = this.options.startupTimeoutMs ?? DEFAULT_STARTUP_TIMEOUT_MS;
|
|
62269
|
-
let timedOut = false;
|
|
62270
|
-
let terminated = false;
|
|
62271
|
-
const terminate = () => {
|
|
62272
|
-
if (terminated)
|
|
62273
|
-
return;
|
|
62274
|
-
terminated = true;
|
|
62275
|
-
child.kill("SIGKILL");
|
|
62276
|
-
};
|
|
62277
|
-
const replied = (error) => {
|
|
62278
|
-
if (error)
|
|
62279
|
-
terminate();
|
|
62280
|
-
};
|
|
62281
|
-
return new Promise((resolve, reject) => {
|
|
62282
|
-
const startup = setTimeout(() => {
|
|
62283
|
-
timedOut = true;
|
|
62284
|
-
terminate();
|
|
62285
|
-
}, startupTimeoutMs);
|
|
62286
|
-
child.on("error", (error) => {
|
|
62287
|
-
clearTimeout(startup);
|
|
62288
|
-
reject(error);
|
|
62289
|
-
terminate();
|
|
62290
|
-
});
|
|
62291
|
-
const closed = new Promise((resolve) => child.once("close", () => resolve()));
|
|
62292
|
-
child.once("exit", () => {
|
|
62293
|
-
Promise.race([closed, delay2(STDERR_DRAIN_GRACE_MS)]).then(() => reject(diagnosed(timedOut ? `Local worker startup timed out after ${startupTimeoutMs} ms` : "Local worker exited before readiness")));
|
|
62294
|
-
this.children.delete(child);
|
|
62295
|
-
this.logs.delete(child);
|
|
62296
|
-
clearTimeout(startup);
|
|
62297
|
-
this.incarnations.delete(worker);
|
|
62298
|
-
this.readyWorkers.delete(worker);
|
|
62299
|
-
this.controlAcks.delete(worker);
|
|
62300
|
-
this.lastHeard.delete(worker);
|
|
62301
|
-
if (work)
|
|
62302
|
-
this.handoff?.fence(work);
|
|
62303
|
-
for (const delivery of deliveries.values())
|
|
62304
|
-
delivery.reject(new Error("Worker exited before delivery acknowledgement"));
|
|
62305
|
-
deliveries.clear();
|
|
62306
|
-
const cleanup = disconnected.then(() => serial).then(async () => {
|
|
62307
|
-
try {
|
|
62308
|
-
if (await this.admission?.releaseStoppedSessions(worker))
|
|
62309
|
-
throw new Error("Local worker process cleanup unconfirmed; permits quarantined");
|
|
62310
|
-
} catch (error) {
|
|
62311
|
-
this.cleanupFailure = error instanceof Error ? error : new Error(String(error));
|
|
62312
|
-
throw this.cleanupFailure;
|
|
62313
|
-
}
|
|
62314
|
-
}).then(async () => {
|
|
62315
|
-
if (this.closed || !ready || this.desiredControl === "quit")
|
|
62316
|
-
return;
|
|
62317
|
-
if (work && await this.queue.result(root, work.id)) {
|
|
62318
|
-
work = undefined;
|
|
62319
|
-
this.replacements = 0;
|
|
62320
|
-
}
|
|
62321
|
-
if (this.closed)
|
|
62322
|
-
return;
|
|
62323
|
-
if (work)
|
|
62324
|
-
await this.queue.releaseExited(this.lease, work);
|
|
62325
|
-
const exhausted = work ? work.generation >= 3 : ++this.replacements > 3;
|
|
62326
|
-
if (exhausted)
|
|
62327
|
-
this.failure = diagnosed("Local worker retry limit");
|
|
62328
|
-
else
|
|
62329
|
-
await this.spawn(root);
|
|
62330
|
-
}).catch((error) => {
|
|
62331
|
-
this.failure = error;
|
|
62332
|
-
});
|
|
62333
|
-
this.cleanups.add(cleanup);
|
|
62334
|
-
cleanup.finally(() => this.cleanups.delete(cleanup));
|
|
62335
|
-
});
|
|
62336
|
-
child.on("message", (message) => {
|
|
62337
|
-
this.observeResponsiveness();
|
|
62338
|
-
this.lastHeard.set(worker, { at: performance.now(), rootStalled: this.rootStalledMs });
|
|
62339
|
-
if (message.kind === "delivery-ack") {
|
|
62340
|
-
const pending = deliveries.get(message.deliveryId);
|
|
62341
|
-
deliveries.delete(message.deliveryId);
|
|
62342
|
-
if (message.error)
|
|
62343
|
-
pending?.reject(new Error(message.error));
|
|
62344
|
-
else
|
|
62345
|
-
pending?.resolve();
|
|
62346
|
-
return;
|
|
62347
|
-
}
|
|
62348
|
-
const response = serial.then(async () => {
|
|
62349
|
-
if (message.kind === "ready") {
|
|
62350
|
-
if (terminated)
|
|
62351
|
-
return;
|
|
62352
|
-
ready = true;
|
|
62353
|
-
this.readyWorkers.add(worker);
|
|
62354
|
-
clearTimeout(startup);
|
|
62355
|
-
resolve();
|
|
62356
|
-
return;
|
|
62357
|
-
}
|
|
62358
|
-
await this.starting;
|
|
62359
|
-
const queue = this.queue;
|
|
62360
|
-
const lease = this.lease;
|
|
62361
|
-
if (message.kind === "control")
|
|
62362
|
-
return queue.control(lease);
|
|
62363
|
-
if (message.kind === "controlled") {
|
|
62364
|
-
const control = await queue.control(lease);
|
|
62365
|
-
if (message.revision > control.revision)
|
|
62366
|
-
throw new Error("Unknown control revision");
|
|
62367
|
-
this.controlAcks.set(worker, Math.max(this.controlAcks.get(worker) ?? -1, message.revision));
|
|
62368
|
-
return;
|
|
62369
|
-
}
|
|
62370
|
-
if (message.kind === "claim") {
|
|
62371
|
-
if (work)
|
|
62372
|
-
throw new Error("Worker already holds an attempt");
|
|
62373
|
-
const decision = this.rootBudget?.checkpoint();
|
|
62374
|
-
if (decision?.kind === "exhausted" || decision?.kind === "wrap_up")
|
|
62375
|
-
return;
|
|
62376
|
-
await this.recoverExpired(root);
|
|
62377
|
-
work = await queue.claim(lease, worker, this.sources.map((source) => ({
|
|
62378
|
-
definition: `${source.path}#${source.exportName}`,
|
|
62379
|
-
digest: source.digest
|
|
62380
|
-
})), this.leaseMs);
|
|
62381
|
-
return work;
|
|
62382
|
-
}
|
|
62383
|
-
if (!work)
|
|
62384
|
-
throw new Error("Worker has no active attempt");
|
|
62385
|
-
if (message.kind === "renew" || work.expiresAt - Date.now() < this.leaseMs / 2)
|
|
62386
|
-
work = await queue.extend(lease, work, this.leaseMs);
|
|
62387
|
-
if (message.kind === "activity")
|
|
62388
|
-
return this.handoff?.publish(work, message.runs);
|
|
62389
|
-
if (message.kind === "route")
|
|
62390
|
-
return this.handoff?.route(work, message.runId);
|
|
62391
|
-
if (message.kind === "delivery-ready")
|
|
62392
|
-
return this.handoff?.ready(work, message.runId, message.stageId, message.stageName, message.sessionId);
|
|
62393
|
-
if (message.kind === "delivery") {
|
|
62394
|
-
if (!message.recipient)
|
|
62395
|
-
throw new Error("Worker delivery requires a session identity");
|
|
62396
|
-
return this.handoff?.deliver(work, message.runId, message.stageId, message.stageName, message.recipient, (from, payload) => new Promise((resolve, reject) => {
|
|
62397
|
-
const id = randomUUID16();
|
|
62398
|
-
deliveries.set(id, { resolve, reject });
|
|
62399
|
-
child.send({ id: message.id, delivery: { id, from, message: payload } }, (error) => {
|
|
62400
|
-
if (error) {
|
|
62401
|
-
deliveries.delete(id);
|
|
62402
|
-
reject(error);
|
|
62403
|
-
}
|
|
62404
|
-
});
|
|
62405
|
-
}));
|
|
62406
|
-
}
|
|
62407
|
-
if (message.kind === "renew")
|
|
62408
|
-
return;
|
|
62409
|
-
if (message.kind === "commit") {
|
|
62410
|
-
const receipt = await queue.commit(lease, work, message.operation, message.mutation);
|
|
62411
|
-
this.usageTotal = await this.usageLedger.total(root);
|
|
62412
|
-
return receipt;
|
|
62413
|
-
}
|
|
62414
|
-
if (message.kind === "usage") {
|
|
62415
|
-
this.usageTotal = await this.usageLedger.report(lease, work, message.reports);
|
|
62416
|
-
return;
|
|
62417
|
-
}
|
|
62418
|
-
if (message.kind === "budget") {
|
|
62419
|
-
await queue.assertCurrent(lease, work);
|
|
62420
|
-
const decision = this.rootBudget?.checkpoint();
|
|
62421
|
-
if (decision?.kind === "exhausted" || decision?.kind === "wrap_up")
|
|
62422
|
-
throw new Error("Root workflow budget exhausted");
|
|
62423
|
-
return queue.control(lease);
|
|
62424
|
-
}
|
|
62425
|
-
if (message.kind !== "complete")
|
|
62426
|
-
throw new Error("Invalid worker operation");
|
|
62427
|
-
await queue.complete(lease, work, message.result);
|
|
62428
|
-
this.replacements = 0;
|
|
62429
|
-
work = undefined;
|
|
62430
|
-
});
|
|
62431
|
-
if (message.kind !== "delivery" && message.kind !== "delivery-ready")
|
|
62432
|
-
serial = response.then(() => {}, () => {});
|
|
62433
|
-
response.then((value2) => child.send({ id: message.id, value: value2 }, replied), (error) => child.send({ id: message.id, error: error.message }, replied));
|
|
62434
|
-
});
|
|
63423
|
+
await this.command({
|
|
63424
|
+
kind: "start",
|
|
63425
|
+
sources: this.sources,
|
|
63426
|
+
cwd,
|
|
63427
|
+
publish: store !== undefined,
|
|
63428
|
+
...this.rootRun()
|
|
62435
63429
|
});
|
|
62436
63430
|
}
|
|
62437
|
-
|
|
62438
|
-
|
|
62439
|
-
|
|
62440
|
-
|
|
62441
|
-
|
|
62442
|
-
|
|
62443
|
-
|
|
62444
|
-
|
|
62445
|
-
|
|
62446
|
-
|
|
62447
|
-
if (work.state !== "leased" || work.expiresAt >= expiredBefore)
|
|
62448
|
-
continue;
|
|
62449
|
-
const child = work.worker === null ? undefined : this.incarnations.get(work.worker);
|
|
62450
|
-
if (child && work.worker !== null && this.isLive(work.worker, child)) {
|
|
62451
|
-
await this.queue.extend(this.lease, work, this.leaseMs).catch((error) => {
|
|
62452
|
-
if (error.message !== STALE_WORK_LEASE)
|
|
62453
|
-
throw error;
|
|
62454
|
-
});
|
|
62455
|
-
continue;
|
|
62456
|
-
}
|
|
62457
|
-
if (child && !await this.stop(child).then(() => true, () => false))
|
|
62458
|
-
continue;
|
|
62459
|
-
await this.queue.expire(this.lease, work);
|
|
62460
|
-
}
|
|
62461
|
-
})().finally(() => {
|
|
62462
|
-
this.recovering = undefined;
|
|
62463
|
-
this.nextRecoveryAt = performance.now() + RECOVERY_INTERVAL_MS;
|
|
62464
|
-
});
|
|
62465
|
-
return this.recovering;
|
|
63431
|
+
queueView() {
|
|
63432
|
+
return {
|
|
63433
|
+
assertCurrent: async (_lease, work) => {
|
|
63434
|
+
const current = this.attempts.get(work.id);
|
|
63435
|
+
if (current === undefined || current.fenced || current.generation !== work.generation || current.worker !== work.worker || current.epoch !== work.epoch)
|
|
63436
|
+
throw new Error(STALE_WORK_LEASE);
|
|
63437
|
+
},
|
|
63438
|
+
withCurrentAppend: (lease, work, assertSession, action) => this.queue.withCurrentAppend(lease, work, assertSession, action),
|
|
63439
|
+
result: async (root, id) => this.results.get(id) ?? this.queue.result(root, id)
|
|
63440
|
+
};
|
|
62466
63441
|
}
|
|
62467
|
-
|
|
62468
|
-
const
|
|
62469
|
-
|
|
62470
|
-
if (gap > ROOT_STALL_THRESHOLD_MS)
|
|
62471
|
-
this.rootStalledMs += gap - ROOT_TICK_INTERVAL_MS;
|
|
62472
|
-
this.lastTick = now;
|
|
63442
|
+
rootProjection() {
|
|
63443
|
+
const run = this.store?.runs().find((candidate) => candidate.id === this.root);
|
|
63444
|
+
return run === undefined ? undefined : projectRootRun(run);
|
|
62473
63445
|
}
|
|
62474
|
-
|
|
62475
|
-
this.
|
|
62476
|
-
|
|
62477
|
-
|
|
63446
|
+
rootRun() {
|
|
63447
|
+
const rootRun = this.rootProjection();
|
|
63448
|
+
if (rootRun === undefined)
|
|
63449
|
+
return {};
|
|
63450
|
+
this.rootSignature = JSON.stringify(rootRun);
|
|
63451
|
+
return { rootRun };
|
|
62478
63452
|
}
|
|
62479
|
-
|
|
62480
|
-
|
|
62481
|
-
|
|
62482
|
-
|
|
62483
|
-
|
|
62484
|
-
|
|
62485
|
-
|
|
62486
|
-
|
|
62487
|
-
|
|
62488
|
-
child.kill("SIGKILL");
|
|
62489
|
-
});
|
|
63453
|
+
refreshRootRun() {
|
|
63454
|
+
const rootRun = this.rootProjection();
|
|
63455
|
+
if (rootRun === undefined)
|
|
63456
|
+
return;
|
|
63457
|
+
const signature = JSON.stringify(rootRun);
|
|
63458
|
+
if (signature === this.rootSignature)
|
|
63459
|
+
return;
|
|
63460
|
+
this.rootSignature = signature;
|
|
63461
|
+
this.notice({ kind: "root", rootRun });
|
|
62490
63462
|
}
|
|
62491
63463
|
async run(definition, inputs, options) {
|
|
62492
63464
|
try {
|
|
@@ -62519,7 +63491,7 @@ class LocalWorkerPool {
|
|
|
62519
63491
|
const selectedSource = this.sources[index];
|
|
62520
63492
|
if (await definitionDigest(selectedSource.path) !== selectedSource.digest)
|
|
62521
63493
|
throw new Error("Worker definition compatibility error");
|
|
62522
|
-
await this.backend.
|
|
63494
|
+
await this.backend.catchUpWorkflow(root);
|
|
62523
63495
|
const boundary = this.backend.listCheckpoints(root).find((entry) => entry.kind === "stage" && entry.topology?.boundary?.event === "start" && entry.topology.stageId === options.parentRun.stageId);
|
|
62524
63496
|
if (boundary?.kind !== "stage")
|
|
62525
63497
|
throw new Error("Missing durable child boundary");
|
|
@@ -62530,39 +63502,232 @@ class LocalWorkerPool {
|
|
|
62530
63502
|
config: options.config,
|
|
62531
63503
|
executionMode: options.executionMode
|
|
62532
63504
|
}));
|
|
62533
|
-
|
|
62534
|
-
|
|
62535
|
-
|
|
62536
|
-
|
|
62537
|
-
|
|
62538
|
-
|
|
62539
|
-
|
|
62540
|
-
|
|
62541
|
-
|
|
62542
|
-
|
|
62543
|
-
|
|
62544
|
-
|
|
63505
|
+
this.evaluateBudget();
|
|
63506
|
+
const enqueued = await this.command({
|
|
63507
|
+
kind: "enqueue",
|
|
63508
|
+
descriptor: {
|
|
63509
|
+
boundaryId: options.parentRun.stageId,
|
|
63510
|
+
childRunId: options.runId,
|
|
63511
|
+
scope: options.durableScope.scopePrefix,
|
|
63512
|
+
definition: `${selectedSource.path}#${selectedSource.exportName}`,
|
|
63513
|
+
digest: selectedSource.digest,
|
|
63514
|
+
inputs,
|
|
63515
|
+
execution
|
|
63516
|
+
},
|
|
63517
|
+
boundary,
|
|
63518
|
+
...this.rootRun()
|
|
63519
|
+
});
|
|
63520
|
+
const work = enqueued.work;
|
|
63521
|
+
this.works.set(work.id, work);
|
|
63522
|
+
if (enqueued.result !== undefined && !this.results.has(work.id))
|
|
63523
|
+
this.results.set(work.id, enqueued.result);
|
|
63524
|
+
this.waiting++;
|
|
63525
|
+
this.ticker ??= setInterval(() => {
|
|
63526
|
+
this.evaluateBudget();
|
|
63527
|
+
this.signalChange();
|
|
63528
|
+
}, POOL_SUMMARY_INTERVAL_MS).unref();
|
|
63529
|
+
try {
|
|
63530
|
+
for (;; ) {
|
|
63531
|
+
options.signal?.throwIfAborted();
|
|
63532
|
+
if (this.closed) {
|
|
63533
|
+
if (this.failure)
|
|
63534
|
+
throw this.failure;
|
|
63535
|
+
if (this.desiredControl === "quit")
|
|
63536
|
+
throw new WorkflowGracefulQuitError(root, "worker pool teardown");
|
|
63537
|
+
throw new Error("Local worker pool is closed");
|
|
63538
|
+
}
|
|
63539
|
+
const result = this.results.get(work.id);
|
|
63540
|
+
if (result) {
|
|
63541
|
+
await this.inbound;
|
|
63542
|
+
await this.backend.catchUpWorkflow(root);
|
|
63543
|
+
await this.handoff?.complete(work);
|
|
63544
|
+
return result;
|
|
63545
|
+
}
|
|
62545
63546
|
if (this.failure)
|
|
62546
63547
|
throw this.failure;
|
|
62547
|
-
if (this.
|
|
62548
|
-
|
|
62549
|
-
|
|
63548
|
+
if (this.budgetBlocked)
|
|
63549
|
+
await this.rootBudget.stopAtBoundaryAsync(definition.name);
|
|
63550
|
+
await this.nextChange(options.signal);
|
|
62550
63551
|
}
|
|
62551
|
-
|
|
62552
|
-
if (
|
|
62553
|
-
|
|
62554
|
-
|
|
62555
|
-
return result;
|
|
63552
|
+
} finally {
|
|
63553
|
+
if (--this.waiting === 0) {
|
|
63554
|
+
clearInterval(this.ticker);
|
|
63555
|
+
this.ticker = undefined;
|
|
62556
63556
|
}
|
|
62557
|
-
if (this.failure)
|
|
62558
|
-
throw this.failure;
|
|
62559
|
-
const decision = this.rootBudget?.checkpoint();
|
|
62560
|
-
if (decision?.kind === "exhausted" || decision?.kind === "wrap_up")
|
|
62561
|
-
await this.rootBudget.stopAtBoundaryAsync(definition.name);
|
|
62562
|
-
await this.recoverExpired(root);
|
|
62563
|
-
await delay2(25, undefined, { signal: options.signal });
|
|
62564
63557
|
}
|
|
62565
63558
|
}
|
|
63559
|
+
evaluateBudget() {
|
|
63560
|
+
const decision = this.rootBudget?.checkpoint();
|
|
63561
|
+
const blocked = decision?.kind === "exhausted" || decision?.kind === "wrap_up";
|
|
63562
|
+
if (blocked === this.budgetBlocked)
|
|
63563
|
+
return;
|
|
63564
|
+
this.budgetBlocked = blocked;
|
|
63565
|
+
this.notice({ kind: "budget", blocked });
|
|
63566
|
+
}
|
|
63567
|
+
signalChange() {
|
|
63568
|
+
const changed = this.changed;
|
|
63569
|
+
this.changed = Promise.withResolvers();
|
|
63570
|
+
changed.resolve();
|
|
63571
|
+
}
|
|
63572
|
+
nextChange(signal) {
|
|
63573
|
+
const changed = this.changed.promise;
|
|
63574
|
+
if (signal === undefined)
|
|
63575
|
+
return changed;
|
|
63576
|
+
return new Promise((resolve, reject) => {
|
|
63577
|
+
const abort = () => reject(signal.reason);
|
|
63578
|
+
signal.addEventListener("abort", abort, { once: true });
|
|
63579
|
+
changed.then(() => {
|
|
63580
|
+
signal.removeEventListener("abort", abort);
|
|
63581
|
+
resolve();
|
|
63582
|
+
});
|
|
63583
|
+
});
|
|
63584
|
+
}
|
|
63585
|
+
connect() {
|
|
63586
|
+
if (this.coordinator !== undefined)
|
|
63587
|
+
return this.coordinator;
|
|
63588
|
+
const coordinator = this.options.coordinator === "in-process" ? launchCoordinatorInProcess() : launchCoordinatorProcess();
|
|
63589
|
+
this.coordinator = coordinator;
|
|
63590
|
+
coordinator.port.onMessage((event) => {
|
|
63591
|
+
this.inbound = this.inbound.then(() => this.receive(event)).catch(() => {});
|
|
63592
|
+
});
|
|
63593
|
+
coordinator.port.onClose((reason) => {
|
|
63594
|
+
this.coordinatorLost = reason;
|
|
63595
|
+
this.inbound = this.inbound.then(() => this.lost(reason));
|
|
63596
|
+
});
|
|
63597
|
+
return coordinator;
|
|
63598
|
+
}
|
|
63599
|
+
lost(reason) {
|
|
63600
|
+
this.pids.clear();
|
|
63601
|
+
if (!this.closed)
|
|
63602
|
+
this.failure ??= reason;
|
|
63603
|
+
for (const pending of this.commands.values())
|
|
63604
|
+
pending.reject(reason);
|
|
63605
|
+
this.commands.clear();
|
|
63606
|
+
this.signalChange();
|
|
63607
|
+
}
|
|
63608
|
+
command(body) {
|
|
63609
|
+
const coordinator = this.connect();
|
|
63610
|
+
if (this.coordinatorLost !== undefined)
|
|
63611
|
+
return Promise.reject(this.coordinatorLost);
|
|
63612
|
+
const id = ++this.commandSequence;
|
|
63613
|
+
return new Promise((resolve, reject) => {
|
|
63614
|
+
this.commands.set(id, { resolve, reject });
|
|
63615
|
+
coordinator.port.send({ ...body, id });
|
|
63616
|
+
});
|
|
63617
|
+
}
|
|
63618
|
+
notice(notice) {
|
|
63619
|
+
this.coordinator?.port.send(notice);
|
|
63620
|
+
}
|
|
63621
|
+
async receive(event) {
|
|
63622
|
+
switch (event.kind) {
|
|
63623
|
+
case "reply": {
|
|
63624
|
+
const pending = this.commands.get(event.id);
|
|
63625
|
+
this.commands.delete(event.id);
|
|
63626
|
+
if (event.error !== undefined)
|
|
63627
|
+
pending?.reject(new Error(event.error));
|
|
63628
|
+
else
|
|
63629
|
+
pending?.resolve(event.value);
|
|
63630
|
+
return;
|
|
63631
|
+
}
|
|
63632
|
+
case "activity":
|
|
63633
|
+
return this.applyActivity(event.batch, event.entries, event.usage);
|
|
63634
|
+
case "spawned":
|
|
63635
|
+
this.pids.add(event.pid);
|
|
63636
|
+
return;
|
|
63637
|
+
case "exited":
|
|
63638
|
+
this.pids.delete(event.pid);
|
|
63639
|
+
return;
|
|
63640
|
+
case "fence": {
|
|
63641
|
+
const current = this.attempts.get(event.work.id);
|
|
63642
|
+
if (current?.generation === event.work.generation)
|
|
63643
|
+
current.fenced = true;
|
|
63644
|
+
this.handoff?.fence(event.work);
|
|
63645
|
+
return;
|
|
63646
|
+
}
|
|
63647
|
+
case "result":
|
|
63648
|
+
if (event.usage !== undefined)
|
|
63649
|
+
this.usageTotal = event.usage;
|
|
63650
|
+
this.results.set(event.workId, event.result);
|
|
63651
|
+
this.signalChange();
|
|
63652
|
+
return;
|
|
63653
|
+
case "failure":
|
|
63654
|
+
this.failure ??= new Error(event.error);
|
|
63655
|
+
this.signalChange();
|
|
63656
|
+
return;
|
|
63657
|
+
case "request":
|
|
63658
|
+
this.serve(event.id, event.request);
|
|
63659
|
+
return;
|
|
63660
|
+
}
|
|
63661
|
+
}
|
|
63662
|
+
async applyActivity(batch, entries, usage) {
|
|
63663
|
+
if (usage !== undefined)
|
|
63664
|
+
this.usageTotal = usage;
|
|
63665
|
+
const rejected = [];
|
|
63666
|
+
const handoff = this.handoff;
|
|
63667
|
+
if (handoff !== undefined) {
|
|
63668
|
+
const publications = entries.flatMap((entry) => {
|
|
63669
|
+
const work = this.track(entry.work);
|
|
63670
|
+
if (work !== undefined)
|
|
63671
|
+
return [{ work, runs: entry.runs }];
|
|
63672
|
+
rejected.push({ workId: entry.work.id, generation: entry.work.generation, error: STALE_WORK_LEASE });
|
|
63673
|
+
return [];
|
|
63674
|
+
});
|
|
63675
|
+
const outcomes = await handoff.publishBatch(publications);
|
|
63676
|
+
outcomes.forEach((error, index) => {
|
|
63677
|
+
const { work } = publications[index];
|
|
63678
|
+
if (error !== undefined)
|
|
63679
|
+
rejected.push({ workId: work.id, generation: work.generation, error: error.message });
|
|
63680
|
+
});
|
|
63681
|
+
}
|
|
63682
|
+
this.notice({ kind: "published", batch, rejected });
|
|
63683
|
+
this.refreshRootRun();
|
|
63684
|
+
}
|
|
63685
|
+
track(attempt) {
|
|
63686
|
+
const work = this.works.get(attempt.id);
|
|
63687
|
+
if (work === undefined)
|
|
63688
|
+
return;
|
|
63689
|
+
const current = this.attempts.get(attempt.id);
|
|
63690
|
+
if (current === undefined || attempt.generation > current.generation)
|
|
63691
|
+
this.attempts.set(attempt.id, { ...attempt, fenced: false });
|
|
63692
|
+
return { ...work, ...attempt, state: "leased" };
|
|
63693
|
+
}
|
|
63694
|
+
async serve(id, request) {
|
|
63695
|
+
try {
|
|
63696
|
+
const value2 = await this.answer(id, request);
|
|
63697
|
+
this.coordinator?.port.send({ kind: "request-reply", id, ...value2 === undefined ? {} : { value: value2 } });
|
|
63698
|
+
} catch (error) {
|
|
63699
|
+
this.coordinator?.port.send({
|
|
63700
|
+
kind: "request-reply",
|
|
63701
|
+
id,
|
|
63702
|
+
error: error instanceof Error ? error.message : String(error)
|
|
63703
|
+
});
|
|
63704
|
+
}
|
|
63705
|
+
}
|
|
63706
|
+
async answer(id, request) {
|
|
63707
|
+
if (request.kind === "pending-stage-message" || request.kind === "sticky-live-delivered") {
|
|
63708
|
+
const router = this.store === undefined ? undefined : pendingStageMessageRouter(this.store);
|
|
63709
|
+
const event = {
|
|
63710
|
+
...request.payload,
|
|
63711
|
+
handled: false
|
|
63712
|
+
};
|
|
63713
|
+
if (request.kind === "pending-stage-message")
|
|
63714
|
+
router?.route(event);
|
|
63715
|
+
else
|
|
63716
|
+
router?.sticky(event);
|
|
63717
|
+
if (event.handled && event.completion !== undefined)
|
|
63718
|
+
return await event.completion;
|
|
63719
|
+
return request.kind === "pending-stage-message" ? { outcome: "refused", reason: "Session not found" } : undefined;
|
|
63720
|
+
}
|
|
63721
|
+
const handoff = this.handoff;
|
|
63722
|
+
const work = this.track(request.work);
|
|
63723
|
+
if (handoff === undefined || work === undefined)
|
|
63724
|
+
throw new Error(STALE_WORK_LEASE);
|
|
63725
|
+
if (request.kind === "ready")
|
|
63726
|
+
return handoff.ready(work, request.runId, request.stageId, request.stageName, request.sessionId);
|
|
63727
|
+
return handoff.deliver(work, request.runId, request.stageId, request.stageName, request.recipient, async (from, message) => {
|
|
63728
|
+
await this.command({ kind: "transport", request: id, from, message });
|
|
63729
|
+
});
|
|
63730
|
+
}
|
|
62566
63731
|
closing;
|
|
62567
63732
|
close() {
|
|
62568
63733
|
this.closing ??= this.dispose();
|
|
@@ -62571,27 +63736,28 @@ class LocalWorkerPool {
|
|
|
62571
63736
|
async dispose() {
|
|
62572
63737
|
this.closed = true;
|
|
62573
63738
|
clearInterval(this.ticker);
|
|
63739
|
+
this.ticker = undefined;
|
|
62574
63740
|
this.releaseBinding();
|
|
62575
63741
|
if (this.options.rootId === undefined)
|
|
62576
63742
|
workflowResourceAdmission.delete(this.backend);
|
|
63743
|
+
this.signalChange();
|
|
62577
63744
|
let failure;
|
|
62578
|
-
|
|
62579
|
-
|
|
62580
|
-
|
|
62581
|
-
|
|
62582
|
-
|
|
62583
|
-
|
|
62584
|
-
|
|
62585
|
-
|
|
62586
|
-
|
|
62587
|
-
|
|
62588
|
-
|
|
62589
|
-
|
|
62590
|
-
const stopped = stopping.flatMap(({ child, log }) => log !== undefined && (child.exitCode !== null || child.signalCode !== null) ? [log] : []);
|
|
62591
|
-
await Promise.all(stopped.map((log) => rm3(log, { force: true }).catch(() => {})));
|
|
62592
|
-
if (this.logDirectory !== undefined)
|
|
62593
|
-
await rmdir(this.logDirectory).catch(() => {});
|
|
63745
|
+
const coordinator = this.coordinator;
|
|
63746
|
+
if (coordinator !== undefined) {
|
|
63747
|
+
if (this.coordinatorLost === undefined) {
|
|
63748
|
+
try {
|
|
63749
|
+
await this.command({ kind: "close" });
|
|
63750
|
+
} catch (error) {
|
|
63751
|
+
failure = error;
|
|
63752
|
+
}
|
|
63753
|
+
}
|
|
63754
|
+
await this.inbound;
|
|
63755
|
+
await coordinator.port.close();
|
|
63756
|
+
this.pids.clear();
|
|
62594
63757
|
}
|
|
63758
|
+
for (const release of this.routeClaims.values())
|
|
63759
|
+
release();
|
|
63760
|
+
this.routeClaims.clear();
|
|
62595
63761
|
try {
|
|
62596
63762
|
await this.releaseRoutes?.();
|
|
62597
63763
|
} catch (error) {
|
|
@@ -62601,27 +63767,26 @@ class LocalWorkerPool {
|
|
|
62601
63767
|
throw failure;
|
|
62602
63768
|
}
|
|
62603
63769
|
}
|
|
62604
|
-
var INTERNAL_WORKFLOW_WORKER_ARG = "--internal-workflow-worker", DEFAULT_WORK_LEASE_MS = 30000, MINIMUM_WORK_LEASE_MS,
|
|
63770
|
+
var INTERNAL_WORKFLOW_WORKER_ARG = "--internal-workflow-worker", DEFAULT_WORK_LEASE_MS = 30000, MINIMUM_WORK_LEASE_MS, DEFAULT_STARTUP_TIMEOUT_MS = 30000, MAX_TIMER_DELAY_MS = 2147483647, COORDINATOR_STDERR_TAIL_CHARACTERS = 8192, COORDINATOR_EXIT_GRACE_MS = 5000;
|
|
62605
63771
|
var init_local_pool = __esm(() => {
|
|
62606
63772
|
init_internal();
|
|
62607
63773
|
init_sql_pool_queue();
|
|
62608
|
-
init_sql_pool_usage();
|
|
62609
63774
|
init_sql_run_journal();
|
|
62610
63775
|
init_workflow_tool_abort();
|
|
62611
63776
|
init_sdk_intercom();
|
|
63777
|
+
init_external_run_routes();
|
|
62612
63778
|
init_workflow_infrastructure_error();
|
|
62613
63779
|
init_admission_binding();
|
|
62614
63780
|
init_definitions();
|
|
62615
63781
|
init_dispatch();
|
|
62616
63782
|
init_host_options();
|
|
62617
|
-
|
|
63783
|
+
init_pool_coordinator();
|
|
62618
63784
|
init_stage_handoff();
|
|
62619
63785
|
MINIMUM_WORK_LEASE_MS = 2 * WORKER_HEARTBEAT_INTERVAL_MS;
|
|
62620
|
-
COORDINATOR_LEASE_MS = 3 * STAGE_SESSION_HEARTBEAT_INTERVAL_MS;
|
|
62621
63786
|
});
|
|
62622
63787
|
|
|
62623
63788
|
// dist/builtin/workflows/src/workers/public-pool.ts
|
|
62624
|
-
import { resolve as
|
|
63789
|
+
import { resolve as resolve13 } from "node:path";
|
|
62625
63790
|
import { isDeepStrictEqual as isDeepStrictEqual2 } from "node:util";
|
|
62626
63791
|
async function restoreWorkerOptions(backend, options) {
|
|
62627
63792
|
const source = options.continuation?.source.id ?? options.runId;
|
|
@@ -62660,11 +63825,11 @@ async function activatePublicPool(backend, options) {
|
|
|
62660
63825
|
...options.localWorkers,
|
|
62661
63826
|
startupTimeoutMs: undefined,
|
|
62662
63827
|
leaseMs: undefined,
|
|
62663
|
-
definitions: options.localWorkers.definitions.map((source) => ({ ...source, path:
|
|
63828
|
+
definitions: options.localWorkers.definitions.map((source) => ({ ...source, path: resolve13(cwd, source.path) }))
|
|
62664
63829
|
};
|
|
62665
63830
|
const admission = options.admission && {
|
|
62666
63831
|
...options.admission,
|
|
62667
|
-
...options.admission.userDatabase ? { userDatabase:
|
|
63832
|
+
...options.admission.userDatabase ? { userDatabase: resolve13(cwd, options.admission.userDatabase) } : {}
|
|
62668
63833
|
};
|
|
62669
63834
|
const sources = localWorkers && await Promise.all(localWorkers.definitions.map(async (source) => ({ ...source, digest: await definitionDigest(source.path) })));
|
|
62670
63835
|
const resources = JSON.parse(JSON.stringify({
|
|
@@ -64010,7 +65175,7 @@ function hasUsableContentBlock(block) {
|
|
|
64010
65175
|
|
|
64011
65176
|
// dist/builtin/workflows/src/shared/stage-session-store.ts
|
|
64012
65177
|
import { randomUUID as randomUUID18 } from "node:crypto";
|
|
64013
|
-
import { dirname as dirname13, resolve as
|
|
65178
|
+
import { dirname as dirname13, resolve as resolve14 } from "node:path";
|
|
64014
65179
|
import {
|
|
64015
65180
|
JsonlSessionStore,
|
|
64016
65181
|
resolveSessionStore,
|
|
@@ -64046,11 +65211,11 @@ async function forkStageSession(store, sessionId, cwd, options, destination = st
|
|
|
64046
65211
|
async function migrateLegacyStageSession(store, sessionId, sessionFile) {
|
|
64047
65212
|
if (sessionFile === undefined || await store.readHeader(sessionId) !== undefined)
|
|
64048
65213
|
return;
|
|
64049
|
-
const location =
|
|
65214
|
+
const location = resolve14(sessionFile);
|
|
64050
65215
|
const legacy = await JsonlSessionStore.open(dirname13(location), { exactDirectory: true, scanExisting: false });
|
|
64051
65216
|
try {
|
|
64052
65217
|
for await (const file of legacy.scanFiles()) {
|
|
64053
|
-
if (
|
|
65218
|
+
if (resolve14(file.location) !== location || file.header.id !== sessionId)
|
|
64054
65219
|
continue;
|
|
64055
65220
|
const snapshot = await file.read();
|
|
64056
65221
|
if (await store.readHeader(sessionId) === undefined)
|
|
@@ -64212,22 +65377,37 @@ function childScopedEvidenceExists(checkpoints, boundaryReplayKey, childRunId) {
|
|
|
64212
65377
|
}
|
|
64213
65378
|
function retainReachableRunGroups(grouped, rootRunId) {
|
|
64214
65379
|
const reachable = new Set([rootRunId]);
|
|
64215
|
-
const
|
|
65380
|
+
const settled = new Set;
|
|
65381
|
+
const pending = [{ runId: rootRunId, live: true }];
|
|
64216
65382
|
while (pending.length > 0) {
|
|
64217
|
-
const
|
|
65383
|
+
const { runId, live } = pending.pop();
|
|
65384
|
+
const drafts = grouped.get(runId);
|
|
64218
65385
|
if (drafts === undefined)
|
|
64219
65386
|
continue;
|
|
64220
65387
|
for (const draft of drafts) {
|
|
64221
|
-
const childRunId =
|
|
64222
|
-
if (childRunId === undefined ||
|
|
65388
|
+
const childRunId = workflowChildFromDraft(draft)?.runId ?? draft.topology?.boundary?.child.runId;
|
|
65389
|
+
if (childRunId === undefined || !grouped.has(childRunId))
|
|
64223
65390
|
continue;
|
|
64224
|
-
|
|
64225
|
-
|
|
65391
|
+
if (live && childRunIdFromDraft(draft) !== undefined) {
|
|
65392
|
+
if (reachable.has(childRunId))
|
|
65393
|
+
continue;
|
|
65394
|
+
reachable.add(childRunId);
|
|
65395
|
+
pending.push({ runId: childRunId, live: true });
|
|
65396
|
+
} else if (!settled.has(childRunId)) {
|
|
65397
|
+
settled.add(childRunId);
|
|
65398
|
+
pending.push({ runId: childRunId, live: false });
|
|
65399
|
+
}
|
|
64226
65400
|
}
|
|
64227
65401
|
}
|
|
64228
|
-
|
|
64229
|
-
|
|
64230
|
-
|
|
65402
|
+
let complete = true;
|
|
65403
|
+
for (const runId of grouped.keys()) {
|
|
65404
|
+
if (reachable.has(runId))
|
|
65405
|
+
continue;
|
|
65406
|
+
if (!settled.has(runId))
|
|
65407
|
+
complete = false;
|
|
65408
|
+
grouped.delete(runId);
|
|
65409
|
+
}
|
|
65410
|
+
return complete;
|
|
64231
65411
|
}
|
|
64232
65412
|
function childRunIdFromDraft(draft) {
|
|
64233
65413
|
const boundary = draft.topology?.boundary;
|
|
@@ -64649,9 +65829,7 @@ function runSnapshotsFromCheckpoints(checkpoints, rootRunId, rootRunName, fallba
|
|
|
64649
65829
|
}
|
|
64650
65830
|
for (const group of grouped.values())
|
|
64651
65831
|
group.sort(compareDraftSourceOrder);
|
|
64652
|
-
|
|
64653
|
-
retainReachableRunGroups(grouped, rootRunId);
|
|
64654
|
-
if (policy.failOnUnreachable && grouped.size !== groupedCount)
|
|
65832
|
+
if (!retainReachableRunGroups(grouped, rootRunId) && policy.failOnUnreachable)
|
|
64655
65833
|
return [];
|
|
64656
65834
|
if (!validateRunGroups(grouped, rootRunId, toolCheckpoints, checkpoints, policy.retainActive))
|
|
64657
65835
|
return [];
|
|
@@ -70309,9 +71487,9 @@ var init_stage_runner_controller = __esm(() => {
|
|
|
70309
71487
|
});
|
|
70310
71488
|
|
|
70311
71489
|
// dist/builtin/workflows/src/runs/foreground/stage-runner-output.ts
|
|
70312
|
-
import { createHash as
|
|
71490
|
+
import { createHash as createHash5 } from "node:crypto";
|
|
70313
71491
|
import { mkdir as mkdir7, writeFile as writeFile10 } from "node:fs/promises";
|
|
70314
|
-
import { basename as basename5, dirname as dirname14, isAbsolute as isAbsolute11, join as join29, resolve as
|
|
71492
|
+
import { basename as basename5, dirname as dirname14, isAbsolute as isAbsolute11, join as join29, resolve as resolve15 } from "node:path";
|
|
70315
71493
|
function normalizeMaxOutput2(maxOutput) {
|
|
70316
71494
|
return {
|
|
70317
71495
|
bytes: maxOutput?.bytes ?? DEFAULT_MAX_OUTPUT_BYTES,
|
|
@@ -70373,7 +71551,7 @@ function formatByteSize(bytes) {
|
|
|
70373
71551
|
return `${value2.toFixed(1)} ${units[unitIndex]}`;
|
|
70374
71552
|
}
|
|
70375
71553
|
function transcriptPath(runId, outputPath) {
|
|
70376
|
-
const digest =
|
|
71554
|
+
const digest = createHash5("sha256").update(outputPath).digest("hex").slice(0, 16);
|
|
70377
71555
|
return join29(workflowArtifactRunPath(runId), "transcripts", `${digest}-${basename5(outputPath)}.transcript.md`);
|
|
70378
71556
|
}
|
|
70379
71557
|
function appendContent(lines, content) {
|
|
@@ -70419,7 +71597,7 @@ function renderTranscript(messages, fallbackOutput) {
|
|
|
70419
71597
|
`);
|
|
70420
71598
|
}
|
|
70421
71599
|
function savedOutputReference(outputPath, fullOutput, transcriptPathValue, transcriptText, transcriptAvailable, warning) {
|
|
70422
|
-
const absolutePath =
|
|
71600
|
+
const absolutePath = resolve15(outputPath);
|
|
70423
71601
|
const bytes = Buffer.byteLength(fullOutput, "utf8");
|
|
70424
71602
|
const lines = countLines(fullOutput);
|
|
70425
71603
|
const transcriptBytes = Buffer.byteLength(transcriptText, "utf8");
|
|
@@ -70434,8 +71612,8 @@ function resolveOutputPath(output, runtimeCwd, requestedCwd) {
|
|
|
70434
71612
|
return;
|
|
70435
71613
|
if (isAbsolute11(output))
|
|
70436
71614
|
return output;
|
|
70437
|
-
const baseCwd = requestedCwd === undefined ? runtimeCwd : isAbsolute11(requestedCwd) ? requestedCwd :
|
|
70438
|
-
return
|
|
71615
|
+
const baseCwd = requestedCwd === undefined ? runtimeCwd : isAbsolute11(requestedCwd) ? requestedCwd : resolve15(runtimeCwd, requestedCwd);
|
|
71616
|
+
return resolve15(baseCwd, output);
|
|
70439
71617
|
}
|
|
70440
71618
|
function degenerateWarning(fullOutput) {
|
|
70441
71619
|
return fullOutput.trim().length === 0 ? "WARNING: the stage artifact is empty; search the companion transcript for this stage's work." : undefined;
|
|
@@ -71444,7 +72622,7 @@ async function run(def, inputs, opts = {}) {
|
|
|
71444
72622
|
if (!isWorkflowDefinition(def))
|
|
71445
72623
|
throw new Error(workflowDefinitionRequirementMessage("run(definition, inputs)", def));
|
|
71446
72624
|
const depth = opts.depth ?? 0;
|
|
71447
|
-
const maxDepth = opts.config?.maxDepth ??
|
|
72625
|
+
const maxDepth = opts.config?.maxDepth ?? 3;
|
|
71448
72626
|
if (depth >= maxDepth) {
|
|
71449
72627
|
return {
|
|
71450
72628
|
runId: opts.runId ?? crypto.randomUUID(),
|
|
@@ -72533,7 +73711,7 @@ init_admission_binding();
|
|
|
72533
73711
|
init_definitions();
|
|
72534
73712
|
init_resource_admission();
|
|
72535
73713
|
import { randomUUID as randomUUID20 } from "node:crypto";
|
|
72536
|
-
import { setTimeout as
|
|
73714
|
+
import { setTimeout as delay4 } from "node:timers/promises";
|
|
72537
73715
|
import { withResourceAdmission as withResourceAdmission2 } from "@bastani/atomic";
|
|
72538
73716
|
import { StorageProvider as StorageProvider3 } from "@bastani/atomic/storage";
|
|
72539
73717
|
|
|
@@ -72542,8 +73720,8 @@ init_stage_topology();
|
|
|
72542
73720
|
init_run_durable_topology();
|
|
72543
73721
|
import { randomUUID as randomUUID19 } from "node:crypto";
|
|
72544
73722
|
function workerSessionStore(input) {
|
|
72545
|
-
const checkpoint =
|
|
72546
|
-
const owner =
|
|
73723
|
+
const checkpoint = (id, header) => {
|
|
73724
|
+
const owner = header.workflow;
|
|
72547
73725
|
const run = input.store.runs().find((run) => run.id === owner?.runId);
|
|
72548
73726
|
const stage = run?.stages.find((stage) => stage.id === owner?.stageId);
|
|
72549
73727
|
if (!run || !stage?.replayKey)
|
|
@@ -72571,11 +73749,11 @@ function workerSessionStore(input) {
|
|
|
72571
73749
|
};
|
|
72572
73750
|
return {
|
|
72573
73751
|
descriptor: input.reader.descriptor,
|
|
72574
|
-
create: async (header, entries) => write({ kind: "session-create", checkpoint:
|
|
72575
|
-
append: async (id, entries, options) => write({ kind: "session-append",
|
|
73752
|
+
create: async (header, entries) => write({ kind: "session-create", checkpoint: checkpoint(header.id, header), header, entries }),
|
|
73753
|
+
append: async (id, entries, options) => write({ kind: "session-append", sessionId: id, entries, expectSeq: options.expectSeq }),
|
|
72576
73754
|
replace: async (id, header, entries, options) => write({
|
|
72577
73755
|
kind: "session-replace",
|
|
72578
|
-
checkpoint:
|
|
73756
|
+
checkpoint: checkpoint(id, header),
|
|
72579
73757
|
header,
|
|
72580
73758
|
entries,
|
|
72581
73759
|
expectSeq: options.expectSeq
|
|
@@ -72603,6 +73781,14 @@ function collectWorkerUsage(runs) {
|
|
|
72603
73781
|
|
|
72604
73782
|
// dist/builtin/workflows/src/workers/local-worker.ts
|
|
72605
73783
|
async function runLocalWorker(start) {
|
|
73784
|
+
if ("role" in start && start.role === POOL_COORDINATOR_ROLE) {
|
|
73785
|
+
await Promise.resolve().then(() => init_pool_coordinator());
|
|
73786
|
+
runPoolCoordinator();
|
|
73787
|
+
return;
|
|
73788
|
+
}
|
|
73789
|
+
await runWorker(start);
|
|
73790
|
+
}
|
|
73791
|
+
async function runWorker(start) {
|
|
72606
73792
|
let sequence = 0;
|
|
72607
73793
|
const pending = new Map;
|
|
72608
73794
|
process.on("message", (reply) => {
|
|
@@ -72726,13 +73912,13 @@ async function runLocalWorker(start) {
|
|
|
72726
73912
|
for (;; ) {
|
|
72727
73913
|
await observeControl();
|
|
72728
73914
|
if (controlState !== "running") {
|
|
72729
|
-
await
|
|
73915
|
+
await delay4(25);
|
|
72730
73916
|
continue;
|
|
72731
73917
|
}
|
|
72732
73918
|
await reader.catchUpWorkflow(start.root);
|
|
72733
73919
|
const work = await request({ kind: "claim" });
|
|
72734
73920
|
if (!work) {
|
|
72735
|
-
await
|
|
73921
|
+
await delay4(25 + Math.random() * 25);
|
|
72736
73922
|
continue;
|
|
72737
73923
|
}
|
|
72738
73924
|
const index = start.definitions.findIndex((source) => `${source.path}#${source.exportName}` === work.definition && source.digest === work.digest);
|
|
@@ -72809,7 +73995,7 @@ async function runLocalWorker(start) {
|
|
|
72809
73995
|
const control = await request({ kind: "budget" });
|
|
72810
73996
|
if (control.state === "running")
|
|
72811
73997
|
return;
|
|
72812
|
-
await
|
|
73998
|
+
await delay4(25, undefined, { signal });
|
|
72813
73999
|
}
|
|
72814
74000
|
};
|
|
72815
74001
|
const resourceAdmission = {
|
|
@@ -72848,7 +74034,7 @@ async function runLocalWorker(start) {
|
|
|
72848
74034
|
await observeControl();
|
|
72849
74035
|
while ((await request({ kind: "control" })).state !== "running") {
|
|
72850
74036
|
controller.signal.throwIfAborted();
|
|
72851
|
-
await
|
|
74037
|
+
await delay4(25);
|
|
72852
74038
|
}
|
|
72853
74039
|
const result = await withResourceAdmission2(resourceAdmission, () => run(definitions[index], work.inputs, {
|
|
72854
74040
|
...work.execution,
|