@bastani/atomic 0.9.43 → 0.9.44-alpha.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +42 -0
- package/dist/builtin/intercom/CHANGELOG.md +22 -0
- package/dist/builtin/intercom/broker/broker.ts +31 -7
- package/dist/builtin/intercom/broker/client-message-validation.ts +6 -0
- package/dist/builtin/intercom/broker/client.ts +69 -0
- package/dist/builtin/intercom/broker/send-signature.ts +4 -1
- package/dist/builtin/intercom/index.bundle.mjs +309 -109
- package/dist/builtin/intercom/live-route-refusal.ts +20 -0
- package/dist/builtin/intercom/package.json +1 -1
- package/dist/builtin/intercom/skills/intercom/SKILL.md +11 -6
- package/dist/builtin/intercom/types.ts +28 -1
- package/dist/builtin/mcp/CHANGELOG.md +7 -0
- package/dist/builtin/mcp/package.json +1 -1
- package/dist/builtin/subagents/CHANGELOG.md +9 -0
- package/dist/builtin/subagents/README.md +8 -8
- package/dist/builtin/subagents/agents/worker.md +1 -1
- package/dist/builtin/subagents/package.json +1 -1
- package/dist/builtin/subagents/skills/agent-browser/SKILL.md +1 -1
- package/dist/builtin/subagents/skills/cua-driver/EMBEDDING.md +21 -2
- package/dist/builtin/subagents/skills/cua-driver/LINUX.md +2 -2
- package/dist/builtin/subagents/skills/cua-driver/MACOS.md +342 -10
- package/dist/builtin/subagents/skills/cua-driver/README.md +4 -3
- package/dist/builtin/subagents/skills/cua-driver/RUNTIME.md +3 -1
- package/dist/builtin/subagents/skills/cua-driver/SKILL.md +7 -4
- package/dist/builtin/subagents/skills/cua-driver/WINDOWS.md +6 -5
- package/dist/builtin/subagents/skills/cua-driver/WORKFLOW.md +15 -0
- package/dist/builtin/subagents/skills/herdr/SKILL.md +22 -10
- package/dist/builtin/subagents/skills/subagent/SKILL.md +8 -5
- package/dist/builtin/subagents/src/extension/index.bundle.mjs +6 -6
- package/dist/builtin/web-access/package.json +1 -1
- package/dist/builtin/workflows/CHANGELOG.md +20 -0
- package/dist/builtin/workflows/README.md +1 -1
- package/dist/builtin/workflows/builtin/{chunk-frqxd4d1.js → chunk-k8d7ehjx.js} +22 -0
- package/dist/builtin/workflows/builtin/{chunk-3s95f7ry.js → chunk-qxpymnaf.js} +1 -1
- package/dist/builtin/workflows/builtin/{chunk-edhf30ef.js → chunk-zjecnjxx.js} +1 -1
- package/dist/builtin/workflows/builtin/goal.js +2 -2
- package/dist/builtin/workflows/builtin/index.js +3 -3
- package/dist/builtin/workflows/builtin/ralph.js +2 -2
- package/dist/builtin/workflows/package.json +1 -1
- package/dist/builtin/workflows/src/extension/index.bundle.mjs +1844 -603
- package/dist/builtin/workflows/src/extension/local-worker-entry.mjs +1880 -605
- package/dist/builtin/workflows/src/index.js +1896 -655
- package/dist/builtin/workflows/src/local-worker-entry.mjs +1880 -605
- package/dist/cli/args.js +1 -1
- package/dist/cli/args.js.map +1 -1
- package/dist/core/agent-session-custom-message-commit.d.ts.map +1 -1
- package/dist/core/agent-session-custom-message-commit.js +59 -0
- package/dist/core/agent-session-custom-message-commit.js.map +1 -1
- package/dist/core/agent-session-methods.d.ts +1 -0
- package/dist/core/agent-session-methods.d.ts.map +1 -1
- package/dist/core/agent-session-methods.js.map +1 -1
- package/dist/core/agent-session-prompt.d.ts +4 -0
- package/dist/core/agent-session-prompt.d.ts.map +1 -1
- package/dist/core/agent-session-prompt.js +10 -2
- package/dist/core/agent-session-prompt.js.map +1 -1
- package/dist/core/agent-session.d.ts +2 -0
- package/dist/core/agent-session.d.ts.map +1 -1
- package/dist/core/agent-session.js +9 -0
- package/dist/core/agent-session.js.map +1 -1
- package/dist/core/builtin-packages.d.ts.map +1 -1
- package/dist/core/builtin-packages.js +13 -13
- package/dist/core/builtin-packages.js.map +1 -1
- package/dist/core/exec.d.ts.map +1 -1
- package/dist/core/exec.js +11 -2
- package/dist/core/exec.js.map +1 -1
- package/dist/core/package-manager-resolver.d.ts.map +1 -1
- package/dist/core/package-manager-resolver.js +25 -14
- package/dist/core/package-manager-resolver.js.map +1 -1
- package/dist/core/package-manager-types.d.ts +8 -0
- package/dist/core/package-manager-types.d.ts.map +1 -1
- package/dist/core/package-manager-types.js.map +1 -1
- package/dist/core/package-manager.d.ts +1 -1
- package/dist/core/package-manager.d.ts.map +1 -1
- package/dist/core/package-manager.js.map +1 -1
- package/dist/core/resource-loader-extension-warnings.d.ts +4 -1
- package/dist/core/resource-loader-extension-warnings.d.ts.map +1 -1
- package/dist/core/resource-loader-extension-warnings.js +15 -0
- package/dist/core/resource-loader-extension-warnings.js.map +1 -1
- package/dist/core/resource-loader-package-resources.d.ts.map +1 -1
- package/dist/core/resource-loader-package-resources.js +1 -0
- package/dist/core/resource-loader-package-resources.js.map +1 -1
- package/dist/core/resource-loader-reload.d.ts.map +1 -1
- package/dist/core/resource-loader-reload.js +5 -3
- package/dist/core/resource-loader-reload.js.map +1 -1
- package/dist/extensions/mcp/client/oauth/flow.d.ts.map +1 -1
- package/dist/extensions/mcp/client/oauth/flow.js +4 -1
- package/dist/extensions/mcp/client/oauth/flow.js.map +1 -1
- package/dist/extensions/mcp/client/oauth/types.d.ts +1 -0
- package/dist/extensions/mcp/client/oauth/types.d.ts.map +1 -1
- package/dist/extensions/mcp/client/oauth/types.js +8 -4
- package/dist/extensions/mcp/client/oauth/types.js.map +1 -1
- package/dist/extensions/mcp/client/transports/streamable-http.d.ts.map +1 -1
- package/dist/extensions/mcp/client/transports/streamable-http.js +2 -1
- package/dist/extensions/mcp/client/transports/streamable-http.js.map +1 -1
- package/dist/index.d.ts +1 -1
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js.map +1 -1
- package/dist/main.d.ts.map +1 -1
- package/dist/main.js +20 -10
- package/dist/main.js.map +1 -1
- package/dist/modes/interactive/components/model-selector.d.ts.map +1 -1
- package/dist/modes/interactive/components/model-selector.js +4 -2
- package/dist/modes/interactive/components/model-selector.js.map +1 -1
- package/dist/modes/interactive/components/scoped-models-selector.d.ts.map +1 -1
- package/dist/modes/interactive/components/scoped-models-selector.js +4 -2
- package/dist/modes/interactive/components/scoped-models-selector.js.map +1 -1
- package/dist/modes/interactive/components/tool-execution.d.ts +6 -0
- package/dist/modes/interactive/components/tool-execution.d.ts.map +1 -1
- package/dist/modes/interactive/components/tool-execution.js +11 -1
- package/dist/modes/interactive/components/tool-execution.js.map +1 -1
- package/dist/modes/interactive/model-search.d.ts +10 -0
- package/dist/modes/interactive/model-search.d.ts.map +1 -1
- package/dist/modes/interactive/model-search.js +21 -0
- package/dist/modes/interactive/model-search.js.map +1 -1
- package/dist/package-manager-cli.d.ts.map +1 -1
- package/dist/package-manager-cli.js +5 -0
- package/dist/package-manager-cli.js.map +1 -1
- package/dist/utils/tools-manager.d.ts.map +1 -1
- package/dist/utils/tools-manager.js +30 -35
- package/dist/utils/tools-manager.js.map +1 -1
- package/docs/computer-use.md +5 -5
- package/docs/custom-provider/streaming.md +9 -14
- package/docs/environment-variables.md +1 -0
- package/docs/intercom/operations.md +1 -1
- package/docs/intercom.md +16 -2
- package/docs/packages.md +2 -0
- package/docs/settings.md +2 -2
- package/docs/skills.md +2 -2
- package/docs/subagents.md +47 -10
- package/docs/workflows/api-reference.md +1 -1
- package/docs/workflows/authoring.md +1 -1
- package/docs/workflows/operations.md +8 -6
- package/docs/workflows/reliable-design.md +23 -5
- package/examples/extensions/custom-provider-anthropic/index.ts +31 -31
- package/examples/extensions/todo.ts +7 -0
- package/npm-shrinkwrap.json +32 -32
- package/package.json +3 -3
|
@@ -1661,6 +1661,13 @@ function removeProcessExitCleanup(cleanup) {
|
|
|
1661
1661
|
if (processExitCleanups.delete(cleanup) && processExitCleanups.size === 0)
|
|
1662
1662
|
process.off("exit", runProcessExitCleanups);
|
|
1663
1663
|
}
|
|
1664
|
+
function snapshotNeedsHistory(tail) {
|
|
1665
|
+
const event = tail.snapshot.event;
|
|
1666
|
+
if (event.type !== RUN_EVENT_TYPES.messageQueued)
|
|
1667
|
+
return false;
|
|
1668
|
+
const payload = event.payload;
|
|
1669
|
+
return hasPendingStageDeliveryDelta(payload.pendingStageMessages);
|
|
1670
|
+
}
|
|
1664
1671
|
function* invoke(action) {
|
|
1665
1672
|
try {
|
|
1666
1673
|
const result = action();
|
|
@@ -1776,8 +1783,11 @@ class JournalDurableBackend {
|
|
|
1776
1783
|
this.projection.applyWorkflowSnapshot(event.payload.handle);
|
|
1777
1784
|
this.prompts.set(entry.runId, new PromptReservationState(event.payload.handle.pendingPrompts, event.payload.prompts));
|
|
1778
1785
|
} else {
|
|
1779
|
-
const handle = event.payload;
|
|
1780
1786
|
const previous = this.projection.getWorkflow(entry.runId);
|
|
1787
|
+
const handle = event.type === RUN_EVENT_TYPES.messageQueued && hasPendingStageDeliveryDelta(event.payload.pendingStageMessages) ? {
|
|
1788
|
+
...event.payload,
|
|
1789
|
+
pendingStageMessages: decodePendingStageDeliveries(previous?.pendingStageMessages ?? [], event.payload.pendingStageMessages ?? [])
|
|
1790
|
+
} : event.payload;
|
|
1781
1791
|
this.projection.applyWorkflowSnapshot(handle);
|
|
1782
1792
|
if (event.type === RUN_EVENT_TYPES.runStarted || previous?.pendingPrompts !== handle.pendingPrompts)
|
|
1783
1793
|
this.prompts.delete(entry.runId);
|
|
@@ -2073,11 +2083,23 @@ class JournalDurableBackend {
|
|
|
2073
2083
|
for (const message of messages)
|
|
2074
2084
|
if (message.runId !== logicalRunId)
|
|
2075
2085
|
throw new JournalMessageScopeError(logicalRunId, message.runId);
|
|
2076
|
-
|
|
2086
|
+
const next = {
|
|
2077
2087
|
...handle,
|
|
2078
2088
|
pendingStageMessages: replacePendingStageMessagesForRun(handle.pendingStageMessages ?? [], logicalRunId, messages),
|
|
2079
2089
|
updatedAt: Math.max(Date.now(), handle.updatedAt + 1)
|
|
2080
|
-
}
|
|
2090
|
+
};
|
|
2091
|
+
const encoded = encodePendingStageDeliveries(handle.pendingStageMessages ?? [], next.pendingStageMessages ?? []);
|
|
2092
|
+
const base = this.sequences.get(runId);
|
|
2093
|
+
if (encoded !== undefined && base !== undefined) {
|
|
2094
|
+
try {
|
|
2095
|
+
yield* this.append(runId, { type: RUN_EVENT_TYPES.messageQueued, payload: { ...next, pendingStageMessages: encoded } }, { expectSeq: base });
|
|
2096
|
+
return true;
|
|
2097
|
+
} catch (error) {
|
|
2098
|
+
if (!(error instanceof JournalCasConflictError))
|
|
2099
|
+
throw error;
|
|
2100
|
+
}
|
|
2101
|
+
}
|
|
2102
|
+
yield* this.snapshot(next, RUN_EVENT_TYPES.messageQueued);
|
|
2081
2103
|
return true;
|
|
2082
2104
|
}
|
|
2083
2105
|
async archivePendingStageMessage(runId, message) {
|
|
@@ -2171,7 +2193,7 @@ class JournalDurableBackend {
|
|
|
2171
2193
|
if (summarized !== undefined && summarized.seq === version.seq && summarized.at === version.at)
|
|
2172
2194
|
return;
|
|
2173
2195
|
const tail = yield* invoke(() => journal.snapshotTail(runId));
|
|
2174
|
-
if (tail !== undefined)
|
|
2196
|
+
if (tail !== undefined && !snapshotNeedsHistory(tail))
|
|
2175
2197
|
return this.summarize(runId, tail);
|
|
2176
2198
|
if (summarized === undefined && version.seq === 0)
|
|
2177
2199
|
return;
|
|
@@ -2397,6 +2419,69 @@ class JournalDurableBackend {
|
|
|
2397
2419
|
}
|
|
2398
2420
|
}
|
|
2399
2421
|
}
|
|
2422
|
+
function identity(message) {
|
|
2423
|
+
return JSON.stringify([message.runId, message.id]);
|
|
2424
|
+
}
|
|
2425
|
+
function uniqueByIdentity(messages) {
|
|
2426
|
+
const result = new Map;
|
|
2427
|
+
const repeated = new Set;
|
|
2428
|
+
for (const message of messages) {
|
|
2429
|
+
const key = identity(message);
|
|
2430
|
+
if (result.has(key))
|
|
2431
|
+
repeated.add(key);
|
|
2432
|
+
result.set(key, message);
|
|
2433
|
+
}
|
|
2434
|
+
for (const key of repeated)
|
|
2435
|
+
result.delete(key);
|
|
2436
|
+
return result;
|
|
2437
|
+
}
|
|
2438
|
+
function isDeltaEncoded(message) {
|
|
2439
|
+
return message.deliveryDelta !== undefined;
|
|
2440
|
+
}
|
|
2441
|
+
function encodePendingStageDeliveries(previous, next) {
|
|
2442
|
+
const bases = uniqueByIdentity(previous);
|
|
2443
|
+
const nextIdentities = uniqueByIdentity(next);
|
|
2444
|
+
let encodedAny = false;
|
|
2445
|
+
const encoded = next.map((message) => {
|
|
2446
|
+
const base = bases.get(identity(message))?.deliveries;
|
|
2447
|
+
const deliveries = message.deliveries;
|
|
2448
|
+
if (base === undefined || deliveries === undefined || !nextIdentities.has(identity(message)))
|
|
2449
|
+
return message;
|
|
2450
|
+
const drop = [];
|
|
2451
|
+
let kept = 0;
|
|
2452
|
+
base.forEach((delivery, index) => {
|
|
2453
|
+
if (kept < deliveries.length && JSON.stringify(delivery) === JSON.stringify(deliveries[kept]))
|
|
2454
|
+
kept++;
|
|
2455
|
+
else
|
|
2456
|
+
drop.push(index);
|
|
2457
|
+
});
|
|
2458
|
+
const add = deliveries.slice(kept);
|
|
2459
|
+
if (drop.length + add.length >= deliveries.length)
|
|
2460
|
+
return message;
|
|
2461
|
+
encodedAny = true;
|
|
2462
|
+
const { deliveries: _full, ...rest } = message;
|
|
2463
|
+
return { ...rest, deliveryDelta: drop.length === 0 ? { add } : { drop, add } };
|
|
2464
|
+
});
|
|
2465
|
+
return encodedAny ? encoded : undefined;
|
|
2466
|
+
}
|
|
2467
|
+
function decodePendingStageDeliveries(previous, messages) {
|
|
2468
|
+
if (!messages.some(isDeltaEncoded))
|
|
2469
|
+
return messages;
|
|
2470
|
+
const bases = uniqueByIdentity(previous);
|
|
2471
|
+
return messages.map((message) => {
|
|
2472
|
+
if (!isDeltaEncoded(message))
|
|
2473
|
+
return message;
|
|
2474
|
+
const base = bases.get(identity(message))?.deliveries;
|
|
2475
|
+
if (base === undefined)
|
|
2476
|
+
throw new Error(`atomic-workflows: journal delivery change for ${identity(message)} has no base entry`);
|
|
2477
|
+
const { deliveryDelta, ...rest } = message;
|
|
2478
|
+
const dropped = new Set(deliveryDelta.drop ?? []);
|
|
2479
|
+
return { ...rest, deliveries: [...base.filter((_, index) => !dropped.has(index)), ...deliveryDelta.add] };
|
|
2480
|
+
});
|
|
2481
|
+
}
|
|
2482
|
+
function hasPendingStageDeliveryDelta(messages) {
|
|
2483
|
+
return messages?.some(isDeltaEncoded) === true;
|
|
2484
|
+
}
|
|
2400
2485
|
var PROCESS_HOLDER_ID, processExitCleanups, JournalOwnershipError, JournalBackendClosedError, JournalOperationAbortedError, JournalExecutionRetiredError, JournalMessageScopeError;
|
|
2401
2486
|
var init_journal_backend = __esm(() => {
|
|
2402
2487
|
init_workflow_instance_ownership_error();
|
|
@@ -8030,7 +8115,13 @@ function createStoreContext(state = createStoreState()) {
|
|
|
8030
8115
|
}
|
|
8031
8116
|
return cachedGraphSnapshot;
|
|
8032
8117
|
}
|
|
8118
|
+
let batchDepth = 0;
|
|
8119
|
+
let batchedNotification = false;
|
|
8033
8120
|
function notify() {
|
|
8121
|
+
if (batchDepth > 0) {
|
|
8122
|
+
batchedNotification = true;
|
|
8123
|
+
return;
|
|
8124
|
+
}
|
|
8034
8125
|
observation.capture();
|
|
8035
8126
|
for (const fn of state.invalidationListeners) {
|
|
8036
8127
|
try {
|
|
@@ -8050,6 +8141,18 @@ function createStoreContext(state = createStoreState()) {
|
|
|
8050
8141
|
state.version++;
|
|
8051
8142
|
notify();
|
|
8052
8143
|
}
|
|
8144
|
+
function batch(update) {
|
|
8145
|
+
batchDepth++;
|
|
8146
|
+
try {
|
|
8147
|
+
return update();
|
|
8148
|
+
} finally {
|
|
8149
|
+
batchDepth--;
|
|
8150
|
+
if (batchDepth === 0 && batchedNotification) {
|
|
8151
|
+
batchedNotification = false;
|
|
8152
|
+
notify();
|
|
8153
|
+
}
|
|
8154
|
+
}
|
|
8155
|
+
}
|
|
8053
8156
|
function findRun(runId) {
|
|
8054
8157
|
return state.runs.find((r) => r.id === runId);
|
|
8055
8158
|
}
|
|
@@ -8103,6 +8206,7 @@ function createStoreContext(state = createStoreState()) {
|
|
|
8103
8206
|
snapshot,
|
|
8104
8207
|
graphSnapshot,
|
|
8105
8208
|
notify,
|
|
8209
|
+
batch,
|
|
8106
8210
|
bumpAndNotify,
|
|
8107
8211
|
findRun,
|
|
8108
8212
|
findStage,
|
|
@@ -8918,6 +9022,9 @@ function createRunStoreMethods(context) {
|
|
|
8918
9022
|
subscribeInvalidation(fn) {
|
|
8919
9023
|
state.invalidationListeners.add(fn);
|
|
8920
9024
|
return () => state.invalidationListeners.delete(fn);
|
|
9025
|
+
},
|
|
9026
|
+
batch(update) {
|
|
9027
|
+
return context.batch(update);
|
|
8921
9028
|
}
|
|
8922
9029
|
};
|
|
8923
9030
|
}
|
|
@@ -13403,7 +13510,7 @@ class ConcurrencyLimiter {
|
|
|
13403
13510
|
}
|
|
13404
13511
|
}
|
|
13405
13512
|
function createRunLimiter(defaultConcurrency) {
|
|
13406
|
-
return new ConcurrencyLimiter(defaultConcurrency ??
|
|
13513
|
+
return new ConcurrencyLimiter(defaultConcurrency ?? 1000);
|
|
13407
13514
|
}
|
|
13408
13515
|
|
|
13409
13516
|
// dist/builtin/workflows/src/shared/abort.ts
|
|
@@ -57173,6 +57280,44 @@ var init_executor = __esm(() => {
|
|
|
57173
57280
|
init_executor_run();
|
|
57174
57281
|
});
|
|
57175
57282
|
|
|
57283
|
+
// dist/builtin/workflows/src/shared/external-run-routes.ts
|
|
57284
|
+
function claimExternalRunRoute(store, runId, owner) {
|
|
57285
|
+
let owners = registry.owners.get(store);
|
|
57286
|
+
if (owners === undefined) {
|
|
57287
|
+
owners = new Map;
|
|
57288
|
+
registry.owners.set(store, owners);
|
|
57289
|
+
}
|
|
57290
|
+
owners.set(runId, owner);
|
|
57291
|
+
return () => {
|
|
57292
|
+
const current = registry.owners.get(store);
|
|
57293
|
+
if (current?.get(runId) !== owner)
|
|
57294
|
+
return;
|
|
57295
|
+
current.delete(runId);
|
|
57296
|
+
if (current.size === 0)
|
|
57297
|
+
registry.owners.delete(store);
|
|
57298
|
+
};
|
|
57299
|
+
}
|
|
57300
|
+
function externalRunRouteOwner(store, runId) {
|
|
57301
|
+
return registry.owners.get(store)?.get(runId);
|
|
57302
|
+
}
|
|
57303
|
+
function registerPendingStageMessageRouter(store, router) {
|
|
57304
|
+
registry.routers.set(store, router);
|
|
57305
|
+
return () => {
|
|
57306
|
+
if (registry.routers.get(store) === router)
|
|
57307
|
+
registry.routers.delete(store);
|
|
57308
|
+
};
|
|
57309
|
+
}
|
|
57310
|
+
function pendingStageMessageRouter(store) {
|
|
57311
|
+
return registry.routers.get(store);
|
|
57312
|
+
}
|
|
57313
|
+
var registryKey, processState, registry;
|
|
57314
|
+
var init_external_run_routes = __esm(() => {
|
|
57315
|
+
registryKey = Symbol.for("atomic-workflows/external-run-routes@1");
|
|
57316
|
+
processState = globalThis;
|
|
57317
|
+
registry = processState[registryKey] ?? { owners: new WeakMap, routers: new WeakMap };
|
|
57318
|
+
processState[registryKey] = registry;
|
|
57319
|
+
});
|
|
57320
|
+
|
|
57176
57321
|
// dist/builtin/workflows/src/shared/pending-stage-route-capability.ts
|
|
57177
57322
|
import { randomUUID as randomUUID10 } from "node:crypto";
|
|
57178
57323
|
function capabilitiesFor(store) {
|
|
@@ -57197,12 +57342,12 @@ function rotateWorkflowPendingStageRouteCapability(store, runId) {
|
|
|
57197
57342
|
capabilitiesFor(store).set(runId, capability);
|
|
57198
57343
|
return capability;
|
|
57199
57344
|
}
|
|
57200
|
-
var routeCapabilitiesKey,
|
|
57345
|
+
var routeCapabilitiesKey, processState2, routeCapabilities;
|
|
57201
57346
|
var init_pending_stage_route_capability = __esm(() => {
|
|
57202
57347
|
routeCapabilitiesKey = Symbol.for("atomic-workflows/pending-stage-route-capabilities@1");
|
|
57203
|
-
|
|
57204
|
-
routeCapabilities =
|
|
57205
|
-
|
|
57348
|
+
processState2 = globalThis;
|
|
57349
|
+
routeCapabilities = processState2[routeCapabilitiesKey] ?? new WeakMap;
|
|
57350
|
+
processState2[routeCapabilitiesKey] = routeCapabilities;
|
|
57206
57351
|
});
|
|
57207
57352
|
|
|
57208
57353
|
// dist/builtin/workflows/src/shared/pending-stage-route-readiness.ts
|
|
@@ -57234,12 +57379,12 @@ function hasWorkflowPendingStageRouteOwner(store) {
|
|
|
57234
57379
|
function workflowPendingStageRouteReady(store, runId) {
|
|
57235
57380
|
return owners.get(store)?.ready(runId);
|
|
57236
57381
|
}
|
|
57237
|
-
var routeOwnersKey,
|
|
57382
|
+
var routeOwnersKey, processState3, owners;
|
|
57238
57383
|
var init_pending_stage_route_readiness = __esm(() => {
|
|
57239
57384
|
routeOwnersKey = Symbol.for("atomic-workflows/pending-stage-route-owners@1");
|
|
57240
|
-
|
|
57241
|
-
owners =
|
|
57242
|
-
|
|
57385
|
+
processState3 = globalThis;
|
|
57386
|
+
owners = processState3[routeOwnersKey] ?? new WeakMap;
|
|
57387
|
+
processState3[routeOwnersKey] = owners;
|
|
57243
57388
|
});
|
|
57244
57389
|
|
|
57245
57390
|
// dist/builtin/workflows/src/shared/workflow-stage-path-matching.ts
|
|
@@ -57540,6 +57685,9 @@ function registerPendingStageIntercomBridge(pi, activeStore, ownsRun = () => tru
|
|
|
57540
57685
|
notificationId,
|
|
57541
57686
|
reason
|
|
57542
57687
|
};
|
|
57688
|
+
const external = externalRunRouteOwner(activeStore, entry.runId);
|
|
57689
|
+
if (external !== undefined)
|
|
57690
|
+
return external.notifyUndeliverable(payload);
|
|
57543
57691
|
pi.events?.emit?.(PENDING_STAGE_UNDELIVERABLE_EVENT, payload);
|
|
57544
57692
|
return payload.handled && await payload.completion === true;
|
|
57545
57693
|
};
|
|
@@ -57551,19 +57699,29 @@ function registerPendingStageIntercomBridge(pi, activeStore, ownsRun = () => tru
|
|
|
57551
57699
|
return;
|
|
57552
57700
|
const ownedRunIds = new Set;
|
|
57553
57701
|
const runs = activeStore.runs();
|
|
57702
|
+
const runById = new Map(runs.map((run) => [run.id, run]));
|
|
57703
|
+
const childrenByBoundary = new Map;
|
|
57704
|
+
for (const run of runs) {
|
|
57705
|
+
if (run.parentRunId === undefined || run.parentStageId === undefined)
|
|
57706
|
+
continue;
|
|
57707
|
+
const key = JSON.stringify([run.parentRunId, run.parentStageId]);
|
|
57708
|
+
const children = childrenByBoundary.get(key) ?? [];
|
|
57709
|
+
children.push(run);
|
|
57710
|
+
childrenByBoundary.set(key, children);
|
|
57711
|
+
}
|
|
57554
57712
|
const resolveBoundarySegments = createWorkflowBoundarySegmentsResolver(runs);
|
|
57555
57713
|
for (const run of runs) {
|
|
57556
|
-
if (!ownsRun(run.id))
|
|
57714
|
+
if (externalRunRouteOwner(activeStore, run.id) !== undefined || !ownsRun(run.id))
|
|
57557
57715
|
continue;
|
|
57558
|
-
const rootRunId =
|
|
57716
|
+
const rootRunId = reciprocalWorkflowRootRunId(runById, run.id);
|
|
57559
57717
|
if (rootRunId === undefined)
|
|
57560
57718
|
continue;
|
|
57561
57719
|
ownedRunIds.add(run.id);
|
|
57562
|
-
const parentRun =
|
|
57720
|
+
const parentRun = run.parentRunId === undefined ? undefined : runById.get(run.parentRunId);
|
|
57563
57721
|
const boundary = parentRun?.stages.find((stage) => stage.id === run.parentStageId);
|
|
57564
57722
|
const parent = parentRun === undefined || boundary === undefined ? undefined : {
|
|
57565
57723
|
runId: parentRun.id,
|
|
57566
|
-
stageKeys: [boundary.id, boundary.name].filter((key) =>
|
|
57724
|
+
stageKeys: [boundary.id, boundary.name].filter((key) => resolveIndexedChildRun(childrenByBoundary, parentRun, key)?.id === run.id)
|
|
57567
57725
|
};
|
|
57568
57726
|
const announcement = {
|
|
57569
57727
|
runId: run.id,
|
|
@@ -57704,16 +57862,22 @@ function registerPendingStageIntercomBridge(pi, activeStore, ownsRun = () => tru
|
|
|
57704
57862
|
payload.handled = true;
|
|
57705
57863
|
payload.completion = queueAndPersist(activeStore, resolvedEvent, workflowInvocationIntercomGroup(rootRunId), stage.pendingStageDeliveryAvailable === true);
|
|
57706
57864
|
};
|
|
57707
|
-
const
|
|
57708
|
-
const stickySubscription = pi.events?.on?.(STICKY_LIVE_DELIVERED_EVENT, (payload) => {
|
|
57865
|
+
const routeStickyLiveDelivered = (payload) => {
|
|
57709
57866
|
if (disposed || !isStickyLiveDeliveredEvent(payload) || payload.handled)
|
|
57710
57867
|
return;
|
|
57711
57868
|
payload.handled = true;
|
|
57712
57869
|
payload.completion = outsideWorkflowExecutions(() => recordConfirmedStickyDeliveries(activeStore, payload));
|
|
57870
|
+
};
|
|
57871
|
+
const subscription = pi.events?.on?.(PENDING_STAGE_MESSAGE_EVENT, (payload) => outsideWorkflowExecutions(() => routePendingStageMessage(payload)));
|
|
57872
|
+
const stickySubscription = pi.events?.on?.(STICKY_LIVE_DELIVERED_EVENT, routeStickyLiveDelivered);
|
|
57873
|
+
const disposeRouter = registerPendingStageMessageRouter(activeStore, {
|
|
57874
|
+
route: (payload) => outsideWorkflowExecutions(() => routePendingStageMessage(payload)),
|
|
57875
|
+
sticky: routeStickyLiveDelivered
|
|
57713
57876
|
});
|
|
57714
57877
|
const dispose = () => {
|
|
57715
57878
|
disposed = true;
|
|
57716
57879
|
disposeReadiness();
|
|
57880
|
+
disposeRouter();
|
|
57717
57881
|
routeCompletions.clear();
|
|
57718
57882
|
unsubscribeStore();
|
|
57719
57883
|
if (typeof subscription === "function")
|
|
@@ -57839,6 +58003,12 @@ function isMaterializedNonAgentTarget(runs, target) {
|
|
|
57839
58003
|
return false;
|
|
57840
58004
|
return nonAgentNodes(run).some((node) => node.id === stageKey || node.name === stageKey);
|
|
57841
58005
|
}
|
|
58006
|
+
function resolveIndexedChildRun(childrenByBoundary, parent, segment) {
|
|
58007
|
+
const boundariesById = parent.stages.filter((stage) => stage.id === segment);
|
|
58008
|
+
const boundaries = boundariesById.length > 0 ? boundariesById : parent.stages.filter((stage) => stage.name === segment);
|
|
58009
|
+
const children = boundaries.flatMap((boundary) => childrenByBoundary.get(JSON.stringify([parent.id, boundary.id])) ?? []);
|
|
58010
|
+
return children.length === 1 ? children[0] : undefined;
|
|
58011
|
+
}
|
|
57842
58012
|
function resolveChildRun(runs, parent, segment) {
|
|
57843
58013
|
const boundariesById = parent.stages.filter((stage) => stage.id === segment);
|
|
57844
58014
|
const boundaries = boundariesById.length > 0 ? boundariesById : parent.stages.filter((stage) => stage.name === segment);
|
|
@@ -58178,6 +58348,7 @@ var init_pending_stage_intercom = __esm(() => {
|
|
|
58178
58348
|
init_execution_context_boundary();
|
|
58179
58349
|
init_run_owner_backend();
|
|
58180
58350
|
init_store_backend();
|
|
58351
|
+
init_external_run_routes();
|
|
58181
58352
|
init_intercom_group();
|
|
58182
58353
|
init_pending_stage_route_capability();
|
|
58183
58354
|
init_pending_stage_route_readiness();
|
|
@@ -58317,7 +58488,8 @@ function buildSendSignature(to, options) {
|
|
|
58317
58488
|
attachments: canonicalizeAttachmentsForSendSignature(options.attachments) ?? [],
|
|
58318
58489
|
replyTo: options.replyTo ?? null,
|
|
58319
58490
|
expectsReply: options.expectsReply ?? false,
|
|
58320
|
-
replyError: options.replyError ?? null
|
|
58491
|
+
replyError: options.replyError ?? null,
|
|
58492
|
+
...options.delivery === undefined || options.delivery === "steer" ? {} : { delivery: options.delivery }
|
|
58321
58493
|
});
|
|
58322
58494
|
}
|
|
58323
58495
|
|
|
@@ -58503,6 +58675,8 @@ function isMessage(value2) {
|
|
|
58503
58675
|
return false;
|
|
58504
58676
|
if (message.expectsReply !== undefined && typeof message.expectsReply !== "boolean")
|
|
58505
58677
|
return false;
|
|
58678
|
+
if (message.delivery !== undefined && message.delivery !== "steer" && message.delivery !== "followUp" && message.delivery !== "interrupt")
|
|
58679
|
+
return false;
|
|
58506
58680
|
if (message.replyError !== undefined && typeof message.replyError !== "string")
|
|
58507
58681
|
return false;
|
|
58508
58682
|
if (message.source !== undefined && !isMessageSource(message.source))
|
|
@@ -58636,7 +58810,7 @@ function isWorkflowFutureStageRosterEntries(value2) {
|
|
|
58636
58810
|
function isWorkflowStageRosterEntries(value2) {
|
|
58637
58811
|
return Array.isArray(value2) && value2.every((entry) => typeof entry === "object" && entry !== null && entry.kind === "workflow-stage" && typeof entry.runId === "string" && typeof entry.stageId === "string" && typeof entry.stageName === "string" && typeof entry.target === "string" && (entry.lifecycle === "pending" || entry.lifecycle === "running") && typeof entry.group === "string" && (entry.sessionId === undefined || typeof entry.sessionId === "string"));
|
|
58638
58812
|
}
|
|
58639
|
-
var BROKER_SOCKET, BROKER_CONNECT_TIMEOUT_MS = 1e4, PENDING_STAGE_ROUTE_ACK_TIMEOUT_MS = 1e4, GROUP_REQUEST_TIMEOUT_MS = 5000, PRESENCE_ACK_TIMEOUT_MS = 5000, RESTORING_RECIPIENT_REQUEST_TIMEOUT_MS = 5000, HELD_SEND_GRACE_MS = 5000, IntercomClient;
|
|
58813
|
+
var BROKER_SOCKET, BROKER_CONNECT_TIMEOUT_MS = 1e4, PENDING_STAGE_ROUTE_ACK_TIMEOUT_MS = 1e4, GROUP_REQUEST_TIMEOUT_MS = 5000, PRESENCE_ACK_TIMEOUT_MS = 5000, RESTORING_RECIPIENT_REQUEST_TIMEOUT_MS = 5000, HELD_SEND_GRACE_MS = 5000, IntercomPendingStageRouteRefusedError, PENDING_STAGE_ROUTE_REFUSAL_CODES, IntercomClient;
|
|
58640
58814
|
var init_client = __esm(() => {
|
|
58641
58815
|
init_paths();
|
|
58642
58816
|
init_pending_send_registry();
|
|
@@ -58645,6 +58819,21 @@ var init_client = __esm(() => {
|
|
|
58645
58819
|
init_live_route_refusal();
|
|
58646
58820
|
init_retry_policy();
|
|
58647
58821
|
BROKER_SOCKET = getBrokerSocketPath();
|
|
58822
|
+
IntercomPendingStageRouteRefusedError = class IntercomPendingStageRouteRefusedError extends Error {
|
|
58823
|
+
connectionKept;
|
|
58824
|
+
code;
|
|
58825
|
+
constructor(reason, refusal = { connectionKept: false }) {
|
|
58826
|
+
super(reason);
|
|
58827
|
+
this.name = "IntercomPendingStageRouteRefusedError";
|
|
58828
|
+
this.connectionKept = refusal.connectionKept;
|
|
58829
|
+
this.code = refusal.code;
|
|
58830
|
+
}
|
|
58831
|
+
};
|
|
58832
|
+
PENDING_STAGE_ROUTE_REFUSAL_CODES = new Set([
|
|
58833
|
+
"not_authorized",
|
|
58834
|
+
"invalid_stage_roster",
|
|
58835
|
+
"invalid_possible_stage_roster"
|
|
58836
|
+
]);
|
|
58648
58837
|
IntercomClient = class IntercomClient extends EventEmitter {
|
|
58649
58838
|
returnAddress;
|
|
58650
58839
|
preferredSessionId;
|
|
@@ -58652,6 +58841,8 @@ var init_client = __esm(() => {
|
|
|
58652
58841
|
connectTimeoutMs;
|
|
58653
58842
|
brokerAcknowledgesReceipt = false;
|
|
58654
58843
|
brokerAcknowledgesPendingStageRoutes = false;
|
|
58844
|
+
multiplexesPendingStageRoutes;
|
|
58845
|
+
brokerRefusesPendingStageRoutesInPlace = false;
|
|
58655
58846
|
brokerAnswersStatus = false;
|
|
58656
58847
|
socket = null;
|
|
58657
58848
|
_sessionId = null;
|
|
@@ -58717,6 +58908,7 @@ var init_client = __esm(() => {
|
|
|
58717
58908
|
this.preferredSessionId = options.preferredSessionId;
|
|
58718
58909
|
this.acknowledgesReceipt = options.acknowledgesReceipt === true;
|
|
58719
58910
|
this.connectTimeoutMs = options.connectTimeoutMs ?? BROKER_CONNECT_TIMEOUT_MS;
|
|
58911
|
+
this.multiplexesPendingStageRoutes = options.multiplexesPendingStageRoutes === true;
|
|
58720
58912
|
}
|
|
58721
58913
|
failPending(error) {
|
|
58722
58914
|
this.pendingSends.rejectAll(error);
|
|
@@ -58829,6 +59021,7 @@ var init_client = __esm(() => {
|
|
|
58829
59021
|
this._supervisorSessionId = null;
|
|
58830
59022
|
this.brokerAcknowledgesReceipt = false;
|
|
58831
59023
|
this.brokerAcknowledgesPendingStageRoutes = false;
|
|
59024
|
+
this.brokerRefusesPendingStageRoutesInPlace = false;
|
|
58832
59025
|
this.brokerAnswersStatus = false;
|
|
58833
59026
|
this._messageSource = undefined;
|
|
58834
59027
|
this.disconnectError = null;
|
|
@@ -58907,7 +59100,7 @@ var init_client = __esm(() => {
|
|
|
58907
59100
|
}
|
|
58908
59101
|
switch (brokerMessage.type) {
|
|
58909
59102
|
case "registered": {
|
|
58910
|
-
if (typeof brokerMessage.sessionId !== "string" || brokerMessage.supervisorSessionId !== undefined && typeof brokerMessage.supervisorSessionId !== "string" || brokerMessage.acknowledgesReceipt !== undefined && brokerMessage.acknowledgesReceipt !== true || brokerMessage.acknowledgesPendingStageRoutes !== undefined && brokerMessage.acknowledgesPendingStageRoutes !== true || brokerMessage.brokerStatus !== undefined && brokerMessage.brokerStatus !== true) {
|
|
59103
|
+
if (typeof brokerMessage.sessionId !== "string" || brokerMessage.supervisorSessionId !== undefined && typeof brokerMessage.supervisorSessionId !== "string" || brokerMessage.acknowledgesReceipt !== undefined && brokerMessage.acknowledgesReceipt !== true || brokerMessage.acknowledgesPendingStageRoutes !== undefined && brokerMessage.acknowledgesPendingStageRoutes !== true || brokerMessage.refusesPendingStageRoutesInPlace !== undefined && brokerMessage.refusesPendingStageRoutesInPlace !== true || brokerMessage.brokerStatus !== undefined && brokerMessage.brokerStatus !== true) {
|
|
58911
59104
|
throw new Error("Invalid registered message");
|
|
58912
59105
|
}
|
|
58913
59106
|
if (this._sessionId !== null) {
|
|
@@ -58917,6 +59110,7 @@ var init_client = __esm(() => {
|
|
|
58917
59110
|
this._supervisorSessionId = typeof brokerMessage.supervisorSessionId === "string" ? brokerMessage.supervisorSessionId : null;
|
|
58918
59111
|
this.brokerAcknowledgesReceipt = this.acknowledgesReceipt && brokerMessage.acknowledgesReceipt === true;
|
|
58919
59112
|
this.brokerAcknowledgesPendingStageRoutes = brokerMessage.acknowledgesPendingStageRoutes === true;
|
|
59113
|
+
this.brokerRefusesPendingStageRoutesInPlace = brokerMessage.refusesPendingStageRoutesInPlace === true;
|
|
58920
59114
|
this.brokerAnswersStatus = brokerMessage.brokerStatus === true;
|
|
58921
59115
|
this.emit("_registered", { type: "registered", sessionId: brokerMessage.sessionId });
|
|
58922
59116
|
break;
|
|
@@ -58929,6 +59123,12 @@ var init_client = __esm(() => {
|
|
|
58929
59123
|
this.emit("_registration_failed", refusal);
|
|
58930
59124
|
break;
|
|
58931
59125
|
}
|
|
59126
|
+
const refusedRouteId = typeof brokerMessage.requestId === "string" ? brokerMessage.requestId : "";
|
|
59127
|
+
const refusedRoute = this.pendingStageRouteRegistrations.get(refusedRouteId);
|
|
59128
|
+
if (refusedRoute !== undefined) {
|
|
59129
|
+
this.pendingStageRouteRegistrations.delete(refusedRouteId);
|
|
59130
|
+
refusedRoute.reject(new IntercomPendingStageRouteRefusedError(brokerMessage.reason));
|
|
59131
|
+
}
|
|
58932
59132
|
this.disconnectError ??= refusal;
|
|
58933
59133
|
this.failPending(this.disconnectError);
|
|
58934
59134
|
this.socket?.destroy();
|
|
@@ -59054,6 +59254,21 @@ var init_client = __esm(() => {
|
|
|
59054
59254
|
this.emit("pending_stage_route_revoked", { runId });
|
|
59055
59255
|
break;
|
|
59056
59256
|
}
|
|
59257
|
+
case "pending_stage_route_refused": {
|
|
59258
|
+
const { requestId, runId, code, reason } = brokerMessage;
|
|
59259
|
+
if (typeof requestId !== "string" || typeof runId !== "string" || typeof reason !== "string" || typeof code !== "string") {
|
|
59260
|
+
throw new Error("Invalid pending-stage route refusal");
|
|
59261
|
+
}
|
|
59262
|
+
const pending = this.pendingStageRouteRegistrations.get(requestId);
|
|
59263
|
+
if (!pending)
|
|
59264
|
+
return;
|
|
59265
|
+
this.pendingStageRouteRegistrations.delete(requestId);
|
|
59266
|
+
pending.reject(new IntercomPendingStageRouteRefusedError(reason, {
|
|
59267
|
+
connectionKept: true,
|
|
59268
|
+
...PENDING_STAGE_ROUTE_REFUSAL_CODES.has(code) ? { code } : {}
|
|
59269
|
+
}));
|
|
59270
|
+
break;
|
|
59271
|
+
}
|
|
59057
59272
|
case "pending_stage_route_registered": {
|
|
59058
59273
|
const { requestId } = brokerMessage;
|
|
59059
59274
|
if (typeof requestId !== "string")
|
|
@@ -59557,7 +59772,8 @@ var init_client = __esm(() => {
|
|
|
59557
59772
|
stages,
|
|
59558
59773
|
possibleStages,
|
|
59559
59774
|
...parent === undefined ? {} : { parent },
|
|
59560
|
-
...dormant === true ? { dormant: true } : {}
|
|
59775
|
+
...dormant === true ? { dormant: true } : {},
|
|
59776
|
+
...this.multiplexesPendingStageRoutes && this.brokerRefusesPendingStageRoutesInPlace ? { refusalKeepsConnection: true } : {}
|
|
59561
59777
|
});
|
|
59562
59778
|
} catch (error) {
|
|
59563
59779
|
clearTimeout(timeout);
|
|
@@ -59670,6 +59886,7 @@ var init_client = __esm(() => {
|
|
|
59670
59886
|
timestamp: Date.now(),
|
|
59671
59887
|
replyTo: options.replyTo,
|
|
59672
59888
|
expectsReply: options.expectsReply,
|
|
59889
|
+
...options.delivery === undefined || options.delivery === "steer" ? {} : { delivery: options.delivery },
|
|
59673
59890
|
replyError: options.replyError,
|
|
59674
59891
|
source: this._messageSource,
|
|
59675
59892
|
content: { text: options.text, attachments: options.attachments }
|
|
@@ -60291,120 +60508,165 @@ var init_reconnect_backoff = __esm(() => {
|
|
|
60291
60508
|
BACKOFF_MS = [1000, 2000, 5000, 1e4, 30000];
|
|
60292
60509
|
});
|
|
60293
60510
|
|
|
60294
|
-
// dist/builtin/intercom/route-connect-gate.ts
|
|
60295
|
-
function createConcurrencyGate(limit) {
|
|
60296
|
-
let active = 0;
|
|
60297
|
-
const waiting = [];
|
|
60298
|
-
const release = () => {
|
|
60299
|
-
const next = waiting.shift();
|
|
60300
|
-
if (next === undefined)
|
|
60301
|
-
active -= 1;
|
|
60302
|
-
else
|
|
60303
|
-
next();
|
|
60304
|
-
};
|
|
60305
|
-
return {
|
|
60306
|
-
async run(operation) {
|
|
60307
|
-
if (active < limit)
|
|
60308
|
-
active += 1;
|
|
60309
|
-
else
|
|
60310
|
-
await new Promise((resolve) => waiting.push(resolve));
|
|
60311
|
-
try {
|
|
60312
|
-
return await operation();
|
|
60313
|
-
} finally {
|
|
60314
|
-
release();
|
|
60315
|
-
}
|
|
60316
|
-
}
|
|
60317
|
-
};
|
|
60318
|
-
}
|
|
60319
|
-
var ROUTE_CLIENT_CONNECT_CONCURRENCY = 4;
|
|
60320
|
-
|
|
60321
60511
|
// dist/builtin/intercom/workflow-route-owner.ts
|
|
60322
60512
|
import { randomUUID as randomUUID13 } from "node:crypto";
|
|
60323
60513
|
import { createEventBus, getBuiltinPackagePaths } from "@bastani/atomic";
|
|
60514
|
+
function closedError() {
|
|
60515
|
+
return new Error("Workflow Intercom route owner closed");
|
|
60516
|
+
}
|
|
60324
60517
|
function createWorkflowRouteOwner(cwd) {
|
|
60325
60518
|
const events = createEventBus();
|
|
60326
60519
|
const config = loadConfig();
|
|
60327
60520
|
const id = randomUUID13();
|
|
60328
60521
|
const routes = new Map;
|
|
60329
|
-
const
|
|
60522
|
+
const connections = new Map;
|
|
60523
|
+
const isCurrent = (state) => routes.get(state.route.runId) === state;
|
|
60524
|
+
const connectionKey = (state) => state.isolated ? state.route.runId : state.route.group;
|
|
60525
|
+
function connectionOf(state) {
|
|
60526
|
+
const key = connectionKey(state);
|
|
60527
|
+
let connection = connections.get(key);
|
|
60528
|
+
if (connection === undefined) {
|
|
60529
|
+
connection = { key, group: state.route.group, acknowledged: false, routes: new Map };
|
|
60530
|
+
connections.set(key, connection);
|
|
60531
|
+
}
|
|
60532
|
+
connection.routes.set(state.route.runId, state);
|
|
60533
|
+
return connection;
|
|
60534
|
+
}
|
|
60535
|
+
function detach(state) {
|
|
60536
|
+
const key = connectionKey(state);
|
|
60537
|
+
const connection = connections.get(key);
|
|
60538
|
+
connection?.routes.delete(state.route.runId);
|
|
60539
|
+
if (!state.isolated || connection === undefined)
|
|
60540
|
+
return;
|
|
60541
|
+
connections.delete(key);
|
|
60542
|
+
connection.client?.disconnect().catch(() => {});
|
|
60543
|
+
}
|
|
60544
|
+
function settle(state, registered) {
|
|
60545
|
+
state.registered = registered;
|
|
60546
|
+
if (!registered && isCurrent(state))
|
|
60547
|
+
return;
|
|
60548
|
+
for (const waiter of state.waiters)
|
|
60549
|
+
waiter(registered);
|
|
60550
|
+
state.waiters.clear();
|
|
60551
|
+
}
|
|
60330
60552
|
function scheduleRetry(state) {
|
|
60331
|
-
if (state
|
|
60553
|
+
if (!isCurrent(state) || state.retry !== undefined)
|
|
60332
60554
|
return;
|
|
60333
60555
|
state.retry = setTimeout(() => {
|
|
60334
60556
|
state.retry = undefined;
|
|
60335
|
-
publish(state, state.route).catch(() =>
|
|
60557
|
+
publish(state, state.route).catch(() => {});
|
|
60336
60558
|
}, jitteredReconnectDelayMs(state.attempt++));
|
|
60337
60559
|
state.retry.unref();
|
|
60338
60560
|
}
|
|
60339
|
-
|
|
60340
|
-
if (
|
|
60341
|
-
|
|
60342
|
-
|
|
60343
|
-
|
|
60344
|
-
|
|
60345
|
-
|
|
60346
|
-
|
|
60347
|
-
}
|
|
60348
|
-
|
|
60349
|
-
|
|
60350
|
-
|
|
60351
|
-
if (
|
|
60352
|
-
|
|
60353
|
-
|
|
60354
|
-
|
|
60355
|
-
|
|
60356
|
-
|
|
60357
|
-
|
|
60358
|
-
|
|
60359
|
-
|
|
60360
|
-
|
|
60361
|
-
|
|
60362
|
-
|
|
60363
|
-
if (
|
|
60364
|
-
|
|
60365
|
-
const
|
|
60366
|
-
|
|
60367
|
-
|
|
60368
|
-
|
|
60369
|
-
|
|
60370
|
-
|
|
60371
|
-
|
|
60372
|
-
|
|
60373
|
-
|
|
60374
|
-
|
|
60375
|
-
|
|
60376
|
-
|
|
60377
|
-
|
|
60378
|
-
|
|
60379
|
-
|
|
60380
|
-
|
|
60381
|
-
|
|
60382
|
-
|
|
60383
|
-
|
|
60561
|
+
function drop(connection, client) {
|
|
60562
|
+
if (connection.client === client) {
|
|
60563
|
+
connection.client = undefined;
|
|
60564
|
+
connection.acknowledged = false;
|
|
60565
|
+
for (const state of connection.routes.values()) {
|
|
60566
|
+
settle(state, false);
|
|
60567
|
+
scheduleRetry(state);
|
|
60568
|
+
}
|
|
60569
|
+
}
|
|
60570
|
+
client.disconnect().catch(() => {});
|
|
60571
|
+
}
|
|
60572
|
+
function connect(connection) {
|
|
60573
|
+
if (connection.client?.isConnected())
|
|
60574
|
+
return Promise.resolve(connection.client);
|
|
60575
|
+
connection.connecting ??= (async () => {
|
|
60576
|
+
const [extensionDir] = getBuiltinPackagePaths({
|
|
60577
|
+
workflows: false,
|
|
60578
|
+
subagents: false,
|
|
60579
|
+
mcp: false,
|
|
60580
|
+
"web-access": false
|
|
60581
|
+
});
|
|
60582
|
+
if (extensionDir === undefined)
|
|
60583
|
+
throw new Error("Intercom builtin package is unavailable");
|
|
60584
|
+
await spawnBrokerIfNeeded(config.brokerCommand, config.brokerArgs, undefined, extensionDir);
|
|
60585
|
+
if (connections.get(connection.key) !== connection)
|
|
60586
|
+
throw closedError();
|
|
60587
|
+
const client = new IntercomClient(`${id}:pending-stage-route:${connection.key}`, {
|
|
60588
|
+
multiplexesPendingStageRoutes: true
|
|
60589
|
+
});
|
|
60590
|
+
connection.client = client;
|
|
60591
|
+
connection.acknowledged = false;
|
|
60592
|
+
client.on("error", () => {});
|
|
60593
|
+
client.on("disconnected", () => drop(connection, client));
|
|
60594
|
+
client.on("pending_stage_message", (request) => {
|
|
60595
|
+
if (connection.client !== client)
|
|
60596
|
+
return;
|
|
60597
|
+
const event = { ...request, handled: false };
|
|
60598
|
+
events.emit("atomic:workflow-pending-stage-message", event);
|
|
60599
|
+
const completion = event.handled && event.completion !== undefined ? event.completion : Promise.resolve({ outcome: "refused", reason: "Session not found" });
|
|
60600
|
+
completion.catch((error) => ({ outcome: "refused", reason: error.message })).then((result) => client.respondPendingStageMessage(request.requestId, result)).catch(() => {});
|
|
60601
|
+
});
|
|
60602
|
+
client.on("sticky_live_delivered", (notice) => {
|
|
60603
|
+
if (connection.client !== client)
|
|
60604
|
+
return;
|
|
60605
|
+
events.emit("atomic:workflow-sticky-live-delivered", { ...notice, handled: false });
|
|
60606
|
+
});
|
|
60607
|
+
try {
|
|
60608
|
+
await client.connect({
|
|
60609
|
+
cwd,
|
|
60610
|
+
model: "workflow",
|
|
60611
|
+
pid: process.pid,
|
|
60612
|
+
startedAt: Date.now(),
|
|
60613
|
+
lastActivity: Date.now(),
|
|
60614
|
+
group: connection.group,
|
|
60615
|
+
groups: [connection.group],
|
|
60616
|
+
recipientPurpose: "control"
|
|
60617
|
+
});
|
|
60618
|
+
if (connections.get(connection.key) !== connection)
|
|
60619
|
+
throw closedError();
|
|
60620
|
+
} catch (error) {
|
|
60621
|
+
drop(connection, client);
|
|
60622
|
+
throw error;
|
|
60623
|
+
}
|
|
60624
|
+
for (const state of connection.routes.values()) {
|
|
60625
|
+
if (state.registered || state.retry === undefined)
|
|
60626
|
+
continue;
|
|
60627
|
+
clearTimeout(state.retry);
|
|
60628
|
+
state.retry = undefined;
|
|
60629
|
+
publish(state, state.route).catch(() => {});
|
|
60630
|
+
}
|
|
60631
|
+
return client;
|
|
60632
|
+
})().finally(() => {
|
|
60633
|
+
connection.connecting = undefined;
|
|
60384
60634
|
});
|
|
60385
|
-
return
|
|
60635
|
+
return connection.connecting;
|
|
60386
60636
|
}
|
|
60387
60637
|
function publish(state, route) {
|
|
60388
|
-
const completion = state.tail.catch(() => {}).then(
|
|
60389
|
-
if (state
|
|
60390
|
-
throw
|
|
60391
|
-
const
|
|
60638
|
+
const completion = state.tail.catch(() => {}).then(async () => {
|
|
60639
|
+
if (!isCurrent(state))
|
|
60640
|
+
throw closedError();
|
|
60641
|
+
const connection = connectionOf(state);
|
|
60642
|
+
const established = connection.acknowledged && connection.client?.isConnected() ? connection.client : undefined;
|
|
60643
|
+
let client;
|
|
60392
60644
|
try {
|
|
60393
|
-
|
|
60645
|
+
client = await connect(connection);
|
|
60394
60646
|
await client.registerPendingStageRouteAcknowledged(route.runId, route.group, route.capability, route.stages, route.possibleStages, route.parent);
|
|
60647
|
+
if (connection.client !== client || !isCurrent(state))
|
|
60648
|
+
return;
|
|
60649
|
+
connection.acknowledged = true;
|
|
60395
60650
|
state.attempt = 0;
|
|
60396
60651
|
clearTimeout(state.retry);
|
|
60397
60652
|
state.retry = undefined;
|
|
60653
|
+
settle(state, true);
|
|
60398
60654
|
} catch (error) {
|
|
60399
|
-
|
|
60400
|
-
|
|
60401
|
-
|
|
60402
|
-
await client?.disconnect();
|
|
60655
|
+
if (error instanceof IntercomPendingStageRouteRefusedError && error.connectionKept) {
|
|
60656
|
+
scheduleRetry(state);
|
|
60657
|
+
throw error;
|
|
60403
60658
|
}
|
|
60659
|
+
if (error instanceof IntercomPendingStageRouteRefusedError && !state.isolated && isCurrent(state)) {
|
|
60660
|
+
detach(state);
|
|
60661
|
+
state.isolated = true;
|
|
60662
|
+
connectionOf(state);
|
|
60663
|
+
}
|
|
60664
|
+
if (client !== undefined && (client !== established || !client.isConnected()))
|
|
60665
|
+
drop(connection, client);
|
|
60404
60666
|
scheduleRetry(state);
|
|
60405
60667
|
throw error;
|
|
60406
60668
|
}
|
|
60407
|
-
})
|
|
60669
|
+
});
|
|
60408
60670
|
state.tail = completion;
|
|
60409
60671
|
completion.catch(() => {});
|
|
60410
60672
|
return completion;
|
|
@@ -60413,10 +60675,15 @@ function createWorkflowRouteOwner(cwd) {
|
|
|
60413
60675
|
const route = payload;
|
|
60414
60676
|
let state = routes.get(route.runId);
|
|
60415
60677
|
if (state === undefined) {
|
|
60416
|
-
state = { route, tail: Promise.resolve(), attempt: 0,
|
|
60678
|
+
state = { route, tail: Promise.resolve(), attempt: 0, registered: false, isolated: false, waiters: new Set };
|
|
60417
60679
|
routes.set(route.runId, state);
|
|
60680
|
+
} else if (state.route.group !== route.group) {
|
|
60681
|
+
detach(state);
|
|
60682
|
+
state.isolated = false;
|
|
60683
|
+
settle(state, false);
|
|
60418
60684
|
}
|
|
60419
60685
|
state.route = route;
|
|
60686
|
+
connectionOf(state);
|
|
60420
60687
|
route.completion = publish(state, route);
|
|
60421
60688
|
});
|
|
60422
60689
|
events.on("atomic:workflow-pending-stage-undeliverable", (payload) => {
|
|
@@ -60427,7 +60694,7 @@ function createWorkflowRouteOwner(cwd) {
|
|
|
60427
60694
|
event.handled = true;
|
|
60428
60695
|
event.completion = (async () => {
|
|
60429
60696
|
await publish(state, state.route);
|
|
60430
|
-
const client = state
|
|
60697
|
+
const client = connections.get(connectionKey(state))?.client;
|
|
60431
60698
|
if (client === undefined || !client.isConnected())
|
|
60432
60699
|
return false;
|
|
60433
60700
|
const text = `Pending workflow stage could not receive intercom message: ${event.reason}`;
|
|
@@ -60437,18 +60704,33 @@ function createWorkflowRouteOwner(cwd) {
|
|
|
60437
60704
|
});
|
|
60438
60705
|
return {
|
|
60439
60706
|
events,
|
|
60707
|
+
registered(runId) {
|
|
60708
|
+
const state = routes.get(runId);
|
|
60709
|
+
if (state === undefined)
|
|
60710
|
+
return Promise.resolve(false);
|
|
60711
|
+
if (state.registered)
|
|
60712
|
+
return Promise.resolve(true);
|
|
60713
|
+
return new Promise((resolve) => state.waiters.add(resolve));
|
|
60714
|
+
},
|
|
60440
60715
|
async closeGroup(group) {
|
|
60441
60716
|
const closing = [...routes.values()].filter((state) => state.route.group === group);
|
|
60717
|
+
const closingConnections = [...connections.values()].filter((connection) => connection.group === group);
|
|
60718
|
+
for (const connection of closingConnections) {
|
|
60719
|
+
connections.delete(connection.key);
|
|
60720
|
+
connection.routes.clear();
|
|
60721
|
+
}
|
|
60442
60722
|
for (const state of closing) {
|
|
60443
|
-
state.closed = true;
|
|
60444
|
-
clearTimeout(state.retry);
|
|
60445
60723
|
routes.delete(state.route.runId);
|
|
60724
|
+
clearTimeout(state.retry);
|
|
60725
|
+
state.retry = undefined;
|
|
60726
|
+
settle(state, false);
|
|
60446
60727
|
}
|
|
60447
|
-
await Promise.all(
|
|
60448
|
-
await
|
|
60449
|
-
await
|
|
60450
|
-
await state.client?.disconnect();
|
|
60728
|
+
await Promise.all(closingConnections.map(async (connection) => {
|
|
60729
|
+
await connection.client?.disconnect();
|
|
60730
|
+
await connection.connecting?.catch(() => {});
|
|
60451
60731
|
}));
|
|
60732
|
+
await Promise.all(closing.map((state) => state.tail.catch(() => {})));
|
|
60733
|
+
await Promise.all(closingConnections.map((connection) => connection.client?.disconnect()));
|
|
60452
60734
|
}
|
|
60453
60735
|
};
|
|
60454
60736
|
}
|
|
@@ -60631,7 +60913,7 @@ var init_workflow_infrastructure_error = __esm(() => {
|
|
|
60631
60913
|
[/Local worker (?:process )?cleanup (?:was not confirmed|unconfirmed)/, "worker_cleanup_unconfirmed"],
|
|
60632
60914
|
[/Local worker retry limit/, "worker_retry_limit"],
|
|
60633
60915
|
[/Local worker (?:startup timed out|exited before readiness)/, "worker_startup_failed"],
|
|
60634
|
-
[/Worker exited before delivery acknowledgement/, "worker_lost"],
|
|
60916
|
+
[/Worker exited before delivery acknowledgement|Local worker pool coordinator exited/, "worker_lost"],
|
|
60635
60917
|
[/Stale work lease/, "stale_work_lease"],
|
|
60636
60918
|
[/Intercom runtime no longer active/, "intercom_unavailable"],
|
|
60637
60919
|
[/Storage unavailable/, "storage_unavailable"]
|
|
@@ -60889,7 +61171,7 @@ var init_definitions = __esm(() => {
|
|
|
60889
61171
|
// dist/builtin/workflows/src/durable/journal/pool-mutations.ts
|
|
60890
61172
|
import { StorageProvider as StorageProvider2 } from "@bastani/atomic/storage";
|
|
60891
61173
|
async function validatePoolMutation(tx, journal, root, work, mutation) {
|
|
60892
|
-
if (!["checkpoint", "session-create", "session-
|
|
61174
|
+
if (!["checkpoint", "session-create", "session-replace"].includes(mutation.kind))
|
|
60893
61175
|
throw new Error("Invalid pool mutation");
|
|
60894
61176
|
const checkpoint = mutation.checkpoint;
|
|
60895
61177
|
const prefix = `${work.scope}:`;
|
|
@@ -60957,10 +61239,18 @@ async function sessionStore(tx, prefix) {
|
|
|
60957
61239
|
return new StorageProvider2().openSessionStore({ kind: "database", database, prefix }, { agentDir: "", domain: "sessions" });
|
|
60958
61240
|
}
|
|
60959
61241
|
async function applyPoolSession(tx, prefix, workId, mutation) {
|
|
61242
|
+
const owners = `${prefix}session_owners`;
|
|
61243
|
+
if (mutation.kind === "session-append") {
|
|
61244
|
+
if (typeof mutation.sessionId !== "string" || mutation.sessionId === "")
|
|
61245
|
+
throw new Error("Pool sessions require database identity");
|
|
61246
|
+
const owner = await tx.get(`SELECT work FROM ${owners} WHERE session = ?`, mutation.sessionId);
|
|
61247
|
+
if (owner?.work !== workId)
|
|
61248
|
+
throw new Error("Pool session scope mismatch");
|
|
61249
|
+
return withSessionStore(tx, prefix, (store) => store.append(mutation.sessionId, mutation.entries, { expectSeq: mutation.expectSeq }));
|
|
61250
|
+
}
|
|
60960
61251
|
const { sessionId, replayKey } = mutation.checkpoint;
|
|
60961
61252
|
if (!sessionId || mutation.checkpoint.sessionFile !== undefined)
|
|
60962
61253
|
throw new Error("Pool sessions require database identity");
|
|
60963
|
-
const owners = `${prefix}session_owners`;
|
|
60964
61254
|
if (mutation.kind === "session-create") {
|
|
60965
61255
|
if (mutation.header.id !== sessionId)
|
|
60966
61256
|
throw new Error("Pool session identity mismatch");
|
|
@@ -60970,10 +61260,12 @@ async function applyPoolSession(tx, prefix, workId, mutation) {
|
|
|
60970
61260
|
if (owner?.work !== workId || owner.replay_key !== replayKey)
|
|
60971
61261
|
throw new Error("Pool session scope mismatch");
|
|
60972
61262
|
}
|
|
61263
|
+
return withSessionStore(tx, prefix, (store) => mutation.kind === "session-create" ? store.create(mutation.header, mutation.entries) : store.replace(sessionId, mutation.header, mutation.entries, { expectSeq: mutation.expectSeq }));
|
|
61264
|
+
}
|
|
61265
|
+
async function withSessionStore(tx, prefix, write) {
|
|
60973
61266
|
const store = await sessionStore(tx, prefix);
|
|
60974
61267
|
try {
|
|
60975
|
-
|
|
60976
|
-
return result.lastSeq;
|
|
61268
|
+
return (await write(store)).lastSeq;
|
|
60977
61269
|
} finally {
|
|
60978
61270
|
await store.close();
|
|
60979
61271
|
}
|
|
@@ -61021,8 +61313,11 @@ var init_pool_stage_usage = __esm(() => {
|
|
|
61021
61313
|
});
|
|
61022
61314
|
|
|
61023
61315
|
// dist/builtin/workflows/src/durable/journal/sql-pool-queue.ts
|
|
61024
|
-
import { randomUUID as randomUUID15 } from "node:crypto";
|
|
61316
|
+
import { createHash as createHash4, randomUUID as randomUUID15 } from "node:crypto";
|
|
61025
61317
|
import { assertStorageSchemaVersion as assertStorageSchemaVersion2 } from "@bastani/atomic/storage";
|
|
61318
|
+
function requestIdentity(request) {
|
|
61319
|
+
return createHash4("sha256").update(request).digest("hex");
|
|
61320
|
+
}
|
|
61026
61321
|
function fromRow(row) {
|
|
61027
61322
|
return {
|
|
61028
61323
|
...JSON.parse(row.descriptor),
|
|
@@ -61078,10 +61373,11 @@ class SqlPoolQueue {
|
|
|
61078
61373
|
const versions = await tx.all(`SELECT version FROM ${pool.meta}`);
|
|
61079
61374
|
if (versions.length > 1)
|
|
61080
61375
|
throw new Error("Corrupt pool schema metadata");
|
|
61081
|
-
|
|
61082
|
-
|
|
61083
|
-
|
|
61084
|
-
|
|
61376
|
+
const version = versions[0]?.version;
|
|
61377
|
+
if (version === undefined)
|
|
61378
|
+
await tx.run(`INSERT INTO ${pool.meta} VALUES (?)`, POOL_SCHEMA_VERSION);
|
|
61379
|
+
else if (version !== 1)
|
|
61380
|
+
assertStorageSchemaVersion2("journal", version, POOL_SCHEMA_VERSION);
|
|
61085
61381
|
await tx.exec(`
|
|
61086
61382
|
CREATE TABLE IF NOT EXISTS ${pool.roots}(root TEXT PRIMARY KEY REFERENCES ${journal.runs}(run_id) ON DELETE CASCADE, timeout INTEGER NOT NULL);
|
|
61087
61383
|
CREATE TABLE IF NOT EXISTS ${pool.work}(
|
|
@@ -61112,9 +61408,23 @@ class SqlPoolQueue {
|
|
|
61112
61408
|
state TEXT NOT NULL DEFAULT 'running', revision INTEGER NOT NULL DEFAULT 0
|
|
61113
61409
|
)`);
|
|
61114
61410
|
await tx.exec(poolUsageSchema(journal.runs));
|
|
61411
|
+
if (version === 1)
|
|
61412
|
+
await pool.migrateFromVersion1(tx);
|
|
61115
61413
|
});
|
|
61116
61414
|
return pool;
|
|
61117
61415
|
}
|
|
61416
|
+
async migrateFromVersion1(tx) {
|
|
61417
|
+
await tx.run(`DELETE FROM ${this.operations} WHERE substr(request, 1, 1) = '['`);
|
|
61418
|
+
for (let after = 0;; ) {
|
|
61419
|
+
const rows = await tx.all(`SELECT rowid, request FROM ${this.operations} WHERE rowid > ? ORDER BY rowid LIMIT 256`, after);
|
|
61420
|
+
if (rows.length === 0)
|
|
61421
|
+
break;
|
|
61422
|
+
for (const row of rows)
|
|
61423
|
+
await tx.run(`UPDATE ${this.operations} SET request = ? WHERE rowid = ?`, requestIdentity(String(row.request)), row.rowid);
|
|
61424
|
+
after = Number(rows.at(-1).rowid);
|
|
61425
|
+
}
|
|
61426
|
+
await tx.run(`UPDATE ${this.meta} SET version = ?`, POOL_SCHEMA_VERSION);
|
|
61427
|
+
}
|
|
61118
61428
|
async authority(tx, lease, timeout) {
|
|
61119
61429
|
const row = await tx.get(`SELECT l.*, r.status, p.timeout, ${STORE_NOW2} AS now FROM ${this.journal.leases} l JOIN ${this.journal.runs} r ON r.run_id = l.run_id LEFT JOIN ${this.roots} p ON p.root = l.run_id WHERE l.run_id = ?`, lease.runId);
|
|
61120
61430
|
if (row?.timeout === null && timeout === undefined)
|
|
@@ -61255,31 +61565,18 @@ class SqlPoolQueue {
|
|
|
61255
61565
|
throw new Error(STALE_WORK_LEASE);
|
|
61256
61566
|
return row;
|
|
61257
61567
|
}
|
|
61258
|
-
async renew(lease, token,
|
|
61568
|
+
async renew(lease, token, durationMs) {
|
|
61259
61569
|
positive(durationMs);
|
|
61260
61570
|
lease = snapshotLease(lease);
|
|
61261
61571
|
token = { ...token };
|
|
61262
|
-
validateText("key", operationId);
|
|
61263
|
-
const request = JSON.stringify([token.epoch, token.generation, token.worker, durationMs]);
|
|
61264
61572
|
return this.transaction(async (tx) => {
|
|
61265
61573
|
const { now } = await this.authority(tx, lease);
|
|
61266
61574
|
const row = await this.current(tx, lease, token);
|
|
61267
61575
|
if (row.epoch !== lease.epoch || row.expires_at < now)
|
|
61268
61576
|
throw new Error(STALE_WORK_LEASE);
|
|
61269
|
-
const
|
|
61270
|
-
|
|
61271
|
-
|
|
61272
|
-
throw new Error("Pool operation identity conflict");
|
|
61273
|
-
return JSON.parse(previous.receipt);
|
|
61274
|
-
}
|
|
61275
|
-
const receipt = {
|
|
61276
|
-
...fromRow(row),
|
|
61277
|
-
expiresAt: Math.max(Number(row.expires_at), expiry(now, durationMs)),
|
|
61278
|
-
renewedAt: now
|
|
61279
|
-
};
|
|
61280
|
-
await tx.run(`UPDATE ${this.work} SET expires_at = ?, renewed_at = ? WHERE id = ?`, receipt.expiresAt, now, row.id);
|
|
61281
|
-
await tx.run(`INSERT INTO ${this.operations} VALUES (?, ?, ?, ?, ?)`, row.id, token.generation, operationId, request, JSON.stringify(receipt));
|
|
61282
|
-
return receipt;
|
|
61577
|
+
const expiresAt = Math.max(Number(row.expires_at), expiry(now, durationMs));
|
|
61578
|
+
await tx.run(`UPDATE ${this.work} SET expires_at = ?, renewed_at = ? WHERE id = ?`, expiresAt, now, row.id);
|
|
61579
|
+
return { ...fromRow(row), expiresAt, renewedAt: now };
|
|
61283
61580
|
});
|
|
61284
61581
|
}
|
|
61285
61582
|
async commit(lease, token, operationId, mutation) {
|
|
@@ -61288,6 +61585,7 @@ class SqlPoolQueue {
|
|
|
61288
61585
|
validateText("key", operationId);
|
|
61289
61586
|
assertWorkflowSerializableObject(mutation, "Pool mutation");
|
|
61290
61587
|
const request = serializeJson({ epoch: token.epoch, worker: token.worker, mutation });
|
|
61588
|
+
const identity = requestIdentity(request);
|
|
61291
61589
|
mutation = JSON.parse(request).mutation;
|
|
61292
61590
|
return this.transaction(async (tx) => {
|
|
61293
61591
|
const { now, status } = await this.authority(tx, lease);
|
|
@@ -61296,7 +61594,7 @@ class SqlPoolQueue {
|
|
|
61296
61594
|
throw new Error(STALE_WORK_LEASE);
|
|
61297
61595
|
const previous = await tx.get(`SELECT * FROM ${this.operations} WHERE work = ? AND generation = ? AND operation = ?`, token.id, token.generation, operationId);
|
|
61298
61596
|
if (previous) {
|
|
61299
|
-
if (previous.request !==
|
|
61597
|
+
if (previous.request !== identity)
|
|
61300
61598
|
throw new Error("Pool operation identity conflict");
|
|
61301
61599
|
return JSON.parse(previous.receipt);
|
|
61302
61600
|
}
|
|
@@ -61304,34 +61602,41 @@ class SqlPoolQueue {
|
|
|
61304
61602
|
throw new Error(STALE_WORK_LEASE);
|
|
61305
61603
|
if (!await this.admitsAttemptWrites(tx, lease, status))
|
|
61306
61604
|
throw new Error("Pool root is not running");
|
|
61307
|
-
const
|
|
61308
|
-
await
|
|
61309
|
-
if (checkpoint.completedAt > Math.max(now, Date.now()) + MAX_WORKER_CLOCK_SKEW_MS)
|
|
61310
|
-
throw new Error("Pool checkpoint timestamp is in the future");
|
|
61311
|
-
assertWorkflowSerializableObject(checkpoint, "Pool checkpoint");
|
|
61312
|
-
const key = JSON.stringify([RUN_EVENT_TYPES.checkpointRecorded, checkpoint.kind, checkpoint.checkpointId]);
|
|
61313
|
-
const payload = serializeJson(checkpoint);
|
|
61314
|
-
const existing = await tx.get(`SELECT seq, at, payload FROM ${this.journal.events} WHERE run_id = ? AND key = ?`, lease.runId, key);
|
|
61315
|
-
if (existing && existing.payload !== payload)
|
|
61316
|
-
throw new Error("Pool checkpoint identity conflict");
|
|
61317
|
-
if (existing && mutation.kind !== "checkpoint")
|
|
61318
|
-
throw new Error("Pool session checkpoint identity conflict");
|
|
61319
|
-
const lastSeq = mutation.kind === "checkpoint" ? undefined : await applyPoolSession(tx, `${this.journal.runs}_pool_`, row.id, mutation);
|
|
61320
|
-
const latest = await tx.get(`SELECT COALESCE(MAX(seq), 0) AS seq FROM ${this.journal.events} WHERE run_id = ?`, lease.runId);
|
|
61321
|
-
const receipt = existing ? { seq: Number(existing.seq), at: Number(existing.at) } : {
|
|
61322
|
-
seq: journalCounter(lease.runId, "seq", Number(latest.seq) + 1, "overflow"),
|
|
61323
|
-
at: now,
|
|
61324
|
-
...lastSeq === undefined ? {} : { lastSeq }
|
|
61325
|
-
};
|
|
61326
|
-
if (!existing)
|
|
61327
|
-
await tx.run(`INSERT INTO ${this.journal.events} VALUES (?, ?, ?, ?, ?, ?, ?)`, lease.runId, receipt.seq, lease.epoch, key, RUN_EVENT_TYPES.checkpointRecorded, payload, now);
|
|
61328
|
-
const billed = poolCheckpointUsage(checkpoint);
|
|
61329
|
-
if (billed && !existing)
|
|
61330
|
-
await upsertPoolUsage(tx, this.journal.runs, row.id, token.generation, billed.key, billed.usage);
|
|
61331
|
-
await tx.run(`INSERT INTO ${this.operations} VALUES (?, ?, ?, ?, ?)`, token.id, token.generation, operationId, request, JSON.stringify(receipt));
|
|
61605
|
+
const receipt = mutation.kind === "session-append" ? await this.appendSession(tx, row, mutation, now) : await this.recordCheckpoint(tx, lease, token, row, mutation, now);
|
|
61606
|
+
await tx.run(`INSERT INTO ${this.operations} VALUES (?, ?, ?, ?, ?)`, token.id, token.generation, operationId, identity, JSON.stringify(receipt));
|
|
61332
61607
|
return receipt;
|
|
61333
61608
|
});
|
|
61334
61609
|
}
|
|
61610
|
+
async appendSession(tx, row, mutation, now) {
|
|
61611
|
+
return { at: now, lastSeq: await applyPoolSession(tx, `${this.journal.runs}_pool_`, row.id, mutation) };
|
|
61612
|
+
}
|
|
61613
|
+
async recordCheckpoint(tx, lease, token, row, mutation, now) {
|
|
61614
|
+
const checkpoint = mutation.checkpoint;
|
|
61615
|
+
await validatePoolMutation(tx, this.journal, lease.runId, fromRow(row), mutation);
|
|
61616
|
+
if (checkpoint.completedAt > Math.max(now, Date.now()) + MAX_WORKER_CLOCK_SKEW_MS)
|
|
61617
|
+
throw new Error("Pool checkpoint timestamp is in the future");
|
|
61618
|
+
assertWorkflowSerializableObject(checkpoint, "Pool checkpoint");
|
|
61619
|
+
const key = JSON.stringify([RUN_EVENT_TYPES.checkpointRecorded, checkpoint.kind, checkpoint.checkpointId]);
|
|
61620
|
+
const payload = serializeJson(checkpoint);
|
|
61621
|
+
const existing = await tx.get(`SELECT seq, at, payload FROM ${this.journal.events} WHERE run_id = ? AND key = ?`, lease.runId, key);
|
|
61622
|
+
if (existing && existing.payload !== payload)
|
|
61623
|
+
throw new Error("Pool checkpoint identity conflict");
|
|
61624
|
+
if (existing && mutation.kind !== "checkpoint")
|
|
61625
|
+
throw new Error("Pool session checkpoint identity conflict");
|
|
61626
|
+
const lastSeq = mutation.kind === "checkpoint" ? undefined : await applyPoolSession(tx, `${this.journal.runs}_pool_`, row.id, mutation);
|
|
61627
|
+
const latest = await tx.get(`SELECT COALESCE(MAX(seq), 0) AS seq FROM ${this.journal.events} WHERE run_id = ?`, lease.runId);
|
|
61628
|
+
const receipt = existing ? { seq: Number(existing.seq), at: Number(existing.at) } : {
|
|
61629
|
+
seq: journalCounter(lease.runId, "seq", Number(latest.seq) + 1, "overflow"),
|
|
61630
|
+
at: now,
|
|
61631
|
+
...lastSeq === undefined ? {} : { lastSeq }
|
|
61632
|
+
};
|
|
61633
|
+
if (!existing)
|
|
61634
|
+
await tx.run(`INSERT INTO ${this.journal.events} VALUES (?, ?, ?, ?, ?, ?, ?)`, lease.runId, receipt.seq, lease.epoch, key, RUN_EVENT_TYPES.checkpointRecorded, payload, now);
|
|
61635
|
+
const billed = poolCheckpointUsage(checkpoint);
|
|
61636
|
+
if (billed && !existing)
|
|
61637
|
+
await upsertPoolUsage(tx, this.journal.runs, row.id, token.generation, billed.key, billed.usage);
|
|
61638
|
+
return receipt;
|
|
61639
|
+
}
|
|
61335
61640
|
async complete(lease, token, result) {
|
|
61336
61641
|
lease = snapshotLease(lease);
|
|
61337
61642
|
token = { ...token };
|
|
@@ -61418,7 +61723,7 @@ function openSqlPoolQueue(journal) {
|
|
|
61418
61723
|
return Promise.reject(new Error("Worker pool requires a SQL run journal"));
|
|
61419
61724
|
return SqlPoolQueue.open(storage);
|
|
61420
61725
|
}
|
|
61421
|
-
var STORE_NOW2 = "CAST(unixepoch('subsec') * 1000 AS INTEGER)", MAX_WORKER_CLOCK_SKEW_MS = 1000, STALE_WORK_LEASE = "Stale work lease";
|
|
61726
|
+
var STORE_NOW2 = "CAST(unixepoch('subsec') * 1000 AS INTEGER)", MAX_WORKER_CLOCK_SKEW_MS = 1000, POOL_SCHEMA_VERSION = 2, STALE_WORK_LEASE = "Stale work lease";
|
|
61422
61727
|
var init_sql_pool_queue = __esm(() => {
|
|
61423
61728
|
init_serializable();
|
|
61424
61729
|
init_boundary_topology();
|
|
@@ -61430,6 +61735,15 @@ var init_sql_pool_queue = __esm(() => {
|
|
|
61430
61735
|
init_values();
|
|
61431
61736
|
});
|
|
61432
61737
|
|
|
61738
|
+
// dist/builtin/workflows/src/workers/coordinator-protocol.ts
|
|
61739
|
+
var POOL_COORDINATOR_ROLE = "pool-coordinator", POOL_SUMMARY_INTERVAL_MS = 250;
|
|
61740
|
+
|
|
61741
|
+
// dist/builtin/workflows/src/workers/host-options.ts
|
|
61742
|
+
var hostWorkerBindings;
|
|
61743
|
+
var init_host_options = __esm(() => {
|
|
61744
|
+
hostWorkerBindings = Symbol("host-worker-bindings");
|
|
61745
|
+
});
|
|
61746
|
+
|
|
61433
61747
|
// dist/builtin/workflows/src/durable/journal/sql-pool-usage.ts
|
|
61434
61748
|
function validate(report) {
|
|
61435
61749
|
validateText("key", report.key);
|
|
@@ -61500,10 +61814,870 @@ var init_sql_pool_usage = __esm(() => {
|
|
|
61500
61814
|
init_values();
|
|
61501
61815
|
});
|
|
61502
61816
|
|
|
61503
|
-
// dist/builtin/workflows/src/workers/
|
|
61504
|
-
|
|
61505
|
-
|
|
61506
|
-
|
|
61817
|
+
// dist/builtin/workflows/src/workers/pool-coordinator.ts
|
|
61818
|
+
import { fork } from "node:child_process";
|
|
61819
|
+
import { randomUUID as randomUUID16 } from "node:crypto";
|
|
61820
|
+
import { createWriteStream } from "node:fs";
|
|
61821
|
+
import { mkdir as mkdir6, rm as rm3, rmdir } from "node:fs/promises";
|
|
61822
|
+
import { dirname as dirname12, join as join28, resolve as resolve11 } from "node:path";
|
|
61823
|
+
import { setTimeout as delay2 } from "node:timers/promises";
|
|
61824
|
+
function workerLogDirectory(journalPath, root) {
|
|
61825
|
+
return join28(dirname12(resolve11(journalPath)), "worker-logs", encodeURIComponent(root).replaceAll(".", "%2E"));
|
|
61826
|
+
}
|
|
61827
|
+
function token(work) {
|
|
61828
|
+
return {
|
|
61829
|
+
id: work.id,
|
|
61830
|
+
generation: work.generation,
|
|
61831
|
+
worker: work.worker,
|
|
61832
|
+
epoch: work.epoch,
|
|
61833
|
+
renewedAt: work.renewedAt,
|
|
61834
|
+
expiresAt: work.expiresAt
|
|
61835
|
+
};
|
|
61836
|
+
}
|
|
61837
|
+
function outside(runId) {
|
|
61838
|
+
return new Error(`Pool stage publication is outside the work subtree: ${runId}`);
|
|
61839
|
+
}
|
|
61840
|
+
function isLiveStage(stage) {
|
|
61841
|
+
return (stage.sessionId !== undefined || stage.sessionFile !== undefined) && stage.status !== "completed" && stage.status !== "failed" && stage.status !== "skipped";
|
|
61842
|
+
}
|
|
61843
|
+
function projectRootRun(run) {
|
|
61844
|
+
return {
|
|
61845
|
+
id: run.id,
|
|
61846
|
+
name: run.name,
|
|
61847
|
+
status: run.status,
|
|
61848
|
+
startedAt: run.startedAt,
|
|
61849
|
+
stages: run.stages.map((stage) => ({
|
|
61850
|
+
id: stage.id,
|
|
61851
|
+
name: stage.name,
|
|
61852
|
+
status: stage.status,
|
|
61853
|
+
parentIds: [],
|
|
61854
|
+
toolEvents: [],
|
|
61855
|
+
...stage.replayKey === undefined ? {} : { replayKey: stage.replayKey },
|
|
61856
|
+
...stage.intercomGroup === undefined ? {} : { intercomGroup: stage.intercomGroup },
|
|
61857
|
+
...stage.workflowChildRun === undefined ? {} : { workflowChildRun: stage.workflowChildRun },
|
|
61858
|
+
...stage.workflowChild === undefined ? {} : { workflowChild: { ...stage.workflowChild, outputs: {} } }
|
|
61859
|
+
}))
|
|
61860
|
+
};
|
|
61861
|
+
}
|
|
61862
|
+
|
|
61863
|
+
class PooledRoutes {
|
|
61864
|
+
root;
|
|
61865
|
+
owner;
|
|
61866
|
+
mirror = createStore();
|
|
61867
|
+
owners = new Map;
|
|
61868
|
+
attempts = new Map;
|
|
61869
|
+
activated = false;
|
|
61870
|
+
bridge;
|
|
61871
|
+
constructor(root, owner) {
|
|
61872
|
+
this.root = root;
|
|
61873
|
+
this.owner = owner;
|
|
61874
|
+
}
|
|
61875
|
+
static async open(root, cwd, rootRun, forward) {
|
|
61876
|
+
await Promise.resolve().then(() => init_workflow_route_owner());
|
|
61877
|
+
const owner = createWorkflowRouteOwner(cwd);
|
|
61878
|
+
const routes = new PooledRoutes(root, owner);
|
|
61879
|
+
if (rootRun !== undefined)
|
|
61880
|
+
routes.updateRoot(rootRun);
|
|
61881
|
+
routes.bridge = registerPendingStageIntercomBridge({
|
|
61882
|
+
events: {
|
|
61883
|
+
emit: (event, payload) => owner.events.emit(event, payload),
|
|
61884
|
+
on: (event, listener) => event === PENDING_STAGE_MESSAGE_EVENT2 || event === STICKY_LIVE_DELIVERED_EVENT2 ? undefined : owner.events.on(event, listener)
|
|
61885
|
+
}
|
|
61886
|
+
}, routes.mirror, (runId) => routes.activated && routes.owners.has(runId));
|
|
61887
|
+
for (const [event, kind] of [
|
|
61888
|
+
[PENDING_STAGE_MESSAGE_EVENT2, "pending-stage-message"],
|
|
61889
|
+
[STICKY_LIVE_DELIVERED_EVENT2, "sticky-live-delivered"]
|
|
61890
|
+
])
|
|
61891
|
+
owner.events.on(event, (payload) => {
|
|
61892
|
+
const claimed = payload;
|
|
61893
|
+
if (claimed.handled !== false)
|
|
61894
|
+
return;
|
|
61895
|
+
claimed.handled = true;
|
|
61896
|
+
const { handled: _handled, completion: _completion, ...forwarded } = claimed;
|
|
61897
|
+
claimed.completion = forward({ kind, payload: forwarded });
|
|
61898
|
+
});
|
|
61899
|
+
return routes;
|
|
61900
|
+
}
|
|
61901
|
+
updateRoot(rootRun) {
|
|
61902
|
+
this.mirror.recordRunStart({ ...rootRun, inputs: {}, toolNodes: [], rootRunId: this.root });
|
|
61903
|
+
}
|
|
61904
|
+
accept(entries) {
|
|
61905
|
+
this.mirror.batch(() => {
|
|
61906
|
+
for (const { work, runs } of entries) {
|
|
61907
|
+
let attempt = this.attempts.get(work.id);
|
|
61908
|
+
if (attempt !== undefined && work.generation < attempt.generation)
|
|
61909
|
+
continue;
|
|
61910
|
+
if (attempt === undefined || attempt.generation !== work.generation) {
|
|
61911
|
+
if (attempt !== undefined)
|
|
61912
|
+
this.release(attempt);
|
|
61913
|
+
attempt = { generation: work.generation, runs: attempt?.runs ?? new Set, fenced: false };
|
|
61914
|
+
this.attempts.set(work.id, attempt);
|
|
61915
|
+
}
|
|
61916
|
+
if (attempt.fenced)
|
|
61917
|
+
continue;
|
|
61918
|
+
for (const run of runs) {
|
|
61919
|
+
attempt.runs.add(run.id);
|
|
61920
|
+
this.owners.set(run.id, work.id);
|
|
61921
|
+
const { pendingStageMessages: _messages, ...published } = run;
|
|
61922
|
+
this.mirror.recordRunStart({ ...published, rootRunId: this.root });
|
|
61923
|
+
if (run.stages.some((stage) => stage.pendingStageDeliveryAvailable === true))
|
|
61924
|
+
this.activated = true;
|
|
61925
|
+
}
|
|
61926
|
+
}
|
|
61927
|
+
});
|
|
61928
|
+
}
|
|
61929
|
+
fence(work) {
|
|
61930
|
+
const attempt = this.attempts.get(work.id);
|
|
61931
|
+
if (attempt === undefined || attempt.generation > work.generation)
|
|
61932
|
+
return;
|
|
61933
|
+
if (attempt.generation < work.generation) {
|
|
61934
|
+
this.release(attempt);
|
|
61935
|
+
this.attempts.set(work.id, { generation: work.generation, runs: attempt.runs, fenced: true });
|
|
61936
|
+
return;
|
|
61937
|
+
}
|
|
61938
|
+
this.release(attempt);
|
|
61939
|
+
attempt.fenced = true;
|
|
61940
|
+
}
|
|
61941
|
+
async route(work, runId) {
|
|
61942
|
+
this.requireOwned(work, runId);
|
|
61943
|
+
const readiness = workflowPendingStageRouteReady(this.mirror, runId);
|
|
61944
|
+
await readiness?.completion.catch(() => {});
|
|
61945
|
+
readiness?.assertCurrent();
|
|
61946
|
+
const waiting = new AbortController;
|
|
61947
|
+
await Promise.race([
|
|
61948
|
+
this.owner.registered(runId),
|
|
61949
|
+
delay2(ROUTE_REGISTRATION_WAIT_MS, undefined, { signal: waiting.signal, ref: false }).catch(() => {})
|
|
61950
|
+
]);
|
|
61951
|
+
waiting.abort();
|
|
61952
|
+
this.requireOwned(work, runId);
|
|
61953
|
+
return workflowPendingStageRouteCapability(this.mirror, runId);
|
|
61954
|
+
}
|
|
61955
|
+
async undeliverable(payload) {
|
|
61956
|
+
const event = { ...payload, handled: false };
|
|
61957
|
+
this.owner.events.emit(PENDING_STAGE_UNDELIVERABLE_EVENT2, event);
|
|
61958
|
+
return event.handled && await event.completion === true;
|
|
61959
|
+
}
|
|
61960
|
+
async close() {
|
|
61961
|
+
const bridge = this.bridge;
|
|
61962
|
+
if (bridge === undefined)
|
|
61963
|
+
return;
|
|
61964
|
+
this.bridge = undefined;
|
|
61965
|
+
try {
|
|
61966
|
+
await bridge.flush();
|
|
61967
|
+
} finally {
|
|
61968
|
+
bridge.refresh();
|
|
61969
|
+
bridge();
|
|
61970
|
+
await this.owner.closeGroup(workflowInvocationIntercomGroup(this.root));
|
|
61971
|
+
}
|
|
61972
|
+
}
|
|
61973
|
+
requireOwned(work, runId) {
|
|
61974
|
+
const attempt = this.attempts.get(work.id);
|
|
61975
|
+
if (attempt === undefined || attempt.generation !== work.generation || attempt.fenced)
|
|
61976
|
+
throw new Error(STALE_WORK_LEASE);
|
|
61977
|
+
if (!attempt.runs.has(runId))
|
|
61978
|
+
throw outside(runId);
|
|
61979
|
+
}
|
|
61980
|
+
release(attempt) {
|
|
61981
|
+
this.mirror.batch(() => {
|
|
61982
|
+
for (const id of attempt.runs) {
|
|
61983
|
+
rotateWorkflowPendingStageRouteCapability(this.mirror, id);
|
|
61984
|
+
const run = this.mirror.runs().find((candidate) => candidate.id === id);
|
|
61985
|
+
if (run?.stages.some(isLiveStage) !== true)
|
|
61986
|
+
continue;
|
|
61987
|
+
this.mirror.recordRunStart({
|
|
61988
|
+
...run,
|
|
61989
|
+
stages: run.stages.map((stage) => {
|
|
61990
|
+
if (!isLiveStage(stage))
|
|
61991
|
+
return stage;
|
|
61992
|
+
const { sessionId: _session, sessionFile: _file, ...rest } = stage;
|
|
61993
|
+
return { ...rest, attachable: false, attached: false };
|
|
61994
|
+
})
|
|
61995
|
+
});
|
|
61996
|
+
}
|
|
61997
|
+
});
|
|
61998
|
+
}
|
|
61999
|
+
}
|
|
62000
|
+
|
|
62001
|
+
class PoolCoordinator {
|
|
62002
|
+
port;
|
|
62003
|
+
children = new Set;
|
|
62004
|
+
incarnations = new Map;
|
|
62005
|
+
readyWorkers = new Set;
|
|
62006
|
+
logs = new Map;
|
|
62007
|
+
cleanups = new Set;
|
|
62008
|
+
controlAcks = new Map;
|
|
62009
|
+
lastHeard = new Map;
|
|
62010
|
+
pendingActivity = new Map;
|
|
62011
|
+
batches = new Map;
|
|
62012
|
+
publicationFailures = new Map;
|
|
62013
|
+
requests = new Map;
|
|
62014
|
+
deliveryRequests = new Map;
|
|
62015
|
+
inflight = new Set;
|
|
62016
|
+
lastTick = performance.now();
|
|
62017
|
+
stalledMs = 0;
|
|
62018
|
+
ticker;
|
|
62019
|
+
recoveryTimer;
|
|
62020
|
+
flushTimer;
|
|
62021
|
+
lastFlushAt = 0;
|
|
62022
|
+
batchSequence = 0;
|
|
62023
|
+
requestSequence = 0;
|
|
62024
|
+
usageChanged = false;
|
|
62025
|
+
budgetBlocked = false;
|
|
62026
|
+
desiredControl;
|
|
62027
|
+
root;
|
|
62028
|
+
lease;
|
|
62029
|
+
options;
|
|
62030
|
+
journal;
|
|
62031
|
+
queue;
|
|
62032
|
+
usageLedger;
|
|
62033
|
+
usageTotal;
|
|
62034
|
+
admission;
|
|
62035
|
+
routes;
|
|
62036
|
+
publishing = false;
|
|
62037
|
+
sources = [];
|
|
62038
|
+
logDirectory;
|
|
62039
|
+
starting;
|
|
62040
|
+
recovering;
|
|
62041
|
+
nextRecoveryAt = 0;
|
|
62042
|
+
cleanupFailure;
|
|
62043
|
+
closed = false;
|
|
62044
|
+
closing;
|
|
62045
|
+
replacements = 0;
|
|
62046
|
+
constructor(port) {
|
|
62047
|
+
this.port = port;
|
|
62048
|
+
port.onMessage((message) => this.receive(message));
|
|
62049
|
+
this.ticker = setInterval(() => this.observeResponsiveness(), TICK_INTERVAL_MS).unref();
|
|
62050
|
+
}
|
|
62051
|
+
async shutdown() {
|
|
62052
|
+
await this.closing?.catch(() => {});
|
|
62053
|
+
while (this.inflight.size > 0)
|
|
62054
|
+
await Promise.allSettled([...this.inflight]);
|
|
62055
|
+
await this.journal?.close().catch(() => {});
|
|
62056
|
+
}
|
|
62057
|
+
receive(message) {
|
|
62058
|
+
if (message.kind === "request-reply") {
|
|
62059
|
+
const pending = this.requests.get(message.id);
|
|
62060
|
+
this.requests.delete(message.id);
|
|
62061
|
+
if (message.error !== undefined)
|
|
62062
|
+
pending?.reject(new Error(message.error));
|
|
62063
|
+
else
|
|
62064
|
+
pending?.resolve(message.value);
|
|
62065
|
+
return;
|
|
62066
|
+
}
|
|
62067
|
+
if ("id" in message) {
|
|
62068
|
+
const command = message;
|
|
62069
|
+
const handled = this.execute(command).then((value2) => this.port.send({ kind: "reply", id: command.id, value: value2 }), (error) => this.port.send({ kind: "reply", id: command.id, error: error.message }));
|
|
62070
|
+
this.inflight.add(handled);
|
|
62071
|
+
handled.finally(() => this.inflight.delete(handled));
|
|
62072
|
+
return;
|
|
62073
|
+
}
|
|
62074
|
+
this.notice(message);
|
|
62075
|
+
}
|
|
62076
|
+
notice(notice) {
|
|
62077
|
+
if (notice.kind === "budget")
|
|
62078
|
+
this.budgetBlocked = notice.blocked;
|
|
62079
|
+
else if (notice.kind === "root")
|
|
62080
|
+
this.routes?.updateRoot(notice.rootRun);
|
|
62081
|
+
else {
|
|
62082
|
+
const batch = this.batches.get(notice.batch);
|
|
62083
|
+
this.batches.delete(notice.batch);
|
|
62084
|
+
const rejected = new Map(notice.rejected.map((entry) => [entry.workId, entry]));
|
|
62085
|
+
for (const entry of rejected.values())
|
|
62086
|
+
this.publicationFailures.set(entry.workId, { generation: entry.generation, error: entry.error });
|
|
62087
|
+
this.routes?.accept((batch?.entries ?? []).filter((entry) => rejected.get(entry.work.id)?.generation !== entry.work.generation));
|
|
62088
|
+
batch?.settle();
|
|
62089
|
+
}
|
|
62090
|
+
}
|
|
62091
|
+
async execute(command) {
|
|
62092
|
+
switch (command.kind) {
|
|
62093
|
+
case "prepare":
|
|
62094
|
+
return this.prepare(command.root, command.lease, command.options, command.control);
|
|
62095
|
+
case "control":
|
|
62096
|
+
return this.control(command.state);
|
|
62097
|
+
case "start":
|
|
62098
|
+
this.starting ??= this.start(command.sources, command.cwd, command.publish, command.rootRun);
|
|
62099
|
+
return this.starting;
|
|
62100
|
+
case "enqueue":
|
|
62101
|
+
return this.enqueue(command);
|
|
62102
|
+
case "transport":
|
|
62103
|
+
return this.transport(command.request, command);
|
|
62104
|
+
case "undeliverable":
|
|
62105
|
+
return await this.routes?.undeliverable(command.payload) ?? false;
|
|
62106
|
+
case "close":
|
|
62107
|
+
this.closing ??= this.close();
|
|
62108
|
+
return this.closing;
|
|
62109
|
+
}
|
|
62110
|
+
}
|
|
62111
|
+
async prepare(root, lease, options, control) {
|
|
62112
|
+
if (this.closed)
|
|
62113
|
+
throw new Error("Local worker pool is closed");
|
|
62114
|
+
if (this.root !== undefined && this.root !== root)
|
|
62115
|
+
throw new Error("Local worker pool belongs to another root");
|
|
62116
|
+
this.root = root;
|
|
62117
|
+
this.options = options;
|
|
62118
|
+
this.desiredControl ??= control;
|
|
62119
|
+
this.journal ??= await resolveRunJournal({ journal: { path: options.journalPath } });
|
|
62120
|
+
this.usageLedger ??= await openSqlPoolUsage(this.journal);
|
|
62121
|
+
this.usageTotal = await this.usageLedger.total(root);
|
|
62122
|
+
this.lease = lease;
|
|
62123
|
+
const queue = await openSqlPoolQueue(this.journal);
|
|
62124
|
+
await queue.initialize(lease, COORDINATOR_LEASE_MS);
|
|
62125
|
+
await queue.releasePreviousIncarnations(lease);
|
|
62126
|
+
for (;; ) {
|
|
62127
|
+
const intended = this.desiredControl ?? "running";
|
|
62128
|
+
await queue.setControl(lease, intended);
|
|
62129
|
+
if (intended === (this.desiredControl ?? "running"))
|
|
62130
|
+
break;
|
|
62131
|
+
}
|
|
62132
|
+
this.queue = queue;
|
|
62133
|
+
return this.usageTotal;
|
|
62134
|
+
}
|
|
62135
|
+
async control(state) {
|
|
62136
|
+
this.desiredControl = state;
|
|
62137
|
+
if (!this.queue || !this.lease)
|
|
62138
|
+
return;
|
|
62139
|
+
const control = await this.queue.setControl(this.lease, state);
|
|
62140
|
+
const deadline = Date.now() + CONTROL_ACKNOWLEDGEMENT_TIMEOUT_MS;
|
|
62141
|
+
while ([...this.readyWorkers].some((worker) => (this.controlAcks.get(worker) ?? -1) < control.revision)) {
|
|
62142
|
+
if (this.desiredControl !== state)
|
|
62143
|
+
return;
|
|
62144
|
+
if (Date.now() >= deadline) {
|
|
62145
|
+
if (state === "quit")
|
|
62146
|
+
break;
|
|
62147
|
+
throw new Error("Worker control acknowledgement unresolved");
|
|
62148
|
+
}
|
|
62149
|
+
await delay2(25);
|
|
62150
|
+
}
|
|
62151
|
+
this.flush();
|
|
62152
|
+
}
|
|
62153
|
+
async start(sources, cwd, publish, rootRun) {
|
|
62154
|
+
if (this.closed)
|
|
62155
|
+
throw new Error("Local worker pool is closed");
|
|
62156
|
+
const root = this.root;
|
|
62157
|
+
if (!this.queue || !this.lease || !this.options || root === undefined || this.lease.runId !== root)
|
|
62158
|
+
throw new Error("Local worker pool requires root preparation");
|
|
62159
|
+
this.sources = sources;
|
|
62160
|
+
this.publishing = publish;
|
|
62161
|
+
if (publish) {
|
|
62162
|
+
this.routes = await PooledRoutes.open(root, cwd, rootRun, (request) => this.request(request));
|
|
62163
|
+
if (this.closed) {
|
|
62164
|
+
await this.routes.close();
|
|
62165
|
+
throw new Error("Local worker pool is closed");
|
|
62166
|
+
}
|
|
62167
|
+
}
|
|
62168
|
+
if (this.options.admission)
|
|
62169
|
+
this.admission = await SqlResourceAdmission.open(this.journal, {
|
|
62170
|
+
...this.options.admission,
|
|
62171
|
+
root,
|
|
62172
|
+
holder: "coordinator"
|
|
62173
|
+
});
|
|
62174
|
+
this.logDirectory = workerLogDirectory(this.options.journalPath, root);
|
|
62175
|
+
await mkdir6(this.logDirectory, { recursive: true }).catch(() => {
|
|
62176
|
+
return;
|
|
62177
|
+
});
|
|
62178
|
+
this.recoveryTimer = setInterval(() => {
|
|
62179
|
+
this.recoverExpired().catch((error) => this.fail(error));
|
|
62180
|
+
}, RECOVERY_INTERVAL_MS).unref();
|
|
62181
|
+
const workers = Array.from({ length: this.options.workers }, () => this.spawn());
|
|
62182
|
+
try {
|
|
62183
|
+
await Promise.any(workers);
|
|
62184
|
+
} catch (error) {
|
|
62185
|
+
throw error.errors[0];
|
|
62186
|
+
}
|
|
62187
|
+
}
|
|
62188
|
+
async enqueue(command) {
|
|
62189
|
+
if (!this.queue || !this.lease || this.root === undefined)
|
|
62190
|
+
throw new Error("Local worker pool requires root preparation");
|
|
62191
|
+
if (command.rootRun !== undefined)
|
|
62192
|
+
this.routes?.updateRoot(command.rootRun);
|
|
62193
|
+
const work = await this.queue.enqueue(this.lease, command.descriptor, command.boundary);
|
|
62194
|
+
const result = await this.queue.result(this.root, work.id);
|
|
62195
|
+
return { work, ...result === undefined ? {} : { result } };
|
|
62196
|
+
}
|
|
62197
|
+
async transport(request, payload) {
|
|
62198
|
+
const target = this.deliveryRequests.get(request);
|
|
62199
|
+
if (target === undefined || !this.incarnations.has(target.slot.worker))
|
|
62200
|
+
throw new Error("Worker exited before delivery acknowledgement");
|
|
62201
|
+
const { slot, workerRequest } = target;
|
|
62202
|
+
await new Promise((resolve, reject) => {
|
|
62203
|
+
const id = randomUUID16();
|
|
62204
|
+
slot.deliveries.set(id, { resolve, reject });
|
|
62205
|
+
slot.child.send({ id: workerRequest, delivery: { id, from: payload.from, message: payload.message } }, (error) => {
|
|
62206
|
+
if (error) {
|
|
62207
|
+
slot.deliveries.delete(id);
|
|
62208
|
+
reject(error);
|
|
62209
|
+
}
|
|
62210
|
+
});
|
|
62211
|
+
});
|
|
62212
|
+
}
|
|
62213
|
+
request(body) {
|
|
62214
|
+
const id = ++this.requestSequence;
|
|
62215
|
+
return new Promise((resolve, reject) => {
|
|
62216
|
+
this.requests.set(id, { resolve, reject });
|
|
62217
|
+
this.port.send({ kind: "request", id, request: body });
|
|
62218
|
+
});
|
|
62219
|
+
}
|
|
62220
|
+
fail(error) {
|
|
62221
|
+
this.port.send({ kind: "failure", error: error.message });
|
|
62222
|
+
}
|
|
62223
|
+
publish(work, runs) {
|
|
62224
|
+
const failure = this.publicationFailures.get(work.id);
|
|
62225
|
+
if (failure?.generation === work.generation)
|
|
62226
|
+
throw new Error(failure.error);
|
|
62227
|
+
if (!this.publishing)
|
|
62228
|
+
return;
|
|
62229
|
+
this.pendingActivity.set(work.id, { work: token(work), runs });
|
|
62230
|
+
this.scheduleFlush();
|
|
62231
|
+
}
|
|
62232
|
+
scheduleFlush() {
|
|
62233
|
+
if (this.flushTimer !== undefined || this.closed)
|
|
62234
|
+
return;
|
|
62235
|
+
const wait = Math.max(0, this.lastFlushAt + POOL_SUMMARY_INTERVAL_MS - performance.now());
|
|
62236
|
+
this.flushTimer = setTimeout(() => {
|
|
62237
|
+
this.flushTimer = undefined;
|
|
62238
|
+
if (this.batches.size > 0) {
|
|
62239
|
+
this.lastFlushAt = performance.now();
|
|
62240
|
+
this.scheduleFlush();
|
|
62241
|
+
return;
|
|
62242
|
+
}
|
|
62243
|
+
this.flush();
|
|
62244
|
+
}, wait);
|
|
62245
|
+
}
|
|
62246
|
+
flush() {
|
|
62247
|
+
clearTimeout(this.flushTimer);
|
|
62248
|
+
this.flushTimer = undefined;
|
|
62249
|
+
if (this.pendingActivity.size === 0 && !this.usageChanged) {
|
|
62250
|
+
const last = [...this.batches.values()].at(-1);
|
|
62251
|
+
return last === undefined ? Promise.resolve() : new Promise((resolve) => this.chainSettle(last, resolve));
|
|
62252
|
+
}
|
|
62253
|
+
this.lastFlushAt = performance.now();
|
|
62254
|
+
const batch = ++this.batchSequence;
|
|
62255
|
+
const entries = [...this.pendingActivity.values()];
|
|
62256
|
+
this.pendingActivity.clear();
|
|
62257
|
+
const usage = this.usageChanged ? this.usageTotal : undefined;
|
|
62258
|
+
this.usageChanged = false;
|
|
62259
|
+
const settled = new Promise((resolve) => this.batches.set(batch, { entries, settle: resolve }));
|
|
62260
|
+
this.port.send({ kind: "activity", batch, entries, ...usage === undefined ? {} : { usage } });
|
|
62261
|
+
return settled;
|
|
62262
|
+
}
|
|
62263
|
+
chainSettle(batch, resolve) {
|
|
62264
|
+
const settle = batch.settle;
|
|
62265
|
+
batch.settle = () => {
|
|
62266
|
+
settle();
|
|
62267
|
+
resolve();
|
|
62268
|
+
};
|
|
62269
|
+
}
|
|
62270
|
+
noteUsage(usage) {
|
|
62271
|
+
this.usageTotal = usage;
|
|
62272
|
+
this.usageChanged = true;
|
|
62273
|
+
this.scheduleFlush();
|
|
62274
|
+
}
|
|
62275
|
+
async sendResult(workId) {
|
|
62276
|
+
const result = await this.queue.result(this.root, workId);
|
|
62277
|
+
if (result === undefined)
|
|
62278
|
+
return false;
|
|
62279
|
+
this.flush();
|
|
62280
|
+
this.port.send({
|
|
62281
|
+
kind: "result",
|
|
62282
|
+
workId,
|
|
62283
|
+
result,
|
|
62284
|
+
...this.usageTotal === undefined ? {} : { usage: this.usageTotal }
|
|
62285
|
+
});
|
|
62286
|
+
return true;
|
|
62287
|
+
}
|
|
62288
|
+
fence(work) {
|
|
62289
|
+
this.pendingActivity.delete(work.id);
|
|
62290
|
+
this.routes?.fence(work);
|
|
62291
|
+
this.port.send({ kind: "fence", work: { id: work.id, generation: work.generation } });
|
|
62292
|
+
}
|
|
62293
|
+
spawn() {
|
|
62294
|
+
if (this.closed)
|
|
62295
|
+
return Promise.reject(new Error("Local worker pool is closed"));
|
|
62296
|
+
const root = this.root;
|
|
62297
|
+
const options = this.options;
|
|
62298
|
+
const child = fork(options.workerEntry ?? process.argv[1], [
|
|
62299
|
+
JSON.stringify({
|
|
62300
|
+
journalPath: options.journalPath,
|
|
62301
|
+
root,
|
|
62302
|
+
definitions: this.sources,
|
|
62303
|
+
admission: options.admission
|
|
62304
|
+
})
|
|
62305
|
+
], { stdio: ["ignore", "ignore", "pipe", "ipc"], execArgv: [] });
|
|
62306
|
+
const disconnected = new Promise((resolve) => child.once("disconnect", () => resolve()));
|
|
62307
|
+
this.children.add(child);
|
|
62308
|
+
if (child.pid !== undefined)
|
|
62309
|
+
this.port.send({ kind: "spawned", pid: child.pid });
|
|
62310
|
+
const worker = randomUUID16();
|
|
62311
|
+
this.incarnations.set(worker, child);
|
|
62312
|
+
const slot = { worker, child, deliveries: new Map };
|
|
62313
|
+
let ready = false;
|
|
62314
|
+
let serial = Promise.resolve();
|
|
62315
|
+
const logPath = join28(this.logDirectory, `${worker}.log`);
|
|
62316
|
+
this.logs.set(child, logPath);
|
|
62317
|
+
let logFailure;
|
|
62318
|
+
const log = createWriteStream(logPath, { flags: "a" });
|
|
62319
|
+
log.on("error", (error) => {
|
|
62320
|
+
logFailure = error.message;
|
|
62321
|
+
});
|
|
62322
|
+
let stderr = "";
|
|
62323
|
+
child.stderr?.on("data", (chunk) => {
|
|
62324
|
+
stderr = (stderr + chunk.toString()).slice(-STDERR_TAIL_CHARACTERS);
|
|
62325
|
+
});
|
|
62326
|
+
child.stderr?.on("error", (error) => {
|
|
62327
|
+
logFailure ??= error.message;
|
|
62328
|
+
log.end();
|
|
62329
|
+
});
|
|
62330
|
+
child.stderr?.pipe(log);
|
|
62331
|
+
const diagnosed = (summary) => new Error(`${summary} (worker stderr log: ${logFailure === undefined ? logPath : `unavailable, ${logFailure}`})${stderr ? `: ${stderr}` : ""}`);
|
|
62332
|
+
const startupTimeoutMs = options.startupTimeoutMs;
|
|
62333
|
+
let timedOut = false;
|
|
62334
|
+
let terminated = false;
|
|
62335
|
+
const terminate = () => {
|
|
62336
|
+
if (terminated)
|
|
62337
|
+
return;
|
|
62338
|
+
terminated = true;
|
|
62339
|
+
child.kill("SIGKILL");
|
|
62340
|
+
};
|
|
62341
|
+
const replied = (error) => {
|
|
62342
|
+
if (error)
|
|
62343
|
+
terminate();
|
|
62344
|
+
};
|
|
62345
|
+
return new Promise((resolve, reject) => {
|
|
62346
|
+
const startup = setTimeout(() => {
|
|
62347
|
+
timedOut = true;
|
|
62348
|
+
terminate();
|
|
62349
|
+
}, startupTimeoutMs);
|
|
62350
|
+
child.on("error", (error) => {
|
|
62351
|
+
clearTimeout(startup);
|
|
62352
|
+
reject(error);
|
|
62353
|
+
terminate();
|
|
62354
|
+
});
|
|
62355
|
+
const closed = new Promise((resolve) => child.once("close", () => resolve()));
|
|
62356
|
+
child.once("exit", () => {
|
|
62357
|
+
Promise.race([closed, delay2(STDERR_DRAIN_GRACE_MS)]).then(() => reject(diagnosed(timedOut ? `Local worker startup timed out after ${startupTimeoutMs} ms` : "Local worker exited before readiness")));
|
|
62358
|
+
this.children.delete(child);
|
|
62359
|
+
this.logs.delete(child);
|
|
62360
|
+
if (child.pid !== undefined)
|
|
62361
|
+
this.port.send({ kind: "exited", pid: child.pid });
|
|
62362
|
+
clearTimeout(startup);
|
|
62363
|
+
this.incarnations.delete(worker);
|
|
62364
|
+
this.readyWorkers.delete(worker);
|
|
62365
|
+
this.controlAcks.delete(worker);
|
|
62366
|
+
this.lastHeard.delete(worker);
|
|
62367
|
+
if (slot.work)
|
|
62368
|
+
this.fence(slot.work);
|
|
62369
|
+
for (const delivery of slot.deliveries.values())
|
|
62370
|
+
delivery.reject(new Error("Worker exited before delivery acknowledgement"));
|
|
62371
|
+
slot.deliveries.clear();
|
|
62372
|
+
const cleanup = disconnected.then(() => serial).then(async () => {
|
|
62373
|
+
try {
|
|
62374
|
+
if (await this.admission?.releaseStoppedSessions(worker))
|
|
62375
|
+
throw new Error("Local worker process cleanup unconfirmed; permits quarantined");
|
|
62376
|
+
} catch (error) {
|
|
62377
|
+
this.cleanupFailure = error instanceof Error ? error : new Error(String(error));
|
|
62378
|
+
throw this.cleanupFailure;
|
|
62379
|
+
}
|
|
62380
|
+
}).then(async () => {
|
|
62381
|
+
if (this.closed || !ready || this.desiredControl === "quit")
|
|
62382
|
+
return;
|
|
62383
|
+
let work = slot.work;
|
|
62384
|
+
if (work && await this.sendResult(work.id)) {
|
|
62385
|
+
work = undefined;
|
|
62386
|
+
slot.work = undefined;
|
|
62387
|
+
this.replacements = 0;
|
|
62388
|
+
}
|
|
62389
|
+
if (this.closed)
|
|
62390
|
+
return;
|
|
62391
|
+
if (work)
|
|
62392
|
+
await this.queue.releaseExited(this.lease, work);
|
|
62393
|
+
const exhausted = work ? work.generation >= 3 : ++this.replacements > 3;
|
|
62394
|
+
if (exhausted)
|
|
62395
|
+
this.fail(diagnosed("Local worker retry limit"));
|
|
62396
|
+
else
|
|
62397
|
+
await this.spawn();
|
|
62398
|
+
}).catch((error) => {
|
|
62399
|
+
this.fail(error);
|
|
62400
|
+
});
|
|
62401
|
+
this.cleanups.add(cleanup);
|
|
62402
|
+
cleanup.finally(() => this.cleanups.delete(cleanup));
|
|
62403
|
+
});
|
|
62404
|
+
child.on("message", (message) => {
|
|
62405
|
+
this.observeResponsiveness();
|
|
62406
|
+
this.lastHeard.set(worker, { at: performance.now(), stalled: this.stalledMs });
|
|
62407
|
+
if (message.kind === "delivery-ack") {
|
|
62408
|
+
const pending = slot.deliveries.get(message.deliveryId);
|
|
62409
|
+
slot.deliveries.delete(message.deliveryId);
|
|
62410
|
+
if (message.error)
|
|
62411
|
+
pending?.reject(new Error(message.error));
|
|
62412
|
+
else
|
|
62413
|
+
pending?.resolve();
|
|
62414
|
+
return;
|
|
62415
|
+
}
|
|
62416
|
+
let routeReady;
|
|
62417
|
+
const handled = serial.then(async () => {
|
|
62418
|
+
if (message.kind === "ready") {
|
|
62419
|
+
if (terminated)
|
|
62420
|
+
return;
|
|
62421
|
+
ready = true;
|
|
62422
|
+
this.readyWorkers.add(worker);
|
|
62423
|
+
clearTimeout(startup);
|
|
62424
|
+
resolve();
|
|
62425
|
+
return;
|
|
62426
|
+
}
|
|
62427
|
+
await this.starting;
|
|
62428
|
+
const queue = this.queue;
|
|
62429
|
+
const lease = this.lease;
|
|
62430
|
+
if (message.kind === "control")
|
|
62431
|
+
return queue.control(lease);
|
|
62432
|
+
if (message.kind === "controlled") {
|
|
62433
|
+
const control = await queue.control(lease);
|
|
62434
|
+
if (message.revision > control.revision)
|
|
62435
|
+
throw new Error("Unknown control revision");
|
|
62436
|
+
this.controlAcks.set(worker, Math.max(this.controlAcks.get(worker) ?? -1, message.revision));
|
|
62437
|
+
return;
|
|
62438
|
+
}
|
|
62439
|
+
if (message.kind === "claim") {
|
|
62440
|
+
if (slot.work)
|
|
62441
|
+
throw new Error("Worker already holds an attempt");
|
|
62442
|
+
if (this.budgetBlocked)
|
|
62443
|
+
return;
|
|
62444
|
+
await this.recoverExpired();
|
|
62445
|
+
slot.work = await queue.claim(lease, worker, this.sources.map((source) => ({
|
|
62446
|
+
definition: `${source.path}#${source.exportName}`,
|
|
62447
|
+
digest: source.digest
|
|
62448
|
+
})), options.leaseMs);
|
|
62449
|
+
return slot.work;
|
|
62450
|
+
}
|
|
62451
|
+
let work = slot.work;
|
|
62452
|
+
if (!work)
|
|
62453
|
+
throw new Error("Worker has no active attempt");
|
|
62454
|
+
if (message.kind === "renew" || work.expiresAt - Date.now() < options.leaseMs / 2) {
|
|
62455
|
+
work = await queue.extend(lease, work, options.leaseMs);
|
|
62456
|
+
slot.work = work;
|
|
62457
|
+
}
|
|
62458
|
+
if (message.kind === "activity")
|
|
62459
|
+
return this.publish(work, message.runs);
|
|
62460
|
+
if (message.kind === "route") {
|
|
62461
|
+
if (this.routes === undefined)
|
|
62462
|
+
return;
|
|
62463
|
+
await this.flush();
|
|
62464
|
+
this.assertPublished(work);
|
|
62465
|
+
routeReady = this.routes.route(work, message.runId);
|
|
62466
|
+
return;
|
|
62467
|
+
}
|
|
62468
|
+
if (message.kind === "delivery-ready") {
|
|
62469
|
+
if (!this.publishing)
|
|
62470
|
+
return;
|
|
62471
|
+
await this.flush();
|
|
62472
|
+
this.assertPublished(work);
|
|
62473
|
+
return this.request({
|
|
62474
|
+
kind: "ready",
|
|
62475
|
+
work: token(work),
|
|
62476
|
+
runId: message.runId,
|
|
62477
|
+
stageId: message.stageId,
|
|
62478
|
+
stageName: message.stageName,
|
|
62479
|
+
...message.sessionId === undefined ? {} : { sessionId: message.sessionId }
|
|
62480
|
+
});
|
|
62481
|
+
}
|
|
62482
|
+
if (message.kind === "delivery") {
|
|
62483
|
+
if (!message.recipient)
|
|
62484
|
+
throw new Error("Worker delivery requires a session identity");
|
|
62485
|
+
if (!this.publishing)
|
|
62486
|
+
return;
|
|
62487
|
+
await this.flush();
|
|
62488
|
+
this.assertPublished(work);
|
|
62489
|
+
const id = ++this.requestSequence;
|
|
62490
|
+
this.deliveryRequests.set(id, { slot, workerRequest: message.id });
|
|
62491
|
+
try {
|
|
62492
|
+
return await new Promise((resolve, reject) => {
|
|
62493
|
+
this.requests.set(id, { resolve, reject });
|
|
62494
|
+
this.port.send({
|
|
62495
|
+
kind: "request",
|
|
62496
|
+
id,
|
|
62497
|
+
request: {
|
|
62498
|
+
kind: "deliver",
|
|
62499
|
+
work: token(work),
|
|
62500
|
+
runId: message.runId,
|
|
62501
|
+
stageId: message.stageId,
|
|
62502
|
+
stageName: message.stageName,
|
|
62503
|
+
recipient: message.recipient
|
|
62504
|
+
}
|
|
62505
|
+
});
|
|
62506
|
+
});
|
|
62507
|
+
} finally {
|
|
62508
|
+
this.deliveryRequests.delete(id);
|
|
62509
|
+
}
|
|
62510
|
+
}
|
|
62511
|
+
if (message.kind === "renew")
|
|
62512
|
+
return;
|
|
62513
|
+
if (message.kind === "commit") {
|
|
62514
|
+
const receipt = await queue.commit(lease, work, message.operation, message.mutation);
|
|
62515
|
+
this.noteUsage(await this.usageLedger.total(root));
|
|
62516
|
+
return receipt;
|
|
62517
|
+
}
|
|
62518
|
+
if (message.kind === "usage") {
|
|
62519
|
+
this.noteUsage(await this.usageLedger.report(lease, work, message.reports));
|
|
62520
|
+
return;
|
|
62521
|
+
}
|
|
62522
|
+
if (message.kind === "budget") {
|
|
62523
|
+
await queue.assertCurrent(lease, work);
|
|
62524
|
+
if (this.budgetBlocked)
|
|
62525
|
+
throw new Error("Root workflow budget exhausted");
|
|
62526
|
+
return queue.control(lease);
|
|
62527
|
+
}
|
|
62528
|
+
if (message.kind !== "complete")
|
|
62529
|
+
throw new Error("Invalid worker operation");
|
|
62530
|
+
await queue.complete(lease, work, message.result);
|
|
62531
|
+
this.replacements = 0;
|
|
62532
|
+
slot.work = undefined;
|
|
62533
|
+
await this.sendResult(work.id);
|
|
62534
|
+
});
|
|
62535
|
+
const response = handled.then((value2) => routeReady ?? value2);
|
|
62536
|
+
if (message.kind !== "delivery" && message.kind !== "delivery-ready")
|
|
62537
|
+
serial = handled.then(() => {}, () => {});
|
|
62538
|
+
response.then((value2) => child.send({ id: message.id, value: value2 }, replied), (error) => child.send({ id: message.id, error: error.message }, replied));
|
|
62539
|
+
});
|
|
62540
|
+
});
|
|
62541
|
+
}
|
|
62542
|
+
assertPublished(work) {
|
|
62543
|
+
const failure = this.publicationFailures.get(work.id);
|
|
62544
|
+
if (failure?.generation === work.generation)
|
|
62545
|
+
throw new Error(failure.error);
|
|
62546
|
+
}
|
|
62547
|
+
recoverExpired() {
|
|
62548
|
+
if (this.recovering)
|
|
62549
|
+
return this.recovering;
|
|
62550
|
+
const root = this.root;
|
|
62551
|
+
if (this.closed || root === undefined || !this.queue || performance.now() < this.nextRecoveryAt)
|
|
62552
|
+
return Promise.resolve();
|
|
62553
|
+
const queue = this.queue;
|
|
62554
|
+
const lease = this.lease;
|
|
62555
|
+
const leaseMs = this.options.leaseMs;
|
|
62556
|
+
this.recovering = (async () => {
|
|
62557
|
+
await new Promise((resolve) => setImmediate(resolve));
|
|
62558
|
+
await this.admission?.reconcile();
|
|
62559
|
+
const expiredBefore = Date.now();
|
|
62560
|
+
for (const work of await queue.inspect(root)) {
|
|
62561
|
+
if (work.state !== "leased" || work.expiresAt >= expiredBefore)
|
|
62562
|
+
continue;
|
|
62563
|
+
const child = work.worker === null ? undefined : this.incarnations.get(work.worker);
|
|
62564
|
+
if (child && work.worker !== null && this.isLive(work.worker, child)) {
|
|
62565
|
+
await queue.extend(lease, work, leaseMs).catch((error) => {
|
|
62566
|
+
if (error.message !== STALE_WORK_LEASE)
|
|
62567
|
+
throw error;
|
|
62568
|
+
});
|
|
62569
|
+
continue;
|
|
62570
|
+
}
|
|
62571
|
+
if (child && !await this.stop(child).then(() => true, () => false))
|
|
62572
|
+
continue;
|
|
62573
|
+
await queue.expire(lease, work);
|
|
62574
|
+
}
|
|
62575
|
+
})().finally(() => {
|
|
62576
|
+
this.recovering = undefined;
|
|
62577
|
+
this.nextRecoveryAt = performance.now() + RECOVERY_INTERVAL_MS;
|
|
62578
|
+
});
|
|
62579
|
+
return this.recovering;
|
|
62580
|
+
}
|
|
62581
|
+
observeResponsiveness() {
|
|
62582
|
+
const now = performance.now();
|
|
62583
|
+
const gap = now - this.lastTick;
|
|
62584
|
+
if (gap > STALL_THRESHOLD_MS)
|
|
62585
|
+
this.stalledMs += gap - TICK_INTERVAL_MS;
|
|
62586
|
+
this.lastTick = now;
|
|
62587
|
+
}
|
|
62588
|
+
isLive(worker, child) {
|
|
62589
|
+
this.observeResponsiveness();
|
|
62590
|
+
const heard = this.lastHeard.get(worker);
|
|
62591
|
+
return child.exitCode === null && child.signalCode === null && child.connected && heard !== undefined && performance.now() - heard.at - (this.stalledMs - heard.stalled) < this.options.leaseMs;
|
|
62592
|
+
}
|
|
62593
|
+
stop(child) {
|
|
62594
|
+
if (child.pid === undefined || child.exitCode !== null || child.signalCode !== null)
|
|
62595
|
+
return Promise.resolve();
|
|
62596
|
+
return new Promise((resolve, reject) => {
|
|
62597
|
+
const timeout = setTimeout(() => reject(new Error("Local worker cleanup was not confirmed")), 5000);
|
|
62598
|
+
child.once("exit", () => {
|
|
62599
|
+
clearTimeout(timeout);
|
|
62600
|
+
resolve();
|
|
62601
|
+
});
|
|
62602
|
+
child.kill("SIGKILL");
|
|
62603
|
+
});
|
|
62604
|
+
}
|
|
62605
|
+
async close() {
|
|
62606
|
+
this.closed = true;
|
|
62607
|
+
clearInterval(this.ticker);
|
|
62608
|
+
clearInterval(this.recoveryTimer);
|
|
62609
|
+
let failure;
|
|
62610
|
+
let stopping = [];
|
|
62611
|
+
try {
|
|
62612
|
+
await this.recovering?.catch(() => {});
|
|
62613
|
+
stopping = [...this.children].map((child) => ({ child, log: this.logs.get(child) }));
|
|
62614
|
+
await Promise.all(stopping.map(({ child }) => this.stop(child)));
|
|
62615
|
+
await Promise.all([...this.cleanups]);
|
|
62616
|
+
if (this.cleanupFailure)
|
|
62617
|
+
throw this.cleanupFailure;
|
|
62618
|
+
} catch (error) {
|
|
62619
|
+
failure = error;
|
|
62620
|
+
}
|
|
62621
|
+
if (failure === undefined) {
|
|
62622
|
+
const stopped = stopping.flatMap(({ child, log }) => log !== undefined && (child.exitCode !== null || child.signalCode !== null) ? [log] : []);
|
|
62623
|
+
await Promise.all(stopped.map((log) => rm3(log, { force: true }).catch(() => {})));
|
|
62624
|
+
if (this.logDirectory !== undefined)
|
|
62625
|
+
await rmdir(this.logDirectory).catch(() => {});
|
|
62626
|
+
}
|
|
62627
|
+
this.flush();
|
|
62628
|
+
try {
|
|
62629
|
+
await this.routes?.close();
|
|
62630
|
+
} catch (error) {
|
|
62631
|
+
failure ??= error;
|
|
62632
|
+
}
|
|
62633
|
+
for (const batch of this.batches.values())
|
|
62634
|
+
batch.settle();
|
|
62635
|
+
this.batches.clear();
|
|
62636
|
+
for (const pending of this.requests.values())
|
|
62637
|
+
pending.reject(new Error("Local worker pool is closed"));
|
|
62638
|
+
this.requests.clear();
|
|
62639
|
+
if (failure !== undefined)
|
|
62640
|
+
throw failure;
|
|
62641
|
+
}
|
|
62642
|
+
}
|
|
62643
|
+
function processPort() {
|
|
62644
|
+
return {
|
|
62645
|
+
send(message) {
|
|
62646
|
+
if (process.connected)
|
|
62647
|
+
process.send?.(message, () => {});
|
|
62648
|
+
},
|
|
62649
|
+
onMessage(listener) {
|
|
62650
|
+
process.on("message", (message) => listener(message));
|
|
62651
|
+
},
|
|
62652
|
+
onClose(listener) {
|
|
62653
|
+
process.once("disconnect", () => listener(new Error("Root host disconnected")));
|
|
62654
|
+
},
|
|
62655
|
+
async close() {
|
|
62656
|
+
if (process.connected)
|
|
62657
|
+
process.disconnect();
|
|
62658
|
+
}
|
|
62659
|
+
};
|
|
62660
|
+
}
|
|
62661
|
+
function runPoolCoordinator() {
|
|
62662
|
+
if (typeof process.send !== "function")
|
|
62663
|
+
throw new Error("Local worker pool coordinator requires an IPC channel");
|
|
62664
|
+
const port = processPort();
|
|
62665
|
+
const coordinator = new PoolCoordinator(port);
|
|
62666
|
+
port.onClose(() => process.exit(0));
|
|
62667
|
+
return coordinator;
|
|
62668
|
+
}
|
|
62669
|
+
var CONTROL_ACKNOWLEDGEMENT_TIMEOUT_MS = 5000, RECOVERY_INTERVAL_MS = 500, TICK_INTERVAL_MS = 100, STALL_THRESHOLD_MS = 500, COORDINATOR_LEASE_MS, STDERR_TAIL_CHARACTERS = 8192, STDERR_DRAIN_GRACE_MS = 1000, ROUTE_REGISTRATION_WAIT_MS = 120000, PENDING_STAGE_MESSAGE_EVENT2 = "atomic:workflow-pending-stage-message", STICKY_LIVE_DELIVERED_EVENT2 = "atomic:workflow-sticky-live-delivered", PENDING_STAGE_UNDELIVERABLE_EVENT2 = "atomic:workflow-pending-stage-undeliverable";
|
|
62670
|
+
var init_pool_coordinator = __esm(() => {
|
|
62671
|
+
init_resolution();
|
|
62672
|
+
init_sql_pool_queue();
|
|
62673
|
+
init_sql_pool_usage();
|
|
62674
|
+
init_pending_stage_intercom();
|
|
62675
|
+
init_intercom_group();
|
|
62676
|
+
init_pending_stage_route_capability();
|
|
62677
|
+
init_pending_stage_route_readiness();
|
|
62678
|
+
init_store();
|
|
62679
|
+
init_resource_admission();
|
|
62680
|
+
COORDINATOR_LEASE_MS = 3 * STAGE_SESSION_HEARTBEAT_INTERVAL_MS;
|
|
61507
62681
|
});
|
|
61508
62682
|
|
|
61509
62683
|
// dist/builtin/workflows/src/workers/protocol.ts
|
|
@@ -61753,21 +62927,26 @@ class PoolStageHandoff {
|
|
|
61753
62927
|
store;
|
|
61754
62928
|
queue;
|
|
61755
62929
|
lease;
|
|
62930
|
+
claimRun;
|
|
61756
62931
|
attempts = new Map;
|
|
61757
62932
|
owners = new Map;
|
|
61758
62933
|
publications = new Map;
|
|
61759
62934
|
drains = new Map;
|
|
61760
|
-
constructor(backend, store, queue, lease) {
|
|
62935
|
+
constructor(backend, store, queue, lease, claimRun) {
|
|
61761
62936
|
this.backend = backend;
|
|
61762
62937
|
this.store = store;
|
|
61763
62938
|
this.queue = queue;
|
|
61764
62939
|
this.lease = lease;
|
|
62940
|
+
this.claimRun = claimRun;
|
|
61765
62941
|
}
|
|
61766
62942
|
publish(work, runs) {
|
|
61767
62943
|
const previous = this.publications.get(work.id) ?? Promise.resolve();
|
|
61768
62944
|
const publication = previous.catch(() => {
|
|
61769
62945
|
return;
|
|
61770
|
-
}).then(() =>
|
|
62946
|
+
}).then(async () => {
|
|
62947
|
+
const commit = await this.prepare(work, runs);
|
|
62948
|
+
this.store.batch(commit);
|
|
62949
|
+
});
|
|
61771
62950
|
const settled = publication.catch(() => {
|
|
61772
62951
|
return;
|
|
61773
62952
|
});
|
|
@@ -61778,6 +62957,21 @@ class PoolStageHandoff {
|
|
|
61778
62957
|
});
|
|
61779
62958
|
return publication;
|
|
61780
62959
|
}
|
|
62960
|
+
async publishBatch(publications) {
|
|
62961
|
+
const prepared = [];
|
|
62962
|
+
for (const { work, runs } of publications)
|
|
62963
|
+
prepared.push(await this.prepare(work, runs).catch((error) => error));
|
|
62964
|
+
return this.store.batch(() => prepared.map((commit) => {
|
|
62965
|
+
if (commit instanceof Error)
|
|
62966
|
+
return commit;
|
|
62967
|
+
try {
|
|
62968
|
+
commit();
|
|
62969
|
+
return;
|
|
62970
|
+
} catch (error) {
|
|
62971
|
+
return error instanceof Error ? error : new Error(String(error));
|
|
62972
|
+
}
|
|
62973
|
+
}));
|
|
62974
|
+
}
|
|
61781
62975
|
async route(work, runId) {
|
|
61782
62976
|
this.requireOwned(await this.begin(work), runId);
|
|
61783
62977
|
const readiness = workflowPendingStageRouteReady(this.store, runId);
|
|
@@ -61828,7 +63022,7 @@ class PoolStageHandoff {
|
|
|
61828
63022
|
const result = await this.queue.result(root, work.id);
|
|
61829
63023
|
if (result === undefined)
|
|
61830
63024
|
return;
|
|
61831
|
-
await this.backend.
|
|
63025
|
+
await this.backend.catchUpWorkflow(root);
|
|
61832
63026
|
const terminals = new Map;
|
|
61833
63027
|
for (const checkpoint of this.backend.listCheckpoints(root)) {
|
|
61834
63028
|
const boundary = checkpoint.kind === "stage" ? checkpoint.topology?.boundary : undefined;
|
|
@@ -61838,6 +63032,9 @@ class PoolStageHandoff {
|
|
|
61838
63032
|
if (child?.runId === boundary.child.runId)
|
|
61839
63033
|
terminals.set(child.runId, { status: child.status, outputs: child.outputs });
|
|
61840
63034
|
}
|
|
63035
|
+
this.store.batch(() => this.settle(work, result, terminals));
|
|
63036
|
+
}
|
|
63037
|
+
settle(work, result, terminals) {
|
|
61841
63038
|
for (const run of this.store.runs()) {
|
|
61842
63039
|
if (run.endedAt !== undefined || this.owners.get(run.id) !== work.id)
|
|
61843
63040
|
continue;
|
|
@@ -61861,22 +63058,31 @@ class PoolStageHandoff {
|
|
|
61861
63058
|
if (current !== undefined && current.generation > work.generation)
|
|
61862
63059
|
return;
|
|
61863
63060
|
const attempt = current?.generation === work.generation ? current : this.start(work, current);
|
|
61864
|
-
this.release(attempt);
|
|
63061
|
+
this.store.batch(() => this.release(attempt));
|
|
61865
63062
|
attempt.fenced = true;
|
|
61866
63063
|
}
|
|
61867
|
-
async
|
|
63064
|
+
async prepare(work, runs) {
|
|
61868
63065
|
const subtree = this.subtree(work, runs);
|
|
61869
63066
|
const root = this.lease.runId;
|
|
61870
|
-
const
|
|
61871
|
-
|
|
61872
|
-
|
|
61873
|
-
|
|
61874
|
-
|
|
61875
|
-
|
|
61876
|
-
|
|
61877
|
-
|
|
61878
|
-
|
|
63067
|
+
const owned = (known) => {
|
|
63068
|
+
for (const run of subtree) {
|
|
63069
|
+
const existing = known.get(run.id);
|
|
63070
|
+
const owner = this.owners.get(run.id);
|
|
63071
|
+
if (owner !== undefined && owner !== work.id || owner === undefined && existing !== undefined && (existing.parentRunId !== run.parentRunId || existing.parentStageId !== run.parentStageId))
|
|
63072
|
+
throw outside2(run.id);
|
|
63073
|
+
}
|
|
63074
|
+
};
|
|
63075
|
+
const before = new Map(this.store.runs().map((run) => [run.id, run]));
|
|
63076
|
+
owned(before);
|
|
63077
|
+
if (subtree.some((run) => before.get(run.id) === undefined))
|
|
63078
|
+
await this.backend.catchUpWorkflow(root);
|
|
61879
63079
|
await this.queue.assertCurrent(this.lease, work);
|
|
63080
|
+
return () => this.commit(work, subtree, owned);
|
|
63081
|
+
}
|
|
63082
|
+
commit(work, subtree, owned) {
|
|
63083
|
+
const root = this.lease.runId;
|
|
63084
|
+
const known = new Map(this.store.runs().map((run) => [run.id, run]));
|
|
63085
|
+
owned(known);
|
|
61880
63086
|
const attempt = this.enter(work);
|
|
61881
63087
|
const execution = {
|
|
61882
63088
|
worker: work.worker ?? "",
|
|
@@ -61888,11 +63094,13 @@ class PoolStageHandoff {
|
|
|
61888
63094
|
...run,
|
|
61889
63095
|
rootRunId: root,
|
|
61890
63096
|
pendingStageMessages: [
|
|
61891
|
-
...
|
|
63097
|
+
...known.get(run.id)?.pendingStageMessages ?? pendingStageMessagesForDurableRun(this.backend, run.id, root)
|
|
61892
63098
|
],
|
|
61893
63099
|
stages: run.stages.map((stage) => ({ ...stage, workerExecution: execution }))
|
|
61894
63100
|
}));
|
|
61895
63101
|
const changed = published.some((run) => activity(known.get(run.id)) !== activity(run));
|
|
63102
|
+
for (const run of published)
|
|
63103
|
+
this.claimRun?.(run.id);
|
|
61896
63104
|
for (const run of published) {
|
|
61897
63105
|
attempt.runs.add(run.id);
|
|
61898
63106
|
this.owners.set(run.id, work.id);
|
|
@@ -61917,7 +63125,7 @@ class PoolStageHandoff {
|
|
|
61917
63125
|
const byId = new Map;
|
|
61918
63126
|
for (const run of runs) {
|
|
61919
63127
|
if (run.id === root || byId.has(run.id))
|
|
61920
|
-
throw
|
|
63128
|
+
throw outside2(run.id);
|
|
61921
63129
|
byId.set(run.id, run);
|
|
61922
63130
|
}
|
|
61923
63131
|
if (runs.length === 0)
|
|
@@ -61930,25 +63138,25 @@ class PoolStageHandoff {
|
|
|
61930
63138
|
const top = byId.get(work.childRunId);
|
|
61931
63139
|
if (top !== undefined) {
|
|
61932
63140
|
if (top.parentRunId !== root || top.parentStageId !== work.boundaryId)
|
|
61933
|
-
throw
|
|
63141
|
+
throw outside2(top.id);
|
|
61934
63142
|
const pending = [top];
|
|
61935
63143
|
for (let run = pending.shift();run !== undefined; run = pending.shift()) {
|
|
61936
63144
|
if (run.rootRunId !== undefined && run.rootRunId !== root)
|
|
61937
|
-
throw
|
|
63145
|
+
throw outside2(run.id);
|
|
61938
63146
|
ordered.push(run);
|
|
61939
63147
|
for (const child of runs) {
|
|
61940
63148
|
if (child.parentRunId !== run.id)
|
|
61941
63149
|
continue;
|
|
61942
63150
|
const parentStage = run.stages.find((stage) => stage.id === child.parentStageId);
|
|
61943
63151
|
if (authoritativeWorkflowChildRunId(parentStage) !== child.id)
|
|
61944
|
-
throw
|
|
63152
|
+
throw outside2(child.id);
|
|
61945
63153
|
pending.push(child);
|
|
61946
63154
|
}
|
|
61947
63155
|
}
|
|
61948
63156
|
}
|
|
61949
63157
|
const stray = runs.find((run) => !ordered.includes(run));
|
|
61950
63158
|
if (stray !== undefined)
|
|
61951
|
-
throw
|
|
63159
|
+
throw outside2(stray.id);
|
|
61952
63160
|
return ordered;
|
|
61953
63161
|
}
|
|
61954
63162
|
async begin(work) {
|
|
@@ -62003,7 +63211,7 @@ class PoolStageHandoff {
|
|
|
62003
63211
|
}
|
|
62004
63212
|
requireOwned(attempt, runId) {
|
|
62005
63213
|
if (!attempt.runs.has(runId))
|
|
62006
|
-
throw
|
|
63214
|
+
throw outside2(runId);
|
|
62007
63215
|
}
|
|
62008
63216
|
delivery(attempt, work, runId, stageId, stageName) {
|
|
62009
63217
|
const key = JSON.stringify([runId, stageId]);
|
|
@@ -62021,7 +63229,7 @@ class PoolStageHandoff {
|
|
|
62021
63229
|
return delivery;
|
|
62022
63230
|
}
|
|
62023
63231
|
}
|
|
62024
|
-
function
|
|
63232
|
+
function outside2(runId) {
|
|
62025
63233
|
return new Error(`Pool stage publication is outside the work subtree: ${runId}`);
|
|
62026
63234
|
}
|
|
62027
63235
|
function isLive(stage) {
|
|
@@ -62046,33 +63254,122 @@ var init_stage_handoff = __esm(() => {
|
|
|
62046
63254
|
});
|
|
62047
63255
|
|
|
62048
63256
|
// dist/builtin/workflows/src/workers/local-pool.ts
|
|
62049
|
-
import { fork } from "node:child_process";
|
|
62050
|
-
import {
|
|
62051
|
-
import {
|
|
62052
|
-
import { mkdir as mkdir6, rm as rm3, rmdir } from "node:fs/promises";
|
|
62053
|
-
import { dirname as dirname12, join as join28, resolve as resolve11 } from "node:path";
|
|
62054
|
-
import { setTimeout as delay2 } from "node:timers/promises";
|
|
63257
|
+
import { fork as fork2 } from "node:child_process";
|
|
63258
|
+
import { resolve as resolve12 } from "node:path";
|
|
63259
|
+
import { setTimeout as delay3 } from "node:timers/promises";
|
|
62055
63260
|
import { fileURLToPath as fileURLToPath5 } from "node:url";
|
|
62056
63261
|
import { isBunBinary as isBunBinary2 } from "@bastani/atomic";
|
|
62057
|
-
function
|
|
62058
|
-
return
|
|
63262
|
+
function defaultWorkerEntry() {
|
|
63263
|
+
return isBunBinary2 ? INTERNAL_WORKFLOW_WORKER_ARG : fileURLToPath5(new URL("./local-worker-entry.mjs", import.meta.url));
|
|
63264
|
+
}
|
|
63265
|
+
function launchCoordinatorProcess() {
|
|
63266
|
+
const child = fork2(defaultWorkerEntry(), [JSON.stringify({ role: POOL_COORDINATOR_ROLE })], {
|
|
63267
|
+
stdio: ["ignore", "ignore", "pipe", "ipc"],
|
|
63268
|
+
execArgv: []
|
|
63269
|
+
});
|
|
63270
|
+
let stderr = "";
|
|
63271
|
+
child.stderr?.on("data", (chunk) => {
|
|
63272
|
+
stderr = (stderr + chunk.toString()).slice(-COORDINATOR_STDERR_TAIL_CHARACTERS);
|
|
63273
|
+
});
|
|
63274
|
+
const listeners = [];
|
|
63275
|
+
let lostReason;
|
|
63276
|
+
const lose = (reason) => {
|
|
63277
|
+
if (lostReason !== undefined)
|
|
63278
|
+
return;
|
|
63279
|
+
lostReason = reason;
|
|
63280
|
+
for (const listener of listeners)
|
|
63281
|
+
listener(reason);
|
|
63282
|
+
};
|
|
63283
|
+
const exited = new Promise((resolve) => child.once("exit", () => resolve()));
|
|
63284
|
+
child.on("error", lose);
|
|
63285
|
+
child.once("exit", (code, signal) => lose(new Error(`Local worker pool coordinator exited (${signal ?? `code ${code}`})${stderr ? `: ${stderr.trim()}` : ""}`)));
|
|
63286
|
+
return {
|
|
63287
|
+
pid: child.pid,
|
|
63288
|
+
port: {
|
|
63289
|
+
send(message) {
|
|
63290
|
+
if (lostReason !== undefined || !child.connected)
|
|
63291
|
+
return;
|
|
63292
|
+
child.send(message, (error) => {
|
|
63293
|
+
if (error)
|
|
63294
|
+
lose(error);
|
|
63295
|
+
});
|
|
63296
|
+
},
|
|
63297
|
+
onMessage(listener) {
|
|
63298
|
+
child.on("message", (message) => listener(message));
|
|
63299
|
+
},
|
|
63300
|
+
onClose(listener) {
|
|
63301
|
+
if (lostReason !== undefined)
|
|
63302
|
+
listener(lostReason);
|
|
63303
|
+
else
|
|
63304
|
+
listeners.push(listener);
|
|
63305
|
+
},
|
|
63306
|
+
async close() {
|
|
63307
|
+
if (child.connected)
|
|
63308
|
+
child.disconnect();
|
|
63309
|
+
if (child.exitCode !== null || child.signalCode !== null)
|
|
63310
|
+
return;
|
|
63311
|
+
const timedOut = await Promise.race([exited.then(() => false), delay3(COORDINATOR_EXIT_GRACE_MS, true)]);
|
|
63312
|
+
if (timedOut) {
|
|
63313
|
+
child.kill("SIGKILL");
|
|
63314
|
+
await exited;
|
|
63315
|
+
}
|
|
63316
|
+
}
|
|
63317
|
+
}
|
|
63318
|
+
};
|
|
63319
|
+
}
|
|
63320
|
+
function launchCoordinatorInProcess() {
|
|
63321
|
+
const mainListeners = [];
|
|
63322
|
+
const coordinatorListeners = [];
|
|
63323
|
+
const copy = (message) => JSON.parse(JSON.stringify(message));
|
|
63324
|
+
const coordinator = new PoolCoordinator({
|
|
63325
|
+
send(message) {
|
|
63326
|
+
const copied = copy(message);
|
|
63327
|
+
setImmediate(() => {
|
|
63328
|
+
for (const listener of mainListeners)
|
|
63329
|
+
listener(copied);
|
|
63330
|
+
});
|
|
63331
|
+
},
|
|
63332
|
+
onMessage(listener) {
|
|
63333
|
+
coordinatorListeners.push(listener);
|
|
63334
|
+
},
|
|
63335
|
+
onClose() {},
|
|
63336
|
+
async close() {}
|
|
63337
|
+
});
|
|
63338
|
+
return {
|
|
63339
|
+
pid: process.pid,
|
|
63340
|
+
port: {
|
|
63341
|
+
send(message) {
|
|
63342
|
+
const copied = copy(message);
|
|
63343
|
+
setImmediate(() => {
|
|
63344
|
+
for (const listener of coordinatorListeners)
|
|
63345
|
+
listener(copied);
|
|
63346
|
+
});
|
|
63347
|
+
},
|
|
63348
|
+
onMessage(listener) {
|
|
63349
|
+
mainListeners.push(listener);
|
|
63350
|
+
},
|
|
63351
|
+
onClose() {},
|
|
63352
|
+
async close() {
|
|
63353
|
+
coordinator.shutdown();
|
|
63354
|
+
}
|
|
63355
|
+
}
|
|
63356
|
+
};
|
|
62059
63357
|
}
|
|
62060
63358
|
|
|
62061
63359
|
class LocalWorkerPool {
|
|
62062
63360
|
backend;
|
|
62063
63361
|
options;
|
|
62064
|
-
|
|
62065
|
-
|
|
62066
|
-
|
|
62067
|
-
|
|
62068
|
-
|
|
62069
|
-
|
|
62070
|
-
|
|
62071
|
-
|
|
62072
|
-
|
|
62073
|
-
|
|
62074
|
-
|
|
62075
|
-
rootStalledMs = 0;
|
|
63362
|
+
pids = new Set;
|
|
63363
|
+
coordinator;
|
|
63364
|
+
coordinatorLost;
|
|
63365
|
+
commands = new Map;
|
|
63366
|
+
commandSequence = 0;
|
|
63367
|
+
inbound = Promise.resolve();
|
|
63368
|
+
works = new Map;
|
|
63369
|
+
attempts = new Map;
|
|
63370
|
+
results = new Map;
|
|
63371
|
+
changed = Promise.withResolvers();
|
|
63372
|
+
waiting = 0;
|
|
62076
63373
|
ticker;
|
|
62077
63374
|
leaseMs;
|
|
62078
63375
|
desiredControl;
|
|
@@ -62080,18 +63377,17 @@ class LocalWorkerPool {
|
|
|
62080
63377
|
store;
|
|
62081
63378
|
handoff;
|
|
62082
63379
|
releaseRoutes;
|
|
63380
|
+
routeClaims = new Map;
|
|
62083
63381
|
sources = [];
|
|
62084
63382
|
queue;
|
|
62085
63383
|
lease;
|
|
62086
|
-
admission;
|
|
62087
|
-
usageLedger;
|
|
62088
63384
|
usageTotal;
|
|
62089
63385
|
rootBudget;
|
|
63386
|
+
budgetBlocked = false;
|
|
63387
|
+
rootSignature;
|
|
62090
63388
|
starting;
|
|
62091
|
-
cleanupFailure;
|
|
62092
63389
|
failure;
|
|
62093
63390
|
closed = false;
|
|
62094
|
-
replacements = 0;
|
|
62095
63391
|
releaseBinding;
|
|
62096
63392
|
constructor(backend, options) {
|
|
62097
63393
|
this.backend = backend;
|
|
@@ -62106,10 +63402,12 @@ class LocalWorkerPool {
|
|
|
62106
63402
|
this.releaseBinding = bindChildWorkerDispatcher(backend, this, options.rootId);
|
|
62107
63403
|
if (options.admission && options.rootId === undefined)
|
|
62108
63404
|
workflowResourceAdmission.set(backend, options.admission);
|
|
62109
|
-
this.ticker = setInterval(() => this.observeResponsiveness(), ROOT_TICK_INTERVAL_MS).unref();
|
|
62110
63405
|
}
|
|
62111
63406
|
get workerPids() {
|
|
62112
|
-
return [...this.
|
|
63407
|
+
return [...this.pids];
|
|
63408
|
+
}
|
|
63409
|
+
get coordinatorPid() {
|
|
63410
|
+
return this.coordinator?.pid;
|
|
62113
63411
|
}
|
|
62114
63412
|
usage() {
|
|
62115
63413
|
return this.usageTotal;
|
|
@@ -62117,47 +63415,39 @@ class LocalWorkerPool {
|
|
|
62117
63415
|
async prepare(root, store) {
|
|
62118
63416
|
if (this.closed)
|
|
62119
63417
|
throw new Error("Local worker pool is closed");
|
|
62120
|
-
if (!(this.backend.journal instanceof SqlRunJournal) ||
|
|
63418
|
+
if (!(this.backend.journal instanceof SqlRunJournal) || resolve12(this.options.journalPath) !== resolve12(this.backend.journal.path) || sqlPoolStorage(this.backend.journal)?.tables.runs !== "wf_runs")
|
|
62121
63419
|
throw new Error("Local workers require the same default-prefix SQLite journal");
|
|
62122
63420
|
if (this.root !== undefined && this.root !== root)
|
|
62123
63421
|
throw new Error("Local worker pool belongs to another root");
|
|
62124
63422
|
this.root = root;
|
|
62125
63423
|
if (store)
|
|
62126
63424
|
this.store = store;
|
|
62127
|
-
this.usageLedger ??= await openSqlPoolUsage(this.backend.journal);
|
|
62128
|
-
this.usageTotal = await this.usageLedger.total(root);
|
|
62129
63425
|
const state = await this.backend.journal.leaseState(root);
|
|
62130
63426
|
if (!state || state.holderId !== this.backend.holderId)
|
|
62131
63427
|
throw new Error("Local worker pool requires the root owner");
|
|
62132
63428
|
this.lease = { runId: root, holderId: state.holderId, epoch: state.epoch };
|
|
62133
|
-
const
|
|
62134
|
-
await
|
|
62135
|
-
|
|
62136
|
-
|
|
62137
|
-
|
|
62138
|
-
|
|
62139
|
-
|
|
62140
|
-
|
|
62141
|
-
|
|
62142
|
-
|
|
63429
|
+
const workerEntry = this.options.workerEntry ?? (isBunBinary2 || this.options.coordinator === "in-process" ? defaultWorkerEntry() : undefined);
|
|
63430
|
+
this.usageTotal = await this.command({
|
|
63431
|
+
kind: "prepare",
|
|
63432
|
+
root,
|
|
63433
|
+
lease: this.lease,
|
|
63434
|
+
options: {
|
|
63435
|
+
journalPath: this.options.journalPath,
|
|
63436
|
+
workers: this.options.workers,
|
|
63437
|
+
leaseMs: this.leaseMs,
|
|
63438
|
+
startupTimeoutMs: this.options.startupTimeoutMs ?? DEFAULT_STARTUP_TIMEOUT_MS,
|
|
63439
|
+
...this.options.admission === undefined ? {} : { admission: this.options.admission },
|
|
63440
|
+
...workerEntry === undefined ? {} : { workerEntry }
|
|
63441
|
+
},
|
|
63442
|
+
...this.desiredControl === undefined ? {} : { control: this.desiredControl }
|
|
63443
|
+
});
|
|
63444
|
+
this.queue ??= await openSqlPoolQueue(this.backend.journal);
|
|
62143
63445
|
}
|
|
62144
63446
|
async control(state) {
|
|
62145
63447
|
this.desiredControl = state;
|
|
62146
63448
|
try {
|
|
62147
|
-
if (
|
|
62148
|
-
|
|
62149
|
-
const control = await this.queue.setControl(this.lease, state);
|
|
62150
|
-
const deadline = Date.now() + CONTROL_ACKNOWLEDGEMENT_TIMEOUT_MS;
|
|
62151
|
-
while ([...this.readyWorkers].some((worker) => (this.controlAcks.get(worker) ?? -1) < control.revision)) {
|
|
62152
|
-
if (this.desiredControl !== state)
|
|
62153
|
-
return;
|
|
62154
|
-
if (Date.now() >= deadline) {
|
|
62155
|
-
if (state === "quit")
|
|
62156
|
-
break;
|
|
62157
|
-
throw new Error("Worker control acknowledgement unresolved");
|
|
62158
|
-
}
|
|
62159
|
-
await delay2(25);
|
|
62160
|
-
}
|
|
63449
|
+
if (this.coordinator !== undefined)
|
|
63450
|
+
await this.command({ kind: "control", state });
|
|
62161
63451
|
} finally {
|
|
62162
63452
|
if (state === "quit")
|
|
62163
63453
|
await this.close();
|
|
@@ -62203,290 +63493,61 @@ class LocalWorkerPool {
|
|
|
62203
63493
|
throw new Error(`Worker definition is not importable: ${binding.exportName}`);
|
|
62204
63494
|
return source;
|
|
62205
63495
|
}));
|
|
62206
|
-
|
|
62207
|
-
|
|
63496
|
+
const cwd = String(this.backend.getWorkflow(root)?.invocationCwd ?? process.cwd());
|
|
63497
|
+
const store = this.store;
|
|
63498
|
+
if (store) {
|
|
63499
|
+
this.releaseRoutes = await acquireStandaloneIntercom(store, root, cwd);
|
|
62208
63500
|
if (this.closed) {
|
|
62209
63501
|
await this.releaseRoutes();
|
|
62210
63502
|
this.releaseRoutes = undefined;
|
|
62211
63503
|
throw new Error("Local worker pool is closed");
|
|
62212
63504
|
}
|
|
62213
|
-
|
|
62214
|
-
|
|
62215
|
-
|
|
62216
|
-
this.
|
|
62217
|
-
|
|
62218
|
-
root,
|
|
62219
|
-
holder: "coordinator"
|
|
63505
|
+
const routeOwner = {
|
|
63506
|
+
notifyUndeliverable: async (payload) => !this.closed && await this.command({ kind: "undeliverable", payload }) === true
|
|
63507
|
+
};
|
|
63508
|
+
this.handoff = new PoolStageHandoff(this.backend, store, this.queueView(), this.lease, (runId) => {
|
|
63509
|
+
this.routeClaims.set(runId, claimExternalRunRoute(store, runId, routeOwner));
|
|
62220
63510
|
});
|
|
62221
|
-
this.logDirectory = workerLogDirectory(this.options.journalPath, root);
|
|
62222
|
-
await mkdir6(this.logDirectory, { recursive: true }).catch(() => {
|
|
62223
|
-
return;
|
|
62224
|
-
});
|
|
62225
|
-
const workers = Array.from({ length: this.options.workers }, () => this.spawn(root));
|
|
62226
|
-
try {
|
|
62227
|
-
await Promise.any(workers);
|
|
62228
|
-
} catch (error) {
|
|
62229
|
-
throw error.errors[0];
|
|
62230
63511
|
}
|
|
62231
|
-
|
|
62232
|
-
|
|
62233
|
-
|
|
62234
|
-
|
|
62235
|
-
|
|
62236
|
-
|
|
62237
|
-
journalPath: this.options.journalPath,
|
|
62238
|
-
root,
|
|
62239
|
-
definitions: this.sources,
|
|
62240
|
-
admission: this.options.admission
|
|
62241
|
-
})
|
|
62242
|
-
], { stdio: ["ignore", "ignore", "pipe", "ipc"], execArgv: [] });
|
|
62243
|
-
const disconnected = new Promise((resolve) => child.once("disconnect", () => resolve()));
|
|
62244
|
-
this.children.add(child);
|
|
62245
|
-
const worker = randomUUID16();
|
|
62246
|
-
this.incarnations.set(worker, child);
|
|
62247
|
-
let ready = false;
|
|
62248
|
-
let serial = Promise.resolve();
|
|
62249
|
-
let work;
|
|
62250
|
-
const deliveries = new Map;
|
|
62251
|
-
const logPath = join28(this.logDirectory, `${worker}.log`);
|
|
62252
|
-
this.logs.set(child, logPath);
|
|
62253
|
-
let logFailure;
|
|
62254
|
-
const log = createWriteStream(logPath, { flags: "a" });
|
|
62255
|
-
log.on("error", (error) => {
|
|
62256
|
-
logFailure = error.message;
|
|
62257
|
-
});
|
|
62258
|
-
let stderr = "";
|
|
62259
|
-
child.stderr?.on("data", (chunk) => {
|
|
62260
|
-
stderr = (stderr + chunk.toString()).slice(-STDERR_TAIL_CHARACTERS);
|
|
62261
|
-
});
|
|
62262
|
-
child.stderr?.on("error", (error) => {
|
|
62263
|
-
logFailure ??= error.message;
|
|
62264
|
-
log.end();
|
|
62265
|
-
});
|
|
62266
|
-
child.stderr?.pipe(log);
|
|
62267
|
-
const diagnosed = (summary) => new Error(`${summary} (worker stderr log: ${logFailure === undefined ? logPath : `unavailable, ${logFailure}`})${stderr ? `: ${stderr}` : ""}`);
|
|
62268
|
-
const startupTimeoutMs = this.options.startupTimeoutMs ?? DEFAULT_STARTUP_TIMEOUT_MS;
|
|
62269
|
-
let timedOut = false;
|
|
62270
|
-
let terminated = false;
|
|
62271
|
-
const terminate = () => {
|
|
62272
|
-
if (terminated)
|
|
62273
|
-
return;
|
|
62274
|
-
terminated = true;
|
|
62275
|
-
child.kill("SIGKILL");
|
|
62276
|
-
};
|
|
62277
|
-
const replied = (error) => {
|
|
62278
|
-
if (error)
|
|
62279
|
-
terminate();
|
|
62280
|
-
};
|
|
62281
|
-
return new Promise((resolve, reject) => {
|
|
62282
|
-
const startup = setTimeout(() => {
|
|
62283
|
-
timedOut = true;
|
|
62284
|
-
terminate();
|
|
62285
|
-
}, startupTimeoutMs);
|
|
62286
|
-
child.on("error", (error) => {
|
|
62287
|
-
clearTimeout(startup);
|
|
62288
|
-
reject(error);
|
|
62289
|
-
terminate();
|
|
62290
|
-
});
|
|
62291
|
-
const closed = new Promise((resolve) => child.once("close", () => resolve()));
|
|
62292
|
-
child.once("exit", () => {
|
|
62293
|
-
Promise.race([closed, delay2(STDERR_DRAIN_GRACE_MS)]).then(() => reject(diagnosed(timedOut ? `Local worker startup timed out after ${startupTimeoutMs} ms` : "Local worker exited before readiness")));
|
|
62294
|
-
this.children.delete(child);
|
|
62295
|
-
this.logs.delete(child);
|
|
62296
|
-
clearTimeout(startup);
|
|
62297
|
-
this.incarnations.delete(worker);
|
|
62298
|
-
this.readyWorkers.delete(worker);
|
|
62299
|
-
this.controlAcks.delete(worker);
|
|
62300
|
-
this.lastHeard.delete(worker);
|
|
62301
|
-
if (work)
|
|
62302
|
-
this.handoff?.fence(work);
|
|
62303
|
-
for (const delivery of deliveries.values())
|
|
62304
|
-
delivery.reject(new Error("Worker exited before delivery acknowledgement"));
|
|
62305
|
-
deliveries.clear();
|
|
62306
|
-
const cleanup = disconnected.then(() => serial).then(async () => {
|
|
62307
|
-
try {
|
|
62308
|
-
if (await this.admission?.releaseStoppedSessions(worker))
|
|
62309
|
-
throw new Error("Local worker process cleanup unconfirmed; permits quarantined");
|
|
62310
|
-
} catch (error) {
|
|
62311
|
-
this.cleanupFailure = error instanceof Error ? error : new Error(String(error));
|
|
62312
|
-
throw this.cleanupFailure;
|
|
62313
|
-
}
|
|
62314
|
-
}).then(async () => {
|
|
62315
|
-
if (this.closed || !ready || this.desiredControl === "quit")
|
|
62316
|
-
return;
|
|
62317
|
-
if (work && await this.queue.result(root, work.id)) {
|
|
62318
|
-
work = undefined;
|
|
62319
|
-
this.replacements = 0;
|
|
62320
|
-
}
|
|
62321
|
-
if (this.closed)
|
|
62322
|
-
return;
|
|
62323
|
-
if (work)
|
|
62324
|
-
await this.queue.releaseExited(this.lease, work);
|
|
62325
|
-
const exhausted = work ? work.generation >= 3 : ++this.replacements > 3;
|
|
62326
|
-
if (exhausted)
|
|
62327
|
-
this.failure = diagnosed("Local worker retry limit");
|
|
62328
|
-
else
|
|
62329
|
-
await this.spawn(root);
|
|
62330
|
-
}).catch((error) => {
|
|
62331
|
-
this.failure = error;
|
|
62332
|
-
});
|
|
62333
|
-
this.cleanups.add(cleanup);
|
|
62334
|
-
cleanup.finally(() => this.cleanups.delete(cleanup));
|
|
62335
|
-
});
|
|
62336
|
-
child.on("message", (message) => {
|
|
62337
|
-
this.observeResponsiveness();
|
|
62338
|
-
this.lastHeard.set(worker, { at: performance.now(), rootStalled: this.rootStalledMs });
|
|
62339
|
-
if (message.kind === "delivery-ack") {
|
|
62340
|
-
const pending = deliveries.get(message.deliveryId);
|
|
62341
|
-
deliveries.delete(message.deliveryId);
|
|
62342
|
-
if (message.error)
|
|
62343
|
-
pending?.reject(new Error(message.error));
|
|
62344
|
-
else
|
|
62345
|
-
pending?.resolve();
|
|
62346
|
-
return;
|
|
62347
|
-
}
|
|
62348
|
-
const response = serial.then(async () => {
|
|
62349
|
-
if (message.kind === "ready") {
|
|
62350
|
-
if (terminated)
|
|
62351
|
-
return;
|
|
62352
|
-
ready = true;
|
|
62353
|
-
this.readyWorkers.add(worker);
|
|
62354
|
-
clearTimeout(startup);
|
|
62355
|
-
resolve();
|
|
62356
|
-
return;
|
|
62357
|
-
}
|
|
62358
|
-
await this.starting;
|
|
62359
|
-
const queue = this.queue;
|
|
62360
|
-
const lease = this.lease;
|
|
62361
|
-
if (message.kind === "control")
|
|
62362
|
-
return queue.control(lease);
|
|
62363
|
-
if (message.kind === "controlled") {
|
|
62364
|
-
const control = await queue.control(lease);
|
|
62365
|
-
if (message.revision > control.revision)
|
|
62366
|
-
throw new Error("Unknown control revision");
|
|
62367
|
-
this.controlAcks.set(worker, Math.max(this.controlAcks.get(worker) ?? -1, message.revision));
|
|
62368
|
-
return;
|
|
62369
|
-
}
|
|
62370
|
-
if (message.kind === "claim") {
|
|
62371
|
-
if (work)
|
|
62372
|
-
throw new Error("Worker already holds an attempt");
|
|
62373
|
-
const decision = this.rootBudget?.checkpoint();
|
|
62374
|
-
if (decision?.kind === "exhausted" || decision?.kind === "wrap_up")
|
|
62375
|
-
return;
|
|
62376
|
-
await this.recoverExpired(root);
|
|
62377
|
-
work = await queue.claim(lease, worker, this.sources.map((source) => ({
|
|
62378
|
-
definition: `${source.path}#${source.exportName}`,
|
|
62379
|
-
digest: source.digest
|
|
62380
|
-
})), this.leaseMs);
|
|
62381
|
-
return work;
|
|
62382
|
-
}
|
|
62383
|
-
if (!work)
|
|
62384
|
-
throw new Error("Worker has no active attempt");
|
|
62385
|
-
if (message.kind === "renew" || work.expiresAt - Date.now() < this.leaseMs / 2)
|
|
62386
|
-
work = await queue.extend(lease, work, this.leaseMs);
|
|
62387
|
-
if (message.kind === "activity")
|
|
62388
|
-
return this.handoff?.publish(work, message.runs);
|
|
62389
|
-
if (message.kind === "route")
|
|
62390
|
-
return this.handoff?.route(work, message.runId);
|
|
62391
|
-
if (message.kind === "delivery-ready")
|
|
62392
|
-
return this.handoff?.ready(work, message.runId, message.stageId, message.stageName, message.sessionId);
|
|
62393
|
-
if (message.kind === "delivery") {
|
|
62394
|
-
if (!message.recipient)
|
|
62395
|
-
throw new Error("Worker delivery requires a session identity");
|
|
62396
|
-
return this.handoff?.deliver(work, message.runId, message.stageId, message.stageName, message.recipient, (from, payload) => new Promise((resolve, reject) => {
|
|
62397
|
-
const id = randomUUID16();
|
|
62398
|
-
deliveries.set(id, { resolve, reject });
|
|
62399
|
-
child.send({ id: message.id, delivery: { id, from, message: payload } }, (error) => {
|
|
62400
|
-
if (error) {
|
|
62401
|
-
deliveries.delete(id);
|
|
62402
|
-
reject(error);
|
|
62403
|
-
}
|
|
62404
|
-
});
|
|
62405
|
-
}));
|
|
62406
|
-
}
|
|
62407
|
-
if (message.kind === "renew")
|
|
62408
|
-
return;
|
|
62409
|
-
if (message.kind === "commit") {
|
|
62410
|
-
const receipt = await queue.commit(lease, work, message.operation, message.mutation);
|
|
62411
|
-
this.usageTotal = await this.usageLedger.total(root);
|
|
62412
|
-
return receipt;
|
|
62413
|
-
}
|
|
62414
|
-
if (message.kind === "usage") {
|
|
62415
|
-
this.usageTotal = await this.usageLedger.report(lease, work, message.reports);
|
|
62416
|
-
return;
|
|
62417
|
-
}
|
|
62418
|
-
if (message.kind === "budget") {
|
|
62419
|
-
await queue.assertCurrent(lease, work);
|
|
62420
|
-
const decision = this.rootBudget?.checkpoint();
|
|
62421
|
-
if (decision?.kind === "exhausted" || decision?.kind === "wrap_up")
|
|
62422
|
-
throw new Error("Root workflow budget exhausted");
|
|
62423
|
-
return queue.control(lease);
|
|
62424
|
-
}
|
|
62425
|
-
if (message.kind !== "complete")
|
|
62426
|
-
throw new Error("Invalid worker operation");
|
|
62427
|
-
await queue.complete(lease, work, message.result);
|
|
62428
|
-
this.replacements = 0;
|
|
62429
|
-
work = undefined;
|
|
62430
|
-
});
|
|
62431
|
-
if (message.kind !== "delivery" && message.kind !== "delivery-ready")
|
|
62432
|
-
serial = response.then(() => {}, () => {});
|
|
62433
|
-
response.then((value2) => child.send({ id: message.id, value: value2 }, replied), (error) => child.send({ id: message.id, error: error.message }, replied));
|
|
62434
|
-
});
|
|
63512
|
+
await this.command({
|
|
63513
|
+
kind: "start",
|
|
63514
|
+
sources: this.sources,
|
|
63515
|
+
cwd,
|
|
63516
|
+
publish: store !== undefined,
|
|
63517
|
+
...this.rootRun()
|
|
62435
63518
|
});
|
|
62436
63519
|
}
|
|
62437
|
-
|
|
62438
|
-
|
|
62439
|
-
|
|
62440
|
-
|
|
62441
|
-
|
|
62442
|
-
|
|
62443
|
-
|
|
62444
|
-
|
|
62445
|
-
|
|
62446
|
-
|
|
62447
|
-
if (work.state !== "leased" || work.expiresAt >= expiredBefore)
|
|
62448
|
-
continue;
|
|
62449
|
-
const child = work.worker === null ? undefined : this.incarnations.get(work.worker);
|
|
62450
|
-
if (child && work.worker !== null && this.isLive(work.worker, child)) {
|
|
62451
|
-
await this.queue.extend(this.lease, work, this.leaseMs).catch((error) => {
|
|
62452
|
-
if (error.message !== STALE_WORK_LEASE)
|
|
62453
|
-
throw error;
|
|
62454
|
-
});
|
|
62455
|
-
continue;
|
|
62456
|
-
}
|
|
62457
|
-
if (child && !await this.stop(child).then(() => true, () => false))
|
|
62458
|
-
continue;
|
|
62459
|
-
await this.queue.expire(this.lease, work);
|
|
62460
|
-
}
|
|
62461
|
-
})().finally(() => {
|
|
62462
|
-
this.recovering = undefined;
|
|
62463
|
-
this.nextRecoveryAt = performance.now() + RECOVERY_INTERVAL_MS;
|
|
62464
|
-
});
|
|
62465
|
-
return this.recovering;
|
|
63520
|
+
queueView() {
|
|
63521
|
+
return {
|
|
63522
|
+
assertCurrent: async (_lease, work) => {
|
|
63523
|
+
const current = this.attempts.get(work.id);
|
|
63524
|
+
if (current === undefined || current.fenced || current.generation !== work.generation || current.worker !== work.worker || current.epoch !== work.epoch)
|
|
63525
|
+
throw new Error(STALE_WORK_LEASE);
|
|
63526
|
+
},
|
|
63527
|
+
withCurrentAppend: (lease, work, assertSession, action) => this.queue.withCurrentAppend(lease, work, assertSession, action),
|
|
63528
|
+
result: async (root, id) => this.results.get(id) ?? this.queue.result(root, id)
|
|
63529
|
+
};
|
|
62466
63530
|
}
|
|
62467
|
-
|
|
62468
|
-
const
|
|
62469
|
-
|
|
62470
|
-
if (gap > ROOT_STALL_THRESHOLD_MS)
|
|
62471
|
-
this.rootStalledMs += gap - ROOT_TICK_INTERVAL_MS;
|
|
62472
|
-
this.lastTick = now;
|
|
63531
|
+
rootProjection() {
|
|
63532
|
+
const run = this.store?.runs().find((candidate) => candidate.id === this.root);
|
|
63533
|
+
return run === undefined ? undefined : projectRootRun(run);
|
|
62473
63534
|
}
|
|
62474
|
-
|
|
62475
|
-
this.
|
|
62476
|
-
|
|
62477
|
-
|
|
63535
|
+
rootRun() {
|
|
63536
|
+
const rootRun = this.rootProjection();
|
|
63537
|
+
if (rootRun === undefined)
|
|
63538
|
+
return {};
|
|
63539
|
+
this.rootSignature = JSON.stringify(rootRun);
|
|
63540
|
+
return { rootRun };
|
|
62478
63541
|
}
|
|
62479
|
-
|
|
62480
|
-
|
|
62481
|
-
|
|
62482
|
-
|
|
62483
|
-
|
|
62484
|
-
|
|
62485
|
-
|
|
62486
|
-
|
|
62487
|
-
|
|
62488
|
-
child.kill("SIGKILL");
|
|
62489
|
-
});
|
|
63542
|
+
refreshRootRun() {
|
|
63543
|
+
const rootRun = this.rootProjection();
|
|
63544
|
+
if (rootRun === undefined)
|
|
63545
|
+
return;
|
|
63546
|
+
const signature = JSON.stringify(rootRun);
|
|
63547
|
+
if (signature === this.rootSignature)
|
|
63548
|
+
return;
|
|
63549
|
+
this.rootSignature = signature;
|
|
63550
|
+
this.notice({ kind: "root", rootRun });
|
|
62490
63551
|
}
|
|
62491
63552
|
async run(definition, inputs, options) {
|
|
62492
63553
|
try {
|
|
@@ -62519,7 +63580,7 @@ class LocalWorkerPool {
|
|
|
62519
63580
|
const selectedSource = this.sources[index];
|
|
62520
63581
|
if (await definitionDigest(selectedSource.path) !== selectedSource.digest)
|
|
62521
63582
|
throw new Error("Worker definition compatibility error");
|
|
62522
|
-
await this.backend.
|
|
63583
|
+
await this.backend.catchUpWorkflow(root);
|
|
62523
63584
|
const boundary = this.backend.listCheckpoints(root).find((entry) => entry.kind === "stage" && entry.topology?.boundary?.event === "start" && entry.topology.stageId === options.parentRun.stageId);
|
|
62524
63585
|
if (boundary?.kind !== "stage")
|
|
62525
63586
|
throw new Error("Missing durable child boundary");
|
|
@@ -62530,39 +63591,232 @@ class LocalWorkerPool {
|
|
|
62530
63591
|
config: options.config,
|
|
62531
63592
|
executionMode: options.executionMode
|
|
62532
63593
|
}));
|
|
62533
|
-
|
|
62534
|
-
|
|
62535
|
-
|
|
62536
|
-
|
|
62537
|
-
|
|
62538
|
-
|
|
62539
|
-
|
|
62540
|
-
|
|
62541
|
-
|
|
62542
|
-
|
|
62543
|
-
|
|
62544
|
-
|
|
63594
|
+
this.evaluateBudget();
|
|
63595
|
+
const enqueued = await this.command({
|
|
63596
|
+
kind: "enqueue",
|
|
63597
|
+
descriptor: {
|
|
63598
|
+
boundaryId: options.parentRun.stageId,
|
|
63599
|
+
childRunId: options.runId,
|
|
63600
|
+
scope: options.durableScope.scopePrefix,
|
|
63601
|
+
definition: `${selectedSource.path}#${selectedSource.exportName}`,
|
|
63602
|
+
digest: selectedSource.digest,
|
|
63603
|
+
inputs,
|
|
63604
|
+
execution
|
|
63605
|
+
},
|
|
63606
|
+
boundary,
|
|
63607
|
+
...this.rootRun()
|
|
63608
|
+
});
|
|
63609
|
+
const work = enqueued.work;
|
|
63610
|
+
this.works.set(work.id, work);
|
|
63611
|
+
if (enqueued.result !== undefined && !this.results.has(work.id))
|
|
63612
|
+
this.results.set(work.id, enqueued.result);
|
|
63613
|
+
this.waiting++;
|
|
63614
|
+
this.ticker ??= setInterval(() => {
|
|
63615
|
+
this.evaluateBudget();
|
|
63616
|
+
this.signalChange();
|
|
63617
|
+
}, POOL_SUMMARY_INTERVAL_MS).unref();
|
|
63618
|
+
try {
|
|
63619
|
+
for (;; ) {
|
|
63620
|
+
options.signal?.throwIfAborted();
|
|
63621
|
+
if (this.closed) {
|
|
63622
|
+
if (this.failure)
|
|
63623
|
+
throw this.failure;
|
|
63624
|
+
if (this.desiredControl === "quit")
|
|
63625
|
+
throw new WorkflowGracefulQuitError(root, "worker pool teardown");
|
|
63626
|
+
throw new Error("Local worker pool is closed");
|
|
63627
|
+
}
|
|
63628
|
+
const result = this.results.get(work.id);
|
|
63629
|
+
if (result) {
|
|
63630
|
+
await this.inbound;
|
|
63631
|
+
await this.backend.catchUpWorkflow(root);
|
|
63632
|
+
await this.handoff?.complete(work);
|
|
63633
|
+
return result;
|
|
63634
|
+
}
|
|
62545
63635
|
if (this.failure)
|
|
62546
63636
|
throw this.failure;
|
|
62547
|
-
if (this.
|
|
62548
|
-
|
|
62549
|
-
|
|
63637
|
+
if (this.budgetBlocked)
|
|
63638
|
+
await this.rootBudget.stopAtBoundaryAsync(definition.name);
|
|
63639
|
+
await this.nextChange(options.signal);
|
|
62550
63640
|
}
|
|
62551
|
-
|
|
62552
|
-
if (
|
|
62553
|
-
|
|
62554
|
-
|
|
62555
|
-
return result;
|
|
63641
|
+
} finally {
|
|
63642
|
+
if (--this.waiting === 0) {
|
|
63643
|
+
clearInterval(this.ticker);
|
|
63644
|
+
this.ticker = undefined;
|
|
62556
63645
|
}
|
|
62557
|
-
if (this.failure)
|
|
62558
|
-
throw this.failure;
|
|
62559
|
-
const decision = this.rootBudget?.checkpoint();
|
|
62560
|
-
if (decision?.kind === "exhausted" || decision?.kind === "wrap_up")
|
|
62561
|
-
await this.rootBudget.stopAtBoundaryAsync(definition.name);
|
|
62562
|
-
await this.recoverExpired(root);
|
|
62563
|
-
await delay2(25, undefined, { signal: options.signal });
|
|
62564
63646
|
}
|
|
62565
63647
|
}
|
|
63648
|
+
evaluateBudget() {
|
|
63649
|
+
const decision = this.rootBudget?.checkpoint();
|
|
63650
|
+
const blocked = decision?.kind === "exhausted" || decision?.kind === "wrap_up";
|
|
63651
|
+
if (blocked === this.budgetBlocked)
|
|
63652
|
+
return;
|
|
63653
|
+
this.budgetBlocked = blocked;
|
|
63654
|
+
this.notice({ kind: "budget", blocked });
|
|
63655
|
+
}
|
|
63656
|
+
signalChange() {
|
|
63657
|
+
const changed = this.changed;
|
|
63658
|
+
this.changed = Promise.withResolvers();
|
|
63659
|
+
changed.resolve();
|
|
63660
|
+
}
|
|
63661
|
+
nextChange(signal) {
|
|
63662
|
+
const changed = this.changed.promise;
|
|
63663
|
+
if (signal === undefined)
|
|
63664
|
+
return changed;
|
|
63665
|
+
return new Promise((resolve, reject) => {
|
|
63666
|
+
const abort = () => reject(signal.reason);
|
|
63667
|
+
signal.addEventListener("abort", abort, { once: true });
|
|
63668
|
+
changed.then(() => {
|
|
63669
|
+
signal.removeEventListener("abort", abort);
|
|
63670
|
+
resolve();
|
|
63671
|
+
});
|
|
63672
|
+
});
|
|
63673
|
+
}
|
|
63674
|
+
connect() {
|
|
63675
|
+
if (this.coordinator !== undefined)
|
|
63676
|
+
return this.coordinator;
|
|
63677
|
+
const coordinator = this.options.coordinator === "in-process" ? launchCoordinatorInProcess() : launchCoordinatorProcess();
|
|
63678
|
+
this.coordinator = coordinator;
|
|
63679
|
+
coordinator.port.onMessage((event) => {
|
|
63680
|
+
this.inbound = this.inbound.then(() => this.receive(event)).catch(() => {});
|
|
63681
|
+
});
|
|
63682
|
+
coordinator.port.onClose((reason) => {
|
|
63683
|
+
this.coordinatorLost = reason;
|
|
63684
|
+
this.inbound = this.inbound.then(() => this.lost(reason));
|
|
63685
|
+
});
|
|
63686
|
+
return coordinator;
|
|
63687
|
+
}
|
|
63688
|
+
lost(reason) {
|
|
63689
|
+
this.pids.clear();
|
|
63690
|
+
if (!this.closed)
|
|
63691
|
+
this.failure ??= reason;
|
|
63692
|
+
for (const pending of this.commands.values())
|
|
63693
|
+
pending.reject(reason);
|
|
63694
|
+
this.commands.clear();
|
|
63695
|
+
this.signalChange();
|
|
63696
|
+
}
|
|
63697
|
+
command(body) {
|
|
63698
|
+
const coordinator = this.connect();
|
|
63699
|
+
if (this.coordinatorLost !== undefined)
|
|
63700
|
+
return Promise.reject(this.coordinatorLost);
|
|
63701
|
+
const id = ++this.commandSequence;
|
|
63702
|
+
return new Promise((resolve, reject) => {
|
|
63703
|
+
this.commands.set(id, { resolve, reject });
|
|
63704
|
+
coordinator.port.send({ ...body, id });
|
|
63705
|
+
});
|
|
63706
|
+
}
|
|
63707
|
+
notice(notice) {
|
|
63708
|
+
this.coordinator?.port.send(notice);
|
|
63709
|
+
}
|
|
63710
|
+
async receive(event) {
|
|
63711
|
+
switch (event.kind) {
|
|
63712
|
+
case "reply": {
|
|
63713
|
+
const pending = this.commands.get(event.id);
|
|
63714
|
+
this.commands.delete(event.id);
|
|
63715
|
+
if (event.error !== undefined)
|
|
63716
|
+
pending?.reject(new Error(event.error));
|
|
63717
|
+
else
|
|
63718
|
+
pending?.resolve(event.value);
|
|
63719
|
+
return;
|
|
63720
|
+
}
|
|
63721
|
+
case "activity":
|
|
63722
|
+
return this.applyActivity(event.batch, event.entries, event.usage);
|
|
63723
|
+
case "spawned":
|
|
63724
|
+
this.pids.add(event.pid);
|
|
63725
|
+
return;
|
|
63726
|
+
case "exited":
|
|
63727
|
+
this.pids.delete(event.pid);
|
|
63728
|
+
return;
|
|
63729
|
+
case "fence": {
|
|
63730
|
+
const current = this.attempts.get(event.work.id);
|
|
63731
|
+
if (current?.generation === event.work.generation)
|
|
63732
|
+
current.fenced = true;
|
|
63733
|
+
this.handoff?.fence(event.work);
|
|
63734
|
+
return;
|
|
63735
|
+
}
|
|
63736
|
+
case "result":
|
|
63737
|
+
if (event.usage !== undefined)
|
|
63738
|
+
this.usageTotal = event.usage;
|
|
63739
|
+
this.results.set(event.workId, event.result);
|
|
63740
|
+
this.signalChange();
|
|
63741
|
+
return;
|
|
63742
|
+
case "failure":
|
|
63743
|
+
this.failure ??= new Error(event.error);
|
|
63744
|
+
this.signalChange();
|
|
63745
|
+
return;
|
|
63746
|
+
case "request":
|
|
63747
|
+
this.serve(event.id, event.request);
|
|
63748
|
+
return;
|
|
63749
|
+
}
|
|
63750
|
+
}
|
|
63751
|
+
async applyActivity(batch, entries, usage) {
|
|
63752
|
+
if (usage !== undefined)
|
|
63753
|
+
this.usageTotal = usage;
|
|
63754
|
+
const rejected = [];
|
|
63755
|
+
const handoff = this.handoff;
|
|
63756
|
+
if (handoff !== undefined) {
|
|
63757
|
+
const publications = entries.flatMap((entry) => {
|
|
63758
|
+
const work = this.track(entry.work);
|
|
63759
|
+
if (work !== undefined)
|
|
63760
|
+
return [{ work, runs: entry.runs }];
|
|
63761
|
+
rejected.push({ workId: entry.work.id, generation: entry.work.generation, error: STALE_WORK_LEASE });
|
|
63762
|
+
return [];
|
|
63763
|
+
});
|
|
63764
|
+
const outcomes = await handoff.publishBatch(publications);
|
|
63765
|
+
outcomes.forEach((error, index) => {
|
|
63766
|
+
const { work } = publications[index];
|
|
63767
|
+
if (error !== undefined)
|
|
63768
|
+
rejected.push({ workId: work.id, generation: work.generation, error: error.message });
|
|
63769
|
+
});
|
|
63770
|
+
}
|
|
63771
|
+
this.notice({ kind: "published", batch, rejected });
|
|
63772
|
+
this.refreshRootRun();
|
|
63773
|
+
}
|
|
63774
|
+
track(attempt) {
|
|
63775
|
+
const work = this.works.get(attempt.id);
|
|
63776
|
+
if (work === undefined)
|
|
63777
|
+
return;
|
|
63778
|
+
const current = this.attempts.get(attempt.id);
|
|
63779
|
+
if (current === undefined || attempt.generation > current.generation)
|
|
63780
|
+
this.attempts.set(attempt.id, { ...attempt, fenced: false });
|
|
63781
|
+
return { ...work, ...attempt, state: "leased" };
|
|
63782
|
+
}
|
|
63783
|
+
async serve(id, request) {
|
|
63784
|
+
try {
|
|
63785
|
+
const value2 = await this.answer(id, request);
|
|
63786
|
+
this.coordinator?.port.send({ kind: "request-reply", id, ...value2 === undefined ? {} : { value: value2 } });
|
|
63787
|
+
} catch (error) {
|
|
63788
|
+
this.coordinator?.port.send({
|
|
63789
|
+
kind: "request-reply",
|
|
63790
|
+
id,
|
|
63791
|
+
error: error instanceof Error ? error.message : String(error)
|
|
63792
|
+
});
|
|
63793
|
+
}
|
|
63794
|
+
}
|
|
63795
|
+
async answer(id, request) {
|
|
63796
|
+
if (request.kind === "pending-stage-message" || request.kind === "sticky-live-delivered") {
|
|
63797
|
+
const router = this.store === undefined ? undefined : pendingStageMessageRouter(this.store);
|
|
63798
|
+
const event = {
|
|
63799
|
+
...request.payload,
|
|
63800
|
+
handled: false
|
|
63801
|
+
};
|
|
63802
|
+
if (request.kind === "pending-stage-message")
|
|
63803
|
+
router?.route(event);
|
|
63804
|
+
else
|
|
63805
|
+
router?.sticky(event);
|
|
63806
|
+
if (event.handled && event.completion !== undefined)
|
|
63807
|
+
return await event.completion;
|
|
63808
|
+
return request.kind === "pending-stage-message" ? { outcome: "refused", reason: "Session not found" } : undefined;
|
|
63809
|
+
}
|
|
63810
|
+
const handoff = this.handoff;
|
|
63811
|
+
const work = this.track(request.work);
|
|
63812
|
+
if (handoff === undefined || work === undefined)
|
|
63813
|
+
throw new Error(STALE_WORK_LEASE);
|
|
63814
|
+
if (request.kind === "ready")
|
|
63815
|
+
return handoff.ready(work, request.runId, request.stageId, request.stageName, request.sessionId);
|
|
63816
|
+
return handoff.deliver(work, request.runId, request.stageId, request.stageName, request.recipient, async (from, message) => {
|
|
63817
|
+
await this.command({ kind: "transport", request: id, from, message });
|
|
63818
|
+
});
|
|
63819
|
+
}
|
|
62566
63820
|
closing;
|
|
62567
63821
|
close() {
|
|
62568
63822
|
this.closing ??= this.dispose();
|
|
@@ -62571,27 +63825,28 @@ class LocalWorkerPool {
|
|
|
62571
63825
|
async dispose() {
|
|
62572
63826
|
this.closed = true;
|
|
62573
63827
|
clearInterval(this.ticker);
|
|
63828
|
+
this.ticker = undefined;
|
|
62574
63829
|
this.releaseBinding();
|
|
62575
63830
|
if (this.options.rootId === undefined)
|
|
62576
63831
|
workflowResourceAdmission.delete(this.backend);
|
|
63832
|
+
this.signalChange();
|
|
62577
63833
|
let failure;
|
|
62578
|
-
|
|
62579
|
-
|
|
62580
|
-
|
|
62581
|
-
|
|
62582
|
-
|
|
62583
|
-
|
|
62584
|
-
|
|
62585
|
-
|
|
62586
|
-
|
|
62587
|
-
|
|
62588
|
-
|
|
62589
|
-
|
|
62590
|
-
const stopped = stopping.flatMap(({ child, log }) => log !== undefined && (child.exitCode !== null || child.signalCode !== null) ? [log] : []);
|
|
62591
|
-
await Promise.all(stopped.map((log) => rm3(log, { force: true }).catch(() => {})));
|
|
62592
|
-
if (this.logDirectory !== undefined)
|
|
62593
|
-
await rmdir(this.logDirectory).catch(() => {});
|
|
63834
|
+
const coordinator = this.coordinator;
|
|
63835
|
+
if (coordinator !== undefined) {
|
|
63836
|
+
if (this.coordinatorLost === undefined) {
|
|
63837
|
+
try {
|
|
63838
|
+
await this.command({ kind: "close" });
|
|
63839
|
+
} catch (error) {
|
|
63840
|
+
failure = error;
|
|
63841
|
+
}
|
|
63842
|
+
}
|
|
63843
|
+
await this.inbound;
|
|
63844
|
+
await coordinator.port.close();
|
|
63845
|
+
this.pids.clear();
|
|
62594
63846
|
}
|
|
63847
|
+
for (const release of this.routeClaims.values())
|
|
63848
|
+
release();
|
|
63849
|
+
this.routeClaims.clear();
|
|
62595
63850
|
try {
|
|
62596
63851
|
await this.releaseRoutes?.();
|
|
62597
63852
|
} catch (error) {
|
|
@@ -62601,27 +63856,26 @@ class LocalWorkerPool {
|
|
|
62601
63856
|
throw failure;
|
|
62602
63857
|
}
|
|
62603
63858
|
}
|
|
62604
|
-
var INTERNAL_WORKFLOW_WORKER_ARG = "--internal-workflow-worker", DEFAULT_WORK_LEASE_MS = 30000, MINIMUM_WORK_LEASE_MS,
|
|
63859
|
+
var INTERNAL_WORKFLOW_WORKER_ARG = "--internal-workflow-worker", DEFAULT_WORK_LEASE_MS = 30000, MINIMUM_WORK_LEASE_MS, DEFAULT_STARTUP_TIMEOUT_MS = 30000, MAX_TIMER_DELAY_MS = 2147483647, COORDINATOR_STDERR_TAIL_CHARACTERS = 8192, COORDINATOR_EXIT_GRACE_MS = 5000;
|
|
62605
63860
|
var init_local_pool = __esm(() => {
|
|
62606
63861
|
init_internal();
|
|
62607
63862
|
init_sql_pool_queue();
|
|
62608
|
-
init_sql_pool_usage();
|
|
62609
63863
|
init_sql_run_journal();
|
|
62610
63864
|
init_workflow_tool_abort();
|
|
62611
63865
|
init_sdk_intercom();
|
|
63866
|
+
init_external_run_routes();
|
|
62612
63867
|
init_workflow_infrastructure_error();
|
|
62613
63868
|
init_admission_binding();
|
|
62614
63869
|
init_definitions();
|
|
62615
63870
|
init_dispatch();
|
|
62616
63871
|
init_host_options();
|
|
62617
|
-
|
|
63872
|
+
init_pool_coordinator();
|
|
62618
63873
|
init_stage_handoff();
|
|
62619
63874
|
MINIMUM_WORK_LEASE_MS = 2 * WORKER_HEARTBEAT_INTERVAL_MS;
|
|
62620
|
-
COORDINATOR_LEASE_MS = 3 * STAGE_SESSION_HEARTBEAT_INTERVAL_MS;
|
|
62621
63875
|
});
|
|
62622
63876
|
|
|
62623
63877
|
// dist/builtin/workflows/src/workers/public-pool.ts
|
|
62624
|
-
import { resolve as
|
|
63878
|
+
import { resolve as resolve13 } from "node:path";
|
|
62625
63879
|
import { isDeepStrictEqual as isDeepStrictEqual2 } from "node:util";
|
|
62626
63880
|
async function restoreWorkerOptions(backend, options) {
|
|
62627
63881
|
const source = options.continuation?.source.id ?? options.runId;
|
|
@@ -62660,11 +63914,11 @@ async function activatePublicPool(backend, options) {
|
|
|
62660
63914
|
...options.localWorkers,
|
|
62661
63915
|
startupTimeoutMs: undefined,
|
|
62662
63916
|
leaseMs: undefined,
|
|
62663
|
-
definitions: options.localWorkers.definitions.map((source) => ({ ...source, path:
|
|
63917
|
+
definitions: options.localWorkers.definitions.map((source) => ({ ...source, path: resolve13(cwd, source.path) }))
|
|
62664
63918
|
};
|
|
62665
63919
|
const admission = options.admission && {
|
|
62666
63920
|
...options.admission,
|
|
62667
|
-
...options.admission.userDatabase ? { userDatabase:
|
|
63921
|
+
...options.admission.userDatabase ? { userDatabase: resolve13(cwd, options.admission.userDatabase) } : {}
|
|
62668
63922
|
};
|
|
62669
63923
|
const sources = localWorkers && await Promise.all(localWorkers.definitions.map(async (source) => ({ ...source, digest: await definitionDigest(source.path) })));
|
|
62670
63924
|
const resources = JSON.parse(JSON.stringify({
|
|
@@ -64010,7 +65264,7 @@ function hasUsableContentBlock(block) {
|
|
|
64010
65264
|
|
|
64011
65265
|
// dist/builtin/workflows/src/shared/stage-session-store.ts
|
|
64012
65266
|
import { randomUUID as randomUUID18 } from "node:crypto";
|
|
64013
|
-
import { dirname as dirname13, resolve as
|
|
65267
|
+
import { dirname as dirname13, resolve as resolve14 } from "node:path";
|
|
64014
65268
|
import {
|
|
64015
65269
|
JsonlSessionStore,
|
|
64016
65270
|
resolveSessionStore,
|
|
@@ -64046,11 +65300,11 @@ async function forkStageSession(store, sessionId, cwd, options, destination = st
|
|
|
64046
65300
|
async function migrateLegacyStageSession(store, sessionId, sessionFile) {
|
|
64047
65301
|
if (sessionFile === undefined || await store.readHeader(sessionId) !== undefined)
|
|
64048
65302
|
return;
|
|
64049
|
-
const location =
|
|
65303
|
+
const location = resolve14(sessionFile);
|
|
64050
65304
|
const legacy = await JsonlSessionStore.open(dirname13(location), { exactDirectory: true, scanExisting: false });
|
|
64051
65305
|
try {
|
|
64052
65306
|
for await (const file of legacy.scanFiles()) {
|
|
64053
|
-
if (
|
|
65307
|
+
if (resolve14(file.location) !== location || file.header.id !== sessionId)
|
|
64054
65308
|
continue;
|
|
64055
65309
|
const snapshot = await file.read();
|
|
64056
65310
|
if (await store.readHeader(sessionId) === undefined)
|
|
@@ -64212,22 +65466,37 @@ function childScopedEvidenceExists(checkpoints, boundaryReplayKey, childRunId) {
|
|
|
64212
65466
|
}
|
|
64213
65467
|
function retainReachableRunGroups(grouped, rootRunId) {
|
|
64214
65468
|
const reachable = new Set([rootRunId]);
|
|
64215
|
-
const
|
|
65469
|
+
const settled = new Set;
|
|
65470
|
+
const pending = [{ runId: rootRunId, live: true }];
|
|
64216
65471
|
while (pending.length > 0) {
|
|
64217
|
-
const
|
|
65472
|
+
const { runId, live } = pending.pop();
|
|
65473
|
+
const drafts = grouped.get(runId);
|
|
64218
65474
|
if (drafts === undefined)
|
|
64219
65475
|
continue;
|
|
64220
65476
|
for (const draft of drafts) {
|
|
64221
|
-
const childRunId =
|
|
64222
|
-
if (childRunId === undefined ||
|
|
65477
|
+
const childRunId = workflowChildFromDraft(draft)?.runId ?? draft.topology?.boundary?.child.runId;
|
|
65478
|
+
if (childRunId === undefined || !grouped.has(childRunId))
|
|
64223
65479
|
continue;
|
|
64224
|
-
|
|
64225
|
-
|
|
65480
|
+
if (live && childRunIdFromDraft(draft) !== undefined) {
|
|
65481
|
+
if (reachable.has(childRunId))
|
|
65482
|
+
continue;
|
|
65483
|
+
reachable.add(childRunId);
|
|
65484
|
+
pending.push({ runId: childRunId, live: true });
|
|
65485
|
+
} else if (!settled.has(childRunId)) {
|
|
65486
|
+
settled.add(childRunId);
|
|
65487
|
+
pending.push({ runId: childRunId, live: false });
|
|
65488
|
+
}
|
|
64226
65489
|
}
|
|
64227
65490
|
}
|
|
64228
|
-
|
|
64229
|
-
|
|
64230
|
-
|
|
65491
|
+
let complete = true;
|
|
65492
|
+
for (const runId of grouped.keys()) {
|
|
65493
|
+
if (reachable.has(runId))
|
|
65494
|
+
continue;
|
|
65495
|
+
if (!settled.has(runId))
|
|
65496
|
+
complete = false;
|
|
65497
|
+
grouped.delete(runId);
|
|
65498
|
+
}
|
|
65499
|
+
return complete;
|
|
64231
65500
|
}
|
|
64232
65501
|
function childRunIdFromDraft(draft) {
|
|
64233
65502
|
const boundary = draft.topology?.boundary;
|
|
@@ -64649,9 +65918,7 @@ function runSnapshotsFromCheckpoints(checkpoints, rootRunId, rootRunName, fallba
|
|
|
64649
65918
|
}
|
|
64650
65919
|
for (const group of grouped.values())
|
|
64651
65920
|
group.sort(compareDraftSourceOrder);
|
|
64652
|
-
|
|
64653
|
-
retainReachableRunGroups(grouped, rootRunId);
|
|
64654
|
-
if (policy.failOnUnreachable && grouped.size !== groupedCount)
|
|
65921
|
+
if (!retainReachableRunGroups(grouped, rootRunId) && policy.failOnUnreachable)
|
|
64655
65922
|
return [];
|
|
64656
65923
|
if (!validateRunGroups(grouped, rootRunId, toolCheckpoints, checkpoints, policy.retainActive))
|
|
64657
65924
|
return [];
|
|
@@ -70309,9 +71576,9 @@ var init_stage_runner_controller = __esm(() => {
|
|
|
70309
71576
|
});
|
|
70310
71577
|
|
|
70311
71578
|
// dist/builtin/workflows/src/runs/foreground/stage-runner-output.ts
|
|
70312
|
-
import { createHash as
|
|
71579
|
+
import { createHash as createHash5 } from "node:crypto";
|
|
70313
71580
|
import { mkdir as mkdir7, writeFile as writeFile10 } from "node:fs/promises";
|
|
70314
|
-
import { basename as basename5, dirname as dirname14, isAbsolute as isAbsolute11, join as join29, resolve as
|
|
71581
|
+
import { basename as basename5, dirname as dirname14, isAbsolute as isAbsolute11, join as join29, resolve as resolve15 } from "node:path";
|
|
70315
71582
|
function normalizeMaxOutput2(maxOutput) {
|
|
70316
71583
|
return {
|
|
70317
71584
|
bytes: maxOutput?.bytes ?? DEFAULT_MAX_OUTPUT_BYTES,
|
|
@@ -70373,7 +71640,7 @@ function formatByteSize(bytes) {
|
|
|
70373
71640
|
return `${value2.toFixed(1)} ${units[unitIndex]}`;
|
|
70374
71641
|
}
|
|
70375
71642
|
function transcriptPath(runId, outputPath) {
|
|
70376
|
-
const digest =
|
|
71643
|
+
const digest = createHash5("sha256").update(outputPath).digest("hex").slice(0, 16);
|
|
70377
71644
|
return join29(workflowArtifactRunPath(runId), "transcripts", `${digest}-${basename5(outputPath)}.transcript.md`);
|
|
70378
71645
|
}
|
|
70379
71646
|
function appendContent(lines, content) {
|
|
@@ -70419,7 +71686,7 @@ function renderTranscript(messages, fallbackOutput) {
|
|
|
70419
71686
|
`);
|
|
70420
71687
|
}
|
|
70421
71688
|
function savedOutputReference(outputPath, fullOutput, transcriptPathValue, transcriptText, transcriptAvailable, warning) {
|
|
70422
|
-
const absolutePath =
|
|
71689
|
+
const absolutePath = resolve15(outputPath);
|
|
70423
71690
|
const bytes = Buffer.byteLength(fullOutput, "utf8");
|
|
70424
71691
|
const lines = countLines(fullOutput);
|
|
70425
71692
|
const transcriptBytes = Buffer.byteLength(transcriptText, "utf8");
|
|
@@ -70434,8 +71701,8 @@ function resolveOutputPath(output, runtimeCwd, requestedCwd) {
|
|
|
70434
71701
|
return;
|
|
70435
71702
|
if (isAbsolute11(output))
|
|
70436
71703
|
return output;
|
|
70437
|
-
const baseCwd = requestedCwd === undefined ? runtimeCwd : isAbsolute11(requestedCwd) ? requestedCwd :
|
|
70438
|
-
return
|
|
71704
|
+
const baseCwd = requestedCwd === undefined ? runtimeCwd : isAbsolute11(requestedCwd) ? requestedCwd : resolve15(runtimeCwd, requestedCwd);
|
|
71705
|
+
return resolve15(baseCwd, output);
|
|
70439
71706
|
}
|
|
70440
71707
|
function degenerateWarning(fullOutput) {
|
|
70441
71708
|
return fullOutput.trim().length === 0 ? "WARNING: the stage artifact is empty; search the companion transcript for this stage's work." : undefined;
|
|
@@ -71444,7 +72711,7 @@ async function run(def, inputs, opts = {}) {
|
|
|
71444
72711
|
if (!isWorkflowDefinition(def))
|
|
71445
72712
|
throw new Error(workflowDefinitionRequirementMessage("run(definition, inputs)", def));
|
|
71446
72713
|
const depth = opts.depth ?? 0;
|
|
71447
|
-
const maxDepth = opts.config?.maxDepth ??
|
|
72714
|
+
const maxDepth = opts.config?.maxDepth ?? 3;
|
|
71448
72715
|
if (depth >= maxDepth) {
|
|
71449
72716
|
return {
|
|
71450
72717
|
runId: opts.runId ?? crypto.randomUUID(),
|
|
@@ -72533,7 +73800,7 @@ init_admission_binding();
|
|
|
72533
73800
|
init_definitions();
|
|
72534
73801
|
init_resource_admission();
|
|
72535
73802
|
import { randomUUID as randomUUID20 } from "node:crypto";
|
|
72536
|
-
import { setTimeout as
|
|
73803
|
+
import { setTimeout as delay4 } from "node:timers/promises";
|
|
72537
73804
|
import { withResourceAdmission as withResourceAdmission2 } from "@bastani/atomic";
|
|
72538
73805
|
import { StorageProvider as StorageProvider3 } from "@bastani/atomic/storage";
|
|
72539
73806
|
|
|
@@ -72542,8 +73809,8 @@ init_stage_topology();
|
|
|
72542
73809
|
init_run_durable_topology();
|
|
72543
73810
|
import { randomUUID as randomUUID19 } from "node:crypto";
|
|
72544
73811
|
function workerSessionStore(input) {
|
|
72545
|
-
const checkpoint =
|
|
72546
|
-
const owner =
|
|
73812
|
+
const checkpoint = (id, header) => {
|
|
73813
|
+
const owner = header.workflow;
|
|
72547
73814
|
const run = input.store.runs().find((run) => run.id === owner?.runId);
|
|
72548
73815
|
const stage = run?.stages.find((stage) => stage.id === owner?.stageId);
|
|
72549
73816
|
if (!run || !stage?.replayKey)
|
|
@@ -72571,11 +73838,11 @@ function workerSessionStore(input) {
|
|
|
72571
73838
|
};
|
|
72572
73839
|
return {
|
|
72573
73840
|
descriptor: input.reader.descriptor,
|
|
72574
|
-
create: async (header, entries) => write({ kind: "session-create", checkpoint:
|
|
72575
|
-
append: async (id, entries, options) => write({ kind: "session-append",
|
|
73841
|
+
create: async (header, entries) => write({ kind: "session-create", checkpoint: checkpoint(header.id, header), header, entries }),
|
|
73842
|
+
append: async (id, entries, options) => write({ kind: "session-append", sessionId: id, entries, expectSeq: options.expectSeq }),
|
|
72576
73843
|
replace: async (id, header, entries, options) => write({
|
|
72577
73844
|
kind: "session-replace",
|
|
72578
|
-
checkpoint:
|
|
73845
|
+
checkpoint: checkpoint(id, header),
|
|
72579
73846
|
header,
|
|
72580
73847
|
entries,
|
|
72581
73848
|
expectSeq: options.expectSeq
|
|
@@ -72603,6 +73870,14 @@ function collectWorkerUsage(runs) {
|
|
|
72603
73870
|
|
|
72604
73871
|
// dist/builtin/workflows/src/workers/local-worker.ts
|
|
72605
73872
|
async function runLocalWorker(start) {
|
|
73873
|
+
if ("role" in start && start.role === POOL_COORDINATOR_ROLE) {
|
|
73874
|
+
await Promise.resolve().then(() => init_pool_coordinator());
|
|
73875
|
+
runPoolCoordinator();
|
|
73876
|
+
return;
|
|
73877
|
+
}
|
|
73878
|
+
await runWorker(start);
|
|
73879
|
+
}
|
|
73880
|
+
async function runWorker(start) {
|
|
72606
73881
|
let sequence = 0;
|
|
72607
73882
|
const pending = new Map;
|
|
72608
73883
|
process.on("message", (reply) => {
|
|
@@ -72726,13 +74001,13 @@ async function runLocalWorker(start) {
|
|
|
72726
74001
|
for (;; ) {
|
|
72727
74002
|
await observeControl();
|
|
72728
74003
|
if (controlState !== "running") {
|
|
72729
|
-
await
|
|
74004
|
+
await delay4(25);
|
|
72730
74005
|
continue;
|
|
72731
74006
|
}
|
|
72732
74007
|
await reader.catchUpWorkflow(start.root);
|
|
72733
74008
|
const work = await request({ kind: "claim" });
|
|
72734
74009
|
if (!work) {
|
|
72735
|
-
await
|
|
74010
|
+
await delay4(25 + Math.random() * 25);
|
|
72736
74011
|
continue;
|
|
72737
74012
|
}
|
|
72738
74013
|
const index = start.definitions.findIndex((source) => `${source.path}#${source.exportName}` === work.definition && source.digest === work.digest);
|
|
@@ -72809,7 +74084,7 @@ async function runLocalWorker(start) {
|
|
|
72809
74084
|
const control = await request({ kind: "budget" });
|
|
72810
74085
|
if (control.state === "running")
|
|
72811
74086
|
return;
|
|
72812
|
-
await
|
|
74087
|
+
await delay4(25, undefined, { signal });
|
|
72813
74088
|
}
|
|
72814
74089
|
};
|
|
72815
74090
|
const resourceAdmission = {
|
|
@@ -72848,7 +74123,7 @@ async function runLocalWorker(start) {
|
|
|
72848
74123
|
await observeControl();
|
|
72849
74124
|
while ((await request({ kind: "control" })).state !== "running") {
|
|
72850
74125
|
controller.signal.throwIfAborted();
|
|
72851
|
-
await
|
|
74126
|
+
await delay4(25);
|
|
72852
74127
|
}
|
|
72853
74128
|
const result = await withResourceAdmission2(resourceAdmission, () => run(definitions[index], work.inputs, {
|
|
72854
74129
|
...work.execution,
|