@bastani/atomic 0.9.43 → 0.9.44-alpha.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +42 -0
- package/dist/builtin/intercom/CHANGELOG.md +22 -0
- package/dist/builtin/intercom/broker/broker.ts +31 -7
- package/dist/builtin/intercom/broker/client-message-validation.ts +6 -0
- package/dist/builtin/intercom/broker/client.ts +69 -0
- package/dist/builtin/intercom/broker/send-signature.ts +4 -1
- package/dist/builtin/intercom/index.bundle.mjs +309 -109
- package/dist/builtin/intercom/live-route-refusal.ts +20 -0
- package/dist/builtin/intercom/package.json +1 -1
- package/dist/builtin/intercom/skills/intercom/SKILL.md +11 -6
- package/dist/builtin/intercom/types.ts +28 -1
- package/dist/builtin/mcp/CHANGELOG.md +7 -0
- package/dist/builtin/mcp/package.json +1 -1
- package/dist/builtin/subagents/CHANGELOG.md +9 -0
- package/dist/builtin/subagents/README.md +8 -8
- package/dist/builtin/subagents/agents/worker.md +1 -1
- package/dist/builtin/subagents/package.json +1 -1
- package/dist/builtin/subagents/skills/agent-browser/SKILL.md +1 -1
- package/dist/builtin/subagents/skills/cua-driver/EMBEDDING.md +21 -2
- package/dist/builtin/subagents/skills/cua-driver/LINUX.md +2 -2
- package/dist/builtin/subagents/skills/cua-driver/MACOS.md +342 -10
- package/dist/builtin/subagents/skills/cua-driver/README.md +4 -3
- package/dist/builtin/subagents/skills/cua-driver/RUNTIME.md +3 -1
- package/dist/builtin/subagents/skills/cua-driver/SKILL.md +7 -4
- package/dist/builtin/subagents/skills/cua-driver/WINDOWS.md +6 -5
- package/dist/builtin/subagents/skills/cua-driver/WORKFLOW.md +15 -0
- package/dist/builtin/subagents/skills/herdr/SKILL.md +22 -10
- package/dist/builtin/subagents/skills/subagent/SKILL.md +8 -5
- package/dist/builtin/subagents/src/extension/index.bundle.mjs +6 -6
- package/dist/builtin/web-access/package.json +1 -1
- package/dist/builtin/workflows/CHANGELOG.md +20 -0
- package/dist/builtin/workflows/README.md +1 -1
- package/dist/builtin/workflows/builtin/{chunk-frqxd4d1.js → chunk-k8d7ehjx.js} +22 -0
- package/dist/builtin/workflows/builtin/{chunk-3s95f7ry.js → chunk-qxpymnaf.js} +1 -1
- package/dist/builtin/workflows/builtin/{chunk-edhf30ef.js → chunk-zjecnjxx.js} +1 -1
- package/dist/builtin/workflows/builtin/goal.js +2 -2
- package/dist/builtin/workflows/builtin/index.js +3 -3
- package/dist/builtin/workflows/builtin/ralph.js +2 -2
- package/dist/builtin/workflows/package.json +1 -1
- package/dist/builtin/workflows/src/extension/index.bundle.mjs +1844 -603
- package/dist/builtin/workflows/src/extension/local-worker-entry.mjs +1880 -605
- package/dist/builtin/workflows/src/index.js +1896 -655
- package/dist/builtin/workflows/src/local-worker-entry.mjs +1880 -605
- package/dist/cli/args.js +1 -1
- package/dist/cli/args.js.map +1 -1
- package/dist/core/agent-session-custom-message-commit.d.ts.map +1 -1
- package/dist/core/agent-session-custom-message-commit.js +59 -0
- package/dist/core/agent-session-custom-message-commit.js.map +1 -1
- package/dist/core/agent-session-methods.d.ts +1 -0
- package/dist/core/agent-session-methods.d.ts.map +1 -1
- package/dist/core/agent-session-methods.js.map +1 -1
- package/dist/core/agent-session-prompt.d.ts +4 -0
- package/dist/core/agent-session-prompt.d.ts.map +1 -1
- package/dist/core/agent-session-prompt.js +10 -2
- package/dist/core/agent-session-prompt.js.map +1 -1
- package/dist/core/agent-session.d.ts +2 -0
- package/dist/core/agent-session.d.ts.map +1 -1
- package/dist/core/agent-session.js +9 -0
- package/dist/core/agent-session.js.map +1 -1
- package/dist/core/builtin-packages.d.ts.map +1 -1
- package/dist/core/builtin-packages.js +13 -13
- package/dist/core/builtin-packages.js.map +1 -1
- package/dist/core/exec.d.ts.map +1 -1
- package/dist/core/exec.js +11 -2
- package/dist/core/exec.js.map +1 -1
- package/dist/core/package-manager-resolver.d.ts.map +1 -1
- package/dist/core/package-manager-resolver.js +25 -14
- package/dist/core/package-manager-resolver.js.map +1 -1
- package/dist/core/package-manager-types.d.ts +8 -0
- package/dist/core/package-manager-types.d.ts.map +1 -1
- package/dist/core/package-manager-types.js.map +1 -1
- package/dist/core/package-manager.d.ts +1 -1
- package/dist/core/package-manager.d.ts.map +1 -1
- package/dist/core/package-manager.js.map +1 -1
- package/dist/core/resource-loader-extension-warnings.d.ts +4 -1
- package/dist/core/resource-loader-extension-warnings.d.ts.map +1 -1
- package/dist/core/resource-loader-extension-warnings.js +15 -0
- package/dist/core/resource-loader-extension-warnings.js.map +1 -1
- package/dist/core/resource-loader-package-resources.d.ts.map +1 -1
- package/dist/core/resource-loader-package-resources.js +1 -0
- package/dist/core/resource-loader-package-resources.js.map +1 -1
- package/dist/core/resource-loader-reload.d.ts.map +1 -1
- package/dist/core/resource-loader-reload.js +5 -3
- package/dist/core/resource-loader-reload.js.map +1 -1
- package/dist/extensions/mcp/client/oauth/flow.d.ts.map +1 -1
- package/dist/extensions/mcp/client/oauth/flow.js +4 -1
- package/dist/extensions/mcp/client/oauth/flow.js.map +1 -1
- package/dist/extensions/mcp/client/oauth/types.d.ts +1 -0
- package/dist/extensions/mcp/client/oauth/types.d.ts.map +1 -1
- package/dist/extensions/mcp/client/oauth/types.js +8 -4
- package/dist/extensions/mcp/client/oauth/types.js.map +1 -1
- package/dist/extensions/mcp/client/transports/streamable-http.d.ts.map +1 -1
- package/dist/extensions/mcp/client/transports/streamable-http.js +2 -1
- package/dist/extensions/mcp/client/transports/streamable-http.js.map +1 -1
- package/dist/index.d.ts +1 -1
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js.map +1 -1
- package/dist/main.d.ts.map +1 -1
- package/dist/main.js +20 -10
- package/dist/main.js.map +1 -1
- package/dist/modes/interactive/components/model-selector.d.ts.map +1 -1
- package/dist/modes/interactive/components/model-selector.js +4 -2
- package/dist/modes/interactive/components/model-selector.js.map +1 -1
- package/dist/modes/interactive/components/scoped-models-selector.d.ts.map +1 -1
- package/dist/modes/interactive/components/scoped-models-selector.js +4 -2
- package/dist/modes/interactive/components/scoped-models-selector.js.map +1 -1
- package/dist/modes/interactive/components/tool-execution.d.ts +6 -0
- package/dist/modes/interactive/components/tool-execution.d.ts.map +1 -1
- package/dist/modes/interactive/components/tool-execution.js +11 -1
- package/dist/modes/interactive/components/tool-execution.js.map +1 -1
- package/dist/modes/interactive/model-search.d.ts +10 -0
- package/dist/modes/interactive/model-search.d.ts.map +1 -1
- package/dist/modes/interactive/model-search.js +21 -0
- package/dist/modes/interactive/model-search.js.map +1 -1
- package/dist/package-manager-cli.d.ts.map +1 -1
- package/dist/package-manager-cli.js +5 -0
- package/dist/package-manager-cli.js.map +1 -1
- package/dist/utils/tools-manager.d.ts.map +1 -1
- package/dist/utils/tools-manager.js +30 -35
- package/dist/utils/tools-manager.js.map +1 -1
- package/docs/computer-use.md +5 -5
- package/docs/custom-provider/streaming.md +9 -14
- package/docs/environment-variables.md +1 -0
- package/docs/intercom/operations.md +1 -1
- package/docs/intercom.md +16 -2
- package/docs/packages.md +2 -0
- package/docs/settings.md +2 -2
- package/docs/skills.md +2 -2
- package/docs/subagents.md +47 -10
- package/docs/workflows/api-reference.md +1 -1
- package/docs/workflows/authoring.md +1 -1
- package/docs/workflows/operations.md +8 -6
- package/docs/workflows/reliable-design.md +23 -5
- package/examples/extensions/custom-provider-anthropic/index.ts +31 -31
- package/examples/extensions/todo.ts +7 -0
- package/npm-shrinkwrap.json +32 -32
- package/package.json +3 -3
|
@@ -3221,7 +3221,13 @@ function createStoreContext(state = createStoreState()) {
|
|
|
3221
3221
|
}
|
|
3222
3222
|
return cachedGraphSnapshot;
|
|
3223
3223
|
}
|
|
3224
|
+
let batchDepth = 0;
|
|
3225
|
+
let batchedNotification = false;
|
|
3224
3226
|
function notify() {
|
|
3227
|
+
if (batchDepth > 0) {
|
|
3228
|
+
batchedNotification = true;
|
|
3229
|
+
return;
|
|
3230
|
+
}
|
|
3225
3231
|
observation.capture();
|
|
3226
3232
|
for (const fn of state.invalidationListeners) {
|
|
3227
3233
|
try {
|
|
@@ -3241,6 +3247,18 @@ function createStoreContext(state = createStoreState()) {
|
|
|
3241
3247
|
state.version++;
|
|
3242
3248
|
notify();
|
|
3243
3249
|
}
|
|
3250
|
+
function batch(update) {
|
|
3251
|
+
batchDepth++;
|
|
3252
|
+
try {
|
|
3253
|
+
return update();
|
|
3254
|
+
} finally {
|
|
3255
|
+
batchDepth--;
|
|
3256
|
+
if (batchDepth === 0 && batchedNotification) {
|
|
3257
|
+
batchedNotification = false;
|
|
3258
|
+
notify();
|
|
3259
|
+
}
|
|
3260
|
+
}
|
|
3261
|
+
}
|
|
3244
3262
|
function findRun(runId) {
|
|
3245
3263
|
return state.runs.find((r) => r.id === runId);
|
|
3246
3264
|
}
|
|
@@ -3294,6 +3312,7 @@ function createStoreContext(state = createStoreState()) {
|
|
|
3294
3312
|
snapshot,
|
|
3295
3313
|
graphSnapshot,
|
|
3296
3314
|
notify,
|
|
3315
|
+
batch,
|
|
3297
3316
|
bumpAndNotify,
|
|
3298
3317
|
findRun,
|
|
3299
3318
|
findStage,
|
|
@@ -4432,6 +4451,9 @@ function createRunStoreMethods(context) {
|
|
|
4432
4451
|
subscribeInvalidation(fn) {
|
|
4433
4452
|
state.invalidationListeners.add(fn);
|
|
4434
4453
|
return () => state.invalidationListeners.delete(fn);
|
|
4454
|
+
},
|
|
4455
|
+
batch(update) {
|
|
4456
|
+
return context.batch(update);
|
|
4435
4457
|
}
|
|
4436
4458
|
};
|
|
4437
4459
|
}
|
|
@@ -5165,7 +5187,7 @@ class ConcurrencyLimiter {
|
|
|
5165
5187
|
}
|
|
5166
5188
|
}
|
|
5167
5189
|
function createRunLimiter(defaultConcurrency) {
|
|
5168
|
-
return new ConcurrencyLimiter(defaultConcurrency ??
|
|
5190
|
+
return new ConcurrencyLimiter(defaultConcurrency ?? 1000);
|
|
5169
5191
|
}
|
|
5170
5192
|
|
|
5171
5193
|
// dist/builtin/workflows/src/extension/atomic-stage-session.ts
|
|
@@ -9848,6 +9870,13 @@ function removeProcessExitCleanup(cleanup) {
|
|
|
9848
9870
|
if (processExitCleanups.delete(cleanup) && processExitCleanups.size === 0)
|
|
9849
9871
|
process.off("exit", runProcessExitCleanups);
|
|
9850
9872
|
}
|
|
9873
|
+
function snapshotNeedsHistory(tail) {
|
|
9874
|
+
const event = tail.snapshot.event;
|
|
9875
|
+
if (event.type !== RUN_EVENT_TYPES.messageQueued)
|
|
9876
|
+
return false;
|
|
9877
|
+
const payload = event.payload;
|
|
9878
|
+
return hasPendingStageDeliveryDelta(payload.pendingStageMessages);
|
|
9879
|
+
}
|
|
9851
9880
|
function* invoke(action) {
|
|
9852
9881
|
try {
|
|
9853
9882
|
const result = action();
|
|
@@ -9963,8 +9992,11 @@ class JournalDurableBackend {
|
|
|
9963
9992
|
this.projection.applyWorkflowSnapshot(event.payload.handle);
|
|
9964
9993
|
this.prompts.set(entry.runId, new PromptReservationState(event.payload.handle.pendingPrompts, event.payload.prompts));
|
|
9965
9994
|
} else {
|
|
9966
|
-
const handle = event.payload;
|
|
9967
9995
|
const previous = this.projection.getWorkflow(entry.runId);
|
|
9996
|
+
const handle = event.type === RUN_EVENT_TYPES.messageQueued && hasPendingStageDeliveryDelta(event.payload.pendingStageMessages) ? {
|
|
9997
|
+
...event.payload,
|
|
9998
|
+
pendingStageMessages: decodePendingStageDeliveries(previous?.pendingStageMessages ?? [], event.payload.pendingStageMessages ?? [])
|
|
9999
|
+
} : event.payload;
|
|
9968
10000
|
this.projection.applyWorkflowSnapshot(handle);
|
|
9969
10001
|
if (event.type === RUN_EVENT_TYPES.runStarted || previous?.pendingPrompts !== handle.pendingPrompts)
|
|
9970
10002
|
this.prompts.delete(entry.runId);
|
|
@@ -10260,11 +10292,23 @@ class JournalDurableBackend {
|
|
|
10260
10292
|
for (const message of messages)
|
|
10261
10293
|
if (message.runId !== logicalRunId)
|
|
10262
10294
|
throw new JournalMessageScopeError(logicalRunId, message.runId);
|
|
10263
|
-
|
|
10295
|
+
const next = {
|
|
10264
10296
|
...handle,
|
|
10265
10297
|
pendingStageMessages: replacePendingStageMessagesForRun(handle.pendingStageMessages ?? [], logicalRunId, messages),
|
|
10266
10298
|
updatedAt: Math.max(Date.now(), handle.updatedAt + 1)
|
|
10267
|
-
}
|
|
10299
|
+
};
|
|
10300
|
+
const encoded = encodePendingStageDeliveries(handle.pendingStageMessages ?? [], next.pendingStageMessages ?? []);
|
|
10301
|
+
const base = this.sequences.get(runId);
|
|
10302
|
+
if (encoded !== undefined && base !== undefined) {
|
|
10303
|
+
try {
|
|
10304
|
+
yield* this.append(runId, { type: RUN_EVENT_TYPES.messageQueued, payload: { ...next, pendingStageMessages: encoded } }, { expectSeq: base });
|
|
10305
|
+
return true;
|
|
10306
|
+
} catch (error) {
|
|
10307
|
+
if (!(error instanceof JournalCasConflictError))
|
|
10308
|
+
throw error;
|
|
10309
|
+
}
|
|
10310
|
+
}
|
|
10311
|
+
yield* this.snapshot(next, RUN_EVENT_TYPES.messageQueued);
|
|
10268
10312
|
return true;
|
|
10269
10313
|
}
|
|
10270
10314
|
async archivePendingStageMessage(runId, message) {
|
|
@@ -10358,7 +10402,7 @@ class JournalDurableBackend {
|
|
|
10358
10402
|
if (summarized !== undefined && summarized.seq === version.seq && summarized.at === version.at)
|
|
10359
10403
|
return;
|
|
10360
10404
|
const tail = yield* invoke(() => journal.snapshotTail(runId));
|
|
10361
|
-
if (tail !== undefined)
|
|
10405
|
+
if (tail !== undefined && !snapshotNeedsHistory(tail))
|
|
10362
10406
|
return this.summarize(runId, tail);
|
|
10363
10407
|
if (summarized === undefined && version.seq === 0)
|
|
10364
10408
|
return;
|
|
@@ -10584,6 +10628,69 @@ class JournalDurableBackend {
|
|
|
10584
10628
|
}
|
|
10585
10629
|
}
|
|
10586
10630
|
}
|
|
10631
|
+
function identity(message) {
|
|
10632
|
+
return JSON.stringify([message.runId, message.id]);
|
|
10633
|
+
}
|
|
10634
|
+
function uniqueByIdentity(messages) {
|
|
10635
|
+
const result = new Map;
|
|
10636
|
+
const repeated = new Set;
|
|
10637
|
+
for (const message of messages) {
|
|
10638
|
+
const key = identity(message);
|
|
10639
|
+
if (result.has(key))
|
|
10640
|
+
repeated.add(key);
|
|
10641
|
+
result.set(key, message);
|
|
10642
|
+
}
|
|
10643
|
+
for (const key of repeated)
|
|
10644
|
+
result.delete(key);
|
|
10645
|
+
return result;
|
|
10646
|
+
}
|
|
10647
|
+
function isDeltaEncoded(message) {
|
|
10648
|
+
return message.deliveryDelta !== undefined;
|
|
10649
|
+
}
|
|
10650
|
+
function encodePendingStageDeliveries(previous, next) {
|
|
10651
|
+
const bases = uniqueByIdentity(previous);
|
|
10652
|
+
const nextIdentities = uniqueByIdentity(next);
|
|
10653
|
+
let encodedAny = false;
|
|
10654
|
+
const encoded = next.map((message) => {
|
|
10655
|
+
const base = bases.get(identity(message))?.deliveries;
|
|
10656
|
+
const deliveries = message.deliveries;
|
|
10657
|
+
if (base === undefined || deliveries === undefined || !nextIdentities.has(identity(message)))
|
|
10658
|
+
return message;
|
|
10659
|
+
const drop = [];
|
|
10660
|
+
let kept = 0;
|
|
10661
|
+
base.forEach((delivery, index) => {
|
|
10662
|
+
if (kept < deliveries.length && JSON.stringify(delivery) === JSON.stringify(deliveries[kept]))
|
|
10663
|
+
kept++;
|
|
10664
|
+
else
|
|
10665
|
+
drop.push(index);
|
|
10666
|
+
});
|
|
10667
|
+
const add = deliveries.slice(kept);
|
|
10668
|
+
if (drop.length + add.length >= deliveries.length)
|
|
10669
|
+
return message;
|
|
10670
|
+
encodedAny = true;
|
|
10671
|
+
const { deliveries: _full, ...rest } = message;
|
|
10672
|
+
return { ...rest, deliveryDelta: drop.length === 0 ? { add } : { drop, add } };
|
|
10673
|
+
});
|
|
10674
|
+
return encodedAny ? encoded : undefined;
|
|
10675
|
+
}
|
|
10676
|
+
function decodePendingStageDeliveries(previous, messages) {
|
|
10677
|
+
if (!messages.some(isDeltaEncoded))
|
|
10678
|
+
return messages;
|
|
10679
|
+
const bases = uniqueByIdentity(previous);
|
|
10680
|
+
return messages.map((message) => {
|
|
10681
|
+
if (!isDeltaEncoded(message))
|
|
10682
|
+
return message;
|
|
10683
|
+
const base = bases.get(identity(message))?.deliveries;
|
|
10684
|
+
if (base === undefined)
|
|
10685
|
+
throw new Error(`atomic-workflows: journal delivery change for ${identity(message)} has no base entry`);
|
|
10686
|
+
const { deliveryDelta, ...rest } = message;
|
|
10687
|
+
const dropped = new Set(deliveryDelta.drop ?? []);
|
|
10688
|
+
return { ...rest, deliveries: [...base.filter((_, index) => !dropped.has(index)), ...deliveryDelta.add] };
|
|
10689
|
+
});
|
|
10690
|
+
}
|
|
10691
|
+
function hasPendingStageDeliveryDelta(messages) {
|
|
10692
|
+
return messages?.some(isDeltaEncoded) === true;
|
|
10693
|
+
}
|
|
10587
10694
|
var PROCESS_HOLDER_ID, processExitCleanups, JournalOwnershipError, JournalBackendClosedError, JournalOperationAbortedError, JournalExecutionRetiredError, JournalMessageScopeError;
|
|
10588
10695
|
var init_journal_backend = __esm(() => {
|
|
10589
10696
|
init_workflow_instance_ownership_error();
|
|
@@ -57292,7 +57399,7 @@ var init_definitions = __esm(() => {
|
|
|
57292
57399
|
// dist/builtin/workflows/src/durable/journal/pool-mutations.ts
|
|
57293
57400
|
import { StorageProvider as StorageProvider2 } from "@bastani/atomic/storage";
|
|
57294
57401
|
async function validatePoolMutation(tx, journal, root, work, mutation) {
|
|
57295
|
-
if (!["checkpoint", "session-create", "session-
|
|
57402
|
+
if (!["checkpoint", "session-create", "session-replace"].includes(mutation.kind))
|
|
57296
57403
|
throw new Error("Invalid pool mutation");
|
|
57297
57404
|
const checkpoint = mutation.checkpoint;
|
|
57298
57405
|
const prefix = `${work.scope}:`;
|
|
@@ -57360,10 +57467,18 @@ async function sessionStore(tx, prefix) {
|
|
|
57360
57467
|
return new StorageProvider2().openSessionStore({ kind: "database", database, prefix }, { agentDir: "", domain: "sessions" });
|
|
57361
57468
|
}
|
|
57362
57469
|
async function applyPoolSession(tx, prefix, workId, mutation) {
|
|
57470
|
+
const owners = `${prefix}session_owners`;
|
|
57471
|
+
if (mutation.kind === "session-append") {
|
|
57472
|
+
if (typeof mutation.sessionId !== "string" || mutation.sessionId === "")
|
|
57473
|
+
throw new Error("Pool sessions require database identity");
|
|
57474
|
+
const owner = await tx.get(`SELECT work FROM ${owners} WHERE session = ?`, mutation.sessionId);
|
|
57475
|
+
if (owner?.work !== workId)
|
|
57476
|
+
throw new Error("Pool session scope mismatch");
|
|
57477
|
+
return withSessionStore(tx, prefix, (store) => store.append(mutation.sessionId, mutation.entries, { expectSeq: mutation.expectSeq }));
|
|
57478
|
+
}
|
|
57363
57479
|
const { sessionId, replayKey } = mutation.checkpoint;
|
|
57364
57480
|
if (!sessionId || mutation.checkpoint.sessionFile !== undefined)
|
|
57365
57481
|
throw new Error("Pool sessions require database identity");
|
|
57366
|
-
const owners = `${prefix}session_owners`;
|
|
57367
57482
|
if (mutation.kind === "session-create") {
|
|
57368
57483
|
if (mutation.header.id !== sessionId)
|
|
57369
57484
|
throw new Error("Pool session identity mismatch");
|
|
@@ -57373,10 +57488,12 @@ async function applyPoolSession(tx, prefix, workId, mutation) {
|
|
|
57373
57488
|
if (owner?.work !== workId || owner.replay_key !== replayKey)
|
|
57374
57489
|
throw new Error("Pool session scope mismatch");
|
|
57375
57490
|
}
|
|
57491
|
+
return withSessionStore(tx, prefix, (store) => mutation.kind === "session-create" ? store.create(mutation.header, mutation.entries) : store.replace(sessionId, mutation.header, mutation.entries, { expectSeq: mutation.expectSeq }));
|
|
57492
|
+
}
|
|
57493
|
+
async function withSessionStore(tx, prefix, write) {
|
|
57376
57494
|
const store = await sessionStore(tx, prefix);
|
|
57377
57495
|
try {
|
|
57378
|
-
|
|
57379
|
-
return result.lastSeq;
|
|
57496
|
+
return (await write(store)).lastSeq;
|
|
57380
57497
|
} finally {
|
|
57381
57498
|
await store.close();
|
|
57382
57499
|
}
|
|
@@ -57424,8 +57541,11 @@ var init_pool_stage_usage = __esm(() => {
|
|
|
57424
57541
|
});
|
|
57425
57542
|
|
|
57426
57543
|
// dist/builtin/workflows/src/durable/journal/sql-pool-queue.ts
|
|
57427
|
-
import { randomUUID as randomUUID10 } from "node:crypto";
|
|
57544
|
+
import { createHash as createHash4, randomUUID as randomUUID10 } from "node:crypto";
|
|
57428
57545
|
import { assertStorageSchemaVersion as assertStorageSchemaVersion2 } from "@bastani/atomic/storage";
|
|
57546
|
+
function requestIdentity(request) {
|
|
57547
|
+
return createHash4("sha256").update(request).digest("hex");
|
|
57548
|
+
}
|
|
57429
57549
|
function fromRow(row) {
|
|
57430
57550
|
return {
|
|
57431
57551
|
...JSON.parse(row.descriptor),
|
|
@@ -57481,10 +57601,11 @@ class SqlPoolQueue {
|
|
|
57481
57601
|
const versions = await tx.all(`SELECT version FROM ${pool.meta}`);
|
|
57482
57602
|
if (versions.length > 1)
|
|
57483
57603
|
throw new Error("Corrupt pool schema metadata");
|
|
57484
|
-
|
|
57485
|
-
|
|
57486
|
-
|
|
57487
|
-
|
|
57604
|
+
const version = versions[0]?.version;
|
|
57605
|
+
if (version === undefined)
|
|
57606
|
+
await tx.run(`INSERT INTO ${pool.meta} VALUES (?)`, POOL_SCHEMA_VERSION);
|
|
57607
|
+
else if (version !== 1)
|
|
57608
|
+
assertStorageSchemaVersion2("journal", version, POOL_SCHEMA_VERSION);
|
|
57488
57609
|
await tx.exec(`
|
|
57489
57610
|
CREATE TABLE IF NOT EXISTS ${pool.roots}(root TEXT PRIMARY KEY REFERENCES ${journal.runs}(run_id) ON DELETE CASCADE, timeout INTEGER NOT NULL);
|
|
57490
57611
|
CREATE TABLE IF NOT EXISTS ${pool.work}(
|
|
@@ -57515,9 +57636,23 @@ class SqlPoolQueue {
|
|
|
57515
57636
|
state TEXT NOT NULL DEFAULT 'running', revision INTEGER NOT NULL DEFAULT 0
|
|
57516
57637
|
)`);
|
|
57517
57638
|
await tx.exec(poolUsageSchema(journal.runs));
|
|
57639
|
+
if (version === 1)
|
|
57640
|
+
await pool.migrateFromVersion1(tx);
|
|
57518
57641
|
});
|
|
57519
57642
|
return pool;
|
|
57520
57643
|
}
|
|
57644
|
+
async migrateFromVersion1(tx) {
|
|
57645
|
+
await tx.run(`DELETE FROM ${this.operations} WHERE substr(request, 1, 1) = '['`);
|
|
57646
|
+
for (let after = 0;; ) {
|
|
57647
|
+
const rows = await tx.all(`SELECT rowid, request FROM ${this.operations} WHERE rowid > ? ORDER BY rowid LIMIT 256`, after);
|
|
57648
|
+
if (rows.length === 0)
|
|
57649
|
+
break;
|
|
57650
|
+
for (const row of rows)
|
|
57651
|
+
await tx.run(`UPDATE ${this.operations} SET request = ? WHERE rowid = ?`, requestIdentity(String(row.request)), row.rowid);
|
|
57652
|
+
after = Number(rows.at(-1).rowid);
|
|
57653
|
+
}
|
|
57654
|
+
await tx.run(`UPDATE ${this.meta} SET version = ?`, POOL_SCHEMA_VERSION);
|
|
57655
|
+
}
|
|
57521
57656
|
async authority(tx, lease, timeout) {
|
|
57522
57657
|
const row = await tx.get(`SELECT l.*, r.status, p.timeout, ${STORE_NOW2} AS now FROM ${this.journal.leases} l JOIN ${this.journal.runs} r ON r.run_id = l.run_id LEFT JOIN ${this.roots} p ON p.root = l.run_id WHERE l.run_id = ?`, lease.runId);
|
|
57523
57658
|
if (row?.timeout === null && timeout === undefined)
|
|
@@ -57658,31 +57793,18 @@ class SqlPoolQueue {
|
|
|
57658
57793
|
throw new Error(STALE_WORK_LEASE);
|
|
57659
57794
|
return row;
|
|
57660
57795
|
}
|
|
57661
|
-
async renew(lease, token,
|
|
57796
|
+
async renew(lease, token, durationMs) {
|
|
57662
57797
|
positive(durationMs);
|
|
57663
57798
|
lease = snapshotLease(lease);
|
|
57664
57799
|
token = { ...token };
|
|
57665
|
-
validateText("key", operationId);
|
|
57666
|
-
const request = JSON.stringify([token.epoch, token.generation, token.worker, durationMs]);
|
|
57667
57800
|
return this.transaction(async (tx) => {
|
|
57668
57801
|
const { now } = await this.authority(tx, lease);
|
|
57669
57802
|
const row = await this.current(tx, lease, token);
|
|
57670
57803
|
if (row.epoch !== lease.epoch || row.expires_at < now)
|
|
57671
57804
|
throw new Error(STALE_WORK_LEASE);
|
|
57672
|
-
const
|
|
57673
|
-
|
|
57674
|
-
|
|
57675
|
-
throw new Error("Pool operation identity conflict");
|
|
57676
|
-
return JSON.parse(previous.receipt);
|
|
57677
|
-
}
|
|
57678
|
-
const receipt = {
|
|
57679
|
-
...fromRow(row),
|
|
57680
|
-
expiresAt: Math.max(Number(row.expires_at), expiry(now, durationMs)),
|
|
57681
|
-
renewedAt: now
|
|
57682
|
-
};
|
|
57683
|
-
await tx.run(`UPDATE ${this.work} SET expires_at = ?, renewed_at = ? WHERE id = ?`, receipt.expiresAt, now, row.id);
|
|
57684
|
-
await tx.run(`INSERT INTO ${this.operations} VALUES (?, ?, ?, ?, ?)`, row.id, token.generation, operationId, request, JSON.stringify(receipt));
|
|
57685
|
-
return receipt;
|
|
57805
|
+
const expiresAt = Math.max(Number(row.expires_at), expiry(now, durationMs));
|
|
57806
|
+
await tx.run(`UPDATE ${this.work} SET expires_at = ?, renewed_at = ? WHERE id = ?`, expiresAt, now, row.id);
|
|
57807
|
+
return { ...fromRow(row), expiresAt, renewedAt: now };
|
|
57686
57808
|
});
|
|
57687
57809
|
}
|
|
57688
57810
|
async commit(lease, token, operationId, mutation) {
|
|
@@ -57691,6 +57813,7 @@ class SqlPoolQueue {
|
|
|
57691
57813
|
validateText("key", operationId);
|
|
57692
57814
|
assertWorkflowSerializableObject(mutation, "Pool mutation");
|
|
57693
57815
|
const request = serializeJson({ epoch: token.epoch, worker: token.worker, mutation });
|
|
57816
|
+
const identity = requestIdentity(request);
|
|
57694
57817
|
mutation = JSON.parse(request).mutation;
|
|
57695
57818
|
return this.transaction(async (tx) => {
|
|
57696
57819
|
const { now, status } = await this.authority(tx, lease);
|
|
@@ -57699,7 +57822,7 @@ class SqlPoolQueue {
|
|
|
57699
57822
|
throw new Error(STALE_WORK_LEASE);
|
|
57700
57823
|
const previous = await tx.get(`SELECT * FROM ${this.operations} WHERE work = ? AND generation = ? AND operation = ?`, token.id, token.generation, operationId);
|
|
57701
57824
|
if (previous) {
|
|
57702
|
-
if (previous.request !==
|
|
57825
|
+
if (previous.request !== identity)
|
|
57703
57826
|
throw new Error("Pool operation identity conflict");
|
|
57704
57827
|
return JSON.parse(previous.receipt);
|
|
57705
57828
|
}
|
|
@@ -57707,34 +57830,41 @@ class SqlPoolQueue {
|
|
|
57707
57830
|
throw new Error(STALE_WORK_LEASE);
|
|
57708
57831
|
if (!await this.admitsAttemptWrites(tx, lease, status))
|
|
57709
57832
|
throw new Error("Pool root is not running");
|
|
57710
|
-
const
|
|
57711
|
-
await
|
|
57712
|
-
if (checkpoint.completedAt > Math.max(now, Date.now()) + MAX_WORKER_CLOCK_SKEW_MS)
|
|
57713
|
-
throw new Error("Pool checkpoint timestamp is in the future");
|
|
57714
|
-
assertWorkflowSerializableObject(checkpoint, "Pool checkpoint");
|
|
57715
|
-
const key = JSON.stringify([RUN_EVENT_TYPES.checkpointRecorded, checkpoint.kind, checkpoint.checkpointId]);
|
|
57716
|
-
const payload = serializeJson(checkpoint);
|
|
57717
|
-
const existing = await tx.get(`SELECT seq, at, payload FROM ${this.journal.events} WHERE run_id = ? AND key = ?`, lease.runId, key);
|
|
57718
|
-
if (existing && existing.payload !== payload)
|
|
57719
|
-
throw new Error("Pool checkpoint identity conflict");
|
|
57720
|
-
if (existing && mutation.kind !== "checkpoint")
|
|
57721
|
-
throw new Error("Pool session checkpoint identity conflict");
|
|
57722
|
-
const lastSeq = mutation.kind === "checkpoint" ? undefined : await applyPoolSession(tx, `${this.journal.runs}_pool_`, row.id, mutation);
|
|
57723
|
-
const latest = await tx.get(`SELECT COALESCE(MAX(seq), 0) AS seq FROM ${this.journal.events} WHERE run_id = ?`, lease.runId);
|
|
57724
|
-
const receipt = existing ? { seq: Number(existing.seq), at: Number(existing.at) } : {
|
|
57725
|
-
seq: journalCounter(lease.runId, "seq", Number(latest.seq) + 1, "overflow"),
|
|
57726
|
-
at: now,
|
|
57727
|
-
...lastSeq === undefined ? {} : { lastSeq }
|
|
57728
|
-
};
|
|
57729
|
-
if (!existing)
|
|
57730
|
-
await tx.run(`INSERT INTO ${this.journal.events} VALUES (?, ?, ?, ?, ?, ?, ?)`, lease.runId, receipt.seq, lease.epoch, key, RUN_EVENT_TYPES.checkpointRecorded, payload, now);
|
|
57731
|
-
const billed = poolCheckpointUsage(checkpoint);
|
|
57732
|
-
if (billed && !existing)
|
|
57733
|
-
await upsertPoolUsage(tx, this.journal.runs, row.id, token.generation, billed.key, billed.usage);
|
|
57734
|
-
await tx.run(`INSERT INTO ${this.operations} VALUES (?, ?, ?, ?, ?)`, token.id, token.generation, operationId, request, JSON.stringify(receipt));
|
|
57833
|
+
const receipt = mutation.kind === "session-append" ? await this.appendSession(tx, row, mutation, now) : await this.recordCheckpoint(tx, lease, token, row, mutation, now);
|
|
57834
|
+
await tx.run(`INSERT INTO ${this.operations} VALUES (?, ?, ?, ?, ?)`, token.id, token.generation, operationId, identity, JSON.stringify(receipt));
|
|
57735
57835
|
return receipt;
|
|
57736
57836
|
});
|
|
57737
57837
|
}
|
|
57838
|
+
async appendSession(tx, row, mutation, now) {
|
|
57839
|
+
return { at: now, lastSeq: await applyPoolSession(tx, `${this.journal.runs}_pool_`, row.id, mutation) };
|
|
57840
|
+
}
|
|
57841
|
+
async recordCheckpoint(tx, lease, token, row, mutation, now) {
|
|
57842
|
+
const checkpoint = mutation.checkpoint;
|
|
57843
|
+
await validatePoolMutation(tx, this.journal, lease.runId, fromRow(row), mutation);
|
|
57844
|
+
if (checkpoint.completedAt > Math.max(now, Date.now()) + MAX_WORKER_CLOCK_SKEW_MS)
|
|
57845
|
+
throw new Error("Pool checkpoint timestamp is in the future");
|
|
57846
|
+
assertWorkflowSerializableObject(checkpoint, "Pool checkpoint");
|
|
57847
|
+
const key = JSON.stringify([RUN_EVENT_TYPES.checkpointRecorded, checkpoint.kind, checkpoint.checkpointId]);
|
|
57848
|
+
const payload = serializeJson(checkpoint);
|
|
57849
|
+
const existing = await tx.get(`SELECT seq, at, payload FROM ${this.journal.events} WHERE run_id = ? AND key = ?`, lease.runId, key);
|
|
57850
|
+
if (existing && existing.payload !== payload)
|
|
57851
|
+
throw new Error("Pool checkpoint identity conflict");
|
|
57852
|
+
if (existing && mutation.kind !== "checkpoint")
|
|
57853
|
+
throw new Error("Pool session checkpoint identity conflict");
|
|
57854
|
+
const lastSeq = mutation.kind === "checkpoint" ? undefined : await applyPoolSession(tx, `${this.journal.runs}_pool_`, row.id, mutation);
|
|
57855
|
+
const latest = await tx.get(`SELECT COALESCE(MAX(seq), 0) AS seq FROM ${this.journal.events} WHERE run_id = ?`, lease.runId);
|
|
57856
|
+
const receipt = existing ? { seq: Number(existing.seq), at: Number(existing.at) } : {
|
|
57857
|
+
seq: journalCounter(lease.runId, "seq", Number(latest.seq) + 1, "overflow"),
|
|
57858
|
+
at: now,
|
|
57859
|
+
...lastSeq === undefined ? {} : { lastSeq }
|
|
57860
|
+
};
|
|
57861
|
+
if (!existing)
|
|
57862
|
+
await tx.run(`INSERT INTO ${this.journal.events} VALUES (?, ?, ?, ?, ?, ?, ?)`, lease.runId, receipt.seq, lease.epoch, key, RUN_EVENT_TYPES.checkpointRecorded, payload, now);
|
|
57863
|
+
const billed = poolCheckpointUsage(checkpoint);
|
|
57864
|
+
if (billed && !existing)
|
|
57865
|
+
await upsertPoolUsage(tx, this.journal.runs, row.id, token.generation, billed.key, billed.usage);
|
|
57866
|
+
return receipt;
|
|
57867
|
+
}
|
|
57738
57868
|
async complete(lease, token, result) {
|
|
57739
57869
|
lease = snapshotLease(lease);
|
|
57740
57870
|
token = { ...token };
|
|
@@ -57821,7 +57951,7 @@ function openSqlPoolQueue(journal) {
|
|
|
57821
57951
|
return Promise.reject(new Error("Worker pool requires a SQL run journal"));
|
|
57822
57952
|
return SqlPoolQueue.open(storage);
|
|
57823
57953
|
}
|
|
57824
|
-
var STORE_NOW2 = "CAST(unixepoch('subsec') * 1000 AS INTEGER)", MAX_WORKER_CLOCK_SKEW_MS = 1000, STALE_WORK_LEASE = "Stale work lease";
|
|
57954
|
+
var STORE_NOW2 = "CAST(unixepoch('subsec') * 1000 AS INTEGER)", MAX_WORKER_CLOCK_SKEW_MS = 1000, POOL_SCHEMA_VERSION = 2, STALE_WORK_LEASE = "Stale work lease";
|
|
57825
57955
|
var init_sql_pool_queue = __esm(() => {
|
|
57826
57956
|
init_serializable();
|
|
57827
57957
|
init_boundary_topology();
|
|
@@ -57833,74 +57963,42 @@ var init_sql_pool_queue = __esm(() => {
|
|
|
57833
57963
|
init_values();
|
|
57834
57964
|
});
|
|
57835
57965
|
|
|
57836
|
-
// dist/builtin/workflows/src/
|
|
57837
|
-
function
|
|
57838
|
-
|
|
57839
|
-
if (
|
|
57840
|
-
|
|
57841
|
-
|
|
57842
|
-
const value2 = report.usage[field];
|
|
57843
|
-
if (typeof value2 !== "number" || !Number.isFinite(value2) || value2 < 0)
|
|
57844
|
-
throw new Error(`Pool usage ${field} must be a finite non-negative number`);
|
|
57966
|
+
// dist/builtin/workflows/src/shared/external-run-routes.ts
|
|
57967
|
+
function claimExternalRunRoute(store, runId, owner) {
|
|
57968
|
+
let owners = registry.owners.get(store);
|
|
57969
|
+
if (owners === undefined) {
|
|
57970
|
+
owners = new Map;
|
|
57971
|
+
registry.owners.set(store, owners);
|
|
57845
57972
|
}
|
|
57973
|
+
owners.set(runId, owner);
|
|
57974
|
+
return () => {
|
|
57975
|
+
const current = registry.owners.get(store);
|
|
57976
|
+
if (current?.get(runId) !== owner)
|
|
57977
|
+
return;
|
|
57978
|
+
current.delete(runId);
|
|
57979
|
+
if (current.size === 0)
|
|
57980
|
+
registry.owners.delete(store);
|
|
57981
|
+
};
|
|
57846
57982
|
}
|
|
57847
|
-
|
|
57848
|
-
|
|
57849
|
-
transaction;
|
|
57850
|
-
journal;
|
|
57851
|
-
table;
|
|
57852
|
-
constructor(transaction, journal) {
|
|
57853
|
-
this.transaction = transaction;
|
|
57854
|
-
this.journal = journal;
|
|
57855
|
-
this.table = poolUsageTable(journal.runs);
|
|
57856
|
-
}
|
|
57857
|
-
static async open(transaction, journal) {
|
|
57858
|
-
const usage = new SqlPoolUsage(transaction, journal);
|
|
57859
|
-
await transaction((tx) => tx.exec(poolUsageSchema(journal.runs)));
|
|
57860
|
-
return usage;
|
|
57861
|
-
}
|
|
57862
|
-
async totalIn(tx, root) {
|
|
57863
|
-
const row = await tx.get(`SELECT ${POOL_USAGE_FIELDS.map((field) => `COALESCE(SUM(u.${field.toLowerCase()}), 0) AS ${field}`).join(", ")}
|
|
57864
|
-
FROM ${this.table} u JOIN ${this.journal.runs}_work w ON w.id = u.work WHERE w.root = ?`, root);
|
|
57865
|
-
return Object.fromEntries(POOL_USAGE_FIELDS.map((field) => [field, Number(row?.[field] ?? 0)]));
|
|
57866
|
-
}
|
|
57867
|
-
async report(lease, token, reports) {
|
|
57868
|
-
validateText("runId", lease.runId);
|
|
57869
|
-
validateText("holderId", lease.holderId);
|
|
57870
|
-
for (const report of reports)
|
|
57871
|
-
validate(report);
|
|
57872
|
-
const snapshot = reports.map((report) => ({ key: report.key, usage: { ...report.usage } }));
|
|
57873
|
-
return this.transaction(async (tx) => {
|
|
57874
|
-
const authority = await tx.get(`SELECT l.epoch, l.holder_id, l.renewed_at, p.timeout, ${STORE_NOW3} AS now FROM ${this.journal.leases} l
|
|
57875
|
-
LEFT JOIN ${this.journal.runs}_pool p ON p.root = l.run_id WHERE l.run_id = ?`, lease.runId);
|
|
57876
|
-
if (!authority || authority.timeout === null || authority.epoch !== lease.epoch || authority.holder_id !== lease.holderId || isWorkflowLeaseExpired(Number(authority.renewed_at), Number(authority.timeout), Number(authority.now)))
|
|
57877
|
-
throw new StaleLeaseError(lease.runId, lease.epoch, Number(authority?.epoch ?? 0));
|
|
57878
|
-
const work = await tx.get(`SELECT * FROM ${this.journal.runs}_work WHERE id = ? AND root = ?`, token.id, lease.runId);
|
|
57879
|
-
if (work?.state !== "leased" && work?.state !== "completed" || work.epoch !== token.epoch || work.epoch !== lease.epoch || work.generation !== token.generation || work.worker !== token.worker || work.state === "leased" && Number(work.expires_at) < Number(authority.now))
|
|
57880
|
-
throw new Error("Stale work lease");
|
|
57881
|
-
for (const { key, usage } of snapshot)
|
|
57882
|
-
await upsertPoolUsage(tx, this.journal.runs, token.id, token.generation, key, usage);
|
|
57883
|
-
return this.totalIn(tx, lease.runId);
|
|
57884
|
-
});
|
|
57885
|
-
}
|
|
57886
|
-
total(root) {
|
|
57887
|
-
return this.transaction((tx) => this.totalIn(tx, root));
|
|
57888
|
-
}
|
|
57983
|
+
function externalRunRouteOwner(store, runId) {
|
|
57984
|
+
return registry.owners.get(store)?.get(runId);
|
|
57889
57985
|
}
|
|
57890
|
-
|
|
57891
|
-
|
|
57892
|
-
|
|
57893
|
-
|
|
57894
|
-
|
|
57895
|
-
|
|
57986
|
+
function registerPendingStageMessageRouter(store, router) {
|
|
57987
|
+
registry.routers.set(store, router);
|
|
57988
|
+
return () => {
|
|
57989
|
+
if (registry.routers.get(store) === router)
|
|
57990
|
+
registry.routers.delete(store);
|
|
57991
|
+
};
|
|
57896
57992
|
}
|
|
57897
|
-
|
|
57898
|
-
|
|
57899
|
-
|
|
57900
|
-
|
|
57901
|
-
|
|
57902
|
-
|
|
57903
|
-
|
|
57993
|
+
function pendingStageMessageRouter(store) {
|
|
57994
|
+
return registry.routers.get(store);
|
|
57995
|
+
}
|
|
57996
|
+
var registryKey, processState, registry;
|
|
57997
|
+
var init_external_run_routes = __esm(() => {
|
|
57998
|
+
registryKey = Symbol.for("atomic-workflows/external-run-routes@1");
|
|
57999
|
+
processState = globalThis;
|
|
58000
|
+
registry = processState[registryKey] ?? { owners: new WeakMap, routers: new WeakMap };
|
|
58001
|
+
processState[registryKey] = registry;
|
|
57904
58002
|
});
|
|
57905
58003
|
|
|
57906
58004
|
// dist/builtin/workflows/src/shared/pending-stage-route-capability.ts
|
|
@@ -57927,12 +58025,12 @@ function rotateWorkflowPendingStageRouteCapability(store, runId) {
|
|
|
57927
58025
|
capabilitiesFor(store).set(runId, capability);
|
|
57928
58026
|
return capability;
|
|
57929
58027
|
}
|
|
57930
|
-
var routeCapabilitiesKey,
|
|
58028
|
+
var routeCapabilitiesKey, processState2, routeCapabilities;
|
|
57931
58029
|
var init_pending_stage_route_capability = __esm(() => {
|
|
57932
58030
|
routeCapabilitiesKey = Symbol.for("atomic-workflows/pending-stage-route-capabilities@1");
|
|
57933
|
-
|
|
57934
|
-
routeCapabilities =
|
|
57935
|
-
|
|
58031
|
+
processState2 = globalThis;
|
|
58032
|
+
routeCapabilities = processState2[routeCapabilitiesKey] ?? new WeakMap;
|
|
58033
|
+
processState2[routeCapabilitiesKey] = routeCapabilities;
|
|
57936
58034
|
});
|
|
57937
58035
|
|
|
57938
58036
|
// dist/builtin/workflows/src/shared/pending-stage-route-readiness.ts
|
|
@@ -57964,12 +58062,12 @@ function hasWorkflowPendingStageRouteOwner(store) {
|
|
|
57964
58062
|
function workflowPendingStageRouteReady(store, runId) {
|
|
57965
58063
|
return owners.get(store)?.ready(runId);
|
|
57966
58064
|
}
|
|
57967
|
-
var routeOwnersKey,
|
|
58065
|
+
var routeOwnersKey, processState3, owners;
|
|
57968
58066
|
var init_pending_stage_route_readiness = __esm(() => {
|
|
57969
58067
|
routeOwnersKey = Symbol.for("atomic-workflows/pending-stage-route-owners@1");
|
|
57970
|
-
|
|
57971
|
-
owners =
|
|
57972
|
-
|
|
58068
|
+
processState3 = globalThis;
|
|
58069
|
+
owners = processState3[routeOwnersKey] ?? new WeakMap;
|
|
58070
|
+
processState3[routeOwnersKey] = owners;
|
|
57973
58071
|
});
|
|
57974
58072
|
|
|
57975
58073
|
// dist/builtin/workflows/src/shared/workflow-stage-path-matching.ts
|
|
@@ -58270,6 +58368,9 @@ function registerPendingStageIntercomBridge(pi, activeStore, ownsRun = () => tru
|
|
|
58270
58368
|
notificationId,
|
|
58271
58369
|
reason
|
|
58272
58370
|
};
|
|
58371
|
+
const external = externalRunRouteOwner(activeStore, entry.runId);
|
|
58372
|
+
if (external !== undefined)
|
|
58373
|
+
return external.notifyUndeliverable(payload);
|
|
58273
58374
|
pi.events?.emit?.(PENDING_STAGE_UNDELIVERABLE_EVENT, payload);
|
|
58274
58375
|
return payload.handled && await payload.completion === true;
|
|
58275
58376
|
};
|
|
@@ -58281,19 +58382,29 @@ function registerPendingStageIntercomBridge(pi, activeStore, ownsRun = () => tru
|
|
|
58281
58382
|
return;
|
|
58282
58383
|
const ownedRunIds = new Set;
|
|
58283
58384
|
const runs = activeStore.runs();
|
|
58385
|
+
const runById = new Map(runs.map((run) => [run.id, run]));
|
|
58386
|
+
const childrenByBoundary = new Map;
|
|
58387
|
+
for (const run of runs) {
|
|
58388
|
+
if (run.parentRunId === undefined || run.parentStageId === undefined)
|
|
58389
|
+
continue;
|
|
58390
|
+
const key = JSON.stringify([run.parentRunId, run.parentStageId]);
|
|
58391
|
+
const children = childrenByBoundary.get(key) ?? [];
|
|
58392
|
+
children.push(run);
|
|
58393
|
+
childrenByBoundary.set(key, children);
|
|
58394
|
+
}
|
|
58284
58395
|
const resolveBoundarySegments = createWorkflowBoundarySegmentsResolver(runs);
|
|
58285
58396
|
for (const run of runs) {
|
|
58286
|
-
if (!ownsRun(run.id))
|
|
58397
|
+
if (externalRunRouteOwner(activeStore, run.id) !== undefined || !ownsRun(run.id))
|
|
58287
58398
|
continue;
|
|
58288
|
-
const rootRunId =
|
|
58399
|
+
const rootRunId = reciprocalWorkflowRootRunId(runById, run.id);
|
|
58289
58400
|
if (rootRunId === undefined)
|
|
58290
58401
|
continue;
|
|
58291
58402
|
ownedRunIds.add(run.id);
|
|
58292
|
-
const parentRun =
|
|
58403
|
+
const parentRun = run.parentRunId === undefined ? undefined : runById.get(run.parentRunId);
|
|
58293
58404
|
const boundary = parentRun?.stages.find((stage) => stage.id === run.parentStageId);
|
|
58294
58405
|
const parent = parentRun === undefined || boundary === undefined ? undefined : {
|
|
58295
58406
|
runId: parentRun.id,
|
|
58296
|
-
stageKeys: [boundary.id, boundary.name].filter((key) =>
|
|
58407
|
+
stageKeys: [boundary.id, boundary.name].filter((key) => resolveIndexedChildRun(childrenByBoundary, parentRun, key)?.id === run.id)
|
|
58297
58408
|
};
|
|
58298
58409
|
const announcement = {
|
|
58299
58410
|
runId: run.id,
|
|
@@ -58434,16 +58545,22 @@ function registerPendingStageIntercomBridge(pi, activeStore, ownsRun = () => tru
|
|
|
58434
58545
|
payload.handled = true;
|
|
58435
58546
|
payload.completion = queueAndPersist(activeStore, resolvedEvent, workflowInvocationIntercomGroup(rootRunId), stage.pendingStageDeliveryAvailable === true);
|
|
58436
58547
|
};
|
|
58437
|
-
const
|
|
58438
|
-
const stickySubscription = pi.events?.on?.(STICKY_LIVE_DELIVERED_EVENT, (payload) => {
|
|
58548
|
+
const routeStickyLiveDelivered = (payload) => {
|
|
58439
58549
|
if (disposed || !isStickyLiveDeliveredEvent(payload) || payload.handled)
|
|
58440
58550
|
return;
|
|
58441
58551
|
payload.handled = true;
|
|
58442
58552
|
payload.completion = outsideWorkflowExecutions(() => recordConfirmedStickyDeliveries(activeStore, payload));
|
|
58553
|
+
};
|
|
58554
|
+
const subscription = pi.events?.on?.(PENDING_STAGE_MESSAGE_EVENT, (payload) => outsideWorkflowExecutions(() => routePendingStageMessage(payload)));
|
|
58555
|
+
const stickySubscription = pi.events?.on?.(STICKY_LIVE_DELIVERED_EVENT, routeStickyLiveDelivered);
|
|
58556
|
+
const disposeRouter = registerPendingStageMessageRouter(activeStore, {
|
|
58557
|
+
route: (payload) => outsideWorkflowExecutions(() => routePendingStageMessage(payload)),
|
|
58558
|
+
sticky: routeStickyLiveDelivered
|
|
58443
58559
|
});
|
|
58444
58560
|
const dispose = () => {
|
|
58445
58561
|
disposed = true;
|
|
58446
58562
|
disposeReadiness();
|
|
58563
|
+
disposeRouter();
|
|
58447
58564
|
routeCompletions.clear();
|
|
58448
58565
|
unsubscribeStore();
|
|
58449
58566
|
if (typeof subscription === "function")
|
|
@@ -58569,6 +58686,12 @@ function isMaterializedNonAgentTarget(runs, target) {
|
|
|
58569
58686
|
return false;
|
|
58570
58687
|
return nonAgentNodes(run).some((node) => node.id === stageKey || node.name === stageKey);
|
|
58571
58688
|
}
|
|
58689
|
+
function resolveIndexedChildRun(childrenByBoundary, parent, segment) {
|
|
58690
|
+
const boundariesById = parent.stages.filter((stage) => stage.id === segment);
|
|
58691
|
+
const boundaries = boundariesById.length > 0 ? boundariesById : parent.stages.filter((stage) => stage.name === segment);
|
|
58692
|
+
const children = boundaries.flatMap((boundary) => childrenByBoundary.get(JSON.stringify([parent.id, boundary.id])) ?? []);
|
|
58693
|
+
return children.length === 1 ? children[0] : undefined;
|
|
58694
|
+
}
|
|
58572
58695
|
function resolveChildRun(runs, parent, segment) {
|
|
58573
58696
|
const boundariesById = parent.stages.filter((stage) => stage.id === segment);
|
|
58574
58697
|
const boundaries = boundariesById.length > 0 ? boundariesById : parent.stages.filter((stage) => stage.name === segment);
|
|
@@ -58908,6 +59031,7 @@ var init_pending_stage_intercom = __esm(() => {
|
|
|
58908
59031
|
init_execution_context_boundary();
|
|
58909
59032
|
init_run_owner_backend();
|
|
58910
59033
|
init_store_backend();
|
|
59034
|
+
init_external_run_routes();
|
|
58911
59035
|
init_intercom_group();
|
|
58912
59036
|
init_pending_stage_route_capability();
|
|
58913
59037
|
init_pending_stage_route_readiness();
|
|
@@ -59047,7 +59171,8 @@ function buildSendSignature(to, options) {
|
|
|
59047
59171
|
attachments: canonicalizeAttachmentsForSendSignature(options.attachments) ?? [],
|
|
59048
59172
|
replyTo: options.replyTo ?? null,
|
|
59049
59173
|
expectsReply: options.expectsReply ?? false,
|
|
59050
|
-
replyError: options.replyError ?? null
|
|
59174
|
+
replyError: options.replyError ?? null,
|
|
59175
|
+
...options.delivery === undefined || options.delivery === "steer" ? {} : { delivery: options.delivery }
|
|
59051
59176
|
});
|
|
59052
59177
|
}
|
|
59053
59178
|
|
|
@@ -59233,6 +59358,8 @@ function isMessage(value2) {
|
|
|
59233
59358
|
return false;
|
|
59234
59359
|
if (message.expectsReply !== undefined && typeof message.expectsReply !== "boolean")
|
|
59235
59360
|
return false;
|
|
59361
|
+
if (message.delivery !== undefined && message.delivery !== "steer" && message.delivery !== "followUp" && message.delivery !== "interrupt")
|
|
59362
|
+
return false;
|
|
59236
59363
|
if (message.replyError !== undefined && typeof message.replyError !== "string")
|
|
59237
59364
|
return false;
|
|
59238
59365
|
if (message.source !== undefined && !isMessageSource(message.source))
|
|
@@ -59366,7 +59493,7 @@ function isWorkflowFutureStageRosterEntries(value2) {
|
|
|
59366
59493
|
function isWorkflowStageRosterEntries(value2) {
|
|
59367
59494
|
return Array.isArray(value2) && value2.every((entry) => typeof entry === "object" && entry !== null && entry.kind === "workflow-stage" && typeof entry.runId === "string" && typeof entry.stageId === "string" && typeof entry.stageName === "string" && typeof entry.target === "string" && (entry.lifecycle === "pending" || entry.lifecycle === "running") && typeof entry.group === "string" && (entry.sessionId === undefined || typeof entry.sessionId === "string"));
|
|
59368
59495
|
}
|
|
59369
|
-
var BROKER_SOCKET, BROKER_CONNECT_TIMEOUT_MS = 1e4, PENDING_STAGE_ROUTE_ACK_TIMEOUT_MS = 1e4, GROUP_REQUEST_TIMEOUT_MS = 5000, PRESENCE_ACK_TIMEOUT_MS = 5000, RESTORING_RECIPIENT_REQUEST_TIMEOUT_MS = 5000, HELD_SEND_GRACE_MS = 5000, IntercomClient;
|
|
59496
|
+
var BROKER_SOCKET, BROKER_CONNECT_TIMEOUT_MS = 1e4, PENDING_STAGE_ROUTE_ACK_TIMEOUT_MS = 1e4, GROUP_REQUEST_TIMEOUT_MS = 5000, PRESENCE_ACK_TIMEOUT_MS = 5000, RESTORING_RECIPIENT_REQUEST_TIMEOUT_MS = 5000, HELD_SEND_GRACE_MS = 5000, IntercomPendingStageRouteRefusedError, PENDING_STAGE_ROUTE_REFUSAL_CODES, IntercomClient;
|
|
59370
59497
|
var init_client = __esm(() => {
|
|
59371
59498
|
init_paths();
|
|
59372
59499
|
init_pending_send_registry();
|
|
@@ -59375,6 +59502,21 @@ var init_client = __esm(() => {
|
|
|
59375
59502
|
init_live_route_refusal();
|
|
59376
59503
|
init_retry_policy();
|
|
59377
59504
|
BROKER_SOCKET = getBrokerSocketPath();
|
|
59505
|
+
IntercomPendingStageRouteRefusedError = class IntercomPendingStageRouteRefusedError extends Error {
|
|
59506
|
+
connectionKept;
|
|
59507
|
+
code;
|
|
59508
|
+
constructor(reason, refusal = { connectionKept: false }) {
|
|
59509
|
+
super(reason);
|
|
59510
|
+
this.name = "IntercomPendingStageRouteRefusedError";
|
|
59511
|
+
this.connectionKept = refusal.connectionKept;
|
|
59512
|
+
this.code = refusal.code;
|
|
59513
|
+
}
|
|
59514
|
+
};
|
|
59515
|
+
PENDING_STAGE_ROUTE_REFUSAL_CODES = new Set([
|
|
59516
|
+
"not_authorized",
|
|
59517
|
+
"invalid_stage_roster",
|
|
59518
|
+
"invalid_possible_stage_roster"
|
|
59519
|
+
]);
|
|
59378
59520
|
IntercomClient = class IntercomClient extends EventEmitter {
|
|
59379
59521
|
returnAddress;
|
|
59380
59522
|
preferredSessionId;
|
|
@@ -59382,6 +59524,8 @@ var init_client = __esm(() => {
|
|
|
59382
59524
|
connectTimeoutMs;
|
|
59383
59525
|
brokerAcknowledgesReceipt = false;
|
|
59384
59526
|
brokerAcknowledgesPendingStageRoutes = false;
|
|
59527
|
+
multiplexesPendingStageRoutes;
|
|
59528
|
+
brokerRefusesPendingStageRoutesInPlace = false;
|
|
59385
59529
|
brokerAnswersStatus = false;
|
|
59386
59530
|
socket = null;
|
|
59387
59531
|
_sessionId = null;
|
|
@@ -59447,6 +59591,7 @@ var init_client = __esm(() => {
|
|
|
59447
59591
|
this.preferredSessionId = options.preferredSessionId;
|
|
59448
59592
|
this.acknowledgesReceipt = options.acknowledgesReceipt === true;
|
|
59449
59593
|
this.connectTimeoutMs = options.connectTimeoutMs ?? BROKER_CONNECT_TIMEOUT_MS;
|
|
59594
|
+
this.multiplexesPendingStageRoutes = options.multiplexesPendingStageRoutes === true;
|
|
59450
59595
|
}
|
|
59451
59596
|
failPending(error) {
|
|
59452
59597
|
this.pendingSends.rejectAll(error);
|
|
@@ -59559,6 +59704,7 @@ var init_client = __esm(() => {
|
|
|
59559
59704
|
this._supervisorSessionId = null;
|
|
59560
59705
|
this.brokerAcknowledgesReceipt = false;
|
|
59561
59706
|
this.brokerAcknowledgesPendingStageRoutes = false;
|
|
59707
|
+
this.brokerRefusesPendingStageRoutesInPlace = false;
|
|
59562
59708
|
this.brokerAnswersStatus = false;
|
|
59563
59709
|
this._messageSource = undefined;
|
|
59564
59710
|
this.disconnectError = null;
|
|
@@ -59637,7 +59783,7 @@ var init_client = __esm(() => {
|
|
|
59637
59783
|
}
|
|
59638
59784
|
switch (brokerMessage.type) {
|
|
59639
59785
|
case "registered": {
|
|
59640
|
-
if (typeof brokerMessage.sessionId !== "string" || brokerMessage.supervisorSessionId !== undefined && typeof brokerMessage.supervisorSessionId !== "string" || brokerMessage.acknowledgesReceipt !== undefined && brokerMessage.acknowledgesReceipt !== true || brokerMessage.acknowledgesPendingStageRoutes !== undefined && brokerMessage.acknowledgesPendingStageRoutes !== true || brokerMessage.brokerStatus !== undefined && brokerMessage.brokerStatus !== true) {
|
|
59786
|
+
if (typeof brokerMessage.sessionId !== "string" || brokerMessage.supervisorSessionId !== undefined && typeof brokerMessage.supervisorSessionId !== "string" || brokerMessage.acknowledgesReceipt !== undefined && brokerMessage.acknowledgesReceipt !== true || brokerMessage.acknowledgesPendingStageRoutes !== undefined && brokerMessage.acknowledgesPendingStageRoutes !== true || brokerMessage.refusesPendingStageRoutesInPlace !== undefined && brokerMessage.refusesPendingStageRoutesInPlace !== true || brokerMessage.brokerStatus !== undefined && brokerMessage.brokerStatus !== true) {
|
|
59641
59787
|
throw new Error("Invalid registered message");
|
|
59642
59788
|
}
|
|
59643
59789
|
if (this._sessionId !== null) {
|
|
@@ -59647,6 +59793,7 @@ var init_client = __esm(() => {
|
|
|
59647
59793
|
this._supervisorSessionId = typeof brokerMessage.supervisorSessionId === "string" ? brokerMessage.supervisorSessionId : null;
|
|
59648
59794
|
this.brokerAcknowledgesReceipt = this.acknowledgesReceipt && brokerMessage.acknowledgesReceipt === true;
|
|
59649
59795
|
this.brokerAcknowledgesPendingStageRoutes = brokerMessage.acknowledgesPendingStageRoutes === true;
|
|
59796
|
+
this.brokerRefusesPendingStageRoutesInPlace = brokerMessage.refusesPendingStageRoutesInPlace === true;
|
|
59650
59797
|
this.brokerAnswersStatus = brokerMessage.brokerStatus === true;
|
|
59651
59798
|
this.emit("_registered", { type: "registered", sessionId: brokerMessage.sessionId });
|
|
59652
59799
|
break;
|
|
@@ -59659,6 +59806,12 @@ var init_client = __esm(() => {
|
|
|
59659
59806
|
this.emit("_registration_failed", refusal);
|
|
59660
59807
|
break;
|
|
59661
59808
|
}
|
|
59809
|
+
const refusedRouteId = typeof brokerMessage.requestId === "string" ? brokerMessage.requestId : "";
|
|
59810
|
+
const refusedRoute = this.pendingStageRouteRegistrations.get(refusedRouteId);
|
|
59811
|
+
if (refusedRoute !== undefined) {
|
|
59812
|
+
this.pendingStageRouteRegistrations.delete(refusedRouteId);
|
|
59813
|
+
refusedRoute.reject(new IntercomPendingStageRouteRefusedError(brokerMessage.reason));
|
|
59814
|
+
}
|
|
59662
59815
|
this.disconnectError ??= refusal;
|
|
59663
59816
|
this.failPending(this.disconnectError);
|
|
59664
59817
|
this.socket?.destroy();
|
|
@@ -59784,6 +59937,21 @@ var init_client = __esm(() => {
|
|
|
59784
59937
|
this.emit("pending_stage_route_revoked", { runId });
|
|
59785
59938
|
break;
|
|
59786
59939
|
}
|
|
59940
|
+
case "pending_stage_route_refused": {
|
|
59941
|
+
const { requestId, runId, code, reason } = brokerMessage;
|
|
59942
|
+
if (typeof requestId !== "string" || typeof runId !== "string" || typeof reason !== "string" || typeof code !== "string") {
|
|
59943
|
+
throw new Error("Invalid pending-stage route refusal");
|
|
59944
|
+
}
|
|
59945
|
+
const pending = this.pendingStageRouteRegistrations.get(requestId);
|
|
59946
|
+
if (!pending)
|
|
59947
|
+
return;
|
|
59948
|
+
this.pendingStageRouteRegistrations.delete(requestId);
|
|
59949
|
+
pending.reject(new IntercomPendingStageRouteRefusedError(reason, {
|
|
59950
|
+
connectionKept: true,
|
|
59951
|
+
...PENDING_STAGE_ROUTE_REFUSAL_CODES.has(code) ? { code } : {}
|
|
59952
|
+
}));
|
|
59953
|
+
break;
|
|
59954
|
+
}
|
|
59787
59955
|
case "pending_stage_route_registered": {
|
|
59788
59956
|
const { requestId } = brokerMessage;
|
|
59789
59957
|
if (typeof requestId !== "string")
|
|
@@ -60287,7 +60455,8 @@ var init_client = __esm(() => {
|
|
|
60287
60455
|
stages,
|
|
60288
60456
|
possibleStages,
|
|
60289
60457
|
...parent === undefined ? {} : { parent },
|
|
60290
|
-
...dormant === true ? { dormant: true } : {}
|
|
60458
|
+
...dormant === true ? { dormant: true } : {},
|
|
60459
|
+
...this.multiplexesPendingStageRoutes && this.brokerRefusesPendingStageRoutesInPlace ? { refusalKeepsConnection: true } : {}
|
|
60291
60460
|
});
|
|
60292
60461
|
} catch (error) {
|
|
60293
60462
|
clearTimeout(timeout);
|
|
@@ -60400,6 +60569,7 @@ var init_client = __esm(() => {
|
|
|
60400
60569
|
timestamp: Date.now(),
|
|
60401
60570
|
replyTo: options.replyTo,
|
|
60402
60571
|
expectsReply: options.expectsReply,
|
|
60572
|
+
...options.delivery === undefined || options.delivery === "steer" ? {} : { delivery: options.delivery },
|
|
60403
60573
|
replyError: options.replyError,
|
|
60404
60574
|
source: this._messageSource,
|
|
60405
60575
|
content: { text: options.text, attachments: options.attachments }
|
|
@@ -61021,120 +61191,165 @@ var init_reconnect_backoff = __esm(() => {
|
|
|
61021
61191
|
BACKOFF_MS = [1000, 2000, 5000, 1e4, 30000];
|
|
61022
61192
|
});
|
|
61023
61193
|
|
|
61024
|
-
// dist/builtin/intercom/route-connect-gate.ts
|
|
61025
|
-
function createConcurrencyGate(limit) {
|
|
61026
|
-
let active = 0;
|
|
61027
|
-
const waiting = [];
|
|
61028
|
-
const release = () => {
|
|
61029
|
-
const next = waiting.shift();
|
|
61030
|
-
if (next === undefined)
|
|
61031
|
-
active -= 1;
|
|
61032
|
-
else
|
|
61033
|
-
next();
|
|
61034
|
-
};
|
|
61035
|
-
return {
|
|
61036
|
-
async run(operation) {
|
|
61037
|
-
if (active < limit)
|
|
61038
|
-
active += 1;
|
|
61039
|
-
else
|
|
61040
|
-
await new Promise((resolve) => waiting.push(resolve));
|
|
61041
|
-
try {
|
|
61042
|
-
return await operation();
|
|
61043
|
-
} finally {
|
|
61044
|
-
release();
|
|
61045
|
-
}
|
|
61046
|
-
}
|
|
61047
|
-
};
|
|
61048
|
-
}
|
|
61049
|
-
var ROUTE_CLIENT_CONNECT_CONCURRENCY = 4;
|
|
61050
|
-
|
|
61051
61194
|
// dist/builtin/intercom/workflow-route-owner.ts
|
|
61052
61195
|
import { randomUUID as randomUUID14 } from "node:crypto";
|
|
61053
61196
|
import { createEventBus, getBuiltinPackagePaths } from "@bastani/atomic";
|
|
61197
|
+
function closedError() {
|
|
61198
|
+
return new Error("Workflow Intercom route owner closed");
|
|
61199
|
+
}
|
|
61054
61200
|
function createWorkflowRouteOwner(cwd) {
|
|
61055
61201
|
const events = createEventBus();
|
|
61056
61202
|
const config = loadConfig();
|
|
61057
61203
|
const id = randomUUID14();
|
|
61058
61204
|
const routes = new Map;
|
|
61059
|
-
const
|
|
61205
|
+
const connections = new Map;
|
|
61206
|
+
const isCurrent = (state) => routes.get(state.route.runId) === state;
|
|
61207
|
+
const connectionKey = (state) => state.isolated ? state.route.runId : state.route.group;
|
|
61208
|
+
function connectionOf(state) {
|
|
61209
|
+
const key = connectionKey(state);
|
|
61210
|
+
let connection = connections.get(key);
|
|
61211
|
+
if (connection === undefined) {
|
|
61212
|
+
connection = { key, group: state.route.group, acknowledged: false, routes: new Map };
|
|
61213
|
+
connections.set(key, connection);
|
|
61214
|
+
}
|
|
61215
|
+
connection.routes.set(state.route.runId, state);
|
|
61216
|
+
return connection;
|
|
61217
|
+
}
|
|
61218
|
+
function detach(state) {
|
|
61219
|
+
const key = connectionKey(state);
|
|
61220
|
+
const connection = connections.get(key);
|
|
61221
|
+
connection?.routes.delete(state.route.runId);
|
|
61222
|
+
if (!state.isolated || connection === undefined)
|
|
61223
|
+
return;
|
|
61224
|
+
connections.delete(key);
|
|
61225
|
+
connection.client?.disconnect().catch(() => {});
|
|
61226
|
+
}
|
|
61227
|
+
function settle(state, registered) {
|
|
61228
|
+
state.registered = registered;
|
|
61229
|
+
if (!registered && isCurrent(state))
|
|
61230
|
+
return;
|
|
61231
|
+
for (const waiter of state.waiters)
|
|
61232
|
+
waiter(registered);
|
|
61233
|
+
state.waiters.clear();
|
|
61234
|
+
}
|
|
61060
61235
|
function scheduleRetry(state) {
|
|
61061
|
-
if (state
|
|
61236
|
+
if (!isCurrent(state) || state.retry !== undefined)
|
|
61062
61237
|
return;
|
|
61063
61238
|
state.retry = setTimeout(() => {
|
|
61064
61239
|
state.retry = undefined;
|
|
61065
|
-
publish(state, state.route).catch(() =>
|
|
61240
|
+
publish(state, state.route).catch(() => {});
|
|
61066
61241
|
}, jitteredReconnectDelayMs(state.attempt++));
|
|
61067
61242
|
state.retry.unref();
|
|
61068
61243
|
}
|
|
61069
|
-
|
|
61070
|
-
if (
|
|
61071
|
-
|
|
61072
|
-
|
|
61073
|
-
|
|
61074
|
-
|
|
61075
|
-
|
|
61076
|
-
|
|
61077
|
-
}
|
|
61078
|
-
|
|
61079
|
-
|
|
61080
|
-
|
|
61081
|
-
if (
|
|
61082
|
-
|
|
61083
|
-
|
|
61084
|
-
|
|
61085
|
-
|
|
61086
|
-
|
|
61087
|
-
|
|
61088
|
-
|
|
61089
|
-
|
|
61090
|
-
|
|
61091
|
-
|
|
61092
|
-
|
|
61093
|
-
if (
|
|
61094
|
-
|
|
61095
|
-
const
|
|
61096
|
-
|
|
61097
|
-
|
|
61098
|
-
|
|
61099
|
-
|
|
61100
|
-
|
|
61101
|
-
|
|
61102
|
-
|
|
61103
|
-
|
|
61104
|
-
|
|
61105
|
-
|
|
61106
|
-
|
|
61107
|
-
|
|
61108
|
-
|
|
61109
|
-
|
|
61110
|
-
|
|
61111
|
-
|
|
61112
|
-
|
|
61113
|
-
|
|
61244
|
+
function drop(connection, client) {
|
|
61245
|
+
if (connection.client === client) {
|
|
61246
|
+
connection.client = undefined;
|
|
61247
|
+
connection.acknowledged = false;
|
|
61248
|
+
for (const state of connection.routes.values()) {
|
|
61249
|
+
settle(state, false);
|
|
61250
|
+
scheduleRetry(state);
|
|
61251
|
+
}
|
|
61252
|
+
}
|
|
61253
|
+
client.disconnect().catch(() => {});
|
|
61254
|
+
}
|
|
61255
|
+
function connect(connection) {
|
|
61256
|
+
if (connection.client?.isConnected())
|
|
61257
|
+
return Promise.resolve(connection.client);
|
|
61258
|
+
connection.connecting ??= (async () => {
|
|
61259
|
+
const [extensionDir] = getBuiltinPackagePaths({
|
|
61260
|
+
workflows: false,
|
|
61261
|
+
subagents: false,
|
|
61262
|
+
mcp: false,
|
|
61263
|
+
"web-access": false
|
|
61264
|
+
});
|
|
61265
|
+
if (extensionDir === undefined)
|
|
61266
|
+
throw new Error("Intercom builtin package is unavailable");
|
|
61267
|
+
await spawnBrokerIfNeeded(config.brokerCommand, config.brokerArgs, undefined, extensionDir);
|
|
61268
|
+
if (connections.get(connection.key) !== connection)
|
|
61269
|
+
throw closedError();
|
|
61270
|
+
const client = new IntercomClient(`${id}:pending-stage-route:${connection.key}`, {
|
|
61271
|
+
multiplexesPendingStageRoutes: true
|
|
61272
|
+
});
|
|
61273
|
+
connection.client = client;
|
|
61274
|
+
connection.acknowledged = false;
|
|
61275
|
+
client.on("error", () => {});
|
|
61276
|
+
client.on("disconnected", () => drop(connection, client));
|
|
61277
|
+
client.on("pending_stage_message", (request) => {
|
|
61278
|
+
if (connection.client !== client)
|
|
61279
|
+
return;
|
|
61280
|
+
const event = { ...request, handled: false };
|
|
61281
|
+
events.emit("atomic:workflow-pending-stage-message", event);
|
|
61282
|
+
const completion = event.handled && event.completion !== undefined ? event.completion : Promise.resolve({ outcome: "refused", reason: "Session not found" });
|
|
61283
|
+
completion.catch((error) => ({ outcome: "refused", reason: error.message })).then((result) => client.respondPendingStageMessage(request.requestId, result)).catch(() => {});
|
|
61284
|
+
});
|
|
61285
|
+
client.on("sticky_live_delivered", (notice) => {
|
|
61286
|
+
if (connection.client !== client)
|
|
61287
|
+
return;
|
|
61288
|
+
events.emit("atomic:workflow-sticky-live-delivered", { ...notice, handled: false });
|
|
61289
|
+
});
|
|
61290
|
+
try {
|
|
61291
|
+
await client.connect({
|
|
61292
|
+
cwd,
|
|
61293
|
+
model: "workflow",
|
|
61294
|
+
pid: process.pid,
|
|
61295
|
+
startedAt: Date.now(),
|
|
61296
|
+
lastActivity: Date.now(),
|
|
61297
|
+
group: connection.group,
|
|
61298
|
+
groups: [connection.group],
|
|
61299
|
+
recipientPurpose: "control"
|
|
61300
|
+
});
|
|
61301
|
+
if (connections.get(connection.key) !== connection)
|
|
61302
|
+
throw closedError();
|
|
61303
|
+
} catch (error) {
|
|
61304
|
+
drop(connection, client);
|
|
61305
|
+
throw error;
|
|
61306
|
+
}
|
|
61307
|
+
for (const state of connection.routes.values()) {
|
|
61308
|
+
if (state.registered || state.retry === undefined)
|
|
61309
|
+
continue;
|
|
61310
|
+
clearTimeout(state.retry);
|
|
61311
|
+
state.retry = undefined;
|
|
61312
|
+
publish(state, state.route).catch(() => {});
|
|
61313
|
+
}
|
|
61314
|
+
return client;
|
|
61315
|
+
})().finally(() => {
|
|
61316
|
+
connection.connecting = undefined;
|
|
61114
61317
|
});
|
|
61115
|
-
return
|
|
61318
|
+
return connection.connecting;
|
|
61116
61319
|
}
|
|
61117
61320
|
function publish(state, route) {
|
|
61118
|
-
const completion = state.tail.catch(() => {}).then(
|
|
61119
|
-
if (state
|
|
61120
|
-
throw
|
|
61121
|
-
const
|
|
61321
|
+
const completion = state.tail.catch(() => {}).then(async () => {
|
|
61322
|
+
if (!isCurrent(state))
|
|
61323
|
+
throw closedError();
|
|
61324
|
+
const connection = connectionOf(state);
|
|
61325
|
+
const established = connection.acknowledged && connection.client?.isConnected() ? connection.client : undefined;
|
|
61326
|
+
let client;
|
|
61122
61327
|
try {
|
|
61123
|
-
|
|
61328
|
+
client = await connect(connection);
|
|
61124
61329
|
await client.registerPendingStageRouteAcknowledged(route.runId, route.group, route.capability, route.stages, route.possibleStages, route.parent);
|
|
61330
|
+
if (connection.client !== client || !isCurrent(state))
|
|
61331
|
+
return;
|
|
61332
|
+
connection.acknowledged = true;
|
|
61125
61333
|
state.attempt = 0;
|
|
61126
61334
|
clearTimeout(state.retry);
|
|
61127
61335
|
state.retry = undefined;
|
|
61336
|
+
settle(state, true);
|
|
61128
61337
|
} catch (error) {
|
|
61129
|
-
|
|
61130
|
-
|
|
61131
|
-
|
|
61132
|
-
await client?.disconnect();
|
|
61338
|
+
if (error instanceof IntercomPendingStageRouteRefusedError && error.connectionKept) {
|
|
61339
|
+
scheduleRetry(state);
|
|
61340
|
+
throw error;
|
|
61133
61341
|
}
|
|
61342
|
+
if (error instanceof IntercomPendingStageRouteRefusedError && !state.isolated && isCurrent(state)) {
|
|
61343
|
+
detach(state);
|
|
61344
|
+
state.isolated = true;
|
|
61345
|
+
connectionOf(state);
|
|
61346
|
+
}
|
|
61347
|
+
if (client !== undefined && (client !== established || !client.isConnected()))
|
|
61348
|
+
drop(connection, client);
|
|
61134
61349
|
scheduleRetry(state);
|
|
61135
61350
|
throw error;
|
|
61136
61351
|
}
|
|
61137
|
-
})
|
|
61352
|
+
});
|
|
61138
61353
|
state.tail = completion;
|
|
61139
61354
|
completion.catch(() => {});
|
|
61140
61355
|
return completion;
|
|
@@ -61143,10 +61358,15 @@ function createWorkflowRouteOwner(cwd) {
|
|
|
61143
61358
|
const route = payload;
|
|
61144
61359
|
let state = routes.get(route.runId);
|
|
61145
61360
|
if (state === undefined) {
|
|
61146
|
-
state = { route, tail: Promise.resolve(), attempt: 0,
|
|
61361
|
+
state = { route, tail: Promise.resolve(), attempt: 0, registered: false, isolated: false, waiters: new Set };
|
|
61147
61362
|
routes.set(route.runId, state);
|
|
61363
|
+
} else if (state.route.group !== route.group) {
|
|
61364
|
+
detach(state);
|
|
61365
|
+
state.isolated = false;
|
|
61366
|
+
settle(state, false);
|
|
61148
61367
|
}
|
|
61149
61368
|
state.route = route;
|
|
61369
|
+
connectionOf(state);
|
|
61150
61370
|
route.completion = publish(state, route);
|
|
61151
61371
|
});
|
|
61152
61372
|
events.on("atomic:workflow-pending-stage-undeliverable", (payload) => {
|
|
@@ -61157,7 +61377,7 @@ function createWorkflowRouteOwner(cwd) {
|
|
|
61157
61377
|
event.handled = true;
|
|
61158
61378
|
event.completion = (async () => {
|
|
61159
61379
|
await publish(state, state.route);
|
|
61160
|
-
const client = state
|
|
61380
|
+
const client = connections.get(connectionKey(state))?.client;
|
|
61161
61381
|
if (client === undefined || !client.isConnected())
|
|
61162
61382
|
return false;
|
|
61163
61383
|
const text = `Pending workflow stage could not receive intercom message: ${event.reason}`;
|
|
@@ -61167,18 +61387,33 @@ function createWorkflowRouteOwner(cwd) {
|
|
|
61167
61387
|
});
|
|
61168
61388
|
return {
|
|
61169
61389
|
events,
|
|
61390
|
+
registered(runId) {
|
|
61391
|
+
const state = routes.get(runId);
|
|
61392
|
+
if (state === undefined)
|
|
61393
|
+
return Promise.resolve(false);
|
|
61394
|
+
if (state.registered)
|
|
61395
|
+
return Promise.resolve(true);
|
|
61396
|
+
return new Promise((resolve) => state.waiters.add(resolve));
|
|
61397
|
+
},
|
|
61170
61398
|
async closeGroup(group) {
|
|
61171
61399
|
const closing = [...routes.values()].filter((state) => state.route.group === group);
|
|
61400
|
+
const closingConnections = [...connections.values()].filter((connection) => connection.group === group);
|
|
61401
|
+
for (const connection of closingConnections) {
|
|
61402
|
+
connections.delete(connection.key);
|
|
61403
|
+
connection.routes.clear();
|
|
61404
|
+
}
|
|
61172
61405
|
for (const state of closing) {
|
|
61173
|
-
state.closed = true;
|
|
61174
|
-
clearTimeout(state.retry);
|
|
61175
61406
|
routes.delete(state.route.runId);
|
|
61407
|
+
clearTimeout(state.retry);
|
|
61408
|
+
state.retry = undefined;
|
|
61409
|
+
settle(state, false);
|
|
61176
61410
|
}
|
|
61177
|
-
await Promise.all(
|
|
61178
|
-
await
|
|
61179
|
-
await
|
|
61180
|
-
await state.client?.disconnect();
|
|
61411
|
+
await Promise.all(closingConnections.map(async (connection) => {
|
|
61412
|
+
await connection.client?.disconnect();
|
|
61413
|
+
await connection.connecting?.catch(() => {});
|
|
61181
61414
|
}));
|
|
61415
|
+
await Promise.all(closing.map((state) => state.tail.catch(() => {})));
|
|
61416
|
+
await Promise.all(closingConnections.map((connection) => connection.client?.disconnect()));
|
|
61182
61417
|
}
|
|
61183
61418
|
};
|
|
61184
61419
|
}
|
|
@@ -61297,7 +61532,7 @@ var init_workflow_infrastructure_error = __esm(() => {
|
|
|
61297
61532
|
[/Local worker (?:process )?cleanup (?:was not confirmed|unconfirmed)/, "worker_cleanup_unconfirmed"],
|
|
61298
61533
|
[/Local worker retry limit/, "worker_retry_limit"],
|
|
61299
61534
|
[/Local worker (?:startup timed out|exited before readiness)/, "worker_startup_failed"],
|
|
61300
|
-
[/Worker exited before delivery acknowledgement/, "worker_lost"],
|
|
61535
|
+
[/Worker exited before delivery acknowledgement|Local worker pool coordinator exited/, "worker_lost"],
|
|
61301
61536
|
[/Stale work lease/, "stale_work_lease"],
|
|
61302
61537
|
[/Intercom runtime no longer active/, "intercom_unavailable"],
|
|
61303
61538
|
[/Storage unavailable/, "storage_unavailable"]
|
|
@@ -61313,12 +61548,925 @@ var init_workflow_infrastructure_error = __esm(() => {
|
|
|
61313
61548
|
};
|
|
61314
61549
|
});
|
|
61315
61550
|
|
|
61551
|
+
// dist/builtin/workflows/src/workers/coordinator-protocol.ts
|
|
61552
|
+
var POOL_COORDINATOR_ROLE = "pool-coordinator", POOL_SUMMARY_INTERVAL_MS = 250;
|
|
61553
|
+
|
|
61316
61554
|
// dist/builtin/workflows/src/workers/host-options.ts
|
|
61317
61555
|
var hostWorkerBindings;
|
|
61318
61556
|
var init_host_options = __esm(() => {
|
|
61319
61557
|
hostWorkerBindings = Symbol("host-worker-bindings");
|
|
61320
61558
|
});
|
|
61321
61559
|
|
|
61560
|
+
// dist/builtin/workflows/src/durable/journal/sql-pool-usage.ts
|
|
61561
|
+
function validate(report) {
|
|
61562
|
+
validateText("key", report.key);
|
|
61563
|
+
if (report.key === "")
|
|
61564
|
+
throw new Error("Pool usage key must not be empty");
|
|
61565
|
+
for (const field of POOL_USAGE_FIELDS) {
|
|
61566
|
+
const value2 = report.usage[field];
|
|
61567
|
+
if (typeof value2 !== "number" || !Number.isFinite(value2) || value2 < 0)
|
|
61568
|
+
throw new Error(`Pool usage ${field} must be a finite non-negative number`);
|
|
61569
|
+
}
|
|
61570
|
+
}
|
|
61571
|
+
|
|
61572
|
+
class SqlPoolUsage {
|
|
61573
|
+
transaction;
|
|
61574
|
+
journal;
|
|
61575
|
+
table;
|
|
61576
|
+
constructor(transaction, journal) {
|
|
61577
|
+
this.transaction = transaction;
|
|
61578
|
+
this.journal = journal;
|
|
61579
|
+
this.table = poolUsageTable(journal.runs);
|
|
61580
|
+
}
|
|
61581
|
+
static async open(transaction, journal) {
|
|
61582
|
+
const usage = new SqlPoolUsage(transaction, journal);
|
|
61583
|
+
await transaction((tx) => tx.exec(poolUsageSchema(journal.runs)));
|
|
61584
|
+
return usage;
|
|
61585
|
+
}
|
|
61586
|
+
async totalIn(tx, root) {
|
|
61587
|
+
const row = await tx.get(`SELECT ${POOL_USAGE_FIELDS.map((field) => `COALESCE(SUM(u.${field.toLowerCase()}), 0) AS ${field}`).join(", ")}
|
|
61588
|
+
FROM ${this.table} u JOIN ${this.journal.runs}_work w ON w.id = u.work WHERE w.root = ?`, root);
|
|
61589
|
+
return Object.fromEntries(POOL_USAGE_FIELDS.map((field) => [field, Number(row?.[field] ?? 0)]));
|
|
61590
|
+
}
|
|
61591
|
+
async report(lease, token, reports) {
|
|
61592
|
+
validateText("runId", lease.runId);
|
|
61593
|
+
validateText("holderId", lease.holderId);
|
|
61594
|
+
for (const report of reports)
|
|
61595
|
+
validate(report);
|
|
61596
|
+
const snapshot = reports.map((report) => ({ key: report.key, usage: { ...report.usage } }));
|
|
61597
|
+
return this.transaction(async (tx) => {
|
|
61598
|
+
const authority = await tx.get(`SELECT l.epoch, l.holder_id, l.renewed_at, p.timeout, ${STORE_NOW3} AS now FROM ${this.journal.leases} l
|
|
61599
|
+
LEFT JOIN ${this.journal.runs}_pool p ON p.root = l.run_id WHERE l.run_id = ?`, lease.runId);
|
|
61600
|
+
if (!authority || authority.timeout === null || authority.epoch !== lease.epoch || authority.holder_id !== lease.holderId || isWorkflowLeaseExpired(Number(authority.renewed_at), Number(authority.timeout), Number(authority.now)))
|
|
61601
|
+
throw new StaleLeaseError(lease.runId, lease.epoch, Number(authority?.epoch ?? 0));
|
|
61602
|
+
const work = await tx.get(`SELECT * FROM ${this.journal.runs}_work WHERE id = ? AND root = ?`, token.id, lease.runId);
|
|
61603
|
+
if (work?.state !== "leased" && work?.state !== "completed" || work.epoch !== token.epoch || work.epoch !== lease.epoch || work.generation !== token.generation || work.worker !== token.worker || work.state === "leased" && Number(work.expires_at) < Number(authority.now))
|
|
61604
|
+
throw new Error("Stale work lease");
|
|
61605
|
+
for (const { key, usage } of snapshot)
|
|
61606
|
+
await upsertPoolUsage(tx, this.journal.runs, token.id, token.generation, key, usage);
|
|
61607
|
+
return this.totalIn(tx, lease.runId);
|
|
61608
|
+
});
|
|
61609
|
+
}
|
|
61610
|
+
total(root) {
|
|
61611
|
+
return this.transaction((tx) => this.totalIn(tx, root));
|
|
61612
|
+
}
|
|
61613
|
+
}
|
|
61614
|
+
async function openSqlPoolUsage(journal) {
|
|
61615
|
+
const storage = sqlPoolStorage(journal);
|
|
61616
|
+
if (!storage)
|
|
61617
|
+
throw new Error("Worker pool requires a SQL run journal");
|
|
61618
|
+
await openSqlPoolQueue(journal);
|
|
61619
|
+
return SqlPoolUsage.open(storage.transaction, storage.tables);
|
|
61620
|
+
}
|
|
61621
|
+
var STORE_NOW3 = "CAST(unixepoch('subsec') * 1000 AS INTEGER)";
|
|
61622
|
+
var init_sql_pool_usage = __esm(() => {
|
|
61623
|
+
init_internal();
|
|
61624
|
+
init_pool_stage_usage();
|
|
61625
|
+
init_sql_pool_queue();
|
|
61626
|
+
init_types();
|
|
61627
|
+
init_values();
|
|
61628
|
+
});
|
|
61629
|
+
|
|
61630
|
+
// dist/builtin/workflows/src/workers/pool-coordinator.ts
|
|
61631
|
+
import { fork } from "node:child_process";
|
|
61632
|
+
import { randomUUID as randomUUID15 } from "node:crypto";
|
|
61633
|
+
import { createWriteStream } from "node:fs";
|
|
61634
|
+
import { mkdir as mkdir6, rm as rm3, rmdir } from "node:fs/promises";
|
|
61635
|
+
import { dirname as dirname12, join as join28, resolve as resolve11 } from "node:path";
|
|
61636
|
+
import { setTimeout as delay2 } from "node:timers/promises";
|
|
61637
|
+
function workerLogDirectory(journalPath, root) {
|
|
61638
|
+
return join28(dirname12(resolve11(journalPath)), "worker-logs", encodeURIComponent(root).replaceAll(".", "%2E"));
|
|
61639
|
+
}
|
|
61640
|
+
function token(work) {
|
|
61641
|
+
return {
|
|
61642
|
+
id: work.id,
|
|
61643
|
+
generation: work.generation,
|
|
61644
|
+
worker: work.worker,
|
|
61645
|
+
epoch: work.epoch,
|
|
61646
|
+
renewedAt: work.renewedAt,
|
|
61647
|
+
expiresAt: work.expiresAt
|
|
61648
|
+
};
|
|
61649
|
+
}
|
|
61650
|
+
function outside(runId) {
|
|
61651
|
+
return new Error(`Pool stage publication is outside the work subtree: ${runId}`);
|
|
61652
|
+
}
|
|
61653
|
+
function isLiveStage(stage) {
|
|
61654
|
+
return (stage.sessionId !== undefined || stage.sessionFile !== undefined) && stage.status !== "completed" && stage.status !== "failed" && stage.status !== "skipped";
|
|
61655
|
+
}
|
|
61656
|
+
function projectRootRun(run) {
|
|
61657
|
+
return {
|
|
61658
|
+
id: run.id,
|
|
61659
|
+
name: run.name,
|
|
61660
|
+
status: run.status,
|
|
61661
|
+
startedAt: run.startedAt,
|
|
61662
|
+
stages: run.stages.map((stage) => ({
|
|
61663
|
+
id: stage.id,
|
|
61664
|
+
name: stage.name,
|
|
61665
|
+
status: stage.status,
|
|
61666
|
+
parentIds: [],
|
|
61667
|
+
toolEvents: [],
|
|
61668
|
+
...stage.replayKey === undefined ? {} : { replayKey: stage.replayKey },
|
|
61669
|
+
...stage.intercomGroup === undefined ? {} : { intercomGroup: stage.intercomGroup },
|
|
61670
|
+
...stage.workflowChildRun === undefined ? {} : { workflowChildRun: stage.workflowChildRun },
|
|
61671
|
+
...stage.workflowChild === undefined ? {} : { workflowChild: { ...stage.workflowChild, outputs: {} } }
|
|
61672
|
+
}))
|
|
61673
|
+
};
|
|
61674
|
+
}
|
|
61675
|
+
|
|
61676
|
+
class PooledRoutes {
|
|
61677
|
+
root;
|
|
61678
|
+
owner;
|
|
61679
|
+
mirror = createStore();
|
|
61680
|
+
owners = new Map;
|
|
61681
|
+
attempts = new Map;
|
|
61682
|
+
activated = false;
|
|
61683
|
+
bridge;
|
|
61684
|
+
constructor(root, owner) {
|
|
61685
|
+
this.root = root;
|
|
61686
|
+
this.owner = owner;
|
|
61687
|
+
}
|
|
61688
|
+
static async open(root, cwd, rootRun, forward) {
|
|
61689
|
+
await Promise.resolve().then(() => init_workflow_route_owner());
|
|
61690
|
+
const owner = createWorkflowRouteOwner(cwd);
|
|
61691
|
+
const routes = new PooledRoutes(root, owner);
|
|
61692
|
+
if (rootRun !== undefined)
|
|
61693
|
+
routes.updateRoot(rootRun);
|
|
61694
|
+
routes.bridge = registerPendingStageIntercomBridge({
|
|
61695
|
+
events: {
|
|
61696
|
+
emit: (event, payload) => owner.events.emit(event, payload),
|
|
61697
|
+
on: (event, listener) => event === PENDING_STAGE_MESSAGE_EVENT2 || event === STICKY_LIVE_DELIVERED_EVENT2 ? undefined : owner.events.on(event, listener)
|
|
61698
|
+
}
|
|
61699
|
+
}, routes.mirror, (runId) => routes.activated && routes.owners.has(runId));
|
|
61700
|
+
for (const [event, kind] of [
|
|
61701
|
+
[PENDING_STAGE_MESSAGE_EVENT2, "pending-stage-message"],
|
|
61702
|
+
[STICKY_LIVE_DELIVERED_EVENT2, "sticky-live-delivered"]
|
|
61703
|
+
])
|
|
61704
|
+
owner.events.on(event, (payload) => {
|
|
61705
|
+
const claimed = payload;
|
|
61706
|
+
if (claimed.handled !== false)
|
|
61707
|
+
return;
|
|
61708
|
+
claimed.handled = true;
|
|
61709
|
+
const { handled: _handled, completion: _completion, ...forwarded } = claimed;
|
|
61710
|
+
claimed.completion = forward({ kind, payload: forwarded });
|
|
61711
|
+
});
|
|
61712
|
+
return routes;
|
|
61713
|
+
}
|
|
61714
|
+
updateRoot(rootRun) {
|
|
61715
|
+
this.mirror.recordRunStart({ ...rootRun, inputs: {}, toolNodes: [], rootRunId: this.root });
|
|
61716
|
+
}
|
|
61717
|
+
accept(entries) {
|
|
61718
|
+
this.mirror.batch(() => {
|
|
61719
|
+
for (const { work, runs } of entries) {
|
|
61720
|
+
let attempt = this.attempts.get(work.id);
|
|
61721
|
+
if (attempt !== undefined && work.generation < attempt.generation)
|
|
61722
|
+
continue;
|
|
61723
|
+
if (attempt === undefined || attempt.generation !== work.generation) {
|
|
61724
|
+
if (attempt !== undefined)
|
|
61725
|
+
this.release(attempt);
|
|
61726
|
+
attempt = { generation: work.generation, runs: attempt?.runs ?? new Set, fenced: false };
|
|
61727
|
+
this.attempts.set(work.id, attempt);
|
|
61728
|
+
}
|
|
61729
|
+
if (attempt.fenced)
|
|
61730
|
+
continue;
|
|
61731
|
+
for (const run of runs) {
|
|
61732
|
+
attempt.runs.add(run.id);
|
|
61733
|
+
this.owners.set(run.id, work.id);
|
|
61734
|
+
const { pendingStageMessages: _messages, ...published } = run;
|
|
61735
|
+
this.mirror.recordRunStart({ ...published, rootRunId: this.root });
|
|
61736
|
+
if (run.stages.some((stage) => stage.pendingStageDeliveryAvailable === true))
|
|
61737
|
+
this.activated = true;
|
|
61738
|
+
}
|
|
61739
|
+
}
|
|
61740
|
+
});
|
|
61741
|
+
}
|
|
61742
|
+
fence(work) {
|
|
61743
|
+
const attempt = this.attempts.get(work.id);
|
|
61744
|
+
if (attempt === undefined || attempt.generation > work.generation)
|
|
61745
|
+
return;
|
|
61746
|
+
if (attempt.generation < work.generation) {
|
|
61747
|
+
this.release(attempt);
|
|
61748
|
+
this.attempts.set(work.id, { generation: work.generation, runs: attempt.runs, fenced: true });
|
|
61749
|
+
return;
|
|
61750
|
+
}
|
|
61751
|
+
this.release(attempt);
|
|
61752
|
+
attempt.fenced = true;
|
|
61753
|
+
}
|
|
61754
|
+
async route(work, runId) {
|
|
61755
|
+
this.requireOwned(work, runId);
|
|
61756
|
+
const readiness = workflowPendingStageRouteReady(this.mirror, runId);
|
|
61757
|
+
await readiness?.completion.catch(() => {});
|
|
61758
|
+
readiness?.assertCurrent();
|
|
61759
|
+
const waiting = new AbortController;
|
|
61760
|
+
await Promise.race([
|
|
61761
|
+
this.owner.registered(runId),
|
|
61762
|
+
delay2(ROUTE_REGISTRATION_WAIT_MS, undefined, { signal: waiting.signal, ref: false }).catch(() => {})
|
|
61763
|
+
]);
|
|
61764
|
+
waiting.abort();
|
|
61765
|
+
this.requireOwned(work, runId);
|
|
61766
|
+
return workflowPendingStageRouteCapability(this.mirror, runId);
|
|
61767
|
+
}
|
|
61768
|
+
async undeliverable(payload) {
|
|
61769
|
+
const event = { ...payload, handled: false };
|
|
61770
|
+
this.owner.events.emit(PENDING_STAGE_UNDELIVERABLE_EVENT2, event);
|
|
61771
|
+
return event.handled && await event.completion === true;
|
|
61772
|
+
}
|
|
61773
|
+
async close() {
|
|
61774
|
+
const bridge = this.bridge;
|
|
61775
|
+
if (bridge === undefined)
|
|
61776
|
+
return;
|
|
61777
|
+
this.bridge = undefined;
|
|
61778
|
+
try {
|
|
61779
|
+
await bridge.flush();
|
|
61780
|
+
} finally {
|
|
61781
|
+
bridge.refresh();
|
|
61782
|
+
bridge();
|
|
61783
|
+
await this.owner.closeGroup(workflowInvocationIntercomGroup(this.root));
|
|
61784
|
+
}
|
|
61785
|
+
}
|
|
61786
|
+
requireOwned(work, runId) {
|
|
61787
|
+
const attempt = this.attempts.get(work.id);
|
|
61788
|
+
if (attempt === undefined || attempt.generation !== work.generation || attempt.fenced)
|
|
61789
|
+
throw new Error(STALE_WORK_LEASE);
|
|
61790
|
+
if (!attempt.runs.has(runId))
|
|
61791
|
+
throw outside(runId);
|
|
61792
|
+
}
|
|
61793
|
+
release(attempt) {
|
|
61794
|
+
this.mirror.batch(() => {
|
|
61795
|
+
for (const id of attempt.runs) {
|
|
61796
|
+
rotateWorkflowPendingStageRouteCapability(this.mirror, id);
|
|
61797
|
+
const run = this.mirror.runs().find((candidate) => candidate.id === id);
|
|
61798
|
+
if (run?.stages.some(isLiveStage) !== true)
|
|
61799
|
+
continue;
|
|
61800
|
+
this.mirror.recordRunStart({
|
|
61801
|
+
...run,
|
|
61802
|
+
stages: run.stages.map((stage) => {
|
|
61803
|
+
if (!isLiveStage(stage))
|
|
61804
|
+
return stage;
|
|
61805
|
+
const { sessionId: _session, sessionFile: _file, ...rest } = stage;
|
|
61806
|
+
return { ...rest, attachable: false, attached: false };
|
|
61807
|
+
})
|
|
61808
|
+
});
|
|
61809
|
+
}
|
|
61810
|
+
});
|
|
61811
|
+
}
|
|
61812
|
+
}
|
|
61813
|
+
|
|
61814
|
+
class PoolCoordinator {
|
|
61815
|
+
port;
|
|
61816
|
+
children = new Set;
|
|
61817
|
+
incarnations = new Map;
|
|
61818
|
+
readyWorkers = new Set;
|
|
61819
|
+
logs = new Map;
|
|
61820
|
+
cleanups = new Set;
|
|
61821
|
+
controlAcks = new Map;
|
|
61822
|
+
lastHeard = new Map;
|
|
61823
|
+
pendingActivity = new Map;
|
|
61824
|
+
batches = new Map;
|
|
61825
|
+
publicationFailures = new Map;
|
|
61826
|
+
requests = new Map;
|
|
61827
|
+
deliveryRequests = new Map;
|
|
61828
|
+
inflight = new Set;
|
|
61829
|
+
lastTick = performance.now();
|
|
61830
|
+
stalledMs = 0;
|
|
61831
|
+
ticker;
|
|
61832
|
+
recoveryTimer;
|
|
61833
|
+
flushTimer;
|
|
61834
|
+
lastFlushAt = 0;
|
|
61835
|
+
batchSequence = 0;
|
|
61836
|
+
requestSequence = 0;
|
|
61837
|
+
usageChanged = false;
|
|
61838
|
+
budgetBlocked = false;
|
|
61839
|
+
desiredControl;
|
|
61840
|
+
root;
|
|
61841
|
+
lease;
|
|
61842
|
+
options;
|
|
61843
|
+
journal;
|
|
61844
|
+
queue;
|
|
61845
|
+
usageLedger;
|
|
61846
|
+
usageTotal;
|
|
61847
|
+
admission;
|
|
61848
|
+
routes;
|
|
61849
|
+
publishing = false;
|
|
61850
|
+
sources = [];
|
|
61851
|
+
logDirectory;
|
|
61852
|
+
starting;
|
|
61853
|
+
recovering;
|
|
61854
|
+
nextRecoveryAt = 0;
|
|
61855
|
+
cleanupFailure;
|
|
61856
|
+
closed = false;
|
|
61857
|
+
closing;
|
|
61858
|
+
replacements = 0;
|
|
61859
|
+
constructor(port) {
|
|
61860
|
+
this.port = port;
|
|
61861
|
+
port.onMessage((message) => this.receive(message));
|
|
61862
|
+
this.ticker = setInterval(() => this.observeResponsiveness(), TICK_INTERVAL_MS).unref();
|
|
61863
|
+
}
|
|
61864
|
+
async shutdown() {
|
|
61865
|
+
await this.closing?.catch(() => {});
|
|
61866
|
+
while (this.inflight.size > 0)
|
|
61867
|
+
await Promise.allSettled([...this.inflight]);
|
|
61868
|
+
await this.journal?.close().catch(() => {});
|
|
61869
|
+
}
|
|
61870
|
+
receive(message) {
|
|
61871
|
+
if (message.kind === "request-reply") {
|
|
61872
|
+
const pending = this.requests.get(message.id);
|
|
61873
|
+
this.requests.delete(message.id);
|
|
61874
|
+
if (message.error !== undefined)
|
|
61875
|
+
pending?.reject(new Error(message.error));
|
|
61876
|
+
else
|
|
61877
|
+
pending?.resolve(message.value);
|
|
61878
|
+
return;
|
|
61879
|
+
}
|
|
61880
|
+
if ("id" in message) {
|
|
61881
|
+
const command = message;
|
|
61882
|
+
const handled = this.execute(command).then((value2) => this.port.send({ kind: "reply", id: command.id, value: value2 }), (error) => this.port.send({ kind: "reply", id: command.id, error: error.message }));
|
|
61883
|
+
this.inflight.add(handled);
|
|
61884
|
+
handled.finally(() => this.inflight.delete(handled));
|
|
61885
|
+
return;
|
|
61886
|
+
}
|
|
61887
|
+
this.notice(message);
|
|
61888
|
+
}
|
|
61889
|
+
notice(notice) {
|
|
61890
|
+
if (notice.kind === "budget")
|
|
61891
|
+
this.budgetBlocked = notice.blocked;
|
|
61892
|
+
else if (notice.kind === "root")
|
|
61893
|
+
this.routes?.updateRoot(notice.rootRun);
|
|
61894
|
+
else {
|
|
61895
|
+
const batch = this.batches.get(notice.batch);
|
|
61896
|
+
this.batches.delete(notice.batch);
|
|
61897
|
+
const rejected = new Map(notice.rejected.map((entry) => [entry.workId, entry]));
|
|
61898
|
+
for (const entry of rejected.values())
|
|
61899
|
+
this.publicationFailures.set(entry.workId, { generation: entry.generation, error: entry.error });
|
|
61900
|
+
this.routes?.accept((batch?.entries ?? []).filter((entry) => rejected.get(entry.work.id)?.generation !== entry.work.generation));
|
|
61901
|
+
batch?.settle();
|
|
61902
|
+
}
|
|
61903
|
+
}
|
|
61904
|
+
async execute(command) {
|
|
61905
|
+
switch (command.kind) {
|
|
61906
|
+
case "prepare":
|
|
61907
|
+
return this.prepare(command.root, command.lease, command.options, command.control);
|
|
61908
|
+
case "control":
|
|
61909
|
+
return this.control(command.state);
|
|
61910
|
+
case "start":
|
|
61911
|
+
this.starting ??= this.start(command.sources, command.cwd, command.publish, command.rootRun);
|
|
61912
|
+
return this.starting;
|
|
61913
|
+
case "enqueue":
|
|
61914
|
+
return this.enqueue(command);
|
|
61915
|
+
case "transport":
|
|
61916
|
+
return this.transport(command.request, command);
|
|
61917
|
+
case "undeliverable":
|
|
61918
|
+
return await this.routes?.undeliverable(command.payload) ?? false;
|
|
61919
|
+
case "close":
|
|
61920
|
+
this.closing ??= this.close();
|
|
61921
|
+
return this.closing;
|
|
61922
|
+
}
|
|
61923
|
+
}
|
|
61924
|
+
async prepare(root, lease, options, control) {
|
|
61925
|
+
if (this.closed)
|
|
61926
|
+
throw new Error("Local worker pool is closed");
|
|
61927
|
+
if (this.root !== undefined && this.root !== root)
|
|
61928
|
+
throw new Error("Local worker pool belongs to another root");
|
|
61929
|
+
this.root = root;
|
|
61930
|
+
this.options = options;
|
|
61931
|
+
this.desiredControl ??= control;
|
|
61932
|
+
this.journal ??= await resolveRunJournal({ journal: { path: options.journalPath } });
|
|
61933
|
+
this.usageLedger ??= await openSqlPoolUsage(this.journal);
|
|
61934
|
+
this.usageTotal = await this.usageLedger.total(root);
|
|
61935
|
+
this.lease = lease;
|
|
61936
|
+
const queue = await openSqlPoolQueue(this.journal);
|
|
61937
|
+
await queue.initialize(lease, COORDINATOR_LEASE_MS);
|
|
61938
|
+
await queue.releasePreviousIncarnations(lease);
|
|
61939
|
+
for (;; ) {
|
|
61940
|
+
const intended = this.desiredControl ?? "running";
|
|
61941
|
+
await queue.setControl(lease, intended);
|
|
61942
|
+
if (intended === (this.desiredControl ?? "running"))
|
|
61943
|
+
break;
|
|
61944
|
+
}
|
|
61945
|
+
this.queue = queue;
|
|
61946
|
+
return this.usageTotal;
|
|
61947
|
+
}
|
|
61948
|
+
async control(state) {
|
|
61949
|
+
this.desiredControl = state;
|
|
61950
|
+
if (!this.queue || !this.lease)
|
|
61951
|
+
return;
|
|
61952
|
+
const control = await this.queue.setControl(this.lease, state);
|
|
61953
|
+
const deadline = Date.now() + CONTROL_ACKNOWLEDGEMENT_TIMEOUT_MS;
|
|
61954
|
+
while ([...this.readyWorkers].some((worker) => (this.controlAcks.get(worker) ?? -1) < control.revision)) {
|
|
61955
|
+
if (this.desiredControl !== state)
|
|
61956
|
+
return;
|
|
61957
|
+
if (Date.now() >= deadline) {
|
|
61958
|
+
if (state === "quit")
|
|
61959
|
+
break;
|
|
61960
|
+
throw new Error("Worker control acknowledgement unresolved");
|
|
61961
|
+
}
|
|
61962
|
+
await delay2(25);
|
|
61963
|
+
}
|
|
61964
|
+
this.flush();
|
|
61965
|
+
}
|
|
61966
|
+
async start(sources, cwd, publish, rootRun) {
|
|
61967
|
+
if (this.closed)
|
|
61968
|
+
throw new Error("Local worker pool is closed");
|
|
61969
|
+
const root = this.root;
|
|
61970
|
+
if (!this.queue || !this.lease || !this.options || root === undefined || this.lease.runId !== root)
|
|
61971
|
+
throw new Error("Local worker pool requires root preparation");
|
|
61972
|
+
this.sources = sources;
|
|
61973
|
+
this.publishing = publish;
|
|
61974
|
+
if (publish) {
|
|
61975
|
+
this.routes = await PooledRoutes.open(root, cwd, rootRun, (request) => this.request(request));
|
|
61976
|
+
if (this.closed) {
|
|
61977
|
+
await this.routes.close();
|
|
61978
|
+
throw new Error("Local worker pool is closed");
|
|
61979
|
+
}
|
|
61980
|
+
}
|
|
61981
|
+
if (this.options.admission)
|
|
61982
|
+
this.admission = await SqlResourceAdmission.open(this.journal, {
|
|
61983
|
+
...this.options.admission,
|
|
61984
|
+
root,
|
|
61985
|
+
holder: "coordinator"
|
|
61986
|
+
});
|
|
61987
|
+
this.logDirectory = workerLogDirectory(this.options.journalPath, root);
|
|
61988
|
+
await mkdir6(this.logDirectory, { recursive: true }).catch(() => {
|
|
61989
|
+
return;
|
|
61990
|
+
});
|
|
61991
|
+
this.recoveryTimer = setInterval(() => {
|
|
61992
|
+
this.recoverExpired().catch((error) => this.fail(error));
|
|
61993
|
+
}, RECOVERY_INTERVAL_MS).unref();
|
|
61994
|
+
const workers = Array.from({ length: this.options.workers }, () => this.spawn());
|
|
61995
|
+
try {
|
|
61996
|
+
await Promise.any(workers);
|
|
61997
|
+
} catch (error) {
|
|
61998
|
+
throw error.errors[0];
|
|
61999
|
+
}
|
|
62000
|
+
}
|
|
62001
|
+
async enqueue(command) {
|
|
62002
|
+
if (!this.queue || !this.lease || this.root === undefined)
|
|
62003
|
+
throw new Error("Local worker pool requires root preparation");
|
|
62004
|
+
if (command.rootRun !== undefined)
|
|
62005
|
+
this.routes?.updateRoot(command.rootRun);
|
|
62006
|
+
const work = await this.queue.enqueue(this.lease, command.descriptor, command.boundary);
|
|
62007
|
+
const result = await this.queue.result(this.root, work.id);
|
|
62008
|
+
return { work, ...result === undefined ? {} : { result } };
|
|
62009
|
+
}
|
|
62010
|
+
async transport(request, payload) {
|
|
62011
|
+
const target = this.deliveryRequests.get(request);
|
|
62012
|
+
if (target === undefined || !this.incarnations.has(target.slot.worker))
|
|
62013
|
+
throw new Error("Worker exited before delivery acknowledgement");
|
|
62014
|
+
const { slot, workerRequest } = target;
|
|
62015
|
+
await new Promise((resolve, reject) => {
|
|
62016
|
+
const id = randomUUID15();
|
|
62017
|
+
slot.deliveries.set(id, { resolve, reject });
|
|
62018
|
+
slot.child.send({ id: workerRequest, delivery: { id, from: payload.from, message: payload.message } }, (error) => {
|
|
62019
|
+
if (error) {
|
|
62020
|
+
slot.deliveries.delete(id);
|
|
62021
|
+
reject(error);
|
|
62022
|
+
}
|
|
62023
|
+
});
|
|
62024
|
+
});
|
|
62025
|
+
}
|
|
62026
|
+
request(body) {
|
|
62027
|
+
const id = ++this.requestSequence;
|
|
62028
|
+
return new Promise((resolve, reject) => {
|
|
62029
|
+
this.requests.set(id, { resolve, reject });
|
|
62030
|
+
this.port.send({ kind: "request", id, request: body });
|
|
62031
|
+
});
|
|
62032
|
+
}
|
|
62033
|
+
fail(error) {
|
|
62034
|
+
this.port.send({ kind: "failure", error: error.message });
|
|
62035
|
+
}
|
|
62036
|
+
publish(work, runs) {
|
|
62037
|
+
const failure = this.publicationFailures.get(work.id);
|
|
62038
|
+
if (failure?.generation === work.generation)
|
|
62039
|
+
throw new Error(failure.error);
|
|
62040
|
+
if (!this.publishing)
|
|
62041
|
+
return;
|
|
62042
|
+
this.pendingActivity.set(work.id, { work: token(work), runs });
|
|
62043
|
+
this.scheduleFlush();
|
|
62044
|
+
}
|
|
62045
|
+
scheduleFlush() {
|
|
62046
|
+
if (this.flushTimer !== undefined || this.closed)
|
|
62047
|
+
return;
|
|
62048
|
+
const wait = Math.max(0, this.lastFlushAt + POOL_SUMMARY_INTERVAL_MS - performance.now());
|
|
62049
|
+
this.flushTimer = setTimeout(() => {
|
|
62050
|
+
this.flushTimer = undefined;
|
|
62051
|
+
if (this.batches.size > 0) {
|
|
62052
|
+
this.lastFlushAt = performance.now();
|
|
62053
|
+
this.scheduleFlush();
|
|
62054
|
+
return;
|
|
62055
|
+
}
|
|
62056
|
+
this.flush();
|
|
62057
|
+
}, wait);
|
|
62058
|
+
}
|
|
62059
|
+
flush() {
|
|
62060
|
+
clearTimeout(this.flushTimer);
|
|
62061
|
+
this.flushTimer = undefined;
|
|
62062
|
+
if (this.pendingActivity.size === 0 && !this.usageChanged) {
|
|
62063
|
+
const last = [...this.batches.values()].at(-1);
|
|
62064
|
+
return last === undefined ? Promise.resolve() : new Promise((resolve) => this.chainSettle(last, resolve));
|
|
62065
|
+
}
|
|
62066
|
+
this.lastFlushAt = performance.now();
|
|
62067
|
+
const batch = ++this.batchSequence;
|
|
62068
|
+
const entries = [...this.pendingActivity.values()];
|
|
62069
|
+
this.pendingActivity.clear();
|
|
62070
|
+
const usage = this.usageChanged ? this.usageTotal : undefined;
|
|
62071
|
+
this.usageChanged = false;
|
|
62072
|
+
const settled = new Promise((resolve) => this.batches.set(batch, { entries, settle: resolve }));
|
|
62073
|
+
this.port.send({ kind: "activity", batch, entries, ...usage === undefined ? {} : { usage } });
|
|
62074
|
+
return settled;
|
|
62075
|
+
}
|
|
62076
|
+
chainSettle(batch, resolve) {
|
|
62077
|
+
const settle = batch.settle;
|
|
62078
|
+
batch.settle = () => {
|
|
62079
|
+
settle();
|
|
62080
|
+
resolve();
|
|
62081
|
+
};
|
|
62082
|
+
}
|
|
62083
|
+
noteUsage(usage) {
|
|
62084
|
+
this.usageTotal = usage;
|
|
62085
|
+
this.usageChanged = true;
|
|
62086
|
+
this.scheduleFlush();
|
|
62087
|
+
}
|
|
62088
|
+
async sendResult(workId) {
|
|
62089
|
+
const result = await this.queue.result(this.root, workId);
|
|
62090
|
+
if (result === undefined)
|
|
62091
|
+
return false;
|
|
62092
|
+
this.flush();
|
|
62093
|
+
this.port.send({
|
|
62094
|
+
kind: "result",
|
|
62095
|
+
workId,
|
|
62096
|
+
result,
|
|
62097
|
+
...this.usageTotal === undefined ? {} : { usage: this.usageTotal }
|
|
62098
|
+
});
|
|
62099
|
+
return true;
|
|
62100
|
+
}
|
|
62101
|
+
fence(work) {
|
|
62102
|
+
this.pendingActivity.delete(work.id);
|
|
62103
|
+
this.routes?.fence(work);
|
|
62104
|
+
this.port.send({ kind: "fence", work: { id: work.id, generation: work.generation } });
|
|
62105
|
+
}
|
|
62106
|
+
spawn() {
|
|
62107
|
+
if (this.closed)
|
|
62108
|
+
return Promise.reject(new Error("Local worker pool is closed"));
|
|
62109
|
+
const root = this.root;
|
|
62110
|
+
const options = this.options;
|
|
62111
|
+
const child = fork(options.workerEntry ?? process.argv[1], [
|
|
62112
|
+
JSON.stringify({
|
|
62113
|
+
journalPath: options.journalPath,
|
|
62114
|
+
root,
|
|
62115
|
+
definitions: this.sources,
|
|
62116
|
+
admission: options.admission
|
|
62117
|
+
})
|
|
62118
|
+
], { stdio: ["ignore", "ignore", "pipe", "ipc"], execArgv: [] });
|
|
62119
|
+
const disconnected = new Promise((resolve) => child.once("disconnect", () => resolve()));
|
|
62120
|
+
this.children.add(child);
|
|
62121
|
+
if (child.pid !== undefined)
|
|
62122
|
+
this.port.send({ kind: "spawned", pid: child.pid });
|
|
62123
|
+
const worker = randomUUID15();
|
|
62124
|
+
this.incarnations.set(worker, child);
|
|
62125
|
+
const slot = { worker, child, deliveries: new Map };
|
|
62126
|
+
let ready = false;
|
|
62127
|
+
let serial = Promise.resolve();
|
|
62128
|
+
const logPath = join28(this.logDirectory, `${worker}.log`);
|
|
62129
|
+
this.logs.set(child, logPath);
|
|
62130
|
+
let logFailure;
|
|
62131
|
+
const log = createWriteStream(logPath, { flags: "a" });
|
|
62132
|
+
log.on("error", (error) => {
|
|
62133
|
+
logFailure = error.message;
|
|
62134
|
+
});
|
|
62135
|
+
let stderr = "";
|
|
62136
|
+
child.stderr?.on("data", (chunk) => {
|
|
62137
|
+
stderr = (stderr + chunk.toString()).slice(-STDERR_TAIL_CHARACTERS);
|
|
62138
|
+
});
|
|
62139
|
+
child.stderr?.on("error", (error) => {
|
|
62140
|
+
logFailure ??= error.message;
|
|
62141
|
+
log.end();
|
|
62142
|
+
});
|
|
62143
|
+
child.stderr?.pipe(log);
|
|
62144
|
+
const diagnosed = (summary) => new Error(`${summary} (worker stderr log: ${logFailure === undefined ? logPath : `unavailable, ${logFailure}`})${stderr ? `: ${stderr}` : ""}`);
|
|
62145
|
+
const startupTimeoutMs = options.startupTimeoutMs;
|
|
62146
|
+
let timedOut = false;
|
|
62147
|
+
let terminated = false;
|
|
62148
|
+
const terminate = () => {
|
|
62149
|
+
if (terminated)
|
|
62150
|
+
return;
|
|
62151
|
+
terminated = true;
|
|
62152
|
+
child.kill("SIGKILL");
|
|
62153
|
+
};
|
|
62154
|
+
const replied = (error) => {
|
|
62155
|
+
if (error)
|
|
62156
|
+
terminate();
|
|
62157
|
+
};
|
|
62158
|
+
return new Promise((resolve, reject) => {
|
|
62159
|
+
const startup = setTimeout(() => {
|
|
62160
|
+
timedOut = true;
|
|
62161
|
+
terminate();
|
|
62162
|
+
}, startupTimeoutMs);
|
|
62163
|
+
child.on("error", (error) => {
|
|
62164
|
+
clearTimeout(startup);
|
|
62165
|
+
reject(error);
|
|
62166
|
+
terminate();
|
|
62167
|
+
});
|
|
62168
|
+
const closed = new Promise((resolve) => child.once("close", () => resolve()));
|
|
62169
|
+
child.once("exit", () => {
|
|
62170
|
+
Promise.race([closed, delay2(STDERR_DRAIN_GRACE_MS)]).then(() => reject(diagnosed(timedOut ? `Local worker startup timed out after ${startupTimeoutMs} ms` : "Local worker exited before readiness")));
|
|
62171
|
+
this.children.delete(child);
|
|
62172
|
+
this.logs.delete(child);
|
|
62173
|
+
if (child.pid !== undefined)
|
|
62174
|
+
this.port.send({ kind: "exited", pid: child.pid });
|
|
62175
|
+
clearTimeout(startup);
|
|
62176
|
+
this.incarnations.delete(worker);
|
|
62177
|
+
this.readyWorkers.delete(worker);
|
|
62178
|
+
this.controlAcks.delete(worker);
|
|
62179
|
+
this.lastHeard.delete(worker);
|
|
62180
|
+
if (slot.work)
|
|
62181
|
+
this.fence(slot.work);
|
|
62182
|
+
for (const delivery of slot.deliveries.values())
|
|
62183
|
+
delivery.reject(new Error("Worker exited before delivery acknowledgement"));
|
|
62184
|
+
slot.deliveries.clear();
|
|
62185
|
+
const cleanup = disconnected.then(() => serial).then(async () => {
|
|
62186
|
+
try {
|
|
62187
|
+
if (await this.admission?.releaseStoppedSessions(worker))
|
|
62188
|
+
throw new Error("Local worker process cleanup unconfirmed; permits quarantined");
|
|
62189
|
+
} catch (error) {
|
|
62190
|
+
this.cleanupFailure = error instanceof Error ? error : new Error(String(error));
|
|
62191
|
+
throw this.cleanupFailure;
|
|
62192
|
+
}
|
|
62193
|
+
}).then(async () => {
|
|
62194
|
+
if (this.closed || !ready || this.desiredControl === "quit")
|
|
62195
|
+
return;
|
|
62196
|
+
let work = slot.work;
|
|
62197
|
+
if (work && await this.sendResult(work.id)) {
|
|
62198
|
+
work = undefined;
|
|
62199
|
+
slot.work = undefined;
|
|
62200
|
+
this.replacements = 0;
|
|
62201
|
+
}
|
|
62202
|
+
if (this.closed)
|
|
62203
|
+
return;
|
|
62204
|
+
if (work)
|
|
62205
|
+
await this.queue.releaseExited(this.lease, work);
|
|
62206
|
+
const exhausted = work ? work.generation >= 3 : ++this.replacements > 3;
|
|
62207
|
+
if (exhausted)
|
|
62208
|
+
this.fail(diagnosed("Local worker retry limit"));
|
|
62209
|
+
else
|
|
62210
|
+
await this.spawn();
|
|
62211
|
+
}).catch((error) => {
|
|
62212
|
+
this.fail(error);
|
|
62213
|
+
});
|
|
62214
|
+
this.cleanups.add(cleanup);
|
|
62215
|
+
cleanup.finally(() => this.cleanups.delete(cleanup));
|
|
62216
|
+
});
|
|
62217
|
+
child.on("message", (message) => {
|
|
62218
|
+
this.observeResponsiveness();
|
|
62219
|
+
this.lastHeard.set(worker, { at: performance.now(), stalled: this.stalledMs });
|
|
62220
|
+
if (message.kind === "delivery-ack") {
|
|
62221
|
+
const pending = slot.deliveries.get(message.deliveryId);
|
|
62222
|
+
slot.deliveries.delete(message.deliveryId);
|
|
62223
|
+
if (message.error)
|
|
62224
|
+
pending?.reject(new Error(message.error));
|
|
62225
|
+
else
|
|
62226
|
+
pending?.resolve();
|
|
62227
|
+
return;
|
|
62228
|
+
}
|
|
62229
|
+
let routeReady;
|
|
62230
|
+
const handled = serial.then(async () => {
|
|
62231
|
+
if (message.kind === "ready") {
|
|
62232
|
+
if (terminated)
|
|
62233
|
+
return;
|
|
62234
|
+
ready = true;
|
|
62235
|
+
this.readyWorkers.add(worker);
|
|
62236
|
+
clearTimeout(startup);
|
|
62237
|
+
resolve();
|
|
62238
|
+
return;
|
|
62239
|
+
}
|
|
62240
|
+
await this.starting;
|
|
62241
|
+
const queue = this.queue;
|
|
62242
|
+
const lease = this.lease;
|
|
62243
|
+
if (message.kind === "control")
|
|
62244
|
+
return queue.control(lease);
|
|
62245
|
+
if (message.kind === "controlled") {
|
|
62246
|
+
const control = await queue.control(lease);
|
|
62247
|
+
if (message.revision > control.revision)
|
|
62248
|
+
throw new Error("Unknown control revision");
|
|
62249
|
+
this.controlAcks.set(worker, Math.max(this.controlAcks.get(worker) ?? -1, message.revision));
|
|
62250
|
+
return;
|
|
62251
|
+
}
|
|
62252
|
+
if (message.kind === "claim") {
|
|
62253
|
+
if (slot.work)
|
|
62254
|
+
throw new Error("Worker already holds an attempt");
|
|
62255
|
+
if (this.budgetBlocked)
|
|
62256
|
+
return;
|
|
62257
|
+
await this.recoverExpired();
|
|
62258
|
+
slot.work = await queue.claim(lease, worker, this.sources.map((source) => ({
|
|
62259
|
+
definition: `${source.path}#${source.exportName}`,
|
|
62260
|
+
digest: source.digest
|
|
62261
|
+
})), options.leaseMs);
|
|
62262
|
+
return slot.work;
|
|
62263
|
+
}
|
|
62264
|
+
let work = slot.work;
|
|
62265
|
+
if (!work)
|
|
62266
|
+
throw new Error("Worker has no active attempt");
|
|
62267
|
+
if (message.kind === "renew" || work.expiresAt - Date.now() < options.leaseMs / 2) {
|
|
62268
|
+
work = await queue.extend(lease, work, options.leaseMs);
|
|
62269
|
+
slot.work = work;
|
|
62270
|
+
}
|
|
62271
|
+
if (message.kind === "activity")
|
|
62272
|
+
return this.publish(work, message.runs);
|
|
62273
|
+
if (message.kind === "route") {
|
|
62274
|
+
if (this.routes === undefined)
|
|
62275
|
+
return;
|
|
62276
|
+
await this.flush();
|
|
62277
|
+
this.assertPublished(work);
|
|
62278
|
+
routeReady = this.routes.route(work, message.runId);
|
|
62279
|
+
return;
|
|
62280
|
+
}
|
|
62281
|
+
if (message.kind === "delivery-ready") {
|
|
62282
|
+
if (!this.publishing)
|
|
62283
|
+
return;
|
|
62284
|
+
await this.flush();
|
|
62285
|
+
this.assertPublished(work);
|
|
62286
|
+
return this.request({
|
|
62287
|
+
kind: "ready",
|
|
62288
|
+
work: token(work),
|
|
62289
|
+
runId: message.runId,
|
|
62290
|
+
stageId: message.stageId,
|
|
62291
|
+
stageName: message.stageName,
|
|
62292
|
+
...message.sessionId === undefined ? {} : { sessionId: message.sessionId }
|
|
62293
|
+
});
|
|
62294
|
+
}
|
|
62295
|
+
if (message.kind === "delivery") {
|
|
62296
|
+
if (!message.recipient)
|
|
62297
|
+
throw new Error("Worker delivery requires a session identity");
|
|
62298
|
+
if (!this.publishing)
|
|
62299
|
+
return;
|
|
62300
|
+
await this.flush();
|
|
62301
|
+
this.assertPublished(work);
|
|
62302
|
+
const id = ++this.requestSequence;
|
|
62303
|
+
this.deliveryRequests.set(id, { slot, workerRequest: message.id });
|
|
62304
|
+
try {
|
|
62305
|
+
return await new Promise((resolve, reject) => {
|
|
62306
|
+
this.requests.set(id, { resolve, reject });
|
|
62307
|
+
this.port.send({
|
|
62308
|
+
kind: "request",
|
|
62309
|
+
id,
|
|
62310
|
+
request: {
|
|
62311
|
+
kind: "deliver",
|
|
62312
|
+
work: token(work),
|
|
62313
|
+
runId: message.runId,
|
|
62314
|
+
stageId: message.stageId,
|
|
62315
|
+
stageName: message.stageName,
|
|
62316
|
+
recipient: message.recipient
|
|
62317
|
+
}
|
|
62318
|
+
});
|
|
62319
|
+
});
|
|
62320
|
+
} finally {
|
|
62321
|
+
this.deliveryRequests.delete(id);
|
|
62322
|
+
}
|
|
62323
|
+
}
|
|
62324
|
+
if (message.kind === "renew")
|
|
62325
|
+
return;
|
|
62326
|
+
if (message.kind === "commit") {
|
|
62327
|
+
const receipt = await queue.commit(lease, work, message.operation, message.mutation);
|
|
62328
|
+
this.noteUsage(await this.usageLedger.total(root));
|
|
62329
|
+
return receipt;
|
|
62330
|
+
}
|
|
62331
|
+
if (message.kind === "usage") {
|
|
62332
|
+
this.noteUsage(await this.usageLedger.report(lease, work, message.reports));
|
|
62333
|
+
return;
|
|
62334
|
+
}
|
|
62335
|
+
if (message.kind === "budget") {
|
|
62336
|
+
await queue.assertCurrent(lease, work);
|
|
62337
|
+
if (this.budgetBlocked)
|
|
62338
|
+
throw new Error("Root workflow budget exhausted");
|
|
62339
|
+
return queue.control(lease);
|
|
62340
|
+
}
|
|
62341
|
+
if (message.kind !== "complete")
|
|
62342
|
+
throw new Error("Invalid worker operation");
|
|
62343
|
+
await queue.complete(lease, work, message.result);
|
|
62344
|
+
this.replacements = 0;
|
|
62345
|
+
slot.work = undefined;
|
|
62346
|
+
await this.sendResult(work.id);
|
|
62347
|
+
});
|
|
62348
|
+
const response = handled.then((value2) => routeReady ?? value2);
|
|
62349
|
+
if (message.kind !== "delivery" && message.kind !== "delivery-ready")
|
|
62350
|
+
serial = handled.then(() => {}, () => {});
|
|
62351
|
+
response.then((value2) => child.send({ id: message.id, value: value2 }, replied), (error) => child.send({ id: message.id, error: error.message }, replied));
|
|
62352
|
+
});
|
|
62353
|
+
});
|
|
62354
|
+
}
|
|
62355
|
+
assertPublished(work) {
|
|
62356
|
+
const failure = this.publicationFailures.get(work.id);
|
|
62357
|
+
if (failure?.generation === work.generation)
|
|
62358
|
+
throw new Error(failure.error);
|
|
62359
|
+
}
|
|
62360
|
+
recoverExpired() {
|
|
62361
|
+
if (this.recovering)
|
|
62362
|
+
return this.recovering;
|
|
62363
|
+
const root = this.root;
|
|
62364
|
+
if (this.closed || root === undefined || !this.queue || performance.now() < this.nextRecoveryAt)
|
|
62365
|
+
return Promise.resolve();
|
|
62366
|
+
const queue = this.queue;
|
|
62367
|
+
const lease = this.lease;
|
|
62368
|
+
const leaseMs = this.options.leaseMs;
|
|
62369
|
+
this.recovering = (async () => {
|
|
62370
|
+
await new Promise((resolve) => setImmediate(resolve));
|
|
62371
|
+
await this.admission?.reconcile();
|
|
62372
|
+
const expiredBefore = Date.now();
|
|
62373
|
+
for (const work of await queue.inspect(root)) {
|
|
62374
|
+
if (work.state !== "leased" || work.expiresAt >= expiredBefore)
|
|
62375
|
+
continue;
|
|
62376
|
+
const child = work.worker === null ? undefined : this.incarnations.get(work.worker);
|
|
62377
|
+
if (child && work.worker !== null && this.isLive(work.worker, child)) {
|
|
62378
|
+
await queue.extend(lease, work, leaseMs).catch((error) => {
|
|
62379
|
+
if (error.message !== STALE_WORK_LEASE)
|
|
62380
|
+
throw error;
|
|
62381
|
+
});
|
|
62382
|
+
continue;
|
|
62383
|
+
}
|
|
62384
|
+
if (child && !await this.stop(child).then(() => true, () => false))
|
|
62385
|
+
continue;
|
|
62386
|
+
await queue.expire(lease, work);
|
|
62387
|
+
}
|
|
62388
|
+
})().finally(() => {
|
|
62389
|
+
this.recovering = undefined;
|
|
62390
|
+
this.nextRecoveryAt = performance.now() + RECOVERY_INTERVAL_MS;
|
|
62391
|
+
});
|
|
62392
|
+
return this.recovering;
|
|
62393
|
+
}
|
|
62394
|
+
observeResponsiveness() {
|
|
62395
|
+
const now = performance.now();
|
|
62396
|
+
const gap = now - this.lastTick;
|
|
62397
|
+
if (gap > STALL_THRESHOLD_MS)
|
|
62398
|
+
this.stalledMs += gap - TICK_INTERVAL_MS;
|
|
62399
|
+
this.lastTick = now;
|
|
62400
|
+
}
|
|
62401
|
+
isLive(worker, child) {
|
|
62402
|
+
this.observeResponsiveness();
|
|
62403
|
+
const heard = this.lastHeard.get(worker);
|
|
62404
|
+
return child.exitCode === null && child.signalCode === null && child.connected && heard !== undefined && performance.now() - heard.at - (this.stalledMs - heard.stalled) < this.options.leaseMs;
|
|
62405
|
+
}
|
|
62406
|
+
stop(child) {
|
|
62407
|
+
if (child.pid === undefined || child.exitCode !== null || child.signalCode !== null)
|
|
62408
|
+
return Promise.resolve();
|
|
62409
|
+
return new Promise((resolve, reject) => {
|
|
62410
|
+
const timeout = setTimeout(() => reject(new Error("Local worker cleanup was not confirmed")), 5000);
|
|
62411
|
+
child.once("exit", () => {
|
|
62412
|
+
clearTimeout(timeout);
|
|
62413
|
+
resolve();
|
|
62414
|
+
});
|
|
62415
|
+
child.kill("SIGKILL");
|
|
62416
|
+
});
|
|
62417
|
+
}
|
|
62418
|
+
async close() {
|
|
62419
|
+
this.closed = true;
|
|
62420
|
+
clearInterval(this.ticker);
|
|
62421
|
+
clearInterval(this.recoveryTimer);
|
|
62422
|
+
let failure;
|
|
62423
|
+
let stopping = [];
|
|
62424
|
+
try {
|
|
62425
|
+
await this.recovering?.catch(() => {});
|
|
62426
|
+
stopping = [...this.children].map((child) => ({ child, log: this.logs.get(child) }));
|
|
62427
|
+
await Promise.all(stopping.map(({ child }) => this.stop(child)));
|
|
62428
|
+
await Promise.all([...this.cleanups]);
|
|
62429
|
+
if (this.cleanupFailure)
|
|
62430
|
+
throw this.cleanupFailure;
|
|
62431
|
+
} catch (error) {
|
|
62432
|
+
failure = error;
|
|
62433
|
+
}
|
|
62434
|
+
if (failure === undefined) {
|
|
62435
|
+
const stopped = stopping.flatMap(({ child, log }) => log !== undefined && (child.exitCode !== null || child.signalCode !== null) ? [log] : []);
|
|
62436
|
+
await Promise.all(stopped.map((log) => rm3(log, { force: true }).catch(() => {})));
|
|
62437
|
+
if (this.logDirectory !== undefined)
|
|
62438
|
+
await rmdir(this.logDirectory).catch(() => {});
|
|
62439
|
+
}
|
|
62440
|
+
this.flush();
|
|
62441
|
+
try {
|
|
62442
|
+
await this.routes?.close();
|
|
62443
|
+
} catch (error) {
|
|
62444
|
+
failure ??= error;
|
|
62445
|
+
}
|
|
62446
|
+
for (const batch of this.batches.values())
|
|
62447
|
+
batch.settle();
|
|
62448
|
+
this.batches.clear();
|
|
62449
|
+
for (const pending of this.requests.values())
|
|
62450
|
+
pending.reject(new Error("Local worker pool is closed"));
|
|
62451
|
+
this.requests.clear();
|
|
62452
|
+
if (failure !== undefined)
|
|
62453
|
+
throw failure;
|
|
62454
|
+
}
|
|
62455
|
+
}
|
|
62456
|
+
var CONTROL_ACKNOWLEDGEMENT_TIMEOUT_MS = 5000, RECOVERY_INTERVAL_MS = 500, TICK_INTERVAL_MS = 100, STALL_THRESHOLD_MS = 500, COORDINATOR_LEASE_MS, STDERR_TAIL_CHARACTERS = 8192, STDERR_DRAIN_GRACE_MS = 1000, ROUTE_REGISTRATION_WAIT_MS = 120000, PENDING_STAGE_MESSAGE_EVENT2 = "atomic:workflow-pending-stage-message", STICKY_LIVE_DELIVERED_EVENT2 = "atomic:workflow-sticky-live-delivered", PENDING_STAGE_UNDELIVERABLE_EVENT2 = "atomic:workflow-pending-stage-undeliverable";
|
|
62457
|
+
var init_pool_coordinator = __esm(() => {
|
|
62458
|
+
init_resolution();
|
|
62459
|
+
init_sql_pool_queue();
|
|
62460
|
+
init_sql_pool_usage();
|
|
62461
|
+
init_pending_stage_intercom();
|
|
62462
|
+
init_intercom_group();
|
|
62463
|
+
init_pending_stage_route_capability();
|
|
62464
|
+
init_pending_stage_route_readiness();
|
|
62465
|
+
init_store();
|
|
62466
|
+
init_resource_admission();
|
|
62467
|
+
COORDINATOR_LEASE_MS = 3 * STAGE_SESSION_HEARTBEAT_INTERVAL_MS;
|
|
62468
|
+
});
|
|
62469
|
+
|
|
61322
62470
|
// dist/builtin/workflows/src/workers/protocol.ts
|
|
61323
62471
|
var WORKER_HEARTBEAT_INTERVAL_MS = 1000;
|
|
61324
62472
|
|
|
@@ -61566,21 +62714,26 @@ class PoolStageHandoff {
|
|
|
61566
62714
|
store;
|
|
61567
62715
|
queue;
|
|
61568
62716
|
lease;
|
|
62717
|
+
claimRun;
|
|
61569
62718
|
attempts = new Map;
|
|
61570
62719
|
owners = new Map;
|
|
61571
62720
|
publications = new Map;
|
|
61572
62721
|
drains = new Map;
|
|
61573
|
-
constructor(backend, store, queue, lease) {
|
|
62722
|
+
constructor(backend, store, queue, lease, claimRun) {
|
|
61574
62723
|
this.backend = backend;
|
|
61575
62724
|
this.store = store;
|
|
61576
62725
|
this.queue = queue;
|
|
61577
62726
|
this.lease = lease;
|
|
62727
|
+
this.claimRun = claimRun;
|
|
61578
62728
|
}
|
|
61579
62729
|
publish(work, runs) {
|
|
61580
62730
|
const previous = this.publications.get(work.id) ?? Promise.resolve();
|
|
61581
62731
|
const publication = previous.catch(() => {
|
|
61582
62732
|
return;
|
|
61583
|
-
}).then(() =>
|
|
62733
|
+
}).then(async () => {
|
|
62734
|
+
const commit = await this.prepare(work, runs);
|
|
62735
|
+
this.store.batch(commit);
|
|
62736
|
+
});
|
|
61584
62737
|
const settled = publication.catch(() => {
|
|
61585
62738
|
return;
|
|
61586
62739
|
});
|
|
@@ -61591,6 +62744,21 @@ class PoolStageHandoff {
|
|
|
61591
62744
|
});
|
|
61592
62745
|
return publication;
|
|
61593
62746
|
}
|
|
62747
|
+
async publishBatch(publications) {
|
|
62748
|
+
const prepared = [];
|
|
62749
|
+
for (const { work, runs } of publications)
|
|
62750
|
+
prepared.push(await this.prepare(work, runs).catch((error) => error));
|
|
62751
|
+
return this.store.batch(() => prepared.map((commit) => {
|
|
62752
|
+
if (commit instanceof Error)
|
|
62753
|
+
return commit;
|
|
62754
|
+
try {
|
|
62755
|
+
commit();
|
|
62756
|
+
return;
|
|
62757
|
+
} catch (error) {
|
|
62758
|
+
return error instanceof Error ? error : new Error(String(error));
|
|
62759
|
+
}
|
|
62760
|
+
}));
|
|
62761
|
+
}
|
|
61594
62762
|
async route(work, runId) {
|
|
61595
62763
|
this.requireOwned(await this.begin(work), runId);
|
|
61596
62764
|
const readiness = workflowPendingStageRouteReady(this.store, runId);
|
|
@@ -61641,7 +62809,7 @@ class PoolStageHandoff {
|
|
|
61641
62809
|
const result = await this.queue.result(root, work.id);
|
|
61642
62810
|
if (result === undefined)
|
|
61643
62811
|
return;
|
|
61644
|
-
await this.backend.
|
|
62812
|
+
await this.backend.catchUpWorkflow(root);
|
|
61645
62813
|
const terminals = new Map;
|
|
61646
62814
|
for (const checkpoint of this.backend.listCheckpoints(root)) {
|
|
61647
62815
|
const boundary = checkpoint.kind === "stage" ? checkpoint.topology?.boundary : undefined;
|
|
@@ -61651,6 +62819,9 @@ class PoolStageHandoff {
|
|
|
61651
62819
|
if (child?.runId === boundary.child.runId)
|
|
61652
62820
|
terminals.set(child.runId, { status: child.status, outputs: child.outputs });
|
|
61653
62821
|
}
|
|
62822
|
+
this.store.batch(() => this.settle(work, result, terminals));
|
|
62823
|
+
}
|
|
62824
|
+
settle(work, result, terminals) {
|
|
61654
62825
|
for (const run of this.store.runs()) {
|
|
61655
62826
|
if (run.endedAt !== undefined || this.owners.get(run.id) !== work.id)
|
|
61656
62827
|
continue;
|
|
@@ -61674,22 +62845,31 @@ class PoolStageHandoff {
|
|
|
61674
62845
|
if (current !== undefined && current.generation > work.generation)
|
|
61675
62846
|
return;
|
|
61676
62847
|
const attempt = current?.generation === work.generation ? current : this.start(work, current);
|
|
61677
|
-
this.release(attempt);
|
|
62848
|
+
this.store.batch(() => this.release(attempt));
|
|
61678
62849
|
attempt.fenced = true;
|
|
61679
62850
|
}
|
|
61680
|
-
async
|
|
62851
|
+
async prepare(work, runs) {
|
|
61681
62852
|
const subtree = this.subtree(work, runs);
|
|
61682
62853
|
const root = this.lease.runId;
|
|
61683
|
-
const
|
|
61684
|
-
|
|
61685
|
-
|
|
61686
|
-
|
|
61687
|
-
|
|
61688
|
-
|
|
61689
|
-
|
|
61690
|
-
|
|
61691
|
-
|
|
62854
|
+
const owned = (known) => {
|
|
62855
|
+
for (const run of subtree) {
|
|
62856
|
+
const existing = known.get(run.id);
|
|
62857
|
+
const owner = this.owners.get(run.id);
|
|
62858
|
+
if (owner !== undefined && owner !== work.id || owner === undefined && existing !== undefined && (existing.parentRunId !== run.parentRunId || existing.parentStageId !== run.parentStageId))
|
|
62859
|
+
throw outside2(run.id);
|
|
62860
|
+
}
|
|
62861
|
+
};
|
|
62862
|
+
const before = new Map(this.store.runs().map((run) => [run.id, run]));
|
|
62863
|
+
owned(before);
|
|
62864
|
+
if (subtree.some((run) => before.get(run.id) === undefined))
|
|
62865
|
+
await this.backend.catchUpWorkflow(root);
|
|
61692
62866
|
await this.queue.assertCurrent(this.lease, work);
|
|
62867
|
+
return () => this.commit(work, subtree, owned);
|
|
62868
|
+
}
|
|
62869
|
+
commit(work, subtree, owned) {
|
|
62870
|
+
const root = this.lease.runId;
|
|
62871
|
+
const known = new Map(this.store.runs().map((run) => [run.id, run]));
|
|
62872
|
+
owned(known);
|
|
61693
62873
|
const attempt = this.enter(work);
|
|
61694
62874
|
const execution = {
|
|
61695
62875
|
worker: work.worker ?? "",
|
|
@@ -61701,11 +62881,13 @@ class PoolStageHandoff {
|
|
|
61701
62881
|
...run,
|
|
61702
62882
|
rootRunId: root,
|
|
61703
62883
|
pendingStageMessages: [
|
|
61704
|
-
...
|
|
62884
|
+
...known.get(run.id)?.pendingStageMessages ?? pendingStageMessagesForDurableRun(this.backend, run.id, root)
|
|
61705
62885
|
],
|
|
61706
62886
|
stages: run.stages.map((stage) => ({ ...stage, workerExecution: execution }))
|
|
61707
62887
|
}));
|
|
61708
62888
|
const changed = published.some((run) => activity(known.get(run.id)) !== activity(run));
|
|
62889
|
+
for (const run of published)
|
|
62890
|
+
this.claimRun?.(run.id);
|
|
61709
62891
|
for (const run of published) {
|
|
61710
62892
|
attempt.runs.add(run.id);
|
|
61711
62893
|
this.owners.set(run.id, work.id);
|
|
@@ -61730,7 +62912,7 @@ class PoolStageHandoff {
|
|
|
61730
62912
|
const byId = new Map;
|
|
61731
62913
|
for (const run of runs) {
|
|
61732
62914
|
if (run.id === root || byId.has(run.id))
|
|
61733
|
-
throw
|
|
62915
|
+
throw outside2(run.id);
|
|
61734
62916
|
byId.set(run.id, run);
|
|
61735
62917
|
}
|
|
61736
62918
|
if (runs.length === 0)
|
|
@@ -61743,25 +62925,25 @@ class PoolStageHandoff {
|
|
|
61743
62925
|
const top = byId.get(work.childRunId);
|
|
61744
62926
|
if (top !== undefined) {
|
|
61745
62927
|
if (top.parentRunId !== root || top.parentStageId !== work.boundaryId)
|
|
61746
|
-
throw
|
|
62928
|
+
throw outside2(top.id);
|
|
61747
62929
|
const pending = [top];
|
|
61748
62930
|
for (let run = pending.shift();run !== undefined; run = pending.shift()) {
|
|
61749
62931
|
if (run.rootRunId !== undefined && run.rootRunId !== root)
|
|
61750
|
-
throw
|
|
62932
|
+
throw outside2(run.id);
|
|
61751
62933
|
ordered.push(run);
|
|
61752
62934
|
for (const child of runs) {
|
|
61753
62935
|
if (child.parentRunId !== run.id)
|
|
61754
62936
|
continue;
|
|
61755
62937
|
const parentStage = run.stages.find((stage) => stage.id === child.parentStageId);
|
|
61756
62938
|
if (authoritativeWorkflowChildRunId(parentStage) !== child.id)
|
|
61757
|
-
throw
|
|
62939
|
+
throw outside2(child.id);
|
|
61758
62940
|
pending.push(child);
|
|
61759
62941
|
}
|
|
61760
62942
|
}
|
|
61761
62943
|
}
|
|
61762
62944
|
const stray = runs.find((run) => !ordered.includes(run));
|
|
61763
62945
|
if (stray !== undefined)
|
|
61764
|
-
throw
|
|
62946
|
+
throw outside2(stray.id);
|
|
61765
62947
|
return ordered;
|
|
61766
62948
|
}
|
|
61767
62949
|
async begin(work) {
|
|
@@ -61816,7 +62998,7 @@ class PoolStageHandoff {
|
|
|
61816
62998
|
}
|
|
61817
62999
|
requireOwned(attempt, runId) {
|
|
61818
63000
|
if (!attempt.runs.has(runId))
|
|
61819
|
-
throw
|
|
63001
|
+
throw outside2(runId);
|
|
61820
63002
|
}
|
|
61821
63003
|
delivery(attempt, work, runId, stageId, stageName) {
|
|
61822
63004
|
const key = JSON.stringify([runId, stageId]);
|
|
@@ -61834,7 +63016,7 @@ class PoolStageHandoff {
|
|
|
61834
63016
|
return delivery;
|
|
61835
63017
|
}
|
|
61836
63018
|
}
|
|
61837
|
-
function
|
|
63019
|
+
function outside2(runId) {
|
|
61838
63020
|
return new Error(`Pool stage publication is outside the work subtree: ${runId}`);
|
|
61839
63021
|
}
|
|
61840
63022
|
function isLive(stage) {
|
|
@@ -61859,33 +63041,122 @@ var init_stage_handoff = __esm(() => {
|
|
|
61859
63041
|
});
|
|
61860
63042
|
|
|
61861
63043
|
// dist/builtin/workflows/src/workers/local-pool.ts
|
|
61862
|
-
import { fork } from "node:child_process";
|
|
61863
|
-
import {
|
|
61864
|
-
import {
|
|
61865
|
-
import { mkdir as mkdir6, rm as rm3, rmdir } from "node:fs/promises";
|
|
61866
|
-
import { dirname as dirname12, join as join28, resolve as resolve11 } from "node:path";
|
|
61867
|
-
import { setTimeout as delay2 } from "node:timers/promises";
|
|
63044
|
+
import { fork as fork2 } from "node:child_process";
|
|
63045
|
+
import { resolve as resolve12 } from "node:path";
|
|
63046
|
+
import { setTimeout as delay3 } from "node:timers/promises";
|
|
61868
63047
|
import { fileURLToPath as fileURLToPath5 } from "node:url";
|
|
61869
63048
|
import { isBunBinary as isBunBinary2 } from "@bastani/atomic";
|
|
61870
|
-
function
|
|
61871
|
-
return
|
|
63049
|
+
function defaultWorkerEntry() {
|
|
63050
|
+
return isBunBinary2 ? INTERNAL_WORKFLOW_WORKER_ARG : fileURLToPath5(new URL("./local-worker-entry.mjs", import.meta.url));
|
|
63051
|
+
}
|
|
63052
|
+
function launchCoordinatorProcess() {
|
|
63053
|
+
const child = fork2(defaultWorkerEntry(), [JSON.stringify({ role: POOL_COORDINATOR_ROLE })], {
|
|
63054
|
+
stdio: ["ignore", "ignore", "pipe", "ipc"],
|
|
63055
|
+
execArgv: []
|
|
63056
|
+
});
|
|
63057
|
+
let stderr = "";
|
|
63058
|
+
child.stderr?.on("data", (chunk) => {
|
|
63059
|
+
stderr = (stderr + chunk.toString()).slice(-COORDINATOR_STDERR_TAIL_CHARACTERS);
|
|
63060
|
+
});
|
|
63061
|
+
const listeners = [];
|
|
63062
|
+
let lostReason;
|
|
63063
|
+
const lose = (reason) => {
|
|
63064
|
+
if (lostReason !== undefined)
|
|
63065
|
+
return;
|
|
63066
|
+
lostReason = reason;
|
|
63067
|
+
for (const listener of listeners)
|
|
63068
|
+
listener(reason);
|
|
63069
|
+
};
|
|
63070
|
+
const exited = new Promise((resolve) => child.once("exit", () => resolve()));
|
|
63071
|
+
child.on("error", lose);
|
|
63072
|
+
child.once("exit", (code, signal) => lose(new Error(`Local worker pool coordinator exited (${signal ?? `code ${code}`})${stderr ? `: ${stderr.trim()}` : ""}`)));
|
|
63073
|
+
return {
|
|
63074
|
+
pid: child.pid,
|
|
63075
|
+
port: {
|
|
63076
|
+
send(message) {
|
|
63077
|
+
if (lostReason !== undefined || !child.connected)
|
|
63078
|
+
return;
|
|
63079
|
+
child.send(message, (error) => {
|
|
63080
|
+
if (error)
|
|
63081
|
+
lose(error);
|
|
63082
|
+
});
|
|
63083
|
+
},
|
|
63084
|
+
onMessage(listener) {
|
|
63085
|
+
child.on("message", (message) => listener(message));
|
|
63086
|
+
},
|
|
63087
|
+
onClose(listener) {
|
|
63088
|
+
if (lostReason !== undefined)
|
|
63089
|
+
listener(lostReason);
|
|
63090
|
+
else
|
|
63091
|
+
listeners.push(listener);
|
|
63092
|
+
},
|
|
63093
|
+
async close() {
|
|
63094
|
+
if (child.connected)
|
|
63095
|
+
child.disconnect();
|
|
63096
|
+
if (child.exitCode !== null || child.signalCode !== null)
|
|
63097
|
+
return;
|
|
63098
|
+
const timedOut = await Promise.race([exited.then(() => false), delay3(COORDINATOR_EXIT_GRACE_MS, true)]);
|
|
63099
|
+
if (timedOut) {
|
|
63100
|
+
child.kill("SIGKILL");
|
|
63101
|
+
await exited;
|
|
63102
|
+
}
|
|
63103
|
+
}
|
|
63104
|
+
}
|
|
63105
|
+
};
|
|
63106
|
+
}
|
|
63107
|
+
function launchCoordinatorInProcess() {
|
|
63108
|
+
const mainListeners = [];
|
|
63109
|
+
const coordinatorListeners = [];
|
|
63110
|
+
const copy = (message) => JSON.parse(JSON.stringify(message));
|
|
63111
|
+
const coordinator = new PoolCoordinator({
|
|
63112
|
+
send(message) {
|
|
63113
|
+
const copied = copy(message);
|
|
63114
|
+
setImmediate(() => {
|
|
63115
|
+
for (const listener of mainListeners)
|
|
63116
|
+
listener(copied);
|
|
63117
|
+
});
|
|
63118
|
+
},
|
|
63119
|
+
onMessage(listener) {
|
|
63120
|
+
coordinatorListeners.push(listener);
|
|
63121
|
+
},
|
|
63122
|
+
onClose() {},
|
|
63123
|
+
async close() {}
|
|
63124
|
+
});
|
|
63125
|
+
return {
|
|
63126
|
+
pid: process.pid,
|
|
63127
|
+
port: {
|
|
63128
|
+
send(message) {
|
|
63129
|
+
const copied = copy(message);
|
|
63130
|
+
setImmediate(() => {
|
|
63131
|
+
for (const listener of coordinatorListeners)
|
|
63132
|
+
listener(copied);
|
|
63133
|
+
});
|
|
63134
|
+
},
|
|
63135
|
+
onMessage(listener) {
|
|
63136
|
+
mainListeners.push(listener);
|
|
63137
|
+
},
|
|
63138
|
+
onClose() {},
|
|
63139
|
+
async close() {
|
|
63140
|
+
coordinator.shutdown();
|
|
63141
|
+
}
|
|
63142
|
+
}
|
|
63143
|
+
};
|
|
61872
63144
|
}
|
|
61873
63145
|
|
|
61874
63146
|
class LocalWorkerPool {
|
|
61875
63147
|
backend;
|
|
61876
63148
|
options;
|
|
61877
|
-
|
|
61878
|
-
|
|
61879
|
-
|
|
61880
|
-
|
|
61881
|
-
|
|
61882
|
-
|
|
61883
|
-
|
|
61884
|
-
|
|
61885
|
-
|
|
61886
|
-
|
|
61887
|
-
|
|
61888
|
-
rootStalledMs = 0;
|
|
63149
|
+
pids = new Set;
|
|
63150
|
+
coordinator;
|
|
63151
|
+
coordinatorLost;
|
|
63152
|
+
commands = new Map;
|
|
63153
|
+
commandSequence = 0;
|
|
63154
|
+
inbound = Promise.resolve();
|
|
63155
|
+
works = new Map;
|
|
63156
|
+
attempts = new Map;
|
|
63157
|
+
results = new Map;
|
|
63158
|
+
changed = Promise.withResolvers();
|
|
63159
|
+
waiting = 0;
|
|
61889
63160
|
ticker;
|
|
61890
63161
|
leaseMs;
|
|
61891
63162
|
desiredControl;
|
|
@@ -61893,18 +63164,17 @@ class LocalWorkerPool {
|
|
|
61893
63164
|
store;
|
|
61894
63165
|
handoff;
|
|
61895
63166
|
releaseRoutes;
|
|
63167
|
+
routeClaims = new Map;
|
|
61896
63168
|
sources = [];
|
|
61897
63169
|
queue;
|
|
61898
63170
|
lease;
|
|
61899
|
-
admission;
|
|
61900
|
-
usageLedger;
|
|
61901
63171
|
usageTotal;
|
|
61902
63172
|
rootBudget;
|
|
63173
|
+
budgetBlocked = false;
|
|
63174
|
+
rootSignature;
|
|
61903
63175
|
starting;
|
|
61904
|
-
cleanupFailure;
|
|
61905
63176
|
failure;
|
|
61906
63177
|
closed = false;
|
|
61907
|
-
replacements = 0;
|
|
61908
63178
|
releaseBinding;
|
|
61909
63179
|
constructor(backend, options) {
|
|
61910
63180
|
this.backend = backend;
|
|
@@ -61919,10 +63189,12 @@ class LocalWorkerPool {
|
|
|
61919
63189
|
this.releaseBinding = bindChildWorkerDispatcher(backend, this, options.rootId);
|
|
61920
63190
|
if (options.admission && options.rootId === undefined)
|
|
61921
63191
|
workflowResourceAdmission.set(backend, options.admission);
|
|
61922
|
-
this.ticker = setInterval(() => this.observeResponsiveness(), ROOT_TICK_INTERVAL_MS).unref();
|
|
61923
63192
|
}
|
|
61924
63193
|
get workerPids() {
|
|
61925
|
-
return [...this.
|
|
63194
|
+
return [...this.pids];
|
|
63195
|
+
}
|
|
63196
|
+
get coordinatorPid() {
|
|
63197
|
+
return this.coordinator?.pid;
|
|
61926
63198
|
}
|
|
61927
63199
|
usage() {
|
|
61928
63200
|
return this.usageTotal;
|
|
@@ -61930,47 +63202,39 @@ class LocalWorkerPool {
|
|
|
61930
63202
|
async prepare(root, store) {
|
|
61931
63203
|
if (this.closed)
|
|
61932
63204
|
throw new Error("Local worker pool is closed");
|
|
61933
|
-
if (!(this.backend.journal instanceof SqlRunJournal) ||
|
|
63205
|
+
if (!(this.backend.journal instanceof SqlRunJournal) || resolve12(this.options.journalPath) !== resolve12(this.backend.journal.path) || sqlPoolStorage(this.backend.journal)?.tables.runs !== "wf_runs")
|
|
61934
63206
|
throw new Error("Local workers require the same default-prefix SQLite journal");
|
|
61935
63207
|
if (this.root !== undefined && this.root !== root)
|
|
61936
63208
|
throw new Error("Local worker pool belongs to another root");
|
|
61937
63209
|
this.root = root;
|
|
61938
63210
|
if (store)
|
|
61939
63211
|
this.store = store;
|
|
61940
|
-
this.usageLedger ??= await openSqlPoolUsage(this.backend.journal);
|
|
61941
|
-
this.usageTotal = await this.usageLedger.total(root);
|
|
61942
63212
|
const state = await this.backend.journal.leaseState(root);
|
|
61943
63213
|
if (!state || state.holderId !== this.backend.holderId)
|
|
61944
63214
|
throw new Error("Local worker pool requires the root owner");
|
|
61945
63215
|
this.lease = { runId: root, holderId: state.holderId, epoch: state.epoch };
|
|
61946
|
-
const
|
|
61947
|
-
await
|
|
61948
|
-
|
|
61949
|
-
|
|
61950
|
-
|
|
61951
|
-
|
|
61952
|
-
|
|
61953
|
-
|
|
61954
|
-
|
|
61955
|
-
|
|
63216
|
+
const workerEntry = this.options.workerEntry ?? (isBunBinary2 || this.options.coordinator === "in-process" ? defaultWorkerEntry() : undefined);
|
|
63217
|
+
this.usageTotal = await this.command({
|
|
63218
|
+
kind: "prepare",
|
|
63219
|
+
root,
|
|
63220
|
+
lease: this.lease,
|
|
63221
|
+
options: {
|
|
63222
|
+
journalPath: this.options.journalPath,
|
|
63223
|
+
workers: this.options.workers,
|
|
63224
|
+
leaseMs: this.leaseMs,
|
|
63225
|
+
startupTimeoutMs: this.options.startupTimeoutMs ?? DEFAULT_STARTUP_TIMEOUT_MS,
|
|
63226
|
+
...this.options.admission === undefined ? {} : { admission: this.options.admission },
|
|
63227
|
+
...workerEntry === undefined ? {} : { workerEntry }
|
|
63228
|
+
},
|
|
63229
|
+
...this.desiredControl === undefined ? {} : { control: this.desiredControl }
|
|
63230
|
+
});
|
|
63231
|
+
this.queue ??= await openSqlPoolQueue(this.backend.journal);
|
|
61956
63232
|
}
|
|
61957
63233
|
async control(state) {
|
|
61958
63234
|
this.desiredControl = state;
|
|
61959
63235
|
try {
|
|
61960
|
-
if (
|
|
61961
|
-
|
|
61962
|
-
const control = await this.queue.setControl(this.lease, state);
|
|
61963
|
-
const deadline = Date.now() + CONTROL_ACKNOWLEDGEMENT_TIMEOUT_MS;
|
|
61964
|
-
while ([...this.readyWorkers].some((worker) => (this.controlAcks.get(worker) ?? -1) < control.revision)) {
|
|
61965
|
-
if (this.desiredControl !== state)
|
|
61966
|
-
return;
|
|
61967
|
-
if (Date.now() >= deadline) {
|
|
61968
|
-
if (state === "quit")
|
|
61969
|
-
break;
|
|
61970
|
-
throw new Error("Worker control acknowledgement unresolved");
|
|
61971
|
-
}
|
|
61972
|
-
await delay2(25);
|
|
61973
|
-
}
|
|
63236
|
+
if (this.coordinator !== undefined)
|
|
63237
|
+
await this.command({ kind: "control", state });
|
|
61974
63238
|
} finally {
|
|
61975
63239
|
if (state === "quit")
|
|
61976
63240
|
await this.close();
|
|
@@ -62016,290 +63280,61 @@ class LocalWorkerPool {
|
|
|
62016
63280
|
throw new Error(`Worker definition is not importable: ${binding.exportName}`);
|
|
62017
63281
|
return source;
|
|
62018
63282
|
}));
|
|
62019
|
-
|
|
62020
|
-
|
|
63283
|
+
const cwd = String(this.backend.getWorkflow(root)?.invocationCwd ?? process.cwd());
|
|
63284
|
+
const store = this.store;
|
|
63285
|
+
if (store) {
|
|
63286
|
+
this.releaseRoutes = await acquireStandaloneIntercom(store, root, cwd);
|
|
62021
63287
|
if (this.closed) {
|
|
62022
63288
|
await this.releaseRoutes();
|
|
62023
63289
|
this.releaseRoutes = undefined;
|
|
62024
63290
|
throw new Error("Local worker pool is closed");
|
|
62025
63291
|
}
|
|
62026
|
-
|
|
62027
|
-
|
|
62028
|
-
|
|
62029
|
-
this.
|
|
62030
|
-
|
|
62031
|
-
root,
|
|
62032
|
-
holder: "coordinator"
|
|
63292
|
+
const routeOwner = {
|
|
63293
|
+
notifyUndeliverable: async (payload) => !this.closed && await this.command({ kind: "undeliverable", payload }) === true
|
|
63294
|
+
};
|
|
63295
|
+
this.handoff = new PoolStageHandoff(this.backend, store, this.queueView(), this.lease, (runId) => {
|
|
63296
|
+
this.routeClaims.set(runId, claimExternalRunRoute(store, runId, routeOwner));
|
|
62033
63297
|
});
|
|
62034
|
-
this.logDirectory = workerLogDirectory(this.options.journalPath, root);
|
|
62035
|
-
await mkdir6(this.logDirectory, { recursive: true }).catch(() => {
|
|
62036
|
-
return;
|
|
62037
|
-
});
|
|
62038
|
-
const workers = Array.from({ length: this.options.workers }, () => this.spawn(root));
|
|
62039
|
-
try {
|
|
62040
|
-
await Promise.any(workers);
|
|
62041
|
-
} catch (error) {
|
|
62042
|
-
throw error.errors[0];
|
|
62043
63298
|
}
|
|
62044
|
-
|
|
62045
|
-
|
|
62046
|
-
|
|
62047
|
-
|
|
62048
|
-
|
|
62049
|
-
|
|
62050
|
-
journalPath: this.options.journalPath,
|
|
62051
|
-
root,
|
|
62052
|
-
definitions: this.sources,
|
|
62053
|
-
admission: this.options.admission
|
|
62054
|
-
})
|
|
62055
|
-
], { stdio: ["ignore", "ignore", "pipe", "ipc"], execArgv: [] });
|
|
62056
|
-
const disconnected = new Promise((resolve) => child.once("disconnect", () => resolve()));
|
|
62057
|
-
this.children.add(child);
|
|
62058
|
-
const worker = randomUUID15();
|
|
62059
|
-
this.incarnations.set(worker, child);
|
|
62060
|
-
let ready = false;
|
|
62061
|
-
let serial = Promise.resolve();
|
|
62062
|
-
let work;
|
|
62063
|
-
const deliveries = new Map;
|
|
62064
|
-
const logPath = join28(this.logDirectory, `${worker}.log`);
|
|
62065
|
-
this.logs.set(child, logPath);
|
|
62066
|
-
let logFailure;
|
|
62067
|
-
const log = createWriteStream(logPath, { flags: "a" });
|
|
62068
|
-
log.on("error", (error) => {
|
|
62069
|
-
logFailure = error.message;
|
|
62070
|
-
});
|
|
62071
|
-
let stderr = "";
|
|
62072
|
-
child.stderr?.on("data", (chunk) => {
|
|
62073
|
-
stderr = (stderr + chunk.toString()).slice(-STDERR_TAIL_CHARACTERS);
|
|
62074
|
-
});
|
|
62075
|
-
child.stderr?.on("error", (error) => {
|
|
62076
|
-
logFailure ??= error.message;
|
|
62077
|
-
log.end();
|
|
62078
|
-
});
|
|
62079
|
-
child.stderr?.pipe(log);
|
|
62080
|
-
const diagnosed = (summary) => new Error(`${summary} (worker stderr log: ${logFailure === undefined ? logPath : `unavailable, ${logFailure}`})${stderr ? `: ${stderr}` : ""}`);
|
|
62081
|
-
const startupTimeoutMs = this.options.startupTimeoutMs ?? DEFAULT_STARTUP_TIMEOUT_MS;
|
|
62082
|
-
let timedOut = false;
|
|
62083
|
-
let terminated = false;
|
|
62084
|
-
const terminate = () => {
|
|
62085
|
-
if (terminated)
|
|
62086
|
-
return;
|
|
62087
|
-
terminated = true;
|
|
62088
|
-
child.kill("SIGKILL");
|
|
62089
|
-
};
|
|
62090
|
-
const replied = (error) => {
|
|
62091
|
-
if (error)
|
|
62092
|
-
terminate();
|
|
62093
|
-
};
|
|
62094
|
-
return new Promise((resolve, reject) => {
|
|
62095
|
-
const startup = setTimeout(() => {
|
|
62096
|
-
timedOut = true;
|
|
62097
|
-
terminate();
|
|
62098
|
-
}, startupTimeoutMs);
|
|
62099
|
-
child.on("error", (error) => {
|
|
62100
|
-
clearTimeout(startup);
|
|
62101
|
-
reject(error);
|
|
62102
|
-
terminate();
|
|
62103
|
-
});
|
|
62104
|
-
const closed = new Promise((resolve) => child.once("close", () => resolve()));
|
|
62105
|
-
child.once("exit", () => {
|
|
62106
|
-
Promise.race([closed, delay2(STDERR_DRAIN_GRACE_MS)]).then(() => reject(diagnosed(timedOut ? `Local worker startup timed out after ${startupTimeoutMs} ms` : "Local worker exited before readiness")));
|
|
62107
|
-
this.children.delete(child);
|
|
62108
|
-
this.logs.delete(child);
|
|
62109
|
-
clearTimeout(startup);
|
|
62110
|
-
this.incarnations.delete(worker);
|
|
62111
|
-
this.readyWorkers.delete(worker);
|
|
62112
|
-
this.controlAcks.delete(worker);
|
|
62113
|
-
this.lastHeard.delete(worker);
|
|
62114
|
-
if (work)
|
|
62115
|
-
this.handoff?.fence(work);
|
|
62116
|
-
for (const delivery of deliveries.values())
|
|
62117
|
-
delivery.reject(new Error("Worker exited before delivery acknowledgement"));
|
|
62118
|
-
deliveries.clear();
|
|
62119
|
-
const cleanup = disconnected.then(() => serial).then(async () => {
|
|
62120
|
-
try {
|
|
62121
|
-
if (await this.admission?.releaseStoppedSessions(worker))
|
|
62122
|
-
throw new Error("Local worker process cleanup unconfirmed; permits quarantined");
|
|
62123
|
-
} catch (error) {
|
|
62124
|
-
this.cleanupFailure = error instanceof Error ? error : new Error(String(error));
|
|
62125
|
-
throw this.cleanupFailure;
|
|
62126
|
-
}
|
|
62127
|
-
}).then(async () => {
|
|
62128
|
-
if (this.closed || !ready || this.desiredControl === "quit")
|
|
62129
|
-
return;
|
|
62130
|
-
if (work && await this.queue.result(root, work.id)) {
|
|
62131
|
-
work = undefined;
|
|
62132
|
-
this.replacements = 0;
|
|
62133
|
-
}
|
|
62134
|
-
if (this.closed)
|
|
62135
|
-
return;
|
|
62136
|
-
if (work)
|
|
62137
|
-
await this.queue.releaseExited(this.lease, work);
|
|
62138
|
-
const exhausted = work ? work.generation >= 3 : ++this.replacements > 3;
|
|
62139
|
-
if (exhausted)
|
|
62140
|
-
this.failure = diagnosed("Local worker retry limit");
|
|
62141
|
-
else
|
|
62142
|
-
await this.spawn(root);
|
|
62143
|
-
}).catch((error) => {
|
|
62144
|
-
this.failure = error;
|
|
62145
|
-
});
|
|
62146
|
-
this.cleanups.add(cleanup);
|
|
62147
|
-
cleanup.finally(() => this.cleanups.delete(cleanup));
|
|
62148
|
-
});
|
|
62149
|
-
child.on("message", (message) => {
|
|
62150
|
-
this.observeResponsiveness();
|
|
62151
|
-
this.lastHeard.set(worker, { at: performance.now(), rootStalled: this.rootStalledMs });
|
|
62152
|
-
if (message.kind === "delivery-ack") {
|
|
62153
|
-
const pending = deliveries.get(message.deliveryId);
|
|
62154
|
-
deliveries.delete(message.deliveryId);
|
|
62155
|
-
if (message.error)
|
|
62156
|
-
pending?.reject(new Error(message.error));
|
|
62157
|
-
else
|
|
62158
|
-
pending?.resolve();
|
|
62159
|
-
return;
|
|
62160
|
-
}
|
|
62161
|
-
const response = serial.then(async () => {
|
|
62162
|
-
if (message.kind === "ready") {
|
|
62163
|
-
if (terminated)
|
|
62164
|
-
return;
|
|
62165
|
-
ready = true;
|
|
62166
|
-
this.readyWorkers.add(worker);
|
|
62167
|
-
clearTimeout(startup);
|
|
62168
|
-
resolve();
|
|
62169
|
-
return;
|
|
62170
|
-
}
|
|
62171
|
-
await this.starting;
|
|
62172
|
-
const queue = this.queue;
|
|
62173
|
-
const lease = this.lease;
|
|
62174
|
-
if (message.kind === "control")
|
|
62175
|
-
return queue.control(lease);
|
|
62176
|
-
if (message.kind === "controlled") {
|
|
62177
|
-
const control = await queue.control(lease);
|
|
62178
|
-
if (message.revision > control.revision)
|
|
62179
|
-
throw new Error("Unknown control revision");
|
|
62180
|
-
this.controlAcks.set(worker, Math.max(this.controlAcks.get(worker) ?? -1, message.revision));
|
|
62181
|
-
return;
|
|
62182
|
-
}
|
|
62183
|
-
if (message.kind === "claim") {
|
|
62184
|
-
if (work)
|
|
62185
|
-
throw new Error("Worker already holds an attempt");
|
|
62186
|
-
const decision = this.rootBudget?.checkpoint();
|
|
62187
|
-
if (decision?.kind === "exhausted" || decision?.kind === "wrap_up")
|
|
62188
|
-
return;
|
|
62189
|
-
await this.recoverExpired(root);
|
|
62190
|
-
work = await queue.claim(lease, worker, this.sources.map((source) => ({
|
|
62191
|
-
definition: `${source.path}#${source.exportName}`,
|
|
62192
|
-
digest: source.digest
|
|
62193
|
-
})), this.leaseMs);
|
|
62194
|
-
return work;
|
|
62195
|
-
}
|
|
62196
|
-
if (!work)
|
|
62197
|
-
throw new Error("Worker has no active attempt");
|
|
62198
|
-
if (message.kind === "renew" || work.expiresAt - Date.now() < this.leaseMs / 2)
|
|
62199
|
-
work = await queue.extend(lease, work, this.leaseMs);
|
|
62200
|
-
if (message.kind === "activity")
|
|
62201
|
-
return this.handoff?.publish(work, message.runs);
|
|
62202
|
-
if (message.kind === "route")
|
|
62203
|
-
return this.handoff?.route(work, message.runId);
|
|
62204
|
-
if (message.kind === "delivery-ready")
|
|
62205
|
-
return this.handoff?.ready(work, message.runId, message.stageId, message.stageName, message.sessionId);
|
|
62206
|
-
if (message.kind === "delivery") {
|
|
62207
|
-
if (!message.recipient)
|
|
62208
|
-
throw new Error("Worker delivery requires a session identity");
|
|
62209
|
-
return this.handoff?.deliver(work, message.runId, message.stageId, message.stageName, message.recipient, (from, payload) => new Promise((resolve, reject) => {
|
|
62210
|
-
const id = randomUUID15();
|
|
62211
|
-
deliveries.set(id, { resolve, reject });
|
|
62212
|
-
child.send({ id: message.id, delivery: { id, from, message: payload } }, (error) => {
|
|
62213
|
-
if (error) {
|
|
62214
|
-
deliveries.delete(id);
|
|
62215
|
-
reject(error);
|
|
62216
|
-
}
|
|
62217
|
-
});
|
|
62218
|
-
}));
|
|
62219
|
-
}
|
|
62220
|
-
if (message.kind === "renew")
|
|
62221
|
-
return;
|
|
62222
|
-
if (message.kind === "commit") {
|
|
62223
|
-
const receipt = await queue.commit(lease, work, message.operation, message.mutation);
|
|
62224
|
-
this.usageTotal = await this.usageLedger.total(root);
|
|
62225
|
-
return receipt;
|
|
62226
|
-
}
|
|
62227
|
-
if (message.kind === "usage") {
|
|
62228
|
-
this.usageTotal = await this.usageLedger.report(lease, work, message.reports);
|
|
62229
|
-
return;
|
|
62230
|
-
}
|
|
62231
|
-
if (message.kind === "budget") {
|
|
62232
|
-
await queue.assertCurrent(lease, work);
|
|
62233
|
-
const decision = this.rootBudget?.checkpoint();
|
|
62234
|
-
if (decision?.kind === "exhausted" || decision?.kind === "wrap_up")
|
|
62235
|
-
throw new Error("Root workflow budget exhausted");
|
|
62236
|
-
return queue.control(lease);
|
|
62237
|
-
}
|
|
62238
|
-
if (message.kind !== "complete")
|
|
62239
|
-
throw new Error("Invalid worker operation");
|
|
62240
|
-
await queue.complete(lease, work, message.result);
|
|
62241
|
-
this.replacements = 0;
|
|
62242
|
-
work = undefined;
|
|
62243
|
-
});
|
|
62244
|
-
if (message.kind !== "delivery" && message.kind !== "delivery-ready")
|
|
62245
|
-
serial = response.then(() => {}, () => {});
|
|
62246
|
-
response.then((value2) => child.send({ id: message.id, value: value2 }, replied), (error) => child.send({ id: message.id, error: error.message }, replied));
|
|
62247
|
-
});
|
|
63299
|
+
await this.command({
|
|
63300
|
+
kind: "start",
|
|
63301
|
+
sources: this.sources,
|
|
63302
|
+
cwd,
|
|
63303
|
+
publish: store !== undefined,
|
|
63304
|
+
...this.rootRun()
|
|
62248
63305
|
});
|
|
62249
63306
|
}
|
|
62250
|
-
|
|
62251
|
-
|
|
62252
|
-
|
|
62253
|
-
|
|
62254
|
-
|
|
62255
|
-
|
|
62256
|
-
|
|
62257
|
-
|
|
62258
|
-
|
|
62259
|
-
|
|
62260
|
-
if (work.state !== "leased" || work.expiresAt >= expiredBefore)
|
|
62261
|
-
continue;
|
|
62262
|
-
const child = work.worker === null ? undefined : this.incarnations.get(work.worker);
|
|
62263
|
-
if (child && work.worker !== null && this.isLive(work.worker, child)) {
|
|
62264
|
-
await this.queue.extend(this.lease, work, this.leaseMs).catch((error) => {
|
|
62265
|
-
if (error.message !== STALE_WORK_LEASE)
|
|
62266
|
-
throw error;
|
|
62267
|
-
});
|
|
62268
|
-
continue;
|
|
62269
|
-
}
|
|
62270
|
-
if (child && !await this.stop(child).then(() => true, () => false))
|
|
62271
|
-
continue;
|
|
62272
|
-
await this.queue.expire(this.lease, work);
|
|
62273
|
-
}
|
|
62274
|
-
})().finally(() => {
|
|
62275
|
-
this.recovering = undefined;
|
|
62276
|
-
this.nextRecoveryAt = performance.now() + RECOVERY_INTERVAL_MS;
|
|
62277
|
-
});
|
|
62278
|
-
return this.recovering;
|
|
63307
|
+
queueView() {
|
|
63308
|
+
return {
|
|
63309
|
+
assertCurrent: async (_lease, work) => {
|
|
63310
|
+
const current = this.attempts.get(work.id);
|
|
63311
|
+
if (current === undefined || current.fenced || current.generation !== work.generation || current.worker !== work.worker || current.epoch !== work.epoch)
|
|
63312
|
+
throw new Error(STALE_WORK_LEASE);
|
|
63313
|
+
},
|
|
63314
|
+
withCurrentAppend: (lease, work, assertSession, action) => this.queue.withCurrentAppend(lease, work, assertSession, action),
|
|
63315
|
+
result: async (root, id) => this.results.get(id) ?? this.queue.result(root, id)
|
|
63316
|
+
};
|
|
62279
63317
|
}
|
|
62280
|
-
|
|
62281
|
-
const
|
|
62282
|
-
|
|
62283
|
-
if (gap > ROOT_STALL_THRESHOLD_MS)
|
|
62284
|
-
this.rootStalledMs += gap - ROOT_TICK_INTERVAL_MS;
|
|
62285
|
-
this.lastTick = now;
|
|
63318
|
+
rootProjection() {
|
|
63319
|
+
const run = this.store?.runs().find((candidate) => candidate.id === this.root);
|
|
63320
|
+
return run === undefined ? undefined : projectRootRun(run);
|
|
62286
63321
|
}
|
|
62287
|
-
|
|
62288
|
-
this.
|
|
62289
|
-
|
|
62290
|
-
|
|
63322
|
+
rootRun() {
|
|
63323
|
+
const rootRun = this.rootProjection();
|
|
63324
|
+
if (rootRun === undefined)
|
|
63325
|
+
return {};
|
|
63326
|
+
this.rootSignature = JSON.stringify(rootRun);
|
|
63327
|
+
return { rootRun };
|
|
62291
63328
|
}
|
|
62292
|
-
|
|
62293
|
-
|
|
62294
|
-
|
|
62295
|
-
|
|
62296
|
-
|
|
62297
|
-
|
|
62298
|
-
|
|
62299
|
-
|
|
62300
|
-
|
|
62301
|
-
child.kill("SIGKILL");
|
|
62302
|
-
});
|
|
63329
|
+
refreshRootRun() {
|
|
63330
|
+
const rootRun = this.rootProjection();
|
|
63331
|
+
if (rootRun === undefined)
|
|
63332
|
+
return;
|
|
63333
|
+
const signature = JSON.stringify(rootRun);
|
|
63334
|
+
if (signature === this.rootSignature)
|
|
63335
|
+
return;
|
|
63336
|
+
this.rootSignature = signature;
|
|
63337
|
+
this.notice({ kind: "root", rootRun });
|
|
62303
63338
|
}
|
|
62304
63339
|
async run(definition, inputs, options) {
|
|
62305
63340
|
try {
|
|
@@ -62332,7 +63367,7 @@ class LocalWorkerPool {
|
|
|
62332
63367
|
const selectedSource = this.sources[index];
|
|
62333
63368
|
if (await definitionDigest(selectedSource.path) !== selectedSource.digest)
|
|
62334
63369
|
throw new Error("Worker definition compatibility error");
|
|
62335
|
-
await this.backend.
|
|
63370
|
+
await this.backend.catchUpWorkflow(root);
|
|
62336
63371
|
const boundary = this.backend.listCheckpoints(root).find((entry) => entry.kind === "stage" && entry.topology?.boundary?.event === "start" && entry.topology.stageId === options.parentRun.stageId);
|
|
62337
63372
|
if (boundary?.kind !== "stage")
|
|
62338
63373
|
throw new Error("Missing durable child boundary");
|
|
@@ -62343,39 +63378,232 @@ class LocalWorkerPool {
|
|
|
62343
63378
|
config: options.config,
|
|
62344
63379
|
executionMode: options.executionMode
|
|
62345
63380
|
}));
|
|
62346
|
-
|
|
62347
|
-
|
|
62348
|
-
|
|
62349
|
-
|
|
62350
|
-
|
|
62351
|
-
|
|
62352
|
-
|
|
62353
|
-
|
|
62354
|
-
|
|
62355
|
-
|
|
62356
|
-
|
|
62357
|
-
|
|
63381
|
+
this.evaluateBudget();
|
|
63382
|
+
const enqueued = await this.command({
|
|
63383
|
+
kind: "enqueue",
|
|
63384
|
+
descriptor: {
|
|
63385
|
+
boundaryId: options.parentRun.stageId,
|
|
63386
|
+
childRunId: options.runId,
|
|
63387
|
+
scope: options.durableScope.scopePrefix,
|
|
63388
|
+
definition: `${selectedSource.path}#${selectedSource.exportName}`,
|
|
63389
|
+
digest: selectedSource.digest,
|
|
63390
|
+
inputs,
|
|
63391
|
+
execution
|
|
63392
|
+
},
|
|
63393
|
+
boundary,
|
|
63394
|
+
...this.rootRun()
|
|
63395
|
+
});
|
|
63396
|
+
const work = enqueued.work;
|
|
63397
|
+
this.works.set(work.id, work);
|
|
63398
|
+
if (enqueued.result !== undefined && !this.results.has(work.id))
|
|
63399
|
+
this.results.set(work.id, enqueued.result);
|
|
63400
|
+
this.waiting++;
|
|
63401
|
+
this.ticker ??= setInterval(() => {
|
|
63402
|
+
this.evaluateBudget();
|
|
63403
|
+
this.signalChange();
|
|
63404
|
+
}, POOL_SUMMARY_INTERVAL_MS).unref();
|
|
63405
|
+
try {
|
|
63406
|
+
for (;; ) {
|
|
63407
|
+
options.signal?.throwIfAborted();
|
|
63408
|
+
if (this.closed) {
|
|
63409
|
+
if (this.failure)
|
|
63410
|
+
throw this.failure;
|
|
63411
|
+
if (this.desiredControl === "quit")
|
|
63412
|
+
throw new WorkflowGracefulQuitError(root, "worker pool teardown");
|
|
63413
|
+
throw new Error("Local worker pool is closed");
|
|
63414
|
+
}
|
|
63415
|
+
const result = this.results.get(work.id);
|
|
63416
|
+
if (result) {
|
|
63417
|
+
await this.inbound;
|
|
63418
|
+
await this.backend.catchUpWorkflow(root);
|
|
63419
|
+
await this.handoff?.complete(work);
|
|
63420
|
+
return result;
|
|
63421
|
+
}
|
|
62358
63422
|
if (this.failure)
|
|
62359
63423
|
throw this.failure;
|
|
62360
|
-
if (this.
|
|
62361
|
-
|
|
62362
|
-
|
|
63424
|
+
if (this.budgetBlocked)
|
|
63425
|
+
await this.rootBudget.stopAtBoundaryAsync(definition.name);
|
|
63426
|
+
await this.nextChange(options.signal);
|
|
62363
63427
|
}
|
|
62364
|
-
|
|
62365
|
-
if (
|
|
62366
|
-
|
|
62367
|
-
|
|
62368
|
-
|
|
63428
|
+
} finally {
|
|
63429
|
+
if (--this.waiting === 0) {
|
|
63430
|
+
clearInterval(this.ticker);
|
|
63431
|
+
this.ticker = undefined;
|
|
63432
|
+
}
|
|
63433
|
+
}
|
|
63434
|
+
}
|
|
63435
|
+
evaluateBudget() {
|
|
63436
|
+
const decision = this.rootBudget?.checkpoint();
|
|
63437
|
+
const blocked = decision?.kind === "exhausted" || decision?.kind === "wrap_up";
|
|
63438
|
+
if (blocked === this.budgetBlocked)
|
|
63439
|
+
return;
|
|
63440
|
+
this.budgetBlocked = blocked;
|
|
63441
|
+
this.notice({ kind: "budget", blocked });
|
|
63442
|
+
}
|
|
63443
|
+
signalChange() {
|
|
63444
|
+
const changed = this.changed;
|
|
63445
|
+
this.changed = Promise.withResolvers();
|
|
63446
|
+
changed.resolve();
|
|
63447
|
+
}
|
|
63448
|
+
nextChange(signal) {
|
|
63449
|
+
const changed = this.changed.promise;
|
|
63450
|
+
if (signal === undefined)
|
|
63451
|
+
return changed;
|
|
63452
|
+
return new Promise((resolve, reject) => {
|
|
63453
|
+
const abort = () => reject(signal.reason);
|
|
63454
|
+
signal.addEventListener("abort", abort, { once: true });
|
|
63455
|
+
changed.then(() => {
|
|
63456
|
+
signal.removeEventListener("abort", abort);
|
|
63457
|
+
resolve();
|
|
63458
|
+
});
|
|
63459
|
+
});
|
|
63460
|
+
}
|
|
63461
|
+
connect() {
|
|
63462
|
+
if (this.coordinator !== undefined)
|
|
63463
|
+
return this.coordinator;
|
|
63464
|
+
const coordinator = this.options.coordinator === "in-process" ? launchCoordinatorInProcess() : launchCoordinatorProcess();
|
|
63465
|
+
this.coordinator = coordinator;
|
|
63466
|
+
coordinator.port.onMessage((event) => {
|
|
63467
|
+
this.inbound = this.inbound.then(() => this.receive(event)).catch(() => {});
|
|
63468
|
+
});
|
|
63469
|
+
coordinator.port.onClose((reason) => {
|
|
63470
|
+
this.coordinatorLost = reason;
|
|
63471
|
+
this.inbound = this.inbound.then(() => this.lost(reason));
|
|
63472
|
+
});
|
|
63473
|
+
return coordinator;
|
|
63474
|
+
}
|
|
63475
|
+
lost(reason) {
|
|
63476
|
+
this.pids.clear();
|
|
63477
|
+
if (!this.closed)
|
|
63478
|
+
this.failure ??= reason;
|
|
63479
|
+
for (const pending of this.commands.values())
|
|
63480
|
+
pending.reject(reason);
|
|
63481
|
+
this.commands.clear();
|
|
63482
|
+
this.signalChange();
|
|
63483
|
+
}
|
|
63484
|
+
command(body) {
|
|
63485
|
+
const coordinator = this.connect();
|
|
63486
|
+
if (this.coordinatorLost !== undefined)
|
|
63487
|
+
return Promise.reject(this.coordinatorLost);
|
|
63488
|
+
const id = ++this.commandSequence;
|
|
63489
|
+
return new Promise((resolve, reject) => {
|
|
63490
|
+
this.commands.set(id, { resolve, reject });
|
|
63491
|
+
coordinator.port.send({ ...body, id });
|
|
63492
|
+
});
|
|
63493
|
+
}
|
|
63494
|
+
notice(notice) {
|
|
63495
|
+
this.coordinator?.port.send(notice);
|
|
63496
|
+
}
|
|
63497
|
+
async receive(event) {
|
|
63498
|
+
switch (event.kind) {
|
|
63499
|
+
case "reply": {
|
|
63500
|
+
const pending = this.commands.get(event.id);
|
|
63501
|
+
this.commands.delete(event.id);
|
|
63502
|
+
if (event.error !== undefined)
|
|
63503
|
+
pending?.reject(new Error(event.error));
|
|
63504
|
+
else
|
|
63505
|
+
pending?.resolve(event.value);
|
|
63506
|
+
return;
|
|
63507
|
+
}
|
|
63508
|
+
case "activity":
|
|
63509
|
+
return this.applyActivity(event.batch, event.entries, event.usage);
|
|
63510
|
+
case "spawned":
|
|
63511
|
+
this.pids.add(event.pid);
|
|
63512
|
+
return;
|
|
63513
|
+
case "exited":
|
|
63514
|
+
this.pids.delete(event.pid);
|
|
63515
|
+
return;
|
|
63516
|
+
case "fence": {
|
|
63517
|
+
const current = this.attempts.get(event.work.id);
|
|
63518
|
+
if (current?.generation === event.work.generation)
|
|
63519
|
+
current.fenced = true;
|
|
63520
|
+
this.handoff?.fence(event.work);
|
|
63521
|
+
return;
|
|
62369
63522
|
}
|
|
62370
|
-
|
|
62371
|
-
|
|
62372
|
-
|
|
62373
|
-
|
|
62374
|
-
|
|
62375
|
-
|
|
62376
|
-
|
|
63523
|
+
case "result":
|
|
63524
|
+
if (event.usage !== undefined)
|
|
63525
|
+
this.usageTotal = event.usage;
|
|
63526
|
+
this.results.set(event.workId, event.result);
|
|
63527
|
+
this.signalChange();
|
|
63528
|
+
return;
|
|
63529
|
+
case "failure":
|
|
63530
|
+
this.failure ??= new Error(event.error);
|
|
63531
|
+
this.signalChange();
|
|
63532
|
+
return;
|
|
63533
|
+
case "request":
|
|
63534
|
+
this.serve(event.id, event.request);
|
|
63535
|
+
return;
|
|
62377
63536
|
}
|
|
62378
63537
|
}
|
|
63538
|
+
async applyActivity(batch, entries, usage) {
|
|
63539
|
+
if (usage !== undefined)
|
|
63540
|
+
this.usageTotal = usage;
|
|
63541
|
+
const rejected = [];
|
|
63542
|
+
const handoff = this.handoff;
|
|
63543
|
+
if (handoff !== undefined) {
|
|
63544
|
+
const publications = entries.flatMap((entry) => {
|
|
63545
|
+
const work = this.track(entry.work);
|
|
63546
|
+
if (work !== undefined)
|
|
63547
|
+
return [{ work, runs: entry.runs }];
|
|
63548
|
+
rejected.push({ workId: entry.work.id, generation: entry.work.generation, error: STALE_WORK_LEASE });
|
|
63549
|
+
return [];
|
|
63550
|
+
});
|
|
63551
|
+
const outcomes = await handoff.publishBatch(publications);
|
|
63552
|
+
outcomes.forEach((error, index) => {
|
|
63553
|
+
const { work } = publications[index];
|
|
63554
|
+
if (error !== undefined)
|
|
63555
|
+
rejected.push({ workId: work.id, generation: work.generation, error: error.message });
|
|
63556
|
+
});
|
|
63557
|
+
}
|
|
63558
|
+
this.notice({ kind: "published", batch, rejected });
|
|
63559
|
+
this.refreshRootRun();
|
|
63560
|
+
}
|
|
63561
|
+
track(attempt) {
|
|
63562
|
+
const work = this.works.get(attempt.id);
|
|
63563
|
+
if (work === undefined)
|
|
63564
|
+
return;
|
|
63565
|
+
const current = this.attempts.get(attempt.id);
|
|
63566
|
+
if (current === undefined || attempt.generation > current.generation)
|
|
63567
|
+
this.attempts.set(attempt.id, { ...attempt, fenced: false });
|
|
63568
|
+
return { ...work, ...attempt, state: "leased" };
|
|
63569
|
+
}
|
|
63570
|
+
async serve(id, request) {
|
|
63571
|
+
try {
|
|
63572
|
+
const value2 = await this.answer(id, request);
|
|
63573
|
+
this.coordinator?.port.send({ kind: "request-reply", id, ...value2 === undefined ? {} : { value: value2 } });
|
|
63574
|
+
} catch (error) {
|
|
63575
|
+
this.coordinator?.port.send({
|
|
63576
|
+
kind: "request-reply",
|
|
63577
|
+
id,
|
|
63578
|
+
error: error instanceof Error ? error.message : String(error)
|
|
63579
|
+
});
|
|
63580
|
+
}
|
|
63581
|
+
}
|
|
63582
|
+
async answer(id, request) {
|
|
63583
|
+
if (request.kind === "pending-stage-message" || request.kind === "sticky-live-delivered") {
|
|
63584
|
+
const router = this.store === undefined ? undefined : pendingStageMessageRouter(this.store);
|
|
63585
|
+
const event = {
|
|
63586
|
+
...request.payload,
|
|
63587
|
+
handled: false
|
|
63588
|
+
};
|
|
63589
|
+
if (request.kind === "pending-stage-message")
|
|
63590
|
+
router?.route(event);
|
|
63591
|
+
else
|
|
63592
|
+
router?.sticky(event);
|
|
63593
|
+
if (event.handled && event.completion !== undefined)
|
|
63594
|
+
return await event.completion;
|
|
63595
|
+
return request.kind === "pending-stage-message" ? { outcome: "refused", reason: "Session not found" } : undefined;
|
|
63596
|
+
}
|
|
63597
|
+
const handoff = this.handoff;
|
|
63598
|
+
const work = this.track(request.work);
|
|
63599
|
+
if (handoff === undefined || work === undefined)
|
|
63600
|
+
throw new Error(STALE_WORK_LEASE);
|
|
63601
|
+
if (request.kind === "ready")
|
|
63602
|
+
return handoff.ready(work, request.runId, request.stageId, request.stageName, request.sessionId);
|
|
63603
|
+
return handoff.deliver(work, request.runId, request.stageId, request.stageName, request.recipient, async (from, message) => {
|
|
63604
|
+
await this.command({ kind: "transport", request: id, from, message });
|
|
63605
|
+
});
|
|
63606
|
+
}
|
|
62379
63607
|
closing;
|
|
62380
63608
|
close() {
|
|
62381
63609
|
this.closing ??= this.dispose();
|
|
@@ -62384,27 +63612,28 @@ class LocalWorkerPool {
|
|
|
62384
63612
|
async dispose() {
|
|
62385
63613
|
this.closed = true;
|
|
62386
63614
|
clearInterval(this.ticker);
|
|
63615
|
+
this.ticker = undefined;
|
|
62387
63616
|
this.releaseBinding();
|
|
62388
63617
|
if (this.options.rootId === undefined)
|
|
62389
63618
|
workflowResourceAdmission.delete(this.backend);
|
|
63619
|
+
this.signalChange();
|
|
62390
63620
|
let failure;
|
|
62391
|
-
|
|
62392
|
-
|
|
62393
|
-
|
|
62394
|
-
|
|
62395
|
-
|
|
62396
|
-
|
|
62397
|
-
|
|
62398
|
-
|
|
62399
|
-
|
|
62400
|
-
|
|
62401
|
-
|
|
62402
|
-
|
|
62403
|
-
const stopped = stopping.flatMap(({ child, log }) => log !== undefined && (child.exitCode !== null || child.signalCode !== null) ? [log] : []);
|
|
62404
|
-
await Promise.all(stopped.map((log) => rm3(log, { force: true }).catch(() => {})));
|
|
62405
|
-
if (this.logDirectory !== undefined)
|
|
62406
|
-
await rmdir(this.logDirectory).catch(() => {});
|
|
63621
|
+
const coordinator = this.coordinator;
|
|
63622
|
+
if (coordinator !== undefined) {
|
|
63623
|
+
if (this.coordinatorLost === undefined) {
|
|
63624
|
+
try {
|
|
63625
|
+
await this.command({ kind: "close" });
|
|
63626
|
+
} catch (error) {
|
|
63627
|
+
failure = error;
|
|
63628
|
+
}
|
|
63629
|
+
}
|
|
63630
|
+
await this.inbound;
|
|
63631
|
+
await coordinator.port.close();
|
|
63632
|
+
this.pids.clear();
|
|
62407
63633
|
}
|
|
63634
|
+
for (const release of this.routeClaims.values())
|
|
63635
|
+
release();
|
|
63636
|
+
this.routeClaims.clear();
|
|
62408
63637
|
try {
|
|
62409
63638
|
await this.releaseRoutes?.();
|
|
62410
63639
|
} catch (error) {
|
|
@@ -62414,27 +63643,26 @@ class LocalWorkerPool {
|
|
|
62414
63643
|
throw failure;
|
|
62415
63644
|
}
|
|
62416
63645
|
}
|
|
62417
|
-
var INTERNAL_WORKFLOW_WORKER_ARG = "--internal-workflow-worker", DEFAULT_WORK_LEASE_MS = 30000, MINIMUM_WORK_LEASE_MS,
|
|
63646
|
+
var INTERNAL_WORKFLOW_WORKER_ARG = "--internal-workflow-worker", DEFAULT_WORK_LEASE_MS = 30000, MINIMUM_WORK_LEASE_MS, DEFAULT_STARTUP_TIMEOUT_MS = 30000, MAX_TIMER_DELAY_MS = 2147483647, COORDINATOR_STDERR_TAIL_CHARACTERS = 8192, COORDINATOR_EXIT_GRACE_MS = 5000;
|
|
62418
63647
|
var init_local_pool = __esm(() => {
|
|
62419
63648
|
init_internal();
|
|
62420
63649
|
init_sql_pool_queue();
|
|
62421
|
-
init_sql_pool_usage();
|
|
62422
63650
|
init_sql_run_journal();
|
|
62423
63651
|
init_workflow_tool_abort();
|
|
62424
63652
|
init_sdk_intercom();
|
|
63653
|
+
init_external_run_routes();
|
|
62425
63654
|
init_workflow_infrastructure_error();
|
|
62426
63655
|
init_admission_binding();
|
|
62427
63656
|
init_definitions();
|
|
62428
63657
|
init_dispatch();
|
|
62429
63658
|
init_host_options();
|
|
62430
|
-
|
|
63659
|
+
init_pool_coordinator();
|
|
62431
63660
|
init_stage_handoff();
|
|
62432
63661
|
MINIMUM_WORK_LEASE_MS = 2 * WORKER_HEARTBEAT_INTERVAL_MS;
|
|
62433
|
-
COORDINATOR_LEASE_MS = 3 * STAGE_SESSION_HEARTBEAT_INTERVAL_MS;
|
|
62434
63662
|
});
|
|
62435
63663
|
|
|
62436
63664
|
// dist/builtin/workflows/src/workers/public-pool.ts
|
|
62437
|
-
import { resolve as
|
|
63665
|
+
import { resolve as resolve13 } from "node:path";
|
|
62438
63666
|
import { isDeepStrictEqual as isDeepStrictEqual2 } from "node:util";
|
|
62439
63667
|
async function restoreWorkerOptions(backend, options) {
|
|
62440
63668
|
const source = options.continuation?.source.id ?? options.runId;
|
|
@@ -62473,11 +63701,11 @@ async function activatePublicPool(backend, options) {
|
|
|
62473
63701
|
...options.localWorkers,
|
|
62474
63702
|
startupTimeoutMs: undefined,
|
|
62475
63703
|
leaseMs: undefined,
|
|
62476
|
-
definitions: options.localWorkers.definitions.map((source) => ({ ...source, path:
|
|
63704
|
+
definitions: options.localWorkers.definitions.map((source) => ({ ...source, path: resolve13(cwd, source.path) }))
|
|
62477
63705
|
};
|
|
62478
63706
|
const admission = options.admission && {
|
|
62479
63707
|
...options.admission,
|
|
62480
|
-
...options.admission.userDatabase ? { userDatabase:
|
|
63708
|
+
...options.admission.userDatabase ? { userDatabase: resolve13(cwd, options.admission.userDatabase) } : {}
|
|
62481
63709
|
};
|
|
62482
63710
|
const sources = localWorkers && await Promise.all(localWorkers.definitions.map(async (source) => ({ ...source, digest: await definitionDigest(source.path) })));
|
|
62483
63711
|
const resources = JSON.parse(JSON.stringify({
|
|
@@ -63823,7 +65051,7 @@ function hasUsableContentBlock(block) {
|
|
|
63823
65051
|
|
|
63824
65052
|
// dist/builtin/workflows/src/shared/stage-session-store.ts
|
|
63825
65053
|
import { randomUUID as randomUUID17 } from "node:crypto";
|
|
63826
|
-
import { dirname as dirname13, resolve as
|
|
65054
|
+
import { dirname as dirname13, resolve as resolve14 } from "node:path";
|
|
63827
65055
|
import {
|
|
63828
65056
|
JsonlSessionStore,
|
|
63829
65057
|
resolveSessionStore,
|
|
@@ -63859,11 +65087,11 @@ async function forkStageSession(store, sessionId, cwd, options, destination = st
|
|
|
63859
65087
|
async function migrateLegacyStageSession(store, sessionId, sessionFile) {
|
|
63860
65088
|
if (sessionFile === undefined || await store.readHeader(sessionId) !== undefined)
|
|
63861
65089
|
return;
|
|
63862
|
-
const location =
|
|
65090
|
+
const location = resolve14(sessionFile);
|
|
63863
65091
|
const legacy = await JsonlSessionStore.open(dirname13(location), { exactDirectory: true, scanExisting: false });
|
|
63864
65092
|
try {
|
|
63865
65093
|
for await (const file of legacy.scanFiles()) {
|
|
63866
|
-
if (
|
|
65094
|
+
if (resolve14(file.location) !== location || file.header.id !== sessionId)
|
|
63867
65095
|
continue;
|
|
63868
65096
|
const snapshot = await file.read();
|
|
63869
65097
|
if (await store.readHeader(sessionId) === undefined)
|
|
@@ -64025,22 +65253,37 @@ function childScopedEvidenceExists(checkpoints, boundaryReplayKey, childRunId) {
|
|
|
64025
65253
|
}
|
|
64026
65254
|
function retainReachableRunGroups(grouped, rootRunId) {
|
|
64027
65255
|
const reachable = new Set([rootRunId]);
|
|
64028
|
-
const
|
|
65256
|
+
const settled = new Set;
|
|
65257
|
+
const pending = [{ runId: rootRunId, live: true }];
|
|
64029
65258
|
while (pending.length > 0) {
|
|
64030
|
-
const
|
|
65259
|
+
const { runId, live } = pending.pop();
|
|
65260
|
+
const drafts = grouped.get(runId);
|
|
64031
65261
|
if (drafts === undefined)
|
|
64032
65262
|
continue;
|
|
64033
65263
|
for (const draft of drafts) {
|
|
64034
|
-
const childRunId =
|
|
64035
|
-
if (childRunId === undefined ||
|
|
65264
|
+
const childRunId = workflowChildFromDraft(draft)?.runId ?? draft.topology?.boundary?.child.runId;
|
|
65265
|
+
if (childRunId === undefined || !grouped.has(childRunId))
|
|
64036
65266
|
continue;
|
|
64037
|
-
|
|
64038
|
-
|
|
65267
|
+
if (live && childRunIdFromDraft(draft) !== undefined) {
|
|
65268
|
+
if (reachable.has(childRunId))
|
|
65269
|
+
continue;
|
|
65270
|
+
reachable.add(childRunId);
|
|
65271
|
+
pending.push({ runId: childRunId, live: true });
|
|
65272
|
+
} else if (!settled.has(childRunId)) {
|
|
65273
|
+
settled.add(childRunId);
|
|
65274
|
+
pending.push({ runId: childRunId, live: false });
|
|
65275
|
+
}
|
|
64039
65276
|
}
|
|
64040
65277
|
}
|
|
64041
|
-
|
|
64042
|
-
|
|
64043
|
-
|
|
65278
|
+
let complete = true;
|
|
65279
|
+
for (const runId of grouped.keys()) {
|
|
65280
|
+
if (reachable.has(runId))
|
|
65281
|
+
continue;
|
|
65282
|
+
if (!settled.has(runId))
|
|
65283
|
+
complete = false;
|
|
65284
|
+
grouped.delete(runId);
|
|
65285
|
+
}
|
|
65286
|
+
return complete;
|
|
64044
65287
|
}
|
|
64045
65288
|
function childRunIdFromDraft(draft) {
|
|
64046
65289
|
const boundary = draft.topology?.boundary;
|
|
@@ -64462,9 +65705,7 @@ function runSnapshotsFromCheckpoints(checkpoints, rootRunId, rootRunName, fallba
|
|
|
64462
65705
|
}
|
|
64463
65706
|
for (const group of grouped.values())
|
|
64464
65707
|
group.sort(compareDraftSourceOrder);
|
|
64465
|
-
|
|
64466
|
-
retainReachableRunGroups(grouped, rootRunId);
|
|
64467
|
-
if (policy.failOnUnreachable && grouped.size !== groupedCount)
|
|
65708
|
+
if (!retainReachableRunGroups(grouped, rootRunId) && policy.failOnUnreachable)
|
|
64468
65709
|
return [];
|
|
64469
65710
|
if (!validateRunGroups(grouped, rootRunId, toolCheckpoints, checkpoints, policy.retainActive))
|
|
64470
65711
|
return [];
|
|
@@ -70122,9 +71363,9 @@ var init_stage_runner_controller = __esm(() => {
|
|
|
70122
71363
|
});
|
|
70123
71364
|
|
|
70124
71365
|
// dist/builtin/workflows/src/runs/foreground/stage-runner-output.ts
|
|
70125
|
-
import { createHash as
|
|
71366
|
+
import { createHash as createHash5 } from "node:crypto";
|
|
70126
71367
|
import { mkdir as mkdir7, writeFile as writeFile10 } from "node:fs/promises";
|
|
70127
|
-
import { basename as basename5, dirname as dirname14, isAbsolute as isAbsolute11, join as join29, resolve as
|
|
71368
|
+
import { basename as basename5, dirname as dirname14, isAbsolute as isAbsolute11, join as join29, resolve as resolve15 } from "node:path";
|
|
70128
71369
|
function normalizeMaxOutput2(maxOutput) {
|
|
70129
71370
|
return {
|
|
70130
71371
|
bytes: maxOutput?.bytes ?? DEFAULT_MAX_OUTPUT_BYTES,
|
|
@@ -70186,7 +71427,7 @@ function formatByteSize(bytes) {
|
|
|
70186
71427
|
return `${value2.toFixed(1)} ${units[unitIndex]}`;
|
|
70187
71428
|
}
|
|
70188
71429
|
function transcriptPath(runId, outputPath) {
|
|
70189
|
-
const digest =
|
|
71430
|
+
const digest = createHash5("sha256").update(outputPath).digest("hex").slice(0, 16);
|
|
70190
71431
|
return join29(workflowArtifactRunPath(runId), "transcripts", `${digest}-${basename5(outputPath)}.transcript.md`);
|
|
70191
71432
|
}
|
|
70192
71433
|
function appendContent(lines, content) {
|
|
@@ -70232,7 +71473,7 @@ function renderTranscript(messages, fallbackOutput) {
|
|
|
70232
71473
|
`);
|
|
70233
71474
|
}
|
|
70234
71475
|
function savedOutputReference(outputPath, fullOutput, transcriptPathValue, transcriptText, transcriptAvailable, warning) {
|
|
70235
|
-
const absolutePath =
|
|
71476
|
+
const absolutePath = resolve15(outputPath);
|
|
70236
71477
|
const bytes = Buffer.byteLength(fullOutput, "utf8");
|
|
70237
71478
|
const lines = countLines(fullOutput);
|
|
70238
71479
|
const transcriptBytes = Buffer.byteLength(transcriptText, "utf8");
|
|
@@ -70247,8 +71488,8 @@ function resolveOutputPath(output, runtimeCwd, requestedCwd) {
|
|
|
70247
71488
|
return;
|
|
70248
71489
|
if (isAbsolute11(output))
|
|
70249
71490
|
return output;
|
|
70250
|
-
const baseCwd = requestedCwd === undefined ? runtimeCwd : isAbsolute11(requestedCwd) ? requestedCwd :
|
|
70251
|
-
return
|
|
71491
|
+
const baseCwd = requestedCwd === undefined ? runtimeCwd : isAbsolute11(requestedCwd) ? requestedCwd : resolve15(runtimeCwd, requestedCwd);
|
|
71492
|
+
return resolve15(baseCwd, output);
|
|
70252
71493
|
}
|
|
70253
71494
|
function degenerateWarning(fullOutput) {
|
|
70254
71495
|
return fullOutput.trim().length === 0 ? "WARNING: the stage artifact is empty; search the companion transcript for this stage's work." : undefined;
|
|
@@ -71257,7 +72498,7 @@ async function run(def, inputs, opts = {}) {
|
|
|
71257
72498
|
if (!isWorkflowDefinition(def))
|
|
71258
72499
|
throw new Error(workflowDefinitionRequirementMessage("run(definition, inputs)", def));
|
|
71259
72500
|
const depth = opts.depth ?? 0;
|
|
71260
|
-
const maxDepth = opts.config?.maxDepth ??
|
|
72501
|
+
const maxDepth = opts.config?.maxDepth ?? 3;
|
|
71261
72502
|
if (depth >= maxDepth) {
|
|
71262
72503
|
return {
|
|
71263
72504
|
runId: opts.runId ?? crypto.randomUUID(),
|