@bastani/atomic 0.9.43 → 0.9.44-alpha.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +26 -0
- package/dist/builtin/intercom/CHANGELOG.md +12 -0
- package/dist/builtin/intercom/broker/broker.ts +31 -7
- package/dist/builtin/intercom/broker/client.ts +65 -0
- package/dist/builtin/intercom/index.bundle.mjs +227 -95
- package/dist/builtin/intercom/live-route-refusal.ts +20 -0
- package/dist/builtin/intercom/package.json +1 -1
- package/dist/builtin/intercom/skills/intercom/SKILL.md +10 -6
- package/dist/builtin/intercom/types.ts +20 -1
- package/dist/builtin/mcp/CHANGELOG.md +7 -0
- package/dist/builtin/mcp/package.json +1 -1
- package/dist/builtin/subagents/CHANGELOG.md +9 -0
- package/dist/builtin/subagents/README.md +8 -8
- package/dist/builtin/subagents/agents/worker.md +1 -1
- package/dist/builtin/subagents/package.json +1 -1
- package/dist/builtin/subagents/skills/agent-browser/SKILL.md +1 -1
- package/dist/builtin/subagents/skills/cua-driver/EMBEDDING.md +21 -2
- package/dist/builtin/subagents/skills/cua-driver/LINUX.md +2 -2
- package/dist/builtin/subagents/skills/cua-driver/MACOS.md +342 -10
- package/dist/builtin/subagents/skills/cua-driver/README.md +4 -3
- package/dist/builtin/subagents/skills/cua-driver/RUNTIME.md +3 -1
- package/dist/builtin/subagents/skills/cua-driver/SKILL.md +7 -4
- package/dist/builtin/subagents/skills/cua-driver/WINDOWS.md +6 -5
- package/dist/builtin/subagents/skills/cua-driver/WORKFLOW.md +15 -0
- package/dist/builtin/subagents/skills/herdr/SKILL.md +22 -10
- package/dist/builtin/subagents/skills/subagent/SKILL.md +8 -5
- package/dist/builtin/subagents/src/extension/index.bundle.mjs +6 -6
- package/dist/builtin/web-access/package.json +1 -1
- package/dist/builtin/workflows/CHANGELOG.md +14 -0
- package/dist/builtin/workflows/README.md +1 -1
- package/dist/builtin/workflows/builtin/{chunk-frqxd4d1.js → chunk-k8d7ehjx.js} +22 -0
- package/dist/builtin/workflows/builtin/{chunk-3s95f7ry.js → chunk-qxpymnaf.js} +1 -1
- package/dist/builtin/workflows/builtin/{chunk-edhf30ef.js → chunk-zjecnjxx.js} +1 -1
- package/dist/builtin/workflows/builtin/goal.js +2 -2
- package/dist/builtin/workflows/builtin/index.js +3 -3
- package/dist/builtin/workflows/builtin/ralph.js +2 -2
- package/dist/builtin/workflows/package.json +1 -1
- package/dist/builtin/workflows/src/extension/index.bundle.mjs +1750 -598
- package/dist/builtin/workflows/src/extension/local-worker-entry.mjs +1786 -600
- package/dist/builtin/workflows/src/index.js +1802 -650
- package/dist/builtin/workflows/src/local-worker-entry.mjs +1786 -600
- package/dist/cli/args.js +1 -1
- package/dist/cli/args.js.map +1 -1
- package/dist/core/exec.d.ts.map +1 -1
- package/dist/core/exec.js +11 -2
- package/dist/core/exec.js.map +1 -1
- package/dist/core/package-manager-resolver.d.ts.map +1 -1
- package/dist/core/package-manager-resolver.js +25 -14
- package/dist/core/package-manager-resolver.js.map +1 -1
- package/dist/core/package-manager-types.d.ts +8 -0
- package/dist/core/package-manager-types.d.ts.map +1 -1
- package/dist/core/package-manager-types.js.map +1 -1
- package/dist/core/package-manager.d.ts +1 -1
- package/dist/core/package-manager.d.ts.map +1 -1
- package/dist/core/package-manager.js.map +1 -1
- package/dist/core/resource-loader-extension-warnings.d.ts +4 -1
- package/dist/core/resource-loader-extension-warnings.d.ts.map +1 -1
- package/dist/core/resource-loader-extension-warnings.js +15 -0
- package/dist/core/resource-loader-extension-warnings.js.map +1 -1
- package/dist/core/resource-loader-package-resources.d.ts.map +1 -1
- package/dist/core/resource-loader-package-resources.js +1 -0
- package/dist/core/resource-loader-package-resources.js.map +1 -1
- package/dist/core/resource-loader-reload.d.ts.map +1 -1
- package/dist/core/resource-loader-reload.js +5 -3
- package/dist/core/resource-loader-reload.js.map +1 -1
- package/dist/extensions/mcp/client/oauth/flow.d.ts.map +1 -1
- package/dist/extensions/mcp/client/oauth/flow.js +4 -1
- package/dist/extensions/mcp/client/oauth/flow.js.map +1 -1
- package/dist/extensions/mcp/client/oauth/types.d.ts +1 -0
- package/dist/extensions/mcp/client/oauth/types.d.ts.map +1 -1
- package/dist/extensions/mcp/client/oauth/types.js +8 -4
- package/dist/extensions/mcp/client/oauth/types.js.map +1 -1
- package/dist/extensions/mcp/client/transports/streamable-http.d.ts.map +1 -1
- package/dist/extensions/mcp/client/transports/streamable-http.js +2 -1
- package/dist/extensions/mcp/client/transports/streamable-http.js.map +1 -1
- package/dist/index.d.ts +1 -1
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js.map +1 -1
- package/dist/main.d.ts.map +1 -1
- package/dist/main.js +20 -10
- package/dist/main.js.map +1 -1
- package/dist/modes/interactive/components/model-selector.d.ts.map +1 -1
- package/dist/modes/interactive/components/model-selector.js +4 -2
- package/dist/modes/interactive/components/model-selector.js.map +1 -1
- package/dist/modes/interactive/components/scoped-models-selector.d.ts.map +1 -1
- package/dist/modes/interactive/components/scoped-models-selector.js +4 -2
- package/dist/modes/interactive/components/scoped-models-selector.js.map +1 -1
- package/dist/modes/interactive/components/tool-execution.d.ts +6 -0
- package/dist/modes/interactive/components/tool-execution.d.ts.map +1 -1
- package/dist/modes/interactive/components/tool-execution.js +11 -1
- package/dist/modes/interactive/components/tool-execution.js.map +1 -1
- package/dist/modes/interactive/model-search.d.ts +10 -0
- package/dist/modes/interactive/model-search.d.ts.map +1 -1
- package/dist/modes/interactive/model-search.js +21 -0
- package/dist/modes/interactive/model-search.js.map +1 -1
- package/dist/package-manager-cli.d.ts.map +1 -1
- package/dist/package-manager-cli.js +5 -0
- package/dist/package-manager-cli.js.map +1 -1
- package/dist/utils/tools-manager.d.ts.map +1 -1
- package/dist/utils/tools-manager.js +30 -35
- package/dist/utils/tools-manager.js.map +1 -1
- package/docs/computer-use.md +5 -5
- package/docs/custom-provider/streaming.md +9 -14
- package/docs/environment-variables.md +1 -0
- package/docs/intercom.md +5 -1
- package/docs/packages.md +2 -0
- package/docs/settings.md +2 -2
- package/docs/skills.md +2 -2
- package/docs/subagents.md +38 -9
- package/docs/workflows/api-reference.md +1 -1
- package/docs/workflows/authoring.md +1 -1
- package/docs/workflows/operations.md +8 -6
- package/docs/workflows/reliable-design.md +23 -5
- package/examples/extensions/custom-provider-anthropic/index.ts +31 -31
- package/examples/extensions/todo.ts +7 -0
- package/npm-shrinkwrap.json +32 -32
- package/package.json +3 -3
|
@@ -3221,7 +3221,13 @@ function createStoreContext(state = createStoreState()) {
|
|
|
3221
3221
|
}
|
|
3222
3222
|
return cachedGraphSnapshot;
|
|
3223
3223
|
}
|
|
3224
|
+
let batchDepth = 0;
|
|
3225
|
+
let batchedNotification = false;
|
|
3224
3226
|
function notify() {
|
|
3227
|
+
if (batchDepth > 0) {
|
|
3228
|
+
batchedNotification = true;
|
|
3229
|
+
return;
|
|
3230
|
+
}
|
|
3225
3231
|
observation.capture();
|
|
3226
3232
|
for (const fn of state.invalidationListeners) {
|
|
3227
3233
|
try {
|
|
@@ -3241,6 +3247,18 @@ function createStoreContext(state = createStoreState()) {
|
|
|
3241
3247
|
state.version++;
|
|
3242
3248
|
notify();
|
|
3243
3249
|
}
|
|
3250
|
+
function batch(update) {
|
|
3251
|
+
batchDepth++;
|
|
3252
|
+
try {
|
|
3253
|
+
return update();
|
|
3254
|
+
} finally {
|
|
3255
|
+
batchDepth--;
|
|
3256
|
+
if (batchDepth === 0 && batchedNotification) {
|
|
3257
|
+
batchedNotification = false;
|
|
3258
|
+
notify();
|
|
3259
|
+
}
|
|
3260
|
+
}
|
|
3261
|
+
}
|
|
3244
3262
|
function findRun(runId) {
|
|
3245
3263
|
return state.runs.find((r) => r.id === runId);
|
|
3246
3264
|
}
|
|
@@ -3294,6 +3312,7 @@ function createStoreContext(state = createStoreState()) {
|
|
|
3294
3312
|
snapshot,
|
|
3295
3313
|
graphSnapshot,
|
|
3296
3314
|
notify,
|
|
3315
|
+
batch,
|
|
3297
3316
|
bumpAndNotify,
|
|
3298
3317
|
findRun,
|
|
3299
3318
|
findStage,
|
|
@@ -4432,6 +4451,9 @@ function createRunStoreMethods(context) {
|
|
|
4432
4451
|
subscribeInvalidation(fn) {
|
|
4433
4452
|
state.invalidationListeners.add(fn);
|
|
4434
4453
|
return () => state.invalidationListeners.delete(fn);
|
|
4454
|
+
},
|
|
4455
|
+
batch(update) {
|
|
4456
|
+
return context.batch(update);
|
|
4435
4457
|
}
|
|
4436
4458
|
};
|
|
4437
4459
|
}
|
|
@@ -5165,7 +5187,7 @@ class ConcurrencyLimiter {
|
|
|
5165
5187
|
}
|
|
5166
5188
|
}
|
|
5167
5189
|
function createRunLimiter(defaultConcurrency) {
|
|
5168
|
-
return new ConcurrencyLimiter(defaultConcurrency ??
|
|
5190
|
+
return new ConcurrencyLimiter(defaultConcurrency ?? 1000);
|
|
5169
5191
|
}
|
|
5170
5192
|
|
|
5171
5193
|
// dist/builtin/workflows/src/extension/atomic-stage-session.ts
|
|
@@ -57292,7 +57314,7 @@ var init_definitions = __esm(() => {
|
|
|
57292
57314
|
// dist/builtin/workflows/src/durable/journal/pool-mutations.ts
|
|
57293
57315
|
import { StorageProvider as StorageProvider2 } from "@bastani/atomic/storage";
|
|
57294
57316
|
async function validatePoolMutation(tx, journal, root, work, mutation) {
|
|
57295
|
-
if (!["checkpoint", "session-create", "session-
|
|
57317
|
+
if (!["checkpoint", "session-create", "session-replace"].includes(mutation.kind))
|
|
57296
57318
|
throw new Error("Invalid pool mutation");
|
|
57297
57319
|
const checkpoint = mutation.checkpoint;
|
|
57298
57320
|
const prefix = `${work.scope}:`;
|
|
@@ -57360,10 +57382,18 @@ async function sessionStore(tx, prefix) {
|
|
|
57360
57382
|
return new StorageProvider2().openSessionStore({ kind: "database", database, prefix }, { agentDir: "", domain: "sessions" });
|
|
57361
57383
|
}
|
|
57362
57384
|
async function applyPoolSession(tx, prefix, workId, mutation) {
|
|
57385
|
+
const owners = `${prefix}session_owners`;
|
|
57386
|
+
if (mutation.kind === "session-append") {
|
|
57387
|
+
if (typeof mutation.sessionId !== "string" || mutation.sessionId === "")
|
|
57388
|
+
throw new Error("Pool sessions require database identity");
|
|
57389
|
+
const owner = await tx.get(`SELECT work FROM ${owners} WHERE session = ?`, mutation.sessionId);
|
|
57390
|
+
if (owner?.work !== workId)
|
|
57391
|
+
throw new Error("Pool session scope mismatch");
|
|
57392
|
+
return withSessionStore(tx, prefix, (store) => store.append(mutation.sessionId, mutation.entries, { expectSeq: mutation.expectSeq }));
|
|
57393
|
+
}
|
|
57363
57394
|
const { sessionId, replayKey } = mutation.checkpoint;
|
|
57364
57395
|
if (!sessionId || mutation.checkpoint.sessionFile !== undefined)
|
|
57365
57396
|
throw new Error("Pool sessions require database identity");
|
|
57366
|
-
const owners = `${prefix}session_owners`;
|
|
57367
57397
|
if (mutation.kind === "session-create") {
|
|
57368
57398
|
if (mutation.header.id !== sessionId)
|
|
57369
57399
|
throw new Error("Pool session identity mismatch");
|
|
@@ -57373,10 +57403,12 @@ async function applyPoolSession(tx, prefix, workId, mutation) {
|
|
|
57373
57403
|
if (owner?.work !== workId || owner.replay_key !== replayKey)
|
|
57374
57404
|
throw new Error("Pool session scope mismatch");
|
|
57375
57405
|
}
|
|
57406
|
+
return withSessionStore(tx, prefix, (store) => mutation.kind === "session-create" ? store.create(mutation.header, mutation.entries) : store.replace(sessionId, mutation.header, mutation.entries, { expectSeq: mutation.expectSeq }));
|
|
57407
|
+
}
|
|
57408
|
+
async function withSessionStore(tx, prefix, write) {
|
|
57376
57409
|
const store = await sessionStore(tx, prefix);
|
|
57377
57410
|
try {
|
|
57378
|
-
|
|
57379
|
-
return result.lastSeq;
|
|
57411
|
+
return (await write(store)).lastSeq;
|
|
57380
57412
|
} finally {
|
|
57381
57413
|
await store.close();
|
|
57382
57414
|
}
|
|
@@ -57424,8 +57456,11 @@ var init_pool_stage_usage = __esm(() => {
|
|
|
57424
57456
|
});
|
|
57425
57457
|
|
|
57426
57458
|
// dist/builtin/workflows/src/durable/journal/sql-pool-queue.ts
|
|
57427
|
-
import { randomUUID as randomUUID10 } from "node:crypto";
|
|
57459
|
+
import { createHash as createHash4, randomUUID as randomUUID10 } from "node:crypto";
|
|
57428
57460
|
import { assertStorageSchemaVersion as assertStorageSchemaVersion2 } from "@bastani/atomic/storage";
|
|
57461
|
+
function requestIdentity(request) {
|
|
57462
|
+
return createHash4("sha256").update(request).digest("hex");
|
|
57463
|
+
}
|
|
57429
57464
|
function fromRow(row) {
|
|
57430
57465
|
return {
|
|
57431
57466
|
...JSON.parse(row.descriptor),
|
|
@@ -57481,10 +57516,11 @@ class SqlPoolQueue {
|
|
|
57481
57516
|
const versions = await tx.all(`SELECT version FROM ${pool.meta}`);
|
|
57482
57517
|
if (versions.length > 1)
|
|
57483
57518
|
throw new Error("Corrupt pool schema metadata");
|
|
57484
|
-
|
|
57485
|
-
|
|
57486
|
-
|
|
57487
|
-
|
|
57519
|
+
const version = versions[0]?.version;
|
|
57520
|
+
if (version === undefined)
|
|
57521
|
+
await tx.run(`INSERT INTO ${pool.meta} VALUES (?)`, POOL_SCHEMA_VERSION);
|
|
57522
|
+
else if (version !== 1)
|
|
57523
|
+
assertStorageSchemaVersion2("journal", version, POOL_SCHEMA_VERSION);
|
|
57488
57524
|
await tx.exec(`
|
|
57489
57525
|
CREATE TABLE IF NOT EXISTS ${pool.roots}(root TEXT PRIMARY KEY REFERENCES ${journal.runs}(run_id) ON DELETE CASCADE, timeout INTEGER NOT NULL);
|
|
57490
57526
|
CREATE TABLE IF NOT EXISTS ${pool.work}(
|
|
@@ -57515,9 +57551,23 @@ class SqlPoolQueue {
|
|
|
57515
57551
|
state TEXT NOT NULL DEFAULT 'running', revision INTEGER NOT NULL DEFAULT 0
|
|
57516
57552
|
)`);
|
|
57517
57553
|
await tx.exec(poolUsageSchema(journal.runs));
|
|
57554
|
+
if (version === 1)
|
|
57555
|
+
await pool.migrateFromVersion1(tx);
|
|
57518
57556
|
});
|
|
57519
57557
|
return pool;
|
|
57520
57558
|
}
|
|
57559
|
+
async migrateFromVersion1(tx) {
|
|
57560
|
+
await tx.run(`DELETE FROM ${this.operations} WHERE substr(request, 1, 1) = '['`);
|
|
57561
|
+
for (let after = 0;; ) {
|
|
57562
|
+
const rows = await tx.all(`SELECT rowid, request FROM ${this.operations} WHERE rowid > ? ORDER BY rowid LIMIT 256`, after);
|
|
57563
|
+
if (rows.length === 0)
|
|
57564
|
+
break;
|
|
57565
|
+
for (const row of rows)
|
|
57566
|
+
await tx.run(`UPDATE ${this.operations} SET request = ? WHERE rowid = ?`, requestIdentity(String(row.request)), row.rowid);
|
|
57567
|
+
after = Number(rows.at(-1).rowid);
|
|
57568
|
+
}
|
|
57569
|
+
await tx.run(`UPDATE ${this.meta} SET version = ?`, POOL_SCHEMA_VERSION);
|
|
57570
|
+
}
|
|
57521
57571
|
async authority(tx, lease, timeout) {
|
|
57522
57572
|
const row = await tx.get(`SELECT l.*, r.status, p.timeout, ${STORE_NOW2} AS now FROM ${this.journal.leases} l JOIN ${this.journal.runs} r ON r.run_id = l.run_id LEFT JOIN ${this.roots} p ON p.root = l.run_id WHERE l.run_id = ?`, lease.runId);
|
|
57523
57573
|
if (row?.timeout === null && timeout === undefined)
|
|
@@ -57658,31 +57708,18 @@ class SqlPoolQueue {
|
|
|
57658
57708
|
throw new Error(STALE_WORK_LEASE);
|
|
57659
57709
|
return row;
|
|
57660
57710
|
}
|
|
57661
|
-
async renew(lease, token,
|
|
57711
|
+
async renew(lease, token, durationMs) {
|
|
57662
57712
|
positive(durationMs);
|
|
57663
57713
|
lease = snapshotLease(lease);
|
|
57664
57714
|
token = { ...token };
|
|
57665
|
-
validateText("key", operationId);
|
|
57666
|
-
const request = JSON.stringify([token.epoch, token.generation, token.worker, durationMs]);
|
|
57667
57715
|
return this.transaction(async (tx) => {
|
|
57668
57716
|
const { now } = await this.authority(tx, lease);
|
|
57669
57717
|
const row = await this.current(tx, lease, token);
|
|
57670
57718
|
if (row.epoch !== lease.epoch || row.expires_at < now)
|
|
57671
57719
|
throw new Error(STALE_WORK_LEASE);
|
|
57672
|
-
const
|
|
57673
|
-
|
|
57674
|
-
|
|
57675
|
-
throw new Error("Pool operation identity conflict");
|
|
57676
|
-
return JSON.parse(previous.receipt);
|
|
57677
|
-
}
|
|
57678
|
-
const receipt = {
|
|
57679
|
-
...fromRow(row),
|
|
57680
|
-
expiresAt: Math.max(Number(row.expires_at), expiry(now, durationMs)),
|
|
57681
|
-
renewedAt: now
|
|
57682
|
-
};
|
|
57683
|
-
await tx.run(`UPDATE ${this.work} SET expires_at = ?, renewed_at = ? WHERE id = ?`, receipt.expiresAt, now, row.id);
|
|
57684
|
-
await tx.run(`INSERT INTO ${this.operations} VALUES (?, ?, ?, ?, ?)`, row.id, token.generation, operationId, request, JSON.stringify(receipt));
|
|
57685
|
-
return receipt;
|
|
57720
|
+
const expiresAt = Math.max(Number(row.expires_at), expiry(now, durationMs));
|
|
57721
|
+
await tx.run(`UPDATE ${this.work} SET expires_at = ?, renewed_at = ? WHERE id = ?`, expiresAt, now, row.id);
|
|
57722
|
+
return { ...fromRow(row), expiresAt, renewedAt: now };
|
|
57686
57723
|
});
|
|
57687
57724
|
}
|
|
57688
57725
|
async commit(lease, token, operationId, mutation) {
|
|
@@ -57691,6 +57728,7 @@ class SqlPoolQueue {
|
|
|
57691
57728
|
validateText("key", operationId);
|
|
57692
57729
|
assertWorkflowSerializableObject(mutation, "Pool mutation");
|
|
57693
57730
|
const request = serializeJson({ epoch: token.epoch, worker: token.worker, mutation });
|
|
57731
|
+
const identity = requestIdentity(request);
|
|
57694
57732
|
mutation = JSON.parse(request).mutation;
|
|
57695
57733
|
return this.transaction(async (tx) => {
|
|
57696
57734
|
const { now, status } = await this.authority(tx, lease);
|
|
@@ -57699,7 +57737,7 @@ class SqlPoolQueue {
|
|
|
57699
57737
|
throw new Error(STALE_WORK_LEASE);
|
|
57700
57738
|
const previous = await tx.get(`SELECT * FROM ${this.operations} WHERE work = ? AND generation = ? AND operation = ?`, token.id, token.generation, operationId);
|
|
57701
57739
|
if (previous) {
|
|
57702
|
-
if (previous.request !==
|
|
57740
|
+
if (previous.request !== identity)
|
|
57703
57741
|
throw new Error("Pool operation identity conflict");
|
|
57704
57742
|
return JSON.parse(previous.receipt);
|
|
57705
57743
|
}
|
|
@@ -57707,34 +57745,41 @@ class SqlPoolQueue {
|
|
|
57707
57745
|
throw new Error(STALE_WORK_LEASE);
|
|
57708
57746
|
if (!await this.admitsAttemptWrites(tx, lease, status))
|
|
57709
57747
|
throw new Error("Pool root is not running");
|
|
57710
|
-
const
|
|
57711
|
-
await
|
|
57712
|
-
if (checkpoint.completedAt > Math.max(now, Date.now()) + MAX_WORKER_CLOCK_SKEW_MS)
|
|
57713
|
-
throw new Error("Pool checkpoint timestamp is in the future");
|
|
57714
|
-
assertWorkflowSerializableObject(checkpoint, "Pool checkpoint");
|
|
57715
|
-
const key = JSON.stringify([RUN_EVENT_TYPES.checkpointRecorded, checkpoint.kind, checkpoint.checkpointId]);
|
|
57716
|
-
const payload = serializeJson(checkpoint);
|
|
57717
|
-
const existing = await tx.get(`SELECT seq, at, payload FROM ${this.journal.events} WHERE run_id = ? AND key = ?`, lease.runId, key);
|
|
57718
|
-
if (existing && existing.payload !== payload)
|
|
57719
|
-
throw new Error("Pool checkpoint identity conflict");
|
|
57720
|
-
if (existing && mutation.kind !== "checkpoint")
|
|
57721
|
-
throw new Error("Pool session checkpoint identity conflict");
|
|
57722
|
-
const lastSeq = mutation.kind === "checkpoint" ? undefined : await applyPoolSession(tx, `${this.journal.runs}_pool_`, row.id, mutation);
|
|
57723
|
-
const latest = await tx.get(`SELECT COALESCE(MAX(seq), 0) AS seq FROM ${this.journal.events} WHERE run_id = ?`, lease.runId);
|
|
57724
|
-
const receipt = existing ? { seq: Number(existing.seq), at: Number(existing.at) } : {
|
|
57725
|
-
seq: journalCounter(lease.runId, "seq", Number(latest.seq) + 1, "overflow"),
|
|
57726
|
-
at: now,
|
|
57727
|
-
...lastSeq === undefined ? {} : { lastSeq }
|
|
57728
|
-
};
|
|
57729
|
-
if (!existing)
|
|
57730
|
-
await tx.run(`INSERT INTO ${this.journal.events} VALUES (?, ?, ?, ?, ?, ?, ?)`, lease.runId, receipt.seq, lease.epoch, key, RUN_EVENT_TYPES.checkpointRecorded, payload, now);
|
|
57731
|
-
const billed = poolCheckpointUsage(checkpoint);
|
|
57732
|
-
if (billed && !existing)
|
|
57733
|
-
await upsertPoolUsage(tx, this.journal.runs, row.id, token.generation, billed.key, billed.usage);
|
|
57734
|
-
await tx.run(`INSERT INTO ${this.operations} VALUES (?, ?, ?, ?, ?)`, token.id, token.generation, operationId, request, JSON.stringify(receipt));
|
|
57748
|
+
const receipt = mutation.kind === "session-append" ? await this.appendSession(tx, row, mutation, now) : await this.recordCheckpoint(tx, lease, token, row, mutation, now);
|
|
57749
|
+
await tx.run(`INSERT INTO ${this.operations} VALUES (?, ?, ?, ?, ?)`, token.id, token.generation, operationId, identity, JSON.stringify(receipt));
|
|
57735
57750
|
return receipt;
|
|
57736
57751
|
});
|
|
57737
57752
|
}
|
|
57753
|
+
async appendSession(tx, row, mutation, now) {
|
|
57754
|
+
return { at: now, lastSeq: await applyPoolSession(tx, `${this.journal.runs}_pool_`, row.id, mutation) };
|
|
57755
|
+
}
|
|
57756
|
+
async recordCheckpoint(tx, lease, token, row, mutation, now) {
|
|
57757
|
+
const checkpoint = mutation.checkpoint;
|
|
57758
|
+
await validatePoolMutation(tx, this.journal, lease.runId, fromRow(row), mutation);
|
|
57759
|
+
if (checkpoint.completedAt > Math.max(now, Date.now()) + MAX_WORKER_CLOCK_SKEW_MS)
|
|
57760
|
+
throw new Error("Pool checkpoint timestamp is in the future");
|
|
57761
|
+
assertWorkflowSerializableObject(checkpoint, "Pool checkpoint");
|
|
57762
|
+
const key = JSON.stringify([RUN_EVENT_TYPES.checkpointRecorded, checkpoint.kind, checkpoint.checkpointId]);
|
|
57763
|
+
const payload = serializeJson(checkpoint);
|
|
57764
|
+
const existing = await tx.get(`SELECT seq, at, payload FROM ${this.journal.events} WHERE run_id = ? AND key = ?`, lease.runId, key);
|
|
57765
|
+
if (existing && existing.payload !== payload)
|
|
57766
|
+
throw new Error("Pool checkpoint identity conflict");
|
|
57767
|
+
if (existing && mutation.kind !== "checkpoint")
|
|
57768
|
+
throw new Error("Pool session checkpoint identity conflict");
|
|
57769
|
+
const lastSeq = mutation.kind === "checkpoint" ? undefined : await applyPoolSession(tx, `${this.journal.runs}_pool_`, row.id, mutation);
|
|
57770
|
+
const latest = await tx.get(`SELECT COALESCE(MAX(seq), 0) AS seq FROM ${this.journal.events} WHERE run_id = ?`, lease.runId);
|
|
57771
|
+
const receipt = existing ? { seq: Number(existing.seq), at: Number(existing.at) } : {
|
|
57772
|
+
seq: journalCounter(lease.runId, "seq", Number(latest.seq) + 1, "overflow"),
|
|
57773
|
+
at: now,
|
|
57774
|
+
...lastSeq === undefined ? {} : { lastSeq }
|
|
57775
|
+
};
|
|
57776
|
+
if (!existing)
|
|
57777
|
+
await tx.run(`INSERT INTO ${this.journal.events} VALUES (?, ?, ?, ?, ?, ?, ?)`, lease.runId, receipt.seq, lease.epoch, key, RUN_EVENT_TYPES.checkpointRecorded, payload, now);
|
|
57778
|
+
const billed = poolCheckpointUsage(checkpoint);
|
|
57779
|
+
if (billed && !existing)
|
|
57780
|
+
await upsertPoolUsage(tx, this.journal.runs, row.id, token.generation, billed.key, billed.usage);
|
|
57781
|
+
return receipt;
|
|
57782
|
+
}
|
|
57738
57783
|
async complete(lease, token, result) {
|
|
57739
57784
|
lease = snapshotLease(lease);
|
|
57740
57785
|
token = { ...token };
|
|
@@ -57821,7 +57866,7 @@ function openSqlPoolQueue(journal) {
|
|
|
57821
57866
|
return Promise.reject(new Error("Worker pool requires a SQL run journal"));
|
|
57822
57867
|
return SqlPoolQueue.open(storage);
|
|
57823
57868
|
}
|
|
57824
|
-
var STORE_NOW2 = "CAST(unixepoch('subsec') * 1000 AS INTEGER)", MAX_WORKER_CLOCK_SKEW_MS = 1000, STALE_WORK_LEASE = "Stale work lease";
|
|
57869
|
+
var STORE_NOW2 = "CAST(unixepoch('subsec') * 1000 AS INTEGER)", MAX_WORKER_CLOCK_SKEW_MS = 1000, POOL_SCHEMA_VERSION = 2, STALE_WORK_LEASE = "Stale work lease";
|
|
57825
57870
|
var init_sql_pool_queue = __esm(() => {
|
|
57826
57871
|
init_serializable();
|
|
57827
57872
|
init_boundary_topology();
|
|
@@ -57833,74 +57878,42 @@ var init_sql_pool_queue = __esm(() => {
|
|
|
57833
57878
|
init_values();
|
|
57834
57879
|
});
|
|
57835
57880
|
|
|
57836
|
-
// dist/builtin/workflows/src/
|
|
57837
|
-
function
|
|
57838
|
-
|
|
57839
|
-
if (
|
|
57840
|
-
|
|
57841
|
-
|
|
57842
|
-
const value2 = report.usage[field];
|
|
57843
|
-
if (typeof value2 !== "number" || !Number.isFinite(value2) || value2 < 0)
|
|
57844
|
-
throw new Error(`Pool usage ${field} must be a finite non-negative number`);
|
|
57881
|
+
// dist/builtin/workflows/src/shared/external-run-routes.ts
|
|
57882
|
+
function claimExternalRunRoute(store, runId, owner) {
|
|
57883
|
+
let owners = registry.owners.get(store);
|
|
57884
|
+
if (owners === undefined) {
|
|
57885
|
+
owners = new Map;
|
|
57886
|
+
registry.owners.set(store, owners);
|
|
57845
57887
|
}
|
|
57888
|
+
owners.set(runId, owner);
|
|
57889
|
+
return () => {
|
|
57890
|
+
const current = registry.owners.get(store);
|
|
57891
|
+
if (current?.get(runId) !== owner)
|
|
57892
|
+
return;
|
|
57893
|
+
current.delete(runId);
|
|
57894
|
+
if (current.size === 0)
|
|
57895
|
+
registry.owners.delete(store);
|
|
57896
|
+
};
|
|
57846
57897
|
}
|
|
57847
|
-
|
|
57848
|
-
|
|
57849
|
-
transaction;
|
|
57850
|
-
journal;
|
|
57851
|
-
table;
|
|
57852
|
-
constructor(transaction, journal) {
|
|
57853
|
-
this.transaction = transaction;
|
|
57854
|
-
this.journal = journal;
|
|
57855
|
-
this.table = poolUsageTable(journal.runs);
|
|
57856
|
-
}
|
|
57857
|
-
static async open(transaction, journal) {
|
|
57858
|
-
const usage = new SqlPoolUsage(transaction, journal);
|
|
57859
|
-
await transaction((tx) => tx.exec(poolUsageSchema(journal.runs)));
|
|
57860
|
-
return usage;
|
|
57861
|
-
}
|
|
57862
|
-
async totalIn(tx, root) {
|
|
57863
|
-
const row = await tx.get(`SELECT ${POOL_USAGE_FIELDS.map((field) => `COALESCE(SUM(u.${field.toLowerCase()}), 0) AS ${field}`).join(", ")}
|
|
57864
|
-
FROM ${this.table} u JOIN ${this.journal.runs}_work w ON w.id = u.work WHERE w.root = ?`, root);
|
|
57865
|
-
return Object.fromEntries(POOL_USAGE_FIELDS.map((field) => [field, Number(row?.[field] ?? 0)]));
|
|
57866
|
-
}
|
|
57867
|
-
async report(lease, token, reports) {
|
|
57868
|
-
validateText("runId", lease.runId);
|
|
57869
|
-
validateText("holderId", lease.holderId);
|
|
57870
|
-
for (const report of reports)
|
|
57871
|
-
validate(report);
|
|
57872
|
-
const snapshot = reports.map((report) => ({ key: report.key, usage: { ...report.usage } }));
|
|
57873
|
-
return this.transaction(async (tx) => {
|
|
57874
|
-
const authority = await tx.get(`SELECT l.epoch, l.holder_id, l.renewed_at, p.timeout, ${STORE_NOW3} AS now FROM ${this.journal.leases} l
|
|
57875
|
-
LEFT JOIN ${this.journal.runs}_pool p ON p.root = l.run_id WHERE l.run_id = ?`, lease.runId);
|
|
57876
|
-
if (!authority || authority.timeout === null || authority.epoch !== lease.epoch || authority.holder_id !== lease.holderId || isWorkflowLeaseExpired(Number(authority.renewed_at), Number(authority.timeout), Number(authority.now)))
|
|
57877
|
-
throw new StaleLeaseError(lease.runId, lease.epoch, Number(authority?.epoch ?? 0));
|
|
57878
|
-
const work = await tx.get(`SELECT * FROM ${this.journal.runs}_work WHERE id = ? AND root = ?`, token.id, lease.runId);
|
|
57879
|
-
if (work?.state !== "leased" && work?.state !== "completed" || work.epoch !== token.epoch || work.epoch !== lease.epoch || work.generation !== token.generation || work.worker !== token.worker || work.state === "leased" && Number(work.expires_at) < Number(authority.now))
|
|
57880
|
-
throw new Error("Stale work lease");
|
|
57881
|
-
for (const { key, usage } of snapshot)
|
|
57882
|
-
await upsertPoolUsage(tx, this.journal.runs, token.id, token.generation, key, usage);
|
|
57883
|
-
return this.totalIn(tx, lease.runId);
|
|
57884
|
-
});
|
|
57885
|
-
}
|
|
57886
|
-
total(root) {
|
|
57887
|
-
return this.transaction((tx) => this.totalIn(tx, root));
|
|
57888
|
-
}
|
|
57898
|
+
function externalRunRouteOwner(store, runId) {
|
|
57899
|
+
return registry.owners.get(store)?.get(runId);
|
|
57889
57900
|
}
|
|
57890
|
-
|
|
57891
|
-
|
|
57892
|
-
|
|
57893
|
-
|
|
57894
|
-
|
|
57895
|
-
|
|
57901
|
+
function registerPendingStageMessageRouter(store, router) {
|
|
57902
|
+
registry.routers.set(store, router);
|
|
57903
|
+
return () => {
|
|
57904
|
+
if (registry.routers.get(store) === router)
|
|
57905
|
+
registry.routers.delete(store);
|
|
57906
|
+
};
|
|
57896
57907
|
}
|
|
57897
|
-
|
|
57898
|
-
|
|
57899
|
-
|
|
57900
|
-
|
|
57901
|
-
|
|
57902
|
-
|
|
57903
|
-
|
|
57908
|
+
function pendingStageMessageRouter(store) {
|
|
57909
|
+
return registry.routers.get(store);
|
|
57910
|
+
}
|
|
57911
|
+
var registryKey, processState, registry;
|
|
57912
|
+
var init_external_run_routes = __esm(() => {
|
|
57913
|
+
registryKey = Symbol.for("atomic-workflows/external-run-routes@1");
|
|
57914
|
+
processState = globalThis;
|
|
57915
|
+
registry = processState[registryKey] ?? { owners: new WeakMap, routers: new WeakMap };
|
|
57916
|
+
processState[registryKey] = registry;
|
|
57904
57917
|
});
|
|
57905
57918
|
|
|
57906
57919
|
// dist/builtin/workflows/src/shared/pending-stage-route-capability.ts
|
|
@@ -57927,12 +57940,12 @@ function rotateWorkflowPendingStageRouteCapability(store, runId) {
|
|
|
57927
57940
|
capabilitiesFor(store).set(runId, capability);
|
|
57928
57941
|
return capability;
|
|
57929
57942
|
}
|
|
57930
|
-
var routeCapabilitiesKey,
|
|
57943
|
+
var routeCapabilitiesKey, processState2, routeCapabilities;
|
|
57931
57944
|
var init_pending_stage_route_capability = __esm(() => {
|
|
57932
57945
|
routeCapabilitiesKey = Symbol.for("atomic-workflows/pending-stage-route-capabilities@1");
|
|
57933
|
-
|
|
57934
|
-
routeCapabilities =
|
|
57935
|
-
|
|
57946
|
+
processState2 = globalThis;
|
|
57947
|
+
routeCapabilities = processState2[routeCapabilitiesKey] ?? new WeakMap;
|
|
57948
|
+
processState2[routeCapabilitiesKey] = routeCapabilities;
|
|
57936
57949
|
});
|
|
57937
57950
|
|
|
57938
57951
|
// dist/builtin/workflows/src/shared/pending-stage-route-readiness.ts
|
|
@@ -57964,12 +57977,12 @@ function hasWorkflowPendingStageRouteOwner(store) {
|
|
|
57964
57977
|
function workflowPendingStageRouteReady(store, runId) {
|
|
57965
57978
|
return owners.get(store)?.ready(runId);
|
|
57966
57979
|
}
|
|
57967
|
-
var routeOwnersKey,
|
|
57980
|
+
var routeOwnersKey, processState3, owners;
|
|
57968
57981
|
var init_pending_stage_route_readiness = __esm(() => {
|
|
57969
57982
|
routeOwnersKey = Symbol.for("atomic-workflows/pending-stage-route-owners@1");
|
|
57970
|
-
|
|
57971
|
-
owners =
|
|
57972
|
-
|
|
57983
|
+
processState3 = globalThis;
|
|
57984
|
+
owners = processState3[routeOwnersKey] ?? new WeakMap;
|
|
57985
|
+
processState3[routeOwnersKey] = owners;
|
|
57973
57986
|
});
|
|
57974
57987
|
|
|
57975
57988
|
// dist/builtin/workflows/src/shared/workflow-stage-path-matching.ts
|
|
@@ -58270,6 +58283,9 @@ function registerPendingStageIntercomBridge(pi, activeStore, ownsRun = () => tru
|
|
|
58270
58283
|
notificationId,
|
|
58271
58284
|
reason
|
|
58272
58285
|
};
|
|
58286
|
+
const external = externalRunRouteOwner(activeStore, entry.runId);
|
|
58287
|
+
if (external !== undefined)
|
|
58288
|
+
return external.notifyUndeliverable(payload);
|
|
58273
58289
|
pi.events?.emit?.(PENDING_STAGE_UNDELIVERABLE_EVENT, payload);
|
|
58274
58290
|
return payload.handled && await payload.completion === true;
|
|
58275
58291
|
};
|
|
@@ -58281,19 +58297,29 @@ function registerPendingStageIntercomBridge(pi, activeStore, ownsRun = () => tru
|
|
|
58281
58297
|
return;
|
|
58282
58298
|
const ownedRunIds = new Set;
|
|
58283
58299
|
const runs = activeStore.runs();
|
|
58300
|
+
const runById = new Map(runs.map((run) => [run.id, run]));
|
|
58301
|
+
const childrenByBoundary = new Map;
|
|
58302
|
+
for (const run of runs) {
|
|
58303
|
+
if (run.parentRunId === undefined || run.parentStageId === undefined)
|
|
58304
|
+
continue;
|
|
58305
|
+
const key = JSON.stringify([run.parentRunId, run.parentStageId]);
|
|
58306
|
+
const children = childrenByBoundary.get(key) ?? [];
|
|
58307
|
+
children.push(run);
|
|
58308
|
+
childrenByBoundary.set(key, children);
|
|
58309
|
+
}
|
|
58284
58310
|
const resolveBoundarySegments = createWorkflowBoundarySegmentsResolver(runs);
|
|
58285
58311
|
for (const run of runs) {
|
|
58286
|
-
if (!ownsRun(run.id))
|
|
58312
|
+
if (externalRunRouteOwner(activeStore, run.id) !== undefined || !ownsRun(run.id))
|
|
58287
58313
|
continue;
|
|
58288
|
-
const rootRunId =
|
|
58314
|
+
const rootRunId = reciprocalWorkflowRootRunId(runById, run.id);
|
|
58289
58315
|
if (rootRunId === undefined)
|
|
58290
58316
|
continue;
|
|
58291
58317
|
ownedRunIds.add(run.id);
|
|
58292
|
-
const parentRun =
|
|
58318
|
+
const parentRun = run.parentRunId === undefined ? undefined : runById.get(run.parentRunId);
|
|
58293
58319
|
const boundary = parentRun?.stages.find((stage) => stage.id === run.parentStageId);
|
|
58294
58320
|
const parent = parentRun === undefined || boundary === undefined ? undefined : {
|
|
58295
58321
|
runId: parentRun.id,
|
|
58296
|
-
stageKeys: [boundary.id, boundary.name].filter((key) =>
|
|
58322
|
+
stageKeys: [boundary.id, boundary.name].filter((key) => resolveIndexedChildRun(childrenByBoundary, parentRun, key)?.id === run.id)
|
|
58297
58323
|
};
|
|
58298
58324
|
const announcement = {
|
|
58299
58325
|
runId: run.id,
|
|
@@ -58434,16 +58460,22 @@ function registerPendingStageIntercomBridge(pi, activeStore, ownsRun = () => tru
|
|
|
58434
58460
|
payload.handled = true;
|
|
58435
58461
|
payload.completion = queueAndPersist(activeStore, resolvedEvent, workflowInvocationIntercomGroup(rootRunId), stage.pendingStageDeliveryAvailable === true);
|
|
58436
58462
|
};
|
|
58437
|
-
const
|
|
58438
|
-
const stickySubscription = pi.events?.on?.(STICKY_LIVE_DELIVERED_EVENT, (payload) => {
|
|
58463
|
+
const routeStickyLiveDelivered = (payload) => {
|
|
58439
58464
|
if (disposed || !isStickyLiveDeliveredEvent(payload) || payload.handled)
|
|
58440
58465
|
return;
|
|
58441
58466
|
payload.handled = true;
|
|
58442
58467
|
payload.completion = outsideWorkflowExecutions(() => recordConfirmedStickyDeliveries(activeStore, payload));
|
|
58468
|
+
};
|
|
58469
|
+
const subscription = pi.events?.on?.(PENDING_STAGE_MESSAGE_EVENT, (payload) => outsideWorkflowExecutions(() => routePendingStageMessage(payload)));
|
|
58470
|
+
const stickySubscription = pi.events?.on?.(STICKY_LIVE_DELIVERED_EVENT, routeStickyLiveDelivered);
|
|
58471
|
+
const disposeRouter = registerPendingStageMessageRouter(activeStore, {
|
|
58472
|
+
route: (payload) => outsideWorkflowExecutions(() => routePendingStageMessage(payload)),
|
|
58473
|
+
sticky: routeStickyLiveDelivered
|
|
58443
58474
|
});
|
|
58444
58475
|
const dispose = () => {
|
|
58445
58476
|
disposed = true;
|
|
58446
58477
|
disposeReadiness();
|
|
58478
|
+
disposeRouter();
|
|
58447
58479
|
routeCompletions.clear();
|
|
58448
58480
|
unsubscribeStore();
|
|
58449
58481
|
if (typeof subscription === "function")
|
|
@@ -58569,6 +58601,12 @@ function isMaterializedNonAgentTarget(runs, target) {
|
|
|
58569
58601
|
return false;
|
|
58570
58602
|
return nonAgentNodes(run).some((node) => node.id === stageKey || node.name === stageKey);
|
|
58571
58603
|
}
|
|
58604
|
+
function resolveIndexedChildRun(childrenByBoundary, parent, segment) {
|
|
58605
|
+
const boundariesById = parent.stages.filter((stage) => stage.id === segment);
|
|
58606
|
+
const boundaries = boundariesById.length > 0 ? boundariesById : parent.stages.filter((stage) => stage.name === segment);
|
|
58607
|
+
const children = boundaries.flatMap((boundary) => childrenByBoundary.get(JSON.stringify([parent.id, boundary.id])) ?? []);
|
|
58608
|
+
return children.length === 1 ? children[0] : undefined;
|
|
58609
|
+
}
|
|
58572
58610
|
function resolveChildRun(runs, parent, segment) {
|
|
58573
58611
|
const boundariesById = parent.stages.filter((stage) => stage.id === segment);
|
|
58574
58612
|
const boundaries = boundariesById.length > 0 ? boundariesById : parent.stages.filter((stage) => stage.name === segment);
|
|
@@ -58908,6 +58946,7 @@ var init_pending_stage_intercom = __esm(() => {
|
|
|
58908
58946
|
init_execution_context_boundary();
|
|
58909
58947
|
init_run_owner_backend();
|
|
58910
58948
|
init_store_backend();
|
|
58949
|
+
init_external_run_routes();
|
|
58911
58950
|
init_intercom_group();
|
|
58912
58951
|
init_pending_stage_route_capability();
|
|
58913
58952
|
init_pending_stage_route_readiness();
|
|
@@ -59366,7 +59405,7 @@ function isWorkflowFutureStageRosterEntries(value2) {
|
|
|
59366
59405
|
function isWorkflowStageRosterEntries(value2) {
|
|
59367
59406
|
return Array.isArray(value2) && value2.every((entry) => typeof entry === "object" && entry !== null && entry.kind === "workflow-stage" && typeof entry.runId === "string" && typeof entry.stageId === "string" && typeof entry.stageName === "string" && typeof entry.target === "string" && (entry.lifecycle === "pending" || entry.lifecycle === "running") && typeof entry.group === "string" && (entry.sessionId === undefined || typeof entry.sessionId === "string"));
|
|
59368
59407
|
}
|
|
59369
|
-
var BROKER_SOCKET, BROKER_CONNECT_TIMEOUT_MS = 1e4, PENDING_STAGE_ROUTE_ACK_TIMEOUT_MS = 1e4, GROUP_REQUEST_TIMEOUT_MS = 5000, PRESENCE_ACK_TIMEOUT_MS = 5000, RESTORING_RECIPIENT_REQUEST_TIMEOUT_MS = 5000, HELD_SEND_GRACE_MS = 5000, IntercomClient;
|
|
59408
|
+
var BROKER_SOCKET, BROKER_CONNECT_TIMEOUT_MS = 1e4, PENDING_STAGE_ROUTE_ACK_TIMEOUT_MS = 1e4, GROUP_REQUEST_TIMEOUT_MS = 5000, PRESENCE_ACK_TIMEOUT_MS = 5000, RESTORING_RECIPIENT_REQUEST_TIMEOUT_MS = 5000, HELD_SEND_GRACE_MS = 5000, IntercomPendingStageRouteRefusedError, PENDING_STAGE_ROUTE_REFUSAL_CODES, IntercomClient;
|
|
59370
59409
|
var init_client = __esm(() => {
|
|
59371
59410
|
init_paths();
|
|
59372
59411
|
init_pending_send_registry();
|
|
@@ -59375,6 +59414,21 @@ var init_client = __esm(() => {
|
|
|
59375
59414
|
init_live_route_refusal();
|
|
59376
59415
|
init_retry_policy();
|
|
59377
59416
|
BROKER_SOCKET = getBrokerSocketPath();
|
|
59417
|
+
IntercomPendingStageRouteRefusedError = class IntercomPendingStageRouteRefusedError extends Error {
|
|
59418
|
+
connectionKept;
|
|
59419
|
+
code;
|
|
59420
|
+
constructor(reason, refusal = { connectionKept: false }) {
|
|
59421
|
+
super(reason);
|
|
59422
|
+
this.name = "IntercomPendingStageRouteRefusedError";
|
|
59423
|
+
this.connectionKept = refusal.connectionKept;
|
|
59424
|
+
this.code = refusal.code;
|
|
59425
|
+
}
|
|
59426
|
+
};
|
|
59427
|
+
PENDING_STAGE_ROUTE_REFUSAL_CODES = new Set([
|
|
59428
|
+
"not_authorized",
|
|
59429
|
+
"invalid_stage_roster",
|
|
59430
|
+
"invalid_possible_stage_roster"
|
|
59431
|
+
]);
|
|
59378
59432
|
IntercomClient = class IntercomClient extends EventEmitter {
|
|
59379
59433
|
returnAddress;
|
|
59380
59434
|
preferredSessionId;
|
|
@@ -59382,6 +59436,8 @@ var init_client = __esm(() => {
|
|
|
59382
59436
|
connectTimeoutMs;
|
|
59383
59437
|
brokerAcknowledgesReceipt = false;
|
|
59384
59438
|
brokerAcknowledgesPendingStageRoutes = false;
|
|
59439
|
+
multiplexesPendingStageRoutes;
|
|
59440
|
+
brokerRefusesPendingStageRoutesInPlace = false;
|
|
59385
59441
|
brokerAnswersStatus = false;
|
|
59386
59442
|
socket = null;
|
|
59387
59443
|
_sessionId = null;
|
|
@@ -59447,6 +59503,7 @@ var init_client = __esm(() => {
|
|
|
59447
59503
|
this.preferredSessionId = options.preferredSessionId;
|
|
59448
59504
|
this.acknowledgesReceipt = options.acknowledgesReceipt === true;
|
|
59449
59505
|
this.connectTimeoutMs = options.connectTimeoutMs ?? BROKER_CONNECT_TIMEOUT_MS;
|
|
59506
|
+
this.multiplexesPendingStageRoutes = options.multiplexesPendingStageRoutes === true;
|
|
59450
59507
|
}
|
|
59451
59508
|
failPending(error) {
|
|
59452
59509
|
this.pendingSends.rejectAll(error);
|
|
@@ -59559,6 +59616,7 @@ var init_client = __esm(() => {
|
|
|
59559
59616
|
this._supervisorSessionId = null;
|
|
59560
59617
|
this.brokerAcknowledgesReceipt = false;
|
|
59561
59618
|
this.brokerAcknowledgesPendingStageRoutes = false;
|
|
59619
|
+
this.brokerRefusesPendingStageRoutesInPlace = false;
|
|
59562
59620
|
this.brokerAnswersStatus = false;
|
|
59563
59621
|
this._messageSource = undefined;
|
|
59564
59622
|
this.disconnectError = null;
|
|
@@ -59637,7 +59695,7 @@ var init_client = __esm(() => {
|
|
|
59637
59695
|
}
|
|
59638
59696
|
switch (brokerMessage.type) {
|
|
59639
59697
|
case "registered": {
|
|
59640
|
-
if (typeof brokerMessage.sessionId !== "string" || brokerMessage.supervisorSessionId !== undefined && typeof brokerMessage.supervisorSessionId !== "string" || brokerMessage.acknowledgesReceipt !== undefined && brokerMessage.acknowledgesReceipt !== true || brokerMessage.acknowledgesPendingStageRoutes !== undefined && brokerMessage.acknowledgesPendingStageRoutes !== true || brokerMessage.brokerStatus !== undefined && brokerMessage.brokerStatus !== true) {
|
|
59698
|
+
if (typeof brokerMessage.sessionId !== "string" || brokerMessage.supervisorSessionId !== undefined && typeof brokerMessage.supervisorSessionId !== "string" || brokerMessage.acknowledgesReceipt !== undefined && brokerMessage.acknowledgesReceipt !== true || brokerMessage.acknowledgesPendingStageRoutes !== undefined && brokerMessage.acknowledgesPendingStageRoutes !== true || brokerMessage.refusesPendingStageRoutesInPlace !== undefined && brokerMessage.refusesPendingStageRoutesInPlace !== true || brokerMessage.brokerStatus !== undefined && brokerMessage.brokerStatus !== true) {
|
|
59641
59699
|
throw new Error("Invalid registered message");
|
|
59642
59700
|
}
|
|
59643
59701
|
if (this._sessionId !== null) {
|
|
@@ -59647,6 +59705,7 @@ var init_client = __esm(() => {
|
|
|
59647
59705
|
this._supervisorSessionId = typeof brokerMessage.supervisorSessionId === "string" ? brokerMessage.supervisorSessionId : null;
|
|
59648
59706
|
this.brokerAcknowledgesReceipt = this.acknowledgesReceipt && brokerMessage.acknowledgesReceipt === true;
|
|
59649
59707
|
this.brokerAcknowledgesPendingStageRoutes = brokerMessage.acknowledgesPendingStageRoutes === true;
|
|
59708
|
+
this.brokerRefusesPendingStageRoutesInPlace = brokerMessage.refusesPendingStageRoutesInPlace === true;
|
|
59650
59709
|
this.brokerAnswersStatus = brokerMessage.brokerStatus === true;
|
|
59651
59710
|
this.emit("_registered", { type: "registered", sessionId: brokerMessage.sessionId });
|
|
59652
59711
|
break;
|
|
@@ -59659,6 +59718,12 @@ var init_client = __esm(() => {
|
|
|
59659
59718
|
this.emit("_registration_failed", refusal);
|
|
59660
59719
|
break;
|
|
59661
59720
|
}
|
|
59721
|
+
const refusedRouteId = typeof brokerMessage.requestId === "string" ? brokerMessage.requestId : "";
|
|
59722
|
+
const refusedRoute = this.pendingStageRouteRegistrations.get(refusedRouteId);
|
|
59723
|
+
if (refusedRoute !== undefined) {
|
|
59724
|
+
this.pendingStageRouteRegistrations.delete(refusedRouteId);
|
|
59725
|
+
refusedRoute.reject(new IntercomPendingStageRouteRefusedError(brokerMessage.reason));
|
|
59726
|
+
}
|
|
59662
59727
|
this.disconnectError ??= refusal;
|
|
59663
59728
|
this.failPending(this.disconnectError);
|
|
59664
59729
|
this.socket?.destroy();
|
|
@@ -59784,6 +59849,21 @@ var init_client = __esm(() => {
|
|
|
59784
59849
|
this.emit("pending_stage_route_revoked", { runId });
|
|
59785
59850
|
break;
|
|
59786
59851
|
}
|
|
59852
|
+
case "pending_stage_route_refused": {
|
|
59853
|
+
const { requestId, runId, code, reason } = brokerMessage;
|
|
59854
|
+
if (typeof requestId !== "string" || typeof runId !== "string" || typeof reason !== "string" || typeof code !== "string") {
|
|
59855
|
+
throw new Error("Invalid pending-stage route refusal");
|
|
59856
|
+
}
|
|
59857
|
+
const pending = this.pendingStageRouteRegistrations.get(requestId);
|
|
59858
|
+
if (!pending)
|
|
59859
|
+
return;
|
|
59860
|
+
this.pendingStageRouteRegistrations.delete(requestId);
|
|
59861
|
+
pending.reject(new IntercomPendingStageRouteRefusedError(reason, {
|
|
59862
|
+
connectionKept: true,
|
|
59863
|
+
...PENDING_STAGE_ROUTE_REFUSAL_CODES.has(code) ? { code } : {}
|
|
59864
|
+
}));
|
|
59865
|
+
break;
|
|
59866
|
+
}
|
|
59787
59867
|
case "pending_stage_route_registered": {
|
|
59788
59868
|
const { requestId } = brokerMessage;
|
|
59789
59869
|
if (typeof requestId !== "string")
|
|
@@ -60287,7 +60367,8 @@ var init_client = __esm(() => {
|
|
|
60287
60367
|
stages,
|
|
60288
60368
|
possibleStages,
|
|
60289
60369
|
...parent === undefined ? {} : { parent },
|
|
60290
|
-
...dormant === true ? { dormant: true } : {}
|
|
60370
|
+
...dormant === true ? { dormant: true } : {},
|
|
60371
|
+
...this.multiplexesPendingStageRoutes && this.brokerRefusesPendingStageRoutesInPlace ? { refusalKeepsConnection: true } : {}
|
|
60291
60372
|
});
|
|
60292
60373
|
} catch (error) {
|
|
60293
60374
|
clearTimeout(timeout);
|
|
@@ -61021,120 +61102,165 @@ var init_reconnect_backoff = __esm(() => {
|
|
|
61021
61102
|
BACKOFF_MS = [1000, 2000, 5000, 1e4, 30000];
|
|
61022
61103
|
});
|
|
61023
61104
|
|
|
61024
|
-
// dist/builtin/intercom/route-connect-gate.ts
|
|
61025
|
-
function createConcurrencyGate(limit) {
|
|
61026
|
-
let active = 0;
|
|
61027
|
-
const waiting = [];
|
|
61028
|
-
const release = () => {
|
|
61029
|
-
const next = waiting.shift();
|
|
61030
|
-
if (next === undefined)
|
|
61031
|
-
active -= 1;
|
|
61032
|
-
else
|
|
61033
|
-
next();
|
|
61034
|
-
};
|
|
61035
|
-
return {
|
|
61036
|
-
async run(operation) {
|
|
61037
|
-
if (active < limit)
|
|
61038
|
-
active += 1;
|
|
61039
|
-
else
|
|
61040
|
-
await new Promise((resolve) => waiting.push(resolve));
|
|
61041
|
-
try {
|
|
61042
|
-
return await operation();
|
|
61043
|
-
} finally {
|
|
61044
|
-
release();
|
|
61045
|
-
}
|
|
61046
|
-
}
|
|
61047
|
-
};
|
|
61048
|
-
}
|
|
61049
|
-
var ROUTE_CLIENT_CONNECT_CONCURRENCY = 4;
|
|
61050
|
-
|
|
61051
61105
|
// dist/builtin/intercom/workflow-route-owner.ts
|
|
61052
61106
|
import { randomUUID as randomUUID14 } from "node:crypto";
|
|
61053
61107
|
import { createEventBus, getBuiltinPackagePaths } from "@bastani/atomic";
|
|
61108
|
+
function closedError() {
|
|
61109
|
+
return new Error("Workflow Intercom route owner closed");
|
|
61110
|
+
}
|
|
61054
61111
|
function createWorkflowRouteOwner(cwd) {
|
|
61055
61112
|
const events = createEventBus();
|
|
61056
61113
|
const config = loadConfig();
|
|
61057
61114
|
const id = randomUUID14();
|
|
61058
61115
|
const routes = new Map;
|
|
61059
|
-
const
|
|
61116
|
+
const connections = new Map;
|
|
61117
|
+
const isCurrent = (state) => routes.get(state.route.runId) === state;
|
|
61118
|
+
const connectionKey = (state) => state.isolated ? state.route.runId : state.route.group;
|
|
61119
|
+
function connectionOf(state) {
|
|
61120
|
+
const key = connectionKey(state);
|
|
61121
|
+
let connection = connections.get(key);
|
|
61122
|
+
if (connection === undefined) {
|
|
61123
|
+
connection = { key, group: state.route.group, acknowledged: false, routes: new Map };
|
|
61124
|
+
connections.set(key, connection);
|
|
61125
|
+
}
|
|
61126
|
+
connection.routes.set(state.route.runId, state);
|
|
61127
|
+
return connection;
|
|
61128
|
+
}
|
|
61129
|
+
function detach(state) {
|
|
61130
|
+
const key = connectionKey(state);
|
|
61131
|
+
const connection = connections.get(key);
|
|
61132
|
+
connection?.routes.delete(state.route.runId);
|
|
61133
|
+
if (!state.isolated || connection === undefined)
|
|
61134
|
+
return;
|
|
61135
|
+
connections.delete(key);
|
|
61136
|
+
connection.client?.disconnect().catch(() => {});
|
|
61137
|
+
}
|
|
61138
|
+
function settle(state, registered) {
|
|
61139
|
+
state.registered = registered;
|
|
61140
|
+
if (!registered && isCurrent(state))
|
|
61141
|
+
return;
|
|
61142
|
+
for (const waiter of state.waiters)
|
|
61143
|
+
waiter(registered);
|
|
61144
|
+
state.waiters.clear();
|
|
61145
|
+
}
|
|
61060
61146
|
function scheduleRetry(state) {
|
|
61061
|
-
if (state
|
|
61147
|
+
if (!isCurrent(state) || state.retry !== undefined)
|
|
61062
61148
|
return;
|
|
61063
61149
|
state.retry = setTimeout(() => {
|
|
61064
61150
|
state.retry = undefined;
|
|
61065
|
-
publish(state, state.route).catch(() =>
|
|
61151
|
+
publish(state, state.route).catch(() => {});
|
|
61066
61152
|
}, jitteredReconnectDelayMs(state.attempt++));
|
|
61067
61153
|
state.retry.unref();
|
|
61068
61154
|
}
|
|
61069
|
-
|
|
61070
|
-
if (
|
|
61071
|
-
|
|
61072
|
-
|
|
61073
|
-
|
|
61074
|
-
|
|
61075
|
-
|
|
61076
|
-
|
|
61077
|
-
}
|
|
61078
|
-
|
|
61079
|
-
|
|
61080
|
-
|
|
61081
|
-
if (
|
|
61082
|
-
|
|
61083
|
-
|
|
61084
|
-
|
|
61085
|
-
|
|
61086
|
-
|
|
61087
|
-
|
|
61088
|
-
|
|
61089
|
-
|
|
61090
|
-
|
|
61091
|
-
|
|
61092
|
-
|
|
61093
|
-
if (
|
|
61094
|
-
|
|
61095
|
-
const
|
|
61096
|
-
|
|
61097
|
-
|
|
61098
|
-
|
|
61099
|
-
|
|
61100
|
-
|
|
61101
|
-
|
|
61102
|
-
|
|
61103
|
-
|
|
61104
|
-
|
|
61105
|
-
|
|
61106
|
-
|
|
61107
|
-
|
|
61108
|
-
|
|
61109
|
-
|
|
61110
|
-
|
|
61111
|
-
|
|
61112
|
-
|
|
61113
|
-
|
|
61155
|
+
function drop(connection, client) {
|
|
61156
|
+
if (connection.client === client) {
|
|
61157
|
+
connection.client = undefined;
|
|
61158
|
+
connection.acknowledged = false;
|
|
61159
|
+
for (const state of connection.routes.values()) {
|
|
61160
|
+
settle(state, false);
|
|
61161
|
+
scheduleRetry(state);
|
|
61162
|
+
}
|
|
61163
|
+
}
|
|
61164
|
+
client.disconnect().catch(() => {});
|
|
61165
|
+
}
|
|
61166
|
+
function connect(connection) {
|
|
61167
|
+
if (connection.client?.isConnected())
|
|
61168
|
+
return Promise.resolve(connection.client);
|
|
61169
|
+
connection.connecting ??= (async () => {
|
|
61170
|
+
const [extensionDir] = getBuiltinPackagePaths({
|
|
61171
|
+
workflows: false,
|
|
61172
|
+
subagents: false,
|
|
61173
|
+
mcp: false,
|
|
61174
|
+
"web-access": false
|
|
61175
|
+
});
|
|
61176
|
+
if (extensionDir === undefined)
|
|
61177
|
+
throw new Error("Intercom builtin package is unavailable");
|
|
61178
|
+
await spawnBrokerIfNeeded(config.brokerCommand, config.brokerArgs, undefined, extensionDir);
|
|
61179
|
+
if (connections.get(connection.key) !== connection)
|
|
61180
|
+
throw closedError();
|
|
61181
|
+
const client = new IntercomClient(`${id}:pending-stage-route:${connection.key}`, {
|
|
61182
|
+
multiplexesPendingStageRoutes: true
|
|
61183
|
+
});
|
|
61184
|
+
connection.client = client;
|
|
61185
|
+
connection.acknowledged = false;
|
|
61186
|
+
client.on("error", () => {});
|
|
61187
|
+
client.on("disconnected", () => drop(connection, client));
|
|
61188
|
+
client.on("pending_stage_message", (request) => {
|
|
61189
|
+
if (connection.client !== client)
|
|
61190
|
+
return;
|
|
61191
|
+
const event = { ...request, handled: false };
|
|
61192
|
+
events.emit("atomic:workflow-pending-stage-message", event);
|
|
61193
|
+
const completion = event.handled && event.completion !== undefined ? event.completion : Promise.resolve({ outcome: "refused", reason: "Session not found" });
|
|
61194
|
+
completion.catch((error) => ({ outcome: "refused", reason: error.message })).then((result) => client.respondPendingStageMessage(request.requestId, result)).catch(() => {});
|
|
61195
|
+
});
|
|
61196
|
+
client.on("sticky_live_delivered", (notice) => {
|
|
61197
|
+
if (connection.client !== client)
|
|
61198
|
+
return;
|
|
61199
|
+
events.emit("atomic:workflow-sticky-live-delivered", { ...notice, handled: false });
|
|
61200
|
+
});
|
|
61201
|
+
try {
|
|
61202
|
+
await client.connect({
|
|
61203
|
+
cwd,
|
|
61204
|
+
model: "workflow",
|
|
61205
|
+
pid: process.pid,
|
|
61206
|
+
startedAt: Date.now(),
|
|
61207
|
+
lastActivity: Date.now(),
|
|
61208
|
+
group: connection.group,
|
|
61209
|
+
groups: [connection.group],
|
|
61210
|
+
recipientPurpose: "control"
|
|
61211
|
+
});
|
|
61212
|
+
if (connections.get(connection.key) !== connection)
|
|
61213
|
+
throw closedError();
|
|
61214
|
+
} catch (error) {
|
|
61215
|
+
drop(connection, client);
|
|
61216
|
+
throw error;
|
|
61217
|
+
}
|
|
61218
|
+
for (const state of connection.routes.values()) {
|
|
61219
|
+
if (state.registered || state.retry === undefined)
|
|
61220
|
+
continue;
|
|
61221
|
+
clearTimeout(state.retry);
|
|
61222
|
+
state.retry = undefined;
|
|
61223
|
+
publish(state, state.route).catch(() => {});
|
|
61224
|
+
}
|
|
61225
|
+
return client;
|
|
61226
|
+
})().finally(() => {
|
|
61227
|
+
connection.connecting = undefined;
|
|
61114
61228
|
});
|
|
61115
|
-
return
|
|
61229
|
+
return connection.connecting;
|
|
61116
61230
|
}
|
|
61117
61231
|
function publish(state, route) {
|
|
61118
|
-
const completion = state.tail.catch(() => {}).then(
|
|
61119
|
-
if (state
|
|
61120
|
-
throw
|
|
61121
|
-
const
|
|
61232
|
+
const completion = state.tail.catch(() => {}).then(async () => {
|
|
61233
|
+
if (!isCurrent(state))
|
|
61234
|
+
throw closedError();
|
|
61235
|
+
const connection = connectionOf(state);
|
|
61236
|
+
const established = connection.acknowledged && connection.client?.isConnected() ? connection.client : undefined;
|
|
61237
|
+
let client;
|
|
61122
61238
|
try {
|
|
61123
|
-
|
|
61239
|
+
client = await connect(connection);
|
|
61124
61240
|
await client.registerPendingStageRouteAcknowledged(route.runId, route.group, route.capability, route.stages, route.possibleStages, route.parent);
|
|
61241
|
+
if (connection.client !== client || !isCurrent(state))
|
|
61242
|
+
return;
|
|
61243
|
+
connection.acknowledged = true;
|
|
61125
61244
|
state.attempt = 0;
|
|
61126
61245
|
clearTimeout(state.retry);
|
|
61127
61246
|
state.retry = undefined;
|
|
61247
|
+
settle(state, true);
|
|
61128
61248
|
} catch (error) {
|
|
61129
|
-
|
|
61130
|
-
|
|
61131
|
-
|
|
61132
|
-
await client?.disconnect();
|
|
61249
|
+
if (error instanceof IntercomPendingStageRouteRefusedError && error.connectionKept) {
|
|
61250
|
+
scheduleRetry(state);
|
|
61251
|
+
throw error;
|
|
61133
61252
|
}
|
|
61253
|
+
if (error instanceof IntercomPendingStageRouteRefusedError && !state.isolated && isCurrent(state)) {
|
|
61254
|
+
detach(state);
|
|
61255
|
+
state.isolated = true;
|
|
61256
|
+
connectionOf(state);
|
|
61257
|
+
}
|
|
61258
|
+
if (client !== undefined && (client !== established || !client.isConnected()))
|
|
61259
|
+
drop(connection, client);
|
|
61134
61260
|
scheduleRetry(state);
|
|
61135
61261
|
throw error;
|
|
61136
61262
|
}
|
|
61137
|
-
})
|
|
61263
|
+
});
|
|
61138
61264
|
state.tail = completion;
|
|
61139
61265
|
completion.catch(() => {});
|
|
61140
61266
|
return completion;
|
|
@@ -61143,10 +61269,15 @@ function createWorkflowRouteOwner(cwd) {
|
|
|
61143
61269
|
const route = payload;
|
|
61144
61270
|
let state = routes.get(route.runId);
|
|
61145
61271
|
if (state === undefined) {
|
|
61146
|
-
state = { route, tail: Promise.resolve(), attempt: 0,
|
|
61272
|
+
state = { route, tail: Promise.resolve(), attempt: 0, registered: false, isolated: false, waiters: new Set };
|
|
61147
61273
|
routes.set(route.runId, state);
|
|
61274
|
+
} else if (state.route.group !== route.group) {
|
|
61275
|
+
detach(state);
|
|
61276
|
+
state.isolated = false;
|
|
61277
|
+
settle(state, false);
|
|
61148
61278
|
}
|
|
61149
61279
|
state.route = route;
|
|
61280
|
+
connectionOf(state);
|
|
61150
61281
|
route.completion = publish(state, route);
|
|
61151
61282
|
});
|
|
61152
61283
|
events.on("atomic:workflow-pending-stage-undeliverable", (payload) => {
|
|
@@ -61157,7 +61288,7 @@ function createWorkflowRouteOwner(cwd) {
|
|
|
61157
61288
|
event.handled = true;
|
|
61158
61289
|
event.completion = (async () => {
|
|
61159
61290
|
await publish(state, state.route);
|
|
61160
|
-
const client = state
|
|
61291
|
+
const client = connections.get(connectionKey(state))?.client;
|
|
61161
61292
|
if (client === undefined || !client.isConnected())
|
|
61162
61293
|
return false;
|
|
61163
61294
|
const text = `Pending workflow stage could not receive intercom message: ${event.reason}`;
|
|
@@ -61167,18 +61298,33 @@ function createWorkflowRouteOwner(cwd) {
|
|
|
61167
61298
|
});
|
|
61168
61299
|
return {
|
|
61169
61300
|
events,
|
|
61301
|
+
registered(runId) {
|
|
61302
|
+
const state = routes.get(runId);
|
|
61303
|
+
if (state === undefined)
|
|
61304
|
+
return Promise.resolve(false);
|
|
61305
|
+
if (state.registered)
|
|
61306
|
+
return Promise.resolve(true);
|
|
61307
|
+
return new Promise((resolve) => state.waiters.add(resolve));
|
|
61308
|
+
},
|
|
61170
61309
|
async closeGroup(group) {
|
|
61171
61310
|
const closing = [...routes.values()].filter((state) => state.route.group === group);
|
|
61311
|
+
const closingConnections = [...connections.values()].filter((connection) => connection.group === group);
|
|
61312
|
+
for (const connection of closingConnections) {
|
|
61313
|
+
connections.delete(connection.key);
|
|
61314
|
+
connection.routes.clear();
|
|
61315
|
+
}
|
|
61172
61316
|
for (const state of closing) {
|
|
61173
|
-
state.closed = true;
|
|
61174
|
-
clearTimeout(state.retry);
|
|
61175
61317
|
routes.delete(state.route.runId);
|
|
61318
|
+
clearTimeout(state.retry);
|
|
61319
|
+
state.retry = undefined;
|
|
61320
|
+
settle(state, false);
|
|
61176
61321
|
}
|
|
61177
|
-
await Promise.all(
|
|
61178
|
-
await
|
|
61179
|
-
await
|
|
61180
|
-
await state.client?.disconnect();
|
|
61322
|
+
await Promise.all(closingConnections.map(async (connection) => {
|
|
61323
|
+
await connection.client?.disconnect();
|
|
61324
|
+
await connection.connecting?.catch(() => {});
|
|
61181
61325
|
}));
|
|
61326
|
+
await Promise.all(closing.map((state) => state.tail.catch(() => {})));
|
|
61327
|
+
await Promise.all(closingConnections.map((connection) => connection.client?.disconnect()));
|
|
61182
61328
|
}
|
|
61183
61329
|
};
|
|
61184
61330
|
}
|
|
@@ -61297,7 +61443,7 @@ var init_workflow_infrastructure_error = __esm(() => {
|
|
|
61297
61443
|
[/Local worker (?:process )?cleanup (?:was not confirmed|unconfirmed)/, "worker_cleanup_unconfirmed"],
|
|
61298
61444
|
[/Local worker retry limit/, "worker_retry_limit"],
|
|
61299
61445
|
[/Local worker (?:startup timed out|exited before readiness)/, "worker_startup_failed"],
|
|
61300
|
-
[/Worker exited before delivery acknowledgement/, "worker_lost"],
|
|
61446
|
+
[/Worker exited before delivery acknowledgement|Local worker pool coordinator exited/, "worker_lost"],
|
|
61301
61447
|
[/Stale work lease/, "stale_work_lease"],
|
|
61302
61448
|
[/Intercom runtime no longer active/, "intercom_unavailable"],
|
|
61303
61449
|
[/Storage unavailable/, "storage_unavailable"]
|
|
@@ -61313,12 +61459,925 @@ var init_workflow_infrastructure_error = __esm(() => {
|
|
|
61313
61459
|
};
|
|
61314
61460
|
});
|
|
61315
61461
|
|
|
61462
|
+
// dist/builtin/workflows/src/workers/coordinator-protocol.ts
|
|
61463
|
+
var POOL_COORDINATOR_ROLE = "pool-coordinator", POOL_SUMMARY_INTERVAL_MS = 250;
|
|
61464
|
+
|
|
61316
61465
|
// dist/builtin/workflows/src/workers/host-options.ts
|
|
61317
61466
|
var hostWorkerBindings;
|
|
61318
61467
|
var init_host_options = __esm(() => {
|
|
61319
61468
|
hostWorkerBindings = Symbol("host-worker-bindings");
|
|
61320
61469
|
});
|
|
61321
61470
|
|
|
61471
|
+
// dist/builtin/workflows/src/durable/journal/sql-pool-usage.ts
|
|
61472
|
+
function validate(report) {
|
|
61473
|
+
validateText("key", report.key);
|
|
61474
|
+
if (report.key === "")
|
|
61475
|
+
throw new Error("Pool usage key must not be empty");
|
|
61476
|
+
for (const field of POOL_USAGE_FIELDS) {
|
|
61477
|
+
const value2 = report.usage[field];
|
|
61478
|
+
if (typeof value2 !== "number" || !Number.isFinite(value2) || value2 < 0)
|
|
61479
|
+
throw new Error(`Pool usage ${field} must be a finite non-negative number`);
|
|
61480
|
+
}
|
|
61481
|
+
}
|
|
61482
|
+
|
|
61483
|
+
class SqlPoolUsage {
|
|
61484
|
+
transaction;
|
|
61485
|
+
journal;
|
|
61486
|
+
table;
|
|
61487
|
+
constructor(transaction, journal) {
|
|
61488
|
+
this.transaction = transaction;
|
|
61489
|
+
this.journal = journal;
|
|
61490
|
+
this.table = poolUsageTable(journal.runs);
|
|
61491
|
+
}
|
|
61492
|
+
static async open(transaction, journal) {
|
|
61493
|
+
const usage = new SqlPoolUsage(transaction, journal);
|
|
61494
|
+
await transaction((tx) => tx.exec(poolUsageSchema(journal.runs)));
|
|
61495
|
+
return usage;
|
|
61496
|
+
}
|
|
61497
|
+
async totalIn(tx, root) {
|
|
61498
|
+
const row = await tx.get(`SELECT ${POOL_USAGE_FIELDS.map((field) => `COALESCE(SUM(u.${field.toLowerCase()}), 0) AS ${field}`).join(", ")}
|
|
61499
|
+
FROM ${this.table} u JOIN ${this.journal.runs}_work w ON w.id = u.work WHERE w.root = ?`, root);
|
|
61500
|
+
return Object.fromEntries(POOL_USAGE_FIELDS.map((field) => [field, Number(row?.[field] ?? 0)]));
|
|
61501
|
+
}
|
|
61502
|
+
async report(lease, token, reports) {
|
|
61503
|
+
validateText("runId", lease.runId);
|
|
61504
|
+
validateText("holderId", lease.holderId);
|
|
61505
|
+
for (const report of reports)
|
|
61506
|
+
validate(report);
|
|
61507
|
+
const snapshot = reports.map((report) => ({ key: report.key, usage: { ...report.usage } }));
|
|
61508
|
+
return this.transaction(async (tx) => {
|
|
61509
|
+
const authority = await tx.get(`SELECT l.epoch, l.holder_id, l.renewed_at, p.timeout, ${STORE_NOW3} AS now FROM ${this.journal.leases} l
|
|
61510
|
+
LEFT JOIN ${this.journal.runs}_pool p ON p.root = l.run_id WHERE l.run_id = ?`, lease.runId);
|
|
61511
|
+
if (!authority || authority.timeout === null || authority.epoch !== lease.epoch || authority.holder_id !== lease.holderId || isWorkflowLeaseExpired(Number(authority.renewed_at), Number(authority.timeout), Number(authority.now)))
|
|
61512
|
+
throw new StaleLeaseError(lease.runId, lease.epoch, Number(authority?.epoch ?? 0));
|
|
61513
|
+
const work = await tx.get(`SELECT * FROM ${this.journal.runs}_work WHERE id = ? AND root = ?`, token.id, lease.runId);
|
|
61514
|
+
if (work?.state !== "leased" && work?.state !== "completed" || work.epoch !== token.epoch || work.epoch !== lease.epoch || work.generation !== token.generation || work.worker !== token.worker || work.state === "leased" && Number(work.expires_at) < Number(authority.now))
|
|
61515
|
+
throw new Error("Stale work lease");
|
|
61516
|
+
for (const { key, usage } of snapshot)
|
|
61517
|
+
await upsertPoolUsage(tx, this.journal.runs, token.id, token.generation, key, usage);
|
|
61518
|
+
return this.totalIn(tx, lease.runId);
|
|
61519
|
+
});
|
|
61520
|
+
}
|
|
61521
|
+
total(root) {
|
|
61522
|
+
return this.transaction((tx) => this.totalIn(tx, root));
|
|
61523
|
+
}
|
|
61524
|
+
}
|
|
61525
|
+
async function openSqlPoolUsage(journal) {
|
|
61526
|
+
const storage = sqlPoolStorage(journal);
|
|
61527
|
+
if (!storage)
|
|
61528
|
+
throw new Error("Worker pool requires a SQL run journal");
|
|
61529
|
+
await openSqlPoolQueue(journal);
|
|
61530
|
+
return SqlPoolUsage.open(storage.transaction, storage.tables);
|
|
61531
|
+
}
|
|
61532
|
+
var STORE_NOW3 = "CAST(unixepoch('subsec') * 1000 AS INTEGER)";
|
|
61533
|
+
var init_sql_pool_usage = __esm(() => {
|
|
61534
|
+
init_internal();
|
|
61535
|
+
init_pool_stage_usage();
|
|
61536
|
+
init_sql_pool_queue();
|
|
61537
|
+
init_types();
|
|
61538
|
+
init_values();
|
|
61539
|
+
});
|
|
61540
|
+
|
|
61541
|
+
// dist/builtin/workflows/src/workers/pool-coordinator.ts
|
|
61542
|
+
import { fork } from "node:child_process";
|
|
61543
|
+
import { randomUUID as randomUUID15 } from "node:crypto";
|
|
61544
|
+
import { createWriteStream } from "node:fs";
|
|
61545
|
+
import { mkdir as mkdir6, rm as rm3, rmdir } from "node:fs/promises";
|
|
61546
|
+
import { dirname as dirname12, join as join28, resolve as resolve11 } from "node:path";
|
|
61547
|
+
import { setTimeout as delay2 } from "node:timers/promises";
|
|
61548
|
+
function workerLogDirectory(journalPath, root) {
|
|
61549
|
+
return join28(dirname12(resolve11(journalPath)), "worker-logs", encodeURIComponent(root).replaceAll(".", "%2E"));
|
|
61550
|
+
}
|
|
61551
|
+
function token(work) {
|
|
61552
|
+
return {
|
|
61553
|
+
id: work.id,
|
|
61554
|
+
generation: work.generation,
|
|
61555
|
+
worker: work.worker,
|
|
61556
|
+
epoch: work.epoch,
|
|
61557
|
+
renewedAt: work.renewedAt,
|
|
61558
|
+
expiresAt: work.expiresAt
|
|
61559
|
+
};
|
|
61560
|
+
}
|
|
61561
|
+
function outside(runId) {
|
|
61562
|
+
return new Error(`Pool stage publication is outside the work subtree: ${runId}`);
|
|
61563
|
+
}
|
|
61564
|
+
function isLiveStage(stage) {
|
|
61565
|
+
return (stage.sessionId !== undefined || stage.sessionFile !== undefined) && stage.status !== "completed" && stage.status !== "failed" && stage.status !== "skipped";
|
|
61566
|
+
}
|
|
61567
|
+
function projectRootRun(run) {
|
|
61568
|
+
return {
|
|
61569
|
+
id: run.id,
|
|
61570
|
+
name: run.name,
|
|
61571
|
+
status: run.status,
|
|
61572
|
+
startedAt: run.startedAt,
|
|
61573
|
+
stages: run.stages.map((stage) => ({
|
|
61574
|
+
id: stage.id,
|
|
61575
|
+
name: stage.name,
|
|
61576
|
+
status: stage.status,
|
|
61577
|
+
parentIds: [],
|
|
61578
|
+
toolEvents: [],
|
|
61579
|
+
...stage.replayKey === undefined ? {} : { replayKey: stage.replayKey },
|
|
61580
|
+
...stage.intercomGroup === undefined ? {} : { intercomGroup: stage.intercomGroup },
|
|
61581
|
+
...stage.workflowChildRun === undefined ? {} : { workflowChildRun: stage.workflowChildRun },
|
|
61582
|
+
...stage.workflowChild === undefined ? {} : { workflowChild: { ...stage.workflowChild, outputs: {} } }
|
|
61583
|
+
}))
|
|
61584
|
+
};
|
|
61585
|
+
}
|
|
61586
|
+
|
|
61587
|
+
class PooledRoutes {
|
|
61588
|
+
root;
|
|
61589
|
+
owner;
|
|
61590
|
+
mirror = createStore();
|
|
61591
|
+
owners = new Map;
|
|
61592
|
+
attempts = new Map;
|
|
61593
|
+
activated = false;
|
|
61594
|
+
bridge;
|
|
61595
|
+
constructor(root, owner) {
|
|
61596
|
+
this.root = root;
|
|
61597
|
+
this.owner = owner;
|
|
61598
|
+
}
|
|
61599
|
+
static async open(root, cwd, rootRun, forward) {
|
|
61600
|
+
await Promise.resolve().then(() => init_workflow_route_owner());
|
|
61601
|
+
const owner = createWorkflowRouteOwner(cwd);
|
|
61602
|
+
const routes = new PooledRoutes(root, owner);
|
|
61603
|
+
if (rootRun !== undefined)
|
|
61604
|
+
routes.updateRoot(rootRun);
|
|
61605
|
+
routes.bridge = registerPendingStageIntercomBridge({
|
|
61606
|
+
events: {
|
|
61607
|
+
emit: (event, payload) => owner.events.emit(event, payload),
|
|
61608
|
+
on: (event, listener) => event === PENDING_STAGE_MESSAGE_EVENT2 || event === STICKY_LIVE_DELIVERED_EVENT2 ? undefined : owner.events.on(event, listener)
|
|
61609
|
+
}
|
|
61610
|
+
}, routes.mirror, (runId) => routes.activated && routes.owners.has(runId));
|
|
61611
|
+
for (const [event, kind] of [
|
|
61612
|
+
[PENDING_STAGE_MESSAGE_EVENT2, "pending-stage-message"],
|
|
61613
|
+
[STICKY_LIVE_DELIVERED_EVENT2, "sticky-live-delivered"]
|
|
61614
|
+
])
|
|
61615
|
+
owner.events.on(event, (payload) => {
|
|
61616
|
+
const claimed = payload;
|
|
61617
|
+
if (claimed.handled !== false)
|
|
61618
|
+
return;
|
|
61619
|
+
claimed.handled = true;
|
|
61620
|
+
const { handled: _handled, completion: _completion, ...forwarded } = claimed;
|
|
61621
|
+
claimed.completion = forward({ kind, payload: forwarded });
|
|
61622
|
+
});
|
|
61623
|
+
return routes;
|
|
61624
|
+
}
|
|
61625
|
+
updateRoot(rootRun) {
|
|
61626
|
+
this.mirror.recordRunStart({ ...rootRun, inputs: {}, toolNodes: [], rootRunId: this.root });
|
|
61627
|
+
}
|
|
61628
|
+
accept(entries) {
|
|
61629
|
+
this.mirror.batch(() => {
|
|
61630
|
+
for (const { work, runs } of entries) {
|
|
61631
|
+
let attempt = this.attempts.get(work.id);
|
|
61632
|
+
if (attempt !== undefined && work.generation < attempt.generation)
|
|
61633
|
+
continue;
|
|
61634
|
+
if (attempt === undefined || attempt.generation !== work.generation) {
|
|
61635
|
+
if (attempt !== undefined)
|
|
61636
|
+
this.release(attempt);
|
|
61637
|
+
attempt = { generation: work.generation, runs: attempt?.runs ?? new Set, fenced: false };
|
|
61638
|
+
this.attempts.set(work.id, attempt);
|
|
61639
|
+
}
|
|
61640
|
+
if (attempt.fenced)
|
|
61641
|
+
continue;
|
|
61642
|
+
for (const run of runs) {
|
|
61643
|
+
attempt.runs.add(run.id);
|
|
61644
|
+
this.owners.set(run.id, work.id);
|
|
61645
|
+
const { pendingStageMessages: _messages, ...published } = run;
|
|
61646
|
+
this.mirror.recordRunStart({ ...published, rootRunId: this.root });
|
|
61647
|
+
if (run.stages.some((stage) => stage.pendingStageDeliveryAvailable === true))
|
|
61648
|
+
this.activated = true;
|
|
61649
|
+
}
|
|
61650
|
+
}
|
|
61651
|
+
});
|
|
61652
|
+
}
|
|
61653
|
+
fence(work) {
|
|
61654
|
+
const attempt = this.attempts.get(work.id);
|
|
61655
|
+
if (attempt === undefined || attempt.generation > work.generation)
|
|
61656
|
+
return;
|
|
61657
|
+
if (attempt.generation < work.generation) {
|
|
61658
|
+
this.release(attempt);
|
|
61659
|
+
this.attempts.set(work.id, { generation: work.generation, runs: attempt.runs, fenced: true });
|
|
61660
|
+
return;
|
|
61661
|
+
}
|
|
61662
|
+
this.release(attempt);
|
|
61663
|
+
attempt.fenced = true;
|
|
61664
|
+
}
|
|
61665
|
+
async route(work, runId) {
|
|
61666
|
+
this.requireOwned(work, runId);
|
|
61667
|
+
const readiness = workflowPendingStageRouteReady(this.mirror, runId);
|
|
61668
|
+
await readiness?.completion.catch(() => {});
|
|
61669
|
+
readiness?.assertCurrent();
|
|
61670
|
+
const waiting = new AbortController;
|
|
61671
|
+
await Promise.race([
|
|
61672
|
+
this.owner.registered(runId),
|
|
61673
|
+
delay2(ROUTE_REGISTRATION_WAIT_MS, undefined, { signal: waiting.signal, ref: false }).catch(() => {})
|
|
61674
|
+
]);
|
|
61675
|
+
waiting.abort();
|
|
61676
|
+
this.requireOwned(work, runId);
|
|
61677
|
+
return workflowPendingStageRouteCapability(this.mirror, runId);
|
|
61678
|
+
}
|
|
61679
|
+
async undeliverable(payload) {
|
|
61680
|
+
const event = { ...payload, handled: false };
|
|
61681
|
+
this.owner.events.emit(PENDING_STAGE_UNDELIVERABLE_EVENT2, event);
|
|
61682
|
+
return event.handled && await event.completion === true;
|
|
61683
|
+
}
|
|
61684
|
+
async close() {
|
|
61685
|
+
const bridge = this.bridge;
|
|
61686
|
+
if (bridge === undefined)
|
|
61687
|
+
return;
|
|
61688
|
+
this.bridge = undefined;
|
|
61689
|
+
try {
|
|
61690
|
+
await bridge.flush();
|
|
61691
|
+
} finally {
|
|
61692
|
+
bridge.refresh();
|
|
61693
|
+
bridge();
|
|
61694
|
+
await this.owner.closeGroup(workflowInvocationIntercomGroup(this.root));
|
|
61695
|
+
}
|
|
61696
|
+
}
|
|
61697
|
+
requireOwned(work, runId) {
|
|
61698
|
+
const attempt = this.attempts.get(work.id);
|
|
61699
|
+
if (attempt === undefined || attempt.generation !== work.generation || attempt.fenced)
|
|
61700
|
+
throw new Error(STALE_WORK_LEASE);
|
|
61701
|
+
if (!attempt.runs.has(runId))
|
|
61702
|
+
throw outside(runId);
|
|
61703
|
+
}
|
|
61704
|
+
release(attempt) {
|
|
61705
|
+
this.mirror.batch(() => {
|
|
61706
|
+
for (const id of attempt.runs) {
|
|
61707
|
+
rotateWorkflowPendingStageRouteCapability(this.mirror, id);
|
|
61708
|
+
const run = this.mirror.runs().find((candidate) => candidate.id === id);
|
|
61709
|
+
if (run?.stages.some(isLiveStage) !== true)
|
|
61710
|
+
continue;
|
|
61711
|
+
this.mirror.recordRunStart({
|
|
61712
|
+
...run,
|
|
61713
|
+
stages: run.stages.map((stage) => {
|
|
61714
|
+
if (!isLiveStage(stage))
|
|
61715
|
+
return stage;
|
|
61716
|
+
const { sessionId: _session, sessionFile: _file, ...rest } = stage;
|
|
61717
|
+
return { ...rest, attachable: false, attached: false };
|
|
61718
|
+
})
|
|
61719
|
+
});
|
|
61720
|
+
}
|
|
61721
|
+
});
|
|
61722
|
+
}
|
|
61723
|
+
}
|
|
61724
|
+
|
|
61725
|
+
class PoolCoordinator {
|
|
61726
|
+
port;
|
|
61727
|
+
children = new Set;
|
|
61728
|
+
incarnations = new Map;
|
|
61729
|
+
readyWorkers = new Set;
|
|
61730
|
+
logs = new Map;
|
|
61731
|
+
cleanups = new Set;
|
|
61732
|
+
controlAcks = new Map;
|
|
61733
|
+
lastHeard = new Map;
|
|
61734
|
+
pendingActivity = new Map;
|
|
61735
|
+
batches = new Map;
|
|
61736
|
+
publicationFailures = new Map;
|
|
61737
|
+
requests = new Map;
|
|
61738
|
+
deliveryRequests = new Map;
|
|
61739
|
+
inflight = new Set;
|
|
61740
|
+
lastTick = performance.now();
|
|
61741
|
+
stalledMs = 0;
|
|
61742
|
+
ticker;
|
|
61743
|
+
recoveryTimer;
|
|
61744
|
+
flushTimer;
|
|
61745
|
+
lastFlushAt = 0;
|
|
61746
|
+
batchSequence = 0;
|
|
61747
|
+
requestSequence = 0;
|
|
61748
|
+
usageChanged = false;
|
|
61749
|
+
budgetBlocked = false;
|
|
61750
|
+
desiredControl;
|
|
61751
|
+
root;
|
|
61752
|
+
lease;
|
|
61753
|
+
options;
|
|
61754
|
+
journal;
|
|
61755
|
+
queue;
|
|
61756
|
+
usageLedger;
|
|
61757
|
+
usageTotal;
|
|
61758
|
+
admission;
|
|
61759
|
+
routes;
|
|
61760
|
+
publishing = false;
|
|
61761
|
+
sources = [];
|
|
61762
|
+
logDirectory;
|
|
61763
|
+
starting;
|
|
61764
|
+
recovering;
|
|
61765
|
+
nextRecoveryAt = 0;
|
|
61766
|
+
cleanupFailure;
|
|
61767
|
+
closed = false;
|
|
61768
|
+
closing;
|
|
61769
|
+
replacements = 0;
|
|
61770
|
+
constructor(port) {
|
|
61771
|
+
this.port = port;
|
|
61772
|
+
port.onMessage((message) => this.receive(message));
|
|
61773
|
+
this.ticker = setInterval(() => this.observeResponsiveness(), TICK_INTERVAL_MS).unref();
|
|
61774
|
+
}
|
|
61775
|
+
async shutdown() {
|
|
61776
|
+
await this.closing?.catch(() => {});
|
|
61777
|
+
while (this.inflight.size > 0)
|
|
61778
|
+
await Promise.allSettled([...this.inflight]);
|
|
61779
|
+
await this.journal?.close().catch(() => {});
|
|
61780
|
+
}
|
|
61781
|
+
receive(message) {
|
|
61782
|
+
if (message.kind === "request-reply") {
|
|
61783
|
+
const pending = this.requests.get(message.id);
|
|
61784
|
+
this.requests.delete(message.id);
|
|
61785
|
+
if (message.error !== undefined)
|
|
61786
|
+
pending?.reject(new Error(message.error));
|
|
61787
|
+
else
|
|
61788
|
+
pending?.resolve(message.value);
|
|
61789
|
+
return;
|
|
61790
|
+
}
|
|
61791
|
+
if ("id" in message) {
|
|
61792
|
+
const command = message;
|
|
61793
|
+
const handled = this.execute(command).then((value2) => this.port.send({ kind: "reply", id: command.id, value: value2 }), (error) => this.port.send({ kind: "reply", id: command.id, error: error.message }));
|
|
61794
|
+
this.inflight.add(handled);
|
|
61795
|
+
handled.finally(() => this.inflight.delete(handled));
|
|
61796
|
+
return;
|
|
61797
|
+
}
|
|
61798
|
+
this.notice(message);
|
|
61799
|
+
}
|
|
61800
|
+
notice(notice) {
|
|
61801
|
+
if (notice.kind === "budget")
|
|
61802
|
+
this.budgetBlocked = notice.blocked;
|
|
61803
|
+
else if (notice.kind === "root")
|
|
61804
|
+
this.routes?.updateRoot(notice.rootRun);
|
|
61805
|
+
else {
|
|
61806
|
+
const batch = this.batches.get(notice.batch);
|
|
61807
|
+
this.batches.delete(notice.batch);
|
|
61808
|
+
const rejected = new Map(notice.rejected.map((entry) => [entry.workId, entry]));
|
|
61809
|
+
for (const entry of rejected.values())
|
|
61810
|
+
this.publicationFailures.set(entry.workId, { generation: entry.generation, error: entry.error });
|
|
61811
|
+
this.routes?.accept((batch?.entries ?? []).filter((entry) => rejected.get(entry.work.id)?.generation !== entry.work.generation));
|
|
61812
|
+
batch?.settle();
|
|
61813
|
+
}
|
|
61814
|
+
}
|
|
61815
|
+
async execute(command) {
|
|
61816
|
+
switch (command.kind) {
|
|
61817
|
+
case "prepare":
|
|
61818
|
+
return this.prepare(command.root, command.lease, command.options, command.control);
|
|
61819
|
+
case "control":
|
|
61820
|
+
return this.control(command.state);
|
|
61821
|
+
case "start":
|
|
61822
|
+
this.starting ??= this.start(command.sources, command.cwd, command.publish, command.rootRun);
|
|
61823
|
+
return this.starting;
|
|
61824
|
+
case "enqueue":
|
|
61825
|
+
return this.enqueue(command);
|
|
61826
|
+
case "transport":
|
|
61827
|
+
return this.transport(command.request, command);
|
|
61828
|
+
case "undeliverable":
|
|
61829
|
+
return await this.routes?.undeliverable(command.payload) ?? false;
|
|
61830
|
+
case "close":
|
|
61831
|
+
this.closing ??= this.close();
|
|
61832
|
+
return this.closing;
|
|
61833
|
+
}
|
|
61834
|
+
}
|
|
61835
|
+
async prepare(root, lease, options, control) {
|
|
61836
|
+
if (this.closed)
|
|
61837
|
+
throw new Error("Local worker pool is closed");
|
|
61838
|
+
if (this.root !== undefined && this.root !== root)
|
|
61839
|
+
throw new Error("Local worker pool belongs to another root");
|
|
61840
|
+
this.root = root;
|
|
61841
|
+
this.options = options;
|
|
61842
|
+
this.desiredControl ??= control;
|
|
61843
|
+
this.journal ??= await resolveRunJournal({ journal: { path: options.journalPath } });
|
|
61844
|
+
this.usageLedger ??= await openSqlPoolUsage(this.journal);
|
|
61845
|
+
this.usageTotal = await this.usageLedger.total(root);
|
|
61846
|
+
this.lease = lease;
|
|
61847
|
+
const queue = await openSqlPoolQueue(this.journal);
|
|
61848
|
+
await queue.initialize(lease, COORDINATOR_LEASE_MS);
|
|
61849
|
+
await queue.releasePreviousIncarnations(lease);
|
|
61850
|
+
for (;; ) {
|
|
61851
|
+
const intended = this.desiredControl ?? "running";
|
|
61852
|
+
await queue.setControl(lease, intended);
|
|
61853
|
+
if (intended === (this.desiredControl ?? "running"))
|
|
61854
|
+
break;
|
|
61855
|
+
}
|
|
61856
|
+
this.queue = queue;
|
|
61857
|
+
return this.usageTotal;
|
|
61858
|
+
}
|
|
61859
|
+
async control(state) {
|
|
61860
|
+
this.desiredControl = state;
|
|
61861
|
+
if (!this.queue || !this.lease)
|
|
61862
|
+
return;
|
|
61863
|
+
const control = await this.queue.setControl(this.lease, state);
|
|
61864
|
+
const deadline = Date.now() + CONTROL_ACKNOWLEDGEMENT_TIMEOUT_MS;
|
|
61865
|
+
while ([...this.readyWorkers].some((worker) => (this.controlAcks.get(worker) ?? -1) < control.revision)) {
|
|
61866
|
+
if (this.desiredControl !== state)
|
|
61867
|
+
return;
|
|
61868
|
+
if (Date.now() >= deadline) {
|
|
61869
|
+
if (state === "quit")
|
|
61870
|
+
break;
|
|
61871
|
+
throw new Error("Worker control acknowledgement unresolved");
|
|
61872
|
+
}
|
|
61873
|
+
await delay2(25);
|
|
61874
|
+
}
|
|
61875
|
+
this.flush();
|
|
61876
|
+
}
|
|
61877
|
+
async start(sources, cwd, publish, rootRun) {
|
|
61878
|
+
if (this.closed)
|
|
61879
|
+
throw new Error("Local worker pool is closed");
|
|
61880
|
+
const root = this.root;
|
|
61881
|
+
if (!this.queue || !this.lease || !this.options || root === undefined || this.lease.runId !== root)
|
|
61882
|
+
throw new Error("Local worker pool requires root preparation");
|
|
61883
|
+
this.sources = sources;
|
|
61884
|
+
this.publishing = publish;
|
|
61885
|
+
if (publish) {
|
|
61886
|
+
this.routes = await PooledRoutes.open(root, cwd, rootRun, (request) => this.request(request));
|
|
61887
|
+
if (this.closed) {
|
|
61888
|
+
await this.routes.close();
|
|
61889
|
+
throw new Error("Local worker pool is closed");
|
|
61890
|
+
}
|
|
61891
|
+
}
|
|
61892
|
+
if (this.options.admission)
|
|
61893
|
+
this.admission = await SqlResourceAdmission.open(this.journal, {
|
|
61894
|
+
...this.options.admission,
|
|
61895
|
+
root,
|
|
61896
|
+
holder: "coordinator"
|
|
61897
|
+
});
|
|
61898
|
+
this.logDirectory = workerLogDirectory(this.options.journalPath, root);
|
|
61899
|
+
await mkdir6(this.logDirectory, { recursive: true }).catch(() => {
|
|
61900
|
+
return;
|
|
61901
|
+
});
|
|
61902
|
+
this.recoveryTimer = setInterval(() => {
|
|
61903
|
+
this.recoverExpired().catch((error) => this.fail(error));
|
|
61904
|
+
}, RECOVERY_INTERVAL_MS).unref();
|
|
61905
|
+
const workers = Array.from({ length: this.options.workers }, () => this.spawn());
|
|
61906
|
+
try {
|
|
61907
|
+
await Promise.any(workers);
|
|
61908
|
+
} catch (error) {
|
|
61909
|
+
throw error.errors[0];
|
|
61910
|
+
}
|
|
61911
|
+
}
|
|
61912
|
+
async enqueue(command) {
|
|
61913
|
+
if (!this.queue || !this.lease || this.root === undefined)
|
|
61914
|
+
throw new Error("Local worker pool requires root preparation");
|
|
61915
|
+
if (command.rootRun !== undefined)
|
|
61916
|
+
this.routes?.updateRoot(command.rootRun);
|
|
61917
|
+
const work = await this.queue.enqueue(this.lease, command.descriptor, command.boundary);
|
|
61918
|
+
const result = await this.queue.result(this.root, work.id);
|
|
61919
|
+
return { work, ...result === undefined ? {} : { result } };
|
|
61920
|
+
}
|
|
61921
|
+
async transport(request, payload) {
|
|
61922
|
+
const target = this.deliveryRequests.get(request);
|
|
61923
|
+
if (target === undefined || !this.incarnations.has(target.slot.worker))
|
|
61924
|
+
throw new Error("Worker exited before delivery acknowledgement");
|
|
61925
|
+
const { slot, workerRequest } = target;
|
|
61926
|
+
await new Promise((resolve, reject) => {
|
|
61927
|
+
const id = randomUUID15();
|
|
61928
|
+
slot.deliveries.set(id, { resolve, reject });
|
|
61929
|
+
slot.child.send({ id: workerRequest, delivery: { id, from: payload.from, message: payload.message } }, (error) => {
|
|
61930
|
+
if (error) {
|
|
61931
|
+
slot.deliveries.delete(id);
|
|
61932
|
+
reject(error);
|
|
61933
|
+
}
|
|
61934
|
+
});
|
|
61935
|
+
});
|
|
61936
|
+
}
|
|
61937
|
+
request(body) {
|
|
61938
|
+
const id = ++this.requestSequence;
|
|
61939
|
+
return new Promise((resolve, reject) => {
|
|
61940
|
+
this.requests.set(id, { resolve, reject });
|
|
61941
|
+
this.port.send({ kind: "request", id, request: body });
|
|
61942
|
+
});
|
|
61943
|
+
}
|
|
61944
|
+
fail(error) {
|
|
61945
|
+
this.port.send({ kind: "failure", error: error.message });
|
|
61946
|
+
}
|
|
61947
|
+
publish(work, runs) {
|
|
61948
|
+
const failure = this.publicationFailures.get(work.id);
|
|
61949
|
+
if (failure?.generation === work.generation)
|
|
61950
|
+
throw new Error(failure.error);
|
|
61951
|
+
if (!this.publishing)
|
|
61952
|
+
return;
|
|
61953
|
+
this.pendingActivity.set(work.id, { work: token(work), runs });
|
|
61954
|
+
this.scheduleFlush();
|
|
61955
|
+
}
|
|
61956
|
+
scheduleFlush() {
|
|
61957
|
+
if (this.flushTimer !== undefined || this.closed)
|
|
61958
|
+
return;
|
|
61959
|
+
const wait = Math.max(0, this.lastFlushAt + POOL_SUMMARY_INTERVAL_MS - performance.now());
|
|
61960
|
+
this.flushTimer = setTimeout(() => {
|
|
61961
|
+
this.flushTimer = undefined;
|
|
61962
|
+
if (this.batches.size > 0) {
|
|
61963
|
+
this.lastFlushAt = performance.now();
|
|
61964
|
+
this.scheduleFlush();
|
|
61965
|
+
return;
|
|
61966
|
+
}
|
|
61967
|
+
this.flush();
|
|
61968
|
+
}, wait);
|
|
61969
|
+
}
|
|
61970
|
+
flush() {
|
|
61971
|
+
clearTimeout(this.flushTimer);
|
|
61972
|
+
this.flushTimer = undefined;
|
|
61973
|
+
if (this.pendingActivity.size === 0 && !this.usageChanged) {
|
|
61974
|
+
const last = [...this.batches.values()].at(-1);
|
|
61975
|
+
return last === undefined ? Promise.resolve() : new Promise((resolve) => this.chainSettle(last, resolve));
|
|
61976
|
+
}
|
|
61977
|
+
this.lastFlushAt = performance.now();
|
|
61978
|
+
const batch = ++this.batchSequence;
|
|
61979
|
+
const entries = [...this.pendingActivity.values()];
|
|
61980
|
+
this.pendingActivity.clear();
|
|
61981
|
+
const usage = this.usageChanged ? this.usageTotal : undefined;
|
|
61982
|
+
this.usageChanged = false;
|
|
61983
|
+
const settled = new Promise((resolve) => this.batches.set(batch, { entries, settle: resolve }));
|
|
61984
|
+
this.port.send({ kind: "activity", batch, entries, ...usage === undefined ? {} : { usage } });
|
|
61985
|
+
return settled;
|
|
61986
|
+
}
|
|
61987
|
+
chainSettle(batch, resolve) {
|
|
61988
|
+
const settle = batch.settle;
|
|
61989
|
+
batch.settle = () => {
|
|
61990
|
+
settle();
|
|
61991
|
+
resolve();
|
|
61992
|
+
};
|
|
61993
|
+
}
|
|
61994
|
+
noteUsage(usage) {
|
|
61995
|
+
this.usageTotal = usage;
|
|
61996
|
+
this.usageChanged = true;
|
|
61997
|
+
this.scheduleFlush();
|
|
61998
|
+
}
|
|
61999
|
+
async sendResult(workId) {
|
|
62000
|
+
const result = await this.queue.result(this.root, workId);
|
|
62001
|
+
if (result === undefined)
|
|
62002
|
+
return false;
|
|
62003
|
+
this.flush();
|
|
62004
|
+
this.port.send({
|
|
62005
|
+
kind: "result",
|
|
62006
|
+
workId,
|
|
62007
|
+
result,
|
|
62008
|
+
...this.usageTotal === undefined ? {} : { usage: this.usageTotal }
|
|
62009
|
+
});
|
|
62010
|
+
return true;
|
|
62011
|
+
}
|
|
62012
|
+
fence(work) {
|
|
62013
|
+
this.pendingActivity.delete(work.id);
|
|
62014
|
+
this.routes?.fence(work);
|
|
62015
|
+
this.port.send({ kind: "fence", work: { id: work.id, generation: work.generation } });
|
|
62016
|
+
}
|
|
62017
|
+
spawn() {
|
|
62018
|
+
if (this.closed)
|
|
62019
|
+
return Promise.reject(new Error("Local worker pool is closed"));
|
|
62020
|
+
const root = this.root;
|
|
62021
|
+
const options = this.options;
|
|
62022
|
+
const child = fork(options.workerEntry ?? process.argv[1], [
|
|
62023
|
+
JSON.stringify({
|
|
62024
|
+
journalPath: options.journalPath,
|
|
62025
|
+
root,
|
|
62026
|
+
definitions: this.sources,
|
|
62027
|
+
admission: options.admission
|
|
62028
|
+
})
|
|
62029
|
+
], { stdio: ["ignore", "ignore", "pipe", "ipc"], execArgv: [] });
|
|
62030
|
+
const disconnected = new Promise((resolve) => child.once("disconnect", () => resolve()));
|
|
62031
|
+
this.children.add(child);
|
|
62032
|
+
if (child.pid !== undefined)
|
|
62033
|
+
this.port.send({ kind: "spawned", pid: child.pid });
|
|
62034
|
+
const worker = randomUUID15();
|
|
62035
|
+
this.incarnations.set(worker, child);
|
|
62036
|
+
const slot = { worker, child, deliveries: new Map };
|
|
62037
|
+
let ready = false;
|
|
62038
|
+
let serial = Promise.resolve();
|
|
62039
|
+
const logPath = join28(this.logDirectory, `${worker}.log`);
|
|
62040
|
+
this.logs.set(child, logPath);
|
|
62041
|
+
let logFailure;
|
|
62042
|
+
const log = createWriteStream(logPath, { flags: "a" });
|
|
62043
|
+
log.on("error", (error) => {
|
|
62044
|
+
logFailure = error.message;
|
|
62045
|
+
});
|
|
62046
|
+
let stderr = "";
|
|
62047
|
+
child.stderr?.on("data", (chunk) => {
|
|
62048
|
+
stderr = (stderr + chunk.toString()).slice(-STDERR_TAIL_CHARACTERS);
|
|
62049
|
+
});
|
|
62050
|
+
child.stderr?.on("error", (error) => {
|
|
62051
|
+
logFailure ??= error.message;
|
|
62052
|
+
log.end();
|
|
62053
|
+
});
|
|
62054
|
+
child.stderr?.pipe(log);
|
|
62055
|
+
const diagnosed = (summary) => new Error(`${summary} (worker stderr log: ${logFailure === undefined ? logPath : `unavailable, ${logFailure}`})${stderr ? `: ${stderr}` : ""}`);
|
|
62056
|
+
const startupTimeoutMs = options.startupTimeoutMs;
|
|
62057
|
+
let timedOut = false;
|
|
62058
|
+
let terminated = false;
|
|
62059
|
+
const terminate = () => {
|
|
62060
|
+
if (terminated)
|
|
62061
|
+
return;
|
|
62062
|
+
terminated = true;
|
|
62063
|
+
child.kill("SIGKILL");
|
|
62064
|
+
};
|
|
62065
|
+
const replied = (error) => {
|
|
62066
|
+
if (error)
|
|
62067
|
+
terminate();
|
|
62068
|
+
};
|
|
62069
|
+
return new Promise((resolve, reject) => {
|
|
62070
|
+
const startup = setTimeout(() => {
|
|
62071
|
+
timedOut = true;
|
|
62072
|
+
terminate();
|
|
62073
|
+
}, startupTimeoutMs);
|
|
62074
|
+
child.on("error", (error) => {
|
|
62075
|
+
clearTimeout(startup);
|
|
62076
|
+
reject(error);
|
|
62077
|
+
terminate();
|
|
62078
|
+
});
|
|
62079
|
+
const closed = new Promise((resolve) => child.once("close", () => resolve()));
|
|
62080
|
+
child.once("exit", () => {
|
|
62081
|
+
Promise.race([closed, delay2(STDERR_DRAIN_GRACE_MS)]).then(() => reject(diagnosed(timedOut ? `Local worker startup timed out after ${startupTimeoutMs} ms` : "Local worker exited before readiness")));
|
|
62082
|
+
this.children.delete(child);
|
|
62083
|
+
this.logs.delete(child);
|
|
62084
|
+
if (child.pid !== undefined)
|
|
62085
|
+
this.port.send({ kind: "exited", pid: child.pid });
|
|
62086
|
+
clearTimeout(startup);
|
|
62087
|
+
this.incarnations.delete(worker);
|
|
62088
|
+
this.readyWorkers.delete(worker);
|
|
62089
|
+
this.controlAcks.delete(worker);
|
|
62090
|
+
this.lastHeard.delete(worker);
|
|
62091
|
+
if (slot.work)
|
|
62092
|
+
this.fence(slot.work);
|
|
62093
|
+
for (const delivery of slot.deliveries.values())
|
|
62094
|
+
delivery.reject(new Error("Worker exited before delivery acknowledgement"));
|
|
62095
|
+
slot.deliveries.clear();
|
|
62096
|
+
const cleanup = disconnected.then(() => serial).then(async () => {
|
|
62097
|
+
try {
|
|
62098
|
+
if (await this.admission?.releaseStoppedSessions(worker))
|
|
62099
|
+
throw new Error("Local worker process cleanup unconfirmed; permits quarantined");
|
|
62100
|
+
} catch (error) {
|
|
62101
|
+
this.cleanupFailure = error instanceof Error ? error : new Error(String(error));
|
|
62102
|
+
throw this.cleanupFailure;
|
|
62103
|
+
}
|
|
62104
|
+
}).then(async () => {
|
|
62105
|
+
if (this.closed || !ready || this.desiredControl === "quit")
|
|
62106
|
+
return;
|
|
62107
|
+
let work = slot.work;
|
|
62108
|
+
if (work && await this.sendResult(work.id)) {
|
|
62109
|
+
work = undefined;
|
|
62110
|
+
slot.work = undefined;
|
|
62111
|
+
this.replacements = 0;
|
|
62112
|
+
}
|
|
62113
|
+
if (this.closed)
|
|
62114
|
+
return;
|
|
62115
|
+
if (work)
|
|
62116
|
+
await this.queue.releaseExited(this.lease, work);
|
|
62117
|
+
const exhausted = work ? work.generation >= 3 : ++this.replacements > 3;
|
|
62118
|
+
if (exhausted)
|
|
62119
|
+
this.fail(diagnosed("Local worker retry limit"));
|
|
62120
|
+
else
|
|
62121
|
+
await this.spawn();
|
|
62122
|
+
}).catch((error) => {
|
|
62123
|
+
this.fail(error);
|
|
62124
|
+
});
|
|
62125
|
+
this.cleanups.add(cleanup);
|
|
62126
|
+
cleanup.finally(() => this.cleanups.delete(cleanup));
|
|
62127
|
+
});
|
|
62128
|
+
child.on("message", (message) => {
|
|
62129
|
+
this.observeResponsiveness();
|
|
62130
|
+
this.lastHeard.set(worker, { at: performance.now(), stalled: this.stalledMs });
|
|
62131
|
+
if (message.kind === "delivery-ack") {
|
|
62132
|
+
const pending = slot.deliveries.get(message.deliveryId);
|
|
62133
|
+
slot.deliveries.delete(message.deliveryId);
|
|
62134
|
+
if (message.error)
|
|
62135
|
+
pending?.reject(new Error(message.error));
|
|
62136
|
+
else
|
|
62137
|
+
pending?.resolve();
|
|
62138
|
+
return;
|
|
62139
|
+
}
|
|
62140
|
+
let routeReady;
|
|
62141
|
+
const handled = serial.then(async () => {
|
|
62142
|
+
if (message.kind === "ready") {
|
|
62143
|
+
if (terminated)
|
|
62144
|
+
return;
|
|
62145
|
+
ready = true;
|
|
62146
|
+
this.readyWorkers.add(worker);
|
|
62147
|
+
clearTimeout(startup);
|
|
62148
|
+
resolve();
|
|
62149
|
+
return;
|
|
62150
|
+
}
|
|
62151
|
+
await this.starting;
|
|
62152
|
+
const queue = this.queue;
|
|
62153
|
+
const lease = this.lease;
|
|
62154
|
+
if (message.kind === "control")
|
|
62155
|
+
return queue.control(lease);
|
|
62156
|
+
if (message.kind === "controlled") {
|
|
62157
|
+
const control = await queue.control(lease);
|
|
62158
|
+
if (message.revision > control.revision)
|
|
62159
|
+
throw new Error("Unknown control revision");
|
|
62160
|
+
this.controlAcks.set(worker, Math.max(this.controlAcks.get(worker) ?? -1, message.revision));
|
|
62161
|
+
return;
|
|
62162
|
+
}
|
|
62163
|
+
if (message.kind === "claim") {
|
|
62164
|
+
if (slot.work)
|
|
62165
|
+
throw new Error("Worker already holds an attempt");
|
|
62166
|
+
if (this.budgetBlocked)
|
|
62167
|
+
return;
|
|
62168
|
+
await this.recoverExpired();
|
|
62169
|
+
slot.work = await queue.claim(lease, worker, this.sources.map((source) => ({
|
|
62170
|
+
definition: `${source.path}#${source.exportName}`,
|
|
62171
|
+
digest: source.digest
|
|
62172
|
+
})), options.leaseMs);
|
|
62173
|
+
return slot.work;
|
|
62174
|
+
}
|
|
62175
|
+
let work = slot.work;
|
|
62176
|
+
if (!work)
|
|
62177
|
+
throw new Error("Worker has no active attempt");
|
|
62178
|
+
if (message.kind === "renew" || work.expiresAt - Date.now() < options.leaseMs / 2) {
|
|
62179
|
+
work = await queue.extend(lease, work, options.leaseMs);
|
|
62180
|
+
slot.work = work;
|
|
62181
|
+
}
|
|
62182
|
+
if (message.kind === "activity")
|
|
62183
|
+
return this.publish(work, message.runs);
|
|
62184
|
+
if (message.kind === "route") {
|
|
62185
|
+
if (this.routes === undefined)
|
|
62186
|
+
return;
|
|
62187
|
+
await this.flush();
|
|
62188
|
+
this.assertPublished(work);
|
|
62189
|
+
routeReady = this.routes.route(work, message.runId);
|
|
62190
|
+
return;
|
|
62191
|
+
}
|
|
62192
|
+
if (message.kind === "delivery-ready") {
|
|
62193
|
+
if (!this.publishing)
|
|
62194
|
+
return;
|
|
62195
|
+
await this.flush();
|
|
62196
|
+
this.assertPublished(work);
|
|
62197
|
+
return this.request({
|
|
62198
|
+
kind: "ready",
|
|
62199
|
+
work: token(work),
|
|
62200
|
+
runId: message.runId,
|
|
62201
|
+
stageId: message.stageId,
|
|
62202
|
+
stageName: message.stageName,
|
|
62203
|
+
...message.sessionId === undefined ? {} : { sessionId: message.sessionId }
|
|
62204
|
+
});
|
|
62205
|
+
}
|
|
62206
|
+
if (message.kind === "delivery") {
|
|
62207
|
+
if (!message.recipient)
|
|
62208
|
+
throw new Error("Worker delivery requires a session identity");
|
|
62209
|
+
if (!this.publishing)
|
|
62210
|
+
return;
|
|
62211
|
+
await this.flush();
|
|
62212
|
+
this.assertPublished(work);
|
|
62213
|
+
const id = ++this.requestSequence;
|
|
62214
|
+
this.deliveryRequests.set(id, { slot, workerRequest: message.id });
|
|
62215
|
+
try {
|
|
62216
|
+
return await new Promise((resolve, reject) => {
|
|
62217
|
+
this.requests.set(id, { resolve, reject });
|
|
62218
|
+
this.port.send({
|
|
62219
|
+
kind: "request",
|
|
62220
|
+
id,
|
|
62221
|
+
request: {
|
|
62222
|
+
kind: "deliver",
|
|
62223
|
+
work: token(work),
|
|
62224
|
+
runId: message.runId,
|
|
62225
|
+
stageId: message.stageId,
|
|
62226
|
+
stageName: message.stageName,
|
|
62227
|
+
recipient: message.recipient
|
|
62228
|
+
}
|
|
62229
|
+
});
|
|
62230
|
+
});
|
|
62231
|
+
} finally {
|
|
62232
|
+
this.deliveryRequests.delete(id);
|
|
62233
|
+
}
|
|
62234
|
+
}
|
|
62235
|
+
if (message.kind === "renew")
|
|
62236
|
+
return;
|
|
62237
|
+
if (message.kind === "commit") {
|
|
62238
|
+
const receipt = await queue.commit(lease, work, message.operation, message.mutation);
|
|
62239
|
+
this.noteUsage(await this.usageLedger.total(root));
|
|
62240
|
+
return receipt;
|
|
62241
|
+
}
|
|
62242
|
+
if (message.kind === "usage") {
|
|
62243
|
+
this.noteUsage(await this.usageLedger.report(lease, work, message.reports));
|
|
62244
|
+
return;
|
|
62245
|
+
}
|
|
62246
|
+
if (message.kind === "budget") {
|
|
62247
|
+
await queue.assertCurrent(lease, work);
|
|
62248
|
+
if (this.budgetBlocked)
|
|
62249
|
+
throw new Error("Root workflow budget exhausted");
|
|
62250
|
+
return queue.control(lease);
|
|
62251
|
+
}
|
|
62252
|
+
if (message.kind !== "complete")
|
|
62253
|
+
throw new Error("Invalid worker operation");
|
|
62254
|
+
await queue.complete(lease, work, message.result);
|
|
62255
|
+
this.replacements = 0;
|
|
62256
|
+
slot.work = undefined;
|
|
62257
|
+
await this.sendResult(work.id);
|
|
62258
|
+
});
|
|
62259
|
+
const response = handled.then((value2) => routeReady ?? value2);
|
|
62260
|
+
if (message.kind !== "delivery" && message.kind !== "delivery-ready")
|
|
62261
|
+
serial = handled.then(() => {}, () => {});
|
|
62262
|
+
response.then((value2) => child.send({ id: message.id, value: value2 }, replied), (error) => child.send({ id: message.id, error: error.message }, replied));
|
|
62263
|
+
});
|
|
62264
|
+
});
|
|
62265
|
+
}
|
|
62266
|
+
assertPublished(work) {
|
|
62267
|
+
const failure = this.publicationFailures.get(work.id);
|
|
62268
|
+
if (failure?.generation === work.generation)
|
|
62269
|
+
throw new Error(failure.error);
|
|
62270
|
+
}
|
|
62271
|
+
recoverExpired() {
|
|
62272
|
+
if (this.recovering)
|
|
62273
|
+
return this.recovering;
|
|
62274
|
+
const root = this.root;
|
|
62275
|
+
if (this.closed || root === undefined || !this.queue || performance.now() < this.nextRecoveryAt)
|
|
62276
|
+
return Promise.resolve();
|
|
62277
|
+
const queue = this.queue;
|
|
62278
|
+
const lease = this.lease;
|
|
62279
|
+
const leaseMs = this.options.leaseMs;
|
|
62280
|
+
this.recovering = (async () => {
|
|
62281
|
+
await new Promise((resolve) => setImmediate(resolve));
|
|
62282
|
+
await this.admission?.reconcile();
|
|
62283
|
+
const expiredBefore = Date.now();
|
|
62284
|
+
for (const work of await queue.inspect(root)) {
|
|
62285
|
+
if (work.state !== "leased" || work.expiresAt >= expiredBefore)
|
|
62286
|
+
continue;
|
|
62287
|
+
const child = work.worker === null ? undefined : this.incarnations.get(work.worker);
|
|
62288
|
+
if (child && work.worker !== null && this.isLive(work.worker, child)) {
|
|
62289
|
+
await queue.extend(lease, work, leaseMs).catch((error) => {
|
|
62290
|
+
if (error.message !== STALE_WORK_LEASE)
|
|
62291
|
+
throw error;
|
|
62292
|
+
});
|
|
62293
|
+
continue;
|
|
62294
|
+
}
|
|
62295
|
+
if (child && !await this.stop(child).then(() => true, () => false))
|
|
62296
|
+
continue;
|
|
62297
|
+
await queue.expire(lease, work);
|
|
62298
|
+
}
|
|
62299
|
+
})().finally(() => {
|
|
62300
|
+
this.recovering = undefined;
|
|
62301
|
+
this.nextRecoveryAt = performance.now() + RECOVERY_INTERVAL_MS;
|
|
62302
|
+
});
|
|
62303
|
+
return this.recovering;
|
|
62304
|
+
}
|
|
62305
|
+
observeResponsiveness() {
|
|
62306
|
+
const now = performance.now();
|
|
62307
|
+
const gap = now - this.lastTick;
|
|
62308
|
+
if (gap > STALL_THRESHOLD_MS)
|
|
62309
|
+
this.stalledMs += gap - TICK_INTERVAL_MS;
|
|
62310
|
+
this.lastTick = now;
|
|
62311
|
+
}
|
|
62312
|
+
isLive(worker, child) {
|
|
62313
|
+
this.observeResponsiveness();
|
|
62314
|
+
const heard = this.lastHeard.get(worker);
|
|
62315
|
+
return child.exitCode === null && child.signalCode === null && child.connected && heard !== undefined && performance.now() - heard.at - (this.stalledMs - heard.stalled) < this.options.leaseMs;
|
|
62316
|
+
}
|
|
62317
|
+
stop(child) {
|
|
62318
|
+
if (child.pid === undefined || child.exitCode !== null || child.signalCode !== null)
|
|
62319
|
+
return Promise.resolve();
|
|
62320
|
+
return new Promise((resolve, reject) => {
|
|
62321
|
+
const timeout = setTimeout(() => reject(new Error("Local worker cleanup was not confirmed")), 5000);
|
|
62322
|
+
child.once("exit", () => {
|
|
62323
|
+
clearTimeout(timeout);
|
|
62324
|
+
resolve();
|
|
62325
|
+
});
|
|
62326
|
+
child.kill("SIGKILL");
|
|
62327
|
+
});
|
|
62328
|
+
}
|
|
62329
|
+
async close() {
|
|
62330
|
+
this.closed = true;
|
|
62331
|
+
clearInterval(this.ticker);
|
|
62332
|
+
clearInterval(this.recoveryTimer);
|
|
62333
|
+
let failure;
|
|
62334
|
+
let stopping = [];
|
|
62335
|
+
try {
|
|
62336
|
+
await this.recovering?.catch(() => {});
|
|
62337
|
+
stopping = [...this.children].map((child) => ({ child, log: this.logs.get(child) }));
|
|
62338
|
+
await Promise.all(stopping.map(({ child }) => this.stop(child)));
|
|
62339
|
+
await Promise.all([...this.cleanups]);
|
|
62340
|
+
if (this.cleanupFailure)
|
|
62341
|
+
throw this.cleanupFailure;
|
|
62342
|
+
} catch (error) {
|
|
62343
|
+
failure = error;
|
|
62344
|
+
}
|
|
62345
|
+
if (failure === undefined) {
|
|
62346
|
+
const stopped = stopping.flatMap(({ child, log }) => log !== undefined && (child.exitCode !== null || child.signalCode !== null) ? [log] : []);
|
|
62347
|
+
await Promise.all(stopped.map((log) => rm3(log, { force: true }).catch(() => {})));
|
|
62348
|
+
if (this.logDirectory !== undefined)
|
|
62349
|
+
await rmdir(this.logDirectory).catch(() => {});
|
|
62350
|
+
}
|
|
62351
|
+
this.flush();
|
|
62352
|
+
try {
|
|
62353
|
+
await this.routes?.close();
|
|
62354
|
+
} catch (error) {
|
|
62355
|
+
failure ??= error;
|
|
62356
|
+
}
|
|
62357
|
+
for (const batch of this.batches.values())
|
|
62358
|
+
batch.settle();
|
|
62359
|
+
this.batches.clear();
|
|
62360
|
+
for (const pending of this.requests.values())
|
|
62361
|
+
pending.reject(new Error("Local worker pool is closed"));
|
|
62362
|
+
this.requests.clear();
|
|
62363
|
+
if (failure !== undefined)
|
|
62364
|
+
throw failure;
|
|
62365
|
+
}
|
|
62366
|
+
}
|
|
62367
|
+
var CONTROL_ACKNOWLEDGEMENT_TIMEOUT_MS = 5000, RECOVERY_INTERVAL_MS = 500, TICK_INTERVAL_MS = 100, STALL_THRESHOLD_MS = 500, COORDINATOR_LEASE_MS, STDERR_TAIL_CHARACTERS = 8192, STDERR_DRAIN_GRACE_MS = 1000, ROUTE_REGISTRATION_WAIT_MS = 120000, PENDING_STAGE_MESSAGE_EVENT2 = "atomic:workflow-pending-stage-message", STICKY_LIVE_DELIVERED_EVENT2 = "atomic:workflow-sticky-live-delivered", PENDING_STAGE_UNDELIVERABLE_EVENT2 = "atomic:workflow-pending-stage-undeliverable";
|
|
62368
|
+
var init_pool_coordinator = __esm(() => {
|
|
62369
|
+
init_resolution();
|
|
62370
|
+
init_sql_pool_queue();
|
|
62371
|
+
init_sql_pool_usage();
|
|
62372
|
+
init_pending_stage_intercom();
|
|
62373
|
+
init_intercom_group();
|
|
62374
|
+
init_pending_stage_route_capability();
|
|
62375
|
+
init_pending_stage_route_readiness();
|
|
62376
|
+
init_store();
|
|
62377
|
+
init_resource_admission();
|
|
62378
|
+
COORDINATOR_LEASE_MS = 3 * STAGE_SESSION_HEARTBEAT_INTERVAL_MS;
|
|
62379
|
+
});
|
|
62380
|
+
|
|
61322
62381
|
// dist/builtin/workflows/src/workers/protocol.ts
|
|
61323
62382
|
var WORKER_HEARTBEAT_INTERVAL_MS = 1000;
|
|
61324
62383
|
|
|
@@ -61566,21 +62625,26 @@ class PoolStageHandoff {
|
|
|
61566
62625
|
store;
|
|
61567
62626
|
queue;
|
|
61568
62627
|
lease;
|
|
62628
|
+
claimRun;
|
|
61569
62629
|
attempts = new Map;
|
|
61570
62630
|
owners = new Map;
|
|
61571
62631
|
publications = new Map;
|
|
61572
62632
|
drains = new Map;
|
|
61573
|
-
constructor(backend, store, queue, lease) {
|
|
62633
|
+
constructor(backend, store, queue, lease, claimRun) {
|
|
61574
62634
|
this.backend = backend;
|
|
61575
62635
|
this.store = store;
|
|
61576
62636
|
this.queue = queue;
|
|
61577
62637
|
this.lease = lease;
|
|
62638
|
+
this.claimRun = claimRun;
|
|
61578
62639
|
}
|
|
61579
62640
|
publish(work, runs) {
|
|
61580
62641
|
const previous = this.publications.get(work.id) ?? Promise.resolve();
|
|
61581
62642
|
const publication = previous.catch(() => {
|
|
61582
62643
|
return;
|
|
61583
|
-
}).then(() =>
|
|
62644
|
+
}).then(async () => {
|
|
62645
|
+
const commit = await this.prepare(work, runs);
|
|
62646
|
+
this.store.batch(commit);
|
|
62647
|
+
});
|
|
61584
62648
|
const settled = publication.catch(() => {
|
|
61585
62649
|
return;
|
|
61586
62650
|
});
|
|
@@ -61591,6 +62655,21 @@ class PoolStageHandoff {
|
|
|
61591
62655
|
});
|
|
61592
62656
|
return publication;
|
|
61593
62657
|
}
|
|
62658
|
+
async publishBatch(publications) {
|
|
62659
|
+
const prepared = [];
|
|
62660
|
+
for (const { work, runs } of publications)
|
|
62661
|
+
prepared.push(await this.prepare(work, runs).catch((error) => error));
|
|
62662
|
+
return this.store.batch(() => prepared.map((commit) => {
|
|
62663
|
+
if (commit instanceof Error)
|
|
62664
|
+
return commit;
|
|
62665
|
+
try {
|
|
62666
|
+
commit();
|
|
62667
|
+
return;
|
|
62668
|
+
} catch (error) {
|
|
62669
|
+
return error instanceof Error ? error : new Error(String(error));
|
|
62670
|
+
}
|
|
62671
|
+
}));
|
|
62672
|
+
}
|
|
61594
62673
|
async route(work, runId) {
|
|
61595
62674
|
this.requireOwned(await this.begin(work), runId);
|
|
61596
62675
|
const readiness = workflowPendingStageRouteReady(this.store, runId);
|
|
@@ -61641,7 +62720,7 @@ class PoolStageHandoff {
|
|
|
61641
62720
|
const result = await this.queue.result(root, work.id);
|
|
61642
62721
|
if (result === undefined)
|
|
61643
62722
|
return;
|
|
61644
|
-
await this.backend.
|
|
62723
|
+
await this.backend.catchUpWorkflow(root);
|
|
61645
62724
|
const terminals = new Map;
|
|
61646
62725
|
for (const checkpoint of this.backend.listCheckpoints(root)) {
|
|
61647
62726
|
const boundary = checkpoint.kind === "stage" ? checkpoint.topology?.boundary : undefined;
|
|
@@ -61651,6 +62730,9 @@ class PoolStageHandoff {
|
|
|
61651
62730
|
if (child?.runId === boundary.child.runId)
|
|
61652
62731
|
terminals.set(child.runId, { status: child.status, outputs: child.outputs });
|
|
61653
62732
|
}
|
|
62733
|
+
this.store.batch(() => this.settle(work, result, terminals));
|
|
62734
|
+
}
|
|
62735
|
+
settle(work, result, terminals) {
|
|
61654
62736
|
for (const run of this.store.runs()) {
|
|
61655
62737
|
if (run.endedAt !== undefined || this.owners.get(run.id) !== work.id)
|
|
61656
62738
|
continue;
|
|
@@ -61674,22 +62756,31 @@ class PoolStageHandoff {
|
|
|
61674
62756
|
if (current !== undefined && current.generation > work.generation)
|
|
61675
62757
|
return;
|
|
61676
62758
|
const attempt = current?.generation === work.generation ? current : this.start(work, current);
|
|
61677
|
-
this.release(attempt);
|
|
62759
|
+
this.store.batch(() => this.release(attempt));
|
|
61678
62760
|
attempt.fenced = true;
|
|
61679
62761
|
}
|
|
61680
|
-
async
|
|
62762
|
+
async prepare(work, runs) {
|
|
61681
62763
|
const subtree = this.subtree(work, runs);
|
|
61682
62764
|
const root = this.lease.runId;
|
|
61683
|
-
const
|
|
61684
|
-
|
|
61685
|
-
|
|
61686
|
-
|
|
61687
|
-
|
|
61688
|
-
|
|
61689
|
-
|
|
61690
|
-
|
|
61691
|
-
|
|
62765
|
+
const owned = (known) => {
|
|
62766
|
+
for (const run of subtree) {
|
|
62767
|
+
const existing = known.get(run.id);
|
|
62768
|
+
const owner = this.owners.get(run.id);
|
|
62769
|
+
if (owner !== undefined && owner !== work.id || owner === undefined && existing !== undefined && (existing.parentRunId !== run.parentRunId || existing.parentStageId !== run.parentStageId))
|
|
62770
|
+
throw outside2(run.id);
|
|
62771
|
+
}
|
|
62772
|
+
};
|
|
62773
|
+
const before = new Map(this.store.runs().map((run) => [run.id, run]));
|
|
62774
|
+
owned(before);
|
|
62775
|
+
if (subtree.some((run) => before.get(run.id) === undefined))
|
|
62776
|
+
await this.backend.catchUpWorkflow(root);
|
|
61692
62777
|
await this.queue.assertCurrent(this.lease, work);
|
|
62778
|
+
return () => this.commit(work, subtree, owned);
|
|
62779
|
+
}
|
|
62780
|
+
commit(work, subtree, owned) {
|
|
62781
|
+
const root = this.lease.runId;
|
|
62782
|
+
const known = new Map(this.store.runs().map((run) => [run.id, run]));
|
|
62783
|
+
owned(known);
|
|
61693
62784
|
const attempt = this.enter(work);
|
|
61694
62785
|
const execution = {
|
|
61695
62786
|
worker: work.worker ?? "",
|
|
@@ -61701,11 +62792,13 @@ class PoolStageHandoff {
|
|
|
61701
62792
|
...run,
|
|
61702
62793
|
rootRunId: root,
|
|
61703
62794
|
pendingStageMessages: [
|
|
61704
|
-
...
|
|
62795
|
+
...known.get(run.id)?.pendingStageMessages ?? pendingStageMessagesForDurableRun(this.backend, run.id, root)
|
|
61705
62796
|
],
|
|
61706
62797
|
stages: run.stages.map((stage) => ({ ...stage, workerExecution: execution }))
|
|
61707
62798
|
}));
|
|
61708
62799
|
const changed = published.some((run) => activity(known.get(run.id)) !== activity(run));
|
|
62800
|
+
for (const run of published)
|
|
62801
|
+
this.claimRun?.(run.id);
|
|
61709
62802
|
for (const run of published) {
|
|
61710
62803
|
attempt.runs.add(run.id);
|
|
61711
62804
|
this.owners.set(run.id, work.id);
|
|
@@ -61730,7 +62823,7 @@ class PoolStageHandoff {
|
|
|
61730
62823
|
const byId = new Map;
|
|
61731
62824
|
for (const run of runs) {
|
|
61732
62825
|
if (run.id === root || byId.has(run.id))
|
|
61733
|
-
throw
|
|
62826
|
+
throw outside2(run.id);
|
|
61734
62827
|
byId.set(run.id, run);
|
|
61735
62828
|
}
|
|
61736
62829
|
if (runs.length === 0)
|
|
@@ -61743,25 +62836,25 @@ class PoolStageHandoff {
|
|
|
61743
62836
|
const top = byId.get(work.childRunId);
|
|
61744
62837
|
if (top !== undefined) {
|
|
61745
62838
|
if (top.parentRunId !== root || top.parentStageId !== work.boundaryId)
|
|
61746
|
-
throw
|
|
62839
|
+
throw outside2(top.id);
|
|
61747
62840
|
const pending = [top];
|
|
61748
62841
|
for (let run = pending.shift();run !== undefined; run = pending.shift()) {
|
|
61749
62842
|
if (run.rootRunId !== undefined && run.rootRunId !== root)
|
|
61750
|
-
throw
|
|
62843
|
+
throw outside2(run.id);
|
|
61751
62844
|
ordered.push(run);
|
|
61752
62845
|
for (const child of runs) {
|
|
61753
62846
|
if (child.parentRunId !== run.id)
|
|
61754
62847
|
continue;
|
|
61755
62848
|
const parentStage = run.stages.find((stage) => stage.id === child.parentStageId);
|
|
61756
62849
|
if (authoritativeWorkflowChildRunId(parentStage) !== child.id)
|
|
61757
|
-
throw
|
|
62850
|
+
throw outside2(child.id);
|
|
61758
62851
|
pending.push(child);
|
|
61759
62852
|
}
|
|
61760
62853
|
}
|
|
61761
62854
|
}
|
|
61762
62855
|
const stray = runs.find((run) => !ordered.includes(run));
|
|
61763
62856
|
if (stray !== undefined)
|
|
61764
|
-
throw
|
|
62857
|
+
throw outside2(stray.id);
|
|
61765
62858
|
return ordered;
|
|
61766
62859
|
}
|
|
61767
62860
|
async begin(work) {
|
|
@@ -61816,7 +62909,7 @@ class PoolStageHandoff {
|
|
|
61816
62909
|
}
|
|
61817
62910
|
requireOwned(attempt, runId) {
|
|
61818
62911
|
if (!attempt.runs.has(runId))
|
|
61819
|
-
throw
|
|
62912
|
+
throw outside2(runId);
|
|
61820
62913
|
}
|
|
61821
62914
|
delivery(attempt, work, runId, stageId, stageName) {
|
|
61822
62915
|
const key = JSON.stringify([runId, stageId]);
|
|
@@ -61834,7 +62927,7 @@ class PoolStageHandoff {
|
|
|
61834
62927
|
return delivery;
|
|
61835
62928
|
}
|
|
61836
62929
|
}
|
|
61837
|
-
function
|
|
62930
|
+
function outside2(runId) {
|
|
61838
62931
|
return new Error(`Pool stage publication is outside the work subtree: ${runId}`);
|
|
61839
62932
|
}
|
|
61840
62933
|
function isLive(stage) {
|
|
@@ -61859,33 +62952,122 @@ var init_stage_handoff = __esm(() => {
|
|
|
61859
62952
|
});
|
|
61860
62953
|
|
|
61861
62954
|
// dist/builtin/workflows/src/workers/local-pool.ts
|
|
61862
|
-
import { fork } from "node:child_process";
|
|
61863
|
-
import {
|
|
61864
|
-
import {
|
|
61865
|
-
import { mkdir as mkdir6, rm as rm3, rmdir } from "node:fs/promises";
|
|
61866
|
-
import { dirname as dirname12, join as join28, resolve as resolve11 } from "node:path";
|
|
61867
|
-
import { setTimeout as delay2 } from "node:timers/promises";
|
|
62955
|
+
import { fork as fork2 } from "node:child_process";
|
|
62956
|
+
import { resolve as resolve12 } from "node:path";
|
|
62957
|
+
import { setTimeout as delay3 } from "node:timers/promises";
|
|
61868
62958
|
import { fileURLToPath as fileURLToPath5 } from "node:url";
|
|
61869
62959
|
import { isBunBinary as isBunBinary2 } from "@bastani/atomic";
|
|
61870
|
-
function
|
|
61871
|
-
return
|
|
62960
|
+
function defaultWorkerEntry() {
|
|
62961
|
+
return isBunBinary2 ? INTERNAL_WORKFLOW_WORKER_ARG : fileURLToPath5(new URL("./local-worker-entry.mjs", import.meta.url));
|
|
62962
|
+
}
|
|
62963
|
+
function launchCoordinatorProcess() {
|
|
62964
|
+
const child = fork2(defaultWorkerEntry(), [JSON.stringify({ role: POOL_COORDINATOR_ROLE })], {
|
|
62965
|
+
stdio: ["ignore", "ignore", "pipe", "ipc"],
|
|
62966
|
+
execArgv: []
|
|
62967
|
+
});
|
|
62968
|
+
let stderr = "";
|
|
62969
|
+
child.stderr?.on("data", (chunk) => {
|
|
62970
|
+
stderr = (stderr + chunk.toString()).slice(-COORDINATOR_STDERR_TAIL_CHARACTERS);
|
|
62971
|
+
});
|
|
62972
|
+
const listeners = [];
|
|
62973
|
+
let lostReason;
|
|
62974
|
+
const lose = (reason) => {
|
|
62975
|
+
if (lostReason !== undefined)
|
|
62976
|
+
return;
|
|
62977
|
+
lostReason = reason;
|
|
62978
|
+
for (const listener of listeners)
|
|
62979
|
+
listener(reason);
|
|
62980
|
+
};
|
|
62981
|
+
const exited = new Promise((resolve) => child.once("exit", () => resolve()));
|
|
62982
|
+
child.on("error", lose);
|
|
62983
|
+
child.once("exit", (code, signal) => lose(new Error(`Local worker pool coordinator exited (${signal ?? `code ${code}`})${stderr ? `: ${stderr.trim()}` : ""}`)));
|
|
62984
|
+
return {
|
|
62985
|
+
pid: child.pid,
|
|
62986
|
+
port: {
|
|
62987
|
+
send(message) {
|
|
62988
|
+
if (lostReason !== undefined || !child.connected)
|
|
62989
|
+
return;
|
|
62990
|
+
child.send(message, (error) => {
|
|
62991
|
+
if (error)
|
|
62992
|
+
lose(error);
|
|
62993
|
+
});
|
|
62994
|
+
},
|
|
62995
|
+
onMessage(listener) {
|
|
62996
|
+
child.on("message", (message) => listener(message));
|
|
62997
|
+
},
|
|
62998
|
+
onClose(listener) {
|
|
62999
|
+
if (lostReason !== undefined)
|
|
63000
|
+
listener(lostReason);
|
|
63001
|
+
else
|
|
63002
|
+
listeners.push(listener);
|
|
63003
|
+
},
|
|
63004
|
+
async close() {
|
|
63005
|
+
if (child.connected)
|
|
63006
|
+
child.disconnect();
|
|
63007
|
+
if (child.exitCode !== null || child.signalCode !== null)
|
|
63008
|
+
return;
|
|
63009
|
+
const timedOut = await Promise.race([exited.then(() => false), delay3(COORDINATOR_EXIT_GRACE_MS, true)]);
|
|
63010
|
+
if (timedOut) {
|
|
63011
|
+
child.kill("SIGKILL");
|
|
63012
|
+
await exited;
|
|
63013
|
+
}
|
|
63014
|
+
}
|
|
63015
|
+
}
|
|
63016
|
+
};
|
|
63017
|
+
}
|
|
63018
|
+
function launchCoordinatorInProcess() {
|
|
63019
|
+
const mainListeners = [];
|
|
63020
|
+
const coordinatorListeners = [];
|
|
63021
|
+
const copy = (message) => JSON.parse(JSON.stringify(message));
|
|
63022
|
+
const coordinator = new PoolCoordinator({
|
|
63023
|
+
send(message) {
|
|
63024
|
+
const copied = copy(message);
|
|
63025
|
+
setImmediate(() => {
|
|
63026
|
+
for (const listener of mainListeners)
|
|
63027
|
+
listener(copied);
|
|
63028
|
+
});
|
|
63029
|
+
},
|
|
63030
|
+
onMessage(listener) {
|
|
63031
|
+
coordinatorListeners.push(listener);
|
|
63032
|
+
},
|
|
63033
|
+
onClose() {},
|
|
63034
|
+
async close() {}
|
|
63035
|
+
});
|
|
63036
|
+
return {
|
|
63037
|
+
pid: process.pid,
|
|
63038
|
+
port: {
|
|
63039
|
+
send(message) {
|
|
63040
|
+
const copied = copy(message);
|
|
63041
|
+
setImmediate(() => {
|
|
63042
|
+
for (const listener of coordinatorListeners)
|
|
63043
|
+
listener(copied);
|
|
63044
|
+
});
|
|
63045
|
+
},
|
|
63046
|
+
onMessage(listener) {
|
|
63047
|
+
mainListeners.push(listener);
|
|
63048
|
+
},
|
|
63049
|
+
onClose() {},
|
|
63050
|
+
async close() {
|
|
63051
|
+
coordinator.shutdown();
|
|
63052
|
+
}
|
|
63053
|
+
}
|
|
63054
|
+
};
|
|
61872
63055
|
}
|
|
61873
63056
|
|
|
61874
63057
|
class LocalWorkerPool {
|
|
61875
63058
|
backend;
|
|
61876
63059
|
options;
|
|
61877
|
-
|
|
61878
|
-
|
|
61879
|
-
|
|
61880
|
-
|
|
61881
|
-
|
|
61882
|
-
|
|
61883
|
-
|
|
61884
|
-
|
|
61885
|
-
|
|
61886
|
-
|
|
61887
|
-
|
|
61888
|
-
rootStalledMs = 0;
|
|
63060
|
+
pids = new Set;
|
|
63061
|
+
coordinator;
|
|
63062
|
+
coordinatorLost;
|
|
63063
|
+
commands = new Map;
|
|
63064
|
+
commandSequence = 0;
|
|
63065
|
+
inbound = Promise.resolve();
|
|
63066
|
+
works = new Map;
|
|
63067
|
+
attempts = new Map;
|
|
63068
|
+
results = new Map;
|
|
63069
|
+
changed = Promise.withResolvers();
|
|
63070
|
+
waiting = 0;
|
|
61889
63071
|
ticker;
|
|
61890
63072
|
leaseMs;
|
|
61891
63073
|
desiredControl;
|
|
@@ -61893,18 +63075,17 @@ class LocalWorkerPool {
|
|
|
61893
63075
|
store;
|
|
61894
63076
|
handoff;
|
|
61895
63077
|
releaseRoutes;
|
|
63078
|
+
routeClaims = new Map;
|
|
61896
63079
|
sources = [];
|
|
61897
63080
|
queue;
|
|
61898
63081
|
lease;
|
|
61899
|
-
admission;
|
|
61900
|
-
usageLedger;
|
|
61901
63082
|
usageTotal;
|
|
61902
63083
|
rootBudget;
|
|
63084
|
+
budgetBlocked = false;
|
|
63085
|
+
rootSignature;
|
|
61903
63086
|
starting;
|
|
61904
|
-
cleanupFailure;
|
|
61905
63087
|
failure;
|
|
61906
63088
|
closed = false;
|
|
61907
|
-
replacements = 0;
|
|
61908
63089
|
releaseBinding;
|
|
61909
63090
|
constructor(backend, options) {
|
|
61910
63091
|
this.backend = backend;
|
|
@@ -61919,10 +63100,12 @@ class LocalWorkerPool {
|
|
|
61919
63100
|
this.releaseBinding = bindChildWorkerDispatcher(backend, this, options.rootId);
|
|
61920
63101
|
if (options.admission && options.rootId === undefined)
|
|
61921
63102
|
workflowResourceAdmission.set(backend, options.admission);
|
|
61922
|
-
this.ticker = setInterval(() => this.observeResponsiveness(), ROOT_TICK_INTERVAL_MS).unref();
|
|
61923
63103
|
}
|
|
61924
63104
|
get workerPids() {
|
|
61925
|
-
return [...this.
|
|
63105
|
+
return [...this.pids];
|
|
63106
|
+
}
|
|
63107
|
+
get coordinatorPid() {
|
|
63108
|
+
return this.coordinator?.pid;
|
|
61926
63109
|
}
|
|
61927
63110
|
usage() {
|
|
61928
63111
|
return this.usageTotal;
|
|
@@ -61930,47 +63113,39 @@ class LocalWorkerPool {
|
|
|
61930
63113
|
async prepare(root, store) {
|
|
61931
63114
|
if (this.closed)
|
|
61932
63115
|
throw new Error("Local worker pool is closed");
|
|
61933
|
-
if (!(this.backend.journal instanceof SqlRunJournal) ||
|
|
63116
|
+
if (!(this.backend.journal instanceof SqlRunJournal) || resolve12(this.options.journalPath) !== resolve12(this.backend.journal.path) || sqlPoolStorage(this.backend.journal)?.tables.runs !== "wf_runs")
|
|
61934
63117
|
throw new Error("Local workers require the same default-prefix SQLite journal");
|
|
61935
63118
|
if (this.root !== undefined && this.root !== root)
|
|
61936
63119
|
throw new Error("Local worker pool belongs to another root");
|
|
61937
63120
|
this.root = root;
|
|
61938
63121
|
if (store)
|
|
61939
63122
|
this.store = store;
|
|
61940
|
-
this.usageLedger ??= await openSqlPoolUsage(this.backend.journal);
|
|
61941
|
-
this.usageTotal = await this.usageLedger.total(root);
|
|
61942
63123
|
const state = await this.backend.journal.leaseState(root);
|
|
61943
63124
|
if (!state || state.holderId !== this.backend.holderId)
|
|
61944
63125
|
throw new Error("Local worker pool requires the root owner");
|
|
61945
63126
|
this.lease = { runId: root, holderId: state.holderId, epoch: state.epoch };
|
|
61946
|
-
const
|
|
61947
|
-
await
|
|
61948
|
-
|
|
61949
|
-
|
|
61950
|
-
|
|
61951
|
-
|
|
61952
|
-
|
|
61953
|
-
|
|
61954
|
-
|
|
61955
|
-
|
|
63127
|
+
const workerEntry = this.options.workerEntry ?? (isBunBinary2 || this.options.coordinator === "in-process" ? defaultWorkerEntry() : undefined);
|
|
63128
|
+
this.usageTotal = await this.command({
|
|
63129
|
+
kind: "prepare",
|
|
63130
|
+
root,
|
|
63131
|
+
lease: this.lease,
|
|
63132
|
+
options: {
|
|
63133
|
+
journalPath: this.options.journalPath,
|
|
63134
|
+
workers: this.options.workers,
|
|
63135
|
+
leaseMs: this.leaseMs,
|
|
63136
|
+
startupTimeoutMs: this.options.startupTimeoutMs ?? DEFAULT_STARTUP_TIMEOUT_MS,
|
|
63137
|
+
...this.options.admission === undefined ? {} : { admission: this.options.admission },
|
|
63138
|
+
...workerEntry === undefined ? {} : { workerEntry }
|
|
63139
|
+
},
|
|
63140
|
+
...this.desiredControl === undefined ? {} : { control: this.desiredControl }
|
|
63141
|
+
});
|
|
63142
|
+
this.queue ??= await openSqlPoolQueue(this.backend.journal);
|
|
61956
63143
|
}
|
|
61957
63144
|
async control(state) {
|
|
61958
63145
|
this.desiredControl = state;
|
|
61959
63146
|
try {
|
|
61960
|
-
if (
|
|
61961
|
-
|
|
61962
|
-
const control = await this.queue.setControl(this.lease, state);
|
|
61963
|
-
const deadline = Date.now() + CONTROL_ACKNOWLEDGEMENT_TIMEOUT_MS;
|
|
61964
|
-
while ([...this.readyWorkers].some((worker) => (this.controlAcks.get(worker) ?? -1) < control.revision)) {
|
|
61965
|
-
if (this.desiredControl !== state)
|
|
61966
|
-
return;
|
|
61967
|
-
if (Date.now() >= deadline) {
|
|
61968
|
-
if (state === "quit")
|
|
61969
|
-
break;
|
|
61970
|
-
throw new Error("Worker control acknowledgement unresolved");
|
|
61971
|
-
}
|
|
61972
|
-
await delay2(25);
|
|
61973
|
-
}
|
|
63147
|
+
if (this.coordinator !== undefined)
|
|
63148
|
+
await this.command({ kind: "control", state });
|
|
61974
63149
|
} finally {
|
|
61975
63150
|
if (state === "quit")
|
|
61976
63151
|
await this.close();
|
|
@@ -62016,290 +63191,61 @@ class LocalWorkerPool {
|
|
|
62016
63191
|
throw new Error(`Worker definition is not importable: ${binding.exportName}`);
|
|
62017
63192
|
return source;
|
|
62018
63193
|
}));
|
|
62019
|
-
|
|
62020
|
-
|
|
63194
|
+
const cwd = String(this.backend.getWorkflow(root)?.invocationCwd ?? process.cwd());
|
|
63195
|
+
const store = this.store;
|
|
63196
|
+
if (store) {
|
|
63197
|
+
this.releaseRoutes = await acquireStandaloneIntercom(store, root, cwd);
|
|
62021
63198
|
if (this.closed) {
|
|
62022
63199
|
await this.releaseRoutes();
|
|
62023
63200
|
this.releaseRoutes = undefined;
|
|
62024
63201
|
throw new Error("Local worker pool is closed");
|
|
62025
63202
|
}
|
|
62026
|
-
|
|
62027
|
-
|
|
62028
|
-
|
|
62029
|
-
this.
|
|
62030
|
-
|
|
62031
|
-
root,
|
|
62032
|
-
holder: "coordinator"
|
|
63203
|
+
const routeOwner = {
|
|
63204
|
+
notifyUndeliverable: async (payload) => !this.closed && await this.command({ kind: "undeliverable", payload }) === true
|
|
63205
|
+
};
|
|
63206
|
+
this.handoff = new PoolStageHandoff(this.backend, store, this.queueView(), this.lease, (runId) => {
|
|
63207
|
+
this.routeClaims.set(runId, claimExternalRunRoute(store, runId, routeOwner));
|
|
62033
63208
|
});
|
|
62034
|
-
this.logDirectory = workerLogDirectory(this.options.journalPath, root);
|
|
62035
|
-
await mkdir6(this.logDirectory, { recursive: true }).catch(() => {
|
|
62036
|
-
return;
|
|
62037
|
-
});
|
|
62038
|
-
const workers = Array.from({ length: this.options.workers }, () => this.spawn(root));
|
|
62039
|
-
try {
|
|
62040
|
-
await Promise.any(workers);
|
|
62041
|
-
} catch (error) {
|
|
62042
|
-
throw error.errors[0];
|
|
62043
63209
|
}
|
|
62044
|
-
|
|
62045
|
-
|
|
62046
|
-
|
|
62047
|
-
|
|
62048
|
-
|
|
62049
|
-
|
|
62050
|
-
journalPath: this.options.journalPath,
|
|
62051
|
-
root,
|
|
62052
|
-
definitions: this.sources,
|
|
62053
|
-
admission: this.options.admission
|
|
62054
|
-
})
|
|
62055
|
-
], { stdio: ["ignore", "ignore", "pipe", "ipc"], execArgv: [] });
|
|
62056
|
-
const disconnected = new Promise((resolve) => child.once("disconnect", () => resolve()));
|
|
62057
|
-
this.children.add(child);
|
|
62058
|
-
const worker = randomUUID15();
|
|
62059
|
-
this.incarnations.set(worker, child);
|
|
62060
|
-
let ready = false;
|
|
62061
|
-
let serial = Promise.resolve();
|
|
62062
|
-
let work;
|
|
62063
|
-
const deliveries = new Map;
|
|
62064
|
-
const logPath = join28(this.logDirectory, `${worker}.log`);
|
|
62065
|
-
this.logs.set(child, logPath);
|
|
62066
|
-
let logFailure;
|
|
62067
|
-
const log = createWriteStream(logPath, { flags: "a" });
|
|
62068
|
-
log.on("error", (error) => {
|
|
62069
|
-
logFailure = error.message;
|
|
62070
|
-
});
|
|
62071
|
-
let stderr = "";
|
|
62072
|
-
child.stderr?.on("data", (chunk) => {
|
|
62073
|
-
stderr = (stderr + chunk.toString()).slice(-STDERR_TAIL_CHARACTERS);
|
|
62074
|
-
});
|
|
62075
|
-
child.stderr?.on("error", (error) => {
|
|
62076
|
-
logFailure ??= error.message;
|
|
62077
|
-
log.end();
|
|
62078
|
-
});
|
|
62079
|
-
child.stderr?.pipe(log);
|
|
62080
|
-
const diagnosed = (summary) => new Error(`${summary} (worker stderr log: ${logFailure === undefined ? logPath : `unavailable, ${logFailure}`})${stderr ? `: ${stderr}` : ""}`);
|
|
62081
|
-
const startupTimeoutMs = this.options.startupTimeoutMs ?? DEFAULT_STARTUP_TIMEOUT_MS;
|
|
62082
|
-
let timedOut = false;
|
|
62083
|
-
let terminated = false;
|
|
62084
|
-
const terminate = () => {
|
|
62085
|
-
if (terminated)
|
|
62086
|
-
return;
|
|
62087
|
-
terminated = true;
|
|
62088
|
-
child.kill("SIGKILL");
|
|
62089
|
-
};
|
|
62090
|
-
const replied = (error) => {
|
|
62091
|
-
if (error)
|
|
62092
|
-
terminate();
|
|
62093
|
-
};
|
|
62094
|
-
return new Promise((resolve, reject) => {
|
|
62095
|
-
const startup = setTimeout(() => {
|
|
62096
|
-
timedOut = true;
|
|
62097
|
-
terminate();
|
|
62098
|
-
}, startupTimeoutMs);
|
|
62099
|
-
child.on("error", (error) => {
|
|
62100
|
-
clearTimeout(startup);
|
|
62101
|
-
reject(error);
|
|
62102
|
-
terminate();
|
|
62103
|
-
});
|
|
62104
|
-
const closed = new Promise((resolve) => child.once("close", () => resolve()));
|
|
62105
|
-
child.once("exit", () => {
|
|
62106
|
-
Promise.race([closed, delay2(STDERR_DRAIN_GRACE_MS)]).then(() => reject(diagnosed(timedOut ? `Local worker startup timed out after ${startupTimeoutMs} ms` : "Local worker exited before readiness")));
|
|
62107
|
-
this.children.delete(child);
|
|
62108
|
-
this.logs.delete(child);
|
|
62109
|
-
clearTimeout(startup);
|
|
62110
|
-
this.incarnations.delete(worker);
|
|
62111
|
-
this.readyWorkers.delete(worker);
|
|
62112
|
-
this.controlAcks.delete(worker);
|
|
62113
|
-
this.lastHeard.delete(worker);
|
|
62114
|
-
if (work)
|
|
62115
|
-
this.handoff?.fence(work);
|
|
62116
|
-
for (const delivery of deliveries.values())
|
|
62117
|
-
delivery.reject(new Error("Worker exited before delivery acknowledgement"));
|
|
62118
|
-
deliveries.clear();
|
|
62119
|
-
const cleanup = disconnected.then(() => serial).then(async () => {
|
|
62120
|
-
try {
|
|
62121
|
-
if (await this.admission?.releaseStoppedSessions(worker))
|
|
62122
|
-
throw new Error("Local worker process cleanup unconfirmed; permits quarantined");
|
|
62123
|
-
} catch (error) {
|
|
62124
|
-
this.cleanupFailure = error instanceof Error ? error : new Error(String(error));
|
|
62125
|
-
throw this.cleanupFailure;
|
|
62126
|
-
}
|
|
62127
|
-
}).then(async () => {
|
|
62128
|
-
if (this.closed || !ready || this.desiredControl === "quit")
|
|
62129
|
-
return;
|
|
62130
|
-
if (work && await this.queue.result(root, work.id)) {
|
|
62131
|
-
work = undefined;
|
|
62132
|
-
this.replacements = 0;
|
|
62133
|
-
}
|
|
62134
|
-
if (this.closed)
|
|
62135
|
-
return;
|
|
62136
|
-
if (work)
|
|
62137
|
-
await this.queue.releaseExited(this.lease, work);
|
|
62138
|
-
const exhausted = work ? work.generation >= 3 : ++this.replacements > 3;
|
|
62139
|
-
if (exhausted)
|
|
62140
|
-
this.failure = diagnosed("Local worker retry limit");
|
|
62141
|
-
else
|
|
62142
|
-
await this.spawn(root);
|
|
62143
|
-
}).catch((error) => {
|
|
62144
|
-
this.failure = error;
|
|
62145
|
-
});
|
|
62146
|
-
this.cleanups.add(cleanup);
|
|
62147
|
-
cleanup.finally(() => this.cleanups.delete(cleanup));
|
|
62148
|
-
});
|
|
62149
|
-
child.on("message", (message) => {
|
|
62150
|
-
this.observeResponsiveness();
|
|
62151
|
-
this.lastHeard.set(worker, { at: performance.now(), rootStalled: this.rootStalledMs });
|
|
62152
|
-
if (message.kind === "delivery-ack") {
|
|
62153
|
-
const pending = deliveries.get(message.deliveryId);
|
|
62154
|
-
deliveries.delete(message.deliveryId);
|
|
62155
|
-
if (message.error)
|
|
62156
|
-
pending?.reject(new Error(message.error));
|
|
62157
|
-
else
|
|
62158
|
-
pending?.resolve();
|
|
62159
|
-
return;
|
|
62160
|
-
}
|
|
62161
|
-
const response = serial.then(async () => {
|
|
62162
|
-
if (message.kind === "ready") {
|
|
62163
|
-
if (terminated)
|
|
62164
|
-
return;
|
|
62165
|
-
ready = true;
|
|
62166
|
-
this.readyWorkers.add(worker);
|
|
62167
|
-
clearTimeout(startup);
|
|
62168
|
-
resolve();
|
|
62169
|
-
return;
|
|
62170
|
-
}
|
|
62171
|
-
await this.starting;
|
|
62172
|
-
const queue = this.queue;
|
|
62173
|
-
const lease = this.lease;
|
|
62174
|
-
if (message.kind === "control")
|
|
62175
|
-
return queue.control(lease);
|
|
62176
|
-
if (message.kind === "controlled") {
|
|
62177
|
-
const control = await queue.control(lease);
|
|
62178
|
-
if (message.revision > control.revision)
|
|
62179
|
-
throw new Error("Unknown control revision");
|
|
62180
|
-
this.controlAcks.set(worker, Math.max(this.controlAcks.get(worker) ?? -1, message.revision));
|
|
62181
|
-
return;
|
|
62182
|
-
}
|
|
62183
|
-
if (message.kind === "claim") {
|
|
62184
|
-
if (work)
|
|
62185
|
-
throw new Error("Worker already holds an attempt");
|
|
62186
|
-
const decision = this.rootBudget?.checkpoint();
|
|
62187
|
-
if (decision?.kind === "exhausted" || decision?.kind === "wrap_up")
|
|
62188
|
-
return;
|
|
62189
|
-
await this.recoverExpired(root);
|
|
62190
|
-
work = await queue.claim(lease, worker, this.sources.map((source) => ({
|
|
62191
|
-
definition: `${source.path}#${source.exportName}`,
|
|
62192
|
-
digest: source.digest
|
|
62193
|
-
})), this.leaseMs);
|
|
62194
|
-
return work;
|
|
62195
|
-
}
|
|
62196
|
-
if (!work)
|
|
62197
|
-
throw new Error("Worker has no active attempt");
|
|
62198
|
-
if (message.kind === "renew" || work.expiresAt - Date.now() < this.leaseMs / 2)
|
|
62199
|
-
work = await queue.extend(lease, work, this.leaseMs);
|
|
62200
|
-
if (message.kind === "activity")
|
|
62201
|
-
return this.handoff?.publish(work, message.runs);
|
|
62202
|
-
if (message.kind === "route")
|
|
62203
|
-
return this.handoff?.route(work, message.runId);
|
|
62204
|
-
if (message.kind === "delivery-ready")
|
|
62205
|
-
return this.handoff?.ready(work, message.runId, message.stageId, message.stageName, message.sessionId);
|
|
62206
|
-
if (message.kind === "delivery") {
|
|
62207
|
-
if (!message.recipient)
|
|
62208
|
-
throw new Error("Worker delivery requires a session identity");
|
|
62209
|
-
return this.handoff?.deliver(work, message.runId, message.stageId, message.stageName, message.recipient, (from, payload) => new Promise((resolve, reject) => {
|
|
62210
|
-
const id = randomUUID15();
|
|
62211
|
-
deliveries.set(id, { resolve, reject });
|
|
62212
|
-
child.send({ id: message.id, delivery: { id, from, message: payload } }, (error) => {
|
|
62213
|
-
if (error) {
|
|
62214
|
-
deliveries.delete(id);
|
|
62215
|
-
reject(error);
|
|
62216
|
-
}
|
|
62217
|
-
});
|
|
62218
|
-
}));
|
|
62219
|
-
}
|
|
62220
|
-
if (message.kind === "renew")
|
|
62221
|
-
return;
|
|
62222
|
-
if (message.kind === "commit") {
|
|
62223
|
-
const receipt = await queue.commit(lease, work, message.operation, message.mutation);
|
|
62224
|
-
this.usageTotal = await this.usageLedger.total(root);
|
|
62225
|
-
return receipt;
|
|
62226
|
-
}
|
|
62227
|
-
if (message.kind === "usage") {
|
|
62228
|
-
this.usageTotal = await this.usageLedger.report(lease, work, message.reports);
|
|
62229
|
-
return;
|
|
62230
|
-
}
|
|
62231
|
-
if (message.kind === "budget") {
|
|
62232
|
-
await queue.assertCurrent(lease, work);
|
|
62233
|
-
const decision = this.rootBudget?.checkpoint();
|
|
62234
|
-
if (decision?.kind === "exhausted" || decision?.kind === "wrap_up")
|
|
62235
|
-
throw new Error("Root workflow budget exhausted");
|
|
62236
|
-
return queue.control(lease);
|
|
62237
|
-
}
|
|
62238
|
-
if (message.kind !== "complete")
|
|
62239
|
-
throw new Error("Invalid worker operation");
|
|
62240
|
-
await queue.complete(lease, work, message.result);
|
|
62241
|
-
this.replacements = 0;
|
|
62242
|
-
work = undefined;
|
|
62243
|
-
});
|
|
62244
|
-
if (message.kind !== "delivery" && message.kind !== "delivery-ready")
|
|
62245
|
-
serial = response.then(() => {}, () => {});
|
|
62246
|
-
response.then((value2) => child.send({ id: message.id, value: value2 }, replied), (error) => child.send({ id: message.id, error: error.message }, replied));
|
|
62247
|
-
});
|
|
63210
|
+
await this.command({
|
|
63211
|
+
kind: "start",
|
|
63212
|
+
sources: this.sources,
|
|
63213
|
+
cwd,
|
|
63214
|
+
publish: store !== undefined,
|
|
63215
|
+
...this.rootRun()
|
|
62248
63216
|
});
|
|
62249
63217
|
}
|
|
62250
|
-
|
|
62251
|
-
|
|
62252
|
-
|
|
62253
|
-
|
|
62254
|
-
|
|
62255
|
-
|
|
62256
|
-
|
|
62257
|
-
|
|
62258
|
-
|
|
62259
|
-
|
|
62260
|
-
if (work.state !== "leased" || work.expiresAt >= expiredBefore)
|
|
62261
|
-
continue;
|
|
62262
|
-
const child = work.worker === null ? undefined : this.incarnations.get(work.worker);
|
|
62263
|
-
if (child && work.worker !== null && this.isLive(work.worker, child)) {
|
|
62264
|
-
await this.queue.extend(this.lease, work, this.leaseMs).catch((error) => {
|
|
62265
|
-
if (error.message !== STALE_WORK_LEASE)
|
|
62266
|
-
throw error;
|
|
62267
|
-
});
|
|
62268
|
-
continue;
|
|
62269
|
-
}
|
|
62270
|
-
if (child && !await this.stop(child).then(() => true, () => false))
|
|
62271
|
-
continue;
|
|
62272
|
-
await this.queue.expire(this.lease, work);
|
|
62273
|
-
}
|
|
62274
|
-
})().finally(() => {
|
|
62275
|
-
this.recovering = undefined;
|
|
62276
|
-
this.nextRecoveryAt = performance.now() + RECOVERY_INTERVAL_MS;
|
|
62277
|
-
});
|
|
62278
|
-
return this.recovering;
|
|
63218
|
+
queueView() {
|
|
63219
|
+
return {
|
|
63220
|
+
assertCurrent: async (_lease, work) => {
|
|
63221
|
+
const current = this.attempts.get(work.id);
|
|
63222
|
+
if (current === undefined || current.fenced || current.generation !== work.generation || current.worker !== work.worker || current.epoch !== work.epoch)
|
|
63223
|
+
throw new Error(STALE_WORK_LEASE);
|
|
63224
|
+
},
|
|
63225
|
+
withCurrentAppend: (lease, work, assertSession, action) => this.queue.withCurrentAppend(lease, work, assertSession, action),
|
|
63226
|
+
result: async (root, id) => this.results.get(id) ?? this.queue.result(root, id)
|
|
63227
|
+
};
|
|
62279
63228
|
}
|
|
62280
|
-
|
|
62281
|
-
const
|
|
62282
|
-
|
|
62283
|
-
if (gap > ROOT_STALL_THRESHOLD_MS)
|
|
62284
|
-
this.rootStalledMs += gap - ROOT_TICK_INTERVAL_MS;
|
|
62285
|
-
this.lastTick = now;
|
|
63229
|
+
rootProjection() {
|
|
63230
|
+
const run = this.store?.runs().find((candidate) => candidate.id === this.root);
|
|
63231
|
+
return run === undefined ? undefined : projectRootRun(run);
|
|
62286
63232
|
}
|
|
62287
|
-
|
|
62288
|
-
this.
|
|
62289
|
-
|
|
62290
|
-
|
|
63233
|
+
rootRun() {
|
|
63234
|
+
const rootRun = this.rootProjection();
|
|
63235
|
+
if (rootRun === undefined)
|
|
63236
|
+
return {};
|
|
63237
|
+
this.rootSignature = JSON.stringify(rootRun);
|
|
63238
|
+
return { rootRun };
|
|
62291
63239
|
}
|
|
62292
|
-
|
|
62293
|
-
|
|
62294
|
-
|
|
62295
|
-
|
|
62296
|
-
|
|
62297
|
-
|
|
62298
|
-
|
|
62299
|
-
|
|
62300
|
-
|
|
62301
|
-
child.kill("SIGKILL");
|
|
62302
|
-
});
|
|
63240
|
+
refreshRootRun() {
|
|
63241
|
+
const rootRun = this.rootProjection();
|
|
63242
|
+
if (rootRun === undefined)
|
|
63243
|
+
return;
|
|
63244
|
+
const signature = JSON.stringify(rootRun);
|
|
63245
|
+
if (signature === this.rootSignature)
|
|
63246
|
+
return;
|
|
63247
|
+
this.rootSignature = signature;
|
|
63248
|
+
this.notice({ kind: "root", rootRun });
|
|
62303
63249
|
}
|
|
62304
63250
|
async run(definition, inputs, options) {
|
|
62305
63251
|
try {
|
|
@@ -62332,7 +63278,7 @@ class LocalWorkerPool {
|
|
|
62332
63278
|
const selectedSource = this.sources[index];
|
|
62333
63279
|
if (await definitionDigest(selectedSource.path) !== selectedSource.digest)
|
|
62334
63280
|
throw new Error("Worker definition compatibility error");
|
|
62335
|
-
await this.backend.
|
|
63281
|
+
await this.backend.catchUpWorkflow(root);
|
|
62336
63282
|
const boundary = this.backend.listCheckpoints(root).find((entry) => entry.kind === "stage" && entry.topology?.boundary?.event === "start" && entry.topology.stageId === options.parentRun.stageId);
|
|
62337
63283
|
if (boundary?.kind !== "stage")
|
|
62338
63284
|
throw new Error("Missing durable child boundary");
|
|
@@ -62343,39 +63289,232 @@ class LocalWorkerPool {
|
|
|
62343
63289
|
config: options.config,
|
|
62344
63290
|
executionMode: options.executionMode
|
|
62345
63291
|
}));
|
|
62346
|
-
|
|
62347
|
-
|
|
62348
|
-
|
|
62349
|
-
|
|
62350
|
-
|
|
62351
|
-
|
|
62352
|
-
|
|
62353
|
-
|
|
62354
|
-
|
|
62355
|
-
|
|
62356
|
-
|
|
62357
|
-
|
|
63292
|
+
this.evaluateBudget();
|
|
63293
|
+
const enqueued = await this.command({
|
|
63294
|
+
kind: "enqueue",
|
|
63295
|
+
descriptor: {
|
|
63296
|
+
boundaryId: options.parentRun.stageId,
|
|
63297
|
+
childRunId: options.runId,
|
|
63298
|
+
scope: options.durableScope.scopePrefix,
|
|
63299
|
+
definition: `${selectedSource.path}#${selectedSource.exportName}`,
|
|
63300
|
+
digest: selectedSource.digest,
|
|
63301
|
+
inputs,
|
|
63302
|
+
execution
|
|
63303
|
+
},
|
|
63304
|
+
boundary,
|
|
63305
|
+
...this.rootRun()
|
|
63306
|
+
});
|
|
63307
|
+
const work = enqueued.work;
|
|
63308
|
+
this.works.set(work.id, work);
|
|
63309
|
+
if (enqueued.result !== undefined && !this.results.has(work.id))
|
|
63310
|
+
this.results.set(work.id, enqueued.result);
|
|
63311
|
+
this.waiting++;
|
|
63312
|
+
this.ticker ??= setInterval(() => {
|
|
63313
|
+
this.evaluateBudget();
|
|
63314
|
+
this.signalChange();
|
|
63315
|
+
}, POOL_SUMMARY_INTERVAL_MS).unref();
|
|
63316
|
+
try {
|
|
63317
|
+
for (;; ) {
|
|
63318
|
+
options.signal?.throwIfAborted();
|
|
63319
|
+
if (this.closed) {
|
|
63320
|
+
if (this.failure)
|
|
63321
|
+
throw this.failure;
|
|
63322
|
+
if (this.desiredControl === "quit")
|
|
63323
|
+
throw new WorkflowGracefulQuitError(root, "worker pool teardown");
|
|
63324
|
+
throw new Error("Local worker pool is closed");
|
|
63325
|
+
}
|
|
63326
|
+
const result = this.results.get(work.id);
|
|
63327
|
+
if (result) {
|
|
63328
|
+
await this.inbound;
|
|
63329
|
+
await this.backend.catchUpWorkflow(root);
|
|
63330
|
+
await this.handoff?.complete(work);
|
|
63331
|
+
return result;
|
|
63332
|
+
}
|
|
62358
63333
|
if (this.failure)
|
|
62359
63334
|
throw this.failure;
|
|
62360
|
-
if (this.
|
|
62361
|
-
|
|
62362
|
-
|
|
63335
|
+
if (this.budgetBlocked)
|
|
63336
|
+
await this.rootBudget.stopAtBoundaryAsync(definition.name);
|
|
63337
|
+
await this.nextChange(options.signal);
|
|
62363
63338
|
}
|
|
62364
|
-
|
|
62365
|
-
if (
|
|
62366
|
-
|
|
62367
|
-
|
|
62368
|
-
|
|
63339
|
+
} finally {
|
|
63340
|
+
if (--this.waiting === 0) {
|
|
63341
|
+
clearInterval(this.ticker);
|
|
63342
|
+
this.ticker = undefined;
|
|
63343
|
+
}
|
|
63344
|
+
}
|
|
63345
|
+
}
|
|
63346
|
+
evaluateBudget() {
|
|
63347
|
+
const decision = this.rootBudget?.checkpoint();
|
|
63348
|
+
const blocked = decision?.kind === "exhausted" || decision?.kind === "wrap_up";
|
|
63349
|
+
if (blocked === this.budgetBlocked)
|
|
63350
|
+
return;
|
|
63351
|
+
this.budgetBlocked = blocked;
|
|
63352
|
+
this.notice({ kind: "budget", blocked });
|
|
63353
|
+
}
|
|
63354
|
+
signalChange() {
|
|
63355
|
+
const changed = this.changed;
|
|
63356
|
+
this.changed = Promise.withResolvers();
|
|
63357
|
+
changed.resolve();
|
|
63358
|
+
}
|
|
63359
|
+
nextChange(signal) {
|
|
63360
|
+
const changed = this.changed.promise;
|
|
63361
|
+
if (signal === undefined)
|
|
63362
|
+
return changed;
|
|
63363
|
+
return new Promise((resolve, reject) => {
|
|
63364
|
+
const abort = () => reject(signal.reason);
|
|
63365
|
+
signal.addEventListener("abort", abort, { once: true });
|
|
63366
|
+
changed.then(() => {
|
|
63367
|
+
signal.removeEventListener("abort", abort);
|
|
63368
|
+
resolve();
|
|
63369
|
+
});
|
|
63370
|
+
});
|
|
63371
|
+
}
|
|
63372
|
+
connect() {
|
|
63373
|
+
if (this.coordinator !== undefined)
|
|
63374
|
+
return this.coordinator;
|
|
63375
|
+
const coordinator = this.options.coordinator === "in-process" ? launchCoordinatorInProcess() : launchCoordinatorProcess();
|
|
63376
|
+
this.coordinator = coordinator;
|
|
63377
|
+
coordinator.port.onMessage((event) => {
|
|
63378
|
+
this.inbound = this.inbound.then(() => this.receive(event)).catch(() => {});
|
|
63379
|
+
});
|
|
63380
|
+
coordinator.port.onClose((reason) => {
|
|
63381
|
+
this.coordinatorLost = reason;
|
|
63382
|
+
this.inbound = this.inbound.then(() => this.lost(reason));
|
|
63383
|
+
});
|
|
63384
|
+
return coordinator;
|
|
63385
|
+
}
|
|
63386
|
+
lost(reason) {
|
|
63387
|
+
this.pids.clear();
|
|
63388
|
+
if (!this.closed)
|
|
63389
|
+
this.failure ??= reason;
|
|
63390
|
+
for (const pending of this.commands.values())
|
|
63391
|
+
pending.reject(reason);
|
|
63392
|
+
this.commands.clear();
|
|
63393
|
+
this.signalChange();
|
|
63394
|
+
}
|
|
63395
|
+
command(body) {
|
|
63396
|
+
const coordinator = this.connect();
|
|
63397
|
+
if (this.coordinatorLost !== undefined)
|
|
63398
|
+
return Promise.reject(this.coordinatorLost);
|
|
63399
|
+
const id = ++this.commandSequence;
|
|
63400
|
+
return new Promise((resolve, reject) => {
|
|
63401
|
+
this.commands.set(id, { resolve, reject });
|
|
63402
|
+
coordinator.port.send({ ...body, id });
|
|
63403
|
+
});
|
|
63404
|
+
}
|
|
63405
|
+
notice(notice) {
|
|
63406
|
+
this.coordinator?.port.send(notice);
|
|
63407
|
+
}
|
|
63408
|
+
async receive(event) {
|
|
63409
|
+
switch (event.kind) {
|
|
63410
|
+
case "reply": {
|
|
63411
|
+
const pending = this.commands.get(event.id);
|
|
63412
|
+
this.commands.delete(event.id);
|
|
63413
|
+
if (event.error !== undefined)
|
|
63414
|
+
pending?.reject(new Error(event.error));
|
|
63415
|
+
else
|
|
63416
|
+
pending?.resolve(event.value);
|
|
63417
|
+
return;
|
|
63418
|
+
}
|
|
63419
|
+
case "activity":
|
|
63420
|
+
return this.applyActivity(event.batch, event.entries, event.usage);
|
|
63421
|
+
case "spawned":
|
|
63422
|
+
this.pids.add(event.pid);
|
|
63423
|
+
return;
|
|
63424
|
+
case "exited":
|
|
63425
|
+
this.pids.delete(event.pid);
|
|
63426
|
+
return;
|
|
63427
|
+
case "fence": {
|
|
63428
|
+
const current = this.attempts.get(event.work.id);
|
|
63429
|
+
if (current?.generation === event.work.generation)
|
|
63430
|
+
current.fenced = true;
|
|
63431
|
+
this.handoff?.fence(event.work);
|
|
63432
|
+
return;
|
|
62369
63433
|
}
|
|
62370
|
-
|
|
62371
|
-
|
|
62372
|
-
|
|
62373
|
-
|
|
62374
|
-
|
|
62375
|
-
|
|
62376
|
-
|
|
63434
|
+
case "result":
|
|
63435
|
+
if (event.usage !== undefined)
|
|
63436
|
+
this.usageTotal = event.usage;
|
|
63437
|
+
this.results.set(event.workId, event.result);
|
|
63438
|
+
this.signalChange();
|
|
63439
|
+
return;
|
|
63440
|
+
case "failure":
|
|
63441
|
+
this.failure ??= new Error(event.error);
|
|
63442
|
+
this.signalChange();
|
|
63443
|
+
return;
|
|
63444
|
+
case "request":
|
|
63445
|
+
this.serve(event.id, event.request);
|
|
63446
|
+
return;
|
|
62377
63447
|
}
|
|
62378
63448
|
}
|
|
63449
|
+
async applyActivity(batch, entries, usage) {
|
|
63450
|
+
if (usage !== undefined)
|
|
63451
|
+
this.usageTotal = usage;
|
|
63452
|
+
const rejected = [];
|
|
63453
|
+
const handoff = this.handoff;
|
|
63454
|
+
if (handoff !== undefined) {
|
|
63455
|
+
const publications = entries.flatMap((entry) => {
|
|
63456
|
+
const work = this.track(entry.work);
|
|
63457
|
+
if (work !== undefined)
|
|
63458
|
+
return [{ work, runs: entry.runs }];
|
|
63459
|
+
rejected.push({ workId: entry.work.id, generation: entry.work.generation, error: STALE_WORK_LEASE });
|
|
63460
|
+
return [];
|
|
63461
|
+
});
|
|
63462
|
+
const outcomes = await handoff.publishBatch(publications);
|
|
63463
|
+
outcomes.forEach((error, index) => {
|
|
63464
|
+
const { work } = publications[index];
|
|
63465
|
+
if (error !== undefined)
|
|
63466
|
+
rejected.push({ workId: work.id, generation: work.generation, error: error.message });
|
|
63467
|
+
});
|
|
63468
|
+
}
|
|
63469
|
+
this.notice({ kind: "published", batch, rejected });
|
|
63470
|
+
this.refreshRootRun();
|
|
63471
|
+
}
|
|
63472
|
+
track(attempt) {
|
|
63473
|
+
const work = this.works.get(attempt.id);
|
|
63474
|
+
if (work === undefined)
|
|
63475
|
+
return;
|
|
63476
|
+
const current = this.attempts.get(attempt.id);
|
|
63477
|
+
if (current === undefined || attempt.generation > current.generation)
|
|
63478
|
+
this.attempts.set(attempt.id, { ...attempt, fenced: false });
|
|
63479
|
+
return { ...work, ...attempt, state: "leased" };
|
|
63480
|
+
}
|
|
63481
|
+
async serve(id, request) {
|
|
63482
|
+
try {
|
|
63483
|
+
const value2 = await this.answer(id, request);
|
|
63484
|
+
this.coordinator?.port.send({ kind: "request-reply", id, ...value2 === undefined ? {} : { value: value2 } });
|
|
63485
|
+
} catch (error) {
|
|
63486
|
+
this.coordinator?.port.send({
|
|
63487
|
+
kind: "request-reply",
|
|
63488
|
+
id,
|
|
63489
|
+
error: error instanceof Error ? error.message : String(error)
|
|
63490
|
+
});
|
|
63491
|
+
}
|
|
63492
|
+
}
|
|
63493
|
+
async answer(id, request) {
|
|
63494
|
+
if (request.kind === "pending-stage-message" || request.kind === "sticky-live-delivered") {
|
|
63495
|
+
const router = this.store === undefined ? undefined : pendingStageMessageRouter(this.store);
|
|
63496
|
+
const event = {
|
|
63497
|
+
...request.payload,
|
|
63498
|
+
handled: false
|
|
63499
|
+
};
|
|
63500
|
+
if (request.kind === "pending-stage-message")
|
|
63501
|
+
router?.route(event);
|
|
63502
|
+
else
|
|
63503
|
+
router?.sticky(event);
|
|
63504
|
+
if (event.handled && event.completion !== undefined)
|
|
63505
|
+
return await event.completion;
|
|
63506
|
+
return request.kind === "pending-stage-message" ? { outcome: "refused", reason: "Session not found" } : undefined;
|
|
63507
|
+
}
|
|
63508
|
+
const handoff = this.handoff;
|
|
63509
|
+
const work = this.track(request.work);
|
|
63510
|
+
if (handoff === undefined || work === undefined)
|
|
63511
|
+
throw new Error(STALE_WORK_LEASE);
|
|
63512
|
+
if (request.kind === "ready")
|
|
63513
|
+
return handoff.ready(work, request.runId, request.stageId, request.stageName, request.sessionId);
|
|
63514
|
+
return handoff.deliver(work, request.runId, request.stageId, request.stageName, request.recipient, async (from, message) => {
|
|
63515
|
+
await this.command({ kind: "transport", request: id, from, message });
|
|
63516
|
+
});
|
|
63517
|
+
}
|
|
62379
63518
|
closing;
|
|
62380
63519
|
close() {
|
|
62381
63520
|
this.closing ??= this.dispose();
|
|
@@ -62384,27 +63523,28 @@ class LocalWorkerPool {
|
|
|
62384
63523
|
async dispose() {
|
|
62385
63524
|
this.closed = true;
|
|
62386
63525
|
clearInterval(this.ticker);
|
|
63526
|
+
this.ticker = undefined;
|
|
62387
63527
|
this.releaseBinding();
|
|
62388
63528
|
if (this.options.rootId === undefined)
|
|
62389
63529
|
workflowResourceAdmission.delete(this.backend);
|
|
63530
|
+
this.signalChange();
|
|
62390
63531
|
let failure;
|
|
62391
|
-
|
|
62392
|
-
|
|
62393
|
-
|
|
62394
|
-
|
|
62395
|
-
|
|
62396
|
-
|
|
62397
|
-
|
|
62398
|
-
|
|
62399
|
-
|
|
62400
|
-
|
|
62401
|
-
|
|
62402
|
-
|
|
62403
|
-
const stopped = stopping.flatMap(({ child, log }) => log !== undefined && (child.exitCode !== null || child.signalCode !== null) ? [log] : []);
|
|
62404
|
-
await Promise.all(stopped.map((log) => rm3(log, { force: true }).catch(() => {})));
|
|
62405
|
-
if (this.logDirectory !== undefined)
|
|
62406
|
-
await rmdir(this.logDirectory).catch(() => {});
|
|
63532
|
+
const coordinator = this.coordinator;
|
|
63533
|
+
if (coordinator !== undefined) {
|
|
63534
|
+
if (this.coordinatorLost === undefined) {
|
|
63535
|
+
try {
|
|
63536
|
+
await this.command({ kind: "close" });
|
|
63537
|
+
} catch (error) {
|
|
63538
|
+
failure = error;
|
|
63539
|
+
}
|
|
63540
|
+
}
|
|
63541
|
+
await this.inbound;
|
|
63542
|
+
await coordinator.port.close();
|
|
63543
|
+
this.pids.clear();
|
|
62407
63544
|
}
|
|
63545
|
+
for (const release of this.routeClaims.values())
|
|
63546
|
+
release();
|
|
63547
|
+
this.routeClaims.clear();
|
|
62408
63548
|
try {
|
|
62409
63549
|
await this.releaseRoutes?.();
|
|
62410
63550
|
} catch (error) {
|
|
@@ -62414,27 +63554,26 @@ class LocalWorkerPool {
|
|
|
62414
63554
|
throw failure;
|
|
62415
63555
|
}
|
|
62416
63556
|
}
|
|
62417
|
-
var INTERNAL_WORKFLOW_WORKER_ARG = "--internal-workflow-worker", DEFAULT_WORK_LEASE_MS = 30000, MINIMUM_WORK_LEASE_MS,
|
|
63557
|
+
var INTERNAL_WORKFLOW_WORKER_ARG = "--internal-workflow-worker", DEFAULT_WORK_LEASE_MS = 30000, MINIMUM_WORK_LEASE_MS, DEFAULT_STARTUP_TIMEOUT_MS = 30000, MAX_TIMER_DELAY_MS = 2147483647, COORDINATOR_STDERR_TAIL_CHARACTERS = 8192, COORDINATOR_EXIT_GRACE_MS = 5000;
|
|
62418
63558
|
var init_local_pool = __esm(() => {
|
|
62419
63559
|
init_internal();
|
|
62420
63560
|
init_sql_pool_queue();
|
|
62421
|
-
init_sql_pool_usage();
|
|
62422
63561
|
init_sql_run_journal();
|
|
62423
63562
|
init_workflow_tool_abort();
|
|
62424
63563
|
init_sdk_intercom();
|
|
63564
|
+
init_external_run_routes();
|
|
62425
63565
|
init_workflow_infrastructure_error();
|
|
62426
63566
|
init_admission_binding();
|
|
62427
63567
|
init_definitions();
|
|
62428
63568
|
init_dispatch();
|
|
62429
63569
|
init_host_options();
|
|
62430
|
-
|
|
63570
|
+
init_pool_coordinator();
|
|
62431
63571
|
init_stage_handoff();
|
|
62432
63572
|
MINIMUM_WORK_LEASE_MS = 2 * WORKER_HEARTBEAT_INTERVAL_MS;
|
|
62433
|
-
COORDINATOR_LEASE_MS = 3 * STAGE_SESSION_HEARTBEAT_INTERVAL_MS;
|
|
62434
63573
|
});
|
|
62435
63574
|
|
|
62436
63575
|
// dist/builtin/workflows/src/workers/public-pool.ts
|
|
62437
|
-
import { resolve as
|
|
63576
|
+
import { resolve as resolve13 } from "node:path";
|
|
62438
63577
|
import { isDeepStrictEqual as isDeepStrictEqual2 } from "node:util";
|
|
62439
63578
|
async function restoreWorkerOptions(backend, options) {
|
|
62440
63579
|
const source = options.continuation?.source.id ?? options.runId;
|
|
@@ -62473,11 +63612,11 @@ async function activatePublicPool(backend, options) {
|
|
|
62473
63612
|
...options.localWorkers,
|
|
62474
63613
|
startupTimeoutMs: undefined,
|
|
62475
63614
|
leaseMs: undefined,
|
|
62476
|
-
definitions: options.localWorkers.definitions.map((source) => ({ ...source, path:
|
|
63615
|
+
definitions: options.localWorkers.definitions.map((source) => ({ ...source, path: resolve13(cwd, source.path) }))
|
|
62477
63616
|
};
|
|
62478
63617
|
const admission = options.admission && {
|
|
62479
63618
|
...options.admission,
|
|
62480
|
-
...options.admission.userDatabase ? { userDatabase:
|
|
63619
|
+
...options.admission.userDatabase ? { userDatabase: resolve13(cwd, options.admission.userDatabase) } : {}
|
|
62481
63620
|
};
|
|
62482
63621
|
const sources = localWorkers && await Promise.all(localWorkers.definitions.map(async (source) => ({ ...source, digest: await definitionDigest(source.path) })));
|
|
62483
63622
|
const resources = JSON.parse(JSON.stringify({
|
|
@@ -63823,7 +64962,7 @@ function hasUsableContentBlock(block) {
|
|
|
63823
64962
|
|
|
63824
64963
|
// dist/builtin/workflows/src/shared/stage-session-store.ts
|
|
63825
64964
|
import { randomUUID as randomUUID17 } from "node:crypto";
|
|
63826
|
-
import { dirname as dirname13, resolve as
|
|
64965
|
+
import { dirname as dirname13, resolve as resolve14 } from "node:path";
|
|
63827
64966
|
import {
|
|
63828
64967
|
JsonlSessionStore,
|
|
63829
64968
|
resolveSessionStore,
|
|
@@ -63859,11 +64998,11 @@ async function forkStageSession(store, sessionId, cwd, options, destination = st
|
|
|
63859
64998
|
async function migrateLegacyStageSession(store, sessionId, sessionFile) {
|
|
63860
64999
|
if (sessionFile === undefined || await store.readHeader(sessionId) !== undefined)
|
|
63861
65000
|
return;
|
|
63862
|
-
const location =
|
|
65001
|
+
const location = resolve14(sessionFile);
|
|
63863
65002
|
const legacy = await JsonlSessionStore.open(dirname13(location), { exactDirectory: true, scanExisting: false });
|
|
63864
65003
|
try {
|
|
63865
65004
|
for await (const file of legacy.scanFiles()) {
|
|
63866
|
-
if (
|
|
65005
|
+
if (resolve14(file.location) !== location || file.header.id !== sessionId)
|
|
63867
65006
|
continue;
|
|
63868
65007
|
const snapshot = await file.read();
|
|
63869
65008
|
if (await store.readHeader(sessionId) === undefined)
|
|
@@ -64025,22 +65164,37 @@ function childScopedEvidenceExists(checkpoints, boundaryReplayKey, childRunId) {
|
|
|
64025
65164
|
}
|
|
64026
65165
|
function retainReachableRunGroups(grouped, rootRunId) {
|
|
64027
65166
|
const reachable = new Set([rootRunId]);
|
|
64028
|
-
const
|
|
65167
|
+
const settled = new Set;
|
|
65168
|
+
const pending = [{ runId: rootRunId, live: true }];
|
|
64029
65169
|
while (pending.length > 0) {
|
|
64030
|
-
const
|
|
65170
|
+
const { runId, live } = pending.pop();
|
|
65171
|
+
const drafts = grouped.get(runId);
|
|
64031
65172
|
if (drafts === undefined)
|
|
64032
65173
|
continue;
|
|
64033
65174
|
for (const draft of drafts) {
|
|
64034
|
-
const childRunId =
|
|
64035
|
-
if (childRunId === undefined ||
|
|
65175
|
+
const childRunId = workflowChildFromDraft(draft)?.runId ?? draft.topology?.boundary?.child.runId;
|
|
65176
|
+
if (childRunId === undefined || !grouped.has(childRunId))
|
|
64036
65177
|
continue;
|
|
64037
|
-
|
|
64038
|
-
|
|
65178
|
+
if (live && childRunIdFromDraft(draft) !== undefined) {
|
|
65179
|
+
if (reachable.has(childRunId))
|
|
65180
|
+
continue;
|
|
65181
|
+
reachable.add(childRunId);
|
|
65182
|
+
pending.push({ runId: childRunId, live: true });
|
|
65183
|
+
} else if (!settled.has(childRunId)) {
|
|
65184
|
+
settled.add(childRunId);
|
|
65185
|
+
pending.push({ runId: childRunId, live: false });
|
|
65186
|
+
}
|
|
64039
65187
|
}
|
|
64040
65188
|
}
|
|
64041
|
-
|
|
64042
|
-
|
|
64043
|
-
|
|
65189
|
+
let complete = true;
|
|
65190
|
+
for (const runId of grouped.keys()) {
|
|
65191
|
+
if (reachable.has(runId))
|
|
65192
|
+
continue;
|
|
65193
|
+
if (!settled.has(runId))
|
|
65194
|
+
complete = false;
|
|
65195
|
+
grouped.delete(runId);
|
|
65196
|
+
}
|
|
65197
|
+
return complete;
|
|
64044
65198
|
}
|
|
64045
65199
|
function childRunIdFromDraft(draft) {
|
|
64046
65200
|
const boundary = draft.topology?.boundary;
|
|
@@ -64462,9 +65616,7 @@ function runSnapshotsFromCheckpoints(checkpoints, rootRunId, rootRunName, fallba
|
|
|
64462
65616
|
}
|
|
64463
65617
|
for (const group of grouped.values())
|
|
64464
65618
|
group.sort(compareDraftSourceOrder);
|
|
64465
|
-
|
|
64466
|
-
retainReachableRunGroups(grouped, rootRunId);
|
|
64467
|
-
if (policy.failOnUnreachable && grouped.size !== groupedCount)
|
|
65619
|
+
if (!retainReachableRunGroups(grouped, rootRunId) && policy.failOnUnreachable)
|
|
64468
65620
|
return [];
|
|
64469
65621
|
if (!validateRunGroups(grouped, rootRunId, toolCheckpoints, checkpoints, policy.retainActive))
|
|
64470
65622
|
return [];
|
|
@@ -70122,9 +71274,9 @@ var init_stage_runner_controller = __esm(() => {
|
|
|
70122
71274
|
});
|
|
70123
71275
|
|
|
70124
71276
|
// dist/builtin/workflows/src/runs/foreground/stage-runner-output.ts
|
|
70125
|
-
import { createHash as
|
|
71277
|
+
import { createHash as createHash5 } from "node:crypto";
|
|
70126
71278
|
import { mkdir as mkdir7, writeFile as writeFile10 } from "node:fs/promises";
|
|
70127
|
-
import { basename as basename5, dirname as dirname14, isAbsolute as isAbsolute11, join as join29, resolve as
|
|
71279
|
+
import { basename as basename5, dirname as dirname14, isAbsolute as isAbsolute11, join as join29, resolve as resolve15 } from "node:path";
|
|
70128
71280
|
function normalizeMaxOutput2(maxOutput) {
|
|
70129
71281
|
return {
|
|
70130
71282
|
bytes: maxOutput?.bytes ?? DEFAULT_MAX_OUTPUT_BYTES,
|
|
@@ -70186,7 +71338,7 @@ function formatByteSize(bytes) {
|
|
|
70186
71338
|
return `${value2.toFixed(1)} ${units[unitIndex]}`;
|
|
70187
71339
|
}
|
|
70188
71340
|
function transcriptPath(runId, outputPath) {
|
|
70189
|
-
const digest =
|
|
71341
|
+
const digest = createHash5("sha256").update(outputPath).digest("hex").slice(0, 16);
|
|
70190
71342
|
return join29(workflowArtifactRunPath(runId), "transcripts", `${digest}-${basename5(outputPath)}.transcript.md`);
|
|
70191
71343
|
}
|
|
70192
71344
|
function appendContent(lines, content) {
|
|
@@ -70232,7 +71384,7 @@ function renderTranscript(messages, fallbackOutput) {
|
|
|
70232
71384
|
`);
|
|
70233
71385
|
}
|
|
70234
71386
|
function savedOutputReference(outputPath, fullOutput, transcriptPathValue, transcriptText, transcriptAvailable, warning) {
|
|
70235
|
-
const absolutePath =
|
|
71387
|
+
const absolutePath = resolve15(outputPath);
|
|
70236
71388
|
const bytes = Buffer.byteLength(fullOutput, "utf8");
|
|
70237
71389
|
const lines = countLines(fullOutput);
|
|
70238
71390
|
const transcriptBytes = Buffer.byteLength(transcriptText, "utf8");
|
|
@@ -70247,8 +71399,8 @@ function resolveOutputPath(output, runtimeCwd, requestedCwd) {
|
|
|
70247
71399
|
return;
|
|
70248
71400
|
if (isAbsolute11(output))
|
|
70249
71401
|
return output;
|
|
70250
|
-
const baseCwd = requestedCwd === undefined ? runtimeCwd : isAbsolute11(requestedCwd) ? requestedCwd :
|
|
70251
|
-
return
|
|
71402
|
+
const baseCwd = requestedCwd === undefined ? runtimeCwd : isAbsolute11(requestedCwd) ? requestedCwd : resolve15(runtimeCwd, requestedCwd);
|
|
71403
|
+
return resolve15(baseCwd, output);
|
|
70252
71404
|
}
|
|
70253
71405
|
function degenerateWarning(fullOutput) {
|
|
70254
71406
|
return fullOutput.trim().length === 0 ? "WARNING: the stage artifact is empty; search the companion transcript for this stage's work." : undefined;
|
|
@@ -71257,7 +72409,7 @@ async function run(def, inputs, opts = {}) {
|
|
|
71257
72409
|
if (!isWorkflowDefinition(def))
|
|
71258
72410
|
throw new Error(workflowDefinitionRequirementMessage("run(definition, inputs)", def));
|
|
71259
72411
|
const depth = opts.depth ?? 0;
|
|
71260
|
-
const maxDepth = opts.config?.maxDepth ??
|
|
72412
|
+
const maxDepth = opts.config?.maxDepth ?? 3;
|
|
71261
72413
|
if (depth >= maxDepth) {
|
|
71262
72414
|
return {
|
|
71263
72415
|
runId: opts.runId ?? crypto.randomUUID(),
|