@bastani/atomic 0.9.61 → 0.9.62-alpha.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +28 -0
- package/dist/builtin/intercom/index.bundle.mjs +14 -0
- package/dist/builtin/intercom/package.json +1 -1
- package/dist/builtin/mcp/package.json +1 -1
- package/dist/builtin/subagents/package.json +1 -1
- package/dist/builtin/web-access/package.json +1 -1
- package/dist/builtin/workflows/package.json +1 -1
- package/dist/builtin/workflows/src/extension/index.bundle.mjs +97 -32
- package/dist/builtin/workflows/src/extension/local-worker-entry.mjs +98 -33
- package/dist/builtin/workflows/src/index.js +97 -32
- package/dist/builtin/workflows/src/local-worker-entry.mjs +98 -33
- package/npm-shrinkwrap.json +32 -32
- package/package.json +3 -3
package/CHANGELOG.md
CHANGED
|
@@ -2,6 +2,34 @@
|
|
|
2
2
|
|
|
3
3
|
## [Unreleased]
|
|
4
4
|
|
|
5
|
+
## [0.9.62] - 2026-11-02
|
|
6
|
+
|
|
7
|
+
Pooled roots that keep dispatching under load, children that survive a stalled worker, and Intercom sends that no longer report a delivered message as failed.
|
|
8
|
+
|
|
9
|
+
- **Pooled roots no longer stall while children start:** checkpoints queued back to back share one journal transaction, so a loop that launches dozens of children keeps to its timers; with 300 busy workers, all children ran in 13 s instead of up to 100 s ([#4181](https://github.com/bastani-inc/atomic/pull/4181))
|
|
10
|
+
- **A stalled worker's waiting admissions survive:** a waiting stage, subagent or process admission fenced while its process stalled rejoins the queue instead of failing its child with `Resource admission holder was stopped`, and an admission released for good is a retryable pool fault ([#4180](https://github.com/bastani-inc/atomic/pull/4180))
|
|
11
|
+
- **Unanswered sends are an unknown outcome:** an Intercom send the broker never answered is reported as possibly delivered, not as `Failed to send`, so it is not resent as a duplicate ([#4178](https://github.com/bastani-inc/atomic/pull/4178))
|
|
12
|
+
|
|
13
|
+
### Fixed
|
|
14
|
+
|
|
15
|
+
- A workflow that launches many child workflows onto local workers (`workflows.localWorkers`) no longer stalls its own next steps while the children start. Every child launch and every `ctx.tool` step wrote its checkpoint in a journal transaction of its own, one after another. So a loop that launched a few dozen children and then saved its state waited for each launch's write in turn, and on a journal busy with hundreds of workers that took minutes: the loop missed its timers and launches crawled. Checkpoints queued back to back now share one transaction. With 300 busy workers, the step after a 48-child burst finished in 78 ms at the 90th percentile (253 ms at most) instead of 3.4 to 9.4 s (14 s at most), and all 300 children were running in 13 s instead of 40 to 100 s. A workflow started from the SDK also no longer pauses its host for seconds at a time when the process holds thousands of finished runs. ([#4174](https://github.com/bastani-inc/atomic/issues/4174))
|
|
16
|
+
- Children on a local worker pool no longer fail with `Resource admission holder was stopped` under heavy load. A waiting resource admission (a stage session, subagent or process start) whose owner lease lapsed while its process stalled was fenced by another worker's cleanup, and the next pass of its wait failed the child. A waiter holds no capacity, so a live one now rejoins the queue. History pruning keeps a fenced waiter's permit for 10 minutes after its owner's lease lapsed, so it can still be taken back. An admission the pool released for good, for example after its worker exited, or whose permit is gone, is now a retryable `stale_work_lease` fault, replayed within the child's attempt limit, rather than a failure of the child's own work; stopping a worker also keeps its just-fenced waiters stopped. ([#4179](https://github.com/bastani-inc/atomic/issues/4179))
|
|
17
|
+
- An Intercom `send` whose broker response timed out, while the broker still answered its health check, is now reported as an unknown outcome instead of `Failed to send: Send timeout`. The broker can still queue or deliver such a message, so the tool now says to check with the recipient before sending again, and marks the result `outcome: "unknown"`. Under heavy load the old wording led to resends that delivered the message twice. ([#4177](https://github.com/bastani-inc/atomic/issues/4177))
|
|
18
|
+
|
|
19
|
+
## [0.9.62-alpha.1] - 2026-11-02
|
|
20
|
+
|
|
21
|
+
Pooled roots that keep dispatching under load, children that survive a stalled worker, and Intercom sends that no longer report a delivered message as failed.
|
|
22
|
+
|
|
23
|
+
- **Pooled roots no longer stall while children start:** checkpoints queued back to back share one journal transaction, so a loop that launches dozens of children keeps to its timers; with 300 busy workers, all children ran in 13 s instead of up to 100 s ([#4181](https://github.com/bastani-inc/atomic/pull/4181))
|
|
24
|
+
- **A stalled worker's waiting admissions survive:** a waiting stage, subagent or process admission fenced while its process stalled rejoins the queue instead of failing its child with `Resource admission holder was stopped`, and an admission released for good is a retryable pool fault ([#4180](https://github.com/bastani-inc/atomic/pull/4180))
|
|
25
|
+
- **Unanswered sends are an unknown outcome:** an Intercom send the broker never answered is reported as possibly delivered, not as `Failed to send`, so it is not resent as a duplicate ([#4178](https://github.com/bastani-inc/atomic/pull/4178))
|
|
26
|
+
|
|
27
|
+
### Fixed
|
|
28
|
+
|
|
29
|
+
- A workflow that launches many child workflows onto local workers (`workflows.localWorkers`) no longer stalls its own next steps while the children start. Every child launch and every `ctx.tool` step wrote its checkpoint in a journal transaction of its own, one after another. So a loop that launched a few dozen children and then saved its state waited for each launch's write in turn, and on a journal busy with hundreds of workers that took minutes: the loop missed its timers and launches crawled. Checkpoints queued back to back now share one transaction. With 300 busy workers, the step after a 48-child burst finished in 78 ms at the 90th percentile (253 ms at most) instead of 3.4 to 9.4 s (14 s at most), and all 300 children were running in 13 s instead of 40 to 100 s. A workflow started from the SDK also no longer pauses its host for seconds at a time when the process holds thousands of finished runs. ([#4174](https://github.com/bastani-inc/atomic/issues/4174))
|
|
30
|
+
- Children on a local worker pool no longer fail with `Resource admission holder was stopped` under heavy load. A waiting resource admission (a stage session, subagent or process start) whose owner lease lapsed while its process stalled was fenced by another worker's cleanup, and the next pass of its wait failed the child. A waiter holds no capacity, so a live one now rejoins the queue. History pruning keeps a fenced waiter's permit for 10 minutes after its owner's lease lapsed, so it can still be taken back. An admission the pool released for good, for example after its worker exited, or whose permit is gone, is now a retryable `stale_work_lease` fault, replayed within the child's attempt limit, rather than a failure of the child's own work; stopping a worker also keeps its just-fenced waiters stopped. ([#4179](https://github.com/bastani-inc/atomic/issues/4179))
|
|
31
|
+
- An Intercom `send` whose broker response timed out, while the broker still answered its health check, is now reported as an unknown outcome instead of `Failed to send: Send timeout`. The broker can still queue or deliver such a message, so the tool now says to check with the recipient before sending again, and marks the result `outcome: "unknown"`. Under heavy load the old wording led to resends that delivered the message twice. ([#4177](https://github.com/bastani-inc/atomic/issues/4177))
|
|
32
|
+
|
|
5
33
|
## [0.9.61] - 2026-11-02
|
|
6
34
|
|
|
7
35
|
Local worker pools that hold up under faults and load, a lighter terminal under heavy workflow load, and workflows that can work in any repository's worktree.
|
|
@@ -4224,6 +4224,9 @@ import { Text as Text3 } from "@earendil-works/pi-tui";
|
|
|
4224
4224
|
function retryErrorDetails(retryToken) {
|
|
4225
4225
|
return retryToken === undefined ? { error: true } : { error: true, retryToken };
|
|
4226
4226
|
}
|
|
4227
|
+
function sendOutcomeUnknown(error) {
|
|
4228
|
+
return error instanceof PendingSendTimeoutError;
|
|
4229
|
+
}
|
|
4227
4230
|
function retainRecoverableRetry(reservations, attempt) {
|
|
4228
4231
|
const retryToken = reservations.retainAfterRecoverableDisconnect(attempt);
|
|
4229
4232
|
return {
|
|
@@ -4809,6 +4812,16 @@ ${message}${attachmentText}`);
|
|
|
4809
4812
|
retryIdentities.release(retryIdentity);
|
|
4810
4813
|
}
|
|
4811
4814
|
}
|
|
4815
|
+
if (failure === error && sendOutcomeUnknown(error)) {
|
|
4816
|
+
return {
|
|
4817
|
+
content: [{
|
|
4818
|
+
type: "text",
|
|
4819
|
+
text: `The broker did not answer the send to "${to}" in time, so whether the message was queued or delivered is unknown: it may still arrive. Check with the recipient before sending it again (for a workflow-stage target, \`intercom list\` shows queued counts), because a new send is a new message.`
|
|
4820
|
+
}],
|
|
4821
|
+
isError: true,
|
|
4822
|
+
details: { error: true, outcome: "unknown" }
|
|
4823
|
+
};
|
|
4824
|
+
}
|
|
4812
4825
|
return {
|
|
4813
4826
|
content: [{
|
|
4814
4827
|
type: "text",
|
|
@@ -5340,6 +5353,7 @@ var init_intercom_tool = __esm(() => {
|
|
|
5340
5353
|
init_recoverable_disconnect();
|
|
5341
5354
|
init_retry_identity();
|
|
5342
5355
|
init_tool_retry();
|
|
5356
|
+
init_pending_send_registry();
|
|
5343
5357
|
});
|
|
5344
5358
|
|
|
5345
5359
|
// dist/builtin/intercom/ui/compose.ts
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@bastani/intercom",
|
|
3
|
-
"version": "0.9.
|
|
3
|
+
"version": "0.9.62-alpha.1",
|
|
4
4
|
"private": true,
|
|
5
5
|
"description": "Atomic extension providing a private coordination channel between parent and child agent sessions. Fork of: https://github.com/nicobailon/pi-intercom",
|
|
6
6
|
"contributors": [
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@bastani/subagents",
|
|
3
|
-
"version": "0.9.
|
|
3
|
+
"version": "0.9.62-alpha.1",
|
|
4
4
|
"private": true,
|
|
5
5
|
"description": "Atomic extension for delegating tasks to subagents with parallel execution. Fork of: https://github.com/nicobailon/pi-subagents",
|
|
6
6
|
"contributors": [
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@bastani/web-access",
|
|
3
|
-
"version": "0.9.
|
|
3
|
+
"version": "0.9.62-alpha.1",
|
|
4
4
|
"private": true,
|
|
5
5
|
"description": "Atomic extension for web search, URL fetching, GitHub repo cloning, PDF/video extraction. Fork of: https://github.com/nicobailon/pi-web-access",
|
|
6
6
|
"contributors": [
|
|
@@ -14289,15 +14289,12 @@ class JournalDurableBackend {
|
|
|
14289
14289
|
commitCheckpoint(checkpoint, options) {
|
|
14290
14290
|
if (options?.signal?.aborted)
|
|
14291
14291
|
return Promise.reject(new JournalOperationAbortedError(checkpoint.workflowId, options.signal.reason instanceof Error ? options.signal.reason : undefined));
|
|
14292
|
-
|
|
14293
|
-
|
|
14294
|
-
|
|
14295
|
-
|
|
14296
|
-
|
|
14297
|
-
|
|
14298
|
-
yield* this.append(checkpoint.workflowId, { type: RUN_EVENT_TYPES.checkpointRecorded, payload: checkpoint }, {
|
|
14299
|
-
key: JSON.stringify([RUN_EVENT_TYPES.checkpointRecorded, checkpoint.kind, checkpoint.checkpointId])
|
|
14300
|
-
});
|
|
14292
|
+
try {
|
|
14293
|
+
validateCheckpointInvocationFingerprint(checkpoint);
|
|
14294
|
+
} catch (error) {
|
|
14295
|
+
return Promise.reject(error);
|
|
14296
|
+
}
|
|
14297
|
+
return this.queueAppend(checkpoint.workflowId, { type: RUN_EVENT_TYPES.checkpointRecorded, payload: checkpoint }, JSON.stringify([RUN_EVENT_TYPES.checkpointRecorded, checkpoint.kind, checkpoint.checkpointId]), true);
|
|
14301
14298
|
}
|
|
14302
14299
|
async commitAdditiveCheckpointBestEffort(checkpoint) {
|
|
14303
14300
|
try {
|
|
@@ -14434,7 +14431,7 @@ class JournalDurableBackend {
|
|
|
14434
14431
|
payload: { messageId, delivery, ...accounting === undefined ? {} : { accounting } }
|
|
14435
14432
|
});
|
|
14436
14433
|
}
|
|
14437
|
-
queueAppend(runId, event, key) {
|
|
14434
|
+
queueAppend(runId, event, key, requiresRegisteredRun = false) {
|
|
14438
14435
|
try {
|
|
14439
14436
|
this.ensureOpen();
|
|
14440
14437
|
this.assertExecutionActive();
|
|
@@ -14447,6 +14444,7 @@ class JournalDurableBackend {
|
|
|
14447
14444
|
batch.push({
|
|
14448
14445
|
event,
|
|
14449
14446
|
...key === undefined ? {} : { key },
|
|
14447
|
+
...requiresRegisteredRun ? { requiresRegisteredRun } : {},
|
|
14450
14448
|
context: AsyncLocalStorage3.snapshot(),
|
|
14451
14449
|
resolve,
|
|
14452
14450
|
reject
|
|
@@ -14461,9 +14459,12 @@ class JournalDurableBackend {
|
|
|
14461
14459
|
}
|
|
14462
14460
|
return queued;
|
|
14463
14461
|
}
|
|
14464
|
-
*appendBatch(runId,
|
|
14465
|
-
if (this.appendBatches.get(runId) ===
|
|
14462
|
+
*appendBatch(runId, queued) {
|
|
14463
|
+
if (this.appendBatches.get(runId) === queued)
|
|
14466
14464
|
this.appendBatches.delete(runId);
|
|
14465
|
+
const batch = yield* this.registeredAppends(runId, queued);
|
|
14466
|
+
if (batch.length === 0)
|
|
14467
|
+
return;
|
|
14467
14468
|
const appendMany = runJournalAppendMany(this.journal);
|
|
14468
14469
|
if (appendMany === undefined || batch.length === 1) {
|
|
14469
14470
|
for (const append of batch) {
|
|
@@ -14501,6 +14502,25 @@ class JournalDurableBackend {
|
|
|
14501
14502
|
append.reject(failure);
|
|
14502
14503
|
}
|
|
14503
14504
|
}
|
|
14505
|
+
*registeredAppends(runId, batch) {
|
|
14506
|
+
if (!batch.some((append) => append.requiresRegisteredRun === true))
|
|
14507
|
+
return [...batch];
|
|
14508
|
+
let registered;
|
|
14509
|
+
try {
|
|
14510
|
+
registered = (yield* this.current(runId)) !== undefined;
|
|
14511
|
+
} catch (error) {
|
|
14512
|
+
for (const append of batch)
|
|
14513
|
+
if (append.requiresRegisteredRun === true)
|
|
14514
|
+
append.reject(error);
|
|
14515
|
+
return batch.filter((append) => append.requiresRegisteredRun !== true);
|
|
14516
|
+
}
|
|
14517
|
+
if (registered)
|
|
14518
|
+
return [...batch];
|
|
14519
|
+
for (const append of batch)
|
|
14520
|
+
if (append.requiresRegisteredRun === true)
|
|
14521
|
+
append.resolve();
|
|
14522
|
+
return batch.filter((append) => append.requiresRegisteredRun !== true);
|
|
14523
|
+
}
|
|
14504
14524
|
getToolOutput(...args) {
|
|
14505
14525
|
return this.projection.getToolOutput(...args);
|
|
14506
14526
|
}
|
|
@@ -20703,7 +20723,7 @@ class SqlResourceAdmission {
|
|
|
20703
20723
|
signal?.throwIfAborted();
|
|
20704
20724
|
const operation = randomUUID7();
|
|
20705
20725
|
const release = (reason = "resource cleanup confirmed") => this.storage.transaction(async (tx) => {
|
|
20706
|
-
await tx.run(`UPDATE ${this.permits} SET state = 'released', reason = ? WHERE operation = ? AND state != 'released'`, reason, operation);
|
|
20726
|
+
await tx.run(`UPDATE ${this.permits} SET state = 'released', reason = ? WHERE operation = ? AND (state != 'released' OR reason = ?)`, reason, operation, FENCED_WAITER_REASON);
|
|
20707
20727
|
});
|
|
20708
20728
|
release.bindProcess = async (identity) => {
|
|
20709
20729
|
if (kind !== "process" || identity.kind !== "unix-process-group" || !Number.isSafeInteger(identity.id) || identity.id <= 0)
|
|
@@ -20715,6 +20735,15 @@ class SqlResourceAdmission {
|
|
|
20715
20735
|
throw new Error("Resource process permit is no longer held");
|
|
20716
20736
|
});
|
|
20717
20737
|
};
|
|
20738
|
+
const keepWaiting = async (tx) => {
|
|
20739
|
+
const permit = await tx.get(`SELECT state, reason FROM ${this.permits} WHERE operation = ?`, operation);
|
|
20740
|
+
if (permit?.state === "released" && permit.reason === FENCED_WAITER_REASON)
|
|
20741
|
+
await tx.run(`UPDATE ${this.permits} SET state = 'waiting', reason = NULL WHERE operation = ?`, operation);
|
|
20742
|
+
else if (permit === undefined)
|
|
20743
|
+
throw new AdmissionHolderStoppedError("permit no longer recorded");
|
|
20744
|
+
else if (permit.state !== "waiting")
|
|
20745
|
+
throw new AdmissionHolderStoppedError(permit.reason === null ? undefined : String(permit.reason));
|
|
20746
|
+
};
|
|
20718
20747
|
await this.storage.transaction(async (tx) => {
|
|
20719
20748
|
await tx.run(`UPDATE ${this.owners} SET expires_at = CAST(unixepoch('subsec') * 1000 AS INTEGER) + ? WHERE id = ?`, OWNER_LEASE_MS, this.owner);
|
|
20720
20749
|
await tx.run(`INSERT INTO ${this.permits}(operation, root, holder, kind, state, owner, attempt) VALUES (?, ?, ?, ?, 'waiting', ?, ?)`, operation, this.options.root, this.options.holder, kind, this.owner, this.options.attempt ? JSON.stringify({
|
|
@@ -20737,11 +20766,12 @@ class SqlResourceAdmission {
|
|
|
20737
20766
|
const granted = await this.storage.transaction(async (tx) => {
|
|
20738
20767
|
quarantined = undefined;
|
|
20739
20768
|
await tx.run(`UPDATE ${this.owners} SET expires_at = CAST(unixepoch('subsec') * 1000 AS INTEGER) + ? WHERE id = ?`, OWNER_LEASE_MS, this.owner);
|
|
20769
|
+
await keepWaiting(tx);
|
|
20740
20770
|
const { root } = this.options;
|
|
20741
20771
|
if (!await this.assertCurrentAttempt(tx)) {
|
|
20742
20772
|
const waiting = await tx.run(`UPDATE ${this.permits} SET reason = ? WHERE operation = ? AND state = 'waiting'`, LATE_ATTEMPT_REASON, operation);
|
|
20743
20773
|
if (waiting.changes !== 1)
|
|
20744
|
-
throw new
|
|
20774
|
+
throw new AdmissionHolderStoppedError(undefined);
|
|
20745
20775
|
return false;
|
|
20746
20776
|
}
|
|
20747
20777
|
let reason = hostReason;
|
|
@@ -20784,7 +20814,7 @@ class SqlResourceAdmission {
|
|
|
20784
20814
|
const stageStart = kind === "session" && request?.delegated !== true;
|
|
20785
20815
|
const updated = await tx.run(`UPDATE ${this.permits} SET state = ?, reason = ?, granted_at = CASE WHEN ? IS NULL AND ? THEN ${NOW_MS} ELSE NULL END WHERE operation = ? AND state = 'waiting'`, reason ? "waiting" : "granted", reason ?? null, reason ?? null, stageStart ? 1 : 0, operation);
|
|
20786
20816
|
if (updated.changes !== 1)
|
|
20787
|
-
throw new
|
|
20817
|
+
throw new AdmissionHolderStoppedError(undefined);
|
|
20788
20818
|
return !reason;
|
|
20789
20819
|
});
|
|
20790
20820
|
if (granted) {
|
|
@@ -20826,7 +20856,7 @@ class SqlResourceAdmission {
|
|
|
20826
20856
|
WHERE p.state IN ('waiting', 'granted')`);
|
|
20827
20857
|
if (live.some((permit) => this.needsFencing(permit, fence)))
|
|
20828
20858
|
return true;
|
|
20829
|
-
return !!await db.get(`SELECT 1 AS prunable FROM ${this.permits} WHERE
|
|
20859
|
+
return !!await db.get(`SELECT 1 AS prunable FROM ${this.permits} r WHERE ${this.prunable} LIMIT 1`, RELEASED_PERMIT_RETENTION, FENCED_WAITER_REASON, FENCED_WAITER_RETENTION_MS);
|
|
20830
20860
|
});
|
|
20831
20861
|
if (!pending)
|
|
20832
20862
|
return;
|
|
@@ -20835,7 +20865,7 @@ class SqlResourceAdmission {
|
|
|
20835
20865
|
const owners = await tx.all(`SELECT o.* FROM ${this.owners} o WHERE o.id IN (SELECT owner FROM ${this.permits} WHERE state IN ('waiting', 'granted'))`);
|
|
20836
20866
|
for (const owner of owners) {
|
|
20837
20867
|
const { rebooted, stopped } = fence(owner);
|
|
20838
|
-
await tx.run(`UPDATE ${this.permits} SET state = 'released', reason =
|
|
20868
|
+
await tx.run(`UPDATE ${this.permits} SET state = 'released', reason = ? WHERE owner = ? AND state = 'waiting' AND (? OR EXISTS (SELECT 1 FROM ${this.owners} WHERE id = ? AND expires_at < ${NOW_MS}))`, FENCED_WAITER_REASON, owner.id, stopped ? 1 : 0, owner.id);
|
|
20839
20869
|
if (rebooted) {
|
|
20840
20870
|
await tx.run(`UPDATE ${this.permits} SET state = 'released', reason = 'host reboot confirmed' WHERE owner = ? AND state = 'granted'`, owner.id);
|
|
20841
20871
|
} else if (stopped) {
|
|
@@ -20853,13 +20883,17 @@ class SqlResourceAdmission {
|
|
|
20853
20883
|
}
|
|
20854
20884
|
await tx.run(`UPDATE ${this.permits} SET state = 'released', reason = 'legacy waiter fenced' WHERE owner IS NULL AND state = 'waiting'`);
|
|
20855
20885
|
await tx.run(`UPDATE ${this.permits} SET reason = ? WHERE owner IS NULL AND state = 'granted' AND reason IS NOT ?`, ANONYMOUS_REASON, ANONYMOUS_REASON);
|
|
20856
|
-
await tx.run(`DELETE FROM ${this.permits} WHERE ticket IN (SELECT ticket FROM ${this.permits} WHERE
|
|
20857
|
-
|
|
20886
|
+
await tx.run(`DELETE FROM ${this.permits} WHERE ticket IN (SELECT ticket FROM ${this.permits} r WHERE ${this.prunable}
|
|
20887
|
+
ORDER BY ticket LIMIT ?)`, RELEASED_PERMIT_RETENTION, FENCED_WAITER_REASON, FENCED_WAITER_RETENTION_MS, RELEASED_PERMIT_PRUNE_BATCH);
|
|
20858
20888
|
});
|
|
20859
20889
|
}
|
|
20860
20890
|
get prunableBefore() {
|
|
20861
20891
|
return `(SELECT ticket FROM ${this.permits} WHERE state = 'released' ORDER BY ticket DESC LIMIT 1 OFFSET ?)`;
|
|
20862
20892
|
}
|
|
20893
|
+
get prunable() {
|
|
20894
|
+
return `+r.state = 'released' AND r.ticket <= ${this.prunableBefore} AND (r.reason IS NOT ? OR NOT EXISTS
|
|
20895
|
+
(SELECT 1 FROM ${this.owners} o WHERE o.id = r.owner AND o.expires_at >= ${NOW_MS} - ?))`;
|
|
20896
|
+
}
|
|
20863
20897
|
needsFencing(permit, fence) {
|
|
20864
20898
|
if (permit.owner === null)
|
|
20865
20899
|
return permit.state === "waiting" || permit.reason !== ANONYMOUS_REASON;
|
|
@@ -20884,6 +20918,7 @@ class SqlResourceAdmission {
|
|
|
20884
20918
|
await this.reconcile();
|
|
20885
20919
|
return this.storage.transaction(async (tx) => {
|
|
20886
20920
|
await tx.run(`UPDATE ${this.permits} SET state = 'released', reason = 'worker exit confirmed' WHERE root = ? AND holder = ? AND state IN ('waiting', 'granted') AND (kind = 'session' OR state = 'waiting')`, this.options.root, holder);
|
|
20921
|
+
await tx.run(`UPDATE ${this.permits} SET reason = 'worker exit confirmed' WHERE root = ? AND holder = ? AND state = 'released' AND reason = ?`, this.options.root, holder, FENCED_WAITER_REASON);
|
|
20887
20922
|
await tx.run(`UPDATE ${this.permits} SET reason = ? WHERE root = ? AND holder = ? AND kind = 'process' AND state = 'granted' AND reason IS NOT ?`, UNCONFIRMED_REASON, this.options.root, holder, QUARANTINED_REASON);
|
|
20888
20923
|
const held = await tx.get(`SELECT COUNT(*) AS count FROM ${this.permits} WHERE root = ? AND holder = ? AND kind = 'process' AND state = 'granted'`, this.options.root, holder);
|
|
20889
20924
|
return Number(held?.count);
|
|
@@ -20956,7 +20991,7 @@ async function reapProcessGroups(journal, root, scope) {
|
|
|
20956
20991
|
return { survivors: [], unprobeable: [] };
|
|
20957
20992
|
return settleProcessGroups(storage, root, scope, true);
|
|
20958
20993
|
}
|
|
20959
|
-
var OWNER_LEASE_MS = 30000, REAP_TERM_GRACE_MS = 5000, REAP_KILL_GRACE_MS = 1e4, REAP_POLL_MS = 50, RELEASED_PERMIT_RETENTION = 256, RELEASED_PERMIT_PRUNE_BATCH = 1000, EXPIRED_REASON = "owner expired; cleanup unconfirmed", UNCONFIRMED_REASON = "cleanup unconfirmed", QUARANTINED_REASON = "cleanup unconfirmed; permit quarantined", ANONYMOUS_REASON = "owner identity unavailable; cleanup unconfirmed", QUARANTINED_CAPACITY_WAIT_MS, QuarantinedCapacityError, NOW_MS = "CAST(unixepoch('subsec') * 1000 AS INTEGER)", LATE_ATTEMPT_POLL_MS = 100, LATE_ATTEMPT_REASON = "waiting for its attempt lease", STAGE_START_SPACING_MS = 5000, STAGE_START_PRESSURE_REASON = "paced for host load";
|
|
20994
|
+
var OWNER_LEASE_MS = 30000, REAP_TERM_GRACE_MS = 5000, REAP_KILL_GRACE_MS = 1e4, REAP_POLL_MS = 50, RELEASED_PERMIT_RETENTION = 256, RELEASED_PERMIT_PRUNE_BATCH = 1000, EXPIRED_REASON = "owner expired; cleanup unconfirmed", UNCONFIRMED_REASON = "cleanup unconfirmed", QUARANTINED_REASON = "cleanup unconfirmed; permit quarantined", ANONYMOUS_REASON = "owner identity unavailable; cleanup unconfirmed", QUARANTINED_CAPACITY_WAIT_MS, QuarantinedCapacityError, FENCED_WAITER_REASON = "abandoned waiter fenced", FENCED_WAITER_RETENTION_MS, AdmissionHolderStoppedError, NOW_MS = "CAST(unixepoch('subsec') * 1000 AS INTEGER)", LATE_ATTEMPT_POLL_MS = 100, LATE_ATTEMPT_REASON = "waiting for its attempt lease", STAGE_START_SPACING_MS = 5000, STAGE_START_PRESSURE_REASON = "paced for host load";
|
|
20960
20995
|
var init_resource_admission = __esm(() => {
|
|
20961
20996
|
init_internal();
|
|
20962
20997
|
init_sql_run_journal();
|
|
@@ -20970,6 +21005,12 @@ var init_resource_admission = __esm(() => {
|
|
|
20970
21005
|
super("worker_cleanup_unconfirmed", `Local worker process cleanup unconfirmed; process permit refused after ${Math.round(waitedMs)} ms: ${limits} ${capacity.length === 1 ? "is" : "are"} held by quarantined permits of process groups ${groups} that survived SIGKILL`);
|
|
20971
21006
|
}
|
|
20972
21007
|
};
|
|
21008
|
+
FENCED_WAITER_RETENTION_MS = 10 * 60000;
|
|
21009
|
+
AdmissionHolderStoppedError = class AdmissionHolderStoppedError extends WorkflowInfrastructureError {
|
|
21010
|
+
constructor(reason) {
|
|
21011
|
+
super("stale_work_lease", `Stale resource admission attempt: the admission holder was stopped${reason === undefined ? "" : ` (${reason})`}`);
|
|
21012
|
+
}
|
|
21013
|
+
};
|
|
20973
21014
|
});
|
|
20974
21015
|
|
|
20975
21016
|
// dist/builtin/workflows/src/workers/admission-binding.ts
|
|
@@ -68191,6 +68232,37 @@ var init_workflow_route_owner = __esm(() => {
|
|
|
68191
68232
|
});
|
|
68192
68233
|
|
|
68193
68234
|
// dist/builtin/workflows/src/runs/foreground/sdk-intercom.ts
|
|
68235
|
+
function indexRoots(runs) {
|
|
68236
|
+
const runById = new Map(runs.map((run) => [run.id, run]));
|
|
68237
|
+
const rootOf = new Map;
|
|
68238
|
+
const deliverable = new Set;
|
|
68239
|
+
for (const run of runs) {
|
|
68240
|
+
const root = reciprocalWorkflowRootRunId(runById, run.id);
|
|
68241
|
+
rootOf.set(run.id, root);
|
|
68242
|
+
if (root !== undefined && run.stages.some((stage) => stage.pendingStageDeliveryAvailable === true))
|
|
68243
|
+
deliverable.add(root);
|
|
68244
|
+
}
|
|
68245
|
+
return { rootOf, deliverable };
|
|
68246
|
+
}
|
|
68247
|
+
function createStandaloneRouteOwnership(store, roots) {
|
|
68248
|
+
const activated = new Set;
|
|
68249
|
+
let index;
|
|
68250
|
+
const dispose = store.subscribeInvalidation(() => {
|
|
68251
|
+
index = undefined;
|
|
68252
|
+
});
|
|
68253
|
+
return {
|
|
68254
|
+
owns: (runId) => {
|
|
68255
|
+
index ??= indexRoots(store.runs());
|
|
68256
|
+
const root = index.rootOf.get(runId) ?? runId;
|
|
68257
|
+
if (!roots.has(root))
|
|
68258
|
+
return false;
|
|
68259
|
+
if (!activated.has(root) && index.deliverable.has(root))
|
|
68260
|
+
activated.add(root);
|
|
68261
|
+
return activated.has(root);
|
|
68262
|
+
},
|
|
68263
|
+
dispose
|
|
68264
|
+
};
|
|
68265
|
+
}
|
|
68194
68266
|
function releaseRoot(owner, runId) {
|
|
68195
68267
|
const remaining = (owner.roots.get(runId) ?? 1) - 1;
|
|
68196
68268
|
if (remaining > 0) {
|
|
@@ -68206,22 +68278,14 @@ async function acquireStandaloneIntercom(store, runId, cwd) {
|
|
|
68206
68278
|
if (hasWorkflowPendingStageRouteOwner(store))
|
|
68207
68279
|
return async () => {};
|
|
68208
68280
|
const roots = new Map;
|
|
68209
|
-
const activated = new Set;
|
|
68210
68281
|
owner = {
|
|
68211
68282
|
roots,
|
|
68212
68283
|
ready: outsideWorkflowExecutions(async () => {
|
|
68213
68284
|
await Promise.resolve().then(() => init_workflow_route_owner());
|
|
68214
68285
|
const routes = createWorkflowRouteOwner(cwd);
|
|
68215
|
-
const
|
|
68216
|
-
|
|
68217
|
-
|
|
68218
|
-
if (!roots.has(root))
|
|
68219
|
-
return false;
|
|
68220
|
-
if (!activated.has(root) && runs.some((run) => durableRootRunIdForRun(runs, run.id) === root && run.stages.some((stage) => stage.pendingStageDeliveryAvailable === true)))
|
|
68221
|
-
activated.add(root);
|
|
68222
|
-
return activated.has(root);
|
|
68223
|
-
});
|
|
68224
|
-
return { routes, bridge };
|
|
68286
|
+
const ownership = createStandaloneRouteOwnership(store, roots);
|
|
68287
|
+
const bridge = registerPendingStageIntercomBridge({ events: routes.events }, store, ownership.owns);
|
|
68288
|
+
return { routes, bridge, unwatch: ownership.dispose };
|
|
68225
68289
|
})
|
|
68226
68290
|
};
|
|
68227
68291
|
owners2.set(store, owner);
|
|
@@ -68245,6 +68309,7 @@ async function acquireStandaloneIntercom(store, runId, cwd) {
|
|
|
68245
68309
|
initialized.bridge.refresh();
|
|
68246
68310
|
if (retained.roots.size === 0) {
|
|
68247
68311
|
initialized.bridge();
|
|
68312
|
+
initialized.unwatch();
|
|
68248
68313
|
if (owners2.get(store) === retained)
|
|
68249
68314
|
owners2.delete(store);
|
|
68250
68315
|
}
|
|
@@ -68256,10 +68321,10 @@ async function acquireStandaloneIntercom(store, runId, cwd) {
|
|
|
68256
68321
|
var owners2;
|
|
68257
68322
|
var init_sdk_intercom = __esm(() => {
|
|
68258
68323
|
init_execution_context_boundary();
|
|
68259
|
-
init_run_owner_backend();
|
|
68260
68324
|
init_pending_stage_intercom();
|
|
68261
68325
|
init_intercom_group();
|
|
68262
68326
|
init_pending_stage_route_readiness();
|
|
68327
|
+
init_workflow_run_ownership();
|
|
68263
68328
|
owners2 = new WeakMap;
|
|
68264
68329
|
});
|
|
68265
68330
|
|