@desplega.ai/agent-swarm 1.108.2 → 1.110.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/dist/{actions-1hxdkcp6.js → actions-837ktebk.js} +6 -6
- package/dist/{app-vn2sy15m.js → app-85h97pg2.js} +3 -3
- package/dist/{assistant-sj71nx88.js → assistant-yhrta0c5.js} +9 -9
- package/dist/{boot-reembed-nc1dpmdj.js → boot-reembed-5sd1t9r4.js} +4 -4
- package/dist/{boot-reembed-nw9vp9ez.js → boot-reembed-8xmfv2pq.js} +3 -3
- package/dist/{boot-scrub-logs-g2mpp0cg.js → boot-scrub-logs-fdw8wbtq.js} +2 -2
- package/dist/{cli-bz6qha8p.js → cli-0pzg1f8z.js} +4 -4
- package/dist/{cli-b7ytq2z8.js → cli-2h86bsfe.js} +2 -2
- package/dist/{cli-b4hwyzev.js → cli-3baz70yp.js} +1 -1
- package/dist/{cli-er5kffrx.js → cli-3t97r3y1.js} +5 -5
- package/dist/{cli-gg6zxrx1.js → cli-4trfcacj.js} +5 -5
- package/dist/{cli-5m4da9px.js → cli-5kav4x1a.js} +1 -1
- package/dist/{cli-cr2wze6h.js → cli-6ttdv68b.js} +1 -1
- package/dist/{cli-3p0e74xv.js → cli-8ps5wsk9.js} +1 -1
- package/dist/{cli-1brke24m.js → cli-8s0swp2t.js} +3 -3
- package/dist/{cli-4wrdrmr3.js → cli-cwety9qq.js} +2 -2
- package/dist/{cli-6jf8xfnt.js → cli-d8vmtyyr.js} +2 -2
- package/dist/{cli-ae9hy8yh.js → cli-dhph6q98.js} +1 -1
- package/dist/{cli-hc7hxy2m.js → cli-dy01kwxk.js} +2 -2
- package/dist/{cli-9zz27c9g.js → cli-g44ffxk5.js} +1 -1
- package/dist/{cli-fxk0as0h.js → cli-gb7dqnna.js} +1 -1
- package/dist/{cli-1wzrmxaj.js → cli-gtt1f14h.js} +3 -3
- package/dist/{cli-27pxayz8.js → cli-k6ssh5dk.js} +15 -5
- package/dist/{cli-tc19ec02.js → cli-kg9v8h1w.js} +2 -2
- package/dist/{cli-xekgq1r8.js → cli-kna0r84e.js} +17 -17
- package/dist/{cli-8kssf4wv.js → cli-r19ttke3.js} +4 -4
- package/dist/{cli-nt60dmfm.js → cli-sjm0e1pa.js} +2 -2
- package/dist/{cli-8gtrpqcw.js → cli-vfyhrk9k.js} +1 -1
- package/dist/{cli-6z0y1339.js → cli-zw3cmjh6.js} +4 -4
- package/dist/cli.js +9 -9
- package/dist/{commands-11d7nc0g.js → commands-3fh75nzv.js} +2 -2
- package/dist/{db-0r0wz8a4.js → db-kwfxkhpw.js} +2 -2
- package/dist/{handlers-v8th05yh.js → handlers-jj8rzetb.js} +9 -9
- package/dist/{hook-9n0np62b.js → hook-1b45zzz1.js} +1 -1
- package/dist/{http-pmaqk60c.js → http-71964f7a.js} +31 -31
- package/dist/{index-9vma4xmv.js → index-464v93y6.js} +9 -9
- package/dist/{index-c9tx34ke.js → index-c35g752k.js} +6 -6
- package/dist/{index-vgvvr6yk.js → index-e01wmtj0.js} +7 -7
- package/dist/{index-35x6ptdt.js → index-zm0x6vmf.js} +8 -8
- package/dist/{keepalive-pjd1kjv7.js → keepalive-2gxqjgrx.js} +4 -4
- package/dist/{lead-czd1xgxr.js → lead-twm4zp8y.js} +17 -17
- package/dist/{maintenance-xjqt3ne7.js → maintenance-1htsnyz6.js} +4 -4
- package/dist/{onboard-whf6w0mr.js → onboard-1y1pdzt5.js} +2 -2
- package/dist/{otel-impl-9n4p19tt.js → otel-impl-pw9tnc80.js} +1 -1
- package/dist/{pricing-refresh-wg80me14.js → pricing-refresh-tqvvrk7a.js} +4 -4
- package/dist/{seed-pricing-garwba27.js → seed-pricing-f23sehzq.js} +3 -3
- package/dist/{setup-dktff2q2.js → setup-jtw9rz9t.js} +2 -2
- package/dist/{worker-t26gwn0n.js → worker-jxrmzwb9.js} +17 -17
- package/openapi.json +1 -1
- package/package.json +2 -2
- package/src/be/db.ts +11 -11
- package/src/tasks/worker-follow-up.ts +47 -23
- package/src/tests/heartbeat-reroute-decision.test.ts +49 -0
- package/src/tests/task-supersede-resume.test.ts +83 -6
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import {
|
|
2
2
|
runAgent
|
|
3
|
-
} from "./cli-
|
|
3
|
+
} from "./cli-0pzg1f8z.js";
|
|
4
4
|
import"./cli-w6zp4qcx.js";
|
|
5
5
|
import"./cli-m3enfr8q.js";
|
|
6
6
|
import"./cli-amncg1dq.js";
|
|
@@ -15,7 +15,7 @@ import"./cli-d6n0thxw.js";
|
|
|
15
15
|
import"./cli-hx1dcevq.js";
|
|
16
16
|
import {
|
|
17
17
|
getEnabledCapabilities
|
|
18
|
-
} from "./cli-
|
|
18
|
+
} from "./cli-kna0r84e.js";
|
|
19
19
|
import"./cli-1z7048dy.js";
|
|
20
20
|
import"./cli-dcfyyh3t.js";
|
|
21
21
|
import"./cli-gz9sbyk3.js";
|
|
@@ -25,32 +25,32 @@ import"./cli-tng3131n.js";
|
|
|
25
25
|
import"./cli-zm7njwzd.js";
|
|
26
26
|
import"./cli-tghre7g6.js";
|
|
27
27
|
import"./cli-tkmm5nf9.js";
|
|
28
|
-
import"./cli-
|
|
29
|
-
import"./cli-
|
|
30
|
-
import"./cli-
|
|
31
|
-
import"./cli-
|
|
28
|
+
import"./cli-6ttdv68b.js";
|
|
29
|
+
import"./cli-sjm0e1pa.js";
|
|
30
|
+
import"./cli-3baz70yp.js";
|
|
31
|
+
import"./cli-k6ssh5dk.js";
|
|
32
32
|
import"./cli-fad8m16k.js";
|
|
33
|
-
import"./cli-
|
|
34
|
-
import"./cli-
|
|
33
|
+
import"./cli-zw3cmjh6.js";
|
|
34
|
+
import"./cli-3t97r3y1.js";
|
|
35
35
|
import"./cli-f14fvzag.js";
|
|
36
|
-
import"./cli-
|
|
37
|
-
import"./cli-
|
|
38
|
-
import"./cli-
|
|
36
|
+
import"./cli-2h86bsfe.js";
|
|
37
|
+
import"./cli-dy01kwxk.js";
|
|
38
|
+
import"./cli-5kav4x1a.js";
|
|
39
39
|
import"./cli-q25ejbyn.js";
|
|
40
|
-
import"./cli-
|
|
41
|
-
import"./cli-
|
|
40
|
+
import"./cli-g44ffxk5.js";
|
|
41
|
+
import"./cli-4trfcacj.js";
|
|
42
42
|
import"./cli-rkndnn90.js";
|
|
43
43
|
import"./cli-b0p7rfnd.js";
|
|
44
|
-
import"./cli-
|
|
44
|
+
import"./cli-gb7dqnna.js";
|
|
45
45
|
import"./cli-6xd0bvf8.js";
|
|
46
46
|
import"./cli-wspgs9bt.js";
|
|
47
|
-
import"./cli-
|
|
47
|
+
import"./cli-8ps5wsk9.js";
|
|
48
48
|
import"./cli-dh55d5fg.js";
|
|
49
|
-
import"./cli-
|
|
49
|
+
import"./cli-d8vmtyyr.js";
|
|
50
50
|
import"./cli-z2zcxes1.js";
|
|
51
51
|
import"./cli-q21d49ac.js";
|
|
52
52
|
import"./cli-anrj584m.js";
|
|
53
|
-
import"./cli-
|
|
53
|
+
import"./cli-kg9v8h1w.js";
|
|
54
54
|
import"./cli-4j2mrzag.js";
|
|
55
55
|
import"./cli-e15jhafb.js";
|
|
56
56
|
import"./cli-9aryf9tv.js";
|
|
@@ -1,13 +1,13 @@
|
|
|
1
1
|
import {
|
|
2
2
|
reembedAllScripts
|
|
3
|
-
} from "./cli-
|
|
4
|
-
import"./cli-
|
|
3
|
+
} from "./cli-dy01kwxk.js";
|
|
4
|
+
import"./cli-5kav4x1a.js";
|
|
5
5
|
import"./cli-q25ejbyn.js";
|
|
6
|
-
import"./cli-
|
|
6
|
+
import"./cli-d8vmtyyr.js";
|
|
7
7
|
import"./cli-z2zcxes1.js";
|
|
8
8
|
import"./cli-q21d49ac.js";
|
|
9
9
|
import"./cli-anrj584m.js";
|
|
10
|
-
import"./cli-
|
|
10
|
+
import"./cli-kg9v8h1w.js";
|
|
11
11
|
import"./cli-4j2mrzag.js";
|
|
12
12
|
import"./cli-p9swy5t3.js";
|
|
13
13
|
|
|
@@ -3,7 +3,7 @@ import {
|
|
|
3
3
|
createDefaultMcpJson,
|
|
4
4
|
createDefaultSettingsLocal,
|
|
5
5
|
createHooksConfig
|
|
6
|
-
} from "./cli-
|
|
6
|
+
} from "./cli-vfyhrk9k.js";
|
|
7
7
|
import {
|
|
8
8
|
require_jsx_dev_runtime
|
|
9
9
|
} from "./cli-wx82tw1k.js";
|
|
@@ -13,7 +13,7 @@ import {
|
|
|
13
13
|
import {
|
|
14
14
|
init_package,
|
|
15
15
|
package_default
|
|
16
|
-
} from "./cli-
|
|
16
|
+
} from "./cli-kg9v8h1w.js";
|
|
17
17
|
import {
|
|
18
18
|
Box_default,
|
|
19
19
|
Select,
|
|
@@ -2,13 +2,13 @@ import {
|
|
|
2
2
|
PRICING_REFRESH_INTERVAL_MS,
|
|
3
3
|
refreshPricingFromModelsDev,
|
|
4
4
|
startPricingRefreshLoop
|
|
5
|
-
} from "./cli-
|
|
6
|
-
import"./cli-
|
|
7
|
-
import"./cli-
|
|
5
|
+
} from "./cli-sjm0e1pa.js";
|
|
6
|
+
import"./cli-3baz70yp.js";
|
|
7
|
+
import"./cli-d8vmtyyr.js";
|
|
8
8
|
import"./cli-z2zcxes1.js";
|
|
9
9
|
import"./cli-q21d49ac.js";
|
|
10
10
|
import"./cli-anrj584m.js";
|
|
11
|
-
import"./cli-
|
|
11
|
+
import"./cli-kg9v8h1w.js";
|
|
12
12
|
import"./cli-4j2mrzag.js";
|
|
13
13
|
import"./cli-p9swy5t3.js";
|
|
14
14
|
export {
|
|
@@ -1,12 +1,12 @@
|
|
|
1
1
|
import {
|
|
2
2
|
buildModelsDevSeedRows,
|
|
3
3
|
seedPricingFromModelsDev
|
|
4
|
-
} from "./cli-
|
|
5
|
-
import"./cli-
|
|
4
|
+
} from "./cli-3baz70yp.js";
|
|
5
|
+
import"./cli-d8vmtyyr.js";
|
|
6
6
|
import"./cli-z2zcxes1.js";
|
|
7
7
|
import"./cli-q21d49ac.js";
|
|
8
8
|
import"./cli-anrj584m.js";
|
|
9
|
-
import"./cli-
|
|
9
|
+
import"./cli-kg9v8h1w.js";
|
|
10
10
|
import"./cli-4j2mrzag.js";
|
|
11
11
|
import"./cli-p9swy5t3.js";
|
|
12
12
|
export {
|
|
@@ -4,14 +4,14 @@ import {
|
|
|
4
4
|
createDefaultMcpJson,
|
|
5
5
|
createDefaultSettingsLocal,
|
|
6
6
|
createHooksConfig
|
|
7
|
-
} from "./cli-
|
|
7
|
+
} from "./cli-vfyhrk9k.js";
|
|
8
8
|
import {
|
|
9
9
|
require_jsx_dev_runtime
|
|
10
10
|
} from "./cli-wx82tw1k.js";
|
|
11
11
|
import {
|
|
12
12
|
getApiKey
|
|
13
13
|
} from "./cli-f14fvzag.js";
|
|
14
|
-
import"./cli-
|
|
14
|
+
import"./cli-kg9v8h1w.js";
|
|
15
15
|
import {
|
|
16
16
|
Box_default,
|
|
17
17
|
Spinner,
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import {
|
|
2
2
|
runAgent
|
|
3
|
-
} from "./cli-
|
|
3
|
+
} from "./cli-0pzg1f8z.js";
|
|
4
4
|
import"./cli-w6zp4qcx.js";
|
|
5
5
|
import"./cli-m3enfr8q.js";
|
|
6
6
|
import"./cli-amncg1dq.js";
|
|
@@ -15,7 +15,7 @@ import"./cli-d6n0thxw.js";
|
|
|
15
15
|
import"./cli-hx1dcevq.js";
|
|
16
16
|
import {
|
|
17
17
|
getEnabledCapabilities
|
|
18
|
-
} from "./cli-
|
|
18
|
+
} from "./cli-kna0r84e.js";
|
|
19
19
|
import"./cli-1z7048dy.js";
|
|
20
20
|
import"./cli-dcfyyh3t.js";
|
|
21
21
|
import"./cli-gz9sbyk3.js";
|
|
@@ -25,32 +25,32 @@ import"./cli-tng3131n.js";
|
|
|
25
25
|
import"./cli-zm7njwzd.js";
|
|
26
26
|
import"./cli-tghre7g6.js";
|
|
27
27
|
import"./cli-tkmm5nf9.js";
|
|
28
|
-
import"./cli-
|
|
29
|
-
import"./cli-
|
|
30
|
-
import"./cli-
|
|
31
|
-
import"./cli-
|
|
28
|
+
import"./cli-6ttdv68b.js";
|
|
29
|
+
import"./cli-sjm0e1pa.js";
|
|
30
|
+
import"./cli-3baz70yp.js";
|
|
31
|
+
import"./cli-k6ssh5dk.js";
|
|
32
32
|
import"./cli-fad8m16k.js";
|
|
33
|
-
import"./cli-
|
|
34
|
-
import"./cli-
|
|
33
|
+
import"./cli-zw3cmjh6.js";
|
|
34
|
+
import"./cli-3t97r3y1.js";
|
|
35
35
|
import"./cli-f14fvzag.js";
|
|
36
|
-
import"./cli-
|
|
37
|
-
import"./cli-
|
|
38
|
-
import"./cli-
|
|
36
|
+
import"./cli-2h86bsfe.js";
|
|
37
|
+
import"./cli-dy01kwxk.js";
|
|
38
|
+
import"./cli-5kav4x1a.js";
|
|
39
39
|
import"./cli-q25ejbyn.js";
|
|
40
|
-
import"./cli-
|
|
41
|
-
import"./cli-
|
|
40
|
+
import"./cli-g44ffxk5.js";
|
|
41
|
+
import"./cli-4trfcacj.js";
|
|
42
42
|
import"./cli-rkndnn90.js";
|
|
43
43
|
import"./cli-b0p7rfnd.js";
|
|
44
|
-
import"./cli-
|
|
44
|
+
import"./cli-gb7dqnna.js";
|
|
45
45
|
import"./cli-6xd0bvf8.js";
|
|
46
46
|
import"./cli-wspgs9bt.js";
|
|
47
|
-
import"./cli-
|
|
47
|
+
import"./cli-8ps5wsk9.js";
|
|
48
48
|
import"./cli-dh55d5fg.js";
|
|
49
|
-
import"./cli-
|
|
49
|
+
import"./cli-d8vmtyyr.js";
|
|
50
50
|
import"./cli-z2zcxes1.js";
|
|
51
51
|
import"./cli-q21d49ac.js";
|
|
52
52
|
import"./cli-anrj584m.js";
|
|
53
|
-
import"./cli-
|
|
53
|
+
import"./cli-kg9v8h1w.js";
|
|
54
54
|
import"./cli-4j2mrzag.js";
|
|
55
55
|
import"./cli-e15jhafb.js";
|
|
56
56
|
import"./cli-9aryf9tv.js";
|
package/openapi.json
CHANGED
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
"openapi": "3.1.0",
|
|
3
3
|
"info": {
|
|
4
4
|
"title": "Agent Swarm API",
|
|
5
|
-
"version": "1.
|
|
5
|
+
"version": "1.110.0",
|
|
6
6
|
"description": "Multi-agent orchestration API for Claude Code, Codex, and Gemini CLI. Enables task distribution, agent communication, and service discovery.\n\nMCP tools are documented separately in [MCP.md](./MCP.md)."
|
|
7
7
|
},
|
|
8
8
|
"servers": [
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@desplega.ai/agent-swarm",
|
|
3
|
-
"version": "1.
|
|
3
|
+
"version": "1.110.0",
|
|
4
4
|
"description": "Multi-agent orchestration for Claude Code, Codex, Gemini CLI, and other AI coding assistants",
|
|
5
5
|
"license": "MIT",
|
|
6
6
|
"author": "desplega.sh <contact@desplega.sh>",
|
|
@@ -146,7 +146,7 @@
|
|
|
146
146
|
"@linear/sdk": "^77.0.0",
|
|
147
147
|
"@modelcontextprotocol/sdk": "^1.25.1",
|
|
148
148
|
"@openai/codex-sdk": "^0.142.5",
|
|
149
|
-
"@opencode-ai/sdk": "^1.17.
|
|
149
|
+
"@opencode-ai/sdk": "^1.17.13",
|
|
150
150
|
"@openfort/openfort-node": "^0.9.1",
|
|
151
151
|
"@opentelemetry/api": "^1.9.1",
|
|
152
152
|
"@opentelemetry/exporter-metrics-otlp-http": "^0.218.0",
|
package/src/be/db.ts
CHANGED
|
@@ -6923,18 +6923,18 @@ export function getStalledInProgressTasks(thresholdMinutes: number = 30): AgentT
|
|
|
6923
6923
|
}
|
|
6924
6924
|
|
|
6925
6925
|
/**
|
|
6926
|
-
* Genuine same-agent
|
|
6927
|
-
*
|
|
6928
|
-
* heartbeat reaper escalates these to a Lead reroute-decision.
|
|
6926
|
+
* Genuine same-agent protected resume PINS (tagged `crash-recovery-pin` or
|
|
6927
|
+
* `graceful-shutdown-pin`) that are still `pending` `graceMin` minutes after
|
|
6928
|
+
* creation — the heartbeat reaper escalates these to a Lead reroute-decision.
|
|
6929
6929
|
*
|
|
6930
6930
|
* Three scoping clauses, each load-bearing:
|
|
6931
|
-
* -
|
|
6932
|
-
*
|
|
6933
|
-
*
|
|
6934
|
-
*
|
|
6935
|
-
*
|
|
6936
|
-
*
|
|
6937
|
-
*
|
|
6931
|
+
* - pin tags — restricts to resumes actually pinned to their original agent on
|
|
6932
|
+
* protected paths. Without this, a *pooled* resume that `autoAssignPoolTasks`
|
|
6933
|
+
* flips to `pending` earlier in the SAME sweep (keeping its old `createdAt`)
|
|
6934
|
+
* would be reaped and cancelled before the assigned worker polls; it also
|
|
6935
|
+
* keeps `context_limits` / `manual_supersede` pins from being escalated under
|
|
6936
|
+
* the protected-pin label. (Literals must match the pin tag constants in
|
|
6937
|
+
* src/tasks/worker-follow-up.ts.)
|
|
6938
6938
|
* - `status = 'pending'` — the "currently unreclaimed" discriminator: when the
|
|
6939
6939
|
* agent reclaims via the normal poll path, `startTask` flips the row to
|
|
6940
6940
|
* `in_progress` and it drops out of this set. (A reclaimed resume whose
|
|
@@ -6954,7 +6954,7 @@ export function getStalePinnedResumes(graceMin: number): AgentTask[] {
|
|
|
6954
6954
|
.prepare<AgentTaskRow, [string]>(
|
|
6955
6955
|
`SELECT * FROM agent_tasks
|
|
6956
6956
|
WHERE taskType = 'resume' AND status = 'pending'
|
|
6957
|
-
AND tags LIKE '%"crash-recovery-pin"%'
|
|
6957
|
+
AND (tags LIKE '%"crash-recovery-pin"%' OR tags LIKE '%"graceful-shutdown-pin"%')
|
|
6958
6958
|
AND createdAt < ?
|
|
6959
6959
|
ORDER BY createdAt ASC`,
|
|
6960
6960
|
)
|
|
@@ -36,6 +36,16 @@ export const WORKER_LIVENESS_WINDOW_SECONDS = Number(
|
|
|
36
36
|
*/
|
|
37
37
|
export const HEARTBEAT_PIN_CRASH_RESUME = process.env.HEARTBEAT_PIN_CRASH_RESUME !== "0";
|
|
38
38
|
|
|
39
|
+
/**
|
|
40
|
+
* Rollback switch for pinning `graceful_shutdown` resumes to their original
|
|
41
|
+
* agent. ON by default: graceful shutdowns happen during deploy/SIGTERM, where
|
|
42
|
+
* the agent ID is stable and the worker normally polls again minutes later.
|
|
43
|
+
* Set `HEARTBEAT_PIN_GRACEFUL_RESUME=0` to restore the old unassigned-pool path.
|
|
44
|
+
*/
|
|
45
|
+
export function isGracefulResumePinEnabled(): boolean {
|
|
46
|
+
return process.env.HEARTBEAT_PIN_GRACEFUL_RESUME !== "0";
|
|
47
|
+
}
|
|
48
|
+
|
|
39
49
|
export const RESUME_GENERATION_TAG_PREFIX = "resume-generation:";
|
|
40
50
|
|
|
41
51
|
/**
|
|
@@ -52,6 +62,13 @@ export const RESUME_GENERATION_TAG_PREFIX = "resume-generation:";
|
|
|
52
62
|
*/
|
|
53
63
|
export const CRASH_RECOVERY_PIN_TAG = "crash-recovery-pin";
|
|
54
64
|
|
|
65
|
+
/**
|
|
66
|
+
* Tag set ONLY on a genuine same-agent `graceful_shutdown` pin. The stale-pin
|
|
67
|
+
* reaper treats it like `CRASH_RECOVERY_PIN_TAG` while preserving which resume
|
|
68
|
+
* path produced the pin. Keep the literal in sync with `getStalePinnedResumes`.
|
|
69
|
+
*/
|
|
70
|
+
export const GRACEFUL_SHUTDOWN_PIN_TAG = "graceful-shutdown-pin";
|
|
71
|
+
|
|
55
72
|
export function getResumeGeneration(task: Pick<AgentTask, "tags">): number {
|
|
56
73
|
const tag = task.tags.find((value) => value.startsWith(RESUME_GENERATION_TAG_PREFIX));
|
|
57
74
|
if (!tag) return 0;
|
|
@@ -189,20 +206,23 @@ export type CreateResumeFollowUpResult =
|
|
|
189
206
|
* (`getActiveTaskCount < agent.maxTasks`). For `crash_recovery` the pin holds
|
|
190
207
|
* regardless of `lastActivityAt` freshness — the agent ID is stable across a
|
|
191
208
|
* restart and the crashed row survives intact, so a stale `lastActivityAt` at
|
|
192
|
-
* the ~5-min crash-detection mark means "restarting", not "gone".
|
|
193
|
-
* the
|
|
194
|
-
*
|
|
195
|
-
*
|
|
196
|
-
*
|
|
197
|
-
* (
|
|
209
|
+
* the ~5-min crash-detection mark means "restarting", not "gone". `graceful_shutdown`
|
|
210
|
+
* follows the same pin path when `HEARTBEAT_PIN_GRACEFUL_RESUME` is enabled:
|
|
211
|
+
* deploy/SIGTERM restarts preserve the stable agent ID and the stale-pin reaper
|
|
212
|
+
* now handles the "agent never returns" case. Pinning keeps the resume off the
|
|
213
|
+
* role-blind unassigned pool so no wrong-specialization worker can grab it
|
|
214
|
+
* (DES-523). For `context_limits` / `manual_supersede` the worker is alive, so
|
|
215
|
+
* `lastActivityAt` freshness is still required. The resume falls back to the
|
|
216
|
+
* unassigned pool only when the agent is genuinely gone (graceful close →
|
|
217
|
+
* `offline`) or its row is absent.
|
|
198
218
|
*
|
|
199
219
|
* Gone-agent / never-reclaimed case: a pin whose agent never returns is NOT
|
|
200
220
|
* re-pooled — the heartbeat's stale-resume reaper (`escalateUnreclaimedResumes`
|
|
201
221
|
* in `src/heartbeat/heartbeat.ts`) escalates it to a Lead re-delegation decision
|
|
202
222
|
* once `HEARTBEAT_RESUME_PIN_GRACE_MIN` lapses.
|
|
203
223
|
*
|
|
204
|
-
* The pin
|
|
205
|
-
*
|
|
224
|
+
* The crash pin is gated by `HEARTBEAT_PIN_CRASH_RESUME`; the graceful shutdown
|
|
225
|
+
* pin is gated by `HEARTBEAT_PIN_GRACEFUL_RESUME`. Both default on.
|
|
206
226
|
*/
|
|
207
227
|
export function createResumeFollowUp(args: {
|
|
208
228
|
parentId: string;
|
|
@@ -218,13 +238,6 @@ export function createResumeFollowUp(args: {
|
|
|
218
238
|
|
|
219
239
|
// Routing decision — same DB process so the read-then-create window is small.
|
|
220
240
|
//
|
|
221
|
-
// For `graceful_shutdown`, force the unassigned-pool path: the parent worker
|
|
222
|
-
// is exiting and will call `closeAgent` (→ offline) moments after the
|
|
223
|
-
// supersede loop. At this check it still looks fresh + has capacity (it just
|
|
224
|
-
// terminal-transitioned), so the liveness branch would pin the resume to a
|
|
225
|
-
// dying worker — orphaning it in `pending` once the worker closes. Pool
|
|
226
|
-
// routing lets any live worker claim it.
|
|
227
|
-
//
|
|
228
241
|
// For `crash_recovery`, deliberately PIN to the same (stable-ID) agent even
|
|
229
242
|
// when `lastActivityAt` is stale. This REVERSES the prior "let staleness pool
|
|
230
243
|
// it" behavior: crash detection only fires after STALL_THRESHOLD_NO_SESSION_MIN
|
|
@@ -237,6 +250,11 @@ export function createResumeFollowUp(args: {
|
|
|
237
250
|
// guard. An unreclaimed pin is escalated to a Lead decision by the heartbeat
|
|
238
251
|
// reaper, never silently re-pooled.
|
|
239
252
|
//
|
|
253
|
+
// `graceful_shutdown` uses the same pin machinery: deploy/SIGTERM restarts
|
|
254
|
+
// preserve the stable agent ID, and the reaper now prevents an unreclaimed pin
|
|
255
|
+
// from being orphaned forever. `HEARTBEAT_PIN_GRACEFUL_RESUME=0` restores the
|
|
256
|
+
// old forced-pool path for this reason.
|
|
257
|
+
//
|
|
240
258
|
// Brittleness note: this relies on a hard crash NEVER marking the agent
|
|
241
259
|
// `offline` (only `POST /close` does). If future code offlines stale agents
|
|
242
260
|
// before remediation, this re-opens the pool path for `crash_recovery` —
|
|
@@ -245,7 +263,7 @@ export function createResumeFollowUp(args: {
|
|
|
245
263
|
// - `context_limits` / `manual_supersede`: the worker is alive and
|
|
246
264
|
// responsive, so keep requiring `fresh`.
|
|
247
265
|
let preferredAgentId: string | undefined;
|
|
248
|
-
if (parent.agentId
|
|
266
|
+
if (parent.agentId) {
|
|
249
267
|
const candidate = getAgentById(parent.agentId);
|
|
250
268
|
if (candidate && candidate.status !== "offline") {
|
|
251
269
|
const lastActivity = candidate.lastActivityAt ? Date.parse(candidate.lastActivityAt) : 0;
|
|
@@ -255,15 +273,18 @@ export function createResumeFollowUp(args: {
|
|
|
255
273
|
const activeCount = getActiveTaskCount(candidate.id);
|
|
256
274
|
const hasCap = activeCount < (candidate.maxTasks ?? 1);
|
|
257
275
|
const isCrashRecovery = args.reason === "crash_recovery" && HEARTBEAT_PIN_CRASH_RESUME;
|
|
258
|
-
|
|
259
|
-
|
|
260
|
-
|
|
276
|
+
const isGracefulShutdown = args.reason === "graceful_shutdown";
|
|
277
|
+
const isGracefulPin = isGracefulShutdown && isGracefulResumePinEnabled();
|
|
278
|
+
const isFreshPinnedReason = !isGracefulShutdown && fresh;
|
|
279
|
+
// crash_recovery and graceful_shutdown pins ignore `fresh` when their
|
|
280
|
+
// kill-switches are on; context_limits/manual_supersede still require it.
|
|
281
|
+
if (hasCap && (isCrashRecovery || isGracefulPin || isFreshPinnedReason)) {
|
|
261
282
|
preferredAgentId = candidate.id;
|
|
262
|
-
} else if (isCrashRecovery && !hasCap) {
|
|
263
|
-
//
|
|
264
|
-
//
|
|
283
|
+
} else if ((isCrashRecovery || isGracefulPin) && !hasCap) {
|
|
284
|
+
// Surface capacity-driven pool fallback instead of letting a protected
|
|
285
|
+
// pinned-reason skip happen silently.
|
|
265
286
|
console.warn(
|
|
266
|
-
`[Heartbeat]
|
|
287
|
+
`[Heartbeat] ${args.reason} resume for task ${parent.id.slice(0, 8)} NOT pinned: agent ${candidate.id.slice(0, 8)} at capacity (${activeCount}/${candidate.maxTasks ?? 1}); falling back to unassigned pool`,
|
|
267
288
|
);
|
|
268
289
|
}
|
|
269
290
|
}
|
|
@@ -296,6 +317,9 @@ export function createResumeFollowUp(args: {
|
|
|
296
317
|
if (args.reason === "crash_recovery" && preferredAgentId !== undefined) {
|
|
297
318
|
tags.push(CRASH_RECOVERY_PIN_TAG);
|
|
298
319
|
}
|
|
320
|
+
if (args.reason === "graceful_shutdown" && preferredAgentId !== undefined) {
|
|
321
|
+
tags.push(GRACEFUL_SHUTDOWN_PIN_TAG);
|
|
322
|
+
}
|
|
299
323
|
|
|
300
324
|
// Identity-shaped fields (dir, VCS provider/repo/number/url/etc.,
|
|
301
325
|
// outputSchema, slack channel/thread/user, agentmail, mention, contextKey,
|
|
@@ -38,6 +38,7 @@ import {
|
|
|
38
38
|
CRASH_RECOVERY_PIN_TAG,
|
|
39
39
|
createRerouteDecisionTask,
|
|
40
40
|
createResumeFollowUp,
|
|
41
|
+
GRACEFUL_SHUTDOWN_PIN_TAG,
|
|
41
42
|
RESUME_GENERATION_TAG_PREFIX,
|
|
42
43
|
} from "../tasks/worker-follow-up";
|
|
43
44
|
import { registerSendTaskTool } from "../tools/send-task";
|
|
@@ -71,6 +72,32 @@ function seedPinnedCrash(agentName: string) {
|
|
|
71
72
|
return { agent, original: getTaskById(original.id)!, r1 };
|
|
72
73
|
}
|
|
73
74
|
|
|
75
|
+
/**
|
|
76
|
+
* Build the graceful-shutdown equivalent: a superseded original + a pending
|
|
77
|
+
* resume R1 (generation 1) pinned to the original agent with its own provenance
|
|
78
|
+
* tag.
|
|
79
|
+
*/
|
|
80
|
+
function seedPinnedGracefulShutdown(agentName: string) {
|
|
81
|
+
const agent = createAgent({ name: agentName, isLead: false, status: "idle" });
|
|
82
|
+
const original = createTaskExtended("Gracefully stopped worker's original work", {
|
|
83
|
+
agentId: agent.id,
|
|
84
|
+
});
|
|
85
|
+
startTask(original.id);
|
|
86
|
+
const r1 = createTaskExtended("Resume of gracefully stopped work", {
|
|
87
|
+
agentId: agent.id,
|
|
88
|
+
parentTaskId: original.id,
|
|
89
|
+
taskType: "resume",
|
|
90
|
+
tags: [
|
|
91
|
+
"auto-resume",
|
|
92
|
+
"reason:graceful_shutdown",
|
|
93
|
+
`${RESUME_GENERATION_TAG_PREFIX}1`,
|
|
94
|
+
GRACEFUL_SHUTDOWN_PIN_TAG,
|
|
95
|
+
],
|
|
96
|
+
});
|
|
97
|
+
supersedeTask(original.id, { reason: "shutdown", resumeTaskId: r1.id });
|
|
98
|
+
return { agent, original: getTaskById(original.id)!, r1 };
|
|
99
|
+
}
|
|
100
|
+
|
|
74
101
|
type RegisteredTool = {
|
|
75
102
|
handler: (args: unknown, extra: unknown) => Promise<CallToolResult>;
|
|
76
103
|
};
|
|
@@ -261,6 +288,28 @@ describe("Heartbeat — reroute-decision fallback (DES-523)", () => {
|
|
|
261
288
|
);
|
|
262
289
|
});
|
|
263
290
|
|
|
291
|
+
test("stale graceful_shutdown pin older than grace → escalated to a Lead decision", async () => {
|
|
292
|
+
const lead = createAgent({ name: "lead", isLead: true, status: "busy" });
|
|
293
|
+
const { original, r1 } = seedPinnedGracefulShutdown("coder-graceful");
|
|
294
|
+
expect(r1.tags).toContain(GRACEFUL_SHUTDOWN_PIN_TAG);
|
|
295
|
+
ageCreatedAtPastGrace(r1.id);
|
|
296
|
+
|
|
297
|
+
const findings = await codeLevelTriage();
|
|
298
|
+
|
|
299
|
+
expect(findings.escalatedReroutes.length).toBe(1);
|
|
300
|
+
expect(findings.escalatedReroutes[0]!.originalTaskId).toBe(original.id);
|
|
301
|
+
expect(getTaskById(r1.id)!.status).not.toBe("pending");
|
|
302
|
+
const decisions = getChildTasks(original.id).filter((c) => c.taskType === "reroute-decision");
|
|
303
|
+
expect(decisions.length).toBe(1);
|
|
304
|
+
expect(decisions[0]!.agentId).toBe(lead.id);
|
|
305
|
+
|
|
306
|
+
const second = await codeLevelTriage();
|
|
307
|
+
expect(second.escalatedReroutes.length).toBe(0);
|
|
308
|
+
expect(getChildTasks(original.id).filter((c) => c.taskType === "reroute-decision").length).toBe(
|
|
309
|
+
1,
|
|
310
|
+
);
|
|
311
|
+
});
|
|
312
|
+
|
|
264
313
|
test("pinned resume reclaimed (in_progress) before the grace window is NOT escalated", async () => {
|
|
265
314
|
createAgent({ name: "lead", isLead: true, status: "busy" });
|
|
266
315
|
const { original, r1 } = seedPinnedCrash("coder-reclaim");
|
|
@@ -21,7 +21,11 @@ import {
|
|
|
21
21
|
getTrackerSyncByExternalId,
|
|
22
22
|
} from "../be/db-queries/tracker";
|
|
23
23
|
import { buildResumeContextPreamble } from "../commands/context-preamble";
|
|
24
|
-
import {
|
|
24
|
+
import {
|
|
25
|
+
CRASH_RECOVERY_PIN_TAG,
|
|
26
|
+
createResumeFollowUp,
|
|
27
|
+
GRACEFUL_SHUTDOWN_PIN_TAG,
|
|
28
|
+
} from "../tasks/worker-follow-up";
|
|
25
29
|
|
|
26
30
|
const TEST_DB_PATH = "./test-task-supersede-resume.sqlite";
|
|
27
31
|
|
|
@@ -398,16 +402,88 @@ describe("Task Supersede + Resume", () => {
|
|
|
398
402
|
expect(afterCount).toBe(beforeCount);
|
|
399
403
|
});
|
|
400
404
|
|
|
401
|
-
test("routing: graceful_shutdown
|
|
402
|
-
// The worker is exiting moments after this check — keeping the resume on
|
|
403
|
-
// the same agent would orphan it once `closeAgent` runs. graceful_shutdown
|
|
404
|
-
// must force the unassigned-pool path regardless of liveness.
|
|
405
|
+
test("routing: graceful_shutdown pins to original online agent with capacity and tags the pin", () => {
|
|
405
406
|
const worker = freshAgent("worker-fresh-shutdown", {
|
|
406
407
|
maxTasks: 5,
|
|
407
|
-
|
|
408
|
+
// Stale activity proves graceful_shutdown skips the fresh-agent gate,
|
|
409
|
+
// matching crash_recovery's stable-agent-ID routing.
|
|
410
|
+
lastActivityAt: new Date(Date.now() - 5 * 60 * 1000).toISOString(),
|
|
408
411
|
});
|
|
409
412
|
const parent = createTaskExtended("Routing graceful_shutdown", { agentId: worker.id });
|
|
410
413
|
startTask(parent.id);
|
|
414
|
+
supersedeTask(parent.id, { reason: "graceful_shutdown", resumeTaskId: null });
|
|
415
|
+
|
|
416
|
+
const result = createResumeFollowUp({
|
|
417
|
+
parentId: parent.id,
|
|
418
|
+
reason: "graceful_shutdown",
|
|
419
|
+
});
|
|
420
|
+
if (result.kind !== "created") throw new Error("expected created");
|
|
421
|
+
expect(result.task.agentId).toBe(worker.id);
|
|
422
|
+
expect(result.task.status).toBe("pending");
|
|
423
|
+
expect(result.task.tags).toContain(GRACEFUL_SHUTDOWN_PIN_TAG);
|
|
424
|
+
expect(result.task.tags).not.toContain(CRASH_RECOVERY_PIN_TAG);
|
|
425
|
+
});
|
|
426
|
+
|
|
427
|
+
test("routing: HEARTBEAT_PIN_GRACEFUL_RESUME=0 restores graceful_shutdown pool behavior", () => {
|
|
428
|
+
const previous = process.env.HEARTBEAT_PIN_GRACEFUL_RESUME;
|
|
429
|
+
process.env.HEARTBEAT_PIN_GRACEFUL_RESUME = "0";
|
|
430
|
+
try {
|
|
431
|
+
const worker = freshAgent("worker-shutdown-killswitch", {
|
|
432
|
+
maxTasks: 5,
|
|
433
|
+
lastActivityAt: new Date().toISOString(),
|
|
434
|
+
});
|
|
435
|
+
const parent = createTaskExtended("Routing graceful_shutdown kill-switch", {
|
|
436
|
+
agentId: worker.id,
|
|
437
|
+
});
|
|
438
|
+
startTask(parent.id);
|
|
439
|
+
supersedeTask(parent.id, { reason: "graceful_shutdown", resumeTaskId: null });
|
|
440
|
+
|
|
441
|
+
const result = createResumeFollowUp({
|
|
442
|
+
parentId: parent.id,
|
|
443
|
+
reason: "graceful_shutdown",
|
|
444
|
+
});
|
|
445
|
+
if (result.kind !== "created") throw new Error("expected created");
|
|
446
|
+
expect(result.task.agentId).toBeNull();
|
|
447
|
+
expect(result.task.status).toBe("unassigned");
|
|
448
|
+
expect(result.task.tags).not.toContain(GRACEFUL_SHUTDOWN_PIN_TAG);
|
|
449
|
+
} finally {
|
|
450
|
+
if (previous === undefined) {
|
|
451
|
+
delete process.env.HEARTBEAT_PIN_GRACEFUL_RESUME;
|
|
452
|
+
} else {
|
|
453
|
+
process.env.HEARTBEAT_PIN_GRACEFUL_RESUME = previous;
|
|
454
|
+
}
|
|
455
|
+
}
|
|
456
|
+
});
|
|
457
|
+
|
|
458
|
+
test("routing: graceful_shutdown at capacity → unassigned pool fail-open", () => {
|
|
459
|
+
const worker = freshAgent("worker-full-shutdown", {
|
|
460
|
+
maxTasks: 1,
|
|
461
|
+
lastActivityAt: new Date().toISOString(),
|
|
462
|
+
});
|
|
463
|
+
// Parent is still in_progress and fills the worker's single slot.
|
|
464
|
+
const parent = createTaskExtended("Routing graceful_shutdown capped", { agentId: worker.id });
|
|
465
|
+
startTask(parent.id);
|
|
466
|
+
|
|
467
|
+
const result = createResumeFollowUp({
|
|
468
|
+
parentId: parent.id,
|
|
469
|
+
reason: "graceful_shutdown",
|
|
470
|
+
});
|
|
471
|
+
if (result.kind !== "created") throw new Error("expected created");
|
|
472
|
+
expect(result.task.agentId).toBeNull();
|
|
473
|
+
expect(result.task.status).toBe("unassigned");
|
|
474
|
+
expect(result.task.tags).not.toContain(GRACEFUL_SHUTDOWN_PIN_TAG);
|
|
475
|
+
});
|
|
476
|
+
|
|
477
|
+
test("routing: graceful_shutdown offline worker → unassigned pool fail-open", () => {
|
|
478
|
+
const worker = freshAgent("worker-offline-shutdown", {
|
|
479
|
+
maxTasks: 5,
|
|
480
|
+
lastActivityAt: new Date().toISOString(),
|
|
481
|
+
});
|
|
482
|
+
updateAgentStatus(worker.id, "offline");
|
|
483
|
+
const parent = createTaskExtended("Routing graceful_shutdown offline", {
|
|
484
|
+
agentId: worker.id,
|
|
485
|
+
});
|
|
486
|
+
startTask(parent.id);
|
|
411
487
|
|
|
412
488
|
const result = createResumeFollowUp({
|
|
413
489
|
parentId: parent.id,
|
|
@@ -416,6 +492,7 @@ describe("Task Supersede + Resume", () => {
|
|
|
416
492
|
if (result.kind !== "created") throw new Error("expected created");
|
|
417
493
|
expect(result.task.agentId).toBeNull();
|
|
418
494
|
expect(result.task.status).toBe("unassigned");
|
|
495
|
+
expect(result.task.tags).not.toContain(GRACEFUL_SHUTDOWN_PIN_TAG);
|
|
419
496
|
});
|
|
420
497
|
|
|
421
498
|
test("routing: fresh worker + capacity (non-shutdown) → resume pre-assigned to same worker", () => {
|