@songsid/agend 2.1.1-beta.16 → 2.1.1-beta.17
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/channel/ipc-timeouts.d.ts +40 -0
- package/dist/channel/ipc-timeouts.js +58 -0
- package/dist/channel/ipc-timeouts.js.map +1 -0
- package/dist/channel/mcp-server.js +5 -4
- package/dist/channel/mcp-server.js.map +1 -1
- package/dist/daemon.d.ts +13 -0
- package/dist/daemon.js +32 -9
- package/dist/daemon.js.map +1 -1
- package/dist/fleet-manager.js +16 -1
- package/dist/fleet-manager.js.map +1 -1
- package/dist/instance-lifecycle.js +9 -0
- package/dist/instance-lifecycle.js.map +1 -1
- package/dist/outbound-handlers.d.ts +7 -0
- package/dist/outbound-handlers.js +62 -4
- package/dist/outbound-handlers.js.map +1 -1
- package/package.json +1 -1
|
@@ -0,0 +1,40 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* IPC timeout budgets, shared by the daemon (which owns the work) and the MCP
|
|
3
|
+
* server (which waits on it).
|
|
4
|
+
*
|
|
5
|
+
* There are two independent timers on every tool call: the MCP server arms one
|
|
6
|
+
* BEFORE sending, and the daemon arms its own ON RECEIPT. With both set to 30s the
|
|
7
|
+
* outer one always fired first, so every carefully worded daemon message —
|
|
8
|
+
* "Schedule operation timed out after 30s", "Cross-instance operation timed out
|
|
9
|
+
* after 30s", "Task operation timed out after 30s", "Fleet outbound timed out after
|
|
10
|
+
* 30s" — was unreachable, and the agent always saw the generic
|
|
11
|
+
* "IPC request timed out after 30000ms" instead.
|
|
12
|
+
*
|
|
13
|
+
* The invariant this module exists to hold: the MCP ceiling is strictly larger than
|
|
14
|
+
* the daemon's budget for the same tool, so the specific message always wins the
|
|
15
|
+
* race. Deriving both from one table is what keeps them from drifting apart again.
|
|
16
|
+
*/
|
|
17
|
+
/** Tools that spawn a tmux window and a CLI before they can answer. */
|
|
18
|
+
export declare const SLOW_TOOLS: ReadonlySet<string>;
|
|
19
|
+
/**
|
|
20
|
+
* Tools that do unbounded work: deploy_template creates instances serially, each
|
|
21
|
+
* spawning a window and a CLI; checkout_repo runs `git worktree add` against a repo
|
|
22
|
+
* of any size. Under the generic 30s budget these timed out on any real workload
|
|
23
|
+
* while the operation kept running in the background — so the agent was told it had
|
|
24
|
+
* failed, and deploy_template's rollback never triggered, leaving a half-built
|
|
25
|
+
* fleet behind.
|
|
26
|
+
*/
|
|
27
|
+
export declare const VERY_SLOW_TOOLS: ReadonlySet<string>;
|
|
28
|
+
export declare const DEFAULT_IPC_BUDGET_MS = 30000;
|
|
29
|
+
export declare const SLOW_IPC_BUDGET_MS = 60000;
|
|
30
|
+
export declare const VERY_SLOW_IPC_BUDGET_MS = 300000;
|
|
31
|
+
/** Headroom for the MCP ceiling over the daemon's budget: transport plus scheduling. */
|
|
32
|
+
export declare const MCP_TIMEOUT_MARGIN_MS = 15000;
|
|
33
|
+
/** How long the daemon gives the fleet to answer for this tool. */
|
|
34
|
+
export declare function daemonBudgetMs(tool: string): number;
|
|
35
|
+
/**
|
|
36
|
+
* How long the MCP server waits for the daemon. Always strictly greater than
|
|
37
|
+
* `daemonBudgetMs(tool)`, so the daemon's specific error message reaches the agent
|
|
38
|
+
* rather than being pre-empted by the generic client-side one.
|
|
39
|
+
*/
|
|
40
|
+
export declare function mcpTimeoutMs(tool: string): number;
|
|
@@ -0,0 +1,58 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* IPC timeout budgets, shared by the daemon (which owns the work) and the MCP
|
|
3
|
+
* server (which waits on it).
|
|
4
|
+
*
|
|
5
|
+
* There are two independent timers on every tool call: the MCP server arms one
|
|
6
|
+
* BEFORE sending, and the daemon arms its own ON RECEIPT. With both set to 30s the
|
|
7
|
+
* outer one always fired first, so every carefully worded daemon message —
|
|
8
|
+
* "Schedule operation timed out after 30s", "Cross-instance operation timed out
|
|
9
|
+
* after 30s", "Task operation timed out after 30s", "Fleet outbound timed out after
|
|
10
|
+
* 30s" — was unreachable, and the agent always saw the generic
|
|
11
|
+
* "IPC request timed out after 30000ms" instead.
|
|
12
|
+
*
|
|
13
|
+
* The invariant this module exists to hold: the MCP ceiling is strictly larger than
|
|
14
|
+
* the daemon's budget for the same tool, so the specific message always wins the
|
|
15
|
+
* race. Deriving both from one table is what keeps them from drifting apart again.
|
|
16
|
+
*/
|
|
17
|
+
/** Tools that spawn a tmux window and a CLI before they can answer. */
|
|
18
|
+
export const SLOW_TOOLS = new Set([
|
|
19
|
+
"start_instance",
|
|
20
|
+
"create_instance",
|
|
21
|
+
"delete_instance",
|
|
22
|
+
"replace_instance",
|
|
23
|
+
]);
|
|
24
|
+
/**
|
|
25
|
+
* Tools that do unbounded work: deploy_template creates instances serially, each
|
|
26
|
+
* spawning a window and a CLI; checkout_repo runs `git worktree add` against a repo
|
|
27
|
+
* of any size. Under the generic 30s budget these timed out on any real workload
|
|
28
|
+
* while the operation kept running in the background — so the agent was told it had
|
|
29
|
+
* failed, and deploy_template's rollback never triggered, leaving a half-built
|
|
30
|
+
* fleet behind.
|
|
31
|
+
*/
|
|
32
|
+
export const VERY_SLOW_TOOLS = new Set([
|
|
33
|
+
"deploy_template",
|
|
34
|
+
"teardown_deployment",
|
|
35
|
+
"checkout_repo",
|
|
36
|
+
]);
|
|
37
|
+
export const DEFAULT_IPC_BUDGET_MS = 30_000;
|
|
38
|
+
export const SLOW_IPC_BUDGET_MS = 60_000;
|
|
39
|
+
export const VERY_SLOW_IPC_BUDGET_MS = 300_000;
|
|
40
|
+
/** Headroom for the MCP ceiling over the daemon's budget: transport plus scheduling. */
|
|
41
|
+
export const MCP_TIMEOUT_MARGIN_MS = 15_000;
|
|
42
|
+
/** How long the daemon gives the fleet to answer for this tool. */
|
|
43
|
+
export function daemonBudgetMs(tool) {
|
|
44
|
+
if (VERY_SLOW_TOOLS.has(tool))
|
|
45
|
+
return VERY_SLOW_IPC_BUDGET_MS;
|
|
46
|
+
if (SLOW_TOOLS.has(tool))
|
|
47
|
+
return SLOW_IPC_BUDGET_MS;
|
|
48
|
+
return DEFAULT_IPC_BUDGET_MS;
|
|
49
|
+
}
|
|
50
|
+
/**
|
|
51
|
+
* How long the MCP server waits for the daemon. Always strictly greater than
|
|
52
|
+
* `daemonBudgetMs(tool)`, so the daemon's specific error message reaches the agent
|
|
53
|
+
* rather than being pre-empted by the generic client-side one.
|
|
54
|
+
*/
|
|
55
|
+
export function mcpTimeoutMs(tool) {
|
|
56
|
+
return daemonBudgetMs(tool) + MCP_TIMEOUT_MARGIN_MS;
|
|
57
|
+
}
|
|
58
|
+
//# sourceMappingURL=ipc-timeouts.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"ipc-timeouts.js","sourceRoot":"","sources":["../../src/channel/ipc-timeouts.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;GAeG;AAEH,uEAAuE;AACvE,MAAM,CAAC,MAAM,UAAU,GAAwB,IAAI,GAAG,CAAC;IACrD,gBAAgB;IAChB,iBAAiB;IACjB,iBAAiB;IACjB,kBAAkB;CACnB,CAAC,CAAC;AAEH;;;;;;;GAOG;AACH,MAAM,CAAC,MAAM,eAAe,GAAwB,IAAI,GAAG,CAAC;IAC1D,iBAAiB;IACjB,qBAAqB;IACrB,eAAe;CAChB,CAAC,CAAC;AAEH,MAAM,CAAC,MAAM,qBAAqB,GAAG,MAAM,CAAC;AAC5C,MAAM,CAAC,MAAM,kBAAkB,GAAG,MAAM,CAAC;AACzC,MAAM,CAAC,MAAM,uBAAuB,GAAG,OAAO,CAAC;AAE/C,wFAAwF;AACxF,MAAM,CAAC,MAAM,qBAAqB,GAAG,MAAM,CAAC;AAE5C,mEAAmE;AACnE,MAAM,UAAU,cAAc,CAAC,IAAY;IACzC,IAAI,eAAe,CAAC,GAAG,CAAC,IAAI,CAAC;QAAE,OAAO,uBAAuB,CAAC;IAC9D,IAAI,UAAU,CAAC,GAAG,CAAC,IAAI,CAAC;QAAE,OAAO,kBAAkB,CAAC;IACpD,OAAO,qBAAqB,CAAC;AAC/B,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,YAAY,CAAC,IAAY;IACvC,OAAO,cAAc,CAAC,IAAI,CAAC,GAAG,qBAAqB,CAAC;AACtD,CAAC"}
|
|
@@ -16,13 +16,14 @@ import { IpcClient } from "./ipc-bridge.js";
|
|
|
16
16
|
import { TOOLS } from "./mcp-tools.js";
|
|
17
17
|
import { buildFleetInstructions } from "../instructions.js";
|
|
18
18
|
import { reconnectDelayMs } from "./reconnect-backoff.js";
|
|
19
|
+
import { mcpTimeoutMs } from "./ipc-timeouts.js";
|
|
19
20
|
// ---------------------------------------------------------------------------
|
|
20
21
|
// Configuration
|
|
21
22
|
// ---------------------------------------------------------------------------
|
|
22
23
|
const SOCKET_PATH = process.env.AGEND_SOCKET_PATH ?? "";
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
24
|
+
// Per-tool ceilings come from the shared table so they stay strictly above the
|
|
25
|
+
// daemon's own budget for the same tool — otherwise this timer pre-empts the
|
|
26
|
+
// daemon's specific error message with a generic one. See ipc-timeouts.ts.
|
|
26
27
|
// ---------------------------------------------------------------------------
|
|
27
28
|
// Safety nets
|
|
28
29
|
// ---------------------------------------------------------------------------
|
|
@@ -142,7 +143,7 @@ function ipcRequest(tool, args) {
|
|
|
142
143
|
reject(new Error("Not connected to daemon IPC (retrying connection in background)"));
|
|
143
144
|
return;
|
|
144
145
|
}
|
|
145
|
-
const timeoutMs =
|
|
146
|
+
const timeoutMs = mcpTimeoutMs(tool);
|
|
146
147
|
const requestId = ++requestCounter;
|
|
147
148
|
const timer = setTimeout(() => {
|
|
148
149
|
pendingRequests.delete(requestId);
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"mcp-server.js","sourceRoot":"","sources":["../../src/channel/mcp-server.ts"],"names":[],"mappings":";AACA;;;;;;;GAOG;AAEH,OAAO,EAAE,MAAM,EAAE,MAAM,2CAA2C,CAAC;AACnE,OAAO,EAAE,oBAAoB,EAAE,MAAM,2CAA2C,CAAC;AACjF,OAAO,EACL,sBAAsB,EACtB,qBAAqB,GACtB,MAAM,oCAAoC,CAAC;AAC5C,OAAO,EAAE,QAAQ,EAAE,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AACpD,OAAO,EAAE,aAAa,EAAE,YAAY,EAAE,UAAU,EAAE,MAAM,SAAS,CAAC;AAClE,OAAO,EAAE,SAAS,EAAE,MAAM,iBAAiB,CAAC;AAC5C,OAAO,EAAE,KAAK,EAAE,MAAM,gBAAgB,CAAC;AACvC,OAAO,EAAE,sBAAsB,EAAE,MAAM,oBAAoB,CAAC;AAC5D,OAAO,EAAE,gBAAgB,EAAE,MAAM,wBAAwB,CAAC;
|
|
1
|
+
{"version":3,"file":"mcp-server.js","sourceRoot":"","sources":["../../src/channel/mcp-server.ts"],"names":[],"mappings":";AACA;;;;;;;GAOG;AAEH,OAAO,EAAE,MAAM,EAAE,MAAM,2CAA2C,CAAC;AACnE,OAAO,EAAE,oBAAoB,EAAE,MAAM,2CAA2C,CAAC;AACjF,OAAO,EACL,sBAAsB,EACtB,qBAAqB,GACtB,MAAM,oCAAoC,CAAC;AAC5C,OAAO,EAAE,QAAQ,EAAE,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AACpD,OAAO,EAAE,aAAa,EAAE,YAAY,EAAE,UAAU,EAAE,MAAM,SAAS,CAAC;AAClE,OAAO,EAAE,SAAS,EAAE,MAAM,iBAAiB,CAAC;AAC5C,OAAO,EAAE,KAAK,EAAE,MAAM,gBAAgB,CAAC;AACvC,OAAO,EAAE,sBAAsB,EAAE,MAAM,oBAAoB,CAAC;AAC5D,OAAO,EAAE,gBAAgB,EAAE,MAAM,wBAAwB,CAAC;AAC1D,OAAO,EAAE,YAAY,EAAE,MAAM,mBAAmB,CAAC;AAEjD,8EAA8E;AAC9E,gBAAgB;AAChB,8EAA8E;AAE9E,MAAM,WAAW,GAAG,OAAO,CAAC,GAAG,CAAC,iBAAiB,IAAI,EAAE,CAAC;AAExD,+EAA+E;AAC/E,6EAA6E;AAC7E,2EAA2E;AAE3E,8EAA8E;AAC9E,cAAc;AACd,8EAA8E;AAE9E,gFAAgF;AAChF,4EAA4E;AAC5E,6EAA6E;AAC7E,0EAA0E;AAC1E,kDAAkD;AAElD,MAAM,UAAU,GAAG,OAAO,CAAC,IAAI,CAAC;AAEhC,SAAS,UAAU;IACjB,OAAO,OAAO,CAAC,IAAI,KAAK,CAAC,IAAI,OAAO,CAAC,IAAI,KAAK,UAAU,CAAC;AAC3D,CAAC;AAED,OAAO,CAAC,EAAE,CAAC,oBAAoB,EAAE,CAAC,GAAG,EAAE,EAAE;IACvC,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,+BAA+B,GAAG,IAAI,CAAC,CAAC;AAC/D,CAAC,CAAC,CAAC;AACH,OAAO,CAAC,EAAE,CAAC,mBAAmB,EAAE,CAAC,GAAG,EAAE,EAAE;IACtC,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,8BAA8B,GAAG,IAAI,CAAC,CAAC;AAC9D,CAAC,CAAC,CAAC;AAEH,8EAA8E;AAC9E,mCAAmC;AACnC,8EAA8E;AAE9E,IAAI,GAAG,GAAqB,IAAI,CAAC;AACjC,IAAI,YAAY,GAAG,KAAK,CAAC;AACzB,IAAI,cAAc,GAAG,CAAC,CAAC;AACvB,IAAI,YAAY,GAAG,KAAK,CAAC;AACzB,IAAI,iBAAiB,GAAG,CAAC,CAAC;AAC1B,iFAAiF;AACjF,kFAAkF;AAClF,0EAA0E;AAC1E,kFAAkF;AAClF,+EAA+E;AAC/E,iFAAiF;AACjF,+BAA+B;AAC/B,MAAM,eAAe,GAAG,IAAI,GAAG,EAG5B,CAAC;AAEJ,SAAS,iBAAiB,CAAC,MAAiB;IAC1C,MAAM,CAAC,EAAE,CAAC,SAAS,EAAE,CAAC,GAA4B,EAAE,EAAE;QACpD,wEAAwE;QACxE,4EAA4E;QAC5E,6EAA6E;QAC7E,6EAA6E;QAC7E,6EAA6E;QAC7E,uDAAuD;QACvD,IAAI,GAAG,CAAC,IAAI,KAAK,UAAU,EAAE,CAAC;YAC5B,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,gEAAgE,CAAC,CAAC;YACvF,YAAY,GAAG,KAAK,CAAC;YACrB,OAAO;QACT,CAAC;QAED,IAAI,OAAO,GAAG,CAAC,SAAS,KAAK,QAAQ,IAAI,eAAe,CAAC,GAAG,CAAC,GAAG,CAAC,SAAS,CAAC,EAAE,CAAC;YAC5E,MAAM,OAAO,GAAG,eAAe,CAAC,GAAG,CAAC,GAAG,CAAC,SAAS,CAAE,CAAC;YACpD,eAAe,CAAC,MAAM,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC;YACtC,YAAY,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;YAC5B,IAAI,GAAG,CAAC,KAAK,EAAE,CAAC;gBACd,OAAO,CAAC,MAAM,CAAC,IAAI,KAAK,CAAC,MAAM,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;YAC/C,CAAC;iBAAM,CAAC;gBACN,OAAO,CAAC,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;YAC9B,CAAC;QACH,CAAC;IACH,CAAC,CAAC,CAAC;IAEH,MAAM,CAAC,EAAE,CAAC,YAAY,EAAE,GAAG,EAAE;QAC3B,YAAY,GAAG,KAAK,CAAC;QACrB,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,4CAA4C,CAAC,CAAC;QACnE,8BAA8B;QAC9B,KAAK,MAAM,CAAC,EAAE,EAAE,OAAO,CAAC,IAAI,eAAe,EAAE,CAAC;YAC5C,YAAY,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;YAC5B,OAAO,CAAC,MAAM,CAAC,IAAI,KAAK,CAAC,kBAAkB,CAAC,CAAC,CAAC;YAC9C,eAAe,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;QAC7B,CAAC;QACD,iBAAiB,EAAE,CAAC;IACtB,CAAC,CAAC,CAAC;AACL,CAAC;AAED,KAAK,UAAU,UAAU;IACvB,IAAI,CAAC;QACH,MAAM,MAAM,GAAG,IAAI,SAAS,CAAC,WAAY,CAAC,CAAC;QAC3C,MAAM,MAAM,CAAC,OAAO,EAAE,CAAC;QACvB,GAAG,GAAG,MAAM,CAAC;QACb,YAAY,GAAG,IAAI,CAAC;QACpB,YAAY,GAAG,KAAK,CAAC;QACrB,iBAAiB,GAAG,CAAC,CAAC;QACtB,iBAAiB,CAAC,MAAM,CAAC,CAAC;QAC1B,sEAAsE;QACtE,qFAAqF;QACrF,4EAA4E;QAC5E,0DAA0D;QAC1D,MAAM,WAAW,GAAG,OAAO,CAAC,GAAG,CAAC,mBAAmB;eAC9C,OAAO,CAAC,GAAG,CAAC,kBAAkB;eAC9B,YAAY,QAAQ,CAAC,OAAO,CAAC,GAAG,EAAE,CAAC,IAAI,OAAO,CAAC,GAAG,EAAE,CAAC;QAC1D,MAAM,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,WAAW,EAAE,WAAW,EAAE,CAAC,CAAC;QAChD,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,kCAAkC,CAAC,CAAC;IAC3D,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,2CAA2C,GAAG,IAAI,CAAC,CAAC;QACzE,YAAY,GAAG,KAAK,CAAC;QACrB,iBAAiB,EAAE,CAAC;IACtB,CAAC;AACH,CAAC;AAED,SAAS,iBAAiB;IACxB,IAAI,YAAY;QAAE,OAAO;IACzB,IAAI,UAAU,EAAE,EAAE,CAAC;QACjB,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,4DAA4D,CAAC,CAAC;QACnF,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC;IACD,iBAAiB,EAAE,CAAC;IACpB,YAAY,GAAG,IAAI,CAAC;IACpB,4EAA4E;IAC5E,MAAM,KAAK,GAAG,gBAAgB,CAAC,iBAAiB,CAAC,CAAC;IAClD,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,0BAA0B,KAAK,eAAe,iBAAiB,QAAQ,CAAC,CAAC;IAC9F,UAAU,CAAC,GAAG,EAAE;QACd,YAAY,GAAG,KAAK,CAAC;QACrB,UAAU,EAAE,CAAC;IACf,CAAC,EAAE,KAAK,CAAC,CAAC;AACZ,CAAC;AAED,SAAS,UAAU,CACjB,IAAY,EACZ,IAA6B;IAE7B,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;QACrC,IAAI,CAAC,YAAY,IAAI,CAAC,GAAG,EAAE,CAAC;YAC1B,MAAM,CAAC,IAAI,KAAK,CAAC,iEAAiE,CAAC,CAAC,CAAC;YACrF,OAAO;QACT,CAAC;QAED,MAAM,SAAS,GAAG,YAAY,CAAC,IAAI,CAAC,CAAC;QACrC,MAAM,SAAS,GAAG,EAAE,cAAc,CAAC;QACnC,MAAM,KAAK,GAAG,UAAU,CAAC,GAAG,EAAE;YAC5B,eAAe,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC;YAClC,MAAM,CAAC,IAAI,KAAK,CAAC,+BAA+B,SAAS,IAAI,CAAC,CAAC,CAAC;QAClE,CAAC,EAAE,SAAS,CAAC,CAAC;QAEd,eAAe,CAAC,GAAG,CAAC,SAAS,EAAE,EAAE,OAAO,EAAE,MAAM,EAAE,KAAK,EAAE,CAAC,CAAC;QAE3D,IAAI,CAAC;YACH,GAAG,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,WAAW,EAAE,IAAI,EAAE,IAAI,EAAE,SAAS,EAAE,CAAC,CAAC;QACzD,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,eAAe,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC;YAClC,YAAY,CAAC,KAAK,CAAC,CAAC;YACpB,YAAY,GAAG,KAAK,CAAC;YACrB,MAAM,CAAC,IAAI,KAAK,CAAC,oBAAoB,GAAG,EAAE,CAAC,CAAC,CAAC;QAC/C,CAAC;IACH,CAAC,CAAC,CAAC;AACL,CAAC;AAED,8EAA8E;AAC9E,aAAa;AACb,8EAA8E;AAE9E,8EAA8E;AAC9E,0EAA0E;AAC1E,4EAA4E;AAC5E,0EAA0E;AAC1E,+DAA+D;AAC/D,8EAA8E;AAE9E,SAAS,oBAAoB;IAC3B,MAAM,WAAW,GAAG,OAAO,CAAC,GAAG,CAAC,cAAc,CAAC;IAC/C,IAAI,QAAoC,CAAC;IACzC,IAAI,WAAW,KAAK,OAAO;QAAE,QAAQ,GAAG,KAAK,CAAC;SACzC,IAAI,WAAW;QAAE,QAAQ,GAAG,WAAW,CAAC;IAE7C,IAAI,SAA2D,CAAC;IAChE,IAAI,OAAO,CAAC,GAAG,CAAC,eAAe,EAAE,CAAC;QAChC,IAAI,CAAC;YAAC,SAAS,GAAG,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,eAAe,CAAC,CAAC;QAAC,CAAC;QAAC,MAAM,CAAC,CAAC,UAAU,CAAC,CAAC;IACnF,CAAC;IAED,OAAO,sBAAsB,CAAC;QAC5B,YAAY,EAAE,OAAO,CAAC,GAAG,CAAC,mBAAmB,IAAI,SAAS;QAC1D,gBAAgB,EAAE,OAAO,CAAC,GAAG,CAAC,iBAAiB,IAAI,OAAO,CAAC,GAAG,EAAE;QAChE,eAAe,EAAE;YACf,IAAI,EAAE,OAAO,CAAC,GAAG,CAAC,mBAAmB,KAAK,SAAS,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,aAAa;YAC/E,OAAO,EAAE,OAAO,CAAC,GAAG,CAAC,aAAa,IAAI,SAAS;YAC/C,KAAK,EAAE,OAAO,CAAC,GAAG,CAAC,WAAW,IAAI,SAAS;SAC5C;QACD,WAAW,EAAE,OAAO,CAAC,GAAG,CAAC,kBAAkB;QAC3C,WAAW,EAAE,OAAO,CAAC,GAAG,CAAC,iBAAiB;QAC1C,YAAY,EAAE,OAAO,CAAC,GAAG,CAAC,mBAAmB;QAC7C,QAAQ;QACR,SAAS;KACV,CAAC,CAAC;AACL,CAAC;AAED,MAAM,GAAG,GAAG,IAAI,MAAM,CACpB,EAAE,IAAI,EAAE,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,EACnC;IACE,YAAY,EAAE;QACZ,KAAK,EAAE,EAAE;KACV;IACD,YAAY,EAAE,oBAAoB,EAAE;CACrC,CACF,CAAC;AAEF,8CAA8C;AAE9C,OAAO,EAAE,SAAS,EAAE,MAAM,gBAAgB,CAAC;AAC3C,OAAO,EAAE,KAAK,EAAE,MAAM,gBAAgB,CAAC;AAEvC,MAAM,OAAO,GAAG,OAAO,CAAC,GAAG,CAAC,cAAc,CAAC;AAC3C,IAAI,WAAyB,CAAC;AAC9B,IAAI,CAAC,OAAO,IAAI,OAAO,KAAK,MAAM,EAAE,CAAC;IACnC,WAAW,GAAG,KAAK,CAAC;AACtB,CAAC;KAAM,IAAI,SAAS,CAAC,OAAO,CAAC,EAAE,CAAC;IAC9B,WAAW,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,SAAS,CAAC,OAAO,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC;AACvE,CAAC;KAAM,CAAC;IACN,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,0CAA0C,OAAO,aAAa,MAAM,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,mBAAmB,CAAC,CAAC;IACzI,WAAW,GAAG,KAAK,CAAC;AACtB,CAAC;AAED,GAAG,CAAC,iBAAiB,CAAC,sBAAsB,EAAE,KAAK,IAAI,EAAE,CAAC,CAAC,EAAE,KAAK,EAAE,WAAW,EAAE,CAAC,CAAC,CAAC;AAEpF,4BAA4B;AAE5B,GAAG,CAAC,iBAAiB,CAAC,qBAAqB,EAAE,KAAK,EAAE,GAAG,EAAE,EAAE;IACzD,MAAM,IAAI,GAAG,CAAC,GAAG,CAAC,MAAM,CAAC,SAAS,IAAI,EAAE,CAA4B,CAAC;IAErE,IAAI,CAAC;QACH,MAAM,MAAM,GAAG,MAAM,UAAU,CAAC,GAAG,CAAC,MAAM,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC;QACvD,MAAM,IAAI,GACR,OAAO,MAAM,KAAK,QAAQ,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,MAAM,IAAI,IAAI,CAAC,CAAC;QACvE,OAAO,EAAE,OAAO,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC,EAAE,CAAC;IAC/C,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,MAAM,OAAO,GAAG,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;QACjE,OAAO;YACL,OAAO,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,UAAU,OAAO,EAAE,EAAE,CAAC;YACtD,OAAO,EAAE,IAAI;SACd,CAAC;IACJ,CAAC;AACH,CAAC,CAAC,CAAC;AAEH,8EAA8E;AAC9E,OAAO;AACP,8EAA8E;AAE9E,KAAK,UAAU,IAAI;IACjB,IAAI,CAAC,WAAW,EAAE,CAAC;QACjB,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,6DAA6D,CAAC,CAAC;QACpF,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC;IAED,uEAAuE;IACvE,iEAAiE;IACjE,MAAM,SAAS,GAAG,IAAI,oBAAoB,EAAE,CAAC;IAC7C,MAAM,GAAG,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC;IAE7B,0BAA0B;IAC1B,+EAA+E;IAC/E,6EAA6E;IAC7E,6EAA6E;IAC7E,+EAA+E;IAC/E,GAAG,CAAC,OAAO,GAAG,GAAG,EAAE;QACjB,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,yCAAyC,CAAC,CAAC;QAChE,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC,CAAC;IACF,OAAO,CAAC,KAAK,CAAC,EAAE,CAAC,KAAK,EAAE,GAAG,EAAE;QAC3B,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,8CAA8C,CAAC,CAAC;QACrE,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC,CAAC,CAAC;IACH,OAAO,CAAC,KAAK,CAAC,EAAE,CAAC,OAAO,EAAE,GAAG,EAAE;QAC7B,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,iDAAiD,CAAC,CAAC;QACxE,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC,CAAC,CAAC;IACH,OAAO,CAAC,KAAK,CAAC,EAAE,CAAC,OAAO,EAAE,GAAG,EAAE;QAC7B,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,gDAAgD,CAAC,CAAC;QACvE,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC,CAAC,CAAC;IAEH,6DAA6D;IAC7D,WAAW,CAAC,GAAG,EAAE;QACf,IAAI,UAAU,EAAE,EAAE,CAAC;YACjB,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,uDAAuD,CAAC,CAAC;YAC9E,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QAClB,CAAC;IACH,CAAC,EAAE,KAAK,CAAC,CAAC;IAEV,yEAAyE;IACzE,6EAA6E;IAC7E,MAAM,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC,WAAW,CAAC,EAAE,iBAAiB,CAAC,CAAC;IAC9D,aAAa,CAAC,OAAO,EAAE,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC;IAC5C,OAAO,CAAC,EAAE,CAAC,MAAM,EAAE,GAAG,EAAE;QACtB,IAAI,CAAC;YACH,IAAI,YAAY,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC,IAAI,EAAE,KAAK,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC;gBAAE,UAAU,CAAC,OAAO,CAAC,CAAC;QACzF,CAAC;QAAC,MAAM,CAAC,CAAC,wBAAwB,CAAC,CAAC;IACtC,CAAC,CAAC,CAAC;IAEH,4DAA4D;IAC5D,MAAM,UAAU,EAAE,CAAC;IAEnB,6EAA6E;IAC7E,IAAI,UAAU,EAAE,EAAE,CAAC;QACjB,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,0CAA0C,CAAC,CAAC;QACjE,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC;IAED,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,6BAA6B,CAAC,CAAC;AACtD,CAAC;AAED,IAAI,EAAE,CAAC,KAAK,CAAC,CAAC,GAAG,EAAE,EAAE;IACnB,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,uBAAuB,GAAG,IAAI,CAAC,CAAC;IACrD,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;AAClB,CAAC,CAAC,CAAC"}
|
package/dist/daemon.d.ts
CHANGED
|
@@ -259,6 +259,19 @@ export declare class Daemon extends EventEmitter {
|
|
|
259
259
|
* A dead remain-on-exit pane still contains the old ready prompt, so allowing
|
|
260
260
|
* the pane monitor to capture it would re-create a false idle state.
|
|
261
261
|
*/
|
|
262
|
+
/**
|
|
263
|
+
* Announce that this instance is no longer being supervised.
|
|
264
|
+
*
|
|
265
|
+
* Four health-check exits set `healthCheckPaused = true` and returned without
|
|
266
|
+
* telling anyone: a clean CLI exit, `max_retries <= 0`, crash retries exhausted,
|
|
267
|
+
* and the instance directory being deleted. Only the crash-LOOP case emitted an
|
|
268
|
+
* event, so a fleet could quietly contain a dead instance that still looked fine
|
|
269
|
+
* on the dashboard — while messages routed to it queued or failed with a bare ❌.
|
|
270
|
+
*
|
|
271
|
+
* `crash_loop` already had this treatment; this is the same bridge for the other
|
|
272
|
+
* four, carrying a human-readable cause and the operator's next step.
|
|
273
|
+
*/
|
|
274
|
+
private emitSupervisionEnded;
|
|
262
275
|
private setProcessStatus;
|
|
263
276
|
/**
|
|
264
277
|
* Periodically scan PTY output for backend-defined error patterns.
|
package/dist/daemon.js
CHANGED
|
@@ -10,6 +10,7 @@ import { TmuxManager, resolveTmuxLogicalSize } from "./tmux-manager.js";
|
|
|
10
10
|
import { TranscriptMonitor } from "./transcript-monitor.js";
|
|
11
11
|
import { ContextGuardian } from "./context-guardian.js";
|
|
12
12
|
import { IpcServer } from "./channel/ipc-bridge.js";
|
|
13
|
+
import { daemonBudgetMs } from "./channel/ipc-timeouts.js";
|
|
13
14
|
import { MessageBus } from "./channel/message-bus.js";
|
|
14
15
|
import { shellQuote } from "./backend/types.js";
|
|
15
16
|
import { getTmuxSession } from "./config.js";
|
|
@@ -848,6 +849,7 @@ export class Daemon extends EventEmitter {
|
|
|
848
849
|
this.logger.warn({ instanceDir: this.instanceDir }, "Instance directory missing — stopping health check");
|
|
849
850
|
this.healthCheckPaused = true;
|
|
850
851
|
this.healthCheckTimer = null;
|
|
852
|
+
this.emitSupervisionEnded("instance directory was removed from disk", "Recreate the instance, or delete it from fleet.yaml.");
|
|
851
853
|
return;
|
|
852
854
|
}
|
|
853
855
|
if (!this.tmux || this.spawning || this.healthCheckPaused || Daemon.tmuxServerPaused) {
|
|
@@ -898,6 +900,7 @@ export class Daemon extends EventEmitter {
|
|
|
898
900
|
this.logger.info("CLI exited normally (code 0) — pausing health check");
|
|
899
901
|
await this.tmux.killWindow();
|
|
900
902
|
this.healthCheckPaused = true;
|
|
903
|
+
this.emitSupervisionEnded("the CLI exited normally (code 0)", "Nothing crashed — start it again when you need it.");
|
|
901
904
|
return;
|
|
902
905
|
}
|
|
903
906
|
this.setProcessStatus("crashed");
|
|
@@ -1007,6 +1010,7 @@ export class Daemon extends EventEmitter {
|
|
|
1007
1010
|
if (max_retries <= 0) {
|
|
1008
1011
|
this.healthCheckPaused = true;
|
|
1009
1012
|
this.logger.warn(`${cliLabel} window died — automatic restart is disabled`);
|
|
1013
|
+
this.emitSupervisionEnded("the CLI died and automatic restart is disabled (restart_policy.max_retries is 0)", "Start it manually, or raise max_retries in fleet.yaml.");
|
|
1010
1014
|
return;
|
|
1011
1015
|
}
|
|
1012
1016
|
// Detect rapid crash: sliding window — 3+ crashes in 5 minutes
|
|
@@ -1036,6 +1040,8 @@ export class Daemon extends EventEmitter {
|
|
|
1036
1040
|
this.lastCrashAt = Date.now();
|
|
1037
1041
|
if (this.crashCount > max_retries) {
|
|
1038
1042
|
this.logger.error({ crashCount: this.crashCount, maxRetries: max_retries }, "Max crash retries exceeded — not respawning");
|
|
1043
|
+
this.healthCheckPaused = true;
|
|
1044
|
+
this.emitSupervisionEnded(`it crashed ${this.crashCount} times, exceeding restart_policy.max_retries (${max_retries})`, "Check the logs for the cause, then restart it.");
|
|
1039
1045
|
return; // don't schedule next — given up
|
|
1040
1046
|
}
|
|
1041
1047
|
// Calculate backoff delay
|
|
@@ -1119,6 +1125,21 @@ export class Daemon extends EventEmitter {
|
|
|
1119
1125
|
* A dead remain-on-exit pane still contains the old ready prompt, so allowing
|
|
1120
1126
|
* the pane monitor to capture it would re-create a false idle state.
|
|
1121
1127
|
*/
|
|
1128
|
+
/**
|
|
1129
|
+
* Announce that this instance is no longer being supervised.
|
|
1130
|
+
*
|
|
1131
|
+
* Four health-check exits set `healthCheckPaused = true` and returned without
|
|
1132
|
+
* telling anyone: a clean CLI exit, `max_retries <= 0`, crash retries exhausted,
|
|
1133
|
+
* and the instance directory being deleted. Only the crash-LOOP case emitted an
|
|
1134
|
+
* event, so a fleet could quietly contain a dead instance that still looked fine
|
|
1135
|
+
* on the dashboard — while messages routed to it queued or failed with a bare ❌.
|
|
1136
|
+
*
|
|
1137
|
+
* `crash_loop` already had this treatment; this is the same bridge for the other
|
|
1138
|
+
* four, carrying a human-readable cause and the operator's next step.
|
|
1139
|
+
*/
|
|
1140
|
+
emitSupervisionEnded(reason, remedy) {
|
|
1141
|
+
this.emit("supervision_ended", { name: this.name, reason, remedy });
|
|
1142
|
+
}
|
|
1122
1143
|
setProcessStatus(status) {
|
|
1123
1144
|
if (this.processStatus === status)
|
|
1124
1145
|
return;
|
|
@@ -2236,8 +2257,8 @@ export class Daemon extends EventEmitter {
|
|
|
2236
2257
|
});
|
|
2237
2258
|
const timeout = setTimeout(() => {
|
|
2238
2259
|
this.pendingIpcRequests.delete(fleetReqId);
|
|
2239
|
-
respond(null,
|
|
2240
|
-
},
|
|
2260
|
+
respond(null, `Task operation timed out after ${daemonBudgetMs(tool) / 1000}s`);
|
|
2261
|
+
}, daemonBudgetMs(tool));
|
|
2241
2262
|
this.pendingIpcRequests.set(fleetReqId, (respMsg) => {
|
|
2242
2263
|
clearTimeout(timeout);
|
|
2243
2264
|
respond(respMsg.result, respMsg.error);
|
|
@@ -2259,8 +2280,8 @@ export class Daemon extends EventEmitter {
|
|
|
2259
2280
|
});
|
|
2260
2281
|
const timeout = setTimeout(() => {
|
|
2261
2282
|
this.pendingIpcRequests.delete(fleetReqId);
|
|
2262
|
-
respond(null,
|
|
2263
|
-
},
|
|
2283
|
+
respond(null, `Decision operation timed out after ${daemonBudgetMs(tool) / 1000}s`);
|
|
2284
|
+
}, daemonBudgetMs(tool));
|
|
2264
2285
|
this.pendingIpcRequests.set(fleetReqId, (respMsg) => {
|
|
2265
2286
|
clearTimeout(timeout);
|
|
2266
2287
|
respond(respMsg.result, respMsg.error);
|
|
@@ -2286,8 +2307,8 @@ export class Daemon extends EventEmitter {
|
|
|
2286
2307
|
// Wait for fleet_schedule_response via pending request map
|
|
2287
2308
|
const timeout = setTimeout(() => {
|
|
2288
2309
|
this.pendingIpcRequests.delete(fleetReqId);
|
|
2289
|
-
respond(null,
|
|
2290
|
-
},
|
|
2310
|
+
respond(null, `Schedule operation timed out after ${daemonBudgetMs(tool) / 1000}s`);
|
|
2311
|
+
}, daemonBudgetMs(tool));
|
|
2291
2312
|
this.pendingIpcRequests.set(fleetReqId, (respMsg) => {
|
|
2292
2313
|
clearTimeout(timeout);
|
|
2293
2314
|
respond(respMsg.result, respMsg.error);
|
|
@@ -2308,7 +2329,9 @@ export class Daemon extends EventEmitter {
|
|
|
2308
2329
|
fleetRequestId: fleetReqId,
|
|
2309
2330
|
senderSessionName,
|
|
2310
2331
|
});
|
|
2311
|
-
|
|
2332
|
+
// Shared table, so delete_instance and the deployment tools get the same
|
|
2333
|
+
// budget here as the MCP client expects (the ad-hoc list omitted them).
|
|
2334
|
+
const crossTimeoutMs = daemonBudgetMs(tool);
|
|
2312
2335
|
const timeout = setTimeout(() => {
|
|
2313
2336
|
this.pendingIpcRequests.delete(fleetReqId);
|
|
2314
2337
|
respond(null, `Cross-instance operation timed out after ${crossTimeoutMs / 1000}s`);
|
|
@@ -2351,8 +2374,8 @@ export class Daemon extends EventEmitter {
|
|
|
2351
2374
|
this.ipcServer?.broadcast({ type: "fleet_outbound", tool, args, fleetRequestId: fleetReqId });
|
|
2352
2375
|
const timeout = setTimeout(() => {
|
|
2353
2376
|
this.pendingIpcRequests.delete(outboundKey);
|
|
2354
|
-
respond(null,
|
|
2355
|
-
},
|
|
2377
|
+
respond(null, `Fleet outbound timed out after ${daemonBudgetMs(tool) / 1000}s`);
|
|
2378
|
+
}, daemonBudgetMs(tool));
|
|
2356
2379
|
this.pendingIpcRequests.set(outboundKey, (respMsg) => {
|
|
2357
2380
|
clearTimeout(timeout);
|
|
2358
2381
|
respond(respMsg.result, respMsg.error);
|