@songsid/agend 2.1.2-beta.3 → 2.1.2-beta.30

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (157) hide show
  1. package/dist/access-path.js +15 -6
  2. package/dist/access-path.js.map +1 -1
  3. package/dist/agent-cli-instructions.md +1 -0
  4. package/dist/agent-cli.js +14 -0
  5. package/dist/agent-cli.js.map +1 -1
  6. package/dist/agent-endpoint.d.ts +2 -0
  7. package/dist/agent-endpoint.js +25 -1
  8. package/dist/agent-endpoint.js.map +1 -1
  9. package/dist/backend/antigravity.d.ts +20 -0
  10. package/dist/backend/antigravity.js +37 -3
  11. package/dist/backend/antigravity.js.map +1 -1
  12. package/dist/backend/claude-code.d.ts +59 -0
  13. package/dist/backend/claude-code.js +67 -1
  14. package/dist/backend/claude-code.js.map +1 -1
  15. package/dist/backend/codex.d.ts +3 -0
  16. package/dist/backend/codex.js +30 -0
  17. package/dist/backend/codex.js.map +1 -1
  18. package/dist/backend/grok.d.ts +25 -0
  19. package/dist/backend/grok.js +51 -0
  20. package/dist/backend/grok.js.map +1 -1
  21. package/dist/backend/kiro.d.ts +22 -0
  22. package/dist/backend/kiro.js +85 -3
  23. package/dist/backend/kiro.js.map +1 -1
  24. package/dist/backend/types.d.ts +58 -0
  25. package/dist/backend/types.js +13 -1
  26. package/dist/backend/types.js.map +1 -1
  27. package/dist/channel/adapters/discord.d.ts +15 -0
  28. package/dist/channel/adapters/discord.js +100 -2
  29. package/dist/channel/adapters/discord.js.map +1 -1
  30. package/dist/channel/adapters/telegram.d.ts +21 -0
  31. package/dist/channel/adapters/telegram.js +86 -0
  32. package/dist/channel/adapters/telegram.js.map +1 -1
  33. package/dist/channel/ipc-bridge.d.ts +9 -1
  34. package/dist/channel/ipc-bridge.js +12 -3
  35. package/dist/channel/ipc-bridge.js.map +1 -1
  36. package/dist/channel/ipc-timeouts.d.ts +46 -0
  37. package/dist/channel/ipc-timeouts.js +65 -0
  38. package/dist/channel/ipc-timeouts.js.map +1 -0
  39. package/dist/channel/mcp-server.js +30 -14
  40. package/dist/channel/mcp-server.js.map +1 -1
  41. package/dist/channel/mcp-tools.js +30 -1
  42. package/dist/channel/mcp-tools.js.map +1 -1
  43. package/dist/channel/reconnect-backoff.d.ts +17 -0
  44. package/dist/channel/reconnect-backoff.js +21 -0
  45. package/dist/channel/reconnect-backoff.js.map +1 -0
  46. package/dist/channel/types.d.ts +29 -0
  47. package/dist/classic-channel-manager.js +1 -5
  48. package/dist/classic-channel-manager.js.map +1 -1
  49. package/dist/cli.js +155 -63
  50. package/dist/cli.js.map +1 -1
  51. package/dist/completion.d.ts +27 -0
  52. package/dist/completion.js +121 -0
  53. package/dist/completion.js.map +1 -0
  54. package/dist/config-validator.js +21 -0
  55. package/dist/config-validator.js.map +1 -1
  56. package/dist/config.js +1 -0
  57. package/dist/config.js.map +1 -1
  58. package/dist/cost-guard.d.ts +3 -1
  59. package/dist/cost-guard.js +3 -1
  60. package/dist/cost-guard.js.map +1 -1
  61. package/dist/daemon.d.ts +240 -14
  62. package/dist/daemon.js +956 -308
  63. package/dist/daemon.js.map +1 -1
  64. package/dist/event-log.d.ts +31 -0
  65. package/dist/event-log.js +96 -0
  66. package/dist/event-log.js.map +1 -1
  67. package/dist/fleet-context.d.ts +9 -0
  68. package/dist/fleet-manager.d.ts +311 -2
  69. package/dist/fleet-manager.js +1369 -108
  70. package/dist/fleet-manager.js.map +1 -1
  71. package/dist/general-knowledge/skills/cross-instance-messaging/SKILL.md +22 -0
  72. package/dist/general-knowledge/skills/model-discovery/SKILL.md +14 -23
  73. package/dist/general-knowledge/skills/session-management/SKILL.md +15 -34
  74. package/dist/hang-detector.d.ts +19 -13
  75. package/dist/hang-detector.js +19 -49
  76. package/dist/hang-detector.js.map +1 -1
  77. package/dist/instance-lifecycle.d.ts +43 -1
  78. package/dist/instance-lifecycle.js +155 -43
  79. package/dist/instance-lifecycle.js.map +1 -1
  80. package/dist/instructions.d.ts +5 -0
  81. package/dist/instructions.js +9 -11
  82. package/dist/instructions.js.map +1 -1
  83. package/dist/locale.js +5 -1
  84. package/dist/locale.js.map +1 -1
  85. package/dist/logger.js +14 -0
  86. package/dist/logger.js.map +1 -1
  87. package/dist/mcp-liveness.d.ts +21 -0
  88. package/dist/mcp-liveness.js +27 -0
  89. package/dist/mcp-liveness.js.map +1 -0
  90. package/dist/outbound-handlers.d.ts +16 -0
  91. package/dist/outbound-handlers.js +151 -27
  92. package/dist/outbound-handlers.js.map +1 -1
  93. package/dist/outbound-schemas.d.ts +11 -3
  94. package/dist/outbound-schemas.js +16 -2
  95. package/dist/outbound-schemas.js.map +1 -1
  96. package/dist/pane-write-lock.d.ts +48 -0
  97. package/dist/pane-write-lock.js +73 -0
  98. package/dist/pane-write-lock.js.map +1 -0
  99. package/dist/process-memory.d.ts +31 -0
  100. package/dist/process-memory.js +79 -0
  101. package/dist/process-memory.js.map +1 -0
  102. package/dist/quickstart.js +17 -16
  103. package/dist/quickstart.js.map +1 -1
  104. package/dist/reply-dedup.d.ts +41 -0
  105. package/dist/reply-dedup.js +0 -0
  106. package/dist/reply-dedup.js.map +1 -0
  107. package/dist/scheduler/db.js +3 -0
  108. package/dist/scheduler/db.js.map +1 -1
  109. package/dist/sd-notify.d.ts +27 -0
  110. package/dist/sd-notify.js +33 -1
  111. package/dist/sd-notify.js.map +1 -1
  112. package/dist/secret-file.d.ts +33 -0
  113. package/dist/secret-file.js +36 -0
  114. package/dist/secret-file.js.map +1 -0
  115. package/dist/setup-wizard.js +9 -7
  116. package/dist/setup-wizard.js.map +1 -1
  117. package/dist/tmux-control.d.ts +58 -5
  118. package/dist/tmux-control.js +102 -14
  119. package/dist/tmux-control.js.map +1 -1
  120. package/dist/tmux-manager.d.ts +31 -1
  121. package/dist/tmux-manager.js +40 -11
  122. package/dist/tmux-manager.js.map +1 -1
  123. package/dist/topic-commands.d.ts +50 -10
  124. package/dist/topic-commands.js +269 -106
  125. package/dist/topic-commands.js.map +1 -1
  126. package/dist/tui-glyphs.d.ts +25 -0
  127. package/dist/tui-glyphs.js +26 -0
  128. package/dist/tui-glyphs.js.map +1 -0
  129. package/dist/types.d.ts +12 -2
  130. package/dist/ui/view.html +123 -2
  131. package/dist/update-marker.d.ts +15 -0
  132. package/dist/update-marker.js +69 -0
  133. package/dist/update-marker.js.map +1 -0
  134. package/dist/usage/format-rich.d.ts +11 -0
  135. package/dist/usage/format-rich.js +121 -0
  136. package/dist/usage/format-rich.js.map +1 -0
  137. package/dist/usage/providers.d.ts +130 -0
  138. package/dist/usage/providers.js +1030 -0
  139. package/dist/usage/providers.js.map +1 -0
  140. package/dist/usage/statusline-usage.d.ts +20 -0
  141. package/dist/usage/statusline-usage.js +120 -0
  142. package/dist/usage/statusline-usage.js.map +1 -0
  143. package/dist/usage/usage-api.d.ts +50 -0
  144. package/dist/usage/usage-api.js +167 -0
  145. package/dist/usage/usage-api.js.map +1 -0
  146. package/dist/view-api.d.ts +3 -0
  147. package/dist/view-api.js +11 -2
  148. package/dist/view-api.js.map +1 -1
  149. package/dist/web-api.js +5 -2
  150. package/dist/web-api.js.map +1 -1
  151. package/package.json +4 -1
  152. package/dist/channel/tool-tracker.d.ts +0 -13
  153. package/dist/channel/tool-tracker.js +0 -58
  154. package/dist/channel/tool-tracker.js.map +0 -1
  155. package/dist/daemon-entry.d.ts +0 -1
  156. package/dist/daemon-entry.js +0 -30
  157. package/dist/daemon-entry.js.map +0 -1
@@ -0,0 +1,22 @@
1
+ ---
2
+ name: cross-instance-messaging
3
+ description: Fire-and-queue cross-instance tools — send once, never resend on queued
4
+ ---
5
+
6
+ ## How to send
7
+
8
+ Use fleet tools only (`send_to_instance`, `delegate_task`, `request_information`, `report_result`, `broadcast`):
9
+ - Call returns immediately: `{ sent: true, queued: true }` — success, fleet owns delivery.
10
+ - **Do not wait** for the target to go idle; **do not** treat 30s IPC timeout as failure to re-send.
11
+ - **Error only if the target does not exist** (or similar hard reject) — then fix the name, don't spam.
12
+ - **Never re-send because the reply said `queued`** — that means the message is already queued.
13
+
14
+ ## requires_reply
15
+
16
+ - Means “target should later answer with `report_result` / a real reply”
17
+ - **Not** a synchronous wait for their turn to finish
18
+
19
+ ## Task flow
20
+
21
+ - `delegate_task` → silent work → `report_result` (zero ack-only pings)
22
+ - Cross-instance traffic is `[from:name]` → answer with `send_to_instance` / `report_result`, never `reply`
@@ -1,36 +1,27 @@
1
1
  ---
2
2
  name: model-discovery
3
- description: List available models per backend, configure model in fleet.yaml
3
+ description: Set and discover models pass-through to the CLI, no AgEnD allowlist gate
4
4
  ---
5
5
 
6
- ## Model Names by Backend
6
+ ## How to set a model
7
7
 
8
- Models are specified in fleet.yaml `defaults.model` or per-instance `model` field.
8
+ - fleet.yaml: `defaults.model` or per-instance `model`
9
+ - **Pass-through:** AgEnD no longer blocks unknown model ids — it may **warn**, then still pass the string to the CLI
10
+ - **CLI is source of truth** — if the model is invalid, the backend CLI errors (fix the name there)
9
11
 
10
- | Backend | How to list models | Default |
11
- |---------|-------------------|---------|
12
- | **kiro-cli** | In tmux: send `/model` + Enter → read model list → Esc to close | auto (latest) |
13
- | **claude-code** | `sonnet`, `opus`, `haiku`, `opusplan`, `best`, `sonnet[1m]`, `opus[1m]` | sonnet |
14
- | **antigravity** | Run `agy models` to see available models | Gemini 3.5 Flash (Medium) |
15
- | **codex** | `gpt-4o`, `o3`, `o4-mini` | gpt-4o |
16
- | **opencode** | `opencode models` | depends on provider |
17
- | **grok** | `grok-4.5`, `grok-4.3`, `grok-code`, `grok-build-0.1` | grok default |
12
+ ## Discover real names
18
13
 
19
- **Important for antigravity (agy):**
20
- - `agy models` shows names like `Gemini 3.5 Flash (Medium)` — the parenthetical suffix (Medium/High/Low/Thinking) is the **effort level**, NOT part of the model name.
21
- - When setting model in fleet.yaml, use only the base name WITHOUT the effort suffix.
22
- - Example: `agy models` shows `Gemini 3.5 Flash (Medium)` set `model: "Gemini 3.5 Flash"`
23
- - Example: `agy models` shows `Claude Opus 4.6 (Thinking)` → set `model: "Claude Opus 4.6"`
14
+ | Backend | How |
15
+ |---------|-----|
16
+ | kiro-cli | In pane: `/model` (gpt-*, deepseek-*, minimax-*, glm-*, qwen* supported) |
17
+ | claude-code | `sonnet` / `opus` / `haiku` / `opusplan` / `Fable` / aliases |
18
+ | codex | pane `/model` or docs (`gpt-*`, `o*`) |
19
+ | grok | `grok models` |
20
+ | antigravity | `agy models` — set **base name only** (drop `(Medium)` / `(Thinking)` effort suffix) |
21
+ | opencode | `opencode models` |
24
22
 
25
- **Important:** Model names vary by backend. Always check the actual CLI output rather than guessing names.
26
-
27
- Example fleet.yaml:
28
23
  ```yaml
29
24
  defaults:
30
25
  backend: kiro-cli
31
26
  model: claude-sonnet-4-20250514
32
-
33
- instances:
34
- heavy-task:
35
- model: claude-opus-4-20250514
36
27
  ```
@@ -1,46 +1,27 @@
1
1
  ---
2
2
  name: session-management
3
- description: Where kiro-cli and claude-code store sessions, and how to fork one to a new instance
3
+ description: Session stores, forking, and auth-pause recovery
4
4
  ---
5
5
 
6
+ ## Auth failure (auto-pause)
7
+
8
+ When AgEnD sees `auth_error` it **pauses** that instance (`pausePending` sticky):
9
+ - One notification **per backend** (not per instance) — log in once, same-backend peers recover.
10
+ - Auth is **per-user global** across backends: claude-code, codex, kiro, grok, opencode, antigravity.
11
+ - Messages while paused stay in the **queue** — do not re-send.
12
+ - After the user re-auths: `wake` / normal wake clears `pausePending`.
13
+
6
14
  ## Where sessions live
7
15
 
8
16
  | | kiro-cli | claude-code |
9
17
  |---|---|---|
10
18
  | Store | `~/.kiro/sessions/cli/<uuid>.json` | `~/.claude/projects/<path-encoded>/*.jsonl` |
11
- | Keyed by | session uuid | project (working) directory |
12
- | Reload | `/chat load <file>` | none — `--continue` (latest for the dir) / `--resume <id>` |
13
- | Text export | — | `/export` (plain text, **not** reloadable) |
14
-
15
- `<path-encoded>` = the absolute working_directory with `/` → `-`
16
- (e.g. `/home/han/Projects/AgEnD` → `-home-han-Projects-AgEnD`).
17
-
18
- ## Fork a session to a new instance
19
-
20
- Confirm the source is **idle** first (`describe_instance` / `get_fleet_status`) — don't fork mid-task.
21
-
22
- **kiro-cli** — save, copy, load:
23
- 1. On the source, save: `/chat save <name>.json -f` (paste via tmux if needed).
24
- 2. `create_instance` (same backend).
25
- 3. `cp ~/.agend/workspaces/<source>/<name>.json ~/.agend/workspaces/<target>/`
26
- 4. Load on the target: `/chat load <name>.json`, or set `pre_task_command: "/chat load <name>.json"`.
27
-
28
- **claude-code** — copy the `.jsonl` (no save/load command):
29
- 1. Newest source session: `ls -lt ~/.claude/projects/<source-encoded>/*.jsonl | head`
30
- 2. `create_instance` (backend `claude-code`); note its working_directory.
31
- 3. Copy into the target's encoded project dir:
32
- ```bash
33
- TARGET_ENC="$(echo '<target-working-dir>' | sed 's#/#-#g')"
34
- mkdir -p ~/.claude/projects/$TARGET_ENC
35
- cp ~/.claude/projects/<source-encoded>/<session>.jsonl ~/.claude/projects/$TARGET_ENC/
36
- ```
37
- 4. Start the target — claude-code resumes the newest `.jsonl` via `--continue`.
19
+ | Reload | `/chat load <file>` | `--continue` / `--resume <id>` |
38
20
 
39
- **Caveats:** a claude-code session only truly makes sense under its original working_directory
40
- (paths inside the transcript refer to it). Pick the right `.jsonl` if several exist (newest by
41
- mtime; compaction/branches create new files). `/export` is text only, not reloadable.
21
+ `<path-encoded>` = absolute cwd with `/` `-`.
42
22
 
43
- ## Backup
23
+ ## Fork (source must be idle)
44
24
 
45
- Sessions are plain files back them up by copying the store paths above
46
- (e.g. `cp ~/.kiro/sessions/cli/*.json <dest>/`). Only copy while the instance is idle.
25
+ - **kiro:** `/chat save name.json -f` `create_instance` copy workspace file `/chat load name.json`
26
+ - **claude-code:** copy newest `*.jsonl` into target's encoded project dir start (uses `--continue`)
27
+ - Prefer `replace_instance` when the whole session is poisoned (see instance-lifecycle)
@@ -1,16 +1,22 @@
1
1
  import { EventEmitter } from "node:events";
2
+ /**
3
+ * Carries "this instance looks hung" from the daemon to the fleet manager.
4
+ *
5
+ * It is only an event bridge. Hang detection itself lives in the daemon's pane-state
6
+ * machine, which emits `hang` directly when a pane stops changing for the configured
7
+ * stuck timeout; `instance-lifecycle` subscribes to that and notifies.
8
+ *
9
+ * It used to also contain a silence-timer state machine — `start()`, `isHung()`,
10
+ * `hungEmitted`, and timestamps fed by `recordActivity` / `recordInbound` /
11
+ * `recordStatuslineUpdate`. None of it ran: `start()` was never called from
12
+ * anywhere, so `isHung()` was unreachable, the timestamps were written and never
13
+ * read, and the constructor's `timeoutMinutes` was ignored (the real stuck timeout
14
+ * is read separately by the pane monitor). Its tests exercised that dead logic —
15
+ * two had identical setup with contradictory expectations, which can only pass
16
+ * unnoticed when neither runs against anything real.
17
+ *
18
+ * Kept as a named class rather than a bare EventEmitter so the daemon → lifecycle
19
+ * wiring stays typed and greppable.
20
+ */
2
21
  export declare class HangDetector extends EventEmitter {
3
- private lastActivityTs;
4
- private lastStatuslineTs;
5
- private lastInboundTs;
6
- private hungEmitted;
7
- private checkTimer;
8
- private timeoutMs;
9
- constructor(timeoutMinutes: number);
10
- recordActivity(): void;
11
- recordInbound(): void;
12
- recordStatuslineUpdate(): void;
13
- isHung(): boolean;
14
- start(intervalMs?: number): void;
15
- stop(): void;
16
22
  }
@@ -1,53 +1,23 @@
1
1
  import { EventEmitter } from "node:events";
2
+ /**
3
+ * Carries "this instance looks hung" from the daemon to the fleet manager.
4
+ *
5
+ * It is only an event bridge. Hang detection itself lives in the daemon's pane-state
6
+ * machine, which emits `hang` directly when a pane stops changing for the configured
7
+ * stuck timeout; `instance-lifecycle` subscribes to that and notifies.
8
+ *
9
+ * It used to also contain a silence-timer state machine — `start()`, `isHung()`,
10
+ * `hungEmitted`, and timestamps fed by `recordActivity` / `recordInbound` /
11
+ * `recordStatuslineUpdate`. None of it ran: `start()` was never called from
12
+ * anywhere, so `isHung()` was unreachable, the timestamps were written and never
13
+ * read, and the constructor's `timeoutMinutes` was ignored (the real stuck timeout
14
+ * is read separately by the pane monitor). Its tests exercised that dead logic —
15
+ * two had identical setup with contradictory expectations, which can only pass
16
+ * unnoticed when neither runs against anything real.
17
+ *
18
+ * Kept as a named class rather than a bare EventEmitter so the daemon → lifecycle
19
+ * wiring stays typed and greppable.
20
+ */
2
21
  export class HangDetector extends EventEmitter {
3
- lastActivityTs = 0;
4
- lastStatuslineTs = 0;
5
- lastInboundTs = 0;
6
- hungEmitted = false;
7
- checkTimer = null;
8
- timeoutMs;
9
- constructor(timeoutMinutes) {
10
- super();
11
- this.timeoutMs = timeoutMinutes * 60 * 1000;
12
- }
13
- recordActivity() {
14
- this.lastActivityTs = Date.now();
15
- if (this.hungEmitted) {
16
- this.hungEmitted = false;
17
- }
18
- }
19
- recordInbound() {
20
- this.lastInboundTs = Date.now();
21
- }
22
- recordStatuslineUpdate() {
23
- this.lastStatuslineTs = Date.now();
24
- }
25
- isHung() {
26
- if (this.lastActivityTs === 0)
27
- return false;
28
- if (this.lastInboundTs === 0)
29
- return false;
30
- const now = Date.now();
31
- // Only flag as hung if:
32
- // 1. There's an inbound that hasn't been answered (no activity since inbound)
33
- // 2. Timeout has elapsed since that inbound
34
- const noActivitySinceInbound = this.lastActivityTs < this.lastInboundTs;
35
- const stale = now - this.lastInboundTs > this.timeoutMs;
36
- return stale && noActivitySinceInbound;
37
- }
38
- start(intervalMs = 60_000) {
39
- this.checkTimer = setInterval(() => {
40
- if (this.isHung() && !this.hungEmitted) {
41
- this.hungEmitted = true;
42
- this.emit("hang");
43
- }
44
- }, intervalMs);
45
- }
46
- stop() {
47
- if (this.checkTimer) {
48
- clearInterval(this.checkTimer);
49
- this.checkTimer = null;
50
- }
51
- }
52
22
  }
53
23
  //# sourceMappingURL=hang-detector.js.map
@@ -1 +1 @@
1
- {"version":3,"file":"hang-detector.js","sourceRoot":"","sources":["../src/hang-detector.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AAE3C,MAAM,OAAO,YAAa,SAAQ,YAAY;IACpC,cAAc,GAAG,CAAC,CAAC;IACnB,gBAAgB,GAAG,CAAC,CAAC;IACrB,aAAa,GAAG,CAAC,CAAC;IAClB,WAAW,GAAG,KAAK,CAAC;IACpB,UAAU,GAA0C,IAAI,CAAC;IACzD,SAAS,CAAS;IAE1B,YAAY,cAAsB;QAChC,KAAK,EAAE,CAAC;QACR,IAAI,CAAC,SAAS,GAAG,cAAc,GAAG,EAAE,GAAG,IAAI,CAAC;IAC9C,CAAC;IAED,cAAc;QACZ,IAAI,CAAC,cAAc,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QACjC,IAAI,IAAI,CAAC,WAAW,EAAE,CAAC;YACrB,IAAI,CAAC,WAAW,GAAG,KAAK,CAAC;QAC3B,CAAC;IACH,CAAC;IAED,aAAa;QACX,IAAI,CAAC,aAAa,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IAClC,CAAC;IAED,sBAAsB;QACpB,IAAI,CAAC,gBAAgB,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IACrC,CAAC;IAED,MAAM;QACJ,IAAI,IAAI,CAAC,cAAc,KAAK,CAAC;YAAE,OAAO,KAAK,CAAC;QAC5C,IAAI,IAAI,CAAC,aAAa,KAAK,CAAC;YAAE,OAAO,KAAK,CAAC;QAC3C,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QACvB,wBAAwB;QACxB,8EAA8E;QAC9E,4CAA4C;QAC5C,MAAM,sBAAsB,GAAG,IAAI,CAAC,cAAc,GAAG,IAAI,CAAC,aAAa,CAAC;QACxE,MAAM,KAAK,GAAG,GAAG,GAAG,IAAI,CAAC,aAAa,GAAG,IAAI,CAAC,SAAS,CAAC;QACxD,OAAO,KAAK,IAAI,sBAAsB,CAAC;IACzC,CAAC;IAED,KAAK,CAAC,UAAU,GAAG,MAAM;QACvB,IAAI,CAAC,UAAU,GAAG,WAAW,CAAC,GAAG,EAAE;YACjC,IAAI,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC,IAAI,CAAC,WAAW,EAAE,CAAC;gBACvC,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC;gBACxB,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;YACpB,CAAC;QACH,CAAC,EAAE,UAAU,CAAC,CAAC;IACjB,CAAC;IAED,IAAI;QACF,IAAI,IAAI,CAAC,UAAU,EAAE,CAAC;YACpB,aAAa,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;YAC/B,IAAI,CAAC,UAAU,GAAG,IAAI,CAAC;QACzB,CAAC;IACH,CAAC;CACF"}
1
+ {"version":3,"file":"hang-detector.js","sourceRoot":"","sources":["../src/hang-detector.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AAE3C;;;;;;;;;;;;;;;;;;GAkBG;AACH,MAAM,OAAO,YAAa,SAAQ,YAAY;CAAG"}
@@ -4,6 +4,7 @@ import type { Logger } from "./logger.js";
4
4
  import type { IpcClient } from "./channel/ipc-bridge.js";
5
5
  import type { EventLog } from "./event-log.js";
6
6
  import type { TmuxControlClient } from "./tmux-control.js";
7
+ import type { FleetInstructionsParams } from "./instructions.js";
7
8
  export interface BackendInstallationInfo {
8
9
  binary: string;
9
10
  install: string;
@@ -49,6 +50,8 @@ export interface LifecycleContext {
49
50
  touchActivity(name: string): void;
50
51
  sendHangNotification(name: string, unchangedForMs?: number): Promise<void>;
51
52
  notifyInstanceTopic(name: string, text: string): void;
53
+ /** True while the fleet is stopping on purpose or an `agend update` is running. */
54
+ isPlannedRestart(): boolean;
52
55
  /** List claimed tasks for an instance (from task board). Returns empty array if unavailable. */
53
56
  listClaimedTasks(assignee: string): Array<{
54
57
  id: string;
@@ -63,6 +66,11 @@ export interface LifecycleContext {
63
66
  reactMessageStatus(instanceName: string, chatId: string, messageId: string, emoji: string): void;
64
67
  startPersistedPausedInstance(name: string): Promise<void>;
65
68
  }
69
+ /** What attachIncidentHandlers needs from a Daemon — the real one satisfies it. */
70
+ export interface IncidentEventSource {
71
+ on(event: string, handler: (...args: any[]) => void): unknown;
72
+ requestPauseWhenIdle(): void;
73
+ }
66
74
  /** Arguments accepted by handleCreate — mirrors CreateInstanceArgs in outbound-schemas.ts
67
75
  * plus internal-only fields forwarded by deploy_template (profile-derived). */
68
76
  export interface LifecycleCreateArgs {
@@ -98,8 +106,42 @@ export declare class InstanceLifecycle {
98
106
  private ctx;
99
107
  /** Active daemon processes: instanceName → Daemon */
100
108
  readonly daemons: Map<string, import("./daemon.js").Daemon>;
109
+ /** backend → last auth-error alert time, so one expiry sends one alert. */
110
+ private lastAuthAlertAt;
101
111
  constructor(ctx: LifecycleContext);
102
- start(name: string, config: InstanceConfig, topicMode: boolean): Promise<void>;
112
+ /**
113
+ * Report an incident to the user — unless the fleet is deliberately going
114
+ * down, in which case the "incident" is the shutdown doing its job.
115
+ *
116
+ * `agend update` stops instances, kills their MCP servers and restarts the
117
+ * daemon; every one of those looked like a crash to the alert path, so an
118
+ * upgrade produced a burst of ⚠️ for things that were working correctly. That
119
+ * burst is worse than noise: it teaches the operator to ignore the same alert
120
+ * that matters when a process really does die on its own.
121
+ *
122
+ * The event log and daemon.log entries still happen at the call sites — this
123
+ * suppresses the chat message, not the record. A crash outside a planned
124
+ * restart notifies exactly as before.
125
+ */
126
+ private notifyIncident;
127
+ /** Backend a running instance uses (config → fleet default). */
128
+ private backendOf;
129
+ /**
130
+ * One alert per backend per cooldown, naming every affected instance — a CLI's
131
+ * credentials are shared, so N instances failing is ONE problem with ONE fix
132
+ * (re-login once). The per-instance daemon cooldown can't dedupe across
133
+ * instances, so the fleet-level map does it here.
134
+ */
135
+ private notifyAuthErrorOnce;
136
+ /**
137
+ * Handlers for the events that mean "something went wrong".
138
+ *
139
+ * Extracted from start() so the planned-restart suppression can be exercised
140
+ * against a plain event emitter — the alternative is a real Daemon, which
141
+ * means a real tmux window, which is why this rule went untested before.
142
+ */
143
+ attachIncidentHandlers(name: string, daemon: IncidentEventSource): void;
144
+ start(name: string, config: InstanceConfig, topicMode: boolean, runtimeIdentity?: FleetInstructionsParams["runtimeIdentity"]): Promise<void>;
103
145
  isPaused(name: string): boolean;
104
146
  getLastPausedAt(name: string): number | null;
105
147
  pause(name: string): Promise<void>;
@@ -76,14 +76,162 @@ export function getUnsafeInstanceDaemonPidReason(pid, dataDir) {
76
76
  }
77
77
  return null;
78
78
  }
79
+ /** Suppress duplicate auth alerts for the same backend within this window. */
80
+ const AUTH_ALERT_COOLDOWN_MS = 5 * 60_000;
79
81
  export class InstanceLifecycle {
80
82
  ctx;
81
83
  /** Active daemon processes: instanceName → Daemon */
82
84
  daemons = new Map();
85
+ /** backend → last auth-error alert time, so one expiry sends one alert. */
86
+ lastAuthAlertAt = new Map();
83
87
  constructor(ctx) {
84
88
  this.ctx = ctx;
85
89
  }
86
- async start(name, config, topicMode) {
90
+ /**
91
+ * Report an incident to the user — unless the fleet is deliberately going
92
+ * down, in which case the "incident" is the shutdown doing its job.
93
+ *
94
+ * `agend update` stops instances, kills their MCP servers and restarts the
95
+ * daemon; every one of those looked like a crash to the alert path, so an
96
+ * upgrade produced a burst of ⚠️ for things that were working correctly. That
97
+ * burst is worse than noise: it teaches the operator to ignore the same alert
98
+ * that matters when a process really does die on its own.
99
+ *
100
+ * The event log and daemon.log entries still happen at the call sites — this
101
+ * suppresses the chat message, not the record. A crash outside a planned
102
+ * restart notifies exactly as before.
103
+ */
104
+ notifyIncident(name, kind, text) {
105
+ if (this.ctx.isPlannedRestart()) {
106
+ this.ctx.logger.info({ name, kind }, "Incident notification suppressed — planned restart in progress");
107
+ return;
108
+ }
109
+ this.ctx.notifyInstanceTopic(name, text);
110
+ }
111
+ /** Backend a running instance uses (config → fleet default). */
112
+ backendOf(name) {
113
+ return this.ctx.fleetConfig?.instances[name]?.backend
114
+ ?? this.ctx.fleetConfig?.defaults?.backend
115
+ ?? "claude-code";
116
+ }
117
+ /**
118
+ * One alert per backend per cooldown, naming every affected instance — a CLI's
119
+ * credentials are shared, so N instances failing is ONE problem with ONE fix
120
+ * (re-login once). The per-instance daemon cooldown can't dedupe across
121
+ * instances, so the fleet-level map does it here.
122
+ */
123
+ notifyAuthErrorOnce(name, message) {
124
+ const backend = this.backendOf(name);
125
+ const now = Date.now();
126
+ const last = this.lastAuthAlertAt.get(backend) ?? 0;
127
+ if (now - last < AUTH_ALERT_COOLDOWN_MS) {
128
+ this.ctx.logger.info({ name, backend }, "auth error suppressed (backend already alerted)");
129
+ return;
130
+ }
131
+ this.lastAuthAlertAt.set(backend, now);
132
+ const affected = [...this.daemons.keys()].filter(n => this.backendOf(n) === backend);
133
+ const others = affected.filter(n => n !== name);
134
+ const scope = others.length
135
+ ? `${affected.length} instances on \`${backend}\`: ${affected.join(", ")}`
136
+ : `\`${name}\` (${backend})`;
137
+ this.notifyIncident(name, "auth_error", `🔑 ${message}\n\nAffects ${scope}. Credentials are shared per backend — one re-login restores all of them; affected instances pause until then.`);
138
+ }
139
+ /**
140
+ * Handlers for the events that mean "something went wrong".
141
+ *
142
+ * Extracted from start() so the planned-restart suppression can be exercised
143
+ * against a plain event emitter — the alternative is a real Daemon, which
144
+ * means a real tmux window, which is why this rule went untested before.
145
+ */
146
+ attachIncidentHandlers(name, daemon) {
147
+ daemon.on("crash_respawn", safeHandler(() => {
148
+ this.ctx.eventLog?.insert(name, "crash_respawn", {});
149
+ this.ctx.logger.warn({ name }, "Instance crashed and respawned");
150
+ this.notifyIncident(name, "crash_respawn", t("inst.crashed_respawned", name));
151
+ const generalName = this.findGeneralInstance();
152
+ if (generalName && generalName !== name) {
153
+ this.notifyIncident(generalName, "crash_respawn", t("inst.crashed_respawned_log", name));
154
+ }
155
+ }, this.ctx.logger, `daemon.crash_respawn[${name}]`));
156
+ daemon.on("snapshot_failed", safeHandler(() => {
157
+ this.ctx.eventLog?.insert(name, "snapshot_failed", {});
158
+ this.notifyIncident(name, "snapshot_failed", t("inst.restarted_no_context", name));
159
+ }, this.ctx.logger, `daemon.snapshot_failed[${name}]`));
160
+ daemon.on("supervision_ended", safeHandler((data) => {
161
+ // The instance is dead and nothing will restart it. Say so where the operator
162
+ // is looking, and mark the topic — otherwise messages routed here just queue
163
+ // or fail with a bare ❌ and the dashboard still looks normal.
164
+ this.ctx.eventLog?.insert(name, "supervision_ended", { reason: data.reason });
165
+ this.ctx.logger.error({ name, reason: data.reason }, "Instance is no longer supervised");
166
+ this.notifyIncident(name, "supervision_ended", `🛑 ${name} is no longer running and will not be restarted automatically — ${data.reason}.\n${data.remedy}`);
167
+ this.ctx.setTopicIcon(name, "red");
168
+ }, this.ctx.logger, `daemon.supervision_ended[${name}]`));
169
+ daemon.on("health_check_error", safeHandler((data) => {
170
+ this.ctx.eventLog?.insert(name, "health_check_error", { message: data.message });
171
+ this.ctx.logger.error({ name, message: data.message }, "Health check failing — instance supervision degraded");
172
+ this.notifyIncident(name, "health_check_error", `⚠️ ${name}: health check is failing (\`${data.message}\`). Crash detection for this instance may be degraded — see daemon.log.`);
173
+ }, this.ctx.logger, `daemon.health_check_error[${name}]`));
174
+ daemon.on("crash_loop", safeHandler(() => {
175
+ this.ctx.eventLog?.insert(name, "crash_loop", {});
176
+ this.ctx.logger.error({ name }, "Instance in crash loop — respawn paused");
177
+ this.notifyIncident(name, "crash_loop", t("inst.respawn_paused", name));
178
+ this.ctx.setTopicIcon(name, "red");
179
+ }, this.ctx.logger, `daemon.crash_loop[${name}]`));
180
+ daemon.on("mcp_died", safeHandler((data) => {
181
+ this.ctx.eventLog?.insert(name, "mcp_died", { pid: data.pid });
182
+ this.ctx.logger.error({ name, pid: data.pid }, "MCP server died — instance cannot use agend tools");
183
+ this.ctx.webhookEmit("mcp_died", name, { pid: data.pid });
184
+ // The CLI owns the MCP server's stdio pipes, so only restarting the CLI can
185
+ // restore its tools — say so instead of implying self-healing. Deliberately
186
+ // NOT auto-restarting: that would interrupt whatever the agent is doing, and
187
+ // an instance whose CLI is otherwise fine may still be doing useful work.
188
+ this.notifyIncident(name, "mcp_died", `⚠️ \`${name}\` 的 MCP server 已終止 — 這個 instance 目前無法使用 agend 工具(無法 reply / 跨 instance 通訊)。\n`
189
+ + `CLI 本身還在執行。工具只能由 CLI 自己重新啟動 MCP server,請用 \`restart_instance("${name}")\` 或 \`/restart\` 恢復。`);
190
+ }, this.ctx.logger, `daemon.mcp_died[${name}]`));
191
+ daemon.on("pty_error", safeHandler((data) => {
192
+ this.ctx.eventLog?.insert(name, "pty_error", { type: data.type, action: data.action });
193
+ this.ctx.logger.warn({ name, errorType: data.type, action: data.action }, `PTY error: ${data.message}`);
194
+ const emoji = data.type === "rate_limit" || data.type === "timeout" ? "⏳" : data.type === "auth_error" ? "🔑" : "⚠️";
195
+ // Auth failures are a property of the BACKEND's shared credentials, not of
196
+ // one instance: every instance on that CLI fails at once, and one re-login
197
+ // fixes them all. Notify once per backend (listing who's affected) instead
198
+ // of N near-identical alerts, and suppress repeats fleet-wide.
199
+ if (data.type === "auth_error") {
200
+ this.notifyAuthErrorOnce(name, data.message);
201
+ }
202
+ else {
203
+ this.notifyIncident(name, "pty_error", t("inst.notification", emoji, name, data.message, data.action));
204
+ }
205
+ this.ctx.webhookEmit("pty_error", name, { type: data.type, action: data.action, message: data.message });
206
+ // The CLI interrupted itself on this error, so any pending Cancel button is
207
+ // now useless — retire it. We only reach here when the error wasn't
208
+ // cooldown-suppressed (the daemon skips the emit during cooldown), so this
209
+ // won't fire on repeat errors within the 5-min window. No-op if no button.
210
+ this.ctx.clearCancelButton(name);
211
+ if (data.action === "failover") {
212
+ this.ctx.checkModelFailover(name, 100); // Force failover trigger
213
+ }
214
+ else if (data.action === "restart") {
215
+ // A broken *resumed* session (e.g. agy pinned to a dead model) can't
216
+ // self-recover; a plain restart would --continue back into it. freshStart
217
+ // makes the respawn skip resume so the CLI starts a clean session on its
218
+ // default (valid) model.
219
+ this.ctx.restartSingleInstance(name, { freshStart: true }).catch(err => this.ctx.logger.error({ err, name }, "pty_error restart failed"));
220
+ }
221
+ else if (data.action === "pause") {
222
+ // Previously unhandled, so an expired session kept receiving messages and
223
+ // re-sending its whole context into a CLI that could only fail — wasted
224
+ // credit and lost work. pause() alone no-ops while the pane is busy (the
225
+ // usual state when auth fails), so mark it to pause as soon as it idles.
226
+ // Queued messages survive: delivery wakes a paused instance.
227
+ void this.pause(name)
228
+ .catch(err => this.ctx.logger.warn({ err, name }, "auth-error pause failed"))
229
+ .finally(() => { if (!this.isPaused(name))
230
+ daemon.requestPauseWhenIdle(); });
231
+ }
232
+ }, this.ctx.logger, `daemon.pty_error[${name}]`));
233
+ }
234
+ async start(name, config, topicMode, runtimeIdentity) {
87
235
  if (this.daemons.has(name)) {
88
236
  this.ctx.logger.info({ name }, "Instance already running, skipping");
89
237
  return;
@@ -116,7 +264,11 @@ export class InstanceLifecycle {
116
264
  return;
117
265
  }
118
266
  const backend = createBackend(backendName, instanceDir);
119
- const daemon = new Daemon(name, config, instanceDir, topicMode, backend, this.ctx.controlClient ?? undefined, this.ctx.logger);
267
+ const daemon = new Daemon(name, config, instanceDir, topicMode, backend, this.ctx.controlClient ?? undefined, this.ctx.logger, runtimeIdentity ?? {
268
+ kind: "fleet-topic",
269
+ backend: backendName,
270
+ model: config.model ?? "default",
271
+ });
120
272
  // Catch errors from daemon internals (e.g. IPC server) to prevent crashing the fleet process
121
273
  daemon.on("error", (err) => {
122
274
  this.ctx.logger.error({ err, name }, "Daemon emitted error — instance isolated");
@@ -161,47 +313,7 @@ export class InstanceLifecycle {
161
313
  this.ctx.setTopicIcon(name, "green");
162
314
  this.ctx.touchActivity(name);
163
315
  });
164
- daemon.on("crash_respawn", safeHandler(() => {
165
- this.ctx.eventLog?.insert(name, "crash_respawn", {});
166
- this.ctx.logger.warn({ name }, "Instance crashed and respawned");
167
- this.ctx.notifyInstanceTopic(name, t("inst.crashed_respawned", name));
168
- const generalName = this.findGeneralInstance();
169
- if (generalName && generalName !== name) {
170
- this.ctx.notifyInstanceTopic(generalName, t("inst.crashed_respawned_log", name));
171
- }
172
- }, this.ctx.logger, `daemon.crash_respawn[${name}]`));
173
- daemon.on("snapshot_failed", safeHandler(() => {
174
- this.ctx.eventLog?.insert(name, "snapshot_failed", {});
175
- this.ctx.notifyInstanceTopic(name, t("inst.restarted_no_context", name));
176
- }, this.ctx.logger, `daemon.snapshot_failed[${name}]`));
177
- daemon.on("crash_loop", safeHandler(() => {
178
- this.ctx.eventLog?.insert(name, "crash_loop", {});
179
- this.ctx.logger.error({ name }, "Instance in crash loop — respawn paused");
180
- this.ctx.notifyInstanceTopic(name, t("inst.respawn_paused", name));
181
- this.ctx.setTopicIcon(name, "red");
182
- }, this.ctx.logger, `daemon.crash_loop[${name}]`));
183
- daemon.on("pty_error", safeHandler((data) => {
184
- this.ctx.eventLog?.insert(name, "pty_error", { type: data.type, action: data.action });
185
- this.ctx.logger.warn({ name, errorType: data.type, action: data.action }, `PTY error: ${data.message}`);
186
- const emoji = data.type === "rate_limit" || data.type === "timeout" ? "⏳" : data.type === "auth_error" ? "🔑" : "⚠️";
187
- this.ctx.notifyInstanceTopic(name, t("inst.notification", emoji, name, data.message, data.action));
188
- this.ctx.webhookEmit("pty_error", name, { type: data.type, action: data.action, message: data.message });
189
- // The CLI interrupted itself on this error, so any pending Cancel button is
190
- // now useless — retire it. We only reach here when the error wasn't
191
- // cooldown-suppressed (the daemon skips the emit during cooldown), so this
192
- // won't fire on repeat errors within the 5-min window. No-op if no button.
193
- this.ctx.clearCancelButton(name);
194
- if (data.action === "failover") {
195
- this.ctx.checkModelFailover(name, 100); // Force failover trigger
196
- }
197
- else if (data.action === "restart") {
198
- // A broken *resumed* session (e.g. agy pinned to a dead model) can't
199
- // self-recover; a plain restart would --continue back into it. freshStart
200
- // makes the respawn skip resume so the CLI starts a clean session on its
201
- // default (valid) model.
202
- this.ctx.restartSingleInstance(name, { freshStart: true }).catch(err => this.ctx.logger.error({ err, name }, "pty_error restart failed"));
203
- }
204
- }, this.ctx.logger, `daemon.pty_error[${name}]`));
316
+ this.attachIncidentHandlers(name, daemon);
205
317
  daemon.on("pty_recovered", safeHandler((data) => {
206
318
  const mins = Math.floor(data.downtime_s / 60);
207
319
  const secs = data.downtime_s % 60;