@songsid/agend 2.1.2-beta.4 → 2.1.2-beta.41
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/access-path.js +15 -6
- package/dist/access-path.js.map +1 -1
- package/dist/agent-cli-instructions.md +1 -0
- package/dist/agent-cli.js +14 -0
- package/dist/agent-cli.js.map +1 -1
- package/dist/agent-endpoint.d.ts +2 -0
- package/dist/agent-endpoint.js +25 -1
- package/dist/agent-endpoint.js.map +1 -1
- package/dist/backend/antigravity.d.ts +20 -0
- package/dist/backend/antigravity.js +51 -4
- package/dist/backend/antigravity.js.map +1 -1
- package/dist/backend/claude-code.d.ts +59 -0
- package/dist/backend/claude-code.js +67 -1
- package/dist/backend/claude-code.js.map +1 -1
- package/dist/backend/codex.d.ts +27 -0
- package/dist/backend/codex.js +93 -0
- package/dist/backend/codex.js.map +1 -1
- package/dist/backend/grok.d.ts +25 -0
- package/dist/backend/grok.js +51 -0
- package/dist/backend/grok.js.map +1 -1
- package/dist/backend/kiro.d.ts +22 -0
- package/dist/backend/kiro.js +79 -1
- package/dist/backend/kiro.js.map +1 -1
- package/dist/backend/types.d.ts +58 -0
- package/dist/backend/types.js +13 -1
- package/dist/backend/types.js.map +1 -1
- package/dist/channel/adapters/discord.d.ts +15 -0
- package/dist/channel/adapters/discord.js +100 -2
- package/dist/channel/adapters/discord.js.map +1 -1
- package/dist/channel/adapters/telegram.d.ts +21 -0
- package/dist/channel/adapters/telegram.js +86 -0
- package/dist/channel/adapters/telegram.js.map +1 -1
- package/dist/channel/ipc-bridge.d.ts +9 -1
- package/dist/channel/ipc-bridge.js +12 -3
- package/dist/channel/ipc-bridge.js.map +1 -1
- package/dist/channel/ipc-timeouts.d.ts +46 -0
- package/dist/channel/ipc-timeouts.js +65 -0
- package/dist/channel/ipc-timeouts.js.map +1 -0
- package/dist/channel/mcp-server.js +30 -14
- package/dist/channel/mcp-server.js.map +1 -1
- package/dist/channel/mcp-tools.js +30 -1
- package/dist/channel/mcp-tools.js.map +1 -1
- package/dist/channel/reconnect-backoff.d.ts +17 -0
- package/dist/channel/reconnect-backoff.js +21 -0
- package/dist/channel/reconnect-backoff.js.map +1 -0
- package/dist/channel/types.d.ts +29 -0
- package/dist/classic-channel-manager.js +1 -5
- package/dist/classic-channel-manager.js.map +1 -1
- package/dist/cli.js +227 -87
- package/dist/cli.js.map +1 -1
- package/dist/completion.d.ts +27 -0
- package/dist/completion.js +121 -0
- package/dist/completion.js.map +1 -0
- package/dist/config-validator.js +21 -0
- package/dist/config-validator.js.map +1 -1
- package/dist/config.js +1 -0
- package/dist/config.js.map +1 -1
- package/dist/cost-guard.d.ts +3 -1
- package/dist/cost-guard.js +3 -1
- package/dist/cost-guard.js.map +1 -1
- package/dist/daemon.d.ts +282 -14
- package/dist/daemon.js +1062 -311
- package/dist/daemon.js.map +1 -1
- package/dist/event-log.d.ts +31 -0
- package/dist/event-log.js +96 -0
- package/dist/event-log.js.map +1 -1
- package/dist/fleet-context.d.ts +9 -0
- package/dist/fleet-lock.d.ts +28 -0
- package/dist/fleet-lock.js +130 -0
- package/dist/fleet-lock.js.map +1 -0
- package/dist/fleet-manager.d.ts +315 -2
- package/dist/fleet-manager.js +1414 -108
- package/dist/fleet-manager.js.map +1 -1
- package/dist/general-knowledge/skills/cross-instance-messaging/SKILL.md +22 -0
- package/dist/general-knowledge/skills/model-discovery/SKILL.md +14 -23
- package/dist/general-knowledge/skills/session-management/SKILL.md +15 -34
- package/dist/hang-detector.d.ts +19 -13
- package/dist/hang-detector.js +19 -49
- package/dist/hang-detector.js.map +1 -1
- package/dist/instance-lifecycle.d.ts +52 -3
- package/dist/instance-lifecycle.js +194 -48
- package/dist/instance-lifecycle.js.map +1 -1
- package/dist/instructions.d.ts +5 -0
- package/dist/instructions.js +9 -11
- package/dist/instructions.js.map +1 -1
- package/dist/locale.js +5 -1
- package/dist/locale.js.map +1 -1
- package/dist/logger.js +14 -0
- package/dist/logger.js.map +1 -1
- package/dist/mcp-liveness.d.ts +21 -0
- package/dist/mcp-liveness.js +27 -0
- package/dist/mcp-liveness.js.map +1 -0
- package/dist/outbound-handlers.d.ts +16 -0
- package/dist/outbound-handlers.js +151 -27
- package/dist/outbound-handlers.js.map +1 -1
- package/dist/outbound-schemas.d.ts +11 -3
- package/dist/outbound-schemas.js +16 -2
- package/dist/outbound-schemas.js.map +1 -1
- package/dist/pane-write-lock.d.ts +48 -0
- package/dist/pane-write-lock.js +73 -0
- package/dist/pane-write-lock.js.map +1 -0
- package/dist/process-memory.d.ts +31 -0
- package/dist/process-memory.js +79 -0
- package/dist/process-memory.js.map +1 -0
- package/dist/quickstart.js +17 -16
- package/dist/quickstart.js.map +1 -1
- package/dist/reply-dedup.d.ts +41 -0
- package/dist/reply-dedup.js +0 -0
- package/dist/reply-dedup.js.map +1 -0
- package/dist/scheduler/db.js +3 -0
- package/dist/scheduler/db.js.map +1 -1
- package/dist/sd-notify.d.ts +27 -0
- package/dist/sd-notify.js +33 -1
- package/dist/sd-notify.js.map +1 -1
- package/dist/secret-file.d.ts +33 -0
- package/dist/secret-file.js +36 -0
- package/dist/secret-file.js.map +1 -0
- package/dist/service-installer.d.ts +16 -0
- package/dist/service-installer.js +56 -1
- package/dist/service-installer.js.map +1 -1
- package/dist/setup-wizard.js +9 -7
- package/dist/setup-wizard.js.map +1 -1
- package/dist/tmux-control.d.ts +58 -5
- package/dist/tmux-control.js +102 -14
- package/dist/tmux-control.js.map +1 -1
- package/dist/tmux-manager.d.ts +31 -1
- package/dist/tmux-manager.js +40 -11
- package/dist/tmux-manager.js.map +1 -1
- package/dist/topic-commands.d.ts +49 -9
- package/dist/topic-commands.js +267 -105
- package/dist/topic-commands.js.map +1 -1
- package/dist/tui-glyphs.d.ts +5 -1
- package/dist/tui-glyphs.js +6 -2
- package/dist/tui-glyphs.js.map +1 -1
- package/dist/types.d.ts +12 -2
- package/dist/ui/view.html +123 -2
- package/dist/update-marker.d.ts +15 -0
- package/dist/update-marker.js +69 -0
- package/dist/update-marker.js.map +1 -0
- package/dist/usage/format-rich.d.ts +11 -0
- package/dist/usage/format-rich.js +121 -0
- package/dist/usage/format-rich.js.map +1 -0
- package/dist/usage/provider-alerts.d.ts +33 -0
- package/dist/usage/provider-alerts.js +57 -0
- package/dist/usage/provider-alerts.js.map +1 -0
- package/dist/usage/providers.d.ts +130 -0
- package/dist/usage/providers.js +1063 -0
- package/dist/usage/providers.js.map +1 -0
- package/dist/usage/statusline-usage.d.ts +20 -0
- package/dist/usage/statusline-usage.js +120 -0
- package/dist/usage/statusline-usage.js.map +1 -0
- package/dist/usage/usage-api.d.ts +50 -0
- package/dist/usage/usage-api.js +197 -0
- package/dist/usage/usage-api.js.map +1 -0
- package/dist/view-api.d.ts +3 -0
- package/dist/view-api.js +11 -2
- package/dist/view-api.js.map +1 -1
- package/dist/web-api.js +5 -2
- package/dist/web-api.js.map +1 -1
- package/package.json +4 -1
- package/dist/channel/tool-tracker.d.ts +0 -13
- package/dist/channel/tool-tracker.js +0 -58
- package/dist/channel/tool-tracker.js.map +0 -1
- package/dist/daemon-entry.d.ts +0 -1
- package/dist/daemon-entry.js +0 -30
- package/dist/daemon-entry.js.map +0 -1
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
---
|
|
2
|
+
name: cross-instance-messaging
|
|
3
|
+
description: Fire-and-queue cross-instance tools — send once, never resend on queued
|
|
4
|
+
---
|
|
5
|
+
|
|
6
|
+
## How to send
|
|
7
|
+
|
|
8
|
+
Use fleet tools only (`send_to_instance`, `delegate_task`, `request_information`, `report_result`, `broadcast`):
|
|
9
|
+
- Call returns immediately: `{ sent: true, queued: true }` — success, fleet owns delivery.
|
|
10
|
+
- **Do not wait** for the target to go idle; **do not** treat 30s IPC timeout as failure to re-send.
|
|
11
|
+
- **Error only if the target does not exist** (or similar hard reject) — then fix the name, don't spam.
|
|
12
|
+
- **Never re-send because the reply said `queued`** — that means the message is already queued.
|
|
13
|
+
|
|
14
|
+
## requires_reply
|
|
15
|
+
|
|
16
|
+
- Means “target should later answer with `report_result` / a real reply”
|
|
17
|
+
- **Not** a synchronous wait for their turn to finish
|
|
18
|
+
|
|
19
|
+
## Task flow
|
|
20
|
+
|
|
21
|
+
- `delegate_task` → silent work → `report_result` (zero ack-only pings)
|
|
22
|
+
- Cross-instance traffic is `[from:name]` → answer with `send_to_instance` / `report_result`, never `reply`
|
|
@@ -1,36 +1,27 @@
|
|
|
1
1
|
---
|
|
2
2
|
name: model-discovery
|
|
3
|
-
description:
|
|
3
|
+
description: Set and discover models — pass-through to the CLI, no AgEnD allowlist gate
|
|
4
4
|
---
|
|
5
5
|
|
|
6
|
-
##
|
|
6
|
+
## How to set a model
|
|
7
7
|
|
|
8
|
-
|
|
8
|
+
- fleet.yaml: `defaults.model` or per-instance `model`
|
|
9
|
+
- **Pass-through:** AgEnD no longer blocks unknown model ids — it may **warn**, then still pass the string to the CLI
|
|
10
|
+
- **CLI is source of truth** — if the model is invalid, the backend CLI errors (fix the name there)
|
|
9
11
|
|
|
10
|
-
|
|
11
|
-
|---------|-------------------|---------|
|
|
12
|
-
| **kiro-cli** | In tmux: send `/model` + Enter → read model list → Esc to close | auto (latest) |
|
|
13
|
-
| **claude-code** | `sonnet`, `opus`, `haiku`, `opusplan`, `best`, `sonnet[1m]`, `opus[1m]` | sonnet |
|
|
14
|
-
| **antigravity** | Run `agy models` to see available models | Gemini 3.5 Flash (Medium) |
|
|
15
|
-
| **codex** | `gpt-4o`, `o3`, `o4-mini` | gpt-4o |
|
|
16
|
-
| **opencode** | `opencode models` | depends on provider |
|
|
17
|
-
| **grok** | `grok-4.5`, `grok-4.3`, `grok-code`, `grok-build-0.1` | grok default |
|
|
12
|
+
## Discover real names
|
|
18
13
|
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
-
|
|
22
|
-
-
|
|
23
|
-
|
|
14
|
+
| Backend | How |
|
|
15
|
+
|---------|-----|
|
|
16
|
+
| kiro-cli | In pane: `/model` (gpt-*, deepseek-*, minimax-*, glm-*, qwen* supported) |
|
|
17
|
+
| claude-code | `sonnet` / `opus` / `haiku` / `opusplan` / `Fable` / aliases |
|
|
18
|
+
| codex | pane `/model` or docs (`gpt-*`, `o*`) |
|
|
19
|
+
| grok | `grok models` |
|
|
20
|
+
| antigravity | `agy models` — set **base name only** (drop `(Medium)` / `(Thinking)` effort suffix) |
|
|
21
|
+
| opencode | `opencode models` |
|
|
24
22
|
|
|
25
|
-
**Important:** Model names vary by backend. Always check the actual CLI output rather than guessing names.
|
|
26
|
-
|
|
27
|
-
Example fleet.yaml:
|
|
28
23
|
```yaml
|
|
29
24
|
defaults:
|
|
30
25
|
backend: kiro-cli
|
|
31
26
|
model: claude-sonnet-4-20250514
|
|
32
|
-
|
|
33
|
-
instances:
|
|
34
|
-
heavy-task:
|
|
35
|
-
model: claude-opus-4-20250514
|
|
36
27
|
```
|
|
@@ -1,46 +1,27 @@
|
|
|
1
1
|
---
|
|
2
2
|
name: session-management
|
|
3
|
-
description:
|
|
3
|
+
description: Session stores, forking, and auth-pause recovery
|
|
4
4
|
---
|
|
5
5
|
|
|
6
|
+
## Auth failure (auto-pause)
|
|
7
|
+
|
|
8
|
+
When AgEnD sees `auth_error` it **pauses** that instance (`pausePending` sticky):
|
|
9
|
+
- One notification **per backend** (not per instance) — log in once, same-backend peers recover.
|
|
10
|
+
- Auth is **per-user global** across backends: claude-code, codex, kiro, grok, opencode, antigravity.
|
|
11
|
+
- Messages while paused stay in the **queue** — do not re-send.
|
|
12
|
+
- After the user re-auths: `wake` / normal wake clears `pausePending`.
|
|
13
|
+
|
|
6
14
|
## Where sessions live
|
|
7
15
|
|
|
8
16
|
| | kiro-cli | claude-code |
|
|
9
17
|
|---|---|---|
|
|
10
18
|
| Store | `~/.kiro/sessions/cli/<uuid>.json` | `~/.claude/projects/<path-encoded>/*.jsonl` |
|
|
11
|
-
|
|
|
12
|
-
| Reload | `/chat load <file>` | none — `--continue` (latest for the dir) / `--resume <id>` |
|
|
13
|
-
| Text export | — | `/export` (plain text, **not** reloadable) |
|
|
14
|
-
|
|
15
|
-
`<path-encoded>` = the absolute working_directory with `/` → `-`
|
|
16
|
-
(e.g. `/home/han/Projects/AgEnD` → `-home-han-Projects-AgEnD`).
|
|
17
|
-
|
|
18
|
-
## Fork a session to a new instance
|
|
19
|
-
|
|
20
|
-
Confirm the source is **idle** first (`describe_instance` / `get_fleet_status`) — don't fork mid-task.
|
|
21
|
-
|
|
22
|
-
**kiro-cli** — save, copy, load:
|
|
23
|
-
1. On the source, save: `/chat save <name>.json -f` (paste via tmux if needed).
|
|
24
|
-
2. `create_instance` (same backend).
|
|
25
|
-
3. `cp ~/.agend/workspaces/<source>/<name>.json ~/.agend/workspaces/<target>/`
|
|
26
|
-
4. Load on the target: `/chat load <name>.json`, or set `pre_task_command: "/chat load <name>.json"`.
|
|
27
|
-
|
|
28
|
-
**claude-code** — copy the `.jsonl` (no save/load command):
|
|
29
|
-
1. Newest source session: `ls -lt ~/.claude/projects/<source-encoded>/*.jsonl | head`
|
|
30
|
-
2. `create_instance` (backend `claude-code`); note its working_directory.
|
|
31
|
-
3. Copy into the target's encoded project dir:
|
|
32
|
-
```bash
|
|
33
|
-
TARGET_ENC="$(echo '<target-working-dir>' | sed 's#/#-#g')"
|
|
34
|
-
mkdir -p ~/.claude/projects/$TARGET_ENC
|
|
35
|
-
cp ~/.claude/projects/<source-encoded>/<session>.jsonl ~/.claude/projects/$TARGET_ENC/
|
|
36
|
-
```
|
|
37
|
-
4. Start the target — claude-code resumes the newest `.jsonl` via `--continue`.
|
|
19
|
+
| Reload | `/chat load <file>` | `--continue` / `--resume <id>` |
|
|
38
20
|
|
|
39
|
-
|
|
40
|
-
(paths inside the transcript refer to it). Pick the right `.jsonl` if several exist (newest by
|
|
41
|
-
mtime; compaction/branches create new files). `/export` is text only, not reloadable.
|
|
21
|
+
`<path-encoded>` = absolute cwd with `/` → `-`.
|
|
42
22
|
|
|
43
|
-
##
|
|
23
|
+
## Fork (source must be idle)
|
|
44
24
|
|
|
45
|
-
|
|
46
|
-
|
|
25
|
+
- **kiro:** `/chat save name.json -f` → `create_instance` → copy workspace file → `/chat load name.json`
|
|
26
|
+
- **claude-code:** copy newest `*.jsonl` into target's encoded project dir → start (uses `--continue`)
|
|
27
|
+
- Prefer `replace_instance` when the whole session is poisoned (see instance-lifecycle)
|
package/dist/hang-detector.d.ts
CHANGED
|
@@ -1,16 +1,22 @@
|
|
|
1
1
|
import { EventEmitter } from "node:events";
|
|
2
|
+
/**
|
|
3
|
+
* Carries "this instance looks hung" from the daemon to the fleet manager.
|
|
4
|
+
*
|
|
5
|
+
* It is only an event bridge. Hang detection itself lives in the daemon's pane-state
|
|
6
|
+
* machine, which emits `hang` directly when a pane stops changing for the configured
|
|
7
|
+
* stuck timeout; `instance-lifecycle` subscribes to that and notifies.
|
|
8
|
+
*
|
|
9
|
+
* It used to also contain a silence-timer state machine — `start()`, `isHung()`,
|
|
10
|
+
* `hungEmitted`, and timestamps fed by `recordActivity` / `recordInbound` /
|
|
11
|
+
* `recordStatuslineUpdate`. None of it ran: `start()` was never called from
|
|
12
|
+
* anywhere, so `isHung()` was unreachable, the timestamps were written and never
|
|
13
|
+
* read, and the constructor's `timeoutMinutes` was ignored (the real stuck timeout
|
|
14
|
+
* is read separately by the pane monitor). Its tests exercised that dead logic —
|
|
15
|
+
* two had identical setup with contradictory expectations, which can only pass
|
|
16
|
+
* unnoticed when neither runs against anything real.
|
|
17
|
+
*
|
|
18
|
+
* Kept as a named class rather than a bare EventEmitter so the daemon → lifecycle
|
|
19
|
+
* wiring stays typed and greppable.
|
|
20
|
+
*/
|
|
2
21
|
export declare class HangDetector extends EventEmitter {
|
|
3
|
-
private lastActivityTs;
|
|
4
|
-
private lastStatuslineTs;
|
|
5
|
-
private lastInboundTs;
|
|
6
|
-
private hungEmitted;
|
|
7
|
-
private checkTimer;
|
|
8
|
-
private timeoutMs;
|
|
9
|
-
constructor(timeoutMinutes: number);
|
|
10
|
-
recordActivity(): void;
|
|
11
|
-
recordInbound(): void;
|
|
12
|
-
recordStatuslineUpdate(): void;
|
|
13
|
-
isHung(): boolean;
|
|
14
|
-
start(intervalMs?: number): void;
|
|
15
|
-
stop(): void;
|
|
16
22
|
}
|
package/dist/hang-detector.js
CHANGED
|
@@ -1,53 +1,23 @@
|
|
|
1
1
|
import { EventEmitter } from "node:events";
|
|
2
|
+
/**
|
|
3
|
+
* Carries "this instance looks hung" from the daemon to the fleet manager.
|
|
4
|
+
*
|
|
5
|
+
* It is only an event bridge. Hang detection itself lives in the daemon's pane-state
|
|
6
|
+
* machine, which emits `hang` directly when a pane stops changing for the configured
|
|
7
|
+
* stuck timeout; `instance-lifecycle` subscribes to that and notifies.
|
|
8
|
+
*
|
|
9
|
+
* It used to also contain a silence-timer state machine — `start()`, `isHung()`,
|
|
10
|
+
* `hungEmitted`, and timestamps fed by `recordActivity` / `recordInbound` /
|
|
11
|
+
* `recordStatuslineUpdate`. None of it ran: `start()` was never called from
|
|
12
|
+
* anywhere, so `isHung()` was unreachable, the timestamps were written and never
|
|
13
|
+
* read, and the constructor's `timeoutMinutes` was ignored (the real stuck timeout
|
|
14
|
+
* is read separately by the pane monitor). Its tests exercised that dead logic —
|
|
15
|
+
* two had identical setup with contradictory expectations, which can only pass
|
|
16
|
+
* unnoticed when neither runs against anything real.
|
|
17
|
+
*
|
|
18
|
+
* Kept as a named class rather than a bare EventEmitter so the daemon → lifecycle
|
|
19
|
+
* wiring stays typed and greppable.
|
|
20
|
+
*/
|
|
2
21
|
export class HangDetector extends EventEmitter {
|
|
3
|
-
lastActivityTs = 0;
|
|
4
|
-
lastStatuslineTs = 0;
|
|
5
|
-
lastInboundTs = 0;
|
|
6
|
-
hungEmitted = false;
|
|
7
|
-
checkTimer = null;
|
|
8
|
-
timeoutMs;
|
|
9
|
-
constructor(timeoutMinutes) {
|
|
10
|
-
super();
|
|
11
|
-
this.timeoutMs = timeoutMinutes * 60 * 1000;
|
|
12
|
-
}
|
|
13
|
-
recordActivity() {
|
|
14
|
-
this.lastActivityTs = Date.now();
|
|
15
|
-
if (this.hungEmitted) {
|
|
16
|
-
this.hungEmitted = false;
|
|
17
|
-
}
|
|
18
|
-
}
|
|
19
|
-
recordInbound() {
|
|
20
|
-
this.lastInboundTs = Date.now();
|
|
21
|
-
}
|
|
22
|
-
recordStatuslineUpdate() {
|
|
23
|
-
this.lastStatuslineTs = Date.now();
|
|
24
|
-
}
|
|
25
|
-
isHung() {
|
|
26
|
-
if (this.lastActivityTs === 0)
|
|
27
|
-
return false;
|
|
28
|
-
if (this.lastInboundTs === 0)
|
|
29
|
-
return false;
|
|
30
|
-
const now = Date.now();
|
|
31
|
-
// Only flag as hung if:
|
|
32
|
-
// 1. There's an inbound that hasn't been answered (no activity since inbound)
|
|
33
|
-
// 2. Timeout has elapsed since that inbound
|
|
34
|
-
const noActivitySinceInbound = this.lastActivityTs < this.lastInboundTs;
|
|
35
|
-
const stale = now - this.lastInboundTs > this.timeoutMs;
|
|
36
|
-
return stale && noActivitySinceInbound;
|
|
37
|
-
}
|
|
38
|
-
start(intervalMs = 60_000) {
|
|
39
|
-
this.checkTimer = setInterval(() => {
|
|
40
|
-
if (this.isHung() && !this.hungEmitted) {
|
|
41
|
-
this.hungEmitted = true;
|
|
42
|
-
this.emit("hang");
|
|
43
|
-
}
|
|
44
|
-
}, intervalMs);
|
|
45
|
-
}
|
|
46
|
-
stop() {
|
|
47
|
-
if (this.checkTimer) {
|
|
48
|
-
clearInterval(this.checkTimer);
|
|
49
|
-
this.checkTimer = null;
|
|
50
|
-
}
|
|
51
|
-
}
|
|
52
22
|
}
|
|
53
23
|
//# sourceMappingURL=hang-detector.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"hang-detector.js","sourceRoot":"","sources":["../src/hang-detector.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AAE3C,MAAM,OAAO,YAAa,SAAQ,YAAY;
|
|
1
|
+
{"version":3,"file":"hang-detector.js","sourceRoot":"","sources":["../src/hang-detector.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AAE3C;;;;;;;;;;;;;;;;;;GAkBG;AACH,MAAM,OAAO,YAAa,SAAQ,YAAY;CAAG"}
|
|
@@ -4,6 +4,8 @@ import type { Logger } from "./logger.js";
|
|
|
4
4
|
import type { IpcClient } from "./channel/ipc-bridge.js";
|
|
5
5
|
import type { EventLog } from "./event-log.js";
|
|
6
6
|
import type { TmuxControlClient } from "./tmux-control.js";
|
|
7
|
+
import type { FleetInstructionsParams } from "./instructions.js";
|
|
8
|
+
export { isFleetStartCommandLine } from "./fleet-lock.js";
|
|
7
9
|
export interface BackendInstallationInfo {
|
|
8
10
|
binary: string;
|
|
9
11
|
install: string;
|
|
@@ -12,8 +14,6 @@ export interface BackendInstallationInfo {
|
|
|
12
14
|
export declare const BACKEND_INSTALLATION_INFO: Readonly<Record<string, BackendInstallationInfo>>;
|
|
13
15
|
/** Check one executable using the same PATH visible to the fleet process. */
|
|
14
16
|
export declare function checkBinaryInstalled(binary: string): boolean;
|
|
15
|
-
/** Whether a process command line identifies the shared AgEnD fleet process. */
|
|
16
|
-
export declare function isFleetStartCommandLine(commandLine: string): boolean;
|
|
17
17
|
/**
|
|
18
18
|
* Return why an instance daemon PID must not receive SIGTERM, or null when the
|
|
19
19
|
* stale-process cleanup may proceed. daemon.pid historically stores the shared
|
|
@@ -49,6 +49,10 @@ export interface LifecycleContext {
|
|
|
49
49
|
touchActivity(name: string): void;
|
|
50
50
|
sendHangNotification(name: string, unchangedForMs?: number): Promise<void>;
|
|
51
51
|
notifyInstanceTopic(name: string, text: string): void;
|
|
52
|
+
/** True for a dynamic ClassicBot channel instance (not a fleet topic). */
|
|
53
|
+
isClassicInstance?(name: string): boolean;
|
|
54
|
+
/** True while the fleet is stopping on purpose or an `agend update` is running. */
|
|
55
|
+
isPlannedRestart(): boolean;
|
|
52
56
|
/** List claimed tasks for an instance (from task board). Returns empty array if unavailable. */
|
|
53
57
|
listClaimedTasks(assignee: string): Array<{
|
|
54
58
|
id: string;
|
|
@@ -63,6 +67,11 @@ export interface LifecycleContext {
|
|
|
63
67
|
reactMessageStatus(instanceName: string, chatId: string, messageId: string, emoji: string): void;
|
|
64
68
|
startPersistedPausedInstance(name: string): Promise<void>;
|
|
65
69
|
}
|
|
70
|
+
/** What attachIncidentHandlers needs from a Daemon — the real one satisfies it. */
|
|
71
|
+
export interface IncidentEventSource {
|
|
72
|
+
on(event: string, handler: (...args: any[]) => void): unknown;
|
|
73
|
+
requestPauseWhenIdle(): void;
|
|
74
|
+
}
|
|
66
75
|
/** Arguments accepted by handleCreate — mirrors CreateInstanceArgs in outbound-schemas.ts
|
|
67
76
|
* plus internal-only fields forwarded by deploy_template (profile-derived). */
|
|
68
77
|
export interface LifecycleCreateArgs {
|
|
@@ -98,8 +107,48 @@ export declare class InstanceLifecycle {
|
|
|
98
107
|
private ctx;
|
|
99
108
|
/** Active daemon processes: instanceName → Daemon */
|
|
100
109
|
readonly daemons: Map<string, import("./daemon.js").Daemon>;
|
|
110
|
+
/** backend → last auth-error alert time, so one expiry sends one alert. */
|
|
111
|
+
private lastAuthAlertAt;
|
|
101
112
|
constructor(ctx: LifecycleContext);
|
|
102
|
-
|
|
113
|
+
/**
|
|
114
|
+
* Report an incident to the user — unless the fleet is deliberately going
|
|
115
|
+
* down, in which case the "incident" is the shutdown doing its job.
|
|
116
|
+
*
|
|
117
|
+
* `agend update` stops instances, kills their MCP servers and restarts the
|
|
118
|
+
* daemon; every one of those looked like a crash to the alert path, so an
|
|
119
|
+
* upgrade produced a burst of ⚠️ for things that were working correctly. That
|
|
120
|
+
* burst is worse than noise: it teaches the operator to ignore the same alert
|
|
121
|
+
* that matters when a process really does die on its own.
|
|
122
|
+
*
|
|
123
|
+
* The event log and daemon.log entries still happen at the call sites — this
|
|
124
|
+
* suppresses the chat message, not the record. A crash outside a planned
|
|
125
|
+
* restart notifies exactly as before.
|
|
126
|
+
*/
|
|
127
|
+
private notifyIncident;
|
|
128
|
+
/** Backend a running instance uses (config → fleet default). */
|
|
129
|
+
private backendOf;
|
|
130
|
+
/**
|
|
131
|
+
* One alert per backend per cooldown, naming every affected instance — a CLI's
|
|
132
|
+
* credentials are shared, so N instances failing is ONE problem with ONE fix
|
|
133
|
+
* (re-login once). The per-instance daemon cooldown can't dedupe across
|
|
134
|
+
* instances, so the fleet-level map does it here.
|
|
135
|
+
*/
|
|
136
|
+
private notifyAuthErrorOnce;
|
|
137
|
+
/**
|
|
138
|
+
* System errors from a ClassicBot belong in the operator's General topic,
|
|
139
|
+
* never in the end user's chat channel. Fleet-topic instances retain their
|
|
140
|
+
* existing local notification target.
|
|
141
|
+
*/
|
|
142
|
+
private ptyErrorNotificationTarget;
|
|
143
|
+
/**
|
|
144
|
+
* Handlers for the events that mean "something went wrong".
|
|
145
|
+
*
|
|
146
|
+
* Extracted from start() so the planned-restart suppression can be exercised
|
|
147
|
+
* against a plain event emitter — the alternative is a real Daemon, which
|
|
148
|
+
* means a real tmux window, which is why this rule went untested before.
|
|
149
|
+
*/
|
|
150
|
+
attachIncidentHandlers(name: string, daemon: IncidentEventSource): void;
|
|
151
|
+
start(name: string, config: InstanceConfig, topicMode: boolean, runtimeIdentity?: FleetInstructionsParams["runtimeIdentity"]): Promise<void>;
|
|
103
152
|
isPaused(name: string): boolean;
|
|
104
153
|
getLastPausedAt(name: string): number | null;
|
|
105
154
|
pause(name: string): Promise<void>;
|
|
@@ -9,6 +9,9 @@ import { isModelCompatible } from "./backend/types.js";
|
|
|
9
9
|
import { safeHandler } from "./safe-async.js";
|
|
10
10
|
import { t } from "./locale.js";
|
|
11
11
|
import { clearPausedMarker, hasPausedMarker, readPausedAt, writePausedMarker } from "./pause-marker.js";
|
|
12
|
+
import { reportProviderRateLimit } from "./usage/provider-alerts.js";
|
|
13
|
+
import { isFleetStartCommandLine } from "./fleet-lock.js";
|
|
14
|
+
export { isFleetStartCommandLine } from "./fleet-lock.js";
|
|
12
15
|
/** Shared CLI metadata used by startup validation and ClassicBot onboarding. */
|
|
13
16
|
export const BACKEND_INSTALLATION_INFO = {
|
|
14
17
|
"claude-code": { binary: "claude", install: "curl -fsSL https://claude.ai/install.sh | bash" },
|
|
@@ -29,11 +32,6 @@ export function checkBinaryInstalled(binary) {
|
|
|
29
32
|
return false;
|
|
30
33
|
}
|
|
31
34
|
}
|
|
32
|
-
/** Whether a process command line identifies the shared AgEnD fleet process. */
|
|
33
|
-
export function isFleetStartCommandLine(commandLine) {
|
|
34
|
-
const normalized = commandLine.replace(/\0/g, " ").replace(/\s+/g, " ").trim();
|
|
35
|
-
return /\b(?:agend|(?:cli|daemon-entry)\.(?:js|ts))\b.*\bfleet\s+start\b/i.test(normalized);
|
|
36
|
-
}
|
|
37
35
|
function readProcessCommandLine(pid) {
|
|
38
36
|
try {
|
|
39
37
|
return readFileSync(`/proc/${pid}/cmdline`, "utf8").replace(/\0/g, " ").trim();
|
|
@@ -76,14 +74,198 @@ export function getUnsafeInstanceDaemonPidReason(pid, dataDir) {
|
|
|
76
74
|
}
|
|
77
75
|
return null;
|
|
78
76
|
}
|
|
77
|
+
/** Suppress duplicate auth alerts for the same backend within this window. */
|
|
78
|
+
const AUTH_ALERT_COOLDOWN_MS = 5 * 60_000;
|
|
79
79
|
export class InstanceLifecycle {
|
|
80
80
|
ctx;
|
|
81
81
|
/** Active daemon processes: instanceName → Daemon */
|
|
82
82
|
daemons = new Map();
|
|
83
|
+
/** backend → last auth-error alert time, so one expiry sends one alert. */
|
|
84
|
+
lastAuthAlertAt = new Map();
|
|
83
85
|
constructor(ctx) {
|
|
84
86
|
this.ctx = ctx;
|
|
85
87
|
}
|
|
86
|
-
|
|
88
|
+
/**
|
|
89
|
+
* Report an incident to the user — unless the fleet is deliberately going
|
|
90
|
+
* down, in which case the "incident" is the shutdown doing its job.
|
|
91
|
+
*
|
|
92
|
+
* `agend update` stops instances, kills their MCP servers and restarts the
|
|
93
|
+
* daemon; every one of those looked like a crash to the alert path, so an
|
|
94
|
+
* upgrade produced a burst of ⚠️ for things that were working correctly. That
|
|
95
|
+
* burst is worse than noise: it teaches the operator to ignore the same alert
|
|
96
|
+
* that matters when a process really does die on its own.
|
|
97
|
+
*
|
|
98
|
+
* The event log and daemon.log entries still happen at the call sites — this
|
|
99
|
+
* suppresses the chat message, not the record. A crash outside a planned
|
|
100
|
+
* restart notifies exactly as before.
|
|
101
|
+
*/
|
|
102
|
+
notifyIncident(name, kind, text) {
|
|
103
|
+
if (this.ctx.isPlannedRestart()) {
|
|
104
|
+
this.ctx.logger.info({ name, kind }, "Incident notification suppressed — planned restart in progress");
|
|
105
|
+
return;
|
|
106
|
+
}
|
|
107
|
+
this.ctx.notifyInstanceTopic(name, text);
|
|
108
|
+
}
|
|
109
|
+
/** Backend a running instance uses (config → fleet default). */
|
|
110
|
+
backendOf(name) {
|
|
111
|
+
return this.ctx.fleetConfig?.instances[name]?.backend
|
|
112
|
+
?? this.ctx.fleetConfig?.defaults?.backend
|
|
113
|
+
?? "claude-code";
|
|
114
|
+
}
|
|
115
|
+
/**
|
|
116
|
+
* One alert per backend per cooldown, naming every affected instance — a CLI's
|
|
117
|
+
* credentials are shared, so N instances failing is ONE problem with ONE fix
|
|
118
|
+
* (re-login once). The per-instance daemon cooldown can't dedupe across
|
|
119
|
+
* instances, so the fleet-level map does it here.
|
|
120
|
+
*/
|
|
121
|
+
notifyAuthErrorOnce(name, message, notificationTarget = name) {
|
|
122
|
+
const backend = this.backendOf(name);
|
|
123
|
+
const now = Date.now();
|
|
124
|
+
const last = this.lastAuthAlertAt.get(backend) ?? 0;
|
|
125
|
+
if (now - last < AUTH_ALERT_COOLDOWN_MS) {
|
|
126
|
+
this.ctx.logger.info({ name, backend }, "auth error suppressed (backend already alerted)");
|
|
127
|
+
return;
|
|
128
|
+
}
|
|
129
|
+
this.lastAuthAlertAt.set(backend, now);
|
|
130
|
+
const affected = [...this.daemons.keys()].filter(n => this.backendOf(n) === backend);
|
|
131
|
+
const others = affected.filter(n => n !== name);
|
|
132
|
+
const scope = others.length
|
|
133
|
+
? `${affected.length} instances on \`${backend}\`: ${affected.join(", ")}`
|
|
134
|
+
: `\`${name}\` (${backend})`;
|
|
135
|
+
this.notifyIncident(notificationTarget, "auth_error", `🔑 ${message}\n\nAffects ${scope}. Credentials are shared per backend — one re-login restores all of them; affected instances pause until then.`);
|
|
136
|
+
}
|
|
137
|
+
/**
|
|
138
|
+
* System errors from a ClassicBot belong in the operator's General topic,
|
|
139
|
+
* never in the end user's chat channel. Fleet-topic instances retain their
|
|
140
|
+
* existing local notification target.
|
|
141
|
+
*/
|
|
142
|
+
ptyErrorNotificationTarget(name) {
|
|
143
|
+
if (!this.ctx.isClassicInstance?.(name))
|
|
144
|
+
return name;
|
|
145
|
+
const general = this.findGeneralInstance();
|
|
146
|
+
if (!general) {
|
|
147
|
+
this.ctx.logger.warn({ name }, "ClassicBot PTY error has no General topic notification target");
|
|
148
|
+
}
|
|
149
|
+
return general;
|
|
150
|
+
}
|
|
151
|
+
/**
|
|
152
|
+
* Handlers for the events that mean "something went wrong".
|
|
153
|
+
*
|
|
154
|
+
* Extracted from start() so the planned-restart suppression can be exercised
|
|
155
|
+
* against a plain event emitter — the alternative is a real Daemon, which
|
|
156
|
+
* means a real tmux window, which is why this rule went untested before.
|
|
157
|
+
*/
|
|
158
|
+
attachIncidentHandlers(name, daemon) {
|
|
159
|
+
daemon.on("crash_respawn", safeHandler(() => {
|
|
160
|
+
this.ctx.eventLog?.insert(name, "crash_respawn", {});
|
|
161
|
+
this.ctx.logger.warn({ name }, "Instance crashed and respawned");
|
|
162
|
+
this.notifyIncident(name, "crash_respawn", t("inst.crashed_respawned", name));
|
|
163
|
+
const generalName = this.findGeneralInstance();
|
|
164
|
+
if (generalName && generalName !== name) {
|
|
165
|
+
this.notifyIncident(generalName, "crash_respawn", t("inst.crashed_respawned_log", name));
|
|
166
|
+
}
|
|
167
|
+
}, this.ctx.logger, `daemon.crash_respawn[${name}]`));
|
|
168
|
+
daemon.on("snapshot_failed", safeHandler(() => {
|
|
169
|
+
this.ctx.eventLog?.insert(name, "snapshot_failed", {});
|
|
170
|
+
this.notifyIncident(name, "snapshot_failed", t("inst.restarted_no_context", name));
|
|
171
|
+
}, this.ctx.logger, `daemon.snapshot_failed[${name}]`));
|
|
172
|
+
daemon.on("supervision_ended", safeHandler((data) => {
|
|
173
|
+
// The instance is dead and nothing will restart it. Say so where the operator
|
|
174
|
+
// is looking, and mark the topic — otherwise messages routed here just queue
|
|
175
|
+
// or fail with a bare ❌ and the dashboard still looks normal.
|
|
176
|
+
this.ctx.eventLog?.insert(name, "supervision_ended", { reason: data.reason });
|
|
177
|
+
this.ctx.logger.error({ name, reason: data.reason }, "Instance is no longer supervised");
|
|
178
|
+
this.notifyIncident(name, "supervision_ended", `🛑 ${name} is no longer running and will not be restarted automatically — ${data.reason}.\n${data.remedy}`);
|
|
179
|
+
this.ctx.setTopicIcon(name, "red");
|
|
180
|
+
}, this.ctx.logger, `daemon.supervision_ended[${name}]`));
|
|
181
|
+
daemon.on("health_check_error", safeHandler((data) => {
|
|
182
|
+
this.ctx.eventLog?.insert(name, "health_check_error", { message: data.message });
|
|
183
|
+
this.ctx.logger.error({ name, message: data.message }, "Health check failing — instance supervision degraded");
|
|
184
|
+
this.notifyIncident(name, "health_check_error", `⚠️ ${name}: health check is failing (\`${data.message}\`). Crash detection for this instance may be degraded — see daemon.log.`);
|
|
185
|
+
}, this.ctx.logger, `daemon.health_check_error[${name}]`));
|
|
186
|
+
daemon.on("crash_loop", safeHandler(() => {
|
|
187
|
+
this.ctx.eventLog?.insert(name, "crash_loop", {});
|
|
188
|
+
this.ctx.logger.error({ name }, "Instance in crash loop — respawn paused");
|
|
189
|
+
this.notifyIncident(name, "crash_loop", t("inst.respawn_paused", name));
|
|
190
|
+
this.ctx.setTopicIcon(name, "red");
|
|
191
|
+
}, this.ctx.logger, `daemon.crash_loop[${name}]`));
|
|
192
|
+
daemon.on("mcp_died", safeHandler((data) => {
|
|
193
|
+
this.ctx.eventLog?.insert(name, "mcp_died", { pid: data.pid });
|
|
194
|
+
this.ctx.logger.error({ name, pid: data.pid }, "MCP server died — instance cannot use agend tools");
|
|
195
|
+
this.ctx.webhookEmit("mcp_died", name, { pid: data.pid });
|
|
196
|
+
// The CLI owns the MCP server's stdio pipes, so only restarting the CLI can
|
|
197
|
+
// restore its tools — say so instead of implying self-healing. Deliberately
|
|
198
|
+
// NOT auto-restarting: that would interrupt whatever the agent is doing, and
|
|
199
|
+
// an instance whose CLI is otherwise fine may still be doing useful work.
|
|
200
|
+
this.notifyIncident(name, "mcp_died", `⚠️ \`${name}\` 的 MCP server 已終止 — 這個 instance 目前無法使用 agend 工具(無法 reply / 跨 instance 通訊)。\n`
|
|
201
|
+
+ `CLI 本身還在執行。工具只能由 CLI 自己重新啟動 MCP server,請用 \`restart_instance("${name}")\` 或 \`/restart\` 恢復。`);
|
|
202
|
+
}, this.ctx.logger, `daemon.mcp_died[${name}]`));
|
|
203
|
+
daemon.on("interactive_prompt", safeHandler((data) => {
|
|
204
|
+
this.ctx.eventLog?.insert(name, "interactive_prompt", { kind: data.kind });
|
|
205
|
+
this.ctx.logger.warn({ name, kind: data.kind, prompt: data.prompt }, "Instance is waiting for interactive terminal input");
|
|
206
|
+
const general = this.findGeneralInstance();
|
|
207
|
+
if (!general) {
|
|
208
|
+
this.ctx.logger.warn({ name }, "Interactive prompt has no General topic notification target");
|
|
209
|
+
return;
|
|
210
|
+
}
|
|
211
|
+
const label = data.kind === "sudo_password" || data.kind === "password"
|
|
212
|
+
? "sudo password"
|
|
213
|
+
: data.kind === "press_enter" ? "Press Enter" : "Y/N confirmation";
|
|
214
|
+
this.notifyIncident(general, "interactive_prompt", `⚠️ \`${name}\` 正在等待人工輸入(${label})。General 可以透過 tmux attach 查看並協助操作。\n請勿在 Telegram/Discord 傳送密碼。`);
|
|
215
|
+
}, this.ctx.logger, `daemon.interactive_prompt[${name}]`));
|
|
216
|
+
daemon.on("pty_error", safeHandler((data) => {
|
|
217
|
+
this.ctx.eventLog?.insert(name, "pty_error", { type: data.type, action: data.action });
|
|
218
|
+
this.ctx.logger.warn({ name, errorType: data.type, action: data.action }, `PTY error: ${data.message}`);
|
|
219
|
+
// Antigravity's account-level cap is visible ONLY here: the quota summary
|
|
220
|
+
// API keeps reporting its buckets as barely used while the CLI is blocked
|
|
221
|
+
// (verified live — 0% used and "Individual quota reached" in the same
|
|
222
|
+
// minute). Remember it so /usage can overlay the truth on that row.
|
|
223
|
+
if (data.type === "quota" && this.backendOf(name) === "antigravity") {
|
|
224
|
+
reportProviderRateLimit("antigravity", data.message);
|
|
225
|
+
}
|
|
226
|
+
const emoji = data.type === "rate_limit" || data.type === "timeout" ? "⏳" : data.type === "auth_error" ? "🔑" : "⚠️";
|
|
227
|
+
const notificationTarget = this.ptyErrorNotificationTarget(name);
|
|
228
|
+
// Auth failures are a property of the BACKEND's shared credentials, not of
|
|
229
|
+
// one instance: every instance on that CLI fails at once, and one re-login
|
|
230
|
+
// fixes them all. Notify once per backend (listing who's affected) instead
|
|
231
|
+
// of N near-identical alerts, and suppress repeats fleet-wide.
|
|
232
|
+
if (data.type === "auth_error") {
|
|
233
|
+
if (notificationTarget)
|
|
234
|
+
this.notifyAuthErrorOnce(name, data.message, notificationTarget);
|
|
235
|
+
}
|
|
236
|
+
else if (notificationTarget) {
|
|
237
|
+
this.notifyIncident(notificationTarget, "pty_error", t("inst.notification", emoji, name, data.message, data.action));
|
|
238
|
+
}
|
|
239
|
+
this.ctx.webhookEmit("pty_error", name, { type: data.type, action: data.action, message: data.message });
|
|
240
|
+
// The CLI interrupted itself on this error, so any pending Cancel button is
|
|
241
|
+
// now useless — retire it. We only reach here when the error wasn't
|
|
242
|
+
// cooldown-suppressed (the daemon skips the emit during cooldown), so this
|
|
243
|
+
// won't fire on repeat errors within the 5-min window. No-op if no button.
|
|
244
|
+
this.ctx.clearCancelButton(name);
|
|
245
|
+
if (data.action === "failover") {
|
|
246
|
+
this.ctx.checkModelFailover(name, 100); // Force failover trigger
|
|
247
|
+
}
|
|
248
|
+
else if (data.action === "restart") {
|
|
249
|
+
// A broken *resumed* session (e.g. agy pinned to a dead model) can't
|
|
250
|
+
// self-recover; a plain restart would --continue back into it. freshStart
|
|
251
|
+
// makes the respawn skip resume so the CLI starts a clean session on its
|
|
252
|
+
// default (valid) model.
|
|
253
|
+
this.ctx.restartSingleInstance(name, { freshStart: true }).catch(err => this.ctx.logger.error({ err, name }, "pty_error restart failed"));
|
|
254
|
+
}
|
|
255
|
+
else if (data.action === "pause") {
|
|
256
|
+
// Previously unhandled, so an expired session kept receiving messages and
|
|
257
|
+
// re-sending its whole context into a CLI that could only fail — wasted
|
|
258
|
+
// credit and lost work. pause() alone no-ops while the pane is busy (the
|
|
259
|
+
// usual state when auth fails), so mark it to pause as soon as it idles.
|
|
260
|
+
// Queued messages survive: delivery wakes a paused instance.
|
|
261
|
+
void this.pause(name)
|
|
262
|
+
.catch(err => this.ctx.logger.warn({ err, name }, "auth-error pause failed"))
|
|
263
|
+
.finally(() => { if (!this.isPaused(name))
|
|
264
|
+
daemon.requestPauseWhenIdle(); });
|
|
265
|
+
}
|
|
266
|
+
}, this.ctx.logger, `daemon.pty_error[${name}]`));
|
|
267
|
+
}
|
|
268
|
+
async start(name, config, topicMode, runtimeIdentity) {
|
|
87
269
|
if (this.daemons.has(name)) {
|
|
88
270
|
this.ctx.logger.info({ name }, "Instance already running, skipping");
|
|
89
271
|
return;
|
|
@@ -116,7 +298,11 @@ export class InstanceLifecycle {
|
|
|
116
298
|
return;
|
|
117
299
|
}
|
|
118
300
|
const backend = createBackend(backendName, instanceDir);
|
|
119
|
-
const daemon = new Daemon(name, config, instanceDir, topicMode, backend, this.ctx.controlClient ?? undefined, this.ctx.logger
|
|
301
|
+
const daemon = new Daemon(name, config, instanceDir, topicMode, backend, this.ctx.controlClient ?? undefined, this.ctx.logger, runtimeIdentity ?? {
|
|
302
|
+
kind: "fleet-topic",
|
|
303
|
+
backend: backendName,
|
|
304
|
+
model: config.model ?? "default",
|
|
305
|
+
});
|
|
120
306
|
// Catch errors from daemon internals (e.g. IPC server) to prevent crashing the fleet process
|
|
121
307
|
daemon.on("error", (err) => {
|
|
122
308
|
this.ctx.logger.error({ err, name }, "Daemon emitted error — instance isolated");
|
|
@@ -161,47 +347,7 @@ export class InstanceLifecycle {
|
|
|
161
347
|
this.ctx.setTopicIcon(name, "green");
|
|
162
348
|
this.ctx.touchActivity(name);
|
|
163
349
|
});
|
|
164
|
-
|
|
165
|
-
this.ctx.eventLog?.insert(name, "crash_respawn", {});
|
|
166
|
-
this.ctx.logger.warn({ name }, "Instance crashed and respawned");
|
|
167
|
-
this.ctx.notifyInstanceTopic(name, t("inst.crashed_respawned", name));
|
|
168
|
-
const generalName = this.findGeneralInstance();
|
|
169
|
-
if (generalName && generalName !== name) {
|
|
170
|
-
this.ctx.notifyInstanceTopic(generalName, t("inst.crashed_respawned_log", name));
|
|
171
|
-
}
|
|
172
|
-
}, this.ctx.logger, `daemon.crash_respawn[${name}]`));
|
|
173
|
-
daemon.on("snapshot_failed", safeHandler(() => {
|
|
174
|
-
this.ctx.eventLog?.insert(name, "snapshot_failed", {});
|
|
175
|
-
this.ctx.notifyInstanceTopic(name, t("inst.restarted_no_context", name));
|
|
176
|
-
}, this.ctx.logger, `daemon.snapshot_failed[${name}]`));
|
|
177
|
-
daemon.on("crash_loop", safeHandler(() => {
|
|
178
|
-
this.ctx.eventLog?.insert(name, "crash_loop", {});
|
|
179
|
-
this.ctx.logger.error({ name }, "Instance in crash loop — respawn paused");
|
|
180
|
-
this.ctx.notifyInstanceTopic(name, t("inst.respawn_paused", name));
|
|
181
|
-
this.ctx.setTopicIcon(name, "red");
|
|
182
|
-
}, this.ctx.logger, `daemon.crash_loop[${name}]`));
|
|
183
|
-
daemon.on("pty_error", safeHandler((data) => {
|
|
184
|
-
this.ctx.eventLog?.insert(name, "pty_error", { type: data.type, action: data.action });
|
|
185
|
-
this.ctx.logger.warn({ name, errorType: data.type, action: data.action }, `PTY error: ${data.message}`);
|
|
186
|
-
const emoji = data.type === "rate_limit" || data.type === "timeout" ? "⏳" : data.type === "auth_error" ? "🔑" : "⚠️";
|
|
187
|
-
this.ctx.notifyInstanceTopic(name, t("inst.notification", emoji, name, data.message, data.action));
|
|
188
|
-
this.ctx.webhookEmit("pty_error", name, { type: data.type, action: data.action, message: data.message });
|
|
189
|
-
// The CLI interrupted itself on this error, so any pending Cancel button is
|
|
190
|
-
// now useless — retire it. We only reach here when the error wasn't
|
|
191
|
-
// cooldown-suppressed (the daemon skips the emit during cooldown), so this
|
|
192
|
-
// won't fire on repeat errors within the 5-min window. No-op if no button.
|
|
193
|
-
this.ctx.clearCancelButton(name);
|
|
194
|
-
if (data.action === "failover") {
|
|
195
|
-
this.ctx.checkModelFailover(name, 100); // Force failover trigger
|
|
196
|
-
}
|
|
197
|
-
else if (data.action === "restart") {
|
|
198
|
-
// A broken *resumed* session (e.g. agy pinned to a dead model) can't
|
|
199
|
-
// self-recover; a plain restart would --continue back into it. freshStart
|
|
200
|
-
// makes the respawn skip resume so the CLI starts a clean session on its
|
|
201
|
-
// default (valid) model.
|
|
202
|
-
this.ctx.restartSingleInstance(name, { freshStart: true }).catch(err => this.ctx.logger.error({ err, name }, "pty_error restart failed"));
|
|
203
|
-
}
|
|
204
|
-
}, this.ctx.logger, `daemon.pty_error[${name}]`));
|
|
350
|
+
this.attachIncidentHandlers(name, daemon);
|
|
205
351
|
daemon.on("pty_recovered", safeHandler((data) => {
|
|
206
352
|
const mins = Math.floor(data.downtime_s / 60);
|
|
207
353
|
const secs = data.downtime_s % 60;
|