@songsid/agend 2.1.2-beta.3 → 2.1.2-beta.30
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/access-path.js +15 -6
- package/dist/access-path.js.map +1 -1
- package/dist/agent-cli-instructions.md +1 -0
- package/dist/agent-cli.js +14 -0
- package/dist/agent-cli.js.map +1 -1
- package/dist/agent-endpoint.d.ts +2 -0
- package/dist/agent-endpoint.js +25 -1
- package/dist/agent-endpoint.js.map +1 -1
- package/dist/backend/antigravity.d.ts +20 -0
- package/dist/backend/antigravity.js +37 -3
- package/dist/backend/antigravity.js.map +1 -1
- package/dist/backend/claude-code.d.ts +59 -0
- package/dist/backend/claude-code.js +67 -1
- package/dist/backend/claude-code.js.map +1 -1
- package/dist/backend/codex.d.ts +3 -0
- package/dist/backend/codex.js +30 -0
- package/dist/backend/codex.js.map +1 -1
- package/dist/backend/grok.d.ts +25 -0
- package/dist/backend/grok.js +51 -0
- package/dist/backend/grok.js.map +1 -1
- package/dist/backend/kiro.d.ts +22 -0
- package/dist/backend/kiro.js +85 -3
- package/dist/backend/kiro.js.map +1 -1
- package/dist/backend/types.d.ts +58 -0
- package/dist/backend/types.js +13 -1
- package/dist/backend/types.js.map +1 -1
- package/dist/channel/adapters/discord.d.ts +15 -0
- package/dist/channel/adapters/discord.js +100 -2
- package/dist/channel/adapters/discord.js.map +1 -1
- package/dist/channel/adapters/telegram.d.ts +21 -0
- package/dist/channel/adapters/telegram.js +86 -0
- package/dist/channel/adapters/telegram.js.map +1 -1
- package/dist/channel/ipc-bridge.d.ts +9 -1
- package/dist/channel/ipc-bridge.js +12 -3
- package/dist/channel/ipc-bridge.js.map +1 -1
- package/dist/channel/ipc-timeouts.d.ts +46 -0
- package/dist/channel/ipc-timeouts.js +65 -0
- package/dist/channel/ipc-timeouts.js.map +1 -0
- package/dist/channel/mcp-server.js +30 -14
- package/dist/channel/mcp-server.js.map +1 -1
- package/dist/channel/mcp-tools.js +30 -1
- package/dist/channel/mcp-tools.js.map +1 -1
- package/dist/channel/reconnect-backoff.d.ts +17 -0
- package/dist/channel/reconnect-backoff.js +21 -0
- package/dist/channel/reconnect-backoff.js.map +1 -0
- package/dist/channel/types.d.ts +29 -0
- package/dist/classic-channel-manager.js +1 -5
- package/dist/classic-channel-manager.js.map +1 -1
- package/dist/cli.js +155 -63
- package/dist/cli.js.map +1 -1
- package/dist/completion.d.ts +27 -0
- package/dist/completion.js +121 -0
- package/dist/completion.js.map +1 -0
- package/dist/config-validator.js +21 -0
- package/dist/config-validator.js.map +1 -1
- package/dist/config.js +1 -0
- package/dist/config.js.map +1 -1
- package/dist/cost-guard.d.ts +3 -1
- package/dist/cost-guard.js +3 -1
- package/dist/cost-guard.js.map +1 -1
- package/dist/daemon.d.ts +240 -14
- package/dist/daemon.js +956 -308
- package/dist/daemon.js.map +1 -1
- package/dist/event-log.d.ts +31 -0
- package/dist/event-log.js +96 -0
- package/dist/event-log.js.map +1 -1
- package/dist/fleet-context.d.ts +9 -0
- package/dist/fleet-manager.d.ts +311 -2
- package/dist/fleet-manager.js +1369 -108
- package/dist/fleet-manager.js.map +1 -1
- package/dist/general-knowledge/skills/cross-instance-messaging/SKILL.md +22 -0
- package/dist/general-knowledge/skills/model-discovery/SKILL.md +14 -23
- package/dist/general-knowledge/skills/session-management/SKILL.md +15 -34
- package/dist/hang-detector.d.ts +19 -13
- package/dist/hang-detector.js +19 -49
- package/dist/hang-detector.js.map +1 -1
- package/dist/instance-lifecycle.d.ts +43 -1
- package/dist/instance-lifecycle.js +155 -43
- package/dist/instance-lifecycle.js.map +1 -1
- package/dist/instructions.d.ts +5 -0
- package/dist/instructions.js +9 -11
- package/dist/instructions.js.map +1 -1
- package/dist/locale.js +5 -1
- package/dist/locale.js.map +1 -1
- package/dist/logger.js +14 -0
- package/dist/logger.js.map +1 -1
- package/dist/mcp-liveness.d.ts +21 -0
- package/dist/mcp-liveness.js +27 -0
- package/dist/mcp-liveness.js.map +1 -0
- package/dist/outbound-handlers.d.ts +16 -0
- package/dist/outbound-handlers.js +151 -27
- package/dist/outbound-handlers.js.map +1 -1
- package/dist/outbound-schemas.d.ts +11 -3
- package/dist/outbound-schemas.js +16 -2
- package/dist/outbound-schemas.js.map +1 -1
- package/dist/pane-write-lock.d.ts +48 -0
- package/dist/pane-write-lock.js +73 -0
- package/dist/pane-write-lock.js.map +1 -0
- package/dist/process-memory.d.ts +31 -0
- package/dist/process-memory.js +79 -0
- package/dist/process-memory.js.map +1 -0
- package/dist/quickstart.js +17 -16
- package/dist/quickstart.js.map +1 -1
- package/dist/reply-dedup.d.ts +41 -0
- package/dist/reply-dedup.js +0 -0
- package/dist/reply-dedup.js.map +1 -0
- package/dist/scheduler/db.js +3 -0
- package/dist/scheduler/db.js.map +1 -1
- package/dist/sd-notify.d.ts +27 -0
- package/dist/sd-notify.js +33 -1
- package/dist/sd-notify.js.map +1 -1
- package/dist/secret-file.d.ts +33 -0
- package/dist/secret-file.js +36 -0
- package/dist/secret-file.js.map +1 -0
- package/dist/setup-wizard.js +9 -7
- package/dist/setup-wizard.js.map +1 -1
- package/dist/tmux-control.d.ts +58 -5
- package/dist/tmux-control.js +102 -14
- package/dist/tmux-control.js.map +1 -1
- package/dist/tmux-manager.d.ts +31 -1
- package/dist/tmux-manager.js +40 -11
- package/dist/tmux-manager.js.map +1 -1
- package/dist/topic-commands.d.ts +50 -10
- package/dist/topic-commands.js +269 -106
- package/dist/topic-commands.js.map +1 -1
- package/dist/tui-glyphs.d.ts +25 -0
- package/dist/tui-glyphs.js +26 -0
- package/dist/tui-glyphs.js.map +1 -0
- package/dist/types.d.ts +12 -2
- package/dist/ui/view.html +123 -2
- package/dist/update-marker.d.ts +15 -0
- package/dist/update-marker.js +69 -0
- package/dist/update-marker.js.map +1 -0
- package/dist/usage/format-rich.d.ts +11 -0
- package/dist/usage/format-rich.js +121 -0
- package/dist/usage/format-rich.js.map +1 -0
- package/dist/usage/providers.d.ts +130 -0
- package/dist/usage/providers.js +1030 -0
- package/dist/usage/providers.js.map +1 -0
- package/dist/usage/statusline-usage.d.ts +20 -0
- package/dist/usage/statusline-usage.js +120 -0
- package/dist/usage/statusline-usage.js.map +1 -0
- package/dist/usage/usage-api.d.ts +50 -0
- package/dist/usage/usage-api.js +167 -0
- package/dist/usage/usage-api.js.map +1 -0
- package/dist/view-api.d.ts +3 -0
- package/dist/view-api.js +11 -2
- package/dist/view-api.js.map +1 -1
- package/dist/web-api.js +5 -2
- package/dist/web-api.js.map +1 -1
- package/package.json +4 -1
- package/dist/channel/tool-tracker.d.ts +0 -13
- package/dist/channel/tool-tracker.js +0 -58
- package/dist/channel/tool-tracker.js.map +0 -1
- package/dist/daemon-entry.d.ts +0 -1
- package/dist/daemon-entry.js +0 -30
- package/dist/daemon-entry.js.map +0 -1
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
---
|
|
2
|
+
name: cross-instance-messaging
|
|
3
|
+
description: Fire-and-queue cross-instance tools — send once, never resend on queued
|
|
4
|
+
---
|
|
5
|
+
|
|
6
|
+
## How to send
|
|
7
|
+
|
|
8
|
+
Use fleet tools only (`send_to_instance`, `delegate_task`, `request_information`, `report_result`, `broadcast`):
|
|
9
|
+
- Call returns immediately: `{ sent: true, queued: true }` — success, fleet owns delivery.
|
|
10
|
+
- **Do not wait** for the target to go idle; **do not** treat 30s IPC timeout as failure to re-send.
|
|
11
|
+
- **Error only if the target does not exist** (or similar hard reject) — then fix the name, don't spam.
|
|
12
|
+
- **Never re-send because the reply said `queued`** — that means the message is already queued.
|
|
13
|
+
|
|
14
|
+
## requires_reply
|
|
15
|
+
|
|
16
|
+
- Means “target should later answer with `report_result` / a real reply”
|
|
17
|
+
- **Not** a synchronous wait for their turn to finish
|
|
18
|
+
|
|
19
|
+
## Task flow
|
|
20
|
+
|
|
21
|
+
- `delegate_task` → silent work → `report_result` (zero ack-only pings)
|
|
22
|
+
- Cross-instance traffic is `[from:name]` → answer with `send_to_instance` / `report_result`, never `reply`
|
|
@@ -1,36 +1,27 @@
|
|
|
1
1
|
---
|
|
2
2
|
name: model-discovery
|
|
3
|
-
description:
|
|
3
|
+
description: Set and discover models — pass-through to the CLI, no AgEnD allowlist gate
|
|
4
4
|
---
|
|
5
5
|
|
|
6
|
-
##
|
|
6
|
+
## How to set a model
|
|
7
7
|
|
|
8
|
-
|
|
8
|
+
- fleet.yaml: `defaults.model` or per-instance `model`
|
|
9
|
+
- **Pass-through:** AgEnD no longer blocks unknown model ids — it may **warn**, then still pass the string to the CLI
|
|
10
|
+
- **CLI is source of truth** — if the model is invalid, the backend CLI errors (fix the name there)
|
|
9
11
|
|
|
10
|
-
|
|
11
|
-
|---------|-------------------|---------|
|
|
12
|
-
| **kiro-cli** | In tmux: send `/model` + Enter → read model list → Esc to close | auto (latest) |
|
|
13
|
-
| **claude-code** | `sonnet`, `opus`, `haiku`, `opusplan`, `best`, `sonnet[1m]`, `opus[1m]` | sonnet |
|
|
14
|
-
| **antigravity** | Run `agy models` to see available models | Gemini 3.5 Flash (Medium) |
|
|
15
|
-
| **codex** | `gpt-4o`, `o3`, `o4-mini` | gpt-4o |
|
|
16
|
-
| **opencode** | `opencode models` | depends on provider |
|
|
17
|
-
| **grok** | `grok-4.5`, `grok-4.3`, `grok-code`, `grok-build-0.1` | grok default |
|
|
12
|
+
## Discover real names
|
|
18
13
|
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
-
|
|
22
|
-
-
|
|
23
|
-
|
|
14
|
+
| Backend | How |
|
|
15
|
+
|---------|-----|
|
|
16
|
+
| kiro-cli | In pane: `/model` (gpt-*, deepseek-*, minimax-*, glm-*, qwen* supported) |
|
|
17
|
+
| claude-code | `sonnet` / `opus` / `haiku` / `opusplan` / `Fable` / aliases |
|
|
18
|
+
| codex | pane `/model` or docs (`gpt-*`, `o*`) |
|
|
19
|
+
| grok | `grok models` |
|
|
20
|
+
| antigravity | `agy models` — set **base name only** (drop `(Medium)` / `(Thinking)` effort suffix) |
|
|
21
|
+
| opencode | `opencode models` |
|
|
24
22
|
|
|
25
|
-
**Important:** Model names vary by backend. Always check the actual CLI output rather than guessing names.
|
|
26
|
-
|
|
27
|
-
Example fleet.yaml:
|
|
28
23
|
```yaml
|
|
29
24
|
defaults:
|
|
30
25
|
backend: kiro-cli
|
|
31
26
|
model: claude-sonnet-4-20250514
|
|
32
|
-
|
|
33
|
-
instances:
|
|
34
|
-
heavy-task:
|
|
35
|
-
model: claude-opus-4-20250514
|
|
36
27
|
```
|
|
@@ -1,46 +1,27 @@
|
|
|
1
1
|
---
|
|
2
2
|
name: session-management
|
|
3
|
-
description:
|
|
3
|
+
description: Session stores, forking, and auth-pause recovery
|
|
4
4
|
---
|
|
5
5
|
|
|
6
|
+
## Auth failure (auto-pause)
|
|
7
|
+
|
|
8
|
+
When AgEnD sees `auth_error` it **pauses** that instance (`pausePending` sticky):
|
|
9
|
+
- One notification **per backend** (not per instance) — log in once, same-backend peers recover.
|
|
10
|
+
- Auth is **per-user global** across backends: claude-code, codex, kiro, grok, opencode, antigravity.
|
|
11
|
+
- Messages while paused stay in the **queue** — do not re-send.
|
|
12
|
+
- After the user re-auths: `wake` / normal wake clears `pausePending`.
|
|
13
|
+
|
|
6
14
|
## Where sessions live
|
|
7
15
|
|
|
8
16
|
| | kiro-cli | claude-code |
|
|
9
17
|
|---|---|---|
|
|
10
18
|
| Store | `~/.kiro/sessions/cli/<uuid>.json` | `~/.claude/projects/<path-encoded>/*.jsonl` |
|
|
11
|
-
|
|
|
12
|
-
| Reload | `/chat load <file>` | none — `--continue` (latest for the dir) / `--resume <id>` |
|
|
13
|
-
| Text export | — | `/export` (plain text, **not** reloadable) |
|
|
14
|
-
|
|
15
|
-
`<path-encoded>` = the absolute working_directory with `/` → `-`
|
|
16
|
-
(e.g. `/home/han/Projects/AgEnD` → `-home-han-Projects-AgEnD`).
|
|
17
|
-
|
|
18
|
-
## Fork a session to a new instance
|
|
19
|
-
|
|
20
|
-
Confirm the source is **idle** first (`describe_instance` / `get_fleet_status`) — don't fork mid-task.
|
|
21
|
-
|
|
22
|
-
**kiro-cli** — save, copy, load:
|
|
23
|
-
1. On the source, save: `/chat save <name>.json -f` (paste via tmux if needed).
|
|
24
|
-
2. `create_instance` (same backend).
|
|
25
|
-
3. `cp ~/.agend/workspaces/<source>/<name>.json ~/.agend/workspaces/<target>/`
|
|
26
|
-
4. Load on the target: `/chat load <name>.json`, or set `pre_task_command: "/chat load <name>.json"`.
|
|
27
|
-
|
|
28
|
-
**claude-code** — copy the `.jsonl` (no save/load command):
|
|
29
|
-
1. Newest source session: `ls -lt ~/.claude/projects/<source-encoded>/*.jsonl | head`
|
|
30
|
-
2. `create_instance` (backend `claude-code`); note its working_directory.
|
|
31
|
-
3. Copy into the target's encoded project dir:
|
|
32
|
-
```bash
|
|
33
|
-
TARGET_ENC="$(echo '<target-working-dir>' | sed 's#/#-#g')"
|
|
34
|
-
mkdir -p ~/.claude/projects/$TARGET_ENC
|
|
35
|
-
cp ~/.claude/projects/<source-encoded>/<session>.jsonl ~/.claude/projects/$TARGET_ENC/
|
|
36
|
-
```
|
|
37
|
-
4. Start the target — claude-code resumes the newest `.jsonl` via `--continue`.
|
|
19
|
+
| Reload | `/chat load <file>` | `--continue` / `--resume <id>` |
|
|
38
20
|
|
|
39
|
-
|
|
40
|
-
(paths inside the transcript refer to it). Pick the right `.jsonl` if several exist (newest by
|
|
41
|
-
mtime; compaction/branches create new files). `/export` is text only, not reloadable.
|
|
21
|
+
`<path-encoded>` = absolute cwd with `/` → `-`.
|
|
42
22
|
|
|
43
|
-
##
|
|
23
|
+
## Fork (source must be idle)
|
|
44
24
|
|
|
45
|
-
|
|
46
|
-
|
|
25
|
+
- **kiro:** `/chat save name.json -f` → `create_instance` → copy workspace file → `/chat load name.json`
|
|
26
|
+
- **claude-code:** copy newest `*.jsonl` into target's encoded project dir → start (uses `--continue`)
|
|
27
|
+
- Prefer `replace_instance` when the whole session is poisoned (see instance-lifecycle)
|
package/dist/hang-detector.d.ts
CHANGED
|
@@ -1,16 +1,22 @@
|
|
|
1
1
|
import { EventEmitter } from "node:events";
|
|
2
|
+
/**
|
|
3
|
+
* Carries "this instance looks hung" from the daemon to the fleet manager.
|
|
4
|
+
*
|
|
5
|
+
* It is only an event bridge. Hang detection itself lives in the daemon's pane-state
|
|
6
|
+
* machine, which emits `hang` directly when a pane stops changing for the configured
|
|
7
|
+
* stuck timeout; `instance-lifecycle` subscribes to that and notifies.
|
|
8
|
+
*
|
|
9
|
+
* It used to also contain a silence-timer state machine — `start()`, `isHung()`,
|
|
10
|
+
* `hungEmitted`, and timestamps fed by `recordActivity` / `recordInbound` /
|
|
11
|
+
* `recordStatuslineUpdate`. None of it ran: `start()` was never called from
|
|
12
|
+
* anywhere, so `isHung()` was unreachable, the timestamps were written and never
|
|
13
|
+
* read, and the constructor's `timeoutMinutes` was ignored (the real stuck timeout
|
|
14
|
+
* is read separately by the pane monitor). Its tests exercised that dead logic —
|
|
15
|
+
* two had identical setup with contradictory expectations, which can only pass
|
|
16
|
+
* unnoticed when neither runs against anything real.
|
|
17
|
+
*
|
|
18
|
+
* Kept as a named class rather than a bare EventEmitter so the daemon → lifecycle
|
|
19
|
+
* wiring stays typed and greppable.
|
|
20
|
+
*/
|
|
2
21
|
export declare class HangDetector extends EventEmitter {
|
|
3
|
-
private lastActivityTs;
|
|
4
|
-
private lastStatuslineTs;
|
|
5
|
-
private lastInboundTs;
|
|
6
|
-
private hungEmitted;
|
|
7
|
-
private checkTimer;
|
|
8
|
-
private timeoutMs;
|
|
9
|
-
constructor(timeoutMinutes: number);
|
|
10
|
-
recordActivity(): void;
|
|
11
|
-
recordInbound(): void;
|
|
12
|
-
recordStatuslineUpdate(): void;
|
|
13
|
-
isHung(): boolean;
|
|
14
|
-
start(intervalMs?: number): void;
|
|
15
|
-
stop(): void;
|
|
16
22
|
}
|
package/dist/hang-detector.js
CHANGED
|
@@ -1,53 +1,23 @@
|
|
|
1
1
|
import { EventEmitter } from "node:events";
|
|
2
|
+
/**
|
|
3
|
+
* Carries "this instance looks hung" from the daemon to the fleet manager.
|
|
4
|
+
*
|
|
5
|
+
* It is only an event bridge. Hang detection itself lives in the daemon's pane-state
|
|
6
|
+
* machine, which emits `hang` directly when a pane stops changing for the configured
|
|
7
|
+
* stuck timeout; `instance-lifecycle` subscribes to that and notifies.
|
|
8
|
+
*
|
|
9
|
+
* It used to also contain a silence-timer state machine — `start()`, `isHung()`,
|
|
10
|
+
* `hungEmitted`, and timestamps fed by `recordActivity` / `recordInbound` /
|
|
11
|
+
* `recordStatuslineUpdate`. None of it ran: `start()` was never called from
|
|
12
|
+
* anywhere, so `isHung()` was unreachable, the timestamps were written and never
|
|
13
|
+
* read, and the constructor's `timeoutMinutes` was ignored (the real stuck timeout
|
|
14
|
+
* is read separately by the pane monitor). Its tests exercised that dead logic —
|
|
15
|
+
* two had identical setup with contradictory expectations, which can only pass
|
|
16
|
+
* unnoticed when neither runs against anything real.
|
|
17
|
+
*
|
|
18
|
+
* Kept as a named class rather than a bare EventEmitter so the daemon → lifecycle
|
|
19
|
+
* wiring stays typed and greppable.
|
|
20
|
+
*/
|
|
2
21
|
export class HangDetector extends EventEmitter {
|
|
3
|
-
lastActivityTs = 0;
|
|
4
|
-
lastStatuslineTs = 0;
|
|
5
|
-
lastInboundTs = 0;
|
|
6
|
-
hungEmitted = false;
|
|
7
|
-
checkTimer = null;
|
|
8
|
-
timeoutMs;
|
|
9
|
-
constructor(timeoutMinutes) {
|
|
10
|
-
super();
|
|
11
|
-
this.timeoutMs = timeoutMinutes * 60 * 1000;
|
|
12
|
-
}
|
|
13
|
-
recordActivity() {
|
|
14
|
-
this.lastActivityTs = Date.now();
|
|
15
|
-
if (this.hungEmitted) {
|
|
16
|
-
this.hungEmitted = false;
|
|
17
|
-
}
|
|
18
|
-
}
|
|
19
|
-
recordInbound() {
|
|
20
|
-
this.lastInboundTs = Date.now();
|
|
21
|
-
}
|
|
22
|
-
recordStatuslineUpdate() {
|
|
23
|
-
this.lastStatuslineTs = Date.now();
|
|
24
|
-
}
|
|
25
|
-
isHung() {
|
|
26
|
-
if (this.lastActivityTs === 0)
|
|
27
|
-
return false;
|
|
28
|
-
if (this.lastInboundTs === 0)
|
|
29
|
-
return false;
|
|
30
|
-
const now = Date.now();
|
|
31
|
-
// Only flag as hung if:
|
|
32
|
-
// 1. There's an inbound that hasn't been answered (no activity since inbound)
|
|
33
|
-
// 2. Timeout has elapsed since that inbound
|
|
34
|
-
const noActivitySinceInbound = this.lastActivityTs < this.lastInboundTs;
|
|
35
|
-
const stale = now - this.lastInboundTs > this.timeoutMs;
|
|
36
|
-
return stale && noActivitySinceInbound;
|
|
37
|
-
}
|
|
38
|
-
start(intervalMs = 60_000) {
|
|
39
|
-
this.checkTimer = setInterval(() => {
|
|
40
|
-
if (this.isHung() && !this.hungEmitted) {
|
|
41
|
-
this.hungEmitted = true;
|
|
42
|
-
this.emit("hang");
|
|
43
|
-
}
|
|
44
|
-
}, intervalMs);
|
|
45
|
-
}
|
|
46
|
-
stop() {
|
|
47
|
-
if (this.checkTimer) {
|
|
48
|
-
clearInterval(this.checkTimer);
|
|
49
|
-
this.checkTimer = null;
|
|
50
|
-
}
|
|
51
|
-
}
|
|
52
22
|
}
|
|
53
23
|
//# sourceMappingURL=hang-detector.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"hang-detector.js","sourceRoot":"","sources":["../src/hang-detector.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AAE3C,MAAM,OAAO,YAAa,SAAQ,YAAY;
|
|
1
|
+
{"version":3,"file":"hang-detector.js","sourceRoot":"","sources":["../src/hang-detector.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AAE3C;;;;;;;;;;;;;;;;;;GAkBG;AACH,MAAM,OAAO,YAAa,SAAQ,YAAY;CAAG"}
|
|
@@ -4,6 +4,7 @@ import type { Logger } from "./logger.js";
|
|
|
4
4
|
import type { IpcClient } from "./channel/ipc-bridge.js";
|
|
5
5
|
import type { EventLog } from "./event-log.js";
|
|
6
6
|
import type { TmuxControlClient } from "./tmux-control.js";
|
|
7
|
+
import type { FleetInstructionsParams } from "./instructions.js";
|
|
7
8
|
export interface BackendInstallationInfo {
|
|
8
9
|
binary: string;
|
|
9
10
|
install: string;
|
|
@@ -49,6 +50,8 @@ export interface LifecycleContext {
|
|
|
49
50
|
touchActivity(name: string): void;
|
|
50
51
|
sendHangNotification(name: string, unchangedForMs?: number): Promise<void>;
|
|
51
52
|
notifyInstanceTopic(name: string, text: string): void;
|
|
53
|
+
/** True while the fleet is stopping on purpose or an `agend update` is running. */
|
|
54
|
+
isPlannedRestart(): boolean;
|
|
52
55
|
/** List claimed tasks for an instance (from task board). Returns empty array if unavailable. */
|
|
53
56
|
listClaimedTasks(assignee: string): Array<{
|
|
54
57
|
id: string;
|
|
@@ -63,6 +66,11 @@ export interface LifecycleContext {
|
|
|
63
66
|
reactMessageStatus(instanceName: string, chatId: string, messageId: string, emoji: string): void;
|
|
64
67
|
startPersistedPausedInstance(name: string): Promise<void>;
|
|
65
68
|
}
|
|
69
|
+
/** What attachIncidentHandlers needs from a Daemon — the real one satisfies it. */
|
|
70
|
+
export interface IncidentEventSource {
|
|
71
|
+
on(event: string, handler: (...args: any[]) => void): unknown;
|
|
72
|
+
requestPauseWhenIdle(): void;
|
|
73
|
+
}
|
|
66
74
|
/** Arguments accepted by handleCreate — mirrors CreateInstanceArgs in outbound-schemas.ts
|
|
67
75
|
* plus internal-only fields forwarded by deploy_template (profile-derived). */
|
|
68
76
|
export interface LifecycleCreateArgs {
|
|
@@ -98,8 +106,42 @@ export declare class InstanceLifecycle {
|
|
|
98
106
|
private ctx;
|
|
99
107
|
/** Active daemon processes: instanceName → Daemon */
|
|
100
108
|
readonly daemons: Map<string, import("./daemon.js").Daemon>;
|
|
109
|
+
/** backend → last auth-error alert time, so one expiry sends one alert. */
|
|
110
|
+
private lastAuthAlertAt;
|
|
101
111
|
constructor(ctx: LifecycleContext);
|
|
102
|
-
|
|
112
|
+
/**
|
|
113
|
+
* Report an incident to the user — unless the fleet is deliberately going
|
|
114
|
+
* down, in which case the "incident" is the shutdown doing its job.
|
|
115
|
+
*
|
|
116
|
+
* `agend update` stops instances, kills their MCP servers and restarts the
|
|
117
|
+
* daemon; every one of those looked like a crash to the alert path, so an
|
|
118
|
+
* upgrade produced a burst of ⚠️ for things that were working correctly. That
|
|
119
|
+
* burst is worse than noise: it teaches the operator to ignore the same alert
|
|
120
|
+
* that matters when a process really does die on its own.
|
|
121
|
+
*
|
|
122
|
+
* The event log and daemon.log entries still happen at the call sites — this
|
|
123
|
+
* suppresses the chat message, not the record. A crash outside a planned
|
|
124
|
+
* restart notifies exactly as before.
|
|
125
|
+
*/
|
|
126
|
+
private notifyIncident;
|
|
127
|
+
/** Backend a running instance uses (config → fleet default). */
|
|
128
|
+
private backendOf;
|
|
129
|
+
/**
|
|
130
|
+
* One alert per backend per cooldown, naming every affected instance — a CLI's
|
|
131
|
+
* credentials are shared, so N instances failing is ONE problem with ONE fix
|
|
132
|
+
* (re-login once). The per-instance daemon cooldown can't dedupe across
|
|
133
|
+
* instances, so the fleet-level map does it here.
|
|
134
|
+
*/
|
|
135
|
+
private notifyAuthErrorOnce;
|
|
136
|
+
/**
|
|
137
|
+
* Handlers for the events that mean "something went wrong".
|
|
138
|
+
*
|
|
139
|
+
* Extracted from start() so the planned-restart suppression can be exercised
|
|
140
|
+
* against a plain event emitter — the alternative is a real Daemon, which
|
|
141
|
+
* means a real tmux window, which is why this rule went untested before.
|
|
142
|
+
*/
|
|
143
|
+
attachIncidentHandlers(name: string, daemon: IncidentEventSource): void;
|
|
144
|
+
start(name: string, config: InstanceConfig, topicMode: boolean, runtimeIdentity?: FleetInstructionsParams["runtimeIdentity"]): Promise<void>;
|
|
103
145
|
isPaused(name: string): boolean;
|
|
104
146
|
getLastPausedAt(name: string): number | null;
|
|
105
147
|
pause(name: string): Promise<void>;
|
|
@@ -76,14 +76,162 @@ export function getUnsafeInstanceDaemonPidReason(pid, dataDir) {
|
|
|
76
76
|
}
|
|
77
77
|
return null;
|
|
78
78
|
}
|
|
79
|
+
/** Suppress duplicate auth alerts for the same backend within this window. */
|
|
80
|
+
const AUTH_ALERT_COOLDOWN_MS = 5 * 60_000;
|
|
79
81
|
export class InstanceLifecycle {
|
|
80
82
|
ctx;
|
|
81
83
|
/** Active daemon processes: instanceName → Daemon */
|
|
82
84
|
daemons = new Map();
|
|
85
|
+
/** backend → last auth-error alert time, so one expiry sends one alert. */
|
|
86
|
+
lastAuthAlertAt = new Map();
|
|
83
87
|
constructor(ctx) {
|
|
84
88
|
this.ctx = ctx;
|
|
85
89
|
}
|
|
86
|
-
|
|
90
|
+
/**
|
|
91
|
+
* Report an incident to the user — unless the fleet is deliberately going
|
|
92
|
+
* down, in which case the "incident" is the shutdown doing its job.
|
|
93
|
+
*
|
|
94
|
+
* `agend update` stops instances, kills their MCP servers and restarts the
|
|
95
|
+
* daemon; every one of those looked like a crash to the alert path, so an
|
|
96
|
+
* upgrade produced a burst of ⚠️ for things that were working correctly. That
|
|
97
|
+
* burst is worse than noise: it teaches the operator to ignore the same alert
|
|
98
|
+
* that matters when a process really does die on its own.
|
|
99
|
+
*
|
|
100
|
+
* The event log and daemon.log entries still happen at the call sites — this
|
|
101
|
+
* suppresses the chat message, not the record. A crash outside a planned
|
|
102
|
+
* restart notifies exactly as before.
|
|
103
|
+
*/
|
|
104
|
+
notifyIncident(name, kind, text) {
|
|
105
|
+
if (this.ctx.isPlannedRestart()) {
|
|
106
|
+
this.ctx.logger.info({ name, kind }, "Incident notification suppressed — planned restart in progress");
|
|
107
|
+
return;
|
|
108
|
+
}
|
|
109
|
+
this.ctx.notifyInstanceTopic(name, text);
|
|
110
|
+
}
|
|
111
|
+
/** Backend a running instance uses (config → fleet default). */
|
|
112
|
+
backendOf(name) {
|
|
113
|
+
return this.ctx.fleetConfig?.instances[name]?.backend
|
|
114
|
+
?? this.ctx.fleetConfig?.defaults?.backend
|
|
115
|
+
?? "claude-code";
|
|
116
|
+
}
|
|
117
|
+
/**
|
|
118
|
+
* One alert per backend per cooldown, naming every affected instance — a CLI's
|
|
119
|
+
* credentials are shared, so N instances failing is ONE problem with ONE fix
|
|
120
|
+
* (re-login once). The per-instance daemon cooldown can't dedupe across
|
|
121
|
+
* instances, so the fleet-level map does it here.
|
|
122
|
+
*/
|
|
123
|
+
notifyAuthErrorOnce(name, message) {
|
|
124
|
+
const backend = this.backendOf(name);
|
|
125
|
+
const now = Date.now();
|
|
126
|
+
const last = this.lastAuthAlertAt.get(backend) ?? 0;
|
|
127
|
+
if (now - last < AUTH_ALERT_COOLDOWN_MS) {
|
|
128
|
+
this.ctx.logger.info({ name, backend }, "auth error suppressed (backend already alerted)");
|
|
129
|
+
return;
|
|
130
|
+
}
|
|
131
|
+
this.lastAuthAlertAt.set(backend, now);
|
|
132
|
+
const affected = [...this.daemons.keys()].filter(n => this.backendOf(n) === backend);
|
|
133
|
+
const others = affected.filter(n => n !== name);
|
|
134
|
+
const scope = others.length
|
|
135
|
+
? `${affected.length} instances on \`${backend}\`: ${affected.join(", ")}`
|
|
136
|
+
: `\`${name}\` (${backend})`;
|
|
137
|
+
this.notifyIncident(name, "auth_error", `🔑 ${message}\n\nAffects ${scope}. Credentials are shared per backend — one re-login restores all of them; affected instances pause until then.`);
|
|
138
|
+
}
|
|
139
|
+
/**
|
|
140
|
+
* Handlers for the events that mean "something went wrong".
|
|
141
|
+
*
|
|
142
|
+
* Extracted from start() so the planned-restart suppression can be exercised
|
|
143
|
+
* against a plain event emitter — the alternative is a real Daemon, which
|
|
144
|
+
* means a real tmux window, which is why this rule went untested before.
|
|
145
|
+
*/
|
|
146
|
+
attachIncidentHandlers(name, daemon) {
|
|
147
|
+
daemon.on("crash_respawn", safeHandler(() => {
|
|
148
|
+
this.ctx.eventLog?.insert(name, "crash_respawn", {});
|
|
149
|
+
this.ctx.logger.warn({ name }, "Instance crashed and respawned");
|
|
150
|
+
this.notifyIncident(name, "crash_respawn", t("inst.crashed_respawned", name));
|
|
151
|
+
const generalName = this.findGeneralInstance();
|
|
152
|
+
if (generalName && generalName !== name) {
|
|
153
|
+
this.notifyIncident(generalName, "crash_respawn", t("inst.crashed_respawned_log", name));
|
|
154
|
+
}
|
|
155
|
+
}, this.ctx.logger, `daemon.crash_respawn[${name}]`));
|
|
156
|
+
daemon.on("snapshot_failed", safeHandler(() => {
|
|
157
|
+
this.ctx.eventLog?.insert(name, "snapshot_failed", {});
|
|
158
|
+
this.notifyIncident(name, "snapshot_failed", t("inst.restarted_no_context", name));
|
|
159
|
+
}, this.ctx.logger, `daemon.snapshot_failed[${name}]`));
|
|
160
|
+
daemon.on("supervision_ended", safeHandler((data) => {
|
|
161
|
+
// The instance is dead and nothing will restart it. Say so where the operator
|
|
162
|
+
// is looking, and mark the topic — otherwise messages routed here just queue
|
|
163
|
+
// or fail with a bare ❌ and the dashboard still looks normal.
|
|
164
|
+
this.ctx.eventLog?.insert(name, "supervision_ended", { reason: data.reason });
|
|
165
|
+
this.ctx.logger.error({ name, reason: data.reason }, "Instance is no longer supervised");
|
|
166
|
+
this.notifyIncident(name, "supervision_ended", `🛑 ${name} is no longer running and will not be restarted automatically — ${data.reason}.\n${data.remedy}`);
|
|
167
|
+
this.ctx.setTopicIcon(name, "red");
|
|
168
|
+
}, this.ctx.logger, `daemon.supervision_ended[${name}]`));
|
|
169
|
+
daemon.on("health_check_error", safeHandler((data) => {
|
|
170
|
+
this.ctx.eventLog?.insert(name, "health_check_error", { message: data.message });
|
|
171
|
+
this.ctx.logger.error({ name, message: data.message }, "Health check failing — instance supervision degraded");
|
|
172
|
+
this.notifyIncident(name, "health_check_error", `⚠️ ${name}: health check is failing (\`${data.message}\`). Crash detection for this instance may be degraded — see daemon.log.`);
|
|
173
|
+
}, this.ctx.logger, `daemon.health_check_error[${name}]`));
|
|
174
|
+
daemon.on("crash_loop", safeHandler(() => {
|
|
175
|
+
this.ctx.eventLog?.insert(name, "crash_loop", {});
|
|
176
|
+
this.ctx.logger.error({ name }, "Instance in crash loop — respawn paused");
|
|
177
|
+
this.notifyIncident(name, "crash_loop", t("inst.respawn_paused", name));
|
|
178
|
+
this.ctx.setTopicIcon(name, "red");
|
|
179
|
+
}, this.ctx.logger, `daemon.crash_loop[${name}]`));
|
|
180
|
+
daemon.on("mcp_died", safeHandler((data) => {
|
|
181
|
+
this.ctx.eventLog?.insert(name, "mcp_died", { pid: data.pid });
|
|
182
|
+
this.ctx.logger.error({ name, pid: data.pid }, "MCP server died — instance cannot use agend tools");
|
|
183
|
+
this.ctx.webhookEmit("mcp_died", name, { pid: data.pid });
|
|
184
|
+
// The CLI owns the MCP server's stdio pipes, so only restarting the CLI can
|
|
185
|
+
// restore its tools — say so instead of implying self-healing. Deliberately
|
|
186
|
+
// NOT auto-restarting: that would interrupt whatever the agent is doing, and
|
|
187
|
+
// an instance whose CLI is otherwise fine may still be doing useful work.
|
|
188
|
+
this.notifyIncident(name, "mcp_died", `⚠️ \`${name}\` 的 MCP server 已終止 — 這個 instance 目前無法使用 agend 工具(無法 reply / 跨 instance 通訊)。\n`
|
|
189
|
+
+ `CLI 本身還在執行。工具只能由 CLI 自己重新啟動 MCP server,請用 \`restart_instance("${name}")\` 或 \`/restart\` 恢復。`);
|
|
190
|
+
}, this.ctx.logger, `daemon.mcp_died[${name}]`));
|
|
191
|
+
daemon.on("pty_error", safeHandler((data) => {
|
|
192
|
+
this.ctx.eventLog?.insert(name, "pty_error", { type: data.type, action: data.action });
|
|
193
|
+
this.ctx.logger.warn({ name, errorType: data.type, action: data.action }, `PTY error: ${data.message}`);
|
|
194
|
+
const emoji = data.type === "rate_limit" || data.type === "timeout" ? "⏳" : data.type === "auth_error" ? "🔑" : "⚠️";
|
|
195
|
+
// Auth failures are a property of the BACKEND's shared credentials, not of
|
|
196
|
+
// one instance: every instance on that CLI fails at once, and one re-login
|
|
197
|
+
// fixes them all. Notify once per backend (listing who's affected) instead
|
|
198
|
+
// of N near-identical alerts, and suppress repeats fleet-wide.
|
|
199
|
+
if (data.type === "auth_error") {
|
|
200
|
+
this.notifyAuthErrorOnce(name, data.message);
|
|
201
|
+
}
|
|
202
|
+
else {
|
|
203
|
+
this.notifyIncident(name, "pty_error", t("inst.notification", emoji, name, data.message, data.action));
|
|
204
|
+
}
|
|
205
|
+
this.ctx.webhookEmit("pty_error", name, { type: data.type, action: data.action, message: data.message });
|
|
206
|
+
// The CLI interrupted itself on this error, so any pending Cancel button is
|
|
207
|
+
// now useless — retire it. We only reach here when the error wasn't
|
|
208
|
+
// cooldown-suppressed (the daemon skips the emit during cooldown), so this
|
|
209
|
+
// won't fire on repeat errors within the 5-min window. No-op if no button.
|
|
210
|
+
this.ctx.clearCancelButton(name);
|
|
211
|
+
if (data.action === "failover") {
|
|
212
|
+
this.ctx.checkModelFailover(name, 100); // Force failover trigger
|
|
213
|
+
}
|
|
214
|
+
else if (data.action === "restart") {
|
|
215
|
+
// A broken *resumed* session (e.g. agy pinned to a dead model) can't
|
|
216
|
+
// self-recover; a plain restart would --continue back into it. freshStart
|
|
217
|
+
// makes the respawn skip resume so the CLI starts a clean session on its
|
|
218
|
+
// default (valid) model.
|
|
219
|
+
this.ctx.restartSingleInstance(name, { freshStart: true }).catch(err => this.ctx.logger.error({ err, name }, "pty_error restart failed"));
|
|
220
|
+
}
|
|
221
|
+
else if (data.action === "pause") {
|
|
222
|
+
// Previously unhandled, so an expired session kept receiving messages and
|
|
223
|
+
// re-sending its whole context into a CLI that could only fail — wasted
|
|
224
|
+
// credit and lost work. pause() alone no-ops while the pane is busy (the
|
|
225
|
+
// usual state when auth fails), so mark it to pause as soon as it idles.
|
|
226
|
+
// Queued messages survive: delivery wakes a paused instance.
|
|
227
|
+
void this.pause(name)
|
|
228
|
+
.catch(err => this.ctx.logger.warn({ err, name }, "auth-error pause failed"))
|
|
229
|
+
.finally(() => { if (!this.isPaused(name))
|
|
230
|
+
daemon.requestPauseWhenIdle(); });
|
|
231
|
+
}
|
|
232
|
+
}, this.ctx.logger, `daemon.pty_error[${name}]`));
|
|
233
|
+
}
|
|
234
|
+
async start(name, config, topicMode, runtimeIdentity) {
|
|
87
235
|
if (this.daemons.has(name)) {
|
|
88
236
|
this.ctx.logger.info({ name }, "Instance already running, skipping");
|
|
89
237
|
return;
|
|
@@ -116,7 +264,11 @@ export class InstanceLifecycle {
|
|
|
116
264
|
return;
|
|
117
265
|
}
|
|
118
266
|
const backend = createBackend(backendName, instanceDir);
|
|
119
|
-
const daemon = new Daemon(name, config, instanceDir, topicMode, backend, this.ctx.controlClient ?? undefined, this.ctx.logger
|
|
267
|
+
const daemon = new Daemon(name, config, instanceDir, topicMode, backend, this.ctx.controlClient ?? undefined, this.ctx.logger, runtimeIdentity ?? {
|
|
268
|
+
kind: "fleet-topic",
|
|
269
|
+
backend: backendName,
|
|
270
|
+
model: config.model ?? "default",
|
|
271
|
+
});
|
|
120
272
|
// Catch errors from daemon internals (e.g. IPC server) to prevent crashing the fleet process
|
|
121
273
|
daemon.on("error", (err) => {
|
|
122
274
|
this.ctx.logger.error({ err, name }, "Daemon emitted error — instance isolated");
|
|
@@ -161,47 +313,7 @@ export class InstanceLifecycle {
|
|
|
161
313
|
this.ctx.setTopicIcon(name, "green");
|
|
162
314
|
this.ctx.touchActivity(name);
|
|
163
315
|
});
|
|
164
|
-
|
|
165
|
-
this.ctx.eventLog?.insert(name, "crash_respawn", {});
|
|
166
|
-
this.ctx.logger.warn({ name }, "Instance crashed and respawned");
|
|
167
|
-
this.ctx.notifyInstanceTopic(name, t("inst.crashed_respawned", name));
|
|
168
|
-
const generalName = this.findGeneralInstance();
|
|
169
|
-
if (generalName && generalName !== name) {
|
|
170
|
-
this.ctx.notifyInstanceTopic(generalName, t("inst.crashed_respawned_log", name));
|
|
171
|
-
}
|
|
172
|
-
}, this.ctx.logger, `daemon.crash_respawn[${name}]`));
|
|
173
|
-
daemon.on("snapshot_failed", safeHandler(() => {
|
|
174
|
-
this.ctx.eventLog?.insert(name, "snapshot_failed", {});
|
|
175
|
-
this.ctx.notifyInstanceTopic(name, t("inst.restarted_no_context", name));
|
|
176
|
-
}, this.ctx.logger, `daemon.snapshot_failed[${name}]`));
|
|
177
|
-
daemon.on("crash_loop", safeHandler(() => {
|
|
178
|
-
this.ctx.eventLog?.insert(name, "crash_loop", {});
|
|
179
|
-
this.ctx.logger.error({ name }, "Instance in crash loop — respawn paused");
|
|
180
|
-
this.ctx.notifyInstanceTopic(name, t("inst.respawn_paused", name));
|
|
181
|
-
this.ctx.setTopicIcon(name, "red");
|
|
182
|
-
}, this.ctx.logger, `daemon.crash_loop[${name}]`));
|
|
183
|
-
daemon.on("pty_error", safeHandler((data) => {
|
|
184
|
-
this.ctx.eventLog?.insert(name, "pty_error", { type: data.type, action: data.action });
|
|
185
|
-
this.ctx.logger.warn({ name, errorType: data.type, action: data.action }, `PTY error: ${data.message}`);
|
|
186
|
-
const emoji = data.type === "rate_limit" || data.type === "timeout" ? "⏳" : data.type === "auth_error" ? "🔑" : "⚠️";
|
|
187
|
-
this.ctx.notifyInstanceTopic(name, t("inst.notification", emoji, name, data.message, data.action));
|
|
188
|
-
this.ctx.webhookEmit("pty_error", name, { type: data.type, action: data.action, message: data.message });
|
|
189
|
-
// The CLI interrupted itself on this error, so any pending Cancel button is
|
|
190
|
-
// now useless — retire it. We only reach here when the error wasn't
|
|
191
|
-
// cooldown-suppressed (the daemon skips the emit during cooldown), so this
|
|
192
|
-
// won't fire on repeat errors within the 5-min window. No-op if no button.
|
|
193
|
-
this.ctx.clearCancelButton(name);
|
|
194
|
-
if (data.action === "failover") {
|
|
195
|
-
this.ctx.checkModelFailover(name, 100); // Force failover trigger
|
|
196
|
-
}
|
|
197
|
-
else if (data.action === "restart") {
|
|
198
|
-
// A broken *resumed* session (e.g. agy pinned to a dead model) can't
|
|
199
|
-
// self-recover; a plain restart would --continue back into it. freshStart
|
|
200
|
-
// makes the respawn skip resume so the CLI starts a clean session on its
|
|
201
|
-
// default (valid) model.
|
|
202
|
-
this.ctx.restartSingleInstance(name, { freshStart: true }).catch(err => this.ctx.logger.error({ err, name }, "pty_error restart failed"));
|
|
203
|
-
}
|
|
204
|
-
}, this.ctx.logger, `daemon.pty_error[${name}]`));
|
|
316
|
+
this.attachIncidentHandlers(name, daemon);
|
|
205
317
|
daemon.on("pty_recovered", safeHandler((data) => {
|
|
206
318
|
const mins = Math.floor(data.downtime_s / 60);
|
|
207
319
|
const secs = data.downtime_s % 60;
|