@animalabs/connectome-host 0.3.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.env.example +20 -0
- package/.github/workflows/publish.yml +65 -0
- package/ARCHITECTURE.md +355 -0
- package/CHANGELOG.md +30 -0
- package/HEADLESS-FLEET-PLAN.md +330 -0
- package/README.md +189 -0
- package/UNIFIED-TREE-PLAN.md +242 -0
- package/bun.lock +288 -0
- package/docs/AGENT-MEMORY-GUIDE.md +214 -0
- package/docs/AGENT-ONBOARDING.md +541 -0
- package/docs/ATTENTION-AND-GATING.md +120 -0
- package/docs/DEPLOYMENTS.md +130 -0
- package/docs/DEV-ENVIRONMENT.md +215 -0
- package/docs/LIBRARY-PIPELINE.md +286 -0
- package/docs/LOCUS-ROUTING-DESIGN.md +115 -0
- package/docs/claudeai-evacuation.md +228 -0
- package/docs/debug-context-api.md +208 -0
- package/docs/webui-deployment.md +219 -0
- package/package.json +33 -0
- package/recipes/SETUP.md +308 -0
- package/recipes/TRIUMVIRATE-SETUP.md +467 -0
- package/recipes/claude-export-revive.json +19 -0
- package/recipes/clerk.json +157 -0
- package/recipes/knowledge-miner.json +174 -0
- package/recipes/knowledge-reviewer.json +76 -0
- package/recipes/mcpl-editor-test.json +38 -0
- package/recipes/prompts/transplant-addendum.md +17 -0
- package/recipes/triumvirate.json +63 -0
- package/recipes/webui-fleet-test.json +27 -0
- package/recipes/webui-test.json +16 -0
- package/recipes/zulip-miner.json +87 -0
- package/scripts/connectome-doctor +373 -0
- package/scripts/evacuator.ts +956 -0
- package/scripts/import-claudeai-export.ts +747 -0
- package/scripts/lib/line-reader.ts +53 -0
- package/scripts/test-historical-thinking.ts +148 -0
- package/scripts/warmup-session.ts +336 -0
- package/src/agent-name.ts +58 -0
- package/src/commands.ts +1074 -0
- package/src/headless.ts +528 -0
- package/src/index.ts +867 -0
- package/src/logging-adapter.ts +208 -0
- package/src/mcpl-config.ts +199 -0
- package/src/modules/activity-module.ts +270 -0
- package/src/modules/channel-mode-module.ts +260 -0
- package/src/modules/fleet-module.ts +1705 -0
- package/src/modules/fleet-types.ts +225 -0
- package/src/modules/lessons-module.ts +465 -0
- package/src/modules/mcpl-admin-module.ts +345 -0
- package/src/modules/retrieval-module.ts +327 -0
- package/src/modules/settings-module.ts +143 -0
- package/src/modules/subagent-module.ts +2074 -0
- package/src/modules/subscription-gc-module.ts +322 -0
- package/src/modules/time-module.ts +85 -0
- package/src/modules/tui-module.ts +76 -0
- package/src/modules/web-ui-curve-page.ts +249 -0
- package/src/modules/web-ui-module.ts +2595 -0
- package/src/recipe.ts +1003 -0
- package/src/session-manager.ts +278 -0
- package/src/state/agent-tree-reducer.ts +431 -0
- package/src/state/fleet-tree-aggregator.ts +195 -0
- package/src/strategies/frontdesk-strategy.ts +364 -0
- package/src/synesthete.ts +68 -0
- package/src/tui.ts +2200 -0
- package/src/types/bun-ffi.d.ts +6 -0
- package/src/web/protocol.ts +648 -0
- package/test/agent-name.test.ts +62 -0
- package/test/agent-tree-fleet-launch.test.ts +64 -0
- package/test/agent-tree-reducer-parity.test.ts +194 -0
- package/test/agent-tree-reducer.test.ts +443 -0
- package/test/agent-tree-rollup.test.ts +109 -0
- package/test/autobio-progress-snapshot.test.ts +40 -0
- package/test/claudeai-export-importer.test.ts +386 -0
- package/test/evacuator.test.ts +332 -0
- package/test/fleet-commands.test.ts +244 -0
- package/test/fleet-no-subfleets.test.ts +147 -0
- package/test/fleet-orchestration.test.ts +353 -0
- package/test/fleet-recipe.test.ts +124 -0
- package/test/fleet-route.test.ts +44 -0
- package/test/fleet-smoke.test.ts +479 -0
- package/test/fleet-subscribe-union-e2e.test.ts +123 -0
- package/test/fleet-subscribe-union.test.ts +85 -0
- package/test/fleet-tree-aggregator-e2e.test.ts +110 -0
- package/test/fleet-tree-aggregator.test.ts +215 -0
- package/test/frontdesk-strategy.test.ts +326 -0
- package/test/headless-describe.test.ts +182 -0
- package/test/headless-smoke.test.ts +190 -0
- package/test/logging-adapter.test.ts +87 -0
- package/test/mcpl-admin-module.test.ts +213 -0
- package/test/mcpl-agent-overlay.test.ts +126 -0
- package/test/mock-headless-child.ts +133 -0
- package/test/recipe-cache-ttl.test.ts +37 -0
- package/test/recipe-env.test.ts +157 -0
- package/test/recipe-path-resolution.test.ts +170 -0
- package/test/subagent-async-timeout.test.ts +381 -0
- package/test/subagent-fork-materialise.test.ts +241 -0
- package/test/subagent-peek-scoping.test.ts +180 -0
- package/test/subagent-peek-zombie.test.ts +148 -0
- package/test/subagent-reaper-in-flight.test.ts +229 -0
- package/test/subscription-gc-module.test.ts +136 -0
- package/test/warmup-handoff.test.ts +150 -0
- package/test/web-ui-bind.test.ts +51 -0
- package/test/web-ui-module.test.ts +246 -0
- package/test/web-ui-protocol.test.ts +0 -0
- package/tsconfig.json +14 -0
- package/web/bun.lock +357 -0
- package/web/index.html +12 -0
- package/web/package.json +24 -0
- package/web/src/App.tsx +1931 -0
- package/web/src/Context.tsx +158 -0
- package/web/src/ContextDocument.tsx +150 -0
- package/web/src/Files.tsx +271 -0
- package/web/src/Lessons.tsx +164 -0
- package/web/src/Mcpl.tsx +310 -0
- package/web/src/Stream.tsx +119 -0
- package/web/src/TreeSidebar.tsx +283 -0
- package/web/src/Usage.tsx +182 -0
- package/web/src/main.tsx +7 -0
- package/web/src/styles.css +26 -0
- package/web/src/tree.ts +268 -0
- package/web/src/wire.ts +120 -0
- package/web/tsconfig.json +21 -0
- package/web/vite.config.ts +32 -0
|
@@ -0,0 +1,330 @@
|
|
|
1
|
+
# Headless Daemon Mode & Fleet Orchestration — Plan
|
|
2
|
+
|
|
3
|
+
Status: draft • Authors: conversation between @tengro and Claude, 2026-04-21
|
|
4
|
+
|
|
5
|
+
## Goal
|
|
6
|
+
|
|
7
|
+
Run the Knowledge Mining Triumvirate (miner / reviewer / clerk) as a single orchestrated system in one terminal instead of three separate TUIs. The parent TUI runs a "conductor" agent with its own LLM that spawns and observes the three child recipes. Children run headless with JSONL IPC over Unix domain sockets. Inter-child communication stays what it is today — Zulip channels and shared workspace mounts — unchanged.
|
|
8
|
+
|
|
9
|
+
## High-level architecture
|
|
10
|
+
|
|
11
|
+
```
|
|
12
|
+
┌───────────── connectome-host (parent TUI) ────────────┐
|
|
13
|
+
│ recipe: triumvirate.json │
|
|
14
|
+
│ agent: "conductor" (LLM, own context) │
|
|
15
|
+
│ modules: │
|
|
16
|
+
│ - FleetModule ──┐ │
|
|
17
|
+
│ - workspace │ tools: fleet--launch / list / │
|
|
18
|
+
│ - lessons │ status / send / command /│
|
|
19
|
+
│ - ... │ peek / kill / restart / │
|
|
20
|
+
│ - ... │ relay / await │
|
|
21
|
+
│ TUI: chat + fleet pane (per-child status + tail) │
|
|
22
|
+
└────────────────┬──────────────────────────────────────┘
|
|
23
|
+
│ spawn (detached), connect Unix socket
|
|
24
|
+
┌───────────┼───────────┐
|
|
25
|
+
▼ ▼ ▼
|
|
26
|
+
┌────────┐ ┌────────┐ ┌────────┐
|
|
27
|
+
│ miner │ │reviewer│ │ clerk │ connectome-host --headless
|
|
28
|
+
│ DATA= │ │ DATA= │ │ DATA= │ recipe per child
|
|
29
|
+
│ data/m │ │ data/r │ │ data/c │ own Chronicle store
|
|
30
|
+
└───┬────┘ └───┬────┘ └───┬────┘ ipc.sock per child
|
|
31
|
+
│ JSONL │ │
|
|
32
|
+
└──────────┴───────────┘
|
|
33
|
+
events out, commands in
|
|
34
|
+
|
|
35
|
+
Cross-child: Zulip + shared workspace mounts (unchanged)
|
|
36
|
+
```
|
|
37
|
+
|
|
38
|
+
### Data flow
|
|
39
|
+
|
|
40
|
+
1. Parent TUI starts with `recipes/triumvirate.json`. FleetModule reads `modules.fleet.children` and spawns each `autoStart` child as a detached subprocess.
|
|
41
|
+
2. Each child runs `connectome-host --headless <recipe>`. It creates a Unix socket at `{DATA_DIR}/ipc.sock`, emits JSONL trace events, accepts JSONL commands.
|
|
42
|
+
3. Parent connects to each child's socket, parses events, maintains per-child state, surfaces it to the conductor agent (context injection) and TUI (fleet pane, peek).
|
|
43
|
+
4. Conductor agent uses `fleet--*` tools to observe / send / kill. User input in TUI routes to conductor by default; `@childname …` routes a line directly to a specific child.
|
|
44
|
+
5. On parent `/quit`: prompt — default stops all children, `/quit --detach` leaves them running.
|
|
45
|
+
6. On parent crash (Ctrl+C, SIGKILL): children survive (detached processes own their own sockets).
|
|
46
|
+
7. On parent re-launch: scan known data dirs for `ipc.sock`, probe liveness, reattach living children, respawn dead ones that are marked `autoStart`.
|
|
47
|
+
|
|
48
|
+
## Decisions (resolved)
|
|
49
|
+
|
|
50
|
+
### 1. Process model
|
|
51
|
+
**Q:** In-process subagents with different recipes, or separate processes?
|
|
52
|
+
**A:** Separate processes. Each child is its own `connectome-host` invocation with its own recipe, data dir, Chronicle store, and lifecycle.
|
|
53
|
+
|
|
54
|
+
### 2. Lifecycle shape
|
|
55
|
+
**Q:** One-shot, event-driven daemon, or task-bounded?
|
|
56
|
+
**A:** Event-driven daemon primarily. One-shot mode also supported via a per-child `--exit-when-idle` flag (or equivalent recipe setting).
|
|
57
|
+
|
|
58
|
+
### 3. Output contract
|
|
59
|
+
**Q:** Plain text, JSONL, or final blob?
|
|
60
|
+
**A:** JSONL. Every framework `TraceEvent` serialized as one line; plus lifecycle events added by the headless runtime. Enables the parent to reuse the same `onTrace` rendering logic it already uses in-process.
|
|
61
|
+
|
|
62
|
+
### 4. Data dir isolation per child
|
|
63
|
+
**Q:** Shared Chronicle under sub-namespaces, or separate fully?
|
|
64
|
+
**A:** Separate fully. Each child gets its own data dir, Chronicle store, sessions index, lessons store. Prevents conflicts and matches the existing separate-TUIs pattern.
|
|
65
|
+
|
|
66
|
+
### 5. Parent TUI shape
|
|
67
|
+
**Q:** New app, or recipe variant with a new module?
|
|
68
|
+
**A:** Recipe variant with a new module (`FleetModule`). The conductor agent has its own LLM and reasons about the fleet.
|
|
69
|
+
|
|
70
|
+
### 6. User interaction model
|
|
71
|
+
**Q:** Dashboard only, routed to specific child, or talk to parent agent?
|
|
72
|
+
**A:** Talk to parent agent by default (option III), `@childname …` as an escape hatch to target a specific child directly (option II). No mode toggle required.
|
|
73
|
+
|
|
74
|
+
### 7. Runtime
|
|
75
|
+
**Q:** Bun or something configurable?
|
|
76
|
+
**A:** Bun. Same binary (`bun src/index.ts --headless …`) for parent and children.
|
|
77
|
+
|
|
78
|
+
### 8. Environment / credentials
|
|
79
|
+
**Q:** Inherit parent env, or per-recipe env files?
|
|
80
|
+
**A:** Inherit parent env by default. Recipe's fleet entry can override via an `env` block.
|
|
81
|
+
|
|
82
|
+
### 9. Parent-restart behavior
|
|
83
|
+
**Q:** Adopt live children, respawn, or refuse to start?
|
|
84
|
+
**A:** Adopt (option a) via Unix socket IPC. Dead orphans (socket exists, PID gone) get cleaned up and respawned if `autoStart`.
|
|
85
|
+
|
|
86
|
+
### 10. Conductor authority to spawn
|
|
87
|
+
**Q:** Recipe-config-only, or LLM-callable?
|
|
88
|
+
**A:** LLM-callable, but bounded by a recipe-level `allowedRecipes` allowlist (defaults to the recipe paths listed under `fleet.children`). Out-of-allowlist spawn calls pause and prompt the user for confirmation via the TUI.
|
|
89
|
+
|
|
90
|
+
### 11. Trace event volume
|
|
91
|
+
**Q:** Filter or send everything?
|
|
92
|
+
**A:** Filter. Subscription at handshake — child only emits event types the parent asked for. Parent also fans out internally so each consumer (conductor context, TUI pane, peek view) can get a different slice from the same subscription.
|
|
93
|
+
|
|
94
|
+
### 12. IPC transport
|
|
95
|
+
**Q:** stdio pipes or sockets? (Derived from #9)
|
|
96
|
+
**A:** Unix domain socket at `{DATA_DIR}/ipc.sock`. Stdio would die with the parent and preclude adoption-on-restart. Children use `detached: true` + `stdio: 'ignore'` so they survive parent death; stdout/stderr go to `{DATA_DIR}/headless.log`.
|
|
97
|
+
|
|
98
|
+
### 13. Nested fleets
|
|
99
|
+
**Q:** Can a fleet child itself declare a `fleet` module and supervise grandchildren?
|
|
100
|
+
**A:** No. Depth-1 invariant enforced by `FleetModule.handleLaunch()`: the child's recipe is loaded and validated before subprocess spawn; if it declares `modules.fleet` (true or object), the launch is rejected with a clear error. This keeps cross-process tree visibility tractable for the unified TUI rendering — the grandparent never needs to peer through the parent to discover grandchildren. See [UNIFIED-TREE-PLAN.md](./UNIFIED-TREE-PLAN.md) §6 for the design rationale and forward-compatibility notes (a future `via:` re-broadcast scheme could lift this invariant if the requirement appears).
|
|
101
|
+
|
|
102
|
+
## Decisions (deferred)
|
|
103
|
+
|
|
104
|
+
- **Log rotation policy** for `{DATA_DIR}/headless.log`. Start with append-only, revisit when files get big.
|
|
105
|
+
- **Conductor wake policies for fleet events** — initial default is "wake on crash / unexpected exit, don't wake on routine child activity" to keep the conductor's context lean. Final policy set to be tuned from real usage.
|
|
106
|
+
- **Windows support** for Unix domain sockets. Project currently Linux/WSL; revisit if needed.
|
|
107
|
+
- **Per-child auto-restart backoff** (default: off; if on, exponential backoff with cap). Ship without, add when we see crash patterns.
|
|
108
|
+
|
|
109
|
+
## Components & files
|
|
110
|
+
|
|
111
|
+
### A. Headless daemon mode (child capability)
|
|
112
|
+
|
|
113
|
+
**Files to touch:**
|
|
114
|
+
- `src/index.ts` — add `--headless` flag parsing; route to new `runHeadless()` instead of `runPiped()`.
|
|
115
|
+
- `src/headless.ts` (new) — headless runtime: socket server, JSONL framing, subscription filter, lifecycle events, graceful shutdown, SIGTERM handler, stderr redirect.
|
|
116
|
+
- `src/modules/tui-module.ts` — accept `source: 'headless'` alongside `cli` / `tui` / `system`.
|
|
117
|
+
|
|
118
|
+
**Flags:**
|
|
119
|
+
- `--headless` — enter headless mode; create socket at `{DATA_DIR}/ipc.sock`.
|
|
120
|
+
- `--exit-when-idle` — one-shot; exit after first `runUntilIdle`-style quiescence (no pending events, no wake subscriptions active).
|
|
121
|
+
- `--socket-path <path>` — override default socket location (optional).
|
|
122
|
+
|
|
123
|
+
### B. FleetModule (parent's new module)
|
|
124
|
+
|
|
125
|
+
**Files:**
|
|
126
|
+
- `src/modules/fleet-module.ts` (new) — module class, tool handlers, subprocess lifecycle manager, socket client, per-child state, Chronicle persistence, restart/adopt logic.
|
|
127
|
+
- `src/modules/fleet-types.ts` (new) — wire protocol types shared with `headless.ts`.
|
|
128
|
+
|
|
129
|
+
**Tools (exposed to conductor agent):**
|
|
130
|
+
- `fleet--launch {name, recipe, dataDir?, env?, autoRestart?}` — launch a child (separate OS process, distinct from `subagent--spawn`). Checks `allowedRecipes`, prompts user if out-of-list.
|
|
131
|
+
- `fleet--list` — enumerate children with status.
|
|
132
|
+
- `fleet--status {name?}` — detailed status for one or all.
|
|
133
|
+
- `fleet--send {name, content}` — send user-like message to child.
|
|
134
|
+
- `fleet--command {name, command}` — send slash command to child.
|
|
135
|
+
- `fleet--peek {name, lines?}` — last N events from child's rolling buffer.
|
|
136
|
+
- `fleet--kill {name}` — graceful shutdown → SIGTERM → SIGKILL escalation.
|
|
137
|
+
- `fleet--restart {name}` — kill then respawn.
|
|
138
|
+
|
|
139
|
+
**Per-child state (persisted to Chronicle module state):**
|
|
140
|
+
`{ name, recipePath, dataDir, pid, status, startedAt, lastEventAt, autoStart, autoRestart, socketPath, subscription }`.
|
|
141
|
+
|
|
142
|
+
**`gatherContext()`** — injects a compact fleet-status block (similar to subagent HUD) before each conductor inference, if enabled.
|
|
143
|
+
|
|
144
|
+
**`onProcess()`** — handles `fleet:child-event` scope events for conductor wake policies.
|
|
145
|
+
|
|
146
|
+
### C. Recipe schema extension
|
|
147
|
+
|
|
148
|
+
**Files:**
|
|
149
|
+
- `src/recipe.ts` — extend `RecipeModules` with `fleet` type, validate, resolve child recipe paths relative to the parent recipe file.
|
|
150
|
+
|
|
151
|
+
**Schema:**
|
|
152
|
+
```jsonc
|
|
153
|
+
"modules": {
|
|
154
|
+
"fleet": {
|
|
155
|
+
"children": [
|
|
156
|
+
{
|
|
157
|
+
"name": "miner",
|
|
158
|
+
"recipe": "recipes/zulip-miner.json",
|
|
159
|
+
"dataDir": "./data/miner",
|
|
160
|
+
"autoStart": true,
|
|
161
|
+
"autoRestart": false,
|
|
162
|
+
"env": { "OPTIONAL_OVERRIDE": "value" },
|
|
163
|
+
"subscription": ["lifecycle", "inference:completed", "tool:completed", "tool:failed", "inference:failed"]
|
|
164
|
+
}
|
|
165
|
+
],
|
|
166
|
+
"allowedRecipes": ["recipes/*.json", "https://trusted.example.com/*"],
|
|
167
|
+
"defaultSubscription": ["lifecycle", "inference:completed", "tool:completed", "tool:failed", "inference:failed"]
|
|
168
|
+
}
|
|
169
|
+
}
|
|
170
|
+
```
|
|
171
|
+
|
|
172
|
+
### D. TUI integration
|
|
173
|
+
|
|
174
|
+
**Files:**
|
|
175
|
+
- `src/tui.ts` — new view mode `fleet-process` (distinct from the existing subagent fleet tree); input routing for `@childname`; peek variant that tails a child process stream.
|
|
176
|
+
- `src/commands.ts` — new slash commands: `/fleet list | start <name> | stop <name> | restart <name> | peek <name>`.
|
|
177
|
+
|
|
178
|
+
**Interaction:**
|
|
179
|
+
- Default: user input → conductor agent (unchanged from today).
|
|
180
|
+
- `@miner look at #router-dev` → `fleet--send {name:"miner", content:"look at #router-dev"}` executed directly, bypassing conductor.
|
|
181
|
+
- Tab cycles view modes: chat → subagent-fleet → process-fleet → peek → chat.
|
|
182
|
+
|
|
183
|
+
### E. Triumvirate meta-recipe
|
|
184
|
+
|
|
185
|
+
**File:**
|
|
186
|
+
- `recipes/triumvirate.json` (new) — conductor system prompt + fleet children entries for miner / reviewer / clerk.
|
|
187
|
+
|
|
188
|
+
Conductor prompt themes:
|
|
189
|
+
- Your three children coordinate among themselves via Zulip and shared workspace mounts; stay out of their way by default.
|
|
190
|
+
- Observe via `fleet--peek`, intervene via `fleet--send` only when the user explicitly asks or a child signals a crash.
|
|
191
|
+
- Spawning recipes outside `allowedRecipes` requires user confirmation; don't surprise the user.
|
|
192
|
+
|
|
193
|
+
## Protocol specs
|
|
194
|
+
|
|
195
|
+
### Child → Parent (events, JSONL)
|
|
196
|
+
|
|
197
|
+
One JSON object per line. All events include `type`; most include `ts` (epoch ms).
|
|
198
|
+
|
|
199
|
+
**Framework trace events (pass-through):**
|
|
200
|
+
```json
|
|
201
|
+
{"type":"inference:started","agentName":"miner","ts":1713700000000}
|
|
202
|
+
{"type":"inference:tokens","agentName":"miner","content":"..."}
|
|
203
|
+
{"type":"inference:tool_calls_yielded","agentName":"miner","calls":[{"id":"...","name":"...","input":{}}]}
|
|
204
|
+
{"type":"tool:started","tool":"workspace--read","callId":"...","input":{}}
|
|
205
|
+
{"type":"tool:completed","callId":"...","tool":"workspace--read","durationMs":142}
|
|
206
|
+
{"type":"tool:failed","callId":"...","tool":"...","error":"..."}
|
|
207
|
+
{"type":"inference:completed","agentName":"miner","tokenUsage":{"input":1234,"output":567}}
|
|
208
|
+
{"type":"inference:failed","agentName":"miner","error":"..."}
|
|
209
|
+
{"type":"message:added","source":"...","participant":"...","content":[...]}
|
|
210
|
+
```
|
|
211
|
+
|
|
212
|
+
**Lifecycle events (added by headless runtime):**
|
|
213
|
+
```json
|
|
214
|
+
{"type":"lifecycle","phase":"ready","pid":12345,"recipe":"recipes/zulip-miner.json","dataDir":"./data/miner"}
|
|
215
|
+
{"type":"lifecycle","phase":"idle"}
|
|
216
|
+
{"type":"lifecycle","phase":"exiting","reason":"shutdown-command|sigterm|crash"}
|
|
217
|
+
```
|
|
218
|
+
|
|
219
|
+
### Parent → Child (commands, JSONL)
|
|
220
|
+
|
|
221
|
+
```json
|
|
222
|
+
{"type":"subscribe","events":["lifecycle","inference:completed","tool:*"]}
|
|
223
|
+
{"type":"text","content":"Please resummarize last week"}
|
|
224
|
+
{"type":"command","command":"/status"}
|
|
225
|
+
{"type":"shutdown","graceful":true}
|
|
226
|
+
```
|
|
227
|
+
|
|
228
|
+
- `subscribe` is idempotent. Typically sent once right after connection. Supports simple glob (`tool:*`, `inference:*`).
|
|
229
|
+
- `text` produces an `external-message` event with `source: 'headless'` — same effect as user typing into the child's own TUI.
|
|
230
|
+
- `command` routes through the child's `commands.ts` handler exactly as if typed locally.
|
|
231
|
+
- `shutdown` sets `graceful`; child completes in-flight inference, then exits. `graceful: false` is equivalent to SIGTERM.
|
|
232
|
+
|
|
233
|
+
### Connection lifecycle
|
|
234
|
+
|
|
235
|
+
1. Parent opens socket at known path. If connect fails: check PID file → alive = retry with backoff; dead = respawn (if autoStart) or mark dead.
|
|
236
|
+
2. Parent sends `subscribe` immediately after connect.
|
|
237
|
+
3. Child sends `lifecycle:ready` once framework is fully up.
|
|
238
|
+
4. Normal steady-state event flow.
|
|
239
|
+
5. On parent `/quit` (default): parent sends `shutdown {graceful:true}` to each child, waits up to 30s, then SIGTERM.
|
|
240
|
+
6. On parent `/quit --detach`: parent closes socket without `shutdown`; children stay up.
|
|
241
|
+
7. On parent crash: sockets get closed at OS level; children notice via `'end'` event on their accept socket, continue running, accept a new connection when parent comes back.
|
|
242
|
+
|
|
243
|
+
## Phased implementation
|
|
244
|
+
|
|
245
|
+
### Phase 1 — Headless daemon mode (child-only capability)
|
|
246
|
+
|
|
247
|
+
**Deliverables:**
|
|
248
|
+
- `src/headless.ts` implementing the protocol above.
|
|
249
|
+
- `--headless` flag in `src/index.ts`.
|
|
250
|
+
- Unit tests: socket server starts, accepts one connection, `subscribe` filters outgoing events, `shutdown` exits cleanly.
|
|
251
|
+
|
|
252
|
+
**Acceptance:**
|
|
253
|
+
- `bun src/index.ts --headless recipes/zulip-miner.json` starts; socket exists at `data/ipc.sock`.
|
|
254
|
+
- Test client connects, sends `{"type":"subscribe","events":["*"]}`, then `{"type":"text","content":"hello"}`; receives inference event stream.
|
|
255
|
+
- Client sends `{"type":"shutdown"}`; child exits 0; socket file removed.
|
|
256
|
+
- SIGTERM: child exits gracefully; socket removed.
|
|
257
|
+
- Client disconnect: child stays up; another client can connect.
|
|
258
|
+
- `stderr` lands in `{DATA_DIR}/headless.log`; `stdout` is JSONL only.
|
|
259
|
+
|
|
260
|
+
### Phase 2 — FleetModule (core, no durability)
|
|
261
|
+
|
|
262
|
+
**Deliverables:**
|
|
263
|
+
- `src/modules/fleet-module.ts` with spawn / list / status / send / command / peek / kill tools.
|
|
264
|
+
- Detached subprocess launch, socket connect, JSONL parse, per-child rolling buffer (last 500 events).
|
|
265
|
+
- No Chronicle persistence, no autoRestart, no reattach yet.
|
|
266
|
+
|
|
267
|
+
**Acceptance:**
|
|
268
|
+
- In a test parent instance, conductor calls `fleet--launch {name:"test", recipe:"recipes/zulip-miner.json"}` → subprocess starts, socket connects.
|
|
269
|
+
- `fleet--status {name:"test"}` returns `ready`.
|
|
270
|
+
- `fleet--send {name:"test", content:"say hi"}` → child runs inference; `fleet--peek {name:"test"}` shows the events.
|
|
271
|
+
- `fleet--kill {name:"test"}` → child exits; status becomes `exited`.
|
|
272
|
+
|
|
273
|
+
### Phase 3 — TUI fleet pane
|
|
274
|
+
|
|
275
|
+
**Deliverables:**
|
|
276
|
+
- New view mode rendering children list + status + last-event snippet.
|
|
277
|
+
- `@childname` input routing in `tui.ts`.
|
|
278
|
+
- Peek mode tailing a child's live stream.
|
|
279
|
+
- `/fleet …` slash commands in `commands.ts`.
|
|
280
|
+
|
|
281
|
+
**Acceptance:**
|
|
282
|
+
- Run a minimal parent that spawns two children; fleet pane shows both, color-coded.
|
|
283
|
+
- `@miner help` routes directly to miner; the conductor does not see or process it.
|
|
284
|
+
- Peek mode updates in real time as the child streams tokens.
|
|
285
|
+
|
|
286
|
+
### Phase 4 — Recipe schema + triumvirate.json
|
|
287
|
+
|
|
288
|
+
**Deliverables:**
|
|
289
|
+
- `fleet` config parsing + validation in `src/recipe.ts`.
|
|
290
|
+
- `autoStart` wiring: children launch when parent framework starts.
|
|
291
|
+
- `allowedRecipes` enforcement with user-prompt flow for out-of-list spawns.
|
|
292
|
+
- `recipes/triumvirate.json` with conductor prompt + three children configured.
|
|
293
|
+
|
|
294
|
+
**Acceptance:**
|
|
295
|
+
- `bun src/index.ts recipes/triumvirate.json` boots parent TUI + three children automatically.
|
|
296
|
+
- TUI shows all three children ready within a few seconds.
|
|
297
|
+
- Conductor can enumerate them via `fleet--list`.
|
|
298
|
+
- Smoke test: ask conductor "are the children healthy?" — gets a sensible answer based on fleet state.
|
|
299
|
+
|
|
300
|
+
### Phase 5 — Durability & polish
|
|
301
|
+
|
|
302
|
+
**Deliverables:**
|
|
303
|
+
- Chronicle persistence of fleet state (per-child record).
|
|
304
|
+
- Reattach-on-restart: scan data dirs, probe sockets + PID files, adopt live children.
|
|
305
|
+
- `autoRestart` on crash with simple policy (immediate retry, up to N times, fail hard after).
|
|
306
|
+
- Event subscription fan-out: different filtered streams for conductor context vs. TUI pane vs. peek.
|
|
307
|
+
- `/quit` prompt: "Stop children? [Y/n/detach]" — default stop.
|
|
308
|
+
- `--exit-when-idle` one-shot mode for children.
|
|
309
|
+
|
|
310
|
+
**Acceptance:**
|
|
311
|
+
- Kill the parent mid-session; restart; children appear re-adopted with live streams resuming.
|
|
312
|
+
- Crash a child (e.g. forced kill); with `autoRestart: true`, it respawns; without, status shows `crashed`.
|
|
313
|
+
- Conductor's context does not include streaming tokens (filtered); TUI peek shows them (not filtered).
|
|
314
|
+
|
|
315
|
+
## Out of scope
|
|
316
|
+
|
|
317
|
+
- Multi-machine orchestration (everything is local, single-host Unix sockets).
|
|
318
|
+
- Children hosting their own fleets (no recursive FleetModule).
|
|
319
|
+
- Config hot-reload (parent restart required to change fleet composition).
|
|
320
|
+
- Windows native support (Linux/WSL2 initially; socket path semantics are the main blocker).
|
|
321
|
+
- Log rotation (append-only until size issues arise).
|
|
322
|
+
- A GUI / web dashboard (terminal only; CLI tools can use the JSONL protocol directly if they want).
|
|
323
|
+
|
|
324
|
+
## References
|
|
325
|
+
|
|
326
|
+
- `src/index.ts` — current `runPiped` path, starting point for `runHeadless`.
|
|
327
|
+
- `src/modules/tui-module.ts` — pattern for external-message-handling modules.
|
|
328
|
+
- `src/modules/subagent-module.ts` — pattern for modules that manage child agents + implement `gatherContext` + persist state to Chronicle.
|
|
329
|
+
- `src/recipe.ts` — schema validation precedent.
|
|
330
|
+
- `agent-framework/src/api/server.ts` — message/event shape precedent (conceptually adjacent to what we need, but over WebSocket instead of Unix socket; not reused directly to keep the FKM work self-contained).
|
package/README.md
ADDED
|
@@ -0,0 +1,189 @@
|
|
|
1
|
+
# connectome-host
|
|
2
|
+
|
|
3
|
+
A general-purpose agent TUI host with recipe-based configuration. Point it at any use case by loading a recipe — a JSON file that defines the system prompt, MCP servers, modules, and agent settings.
|
|
4
|
+
|
|
5
|
+
Built on the Connectome stack: [@animalabs/agent-framework](https://github.com/anima-research/agent-framework) + [@animalabs/context-manager](https://github.com/anima-research/context-manager) + [@animalabs/chronicle](https://github.com/anima-research/chronicle) + [@animalabs/membrane](https://github.com/anima-research/membrane).
|
|
6
|
+
|
|
7
|
+
## Quick start
|
|
8
|
+
|
|
9
|
+
```bash
|
|
10
|
+
# Prerequisites: Bun, Rust toolchain, Anthropic API key
|
|
11
|
+
export ANTHROPIC_API_KEY=sk-ant-...
|
|
12
|
+
|
|
13
|
+
bun install
|
|
14
|
+
bun src/index.ts # generic assistant
|
|
15
|
+
bun src/index.ts recipes/zulip-miner.json # load a recipe
|
|
16
|
+
bun src/index.ts https://example.com/r.json # recipe from URL
|
|
17
|
+
```
|
|
18
|
+
|
|
19
|
+
## Recipes
|
|
20
|
+
|
|
21
|
+
A recipe is a JSON file that configures everything domain-specific:
|
|
22
|
+
|
|
23
|
+
```json
|
|
24
|
+
{
|
|
25
|
+
"name": "My Agent",
|
|
26
|
+
"description": "What this agent does",
|
|
27
|
+
"agent": {
|
|
28
|
+
"name": "researcher",
|
|
29
|
+
"model": "claude-opus-4-6",
|
|
30
|
+
"systemPrompt": "You are a ...",
|
|
31
|
+
"maxTokens": 16384,
|
|
32
|
+
"strategy": {
|
|
33
|
+
"type": "autobiographical",
|
|
34
|
+
"headWindowTokens": 4000,
|
|
35
|
+
"recentWindowTokens": 30000
|
|
36
|
+
}
|
|
37
|
+
},
|
|
38
|
+
"mcpServers": {
|
|
39
|
+
"my-server": {
|
|
40
|
+
"command": "node",
|
|
41
|
+
"args": ["path/to/server.js"],
|
|
42
|
+
"env": { "API_KEY": "..." }
|
|
43
|
+
}
|
|
44
|
+
},
|
|
45
|
+
"modules": {
|
|
46
|
+
"subagents": true,
|
|
47
|
+
"lessons": true,
|
|
48
|
+
"retrieval": true,
|
|
49
|
+
"wake": true,
|
|
50
|
+
"files": { "namespace": "products" }
|
|
51
|
+
},
|
|
52
|
+
"sessionNaming": {
|
|
53
|
+
"examples": ["Thread Archaeology", "Pipeline Debug"]
|
|
54
|
+
}
|
|
55
|
+
}
|
|
56
|
+
```
|
|
57
|
+
|
|
58
|
+
### Recipe loading
|
|
59
|
+
|
|
60
|
+
| Command | Behavior |
|
|
61
|
+
|---------|----------|
|
|
62
|
+
| `bun src/index.ts` | Reuse last saved recipe, or start with generic default |
|
|
63
|
+
| `bun src/index.ts <path>` | Load recipe from local file |
|
|
64
|
+
| `bun src/index.ts <url>` | Fetch recipe from HTTP URL |
|
|
65
|
+
| `bun src/index.ts --no-recipe` | Reset to default generic assistant |
|
|
66
|
+
|
|
67
|
+
The loaded recipe is saved to `data/.recipe.json` and reused on subsequent bare starts.
|
|
68
|
+
|
|
69
|
+
### System prompt from URL
|
|
70
|
+
|
|
71
|
+
If `systemPrompt` is an HTTP(S) URL (no spaces or newlines), it's fetched as plain text:
|
|
72
|
+
|
|
73
|
+
```json
|
|
74
|
+
{
|
|
75
|
+
"agent": {
|
|
76
|
+
"systemPrompt": "https://example.com/prompts/researcher.md"
|
|
77
|
+
}
|
|
78
|
+
}
|
|
79
|
+
```
|
|
80
|
+
|
|
81
|
+
### MCP server merging
|
|
82
|
+
|
|
83
|
+
Recipe servers merge with `mcpl-servers.json`. The file wins on conflict, so users can `/mcp add` extra servers or override recipe defaults.
|
|
84
|
+
|
|
85
|
+
### Included recipes
|
|
86
|
+
|
|
87
|
+
| Recipe | Description |
|
|
88
|
+
|--------|-------------|
|
|
89
|
+
| [`recipes/zulip-miner.json`](recipes/zulip-miner.json) | Knowledge extraction from Zulip workspaces |
|
|
90
|
+
| [`recipes/knowledge-miner.json`](recipes/knowledge-miner.json) | Multi-source extraction from Zulip + Notion + GitLab |
|
|
91
|
+
|
|
92
|
+
See [`recipes/SETUP.md`](recipes/SETUP.md) for a detailed setup guide for the knowledge-miner recipe.
|
|
93
|
+
|
|
94
|
+
## What it provides
|
|
95
|
+
|
|
96
|
+
- **TUI + readline modes**: OpenTUI interactive terminal or `--no-tui` for pipes/CI
|
|
97
|
+
- **Subagent forking**: Spawn/fork parallel agents with fleet tree view (Tab to toggle)
|
|
98
|
+
- **Persistent lessons**: Knowledge store with confidence scores, tags, and semantic retrieval
|
|
99
|
+
- **Time-travel**: Chronicle-backed undo/redo, named checkpoints, branch exploration
|
|
100
|
+
- **Session management**: Isolated sessions with auto-naming
|
|
101
|
+
- **MCPL support**: Connect any MCP/MCPL server; wake subscriptions for selective event triggering
|
|
102
|
+
- **File products**: Write reports and documents, materialize to disk
|
|
103
|
+
|
|
104
|
+
## Prerequisites
|
|
105
|
+
|
|
106
|
+
- [Node.js](https://nodejs.org/) 20+ and [Bun](https://bun.sh/) runtime
|
|
107
|
+
- An Anthropic API key
|
|
108
|
+
|
|
109
|
+
### Install
|
|
110
|
+
|
|
111
|
+
```bash
|
|
112
|
+
npm install
|
|
113
|
+
```
|
|
114
|
+
|
|
115
|
+
## Environment variables
|
|
116
|
+
|
|
117
|
+
| Variable | Default | Description |
|
|
118
|
+
|----------|---------|-------------|
|
|
119
|
+
| `ANTHROPIC_API_KEY` | (required) | Anthropic API key |
|
|
120
|
+
| `MODEL` | from recipe or `claude-opus-4-6` | Override model |
|
|
121
|
+
| `DATA_DIR` | `./data` | Session and recipe storage |
|
|
122
|
+
|
|
123
|
+
## Running
|
|
124
|
+
|
|
125
|
+
```bash
|
|
126
|
+
bun src/index.ts # Interactive TUI
|
|
127
|
+
bun src/index.ts --no-tui # Readline mode
|
|
128
|
+
echo "Hello" | bun src/index.ts # Piped mode
|
|
129
|
+
bun --watch src/index.ts # Dev mode
|
|
130
|
+
```
|
|
131
|
+
|
|
132
|
+
## Slash commands
|
|
133
|
+
|
|
134
|
+
| Command | Effect |
|
|
135
|
+
|---------|--------|
|
|
136
|
+
| `/help` | List all commands |
|
|
137
|
+
| `/recipe` | Show current recipe info |
|
|
138
|
+
| `/status` | Show agent state, branch, queue depth |
|
|
139
|
+
| `/lessons` | Show lesson library sorted by confidence |
|
|
140
|
+
| `/newtopic [context]` | Reset context window for a new topic |
|
|
141
|
+
| `/clear` | Clear conversation display |
|
|
142
|
+
| `/undo` | Revert to state before last agent turn |
|
|
143
|
+
| `/redo` | Re-apply undone action |
|
|
144
|
+
| `/checkpoint <name>` | Save current state |
|
|
145
|
+
| `/restore <name>` | Restore to checkpoint |
|
|
146
|
+
| `/branches` | List Chronicle branches |
|
|
147
|
+
| `/checkout <name>` | Switch to branch |
|
|
148
|
+
| `/history` | Show recent message history |
|
|
149
|
+
| `/mcp list` | List MCPL servers |
|
|
150
|
+
| `/mcp add <id> <cmd> [args...]` | Add or overwrite a server |
|
|
151
|
+
| `/mcp remove <id>` | Remove a server |
|
|
152
|
+
| `/mcp env <id> KEY=VALUE [...]` | Set env vars on a server |
|
|
153
|
+
| `/budget [tokens]` | Show/set stream token budget |
|
|
154
|
+
| `/session list\|new\|switch\|rename\|delete` | Session management |
|
|
155
|
+
| `/quit` | Exit |
|
|
156
|
+
|
|
157
|
+
## TUI controls
|
|
158
|
+
|
|
159
|
+
| Key | Action |
|
|
160
|
+
|-----|--------|
|
|
161
|
+
| `Enter` | Send message or command |
|
|
162
|
+
| `Esc` | Interrupt agent (chat) / back (fleet/peek) |
|
|
163
|
+
| `Tab` | Toggle fleet view (subagent tree) |
|
|
164
|
+
| `Ctrl+V` | Toggle verbose mode |
|
|
165
|
+
| `Ctrl+C` | Exit |
|
|
166
|
+
|
|
167
|
+
**Fleet view** (Tab):
|
|
168
|
+
|
|
169
|
+
| Key | Action |
|
|
170
|
+
|-----|--------|
|
|
171
|
+
| Up/Down | Navigate tree |
|
|
172
|
+
| Enter/Right | Expand/collapse |
|
|
173
|
+
| Left | Collapse |
|
|
174
|
+
| `p` | Peek at running subagent's stream |
|
|
175
|
+
| `Delete` | Stop a running subagent |
|
|
176
|
+
|
|
177
|
+
## Architecture
|
|
178
|
+
|
|
179
|
+
See [ARCHITECTURE.md](ARCHITECTURE.md) for detailed technical documentation.
|
|
180
|
+
|
|
181
|
+
## Dependencies
|
|
182
|
+
|
|
183
|
+
| Package | Source | Role |
|
|
184
|
+
|---------|--------|------|
|
|
185
|
+
| `@animalabs/agent-framework` | [npm](https://www.npmjs.com/package/@animalabs/agent-framework) | Event-driven agent orchestration |
|
|
186
|
+
| `@animalabs/context-manager` | [npm](https://www.npmjs.com/package/@animalabs/context-manager) | Context window management and compression |
|
|
187
|
+
| `@animalabs/chronicle` | [npm](https://www.npmjs.com/package/@animalabs/chronicle) | Branchable event store (Rust + N-API) |
|
|
188
|
+
| `@animalabs/membrane` | [npm](https://www.npmjs.com/package/@animalabs/membrane) | LLM provider abstraction |
|
|
189
|
+
| `@opentui/core` | [npm](https://www.npmjs.com/package/@opentui/core) | Terminal UI (Zig native core) |
|