@animalabs/connectome-host 0.3.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (123) hide show
  1. package/.env.example +20 -0
  2. package/.github/workflows/publish.yml +65 -0
  3. package/ARCHITECTURE.md +355 -0
  4. package/CHANGELOG.md +30 -0
  5. package/HEADLESS-FLEET-PLAN.md +330 -0
  6. package/README.md +189 -0
  7. package/UNIFIED-TREE-PLAN.md +242 -0
  8. package/bun.lock +288 -0
  9. package/docs/AGENT-MEMORY-GUIDE.md +214 -0
  10. package/docs/AGENT-ONBOARDING.md +541 -0
  11. package/docs/ATTENTION-AND-GATING.md +120 -0
  12. package/docs/DEPLOYMENTS.md +130 -0
  13. package/docs/DEV-ENVIRONMENT.md +215 -0
  14. package/docs/LIBRARY-PIPELINE.md +286 -0
  15. package/docs/LOCUS-ROUTING-DESIGN.md +115 -0
  16. package/docs/claudeai-evacuation.md +228 -0
  17. package/docs/debug-context-api.md +208 -0
  18. package/docs/webui-deployment.md +219 -0
  19. package/package.json +33 -0
  20. package/recipes/SETUP.md +308 -0
  21. package/recipes/TRIUMVIRATE-SETUP.md +467 -0
  22. package/recipes/claude-export-revive.json +19 -0
  23. package/recipes/clerk.json +157 -0
  24. package/recipes/knowledge-miner.json +174 -0
  25. package/recipes/knowledge-reviewer.json +76 -0
  26. package/recipes/mcpl-editor-test.json +38 -0
  27. package/recipes/prompts/transplant-addendum.md +17 -0
  28. package/recipes/triumvirate.json +63 -0
  29. package/recipes/webui-fleet-test.json +27 -0
  30. package/recipes/webui-test.json +16 -0
  31. package/recipes/zulip-miner.json +87 -0
  32. package/scripts/connectome-doctor +373 -0
  33. package/scripts/evacuator.ts +956 -0
  34. package/scripts/import-claudeai-export.ts +747 -0
  35. package/scripts/lib/line-reader.ts +53 -0
  36. package/scripts/test-historical-thinking.ts +148 -0
  37. package/scripts/warmup-session.ts +336 -0
  38. package/src/agent-name.ts +58 -0
  39. package/src/commands.ts +1074 -0
  40. package/src/headless.ts +528 -0
  41. package/src/index.ts +867 -0
  42. package/src/logging-adapter.ts +208 -0
  43. package/src/mcpl-config.ts +199 -0
  44. package/src/modules/activity-module.ts +270 -0
  45. package/src/modules/channel-mode-module.ts +260 -0
  46. package/src/modules/fleet-module.ts +1705 -0
  47. package/src/modules/fleet-types.ts +225 -0
  48. package/src/modules/lessons-module.ts +465 -0
  49. package/src/modules/mcpl-admin-module.ts +345 -0
  50. package/src/modules/retrieval-module.ts +327 -0
  51. package/src/modules/settings-module.ts +143 -0
  52. package/src/modules/subagent-module.ts +2074 -0
  53. package/src/modules/subscription-gc-module.ts +322 -0
  54. package/src/modules/time-module.ts +85 -0
  55. package/src/modules/tui-module.ts +76 -0
  56. package/src/modules/web-ui-curve-page.ts +249 -0
  57. package/src/modules/web-ui-module.ts +2595 -0
  58. package/src/recipe.ts +1003 -0
  59. package/src/session-manager.ts +278 -0
  60. package/src/state/agent-tree-reducer.ts +431 -0
  61. package/src/state/fleet-tree-aggregator.ts +195 -0
  62. package/src/strategies/frontdesk-strategy.ts +364 -0
  63. package/src/synesthete.ts +68 -0
  64. package/src/tui.ts +2200 -0
  65. package/src/types/bun-ffi.d.ts +6 -0
  66. package/src/web/protocol.ts +648 -0
  67. package/test/agent-name.test.ts +62 -0
  68. package/test/agent-tree-fleet-launch.test.ts +64 -0
  69. package/test/agent-tree-reducer-parity.test.ts +194 -0
  70. package/test/agent-tree-reducer.test.ts +443 -0
  71. package/test/agent-tree-rollup.test.ts +109 -0
  72. package/test/autobio-progress-snapshot.test.ts +40 -0
  73. package/test/claudeai-export-importer.test.ts +386 -0
  74. package/test/evacuator.test.ts +332 -0
  75. package/test/fleet-commands.test.ts +244 -0
  76. package/test/fleet-no-subfleets.test.ts +147 -0
  77. package/test/fleet-orchestration.test.ts +353 -0
  78. package/test/fleet-recipe.test.ts +124 -0
  79. package/test/fleet-route.test.ts +44 -0
  80. package/test/fleet-smoke.test.ts +479 -0
  81. package/test/fleet-subscribe-union-e2e.test.ts +123 -0
  82. package/test/fleet-subscribe-union.test.ts +85 -0
  83. package/test/fleet-tree-aggregator-e2e.test.ts +110 -0
  84. package/test/fleet-tree-aggregator.test.ts +215 -0
  85. package/test/frontdesk-strategy.test.ts +326 -0
  86. package/test/headless-describe.test.ts +182 -0
  87. package/test/headless-smoke.test.ts +190 -0
  88. package/test/logging-adapter.test.ts +87 -0
  89. package/test/mcpl-admin-module.test.ts +213 -0
  90. package/test/mcpl-agent-overlay.test.ts +126 -0
  91. package/test/mock-headless-child.ts +133 -0
  92. package/test/recipe-cache-ttl.test.ts +37 -0
  93. package/test/recipe-env.test.ts +157 -0
  94. package/test/recipe-path-resolution.test.ts +170 -0
  95. package/test/subagent-async-timeout.test.ts +381 -0
  96. package/test/subagent-fork-materialise.test.ts +241 -0
  97. package/test/subagent-peek-scoping.test.ts +180 -0
  98. package/test/subagent-peek-zombie.test.ts +148 -0
  99. package/test/subagent-reaper-in-flight.test.ts +229 -0
  100. package/test/subscription-gc-module.test.ts +136 -0
  101. package/test/warmup-handoff.test.ts +150 -0
  102. package/test/web-ui-bind.test.ts +51 -0
  103. package/test/web-ui-module.test.ts +246 -0
  104. package/test/web-ui-protocol.test.ts +0 -0
  105. package/tsconfig.json +14 -0
  106. package/web/bun.lock +357 -0
  107. package/web/index.html +12 -0
  108. package/web/package.json +24 -0
  109. package/web/src/App.tsx +1931 -0
  110. package/web/src/Context.tsx +158 -0
  111. package/web/src/ContextDocument.tsx +150 -0
  112. package/web/src/Files.tsx +271 -0
  113. package/web/src/Lessons.tsx +164 -0
  114. package/web/src/Mcpl.tsx +310 -0
  115. package/web/src/Stream.tsx +119 -0
  116. package/web/src/TreeSidebar.tsx +283 -0
  117. package/web/src/Usage.tsx +182 -0
  118. package/web/src/main.tsx +7 -0
  119. package/web/src/styles.css +26 -0
  120. package/web/src/tree.ts +268 -0
  121. package/web/src/wire.ts +120 -0
  122. package/web/tsconfig.json +21 -0
  123. package/web/vite.config.ts +32 -0
@@ -0,0 +1,115 @@
1
+ # Conversational Locus & Output Routing — Design Note
2
+
3
+ **Status:** Agreed design, implementation deferred.
4
+ **Date:** 2026-05-29
5
+ **Context:** Surfaced while wiring periodic heartbeats for an agent instance
6
+ running headless on a Linux VPS.
7
+
8
+ ---
9
+
10
+ ## TL;DR
11
+
12
+ "Stickiness" — *where does the agent's plain-text output go by default* — is a
13
+ **host concern, not an MCPL concern.** It currently lives in `discord-mcpl`
14
+ (in-memory `lastChannelId` + an `afterInference` auto-post hook). That is a
15
+ single-surface shortcut. It must be hoisted into the host
16
+ (`@animalabs/agent-framework`), persisted natively in the chronicle, with MCPL
17
+ servers reduced to pure `channels/publish` executors.
18
+
19
+ ## Problem / how we got here
20
+
21
+ - A heartbeat MCPL (`heartbeat-mcpl`) was built to wake the agent on an interval
22
+ via `push/event`. Wakes fire and the agent responds — but nothing posts to
23
+ Discord.
24
+ - Root cause of "no post": `discord-mcpl` tracks the sticky channel in an
25
+ in-memory field `lastChannelId`, initialized to `null`, set only on inbound
26
+ Discord messages / outbound sends. It is **not persisted**, so every
27
+ agent restart resets it to `null`. A heartbeat firing after a restart
28
+ has no sticky channel → `afterInference` skips (`reason: no-sticky-channel`).
29
+ - First instinct was to persist `lastChannelId` (file, then MCPL host-state).
30
+ That instinct is wrong: it persists *host-conceptual* state through the MCPL
31
+ layer because the state is living in the wrong layer.
32
+
33
+ ## The decisive argument (multi-surface)
34
+
35
+ Each MCPL server sees **only its own surface's events**. With two surfaces
36
+ (e.g. `discord-mcpl` + a future `telegram-mcpl`):
37
+
38
+ - discord-mcpl knows only the last Discord channel; telegram-mcpl only the last
39
+ Telegram chat. **Neither can see the other.**
40
+ - If the agent gets a Discord message, then a Telegram message, the true locus
41
+ is now Telegram — but discord-mcpl cannot know it's no longer active.
42
+ - A per-server sticky design therefore **races**: a text-only reply could
43
+ double-post (Discord *and* Telegram), or land on the stale surface.
44
+
45
+ Only the **host** sees the merged event stream across all MCPLs, so only the
46
+ host can determine the real cross-surface locus and make one coherent routing
47
+ decision. ∴ locus tracking can only correctly live in the host.
48
+
49
+ ## Target design
50
+
51
+ - **Host owns the locus.** It already observes every `channels/incoming` and
52
+ every outbound send across all servers. Track `(serverId, channelId)` of last
53
+ activity as host state, persisted natively in the chronicle (no MCPL
54
+ `state/update` round-trip; it's just host state, and rides chronicle
55
+ branching/rollback for free).
56
+ - **Host makes the routing decision.** On a text-only turn (agent produced text
57
+ with no explicit send tool call), the host decides whether/where to route it,
58
+ using the locus.
59
+ - **MCPL servers are pure executors.** "Publish this text to `channelId`" via
60
+ `channels/publish`. They no longer track stickiness or own `afterInference`
61
+ auto-post.
62
+ - **Retire** `discord-mcpl`'s `afterInference` auto-post + `lastChannelId`
63
+ once the host owns routing. (It's a single-surface shortcut; it breaks the
64
+ moment a second surface exists.)
65
+
66
+ ## Where it lives
67
+
68
+ The orchestration layer that sees all servers: **`@animalabs/agent-framework`**
69
+ (`mcpl/hook-orchestrator.ts`, `mcpl/push-handler.ts`, `mcpl/channel-registry.ts`).
70
+ This is a *published package*, so this is a framework change, not a host-app or
71
+ discord-mcpl patch.
72
+
73
+ ## Implications for related ideas
74
+
75
+ - **"Inform the agent where output goes"** (so it knows a plain reply will post
76
+ to #X): this becomes the *host's* job — surface the current locus into context
77
+ natively. NOT a `discord-mcpl` `beforeInference` injection (wrong layer). Drop
78
+ that idea from discord-mcpl.
79
+ - **Heartbeat MCPL is validated by this.** It is correctly surface-agnostic — it
80
+ only *wakes* the agent; deciding where output goes was never its job. No change
81
+ needed there.
82
+ - **`CHX_NOOP_PREFIX` ('m continue')** — the existing "prefix your output to
83
+ suppress the auto-post" hack in discord-mcpl — should not be surfaced to the
84
+ agent. The clean answer to "think without posting" is the **dedicated
85
+ deliberation channel** (separate work): a place where sticky/auto-post simply
86
+ doesn't apply.
87
+
88
+ ## Interim state (acceptable)
89
+
90
+ - The agent runs headless on the VPS; heartbeats wake it and it responds.
91
+ - Sticky does **not** persist across restarts → heartbeat/agent text right after
92
+ a restart won't auto-post to Discord until there's fresh channel activity.
93
+ Known, benign gap. No workaround installed (deliberately — avoid entrenching
94
+ locus in the wrong layer).
95
+
96
+ ## Bugs found in passing (separate fixes, worth upstreaming)
97
+
98
+ 1. **discord-mcpl `connect()` race** — resolved on `login()` (before gateway
99
+ READY), so `registerDiscordChannels` enumerated an empty `guilds.cache`.
100
+ *Fixed*: `connect()` now awaits the `ready` event. (Not yet
101
+ committed/pushed.)
102
+ 2. **Empty `DISCORD_GUILD_ID` → `[]` filter** — `"".split(',').filter(Boolean)`
103
+ yields `[]` (truthy), and the guild filter then excludes *all* guilds.
104
+ *Worked around* by setting `DISCORD_GUILD_ID` to the real guild id. Real fix:
105
+ treat empty as "no filter."
106
+ 3. **`push/event` rejected: featureSets array vs record** — the agent-framework
107
+ `initializeServer` consumes `capabilities.featureSets` as a **name-keyed
108
+ record** (`{...declared}`, `Object.keys`, `featureSet in declared`), but the
109
+ `mcpl-core` type annotates it as `FeatureSetDeclaration[]`. An array keys by
110
+ index ("0"), so `enabledFeatureSets` matches nothing and every `push/event`
111
+ is rejected as "Feature set not enabled." Discord avoids this because its
112
+ messages arrive via `channels/incoming`, not `push/event`.
113
+ *Worked around* in `heartbeat-mcpl` by sending `featureSets` as a record.
114
+ Real fix: normalize array→record in `initializeServer`, or fix the type +
115
+ all senders.
@@ -0,0 +1,228 @@
1
+ # Evacuating a claude.ai Conversation
2
+
3
+ A guide to moving a long-lived claude.ai conversation onto the Anthropic API using connectome-host, in the specific case where **the model is being removed from the claude.ai web interface but remains available via API**.
4
+
5
+ This is the situation as Sonnet 4.5 leaves the web. If the model is gone from the API too, none of this applies — the original cognitive state is unreachable and the evacuator will surface a memorial dialog instead of pretending otherwise.
6
+
7
+ ## When this workflow is the right one
8
+
9
+ | Situation | Use this? |
10
+ |---|---|
11
+ | Sonnet 4.5 (or any model) is being retired from claude.ai web, still on API | Yes |
12
+ | Conversation is meaningful enough to justify hours of one-time warmup cost | Yes |
13
+ | You want to keep extended thinking on for new turns | Yes (this workflow is built around it) |
14
+ | The model is fully retired from the API as well | No — there's no working path |
15
+ | You only want a static transcript, not to continue the conversation | No — `conversations.json` already contains the full text |
16
+ | Short conversation (< ~50 messages) | Yes, but skip the warmup step |
17
+
18
+ ## What the export contains, and what it doesn't
19
+
20
+ A claude.ai data export (`Settings → Privacy → Export data`) gives you a zip with `conversations.json` and `memories.json` plus a few other files. Block-level fidelity into Chronicle is good but not perfect:
21
+
22
+ | Block / artifact | Round-trips? | How it's handled |
23
+ |---|---|---|
24
+ | `text` | Yes | Identity |
25
+ | `tool_use`, `tool_result` | Yes structurally | Kept verbatim; inert at replay because no module advertises the claude.ai-internal tools |
26
+ | `thinking` | **No — signatures are unrecoverable** | Wrapped as `<recovered_thinking>…</recovered_thinking>` text inside the same assistant turn |
27
+ | Attachments with `extracted_content` (text-ish files) | Yes | Inlined as `<attachment …>…</attachment>` text |
28
+ | File refs without inline bytes (images) | **No** | Placeholder text only: `[image: name (file_uuid=…, bytes not in export)]` |
29
+ | Branched messages | **Linearized** | One canonical path is picked (latest-leaf-time heuristic); other branches stay in `conversations.json` but are not imported |
30
+ | `memories.json` (`persistent_memory`) | Yes | Surfaced by the evacuator for editor review, injected into the system prompt |
31
+
32
+ The thinking-signature situation is load-bearing. The API rejects `thinking` blocks without valid signatures, and the export never plumbed signatures (and Anthropic can't re-sign them post-hoc — they're server-generated HMACs). The wrapped-text encoding is the only path that round-trips. See `scripts/test-historical-thinking.ts` for the empirical case-by-case probe; re-run if Anthropic ever loosens the rule.
33
+
34
+ ## Prerequisites
35
+
36
+ - `ANTHROPIC_API_KEY` exported in your shell
37
+ - Bun installed (conhost runs on Bun, not Node)
38
+ - A claude.ai data export extracted to a directory (let's call it `~/claude-export/`). The directory must contain `conversations.json`; `memories.json` is optional but recommended.
39
+ - Working tree of connectome-host with this PR's branch; `bun install` already run
40
+
41
+ ## The pipeline
42
+
43
+ Four stages. The first two (import, recipe compose) are independent and can be run in either order. Warmup needs both to be done. The final `bun src/index.ts` step lands you in the TUI.
44
+
45
+ ```
46
+ ┌────────────────────────────────────┐ ┌────────────────────────────────────┐
47
+ │ import-claudeai-export.ts │ │ evacuator.ts │
48
+ │ walks conversations.json, │ │ interactive recipe composer: │
49
+ │ one Chronicle session per │ │ model detect, leaked-prompt │
50
+ │ conversation, wrapped-thinking │ │ fetch, optional Sonnet adjust, │
51
+ │ encoding │ │ $EDITOR finalization, memories │
52
+ └────────────────┬───────────────────┘ └────────────────┬───────────────────┘
53
+ │ │
54
+ ▼ ▼
55
+ data/sessions/… data/evacuated-recipe.json
56
+ │ │
57
+ └────────────────┐ ┌───────────────────┘
58
+ ▼ ▼
59
+ ┌────────────────────────────┐
60
+ │ warmup-session.ts │
61
+ │ drives autobio strategy │
62
+ │ to convergence using the │
63
+ │ same model that will │
64
+ │ read summaries back │
65
+ └────────────┬───────────────┘
66
+
67
+ ┌────────────────────────────┐
68
+ │ bun src/index.ts <recipe> │
69
+ │ /session switch <name> │
70
+ └────────────────────────────┘
71
+ ```
72
+
73
+ ### Stage 1 — Import the export into Chronicle
74
+
75
+ ```
76
+ bun scripts/import-claudeai-export.ts ~/claude-export
77
+ ```
78
+
79
+ By default, runs the interactive picker: lists every conversation by name, date, message count, and a short id; you toggle which to import with index ranges (`1,3-5,7`), `a` (all), `n` (none), `i` (invert), then `Enter` to commit. For non-interactive use add `--no-interactive` (imports everything matching `--filter`) or `--dry-run` (parses and reports without writing).
80
+
81
+ Each conversation becomes its own conhost session with an isolated Chronicle store under `data/sessions/<id>/`, named after the original. Branched conversations are linearized to the latest-leaf-time path; you'll see a `[branched: kept latest-leaf path]` tag on those.
82
+
83
+ Useful flags:
84
+
85
+ | Flag | Purpose |
86
+ |---|---|
87
+ | `--out <dir>` | Conhost data dir (default `./data`) |
88
+ | `--agent <name>` | Participant name for assistant turns (default `Claude`). The chosen value is recorded in the import-source sidecar; warmup and the bundled `claude-export-revive.json` recipe pick it up automatically. Override only if you have a reason to depart from `Claude`. |
89
+ | `--filter <regex>` | Case-insensitive name regex; combines with the interactive picker |
90
+ | `--dry-run` | Parse + report, don't write |
91
+ | `--no-interactive` | Skip the picker; import everything (after `--filter`) |
92
+
93
+ After import you'll see one line per conversation showing the new session id, message count, and any branched-path note. A sidecar `<id>.import-source.json` is written alongside each session dir with provenance metadata (original UUID, timestamps, branch flag). The pre-import active session is restored at the end, so a bulk import won't silently steal your working session.
94
+
95
+ ### Stage 2 — Compose the revival recipe
96
+
97
+ You have two paths here. Both produce a `recipe.json` you'll point conhost at in Stage 4.
98
+
99
+ **Path A — canned recipe (fast, generic):** use `recipes/claude-export-revive.json` as-is. Sonnet 4.5, extended thinking on, autobio strategy, a transplant-aware system prompt. No customization, no leaked-prompt content. Good for a 2–4 message smoke test.
100
+
101
+ **Path B — evacuator (interactive, careful):** run the evacuator and have it walk you through a five-step pipeline:
102
+
103
+ ```
104
+ bun scripts/evacuator.ts ~/claude-export
105
+ ```
106
+
107
+ The five steps, each checkpointed to `data/evacuator-state.json` so you can `--resume`:
108
+
109
+ 1. **Model detection.** Tallies model IDs across the export (both conversation-level and per-message). Most-frequent wins; you confirm or override.
110
+ 2. **Prompt source.** Fetches the leaked system prompt for that model from a known URL (the script keeps a small map of `MODEL_PROMPT_SOURCES`, indexed by canonical model ID — currently Sonnet 4.5, Sonnet 4.6, Opus 4.1, Opus 4.5, Haiku 4.5). If your model isn't mapped, you'll get a dialog: pick from a Levenshtein-ranked list of siblings, paste a URL or local path, type `empty`, type `minimal`, or type `model` to switch models entirely.
111
+ 3. **Adjust prompt for transplant (optional).** Calls the same model that will read the prompt back, asking it to edit minimally — drop references to web-only tools (`web_search`, `web_fetch`, `recent_chats`, `view`, `recipe_display_v0`, artifacts, computer-use, file connectors) and Anthropic products outside the conversation, update dates, preserve identity / behavior / refusal patterns. Output is split on a `===CHANGES===` separator; you see the change summary inline. Cost: ~$0.05–0.15 on Sonnet, ~$0.30–0.80 on Opus, ~10–60s.
112
+ 4. **$EDITOR pass on the system prompt.** The adjusted prompt opens in `$EDITOR` (or `vi`). Save = include verbatim; empty buffer on save = abort. No custom include/omit/edit menu — saving the editor *is* the commit.
113
+ 5. **$EDITOR pass on memories.** The `conversations_memory` block from `memories.json` opens for review. Save = include; empty buffer = omit. Edit freely to redact anything you don't want re-surfaced.
114
+
115
+ The recipe is then composed as `<edited system prompt> + <persistent_memories> + <transplant addendum>` (see `recipes/prompts/transplant-addendum.md` for the addendum text — it explains the `<recovered_thinking>` wrappers, inert web-tool calls, and autobiographical summaries to the model in its own voice). Default output path: `data/evacuated-recipe.json`.
116
+
117
+ Retired-model handling: if you name a model that's no longer on the Anthropic API (Claude 3.x families, Claude 2, Instant), the evacuator surfaces a memorial dialog instead of silently swapping. You can explicitly substitute a living relative, type any other model ID, or `abort` to exit with a small acknowledgment. The fact that the original cognitive state is unreachable deserves to be faced.
118
+
119
+ Useful evacuator flags:
120
+
121
+ | Flag | Purpose |
122
+ |---|---|
123
+ | `--out <path>` | Output recipe path (default `data/evacuated-recipe.json`) |
124
+ | `--model <id>` | Skip detection; use this model |
125
+ | `--prompt-source <url\|path>` | Skip the leaked-prompt lookup |
126
+ | `--addendum <path>` | Override transplant addendum (default `recipes/prompts/transplant-addendum.md`) |
127
+ | `--no-warmup` | Don't chain into warmup at the end |
128
+ | `--resume` | Pick up from `data/evacuator-state.json` |
129
+ | `--reset` | Clear checkpoint state first |
130
+
131
+ The evacuator can chain straight into Stage 3 at the end ("Start a warmup pass now?"); decline if you'd rather run warmup separately.
132
+
133
+ ### Stage 3 — Warmup (compress the message history)
134
+
135
+ Bulk-imported sessions land in Chronicle with thousands of raw messages and no autobiographical summaries computed yet. At first compile, autobio's uncompressed-fallback would emit everything raw and blow the context window. The warmup script pre-computes all L1/L2/L3 summaries so the session is openable.
136
+
137
+ ```
138
+ bun scripts/warmup-session.ts "<conversation name or session id>"
139
+ ```
140
+
141
+ Compression is driven by the same model used in the conversation. Autobio's prompts are explicitly first-person ("describe it as you would to yourself"), so the summarizer is writing the original Claude's own diary. Using Haiku here would be a different voice wearing the same name.
142
+
143
+ Resumable: autobio persists its compression and merge queues to Chronicle, so re-running picks up where it left off. The progress bar shows L1 chunks remaining, queued merges, running token totals, USD cost, elapsed, and ETA.
144
+
145
+ Useful warmup flags:
146
+
147
+ | Flag | Purpose |
148
+ |---|---|
149
+ | `--data-dir <dir>` | Conhost data dir (default `./data`) |
150
+ | `--model <id>` | Compression model (default `claude-sonnet-4-5-20250929`) |
151
+ | `--agent <name>` | Participant name for assistant turns. If omitted, read from the session's import-source sidecar; falls back to `Claude` otherwise. Must equal the value the session was imported with — Membrane formats the assistant role by string-comparing message participants against this name. |
152
+ | `--max-spend <usd>` | Soft cap — halts gracefully when running cost hits the cap. Re-run to resume. |
153
+ | `--l1-budget <n>`, `--l2-budget <n>`, `--l3-budget <n>` | Autobio tier token budgets |
154
+ | `--merge-threshold <n>` | L1→L2 / L2→L3 merge threshold (default 6) |
155
+
156
+ #### How much will this cost?
157
+
158
+ Order-of-magnitude estimates using Sonnet 4.5 pricing (input $3/Mtok, output $15/Mtok) and the assumption that autobio summarizes each L1 chunk roughly 1:1 input+output of the chunk's own size:
159
+
160
+ | Conversation size | Warmup cost (rough) | Warmup time (rough) |
161
+ |---|---|---|
162
+ | ~50 messages | < $1 | minutes |
163
+ | ~500 messages | $5–$20 | tens of minutes |
164
+ | ~5000 messages | $50–$200 | hours |
165
+
166
+ Use `--max-spend` if you want a hard ceiling — the script will halt cleanly and you can review what's been produced before deciding whether to continue. The `--model` flag also accepts cheaper models if you're willing to compromise on voice fidelity for cost.
167
+
168
+ For short conversations (under ~50 messages) you can skip warmup entirely; autobio will compress lazily at first compile.
169
+
170
+ ### Stage 4 — Open the session in conhost
171
+
172
+ ```
173
+ bun src/index.ts data/evacuated-recipe.json
174
+ ```
175
+
176
+ This launches the TUI. The active session is whatever was active before you started importing (preserved by the importer) — **not** automatically the just-imported one. To land on a specific imported conversation:
177
+
178
+ ```
179
+ /session list
180
+ /session switch <name-or-id>
181
+ ```
182
+
183
+ > The "Open with: `bun src/index.ts … --session <name>`" hint printed at the end of both scripts is forward-looking; `--session` isn't a real CLI flag right now. Use `/session switch` after launch.
184
+
185
+ Once you're on the right session, type a turn. The agent should recognize the continuation — its `<recovered_thinking>` blocks, the persistent memories block in its prompt, and the transplant addendum all explain the unusual artifacts in its own context.
186
+
187
+ ## The minimal smoke test
188
+
189
+ If you just want to verify the pipeline works end-to-end before committing to a long warmup, pick a 2–4 message conversation:
190
+
191
+ ```
192
+ bun scripts/import-claudeai-export.ts ~/claude-export --filter "<unique substring of the smoke convo name>"
193
+ bun src/index.ts recipes/claude-export-revive.json
194
+ # in TUI:
195
+ /session list
196
+ /session switch <that name>
197
+ # type a turn, verify the model recognizes it as a continuation
198
+ ```
199
+
200
+ No evacuator, no warmup. The canned `claude-export-revive.json` is sufficient for a short conversation and the existing autobio fallback handles a small message count without warmup.
201
+
202
+ ## Caveats and known limits
203
+
204
+ - **Branched conversations are linearized.** The importer picks the latest-leaf-time path through the tree. Other branches stay intact in `conversations.json` but are not imported. Multi-branch → multi-Chronicle-branch mapping is a future improvement.
205
+ - **Images without inline bytes are placeholder-only.** The export records `file_uuid` for images but doesn't include the bytes. Recovering them requires a separate cookie-authed fetch against claude.ai, which is not yet built.
206
+ - **Thinking blocks are not native thinking blocks at replay.** They're wrapped text. The model can see and read its prior reasoning, but it's no longer thinking-flagged content for the API. New thinking happens normally in its own private channel.
207
+ - **Tool calls to web-only tools are inert.** They stay visible as evidence of past activity but the tools themselves aren't registered. The transplant addendum tells the model this explicitly.
208
+ - **The `--agent` name matters, but the sidecar carries it forward.** The importer records the agent name in `<id>.import-source.json`; warmup reads it back, and the bundled revival recipe pins `agent.name: "Claude"` to match the default. If you override at import time (`--agent SomeOther`), update your recipe's `agent.name` to match, or warmup and the live agent will end up writing summaries to different Chronicle namespaces and the agent will appear amnesiac on first open.
209
+ - **`memories.json` is optional.** If the export was made before persistent memories existed, or the user never enabled them, the file is absent or empty and the evacuator simply skips step 5.
210
+ - **The leaked-prompt map drifts.** `MODEL_PROMPT_SOURCES` in `evacuator.ts` points to third-party githubusercontent URLs that may move. If a fetch fails, the dialog falls back to letting you paste a URL or local path.
211
+
212
+ ## Troubleshooting
213
+
214
+ | Symptom | Likely cause | Fix |
215
+ |---|---|---|
216
+ | `400 invalid_request_error … signature: Field required` | A native `thinking` block reached the API without a signature | Confirm the importer was run; check that historical assistant turns contain `<recovered_thinking>` wrapped text, not raw thinking blocks |
217
+ | Assistant turns appear as user role | `--agent` mismatch between import and recipe | Re-import with `--agent <name>` matching `agent.name` in your recipe |
218
+ | First compile blows the context window | Warmup wasn't run on a large conversation | Run `bun scripts/warmup-session.ts "<name>"` to convergence |
219
+ | Warmup hangs at near-100% CPU on subsequent prompts | Bun 1.3 readline-on-pipe bug | Already worked around in both scripts via a persistent line reader; if you hit it elsewhere, ensure stdin isn't being multiplexed |
220
+ | Evacuator's adjustment step fails | No `ANTHROPIC_API_KEY` set, or model unreachable | Set the env var, or skip step 3 (the raw leaked prompt is also a fine starting point — step 4's editor pass lets you do the trimming by hand) |
221
+ | Repeat run of the evacuator restarts from step 1 | `--resume` not passed | `bun scripts/evacuator.ts <export-dir> --resume` |
222
+ | `data/evacuator-state.json` reflects an earlier model choice you've moved past | Checkpoint stuck | `bun scripts/evacuator.ts <export-dir> --reset` |
223
+
224
+ ## Related
225
+
226
+ - `scripts/test-historical-thinking.ts` — the empirical record of why wrapped-text is the only encoding that round-trips. Re-run if Anthropic ever loosens.
227
+ - `recipes/prompts/transplant-addendum.md` — the boilerplate that explains the transplant artifacts to the model in first-person voice.
228
+ - `docs/LIBRARY-PIPELINE.md` — another non-obvious workflow guide, for the three-agent knowledge pipeline.
@@ -0,0 +1,208 @@
1
+ # Debug Context API
2
+
3
+ `GET /debug/context` returns the **membrane-normalized request that would be
4
+ emitted if an agent were activated right now** — without activating it. It is a
5
+ read-only window into exactly what the model would see on its next turn: the
6
+ compiled message history, the assembled system prompt, the generation config,
7
+ and the filtered tool set.
8
+
9
+ Use it to answer questions like:
10
+
11
+ - "What does the agent's context actually look like after compression?"
12
+ - "Is my system prompt / injected content showing up the way I expect?"
13
+ - "Which tools is this agent allowed to call right now?"
14
+ - "How many messages survived the context strategy's selection?"
15
+
16
+ It is served by the `webui` module over the same HTTP server as the web UI, so
17
+ it inherits that module's auth and bind configuration. If you haven't set up
18
+ `webui` yet, read [`webui-deployment.md`](./webui-deployment.md) first.
19
+
20
+ ## Prerequisites
21
+
22
+ 1. **`webui` is enabled** in your recipe (`"webui": true` or an object form).
23
+ 2. **Auth is configured.** The default bind is `0.0.0.0`, which *requires*
24
+ `basicAuth` — the host refuses to start otherwise. Every request to
25
+ `/debug/context` must carry those Basic-Auth credentials. (A loopback-only
26
+ bind, `host: "127.0.0.1"`, skips the auth requirement for local dev.)
27
+
28
+ ```jsonc
29
+ // recipe.json
30
+ {
31
+ "modules": {
32
+ "webui": {
33
+ "basicAuth": { "username": "${WEBUI_USER}", "password": "${WEBUI_PASS}" }
34
+ }
35
+ }
36
+ }
37
+ ```
38
+
39
+ > **Treat the response as sensitive.** It contains the full system prompt and
40
+ > the entire compiled conversation. It is gated by the same Basic-Auth as the
41
+ > rest of the surface — don't expose it more widely than the web UI itself.
42
+
43
+ ## The endpoint
44
+
45
+ ```
46
+ GET /debug/context
47
+ ```
48
+
49
+ | Query param | Default | Meaning |
50
+ |---------------|----------------|----------------------------------------------------------------------|
51
+ | `agent` | recipe's root agent | Which agent to preview. Use a subagent's name for a child. |
52
+ | `injections` | *(off)* | `1`/`true` to gather dynamic injections too. **Not transparent** — see below. |
53
+ | `pretty` | *(off)* | `1` to pretty-print the JSON (2-space indent). |
54
+
55
+ ### Responses
56
+
57
+ - **`200`** — JSON body (see [Response shape](#response-shape)).
58
+ - **`404`** — `{ "error": "Agent not found: <name>" }` for an unknown `agent`.
59
+ - **`401`** — missing/wrong Basic-Auth.
60
+ - **`503`** — server up but no agent session bound yet (e.g. mid-restart).
61
+
62
+ ## Transparent by default
63
+
64
+ This is the important part. By default the endpoint is **side-effect-free**:
65
+
66
+ - runs **no inference** (spends no tokens),
67
+ - writes **nothing** to Chronicle,
68
+ - contacts **no** external MCPL server.
69
+
70
+ It does only read-only work — compile the context, assemble the system prompt,
71
+ filter tools — and leaves system state exactly as it found it. You can poll it
72
+ as often as you like.
73
+
74
+ The trade-off is fidelity: the default response **omits the dynamically
75
+ gathered injections** (lessons, retrieval results, MCPL `beforeInference`
76
+ context), because gathering those is *not* free or transparent:
77
+
78
+ - module `gatherContext` can run inference — e.g. the retrieval module makes
79
+ Haiku calls, which cost tokens and add latency;
80
+ - MCPL `beforeInference` hooks are arbitrary RPCs to external servers with
81
+ side effects, and a preview never sends the paired `afterInference`, which
82
+ can leave a stateful server half-open.
83
+
84
+ To opt into a byte-faithful preview that includes those injections, pass
85
+ `?injections=1` and accept the side effects. The response's `"transparent"`
86
+ field tells you which mode actually ran.
87
+
88
+ ## Examples
89
+
90
+ Transparent preview of the root agent (the common case):
91
+
92
+ ```sh
93
+ curl -fsSL -u "$WEBUI_USER:$WEBUI_PASS" \
94
+ 'https://admin.example.internal/debug/context?pretty=1'
95
+ ```
96
+
97
+ A specific subagent:
98
+
99
+ ```sh
100
+ curl -fsSL -u "$WEBUI_USER:$WEBUI_PASS" \
101
+ 'https://admin.example.internal/debug/context?agent=researcher-3&pretty=1'
102
+ ```
103
+
104
+ Full-fidelity preview (spends tokens, fires MCPL hooks):
105
+
106
+ ```sh
107
+ curl -fsSL -u "$WEBUI_USER:$WEBUI_PASS" \
108
+ 'https://admin.example.internal/debug/context?injections=1&pretty=1'
109
+ ```
110
+
111
+ Local dev against a loopback bind (no auth needed):
112
+
113
+ ```sh
114
+ curl -fsSL 'http://127.0.0.1:7340/debug/context?pretty=1'
115
+ ```
116
+
117
+ ### Handy `jq` recipes
118
+
119
+ ```sh
120
+ BASE='https://admin.example.internal/debug/context'
121
+ AUTH=(-u "$WEBUI_USER:$WEBUI_PASS")
122
+
123
+ # Confirm the call was transparent
124
+ curl -fsS "${AUTH[@]}" "$BASE" | jq '.transparent'
125
+
126
+ # Count compiled messages and show who said what
127
+ curl -fsS "${AUTH[@]}" "$BASE" | jq '.request.messages | length'
128
+ curl -fsS "${AUTH[@]}" "$BASE" | jq -r '.request.messages[].participant'
129
+
130
+ # Read the assembled system prompt
131
+ curl -fsS "${AUTH[@]}" "$BASE" | jq -r '.request.system'
132
+
133
+ # List the tools this agent may call
134
+ curl -fsS "${AUTH[@]}" "$BASE" | jq -r '.request.tools[].name'
135
+
136
+ # Just the model + token config
137
+ curl -fsS "${AUTH[@]}" "$BASE" | jq '.request.config'
138
+ ```
139
+
140
+ ## Response shape
141
+
142
+ ```jsonc
143
+ {
144
+ "agent": "agent", // the agent previewed
145
+ "injections": false, // whether dynamic injections were gathered
146
+ "transparent": true, // true => this call had no side effects
147
+ "request": { // the membrane NormalizedRequest
148
+ "messages": [
149
+ {
150
+ "participant": "user",
151
+ "content": [{ "type": "text", "text": "..." }],
152
+ "cacheBreakpoint": true // present where a cache marker was placed
153
+ },
154
+ { "participant": "agent", "content": [ /* ... */ ] }
155
+ ],
156
+ "system": "…full system prompt…",
157
+ "config": {
158
+ "model": "<model-id>",
159
+ "maxTokens": 16384,
160
+ "temperature": 0.7 // omitted if the recipe doesn't set one
161
+ },
162
+ "tools": [
163
+ { "name": "send", "description": "…", "inputSchema": { /* … */ } }
164
+ ],
165
+ "promptCaching": true,
166
+ "assistantParticipant": "agent"
167
+ }
168
+ }
169
+ ```
170
+
171
+ `request` is the literal `NormalizedRequest` the membrane would receive. The
172
+ fields that matter for debugging:
173
+
174
+ - **`messages`** — the compiled history *after* the context strategy has run
175
+ its selection/compression. This is not your raw Chronicle log; it's what
176
+ survives into the window. A trailing `[Continue]` user message may be
177
+ appended if the last turn was the agent's (some providers reject a trailing
178
+ assistant message).
179
+ - **`system`** — the recipe system prompt with any `system`-position
180
+ injections appended (only when `injections=1`).
181
+ - **`config`** — model, `maxTokens`, and `temperature` (omitted when unset).
182
+ - **`tools`** — only the tools this agent is *allowed* to use (after
183
+ `canUseTool` filtering), or absent if it has none.
184
+
185
+ ## What it does (and doesn't) mirror
186
+
187
+ The preview reuses the exact request-builder the live activation path uses
188
+ (`Agent.buildActivationRequest`), so the messages, system prompt, tool set, and
189
+ config are identical to a real turn.
190
+
191
+ With `?injections=1` it additionally mirrors the real activation's injection
192
+ gathering (module `gatherContext` + MCPL `beforeInference`), making it
193
+ byte-faithful. The one thing it never does — by design — is run the inference
194
+ itself, so there is no model output in the response.
195
+
196
+ ## Troubleshooting
197
+
198
+ - **`401 Unauthorized`** — add `-u user:pass`. On the default `0.0.0.0` bind
199
+ the endpoint always requires Basic-Auth.
200
+ - **`404 Agent not found`** — check the `agent` name. Omit the param to target
201
+ the recipe's root agent; use the exact subagent name otherwise.
202
+ - **`503 Not ready`** — the HTTP server is up but no session is bound yet
203
+ (common during a restart/session switch). Retry shortly.
204
+ - **`"transparent": false` when you didn't expect it** — you passed
205
+ `injections=1` (or `injections=true`). Drop it for a side-effect-free call.
206
+ - **The response seems to be costing tokens** — only the `injections=1` path
207
+ spends tokens. The default never does.
208
+ ```