@animalabs/connectome-host 0.7.3 → 0.8.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.env.example +12 -5
- package/.github/PULL_REQUEST_TEMPLATE.md +3 -2
- package/.github/workflows/changelog.yml +9 -4
- package/.github/workflows/ci.yml +5 -3
- package/.github/workflows/publish.yml +12 -6
- package/CHANGELOG.md +401 -10
- package/CONTRIBUTING.md +47 -19
- package/HEADLESS-FLEET-PLAN.md +22 -0
- package/README.md +39 -1
- package/bun.lock +27 -31
- package/changelog.d/README.md +28 -0
- package/docs/AGENT-ONBOARDING.md +1 -1
- package/docs/debug-context-api.md +2 -2
- package/docs/retrieval-traces.md +173 -0
- package/docs/webui-deployment.md +2 -1
- package/package.json +6 -6
- package/recipes/SETUP.md +11 -5
- package/recipes/TRIUMVIRATE-SETUP.md +68 -14
- package/recipes/knowledge-miner.json +0 -30
- package/recipes/mock-test.json +19 -0
- package/recipes/triumvirate.json +6 -1
- package/scripts/audit-module-optins.ts +288 -0
- package/scripts/release-changelog.ts +210 -21
- package/src/cache-keepalive-log.ts +41 -0
- package/src/commands.ts +96 -0
- package/src/framework-strategy.ts +50 -4
- package/src/gate-telemetry.ts +106 -0
- package/src/headless.ts +24 -0
- package/src/index.ts +179 -64
- package/src/mcpl-config.ts +99 -1
- package/src/modules/fleet-module.ts +60 -1
- package/src/modules/fleet-types.ts +30 -1
- package/src/modules/identity-module.ts +310 -2
- package/src/modules/instructions-module.ts +265 -0
- package/src/modules/mcpl-admin-module.ts +89 -13
- package/src/modules/retrieval-module.ts +249 -51
- package/src/modules/retrieval-trace-page.ts +254 -0
- package/src/modules/retrieval-trace.ts +904 -0
- package/src/modules/subagent-module.ts +18 -0
- package/src/modules/tts-relay-module.ts +33 -18
- package/src/modules/web-ui-module.ts +445 -894
- package/src/recipe.ts +787 -29
- package/src/retrieval-config.ts +39 -0
- package/src/strategies/frontdesk-strategy.ts +34 -125
- package/src/tui.ts +325 -54
- package/src/web/panel-data.ts +1206 -0
- package/src/web/protocol.ts +75 -10
- package/src/workspace-mounts.ts +73 -0
- package/test/audit-module-optins.test.ts +174 -0
- package/test/cache-keepalive-log.test.ts +83 -0
- package/test/conversations-recipe.test.ts +142 -0
- package/test/fleet-panel-request.test.ts +90 -0
- package/test/framework-fkm-composition.test.ts +35 -3
- package/test/framework-strategy-defaults.test.ts +41 -0
- package/test/frontdesk-strategy.test.ts +25 -37
- package/test/gate-telemetry-adapter.test.ts +84 -0
- package/test/gate-telemetry.test.ts +91 -0
- package/test/headless-panel-request.test.ts +201 -0
- package/test/identity-and-surfaces.test.ts +212 -1
- package/test/instructions-module.test.ts +258 -0
- package/test/mcpl-admin-module.test.ts +64 -0
- package/test/mcpl-agent-overlay.test.ts +51 -3
- package/test/mcpl-child-env.test.ts +64 -0
- package/test/mock-headless-child.ts +14 -0
- package/test/nudge-command.test.ts +47 -0
- package/test/recipe-cache-keepalive.test.ts +59 -0
- package/test/recipe-compression-fallback.test.ts +19 -0
- package/test/recipe-hybrid-prose-routing.test.ts +12 -0
- package/test/recipe-instructions.test.ts +176 -0
- package/test/recipe-kv-unified.test.ts +87 -0
- package/test/recipe-mcp-source.test.ts +54 -0
- package/test/recipe-openai-compatible.test.ts +54 -0
- package/test/recipe-path-resolution.test.ts +19 -8
- package/test/recipe-provider.test.ts +14 -0
- package/test/recipe-save-unresolved.test.ts +244 -0
- package/test/recipe-source-only.test.ts +38 -0
- package/test/release-changelog.test.ts +202 -0
- package/test/retrieval-auth-loopback.test.ts +49 -0
- package/test/retrieval-config.test.ts +74 -0
- package/test/retrieval-module.test.ts +821 -0
- package/test/subagent-prose-routing.test.ts +109 -0
- package/test/tui-format.test.ts +106 -0
- package/test/web-ui-context-coverage.test.ts +1 -1
- package/test/web-ui-module.test.ts +189 -3
- package/test/web-ui-observers.test.ts +8 -5
- package/test/web-ui-protocol.test.ts +0 -0
- package/test/workspace-mounts.test.ts +68 -0
- package/web/src/App.tsx +160 -44
- package/web/src/Context.tsx +35 -8
- package/web/src/ContextDocument.tsx +20 -5
- package/web/src/Files.tsx +2 -8
- package/web/src/Health.tsx +61 -1
- package/web/src/Lessons.tsx +2 -38
- package/web/src/Mcpl.tsx +80 -14
- package/web/src/Pins.tsx +5 -0
- package/web/src/Settings.tsx +5 -0
- package/web/vite.config.ts +8 -2
package/package.json
CHANGED
|
@@ -1,23 +1,23 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@animalabs/connectome-host",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.8.0",
|
|
4
4
|
"description": "General-purpose agent TUI host with recipe-based configuration",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"scripts": {
|
|
7
7
|
"start": "bun src/index.ts",
|
|
8
8
|
"dev": "bun --watch src/index.ts",
|
|
9
9
|
"test": "bun test",
|
|
10
|
-
"version": "bun scripts/release-changelog.ts && git add CHANGELOG.md",
|
|
10
|
+
"version": "bun scripts/release-changelog.ts && git add CHANGELOG.md changelog.d",
|
|
11
11
|
"build:web": "npm --prefix web install && npm --prefix web run build",
|
|
12
12
|
"build:web:ci": "npm ci --prefix web && npm --prefix web run build",
|
|
13
13
|
"relock:web": "rm -rf web/node_modules web/package-lock.json && npm --prefix web install",
|
|
14
14
|
"postinstall": "test -d web && npm --prefix web install && npm --prefix web run build || true"
|
|
15
15
|
},
|
|
16
16
|
"dependencies": {
|
|
17
|
-
"@animalabs/agent-framework": "^0.
|
|
18
|
-
"@animalabs/chronicle": "^0.
|
|
19
|
-
"@animalabs/context-manager": "^0.6.
|
|
20
|
-
"@animalabs/membrane": "^0.5.
|
|
17
|
+
"@animalabs/agent-framework": "^0.12.0",
|
|
18
|
+
"@animalabs/chronicle": "^0.3.0",
|
|
19
|
+
"@animalabs/context-manager": "^0.6.3",
|
|
20
|
+
"@animalabs/membrane": "^0.5.82",
|
|
21
21
|
"@opentui/core": "^0.1.82"
|
|
22
22
|
},
|
|
23
23
|
"devDependencies": {
|
package/recipes/SETUP.md
CHANGED
|
@@ -69,17 +69,17 @@ Works with both gitlab.com and self-hosted GitLab instances.
|
|
|
69
69
|
|
|
70
70
|
No separate installation needed — the recipe uses `npx` to run `@zereight/mcp-gitlab` on demand.
|
|
71
71
|
|
|
72
|
-
### Notion (optional, via an MCP server)
|
|
72
|
+
### Notion (optional, via an MCP server — not included by default)
|
|
73
73
|
|
|
74
|
-
If you want the agent to read your Notion workspace,
|
|
74
|
+
The recipe ships **without** a Notion server: the adapter its prompt was developed against (`syncntn`) is not publicly available. If you want the agent to read your Notion workspace, add an `mcpServers` entry pointing at any MCP server that exposes Notion search and page-read tools. The entry name `syncntn` is just a label — any Notion MCP server works, as long as its exposed tool names match what the system prompt references (`syncntn--search_pages`, `syncntn--get_page_markdown`, and friends). If your server uses different tool names, either name the MCP key `syncntn` and update the prompt, or accept that the agent will discover the tools under whatever names they export.
|
|
75
75
|
|
|
76
76
|
Typical setup:
|
|
77
77
|
|
|
78
78
|
1. Install and start your Notion MCP server somewhere the recipe can launch it.
|
|
79
79
|
2. Note any configuration it needs (API credentials, workspace ID, storage URL).
|
|
80
|
-
3.
|
|
80
|
+
3. Add a `syncntn` block with those values to the recipe in Step 3 below.
|
|
81
81
|
|
|
82
|
-
Don't have a Notion MCP server?
|
|
82
|
+
Don't have a Notion MCP server? Skip this — the agent adapts and works with whatever sources remain.
|
|
83
83
|
|
|
84
84
|
### DuckDuckGo web search (optional, but enabled by default)
|
|
85
85
|
|
|
@@ -98,6 +98,10 @@ cd ../connectome-host
|
|
|
98
98
|
|
|
99
99
|
The recipe expects the entry-point script at `../duckduckgo-mcp-server/.venv/bin/duckduckgo-mcp-server`. No API key needed. Don't want public-web access? Remove the `ddg` block from the recipe.
|
|
100
100
|
|
|
101
|
+
### Scribe — audio/video transcription (optional, not included by default)
|
|
102
|
+
|
|
103
|
+
The miner's prompt also knows how to drive [`dariakroshka/scribe-mcp`](https://github.com/dariakroshka/scribe-mcp) for transcribing recordings. It needs a Gemini API key (media is uploaded to Google's Gemini API) and a sibling checkout, so the shipped recipe omits it. To enable: clone scribe-mcp as a sibling of `connectome-host/`, run `bun install` in it, add a `scribe` block under `mcpServers` (see the [Triumvirate guide's Scribe section](./TRIUMVIRATE-SETUP.md#scribe--audiovideo-transcription-optional-off-by-default) for the exact JSON), and set `GEMINI_API_KEY` in `.env`.
|
|
104
|
+
|
|
101
105
|
## Step 3: Configure the recipe
|
|
102
106
|
|
|
103
107
|
Copy the template recipe and fill in your credentials:
|
|
@@ -120,8 +124,10 @@ Edit `my-recipe.json` and replace the placeholder values in `mcpServers`:
|
|
|
120
124
|
"ZULIP_RC_PATH": "./.zuliprc" // path to your .zuliprc
|
|
121
125
|
}
|
|
122
126
|
},
|
|
127
|
+
// Optional — NOT in the shipped recipe. Add only if you set up a
|
|
128
|
+
// Notion MCP server (see Step 2 above):
|
|
123
129
|
"syncntn": {
|
|
124
|
-
"command": "../
|
|
130
|
+
"command": "../your-notion-mcp/start.sh",
|
|
125
131
|
"env": {
|
|
126
132
|
"STORAGE_URL": "http://localhost:8000",
|
|
127
133
|
"WORKSPACE_ID": "YOUR_WORKSPACE_ID" // <-- replace this
|
|
@@ -138,23 +138,26 @@ ANTHROPIC_API_KEY=sk-ant-...
|
|
|
138
138
|
ZULIP_CHANNEL=your-channel-name
|
|
139
139
|
```
|
|
140
140
|
|
|
141
|
-
Optional — **only** if you want the miner to extract from
|
|
141
|
+
Optional — **only** if you want the miner to extract from GitLab (otherwise remove the `gitlab` block from `recipes/knowledge-miner.json` and skip these):
|
|
142
142
|
|
|
143
143
|
```ini
|
|
144
144
|
# GitLab (knowledge-miner.json: gitlab)
|
|
145
145
|
GITLAB_TOKEN=glpat-...
|
|
146
146
|
GITLAB_API_URL=https://gitlab.example.com/api/v4
|
|
147
|
+
```
|
|
147
148
|
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
149
|
+
Optional — the conductor's web UI is protected by Basic-Auth that defaults to `admin` / `admin`. Fine for a laptop; **change it** the moment the machine is reachable by anyone else:
|
|
150
|
+
|
|
151
|
+
```ini
|
|
152
|
+
WEBUI_USERNAME=...
|
|
153
|
+
WEBUI_PASSWORD=...
|
|
151
154
|
```
|
|
152
155
|
|
|
153
156
|
Bun auto-loads `.env`, so nothing else to wire. If a recipe references a `${VAR}` you haven't set, the child's startup will fail with a clear message telling you which variable is missing and which recipe referenced it.
|
|
154
157
|
|
|
155
158
|
## Step 6: Decide which data sources you want
|
|
156
159
|
|
|
157
|
-
The miner child uses `recipes/knowledge-miner.json`, which comes pre-wired to talk to **Zulip,
|
|
160
|
+
The miner child uses `recipes/knowledge-miner.json`, which comes pre-wired to talk to **Zulip, GitLab, and DuckDuckGo** (public web); **Notion** and **Scribe** (audio/video transcription) connections can be added — see their subsections below. The recipe itself references credentials via `${VAR}` placeholders — you don't edit the recipe to fill in secrets; you set the env vars in Step 5 and the framework substitutes at load time.
|
|
158
161
|
|
|
159
162
|
You decide which sources are active by whether you **set the matching env vars** and whether you **keep the matching mcpServers block in the recipe**.
|
|
160
163
|
|
|
@@ -175,17 +178,29 @@ No separate install — the recipe runs `npx @zereight/mcp-gitlab` on demand.
|
|
|
175
178
|
|
|
176
179
|
To disable: remove the `gitlab` block from `recipes/knowledge-miner.json`. If you leave it in but don't set the env vars, the child will fail to start with a message like `Recipe "recipes/knowledge-miner.json" references environment variable ${GITLAB_TOKEN} which is not set.` — that's the system telling you to either fill in the env var or delete the block.
|
|
177
180
|
|
|
178
|
-
### Notion (optional)
|
|
181
|
+
### Notion (optional, off by default)
|
|
182
|
+
|
|
183
|
+
The recipe ships **without** a Notion server — the adapter it was developed against (`syncntn`) is not publicly available, so a default block would only produce a startup failure. The miner's system prompt still describes the `syncntn--*` tools; the agent simply won't have them until you wire a server in.
|
|
179
184
|
|
|
180
|
-
To enable: install a Notion MCP server (
|
|
185
|
+
To enable: install a Notion MCP server (any server whose tool names match what the prompt references — see [SETUP.md → Notion](./SETUP.md#notion-optional-via-an-mcp-server) for selection caveats), then add a block to `recipes/knowledge-miner.json` under `mcpServers`:
|
|
186
|
+
|
|
187
|
+
```jsonc
|
|
188
|
+
"syncntn": {
|
|
189
|
+
"command": "../your-notion-mcp/start.sh", // however your server is launched
|
|
190
|
+
"env": {
|
|
191
|
+
"STORAGE_SERVICE_URL": "${NOTION_STORAGE_URL}",
|
|
192
|
+
"WORKSPACE_ID": "${NOTION_WORKSPACE_ID}"
|
|
193
|
+
}
|
|
194
|
+
}
|
|
195
|
+
```
|
|
196
|
+
|
|
197
|
+
and set the referenced vars in `.env`:
|
|
181
198
|
|
|
182
199
|
```ini
|
|
183
200
|
NOTION_STORAGE_URL=http://localhost:8000
|
|
184
201
|
NOTION_WORKSPACE_ID=...
|
|
185
202
|
```
|
|
186
203
|
|
|
187
|
-
To disable: remove the `syncntn` block from `recipes/knowledge-miner.json`. Same behavior as above — unset env + kept block = startup failure with a clear message.
|
|
188
|
-
|
|
189
204
|
### DuckDuckGo web search (optional, enabled by default)
|
|
190
205
|
|
|
191
206
|
The miner is wired to [`nickclyde/duckduckgo-mcp-server`](https://github.com/nickclyde/duckduckgo-mcp-server) as `ddg`. No API key — DuckDuckGo's public HTML search, scraped at request time.
|
|
@@ -207,14 +222,51 @@ Web hits get tagged `[WEB: <url>]` in mined reports — internal `[SRC]` always
|
|
|
207
222
|
|
|
208
223
|
To disable: remove the `ddg` block from `recipes/knowledge-miner.json`. The agent will skip the public web.
|
|
209
224
|
|
|
225
|
+
### Scribe — audio/video transcription (optional, off by default)
|
|
226
|
+
|
|
227
|
+
The miner's prompt knows how to use [`dariakroshka/scribe-mcp`](https://github.com/dariakroshka/scribe-mcp) to transcribe recordings (via Google's Gemini API — media leaves your machine). The recipe ships without the block: it requires a Gemini API key and a sibling checkout, neither of which a demo should demand.
|
|
228
|
+
|
|
229
|
+
To enable: install the server as a sibling of `connectome-host/`:
|
|
230
|
+
|
|
231
|
+
```bash
|
|
232
|
+
cd ..
|
|
233
|
+
git clone https://github.com/dariakroshka/scribe-mcp.git
|
|
234
|
+
cd scribe-mcp
|
|
235
|
+
bun install
|
|
236
|
+
cd ../connectome-host
|
|
237
|
+
```
|
|
238
|
+
|
|
239
|
+
then add this block to `recipes/knowledge-miner.json` under `mcpServers`:
|
|
240
|
+
|
|
241
|
+
```jsonc
|
|
242
|
+
"scribe": {
|
|
243
|
+
"command": "bun",
|
|
244
|
+
"args": ["../scribe-mcp/src/index.ts"],
|
|
245
|
+
"env": {
|
|
246
|
+
"GEMINI_API_KEY": "${GEMINI_API_KEY}",
|
|
247
|
+
"NOTION_API_KEY": "${NOTION_API_KEY:-}", // only scribe--scribe_notion_page needs it
|
|
248
|
+
"SCRIBE_GLOSSARY_PATH": "./input/glossary.txt",
|
|
249
|
+
"SCRIBE_GLOSSARY_URL": "${SCRIBE_GLOSSARY_URL:-}" // optional domain glossary
|
|
250
|
+
},
|
|
251
|
+
"source": {
|
|
252
|
+
"url": "https://github.com/dariakroshka/scribe-mcp.git",
|
|
253
|
+
"install": { "runtime": "bun", "run": "bun install --frozen-lockfile" },
|
|
254
|
+
"inContainer": { "path": "/scribe-mcp" }
|
|
255
|
+
}
|
|
256
|
+
}
|
|
257
|
+
```
|
|
258
|
+
|
|
259
|
+
and set `GEMINI_API_KEY=...` in `.env`.
|
|
260
|
+
|
|
210
261
|
### Summary table
|
|
211
262
|
|
|
212
|
-
| Source |
|
|
263
|
+
| Source | Block in recipe? | Env vars needed |
|
|
213
264
|
|---|---|---|
|
|
214
|
-
| Zulip | Yes | (configured via `.zuliprc`, no `${VAR}`) |
|
|
215
|
-
| GitLab | Yes
|
|
216
|
-
| Notion |
|
|
217
|
-
| DuckDuckGo | Yes
|
|
265
|
+
| Zulip | Yes (default) | (configured via `.zuliprc`, no `${VAR}`) |
|
|
266
|
+
| GitLab | Yes (default) — remove if not using | `GITLAB_TOKEN`, `GITLAB_API_URL` |
|
|
267
|
+
| Notion | **No** — add a `syncntn` block if using | `NOTION_STORAGE_URL`, `NOTION_WORKSPACE_ID` |
|
|
268
|
+
| DuckDuckGo | Yes (default) — remove if not using | none (no API key) |
|
|
269
|
+
| Scribe | **No** — add a `scribe` block if using | `GEMINI_API_KEY` (+ optional `NOTION_API_KEY`, `SCRIBE_GLOSSARY_URL`) |
|
|
218
270
|
|
|
219
271
|
### Tweaks you can still make to the recipe files
|
|
220
272
|
|
|
@@ -237,6 +289,8 @@ What you'll see:
|
|
|
237
289
|
3. Press **Tab** a couple of times to cycle through view modes. One of them is the **process fleet** view — it lists the three children and their status. All three should reach **ready** (green). If any show **crashed** (red), jump to Troubleshooting.
|
|
238
290
|
4. Ask the conductor `are all three ready?` — it'll run `fleet--list` and confirm. This also serves as a quick "am I set up correctly" smoke test.
|
|
239
291
|
|
|
292
|
+
The conductor also serves a **web UI** on port 7340 (all interfaces, Basic-Auth). Credentials default to `admin` / `admin` unless you set `WEBUI_USERNAME` / `WEBUI_PASSWORD` in `.env` — see Step 5. Open `http://localhost:7340` to watch the fleet from a browser.
|
|
293
|
+
|
|
240
294
|
### The four view modes
|
|
241
295
|
|
|
242
296
|
Press **Tab** to cycle between views. Press **Ctrl+F** to jump straight to the process fleet view from anywhere.
|
|
@@ -32,13 +32,6 @@
|
|
|
32
32
|
}
|
|
33
33
|
}
|
|
34
34
|
},
|
|
35
|
-
"syncntn": {
|
|
36
|
-
"command": "../syncntn/services/mcp/start_mcp_local.sh",
|
|
37
|
-
"env": {
|
|
38
|
-
"STORAGE_SERVICE_URL": "${NOTION_STORAGE_URL}",
|
|
39
|
-
"WORKSPACE_ID": "${NOTION_WORKSPACE_ID}"
|
|
40
|
-
}
|
|
41
|
-
},
|
|
42
35
|
"gitlab": {
|
|
43
36
|
"command": "npx",
|
|
44
37
|
"args": [
|
|
@@ -66,29 +59,6 @@
|
|
|
66
59
|
"path": "/duckduckgo-mcp-server"
|
|
67
60
|
}
|
|
68
61
|
}
|
|
69
|
-
},
|
|
70
|
-
"scribe": {
|
|
71
|
-
"command": "bun",
|
|
72
|
-
"args": [
|
|
73
|
-
"../scribe-mcp/src/index.ts"
|
|
74
|
-
],
|
|
75
|
-
"env": {
|
|
76
|
-
"GEMINI_API_KEY": "${GEMINI_API_KEY}",
|
|
77
|
-
"NOTION_API_KEY": "${NOTION_API_KEY:-}",
|
|
78
|
-
"SCRIBE_GLOSSARY_PATH": "./input/glossary.txt",
|
|
79
|
-
"SCRIBE_GLOSSARY_URL": "${SCRIBE_GLOSSARY_URL:-}"
|
|
80
|
-
},
|
|
81
|
-
"source": {
|
|
82
|
-
"url": "https://github.com/dariakroshka/scribe-mcp.git",
|
|
83
|
-
"ref": "828ae8ba6ea83ce439ce6ba020ed0223cf096f82",
|
|
84
|
-
"install": {
|
|
85
|
-
"runtime": "bun",
|
|
86
|
-
"run": "bun install --frozen-lockfile"
|
|
87
|
-
},
|
|
88
|
-
"inContainer": {
|
|
89
|
-
"path": "/scribe-mcp"
|
|
90
|
-
}
|
|
91
|
-
}
|
|
92
62
|
}
|
|
93
63
|
},
|
|
94
64
|
"modules": {
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
{
|
|
2
|
+
"name": "Mock Test",
|
|
3
|
+
"description": "Zero-cost offline smoke recipe: membrane's mock adapter, no API key, no provider spend. Echoes your messages back through the full host loop.",
|
|
4
|
+
"agent": {
|
|
5
|
+
"name": "agent",
|
|
6
|
+
"provider": "mock",
|
|
7
|
+
"systemPrompt": "You are a mock agent for offline host testing."
|
|
8
|
+
},
|
|
9
|
+
"modules": {
|
|
10
|
+
"subagents": false,
|
|
11
|
+
"lessons": false,
|
|
12
|
+
"retrieval": false,
|
|
13
|
+
"wake": false,
|
|
14
|
+
"workspace": false,
|
|
15
|
+
"webui": {
|
|
16
|
+
"host": "127.0.0.1"
|
|
17
|
+
}
|
|
18
|
+
}
|
|
19
|
+
}
|
package/recipes/triumvirate.json
CHANGED
|
@@ -0,0 +1,288 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Report-only audit of subagents/lessons/retrieval opt-ins across recipes.
|
|
3
|
+
*
|
|
4
|
+
* Background: published v0.7.2 and earlier treated these three modules as
|
|
5
|
+
* opt-OUT (an omitted `modules` key meant enabled), and DEFAULT_RECIPE
|
|
6
|
+
* enabled all three explicitly. Current main treats them as opt-IN
|
|
7
|
+
* (a4bd9fd). That flip is the right default, but it leaves existing
|
|
8
|
+
* deployments with two things only a human can settle:
|
|
9
|
+
*
|
|
10
|
+
* 1. A source recipe that explicitly says `lessons: true` stays enabled
|
|
11
|
+
* after upgrade — intentionally. Whether that `true` was a real choice
|
|
12
|
+
* or boilerplate copied from the old onboarding guide is not something
|
|
13
|
+
* a defaults change (or this script) can infer. It gets reported;
|
|
14
|
+
* the operator decides.
|
|
15
|
+
* 2. A persisted `data/.recipe.json` is a resolved snapshot of whatever
|
|
16
|
+
* was in effect at launch — under the old defaults that includes
|
|
17
|
+
* `subagents/lessons/retrieval: true` the operator never wrote. It is
|
|
18
|
+
* not necessarily the authoritative source recipe, so it's reported
|
|
19
|
+
* separately, as a pointer back to the source, never as a finding in
|
|
20
|
+
* itself.
|
|
21
|
+
*
|
|
22
|
+
* This script reads and reports. It never modifies a file, and it has no
|
|
23
|
+
* flag that would make it modify a file.
|
|
24
|
+
*
|
|
25
|
+
* Usage:
|
|
26
|
+
* bun scripts/audit-module-optins.ts <recipe.json | directory> [...more]
|
|
27
|
+
*
|
|
28
|
+
* Directories are scanned recursively for *.json (including .recipe.json
|
|
29
|
+
* snapshots; node_modules/.git skipped). Fleet children referenced by
|
|
30
|
+
* local path are followed automatically.
|
|
31
|
+
*
|
|
32
|
+
* Exit codes: 0 = nothing needs an operator decision; 2 = explicit enables
|
|
33
|
+
* (or inert-retrieval combinations) found — review the report; 1 = a path
|
|
34
|
+
* argument could not be read.
|
|
35
|
+
*/
|
|
36
|
+
|
|
37
|
+
import { readFileSync, readdirSync, statSync, existsSync } from 'node:fs';
|
|
38
|
+
import { basename, dirname, isAbsolute, join, resolve } from 'node:path';
|
|
39
|
+
|
|
40
|
+
export const AUDITED_MODULES = ['subagents', 'lessons', 'retrieval'] as const;
|
|
41
|
+
export type AuditedModule = (typeof AUDITED_MODULES)[number];
|
|
42
|
+
|
|
43
|
+
export type ModuleState = 'explicit-enable' | 'explicit-disable' | 'omitted';
|
|
44
|
+
|
|
45
|
+
export interface RecipeAudit {
|
|
46
|
+
/** Path as reported (relative to cwd where possible). */
|
|
47
|
+
path: string;
|
|
48
|
+
recipeName: string;
|
|
49
|
+
/** basename === '.recipe.json': a resolved launch snapshot, not a source. */
|
|
50
|
+
isSnapshot: boolean;
|
|
51
|
+
states: Record<AuditedModule, ModuleState>;
|
|
52
|
+
/** Snapshot has all three explicitly true — the old DEFAULT_RECIPE shape.
|
|
53
|
+
* Almost certainly captured pre-flip defaults, not an operator choice. */
|
|
54
|
+
matchesOldDefaultBoilerplate: boolean;
|
|
55
|
+
/** retrieval explicitly enabled while lessons is omitted: worked under the
|
|
56
|
+
* old defaults (omitted lessons = on), silently inert after upgrade. */
|
|
57
|
+
inertRetrieval: boolean;
|
|
58
|
+
/** Local fleet-children recipe paths referenced by this recipe. */
|
|
59
|
+
childRecipePaths: string[];
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
interface RawRecipeShape {
|
|
63
|
+
name?: unknown;
|
|
64
|
+
agent?: unknown;
|
|
65
|
+
modules?: Record<string, unknown>;
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
/** A JSON document we treat as a recipe: object with a name and an agent
|
|
69
|
+
* block. Anything else in a scanned directory is silently skipped. */
|
|
70
|
+
export function looksLikeRecipe(raw: unknown): raw is RawRecipeShape {
|
|
71
|
+
if (!raw || typeof raw !== 'object' || Array.isArray(raw)) return false;
|
|
72
|
+
const o = raw as RawRecipeShape;
|
|
73
|
+
return typeof o.name === 'string' && !!o.agent && typeof o.agent === 'object';
|
|
74
|
+
}
|
|
75
|
+
|
|
76
|
+
export function classifyModule(value: unknown): ModuleState {
|
|
77
|
+
if (value === undefined || value === null) return 'omitted';
|
|
78
|
+
if (value === false) return 'explicit-disable';
|
|
79
|
+
// true or a config object both enable (matches createFramework wiring).
|
|
80
|
+
return 'explicit-enable';
|
|
81
|
+
}
|
|
82
|
+
|
|
83
|
+
export function auditRecipe(raw: RawRecipeShape, path: string): RecipeAudit {
|
|
84
|
+
const modules = (raw.modules && typeof raw.modules === 'object' ? raw.modules : {}) as Record<string, unknown>;
|
|
85
|
+
const states = Object.fromEntries(
|
|
86
|
+
AUDITED_MODULES.map((m) => [m, classifyModule(modules[m])]),
|
|
87
|
+
) as Record<AuditedModule, ModuleState>;
|
|
88
|
+
|
|
89
|
+
const isSnapshot = basename(path) === '.recipe.json';
|
|
90
|
+
|
|
91
|
+
const childRecipePaths: string[] = [];
|
|
92
|
+
const fleet = modules.fleet;
|
|
93
|
+
if (fleet && typeof fleet === 'object') {
|
|
94
|
+
const children = (fleet as { children?: unknown }).children;
|
|
95
|
+
if (Array.isArray(children)) {
|
|
96
|
+
for (const child of children) {
|
|
97
|
+
const ref = (child as { recipe?: unknown })?.recipe;
|
|
98
|
+
if (typeof ref !== 'string' || !ref) continue;
|
|
99
|
+
if (ref.startsWith('http://') || ref.startsWith('https://')) continue;
|
|
100
|
+
childRecipePaths.push(isAbsolute(ref) ? ref : resolve(dirname(resolve(path)), ref));
|
|
101
|
+
}
|
|
102
|
+
}
|
|
103
|
+
}
|
|
104
|
+
|
|
105
|
+
return {
|
|
106
|
+
path,
|
|
107
|
+
recipeName: String(raw.name),
|
|
108
|
+
isSnapshot,
|
|
109
|
+
states,
|
|
110
|
+
matchesOldDefaultBoilerplate:
|
|
111
|
+
isSnapshot &&
|
|
112
|
+
modules.subagents === true && modules.lessons === true && modules.retrieval === true,
|
|
113
|
+
inertRetrieval: states.retrieval === 'explicit-enable' && states.lessons === 'omitted',
|
|
114
|
+
childRecipePaths,
|
|
115
|
+
};
|
|
116
|
+
}
|
|
117
|
+
|
|
118
|
+
// ---------------------------------------------------------------------------
|
|
119
|
+
// Filesystem walk (main-path only; the logic above is what the tests pin)
|
|
120
|
+
// ---------------------------------------------------------------------------
|
|
121
|
+
|
|
122
|
+
const SKIP_DIRS = new Set(['node_modules', '.git', 'web']);
|
|
123
|
+
|
|
124
|
+
function collectJsonFiles(root: string, out: string[]): void {
|
|
125
|
+
for (const entry of readdirSync(root, { withFileTypes: true })) {
|
|
126
|
+
const full = join(root, entry.name);
|
|
127
|
+
if (entry.isDirectory()) {
|
|
128
|
+
if (!SKIP_DIRS.has(entry.name)) collectJsonFiles(full, out);
|
|
129
|
+
} else if (entry.name.endsWith('.json')) {
|
|
130
|
+
out.push(full);
|
|
131
|
+
}
|
|
132
|
+
}
|
|
133
|
+
}
|
|
134
|
+
|
|
135
|
+
export function auditPaths(paths: string[]): { audits: RecipeAudit[]; unreadable: string[] } {
|
|
136
|
+
const files: string[] = [];
|
|
137
|
+
const unreadable: string[] = [];
|
|
138
|
+
for (const p of paths) {
|
|
139
|
+
if (!existsSync(p)) {
|
|
140
|
+
unreadable.push(p);
|
|
141
|
+
continue;
|
|
142
|
+
}
|
|
143
|
+
if (statSync(p).isDirectory()) collectJsonFiles(p, files);
|
|
144
|
+
else files.push(p);
|
|
145
|
+
}
|
|
146
|
+
|
|
147
|
+
const audits: RecipeAudit[] = [];
|
|
148
|
+
const seen = new Set<string>();
|
|
149
|
+
const queue = [...files];
|
|
150
|
+
while (queue.length > 0) {
|
|
151
|
+
const file = queue.shift()!;
|
|
152
|
+
const key = resolve(file);
|
|
153
|
+
if (seen.has(key)) continue;
|
|
154
|
+
seen.add(key);
|
|
155
|
+
let raw: unknown;
|
|
156
|
+
try {
|
|
157
|
+
raw = JSON.parse(readFileSync(file, 'utf-8'));
|
|
158
|
+
} catch {
|
|
159
|
+
continue; // not JSON we can read — not our business to complain about
|
|
160
|
+
}
|
|
161
|
+
if (!looksLikeRecipe(raw)) continue;
|
|
162
|
+
const audit = auditRecipe(raw, file);
|
|
163
|
+
audits.push(audit);
|
|
164
|
+
// Follow local fleet children so a parent path argument covers the tree.
|
|
165
|
+
for (const child of audit.childRecipePaths) {
|
|
166
|
+
if (existsSync(child)) queue.push(child);
|
|
167
|
+
}
|
|
168
|
+
}
|
|
169
|
+
return { audits, unreadable };
|
|
170
|
+
}
|
|
171
|
+
|
|
172
|
+
// ---------------------------------------------------------------------------
|
|
173
|
+
// Report
|
|
174
|
+
// ---------------------------------------------------------------------------
|
|
175
|
+
|
|
176
|
+
const STATE_LINES: Record<AuditedModule, Record<ModuleState, string>> = {
|
|
177
|
+
subagents: {
|
|
178
|
+
'explicit-enable':
|
|
179
|
+
'explicitly enabled — stays on after upgrade. Keep if this agent really forks workers.',
|
|
180
|
+
'explicit-disable': 'explicitly disabled — no change on upgrade (belt-and-braces for old checkouts).',
|
|
181
|
+
omitted:
|
|
182
|
+
'omitted — ON under published ≤0.7.2, OFF after upgrade. Declare `true` only if this agent relied on forking.',
|
|
183
|
+
},
|
|
184
|
+
lessons: {
|
|
185
|
+
'explicit-enable':
|
|
186
|
+
'explicitly enabled — stays on after upgrade. Keep if this agent curates a lesson library.',
|
|
187
|
+
'explicit-disable': 'explicitly disabled — no change on upgrade (belt-and-braces for old checkouts).',
|
|
188
|
+
omitted:
|
|
189
|
+
'omitted — ON under published ≤0.7.2, OFF after upgrade. Declare `true` only if this agent relied on lessons.',
|
|
190
|
+
},
|
|
191
|
+
retrieval: {
|
|
192
|
+
'explicit-enable':
|
|
193
|
+
'explicitly enabled — stays on after upgrade: per-compile injection plus two Haiku calls per turn. Keep only as a real choice.',
|
|
194
|
+
'explicit-disable': 'explicitly disabled — no change on upgrade (belt-and-braces for old checkouts).',
|
|
195
|
+
omitted: 'omitted — ON under published ≤0.7.2 (when lessons ran), OFF after upgrade.',
|
|
196
|
+
},
|
|
197
|
+
};
|
|
198
|
+
|
|
199
|
+
export function renderReport(audits: RecipeAudit[]): { text: string; needsDecision: number } {
|
|
200
|
+
const lines: string[] = [];
|
|
201
|
+
let needsDecision = 0;
|
|
202
|
+
|
|
203
|
+
const sources = audits.filter((a) => !a.isSnapshot);
|
|
204
|
+
const snapshots = audits.filter((a) => a.isSnapshot);
|
|
205
|
+
|
|
206
|
+
for (const a of sources) {
|
|
207
|
+
lines.push(`${a.path} (recipe "${a.recipeName}")`);
|
|
208
|
+
for (const m of AUDITED_MODULES) {
|
|
209
|
+
const state = a.states[m];
|
|
210
|
+
if (state === 'explicit-enable') needsDecision++;
|
|
211
|
+
lines.push(` ${m}: ${STATE_LINES[m][state]}`);
|
|
212
|
+
}
|
|
213
|
+
if (a.inertRetrieval) {
|
|
214
|
+
needsDecision++;
|
|
215
|
+
lines.push(
|
|
216
|
+
' ⚠ retrieval is enabled but lessons is omitted. Under the old defaults omitted lessons still ran,',
|
|
217
|
+
' so retrieval worked; after upgrade lessons is off and retrieval is silently inert.',
|
|
218
|
+
' Either add `lessons: true` (if retrieval is a real choice) or drop `retrieval`.',
|
|
219
|
+
);
|
|
220
|
+
}
|
|
221
|
+
lines.push('');
|
|
222
|
+
}
|
|
223
|
+
|
|
224
|
+
if (snapshots.length > 0) {
|
|
225
|
+
lines.push('Resolved snapshots (data/.recipe.json) — informational only:');
|
|
226
|
+
lines.push(
|
|
227
|
+
' These are launch-time captures, not authoritative sources. Under the old defaults they',
|
|
228
|
+
' include enables the operator never wrote. Audit the source recipe each run was launched',
|
|
229
|
+
' from; the snapshot refreshes on the next launch from source.',
|
|
230
|
+
);
|
|
231
|
+
for (const a of snapshots) {
|
|
232
|
+
const enabled = AUDITED_MODULES.filter((m) => a.states[m] === 'explicit-enable');
|
|
233
|
+
const note = a.matchesOldDefaultBoilerplate
|
|
234
|
+
? 'all three enabled — matches the old DEFAULT_RECIPE shape, almost certainly pre-flip defaults, not a choice'
|
|
235
|
+
: enabled.length > 0
|
|
236
|
+
? `explicitly enabled here: ${enabled.join(', ')}`
|
|
237
|
+
: 'no audited modules enabled';
|
|
238
|
+
lines.push(` ${a.path} (recipe "${a.recipeName}"): ${note}`);
|
|
239
|
+
}
|
|
240
|
+
lines.push('');
|
|
241
|
+
}
|
|
242
|
+
|
|
243
|
+
return { text: lines.join('\n'), needsDecision };
|
|
244
|
+
}
|
|
245
|
+
|
|
246
|
+
// ---------------------------------------------------------------------------
|
|
247
|
+
// Main
|
|
248
|
+
// ---------------------------------------------------------------------------
|
|
249
|
+
|
|
250
|
+
function main(): void {
|
|
251
|
+
const args = process.argv.slice(2).filter((a) => !a.startsWith('--'));
|
|
252
|
+
if (args.length === 0) {
|
|
253
|
+
console.error('Usage: bun scripts/audit-module-optins.ts <recipe.json | directory> [...more]');
|
|
254
|
+
console.error('Report-only: reads recipes, changes nothing.');
|
|
255
|
+
process.exit(1);
|
|
256
|
+
}
|
|
257
|
+
|
|
258
|
+
const { audits, unreadable } = auditPaths(args);
|
|
259
|
+
for (const p of unreadable) console.error(`cannot read: ${p}`);
|
|
260
|
+
|
|
261
|
+
console.log('connectome-host module opt-in audit — report only, nothing is modified\n');
|
|
262
|
+
console.log(
|
|
263
|
+
'Published ≤0.7.2 treated subagents/lessons/retrieval as opt-OUT (omitted = enabled).\n' +
|
|
264
|
+
'Current main treats them as opt-IN (omitted = disabled). Explicit enables survive the\n' +
|
|
265
|
+
'upgrade by design — this report shows where each recipe stands so you can decide which\n' +
|
|
266
|
+
'of those are real choices and which are old boilerplate.\n',
|
|
267
|
+
);
|
|
268
|
+
|
|
269
|
+
let needsDecision = 0;
|
|
270
|
+
if (audits.length === 0) {
|
|
271
|
+
console.log('No recipe-shaped JSON found under the given paths.');
|
|
272
|
+
} else {
|
|
273
|
+
const report = renderReport(audits);
|
|
274
|
+
needsDecision = report.needsDecision;
|
|
275
|
+
console.log(report.text);
|
|
276
|
+
const sources = audits.filter((a) => !a.isSnapshot).length;
|
|
277
|
+
console.log(
|
|
278
|
+
`Summary: ${audits.length} recipe(s) audited (${sources} source, ${audits.length - sources} snapshot), ` +
|
|
279
|
+
`${needsDecision} item(s) need an operator decision.`,
|
|
280
|
+
);
|
|
281
|
+
}
|
|
282
|
+
|
|
283
|
+
// Unreadable path arguments outrank findings in the exit code.
|
|
284
|
+
if (unreadable.length > 0) process.exit(1);
|
|
285
|
+
if (needsDecision > 0) process.exit(2);
|
|
286
|
+
}
|
|
287
|
+
|
|
288
|
+
if (import.meta.main) main();
|