@adhd/agent-mcp 2.0.0 → 2.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/AGENTS.md +236 -0
- package/CHANGELOG.md +115 -0
- package/LICENSE +21 -0
- package/README.md +201 -0
- package/docs/architecture-and-security.md +405 -0
- package/docs/marketing/.catalog/capabilities.md +143 -0
- package/docs/marketing/.catalog/consumer.md +111 -0
- package/docs/marketing/.catalog/distribution.md +223 -0
- package/docs/marketing/.catalog/doc-conformance.md +237 -0
- package/docs/marketing/.catalog/doc-ops.md +98 -0
- package/docs/marketing/.catalog/metrics.md +119 -0
- package/docs/marketing/.catalog/required-tooling.md +224 -0
- package/docs/marketing/.catalog/review.md +164 -0
- package/docs/provider-call-audit.md +254 -0
- package/llms.txt +133 -0
- package/package.json +31 -50
- package/src/__tests__/integration/harness.d.ts +871 -39
- package/src/__tests__/integration/harness.d.ts.map +1 -1
- package/src/__tests__/integration/harness.js +93 -87
- package/src/__tests__/integration/harness.js.map +1 -1
- package/src/__tests__/integration/scripted-provider.d.ts +1 -2
- package/src/__tests__/integration/scripted-provider.d.ts.map +1 -1
- package/src/__tests__/integration/scripted-provider.js +1 -3
- package/src/__tests__/integration/scripted-provider.js.map +1 -1
- package/src/config.d.ts +6 -69
- package/src/config.d.ts.map +1 -1
- package/src/config.js +1 -50
- package/src/config.js.map +1 -1
- package/src/db/client.d.ts +1516 -2
- package/src/db/client.d.ts.map +1 -1
- package/src/db/client.js +4 -2
- package/src/db/client.js.map +1 -1
- package/src/db/migrate-runner.d.ts +0 -28
- package/src/db/migrate-runner.d.ts.map +1 -1
- package/src/db/migrate-runner.js +0 -24
- package/src/db/migrate-runner.js.map +1 -1
- package/src/db/migrate.d.ts +0 -7
- package/src/db/migrate.d.ts.map +1 -1
- package/src/db/migrate.js +0 -7
- package/src/db/migrate.js.map +1 -1
- package/src/db/schema.d.ts +0 -1260
- package/src/db/schema.d.ts.map +1 -1
- package/src/db/schema.js +2 -179
- package/src/db/schema.js.map +1 -1
- package/src/index.d.ts +4 -27
- package/src/index.d.ts.map +1 -1
- package/src/index.js +43 -114
- package/src/index.js.map +1 -1
- package/src/logger.d.ts +0 -8
- package/src/logger.d.ts.map +1 -1
- package/src/logger.js +1 -12
- package/src/logger.js.map +1 -1
- package/src/scripts/agent-mcp-tail.d.ts +3 -0
- package/src/scripts/agent-mcp-tail.d.ts.map +1 -0
- package/src/scripts/agent-mcp-tail.js +227 -0
- package/src/scripts/agent-mcp-tail.js.map +1 -0
- package/src/server.d.ts +13 -34
- package/src/server.d.ts.map +1 -1
- package/src/server.js +234 -326
- package/src/server.js.map +1 -1
- package/src/store/agent-store.d.ts +3 -16
- package/src/store/agent-store.d.ts.map +1 -1
- package/src/store/agent-store.js +38 -41
- package/src/store/agent-store.js.map +1 -1
- package/src/streaming/chat-gateway.d.ts +17 -0
- package/src/streaming/chat-gateway.d.ts.map +1 -0
- package/src/streaming/chat-gateway.js +336 -0
- package/src/streaming/chat-gateway.js.map +1 -0
- package/src/streaming/event-bus.d.ts +4 -0
- package/src/streaming/event-bus.d.ts.map +1 -1
- package/src/streaming/event-bus.js +14 -1
- package/src/streaming/event-bus.js.map +1 -1
- package/src/streaming/sse-server.d.ts +3 -14
- package/src/streaming/sse-server.d.ts.map +1 -1
- package/src/streaming/sse-server.js +37 -37
- package/src/streaming/sse-server.js.map +1 -1
- package/src/utils/load-env.d.ts +0 -14
- package/src/utils/load-env.d.ts.map +1 -1
- package/src/utils/load-env.js +0 -18
- package/src/utils/load-env.js.map +1 -1
- package/drizzle/0000_nifty_sasquatch.sql +0 -55
- package/drizzle/0001_task_usage.sql +0 -16
- package/drizzle/0002_lumpy_silver_centurion.sql +0 -2
- package/drizzle/0003_nifty_shriek.sql +0 -2
- package/drizzle/0004_brief_shocker.sql +0 -4
- package/drizzle/0005_clear_lenny_balinger.sql +0 -25
- package/drizzle/0006_composed_prompts_cache.sql +0 -24
- package/drizzle/meta/0000_snapshot.json +0 -387
- package/drizzle/meta/0001_snapshot.json +0 -495
- package/drizzle/meta/0002_snapshot.json +0 -509
- package/drizzle/meta/0003_snapshot.json +0 -523
- package/drizzle/meta/0004_snapshot.json +0 -551
- package/drizzle/meta/0005_snapshot.json +0 -545
- package/drizzle/meta/0006_snapshot.json +0 -651
- package/drizzle/meta/_journal.json +0 -55
- package/src/clients/http-client.d.ts +0 -24
- package/src/clients/http-client.d.ts.map +0 -1
- package/src/clients/http-client.js +0 -98
- package/src/clients/http-client.js.map +0 -1
- package/src/clients/in-process.d.ts +0 -27
- package/src/clients/in-process.d.ts.map +0 -1
- package/src/clients/in-process.js +0 -37
- package/src/clients/in-process.js.map +0 -1
- package/src/clients/index.d.ts +0 -7
- package/src/clients/index.d.ts.map +0 -1
- package/src/clients/index.js +0 -5
- package/src/clients/index.js.map +0 -1
- package/src/clients/registry.d.ts +0 -54
- package/src/clients/registry.d.ts.map +0 -1
- package/src/clients/registry.js +0 -164
- package/src/clients/registry.js.map +0 -1
- package/src/clients/stdio-client.d.ts +0 -16
- package/src/clients/stdio-client.d.ts.map +0 -1
- package/src/clients/stdio-client.js +0 -97
- package/src/clients/stdio-client.js.map +0 -1
- package/src/clients/tool-naming.d.ts +0 -44
- package/src/clients/tool-naming.d.ts.map +0 -1
- package/src/clients/tool-naming.js +0 -65
- package/src/clients/tool-naming.js.map +0 -1
- package/src/clients/types.d.ts +0 -18
- package/src/clients/types.d.ts.map +0 -1
- package/src/clients/types.js +0 -2
- package/src/clients/types.js.map +0 -1
- package/src/engine/dag-engine.d.ts +0 -62
- package/src/engine/dag-engine.d.ts.map +0 -1
- package/src/engine/dag-engine.js +0 -158
- package/src/engine/dag-engine.js.map +0 -1
- package/src/engine/hooks.d.ts +0 -24
- package/src/engine/hooks.d.ts.map +0 -1
- package/src/engine/hooks.js +0 -50
- package/src/engine/hooks.js.map +0 -1
- package/src/engine/orchestrator.d.ts +0 -40
- package/src/engine/orchestrator.d.ts.map +0 -1
- package/src/engine/orchestrator.js +0 -507
- package/src/engine/orchestrator.js.map +0 -1
- package/src/engine/policy.d.ts +0 -66
- package/src/engine/policy.d.ts.map +0 -1
- package/src/engine/policy.js +0 -112
- package/src/engine/policy.js.map +0 -1
- package/src/engine/prompt-resolver.d.ts +0 -89
- package/src/engine/prompt-resolver.d.ts.map +0 -1
- package/src/engine/prompt-resolver.js +0 -96
- package/src/engine/prompt-resolver.js.map +0 -1
- package/src/engine/queue.d.ts +0 -29
- package/src/engine/queue.d.ts.map +0 -1
- package/src/engine/queue.js +0 -56
- package/src/engine/queue.js.map +0 -1
- package/src/plugins/index.d.ts +0 -2
- package/src/plugins/index.d.ts.map +0 -1
- package/src/plugins/index.js +0 -2
- package/src/plugins/index.js.map +0 -1
- package/src/plugins/loader.d.ts +0 -65
- package/src/plugins/loader.d.ts.map +0 -1
- package/src/plugins/loader.js +0 -222
- package/src/plugins/loader.js.map +0 -1
- package/src/plugins/usage-plugin.d.ts +0 -27
- package/src/plugins/usage-plugin.d.ts.map +0 -1
- package/src/plugins/usage-plugin.js +0 -148
- package/src/plugins/usage-plugin.js.map +0 -1
- package/src/providers/anthropic.d.ts +0 -13
- package/src/providers/anthropic.d.ts.map +0 -1
- package/src/providers/anthropic.js +0 -260
- package/src/providers/anthropic.js.map +0 -1
- package/src/providers/claudecli.d.ts +0 -153
- package/src/providers/claudecli.d.ts.map +0 -1
- package/src/providers/claudecli.js +0 -510
- package/src/providers/claudecli.js.map +0 -1
- package/src/providers/factory.d.ts +0 -10
- package/src/providers/factory.d.ts.map +0 -1
- package/src/providers/factory.js +0 -24
- package/src/providers/factory.js.map +0 -1
- package/src/providers/index.d.ts +0 -6
- package/src/providers/index.d.ts.map +0 -1
- package/src/providers/index.js +0 -5
- package/src/providers/index.js.map +0 -1
- package/src/providers/openai.d.ts +0 -14
- package/src/providers/openai.d.ts.map +0 -1
- package/src/providers/openai.js +0 -162
- package/src/providers/openai.js.map +0 -1
- package/src/providers/types.d.ts +0 -36
- package/src/providers/types.d.ts.map +0 -1
- package/src/providers/types.js +0 -2
- package/src/providers/types.js.map +0 -1
- package/src/store/composed-prompt-store.d.ts +0 -24
- package/src/store/composed-prompt-store.d.ts.map +0 -1
- package/src/store/composed-prompt-store.js +0 -75
- package/src/store/composed-prompt-store.js.map +0 -1
- package/src/store/index.d.ts +0 -5
- package/src/store/index.d.ts.map +0 -1
- package/src/store/index.js +0 -5
- package/src/store/index.js.map +0 -1
- package/src/store/session-store.d.ts +0 -45
- package/src/store/session-store.d.ts.map +0 -1
- package/src/store/session-store.js +0 -197
- package/src/store/session-store.js.map +0 -1
- package/src/store/task-store.d.ts +0 -55
- package/src/store/task-store.d.ts.map +0 -1
- package/src/store/task-store.js +0 -186
- package/src/store/task-store.js.map +0 -1
- package/src/tools/agent-crud.d.ts +0 -15
- package/src/tools/agent-crud.d.ts.map +0 -1
- package/src/tools/agent-crud.js +0 -32
- package/src/tools/agent-crud.js.map +0 -1
- package/src/tools/session.d.ts +0 -33
- package/src/tools/session.d.ts.map +0 -1
- package/src/tools/session.js +0 -73
- package/src/tools/session.js.map +0 -1
- package/src/tools/task.d.ts +0 -86
- package/src/tools/task.d.ts.map +0 -1
- package/src/tools/task.js +0 -406
- package/src/tools/task.js.map +0 -1
- package/src/tools/usage.d.ts +0 -60
- package/src/tools/usage.d.ts.map +0 -1
- package/src/tools/usage.js +0 -174
- package/src/tools/usage.js.map +0 -1
- package/src/types/agent.d.ts +0 -2
- package/src/types/agent.d.ts.map +0 -1
- package/src/types/agent.js +0 -3
- package/src/types/agent.js.map +0 -1
- package/src/types/execution.d.ts +0 -2
- package/src/types/execution.d.ts.map +0 -1
- package/src/types/execution.js +0 -3
- package/src/types/execution.js.map +0 -1
- package/src/types/index.d.ts +0 -2
- package/src/types/index.d.ts.map +0 -1
- package/src/types/index.js +0 -2
- package/src/types/index.js.map +0 -1
- package/src/types/mcp.d.ts +0 -2
- package/src/types/mcp.d.ts.map +0 -1
- package/src/types/mcp.js +0 -3
- package/src/types/mcp.js.map +0 -1
- package/src/types/message.d.ts +0 -2
- package/src/types/message.d.ts.map +0 -1
- package/src/types/message.js +0 -3
- package/src/types/message.js.map +0 -1
- package/src/types/session.d.ts +0 -2
- package/src/types/session.d.ts.map +0 -1
- package/src/types/session.js +0 -3
- package/src/types/session.js.map +0 -1
- package/src/types/task.d.ts +0 -2
- package/src/types/task.d.ts.map +0 -1
- package/src/types/task.js +0 -3
- package/src/types/task.js.map +0 -1
- package/src/utils/ids.d.ts +0 -2
- package/src/utils/ids.d.ts.map +0 -1
- package/src/utils/ids.js +0 -5
- package/src/utils/ids.js.map +0 -1
- package/src/utils/logger.d.ts +0 -2
- package/src/utils/logger.d.ts.map +0 -1
- package/src/utils/logger.js +0 -5
- package/src/utils/logger.js.map +0 -1
- package/src/utils/timestamps.d.ts +0 -2
- package/src/utils/timestamps.d.ts.map +0 -1
- package/src/utils/timestamps.js +0 -4
- package/src/utils/timestamps.js.map +0 -1
- package/src/validation/agent.d.ts +0 -400
- package/src/validation/agent.d.ts.map +0 -1
- package/src/validation/agent.js +0 -251
- package/src/validation/agent.js.map +0 -1
- package/src/validation/agents.d.ts +0 -2
- package/src/validation/agents.d.ts.map +0 -1
- package/src/validation/agents.js +0 -3
- package/src/validation/agents.js.map +0 -1
- package/src/validation/errors.d.ts +0 -32
- package/src/validation/errors.d.ts.map +0 -1
- package/src/validation/errors.js +0 -42
- package/src/validation/errors.js.map +0 -1
- package/src/validation/execution.d.ts +0 -86
- package/src/validation/execution.d.ts.map +0 -1
- package/src/validation/execution.js +0 -28
- package/src/validation/execution.js.map +0 -1
- package/src/validation/index.d.ts +0 -9
- package/src/validation/index.d.ts.map +0 -1
- package/src/validation/index.js +0 -9
- package/src/validation/index.js.map +0 -1
- package/src/validation/mcp.d.ts +0 -42
- package/src/validation/mcp.d.ts.map +0 -1
- package/src/validation/mcp.js +0 -28
- package/src/validation/mcp.js.map +0 -1
- package/src/validation/message.d.ts +0 -43
- package/src/validation/message.d.ts.map +0 -1
- package/src/validation/message.js +0 -23
- package/src/validation/message.js.map +0 -1
- package/src/validation/session.d.ts +0 -49
- package/src/validation/session.d.ts.map +0 -1
- package/src/validation/session.js +0 -34
- package/src/validation/session.js.map +0 -1
- package/src/validation/task.d.ts +0 -146
- package/src/validation/task.d.ts.map +0 -1
- package/src/validation/task.js +0 -95
- package/src/validation/task.js.map +0 -1
- package/src/validation/tasks.d.ts +0 -2
- package/src/validation/tasks.d.ts.map +0 -1
- package/src/validation/tasks.js +0 -3
- package/src/validation/tasks.js.map +0 -1
- package/src/validation/usage.d.ts +0 -89
- package/src/validation/usage.d.ts.map +0 -1
- package/src/validation/usage.js +0 -98
- package/src/validation/usage.js.map +0 -1
|
@@ -0,0 +1,224 @@
|
|
|
1
|
+
# @adhd/agent-mcp — Required Tooling & Unverified Capabilities
|
|
2
|
+
|
|
3
|
+
**Catalog date:** 2026-07-12
|
|
4
|
+
**Total shipped capabilities:** 33
|
|
5
|
+
**Capabilities marked 🔴 UNVERIFIED:** 7
|
|
6
|
+
|
|
7
|
+
---
|
|
8
|
+
|
|
9
|
+
## Summary
|
|
10
|
+
|
|
11
|
+
Seven shipped capabilities cannot be proven with the current tooling setup. Each is marked as shipped in code but lacks a default-running test (either because the test is gated behind an env flag, or the test infrastructure doesn't exist).
|
|
12
|
+
|
|
13
|
+
---
|
|
14
|
+
|
|
15
|
+
## Unverified Capabilities & Root Causes
|
|
16
|
+
|
|
17
|
+
### 1. HTTP OpenAI-Compatible Chat Gateway (`/v1/chat/completions`)
|
|
18
|
+
|
|
19
|
+
**Status:** ✅ Shipped (implemented in src/streaming/chat-gateway.ts + src/streaming/sse-server.ts)
|
|
20
|
+
**Tested:** ❌ No test
|
|
21
|
+
**Reason:** No HTTP client smoke test exists; SSE server is started but never exercised in CI
|
|
22
|
+
|
|
23
|
+
**Needed to verify:**
|
|
24
|
+
- **Tool:** HTTP client (curl, node-fetch, or similar) to POST to /v1/chat/completions
|
|
25
|
+
- **Setup:** Start agent-mcp server with running SQLite DB, create agent, POST request
|
|
26
|
+
- **Assertion:** Parse JSON response, verify status + content + streaming format
|
|
27
|
+
|
|
28
|
+
**Effort:** ~2 hours (write integration test using vitest + node http or got library)
|
|
29
|
+
|
|
30
|
+
**File locations:**
|
|
31
|
+
- Capability: `src/streaming/chat-gateway.ts:handleChatCompletions` (comprehensive implementation)
|
|
32
|
+
- Server wiring: `src/streaming/sse-server.ts:45-50` (route registration)
|
|
33
|
+
- Entry point: `src/index.ts:188` (SSE server startup)
|
|
34
|
+
|
|
35
|
+
---
|
|
36
|
+
|
|
37
|
+
### 2. HTTP Models Endpoint (`GET /v1/models`)
|
|
38
|
+
|
|
39
|
+
**Status:** ✅ Shipped (implemented in src/streaming/chat-gateway.ts)
|
|
40
|
+
**Tested:** ❌ No test
|
|
41
|
+
**Reason:** Part of SSE server, not exercised in tests
|
|
42
|
+
|
|
43
|
+
**Needed to verify:**
|
|
44
|
+
- **Tool:** HTTP GET client (curl -s localhost:ADHD_AGENT_SSE_PORT/v1/models | jq)
|
|
45
|
+
- **Setup:** Start agent-mcp, create agent
|
|
46
|
+
- **Assertion:** Parse JSON response, verify models array
|
|
47
|
+
|
|
48
|
+
**Effort:** ~1 hour (trivial HTTP GET test)
|
|
49
|
+
|
|
50
|
+
**File locations:**
|
|
51
|
+
- Capability: `src/streaming/chat-gateway.ts:handleGetModels`
|
|
52
|
+
- Server wiring: `src/streaming/sse-server.ts:44-45`
|
|
53
|
+
|
|
54
|
+
---
|
|
55
|
+
|
|
56
|
+
### 3. Claude CLI Provider (`claudecli` type)
|
|
57
|
+
|
|
58
|
+
**Status:** ✅ Shipped (configured in src/config.ts, documented in USAGE_GUIDE)
|
|
59
|
+
**Tested:** ❌ No test
|
|
60
|
+
**Reason:** Requires `claude` binary on PATH; not available in CI environment
|
|
61
|
+
|
|
62
|
+
**Needed to verify:**
|
|
63
|
+
- **Tool:** Claude CLI binary (`claude --version` must succeed)
|
|
64
|
+
- **Setup:** Install Claude CLI locally, create agent with type:'claudecli', run task
|
|
65
|
+
- **Assertion:** Task completes, output matches claudecli format
|
|
66
|
+
|
|
67
|
+
**Effort:** ~3 hours (integration test + conditional skip if claude unavailable)
|
|
68
|
+
|
|
69
|
+
**File locations:**
|
|
70
|
+
- Config: `src/config.ts` (no explicit claudecli block; handled separately)
|
|
71
|
+
- Documentation: `src/server.ts:259` (USAGE_GUIDE provider table)
|
|
72
|
+
|
|
73
|
+
---
|
|
74
|
+
|
|
75
|
+
### 4. MCP Server Bindings & Tool Registration
|
|
76
|
+
|
|
77
|
+
**Status:** ✅ Shipped (core feature, wired in orchestrator)
|
|
78
|
+
**Tested:** ❌ Live e2e test exists but skipped (AGENT_MCP_LIVE=1 only)
|
|
79
|
+
**Reason:** Requires real model (LM Studio or Anthropic) to exercise multi-agent recursion
|
|
80
|
+
|
|
81
|
+
**Needed to verify:**
|
|
82
|
+
- **Tool:** LM Studio (`http://localhost:1234/v1`) OR Anthropic API key
|
|
83
|
+
- **Setup:** Start calc-server.mjs (stdio MCP), create agent with mcpServers binding, run task using calc tool
|
|
84
|
+
- **Assertion:** Tool call reaches MCP server, result comes back, persisted in DB
|
|
85
|
+
|
|
86
|
+
**Effort:** ~2 hours (make live-dag.e2e.test.ts run by default with fallback provider detection)
|
|
87
|
+
|
|
88
|
+
**File locations:**
|
|
89
|
+
- Live test: `src/__tests__/integration/live-dag.e2e.test.ts:75-76` (skipIf(!isLive) guard)
|
|
90
|
+
- Calc server fixture: `src/__tests__/integration/fixtures/calc-server.mjs`
|
|
91
|
+
|
|
92
|
+
---
|
|
93
|
+
|
|
94
|
+
### 5. Agent Delegation (Recursive Multi-Agent DAG)
|
|
95
|
+
|
|
96
|
+
**Status:** ✅ Shipped (core feature, tested live)
|
|
97
|
+
**Tested:** ❌ Live e2e test exists but skipped (AGENT_MCP_LIVE=1 only)
|
|
98
|
+
**Reason:** Same as #4 — requires real model to exercise delegation logic
|
|
99
|
+
|
|
100
|
+
**Needed to verify:**
|
|
101
|
+
- **Tool:** Real LLM (LM Studio or Anthropic)
|
|
102
|
+
- **Setup:** Same as #4; test wiring in src/__tests__/integration/live-dag.e2e.test.ts already exists
|
|
103
|
+
- **Assertion:** 3-level recursion happens (coordinator→fanout→workers); every worker invokes calc tool; all tasks persisted
|
|
104
|
+
|
|
105
|
+
**Effort:** ~1 hour (remove skipIf guard + add provider auto-detection fallback)
|
|
106
|
+
|
|
107
|
+
**File locations:**
|
|
108
|
+
- Test: `src/__tests__/integration/live-dag.e2e.test.ts` (complete test, just gated)
|
|
109
|
+
|
|
110
|
+
---
|
|
111
|
+
|
|
112
|
+
### 6. Human-in-the-Loop (HITL) Task Suspension
|
|
113
|
+
|
|
114
|
+
**Status:** ✅ Shipped (awaiting_input status, task_resume tool, documented in USAGE_GUIDE)
|
|
115
|
+
**Tested:** ❌ No test
|
|
116
|
+
**Reason:** No integration test exercises the HITL flow (create task, wait for suspension, resume with input)
|
|
117
|
+
|
|
118
|
+
**Needed to verify:**
|
|
119
|
+
- **Tool:** Real model that calls `request_human_input` tool; test harness to simulate human response
|
|
120
|
+
- **Setup:** Create agent with allowHumanInput:true, run task with model that triggers HITL
|
|
121
|
+
- **Assertion:** Task suspends with awaiting_input status, resumeToken issued, task_resume moves task to running, final result produced
|
|
122
|
+
|
|
123
|
+
**Effort:** ~4 hours (write integration test with mock model or real LM Studio)
|
|
124
|
+
|
|
125
|
+
**File locations:**
|
|
126
|
+
- Tool: `src/server.ts:513-517` (task_resume definition)
|
|
127
|
+
- USAGE_GUIDE: `src/server.ts:269-276` (error codes reference)
|
|
128
|
+
- Config: allowHumanInput field in agent definition
|
|
129
|
+
|
|
130
|
+
---
|
|
131
|
+
|
|
132
|
+
### 7. Server-Sent Events (SSE) Task Streaming
|
|
133
|
+
|
|
134
|
+
**Status:** ✅ Shipped (SSE server on separate port, task/completed notifications)
|
|
135
|
+
**Tested:** ❌ No test
|
|
136
|
+
**Reason:** No SSE client test; SSE server starts but is never polled in CI
|
|
137
|
+
|
|
138
|
+
**Needed to verify:**
|
|
139
|
+
- **Tool:** SSE client (node EventSource or similar) to subscribe to task stream
|
|
140
|
+
- **Setup:** Start agent-mcp, create agent, run background task, subscribe to SSE /tasks endpoint
|
|
141
|
+
- **Assertion:** Receive task/completed event with correct task_id + status
|
|
142
|
+
|
|
143
|
+
**Effort:** ~2 hours (write integration test using EventSource or fetch EventStream)
|
|
144
|
+
|
|
145
|
+
**File locations:**
|
|
146
|
+
- SSE server: `src/streaming/sse-server.ts`
|
|
147
|
+
- Event bus: `src/streaming/event-bus.ts:subscribeToTaskDone`
|
|
148
|
+
- Wiring: `src/index.ts:188` (SSE server startup)
|
|
149
|
+
|
|
150
|
+
---
|
|
151
|
+
|
|
152
|
+
## Tooling Gap Summary
|
|
153
|
+
|
|
154
|
+
| Need | Tool | Status | Impact | Priority |
|
|
155
|
+
|------|------|--------|--------|----------|
|
|
156
|
+
| HTTP client | curl, node-fetch, got | ✅ available | 2 HTTP endpoints unproven | HIGH |
|
|
157
|
+
| Real LLM | LM Studio (localhost) or Anthropic API | ⚠ optional | 2 features unproven (delegation, DAG) | HIGH |
|
|
158
|
+
| Claude CLI binary | claude (system PATH) | ❌ not in CI | 1 provider unproven | MEDIUM |
|
|
159
|
+
| EventSource client | node EventSource | ✅ available | SSE streaming unproven | MEDIUM |
|
|
160
|
+
|
|
161
|
+
---
|
|
162
|
+
|
|
163
|
+
## Capability Impact
|
|
164
|
+
|
|
165
|
+
**If these 7 capabilities are not proven by catalog v2.0.2:**
|
|
166
|
+
|
|
167
|
+
- 🟢 Agent/session/task management: ✅ Proven (19 unit tests pass)
|
|
168
|
+
- 🟢 Provider (OpenAI, Anthropic, DeepSeek): ✅ Proven (config tests pass)
|
|
169
|
+
- 🟢 Tool advertisement (names-only vs full): ✅ Proven (14 tool-advertisement tests pass)
|
|
170
|
+
- 🔴 HTTP API (OpenAI compat): ❌ Unproven — consumers cannot verify it works before deploying
|
|
171
|
+
- 🔴 Agent delegation: ❌ Unproven in CI — works live but no regression guard
|
|
172
|
+
- 🔴 HITL suspension: ❌ Unproven — no evidence it works beyond code review
|
|
173
|
+
- 🔴 SSE streaming: ❌ Unproven — no evidence real-time events flow
|
|
174
|
+
|
|
175
|
+
**Risk:** Publishing 2.0.2 with these 7 gaps means consumers reporting "chat gateway doesn't work" or "delegation failed" after install, with no CI gates to catch regression.
|
|
176
|
+
|
|
177
|
+
---
|
|
178
|
+
|
|
179
|
+
## Action Items for Next Catalog Run
|
|
180
|
+
|
|
181
|
+
### Must-Have (blocker for 2.0.2 publish):
|
|
182
|
+
|
|
183
|
+
1. **Add HTTP smoke test** (curl or node-fetch client) for /v1/chat/completions and /v1/models
|
|
184
|
+
- File: src/__tests__/integration/http-gateway.test.ts (new)
|
|
185
|
+
- Time: ~2 hours
|
|
186
|
+
- Dependency: HTTP client library (already in Nx workspace)
|
|
187
|
+
|
|
188
|
+
2. **Add HITL integration test** (with mock model or real LM Studio)
|
|
189
|
+
- File: src/__tests__/integration/hitl.test.ts (new)
|
|
190
|
+
- Time: ~4 hours
|
|
191
|
+
- Dependency: LM Studio running on localhost:1234
|
|
192
|
+
|
|
193
|
+
### Nice-to-Have (gated by env, runs in CI with fallback):
|
|
194
|
+
|
|
195
|
+
3. **Ungat live-dag.e2e.test.ts** — remove AGENT_MCP_LIVE=1 guard, add provider auto-detection
|
|
196
|
+
- File: src/__tests__/integration/live-dag.e2e.test.ts (modify)
|
|
197
|
+
- Time: ~1 hour
|
|
198
|
+
- Dependency: LM Studio OR Anthropic API key (env var)
|
|
199
|
+
- Fallback: Skip if neither available, warn in logs
|
|
200
|
+
|
|
201
|
+
4. **Add SSE integration test**
|
|
202
|
+
- File: src/__tests__/integration/sse-stream.test.ts (new)
|
|
203
|
+
- Time: ~2 hours
|
|
204
|
+
- Dependency: EventSource client (available)
|
|
205
|
+
|
|
206
|
+
5. **Add Claude CLI test**
|
|
207
|
+
- File: src/__tests__/integration/claudecli.test.ts (new)
|
|
208
|
+
- Time: ~3 hours
|
|
209
|
+
- Dependency: claude CLI binary on PATH
|
|
210
|
+
- Fallback: Skip if not available
|
|
211
|
+
|
|
212
|
+
---
|
|
213
|
+
|
|
214
|
+
## Recommendation
|
|
215
|
+
|
|
216
|
+
**For 2.0.2:**
|
|
217
|
+
- ✅ **Mandatory:** HTTP smoke test (2 hrs) — blocks OpenAI compat consumer trust
|
|
218
|
+
- ✅ **Strongly recommended:** HITL test (4 hrs) — blocks HITL feature launch
|
|
219
|
+
- ⚠ **Nice-to-have:** Ungat live-dag.e2e, add SSE test (3 hrs total) — regression guards
|
|
220
|
+
- ⚠ **Optional:** Claude CLI test (3 hrs) — lower priority (fewer users)
|
|
221
|
+
|
|
222
|
+
**Total effort to verify all 7 capabilities:** ~12 hours across 5 new test files.
|
|
223
|
+
|
|
224
|
+
→ Cartographer recommends 2.0.2 proceed without all 7 verified, but with HTTP + HITL tests added. Delegation + SSE + Claude CLI can be deferred to 2.0.3 with async CI integration.
|
|
@@ -0,0 +1,164 @@
|
|
|
1
|
+
# Doc Review — @adhd/agent-mcp 2.0.2 — 7c400a7
|
|
2
|
+
|
|
3
|
+
**Final Review Gate Run**
|
|
4
|
+
**Timestamp:** 2026-07-12T02:50:00Z
|
|
5
|
+
**Reviewer:** Documentation Assessment Gate (Final)
|
|
6
|
+
|
|
7
|
+
---
|
|
8
|
+
|
|
9
|
+
## VERDICT: FAIL
|
|
10
|
+
|
|
11
|
+
**Reason:** Lens 2 failure — AGENTS.md contains 2 critical inaccuracies referencing removed/renamed tools incompatible with current capabilities.json.
|
|
12
|
+
|
|
13
|
+
---
|
|
14
|
+
|
|
15
|
+
## Lens 1 — Closed-Loop Metrics
|
|
16
|
+
|
|
17
|
+
**STATUS: PASS**
|
|
18
|
+
|
|
19
|
+
### Evidence
|
|
20
|
+
|
|
21
|
+
The steward re-ran the cartographer after fixes; metrics.md now contains a post-fix block (lines 74-119).
|
|
22
|
+
|
|
23
|
+
**Post-fix results (lines 75-89):**
|
|
24
|
+
- **metric_1_eliminated_reader_searches:** 23 → 5 (78% improvement; goal was < 10) ✓ PASS
|
|
25
|
+
- **metric_3_undocumented%:** 76% → 3% (97% improvement; goal was < 50%) ✓ PASS
|
|
26
|
+
- **junk%:** 0% → 0% ✓ PASS
|
|
27
|
+
|
|
28
|
+
**Contradictions with capabilities.json:**
|
|
29
|
+
- All 33 shipped capabilities (per capabilities.json) are status:shipped
|
|
30
|
+
- No deprecated items claimed as current
|
|
31
|
+
- Roadmap items (AGENTS.md lines 134-145) explicitly marked "Do not document these as shipped features"
|
|
32
|
+
- Architecture schema field names verified: snake_case correct (`cache_read_input_tokens`, `cache_creation_input_tokens`, `uncached_input_tokens`, `peak_context_tokens`)
|
|
33
|
+
- `peak_context_at` correctly defined as INTEGER (line 157), not TIMESTAMP
|
|
34
|
+
|
|
35
|
+
✓ **Lens 1 Result: PASS**
|
|
36
|
+
|
|
37
|
+
---
|
|
38
|
+
|
|
39
|
+
## Lens 2 — Template Conformance & Accuracy
|
|
40
|
+
|
|
41
|
+
**STATUS: FAIL**
|
|
42
|
+
|
|
43
|
+
### Per-Document Scores
|
|
44
|
+
|
|
45
|
+
| Doc | Score | Status | Finding |
|
|
46
|
+
|---|---|---|---|
|
|
47
|
+
| README.md | 98/100 | PASS | Quickstart, release notes, env vars all correct; LICENSE file exists |
|
|
48
|
+
| CHANGELOG.md | 100/100 | PASS | Keep-a-Changelog compliant; 6 regression fixes documented |
|
|
49
|
+
| AGENTS.md | 92/100 | **FAIL** | 16 tools in table correct; but example section uses removed tools |
|
|
50
|
+
| llms.txt | 98/100 | PASS | Quick reference accurate; correct 2.0.2 highlights |
|
|
51
|
+
| architecture-and-security.md | 98/100 | PASS | Token field names correct (post-fix); schema accurate |
|
|
52
|
+
|
|
53
|
+
### Critical Failures
|
|
54
|
+
|
|
55
|
+
**FAIL #1: AGENTS.md Example Uses Removed Tool "agent_message"**
|
|
56
|
+
|
|
57
|
+
- **Location:** AGENTS.md, lines 208-213
|
|
58
|
+
- **Issue:** JSON-RPC example calls `"method": "agent_message"` which does NOT exist in capabilities.json or shipped 2.0.2
|
|
59
|
+
- **Evidence:**
|
|
60
|
+
- capabilities.json lists 16 MCP tools; `agent_message` is absent
|
|
61
|
+
- README.md quickstart (lines 82-89) correctly uses `"method": "task"`
|
|
62
|
+
- Consumer test confirms `task` is the correct tool
|
|
63
|
+
- **Impact:** LLM reading AGENTS.md will attempt to use a non-existent tool, breaking playbook integration
|
|
64
|
+
- **Fix:** Replace lines 208-213 example with correct `task` method call (template: README.md lines 82-89)
|
|
65
|
+
|
|
66
|
+
**FAIL #2: AGENTS.md References Renamed Tools in Explanation**
|
|
67
|
+
|
|
68
|
+
- **Location:** AGENTS.md, line 225
|
|
69
|
+
- **Issue:** States "client can later call `task_status(taskId)` or `task_usage(taskId)`" but these tools don't exist; correct names are `result` and `usage_query`
|
|
70
|
+
- **Evidence:**
|
|
71
|
+
- AGENTS.md's own tool table (lines 24-41) correctly lists `result` (line 33) and `usage_query` (line 40)
|
|
72
|
+
- Line 225 contradicts the same document's tool inventory
|
|
73
|
+
- **Impact:** LLM reading explanation section will use wrong tool names, causing failures
|
|
74
|
+
- **Fix:** Replace `task_status(taskId)` → `result(taskId)` and `task_usage(taskId)` → `usage_query({task_id: taskId})`
|
|
75
|
+
|
|
76
|
+
### Conformance Summary
|
|
77
|
+
|
|
78
|
+
✓ **Token field naming:** Correct and consistent across README, AGENTS, CHANGELOG, llms.txt, architecture-and-security.md (cacheReadTokens, uncachedInputTokens, cacheCreationTokens, peakContextTokens)
|
|
79
|
+
✓ **Plugin config chain:** Documented correctly (3 paths in order)
|
|
80
|
+
✓ **All 16 tools:** Listed in AGENTS.md table with correct signatures
|
|
81
|
+
✓ **Link integrity:** All links resolve (LICENSE file verified to exist)
|
|
82
|
+
✓ **Architecture schema:** Snake_case DB names correct; peakContextAt is INTEGER
|
|
83
|
+
✗ **Example consistency:** 2 instances of removed/renamed tools in narrative sections
|
|
84
|
+
|
|
85
|
+
✗ **Lens 2 Result: FAIL** (2 critical inaccuracies prevent PASS)
|
|
86
|
+
|
|
87
|
+
---
|
|
88
|
+
|
|
89
|
+
## Lens 3 — Consumer Usability Test
|
|
90
|
+
|
|
91
|
+
**STATUS: PASS**
|
|
92
|
+
|
|
93
|
+
### Evidence
|
|
94
|
+
|
|
95
|
+
File `/docs/marketing/.catalog/consumer.md` exists and shows completion of all canonical tasks.
|
|
96
|
+
|
|
97
|
+
**Task Results:**
|
|
98
|
+
1. **Quick-Start Installation & First Agent:** ✓ PASS
|
|
99
|
+
- Docs sufficient for install, config, agent creation, task execution, result retrieval
|
|
100
|
+
- No source code required; correct tool names and JSON-RPC format documented
|
|
101
|
+
|
|
102
|
+
2. **Provider Migration (Anthropic → DeepSeek):** ✓ PASS
|
|
103
|
+
- Env vars and token accounting differences clearly explained
|
|
104
|
+
- README, AGENTS.md, CHANGELOG all provide needed information
|
|
105
|
+
|
|
106
|
+
3. **Debugging Token Usage:** ✓ PASS
|
|
107
|
+
- Token field meanings documented; cumulative vs. peak distinction clear
|
|
108
|
+
- AGENTS.md token accounting section and CHANGELOG break down each field
|
|
109
|
+
|
|
110
|
+
**Consumer Verdict:** Documentation is sufficient for independent use; no source-code reading required.
|
|
111
|
+
|
|
112
|
+
✓ **Lens 3 Result: PASS**
|
|
113
|
+
|
|
114
|
+
---
|
|
115
|
+
|
|
116
|
+
## Summary by Lens
|
|
117
|
+
|
|
118
|
+
| Lens | Status | Gate Impact |
|
|
119
|
+
|------|--------|-------------|
|
|
120
|
+
| **Lens 1: Metrics** | PASS | Improvement verified (23→5 searches, 76→3% undoc) |
|
|
121
|
+
| **Lens 2: Conformance** | FAIL | 2 critical tool-name inaccuracies in AGENTS.md |
|
|
122
|
+
| **Lens 3: Consumer** | PASS | All 3 canonical tasks pass; docs are self-sufficient |
|
|
123
|
+
|
|
124
|
+
**Gate Rule:** ALL THREE LENSES must PASS for overall PASS.
|
|
125
|
+
**Result:** 2/3 PASS → **OVERALL FAIL**
|
|
126
|
+
|
|
127
|
+
---
|
|
128
|
+
|
|
129
|
+
## Required Fixes (Before Resubmission)
|
|
130
|
+
|
|
131
|
+
**Fix #1 (CRITICAL):** AGENTS.md lines 208-213 — Remove incorrect tool reference
|
|
132
|
+
- Delete the `agent_message` example block (lines 208-213)
|
|
133
|
+
- Replace with correct `task` method call using README.md lines 82-89 as template:
|
|
134
|
+
```json
|
|
135
|
+
{
|
|
136
|
+
"jsonrpc": "2.0",
|
|
137
|
+
"id": 2,
|
|
138
|
+
"method": "task",
|
|
139
|
+
"params": {
|
|
140
|
+
"agentName": "research-agent",
|
|
141
|
+
"prompt": "Find all TypeScript files in /Users/me/code"
|
|
142
|
+
}
|
|
143
|
+
}
|
|
144
|
+
```
|
|
145
|
+
|
|
146
|
+
**Fix #2 (CRITICAL):** AGENTS.md line 225 — Fix tool name references
|
|
147
|
+
- Current: `"client can later call task_status(taskId) or task_usage(taskId)"`
|
|
148
|
+
- Corrected: `"client can later call result(taskId) to retrieve results, or usage_query(filter, groupBy) to query usage across tasks"`
|
|
149
|
+
- Reason: Aligns with correct tool names from the MCP tool table (lines 33, 40)
|
|
150
|
+
|
|
151
|
+
---
|
|
152
|
+
|
|
153
|
+
## Resubmission Process
|
|
154
|
+
|
|
155
|
+
1. **Apply both fixes to AGENTS.md** (2 surgical edits, ~3 minutes)
|
|
156
|
+
2. **Re-run cartographer** to verify no new broken links: `npx nx run agent-mcp:cartographer`
|
|
157
|
+
3. **Verify consumer test still passes** (it will; consumer uses README examples, not AGENTS.md)
|
|
158
|
+
4. **Re-submit for final gate review**
|
|
159
|
+
|
|
160
|
+
---
|
|
161
|
+
|
|
162
|
+
**Gate Decision:** FAIL — 2 critical inaccuracies in tool naming blocks publication.
|
|
163
|
+
**Resubmit After:** Fixes applied and cartographer re-run (est. 10 minutes).
|
|
164
|
+
**This is the final gate run.** No further iterations after this.
|
|
@@ -0,0 +1,254 @@
|
|
|
1
|
+
# agent-mcp: Provider Call & Event Storage Audit
|
|
2
|
+
|
|
3
|
+
Audit of what agent-mcp sends to the provider API each turn and what it stores in
|
|
4
|
+
the DB. Compared against the standard ReAct loop pattern used by OpenCode and Claude
|
|
5
|
+
Code.
|
|
6
|
+
|
|
7
|
+
**Date:** 2026-07-02
|
|
8
|
+
**Trigger:** investigating 262k input tokens for a 15-turn, 775-line implementation task.
|
|
9
|
+
|
|
10
|
+
---
|
|
11
|
+
|
|
12
|
+
## 1. What Goes to the Provider Every Turn
|
|
13
|
+
|
|
14
|
+
### Orchestrator Loop (`packages/ai/agent-mcp/src/engine/orchestrator.ts`)
|
|
15
|
+
|
|
16
|
+
Every iteration of the while-loop (lines ~130–850):
|
|
17
|
+
|
|
18
|
+
1. **`registry.listAllTools()` (line 176)** — fetches the full tool list from every
|
|
19
|
+
connected MCP server. For each server, this:
|
|
20
|
+
- Calls `getOrCreateClient(serverName)` (stdio clients are cached; the stdio
|
|
21
|
+
subprocess persists across calls)
|
|
22
|
+
- Calls `client.listTools()` which sends an MCP `tools/list` JSON-RPC request
|
|
23
|
+
over stdio and waits for the response
|
|
24
|
+
- Prefixes each tool name as `<server>__<tool>`
|
|
25
|
+
- Returns the complete array of `ToolDefinition[]`
|
|
26
|
+
|
|
27
|
+
2. **HITL append (lines 180-185)** — if the agent has `allowHumanInput: true` and
|
|
28
|
+
the task is durable, appends the built-in `request_human_input` tool.
|
|
29
|
+
|
|
30
|
+
3. **Hook emission (lines 208-226)** — fires `pre:model_request` for observation
|
|
31
|
+
and enforcement, with the full `{messages, tools}` payload.
|
|
32
|
+
|
|
33
|
+
4. **Provider call (lines 235-237)** — passes `tools: tools.length > 0 ? tools : undefined`
|
|
34
|
+
to `provider.chat()`.
|
|
35
|
+
|
|
36
|
+
### OpenAI Provider (`packages/ai/agent-mcp/src/providers/openai.ts`)
|
|
37
|
+
|
|
38
|
+
`toOpenAITools()` (lines 62-76) serializes every `ToolDefinition` as:
|
|
39
|
+
|
|
40
|
+
```typescript
|
|
41
|
+
{
|
|
42
|
+
type: "function",
|
|
43
|
+
function: {
|
|
44
|
+
name: "filesystem__read_text_file",
|
|
45
|
+
description: "Read the complete contents of a file...",
|
|
46
|
+
parameters: { // full JSON Schema — type, properties, required, descriptions
|
|
47
|
+
type: "object",
|
|
48
|
+
properties: {
|
|
49
|
+
path: { type: "string", description: "..." },
|
|
50
|
+
head: { type: "number", description: "..." },
|
|
51
|
+
tail: { type: "number", description: "..." },
|
|
52
|
+
...
|
|
53
|
+
},
|
|
54
|
+
required: ["path"],
|
|
55
|
+
additionalProperties: false
|
|
56
|
+
}
|
|
57
|
+
}
|
|
58
|
+
}
|
|
59
|
+
```
|
|
60
|
+
|
|
61
|
+
The OpenAI client then sends this `tools` array in EVERY `chat.completions.create()`
|
|
62
|
+
call (line 121). The API is stateless — tool definitions must be present in every
|
|
63
|
+
request for the model to emit tool calls.
|
|
64
|
+
|
|
65
|
+
### Measured Overhead (from the 2-tool-call experiment)
|
|
66
|
+
|
|
67
|
+
| Agent | Tools sent per turn | Input tokens (2 turns) | Cache hit | Net paid | Overhead vs shell |
|
|
68
|
+
|---|---|---|---|---|---|
|
|
69
|
+
| Both (fs + shell) | ~10 tools | 5,645 | — | — | 3.0× |
|
|
70
|
+
| Filesystem-only | ~8 tools | 5,126 | 2,560 (50%) | 2,566 | 2.7× |
|
|
71
|
+
| Shell-only | ~1 tool | 1,877 | 768 (41%) | 1,109 | 1.0× baseline |
|
|
72
|
+
|
|
73
|
+
Tool schemas are the dominant per-turn cost. On a 15-turn task:
|
|
74
|
+
- Filesystem-only: ~48k tokens (tool schemas re-sent 15×)
|
|
75
|
+
- Shell-only: ~10k tokens
|
|
76
|
+
- Delta: ~38k tokens (17% of the 262k total on the spec-types task)
|
|
77
|
+
|
|
78
|
+
### Cache Behavior
|
|
79
|
+
|
|
80
|
+
DeepSeek caches repeated prefixes automatically. The 50% cache-hit rate on the
|
|
81
|
+
filesystem agent means:
|
|
82
|
+
- Turn 1: 2,560 tokens paid at full price (system prompt + tool schemas + work order)
|
|
83
|
+
- Turn 2: 2,560 tokens read from cache at ~10% cost (256 billed), 2,566 new tokens
|
|
84
|
+
(tool results from turn 1 + assistant response + next turn's request)
|
|
85
|
+
- The 50% that changes is the growing conversation history — this is inherent and
|
|
86
|
+
cannot be eliminated by any agent-mcp change.
|
|
87
|
+
|
|
88
|
+
---
|
|
89
|
+
|
|
90
|
+
## 2. What Gets Stored in the DB
|
|
91
|
+
|
|
92
|
+
### `task_events` Table
|
|
93
|
+
|
|
94
|
+
Schema: `(id TEXT PK, task_id TEXT, type TEXT, payload TEXT, created_at TEXT)`
|
|
95
|
+
|
|
96
|
+
Event types written per turn:
|
|
97
|
+
- `MODEL_REQUEST` — `{messageCount, toolCount}` (thin summary, no token data)
|
|
98
|
+
- `MODEL_RESPONSE` — `{stopReason, hasContent, toolCallCount}` (thin summary, no token data)
|
|
99
|
+
- `TOOL_CALL` — empty payload
|
|
100
|
+
- `TOOL_RESULT` — empty payload
|
|
101
|
+
|
|
102
|
+
**The payload column is arbitrary JSON but carries no usage data.** Per-turn token
|
|
103
|
+
counts from `providerResponse.usage` are available at the event emission site
|
|
104
|
+
(`orchestrator.ts:303`) but were never written to the payload.
|
|
105
|
+
|
|
106
|
+
### `task_usage` Table
|
|
107
|
+
|
|
108
|
+
Schema includes: `task_id PK, input_tokens, output_tokens, tool_call_count,
|
|
109
|
+
model_calls, cache_read_input_tokens, cache_creation_input_tokens, ...`
|
|
110
|
+
|
|
111
|
+
Updated per turn via UPSERT (accumulate in-place):
|
|
112
|
+
```sql
|
|
113
|
+
INSERT ... ON CONFLICT(task_id) DO UPDATE SET
|
|
114
|
+
input_tokens = input_tokens + <new>,
|
|
115
|
+
output_tokens = output_tokens + <new>,
|
|
116
|
+
model_calls = model_calls + 1,
|
|
117
|
+
cache_read_input_tokens = COALESCE(cache_read_input_tokens, 0) + <new>
|
|
118
|
+
```
|
|
119
|
+
|
|
120
|
+
Single row per task. Per-turn deltas are lost — only the aggregate survives.
|
|
121
|
+
|
|
122
|
+
### `cache_read_input_tokens` Gap (FIXED 2026-07-02)
|
|
123
|
+
|
|
124
|
+
Prior to the fix in `openai.ts:169-170`, `cacheReadTokens` was never extracted from
|
|
125
|
+
the OpenAI SDK response (`sdkUsage.prompt_tokens_details.cached_tokens` existed but
|
|
126
|
+
was ignored). The Anthropic provider had equivalent extraction since inception.
|
|
127
|
+
The fix (3 lines) now captures cache reads for all OpenAI-compatible providers
|
|
128
|
+
(OpenAI, DeepSeek, LM Studio, Ollama).
|
|
129
|
+
|
|
130
|
+
### Per-turn token tracking (PENDING)
|
|
131
|
+
|
|
132
|
+
The MODEL_RESPONSE event payload at `orchestrator.ts:311-315` still writes only
|
|
133
|
+
`{stopReason, hasContent, toolCallCount}`. The fix to add `{inputTokens,
|
|
134
|
+
outputTokens, cacheReadTokens, cacheCreationTokens}` from `providerResponse.usage`
|
|
135
|
+
was applied to source but not yet built/reloaded. Once live, every MODEL_RESPONSE
|
|
136
|
+
row will carry per-turn token counts, enabling reconstruction of exactly where
|
|
137
|
+
tokens went in a multi-turn task.
|
|
138
|
+
|
|
139
|
+
---
|
|
140
|
+
|
|
141
|
+
## 3. Comparison: OpenCode / Claude Code Standard Pattern
|
|
142
|
+
|
|
143
|
+
The standard ReAct (Reasoning + Acting) agent loop sends tools on every request.
|
|
144
|
+
This is not an agent-mcp design flaw — it's how the OpenAI/Anthropic APIs work
|
|
145
|
+
(stateless chat completions).
|
|
146
|
+
|
|
147
|
+
| Aspect | agent-mcp | OpenCode (anomalyco) | Claude Code (Anthropic) |
|
|
148
|
+
|---|---|---|---|
|
|
149
|
+
| Tools fetched per turn | Yes — `listAllTools()` calls MCP `tools/list` every loop iteration | Yes — tool registry queried per turn | Yes — tool schemas sent every request |
|
|
150
|
+
| Full tools sent every turn | Yes — all servers, all tools | Yes — all registered tools | Yes — all tools the model can call |
|
|
151
|
+
| Tool pruning | No — all tools always sent | Partial — tools gated by permission mode | Yes — per-task tool filtering |
|
|
152
|
+
| Tool schema size | Filesystem: ~8 tools, ~3,200 tokens of JSON Schema | ~45 tools, variable | Model-dependent; auto-compacted |
|
|
153
|
+
| Caching behavior | Provider-dependent (DeepSeek: auto-prefix, ~50% hit) | Provider-dependent | Anthropic: explicit breakpoints, 90%+ hit with Sentinel-Fanout |
|
|
154
|
+
|
|
155
|
+
The critical difference: **agent-mcp has no tool pruning.** Claude Code sends only
|
|
156
|
+
the tools relevant to the current task. OpenCode gates tools by permission mode.
|
|
157
|
+
agent-mcp sends every tool from every connected MCP server on every turn.
|
|
158
|
+
|
|
159
|
+
---
|
|
160
|
+
|
|
161
|
+
## 4. Opportunities
|
|
162
|
+
|
|
163
|
+
> **Status update (2026-07-02):** #1, #2, and #3 are SHIPPED. `listAllTools()` is
|
|
164
|
+
> hoisted out of the loop; MODEL_RESPONSE events carry the normalized per-turn
|
|
165
|
+
> token fields plus `rawUsage` (the provider usage object verbatim); and tool
|
|
166
|
+
> advertisement defaults to `"names"` — slim name-only definitions on the wire
|
|
167
|
+
> with full documentation prepended to the system message as a cacheable prefix
|
|
168
|
+
> (per-agent override: `toolAdvertisement: "full"`; claudecli always full). See
|
|
169
|
+
> `engine/tool-advertisement.ts` and `__tests__/tool-advertisement.test.ts`
|
|
170
|
+
> (includes negative control). Remaining: registry-backed tool docs (Plan 8).
|
|
171
|
+
|
|
172
|
+
### Immediate (low effort, high impact)
|
|
173
|
+
|
|
174
|
+
1. **Pull `listAllTools()` out of the loop** — tools don't change mid-task. Call it
|
|
175
|
+
once before the while-loop and reuse the array. Eliminates MCP `tools/list`
|
|
176
|
+
round-trips on every turn. **DONE.**
|
|
177
|
+
|
|
178
|
+
2. **Add `inputTokens/outputTokens/cacheReadTokens/cacheCreationTokens` to
|
|
179
|
+
MODEL_RESPONSE payloads** — fix applied to source, pending build. **DONE, plus
|
|
180
|
+
`rawUsage` verbatim.**
|
|
181
|
+
|
|
182
|
+
### Medium (requires design)
|
|
183
|
+
|
|
184
|
+
3. **Tool pruning** — the agent definition or system prompt should declare which
|
|
185
|
+
tools the work order needs. The `DispatchUnit` from the dispatch-optimizer
|
|
186
|
+
already carries `context_files` and operation shapes — the orchestrator could
|
|
187
|
+
pass a `tools` allowlist. GitHub Engineering cut costs 62% with this approach.
|
|
188
|
+
Per-turn savings for filesystem-only: ~1,600 tokens/turn after caching.
|
|
189
|
+
|
|
190
|
+
4. **Cache-optimized system prompt** — place the largest stable prefix (system
|
|
191
|
+
prompt + tool schemas) first in the messages array to maximize cache-hit
|
|
192
|
+
surface area.
|
|
193
|
+
|
|
194
|
+
### Speculative (would require provider API changes)
|
|
195
|
+
|
|
196
|
+
5. **Omit `tools` on follow-up turns** — some providers tolerate this (model
|
|
197
|
+
already emitted valid tool calls, doesn't need schemas re-validated). Would
|
|
198
|
+
save ~1,600-3,200 tokens/turn. Not standard OpenAI behavior; needs testing
|
|
199
|
+
per provider.
|
|
200
|
+
|
|
201
|
+
---
|
|
202
|
+
|
|
203
|
+
## 5. Verbatim: What the Provider Sees Each Turn
|
|
204
|
+
|
|
205
|
+
A synthesized, token-annotated view of a single turn in the spec-types task:
|
|
206
|
+
|
|
207
|
+
```
|
|
208
|
+
SYSTEM (3,500 tokens — CACHED after turn 1)
|
|
209
|
+
"You are an implementation agent for the ADHD Nx monorepo..."
|
|
210
|
+
+ repo conventions, tool discipline, fail-fast rules
|
|
211
|
+
|
|
212
|
+
TOOLS (3,200 tokens — CACHED after turn 1)
|
|
213
|
+
filesystem__read_text_file { path: string, head?: number, tail?: number }
|
|
214
|
+
filesystem__write_file { path: string, content: string }
|
|
215
|
+
filesystem__edit_file { path: string, old_string: string, new_string: string }
|
|
216
|
+
filesystem__search_files { path: string, pattern: string }
|
|
217
|
+
filesystem__list_directory { path: string }
|
|
218
|
+
shell__shell_exec { command: string }
|
|
219
|
+
... (8 tools * ~400 tokens avg JSON Schema each)
|
|
220
|
+
|
|
221
|
+
USER MESSAGE (1,500 tokens — CACHED after turn 1)
|
|
222
|
+
"SANDBOX MODE — compiler smoke test. Work order compiled from..."
|
|
223
|
+
+ spec-types operations list
|
|
224
|
+
|
|
225
|
+
ASSISTANT (turn 1 — 120 tokens — NOT CACHED)
|
|
226
|
+
Tool call: filesystem__read_text_file(path="packages/dispatch/...")
|
|
227
|
+
|
|
228
|
+
TOOL RESULT (turn 1 — 22,000 tokens — NOT CACHED)
|
|
229
|
+
Full content of types.ts (660 lines)
|
|
230
|
+
|
|
231
|
+
ASSISTANT (turn 2 — 80 tokens — NOT CACHED)
|
|
232
|
+
Tool call: filesystem__create_directory(path="...")
|
|
233
|
+
|
|
234
|
+
TOOL RESULT (turn 2 — 15 tokens — NOT CACHED)
|
|
235
|
+
"Directory created"
|
|
236
|
+
|
|
237
|
+
... (grows each turn — everything below the USER MESSAGE changes)
|
|
238
|
+
|
|
239
|
+
ASSISTANT (turn N — this request)
|
|
240
|
+
"I will now write types.ts..."
|
|
241
|
+
```
|
|
242
|
+
|
|
243
|
+
Every turn after turn 1: ~3,500 system + 3,200 tools + 1,500 work order =
|
|
244
|
+
~8,200 tokens READ FROM CACHE (820 billed), plus the entire growing conversation
|
|
245
|
+
history PAID AT FULL PRICE.
|
|
246
|
+
|
|
247
|
+
For turn 15 with ~30 messages in history (~180k tokens of conversation +
|
|
248
|
+
tool results): 820 (cached) + 180,000 (uncached) = ~180,820 input tokens.
|
|
249
|
+
This matches the observed average of ~17,500/turn × 15 turns = 262,590.
|
|
250
|
+
|
|
251
|
+
**The tool schemas are not the dominant cost on turn 15 — the accumulated
|
|
252
|
+
conversation history is.** But the tool schemas ARE the dominant cost on
|
|
253
|
+
early turns, and they compound the problem by bloating the prefix that
|
|
254
|
+
rides along in every turn.
|