codex-dev-mcp-suite 1.2.0 → 1.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.env.example +19 -0
- package/CHANGELOG.md +64 -0
- package/README.md +179 -0
- package/bin/provider-smoke.mjs +278 -0
- package/bin/prune.mjs +158 -0
- package/bin/stats.mjs +101 -0
- package/lib/provider-smoke.js +234 -0
- package/lib/prune.js +114 -0
- package/lib/stats.js +269 -0
- package/package.json +9 -4
- package/project-memory/embedding.js +108 -30
- package/project-memory/server.js +46 -4
package/.env.example
CHANGED
|
@@ -40,6 +40,25 @@ MCP_RERANK_MODEL=llama3.1:8b
|
|
|
40
40
|
RERANK_TIMEOUT_MS=30000
|
|
41
41
|
RERANK_ENABLED=1
|
|
42
42
|
|
|
43
|
+
|
|
44
|
+
# === Cloudflare Workers AI (REST-only, NOT OpenAI-compatible) ===
|
|
45
|
+
# Free tier: 10,000 neurons/day. No laptop resource usage.
|
|
46
|
+
# To get started: dash.cloudflare.com → Workers & Pages → Create Worker → Workers AI → enable
|
|
47
|
+
# Account ID: from dashboard sidebar. API Token: My Profile → API Tokens → "Edit Cloudflare Workers".
|
|
48
|
+
# Note: base URL has trailing slash; model name goes in URL path, not body.
|
|
49
|
+
# Set CLOUDFLARE_EMBED_MODEL (e.g. @cf/baai/bge-base-en-v1.5) to enable semantic recall
|
|
50
|
+
# via Cloudflare instead of 9router/OpenAI/etc.
|
|
51
|
+
# MCP_EMBED_BASE_URL=https://api.cloudflare.com/client/v4/accounts/<ACCOUNT_ID>/ai/run/
|
|
52
|
+
# MCP_EMBED_API_KEY=<cf-token>
|
|
53
|
+
# MCP_EMBED_MODEL=@cf/baai/bge-base-en-v1.5
|
|
54
|
+
|
|
55
|
+
# === Google Gemini (OpenAI-compatible, generous free tier) ===
|
|
56
|
+
# Get key: https://aistudio.google.com/app/apikey
|
|
57
|
+
# Free tier: 60 req/min, 1500/day for text-embedding-004
|
|
58
|
+
# GEMINI_BASE_URL=https://generativelanguage.googleapis.com/v1beta/openai
|
|
59
|
+
# GEMINI_API_KEY=<gemini-key>
|
|
60
|
+
# GEMINI_MODEL=text-embedding-004
|
|
61
|
+
|
|
43
62
|
# Storage locations (defaults shown; override if you want)
|
|
44
63
|
# MEMORY_VAULT_DIR=~/.codex/memories/vault
|
|
45
64
|
# JOURNAL_DIR=~/.codex/memories/journal
|
package/CHANGELOG.md
CHANGED
|
@@ -1,5 +1,69 @@
|
|
|
1
1
|
# Changelog
|
|
2
2
|
|
|
3
|
+
## 1.4.0 - 2026-06-17
|
|
4
|
+
|
|
5
|
+
### Added
|
|
6
|
+
- **`provider-smoke` CLI** (`bin/provider-smoke.mjs`): probe every configured LLM provider (chat + embeddings) and print a matrix of latency / status / sample. Useful after adding a new API key or comparing providers.
|
|
7
|
+
- Usage: `npx -y -p codex-dev-mcp-suite provider-smoke` (or globally `provider-smoke`)
|
|
8
|
+
- Flags: `--json`, `--markdown`, `--save-md <path>`, `--chat-only`, `--embed-only`, `--env-file <path>`, `--only a,b,c`, `--timeout <ms>`, `--help`, `--version`
|
|
9
|
+
- Auto-detects providers from `MCP_PROVIDER_*` slots + named env (`GROQ_*`, `CEREBRAS_*`, `MISTRAL_*`, `OPENROUTER_*`, `OPENAI_*`, `OLLAMA_*`, `ANTHROPIC_*`, `GEMINI_*`, `COHERE_*`, `VOYAGE_*`, `CLOUDFLARE_*`) + `MCP_LLM_BASE_URL` / `NINEROUTER_URL` fallback.
|
|
10
|
+
- Pure-function library at `lib/provider-smoke.js` (18 offline tests).
|
|
11
|
+
- `KNOWN_PROVIDERS` registry covers 11 providers with `supportsChat` / `supportsEmbed` flags + `endpoint: "openai-compatible" | "cloudflare"`.
|
|
12
|
+
- Cloudflare embed-only models (`@cf/baai/*`, `@cf/google/embedding*`, etc.) are detected and the chat probe is skipped with a clear annotation.
|
|
13
|
+
- **Cloudflare Workers AI support** for embeddings: drop-in alternative when 9router/OpenAI are unavailable. Set `MCP_EMBED_BASE_URL=https://api.cloudflare.com/client/v4/accounts/<ACCOUNT_ID>/ai/run/`, `MCP_EMBED_API_KEY=<token>`, `MCP_EMBED_MODEL=@cf/baai/bge-base-en-v1.5`.
|
|
14
|
+
- Cloudflare's REST API (non-OpenAI-compatible, model-in-path) is auto-detected by URL pattern and dispatched to a separate code path in `project-memory/embedding.js`.
|
|
15
|
+
- New `isCloudflareURL()` + `httpPostJson()` helpers; `embedCloudflare()` handles `{text: [...]}` body and `{result: {data: [[...]]}}` response.
|
|
16
|
+
- `embeddingConfig().mode` and `recallMode()` helpers expose what retrieval mode is active (`semantic` / `keyword` / `deterministic`).
|
|
17
|
+
- **`memory_recall` mode arg** (default `"auto"`): explicit control over fallback behavior. `mode: "semantic"` requires embedding (returns `isError: true` if unavailable), `mode: "keyword"` skips embedding entirely.
|
|
18
|
+
- **Recall output annotation** now shows `[semantic+rerank]` / `[keyword+rerank]` when LLM rerank is active, making the active mode fully transparent.
|
|
19
|
+
- **Top-level test runner hook** (added in 1.3.0, expanded in 1.4.0): `tests/*.test.mjs` are auto-picked-up by `node run-tests.mjs`.
|
|
20
|
+
|
|
21
|
+
### Tests
|
|
22
|
+
- 85 tests pass (was 60 in v1.3.0): 26 project-memory + 6 checkpoint + 6 context-pack + 11 devjournal + 18 provider-smoke + 9 prune + 9 stats.
|
|
23
|
+
- New fallback tests: `mode=keyword` skips semantic, `mode=semantic` without embed key returns isError, default mode=auto, no `+rerank` suffix when rerank disabled.
|
|
24
|
+
|
|
25
|
+
### Docs
|
|
26
|
+
- `docs/providers.md` rewritten: 11-provider matrix, Cloudflare setup caveat, Ollama resource cost, "no API key" graceful degradation notes.
|
|
27
|
+
- `README.md` updated: "Memory Recall Modes" section, Cloudflare + Gemini setup examples, 4-mode annotation examples.
|
|
28
|
+
- `.env.example` updated: copy-pasteable config blocks for Cloudflare Workers AI and Google Gemini.
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
## 1.3.0 - 2026-06-17
|
|
33
|
+
|
|
34
|
+
### Added
|
|
35
|
+
- New `stats` CLI: summarize local memory storage across vault / journal / checkpoints. Shows totals, top projects by notes, most recent activity, and temp-slug cleanup candidates.
|
|
36
|
+
- Usage: `npx -y -p codex-dev-mcp-suite stats` (or globally `stats`)
|
|
37
|
+
- Flags: `--root <path>`, `--json`, `--top N`, `--help`, `--version`
|
|
38
|
+
- Pure-function library at `lib/stats.js` (no MCP / no stdio); tested offline.
|
|
39
|
+
- New top-level test runner hook: `tests/*.test.mjs` are auto-picked-up by `node run-tests.mjs`.
|
|
40
|
+
- New `provider-smoke` CLI: probe every configured LLM provider (chat + embeddings) and print a matrix of latency / status / sample. Useful after adding a new API key or to compare providers.
|
|
41
|
+
- Usage: `npx -y -p codex-dev-mcp-suite provider-smoke`
|
|
42
|
+
- Flags: `--json`, `--markdown`, `--save-md <path>`, `--chat-only`, `--embed-only`, `--env-file <path>`, `--timeout <ms>`
|
|
43
|
+
- Detects providers from `MCP_PROVIDER_*` slots + named env (`GROQ_*`, `CEREBRAS_*`, `MISTRAL_*`, `OPENROUTER_*`, `OPENAI_*`, `OLLAMA_*`, `ANTHROPIC_*`, `GEMINI_*`, `COHERE_*`) + `MCP_LLM_BASE_URL` / `NINEROUTER_URL` fallback.
|
|
44
|
+
- Pure-function library at `lib/provider-smoke.js`; tested offline.
|
|
45
|
+
- New docs page `docs/providers.md` — auto-generated smoke matrix from latest run.
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
### Added
|
|
53
|
+
- New `prune` CLI: remove temp project slugs (prefix `tmp.`) from vault, journal, and checkpoints. Default is DRY-RUN — nothing is deleted without `--yes`.
|
|
54
|
+
- Usage: `npx -y -p codex-dev-mcp-suite prune` (or globally `prune`)
|
|
55
|
+
- Flags: `--root <path>`, `--yes`, `--json`, `--help`, `--version`
|
|
56
|
+
- Pure-function library at `lib/prune.js`; tested offline.
|
|
57
|
+
- Safety: refuses to delete any slug that does not start with `tmp.`.
|
|
58
|
+
- New MCP tool `memory_stats` on the project-memory server: returns the same summary as the `stats` CLI. Useful for in-session introspection from an MCP client.
|
|
59
|
+
- Args: `root?`, `top?` (default 10), `json?` (default false).
|
|
60
|
+
|
|
61
|
+
|
|
62
|
+
|
|
63
|
+
### Added
|
|
64
|
+
- **Cloudflare Workers AI support** for embeddings and chat: drop-in alternative when 9router/OpenAI are unavailable. Set `MCP_EMBED_BASE_URL=https://api.cloudflare.com/client/v4/accounts/<ACCOUNT_ID>/ai/run/`, `MCP_EMBED_API_KEY=<token>`, `MCP_EMBED_MODEL=@cf/baai/bge-base-en-v1.5`. Cloudflare's REST API (non-OpenAI-compatible) is auto-detected by URL pattern and dispatched to a separate code path in `project-memory/embedding.js`. Includes `isCloudflareURL()` detection + `endpoint: "cloudflare"` field in provider matrix.
|
|
65
|
+
- `provider-smoke` CLI: probe Cloudflare alongside OpenAI-compatible providers. Cloudflare chat returns 400 for embed-only models (expected — they don't support chat).
|
|
66
|
+
|
|
3
67
|
## 1.2.0 - 2026-06-15
|
|
4
68
|
|
|
5
69
|
### Added
|
package/README.md
CHANGED
|
@@ -2,6 +2,9 @@
|
|
|
2
2
|
|
|
3
3
|
[](https://github.com/verrysimatupang99/codex-dev-mcp-suite/actions/workflows/ci.yml)
|
|
4
4
|
[](https://www.npmjs.com/package/codex-dev-mcp-suite)
|
|
5
|
+
[](https://www.npmjs.com/package/codex-dev-mcp-suite)
|
|
6
|
+
[](./LICENSE)
|
|
7
|
+
[](https://nodejs.org)
|
|
5
8
|
|
|
6
9
|
Published as `codex-dev-mcp-suite` for backward compatibility.
|
|
7
10
|
|
|
@@ -14,6 +17,51 @@ per-project** by the working directory. Works with any MCP-capable client
|
|
|
14
17
|
(Codex CLI, Claude Code, Cursor, Cline, Gemini-compatible launchers, Hermes,
|
|
15
18
|
or any stdio MCP host).
|
|
16
19
|
|
|
20
|
+
## Quickstart
|
|
21
|
+
|
|
22
|
+
Try it in 10 seconds (no clone, no config):
|
|
23
|
+
|
|
24
|
+
```bash
|
|
25
|
+
npx -y -p codex-dev-mcp-suite project-memory-mcp --help
|
|
26
|
+
```
|
|
27
|
+
|
|
28
|
+
Local-first by default: no hosted backend, no telemetry. Works offline with
|
|
29
|
+
keyword recall; model/provider config is optional. For strict no-network mode,
|
|
30
|
+
set `MCP_DETERMINISTIC_FALLBACK=true`.
|
|
31
|
+
|
|
32
|
+
Register all four servers with an MCP client (Codex CLI shown):
|
|
33
|
+
|
|
34
|
+
```toml
|
|
35
|
+
# ~/.codex/config.toml
|
|
36
|
+
[mcp_servers.project-memory]
|
|
37
|
+
command = "npx"
|
|
38
|
+
args = ["-y", "-p", "codex-dev-mcp-suite", "project-memory-mcp"]
|
|
39
|
+
|
|
40
|
+
[mcp_servers.devjournal]
|
|
41
|
+
command = "npx"
|
|
42
|
+
args = ["-y", "-p", "codex-dev-mcp-suite", "devjournal-mcp"]
|
|
43
|
+
|
|
44
|
+
[mcp_servers.checkpoint]
|
|
45
|
+
command = "npx"
|
|
46
|
+
args = ["-y", "-p", "codex-dev-mcp-suite", "checkpoint-mcp"]
|
|
47
|
+
|
|
48
|
+
[mcp_servers.context-pack]
|
|
49
|
+
command = "npx"
|
|
50
|
+
args = ["-y", "-p", "codex-dev-mcp-suite", "context-pack-mcp"]
|
|
51
|
+
```
|
|
52
|
+
|
|
53
|
+
Inspect any server without starting it:
|
|
54
|
+
|
|
55
|
+
```bash
|
|
56
|
+
project-memory-mcp --version
|
|
57
|
+
project-memory-mcp --doctor # config diagnostics; API keys redacted
|
|
58
|
+
```
|
|
59
|
+
|
|
60
|
+
Other clients (Claude Code, Cursor, Cline, ...): see
|
|
61
|
+
[`docs/clients/`](docs/clients/). Full env reference:
|
|
62
|
+
[`docs/configuration.md`](docs/configuration.md). Data flow:
|
|
63
|
+
[`docs/privacy.md`](docs/privacy.md).
|
|
64
|
+
|
|
17
65
|
## The four servers
|
|
18
66
|
|
|
19
67
|
| Server | What it does | Key tools |
|
|
@@ -120,6 +168,30 @@ idea — use the npm bin commands (`project-memory-mcp`, `devjournal-mcp`,
|
|
|
120
168
|
`checkpoint-mcp`, `context-pack-mcp`) or `node /abs/path/<server>/server.js`,
|
|
121
169
|
with optional `env`.
|
|
122
170
|
|
|
171
|
+
## Memory Recall Modes
|
|
172
|
+
|
|
173
|
+
`memory_recall` supports a `mode` arg (default `"auto"`) that controls fallback behavior:
|
|
174
|
+
|
|
175
|
+
| Mode | Behavior |
|
|
176
|
+
|---|---|
|
|
177
|
+
| `auto` (default) | Smart: try semantic → keyword → LLM rerank |
|
|
178
|
+
| `semantic` | Require embedding. Returns `isError: true` if no embed key configured |
|
|
179
|
+
| `keyword` | Skip embedding entirely (faster, pure keyword scoring) |
|
|
180
|
+
|
|
181
|
+
The recall output annotates the active mode for transparency:
|
|
182
|
+
|
|
183
|
+
```
|
|
184
|
+
Recall for "how do users sign in" in my-project [semantic+rerank]:
|
|
185
|
+
### JWT login flow (id:..., sim:0.603, ...)
|
|
186
|
+
```
|
|
187
|
+
|
|
188
|
+
The `+rerank` suffix is appended when LLM rerank is active (MCP_RERANK_ENABLED + key).
|
|
189
|
+
Without embeddings, recall falls back to `[keyword]` or `[keyword+rerank]`.
|
|
190
|
+
With `MCP_DETERMINISTIC_FALLBACK=true`, the mode is always `[deterministic]`.
|
|
191
|
+
|
|
192
|
+
This means you can run codex-dev-mcp-suite with **zero API keys configured** —
|
|
193
|
+
`memory_recall` still works via keyword scoring, just no semantic similarity.
|
|
194
|
+
|
|
123
195
|
## Daily workflow
|
|
124
196
|
|
|
125
197
|
- Session start: `pack_overview` + `journal_resume` + `memory_recall "<topic>"`
|
|
@@ -138,6 +210,113 @@ node backfill-sessions-v2.mjs --dry --min-prompts 2 # preview
|
|
|
138
210
|
node backfill-sessions-v2.mjs --min-prompts 2 # import
|
|
139
211
|
```
|
|
140
212
|
|
|
213
|
+
## Stats CLI
|
|
214
|
+
|
|
215
|
+
A read-only summary of your local memory storage — totals, top projects,
|
|
216
|
+
recent activity, and temp-slug cleanup candidates.
|
|
217
|
+
|
|
218
|
+
```bash
|
|
219
|
+
npx -y -p codex-dev-mcp-suite stats # human-readable
|
|
220
|
+
npx -y -p codex-dev-mcp-suite stats --json # machine-readable
|
|
221
|
+
npx -y -p codex-dev-mcp-suite stats --root /tmp/mem # different root
|
|
222
|
+
npx -y -p codex-dev-mcp-suite stats --top 5 # trim top lists
|
|
223
|
+
```
|
|
224
|
+
|
|
225
|
+
Example output:
|
|
226
|
+
|
|
227
|
+
```
|
|
228
|
+
Dev MCP Suite — stats
|
|
229
|
+
======================
|
|
230
|
+
Storage root: /home/you/.codex/memories
|
|
231
|
+
|
|
232
|
+
Totals
|
|
233
|
+
------
|
|
234
|
+
Notes: 89
|
|
235
|
+
Journal projects:19
|
|
236
|
+
Checkpoints: 1
|
|
237
|
+
Distinct projects: 25
|
|
238
|
+
|
|
239
|
+
Top projects by notes (top 10)
|
|
240
|
+
------------------------
|
|
241
|
+
24 mrtrickster99-fd1ff0fa
|
|
242
|
+
14 Coding-17e063ef
|
|
243
|
+
...
|
|
244
|
+
|
|
245
|
+
Temp/cleanup candidates (2)
|
|
246
|
+
------------------------
|
|
247
|
+
tmp.itbtDn9eB3-b62798fd
|
|
248
|
+
tmp.iHqM2Uh5K2-8d3660a2
|
|
249
|
+
```
|
|
250
|
+
|
|
251
|
+
The CLI is a thin wrapper around `lib/stats.js`, which is also importable
|
|
252
|
+
directly from your own scripts. The default storage root is
|
|
253
|
+
`~/.codex/memories`, but `--root` (or the existing `MEMORY_VAULT_DIR` /
|
|
254
|
+
`JOURNAL_DIR` / `CHECKPOINT_DIR` env vars) override it.
|
|
255
|
+
|
|
256
|
+
|
|
257
|
+
## Provider Smoke CLI
|
|
258
|
+
|
|
259
|
+
Verify every configured LLM provider (chat + embeddings) with one command.
|
|
260
|
+
Useful after adding a new API key, or when comparing provider latency.
|
|
261
|
+
|
|
262
|
+
```bash
|
|
263
|
+
npx -y -p codex-dev-mcp-suite provider-smoke # uses process.env
|
|
264
|
+
npx -y -p codex-dev-mcp-suite provider-smoke --env-file ~/secrets.env # separate secrets file
|
|
265
|
+
npx -y -p codex-dev-mcp-suite provider-smoke --markdown --save-md docs/providers.md
|
|
266
|
+
```
|
|
267
|
+
|
|
268
|
+
Auto-detects providers from these env vars (highest precedence first):
|
|
269
|
+
|
|
270
|
+
- **Numbered slots**: `MCP_PROVIDER_PRIMARY` / `_CHAIN2` / `_CHAIN3` / ...
|
|
271
|
+
- **Named env**: `GROQ_*`, `CEREBRAS_*`, `MISTRAL_*`, `OPENROUTER_*`, `OPENAI_*`, `GEMINI_*`, `COHERE_*`, `VOYAGE_*`, `OLLAMA_*`, `ANTHROPIC_*`
|
|
272
|
+
- **Catch-all**: `MCP_LLM_BASE_URL` / `MCP_RERANK_BASE_URL` / `MCP_EMBED_BASE_URL` / `NINEROUTER_URL` / `LLM_BASE_URL` (any OpenAI-compatible endpoint)
|
|
273
|
+
|
|
274
|
+
For each detected provider, runs:
|
|
275
|
+
- **chat probe** (`/v1/chat/completions`) — if the provider supports it
|
|
276
|
+
- **embed probe** (`/v1/embeddings`) — if the provider supports it
|
|
277
|
+
|
|
278
|
+
**Inference-only providers** (Groq, Cerebras, Anthropic) skip the embed probe automatically.
|
|
279
|
+
**Non-OpenAI-compatible providers** (Cloudflare Workers AI) require a custom code path in
|
|
280
|
+
`project-memory/embedding.js` — see [docs/providers.md](docs/providers.md).
|
|
281
|
+
|
|
282
|
+
**No provider assumption is baked in**: the tool only probes what's in your env. Zero-config users
|
|
283
|
+
get an empty matrix; pass `--env-file` to point at a secrets file.
|
|
284
|
+
|
|
285
|
+
Chat-only providers (Groq, Cerebras) skip the embeddings probe automatically
|
|
286
|
+
— they don't expose `/v1/embeddings`. Embedding-capable providers (Mistral,
|
|
287
|
+
OpenRouter, OpenAI, Ollama, Gemini, Cohere) run both probes.
|
|
288
|
+
|
|
289
|
+
Example output (Groq + Cerebras, both configured):
|
|
290
|
+
|
|
291
|
+
```
|
|
292
|
+
[groq]
|
|
293
|
+
✓ chat 200 310ms "OK"
|
|
294
|
+
|
|
295
|
+
[cerebras]
|
|
296
|
+
✓ chat 200 476ms
|
|
297
|
+
```
|
|
298
|
+
|
|
299
|
+
See [`docs/providers.md`](docs/providers.md) for a saved smoke matrix.
|
|
300
|
+
|
|
301
|
+
## Prune CLI
|
|
302
|
+
|
|
303
|
+
Remove temp project slugs (prefix `tmp.`) from vault / journal / checkpoints.
|
|
304
|
+
Default is DRY-RUN — nothing is deleted without `--yes`.
|
|
305
|
+
|
|
306
|
+
```bash
|
|
307
|
+
npx -y -p codex-dev-mcp-suite prune # dry-run report
|
|
308
|
+
npx -y -p codex-dev-mcp-suite prune --yes # actually delete
|
|
309
|
+
npx -y -p codex-dev-mcp-suite prune --json # machine-readable
|
|
310
|
+
npx -y -p codex-dev-mcp-suite prune --root /tmp/mem # different root
|
|
311
|
+
```
|
|
312
|
+
|
|
313
|
+
Safety: refuses to delete any slug that does not start with `tmp.`. Useful
|
|
314
|
+
right after heavy refactors or after imports that left tmp scratch dirs.
|
|
315
|
+
|
|
316
|
+
For in-session use from an MCP client, the project-memory server exposes a
|
|
317
|
+
`memory_stats` tool that returns the same summary text (or JSON) as the `stats`
|
|
318
|
+
CLI above.
|
|
319
|
+
|
|
141
320
|
## Tests
|
|
142
321
|
|
|
143
322
|
```bash
|
|
@@ -0,0 +1,278 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
/**
|
|
3
|
+
* bin/provider-smoke.mjs — Dev MCP Suite provider smoke test.
|
|
4
|
+
*
|
|
5
|
+
* Probes every provider that can be derived from the current env:
|
|
6
|
+
* - MCP_LLM_BASE_URL / NINEROUTER_URL / LLM_BASE_URL (9router-like)
|
|
7
|
+
* - MCP_PROVIDER_PRIMARY / _CHAIN2 / _CHAIN3 / ... (numbered slots)
|
|
8
|
+
* - Named: GROQ_*, CEREBRAS_*, MISTRAL_*, OPENROUTER_*, OPENAI_*, OLLAMA_*, ...
|
|
9
|
+
*
|
|
10
|
+
* For each provider, runs:
|
|
11
|
+
* - chat probe (/v1/chat/completions, short prompt) if supportsChat
|
|
12
|
+
* - embed probe (/v1/embeddings, single string) if supportsEmbed
|
|
13
|
+
*
|
|
14
|
+
* Output: human text by default; --json for machines; --markdown for docs.
|
|
15
|
+
*
|
|
16
|
+
* Env override:
|
|
17
|
+
* --env-file <path> Source env from this file before probing (e.g. /tmp/secrets.env)
|
|
18
|
+
*
|
|
19
|
+
* Exit code:
|
|
20
|
+
* 0 = all probes ok
|
|
21
|
+
* 1 = at least one probe failed
|
|
22
|
+
*/
|
|
23
|
+
import fs from "fs";
|
|
24
|
+
import os from "os";
|
|
25
|
+
import path from "path";
|
|
26
|
+
import { fileURLToPath } from "url";
|
|
27
|
+
import { buildProviderMatrix, shapeProbe, formatText, formatJson, formatMarkdown, isCloudflareURL } from "../lib/provider-smoke.js";
|
|
28
|
+
|
|
29
|
+
const __dirname = path.dirname(fileURLToPath(import.meta.url));
|
|
30
|
+
|
|
31
|
+
function pkgVersion() {
|
|
32
|
+
try {
|
|
33
|
+
return JSON.parse(fs.readFileSync(path.join(__dirname, "..", "package.json"), "utf8")).version || "unknown";
|
|
34
|
+
} catch { return "unknown"; }
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
/** Strip a trailing /v1 (only the literal segment, not /openai/v1) so we can re-append /v1/<endpoint>. */
|
|
39
|
+
function stripV1(baseUrl) {
|
|
40
|
+
return String(baseUrl || "").replace(/\/v1\/?$/, "").replace(/\/$/, "");
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
function parseArgs(argv) {
|
|
44
|
+
const opts = {
|
|
45
|
+
help: false, version: false,
|
|
46
|
+
json: false, markdown: false,
|
|
47
|
+
saveMd: null, envFile: null,
|
|
48
|
+
chatOnly: false, embedOnly: false,
|
|
49
|
+
timeoutMs: 15000,
|
|
50
|
+
only: null,
|
|
51
|
+
};
|
|
52
|
+
for (let i = 0; i < argv.length; i++) {
|
|
53
|
+
const a = argv[i];
|
|
54
|
+
if (a === "-h" || a === "--help") opts.help = true;
|
|
55
|
+
else if (a === "-v" || a === "--version") opts.version = true;
|
|
56
|
+
else if (a === "--json") opts.json = true;
|
|
57
|
+
else if (a === "--markdown") opts.markdown = true;
|
|
58
|
+
else if (a === "--save-md") opts.saveMd = argv[++i];
|
|
59
|
+
else if (a === "--env-file") opts.envFile = argv[++i];
|
|
60
|
+
else if (a === "--chat-only") opts.chatOnly = true;
|
|
61
|
+
else if (a === "--embed-only") opts.embedOnly = true;
|
|
62
|
+
else if (a === "--timeout") opts.timeoutMs = parseInt(argv[++i], 10) || 15000;
|
|
63
|
+
else if (a === "--only") opts.only = (argv[++i] || "").split(",").map(s => s.trim()).filter(Boolean);
|
|
64
|
+
else { process.stderr.write(`provider-smoke: unknown flag: ${a}\n`); process.exit(2); }
|
|
65
|
+
}
|
|
66
|
+
return opts;
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
function printHelp() {
|
|
70
|
+
const out = [
|
|
71
|
+
`provider-smoke (Dev MCP Suite) v${pkgVersion()}`,
|
|
72
|
+
"",
|
|
73
|
+
"Probe every configured LLM provider (chat + embeddings) and print a matrix.",
|
|
74
|
+
"",
|
|
75
|
+
"Usage:",
|
|
76
|
+
" provider-smoke [--json] [--markdown] [--save-md <path>]",
|
|
77
|
+
" [--chat-only | --embed-only] [--env-file <path>]",
|
|
78
|
+
" provider-smoke --help | --version",
|
|
79
|
+
"",
|
|
80
|
+
"Options:",
|
|
81
|
+
" --json Machine-readable JSON output",
|
|
82
|
+
" --markdown Markdown table (for docs/)",
|
|
83
|
+
" --save-md <path> Write markdown report to this file",
|
|
84
|
+
" --env-file <path> Source env vars from this file before probing",
|
|
85
|
+
" (KEY=value lines, one per line)",
|
|
86
|
+
" --chat-only Only probe /v1/chat/completions",
|
|
87
|
+
" --embed-only Only probe /v1/embeddings",
|
|
88
|
+
" --timeout <ms> Per-probe timeout (default 15000)",
|
|
89
|
+
" --only a,b,c Only probe these provider ids (comma-separated); others ignored",
|
|
90
|
+
"",
|
|
91
|
+
"Detected providers (precedence):",
|
|
92
|
+
" MCP_PROVIDER_PRIMARY / _CHAIN2 / _CHAIN3 / ... (numbered slots)",
|
|
93
|
+
" GROQ_*, CEREBRAS_*, MISTRAL_*, OPENROUTER_*, (named)",
|
|
94
|
+
" OPENAI_*, OLLAMA_*, ANTHROPIC_*, GEMINI_*, COHERE_*",
|
|
95
|
+
" MCP_LLM_BASE_URL / NINEROUTER_URL / LLM_BASE_URL (9router / agentrouter)",
|
|
96
|
+
"",
|
|
97
|
+
"Examples:",
|
|
98
|
+
" provider-smoke",
|
|
99
|
+
" provider-smoke --env-file /tmp/codex-dev-smoke.env",
|
|
100
|
+
" provider-smoke --json | jq '.[] | select(.ok == false)'",
|
|
101
|
+
" provider-smoke --markdown --save-md docs/providers.md",
|
|
102
|
+
];
|
|
103
|
+
process.stdout.write(out.join("\n") + "\n");
|
|
104
|
+
}
|
|
105
|
+
|
|
106
|
+
function loadEnvFile(file) {
|
|
107
|
+
if (!file) return {};
|
|
108
|
+
const text = fs.readFileSync(file, "utf8");
|
|
109
|
+
const env = {};
|
|
110
|
+
for (const raw of text.split(/\r?\n/)) {
|
|
111
|
+
const line = raw.trim();
|
|
112
|
+
if (!line || line.startsWith("#")) continue;
|
|
113
|
+
const m = line.match(/^([A-Z_][A-Z0-9_]*)=(.*)$/);
|
|
114
|
+
if (m) env[m[1]] = m[2];
|
|
115
|
+
}
|
|
116
|
+
return env;
|
|
117
|
+
}
|
|
118
|
+
|
|
119
|
+
function buildHeaders(provider) {
|
|
120
|
+
const h = { "Content-Type": "application/json" };
|
|
121
|
+
if (provider.apiKey) h["Authorization"] = `Bearer ${provider.apiKey}`;
|
|
122
|
+
return h;
|
|
123
|
+
}
|
|
124
|
+
|
|
125
|
+
|
|
126
|
+
const CF_EMBED_ONLY_PREFIXES = ["@cf/baai/", "@cf/google/embedding", "@cf/baai/sentence-"];
|
|
127
|
+
|
|
128
|
+
function isEmbedOnlyModel(model) {
|
|
129
|
+
if (!model) return false;
|
|
130
|
+
return CF_EMBED_ONLY_PREFIXES.some((p) => model.startsWith(p));
|
|
131
|
+
}
|
|
132
|
+
|
|
133
|
+
async function probeCloudflare(provider, timeoutMs, kind) {
|
|
134
|
+
// Skip chat probe for known embed-only Cloudflare model prefixes (annotate, not error).
|
|
135
|
+
if (kind === "chat" && isEmbedOnlyModel(provider.chatModel || provider.embedModel)) {
|
|
136
|
+
return shapeProbe({ name: provider.id, kind: "chat", ok: false, error: "embed-only model (chat not supported)" });
|
|
137
|
+
}
|
|
138
|
+
// Cloudflare Workers AI REST: POST {baseUrl}/{model}, body shape depends on kind.
|
|
139
|
+
// Cloudflare model names contain "/" (e.g. "@cf/baai/bge-base-en-v1.5"); do NOT URL-encode.
|
|
140
|
+
const url = `${provider.baseUrl.replace(/\/$/, "")}/${provider.chatModel || provider.embedModel}`;
|
|
141
|
+
let body;
|
|
142
|
+
if (kind === "chat") {
|
|
143
|
+
body = JSON.stringify({
|
|
144
|
+
messages: [{ role: "user", content: "Say the single word: OK" }],
|
|
145
|
+
max_tokens: 8,
|
|
146
|
+
});
|
|
147
|
+
} else {
|
|
148
|
+
body = JSON.stringify({ text: ["ping"] });
|
|
149
|
+
}
|
|
150
|
+
const t0 = Date.now();
|
|
151
|
+
try {
|
|
152
|
+
const res = await fetch(url, { method: "POST", headers: buildHeaders(provider), body, signal: AbortSignal.timeout(timeoutMs) });
|
|
153
|
+
const text = await res.text();
|
|
154
|
+
const latencyMs = Date.now() - t0;
|
|
155
|
+
if (res.status !== 200) {
|
|
156
|
+
return shapeProbe({ name: provider.id, kind, ok: false, status: res.status, latencyMs, error: text.slice(0, 200) });
|
|
157
|
+
}
|
|
158
|
+
let sample = null, dim = null;
|
|
159
|
+
try {
|
|
160
|
+
const j = JSON.parse(text);
|
|
161
|
+
if (kind === "embed") {
|
|
162
|
+
const data = j?.result?.data;
|
|
163
|
+
if (Array.isArray(data) && Array.isArray(data[0])) dim = data[0].length;
|
|
164
|
+
} else {
|
|
165
|
+
sample = j?.result?.response ?? null;
|
|
166
|
+
}
|
|
167
|
+
} catch { /* ignore */ }
|
|
168
|
+
return shapeProbe({ name: provider.id, kind, ok: true, status: res.status, latencyMs, sample, dim });
|
|
169
|
+
} catch (e) {
|
|
170
|
+
return shapeProbe({ name: provider.id, kind, ok: false, latencyMs: Date.now() - t0, error: String(e && e.message || e) });
|
|
171
|
+
}
|
|
172
|
+
}
|
|
173
|
+
|
|
174
|
+
async function probeChat(provider, timeoutMs) {
|
|
175
|
+
if (!provider.baseUrl || !provider.chatModel) {
|
|
176
|
+
return shapeProbe({ name: provider.id, kind: "chat", ok: false, error: "missing baseUrl or chatModel" });
|
|
177
|
+
}
|
|
178
|
+
const endpoint = provider.endpoint || (isCloudflareURL(provider.baseUrl) ? "cloudflare" : "openai-compatible");
|
|
179
|
+
if (endpoint === "cloudflare") return probeCloudflare(provider, timeoutMs, "chat");
|
|
180
|
+
const url = `${stripV1(provider.baseUrl)}/v1/chat/completions`;
|
|
181
|
+
const body = JSON.stringify({
|
|
182
|
+
model: provider.chatModel,
|
|
183
|
+
messages: [{ role: "user", content: "Say the single word: OK" }],
|
|
184
|
+
max_tokens: 8,
|
|
185
|
+
temperature: 0,
|
|
186
|
+
});
|
|
187
|
+
const t0 = Date.now();
|
|
188
|
+
try {
|
|
189
|
+
const res = await fetch(url, { method: "POST", headers: buildHeaders(provider), body, signal: AbortSignal.timeout(timeoutMs) });
|
|
190
|
+
const text = await res.text();
|
|
191
|
+
const latencyMs = Date.now() - t0;
|
|
192
|
+
if (res.status !== 200) {
|
|
193
|
+
return shapeProbe({ name: provider.id, kind: "chat", ok: false, status: res.status, latencyMs, error: text.slice(0, 200) });
|
|
194
|
+
}
|
|
195
|
+
let sample = null;
|
|
196
|
+
try {
|
|
197
|
+
const j = JSON.parse(text);
|
|
198
|
+
sample = j?.choices?.[0]?.message?.content ?? null;
|
|
199
|
+
} catch { /* ignore */ }
|
|
200
|
+
return shapeProbe({ name: provider.id, kind: "chat", ok: true, status: res.status, latencyMs, sample });
|
|
201
|
+
} catch (e) {
|
|
202
|
+
return shapeProbe({ name: provider.id, kind: "chat", ok: false, latencyMs: Date.now() - t0, error: String(e && e.message || e) });
|
|
203
|
+
}
|
|
204
|
+
}
|
|
205
|
+
|
|
206
|
+
async function probeEmbed(provider, timeoutMs) {
|
|
207
|
+
if (!provider.supportsEmbed) {
|
|
208
|
+
return shapeProbe({ name: provider.id, kind: "embed", ok: false, error: "provider has no /v1/embeddings endpoint" });
|
|
209
|
+
}
|
|
210
|
+
if (!provider.baseUrl || !provider.embedModel) {
|
|
211
|
+
return shapeProbe({ name: provider.id, kind: "embed", ok: false, error: "missing baseUrl or embedModel" });
|
|
212
|
+
}
|
|
213
|
+
if (provider.endpoint === "cloudflare" || isCloudflareURL(provider.baseUrl)) return probeCloudflare(provider, timeoutMs, "embed");
|
|
214
|
+
const url = `${stripV1(provider.baseUrl)}/v1/embeddings`;
|
|
215
|
+
const body = JSON.stringify({ model: provider.embedModel, input: "ping" });
|
|
216
|
+
const t0 = Date.now();
|
|
217
|
+
try {
|
|
218
|
+
const res = await fetch(url, { method: "POST", headers: buildHeaders(provider), body, signal: AbortSignal.timeout(timeoutMs) });
|
|
219
|
+
const text = await res.text();
|
|
220
|
+
const latencyMs = Date.now() - t0;
|
|
221
|
+
if (res.status !== 200) {
|
|
222
|
+
return shapeProbe({ name: provider.id, kind: "embed", ok: false, status: res.status, latencyMs, error: text.slice(0, 200) });
|
|
223
|
+
}
|
|
224
|
+
let dim = null;
|
|
225
|
+
try {
|
|
226
|
+
const j = JSON.parse(text);
|
|
227
|
+
dim = Array.isArray(j?.data?.[0]?.embedding) ? j.data[0].embedding.length : null;
|
|
228
|
+
} catch { /* ignore */ }
|
|
229
|
+
return shapeProbe({ name: provider.id, kind: "embed", ok: true, status: res.status, latencyMs, dim });
|
|
230
|
+
} catch (e) {
|
|
231
|
+
return shapeProbe({ name: provider.id, kind: "embed", ok: false, latencyMs: Date.now() - t0, error: String(e && e.message || e) });
|
|
232
|
+
}
|
|
233
|
+
}
|
|
234
|
+
|
|
235
|
+
const opts = parseArgs(process.argv.slice(2));
|
|
236
|
+
if (opts.version) { process.stdout.write(`${pkgVersion()}\n`); process.exit(0); }
|
|
237
|
+
if (opts.help) { printHelp(); process.exit(0); }
|
|
238
|
+
|
|
239
|
+
// Merge env: process.env + (optional) env file
|
|
240
|
+
const fileEnv = loadEnvFile(opts.envFile);
|
|
241
|
+
const env = { ...process.env, ...fileEnv };
|
|
242
|
+
|
|
243
|
+
let matrix = buildProviderMatrix(env);
|
|
244
|
+
if (opts.only && opts.only.length) {
|
|
245
|
+
matrix = matrix.filter(p => opts.only.includes(p.id));
|
|
246
|
+
}
|
|
247
|
+
if (matrix.length === 0) {
|
|
248
|
+
const hint = opts.only && opts.only.length ? " (--only filter: " + opts.only.join(",") + ")" : "";
|
|
249
|
+
process.stderr.write("No providers detected. Set MCP_PROVIDER_PRIMARY / GROQ_API_KEY / NINEROUTER_URL etc., or pass --env-file." + hint + "\n");
|
|
250
|
+
process.exit(2);
|
|
251
|
+
}
|
|
252
|
+
|
|
253
|
+
process.stderr.write(`Probing ${matrix.length} provider${matrix.length === 1 ? "" : "s"}...\n`);
|
|
254
|
+
const results = [];
|
|
255
|
+
for (const p of matrix) {
|
|
256
|
+
if (!opts.embedOnly && p.supportsChat) {
|
|
257
|
+
results.push(await probeChat(p, opts.timeoutMs));
|
|
258
|
+
}
|
|
259
|
+
if (!opts.chatOnly && p.supportsEmbed) {
|
|
260
|
+
results.push(await probeEmbed(p, opts.timeoutMs));
|
|
261
|
+
}
|
|
262
|
+
}
|
|
263
|
+
|
|
264
|
+
let out;
|
|
265
|
+
if (opts.markdown) out = formatMarkdown(results);
|
|
266
|
+
else if (opts.json) out = formatJson(results);
|
|
267
|
+
else out = formatText(results);
|
|
268
|
+
|
|
269
|
+
process.stdout.write(out + "\n");
|
|
270
|
+
|
|
271
|
+
if (opts.saveMd) {
|
|
272
|
+
fs.mkdirSync(path.dirname(opts.saveMd), { recursive: true });
|
|
273
|
+
fs.writeFileSync(opts.saveMd, formatMarkdown(results));
|
|
274
|
+
process.stderr.write(`Markdown saved to ${opts.saveMd}\n`);
|
|
275
|
+
}
|
|
276
|
+
|
|
277
|
+
const failed = results.filter((r) => !r.ok).length;
|
|
278
|
+
process.exit(failed === 0 ? 0 : 1);
|