auto-model-router 0.13.1 → 0.15.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.omp-plugin/marketplace.json +2 -2
- package/README.md +12 -2
- package/package.json +1 -1
- package/src/cli/connect.ts +72 -0
- package/src/cli/refresh.ts +3 -0
- package/src/cost/ledger.ts +2 -2
- package/src/cost/views.ts +64 -0
- package/src/index.ts +1 -1
- package/src/lib.ts +1 -1
- package/src/server/http.ts +16 -5
- package/test/mcp-entry.test.ts +157 -0
- package/test/views.test.ts +31 -0
|
@@ -7,14 +7,14 @@
|
|
|
7
7
|
},
|
|
8
8
|
"metadata": {
|
|
9
9
|
"description": "auto-model-router: a local cost/complexity-aware model router for Oh My Pi, backed by OpenRouter",
|
|
10
|
-
"version": "0.
|
|
10
|
+
"version": "0.15.0",
|
|
11
11
|
"pluginRoot": "."
|
|
12
12
|
},
|
|
13
13
|
"plugins": [
|
|
14
14
|
{
|
|
15
15
|
"name": "auto-model-router",
|
|
16
16
|
"description": "Local cost/complexity-aware model router for Oh My Pi, backed by OpenRouter. Runs in-process, routes per turn by price and task complexity, with budget caps, mid-stream escalation, and cache-aware hysteresis.",
|
|
17
|
-
"version": "0.
|
|
17
|
+
"version": "0.15.0",
|
|
18
18
|
"author": {
|
|
19
19
|
"name": "drewappling",
|
|
20
20
|
"email": "drewappling@gmail.com"
|
package/README.md
CHANGED
|
@@ -710,8 +710,12 @@ escalation signal, error. Three views aggregate it, all from the same
|
|
|
710
710
|
and model (dispatches, tokens, spend, escalations, errors) as CSV. Also
|
|
711
711
|
`GET /v1/router/export?days=&harness=[&format=json]`; `GET /v1/router/spend?sinceMs=&harness=`
|
|
712
712
|
gives spend over a harness set since an instant, and `GET /v1/router/feedback?days=&harness=`
|
|
713
|
-
lists verdicts by model and the recent ones with the harness that gave them
|
|
714
|
-
|
|
713
|
+
lists verdicts by model and the recent ones with the harness that gave them, and
|
|
714
|
+
`GET /v1/router/decisions?harness=&days=|since=&slug=&tier=&limit=` is the decision trail
|
|
715
|
+
itself, newest first, each turn with its reasons, the classifier's view, forecast against
|
|
716
|
+
bill, escalation signal and verdicts (`?session=` narrows to one omp session, as `/router
|
|
717
|
+
why` does). These are what a front door such as the team edition reads instead of the
|
|
718
|
+
ledger file.
|
|
715
719
|
- `GET /v1/router/report?days=7&harness=<id>` for dashboards (`harness` may be
|
|
716
720
|
a comma-separated set of ids, for a group).
|
|
717
721
|
- `GET /v1/router/summary?harness=<id>` — the daily summary as JSON (`auto=1`
|
|
@@ -1365,6 +1369,12 @@ skills: Claude Code's `~/.claude/skills/<name>/` and omp's `~/.omp/agent/skills/
|
|
|
1365
1369
|
the bundle no longer carries, and a skill of the same name the member wrote themselves is
|
|
1366
1370
|
left alone with a note. A remote without skills answers 404 and nothing happens.
|
|
1367
1371
|
|
|
1372
|
+
A remote whose `/setup/info` says `mcp: true` (a team edition serving shared context) also
|
|
1373
|
+
gets a `team-context` MCP server written for omp (`~/.omp/agent/mcp.json`) and Claude Code
|
|
1374
|
+
(`~/.claude.json`): `type: http`, the remote's `/mcp`, the member key in the Authorization
|
|
1375
|
+
header. Every refresh rewrites it with the current key, like models.yml; other servers in
|
|
1376
|
+
those files are untouched, and a remote that stops serving MCP has the entry removed.
|
|
1377
|
+
|
|
1368
1378
|
## Direct upstreams: OpenAI, Azure OpenAI, Anthropic, vLLM
|
|
1369
1379
|
|
|
1370
1380
|
OpenRouter and Ollama Cloud are the built-in upstreams. `upstreams:` adds named ones the
|
package/package.json
CHANGED
package/src/cli/connect.ts
CHANGED
|
@@ -68,6 +68,12 @@ export interface ConnectOptions {
|
|
|
68
68
|
exePath?: string;
|
|
69
69
|
/** The remote's skills bundle, installed into every configured harness that reads user-level skills. */
|
|
70
70
|
skills?: SkillsBundle;
|
|
71
|
+
/**
|
|
72
|
+
* The remote's MCP endpoint (a team edition serving shared context): written as the
|
|
73
|
+
* `team-context` server for omp and Claude Code with the member key, rewritten on every
|
|
74
|
+
* refresh like models.yml. `null` removes an entry a previous connect wrote.
|
|
75
|
+
*/
|
|
76
|
+
mcp?: { url: string | null };
|
|
71
77
|
platform: string;
|
|
72
78
|
pathHas: (bin: string) => boolean;
|
|
73
79
|
}
|
|
@@ -80,6 +86,39 @@ export interface ConnectReport {
|
|
|
80
86
|
notes: string[];
|
|
81
87
|
/** What the remote's skills bundle did, when there was one. */
|
|
82
88
|
skills?: SkillsInstallReport;
|
|
89
|
+
/** Files that gained (or lost) the team-context MCP server. */
|
|
90
|
+
mcp?: string[];
|
|
91
|
+
}
|
|
92
|
+
|
|
93
|
+
/** The name of the MCP server connect manages in a harness's config. */
|
|
94
|
+
export const MCP_SERVER_NAME = "team-context";
|
|
95
|
+
|
|
96
|
+
/**
|
|
97
|
+
* Merges the team-context server into an `mcpServers` JSON file (omp's mcp.json, Claude
|
|
98
|
+
* Code's ~/.claude.json), leaving every other key and server alone; `url` null removes it.
|
|
99
|
+
* Returns the new text, or null when nothing changes.
|
|
100
|
+
*/
|
|
101
|
+
export function mergeMcpServers(before: string, url: string | null, key: string): string | null {
|
|
102
|
+
let root: Record<string, unknown> = {};
|
|
103
|
+
if (before.trim() !== "") {
|
|
104
|
+
try {
|
|
105
|
+
const parsed = JSON.parse(before) as unknown;
|
|
106
|
+
if (typeof parsed !== "object" || parsed === null || Array.isArray(parsed)) return null;
|
|
107
|
+
root = parsed as Record<string, unknown>;
|
|
108
|
+
} catch {
|
|
109
|
+
return null; // a file we cannot parse is not ours to rewrite
|
|
110
|
+
}
|
|
111
|
+
}
|
|
112
|
+
const servers = { ...((root.mcpServers as Record<string, unknown> | undefined) ?? {}) };
|
|
113
|
+
if (url === null) {
|
|
114
|
+
if (!(MCP_SERVER_NAME in servers)) return null;
|
|
115
|
+
delete servers[MCP_SERVER_NAME];
|
|
116
|
+
} else {
|
|
117
|
+
const next = { type: "http", url, headers: { Authorization: `Bearer ${key}` } };
|
|
118
|
+
if (JSON.stringify(servers[MCP_SERVER_NAME]) === JSON.stringify(next)) return null;
|
|
119
|
+
servers[MCP_SERVER_NAME] = next;
|
|
120
|
+
}
|
|
121
|
+
return `${JSON.stringify({ ...root, mcpServers: servers }, null, 2)}\n`;
|
|
83
122
|
}
|
|
84
123
|
|
|
85
124
|
const expand = (raw: string, home: string): string => (raw === "~" || raw.startsWith("~/") || raw.startsWith("~\\") ? join(home, raw.slice(1)) : raw);
|
|
@@ -376,6 +415,24 @@ export function connectRemote(o: ConnectOptions): ConnectReport {
|
|
|
376
415
|
for (const s of report.skills.skipped) report.notes.push(`skill ${s}`);
|
|
377
416
|
}
|
|
378
417
|
|
|
418
|
+
// 6c. The remote's MCP endpoint (shared context tools), for the harnesses configured above
|
|
419
|
+
// that read a user-level mcpServers file; the member key travels in the header and is
|
|
420
|
+
// rewritten with every refresh, like models.yml.
|
|
421
|
+
if (o.mcp !== undefined) {
|
|
422
|
+
const files: string[] = [];
|
|
423
|
+
if (report.configured.some((c) => c.startsWith("omp ("))) files.push(join(agentDir, "mcp.json"));
|
|
424
|
+
if (report.configured.some((c) => c.startsWith("Claude Code ("))) files.push(join(o.home, ".claude.json"));
|
|
425
|
+
report.mcp = [];
|
|
426
|
+
for (const path of files) {
|
|
427
|
+
const before = existsSync(path) ? readFileSync(path, "utf8") : "";
|
|
428
|
+
const after = mergeMcpServers(before, o.mcp.url, o.key);
|
|
429
|
+
if (after === null) continue;
|
|
430
|
+
write(path, after);
|
|
431
|
+
report.mcp.push(path);
|
|
432
|
+
}
|
|
433
|
+
if (o.mcp.url !== null && files.length > 0) report.configured.push(`MCP (${MCP_SERVER_NAME} → ${o.mcp.url}: ${files.map((f) => f.replaceAll("\\", "/")).join(", ")})`);
|
|
434
|
+
}
|
|
435
|
+
|
|
379
436
|
// 7. Persist the environment.
|
|
380
437
|
if (o.profile && !o.dryRun) {
|
|
381
438
|
if (o.platform === "win32") {
|
|
@@ -433,6 +490,18 @@ export async function exchangeSetupToken(url: string, token: string, device: str
|
|
|
433
490
|
};
|
|
434
491
|
}
|
|
435
492
|
|
|
493
|
+
/** What a team edition says about itself at /setup/info; a plain router answers nothing. */
|
|
494
|
+
export async function fetchSetupInfo(url: string, fetchImpl: typeof fetch = fetch): Promise<{ mcp: boolean }> {
|
|
495
|
+
try {
|
|
496
|
+
const res = await fetchImpl(`${url}/setup/info`, { signal: AbortSignal.timeout(10_000) });
|
|
497
|
+
if (!res.ok) return { mcp: false };
|
|
498
|
+
const body = (await res.json()) as { mcp?: unknown };
|
|
499
|
+
return { mcp: body.mcp === true };
|
|
500
|
+
} catch {
|
|
501
|
+
return { mcp: false };
|
|
502
|
+
}
|
|
503
|
+
}
|
|
504
|
+
|
|
436
505
|
/** Adds `dir` to the user's PATH on Windows, once, through the registry-backed API rather than setx. */
|
|
437
506
|
function addToUserPathWindows(dir: string): void {
|
|
438
507
|
const quoted = `'${dir.replaceAll("'", "''")}'`;
|
|
@@ -495,6 +564,8 @@ export async function connectCommand(args: CliArgs): Promise<void> {
|
|
|
495
564
|
const scopeFlag = flagString(args, "scope");
|
|
496
565
|
// The remote's skills for the agents on this machine; a remote without any serves 404.
|
|
497
566
|
const skills = await fetchSkills(url, key, fetchImpl);
|
|
567
|
+
// Its shared-context MCP endpoint, when it is a team edition serving one.
|
|
568
|
+
const info = await fetchSetupInfo(url, fetchImpl);
|
|
498
569
|
const report = connectRemote({
|
|
499
570
|
url,
|
|
500
571
|
key,
|
|
@@ -515,6 +586,7 @@ export async function connectCommand(args: CliArgs): Promise<void> {
|
|
|
515
586
|
...(device === "" ? {} : { device }),
|
|
516
587
|
...(exePath === null ? {} : { exePath }),
|
|
517
588
|
...(skills.bundle === null ? {} : { skills: skills.bundle }),
|
|
589
|
+
mcp: { url: info.mcp ? `${url}/mcp` : null },
|
|
518
590
|
});
|
|
519
591
|
if (skills.note !== undefined) report.notes.push(skills.note);
|
|
520
592
|
if (exePath !== null) console.log(`executable ${exePath}; package files under ${packageDir}`);
|
package/src/cli/refresh.ts
CHANGED
|
@@ -14,6 +14,7 @@
|
|
|
14
14
|
*/
|
|
15
15
|
|
|
16
16
|
import { executablePath, materializePackage, readEmbeddedPackage } from "./embedded.ts";
|
|
17
|
+
import { fetchSetupInfo } from "./connect.ts";
|
|
17
18
|
import { fetchSkills } from "./skills.ts";
|
|
18
19
|
import { homedir } from "node:os";
|
|
19
20
|
import { dirname, resolve } from "node:path";
|
|
@@ -99,6 +100,7 @@ export async function refreshAndRewrite(opts: { remote: RemoteRouter; fetchImpl?
|
|
|
99
100
|
const exePath = opts.remote.executable ?? executablePath() ?? undefined;
|
|
100
101
|
// A refresh is when the team's skills reach a machine that has not re-run connect.
|
|
101
102
|
const skills = await fetchSkills(opts.remote.url, fresh.key, opts.fetchImpl ?? fetch);
|
|
103
|
+
const info = await fetchSetupInfo(opts.remote.url, opts.fetchImpl ?? fetch);
|
|
102
104
|
connectRemote({
|
|
103
105
|
url: opts.remote.url,
|
|
104
106
|
key: fresh.key,
|
|
@@ -121,6 +123,7 @@ export async function refreshAndRewrite(opts: { remote: RemoteRouter; fetchImpl?
|
|
|
121
123
|
...(opts.storeDeps !== undefined ? { storeDeps: opts.storeDeps } : {}),
|
|
122
124
|
...(exePath !== undefined ? { exePath } : {}),
|
|
123
125
|
...(skills.bundle === null ? {} : { skills: skills.bundle }),
|
|
126
|
+
mcp: { url: info.mcp ? `${opts.remote.url}/mcp` : null },
|
|
124
127
|
// undefined keeps whatever scope the managed models.yml block already carries.
|
|
125
128
|
});
|
|
126
129
|
return fresh;
|
package/src/cost/ledger.ts
CHANGED
|
@@ -65,7 +65,7 @@ const CACHE_RELIABILITY_MEMO_MS = 60_000;
|
|
|
65
65
|
const DAY_MS = 86_400_000;
|
|
66
66
|
|
|
67
67
|
// Row shapes below are fixed by our own schema in util/sqlite.ts.
|
|
68
|
-
interface LedgerRow {
|
|
68
|
+
export interface LedgerRow {
|
|
69
69
|
id: string;
|
|
70
70
|
created_at_ms: number;
|
|
71
71
|
conversation_key: string;
|
|
@@ -240,7 +240,7 @@ function toLatency(slug: string, row: LatencyRow): ModelLatency | null {
|
|
|
240
240
|
return { slug, samples: row.samples, ttftMs: row.ttft_ms, tokensPerSec };
|
|
241
241
|
}
|
|
242
242
|
|
|
243
|
-
function toEntry(row: LedgerRow): LedgerEntry {
|
|
243
|
+
export function toEntry(row: LedgerRow): LedgerEntry {
|
|
244
244
|
return {
|
|
245
245
|
id: row.id,
|
|
246
246
|
createdAtMs: row.created_at_ms,
|
package/src/cost/views.ts
CHANGED
|
@@ -10,7 +10,9 @@
|
|
|
10
10
|
|
|
11
11
|
import { providerOfSlug } from "./report.ts";
|
|
12
12
|
import type { Database } from "bun:sqlite";
|
|
13
|
+
import { toEntry, type LedgerRow } from "./ledger.ts";
|
|
13
14
|
import { harnessFilter } from "./report.ts";
|
|
15
|
+
import type { LedgerEntry } from "./types.ts";
|
|
14
16
|
|
|
15
17
|
/** `null` means every harness; an empty set matches nothing. */
|
|
16
18
|
export type HarnessScope = readonly string[] | null;
|
|
@@ -61,6 +63,68 @@ function scope(harness: HarnessScope, column: string): { sql: string[]; bind: Re
|
|
|
61
63
|
return { sql: f.sql.map((s) => s.replace(/^harness_id/, column)), bind: f.bind };
|
|
62
64
|
}
|
|
63
65
|
|
|
66
|
+
/** A ledger entry as the decision explorer shows it: the entry itself plus the verdicts given on it. */
|
|
67
|
+
export type DecisionEntry = LedgerEntry & { feedback: { verdict: "good" | "bad"; note: string; createdAtMs: number }[] };
|
|
68
|
+
|
|
69
|
+
export interface DecisionFilter {
|
|
70
|
+
/** Entries at or after this instant; 0 for everything the ledger still holds. */
|
|
71
|
+
sinceMs: number;
|
|
72
|
+
/** The harness set; null for every harness, an empty list for none. */
|
|
73
|
+
harness: HarnessScope;
|
|
74
|
+
/** At most this many, newest first; 1..1000. */
|
|
75
|
+
limit?: number;
|
|
76
|
+
/** Only turns dispatched to (or served by) this slug. */
|
|
77
|
+
slug?: string;
|
|
78
|
+
/** Only turns classified at this tier. */
|
|
79
|
+
tier?: string;
|
|
80
|
+
/** Only one omp session (`/router why`). */
|
|
81
|
+
ompSessionId?: string;
|
|
82
|
+
}
|
|
83
|
+
|
|
84
|
+
/**
|
|
85
|
+
* Turns, newest first, over a harness set: what `GET /v1/router/decisions` serves and what a
|
|
86
|
+
* front door reads from the ledger file. Every field the decision trail needs is here — the
|
|
87
|
+
* reasons, the classifier's view, the cost forecast against the bill, the escalation signal —
|
|
88
|
+
* and the verdicts `/router good|bad` recorded against each turn ride along.
|
|
89
|
+
*/
|
|
90
|
+
export function decisionEntries(db: Database, filter: DecisionFilter): DecisionEntry[] {
|
|
91
|
+
const s = scope(filter.harness, "harness_id");
|
|
92
|
+
if (s === null) return [];
|
|
93
|
+
const where = ["created_at_ms >= $since", ...s.sql];
|
|
94
|
+
const bind: Record<string, string | number> = { $since: filter.sinceMs, ...s.bind };
|
|
95
|
+
if (filter.slug !== undefined && filter.slug !== "") {
|
|
96
|
+
where.push("(slug = $slug OR served_slug = $slug)");
|
|
97
|
+
bind.$slug = filter.slug;
|
|
98
|
+
}
|
|
99
|
+
if (filter.tier !== undefined && filter.tier !== "") {
|
|
100
|
+
where.push("tier = $tier");
|
|
101
|
+
bind.$tier = filter.tier;
|
|
102
|
+
}
|
|
103
|
+
if (filter.ompSessionId !== undefined && filter.ompSessionId !== "") {
|
|
104
|
+
where.push("omp_session_id = $session");
|
|
105
|
+
bind.$session = filter.ompSessionId;
|
|
106
|
+
}
|
|
107
|
+
const limit = Math.min(Math.max(filter.limit ?? 50, 1), 1_000);
|
|
108
|
+
const rows = db.query(`SELECT * FROM ledger WHERE ${where.join(" AND ")} ORDER BY created_at_ms DESC LIMIT ${limit}`).all(bind) as LedgerRow[];
|
|
109
|
+
const entries = rows.map(toEntry);
|
|
110
|
+
if (entries.length === 0) return [];
|
|
111
|
+
// Verdicts, when the feedback table exists (it does not on a ledger no one has judged).
|
|
112
|
+
const hasFeedback = (db.query("SELECT name FROM sqlite_master WHERE type = 'table' AND name = 'feedback'").get() as { name: string } | null) !== null;
|
|
113
|
+
const verdicts = new Map<string, DecisionEntry["feedback"]>();
|
|
114
|
+
if (hasFeedback) {
|
|
115
|
+
const ids = entries.map((e) => e.id);
|
|
116
|
+
const marks = ids.map((_, i) => `$f${i}`).join(", ");
|
|
117
|
+
const fb: Record<string, string> = {};
|
|
118
|
+
ids.forEach((id, i) => (fb[`$f${i}`] = id));
|
|
119
|
+
for (const r of db.query(`SELECT ledger_id, verdict, note, created_at_ms FROM feedback WHERE ledger_id IN (${marks}) ORDER BY created_at_ms ASC`).all(fb) as { ledger_id: string; verdict: string; note: string; created_at_ms: number }[]) {
|
|
120
|
+
const list = verdicts.get(r.ledger_id) ?? [];
|
|
121
|
+
list.push({ verdict: r.verdict === "good" ? "good" : "bad", note: r.note, createdAtMs: r.created_at_ms });
|
|
122
|
+
verdicts.set(r.ledger_id, list);
|
|
123
|
+
}
|
|
124
|
+
}
|
|
125
|
+
return entries.map((e) => ({ ...e, feedback: verdicts.get(e.id) ?? [] }));
|
|
126
|
+
}
|
|
127
|
+
|
|
64
128
|
/** Spend (reported where present, predicted otherwise) since `sinceMs`, digest calls included as the ledger counts them. */
|
|
65
129
|
export function spendUsdSince(db: Database, sinceMs: number, harness: HarnessScope): number {
|
|
66
130
|
const s = scope(harness, "harness_id");
|
package/src/index.ts
CHANGED
|
@@ -28,7 +28,7 @@ Usage: auto-model-router <command> [options]
|
|
|
28
28
|
stats Show routed spend, per-model share, and escalation rates
|
|
29
29
|
report Usage analytics: providers, models, tiers, cost, speed, cache hit rate
|
|
30
30
|
export One row per day, harness and model as CSV (--json for rows)
|
|
31
|
-
connect Point this machine at a remote router (--url with --key[, --refresh-token] or --setup-token <one-time token from a team>; --scope pins one project for the whole machine (default: each workspace's own); --profile persists the environment and, from the compiled executable, PATH; the remote's skills are installed for Claude Code and omp)
|
|
31
|
+
connect Point this machine at a remote router (--url with --key[, --refresh-token] or --setup-token <one-time token from a team>; --scope pins one project for the whole machine (default: each workspace's own); --profile persists the environment and, from the compiled executable, PATH; the remote's skills and its MCP endpoint are installed for Claude Code and omp)
|
|
32
32
|
refresh Trade the refresh token for a new access key and re-write every harness config (--force: even when not near expiry)
|
|
33
33
|
token Print an access key that is good right now, refreshing first if needed (for a harness key-helper)
|
|
34
34
|
models Show what each complexity tier would consider, and why
|
package/src/lib.ts
CHANGED
|
@@ -22,7 +22,7 @@ export type { DeepPartial } from "./config/load.ts";
|
|
|
22
22
|
export { buildUsageReport, renderUsageReport, type UsageReport, type ReportTotals } from "./cost/report.ts";
|
|
23
23
|
export { buildDailySummary, renderDailySummary, type DailySummary } from "./cost/summary.ts";
|
|
24
24
|
export { openDb } from "./util/sqlite.ts";
|
|
25
|
-
export { spendUsdSince, feedbackView, exportRows, exportCsv, harnessScopeParam, type HarnessScope, type ExportRow, type FeedbackRow, type FeedbackByModel, type FeedbackView } from "./cost/views.ts";
|
|
25
|
+
export { spendUsdSince, feedbackView, exportRows, exportCsv, decisionEntries, harnessScopeParam, type HarnessScope, type ExportRow, type FeedbackRow, type FeedbackByModel, type FeedbackView, type DecisionEntry, type DecisionFilter } from "./cost/views.ts";
|
|
26
26
|
export { createLedger } from "./cost/ledger.ts";
|
|
27
27
|
export { createFeedbackStore, type FeedbackStore, type FeedbackRecord } from "./cost/feedback.ts";
|
|
28
28
|
export { buildExecutable, collectPackageFiles, executableFileName, hostTarget, isExecutableTarget, EXECUTABLE_TARGETS, type ExecutableTarget, type BuildExecutableResult } from "./cli/build-executable.ts";
|
package/src/server/http.ts
CHANGED
|
@@ -10,7 +10,7 @@ import { createDigester } from "./digest.ts";
|
|
|
10
10
|
import { advise } from "./advise.ts";
|
|
11
11
|
import { TIER_ORDER, type Tier } from "../router/types.ts";
|
|
12
12
|
import { baselinePrices, buildUsageReport, renderUsageReport } from "../cost/report.ts";
|
|
13
|
-
import { exportCsv, exportRows, feedbackView, harnessScopeParam, spendUsdSince } from "../cost/views.ts";
|
|
13
|
+
import { decisionEntries, exportCsv, exportRows, feedbackView, harnessScopeParam, spendUsdSince } from "../cost/views.ts";
|
|
14
14
|
import { anthropicErrorResponse, countAnthropicTokens, createMessagesWire } from "../wire/anthropic/messages.ts";
|
|
15
15
|
import { buildDailySummary, createKv, markSummaryShown, renderDailySummary, summaryDue, summaryHasNews, type SummaryOllama } from "../cost/summary.ts";
|
|
16
16
|
import type { Ledger, ModelTrust } from "../cost/types.ts";
|
|
@@ -555,13 +555,24 @@ export function startServer(cfg: RouterConfig): StartedServer {
|
|
|
555
555
|
return json({ due: true, summary });
|
|
556
556
|
}
|
|
557
557
|
if (req.method === "GET" && url.pathname === "/v1/router/decisions") {
|
|
558
|
+
// The decision trail, newest first. ?session=<omp session id> narrows to one
|
|
559
|
+
// session (/router why); ?harness=a,b to a harness set (a team's user or group),
|
|
560
|
+
// ?since=<ms> or ?days=N to a window, ?slug= and ?tier= to a model or a tier.
|
|
558
561
|
const rawLimit = url.searchParams.get("limit");
|
|
559
562
|
const parsed = rawLimit === null ? 50 : Number.parseInt(rawLimit, 10);
|
|
560
563
|
const limit = Number.isInteger(parsed) ? Math.min(Math.max(parsed, 1), 1_000) : 50;
|
|
561
|
-
|
|
562
|
-
const
|
|
563
|
-
const
|
|
564
|
-
|
|
564
|
+
const sinceRaw = Number.parseInt(url.searchParams.get("since") ?? "", 10);
|
|
565
|
+
const daysRaw = url.searchParams.get("days");
|
|
566
|
+
const sinceMs = Number.isFinite(sinceRaw) ? sinceRaw : daysRaw === null ? 0 : Date.now() - clampDays(daysRaw, 30) * 86_400_000;
|
|
567
|
+
const entries = decisionEntries(db, {
|
|
568
|
+
sinceMs,
|
|
569
|
+
harness: harnessScopeParam(url.searchParams.get("harness")),
|
|
570
|
+
limit,
|
|
571
|
+
slug: url.searchParams.get("slug") ?? "",
|
|
572
|
+
tier: url.searchParams.get("tier") ?? "",
|
|
573
|
+
ompSessionId: url.searchParams.get("session") ?? "",
|
|
574
|
+
});
|
|
575
|
+
return json({ entries });
|
|
565
576
|
}
|
|
566
577
|
if (url.pathname === "/v1/router/override") {
|
|
567
578
|
// Per-session pin / tier overrides from omp. GET shows, POST sets or clears.
|
|
@@ -0,0 +1,157 @@
|
|
|
1
|
+
import { describe, expect, test } from "bun:test";
|
|
2
|
+
import { existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs";
|
|
3
|
+
import { tmpdir } from "node:os";
|
|
4
|
+
import { join } from "node:path";
|
|
5
|
+
import { connectRemote, fetchSetupInfo, MCP_SERVER_NAME, mergeMcpServers } from "../src/cli/connect.ts";
|
|
6
|
+
import { refreshAndRewrite } from "../src/cli/refresh.ts";
|
|
7
|
+
import { parseRemoteRouter } from "../omp-extension/remote-logic.ts";
|
|
8
|
+
|
|
9
|
+
const NL = "\n";
|
|
10
|
+
const read = (p: string): Record<string, unknown> => JSON.parse(readFileSync(p, "utf8")) as Record<string, unknown>;
|
|
11
|
+
const servers = (p: string): Record<string, unknown> => (read(p).mcpServers as Record<string, unknown>) ?? {};
|
|
12
|
+
const auth = (p: string): string => (servers(p)[MCP_SERVER_NAME] as { headers: { Authorization: string } }).headers.Authorization;
|
|
13
|
+
|
|
14
|
+
describe("mergeMcpServers", () => {
|
|
15
|
+
test("adds the team-context server next to the others and keeps every other key", () => {
|
|
16
|
+
const before = JSON.stringify({ $schema: "https://x/mcp-schema.json", mcpServers: { agentdox: { type: "http", url: "http://localhost:3003/mcp" } }, other: 1 });
|
|
17
|
+
const after = mergeMcpServers(before, "https://team.example/mcp", "amrt_k");
|
|
18
|
+
expect(after).not.toBeNull();
|
|
19
|
+
const doc = JSON.parse(after!) as Record<string, unknown>;
|
|
20
|
+
expect(doc.$schema).toBe("https://x/mcp-schema.json");
|
|
21
|
+
expect(doc.other).toBe(1);
|
|
22
|
+
expect(doc.mcpServers).toEqual({
|
|
23
|
+
agentdox: { type: "http", url: "http://localhost:3003/mcp" },
|
|
24
|
+
[MCP_SERVER_NAME]: { type: "http", url: "https://team.example/mcp", headers: { Authorization: "Bearer amrt_k" } },
|
|
25
|
+
});
|
|
26
|
+
expect(after!.endsWith("\n")).toBe(true);
|
|
27
|
+
});
|
|
28
|
+
|
|
29
|
+
test("an empty or missing file becomes a fresh mcpServers document", () => {
|
|
30
|
+
expect(JSON.parse(mergeMcpServers("", "https://t/mcp", "k")!)).toEqual({ mcpServers: { [MCP_SERVER_NAME]: { type: "http", url: "https://t/mcp", headers: { Authorization: "Bearer k" } } } });
|
|
31
|
+
expect(JSON.parse(mergeMcpServers(" \n", "https://t/mcp", "k")!)).toHaveProperty("mcpServers");
|
|
32
|
+
});
|
|
33
|
+
|
|
34
|
+
test("is idempotent, removes on null, and leaves a file it cannot parse alone", () => {
|
|
35
|
+
const one = mergeMcpServers("", "https://t/mcp", "k")!;
|
|
36
|
+
expect(mergeMcpServers(one, "https://t/mcp", "k")).toBeNull(); // unchanged
|
|
37
|
+
expect(mergeMcpServers(one, "https://t/mcp", "k2")).not.toBeNull(); // a new key rewrites
|
|
38
|
+
const removed = mergeMcpServers(one, null, "k")!;
|
|
39
|
+
expect(JSON.parse(removed)).toEqual({ mcpServers: {} });
|
|
40
|
+
expect(mergeMcpServers(removed, null, "k")).toBeNull(); // nothing to remove
|
|
41
|
+
expect(mergeMcpServers("{ not json", "https://t/mcp", "k")).toBeNull();
|
|
42
|
+
expect(mergeMcpServers("[1,2]", "https://t/mcp", "k")).toBeNull();
|
|
43
|
+
});
|
|
44
|
+
});
|
|
45
|
+
|
|
46
|
+
describe("connect writes the team MCP endpoint", () => {
|
|
47
|
+
function fixture(): { home: string; agent: string; env: Record<string, string> } {
|
|
48
|
+
const home = mkdtempSync(join(tmpdir(), "amr-mcp-connect-"));
|
|
49
|
+
mkdirSync(join(home, ".claude"), { recursive: true });
|
|
50
|
+
const agent = join(home, ".omp", "agent");
|
|
51
|
+
mkdirSync(agent, { recursive: true });
|
|
52
|
+
writeFileSync(join(agent, "config.yml"), `extensions: []${NL}`, "utf8");
|
|
53
|
+
const env = { HOME: home, PI_CODING_AGENT_DIR: agent, AUTO_MODEL_ROUTER_HOME: join(home, ".auto-model-router"), HERMES_HOME: join(home, "no-hermes") };
|
|
54
|
+
return { home, agent, env };
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
test("into omp mcp.json and Claude Code ~/.claude.json, only for the harnesses it configured", () => {
|
|
58
|
+
const { home, agent, env } = fixture();
|
|
59
|
+
try {
|
|
60
|
+
// Pre-existing servers and unrelated settings survive.
|
|
61
|
+
writeFileSync(join(agent, "mcp.json"), JSON.stringify({ $schema: "s", mcpServers: { agentdox: { type: "http", url: "http://localhost:3003/mcp" } } }, null, 2));
|
|
62
|
+
writeFileSync(join(home, ".claude.json"), JSON.stringify({ claudeAiMcpEverConnected: true, mcpServers: {} }));
|
|
63
|
+
const r = connectRemote({ url: "https://team.example", key: "amrt_k", userId: "u", name: "Ada", profile: false, dryRun: false, only: ["omp", "claude"], env, home, packageDir: "/pkg", mcp: { url: "https://team.example/mcp" }, platform: "linux", pathHas: () => false });
|
|
64
|
+
expect(r.mcp?.length).toBe(2);
|
|
65
|
+
expect(r.configured.some((c) => c.startsWith(`MCP (${MCP_SERVER_NAME} → https://team.example/mcp`))).toBe(true);
|
|
66
|
+
const omp = read(join(agent, "mcp.json"));
|
|
67
|
+
expect(omp.$schema).toBe("s");
|
|
68
|
+
expect(servers(join(agent, "mcp.json"))).toEqual({
|
|
69
|
+
agentdox: { type: "http", url: "http://localhost:3003/mcp" },
|
|
70
|
+
[MCP_SERVER_NAME]: { type: "http", url: "https://team.example/mcp", headers: { Authorization: "Bearer amrt_k" } },
|
|
71
|
+
});
|
|
72
|
+
const claude = read(join(home, ".claude.json"));
|
|
73
|
+
expect(claude.claudeAiMcpEverConnected).toBe(true);
|
|
74
|
+
expect(servers(join(home, ".claude.json"))[MCP_SERVER_NAME]).toEqual({ type: "http", url: "https://team.example/mcp", headers: { Authorization: "Bearer amrt_k" } });
|
|
75
|
+
|
|
76
|
+
// A second connect with the same key changes nothing.
|
|
77
|
+
const r2 = connectRemote({ url: "https://team.example", key: "amrt_k", userId: "u", name: "Ada", profile: false, dryRun: false, only: ["omp", "claude"], env, home, packageDir: "/pkg", mcp: { url: "https://team.example/mcp" }, platform: "linux", pathHas: () => false });
|
|
78
|
+
expect(r2.mcp).toEqual([]);
|
|
79
|
+
|
|
80
|
+
// Only Claude Code asked for: the omp file is not touched even though it exists.
|
|
81
|
+
rmSync(join(agent, "mcp.json"));
|
|
82
|
+
const r3 = connectRemote({ url: "https://team.example", key: "amrt_k3", userId: "u", name: "Ada", profile: false, dryRun: false, only: ["claude"], env, home, packageDir: "/pkg", mcp: { url: "https://team.example/mcp" }, platform: "linux", pathHas: () => false });
|
|
83
|
+
expect(r3.mcp).toEqual([join(home, ".claude.json")]);
|
|
84
|
+
expect(existsSync(join(agent, "mcp.json"))).toBe(false);
|
|
85
|
+
expect(auth(join(home, ".claude.json"))).toBe("Bearer amrt_k3");
|
|
86
|
+
|
|
87
|
+
// A remote that stops serving MCP has the entry removed; the neighbours stay.
|
|
88
|
+
const r4 = connectRemote({ url: "https://team.example", key: "amrt_k3", userId: "u", name: "Ada", profile: false, dryRun: false, only: ["claude"], env, home, packageDir: "/pkg", mcp: { url: null }, platform: "linux", pathHas: () => false });
|
|
89
|
+
expect(r4.mcp).toEqual([join(home, ".claude.json")]);
|
|
90
|
+
expect(servers(join(home, ".claude.json"))).toEqual({});
|
|
91
|
+
expect(read(join(home, ".claude.json")).claudeAiMcpEverConnected).toBe(true);
|
|
92
|
+
expect(r4.configured.some((c) => c.startsWith("MCP ("))).toBe(false);
|
|
93
|
+
} finally {
|
|
94
|
+
rmSync(home, { recursive: true, force: true });
|
|
95
|
+
}
|
|
96
|
+
});
|
|
97
|
+
|
|
98
|
+
test("a dry run reports the files without writing them; no mcp option leaves them alone", () => {
|
|
99
|
+
const { home, agent, env } = fixture();
|
|
100
|
+
try {
|
|
101
|
+
const r = connectRemote({ url: "https://team.example", key: "amrt_k", userId: "u", name: "Ada", profile: false, dryRun: true, only: ["omp"], env, home, packageDir: "/pkg", mcp: { url: "https://team.example/mcp" }, platform: "linux", pathHas: () => false });
|
|
102
|
+
expect(r.mcp).toEqual([join(agent, "mcp.json")]);
|
|
103
|
+
expect(existsSync(join(agent, "mcp.json"))).toBe(false);
|
|
104
|
+
const r2 = connectRemote({ url: "https://team.example", key: "amrt_k", userId: "u", name: "Ada", profile: false, dryRun: false, only: ["omp"], env, home, packageDir: "/pkg", platform: "linux", pathHas: () => false });
|
|
105
|
+
expect(r2.mcp).toBeUndefined();
|
|
106
|
+
expect(existsSync(join(agent, "mcp.json"))).toBe(false);
|
|
107
|
+
} finally {
|
|
108
|
+
rmSync(home, { recursive: true, force: true });
|
|
109
|
+
}
|
|
110
|
+
});
|
|
111
|
+
|
|
112
|
+
test("refresh rewrites the entry with the new key, asking the remote whether it still serves MCP", async () => {
|
|
113
|
+
const { home, agent, env } = fixture();
|
|
114
|
+
const routerHome = env.AUTO_MODEL_ROUTER_HOME!;
|
|
115
|
+
try {
|
|
116
|
+
connectRemote({ url: "https://team.example", key: "amrt_old", userId: "u_ada", name: "Ada", refreshToken: "amrr_r1", keyExpiresAtMs: 1, refreshExpiresAtMs: 2, profile: false, dryRun: false, only: ["omp"], env, home, packageDir: process.cwd(), mcp: { url: "https://team.example/mcp" }, platform: "linux", pathHas: () => false });
|
|
117
|
+
expect(auth(join(agent, "mcp.json"))).toBe("Bearer amrt_old");
|
|
118
|
+
const seen: string[] = [];
|
|
119
|
+
const fetchImpl = (async (input: string | URL | Request) => {
|
|
120
|
+
const url = String(input);
|
|
121
|
+
seen.push(url);
|
|
122
|
+
if (url.endsWith("/setup/info")) return Response.json({ version: "0.18.0", mcp: true });
|
|
123
|
+
if (url.endsWith("/setup/skills")) return new Response("", { status: 404 });
|
|
124
|
+
return Response.json({ key: "amrt_new", keyExpiresAtMs: 50, refreshToken: "amrr_r2", refreshExpiresAtMs: 90 });
|
|
125
|
+
}) as unknown as typeof fetch;
|
|
126
|
+
const remote = parseRemoteRouter(readFileSync(join(routerHome, "remote.json"), "utf8"))!;
|
|
127
|
+
await refreshAndRewrite({ remote, fetchImpl, home, packageDir: process.cwd(), env, platform: "linux", pathHas: () => false, routerHome });
|
|
128
|
+
expect(seen.some((u) => u === "https://team.example/setup/info")).toBe(true);
|
|
129
|
+
expect(servers(join(agent, "mcp.json"))[MCP_SERVER_NAME]).toEqual({ type: "http", url: "https://team.example/mcp", headers: { Authorization: "Bearer amrt_new" } });
|
|
130
|
+
// The remote no longer serves MCP (a plain router answers 404): the entry goes.
|
|
131
|
+
const plain = (async (input: string | URL | Request) => {
|
|
132
|
+
const url = String(input);
|
|
133
|
+
if (url.endsWith("/setup/info") || url.endsWith("/setup/skills")) return new Response("", { status: 404 });
|
|
134
|
+
return Response.json({ key: "amrt_new2", keyExpiresAtMs: 60, refreshToken: "amrr_r3", refreshExpiresAtMs: 99 });
|
|
135
|
+
}) as unknown as typeof fetch;
|
|
136
|
+
const remote2 = parseRemoteRouter(readFileSync(join(routerHome, "remote.json"), "utf8"))!;
|
|
137
|
+
await refreshAndRewrite({ remote: remote2, fetchImpl: plain, home, packageDir: process.cwd(), env, platform: "linux", pathHas: () => false, routerHome });
|
|
138
|
+
expect(servers(join(agent, "mcp.json"))).toEqual({});
|
|
139
|
+
} finally {
|
|
140
|
+
rmSync(home, { recursive: true, force: true });
|
|
141
|
+
}
|
|
142
|
+
});
|
|
143
|
+
});
|
|
144
|
+
|
|
145
|
+
describe("fetchSetupInfo", () => {
|
|
146
|
+
test("reads mcp from a team, and answers false for a plain router, a bad body or a dead remote", async () => {
|
|
147
|
+
expect(await fetchSetupInfo("https://t", (async () => Response.json({ mcp: true })) as unknown as typeof fetch)).toEqual({ mcp: true });
|
|
148
|
+
expect(await fetchSetupInfo("https://t", (async () => Response.json({ mcp: "yes" })) as unknown as typeof fetch)).toEqual({ mcp: false });
|
|
149
|
+
expect(await fetchSetupInfo("https://t", (async () => new Response("", { status: 404 })) as unknown as typeof fetch)).toEqual({ mcp: false });
|
|
150
|
+
expect(await fetchSetupInfo("https://t", (async () => new Response("<html>", { status: 200 })) as unknown as typeof fetch)).toEqual({ mcp: false });
|
|
151
|
+
expect(
|
|
152
|
+
await fetchSetupInfo("https://t", (async () => {
|
|
153
|
+
throw new Error("down");
|
|
154
|
+
}) as unknown as typeof fetch),
|
|
155
|
+
).toEqual({ mcp: false });
|
|
156
|
+
});
|
|
157
|
+
});
|
package/test/views.test.ts
CHANGED
|
@@ -148,6 +148,12 @@ describe("view routes", () => {
|
|
|
148
148
|
expect((await fetch(`http://127.0.0.1:${handle.server.port}/v1/router/spend?sinceMs=0`)).status).toBe(401);
|
|
149
149
|
expect((await get("/v1/router/spend")).status).toBe(400);
|
|
150
150
|
expect(((await (await get(`/v1/router/spend?sinceMs=${Date.now() - DAY}&harness=u_x`)).json()) as { usd: number }).usd).toBeCloseTo(0.25, 6);
|
|
151
|
+
// The decision trail over a harness set: a team asks with its user or group ids and sees only theirs.
|
|
152
|
+
const mine = (await (await get("/v1/router/decisions?harness=u_x&days=1")).json()) as { entries: { id: string; feedback: unknown[] }[] };
|
|
153
|
+
expect(mine.entries.map((e) => e.id)).toEqual(["r1"]);
|
|
154
|
+
expect(mine.entries[0]?.feedback).toEqual([]);
|
|
155
|
+
expect((((await (await get("/v1/router/decisions?harness=u_other&days=1")).json()) as { entries: unknown[] }).entries)).toEqual([]);
|
|
156
|
+
expect((((await (await get("/v1/router/decisions?limit=1")).json()) as { entries: { id: string }[] }).entries.map((e) => e.id))).toEqual(["r1"]); // no filter: everything, as before
|
|
151
157
|
expect(((await (await get(`/v1/router/spend?sinceMs=${Date.now() - DAY}&harness=u_other`)).json()) as { usd: number }).usd).toBe(0);
|
|
152
158
|
const fb = (await (await get("/v1/router/feedback?days=7")).json()) as { days: number; byModel: unknown[]; recent: unknown[] };
|
|
153
159
|
expect(fb).toEqual({ days: 7, byModel: [], recent: [] });
|
|
@@ -159,3 +165,28 @@ describe("view routes", () => {
|
|
|
159
165
|
expect(js.rows[0]?.harnessId).toBe("u_x");
|
|
160
166
|
});
|
|
161
167
|
});
|
|
168
|
+
|
|
169
|
+
describe("decision entries", () => {
|
|
170
|
+
const db = seeded();
|
|
171
|
+
const since = NOW - DAY;
|
|
172
|
+
|
|
173
|
+
test("newest first over a harness set, with the verdicts given on each turn", () => {
|
|
174
|
+
const { decisionEntries } = require("../src/cost/views.ts") as typeof import("../src/cost/views.ts");
|
|
175
|
+
const ada = decisionEntries(db, { sinceMs: since, harness: ["u_ada"] });
|
|
176
|
+
expect(ada.map((e) => e.id)).toEqual(["l1", "l2"]); // same instant in the fixture; insertion order within it is stable
|
|
177
|
+
expect(ada.find((e) => e.id === "l1")?.feedback).toEqual([{ verdict: "good", note: "", createdAtMs: NOW - 1000 }]);
|
|
178
|
+
expect(ada.find((e) => e.id === "l2")?.escalationSignal).toBe("circular");
|
|
179
|
+
// Everyone, within the window: the 40-day-old row stays out; the digest row is a turn like any other.
|
|
180
|
+
expect(decisionEntries(db, { sinceMs: since, harness: null }).map((e) => e.id).sort()).toEqual(["l1", "l2", "l3", "l4"]);
|
|
181
|
+
expect(decisionEntries(db, { sinceMs: 0, harness: null }).length).toBe(5);
|
|
182
|
+
// A model, a tier, nobody, and a cap.
|
|
183
|
+
expect(decisionEntries(db, { sinceMs: since, harness: null, slug: "ollama/glm-5.3-flash" }).map((e) => e.id).sort()).toEqual(["l3", "l4"]);
|
|
184
|
+
expect(decisionEntries(db, { sinceMs: since, harness: null, tier: "hard" })).toEqual([]);
|
|
185
|
+
expect(decisionEntries(db, { sinceMs: since, harness: [] })).toEqual([]);
|
|
186
|
+
expect(decisionEntries(db, { sinceMs: since, harness: null, limit: 1 }).length).toBe(1);
|
|
187
|
+
// The error on l3 and its note ride along, so an explorer can show why a turn went wrong.
|
|
188
|
+
const bob = decisionEntries(db, { sinceMs: since, harness: ["u_bob"], slug: "ollama/glm-5.3-flash" });
|
|
189
|
+
expect(bob.find((e) => e.id === "l3")?.error).toBe("boom");
|
|
190
|
+
expect(bob.find((e) => e.id === "l3")?.feedback[0]?.note).toBe("looped");
|
|
191
|
+
});
|
|
192
|
+
});
|