auto-model-router 0.4.12 → 0.4.14
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.omp-plugin/marketplace.json +2 -2
- package/README.md +24 -0
- package/package.json +1 -1
- package/src/cli/args.ts +1 -0
- package/src/cli/export.ts +29 -0
- package/src/cost/views.ts +153 -0
- package/src/index.ts +5 -0
- package/src/lib.ts +3 -0
- package/src/router/index.ts +45 -5
- package/src/server/http.ts +25 -3
- package/src/wire/openai/request.ts +33 -0
- package/src/wire/types.ts +20 -0
- package/test/policy.test.ts +56 -0
- package/test/views.test.ts +161 -0
|
@@ -7,14 +7,14 @@
|
|
|
7
7
|
},
|
|
8
8
|
"metadata": {
|
|
9
9
|
"description": "auto-model-router: a local cost/complexity-aware model router for Oh My Pi, backed by OpenRouter",
|
|
10
|
-
"version": "0.4.
|
|
10
|
+
"version": "0.4.14",
|
|
11
11
|
"pluginRoot": "."
|
|
12
12
|
},
|
|
13
13
|
"plugins": [
|
|
14
14
|
{
|
|
15
15
|
"name": "auto-model-router",
|
|
16
16
|
"description": "Local cost/complexity-aware model router for Oh My Pi, backed by OpenRouter. Runs in-process, routes per turn by price and task complexity, with budget caps, mid-stream escalation, and cache-aware hysteresis.",
|
|
17
|
-
"version": "0.4.
|
|
17
|
+
"version": "0.4.14",
|
|
18
18
|
"author": {
|
|
19
19
|
"name": "drewappling",
|
|
20
20
|
"email": "drewappling@gmail.com"
|
package/README.md
CHANGED
|
@@ -706,6 +706,12 @@ escalation signal, error. Three views aggregate it, all from the same
|
|
|
706
706
|
transcript. Falls back to reading the ledger directly if the router is
|
|
707
707
|
unreachable.
|
|
708
708
|
- `auto-model-router report --days 7 [--harness <id>] [--json]` on the terminal.
|
|
709
|
+
- `auto-model-router export --days 30 [--harness a,b] [--json]`: one row per day, harness
|
|
710
|
+
and model (dispatches, tokens, spend, escalations, errors) as CSV. Also
|
|
711
|
+
`GET /v1/router/export?days=&harness=[&format=json]`; `GET /v1/router/spend?sinceMs=&harness=`
|
|
712
|
+
gives spend over a harness set since an instant, and `GET /v1/router/feedback?days=&harness=`
|
|
713
|
+
lists verdicts by model and the recent ones with the harness that gave them. These are what
|
|
714
|
+
a front door such as the team edition reads instead of the ledger file.
|
|
709
715
|
- `GET /v1/router/report?days=7&harness=<id>` for dashboards (`harness` may be
|
|
710
716
|
a comma-separated set of ids, for a group).
|
|
711
717
|
- `GET /v1/router/summary?harness=<id>` — the daily summary as JSON (`auto=1`
|
|
@@ -1178,6 +1184,24 @@ text, not the conversation, so a hard task that only becomes hard three tool
|
|
|
1178
1184
|
calls in stays on the router (the router's own escalation still applies
|
|
1179
1185
|
there); and the switch happens at prompt boundaries, never mid-turn.
|
|
1180
1186
|
|
|
1187
|
+
## Per-request routing policy
|
|
1188
|
+
|
|
1189
|
+
A front door in front of the router (the team edition, or any proxy that
|
|
1190
|
+
knows who is calling) can constrain one turn with an `X-Omp-Policy` header
|
|
1191
|
+
carrying JSON:
|
|
1192
|
+
|
|
1193
|
+
```json
|
|
1194
|
+
{ "allow": ["anthropic/*", "google/*"], "deny": ["openai/gpt-5-pro"], "minTier": "simple", "maxTier": "moderate", "pin": "anthropic/claude-sonnet-5" }
|
|
1195
|
+
```
|
|
1196
|
+
|
|
1197
|
+
`allow` and `deny` are slug globs like `filters.allow`/`filters.deny`: a
|
|
1198
|
+
request allow list replaces the configured one, a deny list adds to it.
|
|
1199
|
+
`minTier`/`maxTier` narrow the requested profile's tier envelope and never
|
|
1200
|
+
widen it. `pin` forces one model the way `/router pin` does, unless a session
|
|
1201
|
+
override already pinned one. Every field is optional; a malformed header is
|
|
1202
|
+
ignored rather than failing the turn. The decision trail records what the
|
|
1203
|
+
policy changed (`policy: …`).
|
|
1204
|
+
|
|
1181
1205
|
## Multiple coding harnesses, one router
|
|
1182
1206
|
|
|
1183
1207
|
**One router process for everything.** omp's embed extension binds a private
|
package/package.json
CHANGED
package/src/cli/args.ts
CHANGED
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* `auto-model-router export`: the ledger as one row per day, harness and
|
|
3
|
+
* model (dispatches, tokens, spend, escalations, errors), CSV by default or
|
|
4
|
+
* `--json`. The same rows back `GET /v1/router/export`.
|
|
5
|
+
*/
|
|
6
|
+
|
|
7
|
+
import { existsSync } from "node:fs";
|
|
8
|
+
import { Database } from "bun:sqlite";
|
|
9
|
+
import { loadConfig } from "../config/load.ts";
|
|
10
|
+
import { exportCsv, exportRows } from "../cost/views.ts";
|
|
11
|
+
import { configOpts, flagInt, flagString, type CliArgs } from "./args.ts";
|
|
12
|
+
|
|
13
|
+
export async function exportCommand(args: CliArgs): Promise<void> {
|
|
14
|
+
const days = flagInt(args, "days") ?? 30;
|
|
15
|
+
const harness = flagString(args, "harness") ?? "";
|
|
16
|
+
const cfg = loadConfig(configOpts(args));
|
|
17
|
+
if (!existsSync(cfg.ledger.path)) {
|
|
18
|
+
process.stdout.write(args.flags.has("json") ? "[]\n" : exportCsv([]));
|
|
19
|
+
return;
|
|
20
|
+
}
|
|
21
|
+
// Read-only: an export must never create or migrate the ledger.
|
|
22
|
+
const db = new Database(cfg.ledger.path, { readonly: true });
|
|
23
|
+
try {
|
|
24
|
+
const rows = exportRows(db, Date.now() - days * 86_400_000, harness === "" ? null : harness.split(",").map((s) => s.trim()).filter((s) => s !== ""));
|
|
25
|
+
process.stdout.write(args.flags.has("json") ? `${JSON.stringify(rows, null, 2)}\n` : exportCsv(rows));
|
|
26
|
+
} finally {
|
|
27
|
+
db.close();
|
|
28
|
+
}
|
|
29
|
+
}
|
|
@@ -0,0 +1,153 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Ledger views that a front door (the team edition, a dashboard, a script)
|
|
3
|
+
* needs at a scope the report does not offer: spend since an instant over a
|
|
4
|
+
* set of harnesses, feedback verdicts with the harness that gave them, and a
|
|
5
|
+
* cost export by day, harness and model. Served by `/v1/router/spend`,
|
|
6
|
+
* `GET /v1/router/feedback` and `/v1/router/export`, exported from lib.ts, and
|
|
7
|
+
* behind `auto-model-router export`. All of them read only long-stable ledger
|
|
8
|
+
* columns and accept a read-only database handle.
|
|
9
|
+
*/
|
|
10
|
+
|
|
11
|
+
import type { Database } from "bun:sqlite";
|
|
12
|
+
import { harnessFilter } from "./report.ts";
|
|
13
|
+
|
|
14
|
+
/** `null` means every harness; an empty set matches nothing. */
|
|
15
|
+
export type HarnessScope = readonly string[] | null;
|
|
16
|
+
|
|
17
|
+
export interface FeedbackRow {
|
|
18
|
+
atMs: number;
|
|
19
|
+
slug: string;
|
|
20
|
+
tier: string;
|
|
21
|
+
verdict: "good" | "bad";
|
|
22
|
+
note: string;
|
|
23
|
+
/** The harness id of the judged turn (a team user id), or empty. */
|
|
24
|
+
harnessId: string;
|
|
25
|
+
}
|
|
26
|
+
|
|
27
|
+
export interface FeedbackByModel {
|
|
28
|
+
slug: string;
|
|
29
|
+
good: number;
|
|
30
|
+
bad: number;
|
|
31
|
+
/** Distinct harness ids that judged this model. */
|
|
32
|
+
judges: number;
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
export interface FeedbackView {
|
|
36
|
+
byModel: FeedbackByModel[];
|
|
37
|
+
recent: FeedbackRow[];
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
export interface ExportRow {
|
|
41
|
+
day: string;
|
|
42
|
+
harnessId: string;
|
|
43
|
+
slug: string;
|
|
44
|
+
provider: string;
|
|
45
|
+
dispatches: number;
|
|
46
|
+
promptTokens: number;
|
|
47
|
+
cachedTokens: number;
|
|
48
|
+
completionTokens: number;
|
|
49
|
+
spendUsd: number;
|
|
50
|
+
escalations: number;
|
|
51
|
+
errors: number;
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
const USD = "COALESCE(reported_usd, predicted_usd)";
|
|
55
|
+
|
|
56
|
+
function scope(harness: HarnessScope, column: string): { sql: string[]; bind: Record<string, string> } | null {
|
|
57
|
+
if (harness === null) return { sql: [], bind: {} };
|
|
58
|
+
if (harness.length === 0) return null;
|
|
59
|
+
const f = harnessFilter(harness.join(","));
|
|
60
|
+
return { sql: f.sql.map((s) => s.replace(/^harness_id/, column)), bind: f.bind };
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
/** Spend (reported where present, predicted otherwise) since `sinceMs`, digest calls included as the ledger counts them. */
|
|
64
|
+
export function spendUsdSince(db: Database, sinceMs: number, harness: HarnessScope): number {
|
|
65
|
+
const s = scope(harness, "harness_id");
|
|
66
|
+
if (s === null) return 0;
|
|
67
|
+
const where = ["created_at_ms >= $since", ...s.sql].join(" AND ");
|
|
68
|
+
const row = db.query(`SELECT COALESCE(SUM(${USD}), 0) AS usd FROM ledger WHERE ${where}`).get({ $since: sinceMs, ...s.bind }) as { usd: number };
|
|
69
|
+
return row.usd;
|
|
70
|
+
}
|
|
71
|
+
|
|
72
|
+
/** Verdicts since `sinceMs`, by model and the most recent 200, joined to the ledger for the judging harness. */
|
|
73
|
+
export function feedbackView(db: Database, sinceMs: number, harness: HarnessScope): FeedbackView {
|
|
74
|
+
const exists = (db.query("SELECT name FROM sqlite_master WHERE type = 'table' AND name = 'feedback'").get() as { name: string } | null) !== null;
|
|
75
|
+
if (!exists) return { byModel: [], recent: [] };
|
|
76
|
+
const s = scope(harness, "l.harness_id");
|
|
77
|
+
if (s === null) return { byModel: [], recent: [] };
|
|
78
|
+
const where = ["f.created_at_ms >= $since", ...s.sql].join(" AND ");
|
|
79
|
+
const bind = { $since: sinceMs, ...s.bind };
|
|
80
|
+
const recent = (
|
|
81
|
+
db.query(`SELECT f.created_at_ms AS at_ms, f.slug, f.tier, f.verdict, f.note, COALESCE(l.harness_id, '') AS harness_id FROM feedback f LEFT JOIN ledger l ON l.id = f.ledger_id WHERE ${where} ORDER BY f.created_at_ms DESC LIMIT 200`).all(bind) as {
|
|
82
|
+
at_ms: number;
|
|
83
|
+
slug: string;
|
|
84
|
+
tier: string;
|
|
85
|
+
verdict: string;
|
|
86
|
+
note: string;
|
|
87
|
+
harness_id: string;
|
|
88
|
+
}[]
|
|
89
|
+
).map((r) => ({ atMs: r.at_ms, slug: r.slug, tier: r.tier, verdict: (r.verdict === "good" ? "good" : "bad") as "good" | "bad", note: r.note, harnessId: r.harness_id }));
|
|
90
|
+
const byModel = (
|
|
91
|
+
db
|
|
92
|
+
.query(
|
|
93
|
+
`SELECT f.slug, SUM(CASE WHEN f.verdict = 'good' THEN 1 ELSE 0 END) AS good, SUM(CASE WHEN f.verdict = 'bad' THEN 1 ELSE 0 END) AS bad, COUNT(DISTINCT COALESCE(l.harness_id, '')) AS judges
|
|
94
|
+
FROM feedback f LEFT JOIN ledger l ON l.id = f.ledger_id WHERE ${where} GROUP BY f.slug ORDER BY bad DESC, good DESC, f.slug ASC`,
|
|
95
|
+
)
|
|
96
|
+
.all(bind) as { slug: string; good: number; bad: number; judges: number }[]
|
|
97
|
+
).map((r) => ({ slug: r.slug, good: r.good, bad: r.bad, judges: r.judges }));
|
|
98
|
+
return { byModel, recent };
|
|
99
|
+
}
|
|
100
|
+
|
|
101
|
+
/** One row per UTC day, harness and served model since `sinceMs`; digest calls are excluded as in the report. */
|
|
102
|
+
export function exportRows(db: Database, sinceMs: number, harness: HarnessScope): ExportRow[] {
|
|
103
|
+
const s = scope(harness, "harness_id");
|
|
104
|
+
if (s === null) return [];
|
|
105
|
+
const where = ["created_at_ms >= $since", "requested_model <> 'digest'", ...s.sql].join(" AND ");
|
|
106
|
+
const rows = db
|
|
107
|
+
.query(
|
|
108
|
+
`SELECT strftime('%Y-%m-%d', created_at_ms / 1000, 'unixepoch') AS day, harness_id, COALESCE(served_slug, slug) AS slug,
|
|
109
|
+
COUNT(*) AS dispatches,
|
|
110
|
+
COALESCE(SUM(json_extract(usage, '$.promptTokens')), 0) AS prompt_tokens,
|
|
111
|
+
COALESCE(SUM(json_extract(usage, '$.cachedTokens')), 0) AS cached_tokens,
|
|
112
|
+
COALESCE(SUM(json_extract(usage, '$.completionTokens')), 0) AS completion_tokens,
|
|
113
|
+
COALESCE(SUM(${USD}), 0) AS spend,
|
|
114
|
+
SUM(CASE WHEN escalation_signal IS NOT NULL THEN 1 ELSE 0 END) AS escalations,
|
|
115
|
+
SUM(CASE WHEN error IS NOT NULL THEN 1 ELSE 0 END) AS errors
|
|
116
|
+
FROM ledger WHERE ${where} GROUP BY day, harness_id, slug ORDER BY day ASC, harness_id ASC, spend DESC`,
|
|
117
|
+
)
|
|
118
|
+
.all({ $since: sinceMs, ...s.bind }) as { day: string; harness_id: string; slug: string; dispatches: number; prompt_tokens: number; cached_tokens: number; completion_tokens: number; spend: number; escalations: number; errors: number }[];
|
|
119
|
+
return rows.map((r) => ({
|
|
120
|
+
day: r.day,
|
|
121
|
+
harnessId: r.harness_id,
|
|
122
|
+
slug: r.slug,
|
|
123
|
+
provider: r.slug.startsWith("ollama/") ? "ollama" : "openrouter",
|
|
124
|
+
dispatches: r.dispatches,
|
|
125
|
+
promptTokens: r.prompt_tokens,
|
|
126
|
+
cachedTokens: r.cached_tokens,
|
|
127
|
+
completionTokens: r.completion_tokens,
|
|
128
|
+
spendUsd: r.spend,
|
|
129
|
+
escalations: r.escalations,
|
|
130
|
+
errors: r.errors,
|
|
131
|
+
}));
|
|
132
|
+
}
|
|
133
|
+
|
|
134
|
+
export const EXPORT_COLUMNS = ["day", "harness", "model", "provider", "dispatches", "prompt_tokens", "cached_tokens", "completion_tokens", "spend_usd", "escalations", "errors"] as const;
|
|
135
|
+
|
|
136
|
+
export function csvCell(v: string | number): string {
|
|
137
|
+
const s = String(v);
|
|
138
|
+
return /[",\n\r]/.test(s) ? `"${s.replaceAll('"', '""')}"` : s;
|
|
139
|
+
}
|
|
140
|
+
|
|
141
|
+
/** CSV of export rows; spend to 6 decimals so sub-cent rows survive. */
|
|
142
|
+
export function exportCsv(rows: readonly ExportRow[]): string {
|
|
143
|
+
const lines = [EXPORT_COLUMNS.join(",")];
|
|
144
|
+
for (const r of rows) lines.push([r.day, r.harnessId, r.slug, r.provider, r.dispatches, r.promptTokens, r.cachedTokens, r.completionTokens, r.spendUsd.toFixed(6), r.escalations, r.errors].map(csvCell).join(","));
|
|
145
|
+
return `${lines.join("\n")}\n`;
|
|
146
|
+
}
|
|
147
|
+
|
|
148
|
+
/** `?harness=a,b` → scope; absent or empty → everything. */
|
|
149
|
+
export function harnessScopeParam(raw: string | null): HarnessScope {
|
|
150
|
+
if (raw === null) return null;
|
|
151
|
+
const ids = raw.split(",").map((s) => s.trim()).filter((s) => s !== "");
|
|
152
|
+
return ids.length === 0 ? null : ids;
|
|
153
|
+
}
|
package/src/index.ts
CHANGED
|
@@ -11,6 +11,7 @@ import { join } from "node:path";
|
|
|
11
11
|
import { parseArgv } from "./cli/args.ts";
|
|
12
12
|
import { configCommand } from "./cli/config-cmd.ts";
|
|
13
13
|
import { explainCommand } from "./cli/explain.ts";
|
|
14
|
+
import { exportCommand } from "./cli/export.ts";
|
|
14
15
|
import { modelsCommand } from "./cli/models.ts";
|
|
15
16
|
import { reportCommand } from "./cli/report.ts";
|
|
16
17
|
import { serveCommand } from "./cli/serve.ts";
|
|
@@ -23,6 +24,7 @@ Usage: auto-model-router <command> [options]
|
|
|
23
24
|
serve Run the router as a standalone process (for non-omp harnesses)
|
|
24
25
|
stats Show routed spend, per-model share, and escalation rates
|
|
25
26
|
report Usage analytics: providers, models, tiers, cost, speed, cache hit rate
|
|
27
|
+
export One row per day, harness and model as CSV (--json for rows)
|
|
26
28
|
models Show what each complexity tier would consider, and why
|
|
27
29
|
explain Route a saved request without dispatching it, and explain the decision
|
|
28
30
|
config Interactive wizard over the router's own config.yml
|
|
@@ -73,6 +75,9 @@ async function main(): Promise<number> {
|
|
|
73
75
|
case "report":
|
|
74
76
|
await reportCommand(args);
|
|
75
77
|
return 0;
|
|
78
|
+
case "export":
|
|
79
|
+
await exportCommand(args);
|
|
80
|
+
return 0;
|
|
76
81
|
case "models":
|
|
77
82
|
await modelsCommand(args);
|
|
78
83
|
return 0;
|
package/src/lib.ts
CHANGED
|
@@ -19,5 +19,8 @@ export type { RouterConfig } from "./config/types.ts";
|
|
|
19
19
|
export { buildUsageReport, renderUsageReport, type UsageReport, type ReportTotals } from "./cost/report.ts";
|
|
20
20
|
export { buildDailySummary, renderDailySummary, type DailySummary } from "./cost/summary.ts";
|
|
21
21
|
export { openDb } from "./util/sqlite.ts";
|
|
22
|
+
export { spendUsdSince, feedbackView, exportRows, exportCsv, harnessScopeParam, type HarnessScope, type ExportRow, type FeedbackRow, type FeedbackByModel, type FeedbackView } from "./cost/views.ts";
|
|
22
23
|
export { createLedger } from "./cost/ledger.ts";
|
|
24
|
+
export { createFeedbackStore, type FeedbackStore, type FeedbackRecord } from "./cost/feedback.ts";
|
|
25
|
+
export type { RequestPolicy } from "./wire/types.ts";
|
|
23
26
|
export type { Ledger, LedgerEntry } from "./cost/types.ts";
|
package/src/router/index.ts
CHANGED
|
@@ -15,7 +15,7 @@ import type { ProfileConfig, RouterConfig } from "../config/types.ts";
|
|
|
15
15
|
import type { Ledger } from "../cost/types.ts";
|
|
16
16
|
import { estimatePromptTokens } from "../tokens/estimate.ts";
|
|
17
17
|
import type { UpstreamClient } from "../upstream/types.ts";
|
|
18
|
-
import type { NormRequest } from "../wire/types.ts";
|
|
18
|
+
import type { NormRequest, RequestPolicy } from "../wire/types.ts";
|
|
19
19
|
import { classify, classifyTask } from "./classify.ts";
|
|
20
20
|
import { extractFeatures } from "./features.ts";
|
|
21
21
|
import { select } from "./select.ts";
|
|
@@ -38,6 +38,43 @@ export interface RouterDeps {
|
|
|
38
38
|
*/
|
|
39
39
|
const NEUTRAL_TOKENIZER = "gpt";
|
|
40
40
|
|
|
41
|
+
const TIER_RANK: Record<string, number> = { trivial: 0, simple: 1, moderate: 2, hard: 3 };
|
|
42
|
+
|
|
43
|
+
/**
|
|
44
|
+
* Applies a request policy on top of the resolved profile and config: the
|
|
45
|
+
* tier envelope is narrowed (never widened), a request allow list replaces
|
|
46
|
+
* the configured one, a request deny list adds to it, and a pin becomes a
|
|
47
|
+
* forced slug unless a session override already forced one.
|
|
48
|
+
*/
|
|
49
|
+
export function applyRequestPolicy(
|
|
50
|
+
profile: ProfileConfig,
|
|
51
|
+
cfg: RouterConfig,
|
|
52
|
+
policy: RequestPolicy | undefined,
|
|
53
|
+
forceSlug: string | undefined,
|
|
54
|
+
): { profile: ProfileConfig; cfg: RouterConfig; forceSlug: string | undefined; reasons: string[] } {
|
|
55
|
+
if (policy === undefined) return { profile, cfg, forceSlug, reasons: [] };
|
|
56
|
+
const reasons: string[] = [];
|
|
57
|
+
let minTier = profile.minTier;
|
|
58
|
+
let maxTier = profile.maxTier;
|
|
59
|
+
if (policy.minTier !== undefined && TIER_RANK[policy.minTier]! > TIER_RANK[minTier]!) minTier = policy.minTier;
|
|
60
|
+
if (policy.maxTier !== undefined && TIER_RANK[policy.maxTier]! < TIER_RANK[maxTier]!) maxTier = policy.maxTier;
|
|
61
|
+
if (TIER_RANK[minTier]! > TIER_RANK[maxTier]!) minTier = maxTier;
|
|
62
|
+
const narrowed = minTier !== profile.minTier || maxTier !== profile.maxTier;
|
|
63
|
+
const outProfile = narrowed ? { ...profile, id: `${profile.id}+policy`, minTier, maxTier } : profile;
|
|
64
|
+
if (narrowed) reasons.push(`policy: tiers narrowed to [${minTier}..${maxTier}]`);
|
|
65
|
+
let outCfg = cfg;
|
|
66
|
+
if (policy.allow !== undefined || policy.deny !== undefined) {
|
|
67
|
+
outCfg = { ...cfg, filters: { ...cfg.filters, ...(policy.allow === undefined ? {} : { allow: policy.allow }), ...(policy.deny === undefined ? {} : { deny: [...cfg.filters.deny, ...policy.deny] }) } };
|
|
68
|
+
reasons.push(`policy: ${policy.allow === undefined ? "" : `allow ${policy.allow.join("|")} `}${policy.deny === undefined ? "" : `deny ${policy.deny.join("|")}`}`.trim());
|
|
69
|
+
}
|
|
70
|
+
let outForce = forceSlug;
|
|
71
|
+
if (forceSlug === undefined && policy.pin !== undefined) {
|
|
72
|
+
outForce = policy.pin;
|
|
73
|
+
reasons.push(`policy: pinned to ${policy.pin}`);
|
|
74
|
+
}
|
|
75
|
+
return { profile: outProfile, cfg: outCfg, forceSlug: outForce, reasons };
|
|
76
|
+
}
|
|
77
|
+
|
|
41
78
|
export function resolveProfile(cfg: RouterConfig, requestedModel: string, isSubagent = false): ProfileConfig {
|
|
42
79
|
const fallback = cfg.profiles[0];
|
|
43
80
|
if (fallback === undefined) throw new Error("no router profiles configured");
|
|
@@ -99,19 +136,22 @@ export function createRouter(deps: RouterDeps): Router {
|
|
|
99
136
|
classification = await classify(req, features, config, { upstream, ledger, catalog });
|
|
100
137
|
}
|
|
101
138
|
|
|
102
|
-
|
|
139
|
+
const policed = applyRequestPolicy(resolveProfile(config, req.requestedModel, req.isSubagent), config, req.policy, opts.forceSlug);
|
|
140
|
+
const decision = select({
|
|
103
141
|
req,
|
|
104
142
|
features,
|
|
105
143
|
classification,
|
|
106
|
-
profile:
|
|
144
|
+
profile: policed.profile,
|
|
107
145
|
state,
|
|
108
146
|
snapshot,
|
|
109
147
|
ledger,
|
|
110
|
-
cfg:
|
|
148
|
+
cfg: policed.cfg,
|
|
111
149
|
nowMs: Date.now(),
|
|
112
150
|
...(opts.excludeSlugs === undefined ? {} : { excludeSlugs: opts.excludeSlugs }),
|
|
113
|
-
...(
|
|
151
|
+
...(policed.forceSlug === undefined ? {} : { forceSlug: policed.forceSlug }),
|
|
114
152
|
});
|
|
153
|
+
if (policed.reasons.length > 0) decision.reasons.unshift(...policed.reasons);
|
|
154
|
+
return decision;
|
|
115
155
|
},
|
|
116
156
|
};
|
|
117
157
|
}
|
package/src/server/http.ts
CHANGED
|
@@ -10,6 +10,7 @@ import { createDigester } from "./digest.ts";
|
|
|
10
10
|
import { advise } from "./advise.ts";
|
|
11
11
|
import { TIER_ORDER, type Tier } from "../router/types.ts";
|
|
12
12
|
import { baselinePrices, buildUsageReport, renderUsageReport } from "../cost/report.ts";
|
|
13
|
+
import { exportCsv, exportRows, feedbackView, harnessScopeParam, spendUsdSince } from "../cost/views.ts";
|
|
13
14
|
import { buildDailySummary, createKv, markSummaryShown, renderDailySummary, summaryDue, summaryHasNews, type SummaryOllama } from "../cost/summary.ts";
|
|
14
15
|
import type { Ledger, ModelTrust } from "../cost/types.ts";
|
|
15
16
|
import { createRouter } from "../router/index.ts";
|
|
@@ -186,6 +187,12 @@ function isLoopbackHostHeader(hostHeader: string | null): boolean {
|
|
|
186
187
|
return host === "127.0.0.1" || host === "localhost" || host === "::1" || host === "[::1]";
|
|
187
188
|
}
|
|
188
189
|
|
|
190
|
+
/** `?days=` bounded to [1, 365], `dflt` when absent or unparsable. */
|
|
191
|
+
function clampDays(raw: string | null, dflt: number): number {
|
|
192
|
+
const n = raw === null ? dflt : Number.parseInt(raw, 10);
|
|
193
|
+
return Number.isInteger(n) ? Math.min(Math.max(n, 1), 365) : dflt;
|
|
194
|
+
}
|
|
195
|
+
|
|
189
196
|
export function startServer(cfg: RouterConfig): StartedServer {
|
|
190
197
|
const log = createLogger(cfg.logLevel);
|
|
191
198
|
|
|
@@ -422,12 +429,27 @@ export function startServer(cfg: RouterConfig): StartedServer {
|
|
|
422
429
|
if (req.method === "GET" && url.pathname === "/v1/router/stats") {
|
|
423
430
|
return json(computeStats(ledger));
|
|
424
431
|
}
|
|
432
|
+
if (req.method === "GET" && url.pathname === "/v1/router/spend") {
|
|
433
|
+
// Spend since an instant over a harness set: what a front door's
|
|
434
|
+
// budget check needs when it cannot read the ledger file.
|
|
435
|
+
const since = Number.parseInt(url.searchParams.get("sinceMs") ?? "", 10);
|
|
436
|
+
if (!Number.isFinite(since)) return wireErrorResponse({ status: 400, code: "invalid_request_error", message: "sinceMs required" });
|
|
437
|
+
return json({ sinceMs: since, usd: spendUsdSince(db, since, harnessScopeParam(url.searchParams.get("harness"))) });
|
|
438
|
+
}
|
|
439
|
+
if (req.method === "GET" && url.pathname === "/v1/router/feedback") {
|
|
440
|
+
const days = clampDays(url.searchParams.get("days"), 30);
|
|
441
|
+
return json({ days, ...feedbackView(db, Date.now() - days * 86_400_000, harnessScopeParam(url.searchParams.get("harness"))) });
|
|
442
|
+
}
|
|
443
|
+
if (req.method === "GET" && url.pathname === "/v1/router/export") {
|
|
444
|
+
const days = clampDays(url.searchParams.get("days"), 30);
|
|
445
|
+
const rows = exportRows(db, Date.now() - days * 86_400_000, harnessScopeParam(url.searchParams.get("harness")));
|
|
446
|
+
if (url.searchParams.get("format") === "json") return json({ days, rows });
|
|
447
|
+
return new Response(exportCsv(rows), { headers: { "content-type": "text/csv; charset=utf-8", "content-disposition": `attachment; filename="auto-model-router-export-${new Date().toISOString().slice(0, 10)}.csv"` } });
|
|
448
|
+
}
|
|
425
449
|
if (req.method === "GET" && url.pathname === "/v1/router/report") {
|
|
426
450
|
// Usage analytics for `/router report` and the CLI: bounded window,
|
|
427
451
|
// optional harness scope (the X-Omp-Harness header value).
|
|
428
|
-
const
|
|
429
|
-
const parsedDays = rawDays === null ? 7 : Number.parseInt(rawDays, 10);
|
|
430
|
-
const windowDays = Number.isInteger(parsedDays) ? Math.min(Math.max(parsedDays, 1), 365) : 7;
|
|
452
|
+
const windowDays = clampDays(url.searchParams.get("days"), 7);
|
|
431
453
|
const harnessId = url.searchParams.get("harness") ?? "";
|
|
432
454
|
const report = buildUsageReport(db, { windowDays, harnessId, baselines: baselinePrices(cfg.report.baselines, (s) => catalog.find(s)) });
|
|
433
455
|
// ?format=text: the rendered report for harnesses without a renderer of their own (the Hermes plugin).
|
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import type {
|
|
2
|
+
RequestPolicy,
|
|
2
3
|
CompactionEdit,
|
|
3
4
|
NormMessage,
|
|
4
5
|
NormRequest,
|
|
@@ -285,6 +286,34 @@ function renderUpstreamBody(
|
|
|
285
286
|
return body;
|
|
286
287
|
}
|
|
287
288
|
|
|
289
|
+
const TIER_NAMES = new Set(["trivial", "simple", "moderate", "hard"]);
|
|
290
|
+
|
|
291
|
+
/** Parses the X-Omp-Policy header; malformed or empty ⇒ no policy (never a rejected turn). */
|
|
292
|
+
export function parsePolicyHeader(raw: string | null): RequestPolicy | undefined {
|
|
293
|
+
if (raw === null || raw.trim() === "") return undefined;
|
|
294
|
+
let parsed: unknown;
|
|
295
|
+
try {
|
|
296
|
+
parsed = JSON.parse(raw);
|
|
297
|
+
} catch {
|
|
298
|
+
return undefined;
|
|
299
|
+
}
|
|
300
|
+
if (parsed === null || typeof parsed !== "object" || Array.isArray(parsed)) return undefined;
|
|
301
|
+
const p = parsed as Record<string, unknown>;
|
|
302
|
+
const strs = (v: unknown): string[] | undefined => (Array.isArray(v) ? v.filter((s): s is string => typeof s === "string" && s.trim() !== "").map((s) => s.trim()) : undefined);
|
|
303
|
+
const tier = (v: unknown): RequestPolicy["minTier"] => (typeof v === "string" && TIER_NAMES.has(v) ? (v as RequestPolicy["minTier"]) : undefined);
|
|
304
|
+
const out: RequestPolicy = {};
|
|
305
|
+
const allow = strs(p.allow);
|
|
306
|
+
const deny = strs(p.deny);
|
|
307
|
+
if (allow !== undefined && allow.length > 0) out.allow = allow;
|
|
308
|
+
if (deny !== undefined && deny.length > 0) out.deny = deny;
|
|
309
|
+
const min = tier(p.minTier);
|
|
310
|
+
const max = tier(p.maxTier);
|
|
311
|
+
if (min !== undefined) out.minTier = min;
|
|
312
|
+
if (max !== undefined) out.maxTier = max;
|
|
313
|
+
if (typeof p.pin === "string" && p.pin.trim() !== "") out.pin = p.pin.trim();
|
|
314
|
+
return Object.keys(out).length === 0 ? undefined : out;
|
|
315
|
+
}
|
|
316
|
+
|
|
288
317
|
export function parseChatRequest(body: unknown, headers: Headers): NormRequest {
|
|
289
318
|
if (typeof body !== "object" || body === null || Array.isArray(body)) {
|
|
290
319
|
throw invalidRequest("Request body must be a JSON object");
|
|
@@ -306,6 +335,9 @@ export function parseChatRequest(body: unknown, headers: Headers): NormRequest {
|
|
|
306
335
|
// Subagent marker from the embed extension (sessions without a UI).
|
|
307
336
|
const isSubagent = (headers.get("x-omp-subagent") ?? "").trim() === "1";
|
|
308
337
|
|
|
338
|
+
// Per-request routing policy (team edition): JSON in X-Omp-Policy.
|
|
339
|
+
const policy = parsePolicyHeader(headers.get("x-omp-policy"));
|
|
340
|
+
|
|
309
341
|
if (typeof b.model !== "string" || b.model.length === 0) {
|
|
310
342
|
throw invalidRequest("model must be a non-empty string");
|
|
311
343
|
}
|
|
@@ -367,6 +399,7 @@ export function parseChatRequest(body: unknown, headers: Headers): NormRequest {
|
|
|
367
399
|
ompSessionId,
|
|
368
400
|
agentdoxScope,
|
|
369
401
|
isSubagent,
|
|
402
|
+
...(policy === undefined ? {} : { policy }),
|
|
370
403
|
requestedModel,
|
|
371
404
|
messages,
|
|
372
405
|
tools,
|
package/src/wire/types.ts
CHANGED
|
@@ -12,6 +12,20 @@ import type { UsageCounts } from "../cost/types.ts";
|
|
|
12
12
|
|
|
13
13
|
export type WireProtocol = "openai-chat" | "openai-responses" | "pi-native";
|
|
14
14
|
|
|
15
|
+
/**
|
|
16
|
+
* A routing policy attached to one request. `allow`/`deny` are slug globs
|
|
17
|
+
* like `filters.allow`/`filters.deny` (a request allow list replaces the
|
|
18
|
+
* configured one; a deny list adds to it); `minTier`/`maxTier` narrow the
|
|
19
|
+
* profile's tier envelope; `pin` forces one slug, like `/router pin`.
|
|
20
|
+
*/
|
|
21
|
+
export interface RequestPolicy {
|
|
22
|
+
allow?: string[];
|
|
23
|
+
deny?: string[];
|
|
24
|
+
minTier?: "trivial" | "simple" | "moderate" | "hard";
|
|
25
|
+
maxTier?: "trivial" | "simple" | "moderate" | "hard";
|
|
26
|
+
pin?: string;
|
|
27
|
+
}
|
|
28
|
+
|
|
15
29
|
export type Role = "system" | "developer" | "user" | "assistant" | "tool";
|
|
16
30
|
|
|
17
31
|
/** One tool call requested by an assistant turn. */
|
|
@@ -84,6 +98,12 @@ export interface NormRequest {
|
|
|
84
98
|
agentdoxScope: string;
|
|
85
99
|
/** `X-Omp-Subagent: 1`: the caller is an omp subagent (a session without a UI). */
|
|
86
100
|
isSubagent: boolean;
|
|
101
|
+
/**
|
|
102
|
+
* Per-request routing policy from the `X-Omp-Policy` header (JSON), set by
|
|
103
|
+
* a front door such as the team edition: narrows what this turn may route
|
|
104
|
+
* to. Absent ⇒ the configured profile and filters alone.
|
|
105
|
+
*/
|
|
106
|
+
policy?: RequestPolicy;
|
|
87
107
|
/** Virtual model the client selected, e.g. `auto`, `auto-cheap`, `auto-max`. */
|
|
88
108
|
requestedModel: string;
|
|
89
109
|
messages: NormMessage[];
|
|
@@ -0,0 +1,56 @@
|
|
|
1
|
+
import { describe, expect, test } from "bun:test";
|
|
2
|
+
|
|
3
|
+
import { DEFAULT_CONFIG } from "../src/config/defaults.ts";
|
|
4
|
+
import { applyRequestPolicy, resolveProfile } from "../src/router/index.ts";
|
|
5
|
+
import { parseChatRequest, parsePolicyHeader } from "../src/wire/openai/request.ts";
|
|
6
|
+
|
|
7
|
+
/**
|
|
8
|
+
* The per-request routing policy (X-Omp-Policy): parsed defensively from the
|
|
9
|
+
* header, then applied on top of the profile and filters — tiers only narrow,
|
|
10
|
+
* an allow list replaces the configured one, a deny list adds to it, and a
|
|
11
|
+
* pin forces a slug unless a session override already did.
|
|
12
|
+
*/
|
|
13
|
+
|
|
14
|
+
describe("parsePolicyHeader", () => {
|
|
15
|
+
test("accepts the documented fields, drops junk, and never rejects a turn", () => {
|
|
16
|
+
expect(parsePolicyHeader(null)).toBeUndefined();
|
|
17
|
+
expect(parsePolicyHeader("not json")).toBeUndefined();
|
|
18
|
+
expect(parsePolicyHeader("[]")).toBeUndefined();
|
|
19
|
+
expect(parsePolicyHeader("{}")).toBeUndefined();
|
|
20
|
+
expect(parsePolicyHeader(JSON.stringify({ allow: ["anthropic/*", " x/y "], deny: [1, "", "openai/*"], minTier: "simple", maxTier: "nope", pin: " z/w " }))).toEqual({ allow: ["anthropic/*", "x/y"], deny: ["openai/*"], minTier: "simple", pin: "z/w" });
|
|
21
|
+
const req = parseChatRequest({ model: "auto", messages: [{ role: "user", content: "hi" }] }, new Headers({ "X-Omp-Policy": '{"maxTier":"moderate"}' }));
|
|
22
|
+
expect(req.policy).toEqual({ maxTier: "moderate" });
|
|
23
|
+
expect("policy" in parseChatRequest({ model: "auto", messages: [{ role: "user", content: "hi" }] }, new Headers())).toBe(false);
|
|
24
|
+
});
|
|
25
|
+
});
|
|
26
|
+
|
|
27
|
+
describe("applyRequestPolicy", () => {
|
|
28
|
+
const cfg = DEFAULT_CONFIG;
|
|
29
|
+
const profile = resolveProfile(cfg, "auto");
|
|
30
|
+
|
|
31
|
+
test("narrows the tier envelope, never widens it", () => {
|
|
32
|
+
const r = applyRequestPolicy(profile, cfg, { maxTier: "moderate", minTier: "trivial" }, undefined);
|
|
33
|
+
expect(r.profile.maxTier).toBe("moderate");
|
|
34
|
+
expect(r.profile.minTier).toBe(profile.minTier);
|
|
35
|
+
expect(r.profile.id).toBe(`${profile.id}+policy`);
|
|
36
|
+
expect(r.reasons[0]).toContain("tiers narrowed");
|
|
37
|
+
// A cheap profile cannot be raised past its own ceiling.
|
|
38
|
+
const cheap = resolveProfile(cfg, "auto-cheap");
|
|
39
|
+
const up = applyRequestPolicy(cheap, cfg, { minTier: "hard" }, undefined);
|
|
40
|
+
expect(up.profile.minTier).toBe(cheap.maxTier);
|
|
41
|
+
expect(up.profile.maxTier).toBe(cheap.maxTier);
|
|
42
|
+
});
|
|
43
|
+
|
|
44
|
+
test("allow replaces, deny adds, and a pin forces unless a session override already did", () => {
|
|
45
|
+
const base = { ...cfg, filters: { ...cfg.filters, allow: ["x/*"], deny: ["bad/*"] } };
|
|
46
|
+
const r = applyRequestPolicy(profile, base, { allow: ["anthropic/*"], deny: ["openai/*"], pin: "anthropic/claude-sonnet-5" }, undefined);
|
|
47
|
+
expect(r.cfg.filters.allow).toEqual(["anthropic/*"]);
|
|
48
|
+
expect(r.cfg.filters.deny).toEqual(["bad/*", "openai/*"]);
|
|
49
|
+
expect(r.forceSlug).toBe("anthropic/claude-sonnet-5");
|
|
50
|
+
expect(r.profile).toBe(profile); // tiers untouched ⇒ same object
|
|
51
|
+
expect(applyRequestPolicy(profile, base, { pin: "a/b" }, "session/pin").forceSlug).toBe("session/pin");
|
|
52
|
+
expect(applyRequestPolicy(profile, base, undefined, undefined)).toEqual({ profile, cfg: base, forceSlug: undefined, reasons: [] });
|
|
53
|
+
// Untouched config object when the policy carries no filters.
|
|
54
|
+
expect(applyRequestPolicy(profile, base, { maxTier: "hard" }, undefined).cfg).toBe(base);
|
|
55
|
+
});
|
|
56
|
+
});
|
|
@@ -0,0 +1,161 @@
|
|
|
1
|
+
import { afterAll, beforeAll, describe, expect, test } from "bun:test";
|
|
2
|
+
import { mkdtempSync, rmSync } from "node:fs";
|
|
3
|
+
import { tmpdir } from "node:os";
|
|
4
|
+
import { join } from "node:path";
|
|
5
|
+
|
|
6
|
+
import { DEFAULT_CONFIG } from "../src/config/defaults.ts";
|
|
7
|
+
import { createFeedbackStore } from "../src/cost/feedback.ts";
|
|
8
|
+
import { createLedger } from "../src/cost/ledger.ts";
|
|
9
|
+
import type { LedgerEntry } from "../src/cost/types.ts";
|
|
10
|
+
import { exportCsv, exportRows, feedbackView, harnessScopeParam, spendUsdSince } from "../src/cost/views.ts";
|
|
11
|
+
import { startServer, type StartedServer } from "../src/server/http.ts";
|
|
12
|
+
import { openDb } from "../src/util/sqlite.ts";
|
|
13
|
+
|
|
14
|
+
/**
|
|
15
|
+
* The ledger views a front door reads instead of the ledger file: spend over
|
|
16
|
+
* a harness set, feedback with the judging harness, and the day × harness ×
|
|
17
|
+
* model export. Pinned over the public functions and over the HTTP routes.
|
|
18
|
+
*/
|
|
19
|
+
|
|
20
|
+
const DAY = 86_400_000;
|
|
21
|
+
const NOW = Date.UTC(2026, 8, 7, 12, 0, 0);
|
|
22
|
+
|
|
23
|
+
function entry(over: Partial<LedgerEntry>): LedgerEntry {
|
|
24
|
+
return {
|
|
25
|
+
id: crypto.randomUUID(),
|
|
26
|
+
createdAtMs: NOW - 3_600_000,
|
|
27
|
+
conversationKey: "k",
|
|
28
|
+
sessionId: "s",
|
|
29
|
+
turn: 1,
|
|
30
|
+
requestedModel: "auto",
|
|
31
|
+
harnessId: "",
|
|
32
|
+
ompSessionId: "",
|
|
33
|
+
slug: "vendor/model",
|
|
34
|
+
servedSlug: "vendor/model",
|
|
35
|
+
tier: "simple",
|
|
36
|
+
classificationSource: "heuristic",
|
|
37
|
+
reasons: [],
|
|
38
|
+
features: null,
|
|
39
|
+
score: null,
|
|
40
|
+
confidence: null,
|
|
41
|
+
task: null,
|
|
42
|
+
classifierReasons: null,
|
|
43
|
+
exploredFrom: null,
|
|
44
|
+
holdArm: null,
|
|
45
|
+
predictedUsd: 0.001,
|
|
46
|
+
reportedUsd: 0.001,
|
|
47
|
+
usage: { promptTokens: 1000, cachedTokens: 400, cacheWriteTokens: 0, completionTokens: 50, reasoningTokens: 0, images: 0 },
|
|
48
|
+
attempt: 0,
|
|
49
|
+
escalationSignal: null,
|
|
50
|
+
latencyMs: 1_100,
|
|
51
|
+
ttftMs: 100,
|
|
52
|
+
finishReason: "stop",
|
|
53
|
+
wasted: false,
|
|
54
|
+
upstreamGenerationId: null,
|
|
55
|
+
error: null,
|
|
56
|
+
promptTokensSaved: null,
|
|
57
|
+
...over,
|
|
58
|
+
} as LedgerEntry;
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
function seeded() {
|
|
62
|
+
const cfg = structuredClone(DEFAULT_CONFIG);
|
|
63
|
+
cfg.ledger.path = ":memory:";
|
|
64
|
+
const db = openDb(":memory:");
|
|
65
|
+
const ledger = createLedger(db, cfg);
|
|
66
|
+
const feedback = createFeedbackStore(db);
|
|
67
|
+
ledger.record(entry({ id: "l1", harnessId: "u_ada", slug: "anthropic/claude-sonnet-5", servedSlug: "anthropic/claude-sonnet-5", predictedUsd: 0.01, reportedUsd: 0.012 }));
|
|
68
|
+
ledger.record(entry({ id: "l2", harnessId: "u_ada", slug: "anthropic/claude-sonnet-5", servedSlug: null, predictedUsd: 0.01, reportedUsd: null, escalationSignal: "circular" }));
|
|
69
|
+
ledger.record(entry({ id: "l3", harnessId: "u_bob", slug: "ollama/glm-5.3-flash", servedSlug: "ollama/glm-5.3-flash", predictedUsd: 0.001, reportedUsd: 0.001, error: "boom" }));
|
|
70
|
+
ledger.record(entry({ id: "l4", harnessId: "u_bob", requestedModel: "digest", slug: "ollama/glm-5.3-flash", servedSlug: "ollama/glm-5.3-flash", predictedUsd: 0.5, reportedUsd: 0.5 }));
|
|
71
|
+
ledger.record(entry({ id: "l5", harnessId: "u_bob", createdAtMs: NOW - 40 * DAY, slug: "ollama/glm-5.3-flash", predictedUsd: 5, reportedUsd: 5 }));
|
|
72
|
+
feedback.record({ ledgerId: "l1", ompSessionId: "s", slug: "anthropic/claude-sonnet-5", tier: "simple", verdict: "good", note: "" }, NOW - 1000);
|
|
73
|
+
feedback.record({ ledgerId: "l2", ompSessionId: "s", slug: "anthropic/claude-sonnet-5", tier: "simple", verdict: "bad", note: "" }, NOW - 900);
|
|
74
|
+
feedback.record({ ledgerId: "l3", ompSessionId: "s", slug: "ollama/glm-5.3-flash", tier: "simple", verdict: "bad", note: "looped" }, NOW - 800);
|
|
75
|
+
return db;
|
|
76
|
+
}
|
|
77
|
+
|
|
78
|
+
describe("ledger views", () => {
|
|
79
|
+
const db = seeded();
|
|
80
|
+
const since = NOW - DAY;
|
|
81
|
+
|
|
82
|
+
test("spend over a harness set, everything, or nothing", () => {
|
|
83
|
+
expect(spendUsdSince(db, since, ["u_ada"])).toBeCloseTo(0.022, 6); // reported where present, predicted otherwise
|
|
84
|
+
expect(spendUsdSince(db, since, ["u_ada", "u_bob"])).toBeCloseTo(0.523, 6); // the digest row counts as spend
|
|
85
|
+
expect(spendUsdSince(db, since, null)).toBeCloseTo(0.523, 6);
|
|
86
|
+
expect(spendUsdSince(db, NOW - 60 * DAY, null)).toBeCloseTo(5.523, 6);
|
|
87
|
+
expect(spendUsdSince(db, since, [])).toBe(0);
|
|
88
|
+
});
|
|
89
|
+
|
|
90
|
+
test("feedback by model with distinct judges, scoped by harness", () => {
|
|
91
|
+
const all = feedbackView(db, since, null);
|
|
92
|
+
expect(all.byModel).toEqual([
|
|
93
|
+
{ slug: "anthropic/claude-sonnet-5", good: 1, bad: 1, judges: 1 },
|
|
94
|
+
{ slug: "ollama/glm-5.3-flash", good: 0, bad: 1, judges: 1 },
|
|
95
|
+
]);
|
|
96
|
+
expect(all.recent.map((r) => [r.harnessId, r.verdict, r.note])).toEqual([
|
|
97
|
+
["u_bob", "bad", "looped"],
|
|
98
|
+
["u_ada", "bad", ""],
|
|
99
|
+
["u_ada", "good", ""],
|
|
100
|
+
]);
|
|
101
|
+
expect(feedbackView(db, since, ["u_bob"]).byModel).toEqual([{ slug: "ollama/glm-5.3-flash", good: 0, bad: 1, judges: 1 }]);
|
|
102
|
+
expect(feedbackView(db, since, []).recent).toEqual([]);
|
|
103
|
+
});
|
|
104
|
+
|
|
105
|
+
test("export rows by day, harness and served model; digest and old rows out; CSV quoting", () => {
|
|
106
|
+
const rows = exportRows(db, since, null);
|
|
107
|
+
expect(rows).toHaveLength(2);
|
|
108
|
+
expect(rows[0]).toMatchObject({ day: "2026-09-07", harnessId: "u_ada", slug: "anthropic/claude-sonnet-5", provider: "openrouter", dispatches: 2, promptTokens: 2000, cachedTokens: 800, completionTokens: 100, escalations: 1, errors: 0 });
|
|
109
|
+
expect(rows[0]!.spendUsd).toBeCloseTo(0.022, 6);
|
|
110
|
+
expect(rows[1]).toMatchObject({ harnessId: "u_bob", provider: "ollama", dispatches: 1, errors: 1 });
|
|
111
|
+
expect(exportRows(db, since, ["u_bob"])).toHaveLength(1);
|
|
112
|
+
expect(exportRows(db, since, [])).toEqual([]);
|
|
113
|
+
const csv = exportCsv([{ ...rows[0]!, harnessId: 'ada, "L"' }]);
|
|
114
|
+
expect(csv.split("\n")[0]).toBe("day,harness,model,provider,dispatches,prompt_tokens,cached_tokens,completion_tokens,spend_usd,escalations,errors");
|
|
115
|
+
expect(csv.split("\n")[1]).toBe('2026-09-07,"ada, ""L""",anthropic/claude-sonnet-5,openrouter,2,2000,800,100,0.022000,1,0');
|
|
116
|
+
expect(harnessScopeParam(null)).toBeNull();
|
|
117
|
+
expect(harnessScopeParam(" , ")).toBeNull();
|
|
118
|
+
expect(harnessScopeParam("a, b")).toEqual(["a", "b"]);
|
|
119
|
+
});
|
|
120
|
+
});
|
|
121
|
+
|
|
122
|
+
describe("view routes", () => {
|
|
123
|
+
let handle: StartedServer;
|
|
124
|
+
const dir = mkdtempSync(join(tmpdir(), "amr-views-"));
|
|
125
|
+
beforeAll(() => {
|
|
126
|
+
const cfg = structuredClone(DEFAULT_CONFIG);
|
|
127
|
+
cfg.server.host = "127.0.0.1";
|
|
128
|
+
cfg.server.port = 0;
|
|
129
|
+
cfg.server.apiKey = "k";
|
|
130
|
+
cfg.ledger.path = join(dir, "router.db");
|
|
131
|
+
// Seed through the ledger on the same file before the server opens it.
|
|
132
|
+
const db = openDb(cfg.ledger.path);
|
|
133
|
+
createLedger(db, cfg).record(entry({ id: "r1", createdAtMs: Date.now() - 1000, harnessId: "u_x", predictedUsd: 0.2, reportedUsd: 0.25 }));
|
|
134
|
+
db.close();
|
|
135
|
+
handle = startServer(cfg);
|
|
136
|
+
});
|
|
137
|
+
afterAll(async () => {
|
|
138
|
+
await handle.stop();
|
|
139
|
+
try {
|
|
140
|
+
rmSync(dir, { recursive: true, force: true });
|
|
141
|
+
} catch {
|
|
142
|
+
/* Windows may hold the WAL briefly */
|
|
143
|
+
}
|
|
144
|
+
});
|
|
145
|
+
const get = (path: string) => fetch(`http://127.0.0.1:${handle.server.port}${path}`, { headers: { authorization: "Bearer k" } });
|
|
146
|
+
|
|
147
|
+
test("spend, feedback and export answer with the auth every router route needs", async () => {
|
|
148
|
+
expect((await fetch(`http://127.0.0.1:${handle.server.port}/v1/router/spend?sinceMs=0`)).status).toBe(401);
|
|
149
|
+
expect((await get("/v1/router/spend")).status).toBe(400);
|
|
150
|
+
expect(((await (await get(`/v1/router/spend?sinceMs=${Date.now() - DAY}&harness=u_x`)).json()) as { usd: number }).usd).toBeCloseTo(0.25, 6);
|
|
151
|
+
expect(((await (await get(`/v1/router/spend?sinceMs=${Date.now() - DAY}&harness=u_other`)).json()) as { usd: number }).usd).toBe(0);
|
|
152
|
+
const fb = (await (await get("/v1/router/feedback?days=7")).json()) as { days: number; byModel: unknown[]; recent: unknown[] };
|
|
153
|
+
expect(fb).toEqual({ days: 7, byModel: [], recent: [] });
|
|
154
|
+
const csv = await get("/v1/router/export?days=1");
|
|
155
|
+
expect(csv.headers.get("content-type")).toContain("text/csv");
|
|
156
|
+
expect((await csv.text()).split("\n")[1]).toContain("u_x,vendor/model,openrouter,1,1000,400,50,0.250000,0,0");
|
|
157
|
+
const js = (await (await get("/v1/router/export?days=1&format=json&harness=u_x")).json()) as { days: number; rows: { harnessId: string }[] };
|
|
158
|
+
expect(js.days).toBe(1);
|
|
159
|
+
expect(js.rows[0]?.harnessId).toBe("u_x");
|
|
160
|
+
});
|
|
161
|
+
});
|