auto-model-router 0.18.0 → 0.19.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.omp-plugin/marketplace.json +2 -2
- package/README.md +23 -5
- package/package.json +1 -1
- package/src/cost/ledger.ts +9 -2
- package/src/cost/types.ts +7 -0
- package/src/cost/views.ts +27 -10
- package/src/server/http.ts +3 -1
- package/src/server/turn.ts +4 -0
- package/src/util/sqlite.ts +12 -1
- package/test/migrations.test.ts +10 -2
- package/test/trust-attribution.test.ts +2 -2
- package/test/turn.test.ts +23 -0
- package/test/views.test.ts +39 -13
|
@@ -7,14 +7,14 @@
|
|
|
7
7
|
},
|
|
8
8
|
"metadata": {
|
|
9
9
|
"description": "auto-model-router: a local cost/complexity-aware model router for Oh My Pi, backed by OpenRouter",
|
|
10
|
-
"version": "0.
|
|
10
|
+
"version": "0.19.0",
|
|
11
11
|
"pluginRoot": "."
|
|
12
12
|
},
|
|
13
13
|
"plugins": [
|
|
14
14
|
{
|
|
15
15
|
"name": "auto-model-router",
|
|
16
16
|
"description": "Local cost/complexity-aware model router for Oh My Pi, backed by OpenRouter. Runs in-process, routes per turn by price and task complexity, with budget caps, mid-stream escalation, and cache-aware hysteresis.",
|
|
17
|
-
"version": "0.
|
|
17
|
+
"version": "0.19.0",
|
|
18
18
|
"author": {
|
|
19
19
|
"name": "drewappling",
|
|
20
20
|
"email": "drewappling@gmail.com"
|
package/README.md
CHANGED
|
@@ -694,7 +694,9 @@ virtual profile it picked. Every routed response carries
|
|
|
694
694
|
|
|
695
695
|
The ledger records every dispatch: model decided and served, tier, provider,
|
|
696
696
|
tokens (including cached), reported cost, time to first token, total latency,
|
|
697
|
-
escalation signal, error
|
|
697
|
+
escalation signal, error, and the agentdox context scope the turn carried (the
|
|
698
|
+
project it belongs to; NULL for a turn that carried none, and for every row
|
|
699
|
+
written before v0.19.0). Three views aggregate it, all from the same
|
|
698
700
|
`buildUsageReport` in `src/cost/report.ts`:
|
|
699
701
|
|
|
700
702
|
- `/router report` in omp — a fullscreen hub with the `/models` look: views
|
|
@@ -706,10 +708,12 @@ escalation signal, error. Three views aggregate it, all from the same
|
|
|
706
708
|
transcript. Falls back to reading the ledger directly if the router is
|
|
707
709
|
unreachable.
|
|
708
710
|
- `auto-model-router report --days 7 [--harness <id>] [--json]` on the terminal.
|
|
709
|
-
- `auto-model-router export --days 30 [--harness a,b] [--json]`: one row per day, harness
|
|
710
|
-
and
|
|
711
|
-
`
|
|
712
|
-
|
|
711
|
+
- `auto-model-router export --days 30 [--harness a,b] [--json]`: one row per day, harness,
|
|
712
|
+
model and context scope (dispatches, tokens, spend, escalations, errors) as CSV — the
|
|
713
|
+
`scope` column is last, and is empty for a turn that carried none. Also
|
|
714
|
+
`GET /v1/router/export?days=&harness=[&format=json]`; `GET /v1/router/spend?sinceMs=&harness=[&scope=]`
|
|
715
|
+
gives spend over a harness set since an instant, narrowed to one context scope when `scope`
|
|
716
|
+
is given (a project's own bill), and `GET /v1/router/feedback?days=&harness=`
|
|
713
717
|
lists verdicts by model and the recent ones with the harness that gave them, and
|
|
714
718
|
`GET /v1/router/decisions?harness=&days=|since=&slug=&tier=&limit=` is the decision trail
|
|
715
719
|
itself, newest first, each turn with its reasons, the classifier's view, forecast against
|
|
@@ -1555,6 +1559,20 @@ a team posts exactly what it always did, so its block is byte-identical; an olde
|
|
|
1555
1559
|
agentdox ignores the keys it does not know. `context.layers: false` is the kill
|
|
1556
1560
|
switch — the headers are still parsed but nothing new goes to agentdox.
|
|
1557
1561
|
|
|
1562
|
+
### Charging spend back to a project
|
|
1563
|
+
|
|
1564
|
+
Every ledger row records the scope the bridge resolved for that turn — the
|
|
1565
|
+
request's `X-Agentdox-Scope`, or `context.defaultScope` behind it — so the money
|
|
1566
|
+
and the project are the same row. That is what a front door bills from: it names
|
|
1567
|
+
a project's scope on the turns it forwards, then reads its share back with
|
|
1568
|
+
`GET /v1/router/spend?sinceMs=&scope=<scope>` (composable with `harness=`, so one
|
|
1569
|
+
member's spend on one project is one call), or takes the whole split from
|
|
1570
|
+
`GET /v1/router/export`, whose rows now group by day, harness, model **and**
|
|
1571
|
+
scope and carry the scope as their last CSV column. `decisionEntries` (and
|
|
1572
|
+
`GET /v1/router/decisions`) carry it on each turn too. Rows written before
|
|
1573
|
+
v0.19.0, and turns that carried no scope at all, store NULL and export as `""` —
|
|
1574
|
+
old ledgers open and gain the column, they just have nothing to charge.
|
|
1575
|
+
|
|
1558
1576
|
### The origin fingerprint
|
|
1559
1577
|
|
|
1560
1578
|
The scope is the folder's name, and folder names collide: two unrelated repositories
|
package/package.json
CHANGED
package/src/cost/ledger.ts
CHANGED
|
@@ -99,6 +99,7 @@ export interface LedgerRow {
|
|
|
99
99
|
upstream_generation_id: string | null;
|
|
100
100
|
error: string | null;
|
|
101
101
|
prompt_tokens_saved: number | null;
|
|
102
|
+
scope: string | null;
|
|
102
103
|
}
|
|
103
104
|
|
|
104
105
|
interface TrustRow {
|
|
@@ -274,6 +275,9 @@ export function toEntry(row: LedgerRow): LedgerEntry {
|
|
|
274
275
|
upstreamGenerationId: row.upstream_generation_id,
|
|
275
276
|
error: row.error,
|
|
276
277
|
promptTokensSaved: row.prompt_tokens_saved ?? 0,
|
|
278
|
+
// Optional under exactOptionalPropertyTypes: an old row (or a scopeless
|
|
279
|
+
// turn) simply has no `scope`, rather than an explicit undefined.
|
|
280
|
+
...(row.scope === null || row.scope === undefined ? {} : { scope: row.scope }),
|
|
277
281
|
};
|
|
278
282
|
}
|
|
279
283
|
|
|
@@ -319,8 +323,8 @@ export function createLedger(db: Database, cfg: RouterConfig): Ledger {
|
|
|
319
323
|
id, created_at_ms, conversation_key, session_id, turn, requested_model, harness_id, omp_session_id, slug, served_slug,
|
|
320
324
|
tier, classification_source, reasons, predicted_usd, reported_usd, usage, cost_breakdown,
|
|
321
325
|
attempt, escalation_signal, latency_ms, ttft_ms, finish_reason, wasted, upstream_generation_id, error,
|
|
322
|
-
error_kind, features, score, confidence, task, classifier_reasons, explored_from, hold_arm, prompt_tokens_saved
|
|
323
|
-
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,
|
|
326
|
+
error_kind, features, score, confidence, task, classifier_reasons, explored_from, hold_arm, prompt_tokens_saved, scope
|
|
327
|
+
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,
|
|
324
328
|
);
|
|
325
329
|
const calibrationStmt = db.query(
|
|
326
330
|
`INSERT INTO token_calibration (tokenizer, est_bytes, actual_tokens, samples) VALUES (?, ?, ?, 1)
|
|
@@ -473,6 +477,9 @@ export function createLedger(db: Database, cfg: RouterConfig): Ledger {
|
|
|
473
477
|
entry.exploredFrom,
|
|
474
478
|
entry.holdArm,
|
|
475
479
|
entry.promptTokensSaved,
|
|
480
|
+
// A turn that carried no scope stores NULL, exactly as every row
|
|
481
|
+
// written before v18 did; "" and absent are the same fact.
|
|
482
|
+
entry.scope === undefined || entry.scope === "" ? null : entry.scope,
|
|
476
483
|
);
|
|
477
484
|
// Always consume the pending estimate, even when the turn failed, so a
|
|
478
485
|
// dead turn's bytes can never pair with a later turn's tokens. Only
|
package/src/cost/types.ts
CHANGED
|
@@ -146,6 +146,13 @@ export interface LedgerEntry {
|
|
|
146
146
|
error: string | null;
|
|
147
147
|
/** Prompt tokens removed by compaction before dispatch. 0 when none. NULL before v12. */
|
|
148
148
|
promptTokensSaved: number;
|
|
149
|
+
/**
|
|
150
|
+
* The agentdox context scope this turn carried — what the bridge resolved
|
|
151
|
+
* for it (the request's `X-Agentdox-Scope`, or `context.defaultScope`).
|
|
152
|
+
* Absent, or empty, when the turn carried none, and stored as NULL; a front
|
|
153
|
+
* door charges the row's spend back to that project with it. NULL before v18.
|
|
154
|
+
*/
|
|
155
|
+
scope?: string;
|
|
149
156
|
/**
|
|
150
157
|
* The catalog model that served, for the cost split. The ledger can price
|
|
151
158
|
* OpenRouter slugs from its own cached catalog payload; a model from another
|
package/src/cost/views.ts
CHANGED
|
@@ -44,6 +44,8 @@ export interface ExportRow {
|
|
|
44
44
|
day: string;
|
|
45
45
|
harnessId: string;
|
|
46
46
|
slug: string;
|
|
47
|
+
/** The agentdox context scope the turns carried; "" for rows that carried none (every row before v18). */
|
|
48
|
+
scope: string;
|
|
47
49
|
provider: string;
|
|
48
50
|
dispatches: number;
|
|
49
51
|
promptTokens: number;
|
|
@@ -125,12 +127,21 @@ export function decisionEntries(db: Database, filter: DecisionFilter): DecisionE
|
|
|
125
127
|
return entries.map((e) => ({ ...e, feedback: verdicts.get(e.id) ?? [] }));
|
|
126
128
|
}
|
|
127
129
|
|
|
128
|
-
/**
|
|
129
|
-
|
|
130
|
+
/**
|
|
131
|
+
* Spend (reported where present, predicted otherwise) since `sinceMs`, digest calls included
|
|
132
|
+
* as the ledger counts them. `contextScope`, when given, narrows to the turns that carried
|
|
133
|
+
* exactly that agentdox scope — what a front door charges back to one project.
|
|
134
|
+
*/
|
|
135
|
+
export function spendUsdSince(db: Database, sinceMs: number, harness: HarnessScope, contextScope?: string): number {
|
|
130
136
|
const s = scope(harness, "harness_id");
|
|
131
137
|
if (s === null) return 0;
|
|
132
|
-
const where = ["created_at_ms >= $since", ...s.sql]
|
|
133
|
-
const
|
|
138
|
+
const where = ["created_at_ms >= $since", ...s.sql];
|
|
139
|
+
const bind: Record<string, string | number> = { $since: sinceMs, ...s.bind };
|
|
140
|
+
if (contextScope !== undefined && contextScope !== "") {
|
|
141
|
+
where.push("scope = $scope");
|
|
142
|
+
bind.$scope = contextScope;
|
|
143
|
+
}
|
|
144
|
+
const row = db.query(`SELECT COALESCE(SUM(${USD}), 0) AS usd FROM ledger WHERE ${where.join(" AND ")}`).get(bind) as { usd: number };
|
|
134
145
|
return row.usd;
|
|
135
146
|
}
|
|
136
147
|
|
|
@@ -163,14 +174,19 @@ export function feedbackView(db: Database, sinceMs: number, harness: HarnessScop
|
|
|
163
174
|
return { byModel, recent };
|
|
164
175
|
}
|
|
165
176
|
|
|
166
|
-
/**
|
|
177
|
+
/**
|
|
178
|
+
* One row per UTC day, harness, served model and context scope since `sinceMs`; digest calls
|
|
179
|
+
* are excluded as in the report. The scope splits a harness's day by project, so a front door
|
|
180
|
+
* can charge each project its own share; rows from before v18 (and turns that carried no
|
|
181
|
+
* scope) group under "".
|
|
182
|
+
*/
|
|
167
183
|
export function exportRows(db: Database, sinceMs: number, harness: HarnessScope): ExportRow[] {
|
|
168
184
|
const s = scope(harness, "harness_id");
|
|
169
185
|
if (s === null) return [];
|
|
170
186
|
const where = ["created_at_ms >= $since", "requested_model <> 'digest'", ...s.sql].join(" AND ");
|
|
171
187
|
const rows = db
|
|
172
188
|
.query(
|
|
173
|
-
`SELECT strftime('%Y-%m-%d', created_at_ms / 1000, 'unixepoch') AS day, harness_id, COALESCE(served_slug, slug) AS slug,
|
|
189
|
+
`SELECT strftime('%Y-%m-%d', created_at_ms / 1000, 'unixepoch') AS day, harness_id, COALESCE(served_slug, slug) AS slug, COALESCE(scope, '') AS scope,
|
|
174
190
|
COUNT(*) AS dispatches,
|
|
175
191
|
COALESCE(SUM(json_extract(usage, '$.promptTokens')), 0) AS prompt_tokens,
|
|
176
192
|
COALESCE(SUM(json_extract(usage, '$.cachedTokens')), 0) AS cached_tokens,
|
|
@@ -178,13 +194,14 @@ export function exportRows(db: Database, sinceMs: number, harness: HarnessScope)
|
|
|
178
194
|
COALESCE(SUM(${USD}), 0) AS spend,
|
|
179
195
|
SUM(CASE WHEN escalation_signal IS NOT NULL THEN 1 ELSE 0 END) AS escalations,
|
|
180
196
|
SUM(CASE WHEN error IS NOT NULL THEN 1 ELSE 0 END) AS errors
|
|
181
|
-
FROM ledger WHERE ${where} GROUP BY day, harness_id, slug ORDER BY day ASC, harness_id ASC, spend DESC`,
|
|
197
|
+
FROM ledger WHERE ${where} GROUP BY day, harness_id, slug, scope ORDER BY day ASC, harness_id ASC, spend DESC`,
|
|
182
198
|
)
|
|
183
|
-
.all({ $since: sinceMs, ...s.bind }) as { day: string; harness_id: string; slug: string; dispatches: number; prompt_tokens: number; cached_tokens: number; completion_tokens: number; spend: number; escalations: number; errors: number }[];
|
|
199
|
+
.all({ $since: sinceMs, ...s.bind }) as { day: string; harness_id: string; slug: string; scope: string; dispatches: number; prompt_tokens: number; cached_tokens: number; completion_tokens: number; spend: number; escalations: number; errors: number }[];
|
|
184
200
|
return rows.map((r) => ({
|
|
185
201
|
day: r.day,
|
|
186
202
|
harnessId: r.harness_id,
|
|
187
203
|
slug: r.slug,
|
|
204
|
+
scope: r.scope,
|
|
188
205
|
provider: providerOfSlug(r.slug),
|
|
189
206
|
dispatches: r.dispatches,
|
|
190
207
|
promptTokens: r.prompt_tokens,
|
|
@@ -196,7 +213,7 @@ export function exportRows(db: Database, sinceMs: number, harness: HarnessScope)
|
|
|
196
213
|
}));
|
|
197
214
|
}
|
|
198
215
|
|
|
199
|
-
export const EXPORT_COLUMNS = ["day", "harness", "model", "provider", "dispatches", "prompt_tokens", "cached_tokens", "completion_tokens", "spend_usd", "escalations", "errors"] as const;
|
|
216
|
+
export const EXPORT_COLUMNS = ["day", "harness", "model", "provider", "dispatches", "prompt_tokens", "cached_tokens", "completion_tokens", "spend_usd", "escalations", "errors", "scope"] as const;
|
|
200
217
|
|
|
201
218
|
export function csvCell(v: string | number): string {
|
|
202
219
|
const s = String(v);
|
|
@@ -206,7 +223,7 @@ export function csvCell(v: string | number): string {
|
|
|
206
223
|
/** CSV of export rows; spend to 6 decimals so sub-cent rows survive. */
|
|
207
224
|
export function exportCsv(rows: readonly ExportRow[]): string {
|
|
208
225
|
const lines = [EXPORT_COLUMNS.join(",")];
|
|
209
|
-
for (const r of rows) lines.push([r.day, r.harnessId, r.slug, r.provider, r.dispatches, r.promptTokens, r.cachedTokens, r.completionTokens, r.spendUsd.toFixed(6), r.escalations, r.errors].map(csvCell).join(","));
|
|
226
|
+
for (const r of rows) lines.push([r.day, r.harnessId, r.slug, r.provider, r.dispatches, r.promptTokens, r.cachedTokens, r.completionTokens, r.spendUsd.toFixed(6), r.escalations, r.errors, r.scope].map(csvCell).join(","));
|
|
210
227
|
return `${lines.join("\n")}\n`;
|
|
211
228
|
}
|
|
212
229
|
|
package/src/server/http.ts
CHANGED
|
@@ -527,7 +527,9 @@ export function startServer(cfg: RouterConfig): StartedServer {
|
|
|
527
527
|
// budget check needs when it cannot read the ledger file.
|
|
528
528
|
const since = Number.parseInt(url.searchParams.get("sinceMs") ?? "", 10);
|
|
529
529
|
if (!Number.isFinite(since)) return wireErrorResponse({ status: 400, code: "invalid_request_error", message: "sinceMs required" });
|
|
530
|
-
|
|
530
|
+
// `scope` narrows to one agentdox context scope: a project's own spend.
|
|
531
|
+
const contextScope = url.searchParams.get("scope") ?? "";
|
|
532
|
+
return json({ sinceMs: since, usd: spendUsdSince(db, since, harnessScopeParam(url.searchParams.get("harness")), contextScope), ...(contextScope === "" ? {} : { scope: contextScope }) });
|
|
531
533
|
}
|
|
532
534
|
if (req.method === "GET" && url.pathname === "/v1/router/feedback") {
|
|
533
535
|
const days = clampDays(url.searchParams.get("days"), 30);
|
package/src/server/turn.ts
CHANGED
|
@@ -304,6 +304,10 @@ export async function runTurn(
|
|
|
304
304
|
turn: turnNumber,
|
|
305
305
|
requestedModel: req.requestedModel,
|
|
306
306
|
harnessId: req.harnessId,
|
|
307
|
+
// The context scope this turn carried, so a front door can charge the
|
|
308
|
+
// row back to a project. The resolved one the bridge used, header or
|
|
309
|
+
// configured default; "" stores as NULL.
|
|
310
|
+
scope: doxScope,
|
|
307
311
|
ompSessionId: req.ompSessionId,
|
|
308
312
|
slug: decision.slug,
|
|
309
313
|
servedSlug,
|
package/src/util/sqlite.ts
CHANGED
|
@@ -18,7 +18,7 @@ import { mkdirSync } from "node:fs";
|
|
|
18
18
|
import { dirname } from "node:path";
|
|
19
19
|
|
|
20
20
|
/** Bump when a migration is added; guarded below so reopening never regresses it. */
|
|
21
|
-
const USER_VERSION =
|
|
21
|
+
const USER_VERSION = 18;
|
|
22
22
|
|
|
23
23
|
const MIGRATIONS = `
|
|
24
24
|
CREATE TABLE IF NOT EXISTS catalog_cache (
|
|
@@ -286,6 +286,16 @@ const MIGRATE_V17 = `
|
|
|
286
286
|
ALTER TABLE conversations ADD COLUMN upgrade_deferred_tier TEXT;
|
|
287
287
|
`;
|
|
288
288
|
|
|
289
|
+
// v18: ledger records the agentdox context scope the turn carried — the scope
|
|
290
|
+
// the bridge resolved for it (the request header, or the configured default).
|
|
291
|
+
// A front door charges spend back to a project with it: the team edition names
|
|
292
|
+
// a project's scope on every turn, so `scope` is the only column that says
|
|
293
|
+
// which project a row belongs to. NULL on every row written before this, and
|
|
294
|
+
// on any turn that carried no scope at all; there is nothing to backfill from.
|
|
295
|
+
const MIGRATE_V18 = `
|
|
296
|
+
ALTER TABLE ledger ADD COLUMN scope TEXT;
|
|
297
|
+
`;
|
|
298
|
+
|
|
289
299
|
// v9: benchmark_cache holds the external benchmark feeds (Artificial Analysis,
|
|
290
300
|
// BenchLM) that backfill quality scores OpenRouter leaves unpublished. It is a
|
|
291
301
|
// whole new table, created idempotently by the MIGRATIONS block above, so there
|
|
@@ -327,6 +337,7 @@ export function openDb(path: string): Database {
|
|
|
327
337
|
if (!ledgerCols.some((c) => c.name === "explored_from")) db.exec(MIGRATE_V7);
|
|
328
338
|
if (!ledgerCols.some((c) => c.name === "hold_arm")) db.exec(MIGRATE_V8);
|
|
329
339
|
if (!ledgerCols.some((c) => c.name === "prompt_tokens_saved")) db.exec(MIGRATE_V12);
|
|
340
|
+
if (!ledgerCols.some((c) => c.name === "scope")) db.exec(MIGRATE_V18);
|
|
330
341
|
const convCols = db.query("PRAGMA table_info(conversations)").all() as { name: string }[];
|
|
331
342
|
if (!convCols.some((c) => c.name === "context_version")) db.exec(MIGRATE_V11);
|
|
332
343
|
if (!convCols.some((c) => c.name === "compaction_plan")) db.exec(MIGRATE_V13);
|
package/test/migrations.test.ts
CHANGED
|
@@ -8,6 +8,7 @@ import { createFeedbackStore } from "../src/cost/feedback.ts";
|
|
|
8
8
|
import { createLedger } from "../src/cost/ledger.ts";
|
|
9
9
|
import { buildUsageReport } from "../src/cost/report.ts";
|
|
10
10
|
import { buildDailySummary, createKv } from "../src/cost/summary.ts";
|
|
11
|
+
import { exportRows, spendUsdSince } from "../src/cost/views.ts";
|
|
11
12
|
import { createConversationStore } from "../src/router/state.ts";
|
|
12
13
|
import { openDb } from "../src/util/sqlite.ts";
|
|
13
14
|
|
|
@@ -23,7 +24,7 @@ import { openDb } from "../src/util/sqlite.ts";
|
|
|
23
24
|
|
|
24
25
|
const FIXTURES = join(import.meta.dir, "fixtures", "migrations");
|
|
25
26
|
const files = readdirSync(FIXTURES).filter((f) => /^router-v\d+\.db$/.test(f)).sort((a, b) => Number(/\d+/.exec(a)![0]) - Number(/\d+/.exec(b)![0]));
|
|
26
|
-
const CURRENT_VERSION =
|
|
27
|
+
const CURRENT_VERSION = 18;
|
|
27
28
|
|
|
28
29
|
describe("schema migrations from every shipped version", () => {
|
|
29
30
|
test("fixtures exist for the versions that shipped", () => {
|
|
@@ -43,7 +44,7 @@ describe("schema migrations from every shipped version", () => {
|
|
|
43
44
|
expect((db.query("PRAGMA user_version").get() as { user_version: number }).user_version).toBe(CURRENT_VERSION);
|
|
44
45
|
// Every column the current code writes exists after migration.
|
|
45
46
|
const ledgerCols = new Set((db.query("PRAGMA table_info(ledger)").all() as { name: string }[]).map((c) => c.name));
|
|
46
|
-
for (const c of ["harness_id", "error_kind", "omp_session_id", "features", "explored_from", "hold_arm", "prompt_tokens_saved"]) expect(ledgerCols.has(c)).toBe(true);
|
|
47
|
+
for (const c of ["harness_id", "error_kind", "omp_session_id", "features", "explored_from", "hold_arm", "prompt_tokens_saved", "scope"]) expect(ledgerCols.has(c)).toBe(true);
|
|
47
48
|
const convCols = new Set((db.query("PRAGMA table_info(conversations)").all() as { name: string }[]).map((c) => c.name));
|
|
48
49
|
for (const c of ["context_version", "compaction_plan", "compaction_plan_tokens", "upgrade_deferred_tier"]) expect(convCols.has(c)).toBe(true);
|
|
49
50
|
// The fixture's ledger row survived the ALTERs with its values.
|
|
@@ -61,6 +62,11 @@ describe("schema migrations from every shipped version", () => {
|
|
|
61
62
|
expect(conversations.load("fixture-key").key).toBe("fixture-key");
|
|
62
63
|
expect(buildUsageReport(db, { windowDays: 3650 }).totals.dispatches).toBe(1);
|
|
63
64
|
expect(buildDailySummary(db, {}).current.dispatches).toBe(0);
|
|
65
|
+
// v18: the fixture's row predates `scope`, so it exports under "" and no
|
|
66
|
+
// context scope claims its spend.
|
|
67
|
+
expect(exportRows(db, 0, null).map((r) => r.scope)).toEqual([""]);
|
|
68
|
+
expect(spendUsdSince(db, 0, null, "acme.api")).toBe(0);
|
|
69
|
+
expect(spendUsdSince(db, 0, null)).toBeGreaterThanOrEqual(0);
|
|
64
70
|
expect(ledger.prune?.(0)).toBe(0);
|
|
65
71
|
} finally {
|
|
66
72
|
db.close();
|
|
@@ -77,6 +83,8 @@ describe("schema migrations from every shipped version", () => {
|
|
|
77
83
|
const db = openDb(":memory:");
|
|
78
84
|
try {
|
|
79
85
|
expect((db.query("PRAGMA user_version").get() as { user_version: number }).user_version).toBe(CURRENT_VERSION);
|
|
86
|
+
// A fresh ledger has the scope column the bootstrap never spells out in CREATE TABLE.
|
|
87
|
+
expect((db.query("PRAGMA table_info(ledger)").all() as { name: string }[]).some((c) => c.name === "scope")).toBe(true);
|
|
80
88
|
} finally {
|
|
81
89
|
db.close();
|
|
82
90
|
}
|
|
@@ -259,11 +259,11 @@ describe("v4 migration", () => {
|
|
|
259
259
|
db.close();
|
|
260
260
|
});
|
|
261
261
|
|
|
262
|
-
test("schema is at user_version
|
|
262
|
+
test("schema is at user_version 18", () => {
|
|
263
263
|
const db = openDb(":memory:");
|
|
264
264
|
try {
|
|
265
265
|
const row = db.query("PRAGMA user_version").get() as { user_version: number };
|
|
266
|
-
expect(row.user_version).toBe(
|
|
266
|
+
expect(row.user_version).toBe(18);
|
|
267
267
|
} finally {
|
|
268
268
|
db.close();
|
|
269
269
|
}
|
package/test/turn.test.ts
CHANGED
|
@@ -545,6 +545,29 @@ describe("runTurn", () => {
|
|
|
545
545
|
});
|
|
546
546
|
});
|
|
547
547
|
|
|
548
|
+
describe("the context scope reaches the ledger", () => {
|
|
549
|
+
const run = async (req: NormRequest, config: RouterConfig) => {
|
|
550
|
+
const { router } = mkRouter([mkDecision("trivial", "cheap/model", { escalateTo: "simple" })]);
|
|
551
|
+
const { upstream } = mkUpstream([{ kind: "chunks", chunks: [startChunk("cheap/model"), textChunk("hi"), finishChunk("stop"), usageChunk({ promptTokens: 10, completionTokens: 2 }, 0.0001)] }]);
|
|
552
|
+
const { ledger, entries } = mkLedger();
|
|
553
|
+
const { store } = mkConversations();
|
|
554
|
+
const { sink } = mkSink();
|
|
555
|
+
await runTurn(req, sink, { config, router, upstream, ledger, conversations: store, catalog, context: createDisabledBridge() }, new AbortController().signal);
|
|
556
|
+
return entries;
|
|
557
|
+
};
|
|
558
|
+
|
|
559
|
+
test("a turn's scope is recorded, header first and the configured default behind it", async () => {
|
|
560
|
+
// The header the team front door sets: the row can be charged to that project.
|
|
561
|
+
expect((await run({ ...mkReq(), agentdoxScope: "acme.api" }, mkConfig()))[0]!.scope).toBe("acme.api");
|
|
562
|
+
// No header: the resolved scope is the configured default, which is what the bridge would have used.
|
|
563
|
+
const base = mkConfig();
|
|
564
|
+
const withDefault: RouterConfig = { ...base, context: { ...base.context, defaultScope: "solo" } };
|
|
565
|
+
expect((await run(mkReq(), withDefault))[0]!.scope).toBe("solo");
|
|
566
|
+
// Neither: no scope at all, which the ledger stores as NULL.
|
|
567
|
+
expect((await run(mkReq(), mkConfig()))[0]!.scope).toBe("");
|
|
568
|
+
});
|
|
569
|
+
});
|
|
570
|
+
|
|
548
571
|
describe("exploration reaches the ledger", () => {
|
|
549
572
|
test("an explored turn records the tier it was dropped from", async () => {
|
|
550
573
|
const explored = { ...mkDecision("simple", "cheap/model"), explored: { from: "moderate" as Tier, to: "simple" as Tier } };
|
package/test/views.test.ts
CHANGED
|
@@ -64,9 +64,9 @@ function seeded() {
|
|
|
64
64
|
const db = openDb(":memory:");
|
|
65
65
|
const ledger = createLedger(db, cfg);
|
|
66
66
|
const feedback = createFeedbackStore(db);
|
|
67
|
-
ledger.record(entry({ id: "l1", harnessId: "u_ada", slug: "anthropic/claude-sonnet-5", servedSlug: "anthropic/claude-sonnet-5", predictedUsd: 0.01, reportedUsd: 0.012 }));
|
|
68
|
-
ledger.record(entry({ id: "l2", harnessId: "u_ada", slug: "anthropic/claude-sonnet-5", servedSlug: null, predictedUsd: 0.01, reportedUsd: null, escalationSignal: "circular" }));
|
|
69
|
-
ledger.record(entry({ id: "l3", harnessId: "u_bob", slug: "ollama/glm-5.3-flash", servedSlug: "ollama/glm-5.3-flash", predictedUsd: 0.001, reportedUsd: 0.001, error: "boom" }));
|
|
67
|
+
ledger.record(entry({ id: "l1", harnessId: "u_ada", scope: "acme.api", slug: "anthropic/claude-sonnet-5", servedSlug: "anthropic/claude-sonnet-5", predictedUsd: 0.01, reportedUsd: 0.012 }));
|
|
68
|
+
ledger.record(entry({ id: "l2", harnessId: "u_ada", scope: "acme.web", slug: "anthropic/claude-sonnet-5", servedSlug: null, predictedUsd: 0.01, reportedUsd: null, escalationSignal: "circular" }));
|
|
69
|
+
ledger.record(entry({ id: "l3", harnessId: "u_bob", scope: "acme.api", slug: "ollama/glm-5.3-flash", servedSlug: "ollama/glm-5.3-flash", predictedUsd: 0.001, reportedUsd: 0.001, error: "boom" }));
|
|
70
70
|
ledger.record(entry({ id: "l4", harnessId: "u_bob", requestedModel: "digest", slug: "ollama/glm-5.3-flash", servedSlug: "ollama/glm-5.3-flash", predictedUsd: 0.5, reportedUsd: 0.5 }));
|
|
71
71
|
ledger.record(entry({ id: "l5", harnessId: "u_bob", createdAtMs: NOW - 40 * DAY, slug: "ollama/glm-5.3-flash", predictedUsd: 5, reportedUsd: 5 }));
|
|
72
72
|
feedback.record({ ledgerId: "l1", ompSessionId: "s", slug: "anthropic/claude-sonnet-5", tier: "simple", verdict: "good", note: "" }, NOW - 1000);
|
|
@@ -87,6 +87,17 @@ describe("ledger views", () => {
|
|
|
87
87
|
expect(spendUsdSince(db, since, [])).toBe(0);
|
|
88
88
|
});
|
|
89
89
|
|
|
90
|
+
test("spend narrowed to one context scope, so a front door charges a project", () => {
|
|
91
|
+
// l1 (0.012, u_ada) and l3 (0.001, u_bob) carried acme.api; l2 (0.01 predicted) carried acme.web.
|
|
92
|
+
expect(spendUsdSince(db, since, null, "acme.api")).toBeCloseTo(0.013, 6);
|
|
93
|
+
expect(spendUsdSince(db, since, null, "acme.web")).toBeCloseTo(0.01, 6);
|
|
94
|
+
expect(spendUsdSince(db, since, ["u_ada"], "acme.api")).toBeCloseTo(0.012, 6); // harness and scope compose
|
|
95
|
+
expect(spendUsdSince(db, since, ["u_bob"], "acme.web")).toBe(0);
|
|
96
|
+
expect(spendUsdSince(db, since, null, "nope")).toBe(0);
|
|
97
|
+
expect(spendUsdSince(db, since, null, "")).toBeCloseTo(0.523, 6); // no scope given: every turn, scoped or not
|
|
98
|
+
expect(spendUsdSince(db, since, [], "acme.api")).toBe(0);
|
|
99
|
+
});
|
|
100
|
+
|
|
90
101
|
test("feedback by model with distinct judges, scoped by harness", () => {
|
|
91
102
|
const all = feedbackView(db, since, null);
|
|
92
103
|
expect(all.byModel).toEqual([
|
|
@@ -102,17 +113,22 @@ describe("ledger views", () => {
|
|
|
102
113
|
expect(feedbackView(db, since, []).recent).toEqual([]);
|
|
103
114
|
});
|
|
104
115
|
|
|
105
|
-
test("export rows by day, harness
|
|
116
|
+
test("export rows by day, harness, served model and scope; digest and old rows out; CSV quoting", () => {
|
|
106
117
|
const rows = exportRows(db, since, null);
|
|
107
|
-
|
|
108
|
-
expect(rows
|
|
109
|
-
expect(rows[0]
|
|
110
|
-
expect(rows[
|
|
118
|
+
// u_ada's two turns are one model on one day but two projects, so they no longer share a row.
|
|
119
|
+
expect(rows).toHaveLength(3);
|
|
120
|
+
expect(rows[0]).toMatchObject({ day: "2026-09-07", harnessId: "u_ada", slug: "anthropic/claude-sonnet-5", scope: "acme.api", provider: "openrouter", dispatches: 1, promptTokens: 1000, cachedTokens: 400, completionTokens: 50, escalations: 0, errors: 0 });
|
|
121
|
+
expect(rows[0]!.spendUsd).toBeCloseTo(0.012, 6);
|
|
122
|
+
expect(rows[1]).toMatchObject({ harnessId: "u_ada", scope: "acme.web", dispatches: 1, escalations: 1 });
|
|
123
|
+
expect(rows[1]!.spendUsd).toBeCloseTo(0.01, 6);
|
|
124
|
+
expect(rows[2]).toMatchObject({ harnessId: "u_bob", scope: "acme.api", provider: "ollama", dispatches: 1, errors: 1 });
|
|
111
125
|
expect(exportRows(db, since, ["u_bob"])).toHaveLength(1);
|
|
112
126
|
expect(exportRows(db, since, [])).toEqual([]);
|
|
127
|
+
// A turn that carried no scope groups under "": what every row written before v18 does.
|
|
128
|
+
expect(exportRows(db, NOW - 60 * DAY, ["u_bob"]).map((r) => r.scope).sort()).toEqual(["", "acme.api"]);
|
|
113
129
|
const csv = exportCsv([{ ...rows[0]!, harnessId: 'ada, "L"' }]);
|
|
114
|
-
expect(csv.split("\n")[0]).toBe("day,harness,model,provider,dispatches,prompt_tokens,cached_tokens,completion_tokens,spend_usd,escalations,errors");
|
|
115
|
-
expect(csv.split("\n")[1]).toBe('2026-09-07,"ada, ""L""",anthropic/claude-sonnet-5,openrouter,
|
|
130
|
+
expect(csv.split("\n")[0]).toBe("day,harness,model,provider,dispatches,prompt_tokens,cached_tokens,completion_tokens,spend_usd,escalations,errors,scope");
|
|
131
|
+
expect(csv.split("\n")[1]).toBe('2026-09-07,"ada, ""L""",anthropic/claude-sonnet-5,openrouter,1,1000,400,50,0.012000,0,0,acme.api');
|
|
116
132
|
expect(harnessScopeParam(null)).toBeNull();
|
|
117
133
|
expect(harnessScopeParam(" , ")).toBeNull();
|
|
118
134
|
expect(harnessScopeParam("a, b")).toEqual(["a", "b"]);
|
|
@@ -130,7 +146,7 @@ describe("view routes", () => {
|
|
|
130
146
|
cfg.ledger.path = join(dir, "router.db");
|
|
131
147
|
// Seed through the ledger on the same file before the server opens it.
|
|
132
148
|
const db = openDb(cfg.ledger.path);
|
|
133
|
-
createLedger(db, cfg).record(entry({ id: "r1", createdAtMs: Date.now() - 1000, harnessId: "u_x", predictedUsd: 0.2, reportedUsd: 0.25 }));
|
|
149
|
+
createLedger(db, cfg).record(entry({ id: "r1", createdAtMs: Date.now() - 1000, harnessId: "u_x", scope: "acme.api", predictedUsd: 0.2, reportedUsd: 0.25 }));
|
|
134
150
|
db.close();
|
|
135
151
|
handle = startServer(cfg);
|
|
136
152
|
});
|
|
@@ -155,14 +171,21 @@ describe("view routes", () => {
|
|
|
155
171
|
expect((((await (await get("/v1/router/decisions?harness=u_other&days=1")).json()) as { entries: unknown[] }).entries)).toEqual([]);
|
|
156
172
|
expect((((await (await get("/v1/router/decisions?limit=1")).json()) as { entries: { id: string }[] }).entries.map((e) => e.id))).toEqual(["r1"]); // no filter: everything, as before
|
|
157
173
|
expect(((await (await get(`/v1/router/spend?sinceMs=${Date.now() - DAY}&harness=u_other`)).json()) as { usd: number }).usd).toBe(0);
|
|
174
|
+
// ?scope= charges one project: the row carried acme.api, so acme.web sees nothing.
|
|
175
|
+
const scoped = (await (await get(`/v1/router/spend?sinceMs=${Date.now() - DAY}&scope=acme.api`)).json()) as { usd: number; scope: string };
|
|
176
|
+
expect(scoped.usd).toBeCloseTo(0.25, 6);
|
|
177
|
+
expect(scoped.scope).toBe("acme.api");
|
|
178
|
+
expect(((await (await get(`/v1/router/spend?sinceMs=${Date.now() - DAY}&scope=acme.web`)).json()) as { usd: number }).usd).toBe(0);
|
|
179
|
+
expect(((await (await get(`/v1/router/spend?sinceMs=${Date.now() - DAY}&harness=u_x&scope=acme.api`)).json()) as { usd: number }).usd).toBeCloseTo(0.25, 6);
|
|
158
180
|
const fb = (await (await get("/v1/router/feedback?days=7")).json()) as { days: number; byModel: unknown[]; recent: unknown[] };
|
|
159
181
|
expect(fb).toEqual({ days: 7, byModel: [], recent: [] });
|
|
160
182
|
const csv = await get("/v1/router/export?days=1");
|
|
161
183
|
expect(csv.headers.get("content-type")).toContain("text/csv");
|
|
162
|
-
expect((await csv.text()).split("\n")[1]).toContain("u_x,vendor/model,openrouter,1,1000,400,50,0.250000,0,0");
|
|
163
|
-
const js = (await (await get("/v1/router/export?days=1&format=json&harness=u_x")).json()) as { days: number; rows: { harnessId: string }[] };
|
|
184
|
+
expect((await csv.text()).split("\n")[1]).toContain("u_x,vendor/model,openrouter,1,1000,400,50,0.250000,0,0,acme.api");
|
|
185
|
+
const js = (await (await get("/v1/router/export?days=1&format=json&harness=u_x")).json()) as { days: number; rows: { harnessId: string; scope: string }[] };
|
|
164
186
|
expect(js.days).toBe(1);
|
|
165
187
|
expect(js.rows[0]?.harnessId).toBe("u_x");
|
|
188
|
+
expect(js.rows[0]?.scope).toBe("acme.api");
|
|
166
189
|
});
|
|
167
190
|
});
|
|
168
191
|
|
|
@@ -176,6 +199,9 @@ describe("decision entries", () => {
|
|
|
176
199
|
expect(ada.map((e) => e.id)).toEqual(["l1", "l2"]); // same instant in the fixture; insertion order within it is stable
|
|
177
200
|
expect(ada.find((e) => e.id === "l1")?.feedback).toEqual([{ verdict: "good", note: "", createdAtMs: NOW - 1000 }]);
|
|
178
201
|
expect(ada.find((e) => e.id === "l2")?.escalationSignal).toBe("circular");
|
|
202
|
+
// The context scope is a ledger column now, so it rides along on every entry.
|
|
203
|
+
expect(ada.find((e) => e.id === "l1")?.scope).toBe("acme.api");
|
|
204
|
+
expect(ada.find((e) => e.id === "l2")?.scope).toBe("acme.web");
|
|
179
205
|
// Everyone, within the window: the 40-day-old row stays out; the digest row is a turn like any other.
|
|
180
206
|
expect(decisionEntries(db, { sinceMs: since, harness: null }).map((e) => e.id).sort()).toEqual(["l1", "l2", "l3", "l4"]);
|
|
181
207
|
expect(decisionEntries(db, { sinceMs: 0, harness: null }).length).toBe(5);
|