auto-model-router 0.18.0 → 0.20.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.omp-plugin/marketplace.json +2 -2
- package/README.md +117 -17
- package/package.json +1 -1
- package/src/cli/connect.ts +131 -2
- package/src/cli/harnesses.ts +277 -0
- package/src/cost/ledger.ts +9 -2
- package/src/cost/types.ts +7 -0
- package/src/cost/views.ts +27 -10
- package/src/index.ts +1 -0
- package/src/server/http.ts +3 -1
- package/src/server/turn.ts +4 -0
- package/src/util/sqlite.ts +12 -1
- package/test/connect-harnesses.test.ts +299 -0
- package/test/fixtures/connect/cline-providers.json +16 -0
- package/test/fixtures/connect/continue-config.yaml +14 -0
- package/test/fixtures/connect/opencode.json +15 -0
- package/test/fixtures/harness/cline-cli-connected.json +817 -0
- package/test/harness-requests.test.ts +13 -2
- package/test/migrations.test.ts +10 -2
- package/test/trust-attribution.test.ts +2 -2
- package/test/turn.test.ts +23 -0
- package/test/views.test.ts +39 -13
- package/tools/capture-proxy.ts +1 -1
|
@@ -13,8 +13,8 @@ import { RESPONSES_ONLY_PARAMS, parseResponsesRequest } from "../src/wire/openai
|
|
|
13
13
|
* dispatch. The first block holds representative bodies for harnesses that
|
|
14
14
|
* could not be run here (Cline/Roo/Kilo) or that illustrate a header set; the
|
|
15
15
|
* second block parses requests actually captured with tools/capture-proxy.ts
|
|
16
|
-
* (Aider, OpenCode, Codex
|
|
17
|
-
* here as a refreshed capture.
|
|
16
|
+
* (Aider, OpenCode, Codex, and Cline before and after `connect` configured it).
|
|
17
|
+
* A harness release that changes its shape belongs here as a refreshed capture.
|
|
18
18
|
*/
|
|
19
19
|
|
|
20
20
|
const MUT: UpstreamMutations = { slug: "x/y", fallbacks: [], sessionId: "s", cacheBreakpointMessageIndices: [], reasoning: undefined, maxTokens: undefined, stripAssistantReasoning: false };
|
|
@@ -144,6 +144,17 @@ describe("captured harness requests", () => {
|
|
|
144
144
|
expect("stream_options" in req.renderUpstreamBody(MUT)).toBe(false);
|
|
145
145
|
});
|
|
146
146
|
|
|
147
|
+
// The same CLI after `connect --harness cline`. `settings.headers` in Cline's
|
|
148
|
+
// provider store has no CLI flag, so this capture is the proof that writing the
|
|
149
|
+
// file (rather than shelling out to `cline auth`) is what earns the harness id.
|
|
150
|
+
test("cline cli 3.0 configured by connect: the same request, now carrying the harness id and scope", async () => {
|
|
151
|
+
const f = await fixture("cline-cli-connected");
|
|
152
|
+
const req = parseChatRequest(structuredClone(f.body), new Headers(f.headers));
|
|
153
|
+
expect(req.harnessId).toBe("cline");
|
|
154
|
+
expect(f.headers["x-agentdox-scope"]).toBe("omp-router");
|
|
155
|
+
expect(req.tools.map((t) => t.name)).toContain("read_files");
|
|
156
|
+
});
|
|
157
|
+
|
|
147
158
|
test("kilo cli 7.5: an OpenCode-derived request with the harness header and OpenCode tool names", async () => {
|
|
148
159
|
const f = await fixture("kilo");
|
|
149
160
|
const req = parseChatRequest(structuredClone(f.body), new Headers(f.headers));
|
package/test/migrations.test.ts
CHANGED
|
@@ -8,6 +8,7 @@ import { createFeedbackStore } from "../src/cost/feedback.ts";
|
|
|
8
8
|
import { createLedger } from "../src/cost/ledger.ts";
|
|
9
9
|
import { buildUsageReport } from "../src/cost/report.ts";
|
|
10
10
|
import { buildDailySummary, createKv } from "../src/cost/summary.ts";
|
|
11
|
+
import { exportRows, spendUsdSince } from "../src/cost/views.ts";
|
|
11
12
|
import { createConversationStore } from "../src/router/state.ts";
|
|
12
13
|
import { openDb } from "../src/util/sqlite.ts";
|
|
13
14
|
|
|
@@ -23,7 +24,7 @@ import { openDb } from "../src/util/sqlite.ts";
|
|
|
23
24
|
|
|
24
25
|
const FIXTURES = join(import.meta.dir, "fixtures", "migrations");
|
|
25
26
|
const files = readdirSync(FIXTURES).filter((f) => /^router-v\d+\.db$/.test(f)).sort((a, b) => Number(/\d+/.exec(a)![0]) - Number(/\d+/.exec(b)![0]));
|
|
26
|
-
const CURRENT_VERSION =
|
|
27
|
+
const CURRENT_VERSION = 18;
|
|
27
28
|
|
|
28
29
|
describe("schema migrations from every shipped version", () => {
|
|
29
30
|
test("fixtures exist for the versions that shipped", () => {
|
|
@@ -43,7 +44,7 @@ describe("schema migrations from every shipped version", () => {
|
|
|
43
44
|
expect((db.query("PRAGMA user_version").get() as { user_version: number }).user_version).toBe(CURRENT_VERSION);
|
|
44
45
|
// Every column the current code writes exists after migration.
|
|
45
46
|
const ledgerCols = new Set((db.query("PRAGMA table_info(ledger)").all() as { name: string }[]).map((c) => c.name));
|
|
46
|
-
for (const c of ["harness_id", "error_kind", "omp_session_id", "features", "explored_from", "hold_arm", "prompt_tokens_saved"]) expect(ledgerCols.has(c)).toBe(true);
|
|
47
|
+
for (const c of ["harness_id", "error_kind", "omp_session_id", "features", "explored_from", "hold_arm", "prompt_tokens_saved", "scope"]) expect(ledgerCols.has(c)).toBe(true);
|
|
47
48
|
const convCols = new Set((db.query("PRAGMA table_info(conversations)").all() as { name: string }[]).map((c) => c.name));
|
|
48
49
|
for (const c of ["context_version", "compaction_plan", "compaction_plan_tokens", "upgrade_deferred_tier"]) expect(convCols.has(c)).toBe(true);
|
|
49
50
|
// The fixture's ledger row survived the ALTERs with its values.
|
|
@@ -61,6 +62,11 @@ describe("schema migrations from every shipped version", () => {
|
|
|
61
62
|
expect(conversations.load("fixture-key").key).toBe("fixture-key");
|
|
62
63
|
expect(buildUsageReport(db, { windowDays: 3650 }).totals.dispatches).toBe(1);
|
|
63
64
|
expect(buildDailySummary(db, {}).current.dispatches).toBe(0);
|
|
65
|
+
// v18: the fixture's row predates `scope`, so it exports under "" and no
|
|
66
|
+
// context scope claims its spend.
|
|
67
|
+
expect(exportRows(db, 0, null).map((r) => r.scope)).toEqual([""]);
|
|
68
|
+
expect(spendUsdSince(db, 0, null, "acme.api")).toBe(0);
|
|
69
|
+
expect(spendUsdSince(db, 0, null)).toBeGreaterThanOrEqual(0);
|
|
64
70
|
expect(ledger.prune?.(0)).toBe(0);
|
|
65
71
|
} finally {
|
|
66
72
|
db.close();
|
|
@@ -77,6 +83,8 @@ describe("schema migrations from every shipped version", () => {
|
|
|
77
83
|
const db = openDb(":memory:");
|
|
78
84
|
try {
|
|
79
85
|
expect((db.query("PRAGMA user_version").get() as { user_version: number }).user_version).toBe(CURRENT_VERSION);
|
|
86
|
+
// A fresh ledger has the scope column the bootstrap never spells out in CREATE TABLE.
|
|
87
|
+
expect((db.query("PRAGMA table_info(ledger)").all() as { name: string }[]).some((c) => c.name === "scope")).toBe(true);
|
|
80
88
|
} finally {
|
|
81
89
|
db.close();
|
|
82
90
|
}
|
|
@@ -259,11 +259,11 @@ describe("v4 migration", () => {
|
|
|
259
259
|
db.close();
|
|
260
260
|
});
|
|
261
261
|
|
|
262
|
-
test("schema is at user_version
|
|
262
|
+
test("schema is at user_version 18", () => {
|
|
263
263
|
const db = openDb(":memory:");
|
|
264
264
|
try {
|
|
265
265
|
const row = db.query("PRAGMA user_version").get() as { user_version: number };
|
|
266
|
-
expect(row.user_version).toBe(
|
|
266
|
+
expect(row.user_version).toBe(18);
|
|
267
267
|
} finally {
|
|
268
268
|
db.close();
|
|
269
269
|
}
|
package/test/turn.test.ts
CHANGED
|
@@ -545,6 +545,29 @@ describe("runTurn", () => {
|
|
|
545
545
|
});
|
|
546
546
|
});
|
|
547
547
|
|
|
548
|
+
describe("the context scope reaches the ledger", () => {
|
|
549
|
+
const run = async (req: NormRequest, config: RouterConfig) => {
|
|
550
|
+
const { router } = mkRouter([mkDecision("trivial", "cheap/model", { escalateTo: "simple" })]);
|
|
551
|
+
const { upstream } = mkUpstream([{ kind: "chunks", chunks: [startChunk("cheap/model"), textChunk("hi"), finishChunk("stop"), usageChunk({ promptTokens: 10, completionTokens: 2 }, 0.0001)] }]);
|
|
552
|
+
const { ledger, entries } = mkLedger();
|
|
553
|
+
const { store } = mkConversations();
|
|
554
|
+
const { sink } = mkSink();
|
|
555
|
+
await runTurn(req, sink, { config, router, upstream, ledger, conversations: store, catalog, context: createDisabledBridge() }, new AbortController().signal);
|
|
556
|
+
return entries;
|
|
557
|
+
};
|
|
558
|
+
|
|
559
|
+
test("a turn's scope is recorded, header first and the configured default behind it", async () => {
|
|
560
|
+
// The header the team front door sets: the row can be charged to that project.
|
|
561
|
+
expect((await run({ ...mkReq(), agentdoxScope: "acme.api" }, mkConfig()))[0]!.scope).toBe("acme.api");
|
|
562
|
+
// No header: the resolved scope is the configured default, which is what the bridge would have used.
|
|
563
|
+
const base = mkConfig();
|
|
564
|
+
const withDefault: RouterConfig = { ...base, context: { ...base.context, defaultScope: "solo" } };
|
|
565
|
+
expect((await run(mkReq(), withDefault))[0]!.scope).toBe("solo");
|
|
566
|
+
// Neither: no scope at all, which the ledger stores as NULL.
|
|
567
|
+
expect((await run(mkReq(), mkConfig()))[0]!.scope).toBe("");
|
|
568
|
+
});
|
|
569
|
+
});
|
|
570
|
+
|
|
548
571
|
describe("exploration reaches the ledger", () => {
|
|
549
572
|
test("an explored turn records the tier it was dropped from", async () => {
|
|
550
573
|
const explored = { ...mkDecision("simple", "cheap/model"), explored: { from: "moderate" as Tier, to: "simple" as Tier } };
|
package/test/views.test.ts
CHANGED
|
@@ -64,9 +64,9 @@ function seeded() {
|
|
|
64
64
|
const db = openDb(":memory:");
|
|
65
65
|
const ledger = createLedger(db, cfg);
|
|
66
66
|
const feedback = createFeedbackStore(db);
|
|
67
|
-
ledger.record(entry({ id: "l1", harnessId: "u_ada", slug: "anthropic/claude-sonnet-5", servedSlug: "anthropic/claude-sonnet-5", predictedUsd: 0.01, reportedUsd: 0.012 }));
|
|
68
|
-
ledger.record(entry({ id: "l2", harnessId: "u_ada", slug: "anthropic/claude-sonnet-5", servedSlug: null, predictedUsd: 0.01, reportedUsd: null, escalationSignal: "circular" }));
|
|
69
|
-
ledger.record(entry({ id: "l3", harnessId: "u_bob", slug: "ollama/glm-5.3-flash", servedSlug: "ollama/glm-5.3-flash", predictedUsd: 0.001, reportedUsd: 0.001, error: "boom" }));
|
|
67
|
+
ledger.record(entry({ id: "l1", harnessId: "u_ada", scope: "acme.api", slug: "anthropic/claude-sonnet-5", servedSlug: "anthropic/claude-sonnet-5", predictedUsd: 0.01, reportedUsd: 0.012 }));
|
|
68
|
+
ledger.record(entry({ id: "l2", harnessId: "u_ada", scope: "acme.web", slug: "anthropic/claude-sonnet-5", servedSlug: null, predictedUsd: 0.01, reportedUsd: null, escalationSignal: "circular" }));
|
|
69
|
+
ledger.record(entry({ id: "l3", harnessId: "u_bob", scope: "acme.api", slug: "ollama/glm-5.3-flash", servedSlug: "ollama/glm-5.3-flash", predictedUsd: 0.001, reportedUsd: 0.001, error: "boom" }));
|
|
70
70
|
ledger.record(entry({ id: "l4", harnessId: "u_bob", requestedModel: "digest", slug: "ollama/glm-5.3-flash", servedSlug: "ollama/glm-5.3-flash", predictedUsd: 0.5, reportedUsd: 0.5 }));
|
|
71
71
|
ledger.record(entry({ id: "l5", harnessId: "u_bob", createdAtMs: NOW - 40 * DAY, slug: "ollama/glm-5.3-flash", predictedUsd: 5, reportedUsd: 5 }));
|
|
72
72
|
feedback.record({ ledgerId: "l1", ompSessionId: "s", slug: "anthropic/claude-sonnet-5", tier: "simple", verdict: "good", note: "" }, NOW - 1000);
|
|
@@ -87,6 +87,17 @@ describe("ledger views", () => {
|
|
|
87
87
|
expect(spendUsdSince(db, since, [])).toBe(0);
|
|
88
88
|
});
|
|
89
89
|
|
|
90
|
+
test("spend narrowed to one context scope, so a front door charges a project", () => {
|
|
91
|
+
// l1 (0.012, u_ada) and l3 (0.001, u_bob) carried acme.api; l2 (0.01 predicted) carried acme.web.
|
|
92
|
+
expect(spendUsdSince(db, since, null, "acme.api")).toBeCloseTo(0.013, 6);
|
|
93
|
+
expect(spendUsdSince(db, since, null, "acme.web")).toBeCloseTo(0.01, 6);
|
|
94
|
+
expect(spendUsdSince(db, since, ["u_ada"], "acme.api")).toBeCloseTo(0.012, 6); // harness and scope compose
|
|
95
|
+
expect(spendUsdSince(db, since, ["u_bob"], "acme.web")).toBe(0);
|
|
96
|
+
expect(spendUsdSince(db, since, null, "nope")).toBe(0);
|
|
97
|
+
expect(spendUsdSince(db, since, null, "")).toBeCloseTo(0.523, 6); // no scope given: every turn, scoped or not
|
|
98
|
+
expect(spendUsdSince(db, since, [], "acme.api")).toBe(0);
|
|
99
|
+
});
|
|
100
|
+
|
|
90
101
|
test("feedback by model with distinct judges, scoped by harness", () => {
|
|
91
102
|
const all = feedbackView(db, since, null);
|
|
92
103
|
expect(all.byModel).toEqual([
|
|
@@ -102,17 +113,22 @@ describe("ledger views", () => {
|
|
|
102
113
|
expect(feedbackView(db, since, []).recent).toEqual([]);
|
|
103
114
|
});
|
|
104
115
|
|
|
105
|
-
test("export rows by day, harness
|
|
116
|
+
test("export rows by day, harness, served model and scope; digest and old rows out; CSV quoting", () => {
|
|
106
117
|
const rows = exportRows(db, since, null);
|
|
107
|
-
|
|
108
|
-
expect(rows
|
|
109
|
-
expect(rows[0]
|
|
110
|
-
expect(rows[
|
|
118
|
+
// u_ada's two turns are one model on one day but two projects, so they no longer share a row.
|
|
119
|
+
expect(rows).toHaveLength(3);
|
|
120
|
+
expect(rows[0]).toMatchObject({ day: "2026-09-07", harnessId: "u_ada", slug: "anthropic/claude-sonnet-5", scope: "acme.api", provider: "openrouter", dispatches: 1, promptTokens: 1000, cachedTokens: 400, completionTokens: 50, escalations: 0, errors: 0 });
|
|
121
|
+
expect(rows[0]!.spendUsd).toBeCloseTo(0.012, 6);
|
|
122
|
+
expect(rows[1]).toMatchObject({ harnessId: "u_ada", scope: "acme.web", dispatches: 1, escalations: 1 });
|
|
123
|
+
expect(rows[1]!.spendUsd).toBeCloseTo(0.01, 6);
|
|
124
|
+
expect(rows[2]).toMatchObject({ harnessId: "u_bob", scope: "acme.api", provider: "ollama", dispatches: 1, errors: 1 });
|
|
111
125
|
expect(exportRows(db, since, ["u_bob"])).toHaveLength(1);
|
|
112
126
|
expect(exportRows(db, since, [])).toEqual([]);
|
|
127
|
+
// A turn that carried no scope groups under "": what every row written before v18 does.
|
|
128
|
+
expect(exportRows(db, NOW - 60 * DAY, ["u_bob"]).map((r) => r.scope).sort()).toEqual(["", "acme.api"]);
|
|
113
129
|
const csv = exportCsv([{ ...rows[0]!, harnessId: 'ada, "L"' }]);
|
|
114
|
-
expect(csv.split("\n")[0]).toBe("day,harness,model,provider,dispatches,prompt_tokens,cached_tokens,completion_tokens,spend_usd,escalations,errors");
|
|
115
|
-
expect(csv.split("\n")[1]).toBe('2026-09-07,"ada, ""L""",anthropic/claude-sonnet-5,openrouter,
|
|
130
|
+
expect(csv.split("\n")[0]).toBe("day,harness,model,provider,dispatches,prompt_tokens,cached_tokens,completion_tokens,spend_usd,escalations,errors,scope");
|
|
131
|
+
expect(csv.split("\n")[1]).toBe('2026-09-07,"ada, ""L""",anthropic/claude-sonnet-5,openrouter,1,1000,400,50,0.012000,0,0,acme.api');
|
|
116
132
|
expect(harnessScopeParam(null)).toBeNull();
|
|
117
133
|
expect(harnessScopeParam(" , ")).toBeNull();
|
|
118
134
|
expect(harnessScopeParam("a, b")).toEqual(["a", "b"]);
|
|
@@ -130,7 +146,7 @@ describe("view routes", () => {
|
|
|
130
146
|
cfg.ledger.path = join(dir, "router.db");
|
|
131
147
|
// Seed through the ledger on the same file before the server opens it.
|
|
132
148
|
const db = openDb(cfg.ledger.path);
|
|
133
|
-
createLedger(db, cfg).record(entry({ id: "r1", createdAtMs: Date.now() - 1000, harnessId: "u_x", predictedUsd: 0.2, reportedUsd: 0.25 }));
|
|
149
|
+
createLedger(db, cfg).record(entry({ id: "r1", createdAtMs: Date.now() - 1000, harnessId: "u_x", scope: "acme.api", predictedUsd: 0.2, reportedUsd: 0.25 }));
|
|
134
150
|
db.close();
|
|
135
151
|
handle = startServer(cfg);
|
|
136
152
|
});
|
|
@@ -155,14 +171,21 @@ describe("view routes", () => {
|
|
|
155
171
|
expect((((await (await get("/v1/router/decisions?harness=u_other&days=1")).json()) as { entries: unknown[] }).entries)).toEqual([]);
|
|
156
172
|
expect((((await (await get("/v1/router/decisions?limit=1")).json()) as { entries: { id: string }[] }).entries.map((e) => e.id))).toEqual(["r1"]); // no filter: everything, as before
|
|
157
173
|
expect(((await (await get(`/v1/router/spend?sinceMs=${Date.now() - DAY}&harness=u_other`)).json()) as { usd: number }).usd).toBe(0);
|
|
174
|
+
// ?scope= charges one project: the row carried acme.api, so acme.web sees nothing.
|
|
175
|
+
const scoped = (await (await get(`/v1/router/spend?sinceMs=${Date.now() - DAY}&scope=acme.api`)).json()) as { usd: number; scope: string };
|
|
176
|
+
expect(scoped.usd).toBeCloseTo(0.25, 6);
|
|
177
|
+
expect(scoped.scope).toBe("acme.api");
|
|
178
|
+
expect(((await (await get(`/v1/router/spend?sinceMs=${Date.now() - DAY}&scope=acme.web`)).json()) as { usd: number }).usd).toBe(0);
|
|
179
|
+
expect(((await (await get(`/v1/router/spend?sinceMs=${Date.now() - DAY}&harness=u_x&scope=acme.api`)).json()) as { usd: number }).usd).toBeCloseTo(0.25, 6);
|
|
158
180
|
const fb = (await (await get("/v1/router/feedback?days=7")).json()) as { days: number; byModel: unknown[]; recent: unknown[] };
|
|
159
181
|
expect(fb).toEqual({ days: 7, byModel: [], recent: [] });
|
|
160
182
|
const csv = await get("/v1/router/export?days=1");
|
|
161
183
|
expect(csv.headers.get("content-type")).toContain("text/csv");
|
|
162
|
-
expect((await csv.text()).split("\n")[1]).toContain("u_x,vendor/model,openrouter,1,1000,400,50,0.250000,0,0");
|
|
163
|
-
const js = (await (await get("/v1/router/export?days=1&format=json&harness=u_x")).json()) as { days: number; rows: { harnessId: string }[] };
|
|
184
|
+
expect((await csv.text()).split("\n")[1]).toContain("u_x,vendor/model,openrouter,1,1000,400,50,0.250000,0,0,acme.api");
|
|
185
|
+
const js = (await (await get("/v1/router/export?days=1&format=json&harness=u_x")).json()) as { days: number; rows: { harnessId: string; scope: string }[] };
|
|
164
186
|
expect(js.days).toBe(1);
|
|
165
187
|
expect(js.rows[0]?.harnessId).toBe("u_x");
|
|
188
|
+
expect(js.rows[0]?.scope).toBe("acme.api");
|
|
166
189
|
});
|
|
167
190
|
});
|
|
168
191
|
|
|
@@ -176,6 +199,9 @@ describe("decision entries", () => {
|
|
|
176
199
|
expect(ada.map((e) => e.id)).toEqual(["l1", "l2"]); // same instant in the fixture; insertion order within it is stable
|
|
177
200
|
expect(ada.find((e) => e.id === "l1")?.feedback).toEqual([{ verdict: "good", note: "", createdAtMs: NOW - 1000 }]);
|
|
178
201
|
expect(ada.find((e) => e.id === "l2")?.escalationSignal).toBe("circular");
|
|
202
|
+
// The context scope is a ledger column now, so it rides along on every entry.
|
|
203
|
+
expect(ada.find((e) => e.id === "l1")?.scope).toBe("acme.api");
|
|
204
|
+
expect(ada.find((e) => e.id === "l2")?.scope).toBe("acme.web");
|
|
179
205
|
// Everyone, within the window: the 40-day-old row stays out; the digest row is a turn like any other.
|
|
180
206
|
expect(decisionEntries(db, { sinceMs: since, harness: null }).map((e) => e.id).sort()).toEqual(["l1", "l2", "l3", "l4"]);
|
|
181
207
|
expect(decisionEntries(db, { sinceMs: 0, harness: null }).length).toBe(5);
|
package/tools/capture-proxy.ts
CHANGED
|
@@ -30,7 +30,7 @@ const out = flag("--out", "test/fixtures/harness");
|
|
|
30
30
|
const name = flag("--name", "harness");
|
|
31
31
|
mkdirSync(out, { recursive: true });
|
|
32
32
|
|
|
33
|
-
const KEEP_HEADERS = ["content-type", "user-agent", "x-omp-harness", "x-omp-session", "x-omp-subagent", "x-title", "http-referer"];
|
|
33
|
+
const KEEP_HEADERS = ["content-type", "user-agent", "x-omp-harness", "x-omp-session", "x-omp-subagent", "x-agentdox-scope", "x-agentdox-origin", "x-title", "http-referer"];
|
|
34
34
|
let n = 0;
|
|
35
35
|
|
|
36
36
|
/** Large text is not what the fixture guards; cap message text so files stay small. */
|