auto-model-router 0.30.3 → 0.32.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.omp-plugin/marketplace.json +2 -2
- package/README.md +32 -2
- package/omp-extension/router-configure.ts +9 -7
- package/package.json +1 -1
- package/src/cli/config-cmd.ts +8 -7
- package/src/cli/explain.ts +10 -5
- package/src/cli/export.ts +6 -5
- package/src/cli/models.ts +10 -7
- package/src/cli/report.ts +6 -1
- package/src/cli/stats.ts +7 -7
- package/src/config/load.ts +10 -1
- package/src/config/types.ts +10 -1
- package/src/context/bridge.ts +7 -7
- package/src/context/index.ts +3 -3
- package/src/context/store.ts +39 -56
- package/src/context/types.ts +7 -6
- package/src/cost/blended.ts +28 -7
- package/src/cost/feedback.ts +33 -37
- package/src/cost/ledger-sql.ts +547 -0
- package/src/cost/ledger.ts +30 -459
- package/src/cost/report.ts +171 -129
- package/src/cost/retention.ts +10 -10
- package/src/cost/summary.ts +15 -10
- package/src/cost/types.ts +43 -62
- package/src/cost/views.ts +79 -49
- package/src/eval/calibrate.ts +47 -12
- package/src/eval/run.ts +18 -2
- package/src/lib.ts +6 -2
- package/src/router/candidates.ts +7 -15
- package/src/router/classify.ts +6 -4
- package/src/router/index.ts +95 -9
- package/src/router/select.ts +38 -21
- package/src/router/state.ts +90 -102
- package/src/router/types.ts +11 -5
- package/src/server/advise.ts +6 -4
- package/src/server/compaction-digest.ts +1 -1
- package/src/server/digest.ts +9 -10
- package/src/server/http.ts +109 -46
- package/src/server/providers.ts +18 -4
- package/src/server/turn.ts +32 -9
- package/src/tokens/estimate.ts +16 -6
- package/src/upstream/ollama-usage.ts +21 -11
- package/src/util/schema.ts +201 -0
- package/src/util/sql.ts +246 -0
- package/src/wire/anthropic/messages.ts +3 -4
- package/src/wire/openai/request.ts +1 -0
- package/src/wire/types.ts +7 -0
- package/test/anthropic-wire.test.ts +9 -9
- package/test/benchmark-feeds.test.ts +7 -7
- package/test/cache-control.test.ts +7 -7
- package/test/cache-estimate.test.ts +5 -5
- package/test/catalog-view.test.ts +4 -4
- package/test/catalog.test.ts +11 -11
- package/test/classify.test.ts +24 -24
- package/test/compaction.test.ts +20 -20
- package/test/config-wizard.test.ts +32 -32
- package/test/config.test.ts +10 -10
- package/test/connect-harnesses.test.ts +11 -11
- package/test/context-bridge.test.ts +40 -30
- package/test/context-prune.test.ts +43 -36
- package/test/context-query.test.ts +8 -8
- package/test/controls.test.ts +54 -27
- package/test/cost.test.ts +12 -12
- package/test/digest.test.ts +55 -44
- package/test/embed-lifecycle.test.ts +5 -5
- package/test/embed-logic.test.ts +26 -26
- package/test/escalate.test.ts +17 -17
- package/test/eval.test.ts +73 -16
- package/test/executable.test.ts +6 -6
- package/test/exploration.test.ts +19 -20
- package/test/failover.test.ts +22 -21
- package/test/fakes.ts +105 -0
- package/test/features.test.ts +21 -21
- package/test/harness-requests.test.ts +3 -3
- package/test/harness-switch.test.ts +5 -5
- package/test/hold-exploration.test.ts +13 -13
- package/test/hot-reload.test.ts +5 -5
- package/test/learned.test.ts +5 -5
- package/test/ledger-sql.test.ts +342 -0
- package/test/mcp-entry.test.ts +5 -5
- package/test/migrations.test.ts +28 -22
- package/test/models-yml.test.ts +18 -18
- package/test/ollama.test.ts +40 -34
- package/test/omp-credentials.test.ts +16 -16
- package/test/policy.test.ts +3 -3
- package/test/reconfigure.test.ts +4 -4
- package/test/redaction.test.ts +41 -35
- package/test/remote.test.ts +12 -12
- package/test/report-logic.test.ts +8 -8
- package/test/report.test.ts +95 -87
- package/test/retention.test.ts +79 -66
- package/test/schema.test.ts +123 -0
- package/test/scope.test.ts +8 -8
- package/test/select.test.ts +216 -257
- package/test/skills.test.ts +3 -3
- package/test/sql-shim.test.ts +154 -0
- package/test/state.test.ts +43 -36
- package/test/summary.test.ts +38 -27
- package/test/tier-plan.test.ts +45 -62
- package/test/toast-logic.test.ts +31 -31
- package/test/tokens.test.ts +95 -80
- package/test/trust-attribution.test.ts +217 -187
- package/test/trust-window.test.ts +37 -32
- package/test/turn.test.ts +55 -23
- package/test/upstreams.test.ts +13 -13
- package/test/views.test.ts +81 -59
- package/test/wire-request.test.ts +17 -17
- package/test/wire-responses.test.ts +4 -4
- package/tools/agentdox-e2e.ts +5 -2
- package/tools/export-benchmarks.ts +5 -5
- package/tools/ledger-parity.ts +266 -0
- package/tools/replay.ts +16 -8
|
@@ -1,11 +1,15 @@
|
|
|
1
1
|
import { describe, expect, test } from "bun:test";
|
|
2
|
+
import { tmpdir } from "node:os";
|
|
3
|
+
import { join } from "node:path";
|
|
4
|
+
|
|
5
|
+
import { migrateStore } from "../src/util/schema.ts";
|
|
6
|
+
import { openSqlDb } from "../src/util/sql.ts";
|
|
2
7
|
|
|
3
8
|
import { type AgentDoxClient, type AssembleLayers, type AssembleLimits, createAgentDoxClient } from "../src/context/agentdox.ts";
|
|
4
9
|
import { createContextBridge } from "../src/context/bridge.ts";
|
|
5
10
|
import { createContextStore } from "../src/context/store.ts";
|
|
6
11
|
import type { ContextResolveInput, TurnRecord } from "../src/context/types.ts";
|
|
7
12
|
import { createLogger } from "../src/util/log.ts";
|
|
8
|
-
import { openDb } from "../src/util/sqlite.ts";
|
|
9
13
|
import { injectForTest } from "./helpers/inject.ts";
|
|
10
14
|
|
|
11
15
|
const log = createLogger("silent");
|
|
@@ -49,8 +53,11 @@ function mkClient(prompt = "MEMORY: player digs in 3/4 top-down"): FakeClient {
|
|
|
49
53
|
|
|
50
54
|
type BridgeOpts = Parameters<typeof createContextBridge>[0];
|
|
51
55
|
|
|
52
|
-
function mkBridge(client: AgentDoxClient, over: Partial<BridgeOpts> = {}) {
|
|
53
|
-
|
|
56
|
+
async function mkBridge(client: AgentDoxClient, over: Partial<BridgeOpts> = {}) {
|
|
57
|
+
// A file: the shim cannot share `:memory:` between handles. Awaited, so the
|
|
58
|
+
// tables exist before the bridge's first read rather than racing it.
|
|
59
|
+
const db = openSqlDb(join(tmpdir(), `ctxbridge-${process.pid}-${Date.now()}-${Math.random().toString(36).slice(2)}.db`));
|
|
60
|
+
await migrateStore(db);
|
|
54
61
|
const opts: BridgeOpts = {
|
|
55
62
|
client,
|
|
56
63
|
store: createContextStore(db),
|
|
@@ -89,7 +96,7 @@ function input(over: Partial<ContextResolveInput> = {}): ContextResolveInput {
|
|
|
89
96
|
describe("context bridge refresh policy", () => {
|
|
90
97
|
test("recent sessions ride only on a conversation's first block; refreshes ask for none", async () => {
|
|
91
98
|
const client = mkClient();
|
|
92
|
-
const { bridge } = mkBridge(client, { sessionLimit: 6 });
|
|
99
|
+
const { bridge } = await mkBridge(client, { sessionLimit: 6 });
|
|
93
100
|
await bridge.resolve(input({ firstFetch: true }));
|
|
94
101
|
expect(client.lastLimits?.sessionLimit).toBe(6);
|
|
95
102
|
// A refresh (model switch) on a conversation that already had a block.
|
|
@@ -100,7 +107,7 @@ describe("context bridge refresh policy", () => {
|
|
|
100
107
|
|
|
101
108
|
test("fetches on the first turn, then pins without re-fetching", async () => {
|
|
102
109
|
const client = mkClient();
|
|
103
|
-
const { bridge, db } = mkBridge(client);
|
|
110
|
+
const { bridge, db } = await mkBridge(client);
|
|
104
111
|
try {
|
|
105
112
|
const first = await bridge.resolve(input());
|
|
106
113
|
expect(first).not.toBeNull();
|
|
@@ -126,7 +133,7 @@ describe("context bridge refresh policy", () => {
|
|
|
126
133
|
// The REST endpoint also ignores snake_case limit keys, which silently reads
|
|
127
134
|
// as unbounded, so pin that all four limits actually reach the client.
|
|
128
135
|
const client = mkClient();
|
|
129
|
-
const { bridge, db } = mkBridge(client, { memoryLimit: 5, docsLimit: 1, sessionLimit: 2, briefChars: 9000 });
|
|
136
|
+
const { bridge, db } = await mkBridge(client, { memoryLimit: 5, docsLimit: 1, sessionLimit: 2, briefChars: 9000 });
|
|
130
137
|
try {
|
|
131
138
|
await bridge.resolve(input());
|
|
132
139
|
expect(client.lastLimits).toEqual({ memoryLimit: 5, docsLimit: 1, sessionLimit: 2, briefChars: 9000 });
|
|
@@ -137,7 +144,7 @@ describe("context bridge refresh policy", () => {
|
|
|
137
144
|
|
|
138
145
|
test("refreshes when the model switches, because the cache is already forfeit", async () => {
|
|
139
146
|
const client = mkClient();
|
|
140
|
-
const { bridge, db } = mkBridge(client);
|
|
147
|
+
const { bridge, db } = await mkBridge(client);
|
|
141
148
|
try {
|
|
142
149
|
const first = await bridge.resolve(input());
|
|
143
150
|
await bridge.resolve(
|
|
@@ -155,7 +162,7 @@ describe("context bridge refresh policy", () => {
|
|
|
155
162
|
|
|
156
163
|
test("refreshes on a retry", async () => {
|
|
157
164
|
const client = mkClient();
|
|
158
|
-
const { bridge, db } = mkBridge(client);
|
|
165
|
+
const { bridge, db } = await mkBridge(client);
|
|
159
166
|
try {
|
|
160
167
|
const first = await bridge.resolve(input());
|
|
161
168
|
await bridge.resolve(
|
|
@@ -173,7 +180,7 @@ describe("context bridge refresh policy", () => {
|
|
|
173
180
|
|
|
174
181
|
test("refreshes once the staleness TTL elapses", async () => {
|
|
175
182
|
const client = mkClient();
|
|
176
|
-
const { bridge, db } = mkBridge(client, { maxStalenessMs: 1_000 });
|
|
183
|
+
const { bridge, db } = await mkBridge(client, { maxStalenessMs: 1_000 });
|
|
177
184
|
try {
|
|
178
185
|
const first = await bridge.resolve(input());
|
|
179
186
|
await bridge.resolve(input({ pinnedVersion: first?.version ?? null, pinnedFetchedAtMs: Date.now() - 5_000 }));
|
|
@@ -185,7 +192,7 @@ describe("context bridge refresh policy", () => {
|
|
|
185
192
|
|
|
186
193
|
test("version is a content hash, so an unchanged re-assembly keeps the cache warm", async () => {
|
|
187
194
|
const client = mkClient();
|
|
188
|
-
const { bridge, db } = mkBridge(client);
|
|
195
|
+
const { bridge, db } = await mkBridge(client);
|
|
189
196
|
try {
|
|
190
197
|
const first = await bridge.resolve(input());
|
|
191
198
|
// Force a refetch; agentdox returns byte-identical content.
|
|
@@ -206,7 +213,7 @@ describe("context bridge refresh policy", () => {
|
|
|
206
213
|
|
|
207
214
|
test("changed content yields a new version", async () => {
|
|
208
215
|
const client = mkClient();
|
|
209
|
-
const { bridge, db } = mkBridge(client);
|
|
216
|
+
const { bridge, db } = await mkBridge(client);
|
|
210
217
|
try {
|
|
211
218
|
const first = await bridge.resolve(input());
|
|
212
219
|
client.prompt = "MEMORY: player digs in 3/4 top-down; hard edges only";
|
|
@@ -225,7 +232,7 @@ describe("context bridge refresh policy", () => {
|
|
|
225
232
|
|
|
226
233
|
test("an unreachable agentdox keeps serving the pinned block", async () => {
|
|
227
234
|
const client = mkClient();
|
|
228
|
-
const { bridge, db } = mkBridge(client);
|
|
235
|
+
const { bridge, db } = await mkBridge(client);
|
|
229
236
|
try {
|
|
230
237
|
const first = await bridge.resolve(input());
|
|
231
238
|
client.assemble = async () => null; // agentdox goes down
|
|
@@ -244,7 +251,7 @@ describe("context bridge refresh policy", () => {
|
|
|
244
251
|
|
|
245
252
|
test("an empty scope is inert", async () => {
|
|
246
253
|
const client = mkClient();
|
|
247
|
-
const { bridge, db } = mkBridge(client);
|
|
254
|
+
const { bridge, db } = await mkBridge(client);
|
|
248
255
|
try {
|
|
249
256
|
expect(await bridge.resolve(input({ scope: "" }))).toBeNull();
|
|
250
257
|
expect(client.assembleCalls).toBe(0);
|
|
@@ -255,7 +262,10 @@ describe("context bridge refresh policy", () => {
|
|
|
255
262
|
|
|
256
263
|
test("blocks survive a restart, so the same bytes are re-injected", async () => {
|
|
257
264
|
const client = mkClient();
|
|
258
|
-
|
|
265
|
+
// A file, and migrated BEFORE the bridge reads: `void migrateStore(...)`
|
|
266
|
+
// raced the first query and the table did not exist yet.
|
|
267
|
+
const db = openSqlDb(join(tmpdir(), `ctxbridge-restart-${process.pid}-${Date.now()}.db`));
|
|
268
|
+
await migrateStore(db);
|
|
259
269
|
try {
|
|
260
270
|
const opts: BridgeOpts = {
|
|
261
271
|
client,
|
|
@@ -271,9 +281,9 @@ describe("context bridge refresh policy", () => {
|
|
|
271
281
|
recordTurns: true,
|
|
272
282
|
maxQueue: 64,
|
|
273
283
|
};
|
|
274
|
-
const first = await createContextBridge(opts).resolve(input());
|
|
284
|
+
const first = await (await createContextBridge(opts)).resolve(input());
|
|
275
285
|
// A "restart": brand-new bridge over the same store.
|
|
276
|
-
const after = await createContextBridge(opts).resolve(
|
|
286
|
+
const after = await (await createContextBridge(opts)).resolve(
|
|
277
287
|
input({ pinnedVersion: first?.version ?? null, pinnedFetchedAtMs: first?.fetchedAtMs ?? 0 }),
|
|
278
288
|
);
|
|
279
289
|
expect(after?.block).toBe(first?.block ?? "");
|
|
@@ -287,7 +297,7 @@ describe("context bridge refresh policy", () => {
|
|
|
287
297
|
describe("context bridge write-back", () => {
|
|
288
298
|
test("creates one session per conversation and attributes the model", async () => {
|
|
289
299
|
const client = mkClient();
|
|
290
|
-
const { bridge, db } = mkBridge(client);
|
|
300
|
+
const { bridge, db } = await mkBridge(client);
|
|
291
301
|
try {
|
|
292
302
|
bridge.recordTurn({
|
|
293
303
|
scope: "ashlands",
|
|
@@ -325,7 +335,7 @@ describe("context bridge write-back", () => {
|
|
|
325
335
|
|
|
326
336
|
test("recordTurns=false writes nothing", async () => {
|
|
327
337
|
const client = mkClient();
|
|
328
|
-
const { bridge, db } = mkBridge(client, { recordTurns: false });
|
|
338
|
+
const { bridge, db } = await mkBridge(client, { recordTurns: false });
|
|
329
339
|
try {
|
|
330
340
|
bridge.recordTurn({
|
|
331
341
|
scope: "ashlands",
|
|
@@ -362,7 +372,7 @@ describe("context bridge write-back", () => {
|
|
|
362
372
|
|
|
363
373
|
test("a tool loop records one turn, not one record per dispatch", async () => {
|
|
364
374
|
const client = mkClient();
|
|
365
|
-
const { bridge, db } = mkBridge(client);
|
|
375
|
+
const { bridge, db } = await mkBridge(client);
|
|
366
376
|
try {
|
|
367
377
|
// One user-visible turn: five tool round-trips, then the synthesis.
|
|
368
378
|
// Every dispatch carries the SAME unchanged user text — recording per
|
|
@@ -391,7 +401,7 @@ describe("context bridge write-back", () => {
|
|
|
391
401
|
|
|
392
402
|
test("a tool loop still running writes nothing", async () => {
|
|
393
403
|
const client = mkClient();
|
|
394
|
-
const { bridge, db } = mkBridge(client);
|
|
404
|
+
const { bridge, db } = await mkBridge(client);
|
|
395
405
|
try {
|
|
396
406
|
bridge.recordTurn(mkRecord({ assistantText: "let me look", turnEnded: false }));
|
|
397
407
|
await bridge.flush();
|
|
@@ -406,7 +416,7 @@ describe("context bridge write-back", () => {
|
|
|
406
416
|
|
|
407
417
|
test("interleaved conversations buffer independently", async () => {
|
|
408
418
|
const client = mkClient();
|
|
409
|
-
const { bridge, db } = mkBridge(client);
|
|
419
|
+
const { bridge, db } = await mkBridge(client);
|
|
410
420
|
try {
|
|
411
421
|
bridge.recordTurn(mkRecord({ conversationKey: "k1", assistantText: "k1 narration", turnEnded: false }));
|
|
412
422
|
bridge.recordTurn(mkRecord({ conversationKey: "k2", assistantText: "k2 narration", turnEnded: false }));
|
|
@@ -425,7 +435,7 @@ describe("context bridge write-back", () => {
|
|
|
425
435
|
|
|
426
436
|
test("a silent turn still records the user message", async () => {
|
|
427
437
|
const client = mkClient();
|
|
428
|
-
const { bridge, db } = mkBridge(client);
|
|
438
|
+
const { bridge, db } = await mkBridge(client);
|
|
429
439
|
try {
|
|
430
440
|
bridge.recordTurn(mkRecord({ assistantText: "", turnEnded: true }));
|
|
431
441
|
await bridge.flush();
|
|
@@ -444,7 +454,7 @@ describe("context layers a team names (project memory, phase one)", () => {
|
|
|
444
454
|
// layers in headers and the member in the harness id.
|
|
445
455
|
test("resolve passes the group, personal and user layers to assemble; a lone router passes empties", async () => {
|
|
446
456
|
const client = mkClient();
|
|
447
|
-
const { bridge, db } = mkBridge(client);
|
|
457
|
+
const { bridge, db } = await mkBridge(client);
|
|
448
458
|
try {
|
|
449
459
|
await bridge.resolve(input({ group: "group.g1", personal: "ashlands.u.u1", user: "u1" }));
|
|
450
460
|
expect(client.lastLayers).toEqual({ group: "group.g1", personal: "ashlands.u.u1", user: "u1" });
|
|
@@ -464,7 +474,7 @@ describe("context layers a team names (project memory, phase one)", () => {
|
|
|
464
474
|
// agentdox filter the project's recent tail to that harness, dropping
|
|
465
475
|
// every pre-0.16 message and every other harness's turns.
|
|
466
476
|
const client = mkClient();
|
|
467
|
-
const { bridge, db } = mkBridge(client);
|
|
477
|
+
const { bridge, db } = await mkBridge(client);
|
|
468
478
|
try {
|
|
469
479
|
await bridge.resolve(input({ user: "claude-code" }));
|
|
470
480
|
expect(client.lastLayers).toEqual({ group: "", personal: "", user: "" });
|
|
@@ -475,7 +485,7 @@ describe("context layers a team names (project memory, phase one)", () => {
|
|
|
475
485
|
|
|
476
486
|
test("layers: false sends none, whatever the request names", async () => {
|
|
477
487
|
const client = mkClient();
|
|
478
|
-
const { bridge, db } = mkBridge(client, { layers: false });
|
|
488
|
+
const { bridge, db } = await mkBridge(client, { layers: false });
|
|
479
489
|
try {
|
|
480
490
|
const pin = await bridge.resolve(input({ group: "group.g1", personal: "ashlands.u.u1", user: "u1" }));
|
|
481
491
|
expect(pin).not.toBeNull();
|
|
@@ -490,7 +500,7 @@ describe("context layers a team names (project memory, phase one)", () => {
|
|
|
490
500
|
// The version is a hash of the block's content, so two members' blocks
|
|
491
501
|
// never share a version even in one scope; the shared store keys on it.
|
|
492
502
|
const client = mkClient();
|
|
493
|
-
const { bridge, db } = mkBridge(client);
|
|
503
|
+
const { bridge, db } = await mkBridge(client);
|
|
494
504
|
try {
|
|
495
505
|
const ada = await bridge.resolve(input({ conversationKey: "ada", personal: "ashlands.u.ada", user: "ada" }));
|
|
496
506
|
const bob = await bridge.resolve(input({ conversationKey: "bob", personal: "ashlands.u.bob", user: "bob" }));
|
|
@@ -509,7 +519,7 @@ describe("context layers a team names (project memory, phase one)", () => {
|
|
|
509
519
|
|
|
510
520
|
test("the recorded turn carries user:<harnessId> on both messages only when the harness is set", async () => {
|
|
511
521
|
const client = mkClient();
|
|
512
|
-
const { bridge, db } = mkBridge(client);
|
|
522
|
+
const { bridge, db } = await mkBridge(client);
|
|
513
523
|
try {
|
|
514
524
|
const rec = {
|
|
515
525
|
scope: "ashlands",
|
|
@@ -569,7 +579,7 @@ describe("context layers a team names (project memory, phase one)", () => {
|
|
|
569
579
|
});
|
|
570
580
|
|
|
571
581
|
describe("context injection into the wire body", () => {
|
|
572
|
-
test("appends to the last system message, leaving breakpoint indices valid", () => {
|
|
582
|
+
test("appends to the last system message, leaving breakpoint indices valid", async () => {
|
|
573
583
|
const body = {
|
|
574
584
|
model: "auto",
|
|
575
585
|
messages: [
|
|
@@ -589,7 +599,7 @@ describe("context injection into the wire body", () => {
|
|
|
589
599
|
expect(text).toContain("cache_control");
|
|
590
600
|
});
|
|
591
601
|
|
|
592
|
-
test("prepends a system message and shifts breakpoints when there is none", () => {
|
|
602
|
+
test("prepends a system message and shifts breakpoints when there is none", async () => {
|
|
593
603
|
const body = { model: "auto", messages: [{ role: "user", content: "hi" }] };
|
|
594
604
|
const out = injectForTest(body, "BLOCK", [0]);
|
|
595
605
|
const msgs = out.messages as Record<string, unknown>[];
|
|
@@ -600,7 +610,7 @@ describe("context injection into the wire body", () => {
|
|
|
600
610
|
expect(JSON.stringify(msgs[1]?.content ?? "")).toContain("cache_control");
|
|
601
611
|
});
|
|
602
612
|
|
|
603
|
-
test("no contextBlock leaves the body untouched", () => {
|
|
613
|
+
test("no contextBlock leaves the body untouched", async () => {
|
|
604
614
|
const body = {
|
|
605
615
|
model: "auto",
|
|
606
616
|
messages: [
|
|
@@ -2,7 +2,11 @@ import { describe, expect, test } from "bun:test";
|
|
|
2
2
|
|
|
3
3
|
import { createContextStore } from "../src/context/store.ts";
|
|
4
4
|
import { createConversationStore } from "../src/router/state.ts";
|
|
5
|
-
import {
|
|
5
|
+
import { tmpdir } from "node:os";
|
|
6
|
+
import { join } from "node:path";
|
|
7
|
+
|
|
8
|
+
import { migrateStore } from "../src/util/schema.ts";
|
|
9
|
+
import { openSqlDb } from "../src/util/sql.ts";
|
|
6
10
|
|
|
7
11
|
/**
|
|
8
12
|
* `context_blocks` is content-addressed and shared between conversations, so
|
|
@@ -19,66 +23,69 @@ import { openDb } from "../src/util/sqlite.ts";
|
|
|
19
23
|
describe("context block prune", () => {
|
|
20
24
|
const HOUR = 3_600_000;
|
|
21
25
|
|
|
22
|
-
function seed() {
|
|
23
|
-
|
|
26
|
+
async function seed() {
|
|
27
|
+
// A file: the shim cannot share `:memory:` between handles, and both
|
|
28
|
+
// stores here read the same tables.
|
|
29
|
+
const db = openSqlDb(join(tmpdir(), `ctxprune-${process.pid}-${Date.now()}-${Math.random().toString(36).slice(2)}.db`));
|
|
30
|
+
await migrateStore(db);
|
|
24
31
|
const blocks = createContextStore(db);
|
|
25
32
|
const conversations = createConversationStore(db);
|
|
26
33
|
const now = Date.now();
|
|
27
34
|
|
|
28
|
-
blocks.put("scope", { version: "old-pinned", block: "A", fetchedAtMs: now - 5 * HOUR });
|
|
29
|
-
blocks.put("scope", { version: "old-orphan", block: "B", fetchedAtMs: now - 5 * HOUR });
|
|
30
|
-
blocks.put("scope", { version: "fresh-orphan", block: "C", fetchedAtMs: now });
|
|
35
|
+
await blocks.put("scope", { version: "old-pinned", block: "A", fetchedAtMs: now - 5 * HOUR });
|
|
36
|
+
await blocks.put("scope", { version: "old-orphan", block: "B", fetchedAtMs: now - 5 * HOUR });
|
|
37
|
+
await blocks.put("scope", { version: "fresh-orphan", block: "C", fetchedAtMs: now });
|
|
31
38
|
|
|
32
39
|
// One live conversation still pins `old-pinned`.
|
|
33
|
-
const state = conversations.load("conv-1");
|
|
40
|
+
const state = await conversations.load("conv-1");
|
|
34
41
|
state.contextVersion = "old-pinned";
|
|
35
42
|
state.contextFetchedAtMs = now - 5 * HOUR;
|
|
36
|
-
conversations.save(state);
|
|
43
|
+
await conversations.save(state);
|
|
37
44
|
|
|
38
45
|
return { db, blocks };
|
|
39
46
|
}
|
|
40
47
|
|
|
41
|
-
test("drops an old block that nothing references", () => {
|
|
42
|
-
const { db, blocks } = seed();
|
|
43
|
-
expect(blocks.prune(HOUR)).toBe(1);
|
|
44
|
-
expect(blocks.get("old-orphan")).toBeNull();
|
|
45
|
-
db.close();
|
|
48
|
+
test("drops an old block that nothing references", async () => {
|
|
49
|
+
const { db, blocks } = await seed();
|
|
50
|
+
expect(await blocks.prune(HOUR)).toBe(1);
|
|
51
|
+
expect(await blocks.get("old-orphan")).toBeNull();
|
|
52
|
+
await db.close();
|
|
46
53
|
});
|
|
47
54
|
|
|
48
|
-
test("keeps an old block a conversation still pins", () => {
|
|
49
|
-
const { db, blocks } = seed();
|
|
50
|
-
blocks.prune(HOUR);
|
|
55
|
+
test("keeps an old block a conversation still pins", async () => {
|
|
56
|
+
const { db, blocks } = await seed();
|
|
57
|
+
await blocks.prune(HOUR);
|
|
51
58
|
// Deleting this one would cost that conversation its warm prefix.
|
|
52
|
-
expect(blocks.get("old-pinned")?.block).toBe("A");
|
|
53
|
-
db.close();
|
|
59
|
+
expect((await blocks.get("old-pinned"))?.block).toBe("A");
|
|
60
|
+
await db.close();
|
|
54
61
|
});
|
|
55
62
|
|
|
56
|
-
test("keeps a block younger than the age cutoff", () => {
|
|
57
|
-
const { db, blocks } = seed();
|
|
58
|
-
blocks.prune(HOUR);
|
|
59
|
-
expect(blocks.get("fresh-orphan")?.block).toBe("C");
|
|
60
|
-
db.close();
|
|
63
|
+
test("keeps a block younger than the age cutoff", async () => {
|
|
64
|
+
const { db, blocks } = await seed();
|
|
65
|
+
await blocks.prune(HOUR);
|
|
66
|
+
expect((await blocks.get("fresh-orphan"))?.block).toBe("C");
|
|
67
|
+
await db.close();
|
|
61
68
|
});
|
|
62
69
|
|
|
63
|
-
test("is a no-op once the unreferenced blocks are gone", () => {
|
|
64
|
-
const { db, blocks } = seed();
|
|
65
|
-
expect(blocks.prune(HOUR)).toBe(1);
|
|
66
|
-
expect(blocks.prune(HOUR)).toBe(0);
|
|
67
|
-
db.close();
|
|
70
|
+
test("is a no-op once the unreferenced blocks are gone", async () => {
|
|
71
|
+
const { db, blocks } = await seed();
|
|
72
|
+
expect(await blocks.prune(HOUR)).toBe(1);
|
|
73
|
+
expect(await blocks.prune(HOUR)).toBe(0);
|
|
74
|
+
await db.close();
|
|
68
75
|
});
|
|
69
76
|
|
|
70
|
-
test("reclaims a block as soon as its last pin is dropped", () => {
|
|
71
|
-
const { db, blocks } = seed();
|
|
77
|
+
test("reclaims a block as soon as its last pin is dropped", async () => {
|
|
78
|
+
const { db, blocks } = await seed();
|
|
72
79
|
const conversations = createConversationStore(db);
|
|
73
80
|
// The conversation moves to a new context version (a refresh), which is
|
|
74
81
|
// what leaves the old block orphaned in production.
|
|
75
|
-
const state = conversations.load("conv-1");
|
|
82
|
+
const state = await conversations.load("conv-1");
|
|
76
83
|
state.contextVersion = "fresh-orphan";
|
|
77
|
-
conversations.save(state);
|
|
84
|
+
await conversations.save(state);
|
|
78
85
|
|
|
79
|
-
expect(blocks.prune(HOUR)).toBe(2); // old-pinned is now unreferenced too
|
|
80
|
-
expect(blocks.get("old-pinned")).toBeNull();
|
|
81
|
-
expect(blocks.get("fresh-orphan")?.block).toBe("C");
|
|
82
|
-
db.close();
|
|
86
|
+
expect(await blocks.prune(HOUR)).toBe(2); // old-pinned is now unreferenced too
|
|
87
|
+
expect(await blocks.get("old-pinned")).toBeNull();
|
|
88
|
+
expect((await blocks.get("fresh-orphan"))?.block).toBe("C");
|
|
89
|
+
await db.close();
|
|
83
90
|
});
|
|
84
91
|
});
|
|
@@ -11,30 +11,30 @@ import { MAX_QUERY_CHARS, relevanceQuery, userContent } from "../src/context/que
|
|
|
11
11
|
const msg = (role: "user" | "assistant" | "system" | "tool", text: string) => ({ role, text }) as never;
|
|
12
12
|
|
|
13
13
|
describe("userContent", () => {
|
|
14
|
-
test("strips wrapper elements and keeps the user's words", () => {
|
|
14
|
+
test("strips wrapper elements and keeps the user's words", async () => {
|
|
15
15
|
expect(userContent("<system-reminder>\n5 todo items still open.\n</system-reminder>\nplease fix the water tests")).toBe("please fix the water tests");
|
|
16
16
|
expect(userContent("<recap>User stepped away; returning.</recap>")).toBe("");
|
|
17
17
|
expect(userContent("<system-notice>Background job bg_43 completed.</system-notice>")).toBe("");
|
|
18
18
|
});
|
|
19
19
|
|
|
20
|
-
test("an unclosed wrapper at the start is all machinery", () => {
|
|
20
|
+
test("an unclosed wrapper at the start is all machinery", async () => {
|
|
21
21
|
expect(userContent("<chat>\nassistant said things\nuser said things")).toBe("");
|
|
22
22
|
expect(userContent("<system-reminder>Today: 2026-09-07")).toBe("");
|
|
23
23
|
});
|
|
24
24
|
|
|
25
|
-
test("auto-generated stubs are not content", () => {
|
|
25
|
+
test("auto-generated stubs are not content", async () => {
|
|
26
26
|
expect(userContent("Attached image(s) from tool result:")).toBe("");
|
|
27
27
|
expect(userContent("(no output)")).toBe("");
|
|
28
28
|
expect(userContent("continue")).toBe("");
|
|
29
29
|
});
|
|
30
30
|
|
|
31
|
-
test("ordinary text passes through with whitespace collapsed", () => {
|
|
31
|
+
test("ordinary text passes through with whitespace collapsed", async () => {
|
|
32
32
|
expect(userContent(" why does the\n\nrouter pick glm? ")).toBe("why does the router pick glm?");
|
|
33
33
|
});
|
|
34
34
|
});
|
|
35
35
|
|
|
36
36
|
describe("relevanceQuery", () => {
|
|
37
|
-
test("takes the last user message with real content, skipping wrapper-only turns", () => {
|
|
37
|
+
test("takes the last user message with real content, skipping wrapper-only turns", async () => {
|
|
38
38
|
const req = {
|
|
39
39
|
messages: [
|
|
40
40
|
msg("system", "You are omp."),
|
|
@@ -48,19 +48,19 @@ describe("relevanceQuery", () => {
|
|
|
48
48
|
expect(relevanceQuery(req)).toBe("add a /router command with reports");
|
|
49
49
|
});
|
|
50
50
|
|
|
51
|
-
test("prefers the newest real ask over the opening one", () => {
|
|
51
|
+
test("prefers the newest real ask over the opening one", async () => {
|
|
52
52
|
const req = { messages: [msg("user", "first ask"), msg("assistant", "ok"), msg("user", "<recap>x</recap> now fix the report window")] };
|
|
53
53
|
expect(relevanceQuery(req)).toBe("now fix the report window");
|
|
54
54
|
});
|
|
55
55
|
|
|
56
|
-
test("caps long asks so the block header stays a line", () => {
|
|
56
|
+
test("caps long asks so the block header stays a line", async () => {
|
|
57
57
|
const long = "x".repeat(5_000);
|
|
58
58
|
const q = relevanceQuery({ messages: [msg("user", long)] });
|
|
59
59
|
expect(q.length).toBe(MAX_QUERY_CHARS);
|
|
60
60
|
expect(q.endsWith("…")).toBe(true);
|
|
61
61
|
});
|
|
62
62
|
|
|
63
|
-
test("nothing usable yields an empty query", () => {
|
|
63
|
+
test("nothing usable yields an empty query", async () => {
|
|
64
64
|
expect(relevanceQuery({ messages: [msg("user", "<chat>\nlog"), msg("assistant", "hi")] })).toBe("");
|
|
65
65
|
expect(relevanceQuery({ messages: [] })).toBe("");
|
|
66
66
|
});
|
package/test/controls.test.ts
CHANGED
|
@@ -1,15 +1,18 @@
|
|
|
1
1
|
import { afterAll, beforeAll, describe, expect, test } from "bun:test";
|
|
2
|
+
import { tmpdir } from "node:os";
|
|
3
|
+
import { join } from "node:path";
|
|
4
|
+
import { migrateStore } from "../src/util/schema.ts";
|
|
5
|
+
import { openSqlDb } from "../src/util/sql.ts";
|
|
2
6
|
|
|
3
7
|
import { DEFAULT_CONFIG } from "../src/config/defaults.ts";
|
|
4
8
|
import type { RouterConfig } from "../src/config/types.ts";
|
|
5
9
|
import { createFeedbackStore } from "../src/cost/feedback.ts";
|
|
6
|
-
import {
|
|
10
|
+
import { createSqlLedger } from "../src/cost/ledger-sql.ts";
|
|
7
11
|
import { EMPTY_USAGE, type LedgerEntry } from "../src/cost/types.ts";
|
|
8
12
|
import { startServer, type StartedServer } from "../src/server/http.ts";
|
|
9
|
-
import { resolveProfile } from "../src/router/index.ts";
|
|
13
|
+
import { pinForRequestedModel, resolveProfile } from "../src/router/index.ts";
|
|
10
14
|
import { ollamaRunway } from "../src/server/http.ts";
|
|
11
15
|
import { createSessionOverrides, OVERRIDE_TTL_MS } from "../src/server/overrides.ts";
|
|
12
|
-
import { openDb } from "../src/util/sqlite.ts";
|
|
13
16
|
import { describeOverride, parseOverrideArgs, renderWhy, type WhyEntry } from "../omp-extension/report-logic.ts";
|
|
14
17
|
|
|
15
18
|
/**
|
|
@@ -19,7 +22,7 @@ import { describeOverride, parseOverrideArgs, renderWhy, type WhyEntry } from ".
|
|
|
19
22
|
*/
|
|
20
23
|
|
|
21
24
|
describe("session overrides", () => {
|
|
22
|
-
test("set, read, count down, clear, and expire", () => {
|
|
25
|
+
test("set, read, count down, clear, and expire", async () => {
|
|
23
26
|
const o = createSessionOverrides();
|
|
24
27
|
const now = Date.now();
|
|
25
28
|
expect(o.get("s1")).toBeNull();
|
|
@@ -44,7 +47,7 @@ describe("session overrides", () => {
|
|
|
44
47
|
expect(o.get("")).toBeNull();
|
|
45
48
|
});
|
|
46
49
|
|
|
47
|
-
test("clearing both fields reads as no override", () => {
|
|
50
|
+
test("clearing both fields reads as no override", async () => {
|
|
48
51
|
const o = createSessionOverrides();
|
|
49
52
|
o.set("s", { tier: "hard" });
|
|
50
53
|
o.set("s", { tier: null });
|
|
@@ -92,27 +95,29 @@ describe("feedback store", () => {
|
|
|
92
95
|
} as LedgerEntry;
|
|
93
96
|
}
|
|
94
97
|
|
|
95
|
-
test("records verdicts against the session's newest kept turn and counts them by model", () => {
|
|
96
|
-
const db =
|
|
98
|
+
test("records verdicts against the session's newest kept turn and counts them by model", async () => {
|
|
99
|
+
const db = openSqlDb(join(tmpdir(), `t-controls.test-${process.pid}-${Date.now()}-${Math.random().toString(36).slice(2)}.db`));
|
|
100
|
+
await migrateStore(db);
|
|
97
101
|
const cfg = structuredClone(DEFAULT_CONFIG);
|
|
98
102
|
cfg.ledger.path = ":memory:";
|
|
99
|
-
const ledger =
|
|
103
|
+
const ledger = createSqlLedger(db, cfg, { findModel: () => null });
|
|
100
104
|
const fb = createFeedbackStore(db);
|
|
101
|
-
ledger.record(entry({ createdAtMs: 1_000, slug: "a/m", servedSlug: "a/m" }));
|
|
102
|
-
ledger.record(entry({ createdAtMs: 2_000, slug: "b/m", servedSlug: "b/m", wasted: true })); // a wasted probe is never "the turn"
|
|
103
|
-
ledger.record(entry({ createdAtMs: 3_000, slug: "c/m", servedSlug: "c/m" }));
|
|
104
|
-
ledger.record(entry({ createdAtMs: 4_000, ompSessionId: "omp-2", slug: "d/m", servedSlug: "d/m" }));
|
|
105
|
-
const latest = ledger.latestForSession
|
|
106
|
-
expect(latest
|
|
107
|
-
expect(ledger.entriesForSession
|
|
108
|
-
expect(ledger.latestForSession
|
|
105
|
+
await ledger.record(entry({ createdAtMs: 1_000, slug: "a/m", servedSlug: "a/m" }));
|
|
106
|
+
await ledger.record(entry({ createdAtMs: 2_000, slug: "b/m", servedSlug: "b/m", wasted: true })); // a wasted probe is never "the turn"
|
|
107
|
+
await ledger.record(entry({ createdAtMs: 3_000, slug: "c/m", servedSlug: "c/m" }));
|
|
108
|
+
await ledger.record(entry({ createdAtMs: 4_000, ompSessionId: "omp-2", slug: "d/m", servedSlug: "d/m" }));
|
|
109
|
+
const latest = await ledger.latestForSession("omp-1");
|
|
110
|
+
expect(latest?.servedSlug).toBe("c/m");
|
|
111
|
+
expect((await ledger.entriesForSession("omp-1", 10)).map((e) => e.servedSlug)).toEqual(["c/m", "a/m"]);
|
|
112
|
+
expect(await ledger.latestForSession("")).toBeNull();
|
|
109
113
|
|
|
110
|
-
|
|
111
|
-
fb.record({ ledgerId
|
|
112
|
-
|
|
113
|
-
expect(
|
|
114
|
-
expect(fb.countsBySlug(
|
|
115
|
-
|
|
114
|
+
const ledgerId = latest?.id ?? "";
|
|
115
|
+
await fb.record({ ledgerId, ompSessionId: "omp-1", slug: "c/m", tier: "simple", verdict: "bad", note: "wrong file" }, 5_000);
|
|
116
|
+
await fb.record({ ledgerId, ompSessionId: "omp-1", slug: "c/m", tier: "simple", verdict: "good", note: "" }, 6_000);
|
|
117
|
+
expect((await fb.forLedgerId(ledgerId)).map((f) => f.verdict)).toEqual(["good", "bad"]);
|
|
118
|
+
expect([...(await fb.countsBySlug(0)).entries()]).toEqual([["c/m", { good: 1, bad: 1 }]]);
|
|
119
|
+
expect((await fb.countsBySlug(5_500)).get("c/m")).toEqual({ good: 1, bad: 0 });
|
|
120
|
+
await db.close();
|
|
116
121
|
});
|
|
117
122
|
});
|
|
118
123
|
|
|
@@ -184,7 +189,7 @@ describe("/router why and override parsing", () => {
|
|
|
184
189
|
feedback: [{ verdict: "bad", note: "edited the wrong file", createdAtMs: 1_000_500 }],
|
|
185
190
|
};
|
|
186
191
|
|
|
187
|
-
test("renderWhy names the model, tier, confidence, cost, cache, trail and feedback", () => {
|
|
192
|
+
test("renderWhy names the model, tier, confidence, cost, cache, trail and feedback", async () => {
|
|
188
193
|
const text = renderWhy(entry, 1_060_000);
|
|
189
194
|
expect(text).toContain("turn 12 · 1m ago · ollama · ollama/glm-5.3-flash (asked z-ai/glm-5.3-flash) [moderate]");
|
|
190
195
|
expect(text).toContain("classified moderate by heuristic at 42% confidence · task coding");
|
|
@@ -194,7 +199,7 @@ describe("/router why and override parsing", () => {
|
|
|
194
199
|
expect(text).toContain(" - bad: edited the wrong file");
|
|
195
200
|
});
|
|
196
201
|
|
|
197
|
-
test("parseOverrideArgs handles pin, tier with turns, off, and errors", () => {
|
|
202
|
+
test("parseOverrideArgs handles pin, tier with turns, off, and errors", async () => {
|
|
198
203
|
expect(parseOverrideArgs("pin", "ollama/glm-5.3-flash")).toEqual({ kind: "pin", slug: "ollama/glm-5.3-flash" });
|
|
199
204
|
expect(parseOverrideArgs("pin", "off")).toEqual({ kind: "pin", slug: null });
|
|
200
205
|
expect(parseOverrideArgs("pin", "")).toEqual({ kind: "show" });
|
|
@@ -205,7 +210,7 @@ describe("/router why and override parsing", () => {
|
|
|
205
210
|
expect(parseOverrideArgs("tier", "hard x").kind).toBe("error");
|
|
206
211
|
});
|
|
207
212
|
|
|
208
|
-
test("describeOverride summarises what is in force", () => {
|
|
213
|
+
test("describeOverride summarises what is in force", async () => {
|
|
209
214
|
expect(describeOverride(null)).toBe("no override on this session");
|
|
210
215
|
expect(describeOverride({ slug: "a/b", tier: "hard", turnsLeft: 1 })).toBe("pinned to a/b, tier forced to hard · 1 turn left");
|
|
211
216
|
expect(describeOverride({ slug: null, tier: "simple", turnsLeft: 0 })).toBe("tier forced to simple · until cleared");
|
|
@@ -213,7 +218,7 @@ describe("/router why and override parsing", () => {
|
|
|
213
218
|
});
|
|
214
219
|
|
|
215
220
|
describe("ollamaRunway", () => {
|
|
216
|
-
test("days of credits left at the calibrated weekly burn", () => {
|
|
221
|
+
test("days of credits left at the calibrated weekly burn", async () => {
|
|
217
222
|
const r = ollamaRunway({ usedUsd: 6.3, creditsUsd: 60 }, 7, 1.25)!;
|
|
218
223
|
expect(r.dailyBurnUsd).toBeCloseTo(1.25, 6);
|
|
219
224
|
expect(r.creditsLeftUsd).toBeCloseTo(53.7, 6);
|
|
@@ -224,7 +229,7 @@ describe("ollamaRunway", () => {
|
|
|
224
229
|
});
|
|
225
230
|
|
|
226
231
|
describe("subagent profile", () => {
|
|
227
|
-
test("a subagent asking for the default profile is routed under server.subagentProfile; explicit profiles are honoured", () => {
|
|
232
|
+
test("a subagent asking for the default profile is routed under server.subagentProfile; explicit profiles are honoured", async () => {
|
|
228
233
|
const cfg = structuredClone(DEFAULT_CONFIG);
|
|
229
234
|
expect(resolveProfile(cfg, "auto", true).id).toBe("auto-sub");
|
|
230
235
|
expect(resolveProfile(cfg, "auto", false).id).toBe("auto");
|
|
@@ -237,6 +242,28 @@ describe("subagent profile", () => {
|
|
|
237
242
|
});
|
|
238
243
|
});
|
|
239
244
|
|
|
245
|
+
describe("requested model", () => {
|
|
246
|
+
test("a catalog slug pins, a profile does not, and an unknown name is refused rather than substituted", async () => {
|
|
247
|
+
const cfg = structuredClone(DEFAULT_CONFIG);
|
|
248
|
+
const slugs = ["deepseek/deepseek-v4.1-flash", "ollama/deepseek-v4.1-flash"];
|
|
249
|
+
// A profile id routes normally: nothing is pinned.
|
|
250
|
+
expect(pinForRequestedModel(cfg, slugs, "auto", "auto")).toBeUndefined();
|
|
251
|
+
expect(pinForRequestedModel(cfg, slugs, "auto-cheap", "auto-model-router/auto-cheap")).toBeUndefined();
|
|
252
|
+
// A real slug is honoured as a pin, and the vendor prefix decides which
|
|
253
|
+
// one: the stripped name alone is ambiguous between these two.
|
|
254
|
+
expect(pinForRequestedModel(cfg, slugs, "deepseek-v4.1-flash", "deepseek/deepseek-v4.1-flash")).toBe(
|
|
255
|
+
"deepseek/deepseek-v4.1-flash",
|
|
256
|
+
);
|
|
257
|
+
expect(pinForRequestedModel(cfg, slugs, "deepseek-v4.1-flash", "ollama/deepseek-v4.1-flash")).toBe(
|
|
258
|
+
"ollama/deepseek-v4.1-flash",
|
|
259
|
+
);
|
|
260
|
+
// Neither a profile nor a slug: refused. Silently routing `auto` here is
|
|
261
|
+
// what billed a caller for glm-5.3-flash after it asked for v4.1-flash.
|
|
262
|
+
expect(() => pinForRequestedModel(cfg, slugs, "deepseek-v4.1-flash", "deepseek-v4.1-flash")).toThrow();
|
|
263
|
+
expect(() => pinForRequestedModel(cfg, slugs, "nope", "vendor/nope")).toThrow();
|
|
264
|
+
});
|
|
265
|
+
});
|
|
266
|
+
|
|
240
267
|
describe("digest endpoints", () => {
|
|
241
268
|
let handle: StartedServer;
|
|
242
269
|
let baseUrl = "";
|