auto-model-router 0.2.1 → 0.2.7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude/skills/agentdox/SKILL.md +143 -0
- package/.mcp.json +11 -0
- package/.omp-plugin/marketplace.json +2 -2
- package/CLAUDE.md +129 -0
- package/README.md +64 -0
- package/docs/AGENTDOX-BRIDGE.md +132 -0
- package/docs/context-optimization.md +362 -0
- package/omp-extension/embed-logic.ts +31 -0
- package/omp-extension/router-embed.ts +7 -1
- package/package.json +1 -1
- package/src/cli/config-cmd.ts +20 -5
- package/src/cli/explain.ts +1 -0
- package/src/config/defaults.ts +37 -0
- package/src/config/load.ts +13 -0
- package/src/config/schema.ts +30 -0
- package/src/config/types.ts +85 -0
- package/src/context/agentdox.ts +113 -0
- package/src/context/bridge.ts +166 -0
- package/src/context/index.ts +33 -0
- package/src/context/store.ts +82 -0
- package/src/context/types.ts +78 -0
- package/src/cost/ledger.ts +57 -6
- package/src/cost/types.ts +26 -0
- package/src/router/candidates.ts +41 -5
- package/src/router/classify.ts +26 -12
- package/src/router/compaction.ts +163 -0
- package/src/router/features.ts +26 -13
- package/src/router/select.ts +37 -4
- package/src/router/state.ts +12 -2
- package/src/router/types.ts +33 -1
- package/src/server/http.ts +18 -1
- package/src/server/turn.ts +88 -1
- package/src/upstream/openrouter.ts +8 -1
- package/src/util/sqlite.ts +34 -1
- package/src/wire/openai/request.ts +86 -1
- package/src/wire/types.ts +36 -0
- package/test/classify.test.ts +63 -5
- package/test/compaction.test.ts +148 -0
- package/test/context-bridge.test.ts +337 -0
- package/test/embed-logic.test.ts +32 -0
- package/test/escalate.test.ts +1 -0
- package/test/exploration.test.ts +6 -2
- package/test/failover.test.ts +52 -6
- package/test/features.test.ts +45 -0
- package/test/helpers/inject.ts +23 -0
- package/test/hold-exploration.test.ts +4 -2
- package/test/select.test.ts +129 -0
- package/test/tokens.test.ts +1 -0
- package/test/trust-attribution.test.ts +61 -4
- package/test/turn.test.ts +21 -10
- package/tools/agentdox-e2e.ts +123 -0
package/test/failover.test.ts
CHANGED
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import { describe, expect, test } from "bun:test";
|
|
2
|
+
import { createDisabledBridge } from "../src/context/bridge.ts";
|
|
2
3
|
import type { CatalogSource } from "../src/catalog/types.ts";
|
|
3
4
|
import type { EscalationConfig, RouterConfig } from "../src/config/types.ts";
|
|
4
5
|
import { EMPTY_USAGE, type Ledger, type LedgerEntry, type UsageCounts } from "../src/cost/types.ts";
|
|
@@ -43,7 +44,7 @@ function mkConfig(escalation: Partial<EscalationConfig> = {}): RouterConfig {
|
|
|
43
44
|
data: { axis: "intelligence", minQuality: 0 },
|
|
44
45
|
chat: { axis: "intelligence", minQuality: 0 },
|
|
45
46
|
},
|
|
46
|
-
filters: { allow: [], deny: [], includeFree: false, requireToolSupport: true, minTrust: 0.6, minTrustSamples: 5, trustScopedByHarness: false, contextHeadroom: 1.2 },
|
|
47
|
+
filters: { allow: [], deny: [], includeFree: false, requireToolSupport: true, minTrust: 0.6, minTrustSamples: 5, trustScopedByHarness: false, contextHeadroom: 1.2, latencyWeight: 0, latencyReferenceMs: 5000, latencyReferenceTokensPerSec: 30, latencyMinSamples: 20 },
|
|
47
48
|
classifier: {
|
|
48
49
|
ambiguityThreshold: 0,
|
|
49
50
|
model: "test/adjudicator",
|
|
@@ -68,6 +69,8 @@ function mkConfig(escalation: Partial<EscalationConfig> = {}): RouterConfig {
|
|
|
68
69
|
hysteresis: { holdTurns: 2, holdTurnsAfterEscalation: 4, switchMargin: 1.5, cacheWarmTtlMs: 600_000, maxDowngradePerTurn: 1 },
|
|
69
70
|
exploration: { enabled: false, rates: {}, stickyPolicy: "never", holdTurns: { enabled: false, values: [2, 3, 4] } },
|
|
70
71
|
cache: { injectBreakpoints: true, maxBreakpoints: 4, minPromptTokens: 1024 },
|
|
72
|
+
context: { enabled: false, baseUrl: "", token: "", defaultScope: "", timeoutMs: 3_000, maxStalenessMs: 900_000, maxBlockChars: 24_000, recordTurns: false, maxQueue: 64 },
|
|
73
|
+
compaction: { enabled: false, budgetTokens: 40_000, fitToWindow: true, protectRecentTurns: 4, maxToolResultBytes: 4_096, keepHeadBytes: 512, keepTailBytes: 512, elideSupersededReads: true, collapseDuplicateResults: true },
|
|
71
74
|
budget: { onExceeded: "downgrade" },
|
|
72
75
|
profiles: [],
|
|
73
76
|
ledger: { path: ":memory:", blendWindowDays: 7, blendMinSamples: 20, fallbackBlend: { inputPerMtok: 1, outputPerMtok: 4 }, conversationTtlMs: 86_400_000 },
|
|
@@ -83,6 +86,7 @@ function mkReq(): NormRequest {
|
|
|
83
86
|
conversationKey: "conv-test",
|
|
84
87
|
harnessId: "",
|
|
85
88
|
ompSessionId: "",
|
|
89
|
+
agentdoxScope: "",
|
|
86
90
|
requestedModel: "auto",
|
|
87
91
|
messages: [{ role: "user", text: "hi", images: 0, textBytes: 2, toolCalls: [] }],
|
|
88
92
|
tools: [],
|
|
@@ -107,7 +111,9 @@ const FEATURES: Features = {
|
|
|
107
111
|
distinctToolsUsed: 0,
|
|
108
112
|
lastToolFailed: false,
|
|
109
113
|
repeatedToolCall: false,
|
|
114
|
+
circularToolCall: false,
|
|
110
115
|
hasImages: false,
|
|
116
|
+
hasNewImage: false,
|
|
111
117
|
codeBlocks: 0,
|
|
112
118
|
codeBytes: 0,
|
|
113
119
|
looksLikeDiff: false,
|
|
@@ -137,6 +143,8 @@ function mkDecision(tier: Tier, slug: string, probe: Partial<ProbePlan> = {}): D
|
|
|
137
143
|
sessionId: "omp-conv-test",
|
|
138
144
|
sticky: false,
|
|
139
145
|
cacheBreakpointMessageIndices: [],
|
|
146
|
+
compactionPlan: [],
|
|
147
|
+
promptTokensSaved: 0,
|
|
140
148
|
reasoning: undefined,
|
|
141
149
|
maxTokens: undefined,
|
|
142
150
|
stripAssistantReasoning: false,
|
|
@@ -229,6 +237,7 @@ function mkLedger(): { ledger: Ledger; entries: LedgerEntry[] } {
|
|
|
229
237
|
conversationSpend: () => 0,
|
|
230
238
|
spendSince: () => 0,
|
|
231
239
|
blendedRate: () => null,
|
|
240
|
+
latency: () => null,
|
|
232
241
|
trust: () => null,
|
|
233
242
|
allTrust: () => [],
|
|
234
243
|
tokenRatio: () => null,
|
|
@@ -256,6 +265,8 @@ function mkConversations(): { store: ConversationStore; map: Map<string, Convers
|
|
|
256
265
|
lastPromptTokens: 0,
|
|
257
266
|
cacheWarmSlug: null,
|
|
258
267
|
cacheWarmAtMs: 0,
|
|
268
|
+
contextVersion: null,
|
|
269
|
+
contextFetchedAtMs: 0,
|
|
259
270
|
updatedAtMs: 0,
|
|
260
271
|
};
|
|
261
272
|
map.set(k, fresh);
|
|
@@ -319,7 +330,7 @@ describe("same-tier failover", () => {
|
|
|
319
330
|
const { store } = mkConversations();
|
|
320
331
|
const { sink, errors, finishes } = mkSink();
|
|
321
332
|
|
|
322
|
-
await runTurn(mkReq(), sink, { config: mkConfig(), router, upstream, ledger, conversations: store, catalog }, new AbortController().signal);
|
|
333
|
+
await runTurn(mkReq(), sink, { config: mkConfig(), router, upstream, ledger, conversations: store, catalog, context: createDisabledBridge() }, new AbortController().signal);
|
|
323
334
|
|
|
324
335
|
expect(errors).toHaveLength(0);
|
|
325
336
|
expect(finishes).toHaveLength(1);
|
|
@@ -349,6 +360,41 @@ describe("same-tier failover", () => {
|
|
|
349
360
|
expect(finishes[0]!.attempts).toBe(2);
|
|
350
361
|
});
|
|
351
362
|
|
|
363
|
+
test("a 403 moderation block fails over to a different model in the same tier", async () => {
|
|
364
|
+
const { router, calls } = mkRouter([mkDecision("trivial", "a/model"), mkDecision("trivial", "b/model")]);
|
|
365
|
+
const { upstream, calls: dispatches } = mkUpstream([
|
|
366
|
+
{ kind: "fail", error: new UpstreamError("moderation", 403, "Request blocked: prompt injection", true) },
|
|
367
|
+
{ kind: "chunks", chunks: okChunks("b/model") },
|
|
368
|
+
]);
|
|
369
|
+
const { ledger, entries } = mkLedger();
|
|
370
|
+
const { store } = mkConversations();
|
|
371
|
+
const { sink, errors, finishes } = mkSink();
|
|
372
|
+
|
|
373
|
+
await runTurn(mkReq(), sink, { config: mkConfig(), router, upstream, ledger, conversations: store, catalog, context: createDisabledBridge() }, new AbortController().signal);
|
|
374
|
+
|
|
375
|
+
expect(errors).toHaveLength(0);
|
|
376
|
+
expect(finishes).toHaveLength(1);
|
|
377
|
+
|
|
378
|
+
// A per-model policy block indicts the slug, not the tier: the retry
|
|
379
|
+
// re-routes with the blocked slug excluded and serves a sibling.
|
|
380
|
+
expect(calls).toHaveLength(2);
|
|
381
|
+
expect(calls[1]).toEqual({ attempt: 1, excludeSlugs: ["a/model"] });
|
|
382
|
+
expect(dispatches.map((d) => d.body.model)).toEqual(["a/model", "b/model"]);
|
|
383
|
+
|
|
384
|
+
expect(entries).toHaveLength(2);
|
|
385
|
+
expect(entries[0]!.slug).toBe("a/model");
|
|
386
|
+
expect(entries[0]!.wasted).toBe(true);
|
|
387
|
+
expect(entries[0]!.escalationSignal).toBeNull(); // failover, not escalation
|
|
388
|
+
expect(entries[0]!.error).toContain("moderation");
|
|
389
|
+
expect(entries[1]!.slug).toBe("b/model");
|
|
390
|
+
expect(entries[1]!.tier).toBe("trivial");
|
|
391
|
+
expect(entries[1]!.wasted).toBe(false);
|
|
392
|
+
expect(entries[1]!.reasons).toContain("failover: a/model returned moderation; retrying b/model in trivial");
|
|
393
|
+
|
|
394
|
+
expect(finishes[0]!.servedSlug).toBe("b/model");
|
|
395
|
+
expect(finishes[0]!.escalated).toBe(false);
|
|
396
|
+
});
|
|
397
|
+
|
|
352
398
|
test("a tier with no other eligible model falls back to tier escalation", async () => {
|
|
353
399
|
// The router widens to "simple" when "trivial" excludes a/model: the
|
|
354
400
|
// failover probe's decision is discarded and the normal escalation path
|
|
@@ -366,7 +412,7 @@ describe("same-tier failover", () => {
|
|
|
366
412
|
const { store } = mkConversations();
|
|
367
413
|
const { sink, errors, finishes } = mkSink();
|
|
368
414
|
|
|
369
|
-
await runTurn(mkReq(), sink, { config: mkConfig(), router, upstream, ledger, conversations: store, catalog }, new AbortController().signal);
|
|
415
|
+
await runTurn(mkReq(), sink, { config: mkConfig(), router, upstream, ledger, conversations: store, catalog, context: createDisabledBridge() }, new AbortController().signal);
|
|
370
416
|
|
|
371
417
|
expect(errors).toHaveLength(0);
|
|
372
418
|
expect(finishes).toHaveLength(1);
|
|
@@ -400,7 +446,7 @@ describe("same-tier failover", () => {
|
|
|
400
446
|
const { store } = mkConversations();
|
|
401
447
|
const { sink, errors, finishes } = mkSink();
|
|
402
448
|
|
|
403
|
-
await runTurn(mkReq(), sink, { config: mkConfig(), router, upstream, ledger, conversations: store, catalog }, new AbortController().signal);
|
|
449
|
+
await runTurn(mkReq(), sink, { config: mkConfig(), router, upstream, ledger, conversations: store, catalog, context: createDisabledBridge() }, new AbortController().signal);
|
|
404
450
|
|
|
405
451
|
expect(calls).toHaveLength(1);
|
|
406
452
|
expect(calls[0]).toEqual({ attempt: 0 }); // never re-routed, never given excludeSlugs
|
|
@@ -433,7 +479,7 @@ describe("same-tier failover", () => {
|
|
|
433
479
|
await runTurn(
|
|
434
480
|
mkReq(),
|
|
435
481
|
sink,
|
|
436
|
-
{ config: mkConfig({ maxAttempts: 5 }), router, upstream, ledger, conversations: store, catalog },
|
|
482
|
+
{ config: mkConfig({ maxAttempts: 5 }), router, upstream, ledger, conversations: store, catalog, context: createDisabledBridge() },
|
|
437
483
|
new AbortController().signal,
|
|
438
484
|
);
|
|
439
485
|
|
|
@@ -479,7 +525,7 @@ describe("same-tier failover", () => {
|
|
|
479
525
|
const { store } = mkConversations();
|
|
480
526
|
const { sink, chunks, errors, finishes } = mkSink();
|
|
481
527
|
|
|
482
|
-
await runTurn(mkReq(), sink, { config: mkConfig(), router, upstream, ledger, conversations: store, catalog }, new AbortController().signal);
|
|
528
|
+
await runTurn(mkReq(), sink, { config: mkConfig(), router, upstream, ledger, conversations: store, catalog, context: createDisabledBridge() }, new AbortController().signal);
|
|
483
529
|
|
|
484
530
|
expect(errors).toHaveLength(0);
|
|
485
531
|
expect(calls).toHaveLength(2);
|
package/test/features.test.ts
CHANGED
|
@@ -133,6 +133,7 @@ describe("failure and loop signals", () => {
|
|
|
133
133
|
100,
|
|
134
134
|
);
|
|
135
135
|
expect(f.repeatedToolCall).toBe(true);
|
|
136
|
+
expect(f.circularToolCall).toBe(true);
|
|
136
137
|
});
|
|
137
138
|
|
|
138
139
|
test("different arguments to the same tool are not a loop", () => {
|
|
@@ -148,6 +149,26 @@ describe("failure and loop signals", () => {
|
|
|
148
149
|
100,
|
|
149
150
|
);
|
|
150
151
|
expect(f.repeatedToolCall).toBe(false);
|
|
152
|
+
expect(f.circularToolCall).toBe(false);
|
|
153
|
+
});
|
|
154
|
+
|
|
155
|
+
test("a non-adjacent re-issued call is circular but not an adjacent repeat", () => {
|
|
156
|
+
const f = extractFeatures(
|
|
157
|
+
req([
|
|
158
|
+
SYSTEM,
|
|
159
|
+
{ role: "user", content: "go" },
|
|
160
|
+
toolCall("c1", "read", '{"path":"same.ts"}'),
|
|
161
|
+
{ role: "tool", tool_call_id: "c1", content: "x" },
|
|
162
|
+
toolCall("c2", "bash", '{"command":"ls"}'),
|
|
163
|
+
{ role: "tool", tool_call_id: "c2", content: "a.txt" },
|
|
164
|
+
toolCall("c3", "read", '{"path":"same.ts"}'),
|
|
165
|
+
{ role: "tool", tool_call_id: "c3", content: "x" },
|
|
166
|
+
]),
|
|
167
|
+
100,
|
|
168
|
+
);
|
|
169
|
+
// c3 repeats c1 verbatim with c2 in between: not adjacent, but circular.
|
|
170
|
+
expect(f.repeatedToolCall).toBe(false);
|
|
171
|
+
expect(f.circularToolCall).toBe(true);
|
|
151
172
|
});
|
|
152
173
|
});
|
|
153
174
|
|
|
@@ -223,6 +244,30 @@ describe("newest-content scoping", () => {
|
|
|
223
244
|
100,
|
|
224
245
|
);
|
|
225
246
|
expect(f.hasImages).toBe(true);
|
|
247
|
+
expect(f.hasNewImage).toBe(true);
|
|
226
248
|
expect(f.requestedReasoning).toBe("high");
|
|
227
249
|
});
|
|
250
|
+
|
|
251
|
+
test("a stale image in history is not new visual work on a tool continuation", () => {
|
|
252
|
+
const f = extractFeatures(
|
|
253
|
+
req([
|
|
254
|
+
SYSTEM,
|
|
255
|
+
{
|
|
256
|
+
role: "user",
|
|
257
|
+
content: [
|
|
258
|
+
{ type: "text", text: "implement this screen" },
|
|
259
|
+
{ type: "image_url", image_url: { url: "data:image/png;base64,AA" } },
|
|
260
|
+
],
|
|
261
|
+
},
|
|
262
|
+
toolCall("c1", "read", '{"path":"src/app.tsx"}'),
|
|
263
|
+
{ role: "tool", tool_call_id: "c1", name: "read", content: "export const App = () => null;" },
|
|
264
|
+
]),
|
|
265
|
+
100,
|
|
266
|
+
);
|
|
267
|
+
// The image is still in context (capability), but the current turn is a
|
|
268
|
+
// mechanical continuation, not fresh visual work (task axis).
|
|
269
|
+
expect(f.hasImages).toBe(true);
|
|
270
|
+
expect(f.hasNewImage).toBe(false);
|
|
271
|
+
expect(f.isToolResultContinuation).toBe(true);
|
|
272
|
+
});
|
|
228
273
|
});
|
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Exercises the wire's context injection through the public render path, so
|
|
3
|
+
* the test covers the real ordering (inject, then apply cache breakpoints).
|
|
4
|
+
*/
|
|
5
|
+
import { parseChatRequest } from "../../src/wire/openai/request.ts";
|
|
6
|
+
|
|
7
|
+
export function injectForTest(
|
|
8
|
+
body: Record<string, unknown>,
|
|
9
|
+
block: string,
|
|
10
|
+
breakpoints: number[],
|
|
11
|
+
): Record<string, unknown> {
|
|
12
|
+
const req = parseChatRequest(body, new Headers());
|
|
13
|
+
return req.renderUpstreamBody({
|
|
14
|
+
slug: "vendor/model",
|
|
15
|
+
fallbacks: [],
|
|
16
|
+
sessionId: "s",
|
|
17
|
+
cacheBreakpointMessageIndices: breakpoints,
|
|
18
|
+
reasoning: undefined,
|
|
19
|
+
maxTokens: undefined,
|
|
20
|
+
stripAssistantReasoning: false,
|
|
21
|
+
contextBlock: block,
|
|
22
|
+
});
|
|
23
|
+
}
|
|
@@ -1,10 +1,12 @@
|
|
|
1
1
|
import { describe, expect, test } from "bun:test";
|
|
2
2
|
|
|
3
|
-
import {
|
|
3
|
+
import { DEFAULT_CONFIG } from "../src/config/defaults.ts";
|
|
4
4
|
import type { RouterConfig } from "../src/config/types.ts";
|
|
5
5
|
import { explorationDraw, resolveHoldTurns } from "../src/router/explore.ts";
|
|
6
6
|
|
|
7
|
-
|
|
7
|
+
// Shipped defaults, not loadConfig({}) — the latter merges the live home
|
|
8
|
+
// config.yml and makes this suite depend on the developer's local settings.
|
|
9
|
+
const BASE = DEFAULT_CONFIG;
|
|
8
10
|
|
|
9
11
|
function withHold(over: { enabled?: boolean; values?: number[] }, enabled = true): RouterConfig {
|
|
10
12
|
return {
|
package/test/select.test.ts
CHANGED
|
@@ -64,6 +64,8 @@ function state(over: Partial<ConversationState> = {}): ConversationState {
|
|
|
64
64
|
lastPromptTokens: 0,
|
|
65
65
|
cacheWarmSlug: null,
|
|
66
66
|
cacheWarmAtMs: 0,
|
|
67
|
+
contextVersion: null,
|
|
68
|
+
contextFetchedAtMs: 0,
|
|
67
69
|
updatedAtMs: Date.now(),
|
|
68
70
|
...over,
|
|
69
71
|
};
|
|
@@ -257,6 +259,7 @@ describe("budget guard", () => {
|
|
|
257
259
|
conversationSpend: () => 0,
|
|
258
260
|
spendSince: (_sinceMs, harnessId) => (harnessId === undefined ? 1.0 : spendByHarness[harnessId] ?? 0),
|
|
259
261
|
blendedRate: () => null,
|
|
262
|
+
latency: () => null,
|
|
260
263
|
trust: () => null,
|
|
261
264
|
allTrust: () => [],
|
|
262
265
|
tokenRatio: () => null,
|
|
@@ -283,6 +286,7 @@ describe("per-harness trust scoping", () => {
|
|
|
283
286
|
conversationSpend: () => 0,
|
|
284
287
|
spendSince: () => 0,
|
|
285
288
|
blendedRate: () => null,
|
|
289
|
+
latency: () => null,
|
|
286
290
|
trust: (_slug, harnessId) => {
|
|
287
291
|
// Harness A has burned the model; harness B has never tried it.
|
|
288
292
|
if (harnessId === "harness-a") {
|
|
@@ -418,6 +422,7 @@ describe("tier rescue under a guardrail-constrained catalog", () => {
|
|
|
418
422
|
conversationSpend: () => 0,
|
|
419
423
|
spendSince: () => 0,
|
|
420
424
|
blendedRate: () => null,
|
|
425
|
+
latency: () => null,
|
|
421
426
|
trust: (slug) => ({
|
|
422
427
|
slug,
|
|
423
428
|
attempts: 40,
|
|
@@ -536,3 +541,127 @@ describe("task-type routing", () => {
|
|
|
536
541
|
}
|
|
537
542
|
});
|
|
538
543
|
});
|
|
544
|
+
|
|
545
|
+
describe("latency scoring", () => {
|
|
546
|
+
function ledgerWithLatency(bySlug: Record<string, { ttftMs: number; samples: number; tokensPerSec?: number }>): Ledger {
|
|
547
|
+
return {
|
|
548
|
+
record: () => {},
|
|
549
|
+
conversationSpend: () => 0,
|
|
550
|
+
spendSince: () => 0,
|
|
551
|
+
blendedRate: () => null,
|
|
552
|
+
trust: () => null,
|
|
553
|
+
allTrust: () => [],
|
|
554
|
+
latency: (slug) => {
|
|
555
|
+
const v = bySlug[slug];
|
|
556
|
+
// Default throughput is fast, so these cases isolate the TTFT axis
|
|
557
|
+
// unless a test sets tokensPerSec explicitly.
|
|
558
|
+
return v === undefined ? null : { slug, samples: v.samples, ttftMs: v.ttftMs, tokensPerSec: v.tokensPerSec ?? 1000 };
|
|
559
|
+
},
|
|
560
|
+
tokenRatio: () => null,
|
|
561
|
+
recentEntries: () => [],
|
|
562
|
+
};
|
|
563
|
+
}
|
|
564
|
+
|
|
565
|
+
const withWeight = (latencyWeight: number): RouterConfig => ({
|
|
566
|
+
...BASE,
|
|
567
|
+
filters: { ...BASE.filters, latencyWeight, latencyReferenceMs: 5000, latencyMinSamples: 20 },
|
|
568
|
+
});
|
|
569
|
+
|
|
570
|
+
test("penalises a chronically slow model out of the top slot", () => {
|
|
571
|
+
const slow = run({ tier: "simple" }).slug;
|
|
572
|
+
const ledger = ledgerWithLatency({ [slow]: { ttftMs: 60_000, samples: 50 } });
|
|
573
|
+
const d = run({ tier: "simple", cfg: withWeight(2), ledger });
|
|
574
|
+
expect(d.slug).not.toBe(slow);
|
|
575
|
+
});
|
|
576
|
+
|
|
577
|
+
test("latencyWeight 0 disables the penalty", () => {
|
|
578
|
+
const slow = run({ tier: "simple" }).slug;
|
|
579
|
+
const ledger = ledgerWithLatency({ [slow]: { ttftMs: 60_000, samples: 50 } });
|
|
580
|
+
expect(run({ tier: "simple", cfg: withWeight(0), ledger }).slug).toBe(slow);
|
|
581
|
+
});
|
|
582
|
+
|
|
583
|
+
test("a model with too few samples is not penalised", () => {
|
|
584
|
+
const slow = run({ tier: "simple" }).slug;
|
|
585
|
+
const ledger = ledgerWithLatency({ [slow]: { ttftMs: 60_000, samples: 5 } });
|
|
586
|
+
expect(run({ tier: "simple", cfg: withWeight(2), ledger }).slug).toBe(slow);
|
|
587
|
+
});
|
|
588
|
+
|
|
589
|
+
test("penalises a model that starts fast but streams slowly", () => {
|
|
590
|
+
// The case TTFT-only scoring misses: quick first token, slow body.
|
|
591
|
+
const slow = run({ tier: "simple" }).slug;
|
|
592
|
+
const ledger = ledgerWithLatency({ [slow]: { ttftMs: 1500, samples: 50, tokensPerSec: 12 } });
|
|
593
|
+
const d = run({ tier: "simple", cfg: withWeight(2), ledger });
|
|
594
|
+
expect(d.slug).not.toBe(slow);
|
|
595
|
+
});
|
|
596
|
+
});
|
|
597
|
+
|
|
598
|
+
describe("context compaction", () => {
|
|
599
|
+
const COMPACT_CFG: RouterConfig = {
|
|
600
|
+
...BASE,
|
|
601
|
+
compaction: {
|
|
602
|
+
enabled: true,
|
|
603
|
+
budgetTokens: 1_000,
|
|
604
|
+
fitToWindow: false,
|
|
605
|
+
protectRecentTurns: 1,
|
|
606
|
+
maxToolResultBytes: 100,
|
|
607
|
+
keepHeadBytes: 20,
|
|
608
|
+
keepTailBytes: 20,
|
|
609
|
+
elideSupersededReads: true,
|
|
610
|
+
collapseDuplicateResults: true,
|
|
611
|
+
},
|
|
612
|
+
};
|
|
613
|
+
|
|
614
|
+
function loopReq(): NormRequest {
|
|
615
|
+
return parseChatRequest(
|
|
616
|
+
{
|
|
617
|
+
model: "auto",
|
|
618
|
+
tools: TOOLS,
|
|
619
|
+
messages: [
|
|
620
|
+
{ role: "system", content: "You are a coding agent." },
|
|
621
|
+
{ role: "user", content: "read the file" },
|
|
622
|
+
{ role: "assistant", content: null, tool_calls: [{ id: "c1", type: "function", function: { name: "read", arguments: '{"path":"big.ts"}' } }] },
|
|
623
|
+
{ role: "tool", tool_call_id: "c1", content: "x".repeat(4000) },
|
|
624
|
+
{ role: "user", content: "continue" },
|
|
625
|
+
],
|
|
626
|
+
},
|
|
627
|
+
new Headers(),
|
|
628
|
+
);
|
|
629
|
+
}
|
|
630
|
+
|
|
631
|
+
test("an over-budget turn produces a compaction plan and records savings", () => {
|
|
632
|
+
const req = loopReq();
|
|
633
|
+
const features = extractFeatures(req, 5_000); // over budgetTokens=1000
|
|
634
|
+
const d = select({
|
|
635
|
+
req,
|
|
636
|
+
features,
|
|
637
|
+
classification: scoreHeuristic(features, COMPACT_CFG),
|
|
638
|
+
profile: PROFILE,
|
|
639
|
+
state: state(),
|
|
640
|
+
snapshot: SNAPSHOT,
|
|
641
|
+
ledger: null,
|
|
642
|
+
cfg: COMPACT_CFG,
|
|
643
|
+
nowMs: Date.now(),
|
|
644
|
+
});
|
|
645
|
+
expect(d.compactionPlan.length).toBeGreaterThan(0);
|
|
646
|
+
expect(d.promptTokensSaved).toBeGreaterThan(0);
|
|
647
|
+
expect(d.reasons.some((r) => r.startsWith("compaction:"))).toBe(true);
|
|
648
|
+
});
|
|
649
|
+
|
|
650
|
+
test("a small turn is left untouched", () => {
|
|
651
|
+
const req = loopReq();
|
|
652
|
+
const features = extractFeatures(req, 500); // under budgetTokens=1000
|
|
653
|
+
const d = select({
|
|
654
|
+
req,
|
|
655
|
+
features,
|
|
656
|
+
classification: scoreHeuristic(features, COMPACT_CFG),
|
|
657
|
+
profile: PROFILE,
|
|
658
|
+
state: state(),
|
|
659
|
+
snapshot: SNAPSHOT,
|
|
660
|
+
ledger: null,
|
|
661
|
+
cfg: COMPACT_CFG,
|
|
662
|
+
nowMs: Date.now(),
|
|
663
|
+
});
|
|
664
|
+
expect(d.compactionPlan).toEqual([]);
|
|
665
|
+
expect(d.promptTokensSaved).toBe(0);
|
|
666
|
+
});
|
|
667
|
+
});
|
package/test/tokens.test.ts
CHANGED
|
@@ -40,6 +40,7 @@ function entry(over: Partial<LedgerEntry>): LedgerEntry {
|
|
|
40
40
|
wasted: false,
|
|
41
41
|
upstreamGenerationId: null,
|
|
42
42
|
error: null,
|
|
43
|
+
promptTokensSaved: 0,
|
|
43
44
|
...over,
|
|
44
45
|
};
|
|
45
46
|
}
|
|
@@ -75,8 +76,19 @@ describe("trust attribution", () => {
|
|
|
75
76
|
trustAfter([
|
|
76
77
|
null,
|
|
77
78
|
null,
|
|
78
|
-
"auth:
|
|
79
|
-
"auth:
|
|
79
|
+
"auth: No auth credentials found",
|
|
80
|
+
"auth: Insufficient credits",
|
|
81
|
+
]),
|
|
82
|
+
).toBe(CLEAN);
|
|
83
|
+
});
|
|
84
|
+
|
|
85
|
+
test("provider moderation/policy blocks do not count against the model", () => {
|
|
86
|
+
expect(
|
|
87
|
+
trustAfter([
|
|
88
|
+
null,
|
|
89
|
+
null,
|
|
90
|
+
"moderation: Request blocked: prompt injection patterns detected",
|
|
91
|
+
"moderation: This model requires 18+ age confirmation",
|
|
80
92
|
]),
|
|
81
93
|
).toBe(CLEAN);
|
|
82
94
|
});
|
|
@@ -145,6 +157,49 @@ describe("trust attribution", () => {
|
|
|
145
157
|
});
|
|
146
158
|
});
|
|
147
159
|
|
|
160
|
+
describe("latency signal", () => {
|
|
161
|
+
function latencyOf(rows: Array<Partial<LedgerEntry>>): { samples: number; ttftMs: number; tokensPerSec: number } | null {
|
|
162
|
+
const db = openDb(":memory:");
|
|
163
|
+
try {
|
|
164
|
+
const ledger = createLedger(db, cfg);
|
|
165
|
+
for (const r of rows) ledger.record(entry(r));
|
|
166
|
+
const l = ledger.latency("vendor/model");
|
|
167
|
+
return l === null ? null : { samples: l.samples, ttftMs: l.ttftMs, tokensPerSec: l.tokensPerSec };
|
|
168
|
+
} finally {
|
|
169
|
+
db.close();
|
|
170
|
+
}
|
|
171
|
+
}
|
|
172
|
+
|
|
173
|
+
test("averages TTFT over streamed, non-errored turns", () => {
|
|
174
|
+
expect(latencyOf([{ ttftMs: 50 }, { ttftMs: 100 }, { ttftMs: 150 }])).toEqual({ samples: 3, ttftMs: 100, tokensPerSec: 0 });
|
|
175
|
+
});
|
|
176
|
+
|
|
177
|
+
test("excludes errored, aborted, and non-streamed (null TTFT) rows", () => {
|
|
178
|
+
expect(
|
|
179
|
+
latencyOf([
|
|
180
|
+
{ ttftMs: 100 },
|
|
181
|
+
{ ttftMs: 9999, error: "upstream_error: boom" },
|
|
182
|
+
{ ttftMs: 9999, error: "request aborted" },
|
|
183
|
+
{ ttftMs: null },
|
|
184
|
+
]),
|
|
185
|
+
).toEqual({ samples: 1, ttftMs: 100, tokensPerSec: 0 });
|
|
186
|
+
});
|
|
187
|
+
|
|
188
|
+
test("null when no streamed sample exists", () => {
|
|
189
|
+
expect(latencyOf([{ ttftMs: null }, { ttftMs: 0 }])).toBeNull();
|
|
190
|
+
});
|
|
191
|
+
|
|
192
|
+
test("throughput is aggregate completion tokens per post-TTFT second", () => {
|
|
193
|
+
const l = latencyOf([
|
|
194
|
+
{ ttftMs: 1000, latencyMs: 3000, usage: { ...EMPTY_USAGE, completionTokens: 200 } },
|
|
195
|
+
{ ttftMs: 1000, latencyMs: 3000, usage: { ...EMPTY_USAGE, completionTokens: 200 } },
|
|
196
|
+
]);
|
|
197
|
+
// 400 completion tokens over 4000ms of post-TTFT time = 100 tok/s.
|
|
198
|
+
expect(l?.tokensPerSec).toBeCloseTo(100, 5);
|
|
199
|
+
expect(l?.samples).toBe(2);
|
|
200
|
+
});
|
|
201
|
+
});
|
|
202
|
+
|
|
148
203
|
describe("v4 migration", () => {
|
|
149
204
|
test("backfills error_kind from stored error text", () => {
|
|
150
205
|
const db = openDb(":memory:");
|
|
@@ -152,6 +207,7 @@ describe("v4 migration", () => {
|
|
|
152
207
|
const ledger = createLedger(db, cfg);
|
|
153
208
|
ledger.record(entry({ error: "request aborted" }));
|
|
154
209
|
ledger.record(entry({ error: "auth: nope" }));
|
|
210
|
+
ledger.record(entry({ error: "moderation: Request blocked: prompt injection" }));
|
|
155
211
|
ledger.record(entry({ error: "model_unavailable: guardrail" }));
|
|
156
212
|
ledger.record(entry({ error: "upstream_error: boom" }));
|
|
157
213
|
ledger.record(entry({ error: null }));
|
|
@@ -162,6 +218,7 @@ describe("v4 migration", () => {
|
|
|
162
218
|
expect(rows.map((r) => r.error_kind)).toEqual([
|
|
163
219
|
"aborted",
|
|
164
220
|
"auth",
|
|
221
|
+
"moderation",
|
|
165
222
|
"model_unavailable",
|
|
166
223
|
"upstream_error",
|
|
167
224
|
null,
|
|
@@ -171,11 +228,11 @@ describe("v4 migration", () => {
|
|
|
171
228
|
}
|
|
172
229
|
});
|
|
173
230
|
|
|
174
|
-
test("schema is at user_version
|
|
231
|
+
test("schema is at user_version 12", () => {
|
|
175
232
|
const db = openDb(":memory:");
|
|
176
233
|
try {
|
|
177
234
|
const row = db.query("PRAGMA user_version").get() as { user_version: number };
|
|
178
|
-
expect(row.user_version).toBe(
|
|
235
|
+
expect(row.user_version).toBe(12);
|
|
179
236
|
} finally {
|
|
180
237
|
db.close();
|
|
181
238
|
}
|
package/test/turn.test.ts
CHANGED
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import { describe, expect, test } from "bun:test";
|
|
2
|
+
import { createDisabledBridge } from "../src/context/bridge.ts";
|
|
2
3
|
import type { CatalogSource } from "../src/catalog/types.ts";
|
|
3
4
|
import type { EscalationConfig, RouterConfig } from "../src/config/types.ts";
|
|
4
5
|
import { EMPTY_USAGE, type Ledger, type LedgerEntry, type UsageCounts } from "../src/cost/types.ts";
|
|
@@ -43,7 +44,7 @@ function mkConfig(escalation: Partial<EscalationConfig> = {}): RouterConfig {
|
|
|
43
44
|
data: { axis: "intelligence", minQuality: 0 },
|
|
44
45
|
chat: { axis: "intelligence", minQuality: 0 },
|
|
45
46
|
},
|
|
46
|
-
filters: { allow: [], deny: [], includeFree: false, requireToolSupport: true, minTrust: 0.6, minTrustSamples: 5, trustScopedByHarness: false, contextHeadroom: 1.2 },
|
|
47
|
+
filters: { allow: [], deny: [], includeFree: false, requireToolSupport: true, minTrust: 0.6, minTrustSamples: 5, trustScopedByHarness: false, contextHeadroom: 1.2, latencyWeight: 0, latencyReferenceMs: 5000, latencyReferenceTokensPerSec: 30, latencyMinSamples: 20 },
|
|
47
48
|
classifier: {
|
|
48
49
|
ambiguityThreshold: 0,
|
|
49
50
|
model: "test/adjudicator",
|
|
@@ -68,6 +69,8 @@ function mkConfig(escalation: Partial<EscalationConfig> = {}): RouterConfig {
|
|
|
68
69
|
hysteresis: { holdTurns: 2, holdTurnsAfterEscalation: 4, switchMargin: 1.5, cacheWarmTtlMs: 600_000, maxDowngradePerTurn: 1 },
|
|
69
70
|
exploration: { enabled: false, rates: {}, stickyPolicy: "never", holdTurns: { enabled: false, values: [2, 3, 4] } },
|
|
70
71
|
cache: { injectBreakpoints: true, maxBreakpoints: 4, minPromptTokens: 1024 },
|
|
72
|
+
context: { enabled: false, baseUrl: "", token: "", defaultScope: "", timeoutMs: 3_000, maxStalenessMs: 900_000, maxBlockChars: 24_000, recordTurns: false, maxQueue: 64 },
|
|
73
|
+
compaction: { enabled: false, budgetTokens: 40_000, fitToWindow: true, protectRecentTurns: 4, maxToolResultBytes: 4_096, keepHeadBytes: 512, keepTailBytes: 512, elideSupersededReads: true, collapseDuplicateResults: true },
|
|
71
74
|
budget: { onExceeded: "downgrade" },
|
|
72
75
|
profiles: [],
|
|
73
76
|
ledger: { path: ":memory:", blendWindowDays: 7, blendMinSamples: 20, fallbackBlend: { inputPerMtok: 1, outputPerMtok: 4 }, conversationTtlMs: 86_400_000 },
|
|
@@ -83,6 +86,7 @@ function mkReq(): NormRequest {
|
|
|
83
86
|
conversationKey: "conv-test",
|
|
84
87
|
harnessId: "",
|
|
85
88
|
ompSessionId: "",
|
|
89
|
+
agentdoxScope: "",
|
|
86
90
|
requestedModel: "auto",
|
|
87
91
|
messages: [{ role: "user", text: "hi", images: 0, textBytes: 2, toolCalls: [] }],
|
|
88
92
|
tools: [],
|
|
@@ -107,7 +111,9 @@ const FEATURES: Features = {
|
|
|
107
111
|
distinctToolsUsed: 0,
|
|
108
112
|
lastToolFailed: false,
|
|
109
113
|
repeatedToolCall: false,
|
|
114
|
+
circularToolCall: false,
|
|
110
115
|
hasImages: false,
|
|
116
|
+
hasNewImage: false,
|
|
111
117
|
codeBlocks: 0,
|
|
112
118
|
codeBytes: 0,
|
|
113
119
|
looksLikeDiff: false,
|
|
@@ -137,6 +143,8 @@ function mkDecision(tier: Tier, slug: string, probe: Partial<ProbePlan> = {}): D
|
|
|
137
143
|
sessionId: "omp-conv-test",
|
|
138
144
|
sticky: false,
|
|
139
145
|
cacheBreakpointMessageIndices: [],
|
|
146
|
+
compactionPlan: [],
|
|
147
|
+
promptTokensSaved: 0,
|
|
140
148
|
reasoning: undefined,
|
|
141
149
|
maxTokens: undefined,
|
|
142
150
|
stripAssistantReasoning: false,
|
|
@@ -223,6 +231,7 @@ function mkLedger(): { ledger: Ledger; entries: LedgerEntry[] } {
|
|
|
223
231
|
conversationSpend: () => 0,
|
|
224
232
|
spendSince: () => 0,
|
|
225
233
|
blendedRate: () => null,
|
|
234
|
+
latency: () => null,
|
|
226
235
|
trust: () => null,
|
|
227
236
|
allTrust: () => [],
|
|
228
237
|
tokenRatio: () => null,
|
|
@@ -250,6 +259,8 @@ function mkConversations(): { store: ConversationStore; map: Map<string, Convers
|
|
|
250
259
|
lastPromptTokens: 0,
|
|
251
260
|
cacheWarmSlug: null,
|
|
252
261
|
cacheWarmAtMs: 0,
|
|
262
|
+
contextVersion: null,
|
|
263
|
+
contextFetchedAtMs: 0,
|
|
253
264
|
updatedAtMs: 0,
|
|
254
265
|
};
|
|
255
266
|
map.set(k, fresh);
|
|
@@ -316,7 +327,7 @@ describe("runTurn", () => {
|
|
|
316
327
|
const { store, map } = mkConversations();
|
|
317
328
|
const { sink, chunks, errors, finishes } = mkSink();
|
|
318
329
|
|
|
319
|
-
await runTurn(mkReq(), sink, { config: mkConfig(), router, upstream, ledger, conversations: store, catalog }, new AbortController().signal);
|
|
330
|
+
await runTurn(mkReq(), sink, { config: mkConfig(), router, upstream, ledger, conversations: store, catalog, context: createDisabledBridge() }, new AbortController().signal);
|
|
320
331
|
|
|
321
332
|
expect(errors).toHaveLength(0);
|
|
322
333
|
expect(finishes).toHaveLength(1);
|
|
@@ -365,7 +376,7 @@ describe("runTurn", () => {
|
|
|
365
376
|
const { store } = mkConversations();
|
|
366
377
|
const { sink, chunks, errors, finishes } = mkSink();
|
|
367
378
|
|
|
368
|
-
await runTurn(mkReq(), sink, { config: mkConfig(), router, upstream, ledger, conversations: store, catalog }, new AbortController().signal);
|
|
379
|
+
await runTurn(mkReq(), sink, { config: mkConfig(), router, upstream, ledger, conversations: store, catalog, context: createDisabledBridge() }, new AbortController().signal);
|
|
369
380
|
|
|
370
381
|
expect(errors).toHaveLength(0);
|
|
371
382
|
expect(entries).toHaveLength(2);
|
|
@@ -403,7 +414,7 @@ describe("runTurn", () => {
|
|
|
403
414
|
const { store } = mkConversations();
|
|
404
415
|
const { sink, chunks, errors, finishes } = mkSink();
|
|
405
416
|
|
|
406
|
-
await runTurn(mkReq(), sink, { config: mkConfig(), router, upstream, ledger, conversations: store, catalog }, new AbortController().signal);
|
|
417
|
+
await runTurn(mkReq(), sink, { config: mkConfig(), router, upstream, ledger, conversations: store, catalog, context: createDisabledBridge() }, new AbortController().signal);
|
|
407
418
|
|
|
408
419
|
// Bytes reached the client, so the 429 mid-stream is surfaced, not retried.
|
|
409
420
|
expect(calls).toHaveLength(1);
|
|
@@ -423,7 +434,7 @@ describe("runTurn", () => {
|
|
|
423
434
|
const { store } = mkConversations();
|
|
424
435
|
const { sink, errors, finishes } = mkSink();
|
|
425
436
|
|
|
426
|
-
await runTurn(mkReq(), sink, { config: mkConfig(), router, upstream, ledger, conversations: store, catalog }, new AbortController().signal);
|
|
437
|
+
await runTurn(mkReq(), sink, { config: mkConfig(), router, upstream, ledger, conversations: store, catalog, context: createDisabledBridge() }, new AbortController().signal);
|
|
427
438
|
|
|
428
439
|
expect(calls).toHaveLength(1); // no retry, no escalation on auth
|
|
429
440
|
expect(finishes).toHaveLength(0);
|
|
@@ -447,7 +458,7 @@ describe("runTurn", () => {
|
|
|
447
458
|
const { store } = mkConversations();
|
|
448
459
|
const { sink, errors, finishes } = mkSink();
|
|
449
460
|
|
|
450
|
-
await runTurn(mkReq(), sink, { config: mkConfig(), router, upstream, ledger, conversations: store, catalog }, new AbortController().signal);
|
|
461
|
+
await runTurn(mkReq(), sink, { config: mkConfig(), router, upstream, ledger, conversations: store, catalog, context: createDisabledBridge() }, new AbortController().signal);
|
|
451
462
|
|
|
452
463
|
expect(errors).toHaveLength(0);
|
|
453
464
|
expect(finishes).toHaveLength(1);
|
|
@@ -487,14 +498,14 @@ describe("runTurn", () => {
|
|
|
487
498
|
const { sink, errors } = mkSink();
|
|
488
499
|
|
|
489
500
|
// Turn 1: first turn, no prior tier → re-arms (tierChanged true).
|
|
490
|
-
await runTurn(mkReq(), sink, { config: mkConfig(), router, upstream, ledger, conversations: store, catalog }, new AbortController().signal);
|
|
501
|
+
await runTurn(mkReq(), sink, { config: mkConfig(), router, upstream, ledger, conversations: store, catalog, context: createDisabledBridge() }, new AbortController().signal);
|
|
491
502
|
const afterFirst = map.get("conv-test")!;
|
|
492
503
|
expect(afterFirst.currentTier).toBe("hard");
|
|
493
504
|
expect(afterFirst.stickyUntilTurn).toBe(1 + 2); // holdTurns=2
|
|
494
505
|
|
|
495
506
|
// Turn 2: same tier served again → must NOT re-arm. The window should
|
|
496
507
|
// stay at its previous expiry (turn 3), not extend to turn 4.
|
|
497
|
-
await runTurn(mkReq(), sink, { config: mkConfig(), router, upstream, ledger, conversations: store, catalog }, new AbortController().signal);
|
|
508
|
+
await runTurn(mkReq(), sink, { config: mkConfig(), router, upstream, ledger, conversations: store, catalog, context: createDisabledBridge() }, new AbortController().signal);
|
|
498
509
|
const afterSecond = map.get("conv-test")!;
|
|
499
510
|
expect(afterSecond.currentTier).toBe("hard");
|
|
500
511
|
expect(afterSecond.stickyUntilTurn).toBe(3); // unchanged, not 4
|
|
@@ -516,7 +527,7 @@ describe("exploration reaches the ledger", () => {
|
|
|
516
527
|
const { store } = mkConversations();
|
|
517
528
|
const { sink } = mkSink();
|
|
518
529
|
|
|
519
|
-
await runTurn(mkReq(), sink, { config: mkConfig(), router, upstream, ledger, conversations: store, catalog }, new AbortController().signal);
|
|
530
|
+
await runTurn(mkReq(), sink, { config: mkConfig(), router, upstream, ledger, conversations: store, catalog, context: createDisabledBridge() }, new AbortController().signal);
|
|
520
531
|
|
|
521
532
|
expect(entries).toHaveLength(1);
|
|
522
533
|
// The counterfactual pair: what the classifier wanted, and what actually ran.
|
|
@@ -536,7 +547,7 @@ describe("exploration reaches the ledger", () => {
|
|
|
536
547
|
const { store } = mkConversations();
|
|
537
548
|
const { sink } = mkSink();
|
|
538
549
|
|
|
539
|
-
await runTurn(mkReq(), sink, { config: mkConfig(), router, upstream, ledger, conversations: store, catalog }, new AbortController().signal);
|
|
550
|
+
await runTurn(mkReq(), sink, { config: mkConfig(), router, upstream, ledger, conversations: store, catalog, context: createDisabledBridge() }, new AbortController().signal);
|
|
540
551
|
|
|
541
552
|
expect(entries).toHaveLength(1);
|
|
542
553
|
expect(entries[0]?.exploredFrom).toBeNull();
|