auto-model-router 0.2.2 → 0.2.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude/skills/agentdox/SKILL.md +143 -0
- package/.mcp.json +11 -0
- package/.omp-plugin/marketplace.json +2 -2
- package/CLAUDE.md +129 -0
- package/README.md +64 -0
- package/docs/AGENTDOX-BRIDGE.md +132 -0
- package/docs/context-optimization.md +362 -0
- package/omp-extension/embed-logic.ts +31 -0
- package/omp-extension/router-embed.ts +7 -1
- package/package.json +1 -1
- package/src/cli/config-cmd.ts +20 -5
- package/src/cli/explain.ts +1 -0
- package/src/config/defaults.ts +33 -1
- package/src/config/load.ts +13 -0
- package/src/config/schema.ts +27 -0
- package/src/config/types.ts +79 -5
- package/src/context/agentdox.ts +113 -0
- package/src/context/bridge.ts +166 -0
- package/src/context/index.ts +33 -0
- package/src/context/store.ts +82 -0
- package/src/context/types.ts +78 -0
- package/src/cost/ledger.ts +53 -14
- package/src/cost/types.ts +15 -4
- package/src/router/candidates.ts +19 -8
- package/src/router/classify.ts +26 -12
- package/src/router/compaction.ts +163 -0
- package/src/router/features.ts +26 -13
- package/src/router/select.ts +37 -4
- package/src/router/state.ts +12 -2
- package/src/router/types.ts +33 -1
- package/src/server/http.ts +18 -1
- package/src/server/turn.ts +88 -1
- package/src/upstream/openrouter.ts +8 -1
- package/src/util/sqlite.ts +34 -1
- package/src/wire/openai/request.ts +86 -1
- package/src/wire/types.ts +36 -0
- package/test/classify.test.ts +63 -5
- package/test/compaction.test.ts +148 -0
- package/test/context-bridge.test.ts +337 -0
- package/test/embed-logic.test.ts +32 -0
- package/test/escalate.test.ts +1 -0
- package/test/exploration.test.ts +6 -2
- package/test/failover.test.ts +51 -6
- package/test/features.test.ts +45 -0
- package/test/helpers/inject.ts +23 -0
- package/test/hold-exploration.test.ts +4 -2
- package/test/select.test.ts +86 -3
- package/test/tokens.test.ts +1 -0
- package/test/trust-attribution.test.ts +49 -9
- package/test/turn.test.ts +20 -10
- package/tools/agentdox-e2e.ts +123 -0
package/test/failover.test.ts
CHANGED
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import { describe, expect, test } from "bun:test";
|
|
2
|
+
import { createDisabledBridge } from "../src/context/bridge.ts";
|
|
2
3
|
import type { CatalogSource } from "../src/catalog/types.ts";
|
|
3
4
|
import type { EscalationConfig, RouterConfig } from "../src/config/types.ts";
|
|
4
5
|
import { EMPTY_USAGE, type Ledger, type LedgerEntry, type UsageCounts } from "../src/cost/types.ts";
|
|
@@ -43,7 +44,7 @@ function mkConfig(escalation: Partial<EscalationConfig> = {}): RouterConfig {
|
|
|
43
44
|
data: { axis: "intelligence", minQuality: 0 },
|
|
44
45
|
chat: { axis: "intelligence", minQuality: 0 },
|
|
45
46
|
},
|
|
46
|
-
filters: { allow: [], deny: [], includeFree: false, requireToolSupport: true, minTrust: 0.6, minTrustSamples: 5, trustScopedByHarness: false, contextHeadroom: 1.2, latencyWeight: 0, latencyReferenceMs: 5000, latencyMinSamples: 20 },
|
|
47
|
+
filters: { allow: [], deny: [], includeFree: false, requireToolSupport: true, minTrust: 0.6, minTrustSamples: 5, trustScopedByHarness: false, contextHeadroom: 1.2, latencyWeight: 0, latencyReferenceMs: 5000, latencyReferenceTokensPerSec: 30, latencyMinSamples: 20 },
|
|
47
48
|
classifier: {
|
|
48
49
|
ambiguityThreshold: 0,
|
|
49
50
|
model: "test/adjudicator",
|
|
@@ -68,6 +69,8 @@ function mkConfig(escalation: Partial<EscalationConfig> = {}): RouterConfig {
|
|
|
68
69
|
hysteresis: { holdTurns: 2, holdTurnsAfterEscalation: 4, switchMargin: 1.5, cacheWarmTtlMs: 600_000, maxDowngradePerTurn: 1 },
|
|
69
70
|
exploration: { enabled: false, rates: {}, stickyPolicy: "never", holdTurns: { enabled: false, values: [2, 3, 4] } },
|
|
70
71
|
cache: { injectBreakpoints: true, maxBreakpoints: 4, minPromptTokens: 1024 },
|
|
72
|
+
context: { enabled: false, baseUrl: "", token: "", defaultScope: "", timeoutMs: 3_000, maxStalenessMs: 900_000, maxBlockChars: 24_000, recordTurns: false, maxQueue: 64 },
|
|
73
|
+
compaction: { enabled: false, budgetTokens: 40_000, fitToWindow: true, protectRecentTurns: 4, maxToolResultBytes: 4_096, keepHeadBytes: 512, keepTailBytes: 512, elideSupersededReads: true, collapseDuplicateResults: true },
|
|
71
74
|
budget: { onExceeded: "downgrade" },
|
|
72
75
|
profiles: [],
|
|
73
76
|
ledger: { path: ":memory:", blendWindowDays: 7, blendMinSamples: 20, fallbackBlend: { inputPerMtok: 1, outputPerMtok: 4 }, conversationTtlMs: 86_400_000 },
|
|
@@ -83,6 +86,7 @@ function mkReq(): NormRequest {
|
|
|
83
86
|
conversationKey: "conv-test",
|
|
84
87
|
harnessId: "",
|
|
85
88
|
ompSessionId: "",
|
|
89
|
+
agentdoxScope: "",
|
|
86
90
|
requestedModel: "auto",
|
|
87
91
|
messages: [{ role: "user", text: "hi", images: 0, textBytes: 2, toolCalls: [] }],
|
|
88
92
|
tools: [],
|
|
@@ -107,7 +111,9 @@ const FEATURES: Features = {
|
|
|
107
111
|
distinctToolsUsed: 0,
|
|
108
112
|
lastToolFailed: false,
|
|
109
113
|
repeatedToolCall: false,
|
|
114
|
+
circularToolCall: false,
|
|
110
115
|
hasImages: false,
|
|
116
|
+
hasNewImage: false,
|
|
111
117
|
codeBlocks: 0,
|
|
112
118
|
codeBytes: 0,
|
|
113
119
|
looksLikeDiff: false,
|
|
@@ -137,6 +143,8 @@ function mkDecision(tier: Tier, slug: string, probe: Partial<ProbePlan> = {}): D
|
|
|
137
143
|
sessionId: "omp-conv-test",
|
|
138
144
|
sticky: false,
|
|
139
145
|
cacheBreakpointMessageIndices: [],
|
|
146
|
+
compactionPlan: [],
|
|
147
|
+
promptTokensSaved: 0,
|
|
140
148
|
reasoning: undefined,
|
|
141
149
|
maxTokens: undefined,
|
|
142
150
|
stripAssistantReasoning: false,
|
|
@@ -257,6 +265,8 @@ function mkConversations(): { store: ConversationStore; map: Map<string, Convers
|
|
|
257
265
|
lastPromptTokens: 0,
|
|
258
266
|
cacheWarmSlug: null,
|
|
259
267
|
cacheWarmAtMs: 0,
|
|
268
|
+
contextVersion: null,
|
|
269
|
+
contextFetchedAtMs: 0,
|
|
260
270
|
updatedAtMs: 0,
|
|
261
271
|
};
|
|
262
272
|
map.set(k, fresh);
|
|
@@ -320,7 +330,7 @@ describe("same-tier failover", () => {
|
|
|
320
330
|
const { store } = mkConversations();
|
|
321
331
|
const { sink, errors, finishes } = mkSink();
|
|
322
332
|
|
|
323
|
-
await runTurn(mkReq(), sink, { config: mkConfig(), router, upstream, ledger, conversations: store, catalog }, new AbortController().signal);
|
|
333
|
+
await runTurn(mkReq(), sink, { config: mkConfig(), router, upstream, ledger, conversations: store, catalog, context: createDisabledBridge() }, new AbortController().signal);
|
|
324
334
|
|
|
325
335
|
expect(errors).toHaveLength(0);
|
|
326
336
|
expect(finishes).toHaveLength(1);
|
|
@@ -350,6 +360,41 @@ describe("same-tier failover", () => {
|
|
|
350
360
|
expect(finishes[0]!.attempts).toBe(2);
|
|
351
361
|
});
|
|
352
362
|
|
|
363
|
+
test("a 403 moderation block fails over to a different model in the same tier", async () => {
|
|
364
|
+
const { router, calls } = mkRouter([mkDecision("trivial", "a/model"), mkDecision("trivial", "b/model")]);
|
|
365
|
+
const { upstream, calls: dispatches } = mkUpstream([
|
|
366
|
+
{ kind: "fail", error: new UpstreamError("moderation", 403, "Request blocked: prompt injection", true) },
|
|
367
|
+
{ kind: "chunks", chunks: okChunks("b/model") },
|
|
368
|
+
]);
|
|
369
|
+
const { ledger, entries } = mkLedger();
|
|
370
|
+
const { store } = mkConversations();
|
|
371
|
+
const { sink, errors, finishes } = mkSink();
|
|
372
|
+
|
|
373
|
+
await runTurn(mkReq(), sink, { config: mkConfig(), router, upstream, ledger, conversations: store, catalog, context: createDisabledBridge() }, new AbortController().signal);
|
|
374
|
+
|
|
375
|
+
expect(errors).toHaveLength(0);
|
|
376
|
+
expect(finishes).toHaveLength(1);
|
|
377
|
+
|
|
378
|
+
// A per-model policy block indicts the slug, not the tier: the retry
|
|
379
|
+
// re-routes with the blocked slug excluded and serves a sibling.
|
|
380
|
+
expect(calls).toHaveLength(2);
|
|
381
|
+
expect(calls[1]).toEqual({ attempt: 1, excludeSlugs: ["a/model"] });
|
|
382
|
+
expect(dispatches.map((d) => d.body.model)).toEqual(["a/model", "b/model"]);
|
|
383
|
+
|
|
384
|
+
expect(entries).toHaveLength(2);
|
|
385
|
+
expect(entries[0]!.slug).toBe("a/model");
|
|
386
|
+
expect(entries[0]!.wasted).toBe(true);
|
|
387
|
+
expect(entries[0]!.escalationSignal).toBeNull(); // failover, not escalation
|
|
388
|
+
expect(entries[0]!.error).toContain("moderation");
|
|
389
|
+
expect(entries[1]!.slug).toBe("b/model");
|
|
390
|
+
expect(entries[1]!.tier).toBe("trivial");
|
|
391
|
+
expect(entries[1]!.wasted).toBe(false);
|
|
392
|
+
expect(entries[1]!.reasons).toContain("failover: a/model returned moderation; retrying b/model in trivial");
|
|
393
|
+
|
|
394
|
+
expect(finishes[0]!.servedSlug).toBe("b/model");
|
|
395
|
+
expect(finishes[0]!.escalated).toBe(false);
|
|
396
|
+
});
|
|
397
|
+
|
|
353
398
|
test("a tier with no other eligible model falls back to tier escalation", async () => {
|
|
354
399
|
// The router widens to "simple" when "trivial" excludes a/model: the
|
|
355
400
|
// failover probe's decision is discarded and the normal escalation path
|
|
@@ -367,7 +412,7 @@ describe("same-tier failover", () => {
|
|
|
367
412
|
const { store } = mkConversations();
|
|
368
413
|
const { sink, errors, finishes } = mkSink();
|
|
369
414
|
|
|
370
|
-
await runTurn(mkReq(), sink, { config: mkConfig(), router, upstream, ledger, conversations: store, catalog }, new AbortController().signal);
|
|
415
|
+
await runTurn(mkReq(), sink, { config: mkConfig(), router, upstream, ledger, conversations: store, catalog, context: createDisabledBridge() }, new AbortController().signal);
|
|
371
416
|
|
|
372
417
|
expect(errors).toHaveLength(0);
|
|
373
418
|
expect(finishes).toHaveLength(1);
|
|
@@ -401,7 +446,7 @@ describe("same-tier failover", () => {
|
|
|
401
446
|
const { store } = mkConversations();
|
|
402
447
|
const { sink, errors, finishes } = mkSink();
|
|
403
448
|
|
|
404
|
-
await runTurn(mkReq(), sink, { config: mkConfig(), router, upstream, ledger, conversations: store, catalog }, new AbortController().signal);
|
|
449
|
+
await runTurn(mkReq(), sink, { config: mkConfig(), router, upstream, ledger, conversations: store, catalog, context: createDisabledBridge() }, new AbortController().signal);
|
|
405
450
|
|
|
406
451
|
expect(calls).toHaveLength(1);
|
|
407
452
|
expect(calls[0]).toEqual({ attempt: 0 }); // never re-routed, never given excludeSlugs
|
|
@@ -434,7 +479,7 @@ describe("same-tier failover", () => {
|
|
|
434
479
|
await runTurn(
|
|
435
480
|
mkReq(),
|
|
436
481
|
sink,
|
|
437
|
-
{ config: mkConfig({ maxAttempts: 5 }), router, upstream, ledger, conversations: store, catalog },
|
|
482
|
+
{ config: mkConfig({ maxAttempts: 5 }), router, upstream, ledger, conversations: store, catalog, context: createDisabledBridge() },
|
|
438
483
|
new AbortController().signal,
|
|
439
484
|
);
|
|
440
485
|
|
|
@@ -480,7 +525,7 @@ describe("same-tier failover", () => {
|
|
|
480
525
|
const { store } = mkConversations();
|
|
481
526
|
const { sink, chunks, errors, finishes } = mkSink();
|
|
482
527
|
|
|
483
|
-
await runTurn(mkReq(), sink, { config: mkConfig(), router, upstream, ledger, conversations: store, catalog }, new AbortController().signal);
|
|
528
|
+
await runTurn(mkReq(), sink, { config: mkConfig(), router, upstream, ledger, conversations: store, catalog, context: createDisabledBridge() }, new AbortController().signal);
|
|
484
529
|
|
|
485
530
|
expect(errors).toHaveLength(0);
|
|
486
531
|
expect(calls).toHaveLength(2);
|
package/test/features.test.ts
CHANGED
|
@@ -133,6 +133,7 @@ describe("failure and loop signals", () => {
|
|
|
133
133
|
100,
|
|
134
134
|
);
|
|
135
135
|
expect(f.repeatedToolCall).toBe(true);
|
|
136
|
+
expect(f.circularToolCall).toBe(true);
|
|
136
137
|
});
|
|
137
138
|
|
|
138
139
|
test("different arguments to the same tool are not a loop", () => {
|
|
@@ -148,6 +149,26 @@ describe("failure and loop signals", () => {
|
|
|
148
149
|
100,
|
|
149
150
|
);
|
|
150
151
|
expect(f.repeatedToolCall).toBe(false);
|
|
152
|
+
expect(f.circularToolCall).toBe(false);
|
|
153
|
+
});
|
|
154
|
+
|
|
155
|
+
test("a non-adjacent re-issued call is circular but not an adjacent repeat", () => {
|
|
156
|
+
const f = extractFeatures(
|
|
157
|
+
req([
|
|
158
|
+
SYSTEM,
|
|
159
|
+
{ role: "user", content: "go" },
|
|
160
|
+
toolCall("c1", "read", '{"path":"same.ts"}'),
|
|
161
|
+
{ role: "tool", tool_call_id: "c1", content: "x" },
|
|
162
|
+
toolCall("c2", "bash", '{"command":"ls"}'),
|
|
163
|
+
{ role: "tool", tool_call_id: "c2", content: "a.txt" },
|
|
164
|
+
toolCall("c3", "read", '{"path":"same.ts"}'),
|
|
165
|
+
{ role: "tool", tool_call_id: "c3", content: "x" },
|
|
166
|
+
]),
|
|
167
|
+
100,
|
|
168
|
+
);
|
|
169
|
+
// c3 repeats c1 verbatim with c2 in between: not adjacent, but circular.
|
|
170
|
+
expect(f.repeatedToolCall).toBe(false);
|
|
171
|
+
expect(f.circularToolCall).toBe(true);
|
|
151
172
|
});
|
|
152
173
|
});
|
|
153
174
|
|
|
@@ -223,6 +244,30 @@ describe("newest-content scoping", () => {
|
|
|
223
244
|
100,
|
|
224
245
|
);
|
|
225
246
|
expect(f.hasImages).toBe(true);
|
|
247
|
+
expect(f.hasNewImage).toBe(true);
|
|
226
248
|
expect(f.requestedReasoning).toBe("high");
|
|
227
249
|
});
|
|
250
|
+
|
|
251
|
+
test("a stale image in history is not new visual work on a tool continuation", () => {
|
|
252
|
+
const f = extractFeatures(
|
|
253
|
+
req([
|
|
254
|
+
SYSTEM,
|
|
255
|
+
{
|
|
256
|
+
role: "user",
|
|
257
|
+
content: [
|
|
258
|
+
{ type: "text", text: "implement this screen" },
|
|
259
|
+
{ type: "image_url", image_url: { url: "data:image/png;base64,AA" } },
|
|
260
|
+
],
|
|
261
|
+
},
|
|
262
|
+
toolCall("c1", "read", '{"path":"src/app.tsx"}'),
|
|
263
|
+
{ role: "tool", tool_call_id: "c1", name: "read", content: "export const App = () => null;" },
|
|
264
|
+
]),
|
|
265
|
+
100,
|
|
266
|
+
);
|
|
267
|
+
// The image is still in context (capability), but the current turn is a
|
|
268
|
+
// mechanical continuation, not fresh visual work (task axis).
|
|
269
|
+
expect(f.hasImages).toBe(true);
|
|
270
|
+
expect(f.hasNewImage).toBe(false);
|
|
271
|
+
expect(f.isToolResultContinuation).toBe(true);
|
|
272
|
+
});
|
|
228
273
|
});
|
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Exercises the wire's context injection through the public render path, so
|
|
3
|
+
* the test covers the real ordering (inject, then apply cache breakpoints).
|
|
4
|
+
*/
|
|
5
|
+
import { parseChatRequest } from "../../src/wire/openai/request.ts";
|
|
6
|
+
|
|
7
|
+
export function injectForTest(
|
|
8
|
+
body: Record<string, unknown>,
|
|
9
|
+
block: string,
|
|
10
|
+
breakpoints: number[],
|
|
11
|
+
): Record<string, unknown> {
|
|
12
|
+
const req = parseChatRequest(body, new Headers());
|
|
13
|
+
return req.renderUpstreamBody({
|
|
14
|
+
slug: "vendor/model",
|
|
15
|
+
fallbacks: [],
|
|
16
|
+
sessionId: "s",
|
|
17
|
+
cacheBreakpointMessageIndices: breakpoints,
|
|
18
|
+
reasoning: undefined,
|
|
19
|
+
maxTokens: undefined,
|
|
20
|
+
stripAssistantReasoning: false,
|
|
21
|
+
contextBlock: block,
|
|
22
|
+
});
|
|
23
|
+
}
|
|
@@ -1,10 +1,12 @@
|
|
|
1
1
|
import { describe, expect, test } from "bun:test";
|
|
2
2
|
|
|
3
|
-
import {
|
|
3
|
+
import { DEFAULT_CONFIG } from "../src/config/defaults.ts";
|
|
4
4
|
import type { RouterConfig } from "../src/config/types.ts";
|
|
5
5
|
import { explorationDraw, resolveHoldTurns } from "../src/router/explore.ts";
|
|
6
6
|
|
|
7
|
-
|
|
7
|
+
// Shipped defaults, not loadConfig({}) — the latter merges the live home
|
|
8
|
+
// config.yml and makes this suite depend on the developer's local settings.
|
|
9
|
+
const BASE = DEFAULT_CONFIG;
|
|
8
10
|
|
|
9
11
|
function withHold(over: { enabled?: boolean; values?: number[] }, enabled = true): RouterConfig {
|
|
10
12
|
return {
|
package/test/select.test.ts
CHANGED
|
@@ -64,6 +64,8 @@ function state(over: Partial<ConversationState> = {}): ConversationState {
|
|
|
64
64
|
lastPromptTokens: 0,
|
|
65
65
|
cacheWarmSlug: null,
|
|
66
66
|
cacheWarmAtMs: 0,
|
|
67
|
+
contextVersion: null,
|
|
68
|
+
contextFetchedAtMs: 0,
|
|
67
69
|
updatedAtMs: Date.now(),
|
|
68
70
|
...over,
|
|
69
71
|
};
|
|
@@ -541,7 +543,7 @@ describe("task-type routing", () => {
|
|
|
541
543
|
});
|
|
542
544
|
|
|
543
545
|
describe("latency scoring", () => {
|
|
544
|
-
function ledgerWithLatency(
|
|
546
|
+
function ledgerWithLatency(bySlug: Record<string, { ttftMs: number; samples: number; tokensPerSec?: number }>): Ledger {
|
|
545
547
|
return {
|
|
546
548
|
record: () => {},
|
|
547
549
|
conversationSpend: () => 0,
|
|
@@ -550,8 +552,10 @@ describe("latency scoring", () => {
|
|
|
550
552
|
trust: () => null,
|
|
551
553
|
allTrust: () => [],
|
|
552
554
|
latency: (slug) => {
|
|
553
|
-
const v =
|
|
554
|
-
|
|
555
|
+
const v = bySlug[slug];
|
|
556
|
+
// Default throughput is fast, so these cases isolate the TTFT axis
|
|
557
|
+
// unless a test sets tokensPerSec explicitly.
|
|
558
|
+
return v === undefined ? null : { slug, samples: v.samples, ttftMs: v.ttftMs, tokensPerSec: v.tokensPerSec ?? 1000 };
|
|
555
559
|
},
|
|
556
560
|
tokenRatio: () => null,
|
|
557
561
|
recentEntries: () => [],
|
|
@@ -581,4 +585,83 @@ describe("latency scoring", () => {
|
|
|
581
585
|
const ledger = ledgerWithLatency({ [slow]: { ttftMs: 60_000, samples: 5 } });
|
|
582
586
|
expect(run({ tier: "simple", cfg: withWeight(2), ledger }).slug).toBe(slow);
|
|
583
587
|
});
|
|
588
|
+
|
|
589
|
+
test("penalises a model that starts fast but streams slowly", () => {
|
|
590
|
+
// The case TTFT-only scoring misses: quick first token, slow body.
|
|
591
|
+
const slow = run({ tier: "simple" }).slug;
|
|
592
|
+
const ledger = ledgerWithLatency({ [slow]: { ttftMs: 1500, samples: 50, tokensPerSec: 12 } });
|
|
593
|
+
const d = run({ tier: "simple", cfg: withWeight(2), ledger });
|
|
594
|
+
expect(d.slug).not.toBe(slow);
|
|
595
|
+
});
|
|
596
|
+
});
|
|
597
|
+
|
|
598
|
+
describe("context compaction", () => {
|
|
599
|
+
const COMPACT_CFG: RouterConfig = {
|
|
600
|
+
...BASE,
|
|
601
|
+
compaction: {
|
|
602
|
+
enabled: true,
|
|
603
|
+
budgetTokens: 1_000,
|
|
604
|
+
fitToWindow: false,
|
|
605
|
+
protectRecentTurns: 1,
|
|
606
|
+
maxToolResultBytes: 100,
|
|
607
|
+
keepHeadBytes: 20,
|
|
608
|
+
keepTailBytes: 20,
|
|
609
|
+
elideSupersededReads: true,
|
|
610
|
+
collapseDuplicateResults: true,
|
|
611
|
+
},
|
|
612
|
+
};
|
|
613
|
+
|
|
614
|
+
function loopReq(): NormRequest {
|
|
615
|
+
return parseChatRequest(
|
|
616
|
+
{
|
|
617
|
+
model: "auto",
|
|
618
|
+
tools: TOOLS,
|
|
619
|
+
messages: [
|
|
620
|
+
{ role: "system", content: "You are a coding agent." },
|
|
621
|
+
{ role: "user", content: "read the file" },
|
|
622
|
+
{ role: "assistant", content: null, tool_calls: [{ id: "c1", type: "function", function: { name: "read", arguments: '{"path":"big.ts"}' } }] },
|
|
623
|
+
{ role: "tool", tool_call_id: "c1", content: "x".repeat(4000) },
|
|
624
|
+
{ role: "user", content: "continue" },
|
|
625
|
+
],
|
|
626
|
+
},
|
|
627
|
+
new Headers(),
|
|
628
|
+
);
|
|
629
|
+
}
|
|
630
|
+
|
|
631
|
+
test("an over-budget turn produces a compaction plan and records savings", () => {
|
|
632
|
+
const req = loopReq();
|
|
633
|
+
const features = extractFeatures(req, 5_000); // over budgetTokens=1000
|
|
634
|
+
const d = select({
|
|
635
|
+
req,
|
|
636
|
+
features,
|
|
637
|
+
classification: scoreHeuristic(features, COMPACT_CFG),
|
|
638
|
+
profile: PROFILE,
|
|
639
|
+
state: state(),
|
|
640
|
+
snapshot: SNAPSHOT,
|
|
641
|
+
ledger: null,
|
|
642
|
+
cfg: COMPACT_CFG,
|
|
643
|
+
nowMs: Date.now(),
|
|
644
|
+
});
|
|
645
|
+
expect(d.compactionPlan.length).toBeGreaterThan(0);
|
|
646
|
+
expect(d.promptTokensSaved).toBeGreaterThan(0);
|
|
647
|
+
expect(d.reasons.some((r) => r.startsWith("compaction:"))).toBe(true);
|
|
648
|
+
});
|
|
649
|
+
|
|
650
|
+
test("a small turn is left untouched", () => {
|
|
651
|
+
const req = loopReq();
|
|
652
|
+
const features = extractFeatures(req, 500); // under budgetTokens=1000
|
|
653
|
+
const d = select({
|
|
654
|
+
req,
|
|
655
|
+
features,
|
|
656
|
+
classification: scoreHeuristic(features, COMPACT_CFG),
|
|
657
|
+
profile: PROFILE,
|
|
658
|
+
state: state(),
|
|
659
|
+
snapshot: SNAPSHOT,
|
|
660
|
+
ledger: null,
|
|
661
|
+
cfg: COMPACT_CFG,
|
|
662
|
+
nowMs: Date.now(),
|
|
663
|
+
});
|
|
664
|
+
expect(d.compactionPlan).toEqual([]);
|
|
665
|
+
expect(d.promptTokensSaved).toBe(0);
|
|
666
|
+
});
|
|
584
667
|
});
|
package/test/tokens.test.ts
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import { describe, expect, test } from "bun:test";
|
|
2
2
|
|
|
3
3
|
import { loadConfig } from "../src/config/load.ts";
|
|
4
|
-
import { createLedger } from "../src/cost/ledger.ts";
|
|
4
|
+
import { createLedger, LATENCY_WINDOW_ROWS } from "../src/cost/ledger.ts";
|
|
5
5
|
import { EMPTY_USAGE, type LedgerEntry } from "../src/cost/types.ts";
|
|
6
6
|
import { openDb } from "../src/util/sqlite.ts";
|
|
7
7
|
|
|
@@ -40,6 +40,7 @@ function entry(over: Partial<LedgerEntry>): LedgerEntry {
|
|
|
40
40
|
wasted: false,
|
|
41
41
|
upstreamGenerationId: null,
|
|
42
42
|
error: null,
|
|
43
|
+
promptTokensSaved: 0,
|
|
43
44
|
...over,
|
|
44
45
|
};
|
|
45
46
|
}
|
|
@@ -75,8 +76,19 @@ describe("trust attribution", () => {
|
|
|
75
76
|
trustAfter([
|
|
76
77
|
null,
|
|
77
78
|
null,
|
|
78
|
-
"auth:
|
|
79
|
-
"auth:
|
|
79
|
+
"auth: No auth credentials found",
|
|
80
|
+
"auth: Insufficient credits",
|
|
81
|
+
]),
|
|
82
|
+
).toBe(CLEAN);
|
|
83
|
+
});
|
|
84
|
+
|
|
85
|
+
test("provider moderation/policy blocks do not count against the model", () => {
|
|
86
|
+
expect(
|
|
87
|
+
trustAfter([
|
|
88
|
+
null,
|
|
89
|
+
null,
|
|
90
|
+
"moderation: Request blocked: prompt injection patterns detected",
|
|
91
|
+
"moderation: This model requires 18+ age confirmation",
|
|
80
92
|
]),
|
|
81
93
|
).toBe(CLEAN);
|
|
82
94
|
});
|
|
@@ -146,20 +158,20 @@ describe("trust attribution", () => {
|
|
|
146
158
|
});
|
|
147
159
|
|
|
148
160
|
describe("latency signal", () => {
|
|
149
|
-
function latencyOf(rows: Array<Partial<LedgerEntry>>): { samples: number; ttftMs: number } | null {
|
|
161
|
+
function latencyOf(rows: Array<Partial<LedgerEntry>>): { samples: number; ttftMs: number; tokensPerSec: number } | null {
|
|
150
162
|
const db = openDb(":memory:");
|
|
151
163
|
try {
|
|
152
164
|
const ledger = createLedger(db, cfg);
|
|
153
165
|
for (const r of rows) ledger.record(entry(r));
|
|
154
166
|
const l = ledger.latency("vendor/model");
|
|
155
|
-
return l === null ? null : { samples: l.samples, ttftMs: l.ttftMs };
|
|
167
|
+
return l === null ? null : { samples: l.samples, ttftMs: l.ttftMs, tokensPerSec: l.tokensPerSec };
|
|
156
168
|
} finally {
|
|
157
169
|
db.close();
|
|
158
170
|
}
|
|
159
171
|
}
|
|
160
172
|
|
|
161
173
|
test("averages TTFT over streamed, non-errored turns", () => {
|
|
162
|
-
expect(latencyOf([{ ttftMs: 50 }, { ttftMs: 100 }, { ttftMs: 150 }])).toEqual({ samples: 3, ttftMs: 100 });
|
|
174
|
+
expect(latencyOf([{ ttftMs: 50 }, { ttftMs: 100 }, { ttftMs: 150 }])).toEqual({ samples: 3, ttftMs: 100, tokensPerSec: 0 });
|
|
163
175
|
});
|
|
164
176
|
|
|
165
177
|
test("excludes errored, aborted, and non-streamed (null TTFT) rows", () => {
|
|
@@ -170,12 +182,38 @@ describe("latency signal", () => {
|
|
|
170
182
|
{ ttftMs: 9999, error: "request aborted" },
|
|
171
183
|
{ ttftMs: null },
|
|
172
184
|
]),
|
|
173
|
-
).toEqual({ samples: 1, ttftMs: 100 });
|
|
185
|
+
).toEqual({ samples: 1, ttftMs: 100, tokensPerSec: 0 });
|
|
174
186
|
});
|
|
175
187
|
|
|
176
188
|
test("null when no streamed sample exists", () => {
|
|
177
189
|
expect(latencyOf([{ ttftMs: null }, { ttftMs: 0 }])).toBeNull();
|
|
178
190
|
});
|
|
191
|
+
|
|
192
|
+
test("throughput is aggregate completion tokens per post-TTFT second", () => {
|
|
193
|
+
const l = latencyOf([
|
|
194
|
+
{ ttftMs: 1000, latencyMs: 3000, usage: { ...EMPTY_USAGE, completionTokens: 200 } },
|
|
195
|
+
{ ttftMs: 1000, latencyMs: 3000, usage: { ...EMPTY_USAGE, completionTokens: 200 } },
|
|
196
|
+
]);
|
|
197
|
+
// 400 completion tokens over 4000ms of post-TTFT time = 100 tok/s.
|
|
198
|
+
expect(l?.tokensPerSec).toBeCloseTo(100, 5);
|
|
199
|
+
expect(l?.samples).toBe(2);
|
|
200
|
+
});
|
|
201
|
+
|
|
202
|
+
test("throughput and ttft track a recent window, not the lifetime average", () => {
|
|
203
|
+
// Old rows are fast; the recent window is slow. Latency must reflect the
|
|
204
|
+
// recent (slow) behaviour so a degraded model is penalised, not masked by
|
|
205
|
+
// its history. Lifetime blend here would be ~57 tok/s; the window is 10.
|
|
206
|
+
const rows: Array<Partial<LedgerEntry>> = [];
|
|
207
|
+
let t = 1;
|
|
208
|
+
for (let i = 0; i < 50; i++)
|
|
209
|
+
rows.push({ createdAtMs: t++, ttftMs: 200, latencyMs: 1200, usage: { ...EMPTY_USAGE, completionTokens: 1000 } });
|
|
210
|
+
for (let i = 0; i < LATENCY_WINDOW_ROWS; i++)
|
|
211
|
+
rows.push({ createdAtMs: t++, ttftMs: 4000, latencyMs: 14000, usage: { ...EMPTY_USAGE, completionTokens: 100 } });
|
|
212
|
+
const l = latencyOf(rows);
|
|
213
|
+
expect(l?.samples).toBe(LATENCY_WINDOW_ROWS);
|
|
214
|
+
expect(l?.tokensPerSec).toBeCloseTo(10, 0);
|
|
215
|
+
expect(l?.ttftMs).toBeCloseTo(4000, 5);
|
|
216
|
+
});
|
|
179
217
|
});
|
|
180
218
|
|
|
181
219
|
describe("v4 migration", () => {
|
|
@@ -185,6 +223,7 @@ describe("v4 migration", () => {
|
|
|
185
223
|
const ledger = createLedger(db, cfg);
|
|
186
224
|
ledger.record(entry({ error: "request aborted" }));
|
|
187
225
|
ledger.record(entry({ error: "auth: nope" }));
|
|
226
|
+
ledger.record(entry({ error: "moderation: Request blocked: prompt injection" }));
|
|
188
227
|
ledger.record(entry({ error: "model_unavailable: guardrail" }));
|
|
189
228
|
ledger.record(entry({ error: "upstream_error: boom" }));
|
|
190
229
|
ledger.record(entry({ error: null }));
|
|
@@ -195,6 +234,7 @@ describe("v4 migration", () => {
|
|
|
195
234
|
expect(rows.map((r) => r.error_kind)).toEqual([
|
|
196
235
|
"aborted",
|
|
197
236
|
"auth",
|
|
237
|
+
"moderation",
|
|
198
238
|
"model_unavailable",
|
|
199
239
|
"upstream_error",
|
|
200
240
|
null,
|
|
@@ -204,11 +244,11 @@ describe("v4 migration", () => {
|
|
|
204
244
|
}
|
|
205
245
|
});
|
|
206
246
|
|
|
207
|
-
test("schema is at user_version
|
|
247
|
+
test("schema is at user_version 12", () => {
|
|
208
248
|
const db = openDb(":memory:");
|
|
209
249
|
try {
|
|
210
250
|
const row = db.query("PRAGMA user_version").get() as { user_version: number };
|
|
211
|
-
expect(row.user_version).toBe(
|
|
251
|
+
expect(row.user_version).toBe(12);
|
|
212
252
|
} finally {
|
|
213
253
|
db.close();
|
|
214
254
|
}
|
package/test/turn.test.ts
CHANGED
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import { describe, expect, test } from "bun:test";
|
|
2
|
+
import { createDisabledBridge } from "../src/context/bridge.ts";
|
|
2
3
|
import type { CatalogSource } from "../src/catalog/types.ts";
|
|
3
4
|
import type { EscalationConfig, RouterConfig } from "../src/config/types.ts";
|
|
4
5
|
import { EMPTY_USAGE, type Ledger, type LedgerEntry, type UsageCounts } from "../src/cost/types.ts";
|
|
@@ -43,7 +44,7 @@ function mkConfig(escalation: Partial<EscalationConfig> = {}): RouterConfig {
|
|
|
43
44
|
data: { axis: "intelligence", minQuality: 0 },
|
|
44
45
|
chat: { axis: "intelligence", minQuality: 0 },
|
|
45
46
|
},
|
|
46
|
-
filters: { allow: [], deny: [], includeFree: false, requireToolSupport: true, minTrust: 0.6, minTrustSamples: 5, trustScopedByHarness: false, contextHeadroom: 1.2, latencyWeight: 0, latencyReferenceMs: 5000, latencyMinSamples: 20 },
|
|
47
|
+
filters: { allow: [], deny: [], includeFree: false, requireToolSupport: true, minTrust: 0.6, minTrustSamples: 5, trustScopedByHarness: false, contextHeadroom: 1.2, latencyWeight: 0, latencyReferenceMs: 5000, latencyReferenceTokensPerSec: 30, latencyMinSamples: 20 },
|
|
47
48
|
classifier: {
|
|
48
49
|
ambiguityThreshold: 0,
|
|
49
50
|
model: "test/adjudicator",
|
|
@@ -68,6 +69,8 @@ function mkConfig(escalation: Partial<EscalationConfig> = {}): RouterConfig {
|
|
|
68
69
|
hysteresis: { holdTurns: 2, holdTurnsAfterEscalation: 4, switchMargin: 1.5, cacheWarmTtlMs: 600_000, maxDowngradePerTurn: 1 },
|
|
69
70
|
exploration: { enabled: false, rates: {}, stickyPolicy: "never", holdTurns: { enabled: false, values: [2, 3, 4] } },
|
|
70
71
|
cache: { injectBreakpoints: true, maxBreakpoints: 4, minPromptTokens: 1024 },
|
|
72
|
+
context: { enabled: false, baseUrl: "", token: "", defaultScope: "", timeoutMs: 3_000, maxStalenessMs: 900_000, maxBlockChars: 24_000, recordTurns: false, maxQueue: 64 },
|
|
73
|
+
compaction: { enabled: false, budgetTokens: 40_000, fitToWindow: true, protectRecentTurns: 4, maxToolResultBytes: 4_096, keepHeadBytes: 512, keepTailBytes: 512, elideSupersededReads: true, collapseDuplicateResults: true },
|
|
71
74
|
budget: { onExceeded: "downgrade" },
|
|
72
75
|
profiles: [],
|
|
73
76
|
ledger: { path: ":memory:", blendWindowDays: 7, blendMinSamples: 20, fallbackBlend: { inputPerMtok: 1, outputPerMtok: 4 }, conversationTtlMs: 86_400_000 },
|
|
@@ -83,6 +86,7 @@ function mkReq(): NormRequest {
|
|
|
83
86
|
conversationKey: "conv-test",
|
|
84
87
|
harnessId: "",
|
|
85
88
|
ompSessionId: "",
|
|
89
|
+
agentdoxScope: "",
|
|
86
90
|
requestedModel: "auto",
|
|
87
91
|
messages: [{ role: "user", text: "hi", images: 0, textBytes: 2, toolCalls: [] }],
|
|
88
92
|
tools: [],
|
|
@@ -107,7 +111,9 @@ const FEATURES: Features = {
|
|
|
107
111
|
distinctToolsUsed: 0,
|
|
108
112
|
lastToolFailed: false,
|
|
109
113
|
repeatedToolCall: false,
|
|
114
|
+
circularToolCall: false,
|
|
110
115
|
hasImages: false,
|
|
116
|
+
hasNewImage: false,
|
|
111
117
|
codeBlocks: 0,
|
|
112
118
|
codeBytes: 0,
|
|
113
119
|
looksLikeDiff: false,
|
|
@@ -137,6 +143,8 @@ function mkDecision(tier: Tier, slug: string, probe: Partial<ProbePlan> = {}): D
|
|
|
137
143
|
sessionId: "omp-conv-test",
|
|
138
144
|
sticky: false,
|
|
139
145
|
cacheBreakpointMessageIndices: [],
|
|
146
|
+
compactionPlan: [],
|
|
147
|
+
promptTokensSaved: 0,
|
|
140
148
|
reasoning: undefined,
|
|
141
149
|
maxTokens: undefined,
|
|
142
150
|
stripAssistantReasoning: false,
|
|
@@ -251,6 +259,8 @@ function mkConversations(): { store: ConversationStore; map: Map<string, Convers
|
|
|
251
259
|
lastPromptTokens: 0,
|
|
252
260
|
cacheWarmSlug: null,
|
|
253
261
|
cacheWarmAtMs: 0,
|
|
262
|
+
contextVersion: null,
|
|
263
|
+
contextFetchedAtMs: 0,
|
|
254
264
|
updatedAtMs: 0,
|
|
255
265
|
};
|
|
256
266
|
map.set(k, fresh);
|
|
@@ -317,7 +327,7 @@ describe("runTurn", () => {
|
|
|
317
327
|
const { store, map } = mkConversations();
|
|
318
328
|
const { sink, chunks, errors, finishes } = mkSink();
|
|
319
329
|
|
|
320
|
-
await runTurn(mkReq(), sink, { config: mkConfig(), router, upstream, ledger, conversations: store, catalog }, new AbortController().signal);
|
|
330
|
+
await runTurn(mkReq(), sink, { config: mkConfig(), router, upstream, ledger, conversations: store, catalog, context: createDisabledBridge() }, new AbortController().signal);
|
|
321
331
|
|
|
322
332
|
expect(errors).toHaveLength(0);
|
|
323
333
|
expect(finishes).toHaveLength(1);
|
|
@@ -366,7 +376,7 @@ describe("runTurn", () => {
|
|
|
366
376
|
const { store } = mkConversations();
|
|
367
377
|
const { sink, chunks, errors, finishes } = mkSink();
|
|
368
378
|
|
|
369
|
-
await runTurn(mkReq(), sink, { config: mkConfig(), router, upstream, ledger, conversations: store, catalog }, new AbortController().signal);
|
|
379
|
+
await runTurn(mkReq(), sink, { config: mkConfig(), router, upstream, ledger, conversations: store, catalog, context: createDisabledBridge() }, new AbortController().signal);
|
|
370
380
|
|
|
371
381
|
expect(errors).toHaveLength(0);
|
|
372
382
|
expect(entries).toHaveLength(2);
|
|
@@ -404,7 +414,7 @@ describe("runTurn", () => {
|
|
|
404
414
|
const { store } = mkConversations();
|
|
405
415
|
const { sink, chunks, errors, finishes } = mkSink();
|
|
406
416
|
|
|
407
|
-
await runTurn(mkReq(), sink, { config: mkConfig(), router, upstream, ledger, conversations: store, catalog }, new AbortController().signal);
|
|
417
|
+
await runTurn(mkReq(), sink, { config: mkConfig(), router, upstream, ledger, conversations: store, catalog, context: createDisabledBridge() }, new AbortController().signal);
|
|
408
418
|
|
|
409
419
|
// Bytes reached the client, so the 429 mid-stream is surfaced, not retried.
|
|
410
420
|
expect(calls).toHaveLength(1);
|
|
@@ -424,7 +434,7 @@ describe("runTurn", () => {
|
|
|
424
434
|
const { store } = mkConversations();
|
|
425
435
|
const { sink, errors, finishes } = mkSink();
|
|
426
436
|
|
|
427
|
-
await runTurn(mkReq(), sink, { config: mkConfig(), router, upstream, ledger, conversations: store, catalog }, new AbortController().signal);
|
|
437
|
+
await runTurn(mkReq(), sink, { config: mkConfig(), router, upstream, ledger, conversations: store, catalog, context: createDisabledBridge() }, new AbortController().signal);
|
|
428
438
|
|
|
429
439
|
expect(calls).toHaveLength(1); // no retry, no escalation on auth
|
|
430
440
|
expect(finishes).toHaveLength(0);
|
|
@@ -448,7 +458,7 @@ describe("runTurn", () => {
|
|
|
448
458
|
const { store } = mkConversations();
|
|
449
459
|
const { sink, errors, finishes } = mkSink();
|
|
450
460
|
|
|
451
|
-
await runTurn(mkReq(), sink, { config: mkConfig(), router, upstream, ledger, conversations: store, catalog }, new AbortController().signal);
|
|
461
|
+
await runTurn(mkReq(), sink, { config: mkConfig(), router, upstream, ledger, conversations: store, catalog, context: createDisabledBridge() }, new AbortController().signal);
|
|
452
462
|
|
|
453
463
|
expect(errors).toHaveLength(0);
|
|
454
464
|
expect(finishes).toHaveLength(1);
|
|
@@ -488,14 +498,14 @@ describe("runTurn", () => {
|
|
|
488
498
|
const { sink, errors } = mkSink();
|
|
489
499
|
|
|
490
500
|
// Turn 1: first turn, no prior tier → re-arms (tierChanged true).
|
|
491
|
-
await runTurn(mkReq(), sink, { config: mkConfig(), router, upstream, ledger, conversations: store, catalog }, new AbortController().signal);
|
|
501
|
+
await runTurn(mkReq(), sink, { config: mkConfig(), router, upstream, ledger, conversations: store, catalog, context: createDisabledBridge() }, new AbortController().signal);
|
|
492
502
|
const afterFirst = map.get("conv-test")!;
|
|
493
503
|
expect(afterFirst.currentTier).toBe("hard");
|
|
494
504
|
expect(afterFirst.stickyUntilTurn).toBe(1 + 2); // holdTurns=2
|
|
495
505
|
|
|
496
506
|
// Turn 2: same tier served again → must NOT re-arm. The window should
|
|
497
507
|
// stay at its previous expiry (turn 3), not extend to turn 4.
|
|
498
|
-
await runTurn(mkReq(), sink, { config: mkConfig(), router, upstream, ledger, conversations: store, catalog }, new AbortController().signal);
|
|
508
|
+
await runTurn(mkReq(), sink, { config: mkConfig(), router, upstream, ledger, conversations: store, catalog, context: createDisabledBridge() }, new AbortController().signal);
|
|
499
509
|
const afterSecond = map.get("conv-test")!;
|
|
500
510
|
expect(afterSecond.currentTier).toBe("hard");
|
|
501
511
|
expect(afterSecond.stickyUntilTurn).toBe(3); // unchanged, not 4
|
|
@@ -517,7 +527,7 @@ describe("exploration reaches the ledger", () => {
|
|
|
517
527
|
const { store } = mkConversations();
|
|
518
528
|
const { sink } = mkSink();
|
|
519
529
|
|
|
520
|
-
await runTurn(mkReq(), sink, { config: mkConfig(), router, upstream, ledger, conversations: store, catalog }, new AbortController().signal);
|
|
530
|
+
await runTurn(mkReq(), sink, { config: mkConfig(), router, upstream, ledger, conversations: store, catalog, context: createDisabledBridge() }, new AbortController().signal);
|
|
521
531
|
|
|
522
532
|
expect(entries).toHaveLength(1);
|
|
523
533
|
// The counterfactual pair: what the classifier wanted, and what actually ran.
|
|
@@ -537,7 +547,7 @@ describe("exploration reaches the ledger", () => {
|
|
|
537
547
|
const { store } = mkConversations();
|
|
538
548
|
const { sink } = mkSink();
|
|
539
549
|
|
|
540
|
-
await runTurn(mkReq(), sink, { config: mkConfig(), router, upstream, ledger, conversations: store, catalog }, new AbortController().signal);
|
|
550
|
+
await runTurn(mkReq(), sink, { config: mkConfig(), router, upstream, ledger, conversations: store, catalog, context: createDisabledBridge() }, new AbortController().signal);
|
|
541
551
|
|
|
542
552
|
expect(entries).toHaveLength(1);
|
|
543
553
|
expect(entries[0]?.exploredFrom).toBeNull();
|