auto-model-router 0.4.1 → 0.4.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.omp-plugin/marketplace.json +2 -2
- package/README.md +27 -0
- package/omp-extension/digest-logic.ts +53 -0
- package/omp-extension/pi-coding-agent.d.ts +14 -1
- package/omp-extension/router-digest.ts +93 -0
- package/package.json +1 -1
- package/src/cli/config-wizard.ts +15 -0
- package/src/config/defaults.ts +13 -0
- package/src/config/schema.ts +14 -0
- package/src/config/types.ts +27 -0
- package/src/cost/ledger.ts +2 -1
- package/src/cost/report.ts +16 -0
- package/src/server/digest.ts +233 -0
- package/src/server/http.ts +22 -0
- package/test/controls.test.ts +34 -0
- package/test/digest.test.ts +207 -0
- package/test/embed-lifecycle.test.ts +1 -1
- package/test/failover.test.ts +1 -0
- package/test/report-hub.test.ts +3 -0
- package/test/report.test.ts +3 -0
- package/test/turn.test.ts +1 -0
|
@@ -7,14 +7,14 @@
|
|
|
7
7
|
},
|
|
8
8
|
"metadata": {
|
|
9
9
|
"description": "auto-model-router: a local cost/complexity-aware model router for Oh My Pi, backed by OpenRouter",
|
|
10
|
-
"version": "0.4.
|
|
10
|
+
"version": "0.4.2",
|
|
11
11
|
"pluginRoot": "."
|
|
12
12
|
},
|
|
13
13
|
"plugins": [
|
|
14
14
|
{
|
|
15
15
|
"name": "auto-model-router",
|
|
16
16
|
"description": "Local cost/complexity-aware model router for Oh My Pi, backed by OpenRouter. Runs in-process, routes per turn by price and task complexity, with budget caps, mid-stream escalation, and cache-aware hysteresis.",
|
|
17
|
-
"version": "0.4.
|
|
17
|
+
"version": "0.4.2",
|
|
18
18
|
"author": {
|
|
19
19
|
"name": "drewappling",
|
|
20
20
|
"email": "drewappling@gmail.com"
|
package/README.md
CHANGED
|
@@ -213,6 +213,7 @@ extensions:
|
|
|
213
213
|
- auto-model-router/omp-extension/router-embed.ts
|
|
214
214
|
- auto-model-router/omp-extension/router-toast.ts # optional: chosen-model toasts
|
|
215
215
|
- auto-model-router/omp-extension/router-configure.ts # optional: /router config, report, status
|
|
216
|
+
- auto-model-router/omp-extension/router-digest.ts # optional: cheap-model digest of large tool results
|
|
216
217
|
```
|
|
217
218
|
|
|
218
219
|
### From the repo (cross-platform installer)
|
|
@@ -236,6 +237,7 @@ The installer adds:
|
|
|
236
237
|
- `router-embed.ts` — **required**; runs the router in-process.
|
|
237
238
|
- `router-toast.ts` — optional; chosen-model toasts.
|
|
238
239
|
- `router-configure.ts` — optional; the `/router` command (configure, usage reports, status).
|
|
240
|
+
- `router-digest.ts` — optional; condenses large tool results with a cheap model before an expensive one reads them (needs `digest.enabled`).
|
|
239
241
|
|
|
240
242
|
Or add the paths by hand to omp's `~/.omp/agent/config.yml`:
|
|
241
243
|
|
|
@@ -245,6 +247,7 @@ extensions:
|
|
|
245
247
|
- /path/to/auto-model-router/omp-extension/router-embed.ts
|
|
246
248
|
- /path/to/auto-model-router/omp-extension/router-toast.ts # optional: chosen-model toasts
|
|
247
249
|
- /path/to/auto-model-router/omp-extension/router-configure.ts # optional: /router config, report, status
|
|
250
|
+
- /path/to/auto-model-router/omp-extension/router-digest.ts # optional: cheap-model digest of large tool results
|
|
248
251
|
```
|
|
249
252
|
|
|
250
253
|
Then restart the omp session (extensions load at session start).
|
|
@@ -777,6 +780,30 @@ Each profile is a complete entry (arrays replace wholesale):
|
|
|
777
780
|
| `maxTokens` | `32000` | Advertised max output tokens. |
|
|
778
781
|
| `budget` | unset | Per-profile budget overrides. |
|
|
779
782
|
|
|
783
|
+
### `digest` — cheap-model digest of large tool results
|
|
784
|
+
|
|
785
|
+
Tool results are the bulk of every prompt (see the report's prompt anatomy),
|
|
786
|
+
and a prompt is ~96% of spend. With the `router-digest` extension installed
|
|
787
|
+
and `digest.enabled` on, a large read, grep, glob or bash result produced
|
|
788
|
+
while the session's current model is at or above `fromTier` is sent to
|
|
789
|
+
`POST /v1/router/digest`; the cheapest `tier` model rewrites it to what the
|
|
790
|
+
task needs (exact paths, line numbers, names, errors, code to be edited) and
|
|
791
|
+
the digest replaces the tool result. It begins with a marker naming the tool
|
|
792
|
+
and arguments to re-run for the full output, so nothing is lost, only
|
|
793
|
+
deferred. Errors, images, edits and writes are never digested. Every digest
|
|
794
|
+
is a ledger row (`requestedModel` `digest`) and the report totals them.
|
|
795
|
+
|
|
796
|
+
| Key | Default | Meaning |
|
|
797
|
+
| --- | --- | --- |
|
|
798
|
+
| `enabled` | `false` | Master switch; the extension polls it every minute. |
|
|
799
|
+
| `minBytes` / `maxBytes` | `12000` / `400000` | Result size window that gets digested. |
|
|
800
|
+
| `tools` | `read, grep, glob, bash, web_fetch, webfetch, ls, find` | Eligible tool names (lower-case). |
|
|
801
|
+
| `fromTier` | `moderate` | Digest only when the session's current model is at or above this tier. |
|
|
802
|
+
| `tier` / `model` | `simple` / unset | Where the digest model is picked from, or a pinned slug. |
|
|
803
|
+
| `maxOutputTokens` | `700` | Digest length cap. |
|
|
804
|
+
| `maxCostUsd` | `0.02` | Skip when the digest itself would cost more. |
|
|
805
|
+
| `timeoutMs` | `25000` | The raw result stands if the cheap model is slower. |
|
|
806
|
+
|
|
780
807
|
### `report` — usage-report options
|
|
781
808
|
|
|
782
809
|
| Key | Default | Meaning |
|
|
@@ -0,0 +1,53 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Pure logic for the tool-result digest extension: which results to send,
|
|
3
|
+
* how to read a tool result's text, and how to shape the replacement.
|
|
4
|
+
*/
|
|
5
|
+
|
|
6
|
+
export interface DigestPolicy {
|
|
7
|
+
enabled: boolean;
|
|
8
|
+
minBytes: number;
|
|
9
|
+
maxBytes: number;
|
|
10
|
+
tools: string[];
|
|
11
|
+
fromTier: string;
|
|
12
|
+
}
|
|
13
|
+
|
|
14
|
+
export const DISABLED_POLICY: DigestPolicy = { enabled: false, minBytes: 0, maxBytes: 0, tools: [], fromTier: "hard" };
|
|
15
|
+
|
|
16
|
+
/** The text of a tool result's content parts; images are left alone (and block digesting). */
|
|
17
|
+
export function textOf(content: ReadonlyArray<{ type: string; text?: string }>): { text: string; hasImage: boolean } {
|
|
18
|
+
let text = "";
|
|
19
|
+
let hasImage = false;
|
|
20
|
+
for (const part of content) {
|
|
21
|
+
if (part.type === "text" && typeof part.text === "string") text += (text === "" ? "" : "\n") + part.text;
|
|
22
|
+
else if (part.type === "image") hasImage = true;
|
|
23
|
+
}
|
|
24
|
+
return { text, hasImage };
|
|
25
|
+
}
|
|
26
|
+
|
|
27
|
+
/** Client-side gate: cheap checks before anything is sent to the router. */
|
|
28
|
+
export function shouldSend(policy: DigestPolicy, toolName: string, isError: boolean, text: string, hasImage: boolean): boolean {
|
|
29
|
+
if (!policy.enabled || isError || hasImage) return false;
|
|
30
|
+
if (!policy.tools.includes(toolName.toLowerCase())) return false;
|
|
31
|
+
const bytes = Buffer.byteLength(text);
|
|
32
|
+
return bytes >= policy.minBytes && bytes <= policy.maxBytes;
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
/** Parses the router's policy payload defensively; anything odd ⇒ disabled. */
|
|
36
|
+
export function parsePolicy(json: unknown): DigestPolicy {
|
|
37
|
+
if (typeof json !== "object" || json === null) return DISABLED_POLICY;
|
|
38
|
+
const p = json as Record<string, unknown>;
|
|
39
|
+
if (p.enabled !== true) return DISABLED_POLICY;
|
|
40
|
+
return {
|
|
41
|
+
enabled: true,
|
|
42
|
+
minBytes: typeof p.minBytes === "number" ? p.minBytes : 12_000,
|
|
43
|
+
maxBytes: typeof p.maxBytes === "number" ? p.maxBytes : 400_000,
|
|
44
|
+
tools: Array.isArray(p.tools) ? p.tools.filter((t): t is string => typeof t === "string").map((t) => t.toLowerCase()) : [],
|
|
45
|
+
fromTier: typeof p.fromTier === "string" ? p.fromTier : "hard",
|
|
46
|
+
};
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
/** One-line toast for a digest that happened. */
|
|
50
|
+
export function digestToast(toolName: string, inputBytes: number, outputChars: number, model: string, usd: number): string {
|
|
51
|
+
const kb = (n: number): string => `${(n / 1024).toFixed(0)}KB`;
|
|
52
|
+
return `digested ${toolName} ${kb(inputBytes)} → ${kb(outputChars)} via ${model.replace(/^ollama\//, "")} ($${usd.toFixed(4)})`;
|
|
53
|
+
}
|
|
@@ -127,9 +127,22 @@ declare module "@oh-my-pi/pi-coding-agent" {
|
|
|
127
127
|
details?: unknown;
|
|
128
128
|
}
|
|
129
129
|
|
|
130
|
+
/** A tool result's content parts (text and images). */
|
|
131
|
+
export interface ToolResultPart {
|
|
132
|
+
type: string;
|
|
133
|
+
text?: string;
|
|
134
|
+
}
|
|
135
|
+
|
|
136
|
+
/** What a `tool_result` handler may return to replace the result. */
|
|
137
|
+
export interface ToolResultEventResult {
|
|
138
|
+
content?: ToolResultPart[];
|
|
139
|
+
isError?: boolean;
|
|
140
|
+
}
|
|
141
|
+
|
|
130
142
|
export interface ExtensionAPI {
|
|
131
143
|
setLabel(label: string): void;
|
|
132
|
-
|
|
144
|
+
/** Handlers may return an event result (e.g. a `tool_result` replacement); omp ignores it where none applies. */
|
|
145
|
+
on(event: string, handler: (event: unknown, ctx: ExtensionContext) => unknown): void;
|
|
133
146
|
registerProvider(id: string, registration: ProviderRegistration): void;
|
|
134
147
|
unregisterProvider(id: string): void;
|
|
135
148
|
registerCommand(name: string, command: CommandDefinition): void;
|
|
@@ -0,0 +1,93 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* omp extension: condense large tool results with a cheap model before an
|
|
3
|
+
* expensive one reads them.
|
|
4
|
+
*
|
|
5
|
+
* Tool results are the bulk of every prompt, and a prompt is ~96% of spend.
|
|
6
|
+
* When a read, grep, glob or bash result is large and this session's
|
|
7
|
+
* current model sits at or above the router's `digest.fromTier`, the raw
|
|
8
|
+
* text goes to the router's `/v1/router/digest`, a simple-tier model
|
|
9
|
+
* rewrites it to what the task needs, and the digest replaces the tool
|
|
10
|
+
* result. The digest starts with a marker saying how to get the full output
|
|
11
|
+
* back (re-run the tool, or read a line range), so nothing is lost.
|
|
12
|
+
*
|
|
13
|
+
* The router decides (policy, session tier, cost guard); this extension only
|
|
14
|
+
* ships text that passes the cheap client-side checks. Off unless
|
|
15
|
+
* `digest.enabled` is set in the router config.
|
|
16
|
+
*
|
|
17
|
+
* Install beside router-embed.ts:
|
|
18
|
+
*
|
|
19
|
+
* # ~/.omp/agent/config.yml
|
|
20
|
+
* extensions:
|
|
21
|
+
* - /path/to/auto-model-router/omp-extension/router-embed.ts
|
|
22
|
+
* - /path/to/auto-model-router/omp-extension/router-digest.ts
|
|
23
|
+
*/
|
|
24
|
+
|
|
25
|
+
import type { ExtensionAPI } from "@oh-my-pi/pi-coding-agent";
|
|
26
|
+
|
|
27
|
+
import { DISABLED_POLICY, digestToast, parsePolicy, shouldSend, textOf, type DigestPolicy } from "./digest-logic.ts";
|
|
28
|
+
import { routerAuthHeaders, routerBaseUrl } from "./router-url.ts";
|
|
29
|
+
|
|
30
|
+
const HARNESS_ID = process.env.OMP_HARNESS_ID ?? "";
|
|
31
|
+
/** Re-read the policy this often, so a config change lands without a restart. */
|
|
32
|
+
const POLICY_TTL_MS = 60_000;
|
|
33
|
+
|
|
34
|
+
export default function (pi: ExtensionAPI): void {
|
|
35
|
+
pi.setLabel("auto-model-router digest");
|
|
36
|
+
|
|
37
|
+
let policy: DigestPolicy = DISABLED_POLICY;
|
|
38
|
+
let policyAtMs = 0;
|
|
39
|
+
let lastUserText = "";
|
|
40
|
+
|
|
41
|
+
async function refreshPolicy(): Promise<void> {
|
|
42
|
+
if (Date.now() - policyAtMs < POLICY_TTL_MS) return;
|
|
43
|
+
policyAtMs = Date.now();
|
|
44
|
+
try {
|
|
45
|
+
const res = await fetch(`${routerBaseUrl()}/v1/router/digest/policy`, { headers: routerAuthHeaders(), signal: AbortSignal.timeout(2_000) });
|
|
46
|
+
policy = res.ok ? parsePolicy(await res.json()) : DISABLED_POLICY;
|
|
47
|
+
} catch {
|
|
48
|
+
policy = DISABLED_POLICY;
|
|
49
|
+
}
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
pi.on("session_start", async () => {
|
|
53
|
+
policyAtMs = 0;
|
|
54
|
+
await refreshPolicy();
|
|
55
|
+
});
|
|
56
|
+
|
|
57
|
+
// The user's latest ask steers what the digest keeps.
|
|
58
|
+
pi.on("input", (event) => {
|
|
59
|
+
const e = event as { text?: string };
|
|
60
|
+
if (typeof e.text === "string" && e.text.trim() !== "") lastUserText = e.text.trim().slice(0, 400);
|
|
61
|
+
return undefined;
|
|
62
|
+
});
|
|
63
|
+
|
|
64
|
+
pi.on("tool_result", async (event, ctx) => {
|
|
65
|
+
const e = event as { toolName: string; input: Record<string, unknown>; content: Array<{ type: string; text?: string }>; isError: boolean };
|
|
66
|
+
await refreshPolicy();
|
|
67
|
+
const { text, hasImage } = textOf(e.content);
|
|
68
|
+
if (!shouldSend(policy, e.toolName, e.isError, text, hasImage)) return undefined;
|
|
69
|
+
try {
|
|
70
|
+
const res = await fetch(`${routerBaseUrl()}/v1/router/digest`, {
|
|
71
|
+
method: "POST",
|
|
72
|
+
headers: { ...routerAuthHeaders(), "content-type": "application/json" },
|
|
73
|
+
body: JSON.stringify({
|
|
74
|
+
ompSessionId: ctx.sessionManager.getSessionId(),
|
|
75
|
+
harnessId: HARNESS_ID,
|
|
76
|
+
toolName: e.toolName,
|
|
77
|
+
input: e.input,
|
|
78
|
+
content: text,
|
|
79
|
+
query: lastUserText,
|
|
80
|
+
}),
|
|
81
|
+
signal: AbortSignal.timeout(30_000),
|
|
82
|
+
});
|
|
83
|
+
if (!res.ok) return undefined;
|
|
84
|
+
const r = (await res.json()) as { digested: boolean; text?: string; model?: string; usd?: number; inputBytes?: number; outputChars?: number };
|
|
85
|
+
if (!r.digested || typeof r.text !== "string") return undefined;
|
|
86
|
+
if (ctx.hasUI) ctx.ui.notify(digestToast(e.toolName, r.inputBytes ?? 0, r.outputChars ?? 0, r.model ?? "?", r.usd ?? 0), "info");
|
|
87
|
+
return { content: [{ type: "text", text: r.text }] };
|
|
88
|
+
} catch {
|
|
89
|
+
// Router unreachable or slow: the raw result stands.
|
|
90
|
+
return undefined;
|
|
91
|
+
}
|
|
92
|
+
});
|
|
93
|
+
}
|
package/package.json
CHANGED
package/src/cli/config-wizard.ts
CHANGED
|
@@ -310,6 +310,21 @@ export const WIZARD_SECTIONS: readonly SectionSpec[] = [
|
|
|
310
310
|
{ path: "budget.onExceeded", label: "On exceeded", kind: "enum", options: ["downgrade", "reject"] },
|
|
311
311
|
],
|
|
312
312
|
},
|
|
313
|
+
{
|
|
314
|
+
title: "Digest",
|
|
315
|
+
fields: [
|
|
316
|
+
{ path: "digest.enabled", label: "Digest large tool results with a cheap model", kind: "boolean" },
|
|
317
|
+
{ path: "digest.minBytes", label: "Min result bytes", kind: "number", min: 0 },
|
|
318
|
+
{ path: "digest.maxBytes", label: "Max result bytes", kind: "number", min: 1 },
|
|
319
|
+
{ path: "digest.tools", label: "Tools eligible", kind: "stringArray", hint: "comma-separated, lower-case" },
|
|
320
|
+
{ path: "digest.fromTier", label: "Digest when the session is at or above", kind: "enum", options: TIER_NAMES },
|
|
321
|
+
{ path: "digest.tier", label: "Pick the digest model from tier", kind: "enum", options: TIER_NAMES },
|
|
322
|
+
{ path: "digest.model", label: "Pinned digest model", kind: "string", optional: true, hint: "blank = cheapest in tier" },
|
|
323
|
+
{ path: "digest.maxOutputTokens", label: "Max digest tokens", kind: "number", min: 1 },
|
|
324
|
+
{ path: "digest.maxCostUsd", label: "Max cost per digest $", kind: "number", min: 0 },
|
|
325
|
+
{ path: "digest.timeoutMs", label: "Digest timeout", kind: "number", min: 1, hint: "ms" },
|
|
326
|
+
],
|
|
327
|
+
},
|
|
313
328
|
{
|
|
314
329
|
title: "Report",
|
|
315
330
|
fields: [{ path: "report.baselines", label: "Counterfactual baseline models", kind: "stringArray", hint: "comma-separated slugs" }],
|
package/src/config/defaults.ts
CHANGED
|
@@ -292,6 +292,19 @@ export const DEFAULT_CONFIG: RouterConfig = {
|
|
|
292
292
|
elideSupersededReads: true,
|
|
293
293
|
collapseDuplicateResults: true,
|
|
294
294
|
},
|
|
295
|
+
digest: {
|
|
296
|
+
// Off until an operator turns it on: it changes what the model reads.
|
|
297
|
+
enabled: false,
|
|
298
|
+
minBytes: 12_000,
|
|
299
|
+
maxBytes: 400_000,
|
|
300
|
+
tools: ["read", "grep", "glob", "bash", "web_fetch", "webfetch", "ls", "find"],
|
|
301
|
+
fromTier: "moderate",
|
|
302
|
+
tier: "simple",
|
|
303
|
+
model: "",
|
|
304
|
+
maxOutputTokens: 700,
|
|
305
|
+
maxCostUsd: 0.02,
|
|
306
|
+
timeoutMs: 25_000,
|
|
307
|
+
},
|
|
295
308
|
report: {
|
|
296
309
|
// The frontier pair most omp users would otherwise run on.
|
|
297
310
|
baselines: ["anthropic/claude-opus-5", "anthropic/claude-sonnet-5"],
|
package/src/config/schema.ts
CHANGED
|
@@ -273,6 +273,20 @@ export const configInputSchema = z.strictObject({
|
|
|
273
273
|
budget: budget.optional(),
|
|
274
274
|
profiles: z.array(profile).optional(),
|
|
275
275
|
report: z.strictObject({ baselines: z.array(z.string()).optional() }).optional(),
|
|
276
|
+
digest: z
|
|
277
|
+
.strictObject({
|
|
278
|
+
enabled: z.boolean().optional(),
|
|
279
|
+
minBytes: z.number().int().nonnegative().optional(),
|
|
280
|
+
maxBytes: z.number().int().positive().optional(),
|
|
281
|
+
tools: z.array(z.string()).optional(),
|
|
282
|
+
fromTier: tier.optional(),
|
|
283
|
+
tier: tier.optional(),
|
|
284
|
+
model: z.string().optional(),
|
|
285
|
+
maxOutputTokens: z.number().int().positive().optional(),
|
|
286
|
+
maxCostUsd: z.number().nonnegative().optional(),
|
|
287
|
+
timeoutMs: z.number().int().positive().optional(),
|
|
288
|
+
})
|
|
289
|
+
.optional(),
|
|
276
290
|
ledger: ledger.optional(),
|
|
277
291
|
adaptiveTierFloors: z.boolean().optional(),
|
|
278
292
|
adaptivePriceCeilings: z.boolean().optional(),
|
package/src/config/types.ts
CHANGED
|
@@ -554,6 +554,32 @@ export interface CacheConfig {
|
|
|
554
554
|
milestoneTokens: number;
|
|
555
555
|
}
|
|
556
556
|
|
|
557
|
+
/**
|
|
558
|
+
* Tool-result digest: a cheap model condenses large tool outputs before an
|
|
559
|
+
* expensive one reads them (see server/digest.ts and the router-digest omp
|
|
560
|
+
* extension).
|
|
561
|
+
*/
|
|
562
|
+
export interface DigestConfig {
|
|
563
|
+
/** Master switch; the omp extension polls this as its policy. */
|
|
564
|
+
enabled: boolean;
|
|
565
|
+
/** Tool results smaller than this pass through untouched. */
|
|
566
|
+
minBytes: number;
|
|
567
|
+
/** Results larger than this are left alone (too costly even for a cheap model). */
|
|
568
|
+
maxBytes: number;
|
|
569
|
+
/** Tool names (lower-case) whose results may be digested. Never errors, never edits/writes. */
|
|
570
|
+
tools: string[];
|
|
571
|
+
/** Digest only when the session's current model is at or above this tier. */
|
|
572
|
+
fromTier: Tier;
|
|
573
|
+
/** Tier the digest model is picked from (cheapest candidate that fits). */
|
|
574
|
+
tier: Tier;
|
|
575
|
+
/** Pin a specific digest model; empty ⇒ pick from `tier`. */
|
|
576
|
+
model: string;
|
|
577
|
+
maxOutputTokens: number;
|
|
578
|
+
/** Skip when the digest itself would cost more than this, USD. */
|
|
579
|
+
maxCostUsd: number;
|
|
580
|
+
timeoutMs: number;
|
|
581
|
+
}
|
|
582
|
+
|
|
557
583
|
/** Usage-report options. */
|
|
558
584
|
export interface ReportConfig {
|
|
559
585
|
/**
|
|
@@ -741,6 +767,7 @@ export interface RouterConfig {
|
|
|
741
767
|
compaction: CompactionConfig;
|
|
742
768
|
budget: BudgetConfig;
|
|
743
769
|
report: ReportConfig;
|
|
770
|
+
digest: DigestConfig;
|
|
744
771
|
profiles: ProfileConfig[];
|
|
745
772
|
ledger: LedgerConfig;
|
|
746
773
|
/**
|
package/src/cost/ledger.ts
CHANGED
|
@@ -351,7 +351,8 @@ export function createLedger(db: Database, cfg: RouterConfig): Ledger {
|
|
|
351
351
|
const providerSpendStmt = db.query(
|
|
352
352
|
"SELECT COALESCE(SUM(COALESCE(reported_usd, predicted_usd)), 0) AS total FROM ledger WHERE created_at_ms >= ? AND COALESCE(served_slug, slug) LIKE ?",
|
|
353
353
|
);
|
|
354
|
-
|
|
354
|
+
// Digest rows (requested_model 'digest') are side calls, not the session's turns.
|
|
355
|
+
const sessionStmt = db.query("SELECT * FROM ledger WHERE omp_session_id = ? AND wasted = 0 AND requested_model <> 'digest' ORDER BY created_at_ms DESC LIMIT ?");
|
|
355
356
|
// What an escalated retry actually bills, per prompt token, over a window.
|
|
356
357
|
// attempt > 0 rows are the re-dispatches that followed a rejected attempt;
|
|
357
358
|
// errored ones carry no usage and are excluded.
|
package/src/cost/report.ts
CHANGED
|
@@ -30,6 +30,10 @@ export interface ReportTotals {
|
|
|
30
30
|
/** Turns from omp subagents (`features.isSubagent`), and their spend. */
|
|
31
31
|
subagentDispatches: number;
|
|
32
32
|
subagentSpendUsd: number;
|
|
33
|
+
/** Tool-result digests (requestedModel "digest"): count, what they cost, bytes they condensed. */
|
|
34
|
+
digests: number;
|
|
35
|
+
digestSpendUsd: number;
|
|
36
|
+
digestInputTokens: number;
|
|
33
37
|
}
|
|
34
38
|
|
|
35
39
|
export interface ReportRow {
|
|
@@ -201,6 +205,9 @@ export function buildUsageReport(
|
|
|
201
205
|
SUM(CASE WHEN ${EST} THEN 1 ELSE 0 END) AS estimated_rows,
|
|
202
206
|
SUM(CASE WHEN json_extract(features, '$.isSubagent') = 1 THEN 1 ELSE 0 END) AS subagent_rows,
|
|
203
207
|
COALESCE(SUM(CASE WHEN json_extract(features, '$.isSubagent') = 1 THEN ${USD} ELSE 0 END), 0) AS subagent_spend,
|
|
208
|
+
SUM(CASE WHEN requested_model = 'digest' THEN 1 ELSE 0 END) AS digests,
|
|
209
|
+
COALESCE(SUM(CASE WHEN requested_model = 'digest' THEN ${USD} ELSE 0 END), 0) AS digest_spend,
|
|
210
|
+
COALESCE(SUM(CASE WHEN requested_model = 'digest' THEN ${PT} ELSE 0 END), 0) AS digest_input,
|
|
204
211
|
SUM(CASE WHEN escalation_signal IS NOT NULL THEN 1 ELSE 0 END) AS escalations,
|
|
205
212
|
SUM(CASE WHEN instr(reasons, 'failover:') > 0 THEN 1 ELSE 0 END) AS failovers,
|
|
206
213
|
SUM(CASE WHEN error IS NOT NULL THEN 1 ELSE 0 END) AS errors,
|
|
@@ -217,6 +224,9 @@ export function buildUsageReport(
|
|
|
217
224
|
estimated_rows: number | null;
|
|
218
225
|
subagent_rows: number | null;
|
|
219
226
|
subagent_spend: number;
|
|
227
|
+
digests: number | null;
|
|
228
|
+
digest_spend: number;
|
|
229
|
+
digest_input: number;
|
|
220
230
|
escalations: number | null;
|
|
221
231
|
failovers: number | null;
|
|
222
232
|
errors: number | null;
|
|
@@ -336,6 +346,9 @@ export function buildUsageReport(
|
|
|
336
346
|
cacheEstimated: (t.estimated_rows ?? 0) > 0,
|
|
337
347
|
subagentDispatches: t.subagent_rows ?? 0,
|
|
338
348
|
subagentSpendUsd: t.subagent_spend,
|
|
349
|
+
digests: t.digests ?? 0,
|
|
350
|
+
digestSpendUsd: t.digest_spend,
|
|
351
|
+
digestInputTokens: t.digest_input,
|
|
339
352
|
},
|
|
340
353
|
providers,
|
|
341
354
|
models,
|
|
@@ -400,6 +413,9 @@ export function reportView(r: UsageReport, opts: { maxModels?: number } = {}): R
|
|
|
400
413
|
.join(" · ")}`,
|
|
401
414
|
);
|
|
402
415
|
}
|
|
416
|
+
if (t.digests > 0) {
|
|
417
|
+
summary.push(`digests: ${num(t.digests)} tool results condensed (${num(t.digestInputTokens)} tok read by a cheap model) for ${usd(t.digestSpendUsd)}`);
|
|
418
|
+
}
|
|
403
419
|
if (t.subagentDispatches > 0) {
|
|
404
420
|
summary.push(`subagents: ${num(t.subagentDispatches)} dispatches, ${usd(t.subagentSpendUsd)} (${pct(t.spendUsd > 0 ? t.subagentSpendUsd / t.spendUsd : 0)} of spend)`);
|
|
405
421
|
}
|
|
@@ -0,0 +1,233 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Tool-result digest: a cheap model condenses a large tool output before it
|
|
3
|
+
* reaches an expensive one.
|
|
4
|
+
*
|
|
5
|
+
* Prompt anatomy showed tool results are the bulk of every prompt, and a
|
|
6
|
+
* prompt is ~96% of spend. A 60KB file read on a hard-tier turn is re-read
|
|
7
|
+
* by that model on every later turn of the conversation, cached or not.
|
|
8
|
+
* When the omp extension sees a large read/grep/glob/bash result while the
|
|
9
|
+
* session's current model sits at or above `digest.fromTier`, it sends the
|
|
10
|
+
* text here; a simple-tier model rewrites it to what the task needs — exact
|
|
11
|
+
* paths, line numbers, names, errors, code that would be edited — and the
|
|
12
|
+
* digest replaces the tool result. The marker on top says how to get the
|
|
13
|
+
* full output back (re-run the tool, or read a line range), so nothing is
|
|
14
|
+
* lost, only deferred.
|
|
15
|
+
*
|
|
16
|
+
* Guarded: never on errors, never below `minBytes`, never above `maxBytes`,
|
|
17
|
+
* never past `maxCostUsd`, and every digest is a ledger row
|
|
18
|
+
* (requestedModel "digest") so the report shows what it cost and saved.
|
|
19
|
+
*/
|
|
20
|
+
|
|
21
|
+
import type { CatalogModel, CatalogSource } from "../catalog/types.ts";
|
|
22
|
+
import type { DigestConfig, RouterConfig } from "../config/types.ts";
|
|
23
|
+
import { computeCost, forecast } from "../cost/forecast.ts";
|
|
24
|
+
import type { Ledger, LedgerEntry } from "../cost/types.ts";
|
|
25
|
+
import { buildCandidates } from "../router/candidates.ts";
|
|
26
|
+
import { extractFeatures } from "../router/features.ts";
|
|
27
|
+
import { TIER_ORDER, type Tier } from "../router/types.ts";
|
|
28
|
+
import { estimateTokens } from "../tokens/estimate.ts";
|
|
29
|
+
import type { UpstreamClient } from "../upstream/types.ts";
|
|
30
|
+
import type { Logger } from "../util/log.ts";
|
|
31
|
+
import type { NormRequest } from "../wire/types.ts";
|
|
32
|
+
|
|
33
|
+
export interface DigestRequest {
|
|
34
|
+
ompSessionId: string;
|
|
35
|
+
harnessId: string;
|
|
36
|
+
toolName: string;
|
|
37
|
+
/** The tool's arguments, echoed into the marker so the model can re-run it. */
|
|
38
|
+
input: Record<string, unknown>;
|
|
39
|
+
content: string;
|
|
40
|
+
/** The user's current ask, so the digest keeps what matters for it. */
|
|
41
|
+
query: string;
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
export type DigestResult =
|
|
45
|
+
| { digested: true; text: string; model: string; usd: number; inputBytes: number; outputChars: number; ms: number }
|
|
46
|
+
| { digested: false; reason: string };
|
|
47
|
+
|
|
48
|
+
export interface DigesterDeps {
|
|
49
|
+
cfg: RouterConfig;
|
|
50
|
+
catalog: CatalogSource;
|
|
51
|
+
ledger: Ledger;
|
|
52
|
+
upstream: UpstreamClient;
|
|
53
|
+
log: Logger;
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
const DIGEST_SYSTEM = `You condense tool output for a coding agent that is mid-task. Keep everything the task could need: exact file paths, line numbers, identifiers, signatures, error text, counts and values. Quote verbatim, with line numbers, any code the agent is likely to edit or reference. Drop repetition, boilerplate, generated noise and unrelated regions. Never invent content. Plain text only, no preamble. First line: one sentence saying what was omitted and roughly how much.`;
|
|
57
|
+
|
|
58
|
+
const tierIdx = (t: string): number => TIER_ORDER.indexOf(t as Tier);
|
|
59
|
+
|
|
60
|
+
/** Whether a session's current model is expensive enough for a digest to pay off. */
|
|
61
|
+
export function digestApplies(cfg: DigestConfig, toolName: string, bytes: number, isError: boolean, currentTier: string | null): { ok: true } | { ok: false; reason: string } {
|
|
62
|
+
if (!cfg.enabled) return { ok: false, reason: "digest disabled" };
|
|
63
|
+
if (isError) return { ok: false, reason: "error results are never digested" };
|
|
64
|
+
if (!cfg.tools.includes(toolName.toLowerCase())) return { ok: false, reason: `tool ${toolName} not in digest.tools` };
|
|
65
|
+
if (bytes < cfg.minBytes) return { ok: false, reason: `${bytes} bytes < minBytes ${cfg.minBytes}` };
|
|
66
|
+
if (bytes > cfg.maxBytes) return { ok: false, reason: `${bytes} bytes > maxBytes ${cfg.maxBytes}` };
|
|
67
|
+
if (currentTier === null) return { ok: false, reason: "no routed turn in this session yet" };
|
|
68
|
+
if (tierIdx(currentTier) < tierIdx(cfg.fromTier)) return { ok: false, reason: `session is on ${currentTier}, below digest.fromTier ${cfg.fromTier}` };
|
|
69
|
+
return { ok: true };
|
|
70
|
+
}
|
|
71
|
+
|
|
72
|
+
/** The line that replaces the raw output's head: what happened and how to undo it. */
|
|
73
|
+
export function digestMarker(toolName: string, input: Record<string, unknown>, model: string, inputBytes: number, outputChars: number): string {
|
|
74
|
+
const args = JSON.stringify(input);
|
|
75
|
+
const shownArgs = args.length > 160 ? `${args.slice(0, 159)}…` : args;
|
|
76
|
+
return `[digest: ${toolName} output ${inputBytes.toLocaleString("en-US")} bytes → ${outputChars.toLocaleString("en-US")} chars by ${model}. Full output: re-run ${toolName} ${shownArgs}${toolName === "read" ? " (offset/limit for a range)" : ""}]`;
|
|
77
|
+
}
|
|
78
|
+
|
|
79
|
+
function syntheticRequest(req: DigestRequest, promptText: string): NormRequest {
|
|
80
|
+
const bytes = Buffer.byteLength(promptText);
|
|
81
|
+
return {
|
|
82
|
+
protocol: "openai-chat",
|
|
83
|
+
conversationKey: `digest:${req.ompSessionId}`,
|
|
84
|
+
harnessId: req.harnessId,
|
|
85
|
+
ompSessionId: req.ompSessionId,
|
|
86
|
+
agentdoxScope: "",
|
|
87
|
+
isSubagent: true,
|
|
88
|
+
requestedModel: "digest",
|
|
89
|
+
messages: [
|
|
90
|
+
{ role: "system", text: DIGEST_SYSTEM, images: 0, textBytes: Buffer.byteLength(DIGEST_SYSTEM), toolCalls: [] },
|
|
91
|
+
{ role: "user", text: promptText, images: 0, textBytes: bytes, toolCalls: [] },
|
|
92
|
+
],
|
|
93
|
+
tools: [],
|
|
94
|
+
forcedToolChoice: false,
|
|
95
|
+
stream: false,
|
|
96
|
+
hasImages: false,
|
|
97
|
+
promptBytes: bytes + Buffer.byteLength(DIGEST_SYSTEM),
|
|
98
|
+
renderUpstreamBody: () => ({}),
|
|
99
|
+
};
|
|
100
|
+
}
|
|
101
|
+
|
|
102
|
+
export function createDigester(deps: DigesterDeps): { digest(req: DigestRequest): Promise<DigestResult> } {
|
|
103
|
+
const { cfg, catalog, ledger, upstream, log } = deps;
|
|
104
|
+
|
|
105
|
+
/** Cheapest simple-tier model that fits the prompt, or the configured one. */
|
|
106
|
+
async function pickModel(req: NormRequest, promptTokens: number): Promise<CatalogModel | null> {
|
|
107
|
+
const snapshot = await catalog.get();
|
|
108
|
+
if (cfg.digest.model !== "") return snapshot.models.find((m) => m.slug === cfg.digest.model) ?? null;
|
|
109
|
+
const features = extractFeatures(req, promptTokens);
|
|
110
|
+
for (const relaxLevel of [0, 1, 2]) {
|
|
111
|
+
const built = buildCandidates({
|
|
112
|
+
req,
|
|
113
|
+
features,
|
|
114
|
+
tier: cfg.digest.tier,
|
|
115
|
+
task: "documentation",
|
|
116
|
+
snapshot,
|
|
117
|
+
ledger,
|
|
118
|
+
cfg,
|
|
119
|
+
expectedCompletionTokens: cfg.digest.maxOutputTokens,
|
|
120
|
+
warmSlug: null,
|
|
121
|
+
relaxLevel,
|
|
122
|
+
});
|
|
123
|
+
const first = built.candidates[0];
|
|
124
|
+
if (first !== undefined) return first.model;
|
|
125
|
+
}
|
|
126
|
+
return null;
|
|
127
|
+
}
|
|
128
|
+
|
|
129
|
+
return {
|
|
130
|
+
async digest(req) {
|
|
131
|
+
const inputBytes = Buffer.byteLength(req.content);
|
|
132
|
+
const currentTier = ledger.latestForSession?.(req.ompSessionId)?.tier ?? null;
|
|
133
|
+
const applies = digestApplies(cfg.digest, req.toolName, inputBytes, false, currentTier);
|
|
134
|
+
if (!applies.ok) return { digested: false, reason: applies.reason };
|
|
135
|
+
|
|
136
|
+
const promptText = `Task: ${req.query === "" ? "(unknown)" : req.query}\nTool: ${req.toolName} ${JSON.stringify(req.input)}\n--- output ---\n${req.content}`;
|
|
137
|
+
const synthetic = syntheticRequest(req, promptText);
|
|
138
|
+
const promptTokens = estimateTokens(synthetic.promptBytes, "unknown", ledger);
|
|
139
|
+
const model = await pickModel(synthetic, promptTokens);
|
|
140
|
+
if (model === null) return { digested: false, reason: "no digest model available" };
|
|
141
|
+
const est = forecast(model, { promptTokens, completionTokens: cfg.digest.maxOutputTokens, cacheHitRate: 0, images: 0 });
|
|
142
|
+
if (est.coldUsd > cfg.digest.maxCostUsd) {
|
|
143
|
+
return { digested: false, reason: `estimated $${est.coldUsd.toFixed(4)} on ${model.slug} exceeds digest.maxCostUsd $${cfg.digest.maxCostUsd}` };
|
|
144
|
+
}
|
|
145
|
+
|
|
146
|
+
const controller = new AbortController();
|
|
147
|
+
const timer = setTimeout(() => controller.abort(), cfg.digest.timeoutMs);
|
|
148
|
+
const startedAt = Date.now();
|
|
149
|
+
let text = "";
|
|
150
|
+
let costUsd: number | null = null;
|
|
151
|
+
let error: string | null = null;
|
|
152
|
+
try {
|
|
153
|
+
const out = await upstream.complete(
|
|
154
|
+
{
|
|
155
|
+
model: model.slug,
|
|
156
|
+
stream: false,
|
|
157
|
+
max_tokens: cfg.digest.maxOutputTokens,
|
|
158
|
+
temperature: 0,
|
|
159
|
+
messages: [
|
|
160
|
+
{ role: "system", content: DIGEST_SYSTEM },
|
|
161
|
+
{ role: "user", content: promptText },
|
|
162
|
+
],
|
|
163
|
+
},
|
|
164
|
+
controller.signal,
|
|
165
|
+
);
|
|
166
|
+
text = out.text.trim();
|
|
167
|
+
costUsd = out.costUsd;
|
|
168
|
+
} catch (err) {
|
|
169
|
+
error = err instanceof Error ? err.message : String(err);
|
|
170
|
+
} finally {
|
|
171
|
+
clearTimeout(timer);
|
|
172
|
+
}
|
|
173
|
+
const ms = Date.now() - startedAt;
|
|
174
|
+
const completionTokens = estimateTokens(Buffer.byteLength(text), model.tokenizer, ledger);
|
|
175
|
+
const usage = { promptTokens, cachedTokens: 0, cacheWriteTokens: 0, completionTokens, reasoningTokens: 0, images: 0 };
|
|
176
|
+
const usd = costUsd ?? computeCost(model, usage).total;
|
|
177
|
+
|
|
178
|
+
// Every digest is a ledger row: the report shows its cost beside the
|
|
179
|
+
// prompt tokens it kept out of the expensive model.
|
|
180
|
+
const entry: LedgerEntry = {
|
|
181
|
+
id: crypto.randomUUID(),
|
|
182
|
+
createdAtMs: startedAt,
|
|
183
|
+
conversationKey: synthetic.conversationKey,
|
|
184
|
+
sessionId: `digest-${req.ompSessionId}`,
|
|
185
|
+
turn: 1,
|
|
186
|
+
requestedModel: "digest",
|
|
187
|
+
harnessId: req.harnessId,
|
|
188
|
+
ompSessionId: req.ompSessionId,
|
|
189
|
+
slug: model.slug,
|
|
190
|
+
servedSlug: model.slug,
|
|
191
|
+
tier: cfg.digest.tier,
|
|
192
|
+
classificationSource: "forced",
|
|
193
|
+
reasons: [`digest: ${req.toolName} ${inputBytes} bytes → ${text.length} chars for a ${currentTier} session`],
|
|
194
|
+
features: null,
|
|
195
|
+
score: null,
|
|
196
|
+
confidence: null,
|
|
197
|
+
task: "documentation",
|
|
198
|
+
classifierReasons: null,
|
|
199
|
+
exploredFrom: null,
|
|
200
|
+
holdArm: null,
|
|
201
|
+
predictedUsd: est.expectedUsd,
|
|
202
|
+
reportedUsd: error === null ? usd : null,
|
|
203
|
+
usage,
|
|
204
|
+
attempt: 0,
|
|
205
|
+
escalationSignal: null,
|
|
206
|
+
latencyMs: ms,
|
|
207
|
+
ttftMs: null,
|
|
208
|
+
finishReason: error === null ? "stop" : null,
|
|
209
|
+
wasted: false,
|
|
210
|
+
upstreamGenerationId: null,
|
|
211
|
+
error,
|
|
212
|
+
promptTokensSaved: 0,
|
|
213
|
+
priceModel: model,
|
|
214
|
+
};
|
|
215
|
+
try {
|
|
216
|
+
ledger.record(entry);
|
|
217
|
+
} catch (err) {
|
|
218
|
+
log.debug("digest ledger record failed", { error: err instanceof Error ? err.message : String(err) });
|
|
219
|
+
}
|
|
220
|
+
if (error !== null) return { digested: false, reason: `digest model failed: ${error}` };
|
|
221
|
+
if (text === "" || text.length >= inputBytes * 0.9) return { digested: false, reason: "digest did not shrink the output" };
|
|
222
|
+
return {
|
|
223
|
+
digested: true,
|
|
224
|
+
text: `${digestMarker(req.toolName, req.input, model.slug, inputBytes, text.length)}\n${text}`,
|
|
225
|
+
model: model.slug,
|
|
226
|
+
usd,
|
|
227
|
+
inputBytes,
|
|
228
|
+
outputChars: text.length,
|
|
229
|
+
ms,
|
|
230
|
+
};
|
|
231
|
+
},
|
|
232
|
+
};
|
|
233
|
+
}
|
package/src/server/http.ts
CHANGED
|
@@ -6,6 +6,7 @@ import { createBridgeFromConfig } from "../context/index.ts";
|
|
|
6
6
|
import { createFeedbackStore, type Verdict } from "../cost/feedback.ts";
|
|
7
7
|
import { createLedger } from "../cost/ledger.ts";
|
|
8
8
|
import { createSessionOverrides } from "./overrides.ts";
|
|
9
|
+
import { createDigester } from "./digest.ts";
|
|
9
10
|
import { TIER_ORDER, type Tier } from "../router/types.ts";
|
|
10
11
|
import { baselinePrices, buildUsageReport } from "../cost/report.ts";
|
|
11
12
|
import type { Ledger, ModelTrust } from "../cost/types.ts";
|
|
@@ -194,6 +195,7 @@ export function startServer(cfg: RouterConfig): StartedServer {
|
|
|
194
195
|
const context = createBridgeFromConfig(cfg, db);
|
|
195
196
|
const overrides = createSessionOverrides();
|
|
196
197
|
const feedback = createFeedbackStore(db);
|
|
198
|
+
const digester = createDigester({ cfg, catalog, ledger, upstream, log });
|
|
197
199
|
const turnDeps = { config: cfg, router, upstream, ledger, conversations, catalog, context, overrides, ollamaCostScale };
|
|
198
200
|
|
|
199
201
|
// Hot reload: ranking knobs (tiers, filters, escalation, budgets, …) take
|
|
@@ -436,6 +438,26 @@ export function startServer(cfg: RouterConfig): StartedServer {
|
|
|
436
438
|
return json({ override: set });
|
|
437
439
|
}
|
|
438
440
|
}
|
|
441
|
+
if (req.method === "GET" && url.pathname === "/v1/router/digest/policy") {
|
|
442
|
+
const d = cfg.digest;
|
|
443
|
+
return json({ enabled: d.enabled, minBytes: d.minBytes, maxBytes: d.maxBytes, tools: d.tools, fromTier: d.fromTier });
|
|
444
|
+
}
|
|
445
|
+
if (req.method === "POST" && url.pathname === "/v1/router/digest") {
|
|
446
|
+
const body = (await req.json().catch(() => null)) as Record<string, unknown> | null;
|
|
447
|
+
if (body === null || typeof body.content !== "string" || typeof body.toolName !== "string") {
|
|
448
|
+
return wireErrorResponse({ status: 400, code: "invalid_request_error", message: "toolName and content required" });
|
|
449
|
+
}
|
|
450
|
+
return json(
|
|
451
|
+
await digester.digest({
|
|
452
|
+
ompSessionId: typeof body.ompSessionId === "string" ? body.ompSessionId : "",
|
|
453
|
+
harnessId: typeof body.harnessId === "string" ? body.harnessId : "",
|
|
454
|
+
toolName: body.toolName,
|
|
455
|
+
input: typeof body.input === "object" && body.input !== null ? (body.input as Record<string, unknown>) : {},
|
|
456
|
+
content: body.content,
|
|
457
|
+
query: typeof body.query === "string" ? body.query : "",
|
|
458
|
+
}),
|
|
459
|
+
);
|
|
460
|
+
}
|
|
439
461
|
if (req.method === "POST" && url.pathname === "/v1/router/feedback") {
|
|
440
462
|
// A user verdict on the newest routed turn of an omp session.
|
|
441
463
|
const body = (await req.json().catch(() => null)) as Record<string, unknown> | null;
|
package/test/controls.test.ts
CHANGED
|
@@ -236,3 +236,37 @@ describe("subagent profile", () => {
|
|
|
236
236
|
expect(resolveProfile(cfg, "auto", true).id).toBe("auto");
|
|
237
237
|
});
|
|
238
238
|
});
|
|
239
|
+
|
|
240
|
+
describe("digest endpoints", () => {
|
|
241
|
+
let handle: StartedServer;
|
|
242
|
+
let baseUrl = "";
|
|
243
|
+
beforeAll(() => {
|
|
244
|
+
const cfg: RouterConfig = {
|
|
245
|
+
...structuredClone(DEFAULT_CONFIG),
|
|
246
|
+
server: { host: "127.0.0.1", port: 0, maxConcurrentTurns: 24, subagentProfile: "auto-sub" },
|
|
247
|
+
ledger: { ...DEFAULT_CONFIG.ledger, path: ":memory:" },
|
|
248
|
+
logLevel: "silent",
|
|
249
|
+
};
|
|
250
|
+
cfg.digest = { ...cfg.digest, enabled: true, minBytes: 10 };
|
|
251
|
+
handle = startServer(cfg);
|
|
252
|
+
baseUrl = `http://127.0.0.1:${handle.server.port}`;
|
|
253
|
+
});
|
|
254
|
+
afterAll(async () => {
|
|
255
|
+
await handle.stop();
|
|
256
|
+
});
|
|
257
|
+
|
|
258
|
+
test("policy reflects the config; a digest for a session with no turns is declined, a bad body rejected", async () => {
|
|
259
|
+
const policy = (await (await fetch(`${baseUrl}/v1/router/digest/policy`)).json()) as { enabled: boolean; minBytes: number; tools: string[] };
|
|
260
|
+
expect(policy.enabled).toBe(true);
|
|
261
|
+
expect(policy.minBytes).toBe(10);
|
|
262
|
+
expect(policy.tools).toContain("read");
|
|
263
|
+
const bad = await fetch(`${baseUrl}/v1/router/digest`, { method: "POST", headers: { "content-type": "application/json" }, body: JSON.stringify({ toolName: "read" }) });
|
|
264
|
+
expect(bad.status).toBe(400);
|
|
265
|
+
const res = await fetch(`${baseUrl}/v1/router/digest`, {
|
|
266
|
+
method: "POST",
|
|
267
|
+
headers: { "content-type": "application/json" },
|
|
268
|
+
body: JSON.stringify({ ompSessionId: "never", toolName: "read", input: {}, content: "x".repeat(100), query: "q" }),
|
|
269
|
+
});
|
|
270
|
+
expect((await res.json()) as unknown).toMatchObject({ digested: false, reason: "no routed turn in this session yet" });
|
|
271
|
+
});
|
|
272
|
+
});
|
|
@@ -0,0 +1,207 @@
|
|
|
1
|
+
import { describe, expect, test } from "bun:test";
|
|
2
|
+
|
|
3
|
+
import { normalizeCatalogModel } from "../src/catalog/openrouter-catalog.ts";
|
|
4
|
+
import type { CatalogModel, CatalogSnapshot, CatalogSource } from "../src/catalog/types.ts";
|
|
5
|
+
import { DEFAULT_CONFIG } from "../src/config/defaults.ts";
|
|
6
|
+
import type { RouterConfig } from "../src/config/types.ts";
|
|
7
|
+
import { createLedger } from "../src/cost/ledger.ts";
|
|
8
|
+
import type { LedgerEntry } from "../src/cost/types.ts";
|
|
9
|
+
import { createDigester, digestApplies, digestMarker } from "../src/server/digest.ts";
|
|
10
|
+
import type { UpstreamClient } from "../src/upstream/types.ts";
|
|
11
|
+
import { createLogger } from "../src/util/log.ts";
|
|
12
|
+
import { openDb } from "../src/util/sqlite.ts";
|
|
13
|
+
import { digestToast, parsePolicy, shouldSend, textOf } from "../omp-extension/digest-logic.ts";
|
|
14
|
+
|
|
15
|
+
/**
|
|
16
|
+
* The tool-result digest: gates (tool, size, error, session tier, cost),
|
|
17
|
+
* model choice from the cheap tier, the marker that keeps the full output
|
|
18
|
+
* reachable, the ledger row every digest leaves, and the extension's
|
|
19
|
+
* client-side checks.
|
|
20
|
+
*/
|
|
21
|
+
|
|
22
|
+
const FIXTURE = (await Bun.file("test/fixtures/openrouter-models.json").json()) as { data: unknown[] };
|
|
23
|
+
const MODELS: CatalogModel[] = FIXTURE.data.map(normalizeCatalogModel).filter((m): m is CatalogModel => m !== null);
|
|
24
|
+
const SNAPSHOT: CatalogSnapshot = { models: MODELS, fetchedAtMs: Date.now() };
|
|
25
|
+
const catalog: CatalogSource = {
|
|
26
|
+
get: async () => SNAPSHOT,
|
|
27
|
+
refresh: async () => SNAPSHOT,
|
|
28
|
+
peek: () => SNAPSHOT,
|
|
29
|
+
find: (slug) => MODELS.find((m) => m.slug === slug),
|
|
30
|
+
};
|
|
31
|
+
const log = createLogger("silent");
|
|
32
|
+
|
|
33
|
+
function cfgWith(over: Partial<RouterConfig["digest"]> = {}): RouterConfig {
|
|
34
|
+
const cfg = structuredClone(DEFAULT_CONFIG);
|
|
35
|
+
cfg.ledger.path = ":memory:";
|
|
36
|
+
cfg.digest = { ...cfg.digest, enabled: true, minBytes: 100, ...over };
|
|
37
|
+
return cfg;
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
function seedSession(ledger: ReturnType<typeof createLedger>, tier: string): void {
|
|
41
|
+
const e: LedgerEntry = {
|
|
42
|
+
id: crypto.randomUUID(),
|
|
43
|
+
createdAtMs: Date.now(),
|
|
44
|
+
conversationKey: "k",
|
|
45
|
+
sessionId: "s",
|
|
46
|
+
turn: 3,
|
|
47
|
+
requestedModel: "auto",
|
|
48
|
+
harnessId: "",
|
|
49
|
+
ompSessionId: "omp-1",
|
|
50
|
+
slug: "x/y",
|
|
51
|
+
servedSlug: "x/y",
|
|
52
|
+
tier,
|
|
53
|
+
classificationSource: "heuristic",
|
|
54
|
+
reasons: [],
|
|
55
|
+
features: null,
|
|
56
|
+
score: null,
|
|
57
|
+
confidence: null,
|
|
58
|
+
task: null,
|
|
59
|
+
classifierReasons: null,
|
|
60
|
+
exploredFrom: null,
|
|
61
|
+
holdArm: null,
|
|
62
|
+
predictedUsd: 0.01,
|
|
63
|
+
reportedUsd: 0.01,
|
|
64
|
+
usage: { promptTokens: 100, cachedTokens: 0, cacheWriteTokens: 0, completionTokens: 10, reasoningTokens: 0, images: 0 },
|
|
65
|
+
attempt: 0,
|
|
66
|
+
escalationSignal: null,
|
|
67
|
+
latencyMs: 100,
|
|
68
|
+
ttftMs: 50,
|
|
69
|
+
finishReason: "stop",
|
|
70
|
+
wasted: false,
|
|
71
|
+
upstreamGenerationId: null,
|
|
72
|
+
error: null,
|
|
73
|
+
promptTokensSaved: 0,
|
|
74
|
+
};
|
|
75
|
+
ledger.record(e);
|
|
76
|
+
}
|
|
77
|
+
|
|
78
|
+
function fakeUpstream(reply: (body: Record<string, unknown>) => string, costUsd: number | null = 0.0004): { upstream: UpstreamClient; calls: Record<string, unknown>[] } {
|
|
79
|
+
const calls: Record<string, unknown>[] = [];
|
|
80
|
+
return {
|
|
81
|
+
calls,
|
|
82
|
+
upstream: {
|
|
83
|
+
dispatch: () => Promise.reject(new Error("not used")),
|
|
84
|
+
complete: async (body) => {
|
|
85
|
+
calls.push(body);
|
|
86
|
+
return { text: reply(body), costUsd };
|
|
87
|
+
},
|
|
88
|
+
fetchModels: () => Promise.resolve([]),
|
|
89
|
+
fetchModelsForUser: () => Promise.resolve([]),
|
|
90
|
+
},
|
|
91
|
+
};
|
|
92
|
+
}
|
|
93
|
+
|
|
94
|
+
const BIG = Array.from({ length: 400 }, (_, i) => `${i + 1}: export const value${i} = ${i};`).join("\n");
|
|
95
|
+
|
|
96
|
+
describe("digestApplies", () => {
|
|
97
|
+
const d = { ...DEFAULT_CONFIG.digest, enabled: true, minBytes: 100, maxBytes: 1000 };
|
|
98
|
+
test("gates on switch, error, tool, size and session tier", () => {
|
|
99
|
+
expect(digestApplies({ ...d, enabled: false }, "read", 500, false, "hard").ok).toBe(false);
|
|
100
|
+
expect(digestApplies(d, "read", 500, true, "hard").ok).toBe(false);
|
|
101
|
+
expect(digestApplies(d, "edit", 500, false, "hard").ok).toBe(false);
|
|
102
|
+
expect(digestApplies(d, "read", 50, false, "hard").ok).toBe(false);
|
|
103
|
+
expect(digestApplies(d, "read", 5000, false, "hard").ok).toBe(false);
|
|
104
|
+
expect(digestApplies(d, "read", 500, false, null).ok).toBe(false);
|
|
105
|
+
expect(digestApplies(d, "read", 500, false, "simple").ok).toBe(false); // below fromTier moderate
|
|
106
|
+
expect(digestApplies(d, "read", 500, false, "moderate").ok).toBe(true);
|
|
107
|
+
expect(digestApplies(d, "READ", 500, false, "hard").ok).toBe(true);
|
|
108
|
+
});
|
|
109
|
+
});
|
|
110
|
+
|
|
111
|
+
describe("createDigester", () => {
|
|
112
|
+
test("condenses a large read for a hard-tier session on a cheap model and records a ledger row", async () => {
|
|
113
|
+
const cfg = cfgWith();
|
|
114
|
+
const db = openDb(":memory:");
|
|
115
|
+
const ledger = createLedger(db, cfg);
|
|
116
|
+
seedSession(ledger, "hard");
|
|
117
|
+
const { upstream, calls } = fakeUpstream(() => "Omitted 380 trivial constants.\n1: export const value0 = 0;\n...");
|
|
118
|
+
const d = createDigester({ cfg, catalog, ledger, upstream, log });
|
|
119
|
+
const r = await d.digest({ ompSessionId: "omp-1", harnessId: "", toolName: "read", input: { path: "src/values.ts" }, content: BIG, query: "find value0" });
|
|
120
|
+
expect(r.digested).toBe(true);
|
|
121
|
+
if (!r.digested) return;
|
|
122
|
+
expect(r.text.startsWith("[digest: read output")).toBe(true);
|
|
123
|
+
expect(r.text).toContain("re-run read {\"path\":\"src/values.ts\"} (offset/limit for a range)");
|
|
124
|
+
expect(r.text).toContain("Omitted 380 trivial constants.");
|
|
125
|
+
expect(r.usd).toBeCloseTo(0.0004, 6);
|
|
126
|
+
// The cheap tier picked the model; the call carried the task and the raw output.
|
|
127
|
+
const call = calls[0]!;
|
|
128
|
+
expect(typeof call.model).toBe("string");
|
|
129
|
+
expect(catalog.find(call.model as string)?.price.prompt).toBeLessThanOrEqual(cfg.tiers.simple.maxInputPerMtok! / 1e6);
|
|
130
|
+
expect(JSON.stringify(call.messages)).toContain("Task: find value0");
|
|
131
|
+
// A ledger row under requestedModel "digest" with the served model and its cost.
|
|
132
|
+
const rows = ledger.recentEntries(10).filter((e) => e.requestedModel === "digest");
|
|
133
|
+
expect(rows).toHaveLength(1);
|
|
134
|
+
expect(rows[0]!.slug).toBe(call.model as string);
|
|
135
|
+
expect(rows[0]!.reportedUsd).toBeCloseTo(0.0004, 6);
|
|
136
|
+
expect(rows[0]!.ompSessionId).toBe("omp-1");
|
|
137
|
+
db.close();
|
|
138
|
+
});
|
|
139
|
+
|
|
140
|
+
test("declines below the session tier, over the cost guard, when the model fails, or when nothing shrinks", async () => {
|
|
141
|
+
const db = openDb(":memory:");
|
|
142
|
+
const cfg = cfgWith();
|
|
143
|
+
const ledger = createLedger(db, cfg);
|
|
144
|
+
seedSession(ledger, "simple");
|
|
145
|
+
const cheap = createDigester({ cfg, catalog, ledger, upstream: fakeUpstream(() => "short").upstream, log });
|
|
146
|
+
expect(await cheap.digest({ ompSessionId: "omp-1", harnessId: "", toolName: "read", input: {}, content: BIG, query: "" })).toMatchObject({ digested: false, reason: expect.stringContaining("below digest.fromTier") });
|
|
147
|
+
|
|
148
|
+
const db2 = openDb(":memory:");
|
|
149
|
+
const strict = cfgWith({ maxCostUsd: 0 });
|
|
150
|
+
const ledger2 = createLedger(db2, strict);
|
|
151
|
+
seedSession(ledger2, "hard");
|
|
152
|
+
expect(await createDigester({ cfg: strict, catalog, ledger: ledger2, upstream: fakeUpstream(() => "x").upstream, log }).digest({ ompSessionId: "omp-1", harnessId: "", toolName: "read", input: {}, content: BIG, query: "" })).toMatchObject({ digested: false, reason: expect.stringContaining("exceeds digest.maxCostUsd") });
|
|
153
|
+
|
|
154
|
+
const failing: UpstreamClient = { ...fakeUpstream(() => "x").upstream, complete: () => Promise.reject(new Error("boom")) };
|
|
155
|
+
expect(await createDigester({ cfg, catalog, ledger: ledger2, upstream: failing, log }).digest({ ompSessionId: "omp-1", harnessId: "", toolName: "read", input: {}, content: BIG, query: "" })).toMatchObject({ digested: false, reason: "digest model failed: boom" });
|
|
156
|
+
// The failed attempt is still a ledger row, with the error.
|
|
157
|
+
expect(ledger2.recentEntries(5).find((e) => e.requestedModel === "digest")?.error).toBe("boom");
|
|
158
|
+
|
|
159
|
+
const same = createDigester({ cfg, catalog, ledger: ledger2, upstream: fakeUpstream(() => BIG).upstream, log });
|
|
160
|
+
expect(await same.digest({ ompSessionId: "omp-1", harnessId: "", toolName: "read", input: {}, content: BIG, query: "" })).toMatchObject({ digested: false, reason: "digest did not shrink the output" });
|
|
161
|
+
db.close();
|
|
162
|
+
db2.close();
|
|
163
|
+
});
|
|
164
|
+
|
|
165
|
+
test("a pinned digest model is used as-is", async () => {
|
|
166
|
+
const pinned = MODELS.find((m) => m.price.prompt > 0)!.slug;
|
|
167
|
+
const cfg = cfgWith({ model: pinned });
|
|
168
|
+
const db = openDb(":memory:");
|
|
169
|
+
const ledger = createLedger(db, cfg);
|
|
170
|
+
seedSession(ledger, "hard");
|
|
171
|
+
const { upstream, calls } = fakeUpstream(() => "digest");
|
|
172
|
+
await createDigester({ cfg, catalog, ledger, upstream, log }).digest({ ompSessionId: "omp-1", harnessId: "", toolName: "grep", input: { pattern: "x" }, content: BIG, query: "" });
|
|
173
|
+
expect(calls[0]?.model as string).toBe(pinned);
|
|
174
|
+
db.close();
|
|
175
|
+
});
|
|
176
|
+
});
|
|
177
|
+
|
|
178
|
+
describe("digest marker and extension logic", () => {
|
|
179
|
+
test("the marker names the tool, sizes, model and how to get the full output", () => {
|
|
180
|
+
expect(digestMarker("grep", { pattern: "retry" }, "z-ai/glm-5.3-flash", 48_000, 3_000)).toBe(
|
|
181
|
+
'[digest: grep output 48,000 bytes → 3,000 chars by z-ai/glm-5.3-flash. Full output: re-run grep {"pattern":"retry"}]',
|
|
182
|
+
);
|
|
183
|
+
});
|
|
184
|
+
|
|
185
|
+
test("textOf joins text parts and flags images", () => {
|
|
186
|
+
expect(textOf([{ type: "text", text: "a" }, { type: "text", text: "b" }])).toEqual({ text: "a\nb", hasImage: false });
|
|
187
|
+
expect(textOf([{ type: "image" }, { type: "text", text: "a" }])).toEqual({ text: "a", hasImage: true });
|
|
188
|
+
});
|
|
189
|
+
|
|
190
|
+
test("shouldSend applies the client-side gate; parsePolicy is defensive", () => {
|
|
191
|
+
const p = parsePolicy({ enabled: true, minBytes: 10, maxBytes: 100, tools: ["Read", "grep"], fromTier: "hard" });
|
|
192
|
+
expect(p.tools).toEqual(["read", "grep"]);
|
|
193
|
+
expect(shouldSend(p, "read", false, "x".repeat(50), false)).toBe(true);
|
|
194
|
+
expect(shouldSend(p, "read", true, "x".repeat(50), false)).toBe(false);
|
|
195
|
+
expect(shouldSend(p, "read", false, "x".repeat(50), true)).toBe(false);
|
|
196
|
+
expect(shouldSend(p, "edit", false, "x".repeat(50), false)).toBe(false);
|
|
197
|
+
expect(shouldSend(p, "read", false, "x".repeat(5), false)).toBe(false);
|
|
198
|
+
expect(shouldSend(p, "read", false, "x".repeat(500), false)).toBe(false);
|
|
199
|
+
expect(parsePolicy({ enabled: false }).enabled).toBe(false);
|
|
200
|
+
expect(parsePolicy("nope").enabled).toBe(false);
|
|
201
|
+
expect(parsePolicy({ enabled: true }).minBytes).toBe(12_000);
|
|
202
|
+
});
|
|
203
|
+
|
|
204
|
+
test("digestToast is one readable line", () => {
|
|
205
|
+
expect(digestToast("read", 48 * 1024, 3 * 1024, "ollama/glm-5.3-flash", 0.00042)).toBe("digested read 48KB → 3KB via glm-5.3-flash ($0.0004)");
|
|
206
|
+
});
|
|
207
|
+
});
|
|
@@ -29,7 +29,7 @@ import type { ExtensionAPI, ExtensionContext, ProviderRegistration } from "@oh-m
|
|
|
29
29
|
*/
|
|
30
30
|
|
|
31
31
|
const registrations: { id: string; baseUrl: string }[] = [];
|
|
32
|
-
const handlers = new Map<string, ((event: unknown, ctx: ExtensionContext) =>
|
|
32
|
+
const handlers = new Map<string, ((event: unknown, ctx: ExtensionContext) => unknown)[]>();
|
|
33
33
|
|
|
34
34
|
const pi: ExtensionAPI = {
|
|
35
35
|
setLabel: () => {},
|
package/test/failover.test.ts
CHANGED
|
@@ -77,6 +77,7 @@ function mkConfig(escalation: Partial<EscalationConfig> = {}): RouterConfig {
|
|
|
77
77
|
compaction: { enabled: false, budgetTokens: 40_000, floorRatio: 1, fitToWindow: true, protectRecentTurns: 4, maxToolResultBytes: 4_096, keepHeadBytes: 512, keepTailBytes: 512, elideSupersededReads: true, collapseDuplicateResults: true, replanGrowthRatio: 1 },
|
|
78
78
|
budget: { onExceeded: "downgrade" },
|
|
79
79
|
report: { baselines: [] },
|
|
80
|
+
digest: { enabled: false, minBytes: 12_000, maxBytes: 400_000, tools: ["read"], fromTier: "moderate", tier: "simple", model: "", maxOutputTokens: 700, maxCostUsd: 0.02, timeoutMs: 25_000 },
|
|
80
81
|
profiles: [],
|
|
81
82
|
ledger: { path: ":memory:", blendWindowDays: 7, blendMinSamples: 20, fallbackBlend: { inputPerMtok: 1, outputPerMtok: 4 }, conversationTtlMs: 86_400_000 },
|
|
82
83
|
adaptiveTierFloors: true,
|
package/test/report-hub.test.ts
CHANGED
|
@@ -78,6 +78,9 @@ function report(over: Partial<UsageReport> = {}): UsageReport {
|
|
|
78
78
|
cacheEstimated: false,
|
|
79
79
|
subagentDispatches: 0,
|
|
80
80
|
subagentSpendUsd: 0,
|
|
81
|
+
digests: 0,
|
|
82
|
+
digestSpendUsd: 0,
|
|
83
|
+
digestInputTokens: 0,
|
|
81
84
|
},
|
|
82
85
|
providers: [row("openrouter", 2), row("ollama", 1)],
|
|
83
86
|
models: [
|
package/test/report.test.ts
CHANGED
|
@@ -227,6 +227,9 @@ describe("buildUsageReport", () => {
|
|
|
227
227
|
cacheEstimated: false,
|
|
228
228
|
subagentDispatches: 0,
|
|
229
229
|
subagentSpendUsd: 0,
|
|
230
|
+
digests: 0,
|
|
231
|
+
digestSpendUsd: 0,
|
|
232
|
+
digestInputTokens: 0,
|
|
230
233
|
});
|
|
231
234
|
expect(r.providers).toEqual([]);
|
|
232
235
|
expect(r.models).toEqual([]);
|
package/test/turn.test.ts
CHANGED
|
@@ -77,6 +77,7 @@ function mkConfig(escalation: Partial<EscalationConfig> = {}): RouterConfig {
|
|
|
77
77
|
compaction: { enabled: false, budgetTokens: 40_000, floorRatio: 1, fitToWindow: true, protectRecentTurns: 4, maxToolResultBytes: 4_096, keepHeadBytes: 512, keepTailBytes: 512, elideSupersededReads: true, collapseDuplicateResults: true, replanGrowthRatio: 1 },
|
|
78
78
|
budget: { onExceeded: "downgrade" },
|
|
79
79
|
report: { baselines: [] },
|
|
80
|
+
digest: { enabled: false, minBytes: 12_000, maxBytes: 400_000, tools: ["read"], fromTier: "moderate", tier: "simple", model: "", maxOutputTokens: 700, maxCostUsd: 0.02, timeoutMs: 25_000 },
|
|
80
81
|
profiles: [],
|
|
81
82
|
ledger: { path: ":memory:", blendWindowDays: 7, blendMinSamples: 20, fallbackBlend: { inputPerMtok: 1, outputPerMtok: 4 }, conversationTtlMs: 86_400_000 },
|
|
82
83
|
adaptiveTierFloors: true,
|