auto-model-router 0.4.0 → 0.4.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.omp-plugin/marketplace.json +2 -2
- package/README.md +32 -2
- package/omp-extension/digest-logic.ts +53 -0
- package/omp-extension/pi-coding-agent.d.ts +14 -1
- package/omp-extension/router-digest.ts +93 -0
- package/omp-extension/router-embed.ts +9 -6
- package/package.json +1 -1
- package/src/cli/config-wizard.ts +18 -0
- package/src/config/defaults.ts +21 -0
- package/src/config/schema.ts +17 -0
- package/src/config/types.ts +51 -0
- package/src/cost/ledger.ts +39 -6
- package/src/cost/report.ts +28 -0
- package/src/cost/types.ts +4 -1
- package/src/router/classify.ts +3 -0
- package/src/router/features.ts +39 -0
- package/src/router/index.ts +11 -4
- package/src/router/types.ts +8 -0
- package/src/server/digest.ts +233 -0
- package/src/server/http.ts +22 -0
- package/src/wire/openai/request.ts +4 -0
- package/src/wire/types.ts +2 -0
- package/test/classify.test.ts +13 -0
- package/test/config-wizard.test.ts +6 -5
- package/test/controls.test.ts +50 -1
- package/test/digest.test.ts +207 -0
- package/test/embed-lifecycle.test.ts +1 -1
- package/test/escalate.test.ts +1 -0
- package/test/failover.test.ts +5 -3
- package/test/features.test.ts +32 -0
- package/test/http-resilience.test.ts +1 -1
- package/test/report-hub.test.ts +5 -0
- package/test/report.test.ts +17 -0
- package/test/trust-attribution.test.ts +53 -0
- package/test/turn.test.ts +5 -3
- package/tools/replay.ts +1 -0
package/src/cost/ledger.ts
CHANGED
|
@@ -94,6 +94,15 @@ interface TrustRow {
|
|
|
94
94
|
mean_cost_error: number | null;
|
|
95
95
|
}
|
|
96
96
|
|
|
97
|
+
interface FeedbackRow {
|
|
98
|
+
good: number | null;
|
|
99
|
+
bad: number | null;
|
|
100
|
+
}
|
|
101
|
+
|
|
102
|
+
/** Verdict counts per served slug since a cutoff (optionally one harness). */
|
|
103
|
+
const FEEDBACK_SELECT = `COALESCE(SUM(CASE WHEN f.verdict = 'good' THEN 1 ELSE 0 END), 0) AS good,
|
|
104
|
+
COALESCE(SUM(CASE WHEN f.verdict = 'bad' THEN 1 ELSE 0 END), 0) AS bad`;
|
|
105
|
+
|
|
97
106
|
interface LatencyRow {
|
|
98
107
|
samples: number;
|
|
99
108
|
ttft_ms: number | null;
|
|
@@ -186,15 +195,23 @@ function errorKindOf(error: string | null): string | null {
|
|
|
186
195
|
return error.slice(0, sep);
|
|
187
196
|
}
|
|
188
197
|
|
|
189
|
-
function toTrust(slug: string, row: TrustRow): ModelTrust {
|
|
198
|
+
function toTrust(slug: string, row: TrustRow, fb: FeedbackRow | null = null, feedbackWeight = 0): ModelTrust {
|
|
190
199
|
// Laplace smoothing: an untried model scores a neutral 1/2, and a failure
|
|
191
200
|
// is an attempt superseded by an escalation or ended in an upstream error.
|
|
201
|
+
// A user verdict counts as feedbackWeight extra attempts of that outcome.
|
|
202
|
+
const good = fb?.good ?? 0;
|
|
203
|
+
const bad = fb?.bad ?? 0;
|
|
204
|
+
const w = feedbackWeight > 0 ? feedbackWeight : 0;
|
|
205
|
+
const attempts = row.attempts + w * (good + bad);
|
|
206
|
+
const failures = row.failures + w * bad;
|
|
192
207
|
return {
|
|
193
208
|
slug,
|
|
194
209
|
attempts: row.attempts,
|
|
195
210
|
escalations: row.escalations,
|
|
196
211
|
errors: row.errors,
|
|
197
|
-
|
|
212
|
+
feedbackGood: good,
|
|
213
|
+
feedbackBad: bad,
|
|
214
|
+
successRate: (attempts - failures + 1) / (attempts + 2),
|
|
198
215
|
meanCostError: row.mean_cost_error ?? 0,
|
|
199
216
|
};
|
|
200
217
|
}
|
|
@@ -312,6 +329,17 @@ export function createLedger(db: Database, cfg: RouterConfig): Ledger {
|
|
|
312
329
|
const trustStmt = db.query(`SELECT ${TRUST_SELECT} FROM ledger WHERE slug = ? AND created_at_ms > ?`);
|
|
313
330
|
const trustHarnessStmt = db.query(`SELECT ${TRUST_SELECT} FROM ledger WHERE slug = ? AND harness_id = ? AND created_at_ms > ?`);
|
|
314
331
|
const allTrustStmt = db.query(`SELECT slug, ${TRUST_SELECT} FROM ledger WHERE created_at_ms > ? GROUP BY slug`);
|
|
332
|
+
const feedbackStmt = db.query(`SELECT ${FEEDBACK_SELECT} FROM feedback f WHERE f.slug = ? AND f.created_at_ms > ?`);
|
|
333
|
+
const feedbackHarnessStmt = db.query(
|
|
334
|
+
`SELECT ${FEEDBACK_SELECT} FROM feedback f JOIN ledger l ON l.id = f.ledger_id WHERE f.slug = ? AND l.harness_id = ? AND f.created_at_ms > ?`,
|
|
335
|
+
);
|
|
336
|
+
const allFeedbackStmt = db.query(`SELECT f.slug, ${FEEDBACK_SELECT} FROM feedback f WHERE f.created_at_ms > ? GROUP BY f.slug`);
|
|
337
|
+
const feedbackFor = (slug: string, harnessId: string | undefined, cutoff: number): FeedbackRow | null => {
|
|
338
|
+
if (cfg.filters.feedbackWeight <= 0) return null;
|
|
339
|
+
return harnessId !== undefined && harnessId !== ""
|
|
340
|
+
? (feedbackHarnessStmt.get(slug, harnessId, cutoff) as FeedbackRow | null)
|
|
341
|
+
: (feedbackStmt.get(slug, cutoff) as FeedbackRow | null);
|
|
342
|
+
};
|
|
315
343
|
const latencyStmt = db.query(
|
|
316
344
|
`SELECT ${LATENCY_SELECT} FROM (SELECT * FROM ledger WHERE slug = ? ORDER BY created_at_ms DESC LIMIT ${LATENCY_WINDOW_ROWS})`,
|
|
317
345
|
);
|
|
@@ -323,7 +351,8 @@ export function createLedger(db: Database, cfg: RouterConfig): Ledger {
|
|
|
323
351
|
const providerSpendStmt = db.query(
|
|
324
352
|
"SELECT COALESCE(SUM(COALESCE(reported_usd, predicted_usd)), 0) AS total FROM ledger WHERE created_at_ms >= ? AND COALESCE(served_slug, slug) LIKE ?",
|
|
325
353
|
);
|
|
326
|
-
|
|
354
|
+
// Digest rows (requested_model 'digest') are side calls, not the session's turns.
|
|
355
|
+
const sessionStmt = db.query("SELECT * FROM ledger WHERE omp_session_id = ? AND wasted = 0 AND requested_model <> 'digest' ORDER BY created_at_ms DESC LIMIT ?");
|
|
327
356
|
// What an escalated retry actually bills, per prompt token, over a window.
|
|
328
357
|
// attempt > 0 rows are the re-dispatches that followed a rejected attempt;
|
|
329
358
|
// errored ones carry no usage and are excluded.
|
|
@@ -447,13 +476,17 @@ export function createLedger(db: Database, cfg: RouterConfig): Ledger {
|
|
|
447
476
|
? (trustHarnessStmt.get(slug, harnessId, cutoff) as TrustRow | null)
|
|
448
477
|
: (trustStmt.get(slug, cutoff) as TrustRow | null);
|
|
449
478
|
if (row === null || row.attempts === 0) return null;
|
|
450
|
-
return toTrust(slug, row);
|
|
479
|
+
return toTrust(slug, row, feedbackFor(slug, harnessId, cutoff), cfg.filters.feedbackWeight);
|
|
451
480
|
},
|
|
452
481
|
|
|
453
482
|
allTrust(): ModelTrust[] {
|
|
454
483
|
const cutoff = cfg.filters.trustWindowDays > 0 ? Date.now() - cfg.filters.trustWindowDays * DAY_MS : 0;
|
|
455
484
|
const rows = allTrustStmt.all(cutoff) as (TrustRow & { slug: string })[];
|
|
456
|
-
|
|
485
|
+
const fb = new Map<string, FeedbackRow>();
|
|
486
|
+
if (cfg.filters.feedbackWeight > 0) {
|
|
487
|
+
for (const r of allFeedbackStmt.all(cutoff) as (FeedbackRow & { slug: string })[]) fb.set(r.slug, r);
|
|
488
|
+
}
|
|
489
|
+
return rows.map((row) => toTrust(row.slug, row, fb.get(row.slug) ?? null, cfg.filters.feedbackWeight));
|
|
457
490
|
},
|
|
458
491
|
|
|
459
492
|
latency(slug: string, harnessId?: string): ModelLatency | null {
|
|
@@ -476,7 +509,7 @@ export function createLedger(db: Database, cfg: RouterConfig): Ledger {
|
|
|
476
509
|
? (latencyHarnessStmt.get(slug, harnessId) as LatencyRow | null)
|
|
477
510
|
: (latencyStmt.get(slug) as LatencyRow | null);
|
|
478
511
|
out.set(slug, {
|
|
479
|
-
trust: trustRow === null || trustRow.attempts === 0 ? null : toTrust(slug, trustRow),
|
|
512
|
+
trust: trustRow === null || trustRow.attempts === 0 ? null : toTrust(slug, trustRow, feedbackFor(slug, harnessId, cutoff), cfg.filters.feedbackWeight),
|
|
480
513
|
latency: latencyRow === null ? null : toLatency(slug, latencyRow),
|
|
481
514
|
});
|
|
482
515
|
}
|
package/src/cost/report.ts
CHANGED
|
@@ -27,6 +27,13 @@ export interface ReportTotals {
|
|
|
27
27
|
modelSwitches: number;
|
|
28
28
|
/** Any row in the window carries an estimated cache count. */
|
|
29
29
|
cacheEstimated: boolean;
|
|
30
|
+
/** Turns from omp subagents (`features.isSubagent`), and their spend. */
|
|
31
|
+
subagentDispatches: number;
|
|
32
|
+
subagentSpendUsd: number;
|
|
33
|
+
/** Tool-result digests (requestedModel "digest"): count, what they cost, bytes they condensed. */
|
|
34
|
+
digests: number;
|
|
35
|
+
digestSpendUsd: number;
|
|
36
|
+
digestInputTokens: number;
|
|
30
37
|
}
|
|
31
38
|
|
|
32
39
|
export interface ReportRow {
|
|
@@ -196,6 +203,11 @@ export function buildUsageReport(
|
|
|
196
203
|
COALESCE(SUM(${CT}), 0) AS cached_tokens,
|
|
197
204
|
COALESCE(SUM(${COMP}), 0) AS completion_tokens,
|
|
198
205
|
SUM(CASE WHEN ${EST} THEN 1 ELSE 0 END) AS estimated_rows,
|
|
206
|
+
SUM(CASE WHEN json_extract(features, '$.isSubagent') = 1 THEN 1 ELSE 0 END) AS subagent_rows,
|
|
207
|
+
COALESCE(SUM(CASE WHEN json_extract(features, '$.isSubagent') = 1 THEN ${USD} ELSE 0 END), 0) AS subagent_spend,
|
|
208
|
+
SUM(CASE WHEN requested_model = 'digest' THEN 1 ELSE 0 END) AS digests,
|
|
209
|
+
COALESCE(SUM(CASE WHEN requested_model = 'digest' THEN ${USD} ELSE 0 END), 0) AS digest_spend,
|
|
210
|
+
COALESCE(SUM(CASE WHEN requested_model = 'digest' THEN ${PT} ELSE 0 END), 0) AS digest_input,
|
|
199
211
|
SUM(CASE WHEN escalation_signal IS NOT NULL THEN 1 ELSE 0 END) AS escalations,
|
|
200
212
|
SUM(CASE WHEN instr(reasons, 'failover:') > 0 THEN 1 ELSE 0 END) AS failovers,
|
|
201
213
|
SUM(CASE WHEN error IS NOT NULL THEN 1 ELSE 0 END) AS errors,
|
|
@@ -210,6 +222,11 @@ export function buildUsageReport(
|
|
|
210
222
|
cached_tokens: number;
|
|
211
223
|
completion_tokens: number;
|
|
212
224
|
estimated_rows: number | null;
|
|
225
|
+
subagent_rows: number | null;
|
|
226
|
+
subagent_spend: number;
|
|
227
|
+
digests: number | null;
|
|
228
|
+
digest_spend: number;
|
|
229
|
+
digest_input: number;
|
|
213
230
|
escalations: number | null;
|
|
214
231
|
failovers: number | null;
|
|
215
232
|
errors: number | null;
|
|
@@ -327,6 +344,11 @@ export function buildUsageReport(
|
|
|
327
344
|
aborted: t.aborted ?? 0,
|
|
328
345
|
modelSwitches: switches,
|
|
329
346
|
cacheEstimated: (t.estimated_rows ?? 0) > 0,
|
|
347
|
+
subagentDispatches: t.subagent_rows ?? 0,
|
|
348
|
+
subagentSpendUsd: t.subagent_spend,
|
|
349
|
+
digests: t.digests ?? 0,
|
|
350
|
+
digestSpendUsd: t.digest_spend,
|
|
351
|
+
digestInputTokens: t.digest_input,
|
|
330
352
|
},
|
|
331
353
|
providers,
|
|
332
354
|
models,
|
|
@@ -391,6 +413,12 @@ export function reportView(r: UsageReport, opts: { maxModels?: number } = {}): R
|
|
|
391
413
|
.join(" · ")}`,
|
|
392
414
|
);
|
|
393
415
|
}
|
|
416
|
+
if (t.digests > 0) {
|
|
417
|
+
summary.push(`digests: ${num(t.digests)} tool results condensed (${num(t.digestInputTokens)} tok read by a cheap model) for ${usd(t.digestSpendUsd)}`);
|
|
418
|
+
}
|
|
419
|
+
if (t.subagentDispatches > 0) {
|
|
420
|
+
summary.push(`subagents: ${num(t.subagentDispatches)} dispatches, ${usd(t.subagentSpendUsd)} (${pct(t.spendUsd > 0 ? t.subagentSpendUsd / t.spendUsd : 0)} of spend)`);
|
|
421
|
+
}
|
|
394
422
|
const a = r.anatomy;
|
|
395
423
|
if (a !== null) {
|
|
396
424
|
summary.push(
|
package/src/cost/types.ts
CHANGED
|
@@ -177,8 +177,11 @@ export interface ModelTrust {
|
|
|
177
177
|
escalations: number;
|
|
178
178
|
/** Attempts that ended in an upstream error. */
|
|
179
179
|
errors: number;
|
|
180
|
-
/** Laplace-smoothed success rate, 0-1. */
|
|
180
|
+
/** Laplace-smoothed success rate, 0-1; user verdicts weigh in at filters.feedbackWeight. */
|
|
181
181
|
successRate: number;
|
|
182
|
+
/** User verdicts in the window (/router good|bad). */
|
|
183
|
+
feedbackGood?: number;
|
|
184
|
+
feedbackBad?: number;
|
|
182
185
|
/** Mean absolute relative prediction error, for forecast calibration. */
|
|
183
186
|
meanCostError: number;
|
|
184
187
|
}
|
package/src/router/classify.ts
CHANGED
|
@@ -166,6 +166,9 @@ export function scoreHeuristic(f: Features, cfg: RouterConfig): Classification {
|
|
|
166
166
|
// the served model must still accept image input — is enforced separately
|
|
167
167
|
// on `req.hasImages` in candidate selection, exactly as `classifyTask` does.
|
|
168
168
|
if (f.hasNewImage) add(W_IMAGES, "new image input");
|
|
169
|
+
if (f.readOnlyToolTail === true && cfg.classifier.readOnlyToolWeight > 0) {
|
|
170
|
+
add(-cfg.classifier.readOnlyToolWeight, "read-only tool loop (the model is looking, not deciding)");
|
|
171
|
+
}
|
|
169
172
|
if (f.toolCount > 0) add(W_TOOLS_OFFERED, `${f.toolCount} tools offered`);
|
|
170
173
|
|
|
171
174
|
score = Math.min(1, Math.max(0, score));
|
package/src/router/features.ts
CHANGED
|
@@ -242,6 +242,19 @@ export function extractFeatures(req: NormRequest, promptTokens: number): Feature
|
|
|
242
242
|
else anatomy.toolBytes += m.textBytes;
|
|
243
243
|
}
|
|
244
244
|
|
|
245
|
+
// Read-only tool loop: the assistant call behind a tool-result tail used
|
|
246
|
+
// only tools that look at things. Tool names are the harness's own; the
|
|
247
|
+
// set covers omp's built-ins and their common aliases.
|
|
248
|
+
let readOnlyToolTail = false;
|
|
249
|
+
if (isToolResultContinuation) {
|
|
250
|
+
for (let i = messages.length - 1; i >= 0; i--) {
|
|
251
|
+
const m = messages[i];
|
|
252
|
+
if (m === undefined || m.role !== "assistant") continue;
|
|
253
|
+
if (m.toolCalls.length > 0) readOnlyToolTail = m.toolCalls.every((tc) => READ_ONLY_TOOLS.has(tc.name.toLowerCase()));
|
|
254
|
+
break;
|
|
255
|
+
}
|
|
256
|
+
}
|
|
257
|
+
|
|
245
258
|
return {
|
|
246
259
|
promptTokens,
|
|
247
260
|
newContentTokens,
|
|
@@ -265,5 +278,31 @@ export function extractFeatures(req: NormRequest, promptTokens: number): Feature
|
|
|
265
278
|
questionCount,
|
|
266
279
|
isTerseInstruction,
|
|
267
280
|
anatomy,
|
|
281
|
+
isSubagent: req.isSubagent,
|
|
282
|
+
readOnlyToolTail,
|
|
268
283
|
};
|
|
269
284
|
}
|
|
285
|
+
|
|
286
|
+
/** Tools that read state without changing it, in omp, Claude Code and Hermes naming. */
|
|
287
|
+
export const READ_ONLY_TOOLS: ReadonlySet<string> = new Set([
|
|
288
|
+
"read",
|
|
289
|
+
"read_file",
|
|
290
|
+
"grep",
|
|
291
|
+
"glob",
|
|
292
|
+
"ls",
|
|
293
|
+
"list",
|
|
294
|
+
"list_dir",
|
|
295
|
+
"find",
|
|
296
|
+
"lsp",
|
|
297
|
+
"ast_grep",
|
|
298
|
+
"search",
|
|
299
|
+
"web_search",
|
|
300
|
+
"web_fetch",
|
|
301
|
+
"webfetch",
|
|
302
|
+
"websearch",
|
|
303
|
+
"fetch",
|
|
304
|
+
"cat",
|
|
305
|
+
"view",
|
|
306
|
+
"inspect_image",
|
|
307
|
+
"todo",
|
|
308
|
+
]);
|
package/src/router/index.ts
CHANGED
|
@@ -38,11 +38,18 @@ export interface RouterDeps {
|
|
|
38
38
|
*/
|
|
39
39
|
const NEUTRAL_TOKENIZER = "gpt";
|
|
40
40
|
|
|
41
|
-
function resolveProfile(cfg: RouterConfig, requestedModel: string): ProfileConfig {
|
|
42
|
-
const exact = cfg.profiles.find((p) => p.id === requestedModel);
|
|
43
|
-
if (exact !== undefined) return exact;
|
|
41
|
+
export function resolveProfile(cfg: RouterConfig, requestedModel: string, isSubagent = false): ProfileConfig {
|
|
44
42
|
const fallback = cfg.profiles[0];
|
|
45
43
|
if (fallback === undefined) throw new Error("no router profiles configured");
|
|
44
|
+
// A subagent asking for the default profile is routed under the subagent
|
|
45
|
+
// profile when one is configured and exists; an explicit other profile
|
|
46
|
+
// (auto-max, auto-cheap) is honoured as asked.
|
|
47
|
+
const exact = cfg.profiles.find((p) => p.id === requestedModel);
|
|
48
|
+
if (isSubagent && cfg.server.subagentProfile !== "" && (exact === undefined || exact.id === fallback.id)) {
|
|
49
|
+
const sub = cfg.profiles.find((p) => p.id === cfg.server.subagentProfile);
|
|
50
|
+
if (sub !== undefined) return sub;
|
|
51
|
+
}
|
|
52
|
+
if (exact !== undefined) return exact;
|
|
46
53
|
return fallback;
|
|
47
54
|
}
|
|
48
55
|
|
|
@@ -96,7 +103,7 @@ export function createRouter(deps: RouterDeps): Router {
|
|
|
96
103
|
req,
|
|
97
104
|
features,
|
|
98
105
|
classification,
|
|
99
|
-
profile: resolveProfile(config, req.requestedModel),
|
|
106
|
+
profile: resolveProfile(config, req.requestedModel, req.isSubagent),
|
|
100
107
|
state,
|
|
101
108
|
snapshot,
|
|
102
109
|
ledger,
|
package/src/router/types.ts
CHANGED
|
@@ -97,6 +97,14 @@ export interface Features {
|
|
|
97
97
|
* recorded before it existed.
|
|
98
98
|
*/
|
|
99
99
|
anatomy?: PromptAnatomy;
|
|
100
|
+
/** The request came from an omp subagent (`X-Omp-Subagent`). */
|
|
101
|
+
isSubagent?: boolean;
|
|
102
|
+
/**
|
|
103
|
+
* This is a tool-result continuation and the newest assistant turn issued
|
|
104
|
+
* only read-only tools (read, grep, glob, ls, lsp…): the model is looking,
|
|
105
|
+
* not deciding. Recorded for replay; scored at classifier.readOnlyToolWeight.
|
|
106
|
+
*/
|
|
107
|
+
readOnlyToolTail?: boolean;
|
|
100
108
|
}
|
|
101
109
|
|
|
102
110
|
/** Prompt bytes by message role and by age, plus the tool-schema bytes beside them. */
|
|
@@ -0,0 +1,233 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Tool-result digest: a cheap model condenses a large tool output before it
|
|
3
|
+
* reaches an expensive one.
|
|
4
|
+
*
|
|
5
|
+
* Prompt anatomy showed tool results are the bulk of every prompt, and a
|
|
6
|
+
* prompt is ~96% of spend. A 60KB file read on a hard-tier turn is re-read
|
|
7
|
+
* by that model on every later turn of the conversation, cached or not.
|
|
8
|
+
* When the omp extension sees a large read/grep/glob/bash result while the
|
|
9
|
+
* session's current model sits at or above `digest.fromTier`, it sends the
|
|
10
|
+
* text here; a simple-tier model rewrites it to what the task needs — exact
|
|
11
|
+
* paths, line numbers, names, errors, code that would be edited — and the
|
|
12
|
+
* digest replaces the tool result. The marker on top says how to get the
|
|
13
|
+
* full output back (re-run the tool, or read a line range), so nothing is
|
|
14
|
+
* lost, only deferred.
|
|
15
|
+
*
|
|
16
|
+
* Guarded: never on errors, never below `minBytes`, never above `maxBytes`,
|
|
17
|
+
* never past `maxCostUsd`, and every digest is a ledger row
|
|
18
|
+
* (requestedModel "digest") so the report shows what it cost and saved.
|
|
19
|
+
*/
|
|
20
|
+
|
|
21
|
+
import type { CatalogModel, CatalogSource } from "../catalog/types.ts";
|
|
22
|
+
import type { DigestConfig, RouterConfig } from "../config/types.ts";
|
|
23
|
+
import { computeCost, forecast } from "../cost/forecast.ts";
|
|
24
|
+
import type { Ledger, LedgerEntry } from "../cost/types.ts";
|
|
25
|
+
import { buildCandidates } from "../router/candidates.ts";
|
|
26
|
+
import { extractFeatures } from "../router/features.ts";
|
|
27
|
+
import { TIER_ORDER, type Tier } from "../router/types.ts";
|
|
28
|
+
import { estimateTokens } from "../tokens/estimate.ts";
|
|
29
|
+
import type { UpstreamClient } from "../upstream/types.ts";
|
|
30
|
+
import type { Logger } from "../util/log.ts";
|
|
31
|
+
import type { NormRequest } from "../wire/types.ts";
|
|
32
|
+
|
|
33
|
+
export interface DigestRequest {
|
|
34
|
+
ompSessionId: string;
|
|
35
|
+
harnessId: string;
|
|
36
|
+
toolName: string;
|
|
37
|
+
/** The tool's arguments, echoed into the marker so the model can re-run it. */
|
|
38
|
+
input: Record<string, unknown>;
|
|
39
|
+
content: string;
|
|
40
|
+
/** The user's current ask, so the digest keeps what matters for it. */
|
|
41
|
+
query: string;
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
export type DigestResult =
|
|
45
|
+
| { digested: true; text: string; model: string; usd: number; inputBytes: number; outputChars: number; ms: number }
|
|
46
|
+
| { digested: false; reason: string };
|
|
47
|
+
|
|
48
|
+
export interface DigesterDeps {
|
|
49
|
+
cfg: RouterConfig;
|
|
50
|
+
catalog: CatalogSource;
|
|
51
|
+
ledger: Ledger;
|
|
52
|
+
upstream: UpstreamClient;
|
|
53
|
+
log: Logger;
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
const DIGEST_SYSTEM = `You condense tool output for a coding agent that is mid-task. Keep everything the task could need: exact file paths, line numbers, identifiers, signatures, error text, counts and values. Quote verbatim, with line numbers, any code the agent is likely to edit or reference. Drop repetition, boilerplate, generated noise and unrelated regions. Never invent content. Plain text only, no preamble. First line: one sentence saying what was omitted and roughly how much.`;
|
|
57
|
+
|
|
58
|
+
const tierIdx = (t: string): number => TIER_ORDER.indexOf(t as Tier);
|
|
59
|
+
|
|
60
|
+
/** Whether a session's current model is expensive enough for a digest to pay off. */
|
|
61
|
+
export function digestApplies(cfg: DigestConfig, toolName: string, bytes: number, isError: boolean, currentTier: string | null): { ok: true } | { ok: false; reason: string } {
|
|
62
|
+
if (!cfg.enabled) return { ok: false, reason: "digest disabled" };
|
|
63
|
+
if (isError) return { ok: false, reason: "error results are never digested" };
|
|
64
|
+
if (!cfg.tools.includes(toolName.toLowerCase())) return { ok: false, reason: `tool ${toolName} not in digest.tools` };
|
|
65
|
+
if (bytes < cfg.minBytes) return { ok: false, reason: `${bytes} bytes < minBytes ${cfg.minBytes}` };
|
|
66
|
+
if (bytes > cfg.maxBytes) return { ok: false, reason: `${bytes} bytes > maxBytes ${cfg.maxBytes}` };
|
|
67
|
+
if (currentTier === null) return { ok: false, reason: "no routed turn in this session yet" };
|
|
68
|
+
if (tierIdx(currentTier) < tierIdx(cfg.fromTier)) return { ok: false, reason: `session is on ${currentTier}, below digest.fromTier ${cfg.fromTier}` };
|
|
69
|
+
return { ok: true };
|
|
70
|
+
}
|
|
71
|
+
|
|
72
|
+
/** The line that replaces the raw output's head: what happened and how to undo it. */
|
|
73
|
+
export function digestMarker(toolName: string, input: Record<string, unknown>, model: string, inputBytes: number, outputChars: number): string {
|
|
74
|
+
const args = JSON.stringify(input);
|
|
75
|
+
const shownArgs = args.length > 160 ? `${args.slice(0, 159)}…` : args;
|
|
76
|
+
return `[digest: ${toolName} output ${inputBytes.toLocaleString("en-US")} bytes → ${outputChars.toLocaleString("en-US")} chars by ${model}. Full output: re-run ${toolName} ${shownArgs}${toolName === "read" ? " (offset/limit for a range)" : ""}]`;
|
|
77
|
+
}
|
|
78
|
+
|
|
79
|
+
function syntheticRequest(req: DigestRequest, promptText: string): NormRequest {
|
|
80
|
+
const bytes = Buffer.byteLength(promptText);
|
|
81
|
+
return {
|
|
82
|
+
protocol: "openai-chat",
|
|
83
|
+
conversationKey: `digest:${req.ompSessionId}`,
|
|
84
|
+
harnessId: req.harnessId,
|
|
85
|
+
ompSessionId: req.ompSessionId,
|
|
86
|
+
agentdoxScope: "",
|
|
87
|
+
isSubagent: true,
|
|
88
|
+
requestedModel: "digest",
|
|
89
|
+
messages: [
|
|
90
|
+
{ role: "system", text: DIGEST_SYSTEM, images: 0, textBytes: Buffer.byteLength(DIGEST_SYSTEM), toolCalls: [] },
|
|
91
|
+
{ role: "user", text: promptText, images: 0, textBytes: bytes, toolCalls: [] },
|
|
92
|
+
],
|
|
93
|
+
tools: [],
|
|
94
|
+
forcedToolChoice: false,
|
|
95
|
+
stream: false,
|
|
96
|
+
hasImages: false,
|
|
97
|
+
promptBytes: bytes + Buffer.byteLength(DIGEST_SYSTEM),
|
|
98
|
+
renderUpstreamBody: () => ({}),
|
|
99
|
+
};
|
|
100
|
+
}
|
|
101
|
+
|
|
102
|
+
export function createDigester(deps: DigesterDeps): { digest(req: DigestRequest): Promise<DigestResult> } {
|
|
103
|
+
const { cfg, catalog, ledger, upstream, log } = deps;
|
|
104
|
+
|
|
105
|
+
/** Cheapest simple-tier model that fits the prompt, or the configured one. */
|
|
106
|
+
async function pickModel(req: NormRequest, promptTokens: number): Promise<CatalogModel | null> {
|
|
107
|
+
const snapshot = await catalog.get();
|
|
108
|
+
if (cfg.digest.model !== "") return snapshot.models.find((m) => m.slug === cfg.digest.model) ?? null;
|
|
109
|
+
const features = extractFeatures(req, promptTokens);
|
|
110
|
+
for (const relaxLevel of [0, 1, 2]) {
|
|
111
|
+
const built = buildCandidates({
|
|
112
|
+
req,
|
|
113
|
+
features,
|
|
114
|
+
tier: cfg.digest.tier,
|
|
115
|
+
task: "documentation",
|
|
116
|
+
snapshot,
|
|
117
|
+
ledger,
|
|
118
|
+
cfg,
|
|
119
|
+
expectedCompletionTokens: cfg.digest.maxOutputTokens,
|
|
120
|
+
warmSlug: null,
|
|
121
|
+
relaxLevel,
|
|
122
|
+
});
|
|
123
|
+
const first = built.candidates[0];
|
|
124
|
+
if (first !== undefined) return first.model;
|
|
125
|
+
}
|
|
126
|
+
return null;
|
|
127
|
+
}
|
|
128
|
+
|
|
129
|
+
return {
|
|
130
|
+
async digest(req) {
|
|
131
|
+
const inputBytes = Buffer.byteLength(req.content);
|
|
132
|
+
const currentTier = ledger.latestForSession?.(req.ompSessionId)?.tier ?? null;
|
|
133
|
+
const applies = digestApplies(cfg.digest, req.toolName, inputBytes, false, currentTier);
|
|
134
|
+
if (!applies.ok) return { digested: false, reason: applies.reason };
|
|
135
|
+
|
|
136
|
+
const promptText = `Task: ${req.query === "" ? "(unknown)" : req.query}\nTool: ${req.toolName} ${JSON.stringify(req.input)}\n--- output ---\n${req.content}`;
|
|
137
|
+
const synthetic = syntheticRequest(req, promptText);
|
|
138
|
+
const promptTokens = estimateTokens(synthetic.promptBytes, "unknown", ledger);
|
|
139
|
+
const model = await pickModel(synthetic, promptTokens);
|
|
140
|
+
if (model === null) return { digested: false, reason: "no digest model available" };
|
|
141
|
+
const est = forecast(model, { promptTokens, completionTokens: cfg.digest.maxOutputTokens, cacheHitRate: 0, images: 0 });
|
|
142
|
+
if (est.coldUsd > cfg.digest.maxCostUsd) {
|
|
143
|
+
return { digested: false, reason: `estimated $${est.coldUsd.toFixed(4)} on ${model.slug} exceeds digest.maxCostUsd $${cfg.digest.maxCostUsd}` };
|
|
144
|
+
}
|
|
145
|
+
|
|
146
|
+
const controller = new AbortController();
|
|
147
|
+
const timer = setTimeout(() => controller.abort(), cfg.digest.timeoutMs);
|
|
148
|
+
const startedAt = Date.now();
|
|
149
|
+
let text = "";
|
|
150
|
+
let costUsd: number | null = null;
|
|
151
|
+
let error: string | null = null;
|
|
152
|
+
try {
|
|
153
|
+
const out = await upstream.complete(
|
|
154
|
+
{
|
|
155
|
+
model: model.slug,
|
|
156
|
+
stream: false,
|
|
157
|
+
max_tokens: cfg.digest.maxOutputTokens,
|
|
158
|
+
temperature: 0,
|
|
159
|
+
messages: [
|
|
160
|
+
{ role: "system", content: DIGEST_SYSTEM },
|
|
161
|
+
{ role: "user", content: promptText },
|
|
162
|
+
],
|
|
163
|
+
},
|
|
164
|
+
controller.signal,
|
|
165
|
+
);
|
|
166
|
+
text = out.text.trim();
|
|
167
|
+
costUsd = out.costUsd;
|
|
168
|
+
} catch (err) {
|
|
169
|
+
error = err instanceof Error ? err.message : String(err);
|
|
170
|
+
} finally {
|
|
171
|
+
clearTimeout(timer);
|
|
172
|
+
}
|
|
173
|
+
const ms = Date.now() - startedAt;
|
|
174
|
+
const completionTokens = estimateTokens(Buffer.byteLength(text), model.tokenizer, ledger);
|
|
175
|
+
const usage = { promptTokens, cachedTokens: 0, cacheWriteTokens: 0, completionTokens, reasoningTokens: 0, images: 0 };
|
|
176
|
+
const usd = costUsd ?? computeCost(model, usage).total;
|
|
177
|
+
|
|
178
|
+
// Every digest is a ledger row: the report shows its cost beside the
|
|
179
|
+
// prompt tokens it kept out of the expensive model.
|
|
180
|
+
const entry: LedgerEntry = {
|
|
181
|
+
id: crypto.randomUUID(),
|
|
182
|
+
createdAtMs: startedAt,
|
|
183
|
+
conversationKey: synthetic.conversationKey,
|
|
184
|
+
sessionId: `digest-${req.ompSessionId}`,
|
|
185
|
+
turn: 1,
|
|
186
|
+
requestedModel: "digest",
|
|
187
|
+
harnessId: req.harnessId,
|
|
188
|
+
ompSessionId: req.ompSessionId,
|
|
189
|
+
slug: model.slug,
|
|
190
|
+
servedSlug: model.slug,
|
|
191
|
+
tier: cfg.digest.tier,
|
|
192
|
+
classificationSource: "forced",
|
|
193
|
+
reasons: [`digest: ${req.toolName} ${inputBytes} bytes → ${text.length} chars for a ${currentTier} session`],
|
|
194
|
+
features: null,
|
|
195
|
+
score: null,
|
|
196
|
+
confidence: null,
|
|
197
|
+
task: "documentation",
|
|
198
|
+
classifierReasons: null,
|
|
199
|
+
exploredFrom: null,
|
|
200
|
+
holdArm: null,
|
|
201
|
+
predictedUsd: est.expectedUsd,
|
|
202
|
+
reportedUsd: error === null ? usd : null,
|
|
203
|
+
usage,
|
|
204
|
+
attempt: 0,
|
|
205
|
+
escalationSignal: null,
|
|
206
|
+
latencyMs: ms,
|
|
207
|
+
ttftMs: null,
|
|
208
|
+
finishReason: error === null ? "stop" : null,
|
|
209
|
+
wasted: false,
|
|
210
|
+
upstreamGenerationId: null,
|
|
211
|
+
error,
|
|
212
|
+
promptTokensSaved: 0,
|
|
213
|
+
priceModel: model,
|
|
214
|
+
};
|
|
215
|
+
try {
|
|
216
|
+
ledger.record(entry);
|
|
217
|
+
} catch (err) {
|
|
218
|
+
log.debug("digest ledger record failed", { error: err instanceof Error ? err.message : String(err) });
|
|
219
|
+
}
|
|
220
|
+
if (error !== null) return { digested: false, reason: `digest model failed: ${error}` };
|
|
221
|
+
if (text === "" || text.length >= inputBytes * 0.9) return { digested: false, reason: "digest did not shrink the output" };
|
|
222
|
+
return {
|
|
223
|
+
digested: true,
|
|
224
|
+
text: `${digestMarker(req.toolName, req.input, model.slug, inputBytes, text.length)}\n${text}`,
|
|
225
|
+
model: model.slug,
|
|
226
|
+
usd,
|
|
227
|
+
inputBytes,
|
|
228
|
+
outputChars: text.length,
|
|
229
|
+
ms,
|
|
230
|
+
};
|
|
231
|
+
},
|
|
232
|
+
};
|
|
233
|
+
}
|
package/src/server/http.ts
CHANGED
|
@@ -6,6 +6,7 @@ import { createBridgeFromConfig } from "../context/index.ts";
|
|
|
6
6
|
import { createFeedbackStore, type Verdict } from "../cost/feedback.ts";
|
|
7
7
|
import { createLedger } from "../cost/ledger.ts";
|
|
8
8
|
import { createSessionOverrides } from "./overrides.ts";
|
|
9
|
+
import { createDigester } from "./digest.ts";
|
|
9
10
|
import { TIER_ORDER, type Tier } from "../router/types.ts";
|
|
10
11
|
import { baselinePrices, buildUsageReport } from "../cost/report.ts";
|
|
11
12
|
import type { Ledger, ModelTrust } from "../cost/types.ts";
|
|
@@ -194,6 +195,7 @@ export function startServer(cfg: RouterConfig): StartedServer {
|
|
|
194
195
|
const context = createBridgeFromConfig(cfg, db);
|
|
195
196
|
const overrides = createSessionOverrides();
|
|
196
197
|
const feedback = createFeedbackStore(db);
|
|
198
|
+
const digester = createDigester({ cfg, catalog, ledger, upstream, log });
|
|
197
199
|
const turnDeps = { config: cfg, router, upstream, ledger, conversations, catalog, context, overrides, ollamaCostScale };
|
|
198
200
|
|
|
199
201
|
// Hot reload: ranking knobs (tiers, filters, escalation, budgets, …) take
|
|
@@ -436,6 +438,26 @@ export function startServer(cfg: RouterConfig): StartedServer {
|
|
|
436
438
|
return json({ override: set });
|
|
437
439
|
}
|
|
438
440
|
}
|
|
441
|
+
if (req.method === "GET" && url.pathname === "/v1/router/digest/policy") {
|
|
442
|
+
const d = cfg.digest;
|
|
443
|
+
return json({ enabled: d.enabled, minBytes: d.minBytes, maxBytes: d.maxBytes, tools: d.tools, fromTier: d.fromTier });
|
|
444
|
+
}
|
|
445
|
+
if (req.method === "POST" && url.pathname === "/v1/router/digest") {
|
|
446
|
+
const body = (await req.json().catch(() => null)) as Record<string, unknown> | null;
|
|
447
|
+
if (body === null || typeof body.content !== "string" || typeof body.toolName !== "string") {
|
|
448
|
+
return wireErrorResponse({ status: 400, code: "invalid_request_error", message: "toolName and content required" });
|
|
449
|
+
}
|
|
450
|
+
return json(
|
|
451
|
+
await digester.digest({
|
|
452
|
+
ompSessionId: typeof body.ompSessionId === "string" ? body.ompSessionId : "",
|
|
453
|
+
harnessId: typeof body.harnessId === "string" ? body.harnessId : "",
|
|
454
|
+
toolName: body.toolName,
|
|
455
|
+
input: typeof body.input === "object" && body.input !== null ? (body.input as Record<string, unknown>) : {},
|
|
456
|
+
content: body.content,
|
|
457
|
+
query: typeof body.query === "string" ? body.query : "",
|
|
458
|
+
}),
|
|
459
|
+
);
|
|
460
|
+
}
|
|
439
461
|
if (req.method === "POST" && url.pathname === "/v1/router/feedback") {
|
|
440
462
|
// A user verdict on the newest routed turn of an omp session.
|
|
441
463
|
const body = (await req.json().catch(() => null)) as Record<string, unknown> | null;
|
|
@@ -292,6 +292,9 @@ export function parseChatRequest(body: unknown, headers: Headers): NormRequest {
|
|
|
292
292
|
// ⇒ the server falls back to its configured default scope.
|
|
293
293
|
const agentdoxScope = (headers.get("x-agentdox-scope") ?? "").trim();
|
|
294
294
|
|
|
295
|
+
// Subagent marker from the embed extension (sessions without a UI).
|
|
296
|
+
const isSubagent = (headers.get("x-omp-subagent") ?? "").trim() === "1";
|
|
297
|
+
|
|
295
298
|
if (typeof b.model !== "string" || b.model.length === 0) {
|
|
296
299
|
throw invalidRequest("model must be a non-empty string");
|
|
297
300
|
}
|
|
@@ -352,6 +355,7 @@ export function parseChatRequest(body: unknown, headers: Headers): NormRequest {
|
|
|
352
355
|
harnessId,
|
|
353
356
|
ompSessionId,
|
|
354
357
|
agentdoxScope,
|
|
358
|
+
isSubagent,
|
|
355
359
|
requestedModel,
|
|
356
360
|
messages,
|
|
357
361
|
tools,
|
package/src/wire/types.ts
CHANGED
|
@@ -82,6 +82,8 @@ export interface NormRequest {
|
|
|
82
82
|
* and if that is empty too the bridge stays inert for this request.
|
|
83
83
|
*/
|
|
84
84
|
agentdoxScope: string;
|
|
85
|
+
/** `X-Omp-Subagent: 1`: the caller is an omp subagent (a session without a UI). */
|
|
86
|
+
isSubagent: boolean;
|
|
85
87
|
/** Virtual model the client selected, e.g. `auto`, `auto-cheap`, `auto-max`. */
|
|
86
88
|
requestedModel: string;
|
|
87
89
|
messages: NormMessage[];
|
package/test/classify.test.ts
CHANGED
|
@@ -458,3 +458,16 @@ describe("classifyTask", () => {
|
|
|
458
458
|
expect(classifyTask(featuresFor([SYSTEM, { role: "user", content: "explain the architecture of the system" }], []))).toBe("documentation");
|
|
459
459
|
});
|
|
460
460
|
});
|
|
461
|
+
|
|
462
|
+
describe("classifier.readOnlyToolWeight", () => {
|
|
463
|
+
test("subtracts only when enabled and the tail is a read-only loop", () => {
|
|
464
|
+
const base = { ...featuresFor([{ role: "user", content: "look" }]), isToolResultContinuation: true, readOnlyToolTail: true };
|
|
465
|
+
const off = scoreHeuristic(base, DEFAULT_CONFIG);
|
|
466
|
+
const cfg = structuredClone(DEFAULT_CONFIG);
|
|
467
|
+
cfg.classifier.readOnlyToolWeight = 0.1;
|
|
468
|
+
const on = scoreHeuristic(base, cfg);
|
|
469
|
+
expect(on.score).toBeCloseTo(Math.max(0, off.score - 0.1), 6);
|
|
470
|
+
expect(on.reasons.some((r) => r.includes("read-only tool loop"))).toBe(true);
|
|
471
|
+
expect(scoreHeuristic({ ...base, readOnlyToolTail: false }, cfg).score).toBeCloseTo(off.score, 6);
|
|
472
|
+
});
|
|
473
|
+
});
|
|
@@ -280,6 +280,7 @@ describe("runWizard", () => {
|
|
|
280
280
|
"", // keep apiKey
|
|
281
281
|
"", // keep harnessId
|
|
282
282
|
"", // keep maxConcurrentTurns
|
|
283
|
+
"", // keep subagentProfile
|
|
283
284
|
"s",
|
|
284
285
|
]);
|
|
285
286
|
expect(partial).toEqual({ server: { port: 9000 } });
|
|
@@ -358,7 +359,7 @@ describe("runWizard: profiles", () => {
|
|
|
358
359
|
const profiles = (partial ?? {})["profiles"];
|
|
359
360
|
expect(Array.isArray(profiles)).toBe(true);
|
|
360
361
|
if (!Array.isArray(profiles)) return;
|
|
361
|
-
expect(profiles).toHaveLength(
|
|
362
|
+
expect(profiles).toHaveLength(4);
|
|
362
363
|
expect(profiles[0]).toMatchObject({ id: "auto", contextWindow: 500000 });
|
|
363
364
|
expect(profiles[1]).toMatchObject({ id: "auto-cheap", contextWindow: 400000 });
|
|
364
365
|
});
|
|
@@ -379,8 +380,8 @@ describe("runWizard: profiles", () => {
|
|
|
379
380
|
const profiles = (partial ?? {})["profiles"];
|
|
380
381
|
expect(Array.isArray(profiles)).toBe(true);
|
|
381
382
|
if (!Array.isArray(profiles)) return;
|
|
382
|
-
expect(profiles).toHaveLength(
|
|
383
|
-
expect(profiles[
|
|
383
|
+
expect(profiles).toHaveLength(5);
|
|
384
|
+
expect(profiles[4]).toEqual({
|
|
384
385
|
id: "auto-fast",
|
|
385
386
|
name: "Auto Fast",
|
|
386
387
|
minTier: "trivial",
|
|
@@ -400,11 +401,11 @@ describe("runWizard: profiles", () => {
|
|
|
400
401
|
const profiles = (partial ?? {})["profiles"];
|
|
401
402
|
expect(Array.isArray(profiles)).toBe(true);
|
|
402
403
|
if (!Array.isArray(profiles)) return;
|
|
403
|
-
expect(profiles.map((p) => (p as Record<string, unknown>)["id"])).toEqual(["auto", "auto-max"]);
|
|
404
|
+
expect(profiles.map((p) => (p as Record<string, unknown>)["id"])).toEqual(["auto", "auto-max", "auto-sub"]);
|
|
404
405
|
});
|
|
405
406
|
|
|
406
407
|
test("refuses to delete the last remaining profile", async () => {
|
|
407
|
-
const { out } = await drive(["p", "x3", "x2", "x1", "b", "q"]);
|
|
408
|
+
const { out } = await drive(["p", "x4", "x3", "x2", "x1", "b", "q"]);
|
|
408
409
|
expect(out).toContain("cannot delete the last profile");
|
|
409
410
|
});
|
|
410
411
|
|