auto-model-router 0.4.2 → 0.4.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.omp-plugin/marketplace.json +2 -2
- package/README.md +21 -2
- package/omp-extension/report-logic.ts +46 -0
- package/omp-extension/router-configure.ts +55 -3
- package/package.json +1 -1
- package/src/cli/config-wizard.ts +7 -1
- package/src/config/defaults.ts +7 -0
- package/src/config/schema.ts +4 -1
- package/src/config/types.ts +27 -0
- package/src/cost/ledger.ts +75 -8
- package/src/cost/report.ts +8 -3
- package/src/cost/summary.ts +231 -0
- package/src/cost/types.ts +31 -3
- package/src/router/candidates.ts +1 -1
- package/src/router/classify.ts +2 -2
- package/src/router/compaction.ts +1 -0
- package/src/router/learned.ts +11 -1
- package/src/router/select.ts +5 -1
- package/src/server/compaction-digest.ts +127 -0
- package/src/server/digest.ts +13 -3
- package/src/server/http.ts +29 -1
- package/src/server/turn.ts +27 -1
- package/src/util/sqlite.ts +7 -0
- package/src/wire/openai/request.ts +4 -0
- package/src/wire/types.ts +7 -0
- package/test/compaction.test.ts +40 -3
- package/test/digest.test.ts +22 -0
- package/test/failover.test.ts +3 -3
- package/test/learned.test.ts +21 -1
- package/test/report-logic.test.ts +11 -1
- package/test/select.test.ts +2 -2
- package/test/summary.test.ts +171 -0
- package/test/tokens.test.ts +44 -0
- package/test/trust-attribution.test.ts +37 -0
- package/test/turn.test.ts +68 -3
- package/tools/train-classifier.ts +75 -20
|
@@ -0,0 +1,231 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The daily summary: what the router did in the last 24 hours, in a few
|
|
3
|
+
* lines. Posted into the transcript once a day at omp session start
|
|
4
|
+
* (`report.dailySummary`) and on demand via `/router summary` or
|
|
5
|
+
* `GET /v1/router/summary`.
|
|
6
|
+
*
|
|
7
|
+
* Built from the same `buildUsageReport` the report hub uses, over a 1-day
|
|
8
|
+
* window, with the preceding day for comparison, plus the two live signals the
|
|
9
|
+
* report cannot carry: soft-failure spikes (the last hour) and the Ollama
|
|
10
|
+
* meter. The once-a-day gate is a marker in `router_kv`, keyed per harness, so
|
|
11
|
+
* several omp windows on one router share it and a restart does not repeat it.
|
|
12
|
+
*/
|
|
13
|
+
|
|
14
|
+
import type { Database } from "bun:sqlite";
|
|
15
|
+
import { TIER_ORDER } from "../router/types.ts";
|
|
16
|
+
import { buildUsageReport, type BaselinePrice, type BaselineRow, type UsageReport } from "./report.ts";
|
|
17
|
+
import type { SoftFailureSpike } from "./types.ts";
|
|
18
|
+
|
|
19
|
+
/** One 24-hour window's headline numbers. */
|
|
20
|
+
export interface SummaryWindow {
|
|
21
|
+
spendUsd: number;
|
|
22
|
+
dispatches: number;
|
|
23
|
+
conversations: number;
|
|
24
|
+
cacheHitRate: number;
|
|
25
|
+
cacheEstimated: boolean;
|
|
26
|
+
escalations: number;
|
|
27
|
+
errors: number;
|
|
28
|
+
modelSwitches: number;
|
|
29
|
+
digests: number;
|
|
30
|
+
digestSpendUsd: number;
|
|
31
|
+
subagentSpendUsd: number;
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
export interface SummaryModel {
|
|
35
|
+
slug: string;
|
|
36
|
+
spendUsd: number;
|
|
37
|
+
share: number;
|
|
38
|
+
dispatches: number;
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
export interface SummaryOllama {
|
|
42
|
+
plan: string | null;
|
|
43
|
+
usedUsd: number;
|
|
44
|
+
creditsUsd: number;
|
|
45
|
+
/** Days of credits left at the recent burn; null when the burn is zero. */
|
|
46
|
+
runwayDays: number | null;
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
export interface DailySummary {
|
|
50
|
+
generatedAtMs: number;
|
|
51
|
+
sinceMs: number;
|
|
52
|
+
/** Empty ⇒ every harness. */
|
|
53
|
+
harnessId: string;
|
|
54
|
+
current: SummaryWindow;
|
|
55
|
+
/** The 24 hours before `sinceMs`. */
|
|
56
|
+
previous: SummaryWindow;
|
|
57
|
+
/** Top models by spend in the current window. */
|
|
58
|
+
topModels: SummaryModel[];
|
|
59
|
+
/** Tier moves between consecutive kept turns of one conversation. */
|
|
60
|
+
tierChanges: { up: number; down: number };
|
|
61
|
+
/** The first configured baseline the catalog knew, when any. */
|
|
62
|
+
baseline: BaselineRow | null;
|
|
63
|
+
spikes: SoftFailureSpike[];
|
|
64
|
+
ollama: SummaryOllama | null;
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
const DAY_MS = 86_400_000;
|
|
68
|
+
/** How many top models the summary names. */
|
|
69
|
+
const TOP_MODELS = 3;
|
|
70
|
+
|
|
71
|
+
function windowOf(r: UsageReport): SummaryWindow {
|
|
72
|
+
const t = r.totals;
|
|
73
|
+
return {
|
|
74
|
+
spendUsd: t.spendUsd,
|
|
75
|
+
dispatches: t.dispatches,
|
|
76
|
+
conversations: t.conversations,
|
|
77
|
+
cacheHitRate: t.cacheHitRate,
|
|
78
|
+
cacheEstimated: t.cacheEstimated,
|
|
79
|
+
escalations: t.escalations,
|
|
80
|
+
errors: t.errors,
|
|
81
|
+
modelSwitches: t.modelSwitches,
|
|
82
|
+
digests: t.digests,
|
|
83
|
+
digestSpendUsd: t.digestSpendUsd,
|
|
84
|
+
subagentSpendUsd: t.subagentSpendUsd,
|
|
85
|
+
};
|
|
86
|
+
}
|
|
87
|
+
|
|
88
|
+
/** Counts tier moves up and down between consecutive kept turns of each conversation since `sinceMs`. */
|
|
89
|
+
export function countTierChanges(db: Database, sinceMs: number, harnessId: string): { up: number; down: number } {
|
|
90
|
+
const where = harnessId === "" ? "created_at_ms >= $since" : "created_at_ms >= $since AND harness_id = $harness";
|
|
91
|
+
const bind = harnessId === "" ? { $since: sinceMs } : { $since: sinceMs, $harness: harnessId };
|
|
92
|
+
const seq = db
|
|
93
|
+
.query(`SELECT conversation_key AS ck, tier FROM ledger WHERE ${where} AND wasted = 0 AND requested_model <> 'digest' ORDER BY conversation_key, created_at_ms`)
|
|
94
|
+
.all(bind) as { ck: string; tier: string }[];
|
|
95
|
+
let up = 0;
|
|
96
|
+
let down = 0;
|
|
97
|
+
for (let i = 1; i < seq.length; i++) {
|
|
98
|
+
const a = seq[i - 1]!;
|
|
99
|
+
const b = seq[i]!;
|
|
100
|
+
if (a.ck !== b.ck) continue;
|
|
101
|
+
const ra = TIER_ORDER.indexOf(a.tier as (typeof TIER_ORDER)[number]);
|
|
102
|
+
const rb = TIER_ORDER.indexOf(b.tier as (typeof TIER_ORDER)[number]);
|
|
103
|
+
if (ra < 0 || rb < 0 || ra === rb) continue;
|
|
104
|
+
if (rb > ra) up++;
|
|
105
|
+
else down++;
|
|
106
|
+
}
|
|
107
|
+
return { up, down };
|
|
108
|
+
}
|
|
109
|
+
|
|
110
|
+
export function buildDailySummary(
|
|
111
|
+
db: Database,
|
|
112
|
+
opts: {
|
|
113
|
+
harnessId?: string;
|
|
114
|
+
nowMs?: number;
|
|
115
|
+
baselines?: readonly BaselinePrice[];
|
|
116
|
+
spikes?: readonly SoftFailureSpike[];
|
|
117
|
+
ollama?: SummaryOllama | null;
|
|
118
|
+
} = {},
|
|
119
|
+
): DailySummary {
|
|
120
|
+
const nowMs = opts.nowMs ?? Date.now();
|
|
121
|
+
const harnessId = opts.harnessId ?? "";
|
|
122
|
+
const baselines = opts.baselines ?? [];
|
|
123
|
+
const current = buildUsageReport(db, { windowDays: 1, harnessId, nowMs, baselines });
|
|
124
|
+
const previous = buildUsageReport(db, { windowDays: 1, harnessId, nowMs: nowMs - DAY_MS, untilMs: current.sinceMs });
|
|
125
|
+
return {
|
|
126
|
+
generatedAtMs: nowMs,
|
|
127
|
+
sinceMs: current.sinceMs,
|
|
128
|
+
harnessId,
|
|
129
|
+
current: windowOf(current),
|
|
130
|
+
previous: windowOf(previous),
|
|
131
|
+
topModels: current.models.slice(0, TOP_MODELS).map((m) => ({ slug: m.key, spendUsd: m.spendUsd, share: m.share, dispatches: m.dispatches })),
|
|
132
|
+
tierChanges: countTierChanges(db, current.sinceMs, harnessId),
|
|
133
|
+
baseline: current.baselines[0] ?? null,
|
|
134
|
+
spikes: [...(opts.spikes ?? [])],
|
|
135
|
+
ollama: opts.ollama ?? null,
|
|
136
|
+
};
|
|
137
|
+
}
|
|
138
|
+
|
|
139
|
+
/** Whether the once-a-day auto summary is worth posting: something happened, or something is wrong. */
|
|
140
|
+
export function summaryHasNews(s: DailySummary): boolean {
|
|
141
|
+
return s.current.dispatches > 0 || s.spikes.length > 0;
|
|
142
|
+
}
|
|
143
|
+
|
|
144
|
+
const usd = (v: number): string => (v >= 100 ? `$${v.toFixed(0)}` : v >= 1 ? `$${v.toFixed(2)}` : `$${v.toFixed(3)}`);
|
|
145
|
+
const pct = (v: number, estimated = false): string => `${estimated ? "~" : ""}${Math.round(v * 100)}%`;
|
|
146
|
+
|
|
147
|
+
function delta(current: number, previous: number): string {
|
|
148
|
+
if (previous <= 0) return current > 0 ? " (prev 24h: none)" : "";
|
|
149
|
+
const change = (current - previous) / previous;
|
|
150
|
+
const sign = change >= 0 ? "+" : "−";
|
|
151
|
+
return ` (prev 24h ${usd(previous)}, ${sign}${Math.round(Math.abs(change) * 100)}%)`;
|
|
152
|
+
}
|
|
153
|
+
|
|
154
|
+
/** Renders the summary as a few plain lines for the transcript. */
|
|
155
|
+
export function renderDailySummary(s: DailySummary): string {
|
|
156
|
+
const scope = s.harnessId === "" ? "all harnesses" : `harness ${s.harnessId}`;
|
|
157
|
+
const out: string[] = [`auto-model-router daily summary — last 24h (${scope})`];
|
|
158
|
+
const c = s.current;
|
|
159
|
+
if (c.dispatches === 0) {
|
|
160
|
+
out.push("no routed turns in the last 24h");
|
|
161
|
+
} else {
|
|
162
|
+
out.push(
|
|
163
|
+
`spend ${usd(c.spendUsd)}${delta(c.spendUsd, s.previous.spendUsd)} · ${c.dispatches} turns · ${c.conversations} conversations · ${usd(c.spendUsd / c.dispatches)}/turn`,
|
|
164
|
+
);
|
|
165
|
+
const moves = c.modelSwitches > 0 ? ` (${s.tierChanges.up} tier up, ${s.tierChanges.down} down)` : "";
|
|
166
|
+
out.push(`cache hit ${pct(c.cacheHitRate, c.cacheEstimated)} · ${c.escalations} escalations · ${c.errors} errors · ${c.modelSwitches} model switches${moves}`);
|
|
167
|
+
if (s.topModels.length > 0) {
|
|
168
|
+
out.push(`top models: ${s.topModels.map((m) => `${m.slug} ${usd(m.spendUsd)} (${pct(m.share)}, ${m.dispatches} turns)`).join(" · ")}`);
|
|
169
|
+
}
|
|
170
|
+
if (s.baseline !== null && s.baseline.usd > 0) {
|
|
171
|
+
const b = s.baseline;
|
|
172
|
+
out.push(b.savedShare >= 0 ? `saved ${pct(b.savedShare)} vs ${b.slug} (${usd(b.usd)} at list)` : `cost ${pct(-b.savedShare)} MORE than ${b.slug} (${usd(b.usd)} at list)`);
|
|
173
|
+
}
|
|
174
|
+
const extras: string[] = [];
|
|
175
|
+
if (c.digests > 0) extras.push(`${c.digests} digests for ${usd(c.digestSpendUsd)}`);
|
|
176
|
+
if (c.subagentSpendUsd > 0) extras.push(`subagents ${usd(c.subagentSpendUsd)}`);
|
|
177
|
+
if (extras.length > 0) out.push(extras.join(" · "));
|
|
178
|
+
}
|
|
179
|
+
if (s.spikes.length === 0) out.push("soft failures: no model spiking in the last hour");
|
|
180
|
+
else {
|
|
181
|
+
out.push(`soft failures SPIKING (${s.spikes.length}):`);
|
|
182
|
+
for (const sp of s.spikes) {
|
|
183
|
+
out.push(` ${sp.slug}: ${pct(sp.recentRate)} of ${sp.recentDispatches} failed in the last 1h (7d baseline ${pct(sp.baselineRate)} of ${sp.baselineDispatches})`);
|
|
184
|
+
}
|
|
185
|
+
}
|
|
186
|
+
const o = s.ollama;
|
|
187
|
+
if (o !== null) {
|
|
188
|
+
const runway = o.runwayDays === null ? "" : ` · ~${Math.round(o.runwayDays)} days of credits left`;
|
|
189
|
+
out.push(`ollama: ${o.plan === null ? "plan" : `${o.plan} plan`} $${o.usedUsd.toFixed(2)} of $${o.creditsUsd}${runway}`);
|
|
190
|
+
}
|
|
191
|
+
return out.join("\n");
|
|
192
|
+
}
|
|
193
|
+
|
|
194
|
+
// ---------------------------------------------------------------------------
|
|
195
|
+
// Once-a-day gate
|
|
196
|
+
// ---------------------------------------------------------------------------
|
|
197
|
+
|
|
198
|
+
/** A summary posted less than this long ago is not due again. */
|
|
199
|
+
export const DAILY_SUMMARY_INTERVAL_MS = 20 * 3_600_000;
|
|
200
|
+
|
|
201
|
+
export interface KeyValueStore {
|
|
202
|
+
get(key: string): string | null;
|
|
203
|
+
set(key: string, value: string): void;
|
|
204
|
+
}
|
|
205
|
+
|
|
206
|
+
/** A tiny durable key/value store over the `router_kv` table. */
|
|
207
|
+
export function createKv(db: Database): KeyValueStore {
|
|
208
|
+
const getStmt = db.query("SELECT value FROM router_kv WHERE key = $key");
|
|
209
|
+
const setStmt = db.query("INSERT INTO router_kv (key, value, updated_at_ms) VALUES ($key, $value, $at) ON CONFLICT(key) DO UPDATE SET value = excluded.value, updated_at_ms = excluded.updated_at_ms");
|
|
210
|
+
return {
|
|
211
|
+
get(key) {
|
|
212
|
+
const row = getStmt.get({ $key: key }) as { value: string } | null;
|
|
213
|
+
return row === null ? null : row.value;
|
|
214
|
+
},
|
|
215
|
+
set(key, value) {
|
|
216
|
+
setStmt.run({ $key: key, $value: value, $at: Date.now() });
|
|
217
|
+
},
|
|
218
|
+
};
|
|
219
|
+
}
|
|
220
|
+
|
|
221
|
+
const shownKey = (harnessId: string): string => `daily_summary_shown:${harnessId}`;
|
|
222
|
+
|
|
223
|
+
/** True when no auto summary has been posted for this harness within the interval. */
|
|
224
|
+
export function summaryDue(kv: KeyValueStore, harnessId: string, nowMs = Date.now()): boolean {
|
|
225
|
+
const last = Number(kv.get(shownKey(harnessId)) ?? "0");
|
|
226
|
+
return !(Number.isFinite(last) && nowMs - last < DAILY_SUMMARY_INTERVAL_MS);
|
|
227
|
+
}
|
|
228
|
+
|
|
229
|
+
export function markSummaryShown(kv: KeyValueStore, harnessId: string, nowMs = Date.now()): void {
|
|
230
|
+
kv.set(shownKey(harnessId), String(nowMs));
|
|
231
|
+
}
|
package/src/cost/types.ts
CHANGED
|
@@ -230,6 +230,25 @@ export interface EscalationCost {
|
|
|
230
230
|
windowDays: number;
|
|
231
231
|
}
|
|
232
232
|
|
|
233
|
+
/**
|
|
234
|
+
* A model whose recent failure rate (probe rejections OpenRouter counts as
|
|
235
|
+
* success, plus attributable transport errors) is well above its own
|
|
236
|
+
* baseline. Visibility only: the ledger data showed soft failures do not
|
|
237
|
+
* cluster tightly enough for a breaker to save money, so the router reports
|
|
238
|
+
* spikes (/health, /router status, the daily summary) rather than acting.
|
|
239
|
+
*/
|
|
240
|
+
export interface SoftFailureSpike {
|
|
241
|
+
slug: string;
|
|
242
|
+
/** Dispatches and failures in the recent window. */
|
|
243
|
+
recentDispatches: number;
|
|
244
|
+
recentFailures: number;
|
|
245
|
+
recentRate: number;
|
|
246
|
+
/** The same, over the baseline window (recent window excluded). */
|
|
247
|
+
baselineDispatches: number;
|
|
248
|
+
baselineFailures: number;
|
|
249
|
+
baselineRate: number;
|
|
250
|
+
}
|
|
251
|
+
|
|
233
252
|
export interface Ledger {
|
|
234
253
|
record(entry: LedgerEntry): void;
|
|
235
254
|
/** Total reported (or predicted, when reported is null) spend for a conversation. */
|
|
@@ -240,8 +259,12 @@ export interface Ledger {
|
|
|
240
259
|
*/
|
|
241
260
|
spendSince(sinceMs: number, harnessId?: string): number;
|
|
242
261
|
blendedRate(windowDays: number): BlendedRate | null;
|
|
243
|
-
/**
|
|
244
|
-
|
|
262
|
+
/**
|
|
263
|
+
* Per-model reliability over the ledger, optionally scoped to a harness.
|
|
264
|
+
* `task` (with `filters.feedbackByTask`) counts only verdicts given on
|
|
265
|
+
* turns of that task type, plus verdicts on turns with no recorded task.
|
|
266
|
+
*/
|
|
267
|
+
trust(slug: string, harnessId?: string, task?: string): ModelTrust | null;
|
|
245
268
|
allTrust(): ModelTrust[];
|
|
246
269
|
/**
|
|
247
270
|
* Per-model responsiveness (mean TTFT + completion throughput), optionally
|
|
@@ -251,7 +274,7 @@ export interface Ledger {
|
|
|
251
274
|
*/
|
|
252
275
|
latency(slug: string, harnessId?: string): ModelLatency | null;
|
|
253
276
|
/** Batch trust and latency for one candidate set; one query per signal kind. Optional — callers can fall back to per-slug calls. */
|
|
254
|
-
signals?(slugs: readonly string[], harnessId?: string): Map<string, LedgerSignals>;
|
|
277
|
+
signals?(slugs: readonly string[], harnessId?: string, task?: string): Map<string, LedgerSignals>;
|
|
255
278
|
/**
|
|
256
279
|
* What an escalated retry actually bills per prompt token, measured over
|
|
257
280
|
* the last `windowDays` of attempt > 0 rows. Null until enough escalated
|
|
@@ -271,6 +294,11 @@ export interface Ledger {
|
|
|
271
294
|
recentEntries(limit: number): LedgerEntry[];
|
|
272
295
|
/** Spend since an instant on slugs with a prefix (`ollama/`), for provider-level reconciliation. Optional. */
|
|
273
296
|
providerSpendSince?(slugPrefix: string, sinceMs: number): number;
|
|
297
|
+
/**
|
|
298
|
+
* Models whose soft-failure rate over the last `recentMs` is a spike against
|
|
299
|
+
* their own rate over the preceding `baselineMs`. Optional; visibility only.
|
|
300
|
+
*/
|
|
301
|
+
softFailureSpikes?(nowMs?: number, recentMs?: number, baselineMs?: number): SoftFailureSpike[];
|
|
274
302
|
/** Newest kept (non-wasted) entry for an omp session, for /router why and feedback. Optional so fakes need not implement it. */
|
|
275
303
|
latestForSession?(ompSessionId: string): LedgerEntry | null;
|
|
276
304
|
/** Newest entries for an omp session, newest first. Optional. */
|
package/src/router/candidates.ts
CHANGED
|
@@ -263,7 +263,7 @@ export function buildCandidates(args: BuildCandidatesArgs): { candidates: Candid
|
|
|
263
263
|
const signals = args.signals;
|
|
264
264
|
const trust =
|
|
265
265
|
signals?.get(slug)?.trust ??
|
|
266
|
-
ledger?.trust(slug, filters.trustScopedByHarness ? req.harnessId : undefined) ??
|
|
266
|
+
ledger?.trust(slug, filters.trustScopedByHarness ? req.harnessId : undefined, filters.feedbackByTask ? task : undefined) ??
|
|
267
267
|
null;
|
|
268
268
|
if (!relaxTrust && trust !== null && trust.attempts >= filters.minTrustSamples && trust.successRate < filters.minTrust) {
|
|
269
269
|
rejected.push({
|
package/src/router/classify.ts
CHANGED
|
@@ -10,7 +10,7 @@ import type { Ledger } from "../cost/types.ts";
|
|
|
10
10
|
import { estimateTokens } from "../tokens/estimate.ts";
|
|
11
11
|
import type { UpstreamClient } from "../upstream/types.ts";
|
|
12
12
|
import { sha256Hex } from "../util/hash.ts";
|
|
13
|
-
import { loadLearnedModel, predictRisk } from "./learned.ts";
|
|
13
|
+
import { learnedRiskName, loadLearnedModel, predictRisk } from "./learned.ts";
|
|
14
14
|
import type { NormRequest, ReasoningLevel } from "../wire/types.ts";
|
|
15
15
|
import type { Classification, Features, TaskType, Tier } from "./types.ts";
|
|
16
16
|
|
|
@@ -313,7 +313,7 @@ export async function classify(
|
|
|
313
313
|
if (model !== null) {
|
|
314
314
|
const risk = predictRisk(model, f);
|
|
315
315
|
heuristic.learnedRisk = risk;
|
|
316
|
-
heuristic.reasons.push(`learned: p(
|
|
316
|
+
heuristic.reasons.push(`learned: p(${learnedRiskName(model)})=${risk.toFixed(3)}`);
|
|
317
317
|
}
|
|
318
318
|
}
|
|
319
319
|
if (cc.ambiguityThreshold <= 0 || heuristic.confidence >= cc.ambiguityThreshold) return heuristic;
|
package/src/router/compaction.ts
CHANGED
|
@@ -33,6 +33,7 @@ const BREADCRUMB_BYTES = 120;
|
|
|
33
33
|
*/
|
|
34
34
|
export function compactedBytes(originalBytes: number, edit: CompactionEdit | undefined): number {
|
|
35
35
|
if (edit === undefined) return originalBytes;
|
|
36
|
+
if (edit.digest !== undefined) return Math.min(originalBytes, Buffer.byteLength(edit.digest));
|
|
36
37
|
const kept = edit.mode === "stub" ? BREADCRUMB_BYTES : edit.keepHead + edit.keepTail + BREADCRUMB_BYTES;
|
|
37
38
|
return Math.min(originalBytes, kept);
|
|
38
39
|
}
|
package/src/router/learned.ts
CHANGED
|
@@ -77,8 +77,13 @@ export function learnedVector(f: Partial<Features>): number[] {
|
|
|
77
77
|
];
|
|
78
78
|
}
|
|
79
79
|
|
|
80
|
+
/** What a learned model's positive class means. */
|
|
81
|
+
export type LearnedLabel = "escalation" | "feedback";
|
|
82
|
+
|
|
80
83
|
export interface LearnedModel {
|
|
81
84
|
version: number;
|
|
85
|
+
/** Positive class: the turn escalated (default), or the user judged it bad. */
|
|
86
|
+
label?: LearnedLabel;
|
|
82
87
|
trainedAtMs: number;
|
|
83
88
|
rows: number;
|
|
84
89
|
positives: number;
|
|
@@ -93,7 +98,12 @@ export interface LearnedModel {
|
|
|
93
98
|
|
|
94
99
|
const sigmoid = (z: number): number => 1 / (1 + Math.exp(-z));
|
|
95
100
|
|
|
96
|
-
/**
|
|
101
|
+
/** The positive-class name a decision reason should print for a model. */
|
|
102
|
+
export function learnedRiskName(model: LearnedModel): string {
|
|
103
|
+
return model.label === "feedback" ? "bad" : "escalate";
|
|
104
|
+
}
|
|
105
|
+
|
|
106
|
+
/** P(positive class) for one turn under a model: p(escalate), or p(bad) for a feedback-labelled model. */
|
|
97
107
|
export function predictRisk(model: LearnedModel, f: Partial<Features>): number {
|
|
98
108
|
const x = learnedVector(f);
|
|
99
109
|
let z = model.bias;
|
package/src/router/select.ts
CHANGED
|
@@ -361,7 +361,11 @@ export function select(args: SelectArgs): Decision {
|
|
|
361
361
|
// query per signal kind, instead of per-model individual lookups.
|
|
362
362
|
const candidateSignals =
|
|
363
363
|
ledger !== null && snapshot.models.length > 0
|
|
364
|
-
? ledger.signals?.(
|
|
364
|
+
? ledger.signals?.(
|
|
365
|
+
snapshot.models.map((m) => m.slug),
|
|
366
|
+
cfg.filters.trustScopedByHarness ? req.harnessId : undefined,
|
|
367
|
+
cfg.filters.feedbackByTask ? classification.task : undefined,
|
|
368
|
+
)
|
|
365
369
|
: undefined;
|
|
366
370
|
// What an escalated retry has actually been billing per prompt token, for
|
|
367
371
|
// the escalation-cost term in candidate scoring. Read once per turn; null
|
|
@@ -0,0 +1,127 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Summarising compaction: when the compaction plan gains new edits, a cheap
|
|
3
|
+
* model digests the tool results those edits would otherwise truncate or stub,
|
|
4
|
+
* and the digest rides in the persisted plan in place of the breadcrumb.
|
|
5
|
+
*
|
|
6
|
+
* Plain compaction keeps a head and a tail of a stale tool result; a digest
|
|
7
|
+
* keeps what the task needs from all of it (paths, identifiers, errors, the
|
|
8
|
+
* code the agent will edit) in a few hundred chars. The digest is stored on
|
|
9
|
+
* the edit, so the bytes sent stay identical on every later turn until the
|
|
10
|
+
* plan changes — the same byte-stability plain edits have, which is what keeps
|
|
11
|
+
* the prompt cache warm.
|
|
12
|
+
*
|
|
13
|
+
* Bounded: at most `compaction.digestMaxPerTurn` digests per turn, each under
|
|
14
|
+
* the digest's own cost guard and timeout, largest results first. A digest
|
|
15
|
+
* that fails or declines leaves the plain edit in place; nothing is lost.
|
|
16
|
+
*/
|
|
17
|
+
|
|
18
|
+
import type { RouterConfig } from "../config/types.ts";
|
|
19
|
+
import { compactedBytes } from "../router/compaction.ts";
|
|
20
|
+
import type { Logger } from "../util/log.ts";
|
|
21
|
+
import type { CompactionEdit, NormRequest } from "../wire/types.ts";
|
|
22
|
+
import type { DigestRequest, DigestResult } from "./digest.ts";
|
|
23
|
+
|
|
24
|
+
export interface CompactionDigester {
|
|
25
|
+
digest(req: DigestRequest): Promise<DigestResult>;
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
export interface DigestCompactionArgs {
|
|
29
|
+
req: NormRequest;
|
|
30
|
+
/** The plan this turn dispatches with; edits gain `digest` in place. */
|
|
31
|
+
plan: CompactionEdit[];
|
|
32
|
+
/** The tier this turn routed to: the digest pays off only above `digest.fromTier`. */
|
|
33
|
+
tier: string;
|
|
34
|
+
cfg: RouterConfig;
|
|
35
|
+
digester: CompactionDigester;
|
|
36
|
+
/** Per-turn memo (index:bytes → digest) so a retry does not pay twice. */
|
|
37
|
+
memo: Map<string, string>;
|
|
38
|
+
/** The user's current ask, steering what the digest keeps. */
|
|
39
|
+
query: string;
|
|
40
|
+
log: Logger;
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
/** Tool name and parsed arguments for a tool-result message, via its call id. */
|
|
44
|
+
function toolOf(req: NormRequest, index: number): { name: string; input: Record<string, unknown> } | null {
|
|
45
|
+
const m = req.messages[index];
|
|
46
|
+
if (m === undefined || m.role !== "tool") return null;
|
|
47
|
+
let name = m.toolName ?? "";
|
|
48
|
+
let input: Record<string, unknown> = {};
|
|
49
|
+
if (m.toolCallId !== undefined) {
|
|
50
|
+
for (const a of req.messages) {
|
|
51
|
+
if (a.role !== "assistant") continue;
|
|
52
|
+
const tc = a.toolCalls.find((c) => c.id === m.toolCallId);
|
|
53
|
+
if (tc === undefined) continue;
|
|
54
|
+
if (name === "") name = tc.name;
|
|
55
|
+
try {
|
|
56
|
+
const parsed: unknown = JSON.parse(tc.argsJson);
|
|
57
|
+
if (parsed !== null && typeof parsed === "object" && !Array.isArray(parsed)) input = parsed as Record<string, unknown>;
|
|
58
|
+
} catch {
|
|
59
|
+
// Unparseable args: the marker just names the tool.
|
|
60
|
+
}
|
|
61
|
+
break;
|
|
62
|
+
}
|
|
63
|
+
}
|
|
64
|
+
return name === "" ? null : { name, input };
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
/**
|
|
68
|
+
* Digests the plan's new (undigested) edits, largest first, up to the per-turn
|
|
69
|
+
* cap. Returns the change in bytes saved versus the plain edits: positive when
|
|
70
|
+
* the digests are smaller than what truncation would have kept, negative when
|
|
71
|
+
* a digest keeps more than head+tail did (it usually does, and that is the point).
|
|
72
|
+
*/
|
|
73
|
+
export async function digestCompactionEdits(args: DigestCompactionArgs): Promise<number> {
|
|
74
|
+
const { req, plan, tier, cfg, digester, memo, query, log } = args;
|
|
75
|
+
const max = cfg.compaction.digestMaxPerTurn;
|
|
76
|
+
if (max <= 0) return 0;
|
|
77
|
+
const pending = plan.filter((e) => e.digest === undefined).sort((a, b) => b.bytes - a.bytes);
|
|
78
|
+
let delta = 0;
|
|
79
|
+
const work: CompactionEdit[] = [];
|
|
80
|
+
for (const e of pending) {
|
|
81
|
+
const key = `${e.index}:${e.bytes}`;
|
|
82
|
+
const remembered = memo.get(key);
|
|
83
|
+
if (remembered !== undefined) {
|
|
84
|
+
delta += compactedBytes(e.bytes, e) - Buffer.byteLength(remembered);
|
|
85
|
+
e.digest = remembered;
|
|
86
|
+
continue;
|
|
87
|
+
}
|
|
88
|
+
if (work.length >= max) break;
|
|
89
|
+
work.push(e);
|
|
90
|
+
}
|
|
91
|
+
if (work.length === 0) return delta;
|
|
92
|
+
|
|
93
|
+
const results = await Promise.all(
|
|
94
|
+
work.map(async (e): Promise<{ edit: CompactionEdit; result: DigestResult }> => {
|
|
95
|
+
const tool = toolOf(req, e.index);
|
|
96
|
+
const m = req.messages[e.index];
|
|
97
|
+
if (tool === null || m === undefined) return { edit: e, result: { digested: false, reason: "tool result has no tool name" } };
|
|
98
|
+
try {
|
|
99
|
+
const result = await digester.digest({
|
|
100
|
+
ompSessionId: req.ompSessionId,
|
|
101
|
+
harnessId: req.harnessId,
|
|
102
|
+
toolName: tool.name,
|
|
103
|
+
input: tool.input,
|
|
104
|
+
content: m.text,
|
|
105
|
+
query,
|
|
106
|
+
tier,
|
|
107
|
+
source: "compaction",
|
|
108
|
+
});
|
|
109
|
+
return { edit: e, result };
|
|
110
|
+
} catch (err) {
|
|
111
|
+
return { edit: e, result: { digested: false, reason: err instanceof Error ? err.message : String(err) } };
|
|
112
|
+
}
|
|
113
|
+
}),
|
|
114
|
+
);
|
|
115
|
+
for (const { edit, result } of results) {
|
|
116
|
+
if (!result.digested) {
|
|
117
|
+
log.debug("compaction digest declined", { index: edit.index, bytes: edit.bytes, reason: result.reason });
|
|
118
|
+
continue;
|
|
119
|
+
}
|
|
120
|
+
const plain = compactedBytes(edit.bytes, edit);
|
|
121
|
+
edit.digest = result.text;
|
|
122
|
+
memo.set(`${edit.index}:${edit.bytes}`, result.text);
|
|
123
|
+
delta += plain - Buffer.byteLength(result.text);
|
|
124
|
+
log.info("compaction digest", { index: edit.index, bytes: edit.bytes, chars: result.outputChars, model: result.model, usd: result.usd });
|
|
125
|
+
}
|
|
126
|
+
return delta;
|
|
127
|
+
}
|
package/src/server/digest.ts
CHANGED
|
@@ -39,6 +39,14 @@ export interface DigestRequest {
|
|
|
39
39
|
content: string;
|
|
40
40
|
/** The user's current ask, so the digest keeps what matters for it. */
|
|
41
41
|
query: string;
|
|
42
|
+
/** The tier to judge `digest.fromTier` against; default: the session's last routed tier. */
|
|
43
|
+
tier?: string;
|
|
44
|
+
/**
|
|
45
|
+
* Who asked. `tool_result` (default) is the omp extension and is gated on
|
|
46
|
+
* `digest.enabled`; `compaction` is summarising compaction inside a turn
|
|
47
|
+
* and is gated on `compaction.digestToolResults` instead.
|
|
48
|
+
*/
|
|
49
|
+
source?: "tool_result" | "compaction";
|
|
42
50
|
}
|
|
43
51
|
|
|
44
52
|
export type DigestResult =
|
|
@@ -129,8 +137,10 @@ export function createDigester(deps: DigesterDeps): { digest(req: DigestRequest)
|
|
|
129
137
|
return {
|
|
130
138
|
async digest(req) {
|
|
131
139
|
const inputBytes = Buffer.byteLength(req.content);
|
|
132
|
-
const
|
|
133
|
-
const
|
|
140
|
+
const source = req.source ?? "tool_result";
|
|
141
|
+
const currentTier = req.tier ?? ledger.latestForSession?.(req.ompSessionId)?.tier ?? null;
|
|
142
|
+
const gate = source === "compaction" ? { ...cfg.digest, enabled: cfg.compaction.digestToolResults } : cfg.digest;
|
|
143
|
+
const applies = digestApplies(gate, req.toolName, inputBytes, false, currentTier);
|
|
134
144
|
if (!applies.ok) return { digested: false, reason: applies.reason };
|
|
135
145
|
|
|
136
146
|
const promptText = `Task: ${req.query === "" ? "(unknown)" : req.query}\nTool: ${req.toolName} ${JSON.stringify(req.input)}\n--- output ---\n${req.content}`;
|
|
@@ -190,7 +200,7 @@ export function createDigester(deps: DigesterDeps): { digest(req: DigestRequest)
|
|
|
190
200
|
servedSlug: model.slug,
|
|
191
201
|
tier: cfg.digest.tier,
|
|
192
202
|
classificationSource: "forced",
|
|
193
|
-
reasons: [`digest: ${req.toolName} ${inputBytes} bytes → ${text.length} chars for a ${currentTier} session`],
|
|
203
|
+
reasons: [`digest (${source}): ${req.toolName} ${inputBytes} bytes → ${text.length} chars for a ${currentTier} ${source === "compaction" ? "turn" : "session"}`],
|
|
194
204
|
features: null,
|
|
195
205
|
score: null,
|
|
196
206
|
confidence: null,
|
package/src/server/http.ts
CHANGED
|
@@ -9,6 +9,7 @@ import { createSessionOverrides } from "./overrides.ts";
|
|
|
9
9
|
import { createDigester } from "./digest.ts";
|
|
10
10
|
import { TIER_ORDER, type Tier } from "../router/types.ts";
|
|
11
11
|
import { baselinePrices, buildUsageReport } from "../cost/report.ts";
|
|
12
|
+
import { buildDailySummary, createKv, markSummaryShown, summaryDue, summaryHasNews, type SummaryOllama } from "../cost/summary.ts";
|
|
12
13
|
import type { Ledger, ModelTrust } from "../cost/types.ts";
|
|
13
14
|
import { createRouter } from "../router/index.ts";
|
|
14
15
|
import { createConversationStore } from "../router/state.ts";
|
|
@@ -195,8 +196,9 @@ export function startServer(cfg: RouterConfig): StartedServer {
|
|
|
195
196
|
const context = createBridgeFromConfig(cfg, db);
|
|
196
197
|
const overrides = createSessionOverrides();
|
|
197
198
|
const feedback = createFeedbackStore(db);
|
|
199
|
+
const kv = createKv(db);
|
|
198
200
|
const digester = createDigester({ cfg, catalog, ledger, upstream, log });
|
|
199
|
-
const turnDeps = { config: cfg, router, upstream, ledger, conversations, catalog, context, overrides, ollamaCostScale };
|
|
201
|
+
const turnDeps = { config: cfg, router, upstream, ledger, conversations, catalog, context, overrides, ollamaCostScale, digester };
|
|
200
202
|
|
|
201
203
|
// Hot reload: ranking knobs (tiers, filters, escalation, budgets, …) take
|
|
202
204
|
// effect on the next turn without a restart, because every consumer reads
|
|
@@ -398,6 +400,29 @@ export function startServer(cfg: RouterConfig): StartedServer {
|
|
|
398
400
|
const harnessId = url.searchParams.get("harness") ?? "";
|
|
399
401
|
return json(buildUsageReport(db, { windowDays, harnessId, baselines: baselinePrices(cfg.report.baselines, (s) => catalog.find(s)) }));
|
|
400
402
|
}
|
|
403
|
+
if (req.method === "GET" && url.pathname === "/v1/router/summary") {
|
|
404
|
+
// The last 24 hours in a few lines. `auto=1` is the session-start
|
|
405
|
+
// caller: it gets `due: false` unless report.dailySummary is on, no
|
|
406
|
+
// summary was posted for this harness in the last 20h, and there is
|
|
407
|
+
// something to say; posting is then marked so other windows skip it.
|
|
408
|
+
const harnessId = url.searchParams.get("harness") ?? "";
|
|
409
|
+
const auto = url.searchParams.get("auto") === "1";
|
|
410
|
+
if (auto && !cfg.report.dailySummary) return json({ due: false, reason: "report.dailySummary is off", summary: null });
|
|
411
|
+
if (auto && !summaryDue(kv, harnessId)) return json({ due: false, reason: "posted in the last 20h", summary: null });
|
|
412
|
+
const meter = ollamaMeter(ollamaUsage.peek(), cfg.ollama.planCreditsUsd);
|
|
413
|
+
const runway = ollamaRunway(meter, ledger.providerSpendSince?.("ollama/", Date.now() - 7 * 86_400_000) ?? 0, ollamaUsage.calibration()?.factor ?? 1);
|
|
414
|
+
const ollamaSummary: SummaryOllama | null =
|
|
415
|
+
ollama === null || meter === null ? null : { plan: meter.plan ?? null, usedUsd: meter.usedUsd, creditsUsd: meter.creditsUsd, runwayDays: runway?.days ?? null };
|
|
416
|
+
const summary = buildDailySummary(db, {
|
|
417
|
+
harnessId,
|
|
418
|
+
baselines: baselinePrices(cfg.report.baselines, (s) => catalog.find(s)),
|
|
419
|
+
spikes: ledger.softFailureSpikes?.() ?? [],
|
|
420
|
+
ollama: ollamaSummary,
|
|
421
|
+
});
|
|
422
|
+
if (auto && !summaryHasNews(summary)) return json({ due: false, reason: "nothing to report", summary: null });
|
|
423
|
+
if (auto) markSummaryShown(kv, harnessId);
|
|
424
|
+
return json({ due: true, summary });
|
|
425
|
+
}
|
|
401
426
|
if (req.method === "GET" && url.pathname === "/v1/router/decisions") {
|
|
402
427
|
const rawLimit = url.searchParams.get("limit");
|
|
403
428
|
const parsed = rawLimit === null ? 50 : Number.parseInt(rawLimit, 10);
|
|
@@ -514,6 +539,9 @@ export function startServer(cfg: RouterConfig): StartedServer {
|
|
|
514
539
|
runway: ollamaRunway(ollamaMeter(ollamaUsage.peek(), cfg.ollama.planCreditsUsd), ledger.providerSpendSince?.("ollama/", Date.now() - 7 * 86_400_000) ?? 0, ollamaUsage.calibration()?.factor ?? 1),
|
|
515
540
|
costBias: { configured: cfg.ollama.costBias, effective: catalog.ollamaBias?.() ?? cfg.ollama.costBias, biasUntilUsage: cfg.ollama.biasUntilUsage },
|
|
516
541
|
},
|
|
542
|
+
// Models failing well above their own baseline in the last hour.
|
|
543
|
+
// Visibility only: nothing routes around a spike.
|
|
544
|
+
softFailures: { recentMs: 3_600_000, baselineDays: 7, spikes: ledger.softFailureSpikes?.() ?? [] },
|
|
517
545
|
catalog: snap === null
|
|
518
546
|
? null
|
|
519
547
|
: {
|
package/src/server/turn.ts
CHANGED
|
@@ -28,6 +28,7 @@ import {
|
|
|
28
28
|
} from "../router/types.ts";
|
|
29
29
|
import { UpstreamError, type Dispatch, type UpstreamClient } from "../upstream/types.ts";
|
|
30
30
|
import type { SessionOverrides } from "./overrides.ts";
|
|
31
|
+
import { digestCompactionEdits, type CompactionDigester } from "./compaction-digest.ts";
|
|
31
32
|
import { createLogger } from "../util/log.ts";
|
|
32
33
|
import type { NormRequest, ResponseSink, TurnSummary, UpstreamChunk } from "../wire/types.ts";
|
|
33
34
|
|
|
@@ -71,6 +72,8 @@ export interface TurnDeps {
|
|
|
71
72
|
overrides?: SessionOverrides;
|
|
72
73
|
/** Ledger-vs-meter calibration for Ollama's estimated costs; absent ⇒ 1. */
|
|
73
74
|
ollamaCostScale?: () => number;
|
|
75
|
+
/** Cheap-model digester for summarising compaction (`compaction.digestToolResults`). Absent ⇒ plain edits. */
|
|
76
|
+
digester?: CompactionDigester;
|
|
74
77
|
}
|
|
75
78
|
|
|
76
79
|
/** A dead client connection surfaces as the sink throwing mid-stream. */
|
|
@@ -135,6 +138,8 @@ export async function runTurn(
|
|
|
135
138
|
// A failover decision already routed inside onUpstreamError; the next
|
|
136
139
|
// loop iteration dispatches it instead of routing again.
|
|
137
140
|
let pendingDecision: Decision | null = null;
|
|
141
|
+
// Digests made this turn, by edit; a retry re-plans and must not pay twice.
|
|
142
|
+
const digestMemo = new Map<string, string>();
|
|
138
143
|
|
|
139
144
|
for (let attempt = 0; attempt < maxAttempts; attempt++) {
|
|
140
145
|
// Client disconnected before anything was dispatched: spend nothing.
|
|
@@ -160,6 +165,27 @@ export async function runTurn(
|
|
|
160
165
|
}
|
|
161
166
|
}
|
|
162
167
|
|
|
168
|
+
// Summarising compaction: new edits get a cheap-model digest before the
|
|
169
|
+
// plan is applied and persisted. Bounded per turn; a decline leaves the
|
|
170
|
+
// plain edit. Accounted in the estimate adjustment below.
|
|
171
|
+
let compactionSavedBytes = decision.compactionSavedBytes;
|
|
172
|
+
if (deps.digester !== undefined && config.compaction.digestToolResults && decision.compactionPlan.length > 0) {
|
|
173
|
+
try {
|
|
174
|
+
compactionSavedBytes += await digestCompactionEdits({
|
|
175
|
+
req,
|
|
176
|
+
plan: decision.compactionPlan,
|
|
177
|
+
tier: decision.tier,
|
|
178
|
+
cfg: config,
|
|
179
|
+
digester: deps.digester,
|
|
180
|
+
memo: digestMemo,
|
|
181
|
+
query: relevanceQuery(req),
|
|
182
|
+
log,
|
|
183
|
+
});
|
|
184
|
+
} catch (err) {
|
|
185
|
+
log.warn("compaction digest failed; dispatching plain edits", { error: err instanceof Error ? err.message : String(err) });
|
|
186
|
+
}
|
|
187
|
+
}
|
|
188
|
+
|
|
163
189
|
// Resolve the shared context block. The bridge refreshes only when this
|
|
164
190
|
// turn's prefix is already cold — a model switch or a retry — so the
|
|
165
191
|
// injected bytes stay identical while the cache is worth keeping.
|
|
@@ -209,7 +235,7 @@ export async function runTurn(
|
|
|
209
235
|
// — not the raw request the estimate was taken from.
|
|
210
236
|
adjustPendingEstimate(
|
|
211
237
|
req.conversationKey,
|
|
212
|
-
req.promptBytes -
|
|
238
|
+
req.promptBytes - compactionSavedBytes + (contextBlock === undefined ? 0 : Buffer.byteLength(contextBlock)),
|
|
213
239
|
);
|
|
214
240
|
|
|
215
241
|
// Our own abort composes with the client's: escalation teardown and
|
package/src/util/sqlite.ts
CHANGED
|
@@ -141,6 +141,13 @@ CREATE TABLE IF NOT EXISTS feedback (
|
|
|
141
141
|
CREATE INDEX IF NOT EXISTS idx_feedback_created ON feedback (created_at_ms);
|
|
142
142
|
CREATE INDEX IF NOT EXISTS idx_feedback_ledger ON feedback (ledger_id);
|
|
143
143
|
|
|
144
|
+
-- Small durable markers (e.g. when the daily summary was last posted, per harness).
|
|
145
|
+
CREATE TABLE IF NOT EXISTS router_kv (
|
|
146
|
+
key TEXT PRIMARY KEY,
|
|
147
|
+
value TEXT NOT NULL,
|
|
148
|
+
updated_at_ms INTEGER NOT NULL
|
|
149
|
+
);
|
|
150
|
+
|
|
144
151
|
CREATE TABLE IF NOT EXISTS agentdox_sessions (
|
|
145
152
|
conversation_key TEXT PRIMARY KEY,
|
|
146
153
|
scope TEXT NOT NULL,
|