auto-model-router 0.4.1 → 0.4.3

package/test/trust-attribution.test.ts

@@ -463,3 +463,40 @@ describe("feedback in trust", () => {
 		db.close();
 	});
 });
+
+describe("task-scoped feedback (filters.feedbackByTask)", () => {
+	test("a verdict counts only for its task type; an untasked verdict counts everywhere", () => {
+		const db = openDb(":memory:");
+		try {
+			const c = structuredClone(cfg);
+			c.filters.feedbackWeight = 3;
+			c.filters.feedbackByTask = true;
+			const ledger = createLedger(db, c);
+			const fb = createFeedbackStore(db);
+			for (let i = 0; i < 10; i++) ledger.record(entry({ error: null, task: "coding" }));
+			const prose = entry({ error: null, task: "documentation" });
+			ledger.record(prose);
+			const untasked = entry({ error: null, task: null });
+			ledger.record(untasked);
+			fb.record({ ledgerId: prose.id, ompSessionId: "s", slug: "vendor/model", tier: "simple", verdict: "bad", note: "" });
+			// 12 clean attempts, weight 3, one bad verdict on a documentation turn.
+			const pooled = (12 - 0 + 1) / (12 + 2);
+			const withBad = (15 - 3 + 1) / (15 + 2);
+			expect(ledger.trust("vendor/model", undefined, "coding")!.successRate).toBeCloseTo(pooled, 9);
+			expect(ledger.trust("vendor/model", undefined, "documentation")!.successRate).toBeCloseTo(withBad, 9);
+			// No task given (allTrust, reports): pooled behaviour, the verdict counts.
+			expect(ledger.trust("vendor/model")!.successRate).toBeCloseTo(withBad, 9);
+			expect(ledger.allTrust()[0]!.successRate).toBeCloseTo(withBad, 9);
+			// signals() honours the task the same way.
+			expect(ledger.signals?.(["vendor/model"], undefined, "coding").get("vendor/model")!.trust!.successRate).toBeCloseTo(pooled, 9);
+			// A verdict on a turn that recorded no task counts for every task.
+			fb.record({ ledgerId: untasked.id, ompSessionId: "s", slug: "vendor/model", tier: "simple", verdict: "bad", note: "" });
+			expect(ledger.trust("vendor/model", undefined, "coding")!.successRate).toBeCloseTo(withBad, 9);
+			// Off: task is ignored and every verdict pools.
+			c.filters.feedbackByTask = false;
+			expect(ledger.trust("vendor/model", undefined, "coding")!.successRate).toBeCloseTo((18 - 6 + 1) / (18 + 2), 9);
+		} finally {
+			db.close();
+		}
+	});
+});

package/test/turn.test.ts

@@ -46,7 +46,7 @@ function mkConfig(escalation: Partial<EscalationConfig> = {}): RouterConfig {
 			data: { axis: "intelligence", minQuality: 0 },
 			chat: { axis: "intelligence", minQuality: 0 },
 		},
-		filters: { allow: [], deny: [], includeFree: false, requireToolSupport: true, minTrust: 0.6, feedbackWeight: 0, minTrustSamples: 5, trustScopedByHarness: false, trustWindowDays: 0, contextHeadroom: 1.2, latencyWeight: 0, latencyReferenceMs: 5000, latencyReferenceTokensPerSec: 30, cacheReliabilityMinSamples: 10, latencyMinSamples: 20, escalationCostWeight: 0 },
+		filters: { allow: [], deny: [], includeFree: false, requireToolSupport: true, minTrust: 0.6, feedbackWeight: 0, feedbackByTask: false, minTrustSamples: 5, trustScopedByHarness: false, trustWindowDays: 0, contextHeadroom: 1.2, latencyWeight: 0, latencyReferenceMs: 5000, latencyReferenceTokensPerSec: 30, cacheReliabilityMinSamples: 10, latencyMinSamples: 20, escalationCostWeight: 0 },
 		classifier: {
 			ambiguityThreshold: 0,
 			model: "test/adjudicator", learnedModelPath: "",
@@ -74,9 +74,10 @@ function mkConfig(escalation: Partial<EscalationConfig> = {}): RouterConfig {
 		exploration: { enabled: false, rates: {}, stickyPolicy: "never", holdTurns: { enabled: false, values: [2, 3, 4] } },
 		cache: { injectBreakpoints: true, maxBreakpoints: 4, minPromptTokens: 1024, milestoneTokens: 20_000 },
 		context: { enabled: false, baseUrl: "", token: "", defaultScope: "", timeoutMs: 3_000, maxStalenessMs: 900_000, maxBlockChars: 24_000, memoryLimit: 8, docsLimit: 2, sessionLimit: 6, briefChars: 0, recordTurns: false, maxQueue: 64 },
-		compaction: { enabled: false, budgetTokens: 40_000, floorRatio: 1, fitToWindow: true, protectRecentTurns: 4, maxToolResultBytes: 4_096, keepHeadBytes: 512, keepTailBytes: 512, elideSupersededReads: true, collapseDuplicateResults: true, replanGrowthRatio: 1 },
+		compaction: { enabled: false, budgetTokens: 40_000, floorRatio: 1, fitToWindow: true, protectRecentTurns: 4, maxToolResultBytes: 4_096, keepHeadBytes: 512, keepTailBytes: 512, elideSupersededReads: true, collapseDuplicateResults: true, replanGrowthRatio: 1, digestToolResults: false, digestMaxPerTurn: 2 },
 		budget: { onExceeded: "downgrade" },
-		report: { baselines: [] },
+		report: { baselines: [], dailySummary: false },
+		digest: { enabled: false, minBytes: 12_000, maxBytes: 400_000, tools: ["read"], fromTier: "moderate", tier: "simple", model: "", maxOutputTokens: 700, maxCostUsd: 0.02, timeoutMs: 25_000 },
 		profiles: [],
 		ledger: { path: ":memory:", blendWindowDays: 7, blendMinSamples: 20, fallbackBlend: { inputPerMtok: 1, outputPerMtok: 4 }, conversationTtlMs: 86_400_000 },
 		adaptiveTierFloors: true,
@@ -929,3 +930,68 @@ describe("latency measurement covers the work the router actually does", () => {
 	});
 
 });
+
+describe("summarising compaction in a turn", () => {
+	function reqWithTool(): NormRequest {
+		const content = "line\n".repeat(400);
+		const req = mkReq();
+		req.messages = [
+			{ role: "user", text: "fix it", images: 0, textBytes: 6, toolCalls: [] },
+			{ role: "assistant", text: "", images: 0, textBytes: 0, toolCalls: [{ id: "c1", name: "read", argsJson: '{"path":"a.ts"}' }] },
+			{ role: "tool", text: content, images: 0, textBytes: Buffer.byteLength(content), toolCalls: [], toolCallId: "c1" },
+			{ role: "user", text: "go on", images: 0, textBytes: 5, toolCalls: [] },
+		];
+		req.promptBytes = req.messages.reduce((s, m) => s + m.textBytes, 0);
+		return req;
+	}
+	function decisionWithPlan(): Decision {
+		const d = mkDecision("hard", "dear/model", { escalateTo: null });
+		d.compactionPlan = [{ index: 2, mode: "truncate", keepHead: 100, keepTail: 100, note: "large read result", bytes: 2_000 }];
+		d.compactionSavedBytes = 1_700;
+		return d;
+	}
+
+	test("new edits are digested with the turn's tier, persisted with the plan, and not paid twice on a retry", async () => {
+		const seen: { toolName: string; tier?: string; source?: string; content: string; input: Record<string, unknown> }[] = [];
+		const digester = {
+			digest: async (r: { toolName: string; tier?: string; source?: string; content: string; input: Record<string, unknown> }) => {
+				seen.push(r);
+				return { digested: true as const, text: "[digest] the file", model: "cheap/model", usd: 0.0001, inputBytes: r.content.length, outputChars: 17, ms: 5 };
+			},
+		};
+		// First attempt is probe-rejected and escalates; the second dispatches. Both decisions carry the same new edit.
+		const { router } = mkRouter([decisionWithPlan(), decisionWithPlan()]);
+		const { upstream } = mkUpstream([
+			{ kind: "chunks", chunks: [startChunk("dear/model"), textChunk("I'm sorry, but I can't help with that request."), finishChunk("stop")] },
+			{ kind: "chunks", chunks: [startChunk("dear/model"), textChunk("done"), finishChunk("stop"), usageChunk({ promptTokens: 50, completionTokens: 2 }, 0.001)] },
+		]);
+		const { ledger } = mkLedger();
+		const { store, map } = mkConversations();
+		const { sink, errors } = mkSink();
+		const cfg = mkConfig();
+		cfg.compaction.digestToolResults = true;
+		cfg.escalation.maxAttempts = 2;
+
+		await runTurn(reqWithTool(), sink, { config: cfg, router, upstream, ledger, conversations: store, catalog, context: createDisabledBridge(), digester }, new AbortController().signal);
+
+		expect(errors).toHaveLength(0);
+		expect(seen).toHaveLength(1);
+		expect(seen[0]).toMatchObject({ toolName: "read", tier: "hard", source: "compaction", input: { path: "a.ts" } });
+		expect(seen[0]!.content.startsWith("line\n")).toBe(true);
+		const plan = map.get("conv-test")!.compactionPlan!;
+		expect(plan[0]?.digest).toBe("[digest] the file");
+	});
+
+	test("without compaction.digestToolResults the digester is never consulted", async () => {
+		let calls = 0;
+		const digester = { digest: async () => { calls++; return { digested: false as const, reason: "n/a" }; } };
+		const { router } = mkRouter([decisionWithPlan()]);
+		const { upstream } = mkUpstream([{ kind: "chunks", chunks: [startChunk("dear/model"), textChunk("done"), finishChunk("stop")] }]);
+		const { ledger } = mkLedger();
+		const { store, map } = mkConversations();
+		const { sink } = mkSink();
+		await runTurn(reqWithTool(), sink, { config: mkConfig(), router, upstream, ledger, conversations: store, catalog, context: createDisabledBridge(), digester }, new AbortController().signal);
+		expect(calls).toBe(0);
+		expect(map.get("conv-test")!.compactionPlan![0]?.digest).toBeUndefined();
+	});
+});

package/tools/train-classifier.ts

@@ -6,10 +6,15 @@
  *   bun tools/train-classifier.ts --dry-run       # evaluate only
  *   bun tools/train-classifier.ts --days 30       # bound the training window
  *   bun tools/train-classifier.ts --out path.json
+ *   bun tools/train-classifier.ts --label feedback  # learn from /router good|bad verdicts
  *
- * Label: the served turn escalated — a cheaper attempt was probe-rejected
- * first (attempt > 0). Split: the oldest 80% train, the newest 20% test, so
- * the score is what the model would have done on turns it had not seen.
+ * Label (default `escalation`): the served turn escalated — a cheaper attempt
+ * was probe-rejected first (attempt > 0). With `--label feedback` the rows are
+ * the turns a person judged with /router good|bad and the positive class is
+ * `bad`: what a user rejected, which a probe cannot see. Verdicts are scarce,
+ * so this needs at least FEEDBACK_MIN_ROWS of them and prints the count.
+ * Split: the oldest 80% train, the newest 20% test, so the score is what the
+ * model would have done on turns it had not seen.
  * Prints holdout AUC for the learned model and for the heuristic's own score,
  * plus precision/recall at a few thresholds. Read-only on the ledger.
  *
@@ -20,7 +25,7 @@
 import { Database } from "bun:sqlite";
 import { dirname, join } from "node:path";
 import { loadConfig } from "../src/config/load.ts";
-import { auc, FEATURE_NAMES, LEARNED_MODEL_VERSION, learnedVector, type LearnedModel, trainLogistic } from "../src/router/learned.ts";
+import { auc, FEATURE_NAMES, LEARNED_MODEL_VERSION, type LearnedLabel, learnedVector, type LearnedModel, trainLogistic } from "../src/router/learned.ts";
 import type { Features } from "../src/router/types.ts";
 
 const argv = process.argv.slice(2);
@@ -29,6 +34,14 @@ const flag = (name: string): string | undefined => {
 	return i >= 0 ? argv[i + 1] : undefined;
 };
 const dryRun = argv.includes("--dry-run");
+const labelFlag = flag("--label") ?? "escalation";
+if (labelFlag !== "escalation" && labelFlag !== "feedback") {
+	console.error(`--label must be escalation or feedback, not "${labelFlag}"`);
+	process.exit(2);
+}
+const label: LearnedLabel = labelFlag;
+/** Verdict rows are a person's time; a fit on fewer than this is noise dressed as a model. */
+const FEEDBACK_MIN_ROWS = 100;
 const days = Number.parseInt(flag("--days") ?? "0", 10);
 const cfg = loadConfig({});
 const out = flag("--out") ?? join(dirname(cfg.ledger.path), "classifier-learned.json");
@@ -38,28 +51,70 @@ const since = days > 0 ? Date.now() - days * 86_400_000 : 0;
 // For the heuristic baseline on an escalated turn, the score that MATTERS is
 // the one the rejected first attempt was routed on; the served row carries the
 // escalation classification (score 1 by construction), which would leak the label.
-const rows = db
-	.query(
-		`SELECT l.features, l.attempt,
-			COALESCE((SELECT w.score FROM ledger w WHERE w.conversation_key = l.conversation_key AND w.turn = l.turn AND w.attempt = 0 AND w.wasted = 1 LIMIT 1), l.score) AS score
-		 FROM ledger l
-		 WHERE l.features IS NOT NULL AND l.wasted = 0 AND l.error IS NULL AND l.created_at_ms >= ?
-		 ORDER BY l.created_at_ms ASC`,
-	)
-	.all(since) as { features: string; attempt: number; score: number | null }[];
+interface TrainRow {
+	features: string;
+	attempt: number;
+	score: number | null;
+	/** Feedback label only: the person's verdict on this turn. */
+	verdict?: string;
+}
+function feedbackRows(): TrainRow[] {
+	try {
+		return db
+			.query(
+					`SELECT l.features, l.attempt, l.score, f.verdict
+					 FROM feedback f JOIN ledger l ON l.id = f.ledger_id
+					 WHERE l.features IS NOT NULL AND f.created_at_ms >= ?
+				 ORDER BY f.created_at_ms ASC`,
+			)
+			.all(since) as TrainRow[];
+	} catch (err) {
+		if (String(err).includes("no such table: feedback")) {
+			console.error("this ledger has no feedback table yet (a router older than v0.4.1 wrote it); restart the router once, then judge some turns");
+			process.exit(2);
+		}
+		throw err;
+	}
+}
+const rows =
+	label === "feedback"
+		? feedbackRows()
+		: (db
+				.query(
+					`SELECT l.features, l.attempt,
+						COALESCE((SELECT w.score FROM ledger w WHERE w.conversation_key = l.conversation_key AND w.turn = l.turn AND w.attempt = 0 AND w.wasted = 1 LIMIT 1), l.score) AS score
+					 FROM ledger l
+					 WHERE l.features IS NOT NULL AND l.wasted = 0 AND l.error IS NULL AND l.created_at_ms >= ?
+					 ORDER BY l.created_at_ms ASC`,
+				)
+				.all(since) as TrainRow[]);
 db.close();
-if (rows.length < 200) {
-	console.error(`only ${rows.length} rows with features; need a few hundred to fit anything`);
+const minRows = label === "feedback" ? FEEDBACK_MIN_ROWS : 200;
+if (rows.length < minRows) {
+	console.error(
+		label === "feedback"
+			? `only ${rows.length} judged turns with features (need ${minRows}); keep using /router good|bad and try again later`
+			: `only ${rows.length} rows with features; need a few hundred to fit anything`,
+	);
 	process.exit(2);
 }
 
 const xs = rows.map((r) => learnedVector(JSON.parse(r.features) as Partial<Features>));
-const ys: number[] = rows.map((r) => (r.attempt > 0 ? 1 : 0));
+const ys: number[] = rows.map((r) => (label === "feedback" ? (r.verdict === "bad" ? 1 : 0) : r.attempt > 0 ? 1 : 0));
+const positiveName = label === "feedback" ? "bad" : "escalate";
+if (label === "feedback") {
+	const bad = ys.reduce((s, y) => s + y, 0);
+	if (bad === 0 || bad === ys.length) {
+		console.error(`all ${ys.length} verdicts are ${bad === 0 ? "good" : "bad"}; a label with one class cannot be learned`);
+		process.exit(2);
+	}
+}
 const heuristic = rows.map((r) => r.score ?? 0);
 const split = Math.floor(rows.length * 0.8);
 const fit = trainLogistic(xs.slice(0, split), ys.slice(0, split));
 const model: LearnedModel = {
 	version: LEARNED_MODEL_VERSION,
+	label,
 	trainedAtMs: Date.now(),
 	rows: rows.length,
 	positives: ys.reduce((s, y) => s + y, 0),
@@ -82,9 +137,9 @@ const testP = testX.map(score);
 model.auc = auc(testP, testY);
 const heuristicAuc = auc(heuristic.slice(split), testY);
 
-console.log(`rows ${rows.length} (train ${split}, test ${rows.length - split}); positives ${model.positives} (${((100 * model.positives) / rows.length).toFixed(2)}%)`);
+console.log(`label ${label}: rows ${rows.length} (train ${split}, test ${rows.length - split}); positives (${positiveName}) ${model.positives} (${((100 * model.positives) / rows.length).toFixed(2)}%)`);
 console.log(`holdout AUC: learned ${model.auc.toFixed(3)}   heuristic score ${heuristicAuc.toFixed(3)}`);
-console.log("\nprecision / recall on the holdout at p(escalate) thresholds:");
+console.log(`\nprecision / recall on the holdout at p(${positiveName}) thresholds:`);
 for (const t of [0.5, 0.7, 0.8, 0.9]) {
 	let tp = 0;
 	let fp = 0;
@@ -99,7 +154,7 @@ for (const t of [0.5, 0.7, 0.8, 0.9]) {
 	const rec = tp + fn > 0 ? tp / (tp + fn) : 0;
 	console.log(`  p ≥ ${t.toFixed(1)}  flagged ${String(tp + fp).padStart(5)}  precision ${(100 * prec).toFixed(1).padStart(5)}%  recall ${(100 * rec).toFixed(1).padStart(5)}%`);
 }
-console.log("\nweights (standardised; positive ⇒ more likely to escalate):");
+console.log(`\nweights (standardised; positive ⇒ more likely ${positiveName}):`);
 const ranked = model.names.map((n, i) => [n, model.weights[i]!] as const).sort((a, b) => Math.abs(b[1]) - Math.abs(a[1]));
 for (const [n, w] of ranked.slice(0, 12)) console.log(`  ${n.padEnd(30)} ${w >= 0 ? "+" : ""}${w.toFixed(3)}`);
 
@@ -107,5 +162,5 @@ if (dryRun) {
 	console.log("\ndry run: nothing written");
 } else {
 	await Bun.write(out, JSON.stringify(model, null, 1));
-	console.log(`\nwrote ${out}\nset classifier.learnedModelPath to it to record learned risk on every decision (advisory).`);
+	console.log(`\nwrote ${out}\nset classifier.learnedModelPath to it to record learned: p(${positiveName}) on every decision (advisory).`);
 }