auto-model-router 0.3.3 → 0.3.4

package/.omp-plugin/marketplace.json

@@ -7,14 +7,14 @@
   },
   "metadata": {
     "description": "auto-model-router: a local cost/complexity-aware model router for Oh My Pi, backed by OpenRouter",
-    "version": "0.3.3",
+    "version": "0.3.4",
     "pluginRoot": "."
   },
   "plugins": [
     {
       "name": "auto-model-router",
       "description": "Local cost/complexity-aware model router for Oh My Pi, backed by OpenRouter. Runs in-process, routes per turn by price and task complexity, with budget caps, mid-stream escalation, and cache-aware hysteresis.",
-      "version": "0.3.3",
+      "version": "0.3.4",
       "author": {
         "name": "drewappling",
         "email": "drewappling@gmail.com"

package/README.md

@@ -709,6 +709,7 @@ event has already arrived is treated as a completed turn, not an error.
 | `holdTurnsAfterEscalation` | `4` | Hold longer after an escalation. |
 | `switchMargin` | `1.3` | Switching must beat the warm-cache discount by this factor. Lower = switch away from a warm model more readily. |
 | `switchHorizonTurns` | `1` | Turns the stay/switch comparison is amortised over: `H × stayWarm` vs `switchCold + (H − 1) × newWarm`. `1` is the one-turn comparison, which can keep a dear model warm indefinitely when the cheaper winner is itself dear cold; a small `H` lets a switch that pays for itself within a few turns go ahead. |
+| `confirmUpgradesBelowConfidence` | `0.6` | A heuristic tier upgrade classified below this confidence waits one turn while the current model's cache is warm; a second consecutive upgrade classification confirms it. Escalations, explicit high reasoning and failing tool loops bypass the wait. `0` disables. Measured: 65 of 67 moderate→hard upgrades in a week bounced back within 3 turns, each paying a cold hard-tier read of a ~120k prompt. |
 | `cacheWarmTtlMs` | `300000` (5 min) | How long a model's prompt cache is considered warm. |
 | `maxDowngradePerTurn` | `1` | Max tiers a turn may drop in one step (avoids quality cliffs). |
 | `breakHoldOnMechanical` | `false` | Let a tool-result continuation that classifies *below* the held tier escape the hold (still bounded by `maxDowngradePerTurn`). Worth enabling when the held tier is expensive. |

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "auto-model-router",
-  "version": "0.3.3",
+  "version": "0.3.4",
   "private": false,
   "description": "Local cost/complexity-aware model router for Oh My Pi, backed by OpenRouter",
   "type": "module",

package/src/cli/config-wizard.ts

@@ -232,6 +232,7 @@ export const WIZARD_SECTIONS: readonly SectionSpec[] = [
 			{ path: "hysteresis.holdTurnsAfterEscalation", label: "Hold turns after escalation", kind: "number", min: 0 },
 			{ path: "hysteresis.switchMargin", label: "Switch margin", kind: "number", min: 0 },
 			{ path: "hysteresis.switchHorizonTurns", label: "Switch horizon", kind: "number", min: 1, hint: "turns amortised" },
+			{ path: "hysteresis.confirmUpgradesBelowConfidence", label: "Confirm upgrades below confidence", kind: "number", min: 0, max: 1, hint: "0=off; low-confidence tier-ups wait a turn" },
 			{ path: "hysteresis.cacheWarmTtlMs", label: "Cache-warm TTL", kind: "number", min: 0, hint: "ms" },
 			{ path: "hysteresis.maxDowngradePerTurn", label: "Max downgrade per turn", kind: "number", min: 0, hint: "tiers" },
 			{ path: "hysteresis.breakHoldOnMechanical", label: "Break hold on mechanical turns", kind: "boolean" },

package/src/config/defaults.ts

@@ -174,6 +174,9 @@ export const DEFAULT_CONFIG: RouterConfig = {
 		// 1 = the shipped one-turn comparison. Raise to amortise a switch over the
 		// turns that follow it; see HysteresisConfig.switchHorizonTurns.
 		switchHorizonTurns: 1,
+		// Low-confidence heuristic upgrades from a warm model wait one turn; see
+		// HysteresisConfig.confirmUpgradesBelowConfidence for the measurement.
+		confirmUpgradesBelowConfidence: 0.6,
 		// OpenRouter sticky sessions expire in 5-10 minutes.
 		cacheWarmTtlMs: 300_000,
 		maxDowngradePerTurn: 1,

package/src/config/schema.ts

@@ -136,6 +136,7 @@ const hysteresis = z.strictObject({
 	holdTurnsAfterEscalation: z.number().int().nonnegative().optional(),
 	switchMargin: z.number().positive().optional(),
 	switchHorizonTurns: z.number().int().positive().optional(),
+	confirmUpgradesBelowConfidence: z.number().min(0).max(1).optional(),
 	cacheWarmTtlMs: z.number().nonnegative().optional(),
 	maxDowngradePerTurn: z.number().int().nonnegative().optional(),
 	breakHoldOnMechanical: z.boolean().optional(),

package/src/config/types.ts

@@ -392,6 +392,19 @@ export interface HysteresisConfig {
 	 * dispatches per user-visible turn, so single digits are conservative.
 	 */
 	switchHorizonTurns: number;
+	/**
+	 * A heuristic tier UPGRADE whose classification confidence is below this
+	 * waits one turn when the current model's cache is warm; a second
+	 * consecutive upgrade classification confirms it. 0 disables. Escalations,
+	 * explicit high reasoning and failing tool loops bypass the wait.
+	 *
+	 * Measured on 7 days of live traffic: 65 of 67 moderate→hard upgrades
+	 * bounced back within 3 turns, 50 of them below 0.6 confidence, costing
+	 * $17.90 in cold hard-tier prompt reads against $0.23 for staying warm.
+	 * The stay/switch comparison never sees these because the warm cheap
+	 * model is below the new tier's floor.
+	 */
+	confirmUpgradesBelowConfidence: number;
 	/** Assume a warm cache expires after this long. OpenRouter sticky sessions: 5-10 min. */
 	cacheWarmTtlMs: number;
 	/** Downgrade at most this many tiers per turn, so quality never falls off a cliff. */

package/src/router/select.ts

@@ -102,6 +102,12 @@ function wideningOrder(tier: Tier, minTier: Tier, maxTier: Tier): Tier[] {
 	return out;
 }
 
+/** Evidence that an upgrade will stick: the conversation is failing or asked for deep reasoning. */
+function hardSignal(f: Features): boolean {
+	const reasoning: string = f.requestedReasoning ?? "";
+	return f.lastToolFailed || f.circularToolCall || f.repeatedToolCall || reasoning === "high" || reasoning === "xhigh" || reasoning === "max";
+}
+
 export function select(args: SelectArgs): Decision {
 	const { req, features, classification, profile, state, snapshot, ledger, cfg, nowMs } = args;
 	const reasons: string[] = [];
@@ -161,6 +167,42 @@ export function select(args: SelectArgs): Decision {
 	// exploration (2c) and candidate building (3) so both agree on the term.
 	const cacheWarm = state.cacheWarmSlug !== null && nowMs - state.cacheWarmAtMs <= cfg.hysteresis.cacheWarmTtlMs;
 
+	// 2a. Cache-aware upgrade confirmation. A low-confidence heuristic upgrade
+	//     from a warm model waits one turn; the next turn's classification
+	//     confirms or forgets it. Measured on 7 days of live traffic: 65 of 67
+	//     moderate→hard upgrades bounced back within 3 turns, 50 of them below
+	//     0.6 confidence, and each paid a cold hard-tier read of a ~120k prompt
+	//     ($17.90 in total against $0.23 for staying warm). Step 4's stay/switch
+	//     comparison never sees these — the warm cheap model is below the new
+	//     tier's floor, so it is not a candidate there. Escalations, explicit
+	//     high reasoning and failing tool loops bypass the wait: those are the
+	//     upgrades that stick.
+	let upgradeDeferred: Tier | null = null;
+	const confirmBelow = cfg.hysteresis.confirmUpgradesBelowConfidence;
+	if (
+		confirmBelow > 0 &&
+		cls.source === "heuristic" &&
+		classification.confidence < confirmBelow &&
+		state.currentTier !== null &&
+		state.currentSlug !== null &&
+		tierIdx(effective) > tierIdx(clampTier(state.currentTier)) &&
+		cacheWarm &&
+		state.cacheWarmSlug === state.currentSlug &&
+		(args.excludeSlugs === undefined || args.excludeSlugs.length === 0) &&
+		!hardSignal(features)
+	) {
+		const held = clampTier(state.currentTier);
+		if (state.upgradeDeferredTier !== undefined && state.upgradeDeferredTier !== null) {
+			reasons.push(`upgrade ${held} → ${effective} confirmed: classified above ${held} on consecutive turns`);
+		} else {
+			reasons.push(
+				`upgrade ${held} → ${effective} deferred one turn: heuristic confidence ${classification.confidence.toFixed(2)} < ${confirmBelow} with ${state.currentSlug} warm`,
+			);
+			upgradeDeferred = effective;
+			effective = held;
+		}
+	}
+
 	// 2b. Context compaction: shrink stale tool output before dispatch when the
 	//     prompt exceeds the token budget (or would overflow the profile window).
 	//     Deterministic and content-only (never removes a message), so downstream
@@ -531,5 +573,6 @@ export function select(args: SelectArgs): Decision {
 		reasons,
 		explored,
 		budgetDowngraded,
+		upgradeDeferred,
 	};
 }

package/src/router/state.ts

@@ -32,6 +32,7 @@ interface Row {
 	context_fetched_at_ms: number;
 	compaction_plan: string | null;
 	compaction_plan_tokens: number;
+	upgrade_deferred_tier: string | null;
 	updated_at_ms: number;
 }
 
@@ -53,6 +54,7 @@ function toState(row: Row): ConversationState {
 		contextFetchedAtMs: row.context_fetched_at_ms,
 		compactionPlan: row.compaction_plan === null ? null : (JSON.parse(row.compaction_plan) as CompactionEdit[]),
 		compactionPlanTokens: row.compaction_plan_tokens,
+		upgradeDeferredTier: row.upgrade_deferred_tier as Tier | null,
 		updatedAtMs: row.updated_at_ms,
 	};
 }
@@ -71,10 +73,10 @@ export function createConversationStore(db: Database): ConversationStore {
 		INSERT INTO conversations (
 			key, session_id, turn, current_slug, current_tier, sticky_until_turn,
 			last_prompt_tokens, cache_warm_slug, cache_warm_at_ms,
-			context_version, context_fetched_at_ms, compaction_plan, compaction_plan_tokens, updated_at_ms
+			context_version, context_fetched_at_ms, compaction_plan, compaction_plan_tokens, upgrade_deferred_tier, updated_at_ms
 		) VALUES ($key, $sessionId, $turn, $currentSlug, $currentTier, $stickyUntilTurn,
 			$lastPromptTokens, $cacheWarmSlug, $cacheWarmAtMs,
-			$contextVersion, $contextFetchedAtMs, $compactionPlan, $compactionPlanTokens, $updatedAtMs)
+			$contextVersion, $contextFetchedAtMs, $compactionPlan, $compactionPlanTokens, $upgradeDeferredTier, $updatedAtMs)
 		ON CONFLICT(key) DO UPDATE SET
 			session_id = excluded.session_id,
 			turn = excluded.turn,
@@ -88,6 +90,7 @@ export function createConversationStore(db: Database): ConversationStore {
 			context_fetched_at_ms = excluded.context_fetched_at_ms,
 			compaction_plan = excluded.compaction_plan,
 			compaction_plan_tokens = excluded.compaction_plan_tokens,
+			upgrade_deferred_tier = excluded.upgrade_deferred_tier,
 			updated_at_ms = excluded.updated_at_ms
 	`);
 	// Read-modify-write in JS lost money: an aborted or failed dispatch is still
@@ -137,6 +140,7 @@ export function createConversationStore(db: Database): ConversationStore {
 				$lastPromptTokens: state.lastPromptTokens,
 				$compactionPlan: state.compactionPlan === null ? null : JSON.stringify(state.compactionPlan),
 				$compactionPlanTokens: state.compactionPlanTokens ?? 0,
+				$upgradeDeferredTier: state.upgradeDeferredTier ?? null,
 				$cacheWarmSlug: state.cacheWarmSlug,
 				$cacheWarmAtMs: state.cacheWarmAtMs,
 				$contextVersion: state.contextVersion,

package/src/router/types.ts

@@ -186,6 +186,13 @@ export interface ConversationState {
 	compactionPlanTokens?: number;
 	/** When that block was fetched, for the staleness TTL. */
 	contextFetchedAtMs: number;
+	/**
+	 * The tier a low-confidence upgrade was deferred to on the previous turn
+	 * (`hysteresis.confirmUpgradesBelowConfidence`), or null. One-turn memory:
+	 * every turn overwrites it, so a second consecutive upgrade classification
+	 * confirms the switch and anything else forgets it.
+	 */
+	upgradeDeferredTier?: Tier | null;
 	updatedAtMs: number;
 }
 
@@ -270,6 +277,8 @@ export interface Decision {
 	explored: Exploration | null;
 	/** Budget guard forced a cheaper tier than the classifier asked for. */
 	budgetDowngraded: boolean;
+	/** A low-confidence upgrade to this tier was deferred one turn to keep the warm model. */
+	upgradeDeferred: Tier | null;
 }
 
 /** Why a guarded probe rejected an attempt. */

package/src/server/turn.ts

@@ -567,6 +567,8 @@ export async function runTurn(
 		// shrunk so the prompt cache survives and the savings compound.
 		state.compactionPlan = decision.compactionPlan.length > 0 ? decision.compactionPlan : null;
 		state.compactionPlanTokens = decision.compactionPlanTokens;
+		// One-turn memory: a deferred upgrade is confirmed or forgotten next turn.
+		state.upgradeDeferredTier = decision.upgradeDeferred;
 		if (usage.cachedTokens > 0 || usage.cacheWriteTokens > 0) {
 			// Non-zero cache traffic is direct evidence the upstream cache exists.
 			state.cacheWarmSlug = servedSlug ?? decision.slug;

package/src/util/sqlite.ts

@@ -18,7 +18,7 @@ import { mkdirSync } from "node:fs";
 import { dirname } from "node:path";
 
 /** Bump when a migration is added; guarded below so reopening never regresses it. */
-const USER_VERSION = 16;
+const USER_VERSION = 17;
 
 const MIGRATIONS = `
 CREATE TABLE IF NOT EXISTS catalog_cache (
@@ -243,6 +243,12 @@ ALTER TABLE conversations ADD COLUMN compaction_plan_tokens INTEGER NOT NULL DEF
 DELETE FROM token_calibration;
 `;
 
+// v17: one-turn memory of a deferred low-confidence upgrade
+// (hysteresis.confirmUpgradesBelowConfidence).
+const MIGRATE_V17 = `
+ALTER TABLE conversations ADD COLUMN upgrade_deferred_tier TEXT;
+`;
+
 // v9: benchmark_cache holds the external benchmark feeds (Artificial Analysis,
 // BenchLM) that backfill quality scores OpenRouter leaves unpublished. It is a
 // whole new table, created idempotently by the MIGRATIONS block above, so there
@@ -288,6 +294,7 @@ export function openDb(path: string): Database {
 		if (!convCols.some((c) => c.name === "context_version")) db.exec(MIGRATE_V11);
 		if (!convCols.some((c) => c.name === "compaction_plan")) db.exec(MIGRATE_V13);
 		if (!convCols.some((c) => c.name === "compaction_plan_tokens")) db.exec(MIGRATE_V16);
+		if (!convCols.some((c) => c.name === "upgrade_deferred_tier")) db.exec(MIGRATE_V17);
 		db.exec(`PRAGMA user_version = ${USER_VERSION}`);
 	}
 	return db;

package/test/failover.test.ts

@@ -70,7 +70,7 @@ function mkConfig(escalation: Partial<EscalationConfig> = {}): RouterConfig {
 			escalateOnLengthStop: false,
 			...escalation,
 		},
-		hysteresis: { holdTurns: 2, holdTurnsAfterEscalation: 4, switchMargin: 1.5, cacheWarmTtlMs: 600_000, maxDowngradePerTurn: 1, breakHoldOnMechanical: false, switchHorizonTurns: 1 },
+		hysteresis: { holdTurns: 2, holdTurnsAfterEscalation: 4, switchMargin: 1.5, cacheWarmTtlMs: 600_000, maxDowngradePerTurn: 1, breakHoldOnMechanical: false, switchHorizonTurns: 1, confirmUpgradesBelowConfidence: 0.6 },
 		exploration: { enabled: false, rates: {}, stickyPolicy: "never", holdTurns: { enabled: false, values: [2, 3, 4] } },
 		cache: { injectBreakpoints: true, maxBreakpoints: 4, minPromptTokens: 1024, milestoneTokens: 20_000 },
 		context: { enabled: false, baseUrl: "", token: "", defaultScope: "", timeoutMs: 3_000, maxStalenessMs: 900_000, maxBlockChars: 24_000, memoryLimit: 8, docsLimit: 2, sessionLimit: 6, briefChars: 0, recordTurns: false, maxQueue: 64 },
@@ -160,6 +160,7 @@ function mkDecision(tier: Tier, slug: string, probe: Partial<ProbePlan> = {}): D
 		reasons: ["test decision"],
 		explored: null,
 		budgetDowngraded: false,
+		upgradeDeferred: null,
 	};
 }
 

package/test/select.test.ts

@@ -996,3 +996,57 @@ describe("compaction.replanGrowthRatio (review 2026-09-05 §7)", () => {
 	});
 });
 
+
+describe("hysteresis.confirmUpgradesBelowConfidence", () => {
+	// A low-confidence heuristic upgrade from a warm model waits one turn.
+	// Measured: 65 of 67 moderate→hard upgrades in a week bounced back within
+	// 3 turns, each paying a cold hard-tier read of a ~120k prompt.
+	const warmSlug = run({ tier: "moderate" }).slug;
+	function upgrade(opts: { confidence?: number; source?: "heuristic" | "escalation"; st?: Partial<ConversationState>; cfg?: RouterConfig; lastToolFailed?: boolean }) {
+		const cfg = opts.cfg ?? BASE;
+		const req = request("now rework the whole scheduler");
+		const base = extractFeatures(req, 120_000);
+		const features = opts.lastToolFailed === true ? { ...base, lastToolFailed: true } : base;
+		const heuristic = scoreHeuristic(features, cfg);
+		return select({
+			req,
+			features,
+			classification: { ...heuristic, tier: "hard", confidence: opts.confidence ?? 0.45, source: opts.source ?? "heuristic" },
+			profile: PROFILE,
+			state: state({ turn: 4, currentTier: "moderate", currentSlug: warmSlug, cacheWarmSlug: warmSlug, cacheWarmAtMs: Date.now(), lastPromptTokens: 110_000, ...opts.st }),
+			snapshot: SNAPSHOT,
+			ledger: null,
+			cfg,
+			nowMs: Date.now(),
+		});
+	}
+
+	test("a low-confidence upgrade from a warm model is deferred to the held tier", () => {
+		const d = upgrade({});
+		expect(d.tier).toBe("moderate");
+		expect(d.upgradeDeferred).toBe("hard");
+		expect(d.reasons.some((r) => r.includes("upgrade moderate → hard deferred one turn"))).toBe(true);
+	});
+
+	test("a second consecutive upgrade classification confirms it", () => {
+		const d = upgrade({ st: { upgradeDeferredTier: "hard" } });
+		expect(d.tier).toBe("hard");
+		expect(d.upgradeDeferred).toBeNull();
+		expect(d.reasons.some((r) => r.includes("upgrade moderate → hard confirmed"))).toBe(true);
+	});
+
+	test("confident classifications, cold caches, escalations, failing tools and the off switch all upgrade at once", () => {
+		expect(upgrade({ confidence: 0.9 }).tier).toBe("hard");
+		expect(upgrade({ st: { cacheWarmAtMs: Date.now() - 3_600_000 } }).tier).toBe("hard");
+		expect(upgrade({ source: "escalation" }).tier).toBe("hard");
+		expect(upgrade({ lastToolFailed: true }).tier).toBe("hard");
+		const off: RouterConfig = { ...BASE, hysteresis: { ...BASE.hysteresis, confirmUpgradesBelowConfidence: 0 } };
+		expect(upgrade({ cfg: off }).tier).toBe("hard");
+		for (const d of [upgrade({ confidence: 0.9 }), upgrade({ source: "escalation" })]) expect(d.upgradeDeferred).toBeNull();
+	});
+
+	test("a downgrade or a same-tier turn is never deferred", () => {
+		const d = run({ tier: "simple", st: state({ turn: 4, currentTier: "moderate", currentSlug: warmSlug, cacheWarmSlug: warmSlug, cacheWarmAtMs: Date.now() }) });
+		expect(d.upgradeDeferred).toBeNull();
+	});
+});

package/test/trust-attribution.test.ts

@@ -3,6 +3,7 @@ import { describe, expect, test } from "bun:test";
 import { loadConfig } from "../src/config/load.ts";
 import { createLedger, LATENCY_WINDOW_ROWS } from "../src/cost/ledger.ts";
 import { EMPTY_USAGE, type LedgerEntry } from "../src/cost/types.ts";
+import { createConversationStore } from "../src/router/state.ts";
 import { openDb } from "../src/util/sqlite.ts";
 
 const cfg = loadConfig({});
@@ -244,11 +245,24 @@ describe("v4 migration", () => {
 		}
 	});
 
-	test("schema is at user_version 16", () => {
+	test("a deferred upgrade tier survives a save/load round trip", () => {
+		const db = openDb(":memory:");
+		const store = createConversationStore(db);
+		const st = store.load("conv-defer");
+		st.upgradeDeferredTier = "hard";
+		store.save(st);
+		expect(store.load("conv-defer").upgradeDeferredTier).toBe("hard");
+		st.upgradeDeferredTier = null;
+		store.save(st);
+		expect(store.load("conv-defer").upgradeDeferredTier).toBeNull();
+		db.close();
+	});
+
+	test("schema is at user_version 17", () => {
 		const db = openDb(":memory:");
 		try {
 			const row = db.query("PRAGMA user_version").get() as { user_version: number };
-			expect(row.user_version).toBe(16);
+			expect(row.user_version).toBe(17);
 		} finally {
 			db.close();
 		}

package/test/turn.test.ts

@@ -70,7 +70,7 @@ function mkConfig(escalation: Partial<EscalationConfig> = {}): RouterConfig {
 			escalateOnLengthStop: false,
 			...escalation,
 		},
-		hysteresis: { holdTurns: 2, holdTurnsAfterEscalation: 4, switchMargin: 1.5, cacheWarmTtlMs: 600_000, maxDowngradePerTurn: 1, breakHoldOnMechanical: false, switchHorizonTurns: 1 },
+		hysteresis: { holdTurns: 2, holdTurnsAfterEscalation: 4, switchMargin: 1.5, cacheWarmTtlMs: 600_000, maxDowngradePerTurn: 1, breakHoldOnMechanical: false, switchHorizonTurns: 1, confirmUpgradesBelowConfidence: 0.6 },
 		exploration: { enabled: false, rates: {}, stickyPolicy: "never", holdTurns: { enabled: false, values: [2, 3, 4] } },
 		cache: { injectBreakpoints: true, maxBreakpoints: 4, minPromptTokens: 1024, milestoneTokens: 20_000 },
 		context: { enabled: false, baseUrl: "", token: "", defaultScope: "", timeoutMs: 3_000, maxStalenessMs: 900_000, maxBlockChars: 24_000, memoryLimit: 8, docsLimit: 2, sessionLimit: 6, briefChars: 0, recordTurns: false, maxQueue: 64 },
@@ -160,6 +160,7 @@ function mkDecision(tier: Tier, slug: string, probe: Partial<ProbePlan> = {}): D
 		reasons: ["test decision"],
 		explored: null,
 		budgetDowngraded: false,
+		upgradeDeferred: null,
 	};
 }