auto-model-router 0.25.0 → 0.26.0

package/.omp-plugin/marketplace.json

@@ -7,14 +7,14 @@
   },
   "metadata": {
     "description": "auto-model-router: a local cost/complexity-aware model router for Oh My Pi, backed by OpenRouter",
-    "version": "0.25.0",
+    "version": "0.26.0",
     "pluginRoot": "."
   },
   "plugins": [
     {
       "name": "auto-model-router",
       "description": "Local cost/complexity-aware model router for Oh My Pi, backed by OpenRouter. Runs in-process, routes per turn by price and task complexity, with budget caps, mid-stream escalation, and cache-aware hysteresis.",
-      "version": "0.25.0",
+      "version": "0.26.0",
       "author": {
         "name": "drewappling",
         "email": "drewappling@gmail.com"

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "auto-model-router",
-  "version": "0.25.0",
+  "version": "0.26.0",
   "private": false,
   "description": "Local cost/complexity-aware model router for Oh My Pi, backed by OpenRouter",
   "type": "module",

package/src/cli/config-wizard.ts

@@ -182,6 +182,8 @@ export const WIZARD_SECTIONS: readonly SectionSpec[] = [
 			{ path: "filters.deny", label: "Deny globs", kind: "stringArray", hint: "comma-separated" },
 			{ path: "filters.includeFree", label: "Include free models", kind: "boolean" },
 			{ path: "filters.requireToolSupport", label: "Require tool support", kind: "boolean" },
+			{ path: "filters.agenticAxisForToolTurns", label: "Rank tool turns on the agentic axis", kind: "boolean", hint: "a tool loop is won on tool-driving ability, not the task's own axis" },
+			{ path: "filters.minAgenticForToolTurns", label: "Min agentic score for tool turns", kind: "number", min: 0, max: 100, hint: "its own scale, not a tier floor; 0 disables. Models with no agentic score are not filtered" },
 			{ path: "filters.minTrust", label: "Min trust", kind: "number", min: 0, max: 1 },
 			{ path: "filters.feedbackWeight", label: "Feedback weight in trust", kind: "number", min: 0, hint: "0=record only; a bad verdict = this many failures" },
 			{ path: "filters.feedbackByTask", label: "Scope verdicts to the task type", kind: "boolean" },

package/src/config/defaults.ts

@@ -103,6 +103,8 @@ export const DEFAULT_CONFIG: RouterConfig = {
 		// Free models are rate-limited hard enough that retries cost more than they save.
 		includeFree: false,
 		requireToolSupport: true,
+		agenticAxisForToolTurns: true,
+		minAgenticForToolTurns: 25,
 		minTrust: 0.7,
 		// Verdicts are recorded and reported first; weigh them once there are some.
 		feedbackWeight: 0,

package/src/config/schema.ts

@@ -128,6 +128,8 @@ const filters = z.strictObject({
 	deny: z.array(z.string()).optional(),
 	includeFree: z.boolean().optional(),
 	requireToolSupport: z.boolean().optional(),
+	agenticAxisForToolTurns: z.boolean().optional(),
+	minAgenticForToolTurns: z.number().min(0).max(100).optional(),
 	minTrust: z.number().min(0).max(1).optional(),
 	feedbackWeight: z.number().nonnegative().optional(),
 	feedbackByTask: z.boolean().optional(),

package/src/config/types.ts

@@ -221,6 +221,26 @@ export interface FilterConfig {
 	includeFree: boolean;
 	/** Require `supported_parameters` to include `tools` whenever the request offers tools. */
 	requireToolSupport: boolean;
+	/**
+	 * Score a turn that carries tools on the `agentic` axis instead of its task's axis.
+	 * On by default: a tool loop is won or lost on tool-driving ability, and `chat` and
+	 * `documentation` score on `intelligence`, which does not measure it. Off restores
+	 * the task's own axis for every turn.
+	 */
+	agenticAxisForToolTurns: boolean;
+	/**
+	 * Minimum `agentic` score a model needs to be offered a turn that carries tools, 0-100.
+	 * The cheap tiers rank with `qualityExponent: 0` — cheapest above the floor — so on those
+	 * tiers the ranking axis is inert and only a floor keeps a tool-incapable model out.
+	 *
+	 * Judged on its own scale, NOT against a tier's `minQuality`: agentic scores run far lower
+	 * than coding and intelligence, so reusing a tier floor here empties the catalog. Measured:
+	 * 25 drops gpt-oss-20b (1.4) and gemma-3-12b (0.1) while leaving 14 models under $0.30/Mtok.
+	 *
+	 * A model that publishes NO agentic score is not filtered — only 103 of 223 tool-capable
+	 * models carry one, so rejecting the unscored would discard half the catalog. 0 disables.
+	 */
+	minAgenticForToolTurns: number;
 	/**
 	 * Smallest completion budget a REASONING model is dispatched with, tokens.
 	 *

package/src/router/candidates.ts

@@ -169,7 +169,15 @@ export function buildCandidates(args: BuildCandidatesArgs): { candidates: Candid
 	const minContext = Math.ceil(features.promptTokens * filters.contextHeadroom) + expectedCompletionTokens;
 	// Task selects the quality axis and capability filters; the tier still
 	// bounds cost.
-	const effectiveAxis = taskCfg.axis;
+	//
+	// A turn that carries tools is scored on `agentic`, whatever the task's own axis says.
+	// `chat` and `documentation` score on `intelligence`, which says nothing about whether a
+	// model can drive a tool loop: measured here, ollama/gpt-oss:20b (intelligence 9, agentic
+	// 1.4) won a tool-bearing trivial turn on price because every candidate sat under the
+	// tier floor and the adaptive band then relaxed it. The `agentic` score is already in the
+	// catalog for every model and was the one axis nothing routed on.
+	const toolTurn = req.tools.length > 0;
+	const effectiveAxis: QualityAxis = toolTurn && filters.agenticAxisForToolTurns ? "agentic" : taskCfg.axis;
 	// Two floors with different meanings, and only one of them may be relaxed:
 	//  - the TIER floor is an economic envelope tuned against the full catalog,
 	//    so when a guardrail narrows availability below it, relaxing to the
@@ -180,7 +188,7 @@ export function buildCandidates(args: BuildCandidatesArgs): { candidates: Candid
 	const plan = cfg.adaptiveTierFloors || cfg.adaptivePriceCeilings ? tierPlanFor(snapshot, cfg) : null;
 	const adaptiveTierFloor =
 		cfg.adaptiveTierFloors && plan !== null
-			? effectiveQualityFloor(tierCfg.minQuality, tier, effectiveAxis, plan)
+			? effectiveQualityFloor(tierCfg.minQuality, tier, taskCfg.axis, plan)
 			: tierCfg.minQuality;
 	const qualityFloor = Math.max(taskFloor, adaptiveTierFloor);
 	// Input-price ceiling: catalog-derived band when adaptive, else the fixed config.
@@ -229,6 +237,20 @@ export function buildCandidates(args: BuildCandidatesArgs): { candidates: Candid
 			rejected.push({ slug, reason: "no_tool_support" });
 			continue;
 		}
+		// A tool loop needs a model that can drive one. The cheap tiers rank with
+		// `qualityExponent: 0` — cheapest above the floor — so there the ranking axis decides
+		// nothing and only this keeps a tool-incapable model out: ollama/gpt-oss:20b (agentic
+		// 1.4) was winning tool-bearing trivial turns purely on price. Judged on the agentic
+		// scale rather than a tier floor, and never applied to a model that publishes no
+		// agentic score, since most of the catalog does not. Relaxed with the rest under
+		// tier rescue, so a narrowed catalog still gets a turn.
+		if (toolTurn && !relaxQuality && filters.minAgenticForToolTurns > 0) {
+			const agentic = model.quality.agentic;
+			if (agentic !== undefined && agentic < filters.minAgenticForToolTurns) {
+				rejected.push({ slug, reason: "below_quality_floor", detail: `agentic ${agentic} < ${filters.minAgenticForToolTurns} for a tool turn` });
+				continue;
+			}
+		}
 		if ((req.hasImages || taskCfg.requireImage === true) && !model.inputModalities.includes("image")) {
 			rejected.push({ slug, reason: "no_image_support" });
 			continue;
@@ -239,17 +261,28 @@ export function buildCandidates(args: BuildCandidatesArgs): { candidates: Candid
 		}
 
 		const pinned = tierCfg.pin.includes(slug) || taskPins.includes(slug);
-		const quality = resolveQuality(model, effectiveAxis);
+		// Floors are judged on the TASK's axis, always. The `agentic` scores sit on a lower
+		// scale than coding and intelligence (glm-5.3-flash: coding 71.5, agentic 51.2), so
+		// reusing one numeric floor across axes empties the set — measured here, a floor of
+		// 60 on `agentic` admitted nothing, tier rescue then dropped the floor entirely and
+		// the WEAKEST model won. The axis switch below therefore reorders candidates without
+		// touching who is eligible.
+		const quality = resolveQuality(model, taskCfg.axis);
 		if (!pinned && !relaxQuality && qualityFloor > 0) {
 			if (quality === null) {
 				rejected.push({ slug, reason: "below_quality_floor", detail: "no published quality score" });
 				continue;
 			}
 			if (quality.score < qualityFloor) {
-				rejected.push({ slug, reason: "below_quality_floor", detail: `${quality.score} < floor ${qualityFloor}` });
+				rejected.push({ slug, reason: "below_quality_floor", detail: `${quality.score} < floor ${qualityFloor} on ${quality.axis}` });
 				continue;
 			}
 		}
+		// What the candidate is RANKED on: a tool-bearing turn is won or lost on tool-driving
+		// ability, and `chat`/`documentation` score on `intelligence`, which does not measure
+		// it. Ranking is relative within the admitted set, so a lower-scaled axis is safe here
+		// in a way a floor is not.
+		const rankQuality = effectiveAxis === taskCfg.axis ? quality : resolveQuality(model, effectiveAxis);
 
 		// Price ceilings at the ACTUAL prompt size: long-context overrides can
 		// push a model over the ceiling exactly when conversations get long.
@@ -352,11 +385,10 @@ export function buildCandidates(args: BuildCandidatesArgs): { candidates: Candid
 			images,
 		});
 		const trustScore = trust !== null && trust.attempts > 0 ? trust.successRate : UNMEASURED_TRUST;
-		const qualityScore = quality?.score ?? 0;
 		// Shared scoring: trust converts flakiness into money — a model failing
 		// 20% of the time really costs ~25% more in retries. Latency does the same
 		// for slowness (TTFT over the reference). qualityExponent 0 makes this
-		// "cheapest above the floor"; the floor does the quality work.
+		const qualityScore = rankQuality?.score ?? 0;
 		const latencyMult = latencyMultiplier(latency, filters, expectedCompletionTokens, latencyWeightFor(filters, features.isToolResultContinuation));
 		// Escalation-cost term: the trust divisor prices a failure as a retry of
 		// THIS model, but a probe escalation re-dispatches the whole prompt on
@@ -380,9 +412,9 @@ export function buildCandidates(args: BuildCandidatesArgs): { candidates: Candid
 		const score = 0;
 
 		const reasons: string[] = [
-			quality === null
+			rankQuality === null
 				? "unscored on every quality axis"
-				: `quality ${quality.score} on ${quality.axis}${quality.axis === effectiveAxis ? "" : ` (fallback from ${effectiveAxis})`}`,
+				: `quality ${rankQuality.score} on ${rankQuality.axis}${rankQuality.axis === effectiveAxis ? "" : ` (fallback from ${effectiveAxis})`}${effectiveAxis === taskCfg.axis ? "" : ` — ranked on ${effectiveAxis} because the turn carries tools`}`,
 			trust === null || trust.attempts === 0
 				? `trust unmeasured: neutral prior ${UNMEASURED_TRUST}`
 				: `trust ${trustScore.toFixed(2)} over ${trust.attempts} attempts`,

package/test/failover.test.ts

@@ -47,7 +47,7 @@ function mkConfig(escalation: Partial<EscalationConfig> = {}): RouterConfig {
 			data: { axis: "intelligence", minQuality: 0 },
 			chat: { axis: "intelligence", minQuality: 0 },
 		},
-		filters: { allow: [], deny: [], includeFree: false, requireToolSupport: true, minTrust: 0.6, feedbackWeight: 0, feedbackByTask: false, minTrustSamples: 5, trustScopedByHarness: false, trustWindowDays: 0, contextHeadroom: 1.2, reasoningCompletionFloor: 0, latencyWeight: 0, latencyReferenceMs: 5000, latencyReferenceTokensPerSec: 30, cacheReliabilityMinSamples: 10, latencyMinSamples: 20, escalationCostWeight: 0 },
+		filters: { allow: [], deny: [], includeFree: false, requireToolSupport: true, agenticAxisForToolTurns: true, minAgenticForToolTurns: 0, minTrust: 0.6, feedbackWeight: 0, feedbackByTask: false, minTrustSamples: 5, trustScopedByHarness: false, trustWindowDays: 0, contextHeadroom: 1.2, reasoningCompletionFloor: 0, latencyWeight: 0, latencyReferenceMs: 5000, latencyReferenceTokensPerSec: 30, cacheReliabilityMinSamples: 10, latencyMinSamples: 20, escalationCostWeight: 0 },
 		classifier: {
 			ambiguityThreshold: 0,
 			model: "test/adjudicator", learnedModelPath: "",

package/test/select.test.ts

@@ -389,7 +389,10 @@ describe("decision shape", () => {
 	test("carries the session id, features, and a reasoning trail", () => {
 		const d = run({ tier: "simple" });
 		expect(d.sessionId.startsWith("omp-")).toBe(true);
-		expect(d.reasons.length).toBeGreaterThan(0);
+		// `d.reasons` holds decision-level notes — a widening, a hysteresis hold — and is
+		// legitimately empty when a tier serves the turn without incident. The trail that is
+		// always present is the per-candidate one, so that is what a caller can rely on.
+		expect(d.considered[0]!.reasons.length).toBeGreaterThan(0);
 		expect(d.features.toolCount).toBe(1);
 		expect(d.considered.length).toBeGreaterThan(0);
 	});

package/test/tier-plan.test.ts

@@ -374,6 +374,39 @@ describe("adaptive price ceilings", () => {
 		expect(on.candidates.map((c) => c.model.slug)).toContain("a/4");
 		expect(on.rejected.some((r) => r.slug === "a/4")).toBe(false);
 	});
+
+	test("a tool turn excludes tool-incapable models on the cheap tiers and ranks on agentic above them", () => {
+		// Two Ollama-priced models: the cheap one cannot drive a tool loop (agentic 1.4, as
+		// ollama/gpt-oss:20b really scores), the dearer one can (agentic 51.2, glm-5.3-flash).
+		const mk = (slug: string, price: number, intelligence: number, agentic: number) => ({
+			slug, canonicalSlug: slug, name: slug, provider: "ollama", vendor: "ollama",
+			contextLength: 131072, maxCompletionTokens: 32000, supportsTools: true, supportsReasoning: false,
+			reasoningMandatory: false, supportsToolChoice: true, inputModalities: ["text" as const],
+			price: { prompt: price / 1e6, completion: price / 1e6 }, priceTiers: [],
+			quality: { intelligence, coding: intelligence, agentic }, tokenizer: "Other",
+			isFree: false, createdAtMs: 0, author: "ollama",
+		});
+		const weak = mk("ollama/weak", 0.07, 9, 1.4);
+		const capable = mk("ollama/capable", 0.15, 41.9, 51.2);
+		const snap = { models: [weak, capable], fetchedAtMs: Date.now(), keyScoped: false };
+		const run = (tier: "trivial" | "moderate", min: number) =>
+			buildCandidates({
+				req, features, tier, task: "chat", snapshot: snap, ledger: null,
+				cfg: { ...BASE, filters: { ...BASE.filters, minAgenticForToolTurns: min } },
+				expectedCompletionTokens: 512, warmSlug: null,
+			});
+		// `trivial` ranks with qualityExponent 0 — cheapest above the floor — so the axis
+		// decides nothing there and only the agentic floor keeps the weak model out.
+		expect(run("trivial", 0).candidates[0]!.model.slug).toBe("ollama/weak");
+		const floored = run("trivial", 25);
+		expect(floored.candidates.map((c) => c.model.slug)).toEqual(["ollama/capable"]);
+		expect(floored.rejected.some((x) => x.slug === "ollama/weak" && (x.detail ?? "").includes("agentic 1.4 < 25"))).toBe(true);
+		// Above the cheap tiers quality does carry weight, and there the turn is ranked on
+		// agentic: the capable model leads even though it costs more than twice as much.
+		const ranked = run("moderate", 0);
+		expect(ranked.candidates[0]!.model.slug).toBe("ollama/capable");
+		expect(ranked.candidates[0]!.reasons.join(" ")).toContain("ranked on agentic");
+	});
 });
 
 describe("quality normalization and capability floor (benchmark findings 4/6)", () => {

package/test/turn.test.ts

@@ -49,7 +49,7 @@ function mkConfig(escalation: Partial<EscalationConfig> = {}): RouterConfig {
 			data: { axis: "intelligence", minQuality: 0 },
 			chat: { axis: "intelligence", minQuality: 0 },
 		},
-		filters: { allow: [], deny: [], includeFree: false, requireToolSupport: true, minTrust: 0.6, feedbackWeight: 0, feedbackByTask: false, minTrustSamples: 5, trustScopedByHarness: false, trustWindowDays: 0, contextHeadroom: 1.2, reasoningCompletionFloor: 0, latencyWeight: 0, latencyReferenceMs: 5000, latencyReferenceTokensPerSec: 30, cacheReliabilityMinSamples: 10, latencyMinSamples: 20, escalationCostWeight: 0 },
+		filters: { allow: [], deny: [], includeFree: false, requireToolSupport: true, agenticAxisForToolTurns: true, minAgenticForToolTurns: 0, minTrust: 0.6, feedbackWeight: 0, feedbackByTask: false, minTrustSamples: 5, trustScopedByHarness: false, trustWindowDays: 0, contextHeadroom: 1.2, reasoningCompletionFloor: 0, latencyWeight: 0, latencyReferenceMs: 5000, latencyReferenceTokensPerSec: 30, cacheReliabilityMinSamples: 10, latencyMinSamples: 20, escalationCostWeight: 0 },
 		classifier: {
 			ambiguityThreshold: 0,
 			model: "test/adjudicator", learnedModelPath: "",