auto-model-router 0.4.2 → 0.4.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.gitattributes +2 -0
- package/.omp-plugin/marketplace.json +2 -2
- package/README.md +31 -3
- package/bunfig.toml +2 -0
- package/omp-extension/report-logic.ts +46 -0
- package/omp-extension/router-configure.ts +55 -3
- package/package.json +1 -1
- package/src/catalog/composite.ts +4 -1
- package/src/cli/config-wizard.ts +8 -1
- package/src/config/defaults.ts +8 -0
- package/src/config/hot-reload.ts +58 -9
- package/src/config/schema.ts +5 -1
- package/src/config/types.ts +34 -0
- package/src/cost/ledger.ts +84 -8
- package/src/cost/report.ts +34 -4
- package/src/cost/summary.ts +231 -0
- package/src/cost/types.ts +35 -3
- package/src/router/candidates.ts +1 -1
- package/src/router/classify.ts +2 -2
- package/src/router/compaction.ts +2 -1
- package/src/router/learned.ts +11 -1
- package/src/router/select.ts +27 -3
- package/src/server/compaction-digest.ts +129 -0
- package/src/server/digest.ts +68 -4
- package/src/server/http.ts +50 -7
- package/src/server/providers.ts +1 -0
- package/src/server/turn.ts +38 -1
- package/src/util/sqlite.ts +7 -0
- package/src/wire/openai/request.ts +4 -0
- package/src/wire/types.ts +7 -0
- package/test/cache-control.test.ts +1 -1
- package/test/compaction.test.ts +40 -3
- package/test/digest.test.ts +44 -0
- package/test/failover.test.ts +4 -4
- package/test/hot-reload.test.ts +37 -1
- package/test/learned.test.ts +21 -1
- package/test/migrations.test.ts +84 -0
- package/test/report-hub.test.ts +1 -1
- package/test/report-logic.test.ts +11 -1
- package/test/report.test.ts +29 -1
- package/test/select.test.ts +26 -2
- package/test/summary.test.ts +171 -0
- package/test/support/preload.ts +19 -0
- package/test/tokens.test.ts +68 -0
- package/test/trust-attribution.test.ts +37 -0
- package/test/turn.test.ts +69 -4
- package/tools/gen-migration-fixtures.ts +69 -0
- package/tools/train-classifier.ts +75 -20
package/.gitattributes
ADDED
|
@@ -7,14 +7,14 @@
|
|
|
7
7
|
},
|
|
8
8
|
"metadata": {
|
|
9
9
|
"description": "auto-model-router: a local cost/complexity-aware model router for Oh My Pi, backed by OpenRouter",
|
|
10
|
-
"version": "0.4.
|
|
10
|
+
"version": "0.4.4",
|
|
11
11
|
"pluginRoot": "."
|
|
12
12
|
},
|
|
13
13
|
"plugins": [
|
|
14
14
|
{
|
|
15
15
|
"name": "auto-model-router",
|
|
16
16
|
"description": "Local cost/complexity-aware model router for Oh My Pi, backed by OpenRouter. Runs in-process, routes per turn by price and task complexity, with budget caps, mid-stream escalation, and cache-aware hysteresis.",
|
|
17
|
-
"version": "0.4.
|
|
17
|
+
"version": "0.4.4",
|
|
18
18
|
"author": {
|
|
19
19
|
"name": "drewappling",
|
|
20
20
|
"email": "drewappling@gmail.com"
|
package/README.md
CHANGED
|
@@ -497,6 +497,8 @@ escalation signal, error. Three views aggregate it, all from the same
|
|
|
497
497
|
unreachable.
|
|
498
498
|
- `auto-model-router report --days 7 [--harness <id>] [--json]` on the terminal.
|
|
499
499
|
- `GET /v1/router/report?days=7&harness=<id>` for dashboards.
|
|
500
|
+
- `GET /v1/router/summary?harness=<id>` — the daily summary as JSON (`auto=1`
|
|
501
|
+
applies the once-a-day gate and returns `due: false` when nothing is due).
|
|
500
502
|
|
|
501
503
|
What it shows, for the window:
|
|
502
504
|
|
|
@@ -510,6 +512,18 @@ What it shows, for the window:
|
|
|
510
512
|
| by day | UTC calendar days: dispatches, spend, cache hit |
|
|
511
513
|
| same traffic on one model | the window's tokens priced on each `report.baselines` model at list price with the window's cache hit rate, and what share the router saved against it |
|
|
512
514
|
|
|
515
|
+
**Soft-failure spikes.** `/health` (`softFailures.spikes`), `/router status`
|
|
516
|
+
and the daily summary list any model whose failure rate over the last hour —
|
|
517
|
+
probe rejections such as `empty_completion` or `repeat_tool_call` that
|
|
518
|
+
OpenRouter counts as success, plus attributable transport errors — is at
|
|
519
|
+
least 25%, at least twice its own rate over the preceding 7 days, and covers
|
|
520
|
+
at least 5 dispatches with 3 failures. This is visibility only: two weeks of
|
|
521
|
+
ledger data showed soft failures do not cluster tightly enough for a breaker
|
|
522
|
+
to save money (after a burst, the next 15 minutes ran 84–1,577 successes per
|
|
523
|
+
13–50 failures), and OpenRouter's provider failover plus the router's own
|
|
524
|
+
escalation already cover the retry. Use a spike as the cue to `/router pin`
|
|
525
|
+
or deny a model for the session.
|
|
526
|
+
|
|
513
527
|
Spend follows the ledger's rule — the provider's reported cost when it gave
|
|
514
528
|
one, else the usage-priced figure the router computed, else the forecast.
|
|
515
529
|
Speed uses only clean streamed rows (TTFT recorded, no error); tokens/s is
|
|
@@ -534,7 +548,8 @@ Status); the subcommands go straight there:
|
|
|
534
548
|
| --- | --- |
|
|
535
549
|
| `/router config` | Section picker over **every** config key: Server, OpenRouter, Ollama Cloud, Benchmarks, Tiers, Tasks, Filters, Classifier, Escalation, Hysteresis, Exploration, Cache, Compaction, Context (agentdox), Budget, Ledger, Logging, Profiles. Only `ollama.prices` and `ollama.twins` (maps) stay YAML-only. |
|
|
536
550
|
| `/router report` | Usage analytics in a fullscreen hub styled like `/models`: pick a view in the sidebar, set the window (24h / 7d / 30d / 90d) and the harness scope there too. `/router report 30d --all` presets them. See [Usage reports](#usage-reports). |
|
|
537
|
-
| `/router
|
|
551
|
+
| `/router summary [--all]` | The last 24 hours in a few lines: spend against the day before, turns and conversations, cache hit, escalations, errors, model switches with tier moves, top models, savings against the first `report.baselines` model, digests and subagent spend, soft-failure spikes, and the Ollama meter with its runway. Posted automatically once a day at session start when `report.dailySummary` is on (the router keeps a per-harness marker, so several omp windows show it once between them, and a day with no turns and no spikes is skipped). |
|
|
552
|
+
| `/router status` | The router's `/health`: key sources, catalog size and age, Ollama availability, plan usage and cost bias, soft-failure spikes (below), agentdox bridge. |
|
|
538
553
|
| `/router why` | Explain this session's last routed turn: model and provider, tier, classification source and confidence, cost, cache hit, latency, the full decision trail and classifier reasons, any feedback already given. |
|
|
539
554
|
| `/router good` / `/router bad [note]` | Judge that turn. Recorded against the model that served it (`POST /v1/router/feedback`), shown per model in the report's `feedback` column, and the label the de-escalation work needs. `/router feedback good\|bad` is the same. |
|
|
540
555
|
| `/router pin <model\|off>` | Route this session to one model until cleared (admitted past price, quality and trust filters; tool support and context window still apply). Escalations and failovers after the first attempt still run. |
|
|
@@ -547,7 +562,9 @@ as the placeholder — empty input keeps it, `-` clears an optional field,
|
|
|
547
562
|
credentials show as `set`/`unset` and are never echoed. `Save and exit` writes the merged config
|
|
548
563
|
(schema-checked and backed up first). Tier, task, filter, classifier,
|
|
549
564
|
hysteresis, exploration, compaction, cache and budget changes hot-reload;
|
|
550
|
-
restart omp for `server
|
|
565
|
+
restart omp for `server` (except `subagentProfile`), `openrouter`, `context`,
|
|
566
|
+
`ledger.path` and the Ollama connection keys; `ollama.costBias`,
|
|
567
|
+
`ollama.biasUntilUsage` and `ledger.retentionDays` hot-reload too.
|
|
551
568
|
|
|
552
569
|
### Via `auto-model-router config` (text wizard / CLI)
|
|
553
570
|
|
|
@@ -668,6 +685,7 @@ Each task (`coding`, `vision`, `documentation`, `data`, `chat`) is a
|
|
|
668
685
|
| `includeFree` | `false` | Include free models (rate-limited hard; usually excluded). |
|
|
669
686
|
| `requireToolSupport` | `true` | Only models that support tool calls. |
|
|
670
687
|
| `feedbackWeight` | `0` | How much a `/router good\|bad` verdict weighs in a model's trust rate: a bad verdict counts as this many failures, a good one as this many successes. `0` records verdicts without acting on them. |
|
|
688
|
+
| `feedbackByTask` | `false` | Count a verdict only when routing the same task type as the judged turn (coding, vision, documentation, data, chat), so a model that codes well but explains badly keeps its coding trust. Verdicts on turns with no recorded task count everywhere. |
|
|
671
689
|
| `minTrust` | `0.7` | Minimum success rate; models below this (after `minTrustSamples`) are demoted. |
|
|
672
690
|
| `minTrustSamples` | `12` | Attempts before trust is enforced. |
|
|
673
691
|
| `trustScopedByHarness` | `false` | `true` = each harness reads only its own trust rows. |
|
|
@@ -684,7 +702,7 @@ Each task (`coding`, `vision`, `documentation`, `data`, `chat`) is a
|
|
|
684
702
|
| Key | Default | Meaning |
|
|
685
703
|
| --- | --- | --- |
|
|
686
704
|
| `ambiguityThreshold` | `0.6` | Below this heuristic confidence, the adjudicator model decides the tier. |
|
|
687
|
-
| `learnedModelPath` | unset | A model written by `bun tools/train-classifier.ts` (logistic regression over the ledger's recorded features
|
|
705
|
+
| `learnedModelPath` | unset | A model written by `bun tools/train-classifier.ts` (logistic regression over the ledger's recorded features; label = the turn escalated, or with `--label feedback` the turn was judged bad via `/router bad`). When set, every decision records `learned: p(escalate)=…` or `learned: p(bad)=…`. Advisory only: it never moves a tier until replay shows it should. |
|
|
688
706
|
| `model` | `qwen/qwen3.7-flash` | Adjudicator model slug. |
|
|
689
707
|
| `maxCostFraction` | `0.02` | Adjudicator cost cap as a fraction of the turn's budget. |
|
|
690
708
|
| `maxCostUsd` | `0.002` | Absolute adjudicator cost cap, USD. |
|
|
@@ -748,6 +766,8 @@ shrunk results stay shrunk (rewriting them would break the prompt cache).
|
|
|
748
766
|
| `keepHeadBytes` / `keepTailBytes` | `512` / `512` | Bytes kept around the elision breadcrumb. |
|
|
749
767
|
| `elideSupersededReads` | `true` | Stub an older result when a newer call to the same resource supersedes it. |
|
|
750
768
|
| `collapseDuplicateResults` | `true` | Collapse byte-identical repeated results to a single copy. |
|
|
769
|
+
| `digestToolResults` | `false` | Summarising compaction: when the plan gains an edit, a cheap model (`digest.tier`/`digest.model`, under `digest.maxCostUsd` and `digest.timeoutMs`) digests the tool result instead of it being cut to head+tail or a stub. The digest is stored on the edit, so the dispatched bytes stay identical on later turns and the cache holds. Applies when the turn routed at or above `digest.fromTier`; works without `digest.enabled`. |
|
|
770
|
+
| `digestMaxPerTurn` | `2` | Digests per turn at most (largest results first); the rest of a plan's new edits stay plain until a later turn. |
|
|
751
771
|
|
|
752
772
|
### `cache` — prompt-cache breakpoints
|
|
753
773
|
|
|
@@ -804,11 +824,18 @@ is a ledger row (`requestedModel` `digest`) and the report totals them.
|
|
|
804
824
|
| `maxCostUsd` | `0.02` | Skip when the digest itself would cost more. |
|
|
805
825
|
| `timeoutMs` | `25000` | The raw result stands if the cheap model is slower. |
|
|
806
826
|
|
|
827
|
+
Quality signal: when the agent later calls the same tool with the same
|
|
828
|
+
primary argument (re-reads a digested file, re-runs a digested grep), the
|
|
829
|
+
router marks that digest's ledger row wasted. The report's `digests` line
|
|
830
|
+
shows the re-run rate; a high rate means the digest is dropping what the
|
|
831
|
+
task needed, and `digest.maxOutputTokens` or `digest.model` is the lever.
|
|
832
|
+
|
|
807
833
|
### `report` — usage-report options
|
|
808
834
|
|
|
809
835
|
| Key | Default | Meaning |
|
|
810
836
|
| --- | --- | --- |
|
|
811
837
|
| `baselines` | `anthropic/claude-opus-5`, `anthropic/claude-sonnet-5` | Models the report prices the window's traffic on as a single-model counterfactual. Unknown slugs are skipped. |
|
|
838
|
+
| `dailySummary` | `true` | Post the daily summary (below) into the transcript at the first interactive omp session start of each day. Hot-reloads. |
|
|
812
839
|
|
|
813
840
|
### `ledger` — cost measurement
|
|
814
841
|
|
|
@@ -819,6 +846,7 @@ is a ledger row (`requestedModel` `digest`) and the report totals them.
|
|
|
819
846
|
| `blendMinSamples` | `25` | Turns before the measured blend replaces the fallback. |
|
|
820
847
|
| `fallbackBlend` | input `1.5`, output `7.5` | Pre-measurement blend (USD/Mtok) for omp's cost display. |
|
|
821
848
|
| `conversationTtlMs` | `604800000` (7 d) | Drop conversation state untouched this long. |
|
|
849
|
+
| `retentionDays` | `365` | Delete ledger rows older than this, checked hourly; `0` keeps everything. The ledger grows about 2.5 MB a day under steady use. Freed pages are reused, so the file stops growing rather than shrinking. |
|
|
822
850
|
|
|
823
851
|
### Top-level
|
|
824
852
|
|
package/bunfig.toml
ADDED
|
@@ -6,6 +6,7 @@
|
|
|
6
6
|
*/
|
|
7
7
|
|
|
8
8
|
import type { UsageReport } from "../src/cost/report.ts";
|
|
9
|
+
import type { DailySummary } from "../src/cost/summary.ts";
|
|
9
10
|
|
|
10
11
|
export interface ReportRequest {
|
|
11
12
|
windowDays: number;
|
|
@@ -61,6 +62,44 @@ export async function fetchReport(
|
|
|
61
62
|
return (await res.json()) as UsageReport;
|
|
62
63
|
}
|
|
63
64
|
|
|
65
|
+
/** GETs the daily summary; `auto` asks the router whether one is due today. Throws on any failure. */
|
|
66
|
+
export async function fetchSummary(
|
|
67
|
+
baseUrl: string,
|
|
68
|
+
harnessId: string,
|
|
69
|
+
auto: boolean,
|
|
70
|
+
headers: Record<string, string>,
|
|
71
|
+
fetchImpl: FetchLike = fetch,
|
|
72
|
+
timeoutMs = 5_000,
|
|
73
|
+
): Promise<{ due: boolean; reason?: string; summary: DailySummary | null }> {
|
|
74
|
+
const params = new URLSearchParams();
|
|
75
|
+
if (harnessId !== "") params.set("harness", harnessId);
|
|
76
|
+
if (auto) params.set("auto", "1");
|
|
77
|
+
const q = params.toString();
|
|
78
|
+
const res = await fetchImpl(`${baseUrl}/v1/router/summary${q === "" ? "" : `?${q}`}`, { headers, signal: AbortSignal.timeout(timeoutMs) });
|
|
79
|
+
if (!res.ok) throw new Error(`router returned ${res.status}`);
|
|
80
|
+
return (await res.json()) as { due: boolean; reason?: string; summary: DailySummary | null };
|
|
81
|
+
}
|
|
82
|
+
|
|
83
|
+
/** One spiking model as `/health` reports it. */
|
|
84
|
+
export interface SoftFailureSpikeView {
|
|
85
|
+
slug?: string;
|
|
86
|
+
recentDispatches?: number;
|
|
87
|
+
recentFailures?: number;
|
|
88
|
+
recentRate?: number;
|
|
89
|
+
baselineDispatches?: number;
|
|
90
|
+
baselineRate?: number;
|
|
91
|
+
}
|
|
92
|
+
|
|
93
|
+
/** "slug: 40% of 10 failed in the last 1h (7d baseline 8% of 120)" — one line per spiking model. */
|
|
94
|
+
export function renderSoftFailureSpikes(spikes: readonly SoftFailureSpikeView[] | undefined | null, recentMs = 3_600_000, baselineDays = 7): string[] {
|
|
95
|
+
if (spikes === undefined || spikes === null || spikes.length === 0) return [];
|
|
96
|
+
const window = recentMs >= 3_600_000 ? `${(recentMs / 3_600_000).toFixed(recentMs % 3_600_000 === 0 ? 0 : 1)}h` : `${Math.round(recentMs / 60_000)}m`;
|
|
97
|
+
return spikes.map(
|
|
98
|
+
(s) =>
|
|
99
|
+
`${s.slug ?? "?"}: ${((s.recentRate ?? 0) * 100).toFixed(0)}% of ${s.recentDispatches ?? 0} failed in the last ${window} (${baselineDays}d baseline ${((s.baselineRate ?? 0) * 100).toFixed(0)}% of ${s.baselineDispatches ?? 0})`,
|
|
100
|
+
);
|
|
101
|
+
}
|
|
102
|
+
|
|
64
103
|
/** The subset of `/health` the status view renders. */
|
|
65
104
|
export interface HealthSnapshot {
|
|
66
105
|
status?: string;
|
|
@@ -80,6 +119,7 @@ export interface HealthSnapshot {
|
|
|
80
119
|
runway?: { dailyBurnUsd?: number; creditsLeftUsd?: number; days?: number | null } | null;
|
|
81
120
|
costBias?: { configured?: number; effective?: number; biasUntilUsage?: number };
|
|
82
121
|
} | null;
|
|
122
|
+
softFailures?: { recentMs?: number; baselineDays?: number; spikes?: SoftFailureSpikeView[] } | null;
|
|
83
123
|
catalog?: {
|
|
84
124
|
models?: number;
|
|
85
125
|
ageMs?: number;
|
|
@@ -119,6 +159,12 @@ export function renderStatus(baseUrl: string, h: HealthSnapshot, nowMs = Date.no
|
|
|
119
159
|
const rwText = rw !== undefined && rw !== null ? ` · burn $${(rw.dailyBurnUsd ?? 0).toFixed(2)}/day · ${rw.days === null || rw.days === undefined ? "credits left: unknown burn" : `~${Math.round(rw.days)} days of credits left`}` : "";
|
|
120
160
|
if (o.meter !== undefined && o.meter !== null) out.push(`ollama billing: ${calText}${rwText}`);
|
|
121
161
|
}
|
|
162
|
+
const sf = h.softFailures;
|
|
163
|
+
if (sf !== undefined && sf !== null) {
|
|
164
|
+
const lines = renderSoftFailureSpikes(sf.spikes, sf.recentMs, sf.baselineDays);
|
|
165
|
+
if (lines.length === 0) out.push("soft failures: no model spiking in the last hour");
|
|
166
|
+
else out.push(`soft failures SPIKING (${lines.length}):`, ...lines.map((l) => ` ${l}`));
|
|
167
|
+
}
|
|
122
168
|
const a = h.agentdox;
|
|
123
169
|
out.push(a === undefined || a === null ? "agentdox: off" : `agentdox: ${a.url ?? "?"} scope ${a.defaultScope ?? "?"}${a.recordTurns === true ? " · recording turns" : ""}`);
|
|
124
170
|
return out.join("\n");
|
|
@@ -50,6 +50,7 @@ import { routerConfigPath, writeRouterConfig } from "../src/cli/config-cmd.ts";
|
|
|
50
50
|
import { loadConfig } from "../src/config/load.ts";
|
|
51
51
|
import type { RouterConfig } from "../src/config/types.ts";
|
|
52
52
|
import { buildUsageReport, renderUsageReport, type UsageReport } from "../src/cost/report.ts";
|
|
53
|
+
import { buildDailySummary, renderDailySummary } from "../src/cost/summary.ts";
|
|
53
54
|
import { openDb } from "../src/util/sqlite.ts";
|
|
54
55
|
|
|
55
56
|
import type { ExtensionAPI, ExtensionContext } from "@oh-my-pi/pi-coding-agent";
|
|
@@ -61,6 +62,7 @@ import { ReportHub } from "./report-hub.ts";
|
|
|
61
62
|
import {
|
|
62
63
|
describeOverride,
|
|
63
64
|
fetchReport,
|
|
65
|
+
fetchSummary,
|
|
64
66
|
parseOverrideArgs,
|
|
65
67
|
parseReportArgs,
|
|
66
68
|
renderStatus,
|
|
@@ -77,12 +79,32 @@ const HARNESS_ID = process.env.OMP_HARNESS_ID ?? "";
|
|
|
77
79
|
|
|
78
80
|
/** Custom message type for report/status output in the transcript. */
|
|
79
81
|
const MESSAGE_TYPE = "auto-model-router";
|
|
82
|
+
/** The embedded router boots on session_start too; the auto summary waits for it this long. */
|
|
83
|
+
const SUMMARY_TRIES = 8;
|
|
84
|
+
const SUMMARY_RETRY_MS = 1_500;
|
|
80
85
|
|
|
81
86
|
export default function (pi: ExtensionAPI): void {
|
|
82
87
|
pi.setLabel("auto-model-router");
|
|
83
88
|
|
|
89
|
+
// Once a day, the first interactive session posts yesterday's summary. The
|
|
90
|
+
// router decides whether one is due (report.dailySummary, a per-harness
|
|
91
|
+
// marker), so several windows on one router show it once between them.
|
|
92
|
+
pi.on("session_start", async (_event, ctx) => {
|
|
93
|
+
if (!ctx.hasUI) return;
|
|
94
|
+
for (let i = 0; i < SUMMARY_TRIES; i++) {
|
|
95
|
+
try {
|
|
96
|
+
const r = await fetchSummary(routerBaseUrl(), HARNESS_ID, true, routerAuthHeaders(), fetch, 2_000);
|
|
97
|
+
if (r.due && r.summary !== null) post(pi, renderDailySummary(r.summary));
|
|
98
|
+
return;
|
|
99
|
+
} catch {
|
|
100
|
+
// Router still starting (embed) or absent: try again shortly, then give up quietly.
|
|
101
|
+
await new Promise((resolve) => setTimeout(resolve, SUMMARY_RETRY_MS));
|
|
102
|
+
}
|
|
103
|
+
}
|
|
104
|
+
});
|
|
105
|
+
|
|
84
106
|
pi.registerCommand("router", {
|
|
85
|
-
description: "auto-model-router: configure, usage report, status",
|
|
107
|
+
description: "auto-model-router: configure, usage report, status, daily summary",
|
|
86
108
|
handler: async (args, ctx) => {
|
|
87
109
|
const [verb = "", ...rest] = args.trim().split(/\s+/).filter((t) => t !== "");
|
|
88
110
|
const tail = rest.join(" ");
|
|
@@ -95,6 +117,9 @@ export default function (pi: ExtensionAPI): void {
|
|
|
95
117
|
case "status":
|
|
96
118
|
case "health":
|
|
97
119
|
return status(pi, ctx);
|
|
120
|
+
case "summary":
|
|
121
|
+
case "daily":
|
|
122
|
+
return summary(pi, ctx, tail);
|
|
98
123
|
case "why":
|
|
99
124
|
case "explain":
|
|
100
125
|
return why(pi, ctx);
|
|
@@ -116,20 +141,22 @@ export default function (pi: ExtensionAPI): void {
|
|
|
116
141
|
case "":
|
|
117
142
|
break;
|
|
118
143
|
default:
|
|
119
|
-
ctx.ui.notify(`unknown /router subcommand "${verb}" (config | report | status | why | good | bad | pin | tier)`, "warn");
|
|
144
|
+
ctx.ui.notify(`unknown /router subcommand "${verb}" (config | report | summary | status | why | good | bad | pin | tier)`, "warn");
|
|
120
145
|
return;
|
|
121
146
|
}
|
|
122
147
|
|
|
123
148
|
const chosen = await ctx.ui.select("auto-model-router", [
|
|
124
149
|
{ label: "Configure", description: "edit any router setting" },
|
|
125
150
|
{ label: "Report", description: "usage analytics; window and scope adjustable inside" },
|
|
126
|
-
{ label: "
|
|
151
|
+
{ label: "Summary", description: "the last 24h in a few lines" },
|
|
152
|
+
{ label: "Status", description: "keys, catalog, Ollama, agentdox, soft-failure spikes" },
|
|
127
153
|
{ label: "Why", description: "explain this session's last routed turn" },
|
|
128
154
|
{ label: "Override", description: "pin a model or force a tier for this session" },
|
|
129
155
|
]);
|
|
130
156
|
if (chosen === undefined) return;
|
|
131
157
|
if (chosen === "Configure") return configure(ctx);
|
|
132
158
|
if (chosen === "Status") return status(pi, ctx);
|
|
159
|
+
if (chosen === "Summary") return summary(pi, ctx, "");
|
|
133
160
|
if (chosen === "Report") return report(pi, ctx, "");
|
|
134
161
|
if (chosen === "Why") return why(pi, ctx);
|
|
135
162
|
if (chosen === "Override") return override(ctx, "tier", "");
|
|
@@ -299,6 +326,31 @@ async function override(ctx: ExtensionContext, verb: "pin" | "tier", text: strin
|
|
|
299
326
|
}
|
|
300
327
|
}
|
|
301
328
|
|
|
329
|
+
/** `/router summary [--all]`: the last 24h, from the router or (router down) the ledger directly. */
|
|
330
|
+
async function summary(pi: ExtensionAPI, ctx: ExtensionContext, argText: string): Promise<void> {
|
|
331
|
+
const harnessId = parseReportArgs(argText, HARNESS_ID).harnessId;
|
|
332
|
+
try {
|
|
333
|
+
const r = await fetchSummary(routerBaseUrl(), harnessId, false, routerAuthHeaders());
|
|
334
|
+
if (r.summary !== null) {
|
|
335
|
+
post(pi, renderDailySummary(r.summary));
|
|
336
|
+
return;
|
|
337
|
+
}
|
|
338
|
+
} catch {
|
|
339
|
+
// Fall through to the ledger.
|
|
340
|
+
}
|
|
341
|
+
const cfg = loadConfig();
|
|
342
|
+
if (!existsSync(cfg.ledger.path)) {
|
|
343
|
+
ctx.ui.notify(`router unreachable at ${routerBaseUrl()} and no ledger at ${cfg.ledger.path}`, "error");
|
|
344
|
+
return;
|
|
345
|
+
}
|
|
346
|
+
const db = openDb(cfg.ledger.path);
|
|
347
|
+
try {
|
|
348
|
+
post(pi, `${renderDailySummary(buildDailySummary(db, { harnessId }))}\n(router unreachable: read from the ledger; spikes and the Ollama meter need the router)`);
|
|
349
|
+
} finally {
|
|
350
|
+
db.close();
|
|
351
|
+
}
|
|
352
|
+
}
|
|
353
|
+
|
|
302
354
|
async function status(pi: ExtensionAPI, ctx: ExtensionContext): Promise<void> {
|
|
303
355
|
try {
|
|
304
356
|
post(pi, await loadStatus());
|
package/package.json
CHANGED
package/src/catalog/composite.ts
CHANGED
|
@@ -23,6 +23,8 @@ export interface CompositeBias {
|
|
|
23
23
|
/** Plan usage fraction at which the bias switches off (list price). */
|
|
24
24
|
biasUntilUsage: number;
|
|
25
25
|
usage: OllamaUsageSource;
|
|
26
|
+
/** When given, read on every use instead of the static pair, so a config hot reload applies. */
|
|
27
|
+
live?: () => { costBias: number; biasUntilUsage: number };
|
|
26
28
|
}
|
|
27
29
|
|
|
28
30
|
export function createCompositeCatalog(
|
|
@@ -39,7 +41,8 @@ export function createCompositeCatalog(
|
|
|
39
41
|
|
|
40
42
|
/** The multiplier in force from the latest usage reading (no network). */
|
|
41
43
|
function currentBias(): number {
|
|
42
|
-
|
|
44
|
+
const b = bias.live?.() ?? bias;
|
|
45
|
+
return effectiveOllamaBias(b.costBias, b.biasUntilUsage, bias.usage.peek());
|
|
43
46
|
}
|
|
44
47
|
|
|
45
48
|
function combine(base: CatalogSnapshot, models: readonly CatalogModel[]): CatalogSnapshot {
|
package/src/cli/config-wizard.ts
CHANGED
|
@@ -184,6 +184,7 @@ export const WIZARD_SECTIONS: readonly SectionSpec[] = [
|
|
|
184
184
|
{ path: "filters.requireToolSupport", label: "Require tool support", kind: "boolean" },
|
|
185
185
|
{ path: "filters.minTrust", label: "Min trust", kind: "number", min: 0, max: 1 },
|
|
186
186
|
{ path: "filters.feedbackWeight", label: "Feedback weight in trust", kind: "number", min: 0, hint: "0=record only; a bad verdict = this many failures" },
|
|
187
|
+
{ path: "filters.feedbackByTask", label: "Scope verdicts to the task type", kind: "boolean" },
|
|
187
188
|
{ path: "filters.minTrustSamples", label: "Min trust samples", kind: "number", min: 0 },
|
|
188
189
|
{ path: "filters.trustScopedByHarness", label: "Scope trust per harness", kind: "boolean" },
|
|
189
190
|
{ path: "filters.trustWindowDays", label: "Trust window", kind: "number", min: 0, hint: "days, 0=all time" },
|
|
@@ -280,6 +281,8 @@ export const WIZARD_SECTIONS: readonly SectionSpec[] = [
|
|
|
280
281
|
{ path: "compaction.keepTailBytes", label: "Keep tail bytes", kind: "number", min: 0 },
|
|
281
282
|
{ path: "compaction.elideSupersededReads", label: "Elide superseded reads", kind: "boolean" },
|
|
282
283
|
{ path: "compaction.collapseDuplicateResults", label: "Collapse duplicate results", kind: "boolean" },
|
|
284
|
+
{ path: "compaction.digestToolResults", label: "Digest compacted results with a cheap model", kind: "boolean" },
|
|
285
|
+
{ path: "compaction.digestMaxPerTurn", label: "Digests per turn at most", kind: "number", min: 0 },
|
|
283
286
|
],
|
|
284
287
|
},
|
|
285
288
|
{
|
|
@@ -327,7 +330,10 @@ export const WIZARD_SECTIONS: readonly SectionSpec[] = [
|
|
|
327
330
|
},
|
|
328
331
|
{
|
|
329
332
|
title: "Report",
|
|
330
|
-
fields: [
|
|
333
|
+
fields: [
|
|
334
|
+
{ path: "report.baselines", label: "Counterfactual baseline models", kind: "stringArray", hint: "comma-separated slugs" },
|
|
335
|
+
{ path: "report.dailySummary", label: "Daily summary at session start", kind: "boolean" },
|
|
336
|
+
],
|
|
331
337
|
},
|
|
332
338
|
{
|
|
333
339
|
title: "Ledger",
|
|
@@ -338,6 +344,7 @@ export const WIZARD_SECTIONS: readonly SectionSpec[] = [
|
|
|
338
344
|
{ path: "ledger.fallbackBlend.inputPerMtok", label: "Fallback blend input $/Mtok", kind: "number", min: 0 },
|
|
339
345
|
{ path: "ledger.fallbackBlend.outputPerMtok", label: "Fallback blend output $/Mtok", kind: "number", min: 0 },
|
|
340
346
|
{ path: "ledger.conversationTtlMs", label: "Conversation TTL", kind: "number", min: 1, hint: "ms" },
|
|
347
|
+
{ path: "ledger.retentionDays", label: "Ledger retention", kind: "number", min: 0, hint: "days; 0 keeps everything" },
|
|
341
348
|
],
|
|
342
349
|
},
|
|
343
350
|
{
|
package/src/config/defaults.ts
CHANGED
|
@@ -103,6 +103,7 @@ export const DEFAULT_CONFIG: RouterConfig = {
|
|
|
103
103
|
minTrust: 0.7,
|
|
104
104
|
// Verdicts are recorded and reported first; weigh them once there are some.
|
|
105
105
|
feedbackWeight: 0,
|
|
106
|
+
feedbackByTask: false,
|
|
106
107
|
minTrustSamples: 12,
|
|
107
108
|
// Shared trust by default: more samples, demotion guard stays effective
|
|
108
109
|
// even with a tiny guardrail-narrowed catalog.
|
|
@@ -291,6 +292,10 @@ export const DEFAULT_CONFIG: RouterConfig = {
|
|
|
291
292
|
keepTailBytes: 512,
|
|
292
293
|
elideSupersededReads: true,
|
|
293
294
|
collapseDuplicateResults: true,
|
|
295
|
+
// Off: a synchronous cheap-model call before dispatch, only worth it where
|
|
296
|
+
// stale tool output is the prompt and the turn is on a dear model.
|
|
297
|
+
digestToolResults: false,
|
|
298
|
+
digestMaxPerTurn: 2,
|
|
294
299
|
},
|
|
295
300
|
digest: {
|
|
296
301
|
// Off until an operator turns it on: it changes what the model reads.
|
|
@@ -308,6 +313,8 @@ export const DEFAULT_CONFIG: RouterConfig = {
|
|
|
308
313
|
report: {
|
|
309
314
|
// The frontier pair most omp users would otherwise run on.
|
|
310
315
|
baselines: ["anthropic/claude-opus-5", "anthropic/claude-sonnet-5"],
|
|
316
|
+
// One transcript message per day, at the first interactive session start.
|
|
317
|
+
dailySummary: true,
|
|
311
318
|
},
|
|
312
319
|
budget: {
|
|
313
320
|
// No caps by default; at a configured ceiling, downgrade rather than fail.
|
|
@@ -330,6 +337,7 @@ export const DEFAULT_CONFIG: RouterConfig = {
|
|
|
330
337
|
// so early cost reporting never underreports.
|
|
331
338
|
fallbackBlend: { inputPerMtok: 1.5, outputPerMtok: 7.5 },
|
|
332
339
|
conversationTtlMs: 7 * 24 * 60 * 60 * 1000,
|
|
340
|
+
retentionDays: 365,
|
|
333
341
|
},
|
|
334
342
|
// On by default: an absolute floor that no available model meets is how the
|
|
335
343
|
// router ends up serving every turn from the cheapest tier.
|
package/src/config/hot-reload.ts
CHANGED
|
@@ -90,15 +90,47 @@ export interface WatchConfigOptions {
|
|
|
90
90
|
}
|
|
91
91
|
|
|
92
92
|
/**
|
|
93
|
-
*
|
|
94
|
-
*
|
|
95
|
-
*
|
|
93
|
+
* Config paths captured at construction, so a file edit cannot reach the
|
|
94
|
+
* running process: the socket, the upstream clients, the agentdox bridge, the
|
|
95
|
+
* ledger file. Everything else, including `ollama.costBias`,
|
|
96
|
+
* `ollama.biasUntilUsage`, `server.subagentProfile` and `ledger.retentionDays`,
|
|
97
|
+
* is read at call time and hot-reloads. A bare block name pins the whole
|
|
98
|
+
* block; `block.key` pins one key and lets its siblings through.
|
|
99
|
+
*/
|
|
100
|
+
export const PINNED_CONFIG_PATHS: readonly string[] = [
|
|
101
|
+
"server.host",
|
|
102
|
+
"server.port",
|
|
103
|
+
"server.apiKey",
|
|
104
|
+
"server.harnessId",
|
|
105
|
+
"server.maxConcurrentTurns",
|
|
106
|
+
"openrouter",
|
|
107
|
+
"ollama.enabled",
|
|
108
|
+
"ollama.baseUrl",
|
|
109
|
+
"ollama.apiKey",
|
|
110
|
+
"ollama.timeoutMs",
|
|
111
|
+
"ollama.catalogTtlMs",
|
|
112
|
+
"ollama.includeLocal",
|
|
113
|
+
"ollama.prices",
|
|
114
|
+
"ollama.twins",
|
|
115
|
+
"ollama.usagePollMs",
|
|
116
|
+
"ollama.quotaCooldownMs",
|
|
117
|
+
"ollama.rateLimitCooldownMs",
|
|
118
|
+
"ollama.planCreditsUsd",
|
|
119
|
+
"context",
|
|
120
|
+
"ledger.path",
|
|
121
|
+
];
|
|
122
|
+
|
|
123
|
+
/**
|
|
124
|
+
* Watches `path` and applies valid changes to `live` in place. `frozen`
|
|
125
|
+
* entries are re-copied from `pinned` after every reload so file edits to
|
|
126
|
+
* construction-captured settings cannot silently diverge: a top-level name
|
|
127
|
+
* pins the whole block, `block.key` pins one key of it.
|
|
96
128
|
*/
|
|
97
129
|
export function watchConfig(
|
|
98
130
|
path: string,
|
|
99
131
|
live: RouterConfig,
|
|
100
132
|
pinned: RouterConfig,
|
|
101
|
-
frozen: readonly
|
|
133
|
+
frozen: readonly string[],
|
|
102
134
|
opts: WatchConfigOptions = {},
|
|
103
135
|
): ConfigWatcher {
|
|
104
136
|
let closed = false;
|
|
@@ -119,14 +151,31 @@ export function watchConfig(
|
|
|
119
151
|
}
|
|
120
152
|
lastError = "";
|
|
121
153
|
|
|
122
|
-
const
|
|
154
|
+
const frozenBlocks = new Set(frozen.filter((f) => !f.includes(".")));
|
|
155
|
+
const frozenKeys = new Map<string, string[]>();
|
|
156
|
+
for (const f of frozen) {
|
|
157
|
+
const dot = f.indexOf(".");
|
|
158
|
+
if (dot < 0) continue;
|
|
159
|
+
const block = f.slice(0, dot);
|
|
160
|
+
frozenKeys.set(block, [...(frozenKeys.get(block) ?? []), f.slice(dot + 1)]);
|
|
161
|
+
}
|
|
123
162
|
const changed: string[] = [];
|
|
124
163
|
const next = result.cfg as unknown as Record<string, unknown>;
|
|
164
|
+
const pinnedRec = pinned as unknown as Record<string, unknown>;
|
|
125
165
|
for (const key of Object.keys(next)) {
|
|
126
|
-
// Frozen blocks belong to construction: keep the pinned values.
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
166
|
+
// Frozen blocks belong to construction: keep the pinned values. A
|
|
167
|
+
// partially frozen block takes the file's siblings and the pinned keys.
|
|
168
|
+
let value = frozenBlocks.has(key) ? pinnedRec[key] : next[key];
|
|
169
|
+
const keys = frozenKeys.get(key);
|
|
170
|
+
if (keys !== undefined && !frozenBlocks.has(key) && value !== null && typeof value === "object") {
|
|
171
|
+
const merged: Record<string, unknown> = { ...(value as Record<string, unknown>) };
|
|
172
|
+
const pinnedBlock = (pinnedRec[key] ?? {}) as Record<string, unknown>;
|
|
173
|
+
for (const k of keys) {
|
|
174
|
+
if (pinnedBlock[k] === undefined) delete merged[k];
|
|
175
|
+
else merged[k] = pinnedBlock[k];
|
|
176
|
+
}
|
|
177
|
+
value = merged;
|
|
178
|
+
}
|
|
130
179
|
const before = JSON.stringify((live as unknown as Record<string, unknown>)[key]);
|
|
131
180
|
const after = JSON.stringify(value);
|
|
132
181
|
if (before !== after) changed.push(key);
|
package/src/config/schema.ts
CHANGED
|
@@ -90,6 +90,7 @@ const filters = z.strictObject({
|
|
|
90
90
|
requireToolSupport: z.boolean().optional(),
|
|
91
91
|
minTrust: z.number().min(0).max(1).optional(),
|
|
92
92
|
feedbackWeight: z.number().nonnegative().optional(),
|
|
93
|
+
feedbackByTask: z.boolean().optional(),
|
|
93
94
|
minTrustSamples: z.number().int().nonnegative().optional(),
|
|
94
95
|
trustScopedByHarness: z.boolean().optional(),
|
|
95
96
|
trustWindowDays: z.number().nonnegative().optional(),
|
|
@@ -205,6 +206,8 @@ const compaction = z.strictObject({
|
|
|
205
206
|
keepTailBytes: z.number().int().nonnegative().optional(),
|
|
206
207
|
elideSupersededReads: z.boolean().optional(),
|
|
207
208
|
collapseDuplicateResults: z.boolean().optional(),
|
|
209
|
+
digestToolResults: z.boolean().optional(),
|
|
210
|
+
digestMaxPerTurn: z.number().int().nonnegative().optional(),
|
|
208
211
|
});
|
|
209
212
|
|
|
210
213
|
const budget = z.strictObject({
|
|
@@ -226,6 +229,7 @@ const ledger = z.strictObject({
|
|
|
226
229
|
blendMinSamples: z.number().int().nonnegative().optional(),
|
|
227
230
|
fallbackBlend: fallbackBlend.optional(),
|
|
228
231
|
conversationTtlMs: z.number().positive().optional(),
|
|
232
|
+
retentionDays: z.number().int().nonnegative().optional(),
|
|
229
233
|
});
|
|
230
234
|
|
|
231
235
|
// Complete entries: arrays replace wholesale, so a partial profile would
|
|
@@ -272,7 +276,7 @@ export const configInputSchema = z.strictObject({
|
|
|
272
276
|
compaction: compaction.optional(),
|
|
273
277
|
budget: budget.optional(),
|
|
274
278
|
profiles: z.array(profile).optional(),
|
|
275
|
-
report: z.strictObject({ baselines: z.array(z.string()).optional() }).optional(),
|
|
279
|
+
report: z.strictObject({ baselines: z.array(z.string()).optional(), dailySummary: z.boolean().optional() }).optional(),
|
|
276
280
|
digest: z
|
|
277
281
|
.strictObject({
|
|
278
282
|
enabled: z.boolean().optional(),
|
package/src/config/types.ts
CHANGED
|
@@ -232,6 +232,15 @@ export interface FilterConfig {
|
|
|
232
232
|
* once a week of verdicts is in the report.
|
|
233
233
|
*/
|
|
234
234
|
feedbackWeight: number;
|
|
235
|
+
/**
|
|
236
|
+
* Count a verdict toward a model's trust only when routing the same task
|
|
237
|
+
* type the judged turn was (the ledger's `task`: coding, vision,
|
|
238
|
+
* documentation, data, chat). A model that writes good code but bad prose
|
|
239
|
+
* then keeps its coding trust. Verdicts on turns with no recorded task
|
|
240
|
+
* count for every task. Off by default: verdicts are scarce, and pooling
|
|
241
|
+
* them converges sooner.
|
|
242
|
+
*/
|
|
243
|
+
feedbackByTask: boolean;
|
|
235
244
|
/** Attempts required before `minTrust` is enforced against a model. */
|
|
236
245
|
minTrustSamples: number;
|
|
237
246
|
/**
|
|
@@ -589,6 +598,13 @@ export interface ReportConfig {
|
|
|
589
598
|
* are skipped.
|
|
590
599
|
*/
|
|
591
600
|
baselines: string[];
|
|
601
|
+
/**
|
|
602
|
+
* Post a one-screen summary of the last 24 hours (spend, top models, cache
|
|
603
|
+
* hit, escalations, soft-failure spikes, Ollama meter) into the transcript
|
|
604
|
+
* at the first omp session start of each day. `/router summary` shows it
|
|
605
|
+
* on demand regardless.
|
|
606
|
+
*/
|
|
607
|
+
dailySummary: boolean;
|
|
592
608
|
}
|
|
593
609
|
|
|
594
610
|
export interface BudgetConfig {
|
|
@@ -641,6 +657,13 @@ export interface LedgerConfig {
|
|
|
641
657
|
fallbackBlend: { inputPerMtok: number; outputPerMtok: number };
|
|
642
658
|
/** Drop conversation state untouched for longer than this, ms. */
|
|
643
659
|
conversationTtlMs: number;
|
|
660
|
+
/**
|
|
661
|
+
* Delete ledger rows older than this many days (checked hourly). 0 keeps
|
|
662
|
+
* everything. The ledger grows ~2.5 MB a day under steady use; trust,
|
|
663
|
+
* reports and replay only read windows well inside a year. Freed pages
|
|
664
|
+
* are reused, so the file stops growing rather than shrinking.
|
|
665
|
+
*/
|
|
666
|
+
retentionDays: number;
|
|
644
667
|
}
|
|
645
668
|
|
|
646
669
|
/**
|
|
@@ -746,6 +769,17 @@ export interface CompactionConfig {
|
|
|
746
769
|
keepTailBytes: number;
|
|
747
770
|
/** Elide an older tool result when a newer call to the same resource supersedes it. */
|
|
748
771
|
elideSupersededReads: boolean;
|
|
772
|
+
/**
|
|
773
|
+
* Summarising compaction: when the plan gains an edit, a cheap model
|
|
774
|
+
* (`digest.tier` / `digest.model`, under `digest.maxCostUsd` and
|
|
775
|
+
* `digest.timeoutMs`) digests the tool result instead of it being cut to
|
|
776
|
+
* head+tail or a stub. The digest is stored on the edit, so the bytes sent
|
|
777
|
+
* stay identical on later turns. Applies when the turn routed at or above
|
|
778
|
+
* `digest.fromTier`; does not need `digest.enabled`.
|
|
779
|
+
*/
|
|
780
|
+
digestToolResults: boolean;
|
|
781
|
+
/** Digests per turn at most; the rest of a plan's new edits stay plain until a later turn. */
|
|
782
|
+
digestMaxPerTurn: number;
|
|
749
783
|
/** Collapse byte-identical repeated tool results to a single copy. */
|
|
750
784
|
collapseDuplicateResults: boolean;
|
|
751
785
|
}
|