auto-model-router 0.4.2 → 0.4.4

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (48) hide show
  1. package/.gitattributes +2 -0
  2. package/.omp-plugin/marketplace.json +2 -2
  3. package/README.md +31 -3
  4. package/bunfig.toml +2 -0
  5. package/omp-extension/report-logic.ts +46 -0
  6. package/omp-extension/router-configure.ts +55 -3
  7. package/package.json +1 -1
  8. package/src/catalog/composite.ts +4 -1
  9. package/src/cli/config-wizard.ts +8 -1
  10. package/src/config/defaults.ts +8 -0
  11. package/src/config/hot-reload.ts +58 -9
  12. package/src/config/schema.ts +5 -1
  13. package/src/config/types.ts +34 -0
  14. package/src/cost/ledger.ts +84 -8
  15. package/src/cost/report.ts +34 -4
  16. package/src/cost/summary.ts +231 -0
  17. package/src/cost/types.ts +35 -3
  18. package/src/router/candidates.ts +1 -1
  19. package/src/router/classify.ts +2 -2
  20. package/src/router/compaction.ts +2 -1
  21. package/src/router/learned.ts +11 -1
  22. package/src/router/select.ts +27 -3
  23. package/src/server/compaction-digest.ts +129 -0
  24. package/src/server/digest.ts +68 -4
  25. package/src/server/http.ts +50 -7
  26. package/src/server/providers.ts +1 -0
  27. package/src/server/turn.ts +38 -1
  28. package/src/util/sqlite.ts +7 -0
  29. package/src/wire/openai/request.ts +4 -0
  30. package/src/wire/types.ts +7 -0
  31. package/test/cache-control.test.ts +1 -1
  32. package/test/compaction.test.ts +40 -3
  33. package/test/digest.test.ts +44 -0
  34. package/test/failover.test.ts +4 -4
  35. package/test/hot-reload.test.ts +37 -1
  36. package/test/learned.test.ts +21 -1
  37. package/test/migrations.test.ts +84 -0
  38. package/test/report-hub.test.ts +1 -1
  39. package/test/report-logic.test.ts +11 -1
  40. package/test/report.test.ts +29 -1
  41. package/test/select.test.ts +26 -2
  42. package/test/summary.test.ts +171 -0
  43. package/test/support/preload.ts +19 -0
  44. package/test/tokens.test.ts +68 -0
  45. package/test/trust-attribution.test.ts +37 -0
  46. package/test/turn.test.ts +69 -4
  47. package/tools/gen-migration-fixtures.ts +69 -0
  48. package/tools/train-classifier.ts +75 -20
package/.gitattributes ADDED
@@ -0,0 +1,2 @@
1
+ # SQLite fixtures must never get line-ending conversion (core.autocrlf is on for this checkout).
2
+ *.db binary
@@ -7,14 +7,14 @@
7
7
  },
8
8
  "metadata": {
9
9
  "description": "auto-model-router: a local cost/complexity-aware model router for Oh My Pi, backed by OpenRouter",
10
- "version": "0.4.2",
10
+ "version": "0.4.4",
11
11
  "pluginRoot": "."
12
12
  },
13
13
  "plugins": [
14
14
  {
15
15
  "name": "auto-model-router",
16
16
  "description": "Local cost/complexity-aware model router for Oh My Pi, backed by OpenRouter. Runs in-process, routes per turn by price and task complexity, with budget caps, mid-stream escalation, and cache-aware hysteresis.",
17
- "version": "0.4.2",
17
+ "version": "0.4.4",
18
18
  "author": {
19
19
  "name": "drewappling",
20
20
  "email": "drewappling@gmail.com"
package/README.md CHANGED
@@ -497,6 +497,8 @@ escalation signal, error. Three views aggregate it, all from the same
497
497
  unreachable.
498
498
  - `auto-model-router report --days 7 [--harness <id>] [--json]` on the terminal.
499
499
  - `GET /v1/router/report?days=7&harness=<id>` for dashboards.
500
+ - `GET /v1/router/summary?harness=<id>` — the daily summary as JSON (`auto=1`
501
+ applies the once-a-day gate and returns `due: false` when nothing is due).
500
502
 
501
503
  What it shows, for the window:
502
504
 
@@ -510,6 +512,18 @@ What it shows, for the window:
510
512
  | by day | UTC calendar days: dispatches, spend, cache hit |
511
513
  | same traffic on one model | the window's tokens priced on each `report.baselines` model at list price with the window's cache hit rate, and what share the router saved against it |
512
514
 
515
+ **Soft-failure spikes.** `/health` (`softFailures.spikes`), `/router status`
516
+ and the daily summary list any model whose failure rate over the last hour —
517
+ probe rejections such as `empty_completion` or `repeat_tool_call` that
518
+ OpenRouter counts as success, plus attributable transport errors — is at
519
+ least 25%, at least twice its own rate over the preceding 7 days, and covers
520
+ at least 5 dispatches with 3 failures. This is visibility only: two weeks of
521
+ ledger data showed soft failures do not cluster tightly enough for a breaker
522
+ to save money (after a burst, the next 15 minutes ran 84–1,577 successes per
523
+ 13–50 failures), and OpenRouter's provider failover plus the router's own
524
+ escalation already cover the retry. Use a spike as the cue to `/router pin`
525
+ or deny a model for the session.
526
+
513
527
  Spend follows the ledger's rule — the provider's reported cost when it gave
514
528
  one, else the usage-priced figure the router computed, else the forecast.
515
529
  Speed uses only clean streamed rows (TTFT recorded, no error); tokens/s is
@@ -534,7 +548,8 @@ Status); the subcommands go straight there:
534
548
  | --- | --- |
535
549
  | `/router config` | Section picker over **every** config key: Server, OpenRouter, Ollama Cloud, Benchmarks, Tiers, Tasks, Filters, Classifier, Escalation, Hysteresis, Exploration, Cache, Compaction, Context (agentdox), Budget, Ledger, Logging, Profiles. Only `ollama.prices` and `ollama.twins` (maps) stay YAML-only. |
536
550
  | `/router report` | Usage analytics in a fullscreen hub styled like `/models`: pick a view in the sidebar, set the window (24h / 7d / 30d / 90d) and the harness scope there too. `/router report 30d --all` presets them. See [Usage reports](#usage-reports). |
537
- | `/router status` | The router's `/health`: key sources, catalog size and age, Ollama availability, plan usage and cost bias, agentdox bridge. |
551
+ | `/router summary [--all]` | The last 24 hours in a few lines: spend against the day before, turns and conversations, cache hit, escalations, errors, model switches with tier moves, top models, savings against the first `report.baselines` model, digests and subagent spend, soft-failure spikes, and the Ollama meter with its runway. Posted automatically once a day at session start when `report.dailySummary` is on (the router keeps a per-harness marker, so several omp windows show it once between them, and a day with no turns and no spikes is skipped). |
552
+ | `/router status` | The router's `/health`: key sources, catalog size and age, Ollama availability, plan usage and cost bias, soft-failure spikes (below), agentdox bridge. |
538
553
  | `/router why` | Explain this session's last routed turn: model and provider, tier, classification source and confidence, cost, cache hit, latency, the full decision trail and classifier reasons, any feedback already given. |
539
554
  | `/router good` / `/router bad [note]` | Judge that turn. Recorded against the model that served it (`POST /v1/router/feedback`), shown per model in the report's `feedback` column, and the label the de-escalation work needs. `/router feedback good\|bad` is the same. |
540
555
  | `/router pin <model\|off>` | Route this session to one model until cleared (admitted past price, quality and trust filters; tool support and context window still apply). Escalations and failovers after the first attempt still run. |
@@ -547,7 +562,9 @@ as the placeholder — empty input keeps it, `-` clears an optional field,
547
562
  credentials show as `set`/`unset` and are never echoed. `Save and exit` writes the merged config
548
563
  (schema-checked and backed up first). Tier, task, filter, classifier,
549
564
  hysteresis, exploration, compaction, cache and budget changes hot-reload;
550
- restart omp for `server`, `openrouter`, `ollama`, `context` and `ledger`.
565
+ restart omp for `server` (except `subagentProfile`), `openrouter`, `context`,
566
+ `ledger.path` and the Ollama connection keys; `ollama.costBias`,
567
+ `ollama.biasUntilUsage` and `ledger.retentionDays` hot-reload too.
551
568
 
552
569
  ### Via `auto-model-router config` (text wizard / CLI)
553
570
 
@@ -668,6 +685,7 @@ Each task (`coding`, `vision`, `documentation`, `data`, `chat`) is a
668
685
  | `includeFree` | `false` | Include free models (rate-limited hard; usually excluded). |
669
686
  | `requireToolSupport` | `true` | Only models that support tool calls. |
670
687
  | `feedbackWeight` | `0` | How much a `/router good\|bad` verdict weighs in a model's trust rate: a bad verdict counts as this many failures, a good one as this many successes. `0` records verdicts without acting on them. |
688
+ | `feedbackByTask` | `false` | Count a verdict only when routing the same task type as the judged turn (coding, vision, documentation, data, chat), so a model that codes well but explains badly keeps its coding trust. Verdicts on turns with no recorded task count everywhere. |
671
689
  | `minTrust` | `0.7` | Minimum success rate; models below this (after `minTrustSamples`) are demoted. |
672
690
  | `minTrustSamples` | `12` | Attempts before trust is enforced. |
673
691
  | `trustScopedByHarness` | `false` | `true` = each harness reads only its own trust rows. |
@@ -684,7 +702,7 @@ Each task (`coding`, `vision`, `documentation`, `data`, `chat`) is a
684
702
  | Key | Default | Meaning |
685
703
  | --- | --- | --- |
686
704
  | `ambiguityThreshold` | `0.6` | Below this heuristic confidence, the adjudicator model decides the tier. |
687
- | `learnedModelPath` | unset | A model written by `bun tools/train-classifier.ts` (logistic regression over the ledger's recorded features, label = the turn escalated). When set, every decision records `learned: p(escalate)=…`. Advisory only: it never moves a tier until replay shows it should. |
705
+ | `learnedModelPath` | unset | A model written by `bun tools/train-classifier.ts` (logistic regression over the ledger's recorded features; label = the turn escalated, or with `--label feedback` the turn was judged bad via `/router bad`). When set, every decision records `learned: p(escalate)=…` or `learned: p(bad)=…`. Advisory only: it never moves a tier until replay shows it should. |
688
706
  | `model` | `qwen/qwen3.7-flash` | Adjudicator model slug. |
689
707
  | `maxCostFraction` | `0.02` | Adjudicator cost cap as a fraction of the turn's budget. |
690
708
  | `maxCostUsd` | `0.002` | Absolute adjudicator cost cap, USD. |
@@ -748,6 +766,8 @@ shrunk results stay shrunk (rewriting them would break the prompt cache).
748
766
  | `keepHeadBytes` / `keepTailBytes` | `512` / `512` | Bytes kept around the elision breadcrumb. |
749
767
  | `elideSupersededReads` | `true` | Stub an older result when a newer call to the same resource supersedes it. |
750
768
  | `collapseDuplicateResults` | `true` | Collapse byte-identical repeated results to a single copy. |
769
+ | `digestToolResults` | `false` | Summarising compaction: when the plan gains an edit, a cheap model (`digest.tier`/`digest.model`, under `digest.maxCostUsd` and `digest.timeoutMs`) digests the tool result instead of it being cut to head+tail or a stub. The digest is stored on the edit, so the dispatched bytes stay identical on later turns and the cache holds. Applies when the turn routed at or above `digest.fromTier`; works without `digest.enabled`. |
770
+ | `digestMaxPerTurn` | `2` | Digests per turn at most (largest results first); the rest of a plan's new edits stay plain until a later turn. |
751
771
 
752
772
  ### `cache` — prompt-cache breakpoints
753
773
 
@@ -804,11 +824,18 @@ is a ledger row (`requestedModel` `digest`) and the report totals them.
804
824
  | `maxCostUsd` | `0.02` | Skip when the digest itself would cost more. |
805
825
  | `timeoutMs` | `25000` | The raw result stands if the cheap model is slower. |
806
826
 
827
+ Quality signal: when the agent later calls the same tool with the same
828
+ primary argument (re-reads a digested file, re-runs a digested grep), the
829
+ router marks that digest's ledger row wasted. The report's `digests` line
830
+ shows the re-run rate; a high rate means the digest is dropping what the
831
+ task needed, and `digest.maxOutputTokens` or `digest.model` is the lever.
832
+
807
833
  ### `report` — usage-report options
808
834
 
809
835
  | Key | Default | Meaning |
810
836
  | --- | --- | --- |
811
837
  | `baselines` | `anthropic/claude-opus-5`, `anthropic/claude-sonnet-5` | Models the report prices the window's traffic on as a single-model counterfactual. Unknown slugs are skipped. |
838
+ | `dailySummary` | `true` | Post the daily summary (below) into the transcript at the first interactive omp session start of each day. Hot-reloads. |
812
839
 
813
840
  ### `ledger` — cost measurement
814
841
 
@@ -819,6 +846,7 @@ is a ledger row (`requestedModel` `digest`) and the report totals them.
819
846
  | `blendMinSamples` | `25` | Turns before the measured blend replaces the fallback. |
820
847
  | `fallbackBlend` | input `1.5`, output `7.5` | Pre-measurement blend (USD/Mtok) for omp's cost display. |
821
848
  | `conversationTtlMs` | `604800000` (7 d) | Drop conversation state untouched this long. |
849
+ | `retentionDays` | `365` | Delete ledger rows older than this, checked hourly; `0` keeps everything. The ledger grows about 2.5 MB a day under steady use. Freed pages are reused, so the file stops growing rather than shrinking. |
822
850
 
823
851
  ### Top-level
824
852
 
package/bunfig.toml ADDED
@@ -0,0 +1,2 @@
1
+ [test]
2
+ preload = ["./test/support/preload.ts"]
@@ -6,6 +6,7 @@
6
6
  */
7
7
 
8
8
  import type { UsageReport } from "../src/cost/report.ts";
9
+ import type { DailySummary } from "../src/cost/summary.ts";
9
10
 
10
11
  export interface ReportRequest {
11
12
  windowDays: number;
@@ -61,6 +62,44 @@ export async function fetchReport(
61
62
  return (await res.json()) as UsageReport;
62
63
  }
63
64
 
65
+ /** GETs the daily summary; `auto` asks the router whether one is due today. Throws on any failure. */
66
+ export async function fetchSummary(
67
+ baseUrl: string,
68
+ harnessId: string,
69
+ auto: boolean,
70
+ headers: Record<string, string>,
71
+ fetchImpl: FetchLike = fetch,
72
+ timeoutMs = 5_000,
73
+ ): Promise<{ due: boolean; reason?: string; summary: DailySummary | null }> {
74
+ const params = new URLSearchParams();
75
+ if (harnessId !== "") params.set("harness", harnessId);
76
+ if (auto) params.set("auto", "1");
77
+ const q = params.toString();
78
+ const res = await fetchImpl(`${baseUrl}/v1/router/summary${q === "" ? "" : `?${q}`}`, { headers, signal: AbortSignal.timeout(timeoutMs) });
79
+ if (!res.ok) throw new Error(`router returned ${res.status}`);
80
+ return (await res.json()) as { due: boolean; reason?: string; summary: DailySummary | null };
81
+ }
82
+
83
+ /** One spiking model as `/health` reports it. */
84
+ export interface SoftFailureSpikeView {
85
+ slug?: string;
86
+ recentDispatches?: number;
87
+ recentFailures?: number;
88
+ recentRate?: number;
89
+ baselineDispatches?: number;
90
+ baselineRate?: number;
91
+ }
92
+
93
+ /** "slug: 40% of 10 failed in the last 1h (7d baseline 8% of 120)" — one line per spiking model. */
94
+ export function renderSoftFailureSpikes(spikes: readonly SoftFailureSpikeView[] | undefined | null, recentMs = 3_600_000, baselineDays = 7): string[] {
95
+ if (spikes === undefined || spikes === null || spikes.length === 0) return [];
96
+ const window = recentMs >= 3_600_000 ? `${(recentMs / 3_600_000).toFixed(recentMs % 3_600_000 === 0 ? 0 : 1)}h` : `${Math.round(recentMs / 60_000)}m`;
97
+ return spikes.map(
98
+ (s) =>
99
+ `${s.slug ?? "?"}: ${((s.recentRate ?? 0) * 100).toFixed(0)}% of ${s.recentDispatches ?? 0} failed in the last ${window} (${baselineDays}d baseline ${((s.baselineRate ?? 0) * 100).toFixed(0)}% of ${s.baselineDispatches ?? 0})`,
100
+ );
101
+ }
102
+
64
103
  /** The subset of `/health` the status view renders. */
65
104
  export interface HealthSnapshot {
66
105
  status?: string;
@@ -80,6 +119,7 @@ export interface HealthSnapshot {
80
119
  runway?: { dailyBurnUsd?: number; creditsLeftUsd?: number; days?: number | null } | null;
81
120
  costBias?: { configured?: number; effective?: number; biasUntilUsage?: number };
82
121
  } | null;
122
+ softFailures?: { recentMs?: number; baselineDays?: number; spikes?: SoftFailureSpikeView[] } | null;
83
123
  catalog?: {
84
124
  models?: number;
85
125
  ageMs?: number;
@@ -119,6 +159,12 @@ export function renderStatus(baseUrl: string, h: HealthSnapshot, nowMs = Date.no
119
159
  const rwText = rw !== undefined && rw !== null ? ` · burn $${(rw.dailyBurnUsd ?? 0).toFixed(2)}/day · ${rw.days === null || rw.days === undefined ? "credits left: unknown burn" : `~${Math.round(rw.days)} days of credits left`}` : "";
120
160
  if (o.meter !== undefined && o.meter !== null) out.push(`ollama billing: ${calText}${rwText}`);
121
161
  }
162
+ const sf = h.softFailures;
163
+ if (sf !== undefined && sf !== null) {
164
+ const lines = renderSoftFailureSpikes(sf.spikes, sf.recentMs, sf.baselineDays);
165
+ if (lines.length === 0) out.push("soft failures: no model spiking in the last hour");
166
+ else out.push(`soft failures SPIKING (${lines.length}):`, ...lines.map((l) => ` ${l}`));
167
+ }
122
168
  const a = h.agentdox;
123
169
  out.push(a === undefined || a === null ? "agentdox: off" : `agentdox: ${a.url ?? "?"} scope ${a.defaultScope ?? "?"}${a.recordTurns === true ? " · recording turns" : ""}`);
124
170
  return out.join("\n");
@@ -50,6 +50,7 @@ import { routerConfigPath, writeRouterConfig } from "../src/cli/config-cmd.ts";
50
50
  import { loadConfig } from "../src/config/load.ts";
51
51
  import type { RouterConfig } from "../src/config/types.ts";
52
52
  import { buildUsageReport, renderUsageReport, type UsageReport } from "../src/cost/report.ts";
53
+ import { buildDailySummary, renderDailySummary } from "../src/cost/summary.ts";
53
54
  import { openDb } from "../src/util/sqlite.ts";
54
55
 
55
56
  import type { ExtensionAPI, ExtensionContext } from "@oh-my-pi/pi-coding-agent";
@@ -61,6 +62,7 @@ import { ReportHub } from "./report-hub.ts";
61
62
  import {
62
63
  describeOverride,
63
64
  fetchReport,
65
+ fetchSummary,
64
66
  parseOverrideArgs,
65
67
  parseReportArgs,
66
68
  renderStatus,
@@ -77,12 +79,32 @@ const HARNESS_ID = process.env.OMP_HARNESS_ID ?? "";
77
79
 
78
80
  /** Custom message type for report/status output in the transcript. */
79
81
  const MESSAGE_TYPE = "auto-model-router";
82
+ /** The embedded router boots on session_start too; the auto summary waits for it this long. */
83
+ const SUMMARY_TRIES = 8;
84
+ const SUMMARY_RETRY_MS = 1_500;
80
85
 
81
86
  export default function (pi: ExtensionAPI): void {
82
87
  pi.setLabel("auto-model-router");
83
88
 
89
+ // Once a day, the first interactive session posts yesterday's summary. The
90
+ // router decides whether one is due (report.dailySummary, a per-harness
91
+ // marker), so several windows on one router show it once between them.
92
+ pi.on("session_start", async (_event, ctx) => {
93
+ if (!ctx.hasUI) return;
94
+ for (let i = 0; i < SUMMARY_TRIES; i++) {
95
+ try {
96
+ const r = await fetchSummary(routerBaseUrl(), HARNESS_ID, true, routerAuthHeaders(), fetch, 2_000);
97
+ if (r.due && r.summary !== null) post(pi, renderDailySummary(r.summary));
98
+ return;
99
+ } catch {
100
+ // Router still starting (embed) or absent: try again shortly, then give up quietly.
101
+ await new Promise((resolve) => setTimeout(resolve, SUMMARY_RETRY_MS));
102
+ }
103
+ }
104
+ });
105
+
84
106
  pi.registerCommand("router", {
85
- description: "auto-model-router: configure, usage report, status",
107
+ description: "auto-model-router: configure, usage report, status, daily summary",
86
108
  handler: async (args, ctx) => {
87
109
  const [verb = "", ...rest] = args.trim().split(/\s+/).filter((t) => t !== "");
88
110
  const tail = rest.join(" ");
@@ -95,6 +117,9 @@ export default function (pi: ExtensionAPI): void {
95
117
  case "status":
96
118
  case "health":
97
119
  return status(pi, ctx);
120
+ case "summary":
121
+ case "daily":
122
+ return summary(pi, ctx, tail);
98
123
  case "why":
99
124
  case "explain":
100
125
  return why(pi, ctx);
@@ -116,20 +141,22 @@ export default function (pi: ExtensionAPI): void {
116
141
  case "":
117
142
  break;
118
143
  default:
119
- ctx.ui.notify(`unknown /router subcommand "${verb}" (config | report | status | why | good | bad | pin | tier)`, "warn");
144
+ ctx.ui.notify(`unknown /router subcommand "${verb}" (config | report | summary | status | why | good | bad | pin | tier)`, "warn");
120
145
  return;
121
146
  }
122
147
 
123
148
  const chosen = await ctx.ui.select("auto-model-router", [
124
149
  { label: "Configure", description: "edit any router setting" },
125
150
  { label: "Report", description: "usage analytics; window and scope adjustable inside" },
126
- { label: "Status", description: "keys, catalog, Ollama, agentdox" },
151
+ { label: "Summary", description: "the last 24h in a few lines" },
152
+ { label: "Status", description: "keys, catalog, Ollama, agentdox, soft-failure spikes" },
127
153
  { label: "Why", description: "explain this session's last routed turn" },
128
154
  { label: "Override", description: "pin a model or force a tier for this session" },
129
155
  ]);
130
156
  if (chosen === undefined) return;
131
157
  if (chosen === "Configure") return configure(ctx);
132
158
  if (chosen === "Status") return status(pi, ctx);
159
+ if (chosen === "Summary") return summary(pi, ctx, "");
133
160
  if (chosen === "Report") return report(pi, ctx, "");
134
161
  if (chosen === "Why") return why(pi, ctx);
135
162
  if (chosen === "Override") return override(ctx, "tier", "");
@@ -299,6 +326,31 @@ async function override(ctx: ExtensionContext, verb: "pin" | "tier", text: strin
299
326
  }
300
327
  }
301
328
 
329
+ /** `/router summary [--all]`: the last 24h, from the router or (router down) the ledger directly. */
330
+ async function summary(pi: ExtensionAPI, ctx: ExtensionContext, argText: string): Promise<void> {
331
+ const harnessId = parseReportArgs(argText, HARNESS_ID).harnessId;
332
+ try {
333
+ const r = await fetchSummary(routerBaseUrl(), harnessId, false, routerAuthHeaders());
334
+ if (r.summary !== null) {
335
+ post(pi, renderDailySummary(r.summary));
336
+ return;
337
+ }
338
+ } catch {
339
+ // Fall through to the ledger.
340
+ }
341
+ const cfg = loadConfig();
342
+ if (!existsSync(cfg.ledger.path)) {
343
+ ctx.ui.notify(`router unreachable at ${routerBaseUrl()} and no ledger at ${cfg.ledger.path}`, "error");
344
+ return;
345
+ }
346
+ const db = openDb(cfg.ledger.path);
347
+ try {
348
+ post(pi, `${renderDailySummary(buildDailySummary(db, { harnessId }))}\n(router unreachable: read from the ledger; spikes and the Ollama meter need the router)`);
349
+ } finally {
350
+ db.close();
351
+ }
352
+ }
353
+
302
354
  async function status(pi: ExtensionAPI, ctx: ExtensionContext): Promise<void> {
303
355
  try {
304
356
  post(pi, await loadStatus());
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "auto-model-router",
3
- "version": "0.4.2",
3
+ "version": "0.4.4",
4
4
  "private": false,
5
5
  "description": "Local cost/complexity-aware model router for Oh My Pi, backed by OpenRouter",
6
6
  "type": "module",
@@ -23,6 +23,8 @@ export interface CompositeBias {
23
23
  /** Plan usage fraction at which the bias switches off (list price). */
24
24
  biasUntilUsage: number;
25
25
  usage: OllamaUsageSource;
26
+ /** When given, read on every use instead of the static pair, so a config hot reload applies. */
27
+ live?: () => { costBias: number; biasUntilUsage: number };
26
28
  }
27
29
 
28
30
  export function createCompositeCatalog(
@@ -39,7 +41,8 @@ export function createCompositeCatalog(
39
41
 
40
42
  /** The multiplier in force from the latest usage reading (no network). */
41
43
  function currentBias(): number {
42
- return effectiveOllamaBias(bias.costBias, bias.biasUntilUsage, bias.usage.peek());
44
+ const b = bias.live?.() ?? bias;
45
+ return effectiveOllamaBias(b.costBias, b.biasUntilUsage, bias.usage.peek());
43
46
  }
44
47
 
45
48
  function combine(base: CatalogSnapshot, models: readonly CatalogModel[]): CatalogSnapshot {
@@ -184,6 +184,7 @@ export const WIZARD_SECTIONS: readonly SectionSpec[] = [
184
184
  { path: "filters.requireToolSupport", label: "Require tool support", kind: "boolean" },
185
185
  { path: "filters.minTrust", label: "Min trust", kind: "number", min: 0, max: 1 },
186
186
  { path: "filters.feedbackWeight", label: "Feedback weight in trust", kind: "number", min: 0, hint: "0=record only; a bad verdict = this many failures" },
187
+ { path: "filters.feedbackByTask", label: "Scope verdicts to the task type", kind: "boolean" },
187
188
  { path: "filters.minTrustSamples", label: "Min trust samples", kind: "number", min: 0 },
188
189
  { path: "filters.trustScopedByHarness", label: "Scope trust per harness", kind: "boolean" },
189
190
  { path: "filters.trustWindowDays", label: "Trust window", kind: "number", min: 0, hint: "days, 0=all time" },
@@ -280,6 +281,8 @@ export const WIZARD_SECTIONS: readonly SectionSpec[] = [
280
281
  { path: "compaction.keepTailBytes", label: "Keep tail bytes", kind: "number", min: 0 },
281
282
  { path: "compaction.elideSupersededReads", label: "Elide superseded reads", kind: "boolean" },
282
283
  { path: "compaction.collapseDuplicateResults", label: "Collapse duplicate results", kind: "boolean" },
284
+ { path: "compaction.digestToolResults", label: "Digest compacted results with a cheap model", kind: "boolean" },
285
+ { path: "compaction.digestMaxPerTurn", label: "Digests per turn at most", kind: "number", min: 0 },
283
286
  ],
284
287
  },
285
288
  {
@@ -327,7 +330,10 @@ export const WIZARD_SECTIONS: readonly SectionSpec[] = [
327
330
  },
328
331
  {
329
332
  title: "Report",
330
- fields: [{ path: "report.baselines", label: "Counterfactual baseline models", kind: "stringArray", hint: "comma-separated slugs" }],
333
+ fields: [
334
+ { path: "report.baselines", label: "Counterfactual baseline models", kind: "stringArray", hint: "comma-separated slugs" },
335
+ { path: "report.dailySummary", label: "Daily summary at session start", kind: "boolean" },
336
+ ],
331
337
  },
332
338
  {
333
339
  title: "Ledger",
@@ -338,6 +344,7 @@ export const WIZARD_SECTIONS: readonly SectionSpec[] = [
338
344
  { path: "ledger.fallbackBlend.inputPerMtok", label: "Fallback blend input $/Mtok", kind: "number", min: 0 },
339
345
  { path: "ledger.fallbackBlend.outputPerMtok", label: "Fallback blend output $/Mtok", kind: "number", min: 0 },
340
346
  { path: "ledger.conversationTtlMs", label: "Conversation TTL", kind: "number", min: 1, hint: "ms" },
347
+ { path: "ledger.retentionDays", label: "Ledger retention", kind: "number", min: 0, hint: "days; 0 keeps everything" },
341
348
  ],
342
349
  },
343
350
  {
@@ -103,6 +103,7 @@ export const DEFAULT_CONFIG: RouterConfig = {
103
103
  minTrust: 0.7,
104
104
  // Verdicts are recorded and reported first; weigh them once there are some.
105
105
  feedbackWeight: 0,
106
+ feedbackByTask: false,
106
107
  minTrustSamples: 12,
107
108
  // Shared trust by default: more samples, demotion guard stays effective
108
109
  // even with a tiny guardrail-narrowed catalog.
@@ -291,6 +292,10 @@ export const DEFAULT_CONFIG: RouterConfig = {
291
292
  keepTailBytes: 512,
292
293
  elideSupersededReads: true,
293
294
  collapseDuplicateResults: true,
295
+ // Off: a synchronous cheap-model call before dispatch, only worth it where
296
+ // stale tool output is the prompt and the turn is on a dear model.
297
+ digestToolResults: false,
298
+ digestMaxPerTurn: 2,
294
299
  },
295
300
  digest: {
296
301
  // Off until an operator turns it on: it changes what the model reads.
@@ -308,6 +313,8 @@ export const DEFAULT_CONFIG: RouterConfig = {
308
313
  report: {
309
314
  // The frontier pair most omp users would otherwise run on.
310
315
  baselines: ["anthropic/claude-opus-5", "anthropic/claude-sonnet-5"],
316
+ // One transcript message per day, at the first interactive session start.
317
+ dailySummary: true,
311
318
  },
312
319
  budget: {
313
320
  // No caps by default; at a configured ceiling, downgrade rather than fail.
@@ -330,6 +337,7 @@ export const DEFAULT_CONFIG: RouterConfig = {
330
337
  // so early cost reporting never underreports.
331
338
  fallbackBlend: { inputPerMtok: 1.5, outputPerMtok: 7.5 },
332
339
  conversationTtlMs: 7 * 24 * 60 * 60 * 1000,
340
+ retentionDays: 365,
333
341
  },
334
342
  // On by default: an absolute floor that no available model meets is how the
335
343
  // router ends up serving every turn from the cheapest tier.
@@ -90,15 +90,47 @@ export interface WatchConfigOptions {
90
90
  }
91
91
 
92
92
  /**
93
- * Watches `path` and applies valid changes to `live` in place. `frozen` blocks
94
- * (top-level names) are re-copied from `pinned` after every reload so file
95
- * edits to construction-captured blocks cannot silently diverge.
93
+ * Config paths captured at construction, so a file edit cannot reach the
94
+ * running process: the socket, the upstream clients, the agentdox bridge, the
95
+ * ledger file. Everything else, including `ollama.costBias`,
96
+ * `ollama.biasUntilUsage`, `server.subagentProfile` and `ledger.retentionDays`,
97
+ * is read at call time and hot-reloads. A bare block name pins the whole
98
+ * block; `block.key` pins one key and lets its siblings through.
99
+ */
100
+ export const PINNED_CONFIG_PATHS: readonly string[] = [
101
+ "server.host",
102
+ "server.port",
103
+ "server.apiKey",
104
+ "server.harnessId",
105
+ "server.maxConcurrentTurns",
106
+ "openrouter",
107
+ "ollama.enabled",
108
+ "ollama.baseUrl",
109
+ "ollama.apiKey",
110
+ "ollama.timeoutMs",
111
+ "ollama.catalogTtlMs",
112
+ "ollama.includeLocal",
113
+ "ollama.prices",
114
+ "ollama.twins",
115
+ "ollama.usagePollMs",
116
+ "ollama.quotaCooldownMs",
117
+ "ollama.rateLimitCooldownMs",
118
+ "ollama.planCreditsUsd",
119
+ "context",
120
+ "ledger.path",
121
+ ];
122
+
123
+ /**
124
+ * Watches `path` and applies valid changes to `live` in place. `frozen`
125
+ * entries are re-copied from `pinned` after every reload so file edits to
126
+ * construction-captured settings cannot silently diverge: a top-level name
127
+ * pins the whole block, `block.key` pins one key of it.
96
128
  */
97
129
  export function watchConfig(
98
130
  path: string,
99
131
  live: RouterConfig,
100
132
  pinned: RouterConfig,
101
- frozen: readonly (keyof RouterConfig)[],
133
+ frozen: readonly string[],
102
134
  opts: WatchConfigOptions = {},
103
135
  ): ConfigWatcher {
104
136
  let closed = false;
@@ -119,14 +151,31 @@ export function watchConfig(
119
151
  }
120
152
  lastError = "";
121
153
 
122
- const frozenSet = new Set(frozen);
154
+ const frozenBlocks = new Set(frozen.filter((f) => !f.includes(".")));
155
+ const frozenKeys = new Map<string, string[]>();
156
+ for (const f of frozen) {
157
+ const dot = f.indexOf(".");
158
+ if (dot < 0) continue;
159
+ const block = f.slice(0, dot);
160
+ frozenKeys.set(block, [...(frozenKeys.get(block) ?? []), f.slice(dot + 1)]);
161
+ }
123
162
  const changed: string[] = [];
124
163
  const next = result.cfg as unknown as Record<string, unknown>;
164
+ const pinnedRec = pinned as unknown as Record<string, unknown>;
125
165
  for (const key of Object.keys(next)) {
126
- // Frozen blocks belong to construction: keep the pinned values.
127
- const value = frozenSet.has(key as keyof RouterConfig)
128
- ? (pinned as unknown as Record<string, unknown>)[key]
129
- : next[key];
166
+ // Frozen blocks belong to construction: keep the pinned values. A
167
+ // partially frozen block takes the file's siblings and the pinned keys.
168
+ let value = frozenBlocks.has(key) ? pinnedRec[key] : next[key];
169
+ const keys = frozenKeys.get(key);
170
+ if (keys !== undefined && !frozenBlocks.has(key) && value !== null && typeof value === "object") {
171
+ const merged: Record<string, unknown> = { ...(value as Record<string, unknown>) };
172
+ const pinnedBlock = (pinnedRec[key] ?? {}) as Record<string, unknown>;
173
+ for (const k of keys) {
174
+ if (pinnedBlock[k] === undefined) delete merged[k];
175
+ else merged[k] = pinnedBlock[k];
176
+ }
177
+ value = merged;
178
+ }
130
179
  const before = JSON.stringify((live as unknown as Record<string, unknown>)[key]);
131
180
  const after = JSON.stringify(value);
132
181
  if (before !== after) changed.push(key);
@@ -90,6 +90,7 @@ const filters = z.strictObject({
90
90
  requireToolSupport: z.boolean().optional(),
91
91
  minTrust: z.number().min(0).max(1).optional(),
92
92
  feedbackWeight: z.number().nonnegative().optional(),
93
+ feedbackByTask: z.boolean().optional(),
93
94
  minTrustSamples: z.number().int().nonnegative().optional(),
94
95
  trustScopedByHarness: z.boolean().optional(),
95
96
  trustWindowDays: z.number().nonnegative().optional(),
@@ -205,6 +206,8 @@ const compaction = z.strictObject({
205
206
  keepTailBytes: z.number().int().nonnegative().optional(),
206
207
  elideSupersededReads: z.boolean().optional(),
207
208
  collapseDuplicateResults: z.boolean().optional(),
209
+ digestToolResults: z.boolean().optional(),
210
+ digestMaxPerTurn: z.number().int().nonnegative().optional(),
208
211
  });
209
212
 
210
213
  const budget = z.strictObject({
@@ -226,6 +229,7 @@ const ledger = z.strictObject({
226
229
  blendMinSamples: z.number().int().nonnegative().optional(),
227
230
  fallbackBlend: fallbackBlend.optional(),
228
231
  conversationTtlMs: z.number().positive().optional(),
232
+ retentionDays: z.number().int().nonnegative().optional(),
229
233
  });
230
234
 
231
235
  // Complete entries: arrays replace wholesale, so a partial profile would
@@ -272,7 +276,7 @@ export const configInputSchema = z.strictObject({
272
276
  compaction: compaction.optional(),
273
277
  budget: budget.optional(),
274
278
  profiles: z.array(profile).optional(),
275
- report: z.strictObject({ baselines: z.array(z.string()).optional() }).optional(),
279
+ report: z.strictObject({ baselines: z.array(z.string()).optional(), dailySummary: z.boolean().optional() }).optional(),
276
280
  digest: z
277
281
  .strictObject({
278
282
  enabled: z.boolean().optional(),
@@ -232,6 +232,15 @@ export interface FilterConfig {
232
232
  * once a week of verdicts is in the report.
233
233
  */
234
234
  feedbackWeight: number;
235
+ /**
236
+ * Count a verdict toward a model's trust only when routing the same task
237
+ * type the judged turn was (the ledger's `task`: coding, vision,
238
+ * documentation, data, chat). A model that writes good code but bad prose
239
+ * then keeps its coding trust. Verdicts on turns with no recorded task
240
+ * count for every task. Off by default: verdicts are scarce, and pooling
241
+ * them converges sooner.
242
+ */
243
+ feedbackByTask: boolean;
235
244
  /** Attempts required before `minTrust` is enforced against a model. */
236
245
  minTrustSamples: number;
237
246
  /**
@@ -589,6 +598,13 @@ export interface ReportConfig {
589
598
  * are skipped.
590
599
  */
591
600
  baselines: string[];
601
+ /**
602
+ * Post a one-screen summary of the last 24 hours (spend, top models, cache
603
+ * hit, escalations, soft-failure spikes, Ollama meter) into the transcript
604
+ * at the first omp session start of each day. `/router summary` shows it
605
+ * on demand regardless.
606
+ */
607
+ dailySummary: boolean;
592
608
  }
593
609
 
594
610
  export interface BudgetConfig {
@@ -641,6 +657,13 @@ export interface LedgerConfig {
641
657
  fallbackBlend: { inputPerMtok: number; outputPerMtok: number };
642
658
  /** Drop conversation state untouched for longer than this, ms. */
643
659
  conversationTtlMs: number;
660
+ /**
661
+ * Delete ledger rows older than this many days (checked hourly). 0 keeps
662
+ * everything. The ledger grows ~2.5 MB a day under steady use; trust,
663
+ * reports and replay only read windows well inside a year. Freed pages
664
+ * are reused, so the file stops growing rather than shrinking.
665
+ */
666
+ retentionDays: number;
644
667
  }
645
668
 
646
669
  /**
@@ -746,6 +769,17 @@ export interface CompactionConfig {
746
769
  keepTailBytes: number;
747
770
  /** Elide an older tool result when a newer call to the same resource supersedes it. */
748
771
  elideSupersededReads: boolean;
772
+ /**
773
+ * Summarising compaction: when the plan gains an edit, a cheap model
774
+ * (`digest.tier` / `digest.model`, under `digest.maxCostUsd` and
775
+ * `digest.timeoutMs`) digests the tool result instead of it being cut to
776
+ * head+tail or a stub. The digest is stored on the edit, so the bytes sent
777
+ * stay identical on later turns. Applies when the turn routed at or above
778
+ * `digest.fromTier`; does not need `digest.enabled`.
779
+ */
780
+ digestToolResults: boolean;
781
+ /** Digests per turn at most; the rest of a plan's new edits stay plain until a later turn. */
782
+ digestMaxPerTurn: number;
749
783
  /** Collapse byte-identical repeated tool results to a single copy. */
750
784
  collapseDuplicateResults: boolean;
751
785
  }