auto-model-router 0.4.1 → 0.4.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (43) hide show
  1. package/.omp-plugin/marketplace.json +2 -2
  2. package/README.md +48 -2
  3. package/omp-extension/digest-logic.ts +53 -0
  4. package/omp-extension/pi-coding-agent.d.ts +14 -1
  5. package/omp-extension/report-logic.ts +46 -0
  6. package/omp-extension/router-configure.ts +55 -3
  7. package/omp-extension/router-digest.ts +93 -0
  8. package/package.json +1 -1
  9. package/src/cli/config-wizard.ts +22 -1
  10. package/src/config/defaults.ts +20 -0
  11. package/src/config/schema.ts +18 -1
  12. package/src/config/types.ts +54 -0
  13. package/src/cost/ledger.ts +77 -9
  14. package/src/cost/report.ts +24 -3
  15. package/src/cost/summary.ts +231 -0
  16. package/src/cost/types.ts +31 -3
  17. package/src/router/candidates.ts +1 -1
  18. package/src/router/classify.ts +2 -2
  19. package/src/router/compaction.ts +1 -0
  20. package/src/router/learned.ts +11 -1
  21. package/src/router/select.ts +5 -1
  22. package/src/server/compaction-digest.ts +127 -0
  23. package/src/server/digest.ts +243 -0
  24. package/src/server/http.ts +51 -1
  25. package/src/server/turn.ts +27 -1
  26. package/src/util/sqlite.ts +7 -0
  27. package/src/wire/openai/request.ts +4 -0
  28. package/src/wire/types.ts +7 -0
  29. package/test/compaction.test.ts +40 -3
  30. package/test/controls.test.ts +34 -0
  31. package/test/digest.test.ts +229 -0
  32. package/test/embed-lifecycle.test.ts +1 -1
  33. package/test/failover.test.ts +4 -3
  34. package/test/learned.test.ts +21 -1
  35. package/test/report-hub.test.ts +3 -0
  36. package/test/report-logic.test.ts +11 -1
  37. package/test/report.test.ts +3 -0
  38. package/test/select.test.ts +2 -2
  39. package/test/summary.test.ts +171 -0
  40. package/test/tokens.test.ts +44 -0
  41. package/test/trust-attribution.test.ts +37 -0
  42. package/test/turn.test.ts +69 -3
  43. package/tools/train-classifier.ts +75 -20
@@ -7,14 +7,14 @@
7
7
  },
8
8
  "metadata": {
9
9
  "description": "auto-model-router: a local cost/complexity-aware model router for Oh My Pi, backed by OpenRouter",
10
- "version": "0.4.1",
10
+ "version": "0.4.3",
11
11
  "pluginRoot": "."
12
12
  },
13
13
  "plugins": [
14
14
  {
15
15
  "name": "auto-model-router",
16
16
  "description": "Local cost/complexity-aware model router for Oh My Pi, backed by OpenRouter. Runs in-process, routes per turn by price and task complexity, with budget caps, mid-stream escalation, and cache-aware hysteresis.",
17
- "version": "0.4.1",
17
+ "version": "0.4.3",
18
18
  "author": {
19
19
  "name": "drewappling",
20
20
  "email": "drewappling@gmail.com"
package/README.md CHANGED
@@ -213,6 +213,7 @@ extensions:
213
213
  - auto-model-router/omp-extension/router-embed.ts
214
214
  - auto-model-router/omp-extension/router-toast.ts # optional: chosen-model toasts
215
215
  - auto-model-router/omp-extension/router-configure.ts # optional: /router config, report, status
216
+ - auto-model-router/omp-extension/router-digest.ts # optional: cheap-model digest of large tool results
216
217
  ```
217
218
 
218
219
  ### From the repo (cross-platform installer)
@@ -236,6 +237,7 @@ The installer adds:
236
237
  - `router-embed.ts` — **required**; runs the router in-process.
237
238
  - `router-toast.ts` — optional; chosen-model toasts.
238
239
  - `router-configure.ts` — optional; the `/router` command (configure, usage reports, status).
240
+ - `router-digest.ts` — optional; condenses large tool results with a cheap model before an expensive one reads them (needs `digest.enabled`).
239
241
 
240
242
  Or add the paths by hand to omp's `~/.omp/agent/config.yml`:
241
243
 
@@ -245,6 +247,7 @@ extensions:
245
247
  - /path/to/auto-model-router/omp-extension/router-embed.ts
246
248
  - /path/to/auto-model-router/omp-extension/router-toast.ts # optional: chosen-model toasts
247
249
  - /path/to/auto-model-router/omp-extension/router-configure.ts # optional: /router config, report, status
250
+ - /path/to/auto-model-router/omp-extension/router-digest.ts # optional: cheap-model digest of large tool results
248
251
  ```
249
252
 
250
253
  Then restart the omp session (extensions load at session start).
@@ -494,6 +497,8 @@ escalation signal, error. Three views aggregate it, all from the same
494
497
  unreachable.
495
498
  - `auto-model-router report --days 7 [--harness <id>] [--json]` on the terminal.
496
499
  - `GET /v1/router/report?days=7&harness=<id>` for dashboards.
500
+ - `GET /v1/router/summary?harness=<id>` — the daily summary as JSON (`auto=1`
501
+ applies the once-a-day gate and returns `due: false` when nothing is due).
497
502
 
498
503
  What it shows, for the window:
499
504
 
@@ -507,6 +512,18 @@ What it shows, for the window:
507
512
  | by day | UTC calendar days: dispatches, spend, cache hit |
508
513
  | same traffic on one model | the window's tokens priced on each `report.baselines` model at list price with the window's cache hit rate, and what share the router saved against it |
509
514
 
515
+ **Soft-failure spikes.** `/health` (`softFailures.spikes`), `/router status`
516
+ and the daily summary list any model whose failure rate over the last hour —
517
+ probe rejections such as `empty_completion` or `repeat_tool_call` that
518
+ OpenRouter counts as success, plus attributable transport errors — is at
519
+ least 25%, at least twice its own rate over the preceding 7 days, and covers
520
+ at least 5 dispatches with 3 failures. This is visibility only: two weeks of
521
+ ledger data showed soft failures do not cluster tightly enough for a breaker
522
+ to save money (after a burst, the next 15 minutes ran 84–1,577 successes per
523
+ 13–50 failures), and OpenRouter's provider failover plus the router's own
524
+ escalation already cover the retry. Use a spike as the cue to `/router pin`
525
+ or deny a model for the session.
526
+
510
527
  Spend follows the ledger's rule — the provider's reported cost when it gave
511
528
  one, else the usage-priced figure the router computed, else the forecast.
512
529
  Speed uses only clean streamed rows (TTFT recorded, no error); tokens/s is
@@ -531,7 +548,8 @@ Status); the subcommands go straight there:
531
548
  | --- | --- |
532
549
  | `/router config` | Section picker over **every** config key: Server, OpenRouter, Ollama Cloud, Benchmarks, Tiers, Tasks, Filters, Classifier, Escalation, Hysteresis, Exploration, Cache, Compaction, Context (agentdox), Budget, Ledger, Logging, Profiles. Only `ollama.prices` and `ollama.twins` (maps) stay YAML-only. |
533
550
  | `/router report` | Usage analytics in a fullscreen hub styled like `/models`: pick a view in the sidebar, set the window (24h / 7d / 30d / 90d) and the harness scope there too. `/router report 30d --all` presets them. See [Usage reports](#usage-reports). |
534
- | `/router status` | The router's `/health`: key sources, catalog size and age, Ollama availability, plan usage and cost bias, agentdox bridge. |
551
+ | `/router summary [--all]` | The last 24 hours in a few lines: spend against the day before, turns and conversations, cache hit, escalations, errors, model switches with tier moves, top models, savings against the first `report.baselines` model, digests and subagent spend, soft-failure spikes, and the Ollama meter with its runway. Posted automatically once a day at session start when `report.dailySummary` is on (the router keeps a per-harness marker, so several omp windows show it once between them, and a day with no turns and no spikes is skipped). |
552
+ | `/router status` | The router's `/health`: key sources, catalog size and age, Ollama availability, plan usage and cost bias, soft-failure spikes (below), agentdox bridge. |
535
553
  | `/router why` | Explain this session's last routed turn: model and provider, tier, classification source and confidence, cost, cache hit, latency, the full decision trail and classifier reasons, any feedback already given. |
536
554
  | `/router good` / `/router bad [note]` | Judge that turn. Recorded against the model that served it (`POST /v1/router/feedback`), shown per model in the report's `feedback` column, and the label the de-escalation work needs. `/router feedback good\|bad` is the same. |
537
555
  | `/router pin <model\|off>` | Route this session to one model until cleared (admitted past price, quality and trust filters; tool support and context window still apply). Escalations and failovers after the first attempt still run. |
@@ -665,6 +683,7 @@ Each task (`coding`, `vision`, `documentation`, `data`, `chat`) is a
665
683
  | `includeFree` | `false` | Include free models (rate-limited hard; usually excluded). |
666
684
  | `requireToolSupport` | `true` | Only models that support tool calls. |
667
685
  | `feedbackWeight` | `0` | How much a `/router good\|bad` verdict weighs in a model's trust rate: a bad verdict counts as this many failures, a good one as this many successes. `0` records verdicts without acting on them. |
686
+ | `feedbackByTask` | `false` | Count a verdict only when routing the same task type as the judged turn (coding, vision, documentation, data, chat), so a model that codes well but explains badly keeps its coding trust. Verdicts on turns with no recorded task count everywhere. |
668
687
  | `minTrust` | `0.7` | Minimum success rate; models below this (after `minTrustSamples`) are demoted. |
669
688
  | `minTrustSamples` | `12` | Attempts before trust is enforced. |
670
689
  | `trustScopedByHarness` | `false` | `true` = each harness reads only its own trust rows. |
@@ -681,7 +700,7 @@ Each task (`coding`, `vision`, `documentation`, `data`, `chat`) is a
681
700
  | Key | Default | Meaning |
682
701
  | --- | --- | --- |
683
702
  | `ambiguityThreshold` | `0.6` | Below this heuristic confidence, the adjudicator model decides the tier. |
684
- | `learnedModelPath` | unset | A model written by `bun tools/train-classifier.ts` (logistic regression over the ledger's recorded features, label = the turn escalated). When set, every decision records `learned: p(escalate)=…`. Advisory only: it never moves a tier until replay shows it should. |
703
+ | `learnedModelPath` | unset | A model written by `bun tools/train-classifier.ts` (logistic regression over the ledger's recorded features; label = the turn escalated, or with `--label feedback` the turn was judged bad via `/router bad`). When set, every decision records `learned: p(escalate)=…` or `learned: p(bad)=…`. Advisory only: it never moves a tier until replay shows it should. |
685
704
  | `model` | `qwen/qwen3.7-flash` | Adjudicator model slug. |
686
705
  | `maxCostFraction` | `0.02` | Adjudicator cost cap as a fraction of the turn's budget. |
687
706
  | `maxCostUsd` | `0.002` | Absolute adjudicator cost cap, USD. |
@@ -745,6 +764,8 @@ shrunk results stay shrunk (rewriting them would break the prompt cache).
745
764
  | `keepHeadBytes` / `keepTailBytes` | `512` / `512` | Bytes kept around the elision breadcrumb. |
746
765
  | `elideSupersededReads` | `true` | Stub an older result when a newer call to the same resource supersedes it. |
747
766
  | `collapseDuplicateResults` | `true` | Collapse byte-identical repeated results to a single copy. |
767
+ | `digestToolResults` | `false` | Summarising compaction: when the plan gains an edit, a cheap model (`digest.tier`/`digest.model`, under `digest.maxCostUsd` and `digest.timeoutMs`) digests the tool result instead of it being cut to head+tail or a stub. The digest is stored on the edit, so the dispatched bytes stay identical on later turns and the cache holds. Applies when the turn routed at or above `digest.fromTier`; works without `digest.enabled`. |
768
+ | `digestMaxPerTurn` | `2` | Digests per turn at most (largest results first); the rest of a plan's new edits stay plain until a later turn. |
748
769
 
749
770
  ### `cache` — prompt-cache breakpoints
750
771
 
@@ -777,11 +798,36 @@ Each profile is a complete entry (arrays replace wholesale):
777
798
  | `maxTokens` | `32000` | Advertised max output tokens. |
778
799
  | `budget` | unset | Per-profile budget overrides. |
779
800
 
801
+ ### `digest` — cheap-model digest of large tool results
802
+
803
+ Tool results are the bulk of every prompt (see the report's prompt anatomy),
804
+ and a prompt is ~96% of spend. With the `router-digest` extension installed
805
+ and `digest.enabled` on, a large read, grep, glob or bash result produced
806
+ while the session's current model is at or above `fromTier` is sent to
807
+ `POST /v1/router/digest`; the cheapest `tier` model rewrites it to what the
808
+ task needs (exact paths, line numbers, names, errors, code to be edited) and
809
+ the digest replaces the tool result. It begins with a marker naming the tool
810
+ and arguments to re-run for the full output, so nothing is lost, only
811
+ deferred. Errors, images, edits and writes are never digested. Every digest
812
+ is a ledger row (`requestedModel` `digest`) and the report totals them.
813
+
814
+ | Key | Default | Meaning |
815
+ | --- | --- | --- |
816
+ | `enabled` | `false` | Master switch; the extension polls it every minute. |
817
+ | `minBytes` / `maxBytes` | `12000` / `400000` | Result size window that gets digested. |
818
+ | `tools` | `read, grep, glob, bash, web_fetch, webfetch, ls, find` | Eligible tool names (lower-case). |
819
+ | `fromTier` | `moderate` | Digest only when the session's current model is at or above this tier. |
820
+ | `tier` / `model` | `simple` / unset | Where the digest model is picked from, or a pinned slug. |
821
+ | `maxOutputTokens` | `700` | Digest length cap. |
822
+ | `maxCostUsd` | `0.02` | Skip when the digest itself would cost more. |
823
+ | `timeoutMs` | `25000` | The raw result stands if the cheap model is slower. |
824
+
780
825
  ### `report` — usage-report options
781
826
 
782
827
  | Key | Default | Meaning |
783
828
  | --- | --- | --- |
784
829
  | `baselines` | `anthropic/claude-opus-5`, `anthropic/claude-sonnet-5` | Models the report prices the window's traffic on as a single-model counterfactual. Unknown slugs are skipped. |
830
+ | `dailySummary` | `true` | Post the daily summary (below) into the transcript at the first interactive omp session start of each day. Hot-reloads. |
785
831
 
786
832
  ### `ledger` — cost measurement
787
833
 
@@ -0,0 +1,53 @@
1
+ /**
2
+ * Pure logic for the tool-result digest extension: which results to send,
3
+ * how to read a tool result's text, and how to shape the replacement.
4
+ */
5
+
6
+ export interface DigestPolicy {
7
+ enabled: boolean;
8
+ minBytes: number;
9
+ maxBytes: number;
10
+ tools: string[];
11
+ fromTier: string;
12
+ }
13
+
14
+ export const DISABLED_POLICY: DigestPolicy = { enabled: false, minBytes: 0, maxBytes: 0, tools: [], fromTier: "hard" };
15
+
16
+ /** The text of a tool result's content parts; images are left alone (and block digesting). */
17
+ export function textOf(content: ReadonlyArray<{ type: string; text?: string }>): { text: string; hasImage: boolean } {
18
+ let text = "";
19
+ let hasImage = false;
20
+ for (const part of content) {
21
+ if (part.type === "text" && typeof part.text === "string") text += (text === "" ? "" : "\n") + part.text;
22
+ else if (part.type === "image") hasImage = true;
23
+ }
24
+ return { text, hasImage };
25
+ }
26
+
27
+ /** Client-side gate: cheap checks before anything is sent to the router. */
28
+ export function shouldSend(policy: DigestPolicy, toolName: string, isError: boolean, text: string, hasImage: boolean): boolean {
29
+ if (!policy.enabled || isError || hasImage) return false;
30
+ if (!policy.tools.includes(toolName.toLowerCase())) return false;
31
+ const bytes = Buffer.byteLength(text);
32
+ return bytes >= policy.minBytes && bytes <= policy.maxBytes;
33
+ }
34
+
35
+ /** Parses the router's policy payload defensively; anything odd ⇒ disabled. */
36
+ export function parsePolicy(json: unknown): DigestPolicy {
37
+ if (typeof json !== "object" || json === null) return DISABLED_POLICY;
38
+ const p = json as Record<string, unknown>;
39
+ if (p.enabled !== true) return DISABLED_POLICY;
40
+ return {
41
+ enabled: true,
42
+ minBytes: typeof p.minBytes === "number" ? p.minBytes : 12_000,
43
+ maxBytes: typeof p.maxBytes === "number" ? p.maxBytes : 400_000,
44
+ tools: Array.isArray(p.tools) ? p.tools.filter((t): t is string => typeof t === "string").map((t) => t.toLowerCase()) : [],
45
+ fromTier: typeof p.fromTier === "string" ? p.fromTier : "hard",
46
+ };
47
+ }
48
+
49
+ /** One-line toast for a digest that happened. */
50
+ export function digestToast(toolName: string, inputBytes: number, outputChars: number, model: string, usd: number): string {
51
+ const kb = (n: number): string => `${(n / 1024).toFixed(0)}KB`;
52
+ return `digested ${toolName} ${kb(inputBytes)} → ${kb(outputChars)} via ${model.replace(/^ollama\//, "")} ($${usd.toFixed(4)})`;
53
+ }
@@ -127,9 +127,22 @@ declare module "@oh-my-pi/pi-coding-agent" {
127
127
  details?: unknown;
128
128
  }
129
129
 
130
+ /** A tool result's content parts (text and images). */
131
+ export interface ToolResultPart {
132
+ type: string;
133
+ text?: string;
134
+ }
135
+
136
+ /** What a `tool_result` handler may return to replace the result. */
137
+ export interface ToolResultEventResult {
138
+ content?: ToolResultPart[];
139
+ isError?: boolean;
140
+ }
141
+
130
142
  export interface ExtensionAPI {
131
143
  setLabel(label: string): void;
132
- on(event: string, handler: (event: unknown, ctx: ExtensionContext) => void | Promise<void>): void;
144
+ /** Handlers may return an event result (e.g. a `tool_result` replacement); omp ignores it where none applies. */
145
+ on(event: string, handler: (event: unknown, ctx: ExtensionContext) => unknown): void;
133
146
  registerProvider(id: string, registration: ProviderRegistration): void;
134
147
  unregisterProvider(id: string): void;
135
148
  registerCommand(name: string, command: CommandDefinition): void;
@@ -6,6 +6,7 @@
6
6
  */
7
7
 
8
8
  import type { UsageReport } from "../src/cost/report.ts";
9
+ import type { DailySummary } from "../src/cost/summary.ts";
9
10
 
10
11
  export interface ReportRequest {
11
12
  windowDays: number;
@@ -61,6 +62,44 @@ export async function fetchReport(
61
62
  return (await res.json()) as UsageReport;
62
63
  }
63
64
 
65
+ /** GETs the daily summary; `auto` asks the router whether one is due today. Throws on any failure. */
66
+ export async function fetchSummary(
67
+ baseUrl: string,
68
+ harnessId: string,
69
+ auto: boolean,
70
+ headers: Record<string, string>,
71
+ fetchImpl: FetchLike = fetch,
72
+ timeoutMs = 5_000,
73
+ ): Promise<{ due: boolean; reason?: string; summary: DailySummary | null }> {
74
+ const params = new URLSearchParams();
75
+ if (harnessId !== "") params.set("harness", harnessId);
76
+ if (auto) params.set("auto", "1");
77
+ const q = params.toString();
78
+ const res = await fetchImpl(`${baseUrl}/v1/router/summary${q === "" ? "" : `?${q}`}`, { headers, signal: AbortSignal.timeout(timeoutMs) });
79
+ if (!res.ok) throw new Error(`router returned ${res.status}`);
80
+ return (await res.json()) as { due: boolean; reason?: string; summary: DailySummary | null };
81
+ }
82
+
83
+ /** One spiking model as `/health` reports it. */
84
+ export interface SoftFailureSpikeView {
85
+ slug?: string;
86
+ recentDispatches?: number;
87
+ recentFailures?: number;
88
+ recentRate?: number;
89
+ baselineDispatches?: number;
90
+ baselineRate?: number;
91
+ }
92
+
93
+ /** "slug: 40% of 10 failed in the last 1h (7d baseline 8% of 120)" — one line per spiking model. */
94
+ export function renderSoftFailureSpikes(spikes: readonly SoftFailureSpikeView[] | undefined | null, recentMs = 3_600_000, baselineDays = 7): string[] {
95
+ if (spikes === undefined || spikes === null || spikes.length === 0) return [];
96
+ const window = recentMs >= 3_600_000 ? `${(recentMs / 3_600_000).toFixed(recentMs % 3_600_000 === 0 ? 0 : 1)}h` : `${Math.round(recentMs / 60_000)}m`;
97
+ return spikes.map(
98
+ (s) =>
99
+ `${s.slug ?? "?"}: ${((s.recentRate ?? 0) * 100).toFixed(0)}% of ${s.recentDispatches ?? 0} failed in the last ${window} (${baselineDays}d baseline ${((s.baselineRate ?? 0) * 100).toFixed(0)}% of ${s.baselineDispatches ?? 0})`,
100
+ );
101
+ }
102
+
64
103
  /** The subset of `/health` the status view renders. */
65
104
  export interface HealthSnapshot {
66
105
  status?: string;
@@ -80,6 +119,7 @@ export interface HealthSnapshot {
80
119
  runway?: { dailyBurnUsd?: number; creditsLeftUsd?: number; days?: number | null } | null;
81
120
  costBias?: { configured?: number; effective?: number; biasUntilUsage?: number };
82
121
  } | null;
122
+ softFailures?: { recentMs?: number; baselineDays?: number; spikes?: SoftFailureSpikeView[] } | null;
83
123
  catalog?: {
84
124
  models?: number;
85
125
  ageMs?: number;
@@ -119,6 +159,12 @@ export function renderStatus(baseUrl: string, h: HealthSnapshot, nowMs = Date.no
119
159
  const rwText = rw !== undefined && rw !== null ? ` · burn $${(rw.dailyBurnUsd ?? 0).toFixed(2)}/day · ${rw.days === null || rw.days === undefined ? "credits left: unknown burn" : `~${Math.round(rw.days)} days of credits left`}` : "";
120
160
  if (o.meter !== undefined && o.meter !== null) out.push(`ollama billing: ${calText}${rwText}`);
121
161
  }
162
+ const sf = h.softFailures;
163
+ if (sf !== undefined && sf !== null) {
164
+ const lines = renderSoftFailureSpikes(sf.spikes, sf.recentMs, sf.baselineDays);
165
+ if (lines.length === 0) out.push("soft failures: no model spiking in the last hour");
166
+ else out.push(`soft failures SPIKING (${lines.length}):`, ...lines.map((l) => ` ${l}`));
167
+ }
122
168
  const a = h.agentdox;
123
169
  out.push(a === undefined || a === null ? "agentdox: off" : `agentdox: ${a.url ?? "?"} scope ${a.defaultScope ?? "?"}${a.recordTurns === true ? " · recording turns" : ""}`);
124
170
  return out.join("\n");
@@ -50,6 +50,7 @@ import { routerConfigPath, writeRouterConfig } from "../src/cli/config-cmd.ts";
50
50
  import { loadConfig } from "../src/config/load.ts";
51
51
  import type { RouterConfig } from "../src/config/types.ts";
52
52
  import { buildUsageReport, renderUsageReport, type UsageReport } from "../src/cost/report.ts";
53
+ import { buildDailySummary, renderDailySummary } from "../src/cost/summary.ts";
53
54
  import { openDb } from "../src/util/sqlite.ts";
54
55
 
55
56
  import type { ExtensionAPI, ExtensionContext } from "@oh-my-pi/pi-coding-agent";
@@ -61,6 +62,7 @@ import { ReportHub } from "./report-hub.ts";
61
62
  import {
62
63
  describeOverride,
63
64
  fetchReport,
65
+ fetchSummary,
64
66
  parseOverrideArgs,
65
67
  parseReportArgs,
66
68
  renderStatus,
@@ -77,12 +79,32 @@ const HARNESS_ID = process.env.OMP_HARNESS_ID ?? "";
77
79
 
78
80
  /** Custom message type for report/status output in the transcript. */
79
81
  const MESSAGE_TYPE = "auto-model-router";
82
+ /** The embedded router boots on session_start too; the auto summary waits for it this long. */
83
+ const SUMMARY_TRIES = 8;
84
+ const SUMMARY_RETRY_MS = 1_500;
80
85
 
81
86
  export default function (pi: ExtensionAPI): void {
82
87
  pi.setLabel("auto-model-router");
83
88
 
89
+ // Once a day, the first interactive session posts yesterday's summary. The
90
+ // router decides whether one is due (report.dailySummary, a per-harness
91
+ // marker), so several windows on one router show it once between them.
92
+ pi.on("session_start", async (_event, ctx) => {
93
+ if (!ctx.hasUI) return;
94
+ for (let i = 0; i < SUMMARY_TRIES; i++) {
95
+ try {
96
+ const r = await fetchSummary(routerBaseUrl(), HARNESS_ID, true, routerAuthHeaders(), fetch, 2_000);
97
+ if (r.due && r.summary !== null) post(pi, renderDailySummary(r.summary));
98
+ return;
99
+ } catch {
100
+ // Router still starting (embed) or absent: try again shortly, then give up quietly.
101
+ await new Promise((resolve) => setTimeout(resolve, SUMMARY_RETRY_MS));
102
+ }
103
+ }
104
+ });
105
+
84
106
  pi.registerCommand("router", {
85
- description: "auto-model-router: configure, usage report, status",
107
+ description: "auto-model-router: configure, usage report, status, daily summary",
86
108
  handler: async (args, ctx) => {
87
109
  const [verb = "", ...rest] = args.trim().split(/\s+/).filter((t) => t !== "");
88
110
  const tail = rest.join(" ");
@@ -95,6 +117,9 @@ export default function (pi: ExtensionAPI): void {
95
117
  case "status":
96
118
  case "health":
97
119
  return status(pi, ctx);
120
+ case "summary":
121
+ case "daily":
122
+ return summary(pi, ctx, tail);
98
123
  case "why":
99
124
  case "explain":
100
125
  return why(pi, ctx);
@@ -116,20 +141,22 @@ export default function (pi: ExtensionAPI): void {
116
141
  case "":
117
142
  break;
118
143
  default:
119
- ctx.ui.notify(`unknown /router subcommand "${verb}" (config | report | status | why | good | bad | pin | tier)`, "warn");
144
+ ctx.ui.notify(`unknown /router subcommand "${verb}" (config | report | summary | status | why | good | bad | pin | tier)`, "warn");
120
145
  return;
121
146
  }
122
147
 
123
148
  const chosen = await ctx.ui.select("auto-model-router", [
124
149
  { label: "Configure", description: "edit any router setting" },
125
150
  { label: "Report", description: "usage analytics; window and scope adjustable inside" },
126
- { label: "Status", description: "keys, catalog, Ollama, agentdox" },
151
+ { label: "Summary", description: "the last 24h in a few lines" },
152
+ { label: "Status", description: "keys, catalog, Ollama, agentdox, soft-failure spikes" },
127
153
  { label: "Why", description: "explain this session's last routed turn" },
128
154
  { label: "Override", description: "pin a model or force a tier for this session" },
129
155
  ]);
130
156
  if (chosen === undefined) return;
131
157
  if (chosen === "Configure") return configure(ctx);
132
158
  if (chosen === "Status") return status(pi, ctx);
159
+ if (chosen === "Summary") return summary(pi, ctx, "");
133
160
  if (chosen === "Report") return report(pi, ctx, "");
134
161
  if (chosen === "Why") return why(pi, ctx);
135
162
  if (chosen === "Override") return override(ctx, "tier", "");
@@ -299,6 +326,31 @@ async function override(ctx: ExtensionContext, verb: "pin" | "tier", text: strin
299
326
  }
300
327
  }
301
328
 
329
+ /** `/router summary [--all]`: the last 24h, from the router or (router down) the ledger directly. */
330
+ async function summary(pi: ExtensionAPI, ctx: ExtensionContext, argText: string): Promise<void> {
331
+ const harnessId = parseReportArgs(argText, HARNESS_ID).harnessId;
332
+ try {
333
+ const r = await fetchSummary(routerBaseUrl(), harnessId, false, routerAuthHeaders());
334
+ if (r.summary !== null) {
335
+ post(pi, renderDailySummary(r.summary));
336
+ return;
337
+ }
338
+ } catch {
339
+ // Fall through to the ledger.
340
+ }
341
+ const cfg = loadConfig();
342
+ if (!existsSync(cfg.ledger.path)) {
343
+ ctx.ui.notify(`router unreachable at ${routerBaseUrl()} and no ledger at ${cfg.ledger.path}`, "error");
344
+ return;
345
+ }
346
+ const db = openDb(cfg.ledger.path);
347
+ try {
348
+ post(pi, `${renderDailySummary(buildDailySummary(db, { harnessId }))}\n(router unreachable: read from the ledger; spikes and the Ollama meter need the router)`);
349
+ } finally {
350
+ db.close();
351
+ }
352
+ }
353
+
302
354
  async function status(pi: ExtensionAPI, ctx: ExtensionContext): Promise<void> {
303
355
  try {
304
356
  post(pi, await loadStatus());
@@ -0,0 +1,93 @@
1
+ /**
2
+ * omp extension: condense large tool results with a cheap model before an
3
+ * expensive one reads them.
4
+ *
5
+ * Tool results are the bulk of every prompt, and a prompt is ~96% of spend.
6
+ * When a read, grep, glob or bash result is large and this session's
7
+ * current model sits at or above the router's `digest.fromTier`, the raw
8
+ * text goes to the router's `/v1/router/digest`, a simple-tier model
9
+ * rewrites it to what the task needs, and the digest replaces the tool
10
+ * result. The digest starts with a marker saying how to get the full output
11
+ * back (re-run the tool, or read a line range), so nothing is lost.
12
+ *
13
+ * The router decides (policy, session tier, cost guard); this extension only
14
+ * ships text that passes the cheap client-side checks. Off unless
15
+ * `digest.enabled` is set in the router config.
16
+ *
17
+ * Install beside router-embed.ts:
18
+ *
19
+ * # ~/.omp/agent/config.yml
20
+ * extensions:
21
+ * - /path/to/auto-model-router/omp-extension/router-embed.ts
22
+ * - /path/to/auto-model-router/omp-extension/router-digest.ts
23
+ */
24
+
25
+ import type { ExtensionAPI } from "@oh-my-pi/pi-coding-agent";
26
+
27
+ import { DISABLED_POLICY, digestToast, parsePolicy, shouldSend, textOf, type DigestPolicy } from "./digest-logic.ts";
28
+ import { routerAuthHeaders, routerBaseUrl } from "./router-url.ts";
29
+
30
+ const HARNESS_ID = process.env.OMP_HARNESS_ID ?? "";
31
+ /** Re-read the policy this often, so a config change lands without a restart. */
32
+ const POLICY_TTL_MS = 60_000;
33
+
34
+ export default function (pi: ExtensionAPI): void {
35
+ pi.setLabel("auto-model-router digest");
36
+
37
+ let policy: DigestPolicy = DISABLED_POLICY;
38
+ let policyAtMs = 0;
39
+ let lastUserText = "";
40
+
41
+ async function refreshPolicy(): Promise<void> {
42
+ if (Date.now() - policyAtMs < POLICY_TTL_MS) return;
43
+ policyAtMs = Date.now();
44
+ try {
45
+ const res = await fetch(`${routerBaseUrl()}/v1/router/digest/policy`, { headers: routerAuthHeaders(), signal: AbortSignal.timeout(2_000) });
46
+ policy = res.ok ? parsePolicy(await res.json()) : DISABLED_POLICY;
47
+ } catch {
48
+ policy = DISABLED_POLICY;
49
+ }
50
+ }
51
+
52
+ pi.on("session_start", async () => {
53
+ policyAtMs = 0;
54
+ await refreshPolicy();
55
+ });
56
+
57
+ // The user's latest ask steers what the digest keeps.
58
+ pi.on("input", (event) => {
59
+ const e = event as { text?: string };
60
+ if (typeof e.text === "string" && e.text.trim() !== "") lastUserText = e.text.trim().slice(0, 400);
61
+ return undefined;
62
+ });
63
+
64
+ pi.on("tool_result", async (event, ctx) => {
65
+ const e = event as { toolName: string; input: Record<string, unknown>; content: Array<{ type: string; text?: string }>; isError: boolean };
66
+ await refreshPolicy();
67
+ const { text, hasImage } = textOf(e.content);
68
+ if (!shouldSend(policy, e.toolName, e.isError, text, hasImage)) return undefined;
69
+ try {
70
+ const res = await fetch(`${routerBaseUrl()}/v1/router/digest`, {
71
+ method: "POST",
72
+ headers: { ...routerAuthHeaders(), "content-type": "application/json" },
73
+ body: JSON.stringify({
74
+ ompSessionId: ctx.sessionManager.getSessionId(),
75
+ harnessId: HARNESS_ID,
76
+ toolName: e.toolName,
77
+ input: e.input,
78
+ content: text,
79
+ query: lastUserText,
80
+ }),
81
+ signal: AbortSignal.timeout(30_000),
82
+ });
83
+ if (!res.ok) return undefined;
84
+ const r = (await res.json()) as { digested: boolean; text?: string; model?: string; usd?: number; inputBytes?: number; outputChars?: number };
85
+ if (!r.digested || typeof r.text !== "string") return undefined;
86
+ if (ctx.hasUI) ctx.ui.notify(digestToast(e.toolName, r.inputBytes ?? 0, r.outputChars ?? 0, r.model ?? "?", r.usd ?? 0), "info");
87
+ return { content: [{ type: "text", text: r.text }] };
88
+ } catch {
89
+ // Router unreachable or slow: the raw result stands.
90
+ return undefined;
91
+ }
92
+ });
93
+ }
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "auto-model-router",
3
- "version": "0.4.1",
3
+ "version": "0.4.3",
4
4
  "private": false,
5
5
  "description": "Local cost/complexity-aware model router for Oh My Pi, backed by OpenRouter",
6
6
  "type": "module",
@@ -184,6 +184,7 @@ export const WIZARD_SECTIONS: readonly SectionSpec[] = [
184
184
  { path: "filters.requireToolSupport", label: "Require tool support", kind: "boolean" },
185
185
  { path: "filters.minTrust", label: "Min trust", kind: "number", min: 0, max: 1 },
186
186
  { path: "filters.feedbackWeight", label: "Feedback weight in trust", kind: "number", min: 0, hint: "0=record only; a bad verdict = this many failures" },
187
+ { path: "filters.feedbackByTask", label: "Scope verdicts to the task type", kind: "boolean" },
187
188
  { path: "filters.minTrustSamples", label: "Min trust samples", kind: "number", min: 0 },
188
189
  { path: "filters.trustScopedByHarness", label: "Scope trust per harness", kind: "boolean" },
189
190
  { path: "filters.trustWindowDays", label: "Trust window", kind: "number", min: 0, hint: "days, 0=all time" },
@@ -280,6 +281,8 @@ export const WIZARD_SECTIONS: readonly SectionSpec[] = [
280
281
  { path: "compaction.keepTailBytes", label: "Keep tail bytes", kind: "number", min: 0 },
281
282
  { path: "compaction.elideSupersededReads", label: "Elide superseded reads", kind: "boolean" },
282
283
  { path: "compaction.collapseDuplicateResults", label: "Collapse duplicate results", kind: "boolean" },
284
+ { path: "compaction.digestToolResults", label: "Digest compacted results with a cheap model", kind: "boolean" },
285
+ { path: "compaction.digestMaxPerTurn", label: "Digests per turn at most", kind: "number", min: 0 },
283
286
  ],
284
287
  },
285
288
  {
@@ -310,9 +313,27 @@ export const WIZARD_SECTIONS: readonly SectionSpec[] = [
310
313
  { path: "budget.onExceeded", label: "On exceeded", kind: "enum", options: ["downgrade", "reject"] },
311
314
  ],
312
315
  },
316
+ {
317
+ title: "Digest",
318
+ fields: [
319
+ { path: "digest.enabled", label: "Digest large tool results with a cheap model", kind: "boolean" },
320
+ { path: "digest.minBytes", label: "Min result bytes", kind: "number", min: 0 },
321
+ { path: "digest.maxBytes", label: "Max result bytes", kind: "number", min: 1 },
322
+ { path: "digest.tools", label: "Tools eligible", kind: "stringArray", hint: "comma-separated, lower-case" },
323
+ { path: "digest.fromTier", label: "Digest when the session is at or above", kind: "enum", options: TIER_NAMES },
324
+ { path: "digest.tier", label: "Pick the digest model from tier", kind: "enum", options: TIER_NAMES },
325
+ { path: "digest.model", label: "Pinned digest model", kind: "string", optional: true, hint: "blank = cheapest in tier" },
326
+ { path: "digest.maxOutputTokens", label: "Max digest tokens", kind: "number", min: 1 },
327
+ { path: "digest.maxCostUsd", label: "Max cost per digest $", kind: "number", min: 0 },
328
+ { path: "digest.timeoutMs", label: "Digest timeout", kind: "number", min: 1, hint: "ms" },
329
+ ],
330
+ },
313
331
  {
314
332
  title: "Report",
315
- fields: [{ path: "report.baselines", label: "Counterfactual baseline models", kind: "stringArray", hint: "comma-separated slugs" }],
333
+ fields: [
334
+ { path: "report.baselines", label: "Counterfactual baseline models", kind: "stringArray", hint: "comma-separated slugs" },
335
+ { path: "report.dailySummary", label: "Daily summary at session start", kind: "boolean" },
336
+ ],
316
337
  },
317
338
  {
318
339
  title: "Ledger",
@@ -103,6 +103,7 @@ export const DEFAULT_CONFIG: RouterConfig = {
103
103
  minTrust: 0.7,
104
104
  // Verdicts are recorded and reported first; weigh them once there are some.
105
105
  feedbackWeight: 0,
106
+ feedbackByTask: false,
106
107
  minTrustSamples: 12,
107
108
  // Shared trust by default: more samples, demotion guard stays effective
108
109
  // even with a tiny guardrail-narrowed catalog.
@@ -291,10 +292,29 @@ export const DEFAULT_CONFIG: RouterConfig = {
291
292
  keepTailBytes: 512,
292
293
  elideSupersededReads: true,
293
294
  collapseDuplicateResults: true,
295
+ // Off: a synchronous cheap-model call before dispatch, only worth it where
296
+ // stale tool output is the prompt and the turn is on a dear model.
297
+ digestToolResults: false,
298
+ digestMaxPerTurn: 2,
299
+ },
300
+ digest: {
301
+ // Off until an operator turns it on: it changes what the model reads.
302
+ enabled: false,
303
+ minBytes: 12_000,
304
+ maxBytes: 400_000,
305
+ tools: ["read", "grep", "glob", "bash", "web_fetch", "webfetch", "ls", "find"],
306
+ fromTier: "moderate",
307
+ tier: "simple",
308
+ model: "",
309
+ maxOutputTokens: 700,
310
+ maxCostUsd: 0.02,
311
+ timeoutMs: 25_000,
294
312
  },
295
313
  report: {
296
314
  // The frontier pair most omp users would otherwise run on.
297
315
  baselines: ["anthropic/claude-opus-5", "anthropic/claude-sonnet-5"],
316
+ // One transcript message per day, at the first interactive session start.
317
+ dailySummary: true,
298
318
  },
299
319
  budget: {
300
320
  // No caps by default; at a configured ceiling, downgrade rather than fail.
@@ -90,6 +90,7 @@ const filters = z.strictObject({
90
90
  requireToolSupport: z.boolean().optional(),
91
91
  minTrust: z.number().min(0).max(1).optional(),
92
92
  feedbackWeight: z.number().nonnegative().optional(),
93
+ feedbackByTask: z.boolean().optional(),
93
94
  minTrustSamples: z.number().int().nonnegative().optional(),
94
95
  trustScopedByHarness: z.boolean().optional(),
95
96
  trustWindowDays: z.number().nonnegative().optional(),
@@ -205,6 +206,8 @@ const compaction = z.strictObject({
205
206
  keepTailBytes: z.number().int().nonnegative().optional(),
206
207
  elideSupersededReads: z.boolean().optional(),
207
208
  collapseDuplicateResults: z.boolean().optional(),
209
+ digestToolResults: z.boolean().optional(),
210
+ digestMaxPerTurn: z.number().int().nonnegative().optional(),
208
211
  });
209
212
 
210
213
  const budget = z.strictObject({
@@ -272,7 +275,21 @@ export const configInputSchema = z.strictObject({
272
275
  compaction: compaction.optional(),
273
276
  budget: budget.optional(),
274
277
  profiles: z.array(profile).optional(),
275
- report: z.strictObject({ baselines: z.array(z.string()).optional() }).optional(),
278
+ report: z.strictObject({ baselines: z.array(z.string()).optional(), dailySummary: z.boolean().optional() }).optional(),
279
+ digest: z
280
+ .strictObject({
281
+ enabled: z.boolean().optional(),
282
+ minBytes: z.number().int().nonnegative().optional(),
283
+ maxBytes: z.number().int().positive().optional(),
284
+ tools: z.array(z.string()).optional(),
285
+ fromTier: tier.optional(),
286
+ tier: tier.optional(),
287
+ model: z.string().optional(),
288
+ maxOutputTokens: z.number().int().positive().optional(),
289
+ maxCostUsd: z.number().nonnegative().optional(),
290
+ timeoutMs: z.number().int().positive().optional(),
291
+ })
292
+ .optional(),
276
293
  ledger: ledger.optional(),
277
294
  adaptiveTierFloors: z.boolean().optional(),
278
295
  adaptivePriceCeilings: z.boolean().optional(),