auto-model-router 0.30.3 → 0.32.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (112) hide show
  1. package/.omp-plugin/marketplace.json +2 -2
  2. package/README.md +32 -2
  3. package/omp-extension/router-configure.ts +9 -7
  4. package/package.json +1 -1
  5. package/src/cli/config-cmd.ts +8 -7
  6. package/src/cli/explain.ts +10 -5
  7. package/src/cli/export.ts +6 -5
  8. package/src/cli/models.ts +10 -7
  9. package/src/cli/report.ts +6 -1
  10. package/src/cli/stats.ts +7 -7
  11. package/src/config/load.ts +10 -1
  12. package/src/config/types.ts +10 -1
  13. package/src/context/bridge.ts +7 -7
  14. package/src/context/index.ts +3 -3
  15. package/src/context/store.ts +39 -56
  16. package/src/context/types.ts +7 -6
  17. package/src/cost/blended.ts +28 -7
  18. package/src/cost/feedback.ts +33 -37
  19. package/src/cost/ledger-sql.ts +547 -0
  20. package/src/cost/ledger.ts +30 -459
  21. package/src/cost/report.ts +171 -129
  22. package/src/cost/retention.ts +10 -10
  23. package/src/cost/summary.ts +15 -10
  24. package/src/cost/types.ts +43 -62
  25. package/src/cost/views.ts +79 -49
  26. package/src/eval/calibrate.ts +47 -12
  27. package/src/eval/run.ts +18 -2
  28. package/src/lib.ts +6 -2
  29. package/src/router/candidates.ts +7 -15
  30. package/src/router/classify.ts +6 -4
  31. package/src/router/index.ts +95 -9
  32. package/src/router/select.ts +38 -21
  33. package/src/router/state.ts +90 -102
  34. package/src/router/types.ts +11 -5
  35. package/src/server/advise.ts +6 -4
  36. package/src/server/compaction-digest.ts +1 -1
  37. package/src/server/digest.ts +9 -10
  38. package/src/server/http.ts +109 -46
  39. package/src/server/providers.ts +18 -4
  40. package/src/server/turn.ts +32 -9
  41. package/src/tokens/estimate.ts +16 -6
  42. package/src/upstream/ollama-usage.ts +21 -11
  43. package/src/util/schema.ts +201 -0
  44. package/src/util/sql.ts +246 -0
  45. package/src/wire/anthropic/messages.ts +3 -4
  46. package/src/wire/openai/request.ts +1 -0
  47. package/src/wire/types.ts +7 -0
  48. package/test/anthropic-wire.test.ts +9 -9
  49. package/test/benchmark-feeds.test.ts +7 -7
  50. package/test/cache-control.test.ts +7 -7
  51. package/test/cache-estimate.test.ts +5 -5
  52. package/test/catalog-view.test.ts +4 -4
  53. package/test/catalog.test.ts +11 -11
  54. package/test/classify.test.ts +24 -24
  55. package/test/compaction.test.ts +20 -20
  56. package/test/config-wizard.test.ts +32 -32
  57. package/test/config.test.ts +10 -10
  58. package/test/connect-harnesses.test.ts +11 -11
  59. package/test/context-bridge.test.ts +40 -30
  60. package/test/context-prune.test.ts +43 -36
  61. package/test/context-query.test.ts +8 -8
  62. package/test/controls.test.ts +54 -27
  63. package/test/cost.test.ts +12 -12
  64. package/test/digest.test.ts +55 -44
  65. package/test/embed-lifecycle.test.ts +5 -5
  66. package/test/embed-logic.test.ts +26 -26
  67. package/test/escalate.test.ts +17 -17
  68. package/test/eval.test.ts +73 -16
  69. package/test/executable.test.ts +6 -6
  70. package/test/exploration.test.ts +19 -20
  71. package/test/failover.test.ts +22 -21
  72. package/test/fakes.ts +105 -0
  73. package/test/features.test.ts +21 -21
  74. package/test/harness-requests.test.ts +3 -3
  75. package/test/harness-switch.test.ts +5 -5
  76. package/test/hold-exploration.test.ts +13 -13
  77. package/test/hot-reload.test.ts +5 -5
  78. package/test/learned.test.ts +5 -5
  79. package/test/ledger-sql.test.ts +342 -0
  80. package/test/mcp-entry.test.ts +5 -5
  81. package/test/migrations.test.ts +28 -22
  82. package/test/models-yml.test.ts +18 -18
  83. package/test/ollama.test.ts +40 -34
  84. package/test/omp-credentials.test.ts +16 -16
  85. package/test/policy.test.ts +3 -3
  86. package/test/reconfigure.test.ts +4 -4
  87. package/test/redaction.test.ts +41 -35
  88. package/test/remote.test.ts +12 -12
  89. package/test/report-logic.test.ts +8 -8
  90. package/test/report.test.ts +95 -87
  91. package/test/retention.test.ts +79 -66
  92. package/test/schema.test.ts +123 -0
  93. package/test/scope.test.ts +8 -8
  94. package/test/select.test.ts +216 -257
  95. package/test/skills.test.ts +3 -3
  96. package/test/sql-shim.test.ts +154 -0
  97. package/test/state.test.ts +43 -36
  98. package/test/summary.test.ts +38 -27
  99. package/test/tier-plan.test.ts +45 -62
  100. package/test/toast-logic.test.ts +31 -31
  101. package/test/tokens.test.ts +95 -80
  102. package/test/trust-attribution.test.ts +217 -187
  103. package/test/trust-window.test.ts +37 -32
  104. package/test/turn.test.ts +55 -23
  105. package/test/upstreams.test.ts +13 -13
  106. package/test/views.test.ts +81 -59
  107. package/test/wire-request.test.ts +17 -17
  108. package/test/wire-responses.test.ts +4 -4
  109. package/tools/agentdox-e2e.ts +5 -2
  110. package/tools/export-benchmarks.ts +5 -5
  111. package/tools/ledger-parity.ts +266 -0
  112. package/tools/replay.ts +16 -8
@@ -1,43 +1,22 @@
1
1
  /**
2
- * The spend ledger: one row per dispatched generation, plus the two derived
3
- * signals the router consumes per-model trust (Laplace-smoothed success) and
4
- * per-tokenizer-family token calibration.
2
+ * The ledger's shared vocabulary: the SQL fragments its aggregates are built
3
+ * from, the thresholds those aggregates are judged against, and the mappers
4
+ * that turn a stored row into the shapes the router consumes.
5
5
  *
6
- * `record` also persists the cost component split implied by the entry's own
7
- * model pricing (`cost_breakdown`). The split needs catalog prices, which the
8
- * ledger does not receive; it reads them back from the `catalog_cache` row the
9
- * catalog slice already persists, re-normalized lazily and re-read only when
10
- * the cache's `fetched_at_ms` changes. An entry recorded before the first
11
- * catalog fetch simply stores NULL and is skipped by the blended rate.
6
+ * `ledger-sql.ts` holds the one implementation, on either engine. This module
7
+ * exists so the meaning of a term — what counts as an attributable error, when
8
+ * a failure rate is a spike, how a row becomes a `LedgerEntry` is written
9
+ * once. Two copies of that is what this port set out to end: a divergent spike
10
+ * threshold silently changed which models were reported as failing.
12
11
  */
13
12
 
14
- import type { Database } from "bun:sqlite";
15
- import { normalizeCatalogModel } from "../catalog/openrouter-catalog.ts";
16
- import type { CatalogModel } from "../catalog/types.ts";
17
- import type { RouterConfig } from "../config/types.ts";
18
- import { consumePendingEstimate } from "../tokens/estimate.ts";
19
- import { computeBlendedRate } from "./blended.ts";
20
- import { computeCost } from "./forecast.ts";
21
- import type {
22
- BlendedRate,
23
- CostBreakdown,
24
- EscalationCost,
25
- Ledger,
26
- LedgerEntry,
27
- LedgerSignals,
28
- ModelCacheReliability,
29
- ModelLatency,
30
- ModelTrust,
31
- PruneResult,
32
- SoftFailureSpike,
33
- UsageCounts,
34
- } from "./types.ts";
13
+ import type { CostBreakdown, LedgerEntry, ModelLatency, ModelTrust, UsageCounts } from "./types.ts";
35
14
 
36
15
  /** Estimates below this many samples are noise; the default ratio is better. */
37
- const MIN_CALIBRATION_SAMPLES = 20;
16
+ export const MIN_CALIBRATION_SAMPLES = 20;
38
17
  /** Calibration samples outside this bytes-per-token band are provider accounting quirks, not tokenizer facts. */
39
- const MIN_SANE_BYTES_PER_TOKEN = 1.5;
40
- const MAX_SANE_BYTES_PER_TOKEN = 8;
18
+ export const MIN_SANE_BYTES_PER_TOKEN = 1.5;
19
+ export const MAX_SANE_BYTES_PER_TOKEN = 8;
41
20
  /**
42
21
  * Soft-failure spike detection (visibility only). A model is spiking when, over
43
22
  * the recent window, it has at least SPIKE_MIN_DISPATCHES dispatches, at least
@@ -46,25 +25,25 @@ const MAX_SANE_BYTES_PER_TOKEN = 8;
46
25
  * rate over the preceding window (a model with no baseline failures spikes on
47
26
  * the absolute floor alone).
48
27
  */
49
- const SPIKE_RECENT_MS = 60 * 60_000;
50
- const SPIKE_BASELINE_MS = 7 * 24 * 60 * 60_000;
51
- const SPIKE_MIN_DISPATCHES = 5;
52
- const SPIKE_MIN_FAILURES = 3;
53
- const SPIKE_MIN_RATE = 0.25;
54
- const SPIKE_RATIO = 2;
28
+ export const SPIKE_RECENT_MS = 60 * 60_000;
29
+ export const SPIKE_BASELINE_MS = 7 * 24 * 60 * 60_000;
30
+ export const SPIKE_MIN_DISPATCHES = 5;
31
+ export const SPIKE_MIN_FAILURES = 3;
32
+ export const SPIKE_MIN_RATE = 0.25;
33
+ export const SPIKE_RATIO = 2;
55
34
  /** Escalated attempts needed before their measured cost is trusted. */
56
- const MIN_ESCALATION_SAMPLES = 10;
35
+ export const MIN_ESCALATION_SAMPLES = 10;
57
36
  /** The escalation-cost aggregate scans a window of rows; memoised for this long. */
58
- const ESCALATION_COST_MEMO_MS = 60_000;
37
+ export const ESCALATION_COST_MEMO_MS = 60_000;
59
38
 
60
39
  /**
61
40
  * Cache reliability is one window-function pass over the newest rows, memoised
62
41
  * for a minute: the previous kept turn of each conversation is found with LAG,
63
42
  * and a row counts when that turn was on the same model within the warm TTL.
64
43
  */
65
- const CACHE_RELIABILITY_ROWS = 6_000;
66
- const CACHE_RELIABILITY_MEMO_MS = 60_000;
67
- const DAY_MS = 86_400_000;
44
+ export const CACHE_RELIABILITY_ROWS = 6_000;
45
+ export const CACHE_RELIABILITY_MEMO_MS = 60_000;
46
+ export const DAY_MS = 86_400_000;
68
47
 
69
48
  // Row shapes below are fixed by our own schema in util/sqlite.ts.
70
49
  export interface LedgerRow {
@@ -119,7 +98,7 @@ interface FeedbackRow {
119
98
  }
120
99
 
121
100
  /** Verdict counts per served slug since a cutoff (optionally one harness). */
122
- const FEEDBACK_SELECT = `COALESCE(SUM(CASE WHEN f.verdict = 'good' THEN 1 ELSE 0 END), 0) AS good,
101
+ export const FEEDBACK_SELECT = `COALESCE(SUM(CASE WHEN f.verdict = 'good' THEN 1 ELSE 0 END), 0) AS good,
123
102
  COALESCE(SUM(CASE WHEN f.verdict = 'bad' THEN 1 ELSE 0 END), 0) AS bad`;
124
103
 
125
104
  interface LatencyRow {
@@ -129,12 +108,6 @@ interface LatencyRow {
129
108
  elapsed_ms_sum: number | null;
130
109
  }
131
110
 
132
- interface CalibrationRow {
133
- est_bytes: number;
134
- actual_tokens: number;
135
- samples: number;
136
- }
137
-
138
111
  /**
139
112
  * Error kinds that say nothing about a MODEL's reliability, and so must not
140
113
  * count against its trust:
@@ -156,9 +129,9 @@ interface CalibrationRow {
156
129
  // allowance is a fact about the account, identical for every model behind it.
157
130
  const UNATTRIBUTABLE_KINDS = "('aborted', 'auth', 'moderation', 'model_unavailable', 'quota')";
158
131
 
159
- const ATTRIBUTABLE_ERROR = `error IS NOT NULL AND (error_kind IS NULL OR error_kind NOT IN ${UNATTRIBUTABLE_KINDS})`;
132
+ export const ATTRIBUTABLE_ERROR = `error IS NOT NULL AND (error_kind IS NULL OR error_kind NOT IN ${UNATTRIBUTABLE_KINDS})`;
160
133
 
161
- const TRUST_SELECT = `COUNT(*) AS attempts,
134
+ export const TRUST_SELECT = `COUNT(*) AS attempts,
162
135
  COALESCE(SUM(CASE WHEN escalation_signal IS NOT NULL THEN 1 ELSE 0 END), 0) AS escalations,
163
136
  COALESCE(SUM(CASE WHEN ${ATTRIBUTABLE_ERROR} THEN 1 ELSE 0 END), 0) AS errors,
164
137
  COALESCE(SUM(CASE WHEN escalation_signal IS NOT NULL OR (${ATTRIBUTABLE_ERROR}) THEN 1 ELSE 0 END), 0) AS failures,
@@ -181,7 +154,7 @@ const TRUST_SELECT = `COUNT(*) AS attempts,
181
154
  * 100% of coding at ~53s/turn despite latencyWeight=0.75). Trust (reliability)
182
155
  * stays all-time; latency (volatile) is recency-weighted.
183
156
  */
184
- const LATENCY_SELECT = `COUNT(CASE WHEN ttft_ms IS NOT NULL AND ttft_ms > 0 AND error IS NULL THEN 1 END) AS samples,
157
+ export const LATENCY_SELECT = `COUNT(CASE WHEN ttft_ms IS NOT NULL AND ttft_ms > 0 AND error IS NULL THEN 1 END) AS samples,
185
158
  AVG(CASE WHEN ttft_ms IS NOT NULL AND ttft_ms > 0 AND error IS NULL THEN ttft_ms END) AS ttft_ms,
186
159
  SUM(CASE WHEN ttft_ms IS NOT NULL AND ttft_ms > 0 AND error IS NULL AND latency_ms > ttft_ms
187
160
  AND json_extract(usage, '$.completionTokens') > 0
@@ -206,7 +179,7 @@ export const LATENCY_WINDOW_ROWS = 100;
206
179
  * row model-attributable — the stricter reading — rather than quietly
207
180
  * forgiving a failure we cannot classify.
208
181
  */
209
- function errorKindOf(error: string | null): string | null {
182
+ export function errorKindOf(error: string | null): string | null {
210
183
  if (error === null) return null;
211
184
  if (error === "request aborted") return "aborted";
212
185
  const sep = error.indexOf(": ");
@@ -214,7 +187,7 @@ function errorKindOf(error: string | null): string | null {
214
187
  return error.slice(0, sep);
215
188
  }
216
189
 
217
- function toTrust(slug: string, row: TrustRow, fb: FeedbackRow | null = null, feedbackWeight = 0): ModelTrust {
190
+ export function toTrust(slug: string, row: TrustRow, fb: FeedbackRow | null = null, feedbackWeight = 0): ModelTrust {
218
191
  // Laplace smoothing: an untried model scores a neutral 1/2, and a failure
219
192
  // is an attempt superseded by an escalation or ended in an upstream error.
220
193
  // A user verdict counts as feedbackWeight extra attempts of that outcome.
@@ -235,7 +208,7 @@ function toTrust(slug: string, row: TrustRow, fb: FeedbackRow | null = null, fee
235
208
  };
236
209
  }
237
210
 
238
- function toLatency(slug: string, row: LatencyRow): ModelLatency | null {
211
+ export function toLatency(slug: string, row: LatencyRow): ModelLatency | null {
239
212
  if (row.samples <= 0 || row.ttft_ms === null) return null;
240
213
  const tokensPerSec =
241
214
  row.elapsed_ms_sum !== null && row.elapsed_ms_sum > 0 && row.ctok_sum !== null
@@ -290,406 +263,4 @@ export function toEntry(row: LedgerRow): LedgerEntry {
290
263
  };
291
264
  }
292
265
 
293
- export interface CacheReliabilityRow {
294
- slug: string;
295
- samples: number;
296
- hit: number;
297
- }
298
-
299
- /**
300
- * Observed cache hit rates when a warm cache was expected, per served slug.
301
- * `sinceMs` bounds the rows scanned (0 ⇒ the newest `limitRows`). Rows whose
302
- * cache count the router estimated (`usage.cachedEstimated`) are excluded.
303
- */
304
- export function queryCacheReliability(db: Database, opts: { warmTtlMs: number; sinceMs?: number; limitRows?: number }): CacheReliabilityRow[] {
305
- const sinceMs = opts.sinceMs ?? 0;
306
- const limitRows = opts.limitRows ?? CACHE_RELIABILITY_ROWS;
307
- return db
308
- .query(
309
- `WITH recent AS (
310
- SELECT conversation_key AS ck, created_at_ms AS t, COALESCE(served_slug, slug) AS s,
311
- json_extract(usage, '$.promptTokens') AS p, json_extract(usage, '$.cachedTokens') AS c,
312
- COALESCE(json_extract(usage, '$.cachedEstimated'), 0) AS est
313
- FROM ledger WHERE wasted = 0 AND error IS NULL AND created_at_ms >= $since
314
- ORDER BY created_at_ms DESC LIMIT $limit),
315
- seq AS (
316
- SELECT s, p, c, est, t,
317
- LAG(s) OVER w AS prev_s, LAG(p) OVER w AS prev_p, LAG(t) OVER w AS prev_t
318
- FROM recent WINDOW w AS (PARTITION BY ck ORDER BY t))
319
- SELECT s AS slug, COUNT(*) AS samples, AVG(MIN(1.0, c * 1.0 / MIN(prev_p, p))) AS hit
320
- FROM seq
321
- WHERE prev_s = s AND p > 1000 AND prev_p > 1000 AND t - prev_t <= $ttl AND est = 0
322
- GROUP BY s`,
323
- )
324
- .all({ $since: sinceMs, $limit: limitRows, $ttl: opts.warmTtlMs }) as CacheReliabilityRow[];
325
- }
326
-
327
- export function createLedger(db: Database, cfg: RouterConfig): Ledger {
328
- let cacheMemo: { atMs: number; map: Map<string, ModelCacheReliability> } | null = null;
329
- // Prepared once: record() runs on every turn.
330
- const insertStmt = db.query(
331
- `INSERT INTO ledger (
332
- id, created_at_ms, conversation_key, session_id, turn, requested_model, harness_id, omp_session_id, slug, served_slug,
333
- tier, classification_source, reasons, predicted_usd, reported_usd, usage, cost_breakdown,
334
- attempt, escalation_signal, latency_ms, ttft_ms, finish_reason, wasted, upstream_generation_id, error,
335
- error_kind, features, score, confidence, task, classifier_reasons, explored_from, hold_arm, prompt_tokens_saved, scope, redactions
336
- ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,
337
- );
338
- const calibrationStmt = db.query(
339
- `INSERT INTO token_calibration (tokenizer, est_bytes, actual_tokens, samples) VALUES (?, ?, ?, 1)
340
- ON CONFLICT(tokenizer) DO UPDATE SET
341
- est_bytes = est_bytes + excluded.est_bytes,
342
- actual_tokens = actual_tokens + excluded.actual_tokens,
343
- samples = samples + 1`,
344
- );
345
- const spendByConversationStmt = db.query(
346
- "SELECT COALESCE(SUM(COALESCE(reported_usd, predicted_usd)), 0) AS total FROM ledger WHERE conversation_key = ?",
347
- );
348
- const spendSinceStmt = db.query(
349
- "SELECT COALESCE(SUM(COALESCE(reported_usd, predicted_usd)), 0) AS total FROM ledger WHERE created_at_ms >= ?",
350
- );
351
- const spendSinceHarnessStmt = db.query(
352
- "SELECT COALESCE(SUM(COALESCE(reported_usd, predicted_usd)), 0) AS total FROM ledger WHERE created_at_ms >= ? AND harness_id = ?",
353
- );
354
- // `created_at_ms > ?` is always present, with a cutoff of 0 meaning all-time.
355
- // One statement shape rather than two keeps the plan (and the index it uses,
356
- // idx_ledger_slug_created) identical whether or not a window is configured.
357
- const trustStmt = db.query(`SELECT ${TRUST_SELECT} FROM ledger WHERE slug = ? AND created_at_ms > ?`);
358
- const trustHarnessStmt = db.query(`SELECT ${TRUST_SELECT} FROM ledger WHERE slug = ? AND harness_id = ? AND created_at_ms > ?`);
359
- const allTrustStmt = db.query(`SELECT slug, ${TRUST_SELECT} FROM ledger WHERE created_at_ms > ? GROUP BY slug`);
360
- const feedbackStmt = db.query(`SELECT ${FEEDBACK_SELECT} FROM feedback f WHERE f.slug = ? AND f.created_at_ms > ?`);
361
- const feedbackHarnessStmt = db.query(
362
- `SELECT ${FEEDBACK_SELECT} FROM feedback f JOIN ledger l ON l.id = f.ledger_id WHERE f.slug = ? AND l.harness_id = ? AND f.created_at_ms > ?`,
363
- );
364
- const allFeedbackStmt = db.query(`SELECT f.slug, ${FEEDBACK_SELECT} FROM feedback f WHERE f.created_at_ms > ? GROUP BY f.slug`);
365
- // Task-scoped variants (filters.feedbackByTask): the judged turn's task
366
- // must match, or be unrecorded (older rows, or a turn that never classified).
367
- const feedbackTaskStmt = db.query(
368
- `SELECT ${FEEDBACK_SELECT} FROM feedback f JOIN ledger l ON l.id = f.ledger_id WHERE f.slug = ? AND (l.task = ? OR l.task IS NULL) AND f.created_at_ms > ?`,
369
- );
370
- const feedbackHarnessTaskStmt = db.query(
371
- `SELECT ${FEEDBACK_SELECT} FROM feedback f JOIN ledger l ON l.id = f.ledger_id WHERE f.slug = ? AND l.harness_id = ? AND (l.task = ? OR l.task IS NULL) AND f.created_at_ms > ?`,
372
- );
373
- const feedbackFor = (slug: string, harnessId: string | undefined, cutoff: number, task?: string): FeedbackRow | null => {
374
- if (cfg.filters.feedbackWeight <= 0) return null;
375
- const byHarness = harnessId !== undefined && harnessId !== "";
376
- if (cfg.filters.feedbackByTask && task !== undefined && task !== "") {
377
- return byHarness
378
- ? (feedbackHarnessTaskStmt.get(slug, harnessId, task, cutoff) as FeedbackRow | null)
379
- : (feedbackTaskStmt.get(slug, task, cutoff) as FeedbackRow | null);
380
- }
381
- return byHarness ? (feedbackHarnessStmt.get(slug, harnessId, cutoff) as FeedbackRow | null) : (feedbackStmt.get(slug, cutoff) as FeedbackRow | null);
382
- };
383
- const latencyStmt = db.query(
384
- `SELECT ${LATENCY_SELECT} FROM (SELECT * FROM ledger WHERE slug = ? ORDER BY created_at_ms DESC LIMIT ${LATENCY_WINDOW_ROWS})`,
385
- );
386
- const latencyHarnessStmt = db.query(
387
- `SELECT ${LATENCY_SELECT} FROM (SELECT * FROM ledger WHERE slug = ? AND harness_id = ? ORDER BY created_at_ms DESC LIMIT ${LATENCY_WINDOW_ROWS})`,
388
- );
389
- const ratioStmt = db.query("SELECT est_bytes, actual_tokens, samples FROM token_calibration WHERE tokenizer = ?");
390
- const recentStmt = db.query("SELECT * FROM ledger ORDER BY created_at_ms DESC LIMIT ?");
391
- const pruneStmt = db.query("DELETE FROM ledger WHERE created_at_ms < ?");
392
- // Feedback is a verdict ON a ledger row; keeping it past the turn it judges
393
- // would leave a note about a conversation the operator asked us to forget.
394
- // Matched by the row it points at AND by its own age, so verdicts orphaned
395
- // by a prune that ran before v0.21.0 are swept up too.
396
- const pruneFeedbackStmt = db.query(
397
- "DELETE FROM feedback WHERE created_at_ms < ? OR ledger_id IN (SELECT id FROM ledger WHERE created_at_ms < ?)",
398
- );
399
- // Ollama meter samples (one per usage poll) only matter for the current
400
- // billing cycle's calibration; they age out with the ledger rows.
401
- const pruneMeterStmt = db.query("DELETE FROM ollama_meter_samples WHERE at_ms < ?");
402
- const oldestStmt = db.query("SELECT MIN(created_at_ms) AS oldest FROM ledger");
403
- const wasteStmt = db.query("UPDATE ledger SET wasted = 1 WHERE id = ?");
404
- const providerSpendStmt = db.query(
405
- "SELECT COALESCE(SUM(COALESCE(reported_usd, predicted_usd)), 0) AS total FROM ledger WHERE created_at_ms >= ? AND COALESCE(served_slug, slug) LIKE ?",
406
- );
407
- // Digest rows (requested_model 'digest') are side calls, not the session's turns.
408
- const sessionStmt = db.query("SELECT * FROM ledger WHERE omp_session_id = ? AND wasted = 0 AND requested_model <> 'digest' ORDER BY created_at_ms DESC LIMIT ?");
409
- // What an escalated retry actually bills, per prompt token, over a window.
410
- // attempt > 0 rows are the re-dispatches that followed a rejected attempt;
411
- // errored ones carry no usage and are excluded.
412
- const escalationCostStmt = db.query(
413
- `SELECT COUNT(*) AS samples,
414
- COALESCE(SUM(COALESCE(reported_usd, predicted_usd)), 0) AS usd,
415
- COALESCE(SUM(json_extract(usage, '$.promptTokens')), 0) AS prompt_tokens
416
- FROM ledger WHERE attempt > 0 AND error IS NULL AND created_at_ms >= ?`,
417
- );
418
- let escalationMemo: { atMs: number; windowDays: number; value: EscalationCost | null } | null = null;
419
- // Per-model failure counts over two adjacent windows: [recentStart, now] and
420
- // [baselineStart, recentStart). Wasted rows (the failed attempt a retry
421
- // replaced) stay in: they ARE the soft failures being counted. Digest side
422
- // calls are excluded: they are not the session's turns.
423
- const softFailureStmt = db.query(
424
- `SELECT COALESCE(served_slug, slug) AS slug,
425
- SUM(CASE WHEN created_at_ms >= $recentStart THEN 1 ELSE 0 END) AS recent_n,
426
- SUM(CASE WHEN created_at_ms >= $recentStart AND (escalation_signal IS NOT NULL OR (${ATTRIBUTABLE_ERROR})) THEN 1 ELSE 0 END) AS recent_f,
427
- SUM(CASE WHEN created_at_ms < $recentStart THEN 1 ELSE 0 END) AS base_n,
428
- SUM(CASE WHEN created_at_ms < $recentStart AND (escalation_signal IS NOT NULL OR (${ATTRIBUTABLE_ERROR})) THEN 1 ELSE 0 END) AS base_f
429
- FROM ledger WHERE created_at_ms >= $baselineStart AND created_at_ms <= $now AND requested_model <> 'digest'
430
- GROUP BY COALESCE(served_slug, slug)`,
431
- );
432
- const cacheMetaStmt = db.query("SELECT fetched_at_ms FROM catalog_cache WHERE id = 1");
433
- const cachePayloadStmt = db.query("SELECT payload FROM catalog_cache WHERE id = 1");
434
-
435
- let indexFetchedAtMs = -1;
436
- let modelBySlug: Map<string, CatalogModel> | null = null;
437
-
438
- /** Slug → catalog model, rebuilt only when the catalog cache row changes. */
439
- function priceIndex(): Map<string, CatalogModel> | null {
440
- const meta = cacheMetaStmt.get() as { fetched_at_ms: number } | null;
441
- if (meta === null) return null;
442
- if (modelBySlug !== null && indexFetchedAtMs === meta.fetched_at_ms) return modelBySlug;
443
- const row = cachePayloadStmt.get() as { payload: string } | null;
444
- if (row === null) return null;
445
- const payload: unknown = JSON.parse(row.payload);
446
- if (!Array.isArray(payload)) return null;
447
- const map = new Map<string, CatalogModel>();
448
- for (const record of payload) {
449
- const model = normalizeCatalogModel(record);
450
- if (model !== null) map.set(model.slug, model);
451
- }
452
- modelBySlug = map;
453
- indexFetchedAtMs = meta.fetched_at_ms;
454
- return map;
455
- }
456
266
 
457
- return {
458
- record(entry: LedgerEntry): void {
459
- const models = priceIndex();
460
- const model =
461
- entry.priceModel ?? (entry.servedSlug !== null ? models?.get(entry.servedSlug) : undefined) ?? models?.get(entry.slug) ?? null;
462
- insertStmt.run(
463
- entry.id,
464
- entry.createdAtMs,
465
- entry.conversationKey,
466
- entry.sessionId,
467
- entry.turn,
468
- entry.requestedModel,
469
- entry.harnessId,
470
- entry.ompSessionId,
471
- entry.slug,
472
- entry.servedSlug,
473
- entry.tier,
474
- entry.classificationSource,
475
- JSON.stringify(entry.reasons),
476
- entry.predictedUsd,
477
- entry.reportedUsd,
478
- JSON.stringify(entry.usage),
479
- model !== null ? JSON.stringify(computeCost(model, entry.usage)) : null,
480
- entry.attempt,
481
- entry.escalationSignal,
482
- entry.latencyMs,
483
- entry.ttftMs,
484
- entry.finishReason,
485
- entry.wasted ? 1 : 0,
486
- entry.upstreamGenerationId,
487
- entry.error,
488
- errorKindOf(entry.error),
489
- entry.features === null ? null : JSON.stringify(entry.features),
490
- entry.score,
491
- entry.confidence,
492
- entry.task,
493
- entry.classifierReasons === null ? null : JSON.stringify(entry.classifierReasons),
494
- entry.exploredFrom,
495
- entry.holdArm,
496
- entry.promptTokensSaved,
497
- // A turn that carried no scope stores NULL, exactly as every row
498
- // written before v18 did; "" and absent are the same fact.
499
- entry.scope === undefined || entry.scope === "" ? null : entry.scope,
500
- // NULL when redaction was off for this turn; 0 says the rules ran.
501
- entry.redactions ?? null,
502
- );
503
- // Always consume the pending estimate, even when the turn failed, so a
504
- // dead turn's bytes can never pair with a later turn's tokens. Only
505
- // actually-billed prompt tokens calibrate.
506
- const pending = consumePendingEstimate(entry.conversationKey);
507
- if (entry.usage.promptTokens > 0 && pending !== null) {
508
- // The SERVED model's tokenizer produced the billing; the estimate-time
509
- // family is the fallback when the model is unknown to the catalog.
510
- const tokenizer = (model?.tokenizer ?? pending.tokenizer).trim().toLowerCase();
511
- // Reject samples no real tokenizer could produce. The rows are
512
- // running sums, so one provider that reports inflated counts (seen:
513
- // ~8x the bytes-implied tokens, i.e. 0.4 bytes/token) poisons a
514
- // whole family for thousands of samples. Text tokenizers land
515
- // between ~2 and ~5 bytes/token; the band is generous around that.
516
- const bytesPerToken = pending.bytes / entry.usage.promptTokens;
517
- if (bytesPerToken >= MIN_SANE_BYTES_PER_TOKEN && bytesPerToken <= MAX_SANE_BYTES_PER_TOKEN) {
518
- calibrationStmt.run(tokenizer, pending.bytes, entry.usage.promptTokens);
519
- }
520
- }
521
- },
522
-
523
- conversationSpend(conversationKey: string): number {
524
- const row = spendByConversationStmt.get(conversationKey) as { total: number } | null;
525
- return row?.total ?? 0;
526
- },
527
-
528
- spendSince(sinceMs: number, harnessId?: string): number {
529
- const row =
530
- harnessId !== undefined && harnessId !== ""
531
- ? (spendSinceHarnessStmt.get(sinceMs, harnessId) as { total: number } | null)
532
- : (spendSinceStmt.get(sinceMs) as { total: number } | null);
533
- return row?.total ?? 0;
534
- },
535
-
536
- blendedRate(windowDays: number): BlendedRate | null {
537
- return computeBlendedRate(db, cfg, windowDays);
538
- },
539
-
540
- trust(slug: string, harnessId?: string, task?: string): ModelTrust | null {
541
- // Read the window at CALL time, not at construction: hot reload mutates
542
- // the shared config object in place, so a pinned value would ignore an
543
- // edit until restart. 0 => cutoff 0 => every row qualifies.
544
- const cutoff = cfg.filters.trustWindowDays > 0 ? Date.now() - cfg.filters.trustWindowDays * DAY_MS : 0;
545
- const row =
546
- harnessId !== undefined && harnessId !== ""
547
- ? (trustHarnessStmt.get(slug, harnessId, cutoff) as TrustRow | null)
548
- : (trustStmt.get(slug, cutoff) as TrustRow | null);
549
- if (row === null || row.attempts === 0) return null;
550
- return toTrust(slug, row, feedbackFor(slug, harnessId, cutoff, task), cfg.filters.feedbackWeight);
551
- },
552
-
553
- allTrust(): ModelTrust[] {
554
- const cutoff = cfg.filters.trustWindowDays > 0 ? Date.now() - cfg.filters.trustWindowDays * DAY_MS : 0;
555
- const rows = allTrustStmt.all(cutoff) as (TrustRow & { slug: string })[];
556
- const fb = new Map<string, FeedbackRow>();
557
- if (cfg.filters.feedbackWeight > 0) {
558
- for (const r of allFeedbackStmt.all(cutoff) as (FeedbackRow & { slug: string })[]) fb.set(r.slug, r);
559
- }
560
- return rows.map((row) => toTrust(row.slug, row, fb.get(row.slug) ?? null, cfg.filters.feedbackWeight));
561
- },
562
-
563
- latency(slug: string, harnessId?: string): ModelLatency | null {
564
- const row =
565
- harnessId !== undefined && harnessId !== ""
566
- ? (latencyHarnessStmt.get(slug, harnessId) as LatencyRow | null)
567
- : (latencyStmt.get(slug) as LatencyRow | null);
568
- if (row === null) return null;
569
- return toLatency(slug, row);
570
- },
571
- signals(slugs: readonly string[], harnessId?: string, task?: string): Map<string, LedgerSignals> {
572
- const cutoff = cfg.filters.trustWindowDays > 0 ? Date.now() - cfg.filters.trustWindowDays * DAY_MS : 0;
573
- const hasHarness = harnessId !== undefined && harnessId !== "";
574
- const out = new Map<string, LedgerSignals>();
575
- for (const slug of slugs) {
576
- const trustRow = hasHarness
577
- ? (trustHarnessStmt.get(slug, harnessId, cutoff) as TrustRow | null)
578
- : (trustStmt.get(slug, cutoff) as TrustRow | null);
579
- const latencyRow = hasHarness
580
- ? (latencyHarnessStmt.get(slug, harnessId) as LatencyRow | null)
581
- : (latencyStmt.get(slug) as LatencyRow | null);
582
- out.set(slug, {
583
- trust: trustRow === null || trustRow.attempts === 0 ? null : toTrust(slug, trustRow, feedbackFor(slug, harnessId, cutoff, task), cfg.filters.feedbackWeight),
584
- latency: latencyRow === null ? null : toLatency(slug, latencyRow),
585
- });
586
- }
587
- return out;
588
- },
589
-
590
- cacheReliability(slug: string): ModelCacheReliability | null {
591
- const now = Date.now();
592
- if (cacheMemo === null || now - cacheMemo.atMs > CACHE_RELIABILITY_MEMO_MS) {
593
- const map = new Map<string, ModelCacheReliability>();
594
- for (const r of queryCacheReliability(db, { warmTtlMs: cfg.hysteresis.cacheWarmTtlMs })) {
595
- map.set(r.slug, { slug: r.slug, samples: r.samples, hitRate: Math.min(1, Math.max(0, r.hit)) });
596
- }
597
- cacheMemo = { atMs: now, map };
598
- }
599
- return cacheMemo.map.get(slug) ?? null;
600
- },
601
-
602
- escalationCost(windowDays: number): EscalationCost | null {
603
- const now = Date.now();
604
- if (escalationMemo !== null && escalationMemo.windowDays === windowDays && now - escalationMemo.atMs < ESCALATION_COST_MEMO_MS) {
605
- return escalationMemo.value;
606
- }
607
- const row = escalationCostStmt.get(now - windowDays * DAY_MS) as { samples: number; usd: number; prompt_tokens: number } | null;
608
- const value: EscalationCost | null =
609
- row === null || row.samples < MIN_ESCALATION_SAMPLES || row.prompt_tokens <= 0
610
- ? null
611
- : { usdPerPromptToken: row.usd / row.prompt_tokens, samples: row.samples, windowDays };
612
- escalationMemo = { atMs: now, windowDays, value };
613
- return value;
614
- },
615
-
616
- tokenRatio(tokenizer: string): number | null {
617
- const row = ratioStmt.get(tokenizer.trim().toLowerCase()) as CalibrationRow | null;
618
- if (row === null || row.samples < MIN_CALIBRATION_SAMPLES || row.actual_tokens <= 0) return null;
619
- return row.est_bytes / row.actual_tokens;
620
- },
621
-
622
- recentEntries(limit: number): LedgerEntry[] {
623
- const rows = recentStmt.all(limit) as LedgerRow[];
624
- return rows.map(toEntry);
625
- },
626
- softFailureSpikes(nowMs = Date.now(), recentMs = SPIKE_RECENT_MS, baselineMs = SPIKE_BASELINE_MS): SoftFailureSpike[] {
627
- const rows = softFailureStmt.all({ $now: nowMs, $recentStart: nowMs - recentMs, $baselineStart: nowMs - recentMs - baselineMs }) as {
628
- slug: string;
629
- recent_n: number;
630
- recent_f: number;
631
- base_n: number;
632
- base_f: number;
633
- }[];
634
- const spikes: SoftFailureSpike[] = [];
635
- for (const r of rows) {
636
- if (r.recent_n < SPIKE_MIN_DISPATCHES || r.recent_f < SPIKE_MIN_FAILURES) continue;
637
- const recentRate = r.recent_f / r.recent_n;
638
- const baselineRate = r.base_n > 0 ? r.base_f / r.base_n : 0;
639
- if (recentRate < SPIKE_MIN_RATE || recentRate < SPIKE_RATIO * baselineRate) continue;
640
- spikes.push({
641
- slug: r.slug,
642
- recentDispatches: r.recent_n,
643
- recentFailures: r.recent_f,
644
- recentRate,
645
- baselineDispatches: r.base_n,
646
- baselineFailures: r.base_f,
647
- baselineRate,
648
- });
649
- }
650
- spikes.sort((a, b) => b.recentRate - a.recentRate || b.recentFailures - a.recentFailures);
651
- return spikes;
652
- },
653
- providerSpendSince(slugPrefix: string, sinceMs: number): number {
654
- const row = providerSpendStmt.get(sinceMs, `${slugPrefix}%`) as { total: number } | null;
655
- return row?.total ?? 0;
656
- },
657
- prune(retentionDays: number | null, nowMs = Date.now()): PruneResult {
658
- const oldestKeptMs = (): number | null => (oldestStmt.get() as { oldest: number | null } | null)?.oldest ?? null;
659
- // null and 0 are the same instruction: keep everything. Still reports
660
- // how far back the ledger goes, which is what the caller asked.
661
- if (retentionDays === null || retentionDays <= 0) return { deleted: 0, oldestKeptMs: oldestKeptMs() };
662
- const cutoff = nowMs - retentionDays * DAY_MS;
663
- // Dependants first: the feedback statement reads the rows being deleted.
664
- pruneFeedbackStmt.run(cutoff, cutoff);
665
- pruneMeterStmt.run(cutoff);
666
- const deleted = pruneStmt.run(cutoff).changes;
667
- // Hand the freed pages back where the engine can (a ledger created at
668
- // v0.21.0 or later is auto_vacuum=INCREMENTAL; an older file reuses
669
- // them instead), then fold the WAL back so the space is real on disk.
670
- // Best-effort by design: a full ledger that could not shrink is a far
671
- // smaller problem than a prune that throws.
672
- if (deleted > 0) {
673
- try {
674
- db.exec("PRAGMA incremental_vacuum");
675
- db.exec("PRAGMA wal_checkpoint(TRUNCATE)");
676
- } catch {
677
- /* freed pages stay in the file, to be reused by later rows */
678
- }
679
- }
680
- return { deleted, oldestKeptMs: oldestKeptMs() };
681
- },
682
- markWasted(id: string): void {
683
- wasteStmt.run(id);
684
- },
685
- latestForSession(ompSessionId: string): LedgerEntry | null {
686
- if (ompSessionId === "") return null;
687
- const row = sessionStmt.get(ompSessionId, 1) as LedgerRow | null;
688
- return row === null ? null : toEntry(row);
689
- },
690
- entriesForSession(ompSessionId: string, limit: number): LedgerEntry[] {
691
- if (ompSessionId === "") return [];
692
- return (sessionStmt.all(ompSessionId, Math.max(1, limit)) as LedgerRow[]).map(toEntry);
693
- },
694
- };
695
- }