modelrig 0.1.0 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +158 -0
- package/README.md +53 -11
- package/data/PROVENANCE.md +19 -0
- package/data/pricing-manual.json +22 -0
- package/data/pricing-map.json +46362 -0
- package/data/pricing-overlay.json +419 -0
- package/dist/adapters/anthropic.d.ts +50 -6
- package/dist/adapters/anthropic.d.ts.map +1 -1
- package/dist/adapters/anthropic.js +123 -29
- package/dist/adapters/build-adapters.d.ts +11 -0
- package/dist/adapters/build-adapters.d.ts.map +1 -0
- package/dist/adapters/build-adapters.js +45 -0
- package/dist/adapters/deepseek.d.ts +6 -0
- package/dist/adapters/deepseek.d.ts.map +1 -1
- package/dist/adapters/deepseek.js +54 -1
- package/dist/adapters/gemini.d.ts +10 -0
- package/dist/adapters/gemini.d.ts.map +1 -1
- package/dist/adapters/gemini.js +78 -7
- package/dist/adapters/grok.d.ts +9 -0
- package/dist/adapters/grok.d.ts.map +1 -1
- package/dist/adapters/grok.js +67 -3
- package/dist/adapters/openai-compat.d.ts +6 -0
- package/dist/adapters/openai-compat.d.ts.map +1 -1
- package/dist/adapters/openai-compat.js +58 -1
- package/dist/adapters/openai-stream.d.ts +58 -0
- package/dist/adapters/openai-stream.d.ts.map +1 -0
- package/dist/adapters/openai-stream.js +120 -0
- package/dist/adapters/openai.d.ts +6 -0
- package/dist/adapters/openai.d.ts.map +1 -1
- package/dist/adapters/openai.js +60 -1
- package/dist/adapters/sampling.d.ts +28 -0
- package/dist/adapters/sampling.d.ts.map +1 -0
- package/dist/adapters/sampling.js +34 -0
- package/dist/adapters/tool-mapping.d.ts +92 -0
- package/dist/adapters/tool-mapping.d.ts.map +1 -0
- package/dist/adapters/tool-mapping.js +143 -0
- package/dist/artifacts/artifact.d.ts +137 -0
- package/dist/artifacts/artifact.d.ts.map +1 -0
- package/dist/artifacts/artifact.js +377 -0
- package/dist/artifacts/content-uploader.d.ts +87 -0
- package/dist/artifacts/content-uploader.d.ts.map +1 -0
- package/dist/artifacts/content-uploader.js +124 -0
- package/dist/artifacts/index.d.ts +63 -0
- package/dist/artifacts/index.d.ts.map +1 -0
- package/dist/artifacts/index.js +80 -0
- package/dist/artifacts/run-context.d.ts +129 -0
- package/dist/artifacts/run-context.d.ts.map +1 -0
- package/dist/artifacts/run-context.js +208 -0
- package/dist/artifacts/scrub.d.ts +60 -0
- package/dist/artifacts/scrub.d.ts.map +1 -0
- package/dist/artifacts/scrub.js +139 -0
- package/dist/artifacts/step.d.ts +26 -0
- package/dist/artifacts/step.d.ts.map +1 -0
- package/dist/artifacts/step.js +37 -0
- package/dist/artifacts/taint.d.ts +63 -0
- package/dist/artifacts/taint.d.ts.map +1 -0
- package/dist/artifacts/taint.js +76 -0
- package/dist/bakeoff/cache-sim.d.ts +99 -0
- package/dist/bakeoff/cache-sim.d.ts.map +1 -0
- package/dist/bakeoff/cache-sim.js +126 -0
- package/dist/bakeoff/eval-cases.d.ts +96 -0
- package/dist/bakeoff/eval-cases.d.ts.map +1 -0
- package/dist/bakeoff/eval-cases.js +105 -0
- package/dist/bakeoff/report.d.ts +41 -0
- package/dist/bakeoff/report.d.ts.map +1 -0
- package/dist/bakeoff/report.js +142 -0
- package/dist/bakeoff/samples.d.ts +15 -0
- package/dist/bakeoff/samples.d.ts.map +1 -1
- package/dist/bakeoff/samples.js +20 -0
- package/dist/cli.d.ts +1 -3
- package/dist/cli.d.ts.map +1 -1
- package/dist/cli.js +134 -116
- package/dist/config.d.ts +100 -0
- package/dist/config.d.ts.map +1 -1
- package/dist/config.js +104 -0
- package/dist/deprecations.d.ts +32 -0
- package/dist/deprecations.d.ts.map +1 -0
- package/dist/deprecations.js +59 -0
- package/dist/gateway/http-protocol.d.ts +175 -0
- package/dist/gateway/http-protocol.d.ts.map +1 -0
- package/dist/gateway/http-protocol.js +105 -0
- package/dist/index.d.ts +106 -19
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +266 -138
- package/dist/init/finder.d.ts +15 -0
- package/dist/init/finder.d.ts.map +1 -1
- package/dist/init/finder.js +46 -0
- package/dist/init/init-cli.d.ts +10 -0
- package/dist/init/init-cli.d.ts.map +1 -0
- package/dist/init/init-cli.js +86 -0
- package/dist/init/observe-cli.d.ts +19 -0
- package/dist/init/observe-cli.d.ts.map +1 -0
- package/dist/init/observe-cli.js +73 -0
- package/dist/init/observe.d.ts +81 -0
- package/dist/init/observe.d.ts.map +1 -0
- package/dist/init/observe.js +245 -0
- package/dist/registry/build.d.ts +94 -1
- package/dist/registry/build.d.ts.map +1 -1
- package/dist/registry/build.js +125 -4
- package/dist/registry/cache-policy.d.ts +94 -0
- package/dist/registry/cache-policy.d.ts.map +1 -0
- package/dist/registry/cache-policy.js +238 -0
- package/dist/registry/call-notes.d.ts.map +1 -1
- package/dist/registry/call-notes.js +40 -2
- package/dist/registry/effective-rate.d.ts +151 -0
- package/dist/registry/effective-rate.d.ts.map +1 -0
- package/dist/registry/effective-rate.js +156 -0
- package/dist/registry/facts.d.ts.map +1 -1
- package/dist/registry/facts.js +16 -0
- package/dist/registry/parity.d.ts +26 -0
- package/dist/registry/parity.d.ts.map +1 -1
- package/dist/registry/parity.js +43 -2
- package/dist/registry/pricing-shape.d.ts +113 -0
- package/dist/registry/pricing-shape.d.ts.map +1 -0
- package/dist/registry/pricing-shape.js +123 -0
- package/dist/registry/pricing.d.ts +25 -1
- package/dist/registry/pricing.d.ts.map +1 -1
- package/dist/registry/pricing.js +64 -6
- package/dist/rigindex/bootstrap.d.ts +36 -0
- package/dist/rigindex/bootstrap.d.ts.map +1 -0
- package/dist/rigindex/bootstrap.js +97 -0
- package/dist/rigindex/fit.d.ts +24 -0
- package/dist/rigindex/fit.d.ts.map +1 -0
- package/dist/rigindex/fit.js +76 -0
- package/dist/rigindex/gates.d.ts +52 -0
- package/dist/rigindex/gates.d.ts.map +1 -0
- package/dist/rigindex/gates.js +109 -0
- package/dist/rigindex/graph.d.ts +31 -0
- package/dist/rigindex/graph.d.ts.map +1 -0
- package/dist/rigindex/graph.js +120 -0
- package/dist/rigindex/index.d.ts +23 -0
- package/dist/rigindex/index.d.ts.map +1 -0
- package/dist/rigindex/index.js +112 -0
- package/dist/rigindex/types.d.ts +76 -0
- package/dist/rigindex/types.d.ts.map +1 -0
- package/dist/rigindex/types.js +8 -0
- package/dist/routes/cache-lint.d.ts +41 -0
- package/dist/routes/cache-lint.d.ts.map +1 -0
- package/dist/routes/cache-lint.js +81 -0
- package/dist/routes/loader.d.ts +48 -0
- package/dist/routes/loader.d.ts.map +1 -1
- package/dist/routes/loader.js +167 -0
- package/dist/routes/resolved-bundle.d.ts +79 -0
- package/dist/routes/resolved-bundle.d.ts.map +1 -0
- package/dist/routes/resolved-bundle.js +148 -0
- package/dist/routes/sampling-config.d.ts +32 -0
- package/dist/routes/sampling-config.d.ts.map +1 -0
- package/dist/routes/sampling-config.js +91 -0
- package/dist/run/ajv.d.ts +22 -0
- package/dist/run/ajv.d.ts.map +1 -0
- package/dist/run/ajv.js +32 -0
- package/dist/run/cache-handle.d.ts +58 -0
- package/dist/run/cache-handle.d.ts.map +1 -0
- package/dist/run/cache-handle.js +54 -0
- package/dist/run/cache-stamp.d.ts +65 -0
- package/dist/run/cache-stamp.d.ts.map +1 -0
- package/dist/run/cache-stamp.js +0 -0
- package/dist/run/capture-write.d.ts +33 -0
- package/dist/run/capture-write.d.ts.map +1 -0
- package/dist/run/capture-write.js +55 -0
- package/dist/run/cost.d.ts.map +1 -1
- package/dist/run/cost.js +1 -1
- package/dist/run/dispatch.d.ts +99 -0
- package/dist/run/dispatch.d.ts.map +1 -0
- package/dist/run/dispatch.js +191 -0
- package/dist/run/raw.d.ts +130 -0
- package/dist/run/raw.d.ts.map +1 -0
- package/dist/run/raw.js +363 -0
- package/dist/run/record-attempt.d.ts +40 -0
- package/dist/run/record-attempt.d.ts.map +1 -0
- package/dist/run/record-attempt.js +99 -0
- package/dist/run/resolve.d.ts +75 -2
- package/dist/run/resolve.d.ts.map +1 -1
- package/dist/run/resolve.js +126 -4
- package/dist/run/run.d.ts +14 -0
- package/dist/run/run.d.ts.map +1 -1
- package/dist/run/run.js +0 -0
- package/dist/run/stream.d.ts +49 -0
- package/dist/run/stream.d.ts.map +1 -0
- package/dist/run/stream.js +87 -0
- package/dist/run/validate.d.ts.map +1 -1
- package/dist/run/validate.js +6 -7
- package/dist/savings/cache-ledger.d.ts +108 -0
- package/dist/savings/cache-ledger.d.ts.map +1 -0
- package/dist/savings/cache-ledger.js +174 -0
- package/dist/savings/cache-reconcile.d.ts +64 -0
- package/dist/savings/cache-reconcile.d.ts.map +1 -0
- package/dist/savings/cache-reconcile.js +59 -0
- package/dist/savings/cache-stats.d.ts +63 -0
- package/dist/savings/cache-stats.d.ts.map +1 -0
- package/dist/savings/cache-stats.js +108 -0
- package/dist/savings/emit.d.ts +13 -0
- package/dist/savings/emit.d.ts.map +1 -1
- package/dist/savings/emit.js +25 -0
- package/dist/savings/swap-execute.d.ts +14 -0
- package/dist/savings/swap-execute.d.ts.map +1 -1
- package/dist/savings/swap-execute.js +44 -1
- package/dist/savings/swaps-client.d.ts +14 -0
- package/dist/savings/swaps-client.d.ts.map +1 -1
- package/dist/savings/swaps-client.js +16 -2
- package/dist/settings/api-fetcher.d.ts +39 -0
- package/dist/settings/api-fetcher.d.ts.map +1 -0
- package/dist/settings/api-fetcher.js +92 -0
- package/dist/settings/index.d.ts +10 -0
- package/dist/settings/index.d.ts.map +1 -0
- package/dist/settings/index.js +15 -0
- package/dist/settings/task-settings.d.ts +91 -0
- package/dist/settings/task-settings.d.ts.map +1 -0
- package/dist/settings/task-settings.js +116 -0
- package/dist/status.d.ts +77 -0
- package/dist/status.d.ts.map +1 -0
- package/dist/status.js +113 -0
- package/dist/telemetry/artifact-store.d.ts +48 -0
- package/dist/telemetry/artifact-store.d.ts.map +1 -0
- package/dist/telemetry/artifact-store.js +189 -0
- package/dist/telemetry/cache-miss-streak.d.ts +79 -0
- package/dist/telemetry/cache-miss-streak.d.ts.map +1 -0
- package/dist/telemetry/cache-miss-streak.js +99 -0
- package/dist/telemetry/capture.d.ts +3 -0
- package/dist/telemetry/capture.d.ts.map +1 -1
- package/dist/telemetry/capture.js +15 -8
- package/dist/telemetry/exporter-setup.d.ts +28 -0
- package/dist/telemetry/exporter-setup.d.ts.map +1 -0
- package/dist/telemetry/exporter-setup.js +34 -0
- package/dist/telemetry/exporter.d.ts +75 -0
- package/dist/telemetry/exporter.d.ts.map +1 -1
- package/dist/telemetry/exporter.js +324 -47
- package/dist/telemetry/health.d.ts +76 -0
- package/dist/telemetry/health.d.ts.map +1 -0
- package/dist/telemetry/health.js +92 -0
- package/dist/telemetry/ingest-protocol.d.ts +152 -3
- package/dist/telemetry/ingest-protocol.d.ts.map +1 -1
- package/dist/telemetry/ingest-protocol.js +204 -4
- package/dist/telemetry/load-error-report.d.ts +24 -0
- package/dist/telemetry/load-error-report.d.ts.map +1 -0
- package/dist/telemetry/load-error-report.js +50 -0
- package/dist/telemetry/otel-map.d.ts +121 -0
- package/dist/telemetry/otel-map.d.ts.map +1 -0
- package/dist/telemetry/otel-map.js +224 -0
- package/dist/telemetry/otel-normalize.d.ts +104 -0
- package/dist/telemetry/otel-normalize.d.ts.map +1 -0
- package/dist/telemetry/otel-normalize.js +530 -0
- package/dist/telemetry/otlp-emitter.d.ts +52 -0
- package/dist/telemetry/otlp-emitter.d.ts.map +1 -0
- package/dist/telemetry/otlp-emitter.js +144 -0
- package/dist/telemetry/otlp-proto.d.ts +29 -0
- package/dist/telemetry/otlp-proto.d.ts.map +1 -0
- package/dist/telemetry/otlp-proto.js +266 -0
- package/dist/telemetry/sinks.d.ts +37 -1
- package/dist/telemetry/sinks.d.ts.map +1 -1
- package/dist/telemetry/sinks.js +121 -30
- package/dist/telemetry/store.d.ts +28 -1
- package/dist/telemetry/store.d.ts.map +1 -1
- package/dist/telemetry/store.js +218 -3
- package/dist/tenancy/api-key-format.d.ts +15 -5
- package/dist/tenancy/api-key-format.d.ts.map +1 -1
- package/dist/tenancy/api-key-format.js +24 -5
- package/dist/types/config-teaching.d.ts +62 -0
- package/dist/types/config-teaching.d.ts.map +1 -0
- package/dist/types/config-teaching.js +183 -0
- package/dist/types/provider.d.ts +147 -1
- package/dist/types/provider.d.ts.map +1 -1
- package/dist/types/provider.js +21 -1
- package/dist/types/route.d.ts +79 -1
- package/dist/types/route.d.ts.map +1 -1
- package/dist/types/telemetry.d.ts +117 -0
- package/dist/types/telemetry.d.ts.map +1 -1
- package/dist/validate-cli.d.ts +12 -0
- package/dist/validate-cli.d.ts.map +1 -0
- package/dist/validate-cli.js +57 -0
- package/dist/verify/capture.d.ts +66 -0
- package/dist/verify/capture.d.ts.map +1 -0
- package/dist/verify/capture.js +180 -0
- package/dist/verify/cli-run.d.ts +23 -0
- package/dist/verify/cli-run.d.ts.map +1 -0
- package/dist/verify/cli-run.js +57 -0
- package/dist/verify/diff.d.ts +48 -0
- package/dist/verify/diff.d.ts.map +1 -0
- package/dist/verify/diff.js +157 -0
- package/dist/verify/index.d.ts +17 -0
- package/dist/verify/index.d.ts.map +1 -0
- package/dist/verify/index.js +24 -0
- package/dist/verify/report.d.ts +14 -0
- package/dist/verify/report.d.ts.map +1 -0
- package/dist/verify/report.js +88 -0
- package/dist/verify/types.d.ts +101 -0
- package/dist/verify/types.d.ts.map +1 -0
- package/dist/verify/types.js +17 -0
- package/dist/verify/verify.d.ts +29 -0
- package/dist/verify/verify.d.ts.map +1 -0
- package/dist/verify/verify.js +148 -0
- package/dist/watch/evaluate.d.ts +162 -3
- package/dist/watch/evaluate.d.ts.map +1 -1
- package/dist/watch/evaluate.js +61 -0
- package/dist/watch/monitor.d.ts +116 -0
- package/dist/watch/monitor.d.ts.map +1 -0
- package/dist/watch/monitor.js +199 -0
- package/dist/watch/proposals.d.ts +20 -4
- package/dist/watch/proposals.d.ts.map +1 -1
- package/dist/watch/proposals.js +31 -3
- package/dist/watch/run.d.ts +33 -0
- package/dist/watch/run.d.ts.map +1 -1
- package/dist/watch/run.js +39 -0
- package/package.json +7 -1
- package/registry/LICENSE +20 -0
- package/registry/leaderboard.json +3639 -0
- package/registry/registry.json +7263 -0
- package/dist/telemetry/supabase-exporter.d.ts +0 -64
- package/dist/telemetry/supabase-exporter.d.ts.map +0 -1
- package/dist/telemetry/supabase-exporter.js +0 -211
|
@@ -0,0 +1,174 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
/**
|
|
3
|
+
* Cache Ledger — DERIVED in v1 (cache-aware-routing spec §5.2, [Briefing §8]).
|
|
4
|
+
*
|
|
5
|
+
* The Briefing's Cache Ledger is a first-class fast-state store; v1 does NOT
|
|
6
|
+
* build one. It DERIVES warm-set state from telemetry rows at decision time —
|
|
7
|
+
* ZERO new tables (spec schema budget). The `CacheLedgerEntry` TYPE ships now so
|
|
8
|
+
* that when the conversational-gateway phase needs a dedicated store (per-message
|
|
9
|
+
* traffic makes per-call derivation too hot), it is a swap of BACKING, not of
|
|
10
|
+
* SHAPE. Nothing here persists anything.
|
|
11
|
+
*
|
|
12
|
+
* This answers the Briefing's first-milestone question — "which candidate routes
|
|
13
|
+
* appear warm, by how much, at what confidence" [§41] — with a 4-value status:
|
|
14
|
+
*
|
|
15
|
+
* observed_hot — the most recent use within the TTL window actually READ
|
|
16
|
+
* cache (tokensCached>0): the cache is demonstrably warm.
|
|
17
|
+
* probably_hot — last use within the TTL window but no read observed on it:
|
|
18
|
+
* warmth is INFERRED from recency, not seen.
|
|
19
|
+
* probably_expired— last use past the TTL but within a bounded grace: likely
|
|
20
|
+
* cold, yet recent enough that a switch-back could still hit
|
|
21
|
+
* (LiteLLM: 99.3% of switch-backs land within TTL, [§6]).
|
|
22
|
+
* cold — no recent use, well past any grace, or the provider does
|
|
23
|
+
* not cache on our path (fail-closed).
|
|
24
|
+
*
|
|
25
|
+
* ── CONSERVATISM (spec §10 cache-state overconfidence) ────────────────────
|
|
26
|
+
* The TTL comes from the POLICY REGISTRY only. When a provider publishes no TTL
|
|
27
|
+
* (deepseek "hours-to-days", gemini/fireworks best-effort → `default_ttl_seconds:
|
|
28
|
+
* null`), we do NOT invent a warm window: the entry can reach `observed_hot`
|
|
29
|
+
* only via a demonstrated read within a short fallback recency, and otherwise is
|
|
30
|
+
* `cold`. Under-claiming warmth never over-predicts a hit.
|
|
31
|
+
*/
|
|
32
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
33
|
+
exports.PROBABLY_EXPIRED_TTL_MULTIPLE = exports.NULL_TTL_OBSERVED_HOT_SECONDS = void 0;
|
|
34
|
+
exports.classifyWarmStatus = classifyWarmStatus;
|
|
35
|
+
exports.deriveCacheLedger = deriveCacheLedger;
|
|
36
|
+
exports.warmEntries = warmEntries;
|
|
37
|
+
const cache_policy_1 = require("../registry/cache-policy");
|
|
38
|
+
/**
|
|
39
|
+
* Fallback recency window (seconds) for the `observed_hot` decision when a
|
|
40
|
+
* provider publishes NO TTL. NOT a claimed provider TTL — a conservative display
|
|
41
|
+
* bound so an ancient demonstrated read is not still called "hot". 5 minutes is
|
|
42
|
+
* the shortest real cache TTL in the registry (Anthropic 5m), so it never
|
|
43
|
+
* over-claims relative to any documented policy.
|
|
44
|
+
*/
|
|
45
|
+
exports.NULL_TTL_OBSERVED_HOT_SECONDS = 300;
|
|
46
|
+
/**
|
|
47
|
+
* Grace multiplier past the TTL for `probably_expired` before an entry is called
|
|
48
|
+
* `cold`. Beyond `ttl × this`, a switch-back is very unlikely to still hit.
|
|
49
|
+
*/
|
|
50
|
+
exports.PROBABLY_EXPIRED_TTL_MULTIPLE = 3;
|
|
51
|
+
// Grouping key — none of the components contains a newline.
|
|
52
|
+
const KEY_SEP = "\n";
|
|
53
|
+
function isProductionRow(row) {
|
|
54
|
+
try {
|
|
55
|
+
return JSON.parse(row.tagsJson).replay === undefined;
|
|
56
|
+
}
|
|
57
|
+
catch {
|
|
58
|
+
return true;
|
|
59
|
+
}
|
|
60
|
+
}
|
|
61
|
+
/**
|
|
62
|
+
* Classify a group's warm status from its policy TTL, age, and whether the most
|
|
63
|
+
* recent use read cache. Pure and total; the boundary rules are the architect
|
|
64
|
+
* math-review surface, so they are spelled out rather than folded together.
|
|
65
|
+
*/
|
|
66
|
+
function classifyWarmStatus(policy, ageSeconds, lastUseReadCache) {
|
|
67
|
+
if (!policy.cache_supported)
|
|
68
|
+
return "cold";
|
|
69
|
+
const ttl = policy.default_ttl_seconds;
|
|
70
|
+
if (ttl === null) {
|
|
71
|
+
// No documented TTL: only a demonstrated, recent read earns "hot". We never
|
|
72
|
+
// INFER warmth here (no probably_hot) — that would be the overconfidence the
|
|
73
|
+
// spec §10 warns against.
|
|
74
|
+
return lastUseReadCache && ageSeconds <= exports.NULL_TTL_OBSERVED_HOT_SECONDS ? "observed_hot" : "cold";
|
|
75
|
+
}
|
|
76
|
+
if (ageSeconds <= ttl) {
|
|
77
|
+
return lastUseReadCache ? "observed_hot" : "probably_hot";
|
|
78
|
+
}
|
|
79
|
+
if (ageSeconds <= ttl * exports.PROBABLY_EXPIRED_TTL_MULTIPLE) {
|
|
80
|
+
return "probably_expired";
|
|
81
|
+
}
|
|
82
|
+
return "cold";
|
|
83
|
+
}
|
|
84
|
+
/**
|
|
85
|
+
* Derive the warm-set ledger from telemetry rows (spec §5.2). Groups production
|
|
86
|
+
* rows by (route, provider, model, prefix_fingerprint, cache_key), takes each
|
|
87
|
+
* group's most-recent use to classify warmth via the policy TTL, and estimates
|
|
88
|
+
* the warm prefix size. Replay rows are excluded (odometer parity). Output is
|
|
89
|
+
* sorted for deterministic rendering. Nothing is stored.
|
|
90
|
+
*/
|
|
91
|
+
function deriveCacheLedger(rows, opts) {
|
|
92
|
+
const policyFor = opts.policyFor ?? ((p) => (0, cache_policy_1.getCachePolicy)(p));
|
|
93
|
+
const windowStartMs = opts.windowStartMs ?? opts.now - 86400000;
|
|
94
|
+
const byKey = new Map();
|
|
95
|
+
for (const row of rows) {
|
|
96
|
+
if (!isProductionRow(row))
|
|
97
|
+
continue;
|
|
98
|
+
const t = Date.parse(row.ts);
|
|
99
|
+
if (Number.isNaN(t) || t < windowStartMs || t > opts.now)
|
|
100
|
+
continue;
|
|
101
|
+
const fp = row.prefixFingerprint ?? null;
|
|
102
|
+
const ck = row.cacheKey ?? null;
|
|
103
|
+
const key = [row.route, row.provider, row.model, fp ?? "", ck ?? ""].join(KEY_SEP);
|
|
104
|
+
let acc = byKey.get(key);
|
|
105
|
+
if (acc === undefined) {
|
|
106
|
+
acc = {
|
|
107
|
+
route: row.route,
|
|
108
|
+
provider: row.provider,
|
|
109
|
+
model: row.model,
|
|
110
|
+
prefixFingerprint: fp,
|
|
111
|
+
cacheKey: ck,
|
|
112
|
+
lastSeenAtMs: -1,
|
|
113
|
+
lastSeenAt: row.ts,
|
|
114
|
+
lastUseReadCache: false,
|
|
115
|
+
maxReadTokens: 0,
|
|
116
|
+
tokensInSum: 0,
|
|
117
|
+
n: 0,
|
|
118
|
+
};
|
|
119
|
+
byKey.set(key, acc);
|
|
120
|
+
}
|
|
121
|
+
acc.n += 1;
|
|
122
|
+
acc.tokensInSum += Math.max(row.tokensIn, 0);
|
|
123
|
+
acc.maxReadTokens = Math.max(acc.maxReadTokens, Math.max(row.tokensCached, 0));
|
|
124
|
+
if (t > acc.lastSeenAtMs) {
|
|
125
|
+
acc.lastSeenAtMs = t;
|
|
126
|
+
acc.lastSeenAt = row.ts;
|
|
127
|
+
acc.lastUseReadCache = row.tokensCached > 0;
|
|
128
|
+
}
|
|
129
|
+
else if (t === acc.lastSeenAtMs && row.tokensCached > 0) {
|
|
130
|
+
// Same-ms tie: a DEMONSTRATED read at the newest timestamp wins,
|
|
131
|
+
// regardless of row order in the (unsorted) feed. Without this the
|
|
132
|
+
// observed_hot/probably_hot status would depend on array order — a
|
|
133
|
+
// reordered feed of the same rows could flip the status. Preferring the
|
|
134
|
+
// read is the truer signal (the cache WAS warm at that instant).
|
|
135
|
+
acc.lastUseReadCache = true;
|
|
136
|
+
}
|
|
137
|
+
}
|
|
138
|
+
const out = [];
|
|
139
|
+
for (const acc of byKey.values()) {
|
|
140
|
+
const ageSeconds = Math.max(0, (opts.now - acc.lastSeenAtMs) / 1000);
|
|
141
|
+
const policy = policyFor(acc.provider);
|
|
142
|
+
const status = classifyWarmStatus(policy, ageSeconds, acc.lastUseReadCache);
|
|
143
|
+
const meanIn = acc.n > 0 ? acc.tokensInSum / acc.n : 0;
|
|
144
|
+
const warmTokens = acc.maxReadTokens > 0 ? acc.maxReadTokens : meanIn;
|
|
145
|
+
out.push({
|
|
146
|
+
route: acc.route,
|
|
147
|
+
provider: acc.provider,
|
|
148
|
+
model: acc.model,
|
|
149
|
+
prefixFingerprint: acc.prefixFingerprint,
|
|
150
|
+
cacheKey: acc.cacheKey,
|
|
151
|
+
status,
|
|
152
|
+
lastSeenAt: acc.lastSeenAt,
|
|
153
|
+
ageSeconds,
|
|
154
|
+
warmTokens,
|
|
155
|
+
observedReadTokens: acc.maxReadTokens,
|
|
156
|
+
ttlSeconds: policy.cache_supported ? policy.default_ttl_seconds : null,
|
|
157
|
+
workflowId: acc.route,
|
|
158
|
+
});
|
|
159
|
+
}
|
|
160
|
+
return out.sort((a, b) => a.route !== b.route
|
|
161
|
+
? a.route.localeCompare(b.route)
|
|
162
|
+
: a.provider !== b.provider
|
|
163
|
+
? a.provider.localeCompare(b.provider)
|
|
164
|
+
: a.model !== b.model
|
|
165
|
+
? a.model.localeCompare(b.model)
|
|
166
|
+
: (a.prefixFingerprint ?? "").localeCompare(b.prefixFingerprint ?? ""));
|
|
167
|
+
}
|
|
168
|
+
/** The warm candidates for a route — `observed_hot` or `probably_hot` — as the
|
|
169
|
+
* warmth input for shadow-mode ordering (WS-B step 8) and the console warm-set
|
|
170
|
+
* chips (WS-D). `probably_expired`/`cold` are excluded: warmth we would ACT on
|
|
171
|
+
* must be current, not merely recent. */
|
|
172
|
+
function warmEntries(ledger) {
|
|
173
|
+
return ledger.filter((e) => e.status === "observed_hot" || e.status === "probably_hot");
|
|
174
|
+
}
|
|
@@ -0,0 +1,64 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Cache prediction reconciliation — cache-aware-routing spec §5.2 (the
|
|
3
|
+
* [Briefing §8] feedback loop, v1-deterministic). PURE: no store/clock/network.
|
|
4
|
+
*
|
|
5
|
+
* "Predict → route → observe → reconcile" (spec §1.2): the derived ledger + the
|
|
6
|
+
* simulator PREDICT cache reads; provider-reported usage is GROUND TRUTH. When
|
|
7
|
+
* the two diverge beyond tolerance, the estimate is systematically off — a
|
|
8
|
+
* provider rule changed, a fingerprint is unstable, an endpoint lost affinity —
|
|
9
|
+
* and a `cache_prediction_drift` health event fires so the discrepancy is
|
|
10
|
+
* surfaced rather than silently trusted.
|
|
11
|
+
*
|
|
12
|
+
* G-4 PATTERN (telemetry/health.ts): this is a pure aggregation that RETURNS
|
|
13
|
+
* event objects — it never persists. A min-sample gate protects against calling
|
|
14
|
+
* two noisy calls a "drift". WS-D renders the events (console badge); no DDL and
|
|
15
|
+
* no new table are involved (spec schema budget).
|
|
16
|
+
*/
|
|
17
|
+
/** A predicted-vs-observed pair for one (route, candidate) over the window. */
|
|
18
|
+
export interface CachePredictionSample {
|
|
19
|
+
readonly route: string;
|
|
20
|
+
readonly provider: string;
|
|
21
|
+
readonly model: string;
|
|
22
|
+
/** Cache-READ tokens the ledger/simulator predicted over the window. */
|
|
23
|
+
readonly predictedReadTokens: number;
|
|
24
|
+
/** Cache-READ tokens the provider actually reported (Σ tokensCached). */
|
|
25
|
+
readonly observedReadTokens: number;
|
|
26
|
+
/** Total input tokens over the window — the base the drift is normalized by,
|
|
27
|
+
* and the min-sample gate. */
|
|
28
|
+
readonly windowInputTokens: number;
|
|
29
|
+
/** Production rows the pair aggregates (for the gate + the surfaced evidence). */
|
|
30
|
+
readonly sampleN: number;
|
|
31
|
+
}
|
|
32
|
+
/** A fired drift event (G-4 shape). Never persisted; rendered by WS-D. */
|
|
33
|
+
export interface CachePredictionDrift {
|
|
34
|
+
readonly route: string;
|
|
35
|
+
readonly provider: string;
|
|
36
|
+
readonly model: string;
|
|
37
|
+
readonly predictedReadTokens: number;
|
|
38
|
+
readonly observedReadTokens: number;
|
|
39
|
+
/** |predicted − observed| / windowInputTokens — the normalized gap. */
|
|
40
|
+
readonly driftFraction: number;
|
|
41
|
+
/** Signed hit-rate error (predicted_h − observed_h): positive ⇒ we OVER-predicted
|
|
42
|
+
* warmth (the more dangerous direction — over-optimistic routing). */
|
|
43
|
+
readonly signedHitRateError: number;
|
|
44
|
+
readonly sampleN: number;
|
|
45
|
+
}
|
|
46
|
+
export interface ReconcileOptions {
|
|
47
|
+
/** Drift tolerance as a FRACTION of window input tokens (default 0.15 — the
|
|
48
|
+
* estimate must be within 15% of ground truth before it is "off"). */
|
|
49
|
+
readonly toleranceFraction?: number;
|
|
50
|
+
/** Minimum production rows before a pair is judged (default 20 — matches the
|
|
51
|
+
* observed-health min-n; below it the gap is too noisy to trust). */
|
|
52
|
+
readonly minN?: number;
|
|
53
|
+
}
|
|
54
|
+
export declare const DEFAULT_DRIFT_TOLERANCE_FRACTION = 0.15;
|
|
55
|
+
export declare const DEFAULT_DRIFT_MIN_N = 20;
|
|
56
|
+
/**
|
|
57
|
+
* Compare predicted vs observed cache reads per (route, candidate) and emit a
|
|
58
|
+
* drift event for each pair whose normalized gap exceeds tolerance AND clears
|
|
59
|
+
* the min-sample gate. Pure and total; output sorted for deterministic
|
|
60
|
+
* rendering. An empty result means every prediction tracked ground truth (or
|
|
61
|
+
* nothing had enough data) — the healthy case.
|
|
62
|
+
*/
|
|
63
|
+
export declare function reconcileCacheDrift(samples: readonly CachePredictionSample[], opts?: ReconcileOptions): CachePredictionDrift[];
|
|
64
|
+
//# sourceMappingURL=cache-reconcile.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"cache-reconcile.d.ts","sourceRoot":"","sources":["../../src/savings/cache-reconcile.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;GAeG;AAEH,+EAA+E;AAC/E,MAAM,WAAW,qBAAqB;IACpC,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;IAC1B,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,wEAAwE;IACxE,QAAQ,CAAC,mBAAmB,EAAE,MAAM,CAAC;IACrC,yEAAyE;IACzE,QAAQ,CAAC,kBAAkB,EAAE,MAAM,CAAC;IACpC;kCAC8B;IAC9B,QAAQ,CAAC,iBAAiB,EAAE,MAAM,CAAC;IACnC,kFAAkF;IAClF,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;CAC1B;AAED,0EAA0E;AAC1E,MAAM,WAAW,oBAAoB;IACnC,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;IAC1B,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,mBAAmB,EAAE,MAAM,CAAC;IACrC,QAAQ,CAAC,kBAAkB,EAAE,MAAM,CAAC;IACpC,uEAAuE;IACvE,QAAQ,CAAC,aAAa,EAAE,MAAM,CAAC;IAC/B;0EACsE;IACtE,QAAQ,CAAC,kBAAkB,EAAE,MAAM,CAAC;IACpC,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;CAC1B;AAED,MAAM,WAAW,gBAAgB;IAC/B;0EACsE;IACtE,QAAQ,CAAC,iBAAiB,CAAC,EAAE,MAAM,CAAC;IACpC;yEACqE;IACrE,QAAQ,CAAC,IAAI,CAAC,EAAE,MAAM,CAAC;CACxB;AAED,eAAO,MAAM,gCAAgC,OAAO,CAAC;AACrD,eAAO,MAAM,mBAAmB,KAAK,CAAC;AAEtC;;;;;;GAMG;AACH,wBAAgB,mBAAmB,CACjC,OAAO,EAAE,SAAS,qBAAqB,EAAE,EACzC,IAAI,GAAE,gBAAqB,GAC1B,oBAAoB,EAAE,CA+BxB"}
|
|
@@ -0,0 +1,59 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
/**
|
|
3
|
+
* Cache prediction reconciliation — cache-aware-routing spec §5.2 (the
|
|
4
|
+
* [Briefing §8] feedback loop, v1-deterministic). PURE: no store/clock/network.
|
|
5
|
+
*
|
|
6
|
+
* "Predict → route → observe → reconcile" (spec §1.2): the derived ledger + the
|
|
7
|
+
* simulator PREDICT cache reads; provider-reported usage is GROUND TRUTH. When
|
|
8
|
+
* the two diverge beyond tolerance, the estimate is systematically off — a
|
|
9
|
+
* provider rule changed, a fingerprint is unstable, an endpoint lost affinity —
|
|
10
|
+
* and a `cache_prediction_drift` health event fires so the discrepancy is
|
|
11
|
+
* surfaced rather than silently trusted.
|
|
12
|
+
*
|
|
13
|
+
* G-4 PATTERN (telemetry/health.ts): this is a pure aggregation that RETURNS
|
|
14
|
+
* event objects — it never persists. A min-sample gate protects against calling
|
|
15
|
+
* two noisy calls a "drift". WS-D renders the events (console badge); no DDL and
|
|
16
|
+
* no new table are involved (spec schema budget).
|
|
17
|
+
*/
|
|
18
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
19
|
+
exports.DEFAULT_DRIFT_MIN_N = exports.DEFAULT_DRIFT_TOLERANCE_FRACTION = void 0;
|
|
20
|
+
exports.reconcileCacheDrift = reconcileCacheDrift;
|
|
21
|
+
exports.DEFAULT_DRIFT_TOLERANCE_FRACTION = 0.15;
|
|
22
|
+
exports.DEFAULT_DRIFT_MIN_N = 20;
|
|
23
|
+
/**
|
|
24
|
+
* Compare predicted vs observed cache reads per (route, candidate) and emit a
|
|
25
|
+
* drift event for each pair whose normalized gap exceeds tolerance AND clears
|
|
26
|
+
* the min-sample gate. Pure and total; output sorted for deterministic
|
|
27
|
+
* rendering. An empty result means every prediction tracked ground truth (or
|
|
28
|
+
* nothing had enough data) — the healthy case.
|
|
29
|
+
*/
|
|
30
|
+
function reconcileCacheDrift(samples, opts = {}) {
|
|
31
|
+
const tolerance = opts.toleranceFraction ?? exports.DEFAULT_DRIFT_TOLERANCE_FRACTION;
|
|
32
|
+
const minN = opts.minN ?? exports.DEFAULT_DRIFT_MIN_N;
|
|
33
|
+
const out = [];
|
|
34
|
+
for (const s of samples) {
|
|
35
|
+
if (s.sampleN < minN || s.windowInputTokens <= 0)
|
|
36
|
+
continue; // thin data → no judgment
|
|
37
|
+
const gap = Math.abs(s.predictedReadTokens - s.observedReadTokens);
|
|
38
|
+
const driftFraction = gap / s.windowInputTokens;
|
|
39
|
+
if (driftFraction <= tolerance)
|
|
40
|
+
continue;
|
|
41
|
+
const predictedH = s.predictedReadTokens / s.windowInputTokens;
|
|
42
|
+
const observedH = s.observedReadTokens / s.windowInputTokens;
|
|
43
|
+
out.push({
|
|
44
|
+
route: s.route,
|
|
45
|
+
provider: s.provider,
|
|
46
|
+
model: s.model,
|
|
47
|
+
predictedReadTokens: s.predictedReadTokens,
|
|
48
|
+
observedReadTokens: s.observedReadTokens,
|
|
49
|
+
driftFraction,
|
|
50
|
+
signedHitRateError: predictedH - observedH,
|
|
51
|
+
sampleN: s.sampleN,
|
|
52
|
+
});
|
|
53
|
+
}
|
|
54
|
+
return out.sort((a, b) => a.route !== b.route
|
|
55
|
+
? a.route.localeCompare(b.route)
|
|
56
|
+
: a.provider !== b.provider
|
|
57
|
+
? a.provider.localeCompare(b.provider)
|
|
58
|
+
: a.model.localeCompare(b.model));
|
|
59
|
+
}
|
|
@@ -0,0 +1,63 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Cache-hit-rate rollup — cache-aware-routing spec §5.1, WS-B. PURE: a function
|
|
3
|
+
* over telemetry rows, no store/clock/network (an injected `now` bounds the
|
|
4
|
+
* window). Feeds the effective-rate blend (§4.2), the derived ledger (§5.2), and
|
|
5
|
+
* the console (WS-D).
|
|
6
|
+
*
|
|
7
|
+
* ── THE ODOMETER RULE (replay exclusion) ──────────────────────────────────
|
|
8
|
+
* Trailing-window measurements count PRODUCTION serving only: a bake-off replay
|
|
9
|
+
* pass runs real (billed) calls tagged `replay`, and those must NEVER inflate a
|
|
10
|
+
* route's measured hit rate any more than they inflate measured savings. This is
|
|
11
|
+
* the SAME 5-line filter the savings odometer uses — see
|
|
12
|
+
* `savings/odometer.ts` (`JSON.parse(r.tagsJson).replay !== undefined`). It is
|
|
13
|
+
* duplicated (not imported) deliberately: the two live in different modules with
|
|
14
|
+
* different row shapes, and a shared 5-line helper would couple them for no real
|
|
15
|
+
* gain. If the replay tag convention ever changes, change BOTH.
|
|
16
|
+
*
|
|
17
|
+
* ── UNPRICED ROWS ARE KEPT ────────────────────────────────────────────────
|
|
18
|
+
* Unlike the odometer (which drops `pricingMissing` rows so a $0 cannot inflate
|
|
19
|
+
* dollar savings), the hit-rate rollup is a TOKEN measurement, not a dollar one:
|
|
20
|
+
* an unpriced row still carries honest token counts, so it counts toward h/w.
|
|
21
|
+
* Pricing enters later, in the effective-rate blend, where a missing rate
|
|
22
|
+
* fail-closes on its own.
|
|
23
|
+
*/
|
|
24
|
+
import type { TelemetryRow } from "../types/telemetry";
|
|
25
|
+
/** Trailing window for the rollup — 7 days by default (spec §5.1). */
|
|
26
|
+
export declare const DEFAULT_CACHE_WINDOW_DAYS = 7;
|
|
27
|
+
/** Observed cache activity for one (route, provider, model) over the window. */
|
|
28
|
+
export interface CacheStatsRollup {
|
|
29
|
+
readonly route: string;
|
|
30
|
+
readonly provider: string;
|
|
31
|
+
readonly model: string;
|
|
32
|
+
/** Production rows in the window (replay excluded). */
|
|
33
|
+
readonly sampleN: number;
|
|
34
|
+
/** Σ input tokens (the denominator of h and w). */
|
|
35
|
+
readonly tokensIn: number;
|
|
36
|
+
/** Σ cache-READ tokens. */
|
|
37
|
+
readonly tokensCached: number;
|
|
38
|
+
/** Σ cache-WRITE (creation) tokens. */
|
|
39
|
+
readonly tokensCacheWrite: number;
|
|
40
|
+
/** Observed hit rate `h = Σcached/Σin` (0 when `tokensIn` is 0). */
|
|
41
|
+
readonly h: number;
|
|
42
|
+
/** Observed write rate `w = Σwrite/Σin` (0 when `tokensIn` is 0). */
|
|
43
|
+
readonly w: number;
|
|
44
|
+
/** Most recent row ts in the window (ISO), or null when none. */
|
|
45
|
+
readonly lastSeenAt: string | null;
|
|
46
|
+
/** Distinct non-null prefix fingerprints seen — the cacheable prefixes this
|
|
47
|
+
* route×candidate exercised (WS-C fills them; empty in WS-A-only telemetry). */
|
|
48
|
+
readonly prefixFingerprints: readonly string[];
|
|
49
|
+
}
|
|
50
|
+
export interface CacheStatsOptions {
|
|
51
|
+
/** Reference "now" (epoch ms) — injected for deterministic windowing/tests. */
|
|
52
|
+
readonly now: number;
|
|
53
|
+
/** Window width in days (default 7). */
|
|
54
|
+
readonly windowDays?: number;
|
|
55
|
+
}
|
|
56
|
+
/**
|
|
57
|
+
* Roll telemetry rows up into per-(route, provider, model) cache stats over the
|
|
58
|
+
* trailing window. Rows outside `[now − windowDays, now]`, with an unparseable
|
|
59
|
+
* ts, or tagged `replay` are excluded. Output is sorted (route, provider, model)
|
|
60
|
+
* for deterministic downstream rendering and stable snapshots.
|
|
61
|
+
*/
|
|
62
|
+
export declare function rollupCacheStats(rows: readonly TelemetryRow[], opts: CacheStatsOptions): CacheStatsRollup[];
|
|
63
|
+
//# sourceMappingURL=cache-stats.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"cache-stats.d.ts","sourceRoot":"","sources":["../../src/savings/cache-stats.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;GAsBG;AAEH,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC;AAEvD,sEAAsE;AACtE,eAAO,MAAM,yBAAyB,IAAI,CAAC;AAE3C,gFAAgF;AAChF,MAAM,WAAW,gBAAgB;IAC/B,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;IAC1B,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,uDAAuD;IACvD,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;IACzB,mDAAmD;IACnD,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;IAC1B,2BAA2B;IAC3B,QAAQ,CAAC,YAAY,EAAE,MAAM,CAAC;IAC9B,uCAAuC;IACvC,QAAQ,CAAC,gBAAgB,EAAE,MAAM,CAAC;IAClC,oEAAoE;IACpE,QAAQ,CAAC,CAAC,EAAE,MAAM,CAAC;IACnB,qEAAqE;IACrE,QAAQ,CAAC,CAAC,EAAE,MAAM,CAAC;IACnB,iEAAiE;IACjE,QAAQ,CAAC,UAAU,EAAE,MAAM,GAAG,IAAI,CAAC;IACnC;oFACgF;IAChF,QAAQ,CAAC,kBAAkB,EAAE,SAAS,MAAM,EAAE,CAAC;CAChD;AAED,MAAM,WAAW,iBAAiB;IAChC,+EAA+E;IAC/E,QAAQ,CAAC,GAAG,EAAE,MAAM,CAAC;IACrB,wCAAwC;IACxC,QAAQ,CAAC,UAAU,CAAC,EAAE,MAAM,CAAC;CAC9B;AAiBD;;;;;GAKG;AACH,wBAAgB,gBAAgB,CAC9B,IAAI,EAAE,SAAS,YAAY,EAAE,EAC7B,IAAI,EAAE,iBAAiB,GACtB,gBAAgB,EAAE,CAwEpB"}
|
|
@@ -0,0 +1,108 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
/**
|
|
3
|
+
* Cache-hit-rate rollup — cache-aware-routing spec §5.1, WS-B. PURE: a function
|
|
4
|
+
* over telemetry rows, no store/clock/network (an injected `now` bounds the
|
|
5
|
+
* window). Feeds the effective-rate blend (§4.2), the derived ledger (§5.2), and
|
|
6
|
+
* the console (WS-D).
|
|
7
|
+
*
|
|
8
|
+
* ── THE ODOMETER RULE (replay exclusion) ──────────────────────────────────
|
|
9
|
+
* Trailing-window measurements count PRODUCTION serving only: a bake-off replay
|
|
10
|
+
* pass runs real (billed) calls tagged `replay`, and those must NEVER inflate a
|
|
11
|
+
* route's measured hit rate any more than they inflate measured savings. This is
|
|
12
|
+
* the SAME 5-line filter the savings odometer uses — see
|
|
13
|
+
* `savings/odometer.ts` (`JSON.parse(r.tagsJson).replay !== undefined`). It is
|
|
14
|
+
* duplicated (not imported) deliberately: the two live in different modules with
|
|
15
|
+
* different row shapes, and a shared 5-line helper would couple them for no real
|
|
16
|
+
* gain. If the replay tag convention ever changes, change BOTH.
|
|
17
|
+
*
|
|
18
|
+
* ── UNPRICED ROWS ARE KEPT ────────────────────────────────────────────────
|
|
19
|
+
* Unlike the odometer (which drops `pricingMissing` rows so a $0 cannot inflate
|
|
20
|
+
* dollar savings), the hit-rate rollup is a TOKEN measurement, not a dollar one:
|
|
21
|
+
* an unpriced row still carries honest token counts, so it counts toward h/w.
|
|
22
|
+
* Pricing enters later, in the effective-rate blend, where a missing rate
|
|
23
|
+
* fail-closes on its own.
|
|
24
|
+
*/
|
|
25
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
26
|
+
exports.DEFAULT_CACHE_WINDOW_DAYS = void 0;
|
|
27
|
+
exports.rollupCacheStats = rollupCacheStats;
|
|
28
|
+
/** Trailing window for the rollup — 7 days by default (spec §5.1). */
|
|
29
|
+
exports.DEFAULT_CACHE_WINDOW_DAYS = 7;
|
|
30
|
+
/** The replay-exclusion predicate — see the module header (odometer parity).
|
|
31
|
+
* A row is production unless its tags carry a `replay` key. Unparseable tags
|
|
32
|
+
* are treated as production (a malformed tag never hides real traffic). */
|
|
33
|
+
function isProductionRow(row) {
|
|
34
|
+
try {
|
|
35
|
+
return JSON.parse(row.tagsJson).replay === undefined;
|
|
36
|
+
}
|
|
37
|
+
catch {
|
|
38
|
+
return true;
|
|
39
|
+
}
|
|
40
|
+
}
|
|
41
|
+
// Grouping key: none of route / provider / model contains a newline, so a
|
|
42
|
+
// newline separator splits back unambiguously.
|
|
43
|
+
const KEY_SEP = "\n";
|
|
44
|
+
/**
|
|
45
|
+
* Roll telemetry rows up into per-(route, provider, model) cache stats over the
|
|
46
|
+
* trailing window. Rows outside `[now − windowDays, now]`, with an unparseable
|
|
47
|
+
* ts, or tagged `replay` are excluded. Output is sorted (route, provider, model)
|
|
48
|
+
* for deterministic downstream rendering and stable snapshots.
|
|
49
|
+
*/
|
|
50
|
+
function rollupCacheStats(rows, opts) {
|
|
51
|
+
const windowDays = opts.windowDays ?? exports.DEFAULT_CACHE_WINDOW_DAYS;
|
|
52
|
+
const windowStartMs = opts.now - windowDays * 86400000;
|
|
53
|
+
const byKey = new Map();
|
|
54
|
+
for (const row of rows) {
|
|
55
|
+
if (!isProductionRow(row))
|
|
56
|
+
continue;
|
|
57
|
+
const t = Date.parse(row.ts);
|
|
58
|
+
if (Number.isNaN(t) || t < windowStartMs || t > opts.now)
|
|
59
|
+
continue;
|
|
60
|
+
const key = `${row.route}${KEY_SEP}${row.provider}${KEY_SEP}${row.model}`;
|
|
61
|
+
let acc = byKey.get(key);
|
|
62
|
+
if (acc === undefined) {
|
|
63
|
+
acc = {
|
|
64
|
+
sampleN: 0,
|
|
65
|
+
tokensIn: 0,
|
|
66
|
+
tokensCached: 0,
|
|
67
|
+
tokensCacheWrite: 0,
|
|
68
|
+
lastSeenAtMs: -1,
|
|
69
|
+
lastSeenAt: null,
|
|
70
|
+
fingerprints: new Set(),
|
|
71
|
+
};
|
|
72
|
+
byKey.set(key, acc);
|
|
73
|
+
}
|
|
74
|
+
acc.sampleN += 1;
|
|
75
|
+
acc.tokensIn += Math.max(row.tokensIn, 0);
|
|
76
|
+
acc.tokensCached += Math.max(row.tokensCached, 0);
|
|
77
|
+
acc.tokensCacheWrite += Math.max(row.tokensCacheWrite ?? 0, 0);
|
|
78
|
+
if (t > acc.lastSeenAtMs) {
|
|
79
|
+
acc.lastSeenAtMs = t;
|
|
80
|
+
acc.lastSeenAt = row.ts;
|
|
81
|
+
}
|
|
82
|
+
if (row.prefixFingerprint != null && row.prefixFingerprint !== "") {
|
|
83
|
+
acc.fingerprints.add(row.prefixFingerprint);
|
|
84
|
+
}
|
|
85
|
+
}
|
|
86
|
+
const out = [];
|
|
87
|
+
for (const [key, acc] of byKey) {
|
|
88
|
+
const [route, provider, model] = key.split(KEY_SEP);
|
|
89
|
+
out.push({
|
|
90
|
+
route,
|
|
91
|
+
provider,
|
|
92
|
+
model,
|
|
93
|
+
sampleN: acc.sampleN,
|
|
94
|
+
tokensIn: acc.tokensIn,
|
|
95
|
+
tokensCached: acc.tokensCached,
|
|
96
|
+
tokensCacheWrite: acc.tokensCacheWrite,
|
|
97
|
+
h: acc.tokensIn > 0 ? acc.tokensCached / acc.tokensIn : 0,
|
|
98
|
+
w: acc.tokensIn > 0 ? acc.tokensCacheWrite / acc.tokensIn : 0,
|
|
99
|
+
lastSeenAt: acc.lastSeenAt,
|
|
100
|
+
prefixFingerprints: [...acc.fingerprints].sort(),
|
|
101
|
+
});
|
|
102
|
+
}
|
|
103
|
+
return out.sort((a, b) => a.route !== b.route
|
|
104
|
+
? a.route.localeCompare(b.route)
|
|
105
|
+
: a.provider !== b.provider
|
|
106
|
+
? a.provider.localeCompare(b.provider)
|
|
107
|
+
: a.model.localeCompare(b.model));
|
|
108
|
+
}
|
package/dist/savings/emit.d.ts
CHANGED
|
@@ -21,5 +21,18 @@ export interface SwapChange {
|
|
|
21
21
|
* revert can be emitted even without the pre-swap file. */
|
|
22
22
|
readonly incumbentCandidates: readonly CandidateSpec[];
|
|
23
23
|
}
|
|
24
|
+
/**
|
|
25
|
+
* Restore-YAML for a REVERT — managed-loop §1 (`POST /v1/swaps/:id/revert`).
|
|
26
|
+
*
|
|
27
|
+
* A revert has no bake-off winner; it restores the swap's snapshotted incumbent
|
|
28
|
+
* candidates (`swap_events.baseline.incumbent_candidates`) at a version above the
|
|
29
|
+
* route's current one — history moves forward, never rewrites. Same
|
|
30
|
+
* no-filesystem, paste-ready shape as `buildSwapChange`; the server emits it and
|
|
31
|
+
* a human applies it via a PR (never-silent-swap extends to the revert).
|
|
32
|
+
*/
|
|
33
|
+
export declare function buildRestoreYaml(route: string, incumbent: readonly {
|
|
34
|
+
readonly provider: string;
|
|
35
|
+
readonly model: string;
|
|
36
|
+
}[], currentVersion: number): string;
|
|
24
37
|
export declare function buildSwapChange(bundle: RouteBundle, winningVariant: string): SwapChange;
|
|
25
38
|
//# sourceMappingURL=emit.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"emit.d.ts","sourceRoot":"","sources":["../../src/savings/emit.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAEH,OAAO,KAAK,EAAE,aAAa,EAAE,WAAW,EAAgB,MAAM,gBAAgB,CAAC;AAE/E,MAAM,WAAW,UAAU;IACzB,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;IACzB,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,2EAA2E;IAC3E,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB;yCACqC;IACrC,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B;+DAC2D;IAC3D,QAAQ,CAAC,mBAAmB,EAAE,SAAS,aAAa,EAAE,CAAC;CACxD;
|
|
1
|
+
{"version":3,"file":"emit.d.ts","sourceRoot":"","sources":["../../src/savings/emit.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAEH,OAAO,KAAK,EAAE,aAAa,EAAE,WAAW,EAAgB,MAAM,gBAAgB,CAAC;AAE/E,MAAM,WAAW,UAAU;IACzB,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;IACzB,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,2EAA2E;IAC3E,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB;yCACqC;IACrC,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B;+DAC2D;IAC3D,QAAQ,CAAC,mBAAmB,EAAE,SAAS,aAAa,EAAE,CAAC;CACxD;AAmCD;;;;;;;;GAQG;AACH,wBAAgB,gBAAgB,CAC9B,KAAK,EAAE,MAAM,EACb,SAAS,EAAE,SAAS;IAAE,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;IAAC,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAA;CAAE,EAAE,EAC3E,cAAc,EAAE,MAAM,GACrB,MAAM,CAgBR;AAED,wBAAgB,eAAe,CAAC,MAAM,EAAE,WAAW,EAAE,cAAc,EAAE,MAAM,GAAG,UAAU,CA6CvF"}
|
package/dist/savings/emit.js
CHANGED
|
@@ -7,6 +7,7 @@
|
|
|
7
7
|
* greps for it.
|
|
8
8
|
*/
|
|
9
9
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
10
|
+
exports.buildRestoreYaml = buildRestoreYaml;
|
|
10
11
|
exports.buildSwapChange = buildSwapChange;
|
|
11
12
|
function candidateLines(candidates) {
|
|
12
13
|
return candidates.flatMap((c) => [` - provider: ${c.provider}`, ` model: ${c.model}`]);
|
|
@@ -32,6 +33,30 @@ function promotedOrder(declared, winners) {
|
|
|
32
33
|
const winnerKeys = new Set(winners.map(key));
|
|
33
34
|
return [...winners, ...declared.filter((c) => !winnerKeys.has(key(c)))];
|
|
34
35
|
}
|
|
36
|
+
/**
|
|
37
|
+
* Restore-YAML for a REVERT — managed-loop §1 (`POST /v1/swaps/:id/revert`).
|
|
38
|
+
*
|
|
39
|
+
* A revert has no bake-off winner; it restores the swap's snapshotted incumbent
|
|
40
|
+
* candidates (`swap_events.baseline.incumbent_candidates`) at a version above the
|
|
41
|
+
* route's current one — history moves forward, never rewrites. Same
|
|
42
|
+
* no-filesystem, paste-ready shape as `buildSwapChange`; the server emits it and
|
|
43
|
+
* a human applies it via a PR (never-silent-swap extends to the revert).
|
|
44
|
+
*/
|
|
45
|
+
function buildRestoreYaml(route, incumbent, currentVersion) {
|
|
46
|
+
if (incumbent.length === 0) {
|
|
47
|
+
throw new Error(`cannot emit a restore for "${route}" — the swap recorded no incumbent candidates to restore ` +
|
|
48
|
+
"(an older swap predating the baseline snapshot; revert it by hand from git history)");
|
|
49
|
+
}
|
|
50
|
+
const file = `modelrig/routes/${route}.yaml`;
|
|
51
|
+
return [
|
|
52
|
+
`# revert: restore ${route} to ${incumbent[0].provider}/${incumbent[0].model}`,
|
|
53
|
+
`# apply to ${file} — replace the version and candidates fields; nothing else changes`,
|
|
54
|
+
`version: ${currentVersion + 1}`,
|
|
55
|
+
"candidates:",
|
|
56
|
+
...candidateLines(incumbent),
|
|
57
|
+
"",
|
|
58
|
+
].join("\n");
|
|
59
|
+
}
|
|
35
60
|
function buildSwapChange(bundle, winningVariant) {
|
|
36
61
|
const variant = bundle.variants.find((v) => v.name === winningVariant);
|
|
37
62
|
if (variant === undefined) {
|
|
@@ -15,8 +15,16 @@
|
|
|
15
15
|
*/
|
|
16
16
|
import type { SwapChange } from "./emit";
|
|
17
17
|
import type { SwapEventRecord } from "./swaps-client";
|
|
18
|
+
import type { CacheContext } from "../watch/evaluate";
|
|
19
|
+
import type { CacheVerdict } from "../registry/effective-rate";
|
|
18
20
|
import type { ProposalRecord, ProposalStatus } from "../watch/proposals";
|
|
19
21
|
import type { RouteBundle } from "../types/route";
|
|
22
|
+
/** Narrow a proposal's `evidence.cache_context` jsonb (spec §5.5 — additive,
|
|
23
|
+
* present only on cache-aware proposals) to a typed `CacheContext`, or null when
|
|
24
|
+
* absent/malformed. Defensive: an older or non-cache proposal has no block, and
|
|
25
|
+
* a partial block must degrade to "no cache explanation", never throw on the
|
|
26
|
+
* safety-critical execute path. */
|
|
27
|
+
export declare function readCacheContext(evidence: Record<string, unknown> | null): CacheContext | null;
|
|
20
28
|
export interface SwapExecuteDeps {
|
|
21
29
|
readonly proposalId: string;
|
|
22
30
|
/** THE gate — false throws before any read or write. */
|
|
@@ -39,6 +47,12 @@ export interface SwapExecuteResult {
|
|
|
39
47
|
readonly change: SwapChange;
|
|
40
48
|
readonly approvedBy: string;
|
|
41
49
|
readonly executedAt: string;
|
|
50
|
+
/** Cache-aware verdict from the proposal's `cache_context` evidence (spec
|
|
51
|
+
* §5.5), or null on a non-cache proposal. Additive — never gates execution
|
|
52
|
+
* (the human already approved); surfaced so the CLI/console can show WHY. */
|
|
53
|
+
readonly cacheVerdict: CacheVerdict | null;
|
|
54
|
+
/** The [Briefing §30] one-line explanation for the approval prompt, or null. */
|
|
55
|
+
readonly cacheExplanation: string | null;
|
|
42
56
|
}
|
|
43
57
|
export declare function executeSwap(deps: SwapExecuteDeps): Promise<SwapExecuteResult>;
|
|
44
58
|
//# sourceMappingURL=swap-execute.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"swap-execute.d.ts","sourceRoot":"","sources":["../../src/savings/swap-execute.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;GAcG;AAGH,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,QAAQ,CAAC;AACzC,OAAO,KAAK,EAAgB,eAAe,EAAE,MAAM,gBAAgB,CAAC;
|
|
1
|
+
{"version":3,"file":"swap-execute.d.ts","sourceRoot":"","sources":["../../src/savings/swap-execute.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;GAcG;AAGH,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,QAAQ,CAAC;AACzC,OAAO,KAAK,EAAgB,eAAe,EAAE,MAAM,gBAAgB,CAAC;AAEpE,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,mBAAmB,CAAC;AACtD,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,4BAA4B,CAAC;AAC/D,OAAO,KAAK,EAAE,cAAc,EAAE,cAAc,EAAE,MAAM,oBAAoB,CAAC;AACzE,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,gBAAgB,CAAC;AAElD;;;;mCAImC;AACnC,wBAAgB,gBAAgB,CAAC,QAAQ,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,IAAI,GAAG,YAAY,GAAG,IAAI,CAmB9F;AASD,MAAM,WAAW,eAAe;IAC9B,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,wDAAwD;IACxD,QAAQ,CAAC,OAAO,EAAE,OAAO,CAAC;IAC1B,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,SAAS,EAAE;QAClB,GAAG,CAAC,EAAE,EAAE,MAAM,GAAG,OAAO,CAAC,cAAc,GAAG,IAAI,CAAC,CAAC;QAChD,SAAS,CAAC,EAAE,EAAE,MAAM,EAAE,IAAI,EAAE,cAAc,EAAE,EAAE,EAAE,cAAc,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;KAChF,CAAC;IACF,QAAQ,CAAC,KAAK,EAAE;QAAE,MAAM,CAAC,KAAK,EAAE,eAAe,GAAG,OAAO,CAAC,IAAI,CAAC,CAAA;KAAE,CAAC;IAClE,QAAQ,CAAC,UAAU,EAAE,CAAC,KAAK,EAAE,MAAM,KAAK,WAAW,CAAC;IACpD,QAAQ,CAAC,KAAK,EAAE,MAAM,MAAM,CAAC;IAC7B,QAAQ,CAAC,GAAG,EAAE,MAAM,MAAM,CAAC;CAC5B;AAED,MAAM,WAAW,iBAAiB;IAChC,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;IACxB,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,MAAM,EAAE,UAAU,CAAC;IAC5B,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B;;iFAE6E;IAC7E,QAAQ,CAAC,YAAY,EAAE,YAAY,GAAG,IAAI,CAAC;IAC3C,gFAAgF;IAChF,QAAQ,CAAC,gBAAgB,EAAE,MAAM,GAAG,IAAI,CAAC;CAC1C;AAED,wBAAsB,WAAW,CAAC,IAAI,EAAE,eAAe,GAAG,OAAO,CAAC,iBAAiB,CAAC,CAwGnF"}
|
|
@@ -15,8 +15,36 @@
|
|
|
15
15
|
* executed. Applying the YAML stays a human/git action.
|
|
16
16
|
*/
|
|
17
17
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
18
|
+
exports.readCacheContext = readCacheContext;
|
|
18
19
|
exports.executeSwap = executeSwap;
|
|
19
20
|
const emit_1 = require("./emit");
|
|
21
|
+
const evaluate_1 = require("../watch/evaluate");
|
|
22
|
+
/** Narrow a proposal's `evidence.cache_context` jsonb (spec §5.5 — additive,
|
|
23
|
+
* present only on cache-aware proposals) to a typed `CacheContext`, or null when
|
|
24
|
+
* absent/malformed. Defensive: an older or non-cache proposal has no block, and
|
|
25
|
+
* a partial block must degrade to "no cache explanation", never throw on the
|
|
26
|
+
* safety-critical execute path. */
|
|
27
|
+
function readCacheContext(evidence) {
|
|
28
|
+
const raw = evidence?.cache_context;
|
|
29
|
+
if (raw === null || typeof raw !== "object")
|
|
30
|
+
return null;
|
|
31
|
+
const c = raw;
|
|
32
|
+
const verdicts = ["cache_positive", "cache_marginal", "cache_negative", "no_cache_signal"];
|
|
33
|
+
if (typeof c.verdict !== "string" || !verdicts.includes(c.verdict))
|
|
34
|
+
return null;
|
|
35
|
+
const num = (v) => (typeof v === "number" ? v : 0);
|
|
36
|
+
return {
|
|
37
|
+
observed_hit_rate: num(c.observed_hit_rate),
|
|
38
|
+
observed_write_rate: num(c.observed_write_rate),
|
|
39
|
+
window_days: num(c.window_days),
|
|
40
|
+
incumbent_effective_in_per_mtok: num(c.incumbent_effective_in_per_mtok),
|
|
41
|
+
challenger_cold_in_per_mtok: num(c.challenger_cold_in_per_mtok),
|
|
42
|
+
switch_cost_usd: num(c.switch_cost_usd),
|
|
43
|
+
cache_option_value_usd: num(c.cache_option_value_usd),
|
|
44
|
+
breakeven_conformant_calls: typeof c.breakeven_conformant_calls === "number" ? c.breakeven_conformant_calls : Number.POSITIVE_INFINITY,
|
|
45
|
+
verdict: c.verdict,
|
|
46
|
+
};
|
|
47
|
+
}
|
|
20
48
|
async function executeSwap(deps) {
|
|
21
49
|
// The gate first — before any client call, so a non-approved invocation
|
|
22
50
|
// has zero side effects (defense in depth with the CLI's fast-fail check).
|
|
@@ -83,5 +111,20 @@ async function executeSwap(deps) {
|
|
|
83
111
|
revertedAt: null,
|
|
84
112
|
});
|
|
85
113
|
await deps.proposals.setStatus(proposal.id, "approved", "executed");
|
|
86
|
-
|
|
114
|
+
// Cache-aware explainability (spec §5.5) — additive, read-only. The verdict +
|
|
115
|
+
// prose ride along for the approval prompt / audit trail; they NEVER gate the
|
|
116
|
+
// swap (the human decision above is the only gate). Absent on non-cache
|
|
117
|
+
// proposals, in which case both are null and behavior is byte-identical.
|
|
118
|
+
const cacheContext = readCacheContext(proposal.evidence);
|
|
119
|
+
const incumbentLabel = change.incumbentCandidates.map((c) => `${c.provider}/${c.model}`).join(", ") || "incumbent";
|
|
120
|
+
const cacheExplanation = cacheContext !== null ? (0, evaluate_1.explainCacheContext)(cacheContext, incumbentLabel, winner.variant) : null;
|
|
121
|
+
return {
|
|
122
|
+
swapId,
|
|
123
|
+
route: proposal.route,
|
|
124
|
+
change,
|
|
125
|
+
approvedBy: deps.approvedBy,
|
|
126
|
+
executedAt,
|
|
127
|
+
cacheVerdict: cacheContext?.verdict ?? null,
|
|
128
|
+
cacheExplanation,
|
|
129
|
+
};
|
|
87
130
|
}
|