@danypops/jittor 0.17.0 → 0.18.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/package.json CHANGED
@@ -1,11 +1,11 @@
1
1
  {
2
2
  "name": "@danypops/jittor",
3
- "version": "0.17.0",
3
+ "version": "0.18.1",
4
4
  "description": "Just-in-Time Token Optimization Router for Pi -- token and context observability with optimization policies",
5
5
  "type": "module",
6
6
  "main": "./src/index.ts",
7
7
  "types": "./src/index.ts",
8
- "keywords": ["pi-package", "llm-router", "token-budget"],
8
+ "keywords": ["llm-router", "token-budget"],
9
9
  "bin": {
10
10
  "jittor": "src/cli.ts"
11
11
  },
@@ -15,6 +15,17 @@
15
15
  "serve": "bun src/cli.ts serve",
16
16
  "service:install": "bun src/cli.ts service install"
17
17
  },
18
+ "packed": {
19
+ "daemonService": {
20
+ "name": "jittor",
21
+ "displayName": "Jittor",
22
+ "binPath": "src/cli.ts",
23
+ "args": ["serve"],
24
+ "handleFilename": "daemon.json",
25
+ "restartOnFailure": true,
26
+ "restartSec": 2
27
+ }
28
+ },
18
29
  "dependencies": {
19
30
  "@danypops/vehicle-core": "^0.12.1",
20
31
  "@danypops/vehicle-server": "^0.17.0",
@@ -0,0 +1,120 @@
1
+ import type {
2
+ CacheEconomicsAggregateTotals,
3
+ CacheEconomicsModelSummary,
4
+ CacheEconomicsStablePrefixPoint,
5
+ CacheEconomicsSummary,
6
+ CacheEconomicsTaskSummary,
7
+ CacheEconomicsUnattributedActivity,
8
+ } from "../observability/cache-economics.ts";
9
+ import { type CliDependencies, callAndPrint, humanField } from "./support.ts";
10
+
11
+ export const CACHE_USAGE_LINES = [" cache economics --since <ms> --until <ms> [--json]"];
12
+
13
+ interface CacheEconomicsArgs {
14
+ since: number;
15
+ until: number;
16
+ json: boolean;
17
+ }
18
+
19
+ function parseCacheArgs(action: string | undefined, args: string[]): CacheEconomicsArgs | null {
20
+ if (action !== "economics") return null;
21
+ let json = false;
22
+ let since: number | undefined;
23
+ let until: number | undefined;
24
+ for (let index = 0; index < args.length; index += 1) {
25
+ const argument = args[index];
26
+ if (argument === "--json") {
27
+ json = true;
28
+ continue;
29
+ }
30
+ if (argument !== "--since" && argument !== "--until") return null;
31
+ const raw = args[++index];
32
+ const value = raw === undefined ? Number.NaN : Number(raw);
33
+ if (!Number.isSafeInteger(value) || value < 0) return null;
34
+ if (argument === "--since") since = value;
35
+ else until = value;
36
+ }
37
+ if (since === undefined || until === undefined || until < since) return null;
38
+ return { since, until, json };
39
+ }
40
+
41
+ function formatUsdAmount(amount: number): string {
42
+ return `$${amount.toFixed(Math.abs(amount) < 0.01 && amount !== 0 ? 4 : 2)}`;
43
+ }
44
+
45
+ function basisSuffix(basis: "provider-reported" | "catalog-estimate" | "unknown"): string {
46
+ return basis === "provider-reported" ? "" : basis === "catalog-estimate" ? " (catalog estimate)" : "";
47
+ }
48
+
49
+ function formatCostField(amountUsd: number | null, basis: "provider-reported" | "catalog-estimate" | "unknown"): string {
50
+ return amountUsd === null ? "unknown" : `${formatUsdAmount(amountUsd)}${basisSuffix(basis)}`;
51
+ }
52
+
53
+ function formatAggregateFields(totals: CacheEconomicsAggregateTotals): string {
54
+ const payback = totals.paybackAchieved === null ? "n/a" : totals.paybackAchieved ? "yes" : "not yet";
55
+ return [
56
+ `read ${totals.cacheReadTokens.toLocaleString()} tok (${formatCostField(totals.cacheReadCostUsd, totals.cacheReadCostBasis)})`,
57
+ `write ${totals.cacheWriteTokens.toLocaleString()} tok (${formatCostField(totals.cacheWriteCostUsd, totals.cacheWriteCostBasis)})`,
58
+ `savings ${totals.savingsUsd === null ? "unknown" : formatUsdAmount(totals.savingsUsd)}`,
59
+ `premium ${totals.cacheWritePremiumUsd === null ? "unknown" : formatUsdAmount(totals.cacheWritePremiumUsd)}`,
60
+ `break-even ${totals.breakEvenReadTokens === null ? "unknown" : `${totals.breakEvenReadTokens.toLocaleString()} tok`}`,
61
+ `payback ${payback}`,
62
+ ].join(" · ");
63
+ }
64
+
65
+ function formatFreshnessSuffix(catalogFreshness: "fresh" | "stale" | null): string {
66
+ return catalogFreshness === "stale" ? " (stale catalog snapshot used for the estimate(s) above)" : "";
67
+ }
68
+
69
+ function formatModelLine(model: CacheEconomicsModelSummary): string {
70
+ return `- ${humanField(model.provider)}/${humanField(model.model)}: ${formatAggregateFields(model)}${formatFreshnessSuffix(model.catalogFreshness)}`;
71
+ }
72
+
73
+ function formatTaskLine(task: CacheEconomicsTaskSummary): string {
74
+ return `- ${humanField(task.taskId)}: ${formatAggregateFields(task)}${formatFreshnessSuffix(task.catalogFreshness)}`;
75
+ }
76
+
77
+ function formatUnattributedLine(activity: CacheEconomicsUnattributedActivity): string {
78
+ return [
79
+ `Unattributed cache activity (no Papyrus task focused):`,
80
+ `read ${activity.cacheReadTokens.toLocaleString()} tok (${formatCostField(activity.cacheReadCostUsd, activity.cacheReadCostBasis)})`,
81
+ `write ${activity.cacheWriteTokens.toLocaleString()} tok (${formatCostField(activity.cacheWriteCostUsd, activity.cacheWriteCostBasis)})`,
82
+ ].join(" · ");
83
+ }
84
+
85
+ function formatChurnPoint(point: CacheEconomicsStablePrefixPoint): string {
86
+ return `- ${new Date(point.observedAt).toISOString()} session ${humanField(point.sessionId)}: ${point.stablePrefixTokens.toLocaleString()} tok${point.resetReason === null ? "" : ` (${point.resetReason} reset)`}`;
87
+ }
88
+
89
+ export function formatCacheEconomics(summary: CacheEconomicsSummary): string {
90
+ const lines = [
91
+ `Cache economics: ${summary.models.length.toLocaleString()} model(s)${summary.truncated ? " (query limit reached; totals are a lower bound)" : ""}`,
92
+ ...summary.models.map(formatModelLine),
93
+ `By task: ${summary.tasks.length.toLocaleString()} task(s)`,
94
+ ...summary.tasks.map(formatTaskLine),
95
+ formatUnattributedLine(summary.unattributedCacheActivity),
96
+ ...(summary.stablePrefixChurn.length > 0
97
+ ? [
98
+ `Stable-prefix churn (${summary.stablePrefixChurn.length.toLocaleString()} snapshot(s), oldest first):`,
99
+ ...summary.stablePrefixChurn.map(formatChurnPoint),
100
+ ]
101
+ : []),
102
+ `Candidate missed-cache opportunities: ${summary.missedOpportunities.length.toLocaleString()}`,
103
+ ...summary.missedOpportunities.map(
104
+ (candidate) =>
105
+ `- session ${humanField(candidate.sessionId)}: ${candidate.resetReason} reset, then ${candidate.cacheWriteTokens.toLocaleString()} cache-write tok${candidate.cacheWriteCostUsd === null ? "" : ` (${formatUsdAmount(candidate.cacheWriteCostUsd)})`} -- ${candidate.note}`,
106
+ ),
107
+ ];
108
+ return lines.join("\n");
109
+ }
110
+
111
+ export async function runCacheCommand(
112
+ action: string | undefined,
113
+ rest: string[],
114
+ deps: CliDependencies,
115
+ usage: () => number,
116
+ ): Promise<number> {
117
+ const parsed = parseCacheArgs(action, rest);
118
+ if (!parsed) return usage();
119
+ return callAndPrint(deps, "cache.economics", { since: parsed.since, until: parsed.until }, parsed.json, formatCacheEconomics);
120
+ }
package/src/cli.ts CHANGED
@@ -2,6 +2,7 @@
2
2
  import { fileURLToPath } from "node:url";
3
3
  import { BACKFILL_USAGE_LINES, formatUsageImportResult, formatUsageImportStatus, runBackfillCommand } from "./cli-commands/backfill.ts";
4
4
  import { BENCHMARKS_USAGE_LINES, runBenchmarksCommand } from "./cli-commands/benchmarks.ts";
5
+ import { CACHE_USAGE_LINES, formatCacheEconomics, runCacheCommand } from "./cli-commands/cache.ts";
5
6
  import { CATALOG_USAGE_LINES, formatCatalogQuery, formatCatalogStatus, runCatalogCommand } from "./cli-commands/catalog.ts";
6
7
  import { runCompactionCommand } from "./cli-commands/compaction.ts";
7
8
  import { CONTEXT_USAGE_LINES, formatContextAssessment, formatContextDelta, runContextCommand } from "./cli-commands/context.ts";
@@ -19,6 +20,7 @@ import { connectJittorClient } from "./vehicle/client.ts";
19
20
  // directly from cli.ts rather than reaching into src/cli-commands/*.
20
21
  export type { CliDependencies };
21
22
  export {
23
+ formatCacheEconomics,
22
24
  formatCatalogQuery,
23
25
  formatCatalogStatus,
24
26
  formatContextAssessment,
@@ -54,6 +56,7 @@ function usage(stderr: (line: string) => void): number {
54
56
  ...CONTEXT_USAGE_LINES,
55
57
  ...BENCHMARKS_USAGE_LINES,
56
58
  ...CATALOG_USAGE_LINES,
59
+ ...CACHE_USAGE_LINES,
57
60
  ...METRICS_USAGE_LINES,
58
61
  ...ROUTER_USAGE_LINES,
59
62
  ...SESSION_USAGE_LINES,
@@ -96,6 +99,8 @@ export async function runCli(args: string[], deps: CliDependencies = DEFAULT_DEP
96
99
  return runBenchmarksCommand(action, rest, deps, fail);
97
100
  case "catalog":
98
101
  return runCatalogCommand(action, rest, deps, fail);
102
+ case "cache":
103
+ return runCacheCommand(action, rest, deps, fail);
99
104
  case "context":
100
105
  return runContextCommand(action, rest, deps, fail);
101
106
  case "service":
package/src/constants.ts CHANGED
@@ -69,6 +69,19 @@ export const PAPYRUS_TASK_FOCUS_SCHEMA = "papyrus.task-focus/v1";
69
69
  export const TASK_FOCUS_EVENT_MAX_AGE_MS = 5 * MILLISECONDS_PER_MINUTE;
70
70
  export const TASK_FOCUS_ID_MAX_LENGTH = 200;
71
71
  export const TASK_COST_QUERY_LIMIT = 10_000;
72
+ export const CACHE_ECONOMICS_QUERY_LIMIT = 10_000;
73
+ export const CACHE_ECONOMICS_MAX_MODEL_GROUPS = 500;
74
+ /** Bounds the per-task rollup the same way CACHE_ECONOMICS_MAX_MODEL_GROUPS bounds the per-model one -- applied post-hoc (sorted by activity, then sliced) rather than at row-intake time, since realistic task cardinality in one window is small. */
75
+ export const CACHE_ECONOMICS_MAX_TASK_GROUPS = 500;
76
+ /** Bounds the stable-prefix-churn time series; the most recent points are kept (oldest dropped first) since a trend view cares most about what's recent. */
77
+ export const CACHE_ECONOMICS_MAX_STABLE_PREFIX_POINTS = 500;
78
+ export const CACHE_ECONOMICS_MAX_MISSED_OPPORTUNITIES = 100;
79
+ /** How soon after a context-prefix reset (session/provider/model change) a same-session cache-write still counts as a candidate cache-loss consequence of that reset, not an unrelated later write. */
80
+ export const CACHE_ECONOMICS_LOSS_CORRELATION_WINDOW_MS = 5 * MILLISECONDS_PER_MINUTE;
81
+ /** models.dev (and Jittor's catalog) quote pricing per one million tokens; catalog-estimate math divides by this to get a per-token rate. */
82
+ export const CATALOG_PRICE_TOKEN_UNIT = 1_000_000;
83
+ /** models.dev's own long-context pricing convention: a request whose context exceeds this many tokens uses ModelCatalogPricing.contextOver200k instead of the flat/tiered rate. */
84
+ export const CATALOG_LONG_CONTEXT_THRESHOLD_TOKENS = 200_000;
72
85
  /** A `metrics prune --before` cutoff newer than this must pass `force: true`. Guards against accidentally wiping recent/live data with a too-recent cutoff (e.g. "now"), while still allowing routine cleanup of genuinely old rows without ceremony. */
73
86
  export const PRUNE_MIN_AGE_MS = 24 * MILLISECONDS_PER_HOUR;
74
87
  export const CONTEXT_OBSERVATION_MAX_CHARACTERS = 10_000_000;
package/src/index.ts CHANGED
@@ -15,6 +15,20 @@ export {
15
15
  type GoogleVertexMetricSource,
16
16
  googleVertexFailureMetrics,
17
17
  } from "./google-vertex/failures.ts";
18
+ export {
19
+ buildCacheEconomicsSummary,
20
+ type CacheCostBasis,
21
+ type CacheEconomicsAggregateTotals,
22
+ type CacheEconomicsMissedOpportunity,
23
+ type CacheEconomicsModelSummary,
24
+ type CacheEconomicsPricing,
25
+ type CacheEconomicsPricingLookup,
26
+ type CacheEconomicsStablePrefixPoint,
27
+ type CacheEconomicsSummary,
28
+ type CacheEconomicsSummaryOptions,
29
+ type CacheEconomicsTaskSummary,
30
+ type CacheEconomicsUnattributedActivity,
31
+ } from "./observability/cache-economics.ts";
18
32
  export {
19
33
  CONTEXT_SEGMENT_SOURCES,
20
34
  CONTEXT_SEGMENT_STATES,
@@ -0,0 +1,623 @@
1
+ import {
2
+ CACHE_ECONOMICS_LOSS_CORRELATION_WINDOW_MS,
3
+ CACHE_ECONOMICS_MAX_MISSED_OPPORTUNITIES,
4
+ CACHE_ECONOMICS_MAX_MODEL_GROUPS,
5
+ CACHE_ECONOMICS_MAX_STABLE_PREFIX_POINTS,
6
+ CACHE_ECONOMICS_MAX_TASK_GROUPS,
7
+ CATALOG_PRICE_TOKEN_UNIT,
8
+ } from "../constants.ts";
9
+ import type { ContextPrefixResetReason } from "./context-delta.ts";
10
+ import type { StoredMetricObservation } from "./metric.ts";
11
+
12
+ /**
13
+ * How much authority a cache-cost figure carries. `provider-reported` comes straight from the
14
+ * provider's own per-turn `usage.cost.*` breakdown (or is trivially true, e.g. zero tokens cost
15
+ * zero dollars); `catalog-estimate` is derived from models.dev catalog pricing when the provider
16
+ * never itemizes cache cost; `unknown` means neither exists and the figure must stay null rather
17
+ * than being fabricated.
18
+ */
19
+ export type CacheCostBasis = "provider-reported" | "catalog-estimate" | "unknown";
20
+
21
+ /** Flat (already tier-resolved, if applicable) per-token-million catalog prices for one provider/model at one particular request size. */
22
+ export interface CacheEconomicsPricing {
23
+ input?: number;
24
+ cacheRead?: number;
25
+ cacheWrite?: number;
26
+ /** Whether the catalog snapshot this price was resolved from was still fresh at query time. Only meaningful when this pricing is actually used for a catalog-estimate figure; the lookup may omit it (e.g. a fake with no notion of freshness). */
27
+ freshness?: "fresh" | "stale";
28
+ }
29
+
30
+ /**
31
+ * Best-effort catalog pricing lookup; returns null when the catalog has no snapshot or no
32
+ * matching model, never throws. `contextSizeTokens` is the specific request/run's own real size
33
+ * (input + cache-read + cache-write tokens) -- the lookup is free to resolve a tiered or
34
+ * long-context ("contextOver200k") price against it instead of a single flat rate; the domain
35
+ * layer here never assumes which it did.
36
+ */
37
+ export interface CacheEconomicsPricingLookup {
38
+ priceFor(provider: string, model: string, contextSizeTokens: number): CacheEconomicsPricing | null;
39
+ }
40
+
41
+ /**
42
+ * Every derived economics figure this domain produces, independent of which dimension (model,
43
+ * task) it's grouped by -- see CacheEconomicsModelSummary/CacheEconomicsTaskSummary, which each add
44
+ * only their own grouping key on top of this.
45
+ */
46
+ export interface CacheEconomicsAggregateTotals {
47
+ inputTokens: number;
48
+ cacheReadTokens: number;
49
+ cacheWriteTokens: number;
50
+ /** Real dollars actually paid for cache reads in-window, or a catalog estimate, or null when neither is known. */
51
+ cacheReadCostUsd: number | null;
52
+ cacheReadCostBasis: CacheCostBasis;
53
+ /** Real dollars actually paid for cache writes in-window, or a catalog estimate, or null when neither is known. */
54
+ cacheWriteCostUsd: number | null;
55
+ cacheWriteCostBasis: CacheCostBasis;
56
+ /** sum(provider-reported input cost) / sum(input tokens) within the window -- a real per-token rate derived only from provider-reported dollar/token figures, not a catalog guess. Null when the provider never reported itemized input cost in-window. */
57
+ effectiveInputRateUsdPerToken: number | null;
58
+ /** What the cache-read tokens would have cost had they been billed as ordinary input, at the best available rate. */
59
+ counterfactualNoCacheCostUsd: number | null;
60
+ counterfactualBasis: CacheCostBasis;
61
+ /** counterfactualNoCacheCostUsd minus the actual cache-read cost -- the real economic benefit of caching, when derivable. */
62
+ savingsUsd: number | null;
63
+ /** The premium actually paid for cache-write tokens over what plain input billing would have cost. */
64
+ cacheWritePremiumUsd: number | null;
65
+ /** How many cache-read tokens, at the observed/estimated read rate, would be needed to offset the write premium. Zero when no premium was paid; null when undeterminable. An aggregate approximation over the whole window/group, not resolved per run. */
66
+ breakEvenReadTokens: number | null;
67
+ /** Whether observed savings already met or exceeded the write premium. Null when there was no cache write to evaluate, or the comparison is undeterminable. */
68
+ paybackAchieved: boolean | null;
69
+ /** Worst ("stale" wins) freshness among every catalog-estimate figure this group actually used. Null when no catalog estimate was used at all -- everything was provider-reported, or nothing was derivable. Never set from a catalog price that was resolved but ended up unused (e.g. provider-reported cost took precedence). */
70
+ catalogFreshness: "fresh" | "stale" | null;
71
+ }
72
+
73
+ export interface CacheEconomicsModelSummary extends CacheEconomicsAggregateTotals {
74
+ provider: string;
75
+ model: string;
76
+ }
77
+
78
+ /**
79
+ * The same figures as CacheEconomicsModelSummary, rolled up by the Papyrus task focused when each
80
+ * row was recorded instead of by provider/model -- mirrors task-cost.ts's own attributes.taskId
81
+ * grouping for token/cost metrics. A task that stayed on one model gets the same tiered-catalog
82
+ * pricing precision as the model-level rollup; a task that spanned several models still sums
83
+ * correctly (each run was already priced against its own real context size before this rollup ever
84
+ * runs) but skips the single extra break-even catalog lookup that only makes sense for one model.
85
+ */
86
+ export interface CacheEconomicsTaskSummary extends CacheEconomicsAggregateTotals {
87
+ taskId: string;
88
+ }
89
+
90
+ /**
91
+ * Cache activity recorded with no Papyrus task focused. Real spend/activity, just not attributable
92
+ * to any task -- reported separately (mirroring task-cost.ts's own unattributedCostUsd) rather than
93
+ * silently dropped or folded into a fabricated "unknown task" bucket.
94
+ */
95
+ export interface CacheEconomicsUnattributedActivity {
96
+ cacheReadTokens: number;
97
+ cacheWriteTokens: number;
98
+ cacheReadCostUsd: number | null;
99
+ cacheReadCostBasis: CacheCostBasis;
100
+ cacheWriteCostUsd: number | null;
101
+ cacheWriteCostBasis: CacheCostBasis;
102
+ catalogFreshness: "fresh" | "stale" | null;
103
+ }
104
+
105
+ /**
106
+ * One snapshot's own stable-prefix-token measurement at one point in time -- the same raw evidence
107
+ * findMissedOpportunities correlates internally, made directly visible instead of only ever feeding
108
+ * a derived diagnostic. A sharp drop, especially alongside a non-null resetReason, is what a missed
109
+ * cache write actually looks like in this series; still just evidence, not a causal claim.
110
+ */
111
+ export interface CacheEconomicsStablePrefixPoint {
112
+ sessionId: string;
113
+ observedAt: number;
114
+ stablePrefixTokens: number;
115
+ resetReason: ContextPrefixResetReason;
116
+ }
117
+
118
+ /** A context-prefix reset (session/provider/model change) followed shortly by a same-session cache-write is *evidence*, not proof, that the reset forced a cache rewrite -- correlation, never asserted causality. */
119
+ export interface CacheEconomicsMissedOpportunity {
120
+ sessionId: string;
121
+ occurredAt: number;
122
+ resetReason: Exclude<ContextPrefixResetReason, null>;
123
+ cacheWriteTokens: number;
124
+ cacheWriteCostUsd: number | null;
125
+ note: string;
126
+ }
127
+
128
+ export interface CacheEconomicsSummary {
129
+ since: number;
130
+ until: number;
131
+ models: CacheEconomicsModelSummary[];
132
+ tasks: CacheEconomicsTaskSummary[];
133
+ unattributedCacheActivity: CacheEconomicsUnattributedActivity;
134
+ missedOpportunities: CacheEconomicsMissedOpportunity[];
135
+ /** Chronological (oldest first), bounded to the most recent CACHE_ECONOMICS_MAX_STABLE_PREFIX_POINTS. */
136
+ stablePrefixChurn: CacheEconomicsStablePrefixPoint[];
137
+ /** True when the model-group list, the task-group list, the stable-prefix-churn series, or the missed-opportunity list was cut off at its bound. */
138
+ truncated: boolean;
139
+ }
140
+
141
+ export interface CacheEconomicsSummaryOptions {
142
+ since: number;
143
+ until: number;
144
+ }
145
+
146
+ const RESET_REASONS = new Set<Exclude<ContextPrefixResetReason, null>>(["initial", "session-changed", "provider-changed", "model-changed"]);
147
+
148
+ function attributeText(attributes: Record<string, unknown>, key: string): string {
149
+ return typeof attributes[key] === "string" && attributes[key].length > 0 ? (attributes[key] as string) : "unknown";
150
+ }
151
+
152
+ /** A turn's own runId when present, else its shared observedAt timestamp -- rows recorded before runId tagging existed still group correctly as long as they were sent (and therefore timestamped) together. */
153
+ function runKeyFor(row: StoredMetricObservation): string {
154
+ const runId = row.attributes.runId;
155
+ if (typeof runId === "string" && runId.length > 0) return runId;
156
+ return `observedAt:${row.observedAt}`;
157
+ }
158
+
159
+ /** One turn's own token/cost totals -- the unit pricing (including any tiered/long-context catalog rate) is resolved against. */
160
+ interface RunAccumulator {
161
+ provider: string;
162
+ model: string;
163
+ /** The Papyrus task focused when this turn's rows were recorded, or undefined when nothing was focused -- undefined is a real, distinct state from any task id string, never coerced to "unknown". */
164
+ taskId: string | undefined;
165
+ inputTokens: number;
166
+ cacheReadTokens: number;
167
+ cacheWriteTokens: number;
168
+ inputCostUsd: number;
169
+ sawInputCost: boolean;
170
+ cacheReadCostUsd: number;
171
+ sawCacheReadCost: boolean;
172
+ cacheWriteCostUsd: number;
173
+ sawCacheWriteCost: boolean;
174
+ }
175
+
176
+ function newRunAccumulator(provider: string, model: string, taskId: string | undefined): RunAccumulator {
177
+ return {
178
+ provider,
179
+ model,
180
+ taskId,
181
+ inputTokens: 0,
182
+ cacheReadTokens: 0,
183
+ cacheWriteTokens: 0,
184
+ inputCostUsd: 0,
185
+ sawInputCost: false,
186
+ cacheReadCostUsd: 0,
187
+ sawCacheReadCost: false,
188
+ cacheWriteCostUsd: 0,
189
+ sawCacheWriteCost: false,
190
+ };
191
+ }
192
+
193
+ function combineBasis(left: CacheCostBasis, right: CacheCostBasis): CacheCostBasis {
194
+ if (left === "unknown" || right === "unknown") return "unknown";
195
+ if (left === "catalog-estimate" || right === "catalog-estimate") return "catalog-estimate";
196
+ return "provider-reported";
197
+ }
198
+
199
+ /** Sums basis-tagged dollar amounts; a single unknown amount makes the whole sum unknown rather than silently partial. */
200
+ function combineDollarField(entries: Array<{ amount: number | null; basis: CacheCostBasis }>): {
201
+ amount: number | null;
202
+ basis: CacheCostBasis;
203
+ } {
204
+ let total = 0;
205
+ let basis: CacheCostBasis = "provider-reported";
206
+ for (const entry of entries) {
207
+ if (entry.amount === null) return { amount: null, basis: "unknown" };
208
+ total += entry.amount;
209
+ basis = combineBasis(basis, entry.basis);
210
+ }
211
+ return { amount: total, basis };
212
+ }
213
+
214
+ function combineNullableSum(values: Array<number | null>): number | null {
215
+ let total = 0;
216
+ for (const value of values) {
217
+ if (value === null) return null;
218
+ total += value;
219
+ }
220
+ return total;
221
+ }
222
+
223
+ /** "stale" outvotes "fresh"; both outvote "never used a catalog estimate at all" (null). */
224
+ function combineFreshness(values: Array<"fresh" | "stale" | undefined>): "fresh" | "stale" | null {
225
+ let result: "fresh" | "stale" | null = null;
226
+ for (const value of values) {
227
+ if (value === undefined) continue;
228
+ if (value === "stale") return "stale";
229
+ result = "fresh";
230
+ }
231
+ return result;
232
+ }
233
+
234
+ function actualCost(
235
+ tokens: number,
236
+ sawCost: boolean,
237
+ costUsd: number,
238
+ catalogPricePerMillion: number | undefined,
239
+ ): { costUsd: number | null; basis: CacheCostBasis } {
240
+ if (tokens === 0) return { costUsd: 0, basis: "provider-reported" };
241
+ if (sawCost) return { costUsd, basis: "provider-reported" };
242
+ if (catalogPricePerMillion !== undefined)
243
+ return { costUsd: (tokens * catalogPricePerMillion) / CATALOG_PRICE_TOKEN_UNIT, basis: "catalog-estimate" };
244
+ return { costUsd: null, basis: "unknown" };
245
+ }
246
+
247
+ interface RunPricingResult {
248
+ provider: string;
249
+ model: string;
250
+ taskId: string | undefined;
251
+ inputTokens: number;
252
+ cacheReadTokens: number;
253
+ cacheWriteTokens: number;
254
+ inputCostUsd: number;
255
+ sawInputCost: boolean;
256
+ cacheReadCostUsd: number | null;
257
+ cacheReadCostBasis: CacheCostBasis;
258
+ cacheWriteCostUsd: number | null;
259
+ cacheWriteCostBasis: CacheCostBasis;
260
+ counterfactualNoCacheCostUsd: number | null;
261
+ counterfactualBasis: CacheCostBasis;
262
+ cacheWritePremiumUsd: number | null;
263
+ /** This run's own catalog freshness, only when a catalog-estimate price actually ended up used for one of this run's figures; undefined otherwise (never fabricated from an unused lookup result). */
264
+ catalogFreshness: "fresh" | "stale" | undefined;
265
+ }
266
+
267
+ /**
268
+ * Prices exactly one turn, against that turn's own real context size (input + cache-read +
269
+ * cache-write tokens) -- the only level at which a tiered/long-context catalog price can honestly
270
+ * be resolved. Never sums across turns; that happens once, afterward, in aggregateRuns.
271
+ */
272
+ function priceRun(run: RunAccumulator, pricing: CacheEconomicsPricingLookup): RunPricingResult {
273
+ const contextSizeTokens = run.inputTokens + run.cacheReadTokens + run.cacheWriteTokens;
274
+ const catalogPrices = pricing.priceFor(run.provider, run.model, contextSizeTokens);
275
+ const read = actualCost(run.cacheReadTokens, run.sawCacheReadCost, run.cacheReadCostUsd, catalogPrices?.cacheRead);
276
+ const write = actualCost(run.cacheWriteTokens, run.sawCacheWriteCost, run.cacheWriteCostUsd, catalogPrices?.cacheWrite);
277
+
278
+ let baselineRate: number | null = null;
279
+ let baselineBasis: CacheCostBasis = "unknown";
280
+ if (run.sawInputCost && run.inputTokens > 0) {
281
+ baselineRate = run.inputCostUsd / run.inputTokens;
282
+ baselineBasis = "provider-reported";
283
+ } else if (catalogPrices?.input !== undefined) {
284
+ baselineRate = catalogPrices.input / CATALOG_PRICE_TOKEN_UNIT;
285
+ baselineBasis = "catalog-estimate";
286
+ }
287
+
288
+ let counterfactualNoCacheCostUsd: number | null = null;
289
+ let counterfactualBasis: CacheCostBasis = "unknown";
290
+ if (run.cacheReadTokens === 0) {
291
+ counterfactualNoCacheCostUsd = 0;
292
+ counterfactualBasis = "provider-reported";
293
+ } else if (baselineRate !== null) {
294
+ counterfactualNoCacheCostUsd = run.cacheReadTokens * baselineRate;
295
+ counterfactualBasis = baselineBasis;
296
+ }
297
+
298
+ let cacheWritePremiumUsd: number | null = null;
299
+ if (run.cacheWriteTokens === 0) cacheWritePremiumUsd = 0;
300
+ else if (baselineRate !== null && write.costUsd !== null) cacheWritePremiumUsd = write.costUsd - run.cacheWriteTokens * baselineRate;
301
+
302
+ const finalCounterfactualBasis = run.cacheReadTokens === 0 ? "provider-reported" : combineBasis(baselineBasis, counterfactualBasis);
303
+ const usedCatalog =
304
+ read.basis === "catalog-estimate" || write.basis === "catalog-estimate" || finalCounterfactualBasis === "catalog-estimate";
305
+
306
+ return {
307
+ provider: run.provider,
308
+ model: run.model,
309
+ taskId: run.taskId,
310
+ inputTokens: run.inputTokens,
311
+ cacheReadTokens: run.cacheReadTokens,
312
+ cacheWriteTokens: run.cacheWriteTokens,
313
+ inputCostUsd: run.inputCostUsd,
314
+ sawInputCost: run.sawInputCost,
315
+ cacheReadCostUsd: read.costUsd,
316
+ cacheReadCostBasis: read.basis,
317
+ cacheWriteCostUsd: write.costUsd,
318
+ cacheWriteCostBasis: write.basis,
319
+ counterfactualNoCacheCostUsd,
320
+ counterfactualBasis: finalCounterfactualBasis,
321
+ cacheWritePremiumUsd,
322
+ catalogFreshness: usedCatalog ? catalogPrices?.freshness : undefined,
323
+ };
324
+ }
325
+
326
+ /**
327
+ * A single provider/model to resolve one extra, approximate whole-window catalog lookup against
328
+ * for the break-even projection below -- only meaningful when every run being aggregated actually
329
+ * shares this same provider/model (a per-model rollup always does; a per-task rollup only does when
330
+ * that task stayed on one model the whole time). Pass null to skip that refinement rather than
331
+ * guessing which of several different models' rates should stand in for the blend.
332
+ */
333
+ interface SingleModelPricingContext {
334
+ provider: string;
335
+ model: string;
336
+ pricing: CacheEconomicsPricingLookup;
337
+ }
338
+
339
+ /**
340
+ * Sums already-run-priced dollar figures into one group's (model's, or task's) window totals, then
341
+ * derives break-even/payback from those totals plus -- when catalogContext identifies a single real
342
+ * provider/model to resolve against -- one whole-window catalog lookup. This is an intentional,
343
+ * documented approximation: unlike the dollar totals above (correctly tiered per run), a single
344
+ * "how many more read tokens would it take" projection over a blended window has no one real
345
+ * request size to resolve a tier against either.
346
+ */
347
+ function aggregateRunTotals(runs: RunPricingResult[], catalogContext: SingleModelPricingContext | null): CacheEconomicsAggregateTotals {
348
+ const inputTokens = runs.reduce((sum, run) => sum + run.inputTokens, 0);
349
+ const cacheReadTokens = runs.reduce((sum, run) => sum + run.cacheReadTokens, 0);
350
+ const cacheWriteTokens = runs.reduce((sum, run) => sum + run.cacheWriteTokens, 0);
351
+ const read = combineDollarField(runs.map((run) => ({ amount: run.cacheReadCostUsd, basis: run.cacheReadCostBasis })));
352
+ const write = combineDollarField(runs.map((run) => ({ amount: run.cacheWriteCostUsd, basis: run.cacheWriteCostBasis })));
353
+ const counterfactual = combineDollarField(
354
+ runs.map((run) => ({ amount: run.counterfactualNoCacheCostUsd, basis: run.counterfactualBasis })),
355
+ );
356
+ const savingsUsd = counterfactual.amount !== null && read.amount !== null ? counterfactual.amount - read.amount : null;
357
+ const cacheWritePremiumUsd = combineNullableSum(runs.map((run) => run.cacheWritePremiumUsd));
358
+
359
+ const reportingRuns = runs.filter((run) => run.sawInputCost && run.inputTokens > 0);
360
+ const effectiveInputRateUsdPerToken =
361
+ reportingRuns.length > 0
362
+ ? reportingRuns.reduce((sum, run) => sum + run.inputCostUsd, 0) / reportingRuns.reduce((sum, run) => sum + run.inputTokens, 0)
363
+ : null;
364
+
365
+ let breakEvenReadTokens: number | null = null;
366
+ if (cacheWriteTokens === 0) breakEvenReadTokens = 0;
367
+ else if (cacheWritePremiumUsd !== null) {
368
+ const catalogPrices = catalogContext?.pricing.priceFor(
369
+ catalogContext.provider,
370
+ catalogContext.model,
371
+ inputTokens + cacheReadTokens + cacheWriteTokens,
372
+ );
373
+ const baselineRate =
374
+ effectiveInputRateUsdPerToken ?? (catalogPrices?.input !== undefined ? catalogPrices.input / CATALOG_PRICE_TOKEN_UNIT : null);
375
+ const readRate =
376
+ cacheReadTokens > 0 && read.amount !== null
377
+ ? read.amount / cacheReadTokens
378
+ : (catalogPrices?.cacheRead ?? undefined) !== undefined
379
+ ? catalogPrices!.cacheRead! / CATALOG_PRICE_TOKEN_UNIT
380
+ : null;
381
+ const perTokenSavings = baselineRate !== null && readRate !== null ? baselineRate - readRate : null;
382
+ if (cacheWritePremiumUsd <= 0) breakEvenReadTokens = 0;
383
+ else if (perTokenSavings !== null && perTokenSavings > 0) breakEvenReadTokens = Math.ceil(cacheWritePremiumUsd / perTokenSavings);
384
+ }
385
+
386
+ const paybackAchieved =
387
+ cacheWriteTokens === 0 ? null : savingsUsd !== null && cacheWritePremiumUsd !== null ? savingsUsd >= cacheWritePremiumUsd : null;
388
+
389
+ return {
390
+ inputTokens,
391
+ cacheReadTokens,
392
+ cacheWriteTokens,
393
+ cacheReadCostUsd: read.amount,
394
+ cacheReadCostBasis: read.basis,
395
+ cacheWriteCostUsd: write.amount,
396
+ cacheWriteCostBasis: write.basis,
397
+ effectiveInputRateUsdPerToken,
398
+ counterfactualNoCacheCostUsd: counterfactual.amount,
399
+ counterfactualBasis: counterfactual.basis,
400
+ savingsUsd,
401
+ cacheWritePremiumUsd,
402
+ breakEvenReadTokens,
403
+ paybackAchieved,
404
+ catalogFreshness: combineFreshness(runs.map((run) => run.catalogFreshness)),
405
+ };
406
+ }
407
+
408
+ function buildModelSummary(
409
+ provider: string,
410
+ model: string,
411
+ runs: RunPricingResult[],
412
+ pricing: CacheEconomicsPricingLookup,
413
+ ): CacheEconomicsModelSummary {
414
+ return { provider, model, ...aggregateRunTotals(runs, { provider, model, pricing }) };
415
+ }
416
+
417
+ /** A single-model catalog context only when every run in this task's group really did share one provider/model -- never guesses a representative model for a task that switched partway through. */
418
+ function buildTaskSummary(taskId: string, runs: RunPricingResult[], pricing: CacheEconomicsPricingLookup): CacheEconomicsTaskSummary {
419
+ const firstRun = runs[0]!;
420
+ const singleModel = runs.every((run) => run.provider === firstRun.provider && run.model === firstRun.model);
421
+ const catalogContext = singleModel ? { provider: firstRun.provider, model: firstRun.model, pricing } : null;
422
+ return { taskId, ...aggregateRunTotals(runs, catalogContext) };
423
+ }
424
+
425
+ interface ResetEvent {
426
+ sessionId: string;
427
+ occurredAt: number;
428
+ resetReason: Exclude<ContextPrefixResetReason, null>;
429
+ }
430
+
431
+ function resetEventFromRow(row: StoredMetricObservation): ResetEvent | null {
432
+ const resetReason = row.attributes.resetReason;
433
+ if (resetReason === null || resetReason === undefined || !RESET_REASONS.has(resetReason as Exclude<ContextPrefixResetReason, null>))
434
+ return null;
435
+ if (typeof row.scope !== "string" || row.scope.length === 0) return null;
436
+ return { sessionId: row.scope, occurredAt: row.observedAt, resetReason: resetReason as Exclude<ContextPrefixResetReason, null> };
437
+ }
438
+
439
+ function stablePrefixChurnFrom(snapshotRows: StoredMetricObservation[]): CacheEconomicsStablePrefixPoint[] {
440
+ return snapshotRows
441
+ .filter(
442
+ (row): row is StoredMetricObservation =>
443
+ row.source === "pi-context-snapshot" &&
444
+ row.metric === "snapshot" &&
445
+ typeof row.value === "number" &&
446
+ typeof row.scope === "string" &&
447
+ row.scope.length > 0,
448
+ )
449
+ .map((row) => ({
450
+ sessionId: row.scope as string,
451
+ observedAt: row.observedAt,
452
+ stablePrefixTokens: row.value as number,
453
+ resetReason: (row.attributes.resetReason ?? null) as ContextPrefixResetReason,
454
+ }))
455
+ .sort((left, right) => left.observedAt - right.observedAt);
456
+ }
457
+
458
+ function findMissedOpportunities(
459
+ usageRows: StoredMetricObservation[],
460
+ snapshotRows: StoredMetricObservation[],
461
+ ): CacheEconomicsMissedOpportunity[] {
462
+ const resets = snapshotRows
463
+ .filter((row) => row.source === "pi-context-snapshot" && row.metric === "snapshot")
464
+ .map(resetEventFromRow)
465
+ .filter((event): event is ResetEvent => event !== null);
466
+ if (resets.length === 0) return [];
467
+ const writes = usageRows.filter(
468
+ (row) => row.source === "pi" && row.metric === "cache-write-tokens" && typeof row.value === "number" && row.value > 0,
469
+ );
470
+ const found: CacheEconomicsMissedOpportunity[] = [];
471
+ for (const reset of resets) {
472
+ const match = writes
473
+ .filter((row) => attributeText(row.attributes, "sessionId") === reset.sessionId)
474
+ .filter(
475
+ (row) => row.observedAt >= reset.occurredAt && row.observedAt - reset.occurredAt <= CACHE_ECONOMICS_LOSS_CORRELATION_WINDOW_MS,
476
+ )
477
+ .sort((left, right) => left.observedAt - right.observedAt)[0];
478
+ if (!match) continue;
479
+ const costRow = usageRows.find(
480
+ (row) =>
481
+ row.source === "pi" &&
482
+ row.metric === "cache-write-cost" &&
483
+ attributeText(row.attributes, "sessionId") === reset.sessionId &&
484
+ row.observedAt === match.observedAt,
485
+ );
486
+ found.push({
487
+ sessionId: reset.sessionId,
488
+ occurredAt: match.observedAt,
489
+ resetReason: reset.resetReason,
490
+ cacheWriteTokens: match.value as number,
491
+ cacheWriteCostUsd: typeof costRow?.value === "number" ? costRow.value : null,
492
+ note: `Candidate missed-cache opportunity: a ${reset.resetReason} context-prefix reset was followed by a cache write in the same session within the correlation window. This is a correlated pattern, not a proven cause.`,
493
+ });
494
+ }
495
+ return found;
496
+ }
497
+
498
+ /**
499
+ * Pure aggregation over already-fetched, already-bounded rows -- the operation layer owns querying
500
+ * MetricStore and the model catalog; this function only ever combines what it is given. Rows are
501
+ * first grouped into per-turn runs (see runKeyFor) and priced against each run's own real context
502
+ * size, so a tiered/long-context catalog price is resolved honestly instead of guessed against a
503
+ * blended window-wide sum; run-level dollar figures are only summed together afterward. Every
504
+ * derived (non-trivial) dollar figure is explicitly basis-tagged; nothing is fabricated when
505
+ * evidence is absent.
506
+ */
507
+ export function buildCacheEconomicsSummary(
508
+ usageRows: StoredMetricObservation[],
509
+ snapshotRows: StoredMetricObservation[],
510
+ pricing: CacheEconomicsPricingLookup,
511
+ options: CacheEconomicsSummaryOptions,
512
+ ): CacheEconomicsSummary {
513
+ const byRun = new Map<string, RunAccumulator>();
514
+ const admittedModels = new Set<string>();
515
+ let modelGroupsTruncated = false;
516
+ for (const row of usageRows) {
517
+ if (row.source !== "pi" || typeof row.value !== "number" || !Number.isFinite(row.value) || row.value < 0) continue;
518
+ if (row.observedAt < options.since || row.observedAt > options.until) continue;
519
+ const provider = attributeText(row.attributes, "provider");
520
+ const model = attributeText(row.attributes, "model");
521
+ const modelKey = `${provider}\u0000${model}`;
522
+ if (!admittedModels.has(modelKey)) {
523
+ if (admittedModels.size >= CACHE_ECONOMICS_MAX_MODEL_GROUPS) {
524
+ modelGroupsTruncated = true;
525
+ continue;
526
+ }
527
+ admittedModels.add(modelKey);
528
+ }
529
+ const taskId = typeof row.attributes.taskId === "string" && row.attributes.taskId.length > 0 ? row.attributes.taskId : undefined;
530
+ const runKey = `${modelKey}\u0000${runKeyFor(row)}`;
531
+ const run = byRun.get(runKey) ?? newRunAccumulator(provider, model, taskId);
532
+ byRun.set(runKey, run);
533
+ if (row.metric === "input-tokens" && row.unit === "tokens") run.inputTokens += row.value;
534
+ else if (row.metric === "cache-read-tokens" && row.unit === "tokens") run.cacheReadTokens += row.value;
535
+ else if (row.metric === "cache-write-tokens" && row.unit === "tokens") run.cacheWriteTokens += row.value;
536
+ else if (row.metric === "input-cost" && row.unit === "usd") {
537
+ run.inputCostUsd += row.value;
538
+ run.sawInputCost = true;
539
+ } else if (row.metric === "cache-read-cost" && row.unit === "usd") {
540
+ run.cacheReadCostUsd += row.value;
541
+ run.sawCacheReadCost = true;
542
+ } else if (row.metric === "cache-write-cost" && row.unit === "usd") {
543
+ run.cacheWriteCostUsd += row.value;
544
+ run.sawCacheWriteCost = true;
545
+ }
546
+ }
547
+
548
+ const allPriced = [...byRun.values()].map((run) => priceRun(run, pricing));
549
+
550
+ const runsByModel = new Map<string, { provider: string; model: string; runs: RunPricingResult[] }>();
551
+ const runsByTask = new Map<string, RunPricingResult[]>();
552
+ const unattributedRuns: RunPricingResult[] = [];
553
+ for (const priced of allPriced) {
554
+ const modelKey = `${priced.provider}\u0000${priced.model}`;
555
+ const existingModel = runsByModel.get(modelKey);
556
+ if (existingModel) existingModel.runs.push(priced);
557
+ else runsByModel.set(modelKey, { provider: priced.provider, model: priced.model, runs: [priced] });
558
+
559
+ if (priced.taskId === undefined) {
560
+ unattributedRuns.push(priced);
561
+ continue;
562
+ }
563
+ const existingTask = runsByTask.get(priced.taskId);
564
+ if (existingTask) existingTask.push(priced);
565
+ else runsByTask.set(priced.taskId, [priced]);
566
+ }
567
+ const models = [...runsByModel.values()]
568
+ .map(({ provider, model, runs }) => buildModelSummary(provider, model, runs, pricing))
569
+ .sort(
570
+ (left, right) =>
571
+ right.cacheReadTokens + right.cacheWriteTokens - (left.cacheReadTokens + left.cacheWriteTokens) ||
572
+ left.model.localeCompare(right.model),
573
+ );
574
+
575
+ const allTasks = [...runsByTask.entries()]
576
+ .map(([taskId, runs]) => buildTaskSummary(taskId, runs, pricing))
577
+ .sort(
578
+ (left, right) =>
579
+ right.cacheReadTokens + right.cacheWriteTokens - (left.cacheReadTokens + left.cacheWriteTokens) ||
580
+ left.taskId.localeCompare(right.taskId),
581
+ );
582
+ const taskGroupsTruncated = allTasks.length > CACHE_ECONOMICS_MAX_TASK_GROUPS;
583
+ const tasks = allTasks.slice(0, CACHE_ECONOMICS_MAX_TASK_GROUPS);
584
+
585
+ const unattributedCacheRead = combineDollarField(
586
+ unattributedRuns.map((run) => ({ amount: run.cacheReadCostUsd, basis: run.cacheReadCostBasis })),
587
+ );
588
+ const unattributedCacheWrite = combineDollarField(
589
+ unattributedRuns.map((run) => ({ amount: run.cacheWriteCostUsd, basis: run.cacheWriteCostBasis })),
590
+ );
591
+ const unattributedCacheActivity: CacheEconomicsUnattributedActivity = {
592
+ cacheReadTokens: unattributedRuns.reduce((sum, run) => sum + run.cacheReadTokens, 0),
593
+ cacheWriteTokens: unattributedRuns.reduce((sum, run) => sum + run.cacheWriteTokens, 0),
594
+ cacheReadCostUsd: unattributedCacheRead.amount,
595
+ cacheReadCostBasis: unattributedCacheRead.basis,
596
+ cacheWriteCostUsd: unattributedCacheWrite.amount,
597
+ cacheWriteCostBasis: unattributedCacheWrite.basis,
598
+ catalogFreshness: combineFreshness(unattributedRuns.map((run) => run.catalogFreshness)),
599
+ };
600
+
601
+ const windowedSnapshotRows = snapshotRows.filter((row) => row.observedAt >= options.since && row.observedAt <= options.until);
602
+ const allMissed = findMissedOpportunities(
603
+ usageRows.filter((row) => row.observedAt >= options.since && row.observedAt <= options.until),
604
+ windowedSnapshotRows,
605
+ );
606
+ const missedTruncated = allMissed.length > CACHE_ECONOMICS_MAX_MISSED_OPPORTUNITIES;
607
+ const missedOpportunities = allMissed.slice(0, CACHE_ECONOMICS_MAX_MISSED_OPPORTUNITIES);
608
+
609
+ const allChurn = stablePrefixChurnFrom(windowedSnapshotRows);
610
+ const churnTruncated = allChurn.length > CACHE_ECONOMICS_MAX_STABLE_PREFIX_POINTS;
611
+ const stablePrefixChurn = allChurn.slice(-CACHE_ECONOMICS_MAX_STABLE_PREFIX_POINTS);
612
+
613
+ return {
614
+ since: options.since,
615
+ until: options.until,
616
+ models,
617
+ tasks,
618
+ unattributedCacheActivity,
619
+ missedOpportunities,
620
+ stablePrefixChurn,
621
+ truncated: modelGroupsTruncated || taskGroupsTruncated || missedTruncated || churnTruncated,
622
+ };
623
+ }
@@ -0,0 +1,95 @@
1
+ import { CACHE_ECONOMICS_QUERY_LIMIT, CATALOG_LONG_CONTEXT_THRESHOLD_TOKENS } from "../constants.ts";
2
+ import {
3
+ buildCacheEconomicsSummary,
4
+ type CacheEconomicsPricing,
5
+ type CacheEconomicsPricingLookup,
6
+ } from "../observability/cache-economics.ts";
7
+ import type { MetricStore } from "../observability/store.ts";
8
+ import type { ModelCatalogController, ModelCatalogPriceTier, ModelCatalogPricing } from "../optimization/model-selection/catalog.ts";
9
+ import type { OperationHandlerMap } from "./operation-types.ts";
10
+
11
+ function mergeWithBase(
12
+ base: ModelCatalogPricing,
13
+ override: Partial<Pick<ModelCatalogPricing, "input" | "cacheRead" | "cacheWrite">>,
14
+ ): CacheEconomicsPricing {
15
+ return {
16
+ input: override.input ?? base.input,
17
+ cacheRead: override.cacheRead ?? base.cacheRead,
18
+ cacheWrite: override.cacheWrite ?? base.cacheWrite,
19
+ };
20
+ }
21
+
22
+ /**
23
+ * Resolves one flat price for one real request size, honoring models.dev's own long-context and
24
+ * tiered pricing shapes -- the only place tiering can be resolved honestly, since it needs a
25
+ * single request's real context size (see cache-economics.ts's per-run pricing).
26
+ *
27
+ * `contextOver200k` takes priority over `tiers` when both are present (mirrors models.dev's own
28
+ * convention: it is a distinct override for the specific >200k-token case, not one more tier).
29
+ * A field a tier/override omits falls back to the base flat price, never to `undefined`-as-unknown
30
+ * silently -- a tier that only republishes input/output pricing still inherits the model's real
31
+ * cache prices instead of losing them.
32
+ */
33
+ export function resolveTieredCatalogPrice(pricing: ModelCatalogPricing, contextSizeTokens: number): CacheEconomicsPricing {
34
+ if (pricing.contextOver200k && contextSizeTokens > CATALOG_LONG_CONTEXT_THRESHOLD_TOKENS) {
35
+ return mergeWithBase(pricing, pricing.contextOver200k);
36
+ }
37
+ if (pricing.tiers && pricing.tiers.length > 0) {
38
+ const ascending = [...pricing.tiers].sort((left, right) => left.contextSize - right.contextSize);
39
+ const tier: ModelCatalogPriceTier = ascending.find((candidate) => candidate.contextSize >= contextSizeTokens) ?? ascending.at(-1)!;
40
+ return mergeWithBase(pricing, tier);
41
+ }
42
+ return { input: pricing.input, cacheRead: pricing.cacheRead, cacheWrite: pricing.cacheWrite };
43
+ }
44
+
45
+ /**
46
+ * Best-effort catalog-backed pricing lookup: an unconfigured/unavailable catalog, or a model the
47
+ * catalog doesn't carry, returns null rather than throwing -- cache economics degrades that
48
+ * model's pricing to "unknown" instead of failing the whole query.
49
+ */
50
+ class CatalogCacheEconomicsPricing implements CacheEconomicsPricingLookup {
51
+ constructor(private readonly catalog: ModelCatalogController) {}
52
+
53
+ priceFor(provider: string, model: string, contextSizeTokens: number): CacheEconomicsPricing | null {
54
+ try {
55
+ const result = this.catalog.query({ provider, model, limit: 1 });
56
+ const entry = result.entries[0];
57
+ if (!entry?.pricing) return null;
58
+ return { ...resolveTieredCatalogPrice(entry.pricing, contextSizeTokens), freshness: result.freshness };
59
+ } catch {
60
+ return null;
61
+ }
62
+ }
63
+ }
64
+
65
+ /** cache.economics -- the only operation that combines the metric store's "pi"/"pi-context-snapshot" rows with catalog pricing. */
66
+ export function cacheEconomicsOperations(metrics: MetricStore, catalog: ModelCatalogController): OperationHandlerMap {
67
+ const pricing = new CatalogCacheEconomicsPricing(catalog);
68
+ return {
69
+ "cache.economics": (input) => {
70
+ const since = input.since;
71
+ const until = input.until;
72
+ if (!Number.isSafeInteger(since) || !Number.isSafeInteger(until) || (since as number) < 0 || (until as number) < (since as number)) {
73
+ throw new Error("cache economics requires non-negative ordered integer bounds");
74
+ }
75
+ const usageRows = metrics.query({
76
+ source: "pi",
77
+ since: since as number,
78
+ until: until as number,
79
+ order: "desc",
80
+ limit: CACHE_ECONOMICS_QUERY_LIMIT,
81
+ });
82
+ const snapshotRows = metrics.query({
83
+ source: "pi-context-snapshot",
84
+ metric: "snapshot",
85
+ since: since as number,
86
+ until: until as number,
87
+ order: "desc",
88
+ limit: CACHE_ECONOMICS_QUERY_LIMIT,
89
+ });
90
+ const summary = buildCacheEconomicsSummary(usageRows, snapshotRows, pricing, { since: since as number, until: until as number });
91
+ const rowsTruncated = usageRows.length >= CACHE_ECONOMICS_QUERY_LIMIT || snapshotRows.length >= CACHE_ECONOMICS_QUERY_LIMIT;
92
+ return { ...summary, truncated: summary.truncated || rowsTruncated };
93
+ },
94
+ };
95
+ }
@@ -73,6 +73,8 @@ const WRITE: VehicleIdempotency = { mode: "unsafe" };
73
73
  * available_routes: local-write (every one is a real router.set (or
74
74
  * pause/resume) call, despite router.current_route's read-sounding name
75
75
  * -- confirmed directly against router-operations.ts).
76
+ * - cache.economics: read (a pure projection over already-recorded metric
77
+ * rows plus catalog pricing; never mutates anything).
76
78
  */
77
79
  const OPERATION_META: Record<OperationName, OperationMeta> = {
78
80
  "metrics.record": { description: "Records one metric observation.", effect: "local-write" },
@@ -113,6 +115,11 @@ const OPERATION_META: Record<OperationName, OperationMeta> = {
113
115
  "router.clear_override": { description: "Clears a manual route override.", effect: "local-write" },
114
116
  "router.current_route": { description: "Sets the router's currently-active route.", effect: "local-write" },
115
117
  "router.available_routes": { description: "Sets the router's currently-available routes.", effect: "local-write" },
118
+ "cache.economics": {
119
+ description:
120
+ "Reports prompt-cache economics (read/write cost, savings, break-even) and candidate missed-cache opportunities within a bounded time window.",
121
+ effect: "read",
122
+ },
116
123
  };
117
124
 
118
125
  /** Read effects need only jittor:read; every other effect needs both (writes commonly also read first). */
@@ -2,6 +2,7 @@ import { VehicleRegistry } from "@danypops/vehicle-server";
2
2
  import { createVehicleHttpApp } from "@danypops/vehicle-server/http";
3
3
  import { errorResponse, healthResponse, readyResponse, requireBearerToken } from "@danypops/vehicle-server/rpc-http";
4
4
  import { SERVICE_MAX_BODY_BYTES, SERVICE_MAX_RESPONSE_BYTES } from "../constants.ts";
5
+ import type { CacheEconomicsSummary } from "../observability/cache-economics.ts";
5
6
  import type { ContextDelta, ContextSnapshot } from "../observability/context-delta.ts";
6
7
  import { type ContextSnapshotHistory, MetricContextSnapshotHistory } from "../observability/context-snapshot-history.ts";
7
8
  import type { CompactionDurationEstimate, ContextAssessment } from "../observability/context-telemetry.ts";
@@ -27,6 +28,7 @@ import { routerMutationAuthorizer } from "../sessions/router-authorization.ts";
27
28
  import { DisabledObservationExporter, type ObservationExporter, type ObservationExportStatus } from "../telemetry-export/exporter.ts";
28
29
  import { VERSION } from "../version.ts";
29
30
  import { benchmarkOperations } from "./benchmark-operations.ts";
31
+ import { cacheEconomicsOperations } from "./cache-operations.ts";
30
32
  import { catalogOperations } from "./catalog-operations.ts";
31
33
  import { contextOperations } from "./context-operations.ts";
32
34
  import { exportOperations } from "./export-operations.ts";
@@ -74,6 +76,7 @@ export const EXPECTED_OPERATION_NAMES = [
74
76
  "router.clear_override",
75
77
  "router.current_route",
76
78
  "router.available_routes",
79
+ "cache.economics",
77
80
  ] as const;
78
81
 
79
82
  export type OperationName = (typeof EXPECTED_OPERATION_NAMES)[number];
@@ -117,6 +120,7 @@ export interface OperationInputs {
117
120
  "router.clear_override": RouterScopeInput;
118
121
  "router.current_route": Route & RouterScopeInput;
119
122
  "router.available_routes": { routes: Route[] } & RouterScopeInput;
123
+ "cache.economics": { since: number; until: number };
120
124
  }
121
125
  export interface OperationOutputs {
122
126
  "session.register": RegisterSessionIdentityResult;
@@ -154,6 +158,7 @@ export interface OperationOutputs {
154
158
  "router.clear_override": RouterStatus;
155
159
  "router.current_route": RouterStatus;
156
160
  "router.available_routes": RouterStatus;
161
+ "cache.economics": CacheEconomicsSummary;
157
162
  }
158
163
 
159
164
  export class UnknownOperationError extends Error {}
@@ -286,6 +291,7 @@ export class JittorService {
286
291
  ...routerOperations(router, authorize),
287
292
  ...modelRankingOperations(modelRanker, router, authorize),
288
293
  ...sessionIdentityOperations(sessionIdentity),
294
+ ...cacheEconomicsOperations(metrics, catalog),
289
295
  };
290
296
  this.vehicleRegistry = new VehicleRegistry({
291
297
  name: "jittor",