@cliftonc/finius 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (86) hide show
  1. package/LICENSE +21 -0
  2. package/README.md +147 -0
  3. package/dist/branding.js +28 -0
  4. package/dist/cli/backfill.js +122 -0
  5. package/dist/cli/claude-settings.js +54 -0
  6. package/dist/cli/codex-config.js +60 -0
  7. package/dist/cli/codex.js +97 -0
  8. package/dist/cli/config.js +41 -0
  9. package/dist/cli/doctor.js +159 -0
  10. package/dist/cli/hook.js +70 -0
  11. package/dist/cli/identity.js +163 -0
  12. package/dist/cli/import.js +61 -0
  13. package/dist/cli/index.js +70 -0
  14. package/dist/cli/install.js +23 -0
  15. package/dist/cli/password.js +14 -0
  16. package/dist/cli/serve.js +63 -0
  17. package/dist/cli/setup.js +314 -0
  18. package/dist/cli/ui.js +15 -0
  19. package/dist/client/assets/TranscriptView-CBf7-4Bo.css +1 -0
  20. package/dist/client/assets/TranscriptView-CLCPX5bI.js +194 -0
  21. package/dist/client/assets/TranscriptView-D056GDHO.js +194 -0
  22. package/dist/client/assets/TranscriptView-MIgsAwMN.js +194 -0
  23. package/dist/client/assets/index-6OIY_8fO.css +1 -0
  24. package/dist/client/assets/index-9aN8py7_.js +1 -0
  25. package/dist/client/assets/index-B-sjMmTS.js +1636 -0
  26. package/dist/client/assets/index-B4HbP3X6.js +1 -0
  27. package/dist/client/assets/index-B9wgN1BV.js +1636 -0
  28. package/dist/client/assets/index-BHlFz1Th.js +1652 -0
  29. package/dist/client/assets/index-BJyvYca7.js +1636 -0
  30. package/dist/client/assets/index-BKBTeJLz.js +1 -0
  31. package/dist/client/assets/index-BN6CbirS.js +1444 -0
  32. package/dist/client/assets/index-BW4_7xR6.js +1460 -0
  33. package/dist/client/assets/index-BaLElA30.js +1 -0
  34. package/dist/client/assets/index-BaQ02V5d.css +1 -0
  35. package/dist/client/assets/index-Bh0dgUU-.js +1636 -0
  36. package/dist/client/assets/index-Bie86XRc.js +1 -0
  37. package/dist/client/assets/index-Bijt5al-.css +1 -0
  38. package/dist/client/assets/index-BikJP2HS.js +1636 -0
  39. package/dist/client/assets/index-BkwrvP-J.js +1 -0
  40. package/dist/client/assets/index-BwVuUJSv.js +1 -0
  41. package/dist/client/assets/index-BweXI4-D.css +1 -0
  42. package/dist/client/assets/index-BwqdHcDE.js +1 -0
  43. package/dist/client/assets/index-C-Z0w-tQ.js +1652 -0
  44. package/dist/client/assets/index-C2RmKzem.js +1636 -0
  45. package/dist/client/assets/index-CHz-iKIQ.js +1 -0
  46. package/dist/client/assets/index-CIGl5oW_.js +1646 -0
  47. package/dist/client/assets/index-CVYmd4Bm.js +1465 -0
  48. package/dist/client/assets/index-Ca9UVGK1.js +1 -0
  49. package/dist/client/assets/index-CeWDkmJN.js +1 -0
  50. package/dist/client/assets/index-CpsNq0zm.css +1 -0
  51. package/dist/client/assets/index-CrUS6abD.css +1 -0
  52. package/dist/client/assets/index-Ctq8vj2Z.js +1 -0
  53. package/dist/client/assets/index-D1ktp0pp.js +1 -0
  54. package/dist/client/assets/index-D3BoYpFi.css +1 -0
  55. package/dist/client/assets/index-D59GxlrT.js +1636 -0
  56. package/dist/client/assets/index-D5Wkww8x.css +1 -0
  57. package/dist/client/assets/index-DC94jMGe.js +1 -0
  58. package/dist/client/assets/index-DFcIBkv1.js +1652 -0
  59. package/dist/client/assets/index-DmKj5Jqc.css +1 -0
  60. package/dist/client/assets/index-Dx52i05H.js +1465 -0
  61. package/dist/client/assets/index-L3GnPzmU.css +1 -0
  62. package/dist/client/assets/index-OZADsKet.js +1652 -0
  63. package/dist/client/assets/index-Qt124kj1.js +1652 -0
  64. package/dist/client/assets/index-nHzwQ3EM.js +1 -0
  65. package/dist/client/assets/index-s9Mg6LTO.js +1 -0
  66. package/dist/client/assets/index-ye8oxz8P.js +1 -0
  67. package/dist/client/assets/index-yqJS7tUY.css +1 -0
  68. package/dist/client/favicon.svg +35 -0
  69. package/dist/client/finius-dashboard.png +0 -0
  70. package/dist/client/index.html +38 -0
  71. package/dist/server/app.js +285 -0
  72. package/dist/server/claude.js +124 -0
  73. package/dist/server/codex.js +94 -0
  74. package/dist/server/events.js +12 -0
  75. package/dist/server/index.js +119 -0
  76. package/dist/server/otel.js +231 -0
  77. package/dist/server/pricing-backfill.js +41 -0
  78. package/dist/server/pricing.js +138 -0
  79. package/dist/server/queue.js +35 -0
  80. package/dist/server/storage/blob.js +17 -0
  81. package/dist/server/storage/query-helpers.js +104 -0
  82. package/dist/server/storage/sqlite.js +1167 -0
  83. package/dist/server/transcripts.js +46 -0
  84. package/dist/server/types.js +1 -0
  85. package/dist/shared/api-types.js +1 -0
  86. package/package.json +72 -0
@@ -0,0 +1,119 @@
1
+ import { serve } from "@hono/node-server";
2
+ import { serveStatic } from "@hono/node-server/serve-static";
3
+ import { createHash } from "node:crypto";
4
+ import { existsSync } from "node:fs";
5
+ import { dirname, join, resolve } from "node:path";
6
+ import { fileURLToPath } from "node:url";
7
+ import { banner, panel, pc } from "../branding.js";
8
+ import { createApp } from "./app.js";
9
+ import { EventBus } from "./events.js";
10
+ import { normalizeLiteLlm } from "./pricing.js";
11
+ import { githubSnapshot } from "./pricing-backfill.js";
12
+ import { LocalBlobStore } from "./storage/blob.js";
13
+ import { SqliteStorageAdapter } from "./storage/sqlite.js";
14
+ // LiteLLM's community price feed: per-model input/output/cache token costs for Anthropic + OpenAI
15
+ // (and many more). Override with FINIUS_PRICING_URL; disable the fetch with FINIUS_PRICING_FETCH=off.
16
+ const DEFAULT_PRICING_URL = "https://raw.githubusercontent.com/BerriAI/litellm/main/model_prices_and_context_window.json";
17
+ export function startServer(options = {}) {
18
+ const port = options.port ?? Number(process.env.PORT ?? 8787);
19
+ const hostname = options.hostname ?? process.env.FINIUS_HOST ?? "127.0.0.1";
20
+ const dbPath = options.dbPath ?? process.env.FINIUS_DB_PATH ?? "data/finius.sqlite";
21
+ const storeRawPayloads = options.storeRawPayloads ?? (process.env.FINIUS_RAW_PAYLOADS ?? "retain") !== "off";
22
+ const rawRetentionDaysRaw = options.rawRetentionDays ?? Number(process.env.FINIUS_RAW_RETENTION_DAYS ?? 7);
23
+ const rawRetentionDays = Number.isFinite(rawRetentionDaysRaw) ? rawRetentionDaysRaw : 7;
24
+ const blobDir = options.blobDir ?? process.env.FINIUS_BLOB_DIR;
25
+ const blob = blobDir ? new LocalBlobStore(resolve(blobDir)) : undefined;
26
+ const cronToken = options.cronToken ?? process.env.FINIUS_CRON_TOKEN;
27
+ const authSecret = options.authSecret ?? process.env.FINIUS_AUTH_PASSWORD;
28
+ const initialAuthToken = options.initialAuthToken;
29
+ const storage = new SqliteStorageAdapter(resolve(dbPath), { storeRawPayloads, blob });
30
+ if (authSecret && initialAuthToken)
31
+ seedInitialAuthToken(storage, initialAuthToken);
32
+ const events = new EventBus();
33
+ const app = createApp({ storage, events, cronToken, rawRetentionDays, authSecret });
34
+ // Resolve the built client relative to this module so the UI is served no matter the cwd
35
+ // (e.g. when launched via `npx @cliftonc/finius serve`). Falls back to a cwd-relative path for
36
+ // repo-root invocations like `npm start`.
37
+ const moduleDir = dirname(fileURLToPath(import.meta.url));
38
+ const clientDist = [join(moduleDir, "..", "client"), resolve("dist/client")].find((dir) => existsSync(dir));
39
+ if (clientDist) {
40
+ app.use("/*", serveStatic({ root: clientDist }));
41
+ app.get("*", serveStatic({ path: join(clientDist, "index.html") }));
42
+ }
43
+ // Effective transcript location: the explicit blob dir, else the adapter's default of
44
+ // <db-dir>/transcripts. Computed here only so we can report it on startup.
45
+ const resolvedDb = resolve(dbPath);
46
+ const transcriptsDir = blobDir ? resolve(blobDir) : join(dirname(resolvedDb), "transcripts");
47
+ serve({ fetch: app.fetch, hostname, port }, (info) => {
48
+ const url = `http://${hostname}:${info.port}`;
49
+ banner("serve");
50
+ process.stdout.write(`\n ${pc.green("●")} ${pc.bold("Finius is live")} ${pc.dim("·")} ${pc.cyan(url)}\n\n`);
51
+ process.stdout.write(`${panel([
52
+ ["Dashboard", clientDist ? pc.cyan(url) : pc.yellow("not built — run `npm run build` to serve the UI")],
53
+ ["API", `${url}/api`],
54
+ ["Database", pc.dim(resolvedDb)],
55
+ ["Transcripts", pc.dim(transcriptsDir)],
56
+ ["Raw payloads", storeRawPayloads ? `retained, pruned after ${rawRetentionDays}d` : pc.dim("off")],
57
+ ["Auth", authSecret ? `${pc.green("secure mode")} ${pc.dim("· log in with password:")} ${pc.bold(authSecret)}` : pc.dim("open (no auth)")]
58
+ ])}\n\n`);
59
+ });
60
+ // Wire historical pricing backfill: when a JSONL import lands on a day we have no price for, the
61
+ // processing queue fetches that day's rates from LiteLLM's git history. Disabled with
62
+ // FINIUS_PRICING_FETCH=off (keeps the server fully offline).
63
+ if ((process.env.FINIUS_PRICING_FETCH ?? "on") !== "off") {
64
+ storage.setHistoricalPriceFetcher(githubSnapshot);
65
+ }
66
+ // Refresh model pricing in the background so we can compute cost for agents that don't report it
67
+ // (Codex, JSONL Claude). Non-blocking and offline-safe: on failure we keep whatever rows are
68
+ // already cached in model_prices and still recompute from them.
69
+ void syncPricing(storage);
70
+ const shutdown = async () => {
71
+ const timeout = new Promise((resolve) => windowlessSetTimeout(resolve, 5_000));
72
+ await Promise.race([storage.settleIngest(), timeout]);
73
+ storage.close();
74
+ process.exit(0);
75
+ };
76
+ process.on("SIGINT", shutdown);
77
+ process.on("SIGTERM", shutdown);
78
+ return { storage, events, clientDist, hostname, port };
79
+ }
80
+ function windowlessSetTimeout(callback, delay) {
81
+ return globalThis.setTimeout(callback, delay);
82
+ }
83
+ async function syncPricing(storage) {
84
+ if ((process.env.FINIUS_PRICING_FETCH ?? "on") === "off")
85
+ return;
86
+ const url = process.env.FINIUS_PRICING_URL ?? DEFAULT_PRICING_URL;
87
+ try {
88
+ const res = await fetch(url);
89
+ if (!res.ok)
90
+ throw new Error(`HTTP ${res.status}`);
91
+ const prices = normalizeLiteLlm(await res.json(), Date.now());
92
+ if (prices.length === 0)
93
+ throw new Error("no usable prices in feed");
94
+ await storage.importPricing(prices);
95
+ const { costPoints } = await storage.recomputeComputedCost();
96
+ console.log(panel([["Pricing", `${prices.length} models synced ${pc.dim(`(recomputed ${costPoints} cost points)`)}`]]));
97
+ }
98
+ catch (err) {
99
+ console.warn(` ${pc.yellow("▲")} pricing sync failed ${pc.dim(`(${err.message})`)} — using cached pricing`);
100
+ // Re-apply whatever pricing is already cached so a restart still reflects locally-imported rows.
101
+ try {
102
+ await storage.recomputeComputedCost();
103
+ }
104
+ catch {
105
+ /* ignore */
106
+ }
107
+ }
108
+ }
109
+ function seedInitialAuthToken(storage, token) {
110
+ const tokenHash = createHash("sha256").update(token).digest("hex");
111
+ if (storage.findAuthToken(tokenHash))
112
+ return;
113
+ storage.createAuthToken(tokenHash, "owner", Date.now());
114
+ }
115
+ // Auto-start when executed directly (`node dist/server/index.js`), but not when this module is
116
+ // imported by the CLI's `serve` command.
117
+ const invokedDirectly = !!process.argv[1] && resolve(process.argv[1]) === fileURLToPath(import.meta.url);
118
+ if (invokedDirectly)
119
+ startServer();
@@ -0,0 +1,231 @@
1
+ import { createHash } from "node:crypto";
2
+ const TOKEN_METRIC = "claude_code.token.usage";
3
+ const COST_METRIC = "claude_code.cost.usage";
4
+ const LINES_METRIC = "claude_code.lines_of_code.count";
5
+ const DECISION_METRIC = "claude_code.code_edit_tool.decision";
6
+ const ACTIVE_TIME_METRIC = "claude_code.active_time.total";
7
+ const SESSION_METRIC = "claude_code.session.count";
8
+ const PR_METRIC = "claude_code.pull_request.count";
9
+ const COMMIT_METRIC = "claude_code.commit.count";
10
+ // Maps an OTLP metric to our (kind, sub-type) for the metric_points table. Returns null for
11
+ // metrics we don't aggregate (those data points are dropped).
12
+ function classifyMetric(metricName, attributes) {
13
+ switch (metricName) {
14
+ case TOKEN_METRIC:
15
+ return { kind: "tokens", tokenType: normalizeTokenType(stringAttr(attributes, "type")) };
16
+ case COST_METRIC:
17
+ return { kind: "cost", tokenType: null };
18
+ case LINES_METRIC:
19
+ return { kind: "lines", tokenType: stringAttr(attributes, "type") ?? null };
20
+ case DECISION_METRIC:
21
+ return { kind: "decision", tokenType: stringAttr(attributes, "decision") ?? null };
22
+ case ACTIVE_TIME_METRIC:
23
+ return { kind: "active_time", tokenType: stringAttr(attributes, "type") ?? null };
24
+ case SESSION_METRIC:
25
+ return { kind: "session", tokenType: stringAttr(attributes, "start_type") ?? null };
26
+ case PR_METRIC:
27
+ return { kind: "pull_request", tokenType: null };
28
+ case COMMIT_METRIC:
29
+ return { kind: "commit", tokenType: null };
30
+ default:
31
+ return null;
32
+ }
33
+ }
34
+ export function stableHash(signal, batch) {
35
+ return createHash("sha256").update(signal).update(JSON.stringify(batch)).digest("hex");
36
+ }
37
+ export function attributesToObject(attributes) {
38
+ const out = {};
39
+ for (const attr of attributes ?? []) {
40
+ out[attr.key] = decodeAttributeValue(attr.value);
41
+ }
42
+ return out;
43
+ }
44
+ export function decodeAttributeValue(value) {
45
+ if (!value)
46
+ return null;
47
+ if ("stringValue" in value)
48
+ return value.stringValue;
49
+ if ("intValue" in value)
50
+ return Number(value.intValue);
51
+ if ("doubleValue" in value)
52
+ return value.doubleValue;
53
+ if ("boolValue" in value)
54
+ return value.boolValue;
55
+ if ("arrayValue" in value)
56
+ return value.arrayValue?.values?.map(decodeAttributeValue) ?? [];
57
+ if ("kvlistValue" in value) {
58
+ return attributesToObject(value.kvlistValue?.values);
59
+ }
60
+ return null;
61
+ }
62
+ export function parseOtelMetricPoints(batch, source = "claude-code") {
63
+ const points = [];
64
+ const resourceMetrics = asArray(batch?.resourceMetrics);
65
+ for (const resourceMetric of resourceMetrics) {
66
+ const resourceAttrs = attributesToObject(resourceMetric.resource?.attributes);
67
+ const scopeMetrics = asArray(resourceMetric.scopeMetrics);
68
+ for (const scopeMetric of scopeMetrics) {
69
+ const metrics = asArray(scopeMetric.metrics);
70
+ for (const metric of metrics) {
71
+ const metricName = String(metric.name ?? "");
72
+ const dataPoints = getMetricDataPoints(metric);
73
+ for (const dataPoint of dataPoints) {
74
+ const dataPointAttrs = attributesToObject(dataPoint.attributes);
75
+ const attributes = { ...resourceAttrs, ...dataPointAttrs };
76
+ const classified = classifyMetric(metricName, attributes);
77
+ if (!classified)
78
+ continue;
79
+ const sessionId = stringAttr(attributes, "session.id") ?? stringAttr(attributes, "session_id") ?? "unknown-session";
80
+ const timestamp = Number(unixNanoToMs(dataPoint.timeUnixNano ??
81
+ dataPoint.startTimeUnixNano) ?? Date.now());
82
+ const value = numericValue(dataPoint);
83
+ if (!Number.isFinite(value))
84
+ continue;
85
+ points.push({
86
+ source,
87
+ signal: "otlp_metrics",
88
+ sessionId,
89
+ userId: stringAttr(attributes, "user.id") ?? stringAttr(attributes, "enduser.id"),
90
+ userEmail: stringAttr(attributes, "user.email"),
91
+ userAccountId: stringAttr(attributes, "user.account_id") ?? stringAttr(attributes, "user.account_uuid"),
92
+ model: stringAttr(attributes, "model"),
93
+ metricName,
94
+ kind: classified.kind,
95
+ tokenType: classified.tokenType,
96
+ value,
97
+ unit: metric.unit ?? null,
98
+ timestamp,
99
+ attributes
100
+ });
101
+ }
102
+ }
103
+ }
104
+ }
105
+ return points;
106
+ }
107
+ /**
108
+ * Decodes EVERY metric data point in a batch (all metric names, not just token/cost) into a flat
109
+ * list of records, so the raw-events log captures the full telemetry surface. Pure/side-effect-free.
110
+ */
111
+ export function parseOtelMetricRecords(batch, source = "claude-code") {
112
+ const records = [];
113
+ const resourceMetrics = asArray(batch?.resourceMetrics);
114
+ for (const resourceMetric of resourceMetrics) {
115
+ const resourceAttrs = attributesToObject(resourceMetric.resource?.attributes);
116
+ for (const scopeMetric of asArray(resourceMetric.scopeMetrics)) {
117
+ for (const metric of asArray(scopeMetric.metrics)) {
118
+ const m = metric;
119
+ const kind = m.sum ? "sum" : m.gauge ? "gauge" : m.histogram ? "histogram" : "unknown";
120
+ const dataPoints = asArray(m.sum?.dataPoints ?? m.gauge?.dataPoints ?? m.histogram?.dataPoints);
121
+ for (const dataPoint of dataPoints) {
122
+ const dp = dataPoint;
123
+ const rawValue = dp.asDouble ?? (dp.asInt !== undefined ? Number(dp.asInt) : dp.count !== undefined ? Number(dp.count) : null);
124
+ records.push({
125
+ source,
126
+ metricName: String(m.name ?? ""),
127
+ unit: m.unit ?? null,
128
+ kind,
129
+ temporality: m.sum?.aggregationTemporality ?? null,
130
+ isMonotonic: m.sum?.isMonotonic ?? null,
131
+ value: rawValue !== null && Number.isFinite(rawValue) ? rawValue : null,
132
+ startTimeUnixNano: dp.startTimeUnixNano ?? null,
133
+ timeUnixNano: dp.timeUnixNano ?? null,
134
+ timestamp: Number(unixNanoToMs(dp.timeUnixNano ?? dp.startTimeUnixNano) ?? Date.now()),
135
+ attributes: { ...resourceAttrs, ...attributesToObject(dp.attributes) }
136
+ });
137
+ }
138
+ }
139
+ }
140
+ }
141
+ return records;
142
+ }
143
+ // Flattens an OTLP/JSON logs batch into structured records, merging resource + record attributes and
144
+ // decoding the body. Codex's native telemetry is logs-only, so this is the seam through which we
145
+ // capture (and, later, parse) what it sends. Pure / side-effect-free.
146
+ //
147
+ // Codex's native telemetry is logs-only, so this is the visibility path for it: records are indexed
148
+ // into log_events for inspection (GET /api/logs/events), but token/cost for Codex come from the
149
+ // authoritative rollout-JSONL path (`codex-cli-jsonl`), not from these logs — the OTel `codex.sse_event`
150
+ // stream is a partial, cost-less subset of the rollout, so ingesting it as metrics would undercount.
151
+ export function parseOtelLogRecords(batch) {
152
+ const records = [];
153
+ const resourceLogs = asArray(batch?.resourceLogs);
154
+ for (const resourceLog of resourceLogs) {
155
+ const resourceAttrs = attributesToObject(resourceLog.resource?.attributes);
156
+ for (const scopeLog of asArray(resourceLog.scopeLogs)) {
157
+ for (const logRecord of asArray(scopeLog.logRecords)) {
158
+ const lr = logRecord;
159
+ const attributes = { ...resourceAttrs, ...attributesToObject(lr.attributes) };
160
+ // Event name: the semantic `event.name` attribute FIRST (what both Claude Code and Codex
161
+ // actually set to the real id), then the loose top-level `eventName`/`name`. Codex's tracing
162
+ // appender pollutes the top-level `eventName` with a Rust source location (e.g.
163
+ // "event otel/src/.../session_telemetry.rs:778") and carries the true id (`codex.sse_event`, …)
164
+ // only in the attribute — so attribute-first is what keeps records from being mislabeled.
165
+ const eventName = stringAttr(attributes, "event.name") ||
166
+ (typeof lr.eventName === "string" && lr.eventName) ||
167
+ (typeof lr.name === "string" && lr.name) ||
168
+ null;
169
+ records.push({
170
+ eventName,
171
+ severityText: typeof lr.severityText === "string" ? lr.severityText : null,
172
+ timestamp: Number(unixNanoToMs(lr.timeUnixNano ?? lr.observedTimeUnixNano) ?? Date.now()),
173
+ sessionId: stringAttr(attributes, "session.id") ??
174
+ stringAttr(attributes, "session_id") ??
175
+ stringAttr(attributes, "conversation.id") ??
176
+ null,
177
+ attributes,
178
+ body: decodeAttributeValue(lr.body)
179
+ });
180
+ }
181
+ }
182
+ }
183
+ return records;
184
+ }
185
+ export function preferredIdentity(point) {
186
+ return point.userEmail ?? point.userAccountId ?? point.userId ?? "unknown";
187
+ }
188
+ function getMetricDataPoints(metric) {
189
+ const m = metric;
190
+ return asArray(m.sum?.dataPoints ?? m.gauge?.dataPoints ?? m.histogram?.dataPoints);
191
+ }
192
+ function numericValue(dataPoint) {
193
+ const point = dataPoint;
194
+ if (point.asDouble !== undefined)
195
+ return Number(point.asDouble);
196
+ if (point.asInt !== undefined)
197
+ return Number(point.asInt);
198
+ if (point.value !== undefined)
199
+ return Number(point.value);
200
+ return NaN;
201
+ }
202
+ function stringAttr(attributes, name) {
203
+ const value = attributes[name];
204
+ return typeof value === "string" && value.length > 0 ? value : undefined;
205
+ }
206
+ function unixNanoToMs(value) {
207
+ if (value === undefined)
208
+ return undefined;
209
+ const numeric = Number(value);
210
+ if (!Number.isFinite(numeric))
211
+ return undefined;
212
+ return numeric > 10_000_000_000_000 ? Math.floor(numeric / 1_000_000) : numeric;
213
+ }
214
+ function normalizeTokenType(type) {
215
+ if (!type)
216
+ return "total";
217
+ // Claude Code emits camelCase types (cacheCreation, cacheRead) as well as snake/kebab variants.
218
+ const normalized = type.replace(/-/g, "_").toLowerCase();
219
+ if (normalized.includes("cache") && normalized.includes("creation"))
220
+ return "cache_creation";
221
+ if (normalized.includes("cache") && normalized.includes("read"))
222
+ return "cache_read";
223
+ if (normalized.includes("output"))
224
+ return "output";
225
+ if (normalized.includes("input"))
226
+ return "input";
227
+ return normalized;
228
+ }
229
+ function asArray(value) {
230
+ return Array.isArray(value) ? value : [];
231
+ }
@@ -0,0 +1,41 @@
1
+ import { normalizeLiteLlm } from "./pricing.js";
2
+ // Historical-pricing fetcher. When a JSONL import carries usage on a day we have no price for, the
3
+ // processing step asks for the prices in effect on that day; we reconstruct them from LiteLLM's git
4
+ // history on GitHub (it only publishes *current* pricing, but every past version lives in its commits).
5
+ // This is the I/O the storage layer stays out of — it's injected via setHistoricalPriceFetcher.
6
+ const REPO = "BerriAI/litellm";
7
+ const FILE = "model_prices_and_context_window.json";
8
+ // Default fetcher: find the latest commit to the price file at or before the day, fetch that exact
9
+ // version, and stamp its prices with the commit date as effectiveDate so they apply from then on.
10
+ export const githubSnapshot = async (dayIso) => {
11
+ const commit = await latestCommit(`${dayIso}T23:59:59Z`);
12
+ if (!commit)
13
+ return null;
14
+ const json = await fetchJsonAt(commit.sha);
15
+ return normalizeLiteLlm(json, commit.date);
16
+ };
17
+ async function latestCommit(untilIso) {
18
+ const url = `https://api.github.com/repos/${REPO}/commits?path=${encodeURIComponent(FILE)}&until=${encodeURIComponent(untilIso)}&per_page=1`;
19
+ const res = await fetch(url, { headers: githubHeaders() });
20
+ if (!res.ok)
21
+ throw new Error(`GitHub commits HTTP ${res.status}`);
22
+ const commits = (await res.json());
23
+ if (!Array.isArray(commits) || commits.length === 0)
24
+ return null;
25
+ const date = Date.parse(commits[0].commit?.committer?.date ?? untilIso);
26
+ return { sha: commits[0].sha, date: Number.isFinite(date) ? date : Date.parse(untilIso) };
27
+ }
28
+ async function fetchJsonAt(sha) {
29
+ const res = await fetch(`https://raw.githubusercontent.com/${REPO}/${sha}/${FILE}`);
30
+ if (!res.ok)
31
+ throw new Error(`raw file HTTP ${res.status}`);
32
+ return res.json();
33
+ }
34
+ function githubHeaders() {
35
+ const headers = { Accept: "application/vnd.github+json", "User-Agent": "finius" };
36
+ // Optional: raises the GitHub rate limit (60/hr → 5000/hr) for large backfills.
37
+ const token = process.env.FINIUS_GITHUB_TOKEN;
38
+ if (token)
39
+ headers.Authorization = `Bearer ${token}`;
40
+ return headers;
41
+ }
@@ -0,0 +1,138 @@
1
+ // Pure pricing helpers: normalize an external price feed (LiteLLM) into our ModelPrice rows, match a
2
+ // reported model id to a price, and synthesize cost metric points from token points. Side-effect-free
3
+ // so it unit-tests cleanly and is shared by the ingest path and the recompute/backfill path.
4
+ //
5
+ // The metric name used for cost we compute ourselves. It stays kind:"cost" so every existing
6
+ // `SUM(CASE WHEN kind='cost' ...)` picks it up, but is distinguishable from an agent-reported cost
7
+ // (Claude's `claude_code.cost.usage`) in the source-filtered comparison views.
8
+ export const COMPUTED_COST_METRIC = "finius.cost.computed";
9
+ // Token type (our metric_points.token_type) → the ModelPrice rate field it's priced by. Anything not
10
+ // listed here (e.g. the "total" fallback) is intentionally unpriced so we never double-count.
11
+ const RATE_BY_TOKEN_TYPE = {
12
+ input: "inputPerToken",
13
+ output: "outputPerToken",
14
+ cache_read: "cacheReadPerToken",
15
+ cache_creation: "cacheCreationPerToken"
16
+ };
17
+ export function normalizeLiteLlm(json, effectiveDate = 0) {
18
+ if (!json || typeof json !== "object")
19
+ return [];
20
+ const prices = [];
21
+ for (const [model, raw] of Object.entries(json)) {
22
+ if (model === "sample_spec" || !raw || typeof raw !== "object")
23
+ continue;
24
+ const entry = raw;
25
+ const input = numberOr0(entry.input_cost_per_token);
26
+ const output = numberOr0(entry.output_cost_per_token);
27
+ const cacheRead = numberOr0(entry.cache_read_input_token_cost);
28
+ const cacheCreation = numberOr0(entry.cache_creation_input_token_cost);
29
+ // Skip entries with no usable token rates (LiteLLM also lists embedding/audio/etc. models).
30
+ if (input === 0 && output === 0 && cacheRead === 0 && cacheCreation === 0)
31
+ continue;
32
+ prices.push({
33
+ model,
34
+ provider: typeof entry.litellm_provider === "string" ? entry.litellm_provider : inferProvider(model),
35
+ inputPerToken: input,
36
+ outputPerToken: output,
37
+ // LiteLLM omits cache-read for many models; fall back to the input rate so cache reads aren't free.
38
+ cacheReadPerToken: cacheRead || input,
39
+ cacheCreationPerToken: cacheCreation || input,
40
+ effectiveDate
41
+ });
42
+ }
43
+ return prices;
44
+ }
45
+ export function indexPrices(prices) {
46
+ const index = new Map();
47
+ const add = (key, price) => {
48
+ const list = index.get(key);
49
+ if (list)
50
+ list.push(price);
51
+ else
52
+ index.set(key, [price]);
53
+ };
54
+ for (const price of prices) {
55
+ const key = normalizeKey(price.model);
56
+ add(key, price);
57
+ // Also alias under the date-stripped key so a dated feed entry (claude-...-20250929) still
58
+ // matches an undated reported id, and vice versa.
59
+ const undated = key.replace(/-\d{8}$/, "");
60
+ if (undated !== key)
61
+ add(undated, price);
62
+ }
63
+ for (const list of index.values())
64
+ list.sort((a, b) => b.effectiveDate - a.effectiveDate);
65
+ return index;
66
+ }
67
+ // Resolve a reported model id to its candidate price rows. Tries the exact (normalized) id, then the
68
+ // id with any trailing `-YYYYMMDD` date suffix stripped (Claude ships dated and undated variants).
69
+ function matchKeys(model) {
70
+ const exact = normalizeKey(model);
71
+ const undated = exact.replace(/-\d{8}$/, "");
72
+ return undated !== exact ? [exact, undated] : [exact];
73
+ }
74
+ export function priceFor(model, timestamp, index) {
75
+ if (!model)
76
+ return undefined;
77
+ for (const key of matchKeys(model)) {
78
+ const list = index.get(key);
79
+ if (!list || list.length === 0)
80
+ continue;
81
+ // Rows are sorted newest-first; take the newest whose effectiveDate is at or before the usage.
82
+ const inEffect = list.find((p) => p.effectiveDate <= timestamp);
83
+ return inEffect ?? list[list.length - 1]; // else the oldest known rate (better than $0)
84
+ }
85
+ return undefined;
86
+ }
87
+ // Turn token points into synthesized cost points. One cost point per priced token point, copying the
88
+ // source/signal/session/model/timestamp so the existing aggregation + jsonlWins precedence treat the
89
+ // cost exactly like its tokens (and shadow it when the session has authoritative ingested cost).
90
+ export function computeCostPoints(tokenPoints, index) {
91
+ const points = [];
92
+ for (const point of tokenPoints) {
93
+ if (point.kind !== "tokens" || !point.tokenType)
94
+ continue;
95
+ const rateField = RATE_BY_TOKEN_TYPE[point.tokenType];
96
+ if (!rateField)
97
+ continue;
98
+ const price = priceFor(point.model, point.timestamp, index);
99
+ if (!price)
100
+ continue;
101
+ const cost = point.value * price[rateField];
102
+ if (!(cost > 0))
103
+ continue;
104
+ points.push({
105
+ source: point.source,
106
+ signal: point.signal,
107
+ sessionId: point.sessionId,
108
+ userId: point.userId ?? null,
109
+ userEmail: point.userEmail ?? null,
110
+ userAccountId: point.userAccountId ?? null,
111
+ model: point.model ?? null,
112
+ metricName: COMPUTED_COST_METRIC,
113
+ kind: "cost",
114
+ tokenType: null,
115
+ value: cost,
116
+ unit: "USD",
117
+ timestamp: point.timestamp
118
+ });
119
+ }
120
+ return points;
121
+ }
122
+ function normalizeKey(model) {
123
+ // Drop a vendor prefix LiteLLM sometimes carries (e.g. "anthropic/claude-...", "openai/gpt-...").
124
+ const slash = model.lastIndexOf("/");
125
+ return (slash === -1 ? model : model.slice(slash + 1)).toLowerCase();
126
+ }
127
+ function inferProvider(model) {
128
+ const m = model.toLowerCase();
129
+ if (m.includes("claude"))
130
+ return "anthropic";
131
+ if (m.startsWith("gpt") || m.startsWith("o1") || m.startsWith("o3") || m.startsWith("o4") || m.includes("codex"))
132
+ return "openai";
133
+ return null;
134
+ }
135
+ function numberOr0(value) {
136
+ const n = Number(value);
137
+ return Number.isFinite(n) ? n : 0;
138
+ }
@@ -0,0 +1,35 @@
1
+ // The single in-process processing queue that sits behind JSONL uploads. Uploads save their blob and
2
+ // enqueue a task immediately; this drains them ONE AT A TIME in the background (parse → metrics → cost
3
+ // → historical-pricing fetch). Deliberately minimal: no persistence, no priorities, one queue. A task
4
+ // that throws is logged and skipped so one bad transcript can't wedge the queue.
5
+ export class SerialQueue {
6
+ queue = [];
7
+ active = null;
8
+ enqueue(task) {
9
+ this.queue.push(task);
10
+ if (!this.active) {
11
+ this.active = this.drain().finally(() => {
12
+ this.active = null;
13
+ });
14
+ }
15
+ }
16
+ // Resolves once the queue is fully drained — used by tests and graceful shutdown.
17
+ async settle() {
18
+ while (this.active)
19
+ await this.active;
20
+ }
21
+ get size() {
22
+ return this.queue.length + (this.active ? 1 : 0);
23
+ }
24
+ async drain() {
25
+ while (this.queue.length) {
26
+ const task = this.queue.shift();
27
+ try {
28
+ await task();
29
+ }
30
+ catch (err) {
31
+ console.warn(`[finius] processing job failed: ${err.message}`);
32
+ }
33
+ }
34
+ }
35
+ }
@@ -0,0 +1,17 @@
1
+ import { existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs";
2
+ import { dirname, join } from "node:path";
3
+ export class LocalBlobStore {
4
+ dir;
5
+ constructor(dir) {
6
+ this.dir = dir;
7
+ }
8
+ async save(key, bytes) {
9
+ const path = join(this.dir, key);
10
+ mkdirSync(dirname(path), { recursive: true });
11
+ writeFileSync(path, bytes);
12
+ }
13
+ async read(key) {
14
+ const path = join(this.dir, key);
15
+ return existsSync(path) ? readFileSync(path) : null;
16
+ }
17
+ }