@cad0p/pi-usage 0.11.3 → 0.11.4

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,1511 @@
1
+ /**
2
+ * Local token statistics from Pi's own session files.
3
+ *
4
+ * Every assistant message Pi records carries the provider, the model and a
5
+ * usage block, so the tokens a subscription actually burned are already on
6
+ * disk. This module folds those messages into the per-family shape Omarchy's
7
+ * Agents panel renders ("tokens by day", "tokens by model", today's totals),
8
+ * and does it at a bounded cost: the first run streams every session file once,
9
+ * later runs read only the bytes appended since the last scan.
10
+ *
11
+ * Two stores, two lifetimes:
12
+ * - the disposable per-file cache under `~/.cache/pi-quota/local-sessions.json`
13
+ * holds the (size, mtimeMs, inode, offset) of every session file plus that
14
+ * file's aggregates for OPEN days only (normally just today), so an append
15
+ * is read once and nothing is ever parsed twice;
16
+ * - the durable history under `${PI_CODING_AGENT_DIR:-~/.pi/agent}/pi-usage/`
17
+ * (see history.js) holds every closed day. Freezing is one-way: a frozen
18
+ * day is never recomputed from a rescan, never rewritten by a session
19
+ * deletion and never lost with the cache.
20
+ *
21
+ * Hard rules enforced here:
22
+ * - session files are opened strictly read-only and never modified;
23
+ * - nothing but aggregate counts (tokens, prompts, dates, model names) and
24
+ * the file keys needed to scan incrementally reaches a cache, a record or
25
+ * the report, so no prompt text, tool output or credential is ever stored;
26
+ * - the scan is best-effort: a missing root, an unreadable file or a corrupt
27
+ * cache/history degrades to fewer numbers and a warning, never to a
28
+ * provider failure.
29
+ *
30
+ * Duplicates: a fork or rewind is a new session file, so in principle the same
31
+ * assistant message could be counted twice. Measured on a real store (603 files,
32
+ * 185,626 assistant messages) only 11 message ids appeared in more than one file
33
+ * (0.006%), and none was a copied history. An id index would cost megabytes of
34
+ * memory to correct a handful of tokens, and no bounded window can recognise a
35
+ * fork that replays an old history, so every message is counted once per line
36
+ * read. If Pi ever starts copying whole histories into forks, that overstates
37
+ * usage (the conservative direction) instead of hiding it.
38
+ */
39
+
40
+ import { createHash } from "node:crypto";
41
+ import {
42
+ closeSync,
43
+ createReadStream,
44
+ mkdirSync,
45
+ openSync,
46
+ readSync,
47
+ readdirSync,
48
+ readFileSync,
49
+ realpathSync,
50
+ renameSync,
51
+ rmSync,
52
+ statSync,
53
+ writeFileSync,
54
+ } from "node:fs";
55
+ import { homedir } from "node:os";
56
+ import { dirname, join } from "node:path";
57
+
58
+ import { FAMILY_BY_PROVIDER, resolveProviderSlot } from "../auth/pi-auth.js";
59
+ import { redact } from "../http.js";
60
+ import {
61
+ acquireHistoryLock,
62
+ addToModels,
63
+ applyDurable,
64
+ bucketMessages,
65
+ bucketTokens,
66
+ emptyBucket,
67
+ emptyTotals,
68
+ familyRowMessages,
69
+ familyRowTokens,
70
+ loadTotals,
71
+ loadWindowDays,
72
+ refreshHistoryLock,
73
+ releaseHistoryLock,
74
+ resolveHistoryDir,
75
+ } from "./history.js";
76
+
77
+ /** Version of the disposable per-file cache; a foreign version is a cache miss. */
78
+ export const LOCAL_STATS_SCHEMA_VERSION = 2;
79
+
80
+ /**
81
+ * How long a scan may run before it returns what it has. The /usage extension
82
+ * runs the CLI with a 30 s timeout and a first full scan of ~1.5 GB costs a few
83
+ * seconds, so this leaves room for the quota calls that run beside it. A
84
+ * truncated scan never freezes a day: history is written only from complete runs.
85
+ */
86
+ export const DEFAULT_SCAN_BUDGET_MS = 12_000;
87
+
88
+ /** Session formats we can fold: only JSONL files, only message entries. */
89
+ const SESSION_SUFFIX = ".jsonl";
90
+
91
+ /** Number of local days "recentDays" carries, oldest first, today last. */
92
+ const RECENT_DAYS = 7;
93
+
94
+ /**
95
+ * @typedef {import("./history.js").Bucket} Bucket
96
+ * @typedef {import("./history.js").DayRows} DayRows
97
+ */
98
+
99
+ /**
100
+ * The Omarchy-shaped local statistics for one family, plus the per-account
101
+ * breakdown pi's renderers use next to each account line.
102
+ *
103
+ * @typedef {Object} LocalFamilyStats
104
+ * @property {true} hasLocalStats
105
+ * @property {true} hasPromptStats
106
+ * @property {number} todayPrompts
107
+ * @property {number} todaySessions
108
+ * @property {number} todayTotalTokens
109
+ * @property {Record<string, number>} todayTokensByModel
110
+ * @property {Record<string, number>} weekTokensByModel Same, over the last 7 local days.
111
+ * @property {Array<{ date: string, messageCount: number }>} recentDays
112
+ * @property {number} totalPrompts
113
+ * @property {number} totalSessions Session-days: a session spanning midnight counts once per day.
114
+ * @property {number} activeDays
115
+ * @property {string[]} activeDates
116
+ * @property {Record<string, { inputTokens: number, outputTokens: number, cacheReadInputTokens: number, cacheCreationInputTokens: number }>} modelUsage
117
+ * @property {Record<string, { todayTokens: number, weekTokens: number, todayPrompts: number, totalPrompts: number, totalTokens: number }>} byAccount
118
+ */
119
+
120
+ /**
121
+ * How the scan went, for `localStatsMeta` in the report.
122
+ *
123
+ * @typedef {Object} LocalScanMeta
124
+ * @property {number} files Session files found.
125
+ * @property {number} scanned Files (re)read in this run.
126
+ * @property {number} reused Files whose cached offsets still matched.
127
+ * @property {number} bytesRead Bytes parsed in this run.
128
+ * @property {number} durationMs Scan wall time.
129
+ * @property {boolean} partial The budget stopped the scan; nothing was frozen.
130
+ * @property {number} budgetMs The budget that applied.
131
+ * @property {number} frozenDays Days moved into the durable history by this run.
132
+ * @property {boolean} locked Another run held the history lock; nothing was written.
133
+ */
134
+
135
+ /**
136
+ * Options for a local-stats scan, as accepted by the engine's `localStats` option.
137
+ *
138
+ * @typedef {Object} LocalStatsOptions
139
+ * @property {string[]} [roots] Session roots to scan; overrides the Pi/omp defaults.
140
+ * @property {string} [cachePath] Scan cache location; overrides ~/.cache/pi-quota/local-sessions.json.
141
+ * @property {string} [historyDir] Durable history directory; overrides ${PI_CODING_AGENT_DIR:-~/.pi/agent}/pi-usage.
142
+ * @property {Record<string, string | undefined>} [env]
143
+ * @property {string} [home]
144
+ * @property {number} [now] Report time; decides which local day is "today".
145
+ * @property {number} [budgetMs] Wall-clock budget for this scan.
146
+ * @property {() => number} [clock] Time source; tests inject one to drive the budget.
147
+ */
148
+
149
+ /**
150
+ * One session file's cache entry.
151
+ *
152
+ * `offset` is the byte offset of the first byte that is not part of a complete
153
+ * line (end of file when the last line ended with a newline). `complete` is
154
+ * false when the scan budget stopped the read mid-file; the next run continues
155
+ * from `offset` instead of assuming the file is finished. `days` holds open
156
+ * days only: a frozen day is removed from the entry once the durable write
157
+ * succeeded, so a rescan can never recompute it.
158
+ *
159
+ * @typedef {Object} SessionFileEntry
160
+ * @property {number} size
161
+ * @property {number} mtimeMs
162
+ * @property {number} ino
163
+ * @property {number} offset
164
+ * @property {boolean} complete
165
+ * @property {string} [fingerprint] Copy fingerprint of the content at `size`
166
+ * (see fingerprintFile), only set after a full read.
167
+ * @property {string} [dedupedBy] Path of the file this one was verified to be
168
+ * a byte-identical copy of; it holds no rows of
169
+ * its own while that path exists.
170
+ * @property {Record<string, Array<[string, string, string, number, number, number, number, number]>>} days
171
+ */
172
+
173
+ /**
174
+ * @param {Record<string, string | undefined>} env
175
+ * @param {string | undefined} home
176
+ * @returns {string}
177
+ */
178
+ function cacheHome(env, home) {
179
+ return env.XDG_CACHE_HOME || join(home ?? homedir(), ".cache");
180
+ }
181
+
182
+ /**
183
+ * Where the disposable per-file scan cache lives.
184
+ *
185
+ * @param {{ env?: Record<string, string | undefined>, home?: string }} [options]
186
+ * @returns {string}
187
+ */
188
+ export function resolveLocalStatsCachePath(options = {}) {
189
+ const env = options.env ?? process.env;
190
+ return join(cacheHome(env, options.home), "pi-quota", "local-sessions.json");
191
+ }
192
+
193
+ /**
194
+ * Session roots to scan, in the order Pi itself writes them:
195
+ * - `$PI_CODING_AGENT_DIR/sessions` when the env var is set;
196
+ * - `~/.pi/agent/sessions` always;
197
+ * - `~/.omp/agent/sessions` when the omp fork is installed.
198
+ *
199
+ * Only directories that exist are returned, so a machine without Pi simply
200
+ * reports no local statistics instead of a warning.
201
+ *
202
+ * @param {{ env?: Record<string, string | undefined>, home?: string }} [options]
203
+ * @returns {string[]}
204
+ */
205
+ export function resolveSessionRoots(options = {}) {
206
+ const env = options.env ?? process.env;
207
+ const home = options.home ?? homedir();
208
+ /** @type {string[]} */
209
+ const candidates = [];
210
+ if (env.PI_CODING_AGENT_DIR) candidates.push(join(env.PI_CODING_AGENT_DIR, "sessions"));
211
+ candidates.push(join(home, ".pi", "agent", "sessions"));
212
+ candidates.push(join(home, ".omp", "agent", "sessions"));
213
+
214
+ /** @type {string[]} */
215
+ const roots = [];
216
+ const seen = new Set();
217
+ for (const candidate of candidates) {
218
+ let key = candidate;
219
+ try {
220
+ // Two roots can name the same directory: `PI_CODING_AGENT_DIR` may point
221
+ // at a symlink of `~/.pi/agent`, and scanning that twice would count
222
+ // every message twice.
223
+ key = realpathSync(candidate);
224
+ if (seen.has(key)) continue;
225
+ if (!statSync(candidate).isDirectory()) continue;
226
+ } catch {
227
+ // Missing root: not an error, just nothing to scan there.
228
+ continue;
229
+ }
230
+ seen.add(key);
231
+ roots.push(candidate);
232
+ }
233
+ return roots;
234
+ }
235
+
236
+ /**
237
+ * Local calendar date for an epoch value, as YYYY-MM-DD. Dates are local
238
+ * because the panel draws a local week and calls the last row "Today".
239
+ *
240
+ * @param {number} ms
241
+ * @returns {string}
242
+ */
243
+ export function localDayKey(ms) {
244
+ const date = new Date(ms);
245
+ const year = date.getFullYear();
246
+ const month = String(date.getMonth() + 1).padStart(2, "0");
247
+ const day = String(date.getDate()).padStart(2, "0");
248
+ return `${year}-${month}-${day}`;
249
+ }
250
+
251
+ /**
252
+ * Explicit roots bypass `resolveSessionRoots`, so de-duplicate them by real path
253
+ * here too: a caller that names the same store twice must not double count.
254
+ *
255
+ * @param {string[]} roots
256
+ * @returns {string[]}
257
+ */
258
+ function dedupeRoots(roots) {
259
+ /** @type {string[]} */
260
+ const unique = [];
261
+ const seen = new Set();
262
+ for (const root of roots) {
263
+ let key = root;
264
+ try {
265
+ key = realpathSync(root);
266
+ } catch {
267
+ // Keep the unresolved path; listSessionFiles skips missing roots.
268
+ }
269
+ if (seen.has(key)) continue;
270
+ seen.add(key);
271
+ unique.push(root);
272
+ }
273
+ return unique;
274
+ }
275
+
276
+ /**
277
+ * @param {number} now
278
+ * @param {number} count
279
+ * @returns {string[]} Oldest first, today last.
280
+ */
281
+ function recentLocalDates(now, count) {
282
+ const cursor = new Date(now);
283
+ // Noon keeps the walk away from DST midnights, where "one day back" can land
284
+ // on the same calendar date.
285
+ cursor.setHours(12, 0, 0, 0);
286
+ /** @type {string[]} */
287
+ const dates = [];
288
+ for (let offset = count - 1; offset >= 0; offset -= 1) {
289
+ const date = new Date(cursor);
290
+ date.setDate(cursor.getDate() - offset);
291
+ dates.push(localDayKey(date.getTime()));
292
+ }
293
+ return dates;
294
+ }
295
+
296
+ /**
297
+ * @param {unknown} value
298
+ * @returns {number}
299
+ */
300
+ function count(value) {
301
+ const number = Number(value);
302
+ if (!Number.isFinite(number) || number <= 0) return 0;
303
+ return Math.trunc(number);
304
+ }
305
+
306
+ /**
307
+ * @param {unknown} value
308
+ * @returns {string}
309
+ */
310
+ function modelName(value) {
311
+ if (typeof value !== "string") return "unknown";
312
+ const text = value.trim().replace(/[\u0000-\u001f]/g, "");
313
+ return text === "" ? "unknown" : text.slice(0, 120);
314
+ }
315
+
316
+ /**
317
+ * Map a message onto a quota family and account slot.
318
+ *
319
+ * Pi's own provider id is authoritative: it is the only thing that identifies a
320
+ * numbered sibling (`opencode-go-7`). The `api` string is the fallback Pi's
321
+ * Omarchy collector uses for Codex, generalised: it only ever names a base
322
+ * provider, never a slot.
323
+ *
324
+ * @param {unknown} provider
325
+ * @param {unknown} api
326
+ * @returns {{ family: string, slot: string | null } | null}
327
+ */
328
+ function familyForMessage(provider, api) {
329
+ const direct = resolveProviderSlot(typeof provider === "string" ? provider : "");
330
+ if (direct) return { family: direct.family, slot: direct.slot };
331
+ if (typeof api !== "string" || api === "") return null;
332
+ for (const base of Object.keys(FAMILY_BY_PROVIDER)) {
333
+ if (api === base || api.startsWith(`${base}-`) || api.startsWith(`${base}:`)) {
334
+ const resolved = resolveProviderSlot(base);
335
+ if (resolved) return { family: resolved.family, slot: resolved.slot };
336
+ }
337
+ }
338
+ return null;
339
+ }
340
+
341
+ /**
342
+ * Timestamp of a message entry. Pi writes an ISO string on the entry and epoch
343
+ * milliseconds on the message; second-epoch values are accepted because a
344
+ * hand-written or imported session can carry them.
345
+ *
346
+ * @param {Record<string, any>} entry
347
+ * @param {number} fallback
348
+ * @returns {number}
349
+ */
350
+ function entryTimeMs(entry, fallback) {
351
+ const candidates = [entry.timestamp, entry.message?.timestamp];
352
+ for (const candidate of candidates) {
353
+ if (typeof candidate === "number" && Number.isFinite(candidate) && candidate > 0) {
354
+ return candidate > 1e11 ? candidate : candidate * 1000;
355
+ }
356
+ if (typeof candidate === "string" && candidate.trim() !== "") {
357
+ const parsed = Date.parse(candidate);
358
+ if (!Number.isNaN(parsed)) return parsed;
359
+ }
360
+ }
361
+ return fallback;
362
+ }
363
+
364
+ /**
365
+ * Parse one session line into a usage record, or null when the line is not an
366
+ * assistant message this module counts.
367
+ *
368
+ * A message with no tokens at all is not a prompt: Pi records aborted turns and
369
+ * provider errors this way, and counting them would inflate "today's prompts"
370
+ * without adding a single token. Omarchy's collector skips them for the same
371
+ * reason. When only `totalTokens` is present, it is attributed to input, which
372
+ * is how the panel's own totals stay non-zero.
373
+ *
374
+ * @param {string} line
375
+ * @param {number} now
376
+ * @returns {{ day: string, family: string, slot: string, model: string, input: number, output: number, cacheRead: number, cacheWrite: number } | null}
377
+ */
378
+ export function usageFromLine(line, now = Date.now()) {
379
+ // A cheap pre-filter before JSON.parse: the session files are ~1.5 GB and
380
+ // most lines are custom/context entries that can never be an assistant
381
+ // message. Safe because an assistant message always serializes the literal.
382
+ if (!line.includes('"assistant"')) return null;
383
+
384
+ let entry;
385
+ try {
386
+ entry = JSON.parse(line);
387
+ } catch {
388
+ return null;
389
+ }
390
+ if (!entry || entry.type !== "message") return null;
391
+ const message = entry.message;
392
+ if (!message || message.role !== "assistant") return null;
393
+ const usage = message.usage;
394
+ if (!usage || typeof usage !== "object" || Array.isArray(usage)) return null;
395
+
396
+ const resolved = familyForMessage(message.provider, message.api);
397
+ if (!resolved) return null;
398
+
399
+ let input = count(usage.input);
400
+ const output = count(usage.output);
401
+ const cacheRead = count(usage.cacheRead);
402
+ const cacheWrite = count(usage.cacheWrite);
403
+ if (input + output + cacheRead + cacheWrite === 0) {
404
+ const reported = count(usage.totalTokens);
405
+ if (reported === 0) return null;
406
+ input = reported;
407
+ }
408
+
409
+ return {
410
+ day: localDayKey(entryTimeMs(entry, now)),
411
+ family: resolved.family,
412
+ slot: resolved.slot ?? "",
413
+ model: modelName(message.model),
414
+ input,
415
+ output,
416
+ cacheRead,
417
+ cacheWrite,
418
+ };
419
+ }
420
+
421
+ /**
422
+ * Per-file open-day aggregates while scanning: day -> key -> tuple.
423
+ * The tuple is the serialized form `[family, slot, model, input, output,
424
+ * cacheRead, cacheWrite, prompts]`, which keeps the cache compact.
425
+ *
426
+ * @typedef {Map<string, Map<string, [string, string, string, number, number, number, number, number]>>} DaysMap
427
+ */
428
+
429
+ /**
430
+ * @returns {DaysMap}
431
+ */
432
+ function emptyDays() {
433
+ return new Map();
434
+ }
435
+
436
+ /**
437
+ * @param {DaysMap} days
438
+ * @param {NonNullable<ReturnType<typeof usageFromLine>>} usage
439
+ */
440
+ function addUsage(days, usage) {
441
+ let bucket = days.get(usage.day);
442
+ if (!bucket) {
443
+ bucket = new Map();
444
+ days.set(usage.day, bucket);
445
+ }
446
+ const key = `${usage.family}\u0000${usage.slot}\u0000${usage.model}`;
447
+ const tuple = bucket.get(key);
448
+ if (tuple) {
449
+ tuple[3] += usage.input;
450
+ tuple[4] += usage.output;
451
+ tuple[5] += usage.cacheRead;
452
+ tuple[6] += usage.cacheWrite;
453
+ tuple[7] += 1;
454
+ return;
455
+ }
456
+ bucket.set(key, [usage.family, usage.slot, usage.model, usage.input, usage.output, usage.cacheRead, usage.cacheWrite, 1]);
457
+ }
458
+
459
+ /**
460
+ * @param {DaysMap} days
461
+ * @returns {SessionFileEntry["days"]}
462
+ */
463
+ function encodeDays(days) {
464
+ /** @type {SessionFileEntry["days"]} */
465
+ const encoded = {};
466
+ for (const [day, bucket] of days) encoded[day] = Array.from(bucket.values());
467
+ return encoded;
468
+ }
469
+
470
+ /**
471
+ * @param {unknown} raw
472
+ * @returns {DaysMap}
473
+ */
474
+ function decodeDays(raw) {
475
+ const days = emptyDays();
476
+ if (!raw || typeof raw !== "object" || Array.isArray(raw)) return days;
477
+ for (const [day, tuples] of Object.entries(raw)) {
478
+ if (!Array.isArray(tuples)) continue;
479
+ const bucket = new Map();
480
+ for (const tuple of tuples) {
481
+ if (!Array.isArray(tuple) || tuple.length < 8) continue;
482
+ const [family, slot, model] = tuple;
483
+ if (typeof family !== "string" || typeof slot !== "string" || typeof model !== "string") continue;
484
+ const numbers = [tuple[3], tuple[4], tuple[5], tuple[6], tuple[7]].map(count);
485
+ bucket.set(`${family}\u0000${slot}\u0000${model}`, [
486
+ family,
487
+ slot,
488
+ model,
489
+ numbers[0],
490
+ numbers[1],
491
+ numbers[2],
492
+ numbers[3],
493
+ numbers[4],
494
+ ]);
495
+ }
496
+ if (bucket.size > 0) days.set(day, bucket);
497
+ }
498
+ return days;
499
+ }
500
+
501
+ /**
502
+ * Stream one session file from `startOffset`, handing every complete line to
503
+ * `onLine` and stopping early when `shouldStop` asks.
504
+ *
505
+ * The file is sliced at the newline byte (0x0A), which cannot occur inside a
506
+ * multi-byte UTF-8 sequence, so decoding per complete line is safe. The
507
+ * returned `offset` points at the first byte not part of a processed line,
508
+ * which is exactly where the next incremental run must resume.
509
+ *
510
+ * @param {{
511
+ * path: string,
512
+ * startOffset: number,
513
+ * onLine: (line: string) => void,
514
+ * shouldStop?: () => boolean,
515
+ * }} input
516
+ * @returns {Promise<{ offset: number, aborted: boolean }>}
517
+ */
518
+ async function readLines(input) {
519
+ const stream = createReadStream(input.path, { start: input.startOffset, highWaterMark: 1 << 20 });
520
+ /** @type {Buffer | null} */
521
+ let pending = null;
522
+ let consumed = 0;
523
+ let aborted = false;
524
+ let newlineSeen = 0;
525
+
526
+ try {
527
+ for await (const chunk of stream) {
528
+ const buffer = pending && pending.length > 0 ? Buffer.concat([pending, chunk]) : chunk;
529
+ pending = null;
530
+ let cursor = 0;
531
+ let newline = buffer.indexOf(10, cursor);
532
+ while (newline !== -1) {
533
+ if (newline > cursor) input.onLine(buffer.toString("utf8", cursor, newline));
534
+ consumed += newline - cursor + 1;
535
+ cursor = newline + 1;
536
+ newlineSeen += 1;
537
+ // Checking the clock every 16k lines keeps a single huge file inside
538
+ // the budget without calling Date.now() per line.
539
+ if ((newlineSeen & 0x3fff) === 0 && input.shouldStop?.()) {
540
+ aborted = true;
541
+ break;
542
+ }
543
+ newline = buffer.indexOf(10, cursor);
544
+ }
545
+ if (aborted) break;
546
+ pending = cursor < buffer.length ? Buffer.from(buffer.subarray(cursor)) : null;
547
+ }
548
+ } finally {
549
+ // Breaking out of `for await` destroys the stream; this also covers a throw
550
+ // from onLine.
551
+ stream.destroy();
552
+ }
553
+
554
+ return { offset: input.startOffset + consumed, aborted };
555
+ }
556
+
557
+ /**
558
+ * Read one session file into open-day aggregates, routing each message.
559
+ *
560
+ * @param {{
561
+ * path: string,
562
+ * startOffset: number,
563
+ * days: DaysMap,
564
+ * now: number,
565
+ * onUsage: (usage: NonNullable<ReturnType<typeof usageFromLine>>) => void,
566
+ * shouldStop: () => boolean,
567
+ * }} input
568
+ * @returns {Promise<{ offset: number, complete: boolean }>}
569
+ */
570
+ async function scanFile(input) {
571
+ const result = await readLines({
572
+ path: input.path,
573
+ startOffset: input.startOffset,
574
+ shouldStop: input.shouldStop,
575
+ onLine: (line) => {
576
+ const usage = usageFromLine(line, input.now);
577
+ if (usage) input.onUsage(usage);
578
+ },
579
+ });
580
+ return { offset: result.offset, complete: !result.aborted };
581
+ }
582
+
583
+ /** Bytes hashed from each end of a session file for the copy fingerprint. */
584
+ const FINGERPRINT_BYTES = 32 * 1024;
585
+
586
+ /**
587
+ * Fingerprint a session file as (size, first 32 KB, last 32 KB).
588
+ *
589
+ * This exists to recognise a byte-identical copy of a session that is already
590
+ * counted: a `cp`, a restored backup or a sync tool would otherwise add the
591
+ * same messages a second time, and for today's messages that is a real
592
+ * overcount (a frozen day is protected by the frozen-day rule instead).
593
+ *
594
+ * Only two small reads, so a duplicate 240 MB session costs 64 KB rather than
595
+ * a full parse. A *fork* (which copies a prefix and then diverges) has a
596
+ * different size and tail, so it is never mistaken for a copy and never loses
597
+ * the messages it added after forking.
598
+ *
599
+ * @param {string} path
600
+ * @param {number} size
601
+ * @returns {string | null} Hex digest, or null when the file cannot be read.
602
+ */
603
+ function fingerprintFile(path, size) {
604
+ let fd = null;
605
+ try {
606
+ fd = openSync(path, "r");
607
+ const head = Buffer.alloc(Math.min(FINGERPRINT_BYTES, size));
608
+ if (head.length > 0) readSync(fd, head, 0, head.length, 0);
609
+ const tailLength = Math.min(FINGERPRINT_BYTES, Math.max(0, size - head.length));
610
+ let tail = Buffer.alloc(0);
611
+ if (tailLength > 0) {
612
+ tail = Buffer.alloc(tailLength);
613
+ readSync(fd, tail, 0, tailLength, size - tailLength);
614
+ }
615
+ const hash = createHash("sha256");
616
+ hash.update(`${size}\u0000`);
617
+ hash.update(head);
618
+ hash.update(tail);
619
+ return hash.digest("hex").slice(0, 32);
620
+ } catch {
621
+ return null;
622
+ } finally {
623
+ if (fd !== null) {
624
+ try {
625
+ closeSync(fd);
626
+ } catch {
627
+ // The fingerprint is an optimization; a close failure changes nothing.
628
+ }
629
+ }
630
+ }
631
+ }
632
+
633
+ /**
634
+ * Full-content comparison of two session files, streamed in 1 MB chunks.
635
+ *
636
+ * Only called when a cheap fingerprint already matched, so the expensive path is
637
+ * the rare one: a real duplicate. Streams rather than buffering, because the
638
+ * sessions on this machine reach 240 MB.
639
+ *
640
+ * @param {string} a
641
+ * @param {string} b
642
+ * @param {number} size Expected size of both.
643
+ * @returns {boolean}
644
+ */
645
+ function sameContent(a, b, size) {
646
+ let left = null;
647
+ let right = null;
648
+ try {
649
+ left = openSync(a, "r");
650
+ right = openSync(b, "r");
651
+ const leftBuffer = Buffer.alloc(1 << 20);
652
+ const rightBuffer = Buffer.alloc(1 << 20);
653
+ let offset = 0;
654
+ while (offset < size) {
655
+ const wanted = Math.min(leftBuffer.length, size - offset);
656
+ const leftRead = readSync(left, leftBuffer, 0, wanted, offset);
657
+ const rightRead = readSync(right, rightBuffer, 0, wanted, offset);
658
+ if (leftRead !== rightRead) return false;
659
+ if (leftBuffer.compare(rightBuffer, 0, rightRead, 0, leftRead) !== 0) return false;
660
+ if (leftRead <= 0) break;
661
+ offset += leftRead;
662
+ }
663
+ return true;
664
+ } catch {
665
+ // An unreadable candidate is not a duplicate: counting it twice is the safer
666
+ // error, since dropping a session silently loses tokens.
667
+ return false;
668
+ } finally {
669
+ for (const fd of [left, right]) {
670
+ if (fd !== null) {
671
+ try {
672
+ closeSync(fd);
673
+ } catch {
674
+ // Closing is best-effort.
675
+ }
676
+ }
677
+ }
678
+ }
679
+ }
680
+
681
+ /**
682
+ * @param {string[]} roots
683
+ * @returns {string[]}
684
+ */
685
+ function listSessionFiles(roots) {
686
+ /** @type {string[]} */
687
+ const files = [];
688
+ const seen = new Set();
689
+ for (const root of roots) {
690
+ /** @type {Array<[string, import("node:fs").Dirent[]]>} */
691
+ const stack = [];
692
+ try {
693
+ stack.push([root, readdirSync(root, { withFileTypes: true })]);
694
+ } catch {
695
+ continue;
696
+ }
697
+ while (stack.length > 0) {
698
+ const [dir, entries] = /** @type {[string, import("node:fs").Dirent[]]} */ (stack.pop());
699
+ for (const entry of entries) {
700
+ const full = join(dir, entry.name);
701
+ if (entry.isDirectory()) {
702
+ try {
703
+ stack.push([full, readdirSync(full, { withFileTypes: true })]);
704
+ } catch {
705
+ // An unreadable directory only hides its own sessions.
706
+ }
707
+ } else if (entry.isFile() && entry.name.endsWith(SESSION_SUFFIX)) {
708
+ // Same file through two roots (symlinked session dirs, a shared
709
+ // store) must be read once per run.
710
+ let key = full;
711
+ try {
712
+ key = realpathSync(full);
713
+ } catch {
714
+ // A vanished file is caught by statSync below and skipped.
715
+ }
716
+ if (seen.has(key)) continue;
717
+ seen.add(key);
718
+ files.push(full);
719
+ }
720
+ }
721
+ }
722
+ }
723
+ return files.sort();
724
+ }
725
+
726
+ /**
727
+ * Read the disposable cache envelope. A missing, corrupt or foreign-shaped file
728
+ * is a cache miss (a full rescan), never a crash.
729
+ *
730
+ * @param {string} path
731
+ * @returns {{ files: Record<string, SessionFileEntry>, hit: boolean, pendingLate: DaysMap }}
732
+ */
733
+ function readCacheFile(path) {
734
+ let parsed;
735
+ try {
736
+ parsed = JSON.parse(readFileSync(path, "utf-8"));
737
+ } catch {
738
+ return { files: {}, hit: false, historyWatermark: "", pendingLate: emptyDays() };
739
+ }
740
+ if (!parsed || typeof parsed !== "object" || parsed.schemaVersion !== LOCAL_STATS_SCHEMA_VERSION) {
741
+ return { files: {}, hit: false, historyWatermark: "", pendingLate: emptyDays() };
742
+ }
743
+ if (!parsed.files || typeof parsed.files !== "object" || Array.isArray(parsed.files)) {
744
+ return { files: {}, hit: false, historyWatermark: "", pendingLate: emptyDays() };
745
+ }
746
+ // The last `lastFrozenDay` this cache was written against. An empty rollup
747
+ // next to a non-empty watermark is a lost history, not a fresh one.
748
+ const historyWatermark = typeof parsed.historyWatermark === "string" ? parsed.historyWatermark : "";
749
+ const pendingLate = decodeDays(parsed.pendingLate);
750
+
751
+ /** @type {Record<string, SessionFileEntry>} */
752
+ const files = {};
753
+ for (const [file, value] of Object.entries(parsed.files)) {
754
+ if (!value || typeof value !== "object") continue;
755
+ const entry = /** @type {Record<string, unknown>} */ (value);
756
+ if (typeof entry.size !== "number" || !Number.isFinite(entry.size)) continue;
757
+ if (typeof entry.mtimeMs !== "number" || !Number.isFinite(entry.mtimeMs)) continue;
758
+ if (typeof entry.offset !== "number" || !Number.isFinite(entry.offset)) continue;
759
+ if (!entry.days || typeof entry.days !== "object" || Array.isArray(entry.days)) continue;
760
+ files[file] = {
761
+ size: entry.size,
762
+ mtimeMs: entry.mtimeMs,
763
+ ino: typeof entry.ino === "number" && Number.isFinite(entry.ino) ? entry.ino : 0,
764
+ offset: Math.max(0, entry.offset),
765
+ complete: entry.complete !== false,
766
+ ...(typeof entry.fingerprint === "string" && entry.fingerprint !== "" ? { fingerprint: entry.fingerprint } : {}),
767
+ ...(typeof entry.dedupedBy === "string" && entry.dedupedBy !== "" ? { dedupedBy: entry.dedupedBy } : {}),
768
+ days: /** @type {SessionFileEntry["days"]} */ (entry.days),
769
+ };
770
+ }
771
+ return { files, hit: true, historyWatermark, pendingLate };
772
+ }
773
+
774
+ /**
775
+ * @param {string} path
776
+ * @param {Record<string, SessionFileEntry>} files
777
+ * @param {string} historyWatermark The `lastFrozenDay` this cache is written against.
778
+ * @param {SessionFileEntry["days"]} [pendingLate] Late deltas waiting for an unreadable month.
779
+ * @returns {{ ok: boolean, error?: string }}
780
+ */
781
+ function writeCacheFile(path, files, historyWatermark = "", pendingLate = {}) {
782
+ // `historyWatermark` records which `lastFrozenDay` this cache belongs to, so a
783
+ // later run can tell a lost rollup from a fresh install. Pending late rows are
784
+ // aggregate deltas retained until their month can be updated.
785
+ const payload = `${JSON.stringify({ schemaVersion: LOCAL_STATS_SCHEMA_VERSION, historyWatermark, files, ...(Object.keys(pendingLate).length > 0 ? { pendingLate } : {}) })}\n`;
786
+ const temporary = `${path}.${process.pid}.tmp`;
787
+ try {
788
+ mkdirSync(dirname(path), { recursive: true, mode: 0o700 });
789
+ writeFileSync(temporary, payload, { encoding: "utf-8", mode: 0o600 });
790
+ renameSync(temporary, path);
791
+ return { ok: true };
792
+ } catch (error) {
793
+ rmSync(temporary, { force: true });
794
+ return { ok: false, error: redact(/** @type {{ message?: string }} */ (error)?.message ?? String(error)) };
795
+ }
796
+ }
797
+
798
+ /**
799
+ * Fold every session file's open-day tuples into one `DayRows` per day. Session
800
+ * counts are distinct files, which is what "today: N sessions" means.
801
+ *
802
+ * @param {Record<string, SessionFileEntry>} files
803
+ * @returns {Record<string, DayRows>}
804
+ */
805
+ function buildOpenRows(files) {
806
+ /** @type {Map<string, Map<string, { files: Set<string>, slots: Map<string, { files: Set<string>, models: Map<string, Bucket> }> }>>} */
807
+ const days = new Map();
808
+ for (const [file, entry] of Object.entries(files ?? {})) {
809
+ for (const [day, tuples] of Object.entries(entry.days ?? {})) {
810
+ if (!Array.isArray(tuples) || tuples.length === 0) continue;
811
+ for (const tuple of tuples) {
812
+ if (!Array.isArray(tuple) || tuple.length < 8) continue;
813
+ const [family, slot, model] = tuple;
814
+ if (typeof family !== "string" || typeof slot !== "string" || typeof model !== "string") continue;
815
+ const bucket = {
816
+ input: count(tuple[3]),
817
+ output: count(tuple[4]),
818
+ cacheRead: count(tuple[5]),
819
+ cacheWrite: count(tuple[6]),
820
+ messages: count(tuple[7]),
821
+ };
822
+ if (bucket.messages === 0 && bucketTokens(bucket) === 0) continue;
823
+
824
+ const dayMap = days.get(day) ?? new Map();
825
+ days.set(day, dayMap);
826
+ const familyAgg = dayMap.get(family) ?? { files: new Set(), slots: new Map() };
827
+ dayMap.set(family, familyAgg);
828
+ familyAgg.files.add(file);
829
+ const slotAgg = familyAgg.slots.get(slot) ?? { files: new Set(), models: new Map() };
830
+ familyAgg.slots.set(slot, slotAgg);
831
+ slotAgg.files.add(file);
832
+ const known = slotAgg.models.get(model);
833
+ if (known) {
834
+ known.input += bucket.input;
835
+ known.output += bucket.output;
836
+ known.cacheRead += bucket.cacheRead;
837
+ known.cacheWrite += bucket.cacheWrite;
838
+ known.messages += bucket.messages;
839
+ } else {
840
+ slotAgg.models.set(model, { ...bucket });
841
+ }
842
+ }
843
+ }
844
+ }
845
+
846
+ /** @type {Record<string, DayRows>} */
847
+ const rows = {};
848
+ for (const [day, families] of days) {
849
+ /** @type {DayRows} */
850
+ const dayRows = {};
851
+ for (const [family, familyAgg] of families) {
852
+ /** @type {Record<string, { sessions: number, models: Record<string, Bucket> }>} */
853
+ const slots = {};
854
+ for (const [slot, slotAgg] of familyAgg.slots) {
855
+ /** @type {Record<string, Bucket>} */
856
+ const models = {};
857
+ for (const [model, bucket] of slotAgg.models) models[model] = { ...bucket };
858
+ slots[slot] = { sessions: slotAgg.files.size, models };
859
+ }
860
+ dayRows[family] = { sessions: familyAgg.files.size, slots };
861
+ }
862
+ rows[day] = dayRows;
863
+ }
864
+ return rows;
865
+ }
866
+
867
+ /**
868
+ * Turn cached, not-yet-applied late deltas into durable-history rows.
869
+ * Session counts stay zero because the original frozen day already counted them.
870
+ *
871
+ * @param {DaysMap} days
872
+ * @returns {Record<string, DayRows>}
873
+ */
874
+ function buildLateRows(days) {
875
+ /** @type {Record<string, DayRows>} */
876
+ const rows = {};
877
+ for (const [day, tuples] of days) {
878
+ const dayRows = (rows[day] ??= {});
879
+ for (const tuple of tuples.values()) {
880
+ const [family, slot, model] = tuple;
881
+ const familyRow = (dayRows[family] ??= { sessions: 0, slots: {} });
882
+ const slotRow = (familyRow.slots[slot] ??= { sessions: 0, models: {} });
883
+ const bucket = (slotRow.models[model] ??= emptyBucket());
884
+ bucket.input += count(tuple[3]);
885
+ bucket.output += count(tuple[4]);
886
+ bucket.cacheRead += count(tuple[5]);
887
+ bucket.cacheWrite += count(tuple[6]);
888
+ bucket.messages += count(tuple[7]);
889
+ }
890
+ }
891
+ return rows;
892
+ }
893
+
894
+ /**
895
+ * @param {Map<string, number>} models Token totals by model id.
896
+ * @returns {Record<string, number>}
897
+ */
898
+ function tokensByModel(models) {
899
+ /** @type {Array<[string, number]>} */
900
+ const entries = [];
901
+ for (const [model, tokens] of models) {
902
+ if (tokens > 0) entries.push([model, tokens]);
903
+ }
904
+ entries.sort((a, b) => b[1] - a[1] || a[0].localeCompare(b[0]));
905
+ return Object.fromEntries(entries);
906
+ }
907
+
908
+ /**
909
+ * @param {string} label
910
+ * @returns {number}
911
+ */
912
+ function labelOrder(label) {
913
+ if (label === "#1") return -1;
914
+ const numeric = Number.parseInt(String(label).replace(/^#/, ""), 10);
915
+ return Number.isFinite(numeric) ? numeric : Number.MAX_SAFE_INTEGER;
916
+ }
917
+
918
+ /**
919
+ * Fold the durable rollup, the frozen days of the last seven days, and the open
920
+ * days into the shape the Omarchy panel and /usage read. Today is always taken
921
+ * from open days; the week prefers the open copy of a day and falls back to the
922
+ * frozen month file, so a crash between a month write and the totals write never
923
+ * double counts.
924
+ *
925
+ * @param {{
926
+ * totals?: import("./history.js").Totals,
927
+ * openRows?: Record<string, DayRows>,
928
+ * windowDays?: Record<string, DayRows>,
929
+ * now?: number,
930
+ * }} input
931
+ * @returns {Record<string, LocalFamilyStats>}
932
+ */
933
+ export function buildLocalStats(input = {}) {
934
+ const now = input.now ?? Date.now();
935
+ const today = localDayKey(now);
936
+ const recentDates = recentLocalDates(now, RECENT_DAYS);
937
+ const openRows = input.openRows ?? {};
938
+ const windowDays = input.windowDays ?? {};
939
+ const totals = input.totals ?? emptyTotals();
940
+
941
+ const families = new Set(totals.families.keys());
942
+ for (const rows of Object.values(openRows)) for (const family of Object.keys(rows)) families.add(family);
943
+ for (const rows of Object.values(windowDays)) for (const family of Object.keys(rows)) families.add(family);
944
+
945
+ /** @type {Record<string, LocalFamilyStats>} */
946
+ const byFamily = {};
947
+ for (const family of families) {
948
+ const history = totals.families.get(family) ?? null;
949
+
950
+ // Open days (normally just today, plus anything a truncated run left open).
951
+ let openPrompts = 0;
952
+ let openSessions = 0;
953
+ const openModels = new Map();
954
+ const openAccounts = new Map();
955
+ const openDates = new Set();
956
+ for (const [day, rows] of Object.entries(openRows)) {
957
+ const familyRow = rows?.[family];
958
+ if (!familyRow) continue;
959
+ openDates.add(day);
960
+ openSessions += count(familyRow.sessions);
961
+ for (const [slot, slotRow] of Object.entries(familyRow.slots ?? {})) {
962
+ const label = slot === "" ? "#1" : `#${slot}`;
963
+ const account = openAccounts.get(label) ?? { prompts: 0, sessions: 0, models: new Map() };
964
+ account.sessions += count(slotRow.sessions);
965
+ for (const [model, bucket] of Object.entries(slotRow.models ?? {})) {
966
+ addToModels(openModels, model, bucket);
967
+ addToModels(account.models, model, bucket);
968
+ openPrompts += bucketMessages(bucket);
969
+ account.prompts += bucketMessages(bucket);
970
+ }
971
+ openAccounts.set(label, account);
972
+ }
973
+ }
974
+
975
+ // The seven-day window: open days override frozen ones.
976
+ const dayTokens = new Map();
977
+ const dayModels = new Map();
978
+ const daySlots = new Map();
979
+ const frozenThrough = totals.lastFrozenDay;
980
+ for (const date of recentDates) {
981
+ // A frozen day belongs to the month file, which carries its late writes;
982
+ // an open day belongs to the cache. Preferring the cache for a frozen day
983
+ // would hide a late append until the day left the window.
984
+ const frozen = frozenThrough !== "" && date <= frozenThrough;
985
+ const rows = frozen ? (windowDays[date] ?? openRows[date]) : (openRows[date] ?? windowDays[date]);
986
+ const familyRow = rows?.[family];
987
+ if (!familyRow) {
988
+ dayTokens.set(date, 0);
989
+ continue;
990
+ }
991
+ dayTokens.set(date, familyRowTokens(familyRow));
992
+ /** @type {Map<string, number>} */
993
+ const models = new Map();
994
+ /** @type {Map<string, { tokens: number, prompts: number }>} */
995
+ const slots = new Map();
996
+ for (const [slot, slotRow] of Object.entries(familyRow.slots ?? {})) {
997
+ const label = slot === "" ? "#1" : `#${slot}`;
998
+ let tokens = 0;
999
+ let prompts = 0;
1000
+ for (const [model, bucket] of Object.entries(slotRow.models ?? {})) {
1001
+ const modelTokens = bucketTokens(bucket);
1002
+ tokens += modelTokens;
1003
+ prompts += bucketMessages(bucket);
1004
+ models.set(model, (models.get(model) ?? 0) + modelTokens);
1005
+ }
1006
+ const known = slots.get(label) ?? { tokens: 0, prompts: 0 };
1007
+ known.tokens += tokens;
1008
+ known.prompts += prompts;
1009
+ slots.set(label, known);
1010
+ }
1011
+ dayModels.set(date, models);
1012
+ daySlots.set(date, slots);
1013
+ }
1014
+
1015
+ const todayRows = openRows[today]?.[family] ?? null;
1016
+ const todayPrompts = todayRows ? familyRowMessages(todayRows) : 0;
1017
+ const todayTotalTokens = todayRows ? familyRowTokens(todayRows) : 0;
1018
+ /** @type {Map<string, number>} */
1019
+ const todayModels = new Map();
1020
+ for (const slotRow of Object.values(todayRows?.slots ?? {})) {
1021
+ for (const [model, bucket] of Object.entries(slotRow.models ?? {})) {
1022
+ todayModels.set(model, (todayModels.get(model) ?? 0) + bucketTokens(bucket));
1023
+ }
1024
+ }
1025
+
1026
+ /** @type {Map<string, number>} */
1027
+ const weekModels = new Map();
1028
+ for (const models of dayModels.values()) {
1029
+ for (const [model, tokens] of models) weekModels.set(model, (weekModels.get(model) ?? 0) + tokens);
1030
+ }
1031
+
1032
+ /** @type {Map<string, Bucket>} */
1033
+ const allModels = new Map();
1034
+ for (const [model, bucket] of history?.models ?? []) allModels.set(model, { ...bucket });
1035
+ for (const [model, bucket] of openModels) addToModels(allModels, model, bucket);
1036
+
1037
+ const activeDates = new Set(history?.activeDates ?? []);
1038
+ for (const date of openDates) activeDates.add(date);
1039
+
1040
+ /** @type {Record<string, { todayTokens: number, weekTokens: number, todayPrompts: number, totalPrompts: number, totalTokens: number }>} */
1041
+ const byAccount = {};
1042
+ const labels = new Set([...(history?.accounts.keys() ?? []), ...openAccounts.keys()]);
1043
+ for (const label of Array.from(labels).sort((a, b) => labelOrder(a) - labelOrder(b))) {
1044
+ const historyAccount = history?.accounts.get(label);
1045
+ const openAccount = openAccounts.get(label);
1046
+ /** @type {Map<string, Bucket>} */
1047
+ const accountModels = new Map();
1048
+ for (const [model, bucket] of historyAccount?.models ?? []) accountModels.set(model, { ...bucket });
1049
+ for (const [model, bucket] of openAccount?.models ?? []) addToModels(accountModels, model, bucket);
1050
+ let totalTokens = 0;
1051
+ for (const bucket of accountModels.values()) totalTokens += bucketTokens(bucket);
1052
+ let weekTokens = 0;
1053
+ for (const date of recentDates) weekTokens += daySlots.get(date)?.get(label)?.tokens ?? 0;
1054
+ byAccount[label] = {
1055
+ todayTokens: daySlots.get(today)?.get(label)?.tokens ?? 0,
1056
+ weekTokens,
1057
+ todayPrompts: daySlots.get(today)?.get(label)?.prompts ?? 0,
1058
+ totalPrompts: (historyAccount?.totalPrompts ?? 0) + (openAccount?.prompts ?? 0),
1059
+ totalTokens,
1060
+ };
1061
+ }
1062
+
1063
+ const totalPrompts = (history?.totalPrompts ?? 0) + openPrompts;
1064
+ if (totalPrompts === 0 && allModels.size === 0) continue;
1065
+
1066
+ /** @type {Record<string, { inputTokens: number, outputTokens: number, cacheReadInputTokens: number, cacheCreationInputTokens: number }>} */
1067
+ const modelUsage = {};
1068
+ const sortedModels = Array.from(allModels.entries()).sort((a, b) => bucketTokens(b[1]) - bucketTokens(a[1]) || a[0].localeCompare(b[0]));
1069
+ for (const [model, bucket] of sortedModels) {
1070
+ modelUsage[model] = {
1071
+ inputTokens: bucket.input,
1072
+ outputTokens: bucket.output,
1073
+ cacheReadInputTokens: bucket.cacheRead,
1074
+ cacheCreationInputTokens: bucket.cacheWrite,
1075
+ };
1076
+ }
1077
+
1078
+ byFamily[family] = {
1079
+ hasLocalStats: true,
1080
+ hasPromptStats: true,
1081
+ todayPrompts,
1082
+ todaySessions: todayRows ? count(todayRows.sessions) : 0,
1083
+ todayTotalTokens,
1084
+ todayTokensByModel: tokensByModel(todayModels),
1085
+ weekTokensByModel: tokensByModel(weekModels),
1086
+ recentDays: recentDates.map((date) => ({ date, messageCount: dayTokens.get(date) ?? 0 })),
1087
+ totalPrompts,
1088
+ totalSessions: (history?.totalSessions ?? 0) + openSessions,
1089
+ activeDays: activeDates.size,
1090
+ activeDates: Array.from(activeDates).sort(),
1091
+ modelUsage,
1092
+ byAccount,
1093
+ };
1094
+ }
1095
+ return byFamily;
1096
+ }
1097
+
1098
+ /**
1099
+ * Scan Pi's session files, freeze closed days into the durable history, and fold
1100
+ * everything into local statistics.
1101
+ *
1102
+ * Never throws. Everything unexpected (unreadable file, unwritable cache or
1103
+ * history) becomes a warning; the numbers simply cover fewer sessions.
1104
+ *
1105
+ * @param {{
1106
+ * roots?: string[],
1107
+ * cachePath?: string,
1108
+ * historyDir?: string,
1109
+ * env?: Record<string, string | undefined>,
1110
+ * home?: string,
1111
+ * now?: number,
1112
+ * budgetMs?: number,
1113
+ * clock?: () => number,
1114
+ * }} [options]
1115
+ * @returns {Promise<{
1116
+ * byFamily: Record<string, LocalFamilyStats>,
1117
+ * warnings: string[],
1118
+ * scan: LocalScanMeta,
1119
+ * }>}
1120
+ */export async function collectLocalStats(options = {}) {
1121
+ const clock = options.clock ?? Date.now;
1122
+ const startedAt = clock();
1123
+ const now = options.now ?? Date.now();
1124
+ const today = localDayKey(now);
1125
+ const budgetMs = Number.isFinite(options.budgetMs) ? /** @type {number} */ (options.budgetMs) : DEFAULT_SCAN_BUDGET_MS;
1126
+ const roots = dedupeRoots(options.roots ?? resolveSessionRoots(options));
1127
+ const cachePath = options.cachePath ?? resolveLocalStatsCachePath(options);
1128
+ const historyDir = options.historyDir ?? resolveHistoryDir(options);
1129
+ /** @type {string[]} */
1130
+ const warnings = [];
1131
+ const listen = listSessionFiles(roots);
1132
+
1133
+ /** @type {LocalScanMeta} */
1134
+ const emptyScan = {
1135
+ files: listen.length,
1136
+ scanned: 0,
1137
+ reused: 0,
1138
+ bytesRead: 0,
1139
+ durationMs: clock() - startedAt,
1140
+ partial: false,
1141
+ budgetMs,
1142
+ frozenDays: 0,
1143
+ locked: false,
1144
+ };
1145
+
1146
+ // The lock comes before the reads, not after: two writers (the systemd timer
1147
+ // and /usage) that both read then both write would apply the same late-write
1148
+ // delta twice. A run that cannot take it still answers, from what is on disk.
1149
+ const lockPath = `${cachePath}.lock`;
1150
+ const lock = acquireHistoryLock({ lockPath, now: clock() });
1151
+ if (!lock.acquired) {
1152
+ // Read-only path: `canWrite: false` keeps it from rebuilding or repairing
1153
+ // anything the run that does hold the lock is writing right now.
1154
+ const history = loadTotals({ historyDir, warnings, canWrite: false });
1155
+ const cached = readCacheFile(cachePath);
1156
+ // Nothing here may write: not the history, not the cache, and not a repair
1157
+ // of a corrupt month file (review findings N8, N10).
1158
+ warnings.push(
1159
+ lock.reason
1160
+ ? `local token stats: could not take the history lock (${lock.reason}); showing the numbers already on disk and writing nothing`
1161
+ : "local token stats: another piquota run is updating the token history; showing the numbers already on disk",
1162
+ );
1163
+ return {
1164
+ byFamily: buildLocalStats({
1165
+ totals: history.totals,
1166
+ openRows: buildOpenRows(cached.files),
1167
+ windowDays: loadWindowDays({
1168
+ historyDir,
1169
+ dates: recentLocalDates(now, RECENT_DAYS),
1170
+ warnings,
1171
+ canWrite: false,
1172
+ }),
1173
+ now,
1174
+ }),
1175
+ warnings,
1176
+ scan: { ...emptyScan, locked: true },
1177
+ };
1178
+ }
1179
+
1180
+ try {
1181
+ // A missing history or a missing cache means every day still on disk has to
1182
+ // be rebuildable, so both force a rescan. Frozen days are skipped in that
1183
+ // case: they were already counted and a rescan must never add them again.
1184
+ const history = loadTotals({ historyDir, warnings, canWrite: true });
1185
+ const cached = readCacheFile(cachePath);
1186
+ // A rollup that lost its numbers while the cache remembers having seen
1187
+ // frozen days is a lost history: rebuild whatever the session files still
1188
+ // hold instead of leaving the all-time totals empty.
1189
+ const lostHistory =
1190
+ history.totals.families.size === 0 &&
1191
+ history.totals.lastFrozenDay === "" &&
1192
+ typeof cached.historyWatermark === "string" &&
1193
+ cached.historyWatermark !== "";
1194
+ // Damaged or unreadable month files affect only their daily detail. They
1195
+ // never trigger a session rescan: frozen days are not reconstructed from
1196
+ // surviving session files.
1197
+ const windowDates = recentLocalDates(now, RECENT_DAYS);
1198
+ const rescanAll = !cached.hit || history.rescanAll || lostHistory;
1199
+ const lastFrozenDay = history.totals.lastFrozenDay;
1200
+
1201
+ const overBudget = () => clock() - startedAt > budgetMs;
1202
+ const listedSet = new Set(listen);
1203
+
1204
+ // Fingerprint -> the live path already counted under it. Built from the
1205
+ // cache (so a copy made after the original was scanned is still caught) and
1206
+ // only for files that still exist: restoring a backup *after* deleting the
1207
+ // original must count, not be discarded as a duplicate of a ghost.
1208
+ /** @type {Map<string, string>} */
1209
+ const counted = new Map();
1210
+ for (const path of listen) {
1211
+ const entry = cached.files[path];
1212
+ if (entry?.fingerprint) counted.set(entry.fingerprint, path);
1213
+ }
1214
+
1215
+ /**
1216
+ * The surviving file a copy chain bottoms out in.
1217
+ *
1218
+ * A copy can itself be copied (A -> B -> C), and an older cache may hold a
1219
+ * chain. Following `dedupedBy` to its root means deleting a middle copy
1220
+ * cannot make the tail look orphaned while the root is still counted
1221
+ * (review finding N11).
1222
+ *
1223
+ * @param {string} path
1224
+ * @returns {string | null} A path outside the chain that still exists.
1225
+ */
1226
+ const rootOriginal = (path) => {
1227
+ const seen = new Set([path]);
1228
+ let current = cached.files[path]?.dedupedBy;
1229
+ while (current) {
1230
+ if (seen.has(current)) return null;
1231
+ seen.add(current);
1232
+ const next = cached.files[current]?.dedupedBy;
1233
+ if (!next) return current;
1234
+ current = next;
1235
+ }
1236
+ return null;
1237
+ };
1238
+
1239
+ /** @type {Record<string, SessionFileEntry>} */
1240
+ const next = {};
1241
+ const pendingLate = cached.pendingLate;
1242
+ let changed = !cached.hit;
1243
+ let scanned = 0;
1244
+ let reused = 0;
1245
+ let bytesRead = 0;
1246
+ let partial = overBudget();
1247
+
1248
+ // The lock is refreshed as the scan walks the store: a cold scan of 1.5 GB
1249
+ // takes seconds, but a slower machine or a bigger store could exceed the
1250
+ // stale timeout, and another run would then take the lock over while this
1251
+ // one kept writing. `lostLock` makes that impossible.
1252
+ let lostLock = false;
1253
+ let filesSinceRefresh = 0;
1254
+ const keepLock = () => {
1255
+ filesSinceRefresh += 1;
1256
+ if (filesSinceRefresh < 8) return true;
1257
+ filesSinceRefresh = 0;
1258
+ if (refreshHistoryLock(lock)) return true;
1259
+ lostLock = true;
1260
+ return false;
1261
+ };
1262
+
1263
+ for (const path of listen) {
1264
+ if (!keepLock()) break;
1265
+ if (!partial && overBudget()) partial = true;
1266
+ if (partial) break;
1267
+
1268
+ let stats;
1269
+ try {
1270
+ stats = statSync(path);
1271
+ } catch (error) {
1272
+ if (cached.files[path]) next[path] = cached.files[path];
1273
+ warnings.push(`local token stats: could not stat ${path}: ${redact(/** @type {{ message?: string }} */ (error)?.message)}`);
1274
+ continue;
1275
+ }
1276
+
1277
+ // Reuse only a file that was read to its last complete line at exactly
1278
+ // this size and mtime. `offset < size` means a trailing partial line (or
1279
+ // bytes that arrived after the read reached EOF), and those must be
1280
+ // re-examined before the run can call the file unchanged.
1281
+ // A file that was skipped as a copy of another path has no rows of its
1282
+ // own: if the original is gone, this file is now the only holder of those
1283
+ // open days and must be read again (review finding N4).
1284
+ const cachedEntry = cached.files[path];
1285
+ const orphanedCopy = cachedEntry?.dedupedBy !== undefined && !listedSet.has(rootOriginal(path) ?? cachedEntry.dedupedBy);
1286
+ const previous = orphanedCopy ? undefined : cachedEntry;
1287
+ if (
1288
+ previous &&
1289
+ !rescanAll &&
1290
+ previous.size === stats.size &&
1291
+ previous.mtimeMs === stats.mtimeMs &&
1292
+ previous.ino === (stats.ino ?? 0) &&
1293
+ previous.complete &&
1294
+ previous.offset === previous.size
1295
+ ) {
1296
+ next[path] = previous;
1297
+ reused += 1;
1298
+ continue;
1299
+ }
1300
+
1301
+ // Reading from zero is the rescan case, and every one of these reasons
1302
+ // means "count the open days again, never the frozen ones":
1303
+ // - the cache is gone or unreadable (rescanAll);
1304
+ // - the file is a different inode or now shorter than what was read
1305
+ // (replaced, truncated);
1306
+ // - the cache has never seen this path. A copy, a move or a restored
1307
+ // backup looks exactly like a brand-new session, and its old-dated
1308
+ // messages were already frozen under the path it came from;
1309
+ // - the size is unchanged but the mtime moved: an in-place rewrite of
1310
+ // the same length, which reading from the old offset would miss.
1311
+ const seen = previous !== undefined;
1312
+ const replaced = seen && (previous.ino !== (stats.ino ?? 0) || stats.size < previous.offset);
1313
+ const rewritten = seen && previous.size === stats.size && previous.mtimeMs !== stats.mtimeMs;
1314
+ const rescan = rescanAll || !seen || replaced || rewritten;
1315
+
1316
+ // A byte-identical copy of a file that is already counted contributes
1317
+ // nothing: its messages sit in the original's rows. The entry is still
1318
+ // recorded at the copy's end so anything appended to it later is counted.
1319
+ const fingerprint = rescan ? fingerprintFile(path, stats.size) : (previous?.fingerprint ?? null);
1320
+ // Its own fingerprint in the map is not a copy: a rescan of the same file
1321
+ // (a lost history, a rewritten file) must fold its days again.
1322
+ const twin = rescan && fingerprint ? counted.get(fingerprint) : undefined;
1323
+ const candidate = twin === path ? undefined : twin;
1324
+ // The fingerprint samples the head and the tail, which is cheap but can
1325
+ // collide: two sessions of the same size that differ only in the middle
1326
+ // look identical (review finding N1). Only a candidate that already
1327
+ // matched the sample is compared in full, so the expensive hash runs on
1328
+ // the rare duplicate and never on a normal run.
1329
+ const duplicateOf = candidate !== undefined && sameContent(candidate, path, stats.size) ? candidate : undefined;
1330
+ if (rescan && fingerprint && candidate === undefined) counted.set(fingerprint, path);
1331
+ if (duplicateOf !== undefined) {
1332
+ // Point at the root of the chain, not at whatever file happened to
1333
+ // match: a copy of a copy must survive the middle link being deleted.
1334
+ const root = rootOriginal(duplicateOf) ?? duplicateOf;
1335
+ next[path] = {
1336
+ size: stats.size,
1337
+ mtimeMs: stats.mtimeMs,
1338
+ ino: stats.ino ?? 0,
1339
+ offset: stats.size,
1340
+ complete: true,
1341
+ fingerprint,
1342
+ dedupedBy: root,
1343
+ days: {},
1344
+ };
1345
+ changed = true;
1346
+ reused += 1;
1347
+ continue;
1348
+ }
1349
+
1350
+ const days = rescan ? emptyDays() : decodeDays(previous.days);
1351
+ const startOffset = rescan ? 0 : previous.offset;
1352
+ let offset;
1353
+ let complete;
1354
+ try {
1355
+ const result = await scanFile({
1356
+ path,
1357
+ startOffset,
1358
+ days,
1359
+ now,
1360
+ shouldStop: overBudget,
1361
+ onUsage: (usage) => {
1362
+ if (lastFrozenDay && usage.day <= lastFrozenDay) {
1363
+ // Frozen days are never reconstructed on a rescan. Incremental
1364
+ // late writes stay in the cache until their month accepts them.
1365
+ if (rescan) return;
1366
+ addUsage(pendingLate, usage);
1367
+ return;
1368
+ }
1369
+ addUsage(days, usage);
1370
+ },
1371
+ });
1372
+ offset = result.offset;
1373
+ complete = result.complete;
1374
+ bytesRead += offset - startOffset;
1375
+ } catch (error) {
1376
+ // A transient read error keeps the last known contribution instead of
1377
+ // silently deleting history for that session.
1378
+ if (previous) next[path] = previous;
1379
+ warnings.push(`local token stats: could not read ${path}: ${redact(/** @type {{ message?: string }} */ (error)?.message)}`);
1380
+ continue;
1381
+ }
1382
+
1383
+ let finalStats = stats;
1384
+ if (complete) {
1385
+ try {
1386
+ finalStats = statSync(path);
1387
+ } catch {
1388
+ // Keep the pre-scan stat; the next run detects the change.
1389
+ }
1390
+ }
1391
+ next[path] = {
1392
+ size: finalStats.size,
1393
+ mtimeMs: finalStats.mtimeMs,
1394
+ ino: finalStats.ino ?? 0,
1395
+ offset,
1396
+ complete,
1397
+ ...(fingerprint ? { fingerprint } : {}),
1398
+ days: encodeDays(days),
1399
+ };
1400
+ changed = true;
1401
+ scanned += 1;
1402
+ if (!complete) {
1403
+ partial = true;
1404
+ break;
1405
+ }
1406
+ }
1407
+
1408
+ // Files the budget skipped keep their previous entry; files that are gone
1409
+ // lose theirs on this run.
1410
+ for (const path of listen) {
1411
+ if (!next[path] && cached.files[path]) next[path] = cached.files[path];
1412
+ }
1413
+ for (const path of Object.keys(cached.files)) {
1414
+ if (!listedSet.has(path)) changed = true;
1415
+ }
1416
+
1417
+ // Ownership is re-checked here, not only every eight files: a scan of a
1418
+ // handful of small files reaches this point without a refresh point, and a
1419
+ // run that lost the lock must not write (review finding N8).
1420
+ if (!lostLock && !refreshHistoryLock(lock)) lostLock = true;
1421
+
1422
+ // A run that lost the lock must not write anything: another run now owns
1423
+ // the history, and a second write is exactly the double count the lock
1424
+ // exists to prevent.
1425
+ if (lostLock) {
1426
+ warnings.push(
1427
+ "local token stats: another piquota run took the history lock over during this scan; the numbers below are read-only",
1428
+ );
1429
+ const windowDays = loadWindowDays({
1430
+ historyDir,
1431
+ dates: recentLocalDates(now, RECENT_DAYS),
1432
+ warnings,
1433
+ canWrite: false,
1434
+ });
1435
+ return {
1436
+ byFamily: buildLocalStats({ totals: history.totals, openRows: buildOpenRows(next), windowDays, now }),
1437
+ warnings,
1438
+ scan: {
1439
+ files: listen.length,
1440
+ scanned,
1441
+ reused,
1442
+ bytesRead,
1443
+ durationMs: clock() - startedAt,
1444
+ partial: true,
1445
+ budgetMs,
1446
+ frozenDays: 0,
1447
+ locked: true,
1448
+ },
1449
+ };
1450
+ }
1451
+
1452
+ const openRows = buildOpenRows(next);
1453
+ const pendingBeforeApply = pendingLate.size;
1454
+ const applied = applyDurable({
1455
+ historyDir,
1456
+ totals: history.totals,
1457
+ openRows,
1458
+ lateRows: buildLateRows(pendingLate),
1459
+ today,
1460
+ partial,
1461
+ writable: history.writable,
1462
+ ensure: !history.exists,
1463
+ warnings,
1464
+ });
1465
+ for (const day of pendingLate.keys()) {
1466
+ if (!applied.pendingLateDays.has(day)) pendingLate.delete(day);
1467
+ }
1468
+ if (pendingLate.size !== pendingBeforeApply) changed = true;
1469
+
1470
+ // A day that is in the rollup (frozen now, or already frozen before a crash)
1471
+ // must leave the disposable cache; otherwise a rescan would see it twice.
1472
+ if (applied.droppedDays.size > 0) {
1473
+ for (const entry of Object.values(next)) {
1474
+ for (const day of applied.droppedDays) {
1475
+ if (entry.days && day in entry.days) delete entry.days[day];
1476
+ }
1477
+ }
1478
+ for (const day of applied.droppedDays) delete openRows[day];
1479
+ changed = true;
1480
+ }
1481
+
1482
+ if (changed) {
1483
+ const written = refreshHistoryLock(lock)
1484
+ ? writeCacheFile(cachePath, next, history.totals.lastFrozenDay, encodeDays(pendingLate))
1485
+ : { ok: true };
1486
+ if (!written.ok) {
1487
+ warnings.push(`local token stats: could not write the scan cache at ${cachePath}: ${written.error}`);
1488
+ }
1489
+ }
1490
+
1491
+ const windowDays = loadWindowDays({ historyDir, dates: recentLocalDates(now, RECENT_DAYS), warnings });
1492
+
1493
+ return {
1494
+ byFamily: buildLocalStats({ totals: history.totals, openRows, windowDays, now }),
1495
+ warnings,
1496
+ scan: {
1497
+ files: listen.length,
1498
+ scanned,
1499
+ reused,
1500
+ bytesRead,
1501
+ durationMs: clock() - startedAt,
1502
+ partial,
1503
+ budgetMs,
1504
+ frozenDays: applied.frozenDays.length,
1505
+ locked: false,
1506
+ },
1507
+ };
1508
+ } finally {
1509
+ releaseHistoryLock(lock);
1510
+ }
1511
+ }