@cad0p/pi-usage 0.11.3 → 0.11.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +28 -0
- package/README.md +274 -12
- package/bin/piquota.js +49 -4
- package/extensions/quota-panel.ts +188 -31
- package/omarchy/cad0p.pi-usage/Agent.qml +33 -0
- package/omarchy/cad0p.pi-usage/Main.qml +333 -0
- package/omarchy/cad0p.pi-usage/Panel.qml +136 -0
- package/omarchy/cad0p.pi-usage/README.md +40 -0
- package/omarchy/cad0p.pi-usage/UsageContent.qml +1001 -0
- package/omarchy/cad0p.pi-usage/UsageLogic.js +603 -0
- package/omarchy/cad0p.pi-usage/assets/antigravity.svg +3 -0
- package/omarchy/cad0p.pi-usage/assets/claude.svg +1 -0
- package/omarchy/cad0p.pi-usage/assets/codex-light.svg +1 -0
- package/omarchy/cad0p.pi-usage/assets/codex.svg +1 -0
- package/omarchy/cad0p.pi-usage/assets/fireworks.svg +7 -0
- package/omarchy/cad0p.pi-usage/assets/github-actions.svg +3 -0
- package/omarchy/cad0p.pi-usage/assets/opencode-go.svg +3 -0
- package/omarchy/cad0p.pi-usage/manifest.json +56 -0
- package/package.json +2 -1
- package/src/auth/pi-auth.js +3 -0
- package/src/cli/args.js +5 -0
- package/src/cli/commands/stats.js +76 -0
- package/src/engine.js +44 -0
- package/src/local/history.js +1160 -0
- package/src/local/pi-sessions.js +1511 -0
- package/src/omarchy/publish.js +1 -65
- package/src/omarchy/record.js +82 -7
|
@@ -0,0 +1,1511 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Local token statistics from Pi's own session files.
|
|
3
|
+
*
|
|
4
|
+
* Every assistant message Pi records carries the provider, the model and a
|
|
5
|
+
* usage block, so the tokens a subscription actually burned are already on
|
|
6
|
+
* disk. This module folds those messages into the per-family shape Omarchy's
|
|
7
|
+
* Agents panel renders ("tokens by day", "tokens by model", today's totals),
|
|
8
|
+
* and does it at a bounded cost: the first run streams every session file once,
|
|
9
|
+
* later runs read only the bytes appended since the last scan.
|
|
10
|
+
*
|
|
11
|
+
* Two stores, two lifetimes:
|
|
12
|
+
* - the disposable per-file cache under `~/.cache/pi-quota/local-sessions.json`
|
|
13
|
+
* holds the (size, mtimeMs, inode, offset) of every session file plus that
|
|
14
|
+
* file's aggregates for OPEN days only (normally just today), so an append
|
|
15
|
+
* is read once and nothing is ever parsed twice;
|
|
16
|
+
* - the durable history under `${PI_CODING_AGENT_DIR:-~/.pi/agent}/pi-usage/`
|
|
17
|
+
* (see history.js) holds every closed day. Freezing is one-way: a frozen
|
|
18
|
+
* day is never recomputed from a rescan, never rewritten by a session
|
|
19
|
+
* deletion and never lost with the cache.
|
|
20
|
+
*
|
|
21
|
+
* Hard rules enforced here:
|
|
22
|
+
* - session files are opened strictly read-only and never modified;
|
|
23
|
+
* - nothing but aggregate counts (tokens, prompts, dates, model names) and
|
|
24
|
+
* the file keys needed to scan incrementally reaches a cache, a record or
|
|
25
|
+
* the report, so no prompt text, tool output or credential is ever stored;
|
|
26
|
+
* - the scan is best-effort: a missing root, an unreadable file or a corrupt
|
|
27
|
+
* cache/history degrades to fewer numbers and a warning, never to a
|
|
28
|
+
* provider failure.
|
|
29
|
+
*
|
|
30
|
+
* Duplicates: a fork or rewind is a new session file, so in principle the same
|
|
31
|
+
* assistant message could be counted twice. Measured on a real store (603 files,
|
|
32
|
+
* 185,626 assistant messages) only 11 message ids appeared in more than one file
|
|
33
|
+
* (0.006%), and none was a copied history. An id index would cost megabytes of
|
|
34
|
+
* memory to correct a handful of tokens, and no bounded window can recognise a
|
|
35
|
+
* fork that replays an old history, so every message is counted once per line
|
|
36
|
+
* read. If Pi ever starts copying whole histories into forks, that overstates
|
|
37
|
+
* usage (the conservative direction) instead of hiding it.
|
|
38
|
+
*/
|
|
39
|
+
|
|
40
|
+
import { createHash } from "node:crypto";
|
|
41
|
+
import {
|
|
42
|
+
closeSync,
|
|
43
|
+
createReadStream,
|
|
44
|
+
mkdirSync,
|
|
45
|
+
openSync,
|
|
46
|
+
readSync,
|
|
47
|
+
readdirSync,
|
|
48
|
+
readFileSync,
|
|
49
|
+
realpathSync,
|
|
50
|
+
renameSync,
|
|
51
|
+
rmSync,
|
|
52
|
+
statSync,
|
|
53
|
+
writeFileSync,
|
|
54
|
+
} from "node:fs";
|
|
55
|
+
import { homedir } from "node:os";
|
|
56
|
+
import { dirname, join } from "node:path";
|
|
57
|
+
|
|
58
|
+
import { FAMILY_BY_PROVIDER, resolveProviderSlot } from "../auth/pi-auth.js";
|
|
59
|
+
import { redact } from "../http.js";
|
|
60
|
+
import {
|
|
61
|
+
acquireHistoryLock,
|
|
62
|
+
addToModels,
|
|
63
|
+
applyDurable,
|
|
64
|
+
bucketMessages,
|
|
65
|
+
bucketTokens,
|
|
66
|
+
emptyBucket,
|
|
67
|
+
emptyTotals,
|
|
68
|
+
familyRowMessages,
|
|
69
|
+
familyRowTokens,
|
|
70
|
+
loadTotals,
|
|
71
|
+
loadWindowDays,
|
|
72
|
+
refreshHistoryLock,
|
|
73
|
+
releaseHistoryLock,
|
|
74
|
+
resolveHistoryDir,
|
|
75
|
+
} from "./history.js";
|
|
76
|
+
|
|
77
|
+
/** Version of the disposable per-file cache; a foreign version is a cache miss. */
|
|
78
|
+
export const LOCAL_STATS_SCHEMA_VERSION = 2;
|
|
79
|
+
|
|
80
|
+
/**
|
|
81
|
+
* How long a scan may run before it returns what it has. The /usage extension
|
|
82
|
+
* runs the CLI with a 30 s timeout and a first full scan of ~1.5 GB costs a few
|
|
83
|
+
* seconds, so this leaves room for the quota calls that run beside it. A
|
|
84
|
+
* truncated scan never freezes a day: history is written only from complete runs.
|
|
85
|
+
*/
|
|
86
|
+
export const DEFAULT_SCAN_BUDGET_MS = 12_000;
|
|
87
|
+
|
|
88
|
+
/** Session formats we can fold: only JSONL files, only message entries. */
|
|
89
|
+
const SESSION_SUFFIX = ".jsonl";
|
|
90
|
+
|
|
91
|
+
/** Number of local days "recentDays" carries, oldest first, today last. */
|
|
92
|
+
const RECENT_DAYS = 7;
|
|
93
|
+
|
|
94
|
+
/**
|
|
95
|
+
* @typedef {import("./history.js").Bucket} Bucket
|
|
96
|
+
* @typedef {import("./history.js").DayRows} DayRows
|
|
97
|
+
*/
|
|
98
|
+
|
|
99
|
+
/**
|
|
100
|
+
* The Omarchy-shaped local statistics for one family, plus the per-account
|
|
101
|
+
* breakdown pi's renderers use next to each account line.
|
|
102
|
+
*
|
|
103
|
+
* @typedef {Object} LocalFamilyStats
|
|
104
|
+
* @property {true} hasLocalStats
|
|
105
|
+
* @property {true} hasPromptStats
|
|
106
|
+
* @property {number} todayPrompts
|
|
107
|
+
* @property {number} todaySessions
|
|
108
|
+
* @property {number} todayTotalTokens
|
|
109
|
+
* @property {Record<string, number>} todayTokensByModel
|
|
110
|
+
* @property {Record<string, number>} weekTokensByModel Same, over the last 7 local days.
|
|
111
|
+
* @property {Array<{ date: string, messageCount: number }>} recentDays
|
|
112
|
+
* @property {number} totalPrompts
|
|
113
|
+
* @property {number} totalSessions Session-days: a session spanning midnight counts once per day.
|
|
114
|
+
* @property {number} activeDays
|
|
115
|
+
* @property {string[]} activeDates
|
|
116
|
+
* @property {Record<string, { inputTokens: number, outputTokens: number, cacheReadInputTokens: number, cacheCreationInputTokens: number }>} modelUsage
|
|
117
|
+
* @property {Record<string, { todayTokens: number, weekTokens: number, todayPrompts: number, totalPrompts: number, totalTokens: number }>} byAccount
|
|
118
|
+
*/
|
|
119
|
+
|
|
120
|
+
/**
|
|
121
|
+
* How the scan went, for `localStatsMeta` in the report.
|
|
122
|
+
*
|
|
123
|
+
* @typedef {Object} LocalScanMeta
|
|
124
|
+
* @property {number} files Session files found.
|
|
125
|
+
* @property {number} scanned Files (re)read in this run.
|
|
126
|
+
* @property {number} reused Files whose cached offsets still matched.
|
|
127
|
+
* @property {number} bytesRead Bytes parsed in this run.
|
|
128
|
+
* @property {number} durationMs Scan wall time.
|
|
129
|
+
* @property {boolean} partial The budget stopped the scan; nothing was frozen.
|
|
130
|
+
* @property {number} budgetMs The budget that applied.
|
|
131
|
+
* @property {number} frozenDays Days moved into the durable history by this run.
|
|
132
|
+
* @property {boolean} locked Another run held the history lock; nothing was written.
|
|
133
|
+
*/
|
|
134
|
+
|
|
135
|
+
/**
|
|
136
|
+
* Options for a local-stats scan, as accepted by the engine's `localStats` option.
|
|
137
|
+
*
|
|
138
|
+
* @typedef {Object} LocalStatsOptions
|
|
139
|
+
* @property {string[]} [roots] Session roots to scan; overrides the Pi/omp defaults.
|
|
140
|
+
* @property {string} [cachePath] Scan cache location; overrides ~/.cache/pi-quota/local-sessions.json.
|
|
141
|
+
* @property {string} [historyDir] Durable history directory; overrides ${PI_CODING_AGENT_DIR:-~/.pi/agent}/pi-usage.
|
|
142
|
+
* @property {Record<string, string | undefined>} [env]
|
|
143
|
+
* @property {string} [home]
|
|
144
|
+
* @property {number} [now] Report time; decides which local day is "today".
|
|
145
|
+
* @property {number} [budgetMs] Wall-clock budget for this scan.
|
|
146
|
+
* @property {() => number} [clock] Time source; tests inject one to drive the budget.
|
|
147
|
+
*/
|
|
148
|
+
|
|
149
|
+
/**
|
|
150
|
+
* One session file's cache entry.
|
|
151
|
+
*
|
|
152
|
+
* `offset` is the byte offset of the first byte that is not part of a complete
|
|
153
|
+
* line (end of file when the last line ended with a newline). `complete` is
|
|
154
|
+
* false when the scan budget stopped the read mid-file; the next run continues
|
|
155
|
+
* from `offset` instead of assuming the file is finished. `days` holds open
|
|
156
|
+
* days only: a frozen day is removed from the entry once the durable write
|
|
157
|
+
* succeeded, so a rescan can never recompute it.
|
|
158
|
+
*
|
|
159
|
+
* @typedef {Object} SessionFileEntry
|
|
160
|
+
* @property {number} size
|
|
161
|
+
* @property {number} mtimeMs
|
|
162
|
+
* @property {number} ino
|
|
163
|
+
* @property {number} offset
|
|
164
|
+
* @property {boolean} complete
|
|
165
|
+
* @property {string} [fingerprint] Copy fingerprint of the content at `size`
|
|
166
|
+
* (see fingerprintFile), only set after a full read.
|
|
167
|
+
* @property {string} [dedupedBy] Path of the file this one was verified to be
|
|
168
|
+
* a byte-identical copy of; it holds no rows of
|
|
169
|
+
* its own while that path exists.
|
|
170
|
+
* @property {Record<string, Array<[string, string, string, number, number, number, number, number]>>} days
|
|
171
|
+
*/
|
|
172
|
+
|
|
173
|
+
/**
|
|
174
|
+
* @param {Record<string, string | undefined>} env
|
|
175
|
+
* @param {string | undefined} home
|
|
176
|
+
* @returns {string}
|
|
177
|
+
*/
|
|
178
|
+
function cacheHome(env, home) {
|
|
179
|
+
return env.XDG_CACHE_HOME || join(home ?? homedir(), ".cache");
|
|
180
|
+
}
|
|
181
|
+
|
|
182
|
+
/**
|
|
183
|
+
* Where the disposable per-file scan cache lives.
|
|
184
|
+
*
|
|
185
|
+
* @param {{ env?: Record<string, string | undefined>, home?: string }} [options]
|
|
186
|
+
* @returns {string}
|
|
187
|
+
*/
|
|
188
|
+
export function resolveLocalStatsCachePath(options = {}) {
|
|
189
|
+
const env = options.env ?? process.env;
|
|
190
|
+
return join(cacheHome(env, options.home), "pi-quota", "local-sessions.json");
|
|
191
|
+
}
|
|
192
|
+
|
|
193
|
+
/**
|
|
194
|
+
* Session roots to scan, in the order Pi itself writes them:
|
|
195
|
+
* - `$PI_CODING_AGENT_DIR/sessions` when the env var is set;
|
|
196
|
+
* - `~/.pi/agent/sessions` always;
|
|
197
|
+
* - `~/.omp/agent/sessions` when the omp fork is installed.
|
|
198
|
+
*
|
|
199
|
+
* Only directories that exist are returned, so a machine without Pi simply
|
|
200
|
+
* reports no local statistics instead of a warning.
|
|
201
|
+
*
|
|
202
|
+
* @param {{ env?: Record<string, string | undefined>, home?: string }} [options]
|
|
203
|
+
* @returns {string[]}
|
|
204
|
+
*/
|
|
205
|
+
export function resolveSessionRoots(options = {}) {
|
|
206
|
+
const env = options.env ?? process.env;
|
|
207
|
+
const home = options.home ?? homedir();
|
|
208
|
+
/** @type {string[]} */
|
|
209
|
+
const candidates = [];
|
|
210
|
+
if (env.PI_CODING_AGENT_DIR) candidates.push(join(env.PI_CODING_AGENT_DIR, "sessions"));
|
|
211
|
+
candidates.push(join(home, ".pi", "agent", "sessions"));
|
|
212
|
+
candidates.push(join(home, ".omp", "agent", "sessions"));
|
|
213
|
+
|
|
214
|
+
/** @type {string[]} */
|
|
215
|
+
const roots = [];
|
|
216
|
+
const seen = new Set();
|
|
217
|
+
for (const candidate of candidates) {
|
|
218
|
+
let key = candidate;
|
|
219
|
+
try {
|
|
220
|
+
// Two roots can name the same directory: `PI_CODING_AGENT_DIR` may point
|
|
221
|
+
// at a symlink of `~/.pi/agent`, and scanning that twice would count
|
|
222
|
+
// every message twice.
|
|
223
|
+
key = realpathSync(candidate);
|
|
224
|
+
if (seen.has(key)) continue;
|
|
225
|
+
if (!statSync(candidate).isDirectory()) continue;
|
|
226
|
+
} catch {
|
|
227
|
+
// Missing root: not an error, just nothing to scan there.
|
|
228
|
+
continue;
|
|
229
|
+
}
|
|
230
|
+
seen.add(key);
|
|
231
|
+
roots.push(candidate);
|
|
232
|
+
}
|
|
233
|
+
return roots;
|
|
234
|
+
}
|
|
235
|
+
|
|
236
|
+
/**
|
|
237
|
+
* Local calendar date for an epoch value, as YYYY-MM-DD. Dates are local
|
|
238
|
+
* because the panel draws a local week and calls the last row "Today".
|
|
239
|
+
*
|
|
240
|
+
* @param {number} ms
|
|
241
|
+
* @returns {string}
|
|
242
|
+
*/
|
|
243
|
+
export function localDayKey(ms) {
|
|
244
|
+
const date = new Date(ms);
|
|
245
|
+
const year = date.getFullYear();
|
|
246
|
+
const month = String(date.getMonth() + 1).padStart(2, "0");
|
|
247
|
+
const day = String(date.getDate()).padStart(2, "0");
|
|
248
|
+
return `${year}-${month}-${day}`;
|
|
249
|
+
}
|
|
250
|
+
|
|
251
|
+
/**
|
|
252
|
+
* Explicit roots bypass `resolveSessionRoots`, so de-duplicate them by real path
|
|
253
|
+
* here too: a caller that names the same store twice must not double count.
|
|
254
|
+
*
|
|
255
|
+
* @param {string[]} roots
|
|
256
|
+
* @returns {string[]}
|
|
257
|
+
*/
|
|
258
|
+
function dedupeRoots(roots) {
|
|
259
|
+
/** @type {string[]} */
|
|
260
|
+
const unique = [];
|
|
261
|
+
const seen = new Set();
|
|
262
|
+
for (const root of roots) {
|
|
263
|
+
let key = root;
|
|
264
|
+
try {
|
|
265
|
+
key = realpathSync(root);
|
|
266
|
+
} catch {
|
|
267
|
+
// Keep the unresolved path; listSessionFiles skips missing roots.
|
|
268
|
+
}
|
|
269
|
+
if (seen.has(key)) continue;
|
|
270
|
+
seen.add(key);
|
|
271
|
+
unique.push(root);
|
|
272
|
+
}
|
|
273
|
+
return unique;
|
|
274
|
+
}
|
|
275
|
+
|
|
276
|
+
/**
|
|
277
|
+
* @param {number} now
|
|
278
|
+
* @param {number} count
|
|
279
|
+
* @returns {string[]} Oldest first, today last.
|
|
280
|
+
*/
|
|
281
|
+
function recentLocalDates(now, count) {
|
|
282
|
+
const cursor = new Date(now);
|
|
283
|
+
// Noon keeps the walk away from DST midnights, where "one day back" can land
|
|
284
|
+
// on the same calendar date.
|
|
285
|
+
cursor.setHours(12, 0, 0, 0);
|
|
286
|
+
/** @type {string[]} */
|
|
287
|
+
const dates = [];
|
|
288
|
+
for (let offset = count - 1; offset >= 0; offset -= 1) {
|
|
289
|
+
const date = new Date(cursor);
|
|
290
|
+
date.setDate(cursor.getDate() - offset);
|
|
291
|
+
dates.push(localDayKey(date.getTime()));
|
|
292
|
+
}
|
|
293
|
+
return dates;
|
|
294
|
+
}
|
|
295
|
+
|
|
296
|
+
/**
|
|
297
|
+
* @param {unknown} value
|
|
298
|
+
* @returns {number}
|
|
299
|
+
*/
|
|
300
|
+
function count(value) {
|
|
301
|
+
const number = Number(value);
|
|
302
|
+
if (!Number.isFinite(number) || number <= 0) return 0;
|
|
303
|
+
return Math.trunc(number);
|
|
304
|
+
}
|
|
305
|
+
|
|
306
|
+
/**
|
|
307
|
+
* @param {unknown} value
|
|
308
|
+
* @returns {string}
|
|
309
|
+
*/
|
|
310
|
+
function modelName(value) {
|
|
311
|
+
if (typeof value !== "string") return "unknown";
|
|
312
|
+
const text = value.trim().replace(/[\u0000-\u001f]/g, "");
|
|
313
|
+
return text === "" ? "unknown" : text.slice(0, 120);
|
|
314
|
+
}
|
|
315
|
+
|
|
316
|
+
/**
|
|
317
|
+
* Map a message onto a quota family and account slot.
|
|
318
|
+
*
|
|
319
|
+
* Pi's own provider id is authoritative: it is the only thing that identifies a
|
|
320
|
+
* numbered sibling (`opencode-go-7`). The `api` string is the fallback Pi's
|
|
321
|
+
* Omarchy collector uses for Codex, generalised: it only ever names a base
|
|
322
|
+
* provider, never a slot.
|
|
323
|
+
*
|
|
324
|
+
* @param {unknown} provider
|
|
325
|
+
* @param {unknown} api
|
|
326
|
+
* @returns {{ family: string, slot: string | null } | null}
|
|
327
|
+
*/
|
|
328
|
+
function familyForMessage(provider, api) {
|
|
329
|
+
const direct = resolveProviderSlot(typeof provider === "string" ? provider : "");
|
|
330
|
+
if (direct) return { family: direct.family, slot: direct.slot };
|
|
331
|
+
if (typeof api !== "string" || api === "") return null;
|
|
332
|
+
for (const base of Object.keys(FAMILY_BY_PROVIDER)) {
|
|
333
|
+
if (api === base || api.startsWith(`${base}-`) || api.startsWith(`${base}:`)) {
|
|
334
|
+
const resolved = resolveProviderSlot(base);
|
|
335
|
+
if (resolved) return { family: resolved.family, slot: resolved.slot };
|
|
336
|
+
}
|
|
337
|
+
}
|
|
338
|
+
return null;
|
|
339
|
+
}
|
|
340
|
+
|
|
341
|
+
/**
|
|
342
|
+
* Timestamp of a message entry. Pi writes an ISO string on the entry and epoch
|
|
343
|
+
* milliseconds on the message; second-epoch values are accepted because a
|
|
344
|
+
* hand-written or imported session can carry them.
|
|
345
|
+
*
|
|
346
|
+
* @param {Record<string, any>} entry
|
|
347
|
+
* @param {number} fallback
|
|
348
|
+
* @returns {number}
|
|
349
|
+
*/
|
|
350
|
+
function entryTimeMs(entry, fallback) {
|
|
351
|
+
const candidates = [entry.timestamp, entry.message?.timestamp];
|
|
352
|
+
for (const candidate of candidates) {
|
|
353
|
+
if (typeof candidate === "number" && Number.isFinite(candidate) && candidate > 0) {
|
|
354
|
+
return candidate > 1e11 ? candidate : candidate * 1000;
|
|
355
|
+
}
|
|
356
|
+
if (typeof candidate === "string" && candidate.trim() !== "") {
|
|
357
|
+
const parsed = Date.parse(candidate);
|
|
358
|
+
if (!Number.isNaN(parsed)) return parsed;
|
|
359
|
+
}
|
|
360
|
+
}
|
|
361
|
+
return fallback;
|
|
362
|
+
}
|
|
363
|
+
|
|
364
|
+
/**
|
|
365
|
+
* Parse one session line into a usage record, or null when the line is not an
|
|
366
|
+
* assistant message this module counts.
|
|
367
|
+
*
|
|
368
|
+
* A message with no tokens at all is not a prompt: Pi records aborted turns and
|
|
369
|
+
* provider errors this way, and counting them would inflate "today's prompts"
|
|
370
|
+
* without adding a single token. Omarchy's collector skips them for the same
|
|
371
|
+
* reason. When only `totalTokens` is present, it is attributed to input, which
|
|
372
|
+
* is how the panel's own totals stay non-zero.
|
|
373
|
+
*
|
|
374
|
+
* @param {string} line
|
|
375
|
+
* @param {number} now
|
|
376
|
+
* @returns {{ day: string, family: string, slot: string, model: string, input: number, output: number, cacheRead: number, cacheWrite: number } | null}
|
|
377
|
+
*/
|
|
378
|
+
export function usageFromLine(line, now = Date.now()) {
|
|
379
|
+
// A cheap pre-filter before JSON.parse: the session files are ~1.5 GB and
|
|
380
|
+
// most lines are custom/context entries that can never be an assistant
|
|
381
|
+
// message. Safe because an assistant message always serializes the literal.
|
|
382
|
+
if (!line.includes('"assistant"')) return null;
|
|
383
|
+
|
|
384
|
+
let entry;
|
|
385
|
+
try {
|
|
386
|
+
entry = JSON.parse(line);
|
|
387
|
+
} catch {
|
|
388
|
+
return null;
|
|
389
|
+
}
|
|
390
|
+
if (!entry || entry.type !== "message") return null;
|
|
391
|
+
const message = entry.message;
|
|
392
|
+
if (!message || message.role !== "assistant") return null;
|
|
393
|
+
const usage = message.usage;
|
|
394
|
+
if (!usage || typeof usage !== "object" || Array.isArray(usage)) return null;
|
|
395
|
+
|
|
396
|
+
const resolved = familyForMessage(message.provider, message.api);
|
|
397
|
+
if (!resolved) return null;
|
|
398
|
+
|
|
399
|
+
let input = count(usage.input);
|
|
400
|
+
const output = count(usage.output);
|
|
401
|
+
const cacheRead = count(usage.cacheRead);
|
|
402
|
+
const cacheWrite = count(usage.cacheWrite);
|
|
403
|
+
if (input + output + cacheRead + cacheWrite === 0) {
|
|
404
|
+
const reported = count(usage.totalTokens);
|
|
405
|
+
if (reported === 0) return null;
|
|
406
|
+
input = reported;
|
|
407
|
+
}
|
|
408
|
+
|
|
409
|
+
return {
|
|
410
|
+
day: localDayKey(entryTimeMs(entry, now)),
|
|
411
|
+
family: resolved.family,
|
|
412
|
+
slot: resolved.slot ?? "",
|
|
413
|
+
model: modelName(message.model),
|
|
414
|
+
input,
|
|
415
|
+
output,
|
|
416
|
+
cacheRead,
|
|
417
|
+
cacheWrite,
|
|
418
|
+
};
|
|
419
|
+
}
|
|
420
|
+
|
|
421
|
+
/**
|
|
422
|
+
* Per-file open-day aggregates while scanning: day -> key -> tuple.
|
|
423
|
+
* The tuple is the serialized form `[family, slot, model, input, output,
|
|
424
|
+
* cacheRead, cacheWrite, prompts]`, which keeps the cache compact.
|
|
425
|
+
*
|
|
426
|
+
* @typedef {Map<string, Map<string, [string, string, string, number, number, number, number, number]>>} DaysMap
|
|
427
|
+
*/
|
|
428
|
+
|
|
429
|
+
/**
|
|
430
|
+
* @returns {DaysMap}
|
|
431
|
+
*/
|
|
432
|
+
function emptyDays() {
|
|
433
|
+
return new Map();
|
|
434
|
+
}
|
|
435
|
+
|
|
436
|
+
/**
|
|
437
|
+
* @param {DaysMap} days
|
|
438
|
+
* @param {NonNullable<ReturnType<typeof usageFromLine>>} usage
|
|
439
|
+
*/
|
|
440
|
+
function addUsage(days, usage) {
|
|
441
|
+
let bucket = days.get(usage.day);
|
|
442
|
+
if (!bucket) {
|
|
443
|
+
bucket = new Map();
|
|
444
|
+
days.set(usage.day, bucket);
|
|
445
|
+
}
|
|
446
|
+
const key = `${usage.family}\u0000${usage.slot}\u0000${usage.model}`;
|
|
447
|
+
const tuple = bucket.get(key);
|
|
448
|
+
if (tuple) {
|
|
449
|
+
tuple[3] += usage.input;
|
|
450
|
+
tuple[4] += usage.output;
|
|
451
|
+
tuple[5] += usage.cacheRead;
|
|
452
|
+
tuple[6] += usage.cacheWrite;
|
|
453
|
+
tuple[7] += 1;
|
|
454
|
+
return;
|
|
455
|
+
}
|
|
456
|
+
bucket.set(key, [usage.family, usage.slot, usage.model, usage.input, usage.output, usage.cacheRead, usage.cacheWrite, 1]);
|
|
457
|
+
}
|
|
458
|
+
|
|
459
|
+
/**
|
|
460
|
+
* @param {DaysMap} days
|
|
461
|
+
* @returns {SessionFileEntry["days"]}
|
|
462
|
+
*/
|
|
463
|
+
function encodeDays(days) {
|
|
464
|
+
/** @type {SessionFileEntry["days"]} */
|
|
465
|
+
const encoded = {};
|
|
466
|
+
for (const [day, bucket] of days) encoded[day] = Array.from(bucket.values());
|
|
467
|
+
return encoded;
|
|
468
|
+
}
|
|
469
|
+
|
|
470
|
+
/**
|
|
471
|
+
* @param {unknown} raw
|
|
472
|
+
* @returns {DaysMap}
|
|
473
|
+
*/
|
|
474
|
+
function decodeDays(raw) {
|
|
475
|
+
const days = emptyDays();
|
|
476
|
+
if (!raw || typeof raw !== "object" || Array.isArray(raw)) return days;
|
|
477
|
+
for (const [day, tuples] of Object.entries(raw)) {
|
|
478
|
+
if (!Array.isArray(tuples)) continue;
|
|
479
|
+
const bucket = new Map();
|
|
480
|
+
for (const tuple of tuples) {
|
|
481
|
+
if (!Array.isArray(tuple) || tuple.length < 8) continue;
|
|
482
|
+
const [family, slot, model] = tuple;
|
|
483
|
+
if (typeof family !== "string" || typeof slot !== "string" || typeof model !== "string") continue;
|
|
484
|
+
const numbers = [tuple[3], tuple[4], tuple[5], tuple[6], tuple[7]].map(count);
|
|
485
|
+
bucket.set(`${family}\u0000${slot}\u0000${model}`, [
|
|
486
|
+
family,
|
|
487
|
+
slot,
|
|
488
|
+
model,
|
|
489
|
+
numbers[0],
|
|
490
|
+
numbers[1],
|
|
491
|
+
numbers[2],
|
|
492
|
+
numbers[3],
|
|
493
|
+
numbers[4],
|
|
494
|
+
]);
|
|
495
|
+
}
|
|
496
|
+
if (bucket.size > 0) days.set(day, bucket);
|
|
497
|
+
}
|
|
498
|
+
return days;
|
|
499
|
+
}
|
|
500
|
+
|
|
501
|
+
/**
|
|
502
|
+
* Stream one session file from `startOffset`, handing every complete line to
|
|
503
|
+
* `onLine` and stopping early when `shouldStop` asks.
|
|
504
|
+
*
|
|
505
|
+
* The file is sliced at the newline byte (0x0A), which cannot occur inside a
|
|
506
|
+
* multi-byte UTF-8 sequence, so decoding per complete line is safe. The
|
|
507
|
+
* returned `offset` points at the first byte not part of a processed line,
|
|
508
|
+
* which is exactly where the next incremental run must resume.
|
|
509
|
+
*
|
|
510
|
+
* @param {{
|
|
511
|
+
* path: string,
|
|
512
|
+
* startOffset: number,
|
|
513
|
+
* onLine: (line: string) => void,
|
|
514
|
+
* shouldStop?: () => boolean,
|
|
515
|
+
* }} input
|
|
516
|
+
* @returns {Promise<{ offset: number, aborted: boolean }>}
|
|
517
|
+
*/
|
|
518
|
+
async function readLines(input) {
|
|
519
|
+
const stream = createReadStream(input.path, { start: input.startOffset, highWaterMark: 1 << 20 });
|
|
520
|
+
/** @type {Buffer | null} */
|
|
521
|
+
let pending = null;
|
|
522
|
+
let consumed = 0;
|
|
523
|
+
let aborted = false;
|
|
524
|
+
let newlineSeen = 0;
|
|
525
|
+
|
|
526
|
+
try {
|
|
527
|
+
for await (const chunk of stream) {
|
|
528
|
+
const buffer = pending && pending.length > 0 ? Buffer.concat([pending, chunk]) : chunk;
|
|
529
|
+
pending = null;
|
|
530
|
+
let cursor = 0;
|
|
531
|
+
let newline = buffer.indexOf(10, cursor);
|
|
532
|
+
while (newline !== -1) {
|
|
533
|
+
if (newline > cursor) input.onLine(buffer.toString("utf8", cursor, newline));
|
|
534
|
+
consumed += newline - cursor + 1;
|
|
535
|
+
cursor = newline + 1;
|
|
536
|
+
newlineSeen += 1;
|
|
537
|
+
// Checking the clock every 16k lines keeps a single huge file inside
|
|
538
|
+
// the budget without calling Date.now() per line.
|
|
539
|
+
if ((newlineSeen & 0x3fff) === 0 && input.shouldStop?.()) {
|
|
540
|
+
aborted = true;
|
|
541
|
+
break;
|
|
542
|
+
}
|
|
543
|
+
newline = buffer.indexOf(10, cursor);
|
|
544
|
+
}
|
|
545
|
+
if (aborted) break;
|
|
546
|
+
pending = cursor < buffer.length ? Buffer.from(buffer.subarray(cursor)) : null;
|
|
547
|
+
}
|
|
548
|
+
} finally {
|
|
549
|
+
// Breaking out of `for await` destroys the stream; this also covers a throw
|
|
550
|
+
// from onLine.
|
|
551
|
+
stream.destroy();
|
|
552
|
+
}
|
|
553
|
+
|
|
554
|
+
return { offset: input.startOffset + consumed, aborted };
|
|
555
|
+
}
|
|
556
|
+
|
|
557
|
+
/**
|
|
558
|
+
* Read one session file into open-day aggregates, routing each message.
|
|
559
|
+
*
|
|
560
|
+
* @param {{
|
|
561
|
+
* path: string,
|
|
562
|
+
* startOffset: number,
|
|
563
|
+
* days: DaysMap,
|
|
564
|
+
* now: number,
|
|
565
|
+
* onUsage: (usage: NonNullable<ReturnType<typeof usageFromLine>>) => void,
|
|
566
|
+
* shouldStop: () => boolean,
|
|
567
|
+
* }} input
|
|
568
|
+
* @returns {Promise<{ offset: number, complete: boolean }>}
|
|
569
|
+
*/
|
|
570
|
+
async function scanFile(input) {
|
|
571
|
+
const result = await readLines({
|
|
572
|
+
path: input.path,
|
|
573
|
+
startOffset: input.startOffset,
|
|
574
|
+
shouldStop: input.shouldStop,
|
|
575
|
+
onLine: (line) => {
|
|
576
|
+
const usage = usageFromLine(line, input.now);
|
|
577
|
+
if (usage) input.onUsage(usage);
|
|
578
|
+
},
|
|
579
|
+
});
|
|
580
|
+
return { offset: result.offset, complete: !result.aborted };
|
|
581
|
+
}
|
|
582
|
+
|
|
583
|
+
/** Bytes hashed from each end of a session file for the copy fingerprint. */
|
|
584
|
+
const FINGERPRINT_BYTES = 32 * 1024;
|
|
585
|
+
|
|
586
|
+
/**
|
|
587
|
+
* Fingerprint a session file as (size, first 32 KB, last 32 KB).
|
|
588
|
+
*
|
|
589
|
+
* This exists to recognise a byte-identical copy of a session that is already
|
|
590
|
+
* counted: a `cp`, a restored backup or a sync tool would otherwise add the
|
|
591
|
+
* same messages a second time, and for today's messages that is a real
|
|
592
|
+
* overcount (a frozen day is protected by the frozen-day rule instead).
|
|
593
|
+
*
|
|
594
|
+
* Only two small reads, so a duplicate 240 MB session costs 64 KB rather than
|
|
595
|
+
* a full parse. A *fork* (which copies a prefix and then diverges) has a
|
|
596
|
+
* different size and tail, so it is never mistaken for a copy and never loses
|
|
597
|
+
* the messages it added after forking.
|
|
598
|
+
*
|
|
599
|
+
* @param {string} path
|
|
600
|
+
* @param {number} size
|
|
601
|
+
* @returns {string | null} Hex digest, or null when the file cannot be read.
|
|
602
|
+
*/
|
|
603
|
+
function fingerprintFile(path, size) {
|
|
604
|
+
let fd = null;
|
|
605
|
+
try {
|
|
606
|
+
fd = openSync(path, "r");
|
|
607
|
+
const head = Buffer.alloc(Math.min(FINGERPRINT_BYTES, size));
|
|
608
|
+
if (head.length > 0) readSync(fd, head, 0, head.length, 0);
|
|
609
|
+
const tailLength = Math.min(FINGERPRINT_BYTES, Math.max(0, size - head.length));
|
|
610
|
+
let tail = Buffer.alloc(0);
|
|
611
|
+
if (tailLength > 0) {
|
|
612
|
+
tail = Buffer.alloc(tailLength);
|
|
613
|
+
readSync(fd, tail, 0, tailLength, size - tailLength);
|
|
614
|
+
}
|
|
615
|
+
const hash = createHash("sha256");
|
|
616
|
+
hash.update(`${size}\u0000`);
|
|
617
|
+
hash.update(head);
|
|
618
|
+
hash.update(tail);
|
|
619
|
+
return hash.digest("hex").slice(0, 32);
|
|
620
|
+
} catch {
|
|
621
|
+
return null;
|
|
622
|
+
} finally {
|
|
623
|
+
if (fd !== null) {
|
|
624
|
+
try {
|
|
625
|
+
closeSync(fd);
|
|
626
|
+
} catch {
|
|
627
|
+
// The fingerprint is an optimization; a close failure changes nothing.
|
|
628
|
+
}
|
|
629
|
+
}
|
|
630
|
+
}
|
|
631
|
+
}
|
|
632
|
+
|
|
633
|
+
/**
|
|
634
|
+
* Full-content comparison of two session files, streamed in 1 MB chunks.
|
|
635
|
+
*
|
|
636
|
+
* Only called when a cheap fingerprint already matched, so the expensive path is
|
|
637
|
+
* the rare one: a real duplicate. Streams rather than buffering, because the
|
|
638
|
+
* sessions on this machine reach 240 MB.
|
|
639
|
+
*
|
|
640
|
+
* @param {string} a
|
|
641
|
+
* @param {string} b
|
|
642
|
+
* @param {number} size Expected size of both.
|
|
643
|
+
* @returns {boolean}
|
|
644
|
+
*/
|
|
645
|
+
function sameContent(a, b, size) {
|
|
646
|
+
let left = null;
|
|
647
|
+
let right = null;
|
|
648
|
+
try {
|
|
649
|
+
left = openSync(a, "r");
|
|
650
|
+
right = openSync(b, "r");
|
|
651
|
+
const leftBuffer = Buffer.alloc(1 << 20);
|
|
652
|
+
const rightBuffer = Buffer.alloc(1 << 20);
|
|
653
|
+
let offset = 0;
|
|
654
|
+
while (offset < size) {
|
|
655
|
+
const wanted = Math.min(leftBuffer.length, size - offset);
|
|
656
|
+
const leftRead = readSync(left, leftBuffer, 0, wanted, offset);
|
|
657
|
+
const rightRead = readSync(right, rightBuffer, 0, wanted, offset);
|
|
658
|
+
if (leftRead !== rightRead) return false;
|
|
659
|
+
if (leftBuffer.compare(rightBuffer, 0, rightRead, 0, leftRead) !== 0) return false;
|
|
660
|
+
if (leftRead <= 0) break;
|
|
661
|
+
offset += leftRead;
|
|
662
|
+
}
|
|
663
|
+
return true;
|
|
664
|
+
} catch {
|
|
665
|
+
// An unreadable candidate is not a duplicate: counting it twice is the safer
|
|
666
|
+
// error, since dropping a session silently loses tokens.
|
|
667
|
+
return false;
|
|
668
|
+
} finally {
|
|
669
|
+
for (const fd of [left, right]) {
|
|
670
|
+
if (fd !== null) {
|
|
671
|
+
try {
|
|
672
|
+
closeSync(fd);
|
|
673
|
+
} catch {
|
|
674
|
+
// Closing is best-effort.
|
|
675
|
+
}
|
|
676
|
+
}
|
|
677
|
+
}
|
|
678
|
+
}
|
|
679
|
+
}
|
|
680
|
+
|
|
681
|
+
/**
|
|
682
|
+
* @param {string[]} roots
|
|
683
|
+
* @returns {string[]}
|
|
684
|
+
*/
|
|
685
|
+
function listSessionFiles(roots) {
|
|
686
|
+
/** @type {string[]} */
|
|
687
|
+
const files = [];
|
|
688
|
+
const seen = new Set();
|
|
689
|
+
for (const root of roots) {
|
|
690
|
+
/** @type {Array<[string, import("node:fs").Dirent[]]>} */
|
|
691
|
+
const stack = [];
|
|
692
|
+
try {
|
|
693
|
+
stack.push([root, readdirSync(root, { withFileTypes: true })]);
|
|
694
|
+
} catch {
|
|
695
|
+
continue;
|
|
696
|
+
}
|
|
697
|
+
while (stack.length > 0) {
|
|
698
|
+
const [dir, entries] = /** @type {[string, import("node:fs").Dirent[]]} */ (stack.pop());
|
|
699
|
+
for (const entry of entries) {
|
|
700
|
+
const full = join(dir, entry.name);
|
|
701
|
+
if (entry.isDirectory()) {
|
|
702
|
+
try {
|
|
703
|
+
stack.push([full, readdirSync(full, { withFileTypes: true })]);
|
|
704
|
+
} catch {
|
|
705
|
+
// An unreadable directory only hides its own sessions.
|
|
706
|
+
}
|
|
707
|
+
} else if (entry.isFile() && entry.name.endsWith(SESSION_SUFFIX)) {
|
|
708
|
+
// Same file through two roots (symlinked session dirs, a shared
|
|
709
|
+
// store) must be read once per run.
|
|
710
|
+
let key = full;
|
|
711
|
+
try {
|
|
712
|
+
key = realpathSync(full);
|
|
713
|
+
} catch {
|
|
714
|
+
// A vanished file is caught by statSync below and skipped.
|
|
715
|
+
}
|
|
716
|
+
if (seen.has(key)) continue;
|
|
717
|
+
seen.add(key);
|
|
718
|
+
files.push(full);
|
|
719
|
+
}
|
|
720
|
+
}
|
|
721
|
+
}
|
|
722
|
+
}
|
|
723
|
+
return files.sort();
|
|
724
|
+
}
|
|
725
|
+
|
|
726
|
+
/**
|
|
727
|
+
* Read the disposable cache envelope. A missing, corrupt or foreign-shaped file
|
|
728
|
+
* is a cache miss (a full rescan), never a crash.
|
|
729
|
+
*
|
|
730
|
+
* @param {string} path
|
|
731
|
+
* @returns {{ files: Record<string, SessionFileEntry>, hit: boolean, pendingLate: DaysMap }}
|
|
732
|
+
*/
|
|
733
|
+
function readCacheFile(path) {
|
|
734
|
+
let parsed;
|
|
735
|
+
try {
|
|
736
|
+
parsed = JSON.parse(readFileSync(path, "utf-8"));
|
|
737
|
+
} catch {
|
|
738
|
+
return { files: {}, hit: false, historyWatermark: "", pendingLate: emptyDays() };
|
|
739
|
+
}
|
|
740
|
+
if (!parsed || typeof parsed !== "object" || parsed.schemaVersion !== LOCAL_STATS_SCHEMA_VERSION) {
|
|
741
|
+
return { files: {}, hit: false, historyWatermark: "", pendingLate: emptyDays() };
|
|
742
|
+
}
|
|
743
|
+
if (!parsed.files || typeof parsed.files !== "object" || Array.isArray(parsed.files)) {
|
|
744
|
+
return { files: {}, hit: false, historyWatermark: "", pendingLate: emptyDays() };
|
|
745
|
+
}
|
|
746
|
+
// The last `lastFrozenDay` this cache was written against. An empty rollup
|
|
747
|
+
// next to a non-empty watermark is a lost history, not a fresh one.
|
|
748
|
+
const historyWatermark = typeof parsed.historyWatermark === "string" ? parsed.historyWatermark : "";
|
|
749
|
+
const pendingLate = decodeDays(parsed.pendingLate);
|
|
750
|
+
|
|
751
|
+
/** @type {Record<string, SessionFileEntry>} */
|
|
752
|
+
const files = {};
|
|
753
|
+
for (const [file, value] of Object.entries(parsed.files)) {
|
|
754
|
+
if (!value || typeof value !== "object") continue;
|
|
755
|
+
const entry = /** @type {Record<string, unknown>} */ (value);
|
|
756
|
+
if (typeof entry.size !== "number" || !Number.isFinite(entry.size)) continue;
|
|
757
|
+
if (typeof entry.mtimeMs !== "number" || !Number.isFinite(entry.mtimeMs)) continue;
|
|
758
|
+
if (typeof entry.offset !== "number" || !Number.isFinite(entry.offset)) continue;
|
|
759
|
+
if (!entry.days || typeof entry.days !== "object" || Array.isArray(entry.days)) continue;
|
|
760
|
+
files[file] = {
|
|
761
|
+
size: entry.size,
|
|
762
|
+
mtimeMs: entry.mtimeMs,
|
|
763
|
+
ino: typeof entry.ino === "number" && Number.isFinite(entry.ino) ? entry.ino : 0,
|
|
764
|
+
offset: Math.max(0, entry.offset),
|
|
765
|
+
complete: entry.complete !== false,
|
|
766
|
+
...(typeof entry.fingerprint === "string" && entry.fingerprint !== "" ? { fingerprint: entry.fingerprint } : {}),
|
|
767
|
+
...(typeof entry.dedupedBy === "string" && entry.dedupedBy !== "" ? { dedupedBy: entry.dedupedBy } : {}),
|
|
768
|
+
days: /** @type {SessionFileEntry["days"]} */ (entry.days),
|
|
769
|
+
};
|
|
770
|
+
}
|
|
771
|
+
return { files, hit: true, historyWatermark, pendingLate };
|
|
772
|
+
}
|
|
773
|
+
|
|
774
|
+
/**
|
|
775
|
+
* @param {string} path
|
|
776
|
+
* @param {Record<string, SessionFileEntry>} files
|
|
777
|
+
* @param {string} historyWatermark The `lastFrozenDay` this cache is written against.
|
|
778
|
+
* @param {SessionFileEntry["days"]} [pendingLate] Late deltas waiting for an unreadable month.
|
|
779
|
+
* @returns {{ ok: boolean, error?: string }}
|
|
780
|
+
*/
|
|
781
|
+
function writeCacheFile(path, files, historyWatermark = "", pendingLate = {}) {
|
|
782
|
+
// `historyWatermark` records which `lastFrozenDay` this cache belongs to, so a
|
|
783
|
+
// later run can tell a lost rollup from a fresh install. Pending late rows are
|
|
784
|
+
// aggregate deltas retained until their month can be updated.
|
|
785
|
+
const payload = `${JSON.stringify({ schemaVersion: LOCAL_STATS_SCHEMA_VERSION, historyWatermark, files, ...(Object.keys(pendingLate).length > 0 ? { pendingLate } : {}) })}\n`;
|
|
786
|
+
const temporary = `${path}.${process.pid}.tmp`;
|
|
787
|
+
try {
|
|
788
|
+
mkdirSync(dirname(path), { recursive: true, mode: 0o700 });
|
|
789
|
+
writeFileSync(temporary, payload, { encoding: "utf-8", mode: 0o600 });
|
|
790
|
+
renameSync(temporary, path);
|
|
791
|
+
return { ok: true };
|
|
792
|
+
} catch (error) {
|
|
793
|
+
rmSync(temporary, { force: true });
|
|
794
|
+
return { ok: false, error: redact(/** @type {{ message?: string }} */ (error)?.message ?? String(error)) };
|
|
795
|
+
}
|
|
796
|
+
}
|
|
797
|
+
|
|
798
|
+
/**
|
|
799
|
+
* Fold every session file's open-day tuples into one `DayRows` per day. Session
|
|
800
|
+
* counts are distinct files, which is what "today: N sessions" means.
|
|
801
|
+
*
|
|
802
|
+
* @param {Record<string, SessionFileEntry>} files
|
|
803
|
+
* @returns {Record<string, DayRows>}
|
|
804
|
+
*/
|
|
805
|
+
function buildOpenRows(files) {
|
|
806
|
+
/** @type {Map<string, Map<string, { files: Set<string>, slots: Map<string, { files: Set<string>, models: Map<string, Bucket> }> }>>} */
|
|
807
|
+
const days = new Map();
|
|
808
|
+
for (const [file, entry] of Object.entries(files ?? {})) {
|
|
809
|
+
for (const [day, tuples] of Object.entries(entry.days ?? {})) {
|
|
810
|
+
if (!Array.isArray(tuples) || tuples.length === 0) continue;
|
|
811
|
+
for (const tuple of tuples) {
|
|
812
|
+
if (!Array.isArray(tuple) || tuple.length < 8) continue;
|
|
813
|
+
const [family, slot, model] = tuple;
|
|
814
|
+
if (typeof family !== "string" || typeof slot !== "string" || typeof model !== "string") continue;
|
|
815
|
+
const bucket = {
|
|
816
|
+
input: count(tuple[3]),
|
|
817
|
+
output: count(tuple[4]),
|
|
818
|
+
cacheRead: count(tuple[5]),
|
|
819
|
+
cacheWrite: count(tuple[6]),
|
|
820
|
+
messages: count(tuple[7]),
|
|
821
|
+
};
|
|
822
|
+
if (bucket.messages === 0 && bucketTokens(bucket) === 0) continue;
|
|
823
|
+
|
|
824
|
+
const dayMap = days.get(day) ?? new Map();
|
|
825
|
+
days.set(day, dayMap);
|
|
826
|
+
const familyAgg = dayMap.get(family) ?? { files: new Set(), slots: new Map() };
|
|
827
|
+
dayMap.set(family, familyAgg);
|
|
828
|
+
familyAgg.files.add(file);
|
|
829
|
+
const slotAgg = familyAgg.slots.get(slot) ?? { files: new Set(), models: new Map() };
|
|
830
|
+
familyAgg.slots.set(slot, slotAgg);
|
|
831
|
+
slotAgg.files.add(file);
|
|
832
|
+
const known = slotAgg.models.get(model);
|
|
833
|
+
if (known) {
|
|
834
|
+
known.input += bucket.input;
|
|
835
|
+
known.output += bucket.output;
|
|
836
|
+
known.cacheRead += bucket.cacheRead;
|
|
837
|
+
known.cacheWrite += bucket.cacheWrite;
|
|
838
|
+
known.messages += bucket.messages;
|
|
839
|
+
} else {
|
|
840
|
+
slotAgg.models.set(model, { ...bucket });
|
|
841
|
+
}
|
|
842
|
+
}
|
|
843
|
+
}
|
|
844
|
+
}
|
|
845
|
+
|
|
846
|
+
/** @type {Record<string, DayRows>} */
|
|
847
|
+
const rows = {};
|
|
848
|
+
for (const [day, families] of days) {
|
|
849
|
+
/** @type {DayRows} */
|
|
850
|
+
const dayRows = {};
|
|
851
|
+
for (const [family, familyAgg] of families) {
|
|
852
|
+
/** @type {Record<string, { sessions: number, models: Record<string, Bucket> }>} */
|
|
853
|
+
const slots = {};
|
|
854
|
+
for (const [slot, slotAgg] of familyAgg.slots) {
|
|
855
|
+
/** @type {Record<string, Bucket>} */
|
|
856
|
+
const models = {};
|
|
857
|
+
for (const [model, bucket] of slotAgg.models) models[model] = { ...bucket };
|
|
858
|
+
slots[slot] = { sessions: slotAgg.files.size, models };
|
|
859
|
+
}
|
|
860
|
+
dayRows[family] = { sessions: familyAgg.files.size, slots };
|
|
861
|
+
}
|
|
862
|
+
rows[day] = dayRows;
|
|
863
|
+
}
|
|
864
|
+
return rows;
|
|
865
|
+
}
|
|
866
|
+
|
|
867
|
+
/**
|
|
868
|
+
* Turn cached, not-yet-applied late deltas into durable-history rows.
|
|
869
|
+
* Session counts stay zero because the original frozen day already counted them.
|
|
870
|
+
*
|
|
871
|
+
* @param {DaysMap} days
|
|
872
|
+
* @returns {Record<string, DayRows>}
|
|
873
|
+
*/
|
|
874
|
+
function buildLateRows(days) {
|
|
875
|
+
/** @type {Record<string, DayRows>} */
|
|
876
|
+
const rows = {};
|
|
877
|
+
for (const [day, tuples] of days) {
|
|
878
|
+
const dayRows = (rows[day] ??= {});
|
|
879
|
+
for (const tuple of tuples.values()) {
|
|
880
|
+
const [family, slot, model] = tuple;
|
|
881
|
+
const familyRow = (dayRows[family] ??= { sessions: 0, slots: {} });
|
|
882
|
+
const slotRow = (familyRow.slots[slot] ??= { sessions: 0, models: {} });
|
|
883
|
+
const bucket = (slotRow.models[model] ??= emptyBucket());
|
|
884
|
+
bucket.input += count(tuple[3]);
|
|
885
|
+
bucket.output += count(tuple[4]);
|
|
886
|
+
bucket.cacheRead += count(tuple[5]);
|
|
887
|
+
bucket.cacheWrite += count(tuple[6]);
|
|
888
|
+
bucket.messages += count(tuple[7]);
|
|
889
|
+
}
|
|
890
|
+
}
|
|
891
|
+
return rows;
|
|
892
|
+
}
|
|
893
|
+
|
|
894
|
+
/**
|
|
895
|
+
* @param {Map<string, number>} models Token totals by model id.
|
|
896
|
+
* @returns {Record<string, number>}
|
|
897
|
+
*/
|
|
898
|
+
function tokensByModel(models) {
|
|
899
|
+
/** @type {Array<[string, number]>} */
|
|
900
|
+
const entries = [];
|
|
901
|
+
for (const [model, tokens] of models) {
|
|
902
|
+
if (tokens > 0) entries.push([model, tokens]);
|
|
903
|
+
}
|
|
904
|
+
entries.sort((a, b) => b[1] - a[1] || a[0].localeCompare(b[0]));
|
|
905
|
+
return Object.fromEntries(entries);
|
|
906
|
+
}
|
|
907
|
+
|
|
908
|
+
/**
|
|
909
|
+
* @param {string} label
|
|
910
|
+
* @returns {number}
|
|
911
|
+
*/
|
|
912
|
+
function labelOrder(label) {
|
|
913
|
+
if (label === "#1") return -1;
|
|
914
|
+
const numeric = Number.parseInt(String(label).replace(/^#/, ""), 10);
|
|
915
|
+
return Number.isFinite(numeric) ? numeric : Number.MAX_SAFE_INTEGER;
|
|
916
|
+
}
|
|
917
|
+
|
|
918
|
+
/**
|
|
919
|
+
* Fold the durable rollup, the frozen days of the last seven days, and the open
|
|
920
|
+
* days into the shape the Omarchy panel and /usage read. Today is always taken
|
|
921
|
+
* from open days; the week prefers the open copy of a day and falls back to the
|
|
922
|
+
* frozen month file, so a crash between a month write and the totals write never
|
|
923
|
+
* double counts.
|
|
924
|
+
*
|
|
925
|
+
* @param {{
|
|
926
|
+
* totals?: import("./history.js").Totals,
|
|
927
|
+
* openRows?: Record<string, DayRows>,
|
|
928
|
+
* windowDays?: Record<string, DayRows>,
|
|
929
|
+
* now?: number,
|
|
930
|
+
* }} input
|
|
931
|
+
* @returns {Record<string, LocalFamilyStats>}
|
|
932
|
+
*/
|
|
933
|
+
export function buildLocalStats(input = {}) {
|
|
934
|
+
const now = input.now ?? Date.now();
|
|
935
|
+
const today = localDayKey(now);
|
|
936
|
+
const recentDates = recentLocalDates(now, RECENT_DAYS);
|
|
937
|
+
const openRows = input.openRows ?? {};
|
|
938
|
+
const windowDays = input.windowDays ?? {};
|
|
939
|
+
const totals = input.totals ?? emptyTotals();
|
|
940
|
+
|
|
941
|
+
const families = new Set(totals.families.keys());
|
|
942
|
+
for (const rows of Object.values(openRows)) for (const family of Object.keys(rows)) families.add(family);
|
|
943
|
+
for (const rows of Object.values(windowDays)) for (const family of Object.keys(rows)) families.add(family);
|
|
944
|
+
|
|
945
|
+
/** @type {Record<string, LocalFamilyStats>} */
|
|
946
|
+
const byFamily = {};
|
|
947
|
+
for (const family of families) {
|
|
948
|
+
const history = totals.families.get(family) ?? null;
|
|
949
|
+
|
|
950
|
+
// Open days (normally just today, plus anything a truncated run left open).
|
|
951
|
+
let openPrompts = 0;
|
|
952
|
+
let openSessions = 0;
|
|
953
|
+
const openModels = new Map();
|
|
954
|
+
const openAccounts = new Map();
|
|
955
|
+
const openDates = new Set();
|
|
956
|
+
for (const [day, rows] of Object.entries(openRows)) {
|
|
957
|
+
const familyRow = rows?.[family];
|
|
958
|
+
if (!familyRow) continue;
|
|
959
|
+
openDates.add(day);
|
|
960
|
+
openSessions += count(familyRow.sessions);
|
|
961
|
+
for (const [slot, slotRow] of Object.entries(familyRow.slots ?? {})) {
|
|
962
|
+
const label = slot === "" ? "#1" : `#${slot}`;
|
|
963
|
+
const account = openAccounts.get(label) ?? { prompts: 0, sessions: 0, models: new Map() };
|
|
964
|
+
account.sessions += count(slotRow.sessions);
|
|
965
|
+
for (const [model, bucket] of Object.entries(slotRow.models ?? {})) {
|
|
966
|
+
addToModels(openModels, model, bucket);
|
|
967
|
+
addToModels(account.models, model, bucket);
|
|
968
|
+
openPrompts += bucketMessages(bucket);
|
|
969
|
+
account.prompts += bucketMessages(bucket);
|
|
970
|
+
}
|
|
971
|
+
openAccounts.set(label, account);
|
|
972
|
+
}
|
|
973
|
+
}
|
|
974
|
+
|
|
975
|
+
// The seven-day window: open days override frozen ones.
|
|
976
|
+
const dayTokens = new Map();
|
|
977
|
+
const dayModels = new Map();
|
|
978
|
+
const daySlots = new Map();
|
|
979
|
+
const frozenThrough = totals.lastFrozenDay;
|
|
980
|
+
for (const date of recentDates) {
|
|
981
|
+
// A frozen day belongs to the month file, which carries its late writes;
|
|
982
|
+
// an open day belongs to the cache. Preferring the cache for a frozen day
|
|
983
|
+
// would hide a late append until the day left the window.
|
|
984
|
+
const frozen = frozenThrough !== "" && date <= frozenThrough;
|
|
985
|
+
const rows = frozen ? (windowDays[date] ?? openRows[date]) : (openRows[date] ?? windowDays[date]);
|
|
986
|
+
const familyRow = rows?.[family];
|
|
987
|
+
if (!familyRow) {
|
|
988
|
+
dayTokens.set(date, 0);
|
|
989
|
+
continue;
|
|
990
|
+
}
|
|
991
|
+
dayTokens.set(date, familyRowTokens(familyRow));
|
|
992
|
+
/** @type {Map<string, number>} */
|
|
993
|
+
const models = new Map();
|
|
994
|
+
/** @type {Map<string, { tokens: number, prompts: number }>} */
|
|
995
|
+
const slots = new Map();
|
|
996
|
+
for (const [slot, slotRow] of Object.entries(familyRow.slots ?? {})) {
|
|
997
|
+
const label = slot === "" ? "#1" : `#${slot}`;
|
|
998
|
+
let tokens = 0;
|
|
999
|
+
let prompts = 0;
|
|
1000
|
+
for (const [model, bucket] of Object.entries(slotRow.models ?? {})) {
|
|
1001
|
+
const modelTokens = bucketTokens(bucket);
|
|
1002
|
+
tokens += modelTokens;
|
|
1003
|
+
prompts += bucketMessages(bucket);
|
|
1004
|
+
models.set(model, (models.get(model) ?? 0) + modelTokens);
|
|
1005
|
+
}
|
|
1006
|
+
const known = slots.get(label) ?? { tokens: 0, prompts: 0 };
|
|
1007
|
+
known.tokens += tokens;
|
|
1008
|
+
known.prompts += prompts;
|
|
1009
|
+
slots.set(label, known);
|
|
1010
|
+
}
|
|
1011
|
+
dayModels.set(date, models);
|
|
1012
|
+
daySlots.set(date, slots);
|
|
1013
|
+
}
|
|
1014
|
+
|
|
1015
|
+
const todayRows = openRows[today]?.[family] ?? null;
|
|
1016
|
+
const todayPrompts = todayRows ? familyRowMessages(todayRows) : 0;
|
|
1017
|
+
const todayTotalTokens = todayRows ? familyRowTokens(todayRows) : 0;
|
|
1018
|
+
/** @type {Map<string, number>} */
|
|
1019
|
+
const todayModels = new Map();
|
|
1020
|
+
for (const slotRow of Object.values(todayRows?.slots ?? {})) {
|
|
1021
|
+
for (const [model, bucket] of Object.entries(slotRow.models ?? {})) {
|
|
1022
|
+
todayModels.set(model, (todayModels.get(model) ?? 0) + bucketTokens(bucket));
|
|
1023
|
+
}
|
|
1024
|
+
}
|
|
1025
|
+
|
|
1026
|
+
/** @type {Map<string, number>} */
|
|
1027
|
+
const weekModels = new Map();
|
|
1028
|
+
for (const models of dayModels.values()) {
|
|
1029
|
+
for (const [model, tokens] of models) weekModels.set(model, (weekModels.get(model) ?? 0) + tokens);
|
|
1030
|
+
}
|
|
1031
|
+
|
|
1032
|
+
/** @type {Map<string, Bucket>} */
|
|
1033
|
+
const allModels = new Map();
|
|
1034
|
+
for (const [model, bucket] of history?.models ?? []) allModels.set(model, { ...bucket });
|
|
1035
|
+
for (const [model, bucket] of openModels) addToModels(allModels, model, bucket);
|
|
1036
|
+
|
|
1037
|
+
const activeDates = new Set(history?.activeDates ?? []);
|
|
1038
|
+
for (const date of openDates) activeDates.add(date);
|
|
1039
|
+
|
|
1040
|
+
/** @type {Record<string, { todayTokens: number, weekTokens: number, todayPrompts: number, totalPrompts: number, totalTokens: number }>} */
|
|
1041
|
+
const byAccount = {};
|
|
1042
|
+
const labels = new Set([...(history?.accounts.keys() ?? []), ...openAccounts.keys()]);
|
|
1043
|
+
for (const label of Array.from(labels).sort((a, b) => labelOrder(a) - labelOrder(b))) {
|
|
1044
|
+
const historyAccount = history?.accounts.get(label);
|
|
1045
|
+
const openAccount = openAccounts.get(label);
|
|
1046
|
+
/** @type {Map<string, Bucket>} */
|
|
1047
|
+
const accountModels = new Map();
|
|
1048
|
+
for (const [model, bucket] of historyAccount?.models ?? []) accountModels.set(model, { ...bucket });
|
|
1049
|
+
for (const [model, bucket] of openAccount?.models ?? []) addToModels(accountModels, model, bucket);
|
|
1050
|
+
let totalTokens = 0;
|
|
1051
|
+
for (const bucket of accountModels.values()) totalTokens += bucketTokens(bucket);
|
|
1052
|
+
let weekTokens = 0;
|
|
1053
|
+
for (const date of recentDates) weekTokens += daySlots.get(date)?.get(label)?.tokens ?? 0;
|
|
1054
|
+
byAccount[label] = {
|
|
1055
|
+
todayTokens: daySlots.get(today)?.get(label)?.tokens ?? 0,
|
|
1056
|
+
weekTokens,
|
|
1057
|
+
todayPrompts: daySlots.get(today)?.get(label)?.prompts ?? 0,
|
|
1058
|
+
totalPrompts: (historyAccount?.totalPrompts ?? 0) + (openAccount?.prompts ?? 0),
|
|
1059
|
+
totalTokens,
|
|
1060
|
+
};
|
|
1061
|
+
}
|
|
1062
|
+
|
|
1063
|
+
const totalPrompts = (history?.totalPrompts ?? 0) + openPrompts;
|
|
1064
|
+
if (totalPrompts === 0 && allModels.size === 0) continue;
|
|
1065
|
+
|
|
1066
|
+
/** @type {Record<string, { inputTokens: number, outputTokens: number, cacheReadInputTokens: number, cacheCreationInputTokens: number }>} */
|
|
1067
|
+
const modelUsage = {};
|
|
1068
|
+
const sortedModels = Array.from(allModels.entries()).sort((a, b) => bucketTokens(b[1]) - bucketTokens(a[1]) || a[0].localeCompare(b[0]));
|
|
1069
|
+
for (const [model, bucket] of sortedModels) {
|
|
1070
|
+
modelUsage[model] = {
|
|
1071
|
+
inputTokens: bucket.input,
|
|
1072
|
+
outputTokens: bucket.output,
|
|
1073
|
+
cacheReadInputTokens: bucket.cacheRead,
|
|
1074
|
+
cacheCreationInputTokens: bucket.cacheWrite,
|
|
1075
|
+
};
|
|
1076
|
+
}
|
|
1077
|
+
|
|
1078
|
+
byFamily[family] = {
|
|
1079
|
+
hasLocalStats: true,
|
|
1080
|
+
hasPromptStats: true,
|
|
1081
|
+
todayPrompts,
|
|
1082
|
+
todaySessions: todayRows ? count(todayRows.sessions) : 0,
|
|
1083
|
+
todayTotalTokens,
|
|
1084
|
+
todayTokensByModel: tokensByModel(todayModels),
|
|
1085
|
+
weekTokensByModel: tokensByModel(weekModels),
|
|
1086
|
+
recentDays: recentDates.map((date) => ({ date, messageCount: dayTokens.get(date) ?? 0 })),
|
|
1087
|
+
totalPrompts,
|
|
1088
|
+
totalSessions: (history?.totalSessions ?? 0) + openSessions,
|
|
1089
|
+
activeDays: activeDates.size,
|
|
1090
|
+
activeDates: Array.from(activeDates).sort(),
|
|
1091
|
+
modelUsage,
|
|
1092
|
+
byAccount,
|
|
1093
|
+
};
|
|
1094
|
+
}
|
|
1095
|
+
return byFamily;
|
|
1096
|
+
}
|
|
1097
|
+
|
|
1098
|
+
/**
|
|
1099
|
+
* Scan Pi's session files, freeze closed days into the durable history, and fold
|
|
1100
|
+
* everything into local statistics.
|
|
1101
|
+
*
|
|
1102
|
+
* Never throws. Everything unexpected (unreadable file, unwritable cache or
|
|
1103
|
+
* history) becomes a warning; the numbers simply cover fewer sessions.
|
|
1104
|
+
*
|
|
1105
|
+
* @param {{
|
|
1106
|
+
* roots?: string[],
|
|
1107
|
+
* cachePath?: string,
|
|
1108
|
+
* historyDir?: string,
|
|
1109
|
+
* env?: Record<string, string | undefined>,
|
|
1110
|
+
* home?: string,
|
|
1111
|
+
* now?: number,
|
|
1112
|
+
* budgetMs?: number,
|
|
1113
|
+
* clock?: () => number,
|
|
1114
|
+
* }} [options]
|
|
1115
|
+
* @returns {Promise<{
|
|
1116
|
+
* byFamily: Record<string, LocalFamilyStats>,
|
|
1117
|
+
* warnings: string[],
|
|
1118
|
+
* scan: LocalScanMeta,
|
|
1119
|
+
* }>}
|
|
1120
|
+
*/export async function collectLocalStats(options = {}) {
|
|
1121
|
+
const clock = options.clock ?? Date.now;
|
|
1122
|
+
const startedAt = clock();
|
|
1123
|
+
const now = options.now ?? Date.now();
|
|
1124
|
+
const today = localDayKey(now);
|
|
1125
|
+
const budgetMs = Number.isFinite(options.budgetMs) ? /** @type {number} */ (options.budgetMs) : DEFAULT_SCAN_BUDGET_MS;
|
|
1126
|
+
const roots = dedupeRoots(options.roots ?? resolveSessionRoots(options));
|
|
1127
|
+
const cachePath = options.cachePath ?? resolveLocalStatsCachePath(options);
|
|
1128
|
+
const historyDir = options.historyDir ?? resolveHistoryDir(options);
|
|
1129
|
+
/** @type {string[]} */
|
|
1130
|
+
const warnings = [];
|
|
1131
|
+
const listen = listSessionFiles(roots);
|
|
1132
|
+
|
|
1133
|
+
/** @type {LocalScanMeta} */
|
|
1134
|
+
const emptyScan = {
|
|
1135
|
+
files: listen.length,
|
|
1136
|
+
scanned: 0,
|
|
1137
|
+
reused: 0,
|
|
1138
|
+
bytesRead: 0,
|
|
1139
|
+
durationMs: clock() - startedAt,
|
|
1140
|
+
partial: false,
|
|
1141
|
+
budgetMs,
|
|
1142
|
+
frozenDays: 0,
|
|
1143
|
+
locked: false,
|
|
1144
|
+
};
|
|
1145
|
+
|
|
1146
|
+
// The lock comes before the reads, not after: two writers (the systemd timer
|
|
1147
|
+
// and /usage) that both read then both write would apply the same late-write
|
|
1148
|
+
// delta twice. A run that cannot take it still answers, from what is on disk.
|
|
1149
|
+
const lockPath = `${cachePath}.lock`;
|
|
1150
|
+
const lock = acquireHistoryLock({ lockPath, now: clock() });
|
|
1151
|
+
if (!lock.acquired) {
|
|
1152
|
+
// Read-only path: `canWrite: false` keeps it from rebuilding or repairing
|
|
1153
|
+
// anything the run that does hold the lock is writing right now.
|
|
1154
|
+
const history = loadTotals({ historyDir, warnings, canWrite: false });
|
|
1155
|
+
const cached = readCacheFile(cachePath);
|
|
1156
|
+
// Nothing here may write: not the history, not the cache, and not a repair
|
|
1157
|
+
// of a corrupt month file (review findings N8, N10).
|
|
1158
|
+
warnings.push(
|
|
1159
|
+
lock.reason
|
|
1160
|
+
? `local token stats: could not take the history lock (${lock.reason}); showing the numbers already on disk and writing nothing`
|
|
1161
|
+
: "local token stats: another piquota run is updating the token history; showing the numbers already on disk",
|
|
1162
|
+
);
|
|
1163
|
+
return {
|
|
1164
|
+
byFamily: buildLocalStats({
|
|
1165
|
+
totals: history.totals,
|
|
1166
|
+
openRows: buildOpenRows(cached.files),
|
|
1167
|
+
windowDays: loadWindowDays({
|
|
1168
|
+
historyDir,
|
|
1169
|
+
dates: recentLocalDates(now, RECENT_DAYS),
|
|
1170
|
+
warnings,
|
|
1171
|
+
canWrite: false,
|
|
1172
|
+
}),
|
|
1173
|
+
now,
|
|
1174
|
+
}),
|
|
1175
|
+
warnings,
|
|
1176
|
+
scan: { ...emptyScan, locked: true },
|
|
1177
|
+
};
|
|
1178
|
+
}
|
|
1179
|
+
|
|
1180
|
+
try {
|
|
1181
|
+
// A missing history or a missing cache means every day still on disk has to
|
|
1182
|
+
// be rebuildable, so both force a rescan. Frozen days are skipped in that
|
|
1183
|
+
// case: they were already counted and a rescan must never add them again.
|
|
1184
|
+
const history = loadTotals({ historyDir, warnings, canWrite: true });
|
|
1185
|
+
const cached = readCacheFile(cachePath);
|
|
1186
|
+
// A rollup that lost its numbers while the cache remembers having seen
|
|
1187
|
+
// frozen days is a lost history: rebuild whatever the session files still
|
|
1188
|
+
// hold instead of leaving the all-time totals empty.
|
|
1189
|
+
const lostHistory =
|
|
1190
|
+
history.totals.families.size === 0 &&
|
|
1191
|
+
history.totals.lastFrozenDay === "" &&
|
|
1192
|
+
typeof cached.historyWatermark === "string" &&
|
|
1193
|
+
cached.historyWatermark !== "";
|
|
1194
|
+
// Damaged or unreadable month files affect only their daily detail. They
|
|
1195
|
+
// never trigger a session rescan: frozen days are not reconstructed from
|
|
1196
|
+
// surviving session files.
|
|
1197
|
+
const windowDates = recentLocalDates(now, RECENT_DAYS);
|
|
1198
|
+
const rescanAll = !cached.hit || history.rescanAll || lostHistory;
|
|
1199
|
+
const lastFrozenDay = history.totals.lastFrozenDay;
|
|
1200
|
+
|
|
1201
|
+
const overBudget = () => clock() - startedAt > budgetMs;
|
|
1202
|
+
const listedSet = new Set(listen);
|
|
1203
|
+
|
|
1204
|
+
// Fingerprint -> the live path already counted under it. Built from the
|
|
1205
|
+
// cache (so a copy made after the original was scanned is still caught) and
|
|
1206
|
+
// only for files that still exist: restoring a backup *after* deleting the
|
|
1207
|
+
// original must count, not be discarded as a duplicate of a ghost.
|
|
1208
|
+
/** @type {Map<string, string>} */
|
|
1209
|
+
const counted = new Map();
|
|
1210
|
+
for (const path of listen) {
|
|
1211
|
+
const entry = cached.files[path];
|
|
1212
|
+
if (entry?.fingerprint) counted.set(entry.fingerprint, path);
|
|
1213
|
+
}
|
|
1214
|
+
|
|
1215
|
+
/**
|
|
1216
|
+
* The surviving file a copy chain bottoms out in.
|
|
1217
|
+
*
|
|
1218
|
+
* A copy can itself be copied (A -> B -> C), and an older cache may hold a
|
|
1219
|
+
* chain. Following `dedupedBy` to its root means deleting a middle copy
|
|
1220
|
+
* cannot make the tail look orphaned while the root is still counted
|
|
1221
|
+
* (review finding N11).
|
|
1222
|
+
*
|
|
1223
|
+
* @param {string} path
|
|
1224
|
+
* @returns {string | null} A path outside the chain that still exists.
|
|
1225
|
+
*/
|
|
1226
|
+
const rootOriginal = (path) => {
|
|
1227
|
+
const seen = new Set([path]);
|
|
1228
|
+
let current = cached.files[path]?.dedupedBy;
|
|
1229
|
+
while (current) {
|
|
1230
|
+
if (seen.has(current)) return null;
|
|
1231
|
+
seen.add(current);
|
|
1232
|
+
const next = cached.files[current]?.dedupedBy;
|
|
1233
|
+
if (!next) return current;
|
|
1234
|
+
current = next;
|
|
1235
|
+
}
|
|
1236
|
+
return null;
|
|
1237
|
+
};
|
|
1238
|
+
|
|
1239
|
+
/** @type {Record<string, SessionFileEntry>} */
|
|
1240
|
+
const next = {};
|
|
1241
|
+
const pendingLate = cached.pendingLate;
|
|
1242
|
+
let changed = !cached.hit;
|
|
1243
|
+
let scanned = 0;
|
|
1244
|
+
let reused = 0;
|
|
1245
|
+
let bytesRead = 0;
|
|
1246
|
+
let partial = overBudget();
|
|
1247
|
+
|
|
1248
|
+
// The lock is refreshed as the scan walks the store: a cold scan of 1.5 GB
|
|
1249
|
+
// takes seconds, but a slower machine or a bigger store could exceed the
|
|
1250
|
+
// stale timeout, and another run would then take the lock over while this
|
|
1251
|
+
// one kept writing. `lostLock` makes that impossible.
|
|
1252
|
+
let lostLock = false;
|
|
1253
|
+
let filesSinceRefresh = 0;
|
|
1254
|
+
const keepLock = () => {
|
|
1255
|
+
filesSinceRefresh += 1;
|
|
1256
|
+
if (filesSinceRefresh < 8) return true;
|
|
1257
|
+
filesSinceRefresh = 0;
|
|
1258
|
+
if (refreshHistoryLock(lock)) return true;
|
|
1259
|
+
lostLock = true;
|
|
1260
|
+
return false;
|
|
1261
|
+
};
|
|
1262
|
+
|
|
1263
|
+
for (const path of listen) {
|
|
1264
|
+
if (!keepLock()) break;
|
|
1265
|
+
if (!partial && overBudget()) partial = true;
|
|
1266
|
+
if (partial) break;
|
|
1267
|
+
|
|
1268
|
+
let stats;
|
|
1269
|
+
try {
|
|
1270
|
+
stats = statSync(path);
|
|
1271
|
+
} catch (error) {
|
|
1272
|
+
if (cached.files[path]) next[path] = cached.files[path];
|
|
1273
|
+
warnings.push(`local token stats: could not stat ${path}: ${redact(/** @type {{ message?: string }} */ (error)?.message)}`);
|
|
1274
|
+
continue;
|
|
1275
|
+
}
|
|
1276
|
+
|
|
1277
|
+
// Reuse only a file that was read to its last complete line at exactly
|
|
1278
|
+
// this size and mtime. `offset < size` means a trailing partial line (or
|
|
1279
|
+
// bytes that arrived after the read reached EOF), and those must be
|
|
1280
|
+
// re-examined before the run can call the file unchanged.
|
|
1281
|
+
// A file that was skipped as a copy of another path has no rows of its
|
|
1282
|
+
// own: if the original is gone, this file is now the only holder of those
|
|
1283
|
+
// open days and must be read again (review finding N4).
|
|
1284
|
+
const cachedEntry = cached.files[path];
|
|
1285
|
+
const orphanedCopy = cachedEntry?.dedupedBy !== undefined && !listedSet.has(rootOriginal(path) ?? cachedEntry.dedupedBy);
|
|
1286
|
+
const previous = orphanedCopy ? undefined : cachedEntry;
|
|
1287
|
+
if (
|
|
1288
|
+
previous &&
|
|
1289
|
+
!rescanAll &&
|
|
1290
|
+
previous.size === stats.size &&
|
|
1291
|
+
previous.mtimeMs === stats.mtimeMs &&
|
|
1292
|
+
previous.ino === (stats.ino ?? 0) &&
|
|
1293
|
+
previous.complete &&
|
|
1294
|
+
previous.offset === previous.size
|
|
1295
|
+
) {
|
|
1296
|
+
next[path] = previous;
|
|
1297
|
+
reused += 1;
|
|
1298
|
+
continue;
|
|
1299
|
+
}
|
|
1300
|
+
|
|
1301
|
+
// Reading from zero is the rescan case, and every one of these reasons
|
|
1302
|
+
// means "count the open days again, never the frozen ones":
|
|
1303
|
+
// - the cache is gone or unreadable (rescanAll);
|
|
1304
|
+
// - the file is a different inode or now shorter than what was read
|
|
1305
|
+
// (replaced, truncated);
|
|
1306
|
+
// - the cache has never seen this path. A copy, a move or a restored
|
|
1307
|
+
// backup looks exactly like a brand-new session, and its old-dated
|
|
1308
|
+
// messages were already frozen under the path it came from;
|
|
1309
|
+
// - the size is unchanged but the mtime moved: an in-place rewrite of
|
|
1310
|
+
// the same length, which reading from the old offset would miss.
|
|
1311
|
+
const seen = previous !== undefined;
|
|
1312
|
+
const replaced = seen && (previous.ino !== (stats.ino ?? 0) || stats.size < previous.offset);
|
|
1313
|
+
const rewritten = seen && previous.size === stats.size && previous.mtimeMs !== stats.mtimeMs;
|
|
1314
|
+
const rescan = rescanAll || !seen || replaced || rewritten;
|
|
1315
|
+
|
|
1316
|
+
// A byte-identical copy of a file that is already counted contributes
|
|
1317
|
+
// nothing: its messages sit in the original's rows. The entry is still
|
|
1318
|
+
// recorded at the copy's end so anything appended to it later is counted.
|
|
1319
|
+
const fingerprint = rescan ? fingerprintFile(path, stats.size) : (previous?.fingerprint ?? null);
|
|
1320
|
+
// Its own fingerprint in the map is not a copy: a rescan of the same file
|
|
1321
|
+
// (a lost history, a rewritten file) must fold its days again.
|
|
1322
|
+
const twin = rescan && fingerprint ? counted.get(fingerprint) : undefined;
|
|
1323
|
+
const candidate = twin === path ? undefined : twin;
|
|
1324
|
+
// The fingerprint samples the head and the tail, which is cheap but can
|
|
1325
|
+
// collide: two sessions of the same size that differ only in the middle
|
|
1326
|
+
// look identical (review finding N1). Only a candidate that already
|
|
1327
|
+
// matched the sample is compared in full, so the expensive hash runs on
|
|
1328
|
+
// the rare duplicate and never on a normal run.
|
|
1329
|
+
const duplicateOf = candidate !== undefined && sameContent(candidate, path, stats.size) ? candidate : undefined;
|
|
1330
|
+
if (rescan && fingerprint && candidate === undefined) counted.set(fingerprint, path);
|
|
1331
|
+
if (duplicateOf !== undefined) {
|
|
1332
|
+
// Point at the root of the chain, not at whatever file happened to
|
|
1333
|
+
// match: a copy of a copy must survive the middle link being deleted.
|
|
1334
|
+
const root = rootOriginal(duplicateOf) ?? duplicateOf;
|
|
1335
|
+
next[path] = {
|
|
1336
|
+
size: stats.size,
|
|
1337
|
+
mtimeMs: stats.mtimeMs,
|
|
1338
|
+
ino: stats.ino ?? 0,
|
|
1339
|
+
offset: stats.size,
|
|
1340
|
+
complete: true,
|
|
1341
|
+
fingerprint,
|
|
1342
|
+
dedupedBy: root,
|
|
1343
|
+
days: {},
|
|
1344
|
+
};
|
|
1345
|
+
changed = true;
|
|
1346
|
+
reused += 1;
|
|
1347
|
+
continue;
|
|
1348
|
+
}
|
|
1349
|
+
|
|
1350
|
+
const days = rescan ? emptyDays() : decodeDays(previous.days);
|
|
1351
|
+
const startOffset = rescan ? 0 : previous.offset;
|
|
1352
|
+
let offset;
|
|
1353
|
+
let complete;
|
|
1354
|
+
try {
|
|
1355
|
+
const result = await scanFile({
|
|
1356
|
+
path,
|
|
1357
|
+
startOffset,
|
|
1358
|
+
days,
|
|
1359
|
+
now,
|
|
1360
|
+
shouldStop: overBudget,
|
|
1361
|
+
onUsage: (usage) => {
|
|
1362
|
+
if (lastFrozenDay && usage.day <= lastFrozenDay) {
|
|
1363
|
+
// Frozen days are never reconstructed on a rescan. Incremental
|
|
1364
|
+
// late writes stay in the cache until their month accepts them.
|
|
1365
|
+
if (rescan) return;
|
|
1366
|
+
addUsage(pendingLate, usage);
|
|
1367
|
+
return;
|
|
1368
|
+
}
|
|
1369
|
+
addUsage(days, usage);
|
|
1370
|
+
},
|
|
1371
|
+
});
|
|
1372
|
+
offset = result.offset;
|
|
1373
|
+
complete = result.complete;
|
|
1374
|
+
bytesRead += offset - startOffset;
|
|
1375
|
+
} catch (error) {
|
|
1376
|
+
// A transient read error keeps the last known contribution instead of
|
|
1377
|
+
// silently deleting history for that session.
|
|
1378
|
+
if (previous) next[path] = previous;
|
|
1379
|
+
warnings.push(`local token stats: could not read ${path}: ${redact(/** @type {{ message?: string }} */ (error)?.message)}`);
|
|
1380
|
+
continue;
|
|
1381
|
+
}
|
|
1382
|
+
|
|
1383
|
+
let finalStats = stats;
|
|
1384
|
+
if (complete) {
|
|
1385
|
+
try {
|
|
1386
|
+
finalStats = statSync(path);
|
|
1387
|
+
} catch {
|
|
1388
|
+
// Keep the pre-scan stat; the next run detects the change.
|
|
1389
|
+
}
|
|
1390
|
+
}
|
|
1391
|
+
next[path] = {
|
|
1392
|
+
size: finalStats.size,
|
|
1393
|
+
mtimeMs: finalStats.mtimeMs,
|
|
1394
|
+
ino: finalStats.ino ?? 0,
|
|
1395
|
+
offset,
|
|
1396
|
+
complete,
|
|
1397
|
+
...(fingerprint ? { fingerprint } : {}),
|
|
1398
|
+
days: encodeDays(days),
|
|
1399
|
+
};
|
|
1400
|
+
changed = true;
|
|
1401
|
+
scanned += 1;
|
|
1402
|
+
if (!complete) {
|
|
1403
|
+
partial = true;
|
|
1404
|
+
break;
|
|
1405
|
+
}
|
|
1406
|
+
}
|
|
1407
|
+
|
|
1408
|
+
// Files the budget skipped keep their previous entry; files that are gone
|
|
1409
|
+
// lose theirs on this run.
|
|
1410
|
+
for (const path of listen) {
|
|
1411
|
+
if (!next[path] && cached.files[path]) next[path] = cached.files[path];
|
|
1412
|
+
}
|
|
1413
|
+
for (const path of Object.keys(cached.files)) {
|
|
1414
|
+
if (!listedSet.has(path)) changed = true;
|
|
1415
|
+
}
|
|
1416
|
+
|
|
1417
|
+
// Ownership is re-checked here, not only every eight files: a scan of a
|
|
1418
|
+
// handful of small files reaches this point without a refresh point, and a
|
|
1419
|
+
// run that lost the lock must not write (review finding N8).
|
|
1420
|
+
if (!lostLock && !refreshHistoryLock(lock)) lostLock = true;
|
|
1421
|
+
|
|
1422
|
+
// A run that lost the lock must not write anything: another run now owns
|
|
1423
|
+
// the history, and a second write is exactly the double count the lock
|
|
1424
|
+
// exists to prevent.
|
|
1425
|
+
if (lostLock) {
|
|
1426
|
+
warnings.push(
|
|
1427
|
+
"local token stats: another piquota run took the history lock over during this scan; the numbers below are read-only",
|
|
1428
|
+
);
|
|
1429
|
+
const windowDays = loadWindowDays({
|
|
1430
|
+
historyDir,
|
|
1431
|
+
dates: recentLocalDates(now, RECENT_DAYS),
|
|
1432
|
+
warnings,
|
|
1433
|
+
canWrite: false,
|
|
1434
|
+
});
|
|
1435
|
+
return {
|
|
1436
|
+
byFamily: buildLocalStats({ totals: history.totals, openRows: buildOpenRows(next), windowDays, now }),
|
|
1437
|
+
warnings,
|
|
1438
|
+
scan: {
|
|
1439
|
+
files: listen.length,
|
|
1440
|
+
scanned,
|
|
1441
|
+
reused,
|
|
1442
|
+
bytesRead,
|
|
1443
|
+
durationMs: clock() - startedAt,
|
|
1444
|
+
partial: true,
|
|
1445
|
+
budgetMs,
|
|
1446
|
+
frozenDays: 0,
|
|
1447
|
+
locked: true,
|
|
1448
|
+
},
|
|
1449
|
+
};
|
|
1450
|
+
}
|
|
1451
|
+
|
|
1452
|
+
const openRows = buildOpenRows(next);
|
|
1453
|
+
const pendingBeforeApply = pendingLate.size;
|
|
1454
|
+
const applied = applyDurable({
|
|
1455
|
+
historyDir,
|
|
1456
|
+
totals: history.totals,
|
|
1457
|
+
openRows,
|
|
1458
|
+
lateRows: buildLateRows(pendingLate),
|
|
1459
|
+
today,
|
|
1460
|
+
partial,
|
|
1461
|
+
writable: history.writable,
|
|
1462
|
+
ensure: !history.exists,
|
|
1463
|
+
warnings,
|
|
1464
|
+
});
|
|
1465
|
+
for (const day of pendingLate.keys()) {
|
|
1466
|
+
if (!applied.pendingLateDays.has(day)) pendingLate.delete(day);
|
|
1467
|
+
}
|
|
1468
|
+
if (pendingLate.size !== pendingBeforeApply) changed = true;
|
|
1469
|
+
|
|
1470
|
+
// A day that is in the rollup (frozen now, or already frozen before a crash)
|
|
1471
|
+
// must leave the disposable cache; otherwise a rescan would see it twice.
|
|
1472
|
+
if (applied.droppedDays.size > 0) {
|
|
1473
|
+
for (const entry of Object.values(next)) {
|
|
1474
|
+
for (const day of applied.droppedDays) {
|
|
1475
|
+
if (entry.days && day in entry.days) delete entry.days[day];
|
|
1476
|
+
}
|
|
1477
|
+
}
|
|
1478
|
+
for (const day of applied.droppedDays) delete openRows[day];
|
|
1479
|
+
changed = true;
|
|
1480
|
+
}
|
|
1481
|
+
|
|
1482
|
+
if (changed) {
|
|
1483
|
+
const written = refreshHistoryLock(lock)
|
|
1484
|
+
? writeCacheFile(cachePath, next, history.totals.lastFrozenDay, encodeDays(pendingLate))
|
|
1485
|
+
: { ok: true };
|
|
1486
|
+
if (!written.ok) {
|
|
1487
|
+
warnings.push(`local token stats: could not write the scan cache at ${cachePath}: ${written.error}`);
|
|
1488
|
+
}
|
|
1489
|
+
}
|
|
1490
|
+
|
|
1491
|
+
const windowDays = loadWindowDays({ historyDir, dates: recentLocalDates(now, RECENT_DAYS), warnings });
|
|
1492
|
+
|
|
1493
|
+
return {
|
|
1494
|
+
byFamily: buildLocalStats({ totals: history.totals, openRows, windowDays, now }),
|
|
1495
|
+
warnings,
|
|
1496
|
+
scan: {
|
|
1497
|
+
files: listen.length,
|
|
1498
|
+
scanned,
|
|
1499
|
+
reused,
|
|
1500
|
+
bytesRead,
|
|
1501
|
+
durationMs: clock() - startedAt,
|
|
1502
|
+
partial,
|
|
1503
|
+
budgetMs,
|
|
1504
|
+
frozenDays: applied.frozenDays.length,
|
|
1505
|
+
locked: false,
|
|
1506
|
+
},
|
|
1507
|
+
};
|
|
1508
|
+
} finally {
|
|
1509
|
+
releaseHistoryLock(lock);
|
|
1510
|
+
}
|
|
1511
|
+
}
|