@vibe-cafe/vibe-usage 0.9.15 → 0.10.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +4 -2
- package/package.json +1 -1
- package/src/parsers/codex-cache.js +138 -0
- package/src/parsers/codex.js +664 -142
- package/src/sync.js +21 -5
package/README.md
CHANGED
|
@@ -51,7 +51,7 @@ npx @vibe-cafe/vibe-usage status # Show config & detected tools
|
|
|
51
51
|
| Tool | Data Location |
|
|
52
52
|
|------|---------------|
|
|
53
53
|
| Claude Code | `~/.claude/projects/` (tokens + sessions), `~/.claude/transcripts/` (sessions only); also scans `$CLAUDE_CONFIG_DIR` when set (deduped), so relocated configs and GUI/CLI env mismatches are both covered |
|
|
54
|
-
| Codex CLI | `$CODEX_HOME/sessions/` and `$CODEX_HOME/archived_sessions/` (default `~/.codex`);
|
|
54
|
+
| Codex CLI | `$CODEX_HOME/sessions/` and `$CODEX_HOME/archived_sessions/` (default `~/.codex`); a versioned local index avoids re-reading unchanged rollouts and reads only safe append tails for ordinary sessions, while fork/sub-agent replay matching, duplicate suppression, and live/archive deduplication retain their existing semantics |
|
|
55
55
|
| Grok | `$GROK_HOME/sessions/<encoded-cwd>/<session-id>/` (default `~/.grok`); token usage from `updates.jsonl` `turn_completed.usage` (per-model `modelUsage`, cache reads, reasoning); project from `summary.json` cwd; honors `GROK_HOME` |
|
|
56
56
|
| GitHub Copilot CLI | `~/.copilot/session-state/*/events.jsonl` |
|
|
57
57
|
| Cursor | `state.vscdb` (SQLite, reads `cursorAuth/accessToken`, fetches CSV from `cursor.com`); cloud data is stamped with a fixed `cursor-cloud` hostname so multi-machine setups don't double-count |
|
|
@@ -77,7 +77,9 @@ npx @vibe-cafe/vibe-usage status # Show config & detected tools
|
|
|
77
77
|
- Aggregates token usage into 30-minute buckets
|
|
78
78
|
- Extracts session metadata from all parsers: active time (AI generation time, excluding queue/TTFT wait), total duration, message counts
|
|
79
79
|
- Uploads buckets + sessions to your vibecafe.ai dashboard (always gzip-compressed, ~94% smaller)
|
|
80
|
-
- Incremental:
|
|
80
|
+
- Incremental upload: every parser emits a complete local snapshot, then only buckets/sessions that are new or changed since the last successful upload are sent — a quiet machine uploads nothing. Upload state remains in `~/.vibe-usage/state.json`; failed or still-indexing parsers retain their prior state, while deleted local logs are pruned. Deleting the state file triggers a one-time full re-upload, and `reset` clears it automatically after deleting cloud data
|
|
81
|
+
- Incremental Codex parsing: a versioned, disposable cache under `~/.vibe-usage/cache/codex/` stores per-rollout aggregate results and parser continuation state. Unchanged rollouts require no raw-log reads; an ordinary append reads only the new tail; forks, sub-agents, replacements, truncations, and failed safety checks fall back to the full correctness path. A bounded rolling audit occasionally re-reads one historical file. Very large first-time indexes checkpoint before the Mac app timeout and resume on the next sync instead of restarting
|
|
82
|
+
- The Codex parser cache contains derived aggregates and replay metadata, not raw prompt or response text. It is independent of upload state and can be deleted safely (the next sync rebuilds it). `reset` intentionally keeps it so the required full re-upload does not also require a full disk rescan. Set `VIBE_USAGE_CODEX_CACHE=0` to disable the optimization for diagnosis
|
|
81
83
|
- SQLite-backed tools (Cursor, OpenCode, Kiro, Hermes) are read via Node's built-in `node:sqlite` on Node ≥ 22.5 — no `sqlite3` binary needed (works on Windows out of the box); on older Node it falls back to the system `sqlite3` CLI
|
|
82
84
|
- For continuous syncing, use `npx @vibe-cafe/vibe-usage daemon` or the [Vibe Usage Mac app](https://github.com/vibe-cafe/vibe-usage-app)
|
|
83
85
|
|
package/package.json
CHANGED
|
@@ -0,0 +1,138 @@
|
|
|
1
|
+
import {
|
|
2
|
+
existsSync,
|
|
3
|
+
mkdirSync,
|
|
4
|
+
readFileSync,
|
|
5
|
+
renameSync,
|
|
6
|
+
rmSync,
|
|
7
|
+
writeFileSync,
|
|
8
|
+
} from 'node:fs';
|
|
9
|
+
import { createHash, randomBytes } from 'node:crypto';
|
|
10
|
+
import { homedir } from 'node:os';
|
|
11
|
+
import { join } from 'node:path';
|
|
12
|
+
|
|
13
|
+
// Parser caches are disposable derived data. Keep their schema/version fully
|
|
14
|
+
// separate from ~/.vibe-usage/state.json, whose hashes are the authoritative
|
|
15
|
+
// record of successful uploads and must remain backward-compatible.
|
|
16
|
+
export const CODEX_CACHE_SCHEMA_VERSION = 1;
|
|
17
|
+
export const CODEX_PARSER_ALGORITHM_VERSION = 1;
|
|
18
|
+
|
|
19
|
+
function hash(value, length = 24) {
|
|
20
|
+
return createHash('sha256').update(value).digest('hex').slice(0, length);
|
|
21
|
+
}
|
|
22
|
+
|
|
23
|
+
export function codexCacheEnabled() {
|
|
24
|
+
return process.env.VIBE_USAGE_CODEX_CACHE !== '0';
|
|
25
|
+
}
|
|
26
|
+
|
|
27
|
+
export function fileSignature(stat) {
|
|
28
|
+
return {
|
|
29
|
+
size: stat.size,
|
|
30
|
+
mtimeMs: stat.mtimeMs,
|
|
31
|
+
dev: String(stat.dev),
|
|
32
|
+
ino: String(stat.ino),
|
|
33
|
+
};
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
function sameSignature(a, b) {
|
|
37
|
+
return a?.size === b.size
|
|
38
|
+
&& a?.mtimeMs === b.mtimeMs
|
|
39
|
+
&& a?.dev === b.dev
|
|
40
|
+
&& a?.ino === b.ino;
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
export function codexCacheDir(codexHome) {
|
|
44
|
+
const base = process.env.VIBE_USAGE_CACHE_DIR?.trim()
|
|
45
|
+
|| join(homedir(), '.vibe-usage', 'cache');
|
|
46
|
+
return join(base, 'codex', `root-${hash(codexHome)}`);
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
function entryPath(codexHome, filePath) {
|
|
50
|
+
return join(codexCacheDir(codexHome), `${hash(filePath, 32)}.json`);
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
function tailPath(codexHome, filePath) {
|
|
54
|
+
return join(codexCacheDir(codexHome), `${hash(filePath, 32)}.tail.json`);
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
export function loadCodexFileCache(codexHome, filePath, signature) {
|
|
58
|
+
if (!codexCacheEnabled()) return null;
|
|
59
|
+
const path = entryPath(codexHome, filePath);
|
|
60
|
+
if (!existsSync(path)) return null;
|
|
61
|
+
try {
|
|
62
|
+
const parsed = JSON.parse(readFileSync(path, 'utf8'));
|
|
63
|
+
if (parsed.schemaVersion !== CODEX_CACHE_SCHEMA_VERSION) return null;
|
|
64
|
+
if (parsed.algorithmVersion !== CODEX_PARSER_ALGORITHM_VERSION) return null;
|
|
65
|
+
if (parsed.filePath !== filePath) return null;
|
|
66
|
+
if (signature && !sameSignature(parsed.signature, signature)) return null;
|
|
67
|
+
return parsed;
|
|
68
|
+
} catch {
|
|
69
|
+
// Cache corruption is a performance miss, never a correctness failure.
|
|
70
|
+
return null;
|
|
71
|
+
}
|
|
72
|
+
}
|
|
73
|
+
|
|
74
|
+
export function saveCodexFileCache(codexHome, filePath, signature, data) {
|
|
75
|
+
if (!codexCacheEnabled()) return;
|
|
76
|
+
const dir = codexCacheDir(codexHome);
|
|
77
|
+
mkdirSync(dir, { recursive: true, mode: 0o700 });
|
|
78
|
+
const path = entryPath(codexHome, filePath);
|
|
79
|
+
const tempPath = `${path}.${process.pid}.${randomBytes(6).toString('hex')}.tmp`;
|
|
80
|
+
const payload = {
|
|
81
|
+
schemaVersion: CODEX_CACHE_SCHEMA_VERSION,
|
|
82
|
+
algorithmVersion: CODEX_PARSER_ALGORITHM_VERSION,
|
|
83
|
+
filePath,
|
|
84
|
+
signature,
|
|
85
|
+
...data,
|
|
86
|
+
};
|
|
87
|
+
try {
|
|
88
|
+
writeFileSync(tempPath, `${JSON.stringify(payload)}\n`, { encoding: 'utf8', mode: 0o600 });
|
|
89
|
+
renameSync(tempPath, path);
|
|
90
|
+
} finally {
|
|
91
|
+
// A killed writer can leave a unique temp file; a normal failed writer
|
|
92
|
+
// should not. rmSync is safe here because the target is this call's exact,
|
|
93
|
+
// random temporary path inside the versioned cache directory.
|
|
94
|
+
rmSync(tempPath, { force: true });
|
|
95
|
+
}
|
|
96
|
+
}
|
|
97
|
+
|
|
98
|
+
export function loadCodexFileTail(codexHome, filePath) {
|
|
99
|
+
if (!codexCacheEnabled()) return null;
|
|
100
|
+
const path = tailPath(codexHome, filePath);
|
|
101
|
+
if (!existsSync(path)) return null;
|
|
102
|
+
try {
|
|
103
|
+
const parsed = JSON.parse(readFileSync(path, 'utf8'));
|
|
104
|
+
if (parsed.schemaVersion !== CODEX_CACHE_SCHEMA_VERSION) return null;
|
|
105
|
+
if (parsed.algorithmVersion !== CODEX_PARSER_ALGORITHM_VERSION) return null;
|
|
106
|
+
if (parsed.filePath !== filePath || !parsed.signature || !parsed.tail) return null;
|
|
107
|
+
return parsed;
|
|
108
|
+
} catch {
|
|
109
|
+
return null;
|
|
110
|
+
}
|
|
111
|
+
}
|
|
112
|
+
|
|
113
|
+
export function saveCodexFileTail(codexHome, filePath, signature, tail) {
|
|
114
|
+
if (!codexCacheEnabled() || !tail) return;
|
|
115
|
+
const dir = codexCacheDir(codexHome);
|
|
116
|
+
mkdirSync(dir, { recursive: true, mode: 0o700 });
|
|
117
|
+
const path = tailPath(codexHome, filePath);
|
|
118
|
+
const tempPath = `${path}.${process.pid}.${randomBytes(6).toString('hex')}.tmp`;
|
|
119
|
+
const payload = {
|
|
120
|
+
schemaVersion: CODEX_CACHE_SCHEMA_VERSION,
|
|
121
|
+
algorithmVersion: CODEX_PARSER_ALGORITHM_VERSION,
|
|
122
|
+
filePath,
|
|
123
|
+
signature,
|
|
124
|
+
tail,
|
|
125
|
+
};
|
|
126
|
+
try {
|
|
127
|
+
writeFileSync(tempPath, `${JSON.stringify(payload)}\n`, { encoding: 'utf8', mode: 0o600 });
|
|
128
|
+
renameSync(tempPath, path);
|
|
129
|
+
} finally {
|
|
130
|
+
rmSync(tempPath, { force: true });
|
|
131
|
+
}
|
|
132
|
+
}
|
|
133
|
+
|
|
134
|
+
export function removeCodexFileCache(codexHome, filePath) {
|
|
135
|
+
if (!codexCacheEnabled()) return;
|
|
136
|
+
rmSync(entryPath(codexHome, filePath), { force: true });
|
|
137
|
+
rmSync(tailPath(codexHome, filePath), { force: true });
|
|
138
|
+
}
|
package/src/parsers/codex.js
CHANGED
|
@@ -1,9 +1,25 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import {
|
|
2
|
+
closeSync,
|
|
3
|
+
createReadStream,
|
|
4
|
+
existsSync,
|
|
5
|
+
openSync,
|
|
6
|
+
readSync,
|
|
7
|
+
readdirSync,
|
|
8
|
+
statSync,
|
|
9
|
+
} from 'node:fs';
|
|
2
10
|
import { join } from 'node:path';
|
|
3
11
|
import { homedir } from 'node:os';
|
|
4
12
|
import { createInterface } from 'node:readline';
|
|
5
13
|
import { createHash } from 'node:crypto';
|
|
6
|
-
import { aggregateToBuckets
|
|
14
|
+
import { aggregateToBuckets } from './index.js';
|
|
15
|
+
import {
|
|
16
|
+
codexCacheEnabled,
|
|
17
|
+
fileSignature,
|
|
18
|
+
loadCodexFileCache,
|
|
19
|
+
loadCodexFileTail,
|
|
20
|
+
saveCodexFileCache,
|
|
21
|
+
saveCodexFileTail,
|
|
22
|
+
} from './codex-cache.js';
|
|
7
23
|
|
|
8
24
|
// Codex stores live sessions in $CODEX_HOME/sessions (default ~/.codex) and,
|
|
9
25
|
// once a session is "completed", moves its rollout file verbatim into
|
|
@@ -12,8 +28,11 @@ import { aggregateToBuckets, extractSessions } from './index.js';
|
|
|
12
28
|
// both, index them together so fork replay-skip works across directories, and
|
|
13
29
|
// select the most complete physical file when the same session briefly exists
|
|
14
30
|
// in both locations during an archive move.
|
|
15
|
-
function
|
|
16
|
-
|
|
31
|
+
function getCodexHome() {
|
|
32
|
+
return process.env.CODEX_HOME?.trim() || join(homedir(), '.codex');
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
function sessionsDirs(codexHome) {
|
|
17
36
|
return [
|
|
18
37
|
join(codexHome, 'sessions'),
|
|
19
38
|
join(codexHome, 'archived_sessions'),
|
|
@@ -42,9 +61,16 @@ function findJsonlFiles(dir) {
|
|
|
42
61
|
return results;
|
|
43
62
|
}
|
|
44
63
|
|
|
45
|
-
function readLines(filePath) {
|
|
64
|
+
function readLines(filePath, snapshotSize, start = 0) {
|
|
46
65
|
return createInterface({
|
|
47
|
-
input: createReadStream(filePath, {
|
|
66
|
+
input: createReadStream(filePath, {
|
|
67
|
+
encoding: 'utf-8',
|
|
68
|
+
// Rollouts are append-only while Codex is working. Bound both parser
|
|
69
|
+
// passes to the size captured before pass 1 so they see the same prefix
|
|
70
|
+
// even when the live file grows between reads.
|
|
71
|
+
...(start > 0 ? { start } : {}),
|
|
72
|
+
...(snapshotSize == null ? {} : { end: snapshotSize - 1 }),
|
|
73
|
+
}),
|
|
48
74
|
crlfDelay: Infinity,
|
|
49
75
|
});
|
|
50
76
|
}
|
|
@@ -79,6 +105,84 @@ function extractParentThreadId(meta) {
|
|
|
79
105
|
|| null;
|
|
80
106
|
}
|
|
81
107
|
|
|
108
|
+
/**
|
|
109
|
+
* Read only far enough to find the canonical (first) session_meta. This cheap
|
|
110
|
+
* discovery pass lets ordinary sessions skip the old full-file index pass;
|
|
111
|
+
* only forks, sub-agents, and parents referenced by them need replay indexes.
|
|
112
|
+
*/
|
|
113
|
+
async function readSessionHeader(filePath, snapshotSize) {
|
|
114
|
+
for await (const line of readLines(filePath, snapshotSize)) {
|
|
115
|
+
if (!line.trim()) continue;
|
|
116
|
+
try {
|
|
117
|
+
const obj = JSON.parse(line);
|
|
118
|
+
if (obj.type !== 'session_meta' || !obj.payload) continue;
|
|
119
|
+
const meta = obj.payload;
|
|
120
|
+
return {
|
|
121
|
+
sessionId: meta.id || null,
|
|
122
|
+
forkedFromId: meta.forked_from_id || null,
|
|
123
|
+
parentThreadId: extractParentThreadId(meta),
|
|
124
|
+
sessionProject: extractProject(meta),
|
|
125
|
+
sessionStartedAtMs: timestampMs(meta.timestamp) ?? timestampMs(obj.timestamp),
|
|
126
|
+
isSubagent: isSubagentMeta(meta),
|
|
127
|
+
};
|
|
128
|
+
} catch {
|
|
129
|
+
continue;
|
|
130
|
+
}
|
|
131
|
+
}
|
|
132
|
+
return {
|
|
133
|
+
sessionId: null,
|
|
134
|
+
forkedFromId: null,
|
|
135
|
+
parentThreadId: null,
|
|
136
|
+
sessionProject: 'unknown',
|
|
137
|
+
sessionStartedAtMs: null,
|
|
138
|
+
isSubagent: false,
|
|
139
|
+
};
|
|
140
|
+
}
|
|
141
|
+
|
|
142
|
+
function cacheData(cache, changes = {}) {
|
|
143
|
+
return {
|
|
144
|
+
header: changes.header ?? cache?.header ?? null,
|
|
145
|
+
index: changes.index ?? cache?.index ?? null,
|
|
146
|
+
result: changes.result ?? cache?.result ?? null,
|
|
147
|
+
lastAuditedAt: changes.lastAuditedAt ?? cache?.lastAuditedAt ?? null,
|
|
148
|
+
};
|
|
149
|
+
}
|
|
150
|
+
|
|
151
|
+
const TAIL_GUARD_BYTES = 4096;
|
|
152
|
+
|
|
153
|
+
function snapshotGuard(filePath, size) {
|
|
154
|
+
if (size <= 0) return { hash: null, endsWithNewline: false };
|
|
155
|
+
const length = Math.min(size, TAIL_GUARD_BYTES);
|
|
156
|
+
const buffer = Buffer.allocUnsafe(length);
|
|
157
|
+
const fd = openSync(filePath, 'r');
|
|
158
|
+
try {
|
|
159
|
+
const read = readSync(fd, buffer, 0, length, size - length);
|
|
160
|
+
const slice = buffer.subarray(0, read);
|
|
161
|
+
return {
|
|
162
|
+
hash: createHash('sha256').update(slice).digest('base64url').slice(0, 20),
|
|
163
|
+
endsWithNewline: slice.at(-1) === 0x0a,
|
|
164
|
+
};
|
|
165
|
+
} finally {
|
|
166
|
+
closeSync(fd);
|
|
167
|
+
}
|
|
168
|
+
}
|
|
169
|
+
|
|
170
|
+
function tailStateFor(file) {
|
|
171
|
+
const prior = file.priorTail;
|
|
172
|
+
const tail = prior?.tail;
|
|
173
|
+
if (!tail || !prior.signature) return null;
|
|
174
|
+
if (prior.signature.size <= 0 || prior.signature.size >= file.signature.size) return null;
|
|
175
|
+
if (prior.signature.dev !== file.signature.dev || prior.signature.ino !== file.signature.ino) return null;
|
|
176
|
+
if (prior.signature.mtimeMs > file.signature.mtimeMs) return null;
|
|
177
|
+
if (tail.parsedBytes !== prior.signature.size || !tail.endsWithNewline || !tail.guardHash) return null;
|
|
178
|
+
try {
|
|
179
|
+
const guard = snapshotGuard(file.filePath, prior.signature.size);
|
|
180
|
+
return guard.hash === tail.guardHash ? tail : null;
|
|
181
|
+
} catch {
|
|
182
|
+
return null;
|
|
183
|
+
}
|
|
184
|
+
}
|
|
185
|
+
|
|
82
186
|
function timestampMs(value) {
|
|
83
187
|
if (value == null || value === '') return null;
|
|
84
188
|
const n = new Date(value).getTime();
|
|
@@ -143,6 +247,34 @@ function longestReplayPrefix(child, parent) {
|
|
|
143
247
|
return matched;
|
|
144
248
|
}
|
|
145
249
|
|
|
250
|
+
/**
|
|
251
|
+
* Return the longest prefix of `child` found contiguously anywhere in
|
|
252
|
+
* `parent`. A live sub-agent rollout can be observed while Codex is still
|
|
253
|
+
* copying the parent block, before that copy reaches the parent snapshot's
|
|
254
|
+
* end. In that state the exact records are inherited history even though the
|
|
255
|
+
* stricter completed-replay suffix match above deliberately rejects them.
|
|
256
|
+
*/
|
|
257
|
+
function longestPartialReplayPrefix(child, parent) {
|
|
258
|
+
if (child.length === 0 || parent.length === 0) return 0;
|
|
259
|
+
|
|
260
|
+
const prefix = new Array(child.length).fill(0);
|
|
261
|
+
for (let i = 1, matched = 0; i < child.length; i++) {
|
|
262
|
+
while (matched > 0 && child[i] !== child[matched]) matched = prefix[matched - 1];
|
|
263
|
+
if (child[i] === child[matched]) matched++;
|
|
264
|
+
prefix[i] = matched;
|
|
265
|
+
}
|
|
266
|
+
|
|
267
|
+
let matched = 0;
|
|
268
|
+
let longest = 0;
|
|
269
|
+
for (const fingerprint of parent) {
|
|
270
|
+
while (matched > 0 && fingerprint !== child[matched]) matched = prefix[matched - 1];
|
|
271
|
+
if (fingerprint === child[matched]) matched++;
|
|
272
|
+
longest = Math.max(longest, matched);
|
|
273
|
+
if (matched === child.length) matched = prefix[matched - 1];
|
|
274
|
+
}
|
|
275
|
+
return longest;
|
|
276
|
+
}
|
|
277
|
+
|
|
146
278
|
// `task_started.started_at` is stored at one-second precision while the
|
|
147
279
|
// canonical session timestamp has milliseconds. Real Codex Desktop rollouts
|
|
148
280
|
// start the child task within a few seconds of creating the child session.
|
|
@@ -160,7 +292,7 @@ const OWN_TASK_START_WINDOW_MS = 5_000;
|
|
|
160
292
|
* full prefix. Together they bound matching to source records that existed at
|
|
161
293
|
* spawn without over-skipping child work when the parent later grows.
|
|
162
294
|
*/
|
|
163
|
-
async function indexSessionFile(filePath) {
|
|
295
|
+
async function indexSessionFile(filePath, snapshotSize) {
|
|
164
296
|
let sessionId = null;
|
|
165
297
|
let forkedFromId = null;
|
|
166
298
|
let parentThreadId = null;
|
|
@@ -178,7 +310,7 @@ async function indexSessionFile(filePath) {
|
|
|
178
310
|
let firstTaskBoundary = null;
|
|
179
311
|
let ownTaskBoundary = null;
|
|
180
312
|
|
|
181
|
-
for await (const line of readLines(filePath)) {
|
|
313
|
+
for await (const line of readLines(filePath, snapshotSize)) {
|
|
182
314
|
if (!line.trim()) continue;
|
|
183
315
|
try {
|
|
184
316
|
const obj = JSON.parse(line);
|
|
@@ -261,12 +393,13 @@ function replayBoundary(meta, sessionById) {
|
|
|
261
393
|
const parentAtSpawn = parent && meta.sessionStartedAtMs != null
|
|
262
394
|
? upperBound(parent.tokenTimes, meta.sessionStartedAtMs)
|
|
263
395
|
: null;
|
|
264
|
-
const
|
|
265
|
-
?
|
|
266
|
-
:
|
|
267
|
-
|
|
268
|
-
|
|
269
|
-
)
|
|
396
|
+
const parentSnapshot = parentAtSpawn == null
|
|
397
|
+
? []
|
|
398
|
+
: parent.tokenFingerprints.slice(0, parentAtSpawn);
|
|
399
|
+
const replayTokenCount = longestReplayPrefix(meta.tokenFingerprints, parentSnapshot);
|
|
400
|
+
const partialReplayTokenCount = meta.isSubagent
|
|
401
|
+
? longestPartialReplayPrefix(meta.tokenFingerprints, parentSnapshot)
|
|
402
|
+
: 0;
|
|
270
403
|
|
|
271
404
|
if (meta.isSubagent) {
|
|
272
405
|
// Direct evidence inside the child wins. Legacy single-meta rollouts did
|
|
@@ -292,11 +425,25 @@ function replayBoundary(meta, sessionById) {
|
|
|
292
425
|
: null);
|
|
293
426
|
if (direct) {
|
|
294
427
|
return {
|
|
295
|
-
rawTokenCount: Math.max(
|
|
428
|
+
rawTokenCount: Math.max(
|
|
429
|
+
replayTokenCount,
|
|
430
|
+
partialReplayTokenCount,
|
|
431
|
+
direct.rawTokenCount
|
|
432
|
+
),
|
|
296
433
|
recordIndex: direct.recordIndex,
|
|
297
434
|
};
|
|
298
435
|
}
|
|
299
|
-
|
|
436
|
+
|
|
437
|
+
// A recognized sub-agent can be synced while Codex is only partway
|
|
438
|
+
// through appending the copied parent block. The completed-replay matcher
|
|
439
|
+
// correctly rejects that interior slice, but counting it would create a
|
|
440
|
+
// temporary spike that disappears on the next sync. Exact payload overlap
|
|
441
|
+
// with the known parent is sufficient evidence to defer those leading
|
|
442
|
+
// records until the rollout reaches a stable suffix or task boundary.
|
|
443
|
+
return {
|
|
444
|
+
rawTokenCount: Math.max(replayTokenCount, partialReplayTokenCount),
|
|
445
|
+
recordIndex: null,
|
|
446
|
+
};
|
|
300
447
|
}
|
|
301
448
|
|
|
302
449
|
if (meta.forkedFromId) {
|
|
@@ -305,118 +452,223 @@ function replayBoundary(meta, sessionById) {
|
|
|
305
452
|
return { rawTokenCount: 0, recordIndex: null };
|
|
306
453
|
}
|
|
307
454
|
|
|
308
|
-
|
|
309
|
-
|
|
310
|
-
|
|
455
|
+
function boundaryKey(boundary) {
|
|
456
|
+
return `${boundary.rawTokenCount}:${boundary.recordIndex ?? ''}`;
|
|
457
|
+
}
|
|
311
458
|
|
|
312
|
-
|
|
313
|
-
const
|
|
314
|
-
|
|
315
|
-
|
|
459
|
+
function updateFileCache(codexHome, file, changes) {
|
|
460
|
+
const data = cacheData(file.cache, changes);
|
|
461
|
+
try {
|
|
462
|
+
saveCodexFileCache(codexHome, file.filePath, file.signature, data);
|
|
463
|
+
} catch {
|
|
464
|
+
// A read-only home, full disk, or antivirus race must only disable the
|
|
465
|
+
// optimization for this run. Raw-log parsing remains the source of truth.
|
|
466
|
+
}
|
|
467
|
+
file.cache = { ...(file.cache || {}), ...data };
|
|
468
|
+
}
|
|
316
469
|
|
|
317
|
-
|
|
318
|
-
|
|
319
|
-
|
|
320
|
-
|
|
321
|
-
|
|
322
|
-
|
|
323
|
-
|
|
324
|
-
|
|
325
|
-
|
|
326
|
-
|
|
327
|
-
|
|
470
|
+
function workBudgetMs() {
|
|
471
|
+
const configured = Number(process.env.VIBE_USAGE_CODEX_WORK_BUDGET_MS);
|
|
472
|
+
if (Number.isFinite(configured) && configured > 0) return configured;
|
|
473
|
+
// The macOS app terminates its child after 120 seconds. Cache-building work
|
|
474
|
+
// in a non-interactive child therefore checkpoints before that wall so the
|
|
475
|
+
// next invocation resumes instead of starting from zero. Interactive users
|
|
476
|
+
// can let a cold build finish in one run (and can interrupt it safely).
|
|
477
|
+
if (codexCacheEnabled() && !process.stdout.isTTY) return 105_000;
|
|
478
|
+
return Number.POSITIVE_INFINITY;
|
|
479
|
+
}
|
|
480
|
+
|
|
481
|
+
function auditIntervalMs() {
|
|
482
|
+
const configured = Number(process.env.VIBE_USAGE_CODEX_AUDIT_INTERVAL_MS);
|
|
483
|
+
if (Number.isFinite(configured) && configured >= 0) return configured;
|
|
484
|
+
return 30 * 24 * 60 * 60 * 1000;
|
|
485
|
+
}
|
|
486
|
+
|
|
487
|
+
function auditMaxBytes() {
|
|
488
|
+
const configured = Number(process.env.VIBE_USAGE_CODEX_AUDIT_MAX_BYTES);
|
|
489
|
+
if (Number.isFinite(configured) && configured > 0) return configured;
|
|
490
|
+
// Keep the background audit bounded below the app's wall timeout. Larger
|
|
491
|
+
// active files are still invalidated immediately by their stat signature,
|
|
492
|
+
// and every cache generation is rebuilt after parser-algorithm changes.
|
|
493
|
+
return 64 * 1024 * 1024;
|
|
494
|
+
}
|
|
495
|
+
|
|
496
|
+
function applySessionEvent(acc, event) {
|
|
497
|
+
const timestampMsValue = event.timestamp.getTime();
|
|
498
|
+
if (acc.lastTimestampMs != null && timestampMsValue < acc.lastTimestampMs) return false;
|
|
499
|
+
acc.firstTimestampMs ??= timestampMsValue;
|
|
500
|
+
acc.lastTimestampMs = timestampMsValue;
|
|
501
|
+
acc.messageCount++;
|
|
502
|
+
|
|
503
|
+
if (event.role === 'user') {
|
|
504
|
+
if (acc.turnStartMs != null && acc.turnEndMs != null && acc.turnEndMs > acc.turnStartMs) {
|
|
505
|
+
acc.completedActiveSeconds += Math.round((acc.turnEndMs - acc.turnStartMs) / 1000);
|
|
328
506
|
}
|
|
329
|
-
|
|
330
|
-
|
|
331
|
-
|
|
332
|
-
|
|
333
|
-
|
|
507
|
+
acc.turnStartMs = null;
|
|
508
|
+
acc.turnEndMs = null;
|
|
509
|
+
acc.waitingForFirstResponse = true;
|
|
510
|
+
acc.userMessageCount++;
|
|
511
|
+
acc.userPromptHours[event.timestamp.getUTCHours()]++;
|
|
512
|
+
} else if (acc.waitingForFirstResponse) {
|
|
513
|
+
acc.turnStartMs = timestampMsValue;
|
|
514
|
+
acc.turnEndMs = timestampMsValue;
|
|
515
|
+
acc.waitingForFirstResponse = false;
|
|
516
|
+
} else if (acc.turnStartMs != null) {
|
|
517
|
+
acc.turnEndMs = timestampMsValue;
|
|
518
|
+
}
|
|
519
|
+
return true;
|
|
520
|
+
}
|
|
521
|
+
|
|
522
|
+
function buildSessionAccumulator(events, previous = null) {
|
|
523
|
+
const sorted = [...events].sort((a, b) => a.timestamp - b.timestamp);
|
|
524
|
+
const first = sorted[0];
|
|
525
|
+
const acc = previous
|
|
526
|
+
? {
|
|
527
|
+
...previous,
|
|
528
|
+
userPromptHours: [...previous.userPromptHours],
|
|
334
529
|
}
|
|
530
|
+
: {
|
|
531
|
+
sessionId: first?.sessionId || null,
|
|
532
|
+
source: first?.source || null,
|
|
533
|
+
project: first?.project || 'unknown',
|
|
534
|
+
firstTimestampMs: null,
|
|
535
|
+
lastTimestampMs: null,
|
|
536
|
+
completedActiveSeconds: 0,
|
|
537
|
+
turnStartMs: null,
|
|
538
|
+
turnEndMs: null,
|
|
539
|
+
waitingForFirstResponse: false,
|
|
540
|
+
messageCount: 0,
|
|
541
|
+
userMessageCount: 0,
|
|
542
|
+
userPromptHours: new Array(24).fill(0),
|
|
543
|
+
};
|
|
544
|
+
for (const event of sorted) {
|
|
545
|
+
if (!applySessionEvent(acc, event)) return null;
|
|
546
|
+
}
|
|
547
|
+
return acc.sessionId ? acc : null;
|
|
548
|
+
}
|
|
549
|
+
|
|
550
|
+
function sessionFromAccumulator(acc) {
|
|
551
|
+
if (!acc?.sessionId || acc.firstTimestampMs == null || acc.lastTimestampMs == null) return null;
|
|
552
|
+
let activeSeconds = acc.completedActiveSeconds;
|
|
553
|
+
if (acc.turnStartMs != null && acc.turnEndMs != null && acc.turnEndMs > acc.turnStartMs) {
|
|
554
|
+
activeSeconds += Math.round((acc.turnEndMs - acc.turnStartMs) / 1000);
|
|
555
|
+
}
|
|
556
|
+
return {
|
|
557
|
+
source: acc.source,
|
|
558
|
+
project: acc.project || 'unknown',
|
|
559
|
+
sessionHash: createHash('sha256').update(acc.sessionId).digest('hex').slice(0, 16),
|
|
560
|
+
firstMessageAt: new Date(acc.firstTimestampMs).toISOString(),
|
|
561
|
+
lastMessageAt: new Date(acc.lastTimestampMs).toISOString(),
|
|
562
|
+
durationSeconds: Math.round((acc.lastTimestampMs - acc.firstTimestampMs) / 1000),
|
|
563
|
+
activeSeconds,
|
|
564
|
+
messageCount: acc.messageCount,
|
|
565
|
+
userMessageCount: acc.userMessageCount,
|
|
566
|
+
userPromptHours: acc.userPromptHours,
|
|
567
|
+
};
|
|
568
|
+
}
|
|
569
|
+
|
|
570
|
+
function mergeBucketLists(lists) {
|
|
571
|
+
const entries = [];
|
|
572
|
+
for (const buckets of lists) {
|
|
573
|
+
for (const bucket of buckets || []) {
|
|
574
|
+
entries.push({
|
|
575
|
+
source: bucket.source,
|
|
576
|
+
model: bucket.model,
|
|
577
|
+
project: bucket.project,
|
|
578
|
+
...(bucket.hostname ? { hostname: bucket.hostname } : {}),
|
|
579
|
+
timestamp: new Date(bucket.bucketStart),
|
|
580
|
+
inputTokens: bucket.inputTokens,
|
|
581
|
+
outputTokens: bucket.outputTokens,
|
|
582
|
+
cachedInputTokens: bucket.cachedInputTokens,
|
|
583
|
+
reasoningOutputTokens: bucket.reasoningOutputTokens,
|
|
584
|
+
});
|
|
335
585
|
}
|
|
336
586
|
}
|
|
587
|
+
return aggregateToBuckets(entries);
|
|
588
|
+
}
|
|
337
589
|
|
|
338
|
-
|
|
339
|
-
|
|
340
|
-
|
|
341
|
-
|
|
342
|
-
|
|
343
|
-
|
|
344
|
-
|
|
345
|
-
|
|
346
|
-
|
|
347
|
-
|
|
348
|
-
|
|
349
|
-
|
|
350
|
-
|
|
351
|
-
|
|
352
|
-
|
|
353
|
-
|
|
354
|
-
|
|
355
|
-
|
|
356
|
-
|
|
357
|
-
|
|
358
|
-
|
|
359
|
-
|
|
360
|
-
|
|
361
|
-
|
|
362
|
-
|
|
363
|
-
|
|
364
|
-
|
|
590
|
+
async function parseSessionFile(filePath, snapshotSize, fm, boundary, {
|
|
591
|
+
previousTail = null,
|
|
592
|
+
captureTail = false,
|
|
593
|
+
} = {}) {
|
|
594
|
+
const entries = [];
|
|
595
|
+
const sessionEvents = [];
|
|
596
|
+
let rawTokenSeen = previousTail?.rawTokenSeen || 0;
|
|
597
|
+
let parsedRecordIndex = previousTail?.parsedRecordIndex || 0;
|
|
598
|
+
let firstSessionMetaSeen = previousTail?.firstSessionMetaSeen || false;
|
|
599
|
+
|
|
600
|
+
const sessionProject = fm.sessionProject;
|
|
601
|
+
// Group timing events by the real Codex session id, not the file path: the
|
|
602
|
+
// same session can briefly exist in both sessions/ and archived_sessions/
|
|
603
|
+
// (mid-archive, or a re-synced archive). Path-keyed grouping would emit it
|
|
604
|
+
// as two different sessionHashes and double-count its session stats. Fall
|
|
605
|
+
// back to the path only when the id is unknown (corrupt/missing meta).
|
|
606
|
+
const sessionKey = fm.sessionId || filePath;
|
|
607
|
+
|
|
608
|
+
let turnContextModel = previousTail?.turnContextModel || 'unknown';
|
|
609
|
+
let prevTotal = previousTail?.prevTotal || null;
|
|
610
|
+
let prevCumulativeTotal = previousTail?.prevCumulativeTotal ?? null;
|
|
611
|
+
const start = previousTail?.parsedBytes || 0;
|
|
612
|
+
for await (const line of readLines(filePath, snapshotSize, start)) {
|
|
613
|
+
if (!line.trim()) continue;
|
|
614
|
+
try {
|
|
615
|
+
const obj = JSON.parse(line);
|
|
616
|
+
parsedRecordIndex++;
|
|
365
617
|
|
|
366
618
|
// A direct child task boundary covers every copied record, including
|
|
367
619
|
// timing/meta events. The raw-token ordinal covers full-history and
|
|
368
620
|
// last-N-turn forks whose exact payload sequence was matched in pass 1.
|
|
369
|
-
|
|
370
|
-
|
|
371
|
-
|
|
372
|
-
|
|
373
|
-
|
|
374
|
-
|
|
375
|
-
|
|
376
|
-
|
|
377
|
-
|
|
378
|
-
|
|
379
|
-
|
|
380
|
-
|
|
381
|
-
|
|
382
|
-
|
|
621
|
+
const beforeOwnTask = boundary.recordIndex != null
|
|
622
|
+
&& parsedRecordIndex < boundary.recordIndex;
|
|
623
|
+
const inReplayBlock = beforeOwnTask || rawTokenSeen < boundary.rawTokenCount;
|
|
624
|
+
|
|
625
|
+
const isSessionMeta = obj.type === 'session_meta';
|
|
626
|
+
const isCanonicalSessionMeta = isSessionMeta && !firstSessionMetaSeen;
|
|
627
|
+
const isOwnSessionMeta = isSessionMeta
|
|
628
|
+
&& obj.payload?.id != null
|
|
629
|
+
&& obj.payload.id === fm.sessionId;
|
|
630
|
+
if (isSessionMeta) firstSessionMetaSeen = true;
|
|
631
|
+
|
|
632
|
+
if (obj.timestamp) {
|
|
633
|
+
const evTs = new Date(obj.timestamp);
|
|
634
|
+
if (!isNaN(evTs.getTime())) {
|
|
383
635
|
// Repeated same-id metadata can be appended on resume/config
|
|
384
636
|
// updates and belongs to this logical session. A different-id meta
|
|
385
637
|
// is copied parent history and must not inflate timing stats.
|
|
386
|
-
|
|
387
|
-
|
|
388
|
-
|
|
389
|
-
|
|
390
|
-
|
|
391
|
-
|
|
392
|
-
|
|
393
|
-
|
|
394
|
-
|
|
395
|
-
|
|
396
|
-
|
|
397
|
-
}
|
|
638
|
+
const keepSessionMeta = isCanonicalSessionMeta
|
|
639
|
+
|| (isOwnSessionMeta && !inReplayBlock);
|
|
640
|
+
if (keepSessionMeta || (!isSessionMeta && !inReplayBlock)) {
|
|
641
|
+
const isUserTurn = obj.type === 'turn_context' || obj.type === 'session_meta';
|
|
642
|
+
sessionEvents.push({
|
|
643
|
+
sessionId: sessionKey,
|
|
644
|
+
source: 'codex',
|
|
645
|
+
project: sessionProject,
|
|
646
|
+
timestamp: evTs,
|
|
647
|
+
role: isUserTurn ? 'user' : 'assistant',
|
|
648
|
+
});
|
|
398
649
|
}
|
|
399
650
|
}
|
|
651
|
+
}
|
|
400
652
|
|
|
401
|
-
|
|
402
|
-
|
|
403
|
-
|
|
404
|
-
|
|
653
|
+
if (obj.type === 'turn_context' && obj.payload?.model) {
|
|
654
|
+
turnContextModel = obj.payload.model;
|
|
655
|
+
continue;
|
|
656
|
+
}
|
|
405
657
|
|
|
406
|
-
|
|
658
|
+
if (obj.type !== 'event_msg') continue;
|
|
407
659
|
|
|
408
|
-
|
|
409
|
-
|
|
660
|
+
const payload = obj.payload;
|
|
661
|
+
if (!payload) continue;
|
|
410
662
|
|
|
411
|
-
|
|
663
|
+
if (payload.type !== 'token_count') continue;
|
|
412
664
|
|
|
413
665
|
// Raw ordinals advance before validating usage/timestamp so pass 1 and
|
|
414
666
|
// pass 2 cannot drift on a malformed copied token_count record.
|
|
415
|
-
|
|
416
|
-
|
|
667
|
+
const isReplayedHistory = inReplayBlock;
|
|
668
|
+
rawTokenSeen++;
|
|
417
669
|
|
|
418
|
-
|
|
419
|
-
|
|
670
|
+
const info = payload.info;
|
|
671
|
+
if (!info) continue;
|
|
420
672
|
|
|
421
673
|
// Codex sometimes writes the same token_count twice back-to-back:
|
|
422
674
|
// identical last_token_usage with an unchanged cumulative total. A
|
|
@@ -426,64 +678,334 @@ export async function parse() {
|
|
|
426
678
|
// compaction — and must count as zero, not a second copy of
|
|
427
679
|
// last_token_usage. Guarded to positive totals so builds that leave
|
|
428
680
|
// total_token_usage all-zero can't suppress real usage.
|
|
429
|
-
|
|
430
|
-
|
|
431
|
-
|
|
432
|
-
|
|
433
|
-
|
|
681
|
+
const cumulativeTotal = info.total_token_usage?.total_tokens;
|
|
682
|
+
const isDuplicateEmission = typeof cumulativeTotal === 'number'
|
|
683
|
+
&& cumulativeTotal > 0
|
|
684
|
+
&& cumulativeTotal === prevCumulativeTotal;
|
|
685
|
+
if (typeof cumulativeTotal === 'number') prevCumulativeTotal = cumulativeTotal;
|
|
434
686
|
|
|
435
687
|
// Prefer incremental per-request usage; compute delta from cumulative
|
|
436
688
|
// totals as fallback. Always advance the cumulative baseline, even
|
|
437
689
|
// when last_token_usage exists or the record belongs to a replay.
|
|
438
|
-
|
|
439
|
-
|
|
440
|
-
|
|
441
|
-
|
|
442
|
-
|
|
443
|
-
|
|
444
|
-
|
|
445
|
-
|
|
446
|
-
|
|
447
|
-
|
|
690
|
+
const curr = info.total_token_usage;
|
|
691
|
+
let usage = info.last_token_usage;
|
|
692
|
+
if (!usage && curr) {
|
|
693
|
+
if (prevTotal) {
|
|
694
|
+
const delta = {
|
|
695
|
+
input_tokens: (curr.input_tokens || 0) - (prevTotal.input_tokens || 0),
|
|
696
|
+
output_tokens: (curr.output_tokens || 0) - (prevTotal.output_tokens || 0),
|
|
697
|
+
cached_input_tokens: (curr.cached_input_tokens || 0) - (prevTotal.cached_input_tokens || 0),
|
|
698
|
+
reasoning_output_tokens: (curr.reasoning_output_tokens || 0) - (prevTotal.reasoning_output_tokens || 0),
|
|
699
|
+
};
|
|
448
700
|
// Cumulative counters can reset after compaction or a new usage
|
|
449
701
|
// window. Treat the first post-reset total as a fresh baseline;
|
|
450
702
|
// allowing a negative delta would cancel legitimate bucket usage.
|
|
451
|
-
|
|
452
|
-
|
|
703
|
+
usage = Object.values(delta).some(value => value < 0) ? curr : delta;
|
|
704
|
+
} else {
|
|
453
705
|
// First cumulative entry — use as-is (it's the first event's total)
|
|
454
|
-
|
|
455
|
-
}
|
|
706
|
+
usage = curr;
|
|
456
707
|
}
|
|
708
|
+
}
|
|
457
709
|
// total_token_usage is session-wide, not per model. A global baseline
|
|
458
710
|
// avoids counting the full cumulative total again after a model switch.
|
|
459
|
-
|
|
460
|
-
|
|
461
|
-
|
|
711
|
+
if (curr) prevTotal = { ...curr };
|
|
712
|
+
if (!usage) continue;
|
|
713
|
+
if (isReplayedHistory || isDuplicateEmission) continue;
|
|
462
714
|
|
|
463
|
-
|
|
464
|
-
|
|
715
|
+
const timestamp = obj.timestamp ? new Date(obj.timestamp) : null;
|
|
716
|
+
if (!timestamp || isNaN(timestamp.getTime())) continue;
|
|
465
717
|
|
|
466
|
-
|
|
718
|
+
const model = info.model || payload.model || turnContextModel || 'unknown';
|
|
467
719
|
|
|
468
720
|
// OpenAI API: input_tokens INCLUDES cached, output_tokens INCLUDES reasoning.
|
|
469
721
|
// Normalize to Anthropic-style semantics where each field is non-overlapping.
|
|
470
|
-
|
|
471
|
-
|
|
472
|
-
|
|
473
|
-
|
|
474
|
-
|
|
475
|
-
|
|
476
|
-
|
|
477
|
-
|
|
478
|
-
|
|
479
|
-
|
|
480
|
-
|
|
481
|
-
|
|
722
|
+
const cachedInput = usage.cached_input_tokens || usage.cache_read_input_tokens || 0;
|
|
723
|
+
const reasoningOutput = usage.reasoning_output_tokens || 0;
|
|
724
|
+
entries.push({
|
|
725
|
+
source: 'codex',
|
|
726
|
+
model,
|
|
727
|
+
project: sessionProject,
|
|
728
|
+
timestamp,
|
|
729
|
+
inputTokens: (usage.input_tokens || 0) - cachedInput,
|
|
730
|
+
outputTokens: (usage.output_tokens || 0) - reasoningOutput,
|
|
731
|
+
cachedInputTokens: cachedInput,
|
|
732
|
+
reasoningOutputTokens: reasoningOutput,
|
|
733
|
+
});
|
|
734
|
+
} catch {
|
|
735
|
+
continue;
|
|
736
|
+
}
|
|
737
|
+
}
|
|
738
|
+
|
|
739
|
+
// Indexed files must match both passes exactly. Ordinary sessions take the
|
|
740
|
+
// single-pass fast path and have no expected counts; their byte-bounded
|
|
741
|
+
// snapshot is still stable, and any append invalidates the stat signature on
|
|
742
|
+
// the next sync.
|
|
743
|
+
if (fm.parsedRecordCount != null && fm.rawTokenCount != null) {
|
|
744
|
+
if (parsedRecordIndex !== fm.parsedRecordCount || rawTokenSeen !== fm.rawTokenCount) {
|
|
745
|
+
throw new Error('Codex rollout changed while syncing; retry on the next sync');
|
|
746
|
+
}
|
|
747
|
+
}
|
|
748
|
+
|
|
749
|
+
const buckets = mergeBucketLists([
|
|
750
|
+
previousTail?.buckets || [],
|
|
751
|
+
aggregateToBuckets(entries),
|
|
752
|
+
]);
|
|
753
|
+
const sessionAccumulator = buildSessionAccumulator(
|
|
754
|
+
sessionEvents,
|
|
755
|
+
previousTail?.sessionAccumulator || null
|
|
756
|
+
);
|
|
757
|
+
// Appended records should be chronological. If an app version inserts an
|
|
758
|
+
// older event into the tail, the compact accumulator cannot reproduce the
|
|
759
|
+
// global sort exactly, so discard the optimization and rebuild this file.
|
|
760
|
+
if (previousTail && sessionEvents.length > 0 && !sessionAccumulator) {
|
|
761
|
+
return parseSessionFile(filePath, snapshotSize, fm, boundary, { captureTail });
|
|
762
|
+
}
|
|
763
|
+
const session = sessionFromAccumulator(sessionAccumulator);
|
|
764
|
+
const result = { buckets, sessions: session ? [session] : [] };
|
|
765
|
+
if (captureTail) {
|
|
766
|
+
const guard = snapshotGuard(filePath, snapshotSize);
|
|
767
|
+
result.tail = {
|
|
768
|
+
parsedBytes: snapshotSize,
|
|
769
|
+
parsedRecordIndex,
|
|
770
|
+
rawTokenSeen,
|
|
771
|
+
firstSessionMetaSeen,
|
|
772
|
+
turnContextModel,
|
|
773
|
+
prevTotal,
|
|
774
|
+
prevCumulativeTotal,
|
|
775
|
+
buckets,
|
|
776
|
+
sessionAccumulator,
|
|
777
|
+
guardHash: guard.hash,
|
|
778
|
+
endsWithNewline: guard.endsWithNewline,
|
|
779
|
+
};
|
|
780
|
+
}
|
|
781
|
+
return result;
|
|
782
|
+
}
|
|
783
|
+
|
|
784
|
+
function mergeFileResults(results) {
|
|
785
|
+
const entries = [];
|
|
786
|
+
const sessions = [];
|
|
787
|
+
for (const result of results) {
|
|
788
|
+
for (const bucket of result.buckets || []) {
|
|
789
|
+
entries.push({
|
|
790
|
+
source: bucket.source,
|
|
791
|
+
model: bucket.model,
|
|
792
|
+
project: bucket.project,
|
|
793
|
+
...(bucket.hostname ? { hostname: bucket.hostname } : {}),
|
|
794
|
+
timestamp: new Date(bucket.bucketStart),
|
|
795
|
+
inputTokens: bucket.inputTokens,
|
|
796
|
+
outputTokens: bucket.outputTokens,
|
|
797
|
+
cachedInputTokens: bucket.cachedInputTokens,
|
|
798
|
+
reasoningOutputTokens: bucket.reasoningOutputTokens,
|
|
799
|
+
});
|
|
800
|
+
}
|
|
801
|
+
sessions.push(...(result.sessions || []));
|
|
802
|
+
}
|
|
803
|
+
return { buckets: aggregateToBuckets(entries), sessions };
|
|
804
|
+
}
|
|
805
|
+
|
|
806
|
+
export async function parse() {
|
|
807
|
+
const codexHome = getCodexHome();
|
|
808
|
+
const dirs = sessionsDirs(codexHome);
|
|
809
|
+
if (!dirs.some(existsSync)) return { buckets: [], sessions: [] };
|
|
810
|
+
|
|
811
|
+
const startedAt = Date.now();
|
|
812
|
+
const budget = workBudgetMs();
|
|
813
|
+
const overBudget = () => Date.now() - startedAt >= budget;
|
|
814
|
+
const cacheStats = {
|
|
815
|
+
headerHits: 0,
|
|
816
|
+
indexHits: 0,
|
|
817
|
+
resultHits: 0,
|
|
818
|
+
tailHits: 0,
|
|
819
|
+
filesRead: 0,
|
|
820
|
+
audited: 0,
|
|
821
|
+
};
|
|
822
|
+
const files = [];
|
|
823
|
+
for (const filePath of dirs.flatMap(findJsonlFiles)) {
|
|
824
|
+
try {
|
|
825
|
+
const stat = statSync(filePath);
|
|
826
|
+
if (stat.size <= 0) continue;
|
|
827
|
+
const signature = fileSignature(stat);
|
|
828
|
+
const cache = loadCodexFileCache(codexHome, filePath, signature);
|
|
829
|
+
const priorCache = cache || loadCodexFileCache(codexHome, filePath);
|
|
830
|
+
const priorTail = cache ? null : loadCodexFileTail(codexHome, filePath);
|
|
831
|
+
const file = {
|
|
832
|
+
filePath,
|
|
833
|
+
snapshotSize: stat.size,
|
|
834
|
+
signature,
|
|
835
|
+
cache,
|
|
836
|
+
priorCache,
|
|
837
|
+
priorTail,
|
|
838
|
+
header: null,
|
|
839
|
+
appendTail: null,
|
|
840
|
+
};
|
|
841
|
+
if (!cache && priorCache) file.appendTail = tailStateFor(file);
|
|
842
|
+
files.push(file);
|
|
843
|
+
} catch {
|
|
844
|
+
// The file may move to archived_sessions between discovery and stat.
|
|
845
|
+
// Its archived copy will be picked up on the next sync.
|
|
846
|
+
}
|
|
847
|
+
}
|
|
848
|
+
if (files.length === 0) return { buckets: [], sessions: [] };
|
|
849
|
+
|
|
850
|
+
// A warm cache gets a tiny rolling correctness audit. Never mix this into a
|
|
851
|
+
// cold/resumed build: all files must already have complete results, and at
|
|
852
|
+
// most one bounded file is re-read per invocation.
|
|
853
|
+
const auditPaths = new Set();
|
|
854
|
+
if (files.every(file => file.cache?.header && file.cache?.result)) {
|
|
855
|
+
const cutoff = Date.now() - auditIntervalMs();
|
|
856
|
+
const candidate = files
|
|
857
|
+
.filter(file => file.snapshotSize <= auditMaxBytes())
|
|
858
|
+
.filter(file => (file.cache.lastAuditedAt || 0) <= cutoff)
|
|
859
|
+
.sort((a, b) => (a.cache.lastAuditedAt || 0) - (b.cache.lastAuditedAt || 0))[0];
|
|
860
|
+
if (candidate) auditPaths.add(candidate.filePath);
|
|
861
|
+
}
|
|
862
|
+
|
|
863
|
+
// Cheap discovery: cached headers require no rollout read. On a cold build,
|
|
864
|
+
// read only through the first session_meta so ordinary sessions can avoid
|
|
865
|
+
// the former all-files replay-index pass.
|
|
866
|
+
const candidatesById = new Map();
|
|
867
|
+
for (let i = 0; i < files.length; i++) {
|
|
868
|
+
const file = files[i];
|
|
869
|
+
const reusableHeader = file.cache?.header || (file.appendTail ? file.priorCache?.header : null);
|
|
870
|
+
if (reusableHeader && !auditPaths.has(file.filePath)) {
|
|
871
|
+
file.header = reusableHeader;
|
|
872
|
+
if (!file.cache) file.cache = { header: reusableHeader };
|
|
873
|
+
cacheStats.headerHits++;
|
|
874
|
+
} else {
|
|
875
|
+
try {
|
|
876
|
+
file.header = await readSessionHeader(file.filePath, file.snapshotSize);
|
|
877
|
+
cacheStats.filesRead++;
|
|
878
|
+
updateFileCache(codexHome, file, { header: file.header });
|
|
482
879
|
} catch {
|
|
483
880
|
continue;
|
|
484
881
|
}
|
|
485
882
|
}
|
|
883
|
+
if (file.header.sessionId) {
|
|
884
|
+
if (!candidatesById.has(file.header.sessionId)) candidatesById.set(file.header.sessionId, []);
|
|
885
|
+
candidatesById.get(file.header.sessionId).push(file);
|
|
886
|
+
}
|
|
887
|
+
if (overBudget() && i < files.length - 1) {
|
|
888
|
+
return {
|
|
889
|
+
buckets: [], sessions: [], skipped: true,
|
|
890
|
+
indexing: { phase: 'discovery', completed: i + 1, total: files.length },
|
|
891
|
+
cache: cacheStats,
|
|
892
|
+
};
|
|
893
|
+
}
|
|
894
|
+
}
|
|
895
|
+
|
|
896
|
+
const duplicateIds = new Set(
|
|
897
|
+
[...candidatesById].filter(([, candidates]) => candidates.length > 1).map(([id]) => id)
|
|
898
|
+
);
|
|
899
|
+
const referencedParentIds = new Set();
|
|
900
|
+
for (const file of files) {
|
|
901
|
+
const parentId = file.header?.forkedFromId
|
|
902
|
+
|| (file.header?.isSubagent ? file.header.parentThreadId : null);
|
|
903
|
+
if (parentId) referencedParentIds.add(parentId);
|
|
904
|
+
}
|
|
905
|
+
|
|
906
|
+
// Only replay participants, their parents, corrupt-header files, and
|
|
907
|
+
// duplicate physical copies need the full compact token index.
|
|
908
|
+
const fileMeta = new Map();
|
|
909
|
+
const needsIndex = new Set();
|
|
910
|
+
for (const file of files) {
|
|
911
|
+
const header = file.header;
|
|
912
|
+
const required = !header
|
|
913
|
+
|| !header.sessionId
|
|
914
|
+
|| header.isSubagent
|
|
915
|
+
|| header.forkedFromId != null
|
|
916
|
+
|| header.parentThreadId != null
|
|
917
|
+
|| referencedParentIds.has(header.sessionId)
|
|
918
|
+
|| duplicateIds.has(header.sessionId);
|
|
919
|
+
if (!required) {
|
|
920
|
+
fileMeta.set(file.filePath, { ...header, filePath: file.filePath });
|
|
921
|
+
continue;
|
|
922
|
+
}
|
|
923
|
+
needsIndex.add(file.filePath);
|
|
924
|
+
let meta = auditPaths.has(file.filePath) ? null : file.cache?.index;
|
|
925
|
+
if (meta) {
|
|
926
|
+
cacheStats.indexHits++;
|
|
927
|
+
} else {
|
|
928
|
+
try {
|
|
929
|
+
meta = await indexSessionFile(file.filePath, file.snapshotSize);
|
|
930
|
+
cacheStats.filesRead++;
|
|
931
|
+
updateFileCache(codexHome, file, { index: meta });
|
|
932
|
+
} catch {
|
|
933
|
+
continue;
|
|
934
|
+
}
|
|
935
|
+
}
|
|
936
|
+
fileMeta.set(file.filePath, meta);
|
|
937
|
+
if (overBudget()) {
|
|
938
|
+
return {
|
|
939
|
+
buckets: [], sessions: [], skipped: true,
|
|
940
|
+
indexing: { phase: 'replay-index', completed: fileMeta.size, total: files.length },
|
|
941
|
+
cache: cacheStats,
|
|
942
|
+
};
|
|
943
|
+
}
|
|
944
|
+
}
|
|
945
|
+
|
|
946
|
+
// Select the most complete physical copy exactly as before. Unique ordinary
|
|
947
|
+
// sessions have no full record count, but cannot compete with another copy.
|
|
948
|
+
const sessionById = new Map();
|
|
949
|
+
for (const file of files) {
|
|
950
|
+
const meta = fileMeta.get(file.filePath);
|
|
951
|
+
if (!meta?.sessionId) continue;
|
|
952
|
+
const existing = sessionById.get(meta.sessionId);
|
|
953
|
+
const count = meta.parsedRecordCount ?? 0;
|
|
954
|
+
const existingCount = existing?.parsedRecordCount ?? 0;
|
|
955
|
+
if (!existing || count > existingCount) sessionById.set(meta.sessionId, meta);
|
|
956
|
+
}
|
|
957
|
+
|
|
958
|
+
const results = [];
|
|
959
|
+
for (let i = 0; i < files.length; i++) {
|
|
960
|
+
const file = files[i];
|
|
961
|
+
const fm = fileMeta.get(file.filePath);
|
|
962
|
+
if (!fm) continue;
|
|
963
|
+
if (fm.sessionId && sessionById.get(fm.sessionId)?.filePath !== file.filePath) continue;
|
|
964
|
+
|
|
965
|
+
const boundary = needsIndex.has(file.filePath)
|
|
966
|
+
? replayBoundary(fm, sessionById)
|
|
967
|
+
: { rawTokenCount: 0, recordIndex: null };
|
|
968
|
+
const key = boundaryKey(boundary);
|
|
969
|
+
let result = !auditPaths.has(file.filePath) && file.cache?.result?.boundaryKey === key
|
|
970
|
+
? file.cache.result
|
|
971
|
+
: null;
|
|
972
|
+
if (result) {
|
|
973
|
+
cacheStats.resultHits++;
|
|
974
|
+
} else {
|
|
975
|
+
const previousTail = !needsIndex.has(file.filePath) && !auditPaths.has(file.filePath)
|
|
976
|
+
? file.appendTail
|
|
977
|
+
: null;
|
|
978
|
+
const parsed = await parseSessionFile(file.filePath, file.snapshotSize, fm, boundary, {
|
|
979
|
+
previousTail,
|
|
980
|
+
captureTail: !needsIndex.has(file.filePath),
|
|
981
|
+
});
|
|
982
|
+
cacheStats.filesRead++;
|
|
983
|
+
if (previousTail) cacheStats.tailHits++;
|
|
984
|
+
const { tail, ...summary } = parsed;
|
|
985
|
+
if (tail) {
|
|
986
|
+
try {
|
|
987
|
+
saveCodexFileTail(codexHome, file.filePath, file.signature, tail);
|
|
988
|
+
} catch {
|
|
989
|
+
// Same fail-open rule as the summary cache: tail acceleration is
|
|
990
|
+
// optional and a write failure must not fail the parser.
|
|
991
|
+
}
|
|
992
|
+
}
|
|
993
|
+
result = { boundaryKey: key, ...summary };
|
|
994
|
+
updateFileCache(codexHome, file, { result, lastAuditedAt: Date.now() });
|
|
995
|
+
file.appendTail = null;
|
|
996
|
+
file.priorTail = null;
|
|
997
|
+
if (auditPaths.has(file.filePath)) cacheStats.audited++;
|
|
998
|
+
}
|
|
999
|
+
results.push(result);
|
|
1000
|
+
|
|
1001
|
+
if (overBudget() && i < files.length - 1) {
|
|
1002
|
+
return {
|
|
1003
|
+
buckets: [], sessions: [], skipped: true,
|
|
1004
|
+
indexing: { phase: 'usage', completed: i + 1, total: files.length },
|
|
1005
|
+
cache: cacheStats,
|
|
1006
|
+
};
|
|
1007
|
+
}
|
|
486
1008
|
}
|
|
487
1009
|
|
|
488
|
-
return {
|
|
1010
|
+
return { ...mergeFileResults(results), cache: cacheStats };
|
|
489
1011
|
}
|
package/src/sync.js
CHANGED
|
@@ -34,6 +34,7 @@ export async function runSync({ throws = false, quiet = false } = {}) {
|
|
|
34
34
|
const allBuckets = [];
|
|
35
35
|
const allSessions = [];
|
|
36
36
|
const parserResults = [];
|
|
37
|
+
const parserProgress = [];
|
|
37
38
|
// Sources whose parser ran to completion this sync. pruneState() below is
|
|
38
39
|
// scoped to these so a transient parser failure doesn't evict that tool's
|
|
39
40
|
// state and force a full re-upload next run.
|
|
@@ -47,6 +48,9 @@ export async function runSync({ throws = false, quiet = false } = {}) {
|
|
|
47
48
|
if (!Array.isArray(buckets) || !Array.isArray(sessions)) {
|
|
48
49
|
throw new TypeError('Parser returned an invalid result');
|
|
49
50
|
}
|
|
51
|
+
if (result?.indexing) {
|
|
52
|
+
parserProgress.push({ source, ...result.indexing });
|
|
53
|
+
}
|
|
50
54
|
// A parser may deliberately suppress a transient error (Cursor network
|
|
51
55
|
// timeout) to keep daemon logs quiet. Its empty result is not proof that
|
|
52
56
|
// its prior data disappeared, so it must not be pruned this run.
|
|
@@ -71,7 +75,13 @@ export async function runSync({ throws = false, quiet = false } = {}) {
|
|
|
71
75
|
pruneState(state, new Set(), new Set(), okSources);
|
|
72
76
|
const pruned = before - (Object.keys(state.buckets).length + Object.keys(state.sessions).length);
|
|
73
77
|
if (pruned > 0) saveState(state);
|
|
74
|
-
if (!quiet
|
|
78
|
+
if (!quiet && parserProgress.length > 0) {
|
|
79
|
+
for (const p of parserProgress) {
|
|
80
|
+
console.log(dim(` ${p.source}: 正在建立本地索引 ${p.completed}/${p.total}(下次同步继续)`));
|
|
81
|
+
}
|
|
82
|
+
} else if (!quiet) {
|
|
83
|
+
console.log(dim('暂无新数据。'));
|
|
84
|
+
}
|
|
75
85
|
return 0;
|
|
76
86
|
}
|
|
77
87
|
|
|
@@ -83,6 +93,11 @@ export async function runSync({ throws = false, quiet = false } = {}) {
|
|
|
83
93
|
console.log(` ${dim(p.source.padEnd(14))}${parts.join(' · ')}`);
|
|
84
94
|
}
|
|
85
95
|
}
|
|
96
|
+
if (!quiet && parserProgress.length > 0) {
|
|
97
|
+
for (const p of parserProgress) {
|
|
98
|
+
console.log(dim(` ${p.source}: 正在建立本地索引 ${p.completed}/${p.total}(下次同步继续)`));
|
|
99
|
+
}
|
|
100
|
+
}
|
|
86
101
|
|
|
87
102
|
let host = config.hostname;
|
|
88
103
|
if (!host) {
|
|
@@ -112,10 +127,11 @@ export async function runSync({ throws = false, quiet = false } = {}) {
|
|
|
112
127
|
for (const s of allSessions) s.project = 'unknown';
|
|
113
128
|
}
|
|
114
129
|
|
|
115
|
-
// Incremental diff: parsers above
|
|
116
|
-
//
|
|
117
|
-
//
|
|
118
|
-
//
|
|
130
|
+
// Incremental upload diff: parsers above emit a complete view of live local
|
|
131
|
+
// data (Codex may assemble that view from its disposable parser cache). Here
|
|
132
|
+
// we drop anything whose content matches what we already uploaded, so only
|
|
133
|
+
// new/changed items go over the network. A quiet machine sends zero bytes;
|
|
134
|
+
// an active one sends just the current 30-min bucket.
|
|
119
135
|
// Missing/corrupt state.json => empty maps => one-time full upload, then
|
|
120
136
|
// incremental forever after.
|
|
121
137
|
const state = loadState();
|