@phnx-labs/agents-cli 1.20.74 → 1.20.76
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +893 -0
- package/README.md +36 -10
- package/dist/bin/agents +0 -0
- package/dist/commands/apply.js +21 -3
- package/dist/commands/attach.d.ts +10 -0
- package/dist/commands/attach.js +41 -0
- package/dist/commands/computer.js +3 -3
- package/dist/commands/detach-core.d.ts +46 -0
- package/dist/commands/detach-core.js +61 -0
- package/dist/commands/detach.d.ts +5 -0
- package/dist/commands/detach.js +171 -0
- package/dist/commands/doctor.js +18 -0
- package/dist/commands/exec.js +31 -3
- package/dist/commands/feed.d.ts +7 -1
- package/dist/commands/feed.js +47 -4
- package/dist/commands/go.d.ts +9 -2
- package/dist/commands/go.js +16 -6
- package/dist/commands/harness.d.ts +14 -0
- package/dist/commands/harness.js +145 -0
- package/dist/commands/import.js +38 -8
- package/dist/commands/inspect.js +5 -2
- package/dist/commands/profiles.d.ts +20 -0
- package/dist/commands/profiles.js +68 -21
- package/dist/commands/repo.js +14 -0
- package/dist/commands/routines.js +15 -2
- package/dist/commands/secrets.js +44 -21
- package/dist/commands/sessions-browser.js +1 -1
- package/dist/commands/sessions-export.js +19 -5
- package/dist/commands/sessions-migrate.d.ts +29 -0
- package/dist/commands/sessions-migrate.js +596 -0
- package/dist/commands/sessions-picker.js +16 -1
- package/dist/commands/sessions.d.ts +36 -13
- package/dist/commands/sessions.js +210 -40
- package/dist/commands/setup-computer.js +2 -2
- package/dist/commands/ssh.js +71 -35
- package/dist/commands/status.js +10 -2
- package/dist/commands/versions.js +15 -6
- package/dist/commands/view.d.ts +5 -0
- package/dist/commands/view.js +24 -4
- package/dist/commands/watchdog.d.ts +7 -2
- package/dist/commands/watchdog.js +73 -57
- package/dist/index.js +58 -7
- package/dist/lib/activity.d.ts +14 -4
- package/dist/lib/activity.js +17 -5
- package/dist/lib/actor.d.ts +50 -0
- package/dist/lib/actor.js +166 -0
- package/dist/lib/agent-spec/provider.js +2 -1
- package/dist/lib/agent-spec/resolve.js +19 -5
- package/dist/lib/agent-spec/types.d.ts +9 -1
- package/dist/lib/agents.d.ts +21 -0
- package/dist/lib/agents.js +70 -6
- package/dist/lib/cloud/codex.d.ts +2 -0
- package/dist/lib/cloud/codex.js +14 -3
- package/dist/lib/cloud/session-index.d.ts +32 -0
- package/dist/lib/cloud/session-index.js +58 -0
- package/dist/lib/cloud/store.d.ts +7 -0
- package/dist/lib/cloud/store.js +25 -0
- package/dist/lib/config-transfer.js +4 -0
- package/dist/lib/daemon.js +18 -2
- package/dist/lib/devices/connect.d.ts +15 -1
- package/dist/lib/devices/connect.js +15 -1
- package/dist/lib/devices/fleet.d.ts +21 -0
- package/dist/lib/devices/fleet.js +26 -0
- package/dist/lib/devices/health-report.d.ts +38 -0
- package/dist/lib/devices/health-report.js +214 -0
- package/dist/lib/devices/health.js +4 -1
- package/dist/lib/devices/reachability.d.ts +31 -0
- package/dist/lib/devices/reachability.js +40 -0
- package/dist/lib/devices/registry.d.ts +33 -0
- package/dist/lib/devices/registry.js +37 -0
- package/dist/lib/devices/resolve-target.d.ts +15 -27
- package/dist/lib/devices/resolve-target.js +63 -102
- package/dist/lib/devices/sync.d.ts +18 -0
- package/dist/lib/devices/sync.js +23 -1
- package/dist/lib/devices/tailscale.d.ts +3 -0
- package/dist/lib/devices/tailscale.js +1 -0
- package/dist/lib/events.d.ts +1 -1
- package/dist/lib/exec.d.ts +66 -0
- package/dist/lib/exec.js +195 -6
- package/dist/lib/feed-post.d.ts +63 -0
- package/dist/lib/feed-post.js +204 -0
- package/dist/lib/fleet/apply.d.ts +32 -2
- package/dist/lib/fleet/apply.js +97 -10
- package/dist/lib/fleet/types.d.ts +11 -0
- package/dist/lib/fs-walk.d.ts +13 -0
- package/dist/lib/fs-walk.js +16 -7
- package/dist/lib/hooks.d.ts +1 -0
- package/dist/lib/hooks.js +156 -0
- package/dist/lib/hosts/dispatch.d.ts +7 -0
- package/dist/lib/hosts/dispatch.js +2 -0
- package/dist/lib/hosts/passthrough.js +2 -0
- package/dist/lib/hosts/reconnect.d.ts +78 -0
- package/dist/lib/hosts/reconnect.js +127 -0
- package/dist/lib/hosts/registry.d.ts +75 -14
- package/dist/lib/hosts/registry.js +205 -30
- package/dist/lib/hosts/remote-cmd.js +1 -0
- package/dist/lib/hosts/run-target.d.ts +13 -5
- package/dist/lib/hosts/run-target.js +26 -10
- package/dist/lib/hosts/session-index.d.ts +18 -1
- package/dist/lib/hosts/session-index.js +37 -2
- package/dist/lib/hosts/session-marker.d.ts +33 -0
- package/dist/lib/hosts/session-marker.js +51 -0
- package/dist/lib/hosts/tasks.d.ts +8 -4
- package/dist/lib/import.d.ts +2 -0
- package/dist/lib/import.js +35 -2
- package/dist/lib/menubar/install-menubar.d.ts +20 -0
- package/dist/lib/menubar/install-menubar.js +45 -1
- package/dist/lib/migrate.d.ts +16 -0
- package/dist/lib/migrate.js +36 -0
- package/dist/lib/models.d.ts +27 -0
- package/dist/lib/models.js +54 -1
- package/dist/lib/overdue.d.ts +7 -2
- package/dist/lib/overdue.js +18 -11
- package/dist/lib/picker.js +4 -1
- package/dist/lib/platform/process.d.ts +17 -0
- package/dist/lib/platform/process.js +70 -0
- package/dist/lib/profiles-presets.js +9 -7
- package/dist/lib/profiles.d.ts +31 -0
- package/dist/lib/profiles.js +70 -0
- package/dist/lib/pty-server.d.ts +2 -10
- package/dist/lib/pty-server.js +4 -38
- package/dist/lib/registry.d.ts +1 -1
- package/dist/lib/registry.js +48 -8
- package/dist/lib/rotate.d.ts +18 -0
- package/dist/lib/rotate.js +28 -0
- package/dist/lib/routines.d.ts +26 -0
- package/dist/lib/routines.js +19 -2
- package/dist/lib/runner.js +104 -7
- package/dist/lib/secrets/agent.d.ts +66 -1
- package/dist/lib/secrets/agent.js +154 -90
- package/dist/lib/secrets/bundles.d.ts +34 -0
- package/dist/lib/secrets/bundles.js +179 -19
- package/dist/lib/secrets/index.d.ts +15 -1
- package/dist/lib/secrets/index.js +21 -3
- package/dist/lib/secrets/rc-hygiene.d.ts +55 -0
- package/dist/lib/secrets/rc-hygiene.js +154 -0
- package/dist/lib/secrets/remote.d.ts +7 -4
- package/dist/lib/secrets/remote.js +7 -4
- package/dist/lib/secrets/sync-commands.d.ts +21 -0
- package/dist/lib/secrets/sync-commands.js +21 -0
- package/dist/lib/self-update.d.ts +23 -2
- package/dist/lib/self-update.js +86 -5
- package/dist/lib/session/active.d.ts +114 -7
- package/dist/lib/session/active.js +323 -52
- package/dist/lib/session/cloud.js +2 -1
- package/dist/lib/session/db.d.ts +48 -0
- package/dist/lib/session/db.js +238 -14
- package/dist/lib/session/detached.d.ts +30 -0
- package/dist/lib/session/detached.js +92 -0
- package/dist/lib/session/discover.d.ts +465 -2
- package/dist/lib/session/discover.js +1648 -468
- package/dist/lib/session/fork.js +2 -1
- package/dist/lib/session/hook-sessions.d.ts +43 -0
- package/dist/lib/session/hook-sessions.js +135 -0
- package/dist/lib/session/linear.d.ts +6 -0
- package/dist/lib/session/linear.js +55 -0
- package/dist/lib/session/migrate-targets.d.ts +65 -0
- package/dist/lib/session/migrate-targets.js +94 -0
- package/dist/lib/session/migrations.d.ts +37 -0
- package/dist/lib/session/migrations.js +60 -0
- package/dist/lib/session/parse.d.ts +18 -0
- package/dist/lib/session/parse.js +130 -30
- package/dist/lib/session/pid-registry.d.ts +34 -2
- package/dist/lib/session/pid-registry.js +49 -2
- package/dist/lib/session/render.d.ts +7 -2
- package/dist/lib/session/render.js +22 -10
- package/dist/lib/session/short-id.d.ts +17 -0
- package/dist/lib/session/short-id.js +20 -0
- package/dist/lib/session/state.js +6 -2
- package/dist/lib/shims.d.ts +1 -1
- package/dist/lib/shims.js +20 -6
- package/dist/lib/sqlite.d.ts +3 -2
- package/dist/lib/sqlite.js +27 -4
- package/dist/lib/staleness/detectors/commands.js +1 -1
- package/dist/lib/staleness/detectors/workflows.js +13 -2
- package/dist/lib/staleness/writers/commands.js +7 -7
- package/dist/lib/staleness/writers/workflows.d.ts +4 -2
- package/dist/lib/startup/command-registry.d.ts +1 -0
- package/dist/lib/startup/command-registry.js +3 -0
- package/dist/lib/state.d.ts +1 -1
- package/dist/lib/state.js +1 -32
- package/dist/lib/teams/agents.d.ts +19 -10
- package/dist/lib/teams/agents.js +40 -39
- package/dist/lib/types.d.ts +37 -2
- package/dist/lib/usage.d.ts +33 -1
- package/dist/lib/usage.js +172 -12
- package/dist/lib/versions.d.ts +9 -2
- package/dist/lib/versions.js +37 -8
- package/dist/lib/watchdog/log.d.ts +43 -0
- package/dist/lib/watchdog/log.js +69 -0
- package/dist/lib/watchdog/routine.d.ts +44 -0
- package/dist/lib/watchdog/routine.js +69 -0
- package/dist/lib/watchdog/runner.d.ts +51 -7
- package/dist/lib/watchdog/runner.js +239 -64
- package/dist/lib/watchdog/watchdog.d.ts +1 -1
- package/dist/lib/watchdog/watchdog.js +31 -16
- package/dist/lib/workflows.d.ts +16 -0
- package/dist/lib/workflows.js +110 -1
- package/package.json +1 -1
|
@@ -14,19 +14,21 @@ import * as readline from 'readline';
|
|
|
14
14
|
import { execFile } from 'child_process';
|
|
15
15
|
import { promisify } from 'util';
|
|
16
16
|
import Database from '../sqlite.js';
|
|
17
|
-
import { getAgentsDir, getHistoryDir, getRunsDir } from '../state.js';
|
|
17
|
+
import { getAgentsDir, getUserAgentsDir, getHistoryDir, getRunsDir } from '../state.js';
|
|
18
|
+
import { shortCodexHome } from '../codex-home.js';
|
|
18
19
|
const execFileAsync = promisify(execFile);
|
|
19
20
|
import { AGENTS, agentConfigDirName, getCliVersion } from '../agents.js';
|
|
20
|
-
import {
|
|
21
|
+
import { walkForFilesWithStat } from '../fs-walk.js';
|
|
21
22
|
import { getConfigSymlinkVersion } from '../shims.js';
|
|
22
23
|
import { SESSION_AGENTS } from './types.js';
|
|
24
|
+
import { deriveShortId } from './short-id.js';
|
|
23
25
|
import { extractSessionTopic } from './prompt.js';
|
|
24
26
|
import { parseAntigravity } from './parse.js';
|
|
25
27
|
import { extractPrUrl, detectWorktree, detectTicket, isPrCreateCommand, detectSpawnedTeam, isTicketCreateTool, extractCreatedTicket } from './state.js';
|
|
26
28
|
import { costOfUsage } from '../pricing/index.js';
|
|
27
29
|
import { machineId } from './sync/config.js';
|
|
28
30
|
import { mapBounded } from '../concurrency.js';
|
|
29
|
-
import { getDB, getScanStampByPath, getScanStampsForPaths, recordScans, syncLabels, seedLabelsFromNames, syncTopics, upsertSessionsBatch, querySessions, countSessions, ftsSearch, tryClaimScan, releaseScan, } from './db.js';
|
|
31
|
+
import { getDB, getScanStampByPath, getScanStampsForPaths, getParserStatesForPaths, getDirLedgerForPaths, recordDirScans, recordScans, syncLabels, seedLabelsFromNames, syncTopics, upsertSessionsBatch, querySessions, countSessions, ftsSearch, tryClaimScan, releaseScan, } from './db.js';
|
|
30
32
|
import { buildRunNameMap } from './run-names.js';
|
|
31
33
|
const HOME = os.homedir();
|
|
32
34
|
// Versions can live under either repo: the user repo (current canonical
|
|
@@ -41,6 +43,26 @@ const HERMES_SESSIONS_DIR = path.join(HOME, '.hermes', 'sessions');
|
|
|
41
43
|
/** How long OpenClaw channel/cron snapshots stay valid before we re-shell-out. */
|
|
42
44
|
const OPENCLAW_TTL_MS = 60_000;
|
|
43
45
|
const ACTIVE_APPEND_RESCAN_DEBOUNCE_MS = 5_000;
|
|
46
|
+
/**
|
|
47
|
+
* How recently a file must have been scanned to be treated as "hot" — a
|
|
48
|
+
* candidate for an in-place append even when its parent dir's mtime hasn't
|
|
49
|
+
* moved. A dir-ledger match lets us skip the per-file stat of everything in a
|
|
50
|
+
* leaf dir EXCEPT its hot set; a file is hot if it lives under the agent's live
|
|
51
|
+
* `~/.<agent>` root (the only tree an agent appends to live) or was scanned
|
|
52
|
+
* within this window. 10 minutes comfortably covers a session that paused
|
|
53
|
+
* between `agents sessions` calls but is still being written to.
|
|
54
|
+
*/
|
|
55
|
+
const HOT_FILE_WINDOW_MS = 600_000;
|
|
56
|
+
/**
|
|
57
|
+
* Kill-switch: set `AGENTS_SESSIONS_NO_DIR_LEDGER=1` to force the old full-walk
|
|
58
|
+
* path (readdir + per-file stat every dir, every run — the pre-A-2 behavior),
|
|
59
|
+
* skipping the dir_ledger short-circuit entirely. One env var reverts a field
|
|
60
|
+
* regression to today's behavior.
|
|
61
|
+
*/
|
|
62
|
+
function dirLedgerDisabled() {
|
|
63
|
+
const v = process.env.AGENTS_SESSIONS_NO_DIR_LEDGER;
|
|
64
|
+
return v === '1' || v === 'true';
|
|
65
|
+
}
|
|
44
66
|
let cachedOpenClawWorkspaces = null;
|
|
45
67
|
const cachedAgentVersions = new Map();
|
|
46
68
|
/**
|
|
@@ -79,7 +101,56 @@ export async function discoverSessions(options) {
|
|
|
79
101
|
const sessions = querySessions(buildQueryOptions(options, agents, { includeLimit: true }));
|
|
80
102
|
for (const s of sessions)
|
|
81
103
|
s.machine = machineForSessionFile(s.filePath, s.agent);
|
|
82
|
-
return sessions;
|
|
104
|
+
return scopeToManaged(sessions, agents, options);
|
|
105
|
+
}
|
|
106
|
+
/**
|
|
107
|
+
* Drop unmanaged rows for agents that HAVE managed versions.
|
|
108
|
+
*
|
|
109
|
+
* Scoping happens here, at query time, rather than by narrowing the scan: the index
|
|
110
|
+
* stays complete, so `--unmanaged` needs no re-scan and every other consumer of the
|
|
111
|
+
* DB (watchdog, the Factory watcher, `--roots`) is unaffected.
|
|
112
|
+
*
|
|
113
|
+
* An agent with no managed versions is left alone entirely — someone who has never
|
|
114
|
+
* run `agents add` sees exactly what they saw before.
|
|
115
|
+
*/
|
|
116
|
+
function scopeToManaged(sessions, agents, options) {
|
|
117
|
+
if (options?.includeUnmanaged)
|
|
118
|
+
return sessions;
|
|
119
|
+
if (!anyManagedVersions())
|
|
120
|
+
return sessions;
|
|
121
|
+
const kept = sessions.filter((s) => isManagedSessionFile(s.filePath));
|
|
122
|
+
const hidden = sessions.length - kept.length;
|
|
123
|
+
if (hidden > 0)
|
|
124
|
+
options?.onHiddenUnmanaged?.(hidden);
|
|
125
|
+
return kept;
|
|
126
|
+
}
|
|
127
|
+
/**
|
|
128
|
+
* True once agents-cli manages ANY agent version. Until then it manages nothing, so
|
|
129
|
+
* scoping to "managed only" would leave the listing empty for a user who has never
|
|
130
|
+
* run `agents add` — the browser is most of the tool's value before you install
|
|
131
|
+
* anything through it.
|
|
132
|
+
*/
|
|
133
|
+
function anyManagedVersions() {
|
|
134
|
+
for (const root of VERSIONS_ROOTS) {
|
|
135
|
+
const base = path.join(root, 'versions');
|
|
136
|
+
let agentDirs;
|
|
137
|
+
try {
|
|
138
|
+
agentDirs = fs.readdirSync(base, { withFileTypes: true });
|
|
139
|
+
}
|
|
140
|
+
catch {
|
|
141
|
+
continue;
|
|
142
|
+
}
|
|
143
|
+
for (const a of agentDirs) {
|
|
144
|
+
if (!a.isDirectory())
|
|
145
|
+
continue;
|
|
146
|
+
try {
|
|
147
|
+
if (fs.readdirSync(path.join(base, a.name), { withFileTypes: true }).some((e) => e.isDirectory()))
|
|
148
|
+
return true;
|
|
149
|
+
}
|
|
150
|
+
catch { /* unreadable */ }
|
|
151
|
+
}
|
|
152
|
+
}
|
|
153
|
+
return false;
|
|
83
154
|
}
|
|
84
155
|
/**
|
|
85
156
|
* How many agents' dotfile dirs we scan at once, and the minimum spacing between
|
|
@@ -108,6 +179,7 @@ function dispatchAgentScan(agent, onProgress) {
|
|
|
108
179
|
case 'hermes': return scanHermesIncremental(onProgress);
|
|
109
180
|
case 'kimi': return scanKimiIncremental(onProgress);
|
|
110
181
|
case 'droid': return scanDroidIncremental(onProgress);
|
|
182
|
+
case 'grok': return scanGrokIncremental(onProgress);
|
|
111
183
|
default: return Promise.resolve();
|
|
112
184
|
}
|
|
113
185
|
}
|
|
@@ -119,6 +191,45 @@ let _localMachineId;
|
|
|
119
191
|
* when the path sits under the agent's backups root, the first segment below it
|
|
120
192
|
* is the origin machine id; otherwise it's the local machine.
|
|
121
193
|
*/
|
|
194
|
+
/**
|
|
195
|
+
* True when this transcript belongs to a version agents-cli manages — i.e. it lives
|
|
196
|
+
* under a version home (or a backup mirror of one) rather than in the user's own
|
|
197
|
+
* `~/.<agent>`.
|
|
198
|
+
*
|
|
199
|
+
* `agents sessions` scans both, which is right for indexing: the DB stays a complete
|
|
200
|
+
* picture and `--unmanaged` can surface everything without a re-scan. But listing
|
|
201
|
+
* *by default* is a different question. Once you have managed versions, an unmanaged
|
|
202
|
+
* install's history is not really agents-cli's to show — most visibly after
|
|
203
|
+
* `agents add --isolated`, where the whole point was to keep the two apart.
|
|
204
|
+
*/
|
|
205
|
+
export function isManagedSessionFile(filePath) {
|
|
206
|
+
// Synthetic rows (OpenClaw workspace sessions, cloud/remote entries) have no local
|
|
207
|
+
// transcript to classify. They are produced BY agents-cli rather than read out of
|
|
208
|
+
// someone's dotfile dir, so scoping must not silently swallow them.
|
|
209
|
+
if (!filePath || !path.isAbsolute(filePath))
|
|
210
|
+
return true;
|
|
211
|
+
const roots = [
|
|
212
|
+
...VERSIONS_ROOTS.map((root) => path.join(root, 'versions')),
|
|
213
|
+
path.join(getHistoryDir(), 'backups'),
|
|
214
|
+
// Codex's managed home is not always under versions/. On macOS the versioned path
|
|
215
|
+
// overflows SUN_LEN for codex's control socket, so the shim relocates it to
|
|
216
|
+
// `<agentsUserDir>/.codex-homes/<version>/` (lib/codex-home.ts).
|
|
217
|
+
path.join(getUserAgentsDir(), '.codex-homes'),
|
|
218
|
+
// Routine archives are agents-cli's OWN run output — managed by definition.
|
|
219
|
+
getRunsDir(),
|
|
220
|
+
];
|
|
221
|
+
// Compare realpaths as well as the literal roots. A transcript's stored path is
|
|
222
|
+
// resolved, so on macOS (`/var` -> `/private/var`) a temp-dir HOME yields
|
|
223
|
+
// `/private/var/...` for the file and `/var/...` for the root, and a plain prefix
|
|
224
|
+
// test silently classifies every managed session as the user's own.
|
|
225
|
+
const real = safeRealpathSync(filePath) || filePath;
|
|
226
|
+
return roots.some((root) => {
|
|
227
|
+
if (filePath.startsWith(root + path.sep))
|
|
228
|
+
return true;
|
|
229
|
+
const realRoot = safeRealpathSync(root);
|
|
230
|
+
return !!realRoot && real.startsWith(realRoot + path.sep);
|
|
231
|
+
});
|
|
232
|
+
}
|
|
122
233
|
export function machineForSessionFile(filePath, agent) {
|
|
123
234
|
const base = path.join(getHistoryDir(), 'backups', agent) + path.sep;
|
|
124
235
|
if (filePath.startsWith(base)) {
|
|
@@ -168,12 +279,74 @@ function buildQueryOptions(options, agents, opts) {
|
|
|
168
279
|
sortBy: options?.sortBy,
|
|
169
280
|
};
|
|
170
281
|
}
|
|
171
|
-
/**
|
|
282
|
+
/**
|
|
283
|
+
* Canonicalize a working directory path (follows symlinks when it is local).
|
|
284
|
+
*
|
|
285
|
+
* Most callers pass a cwd RECORDED in a transcript, which may name a directory
|
|
286
|
+
* on another machine — a POSIX path read on a Windows host, say. `path.resolve()`
|
|
287
|
+
* rebases such a path onto the current drive (`/Users/me` -> `D:\Users\me`),
|
|
288
|
+
* inventing a location that never existed. So an absolute path is normalized but
|
|
289
|
+
* never rebased; only a genuinely relative one resolves against the process cwd.
|
|
290
|
+
*
|
|
291
|
+
* `path.normalize()` still runs on every branch: it collapses `.`, `..`, and
|
|
292
|
+
* duplicate separators, and folds separators on Windows. Both sides of the cwd
|
|
293
|
+
* filter in `db.ts` (`cwd = ?` and `cwd LIKE ? || path.sep || '%'`) come through
|
|
294
|
+
* here, so dropping that would leave a trailing slash or a `..` segment in one
|
|
295
|
+
* side and match nothing.
|
|
296
|
+
*
|
|
297
|
+
* Realpath is attempted only for a path that is absolute in THIS platform's
|
|
298
|
+
* terms. A POSIX-rooted path on Windows is drive-relative to `fs.realpathSync`,
|
|
299
|
+
* which would resolve `/Users/me` against the current drive and reintroduce the
|
|
300
|
+
* graft for any path that happens to exist locally.
|
|
301
|
+
*/
|
|
302
|
+
export function _normalizeCwdForTest(cwd) {
|
|
303
|
+
return normalizeCwd(cwd);
|
|
304
|
+
}
|
|
172
305
|
function normalizeCwd(cwd) {
|
|
173
306
|
if (!cwd)
|
|
174
307
|
return '';
|
|
175
|
-
|
|
176
|
-
|
|
308
|
+
// A POSIX-rooted path on Windows belongs to another machine. Normalize it with
|
|
309
|
+
// POSIX rules so its separators survive — path.win32.normalize would fold them
|
|
310
|
+
// to backslashes, mangling the very path we are trying to preserve — and never
|
|
311
|
+
// realpath it, since fs.realpathSync would resolve it against the current drive.
|
|
312
|
+
if (process.platform === 'win32' && /^\//.test(cwd) && !/^[a-zA-Z]:/.test(cwd)) {
|
|
313
|
+
return stripTrailingSep(path.posix.normalize(cwd));
|
|
314
|
+
}
|
|
315
|
+
const normalized = path.isAbsolute(cwd) ? stripTrailingSep(path.normalize(cwd)) : path.resolve(cwd);
|
|
316
|
+
return safeRealpathSync(normalized) || normalized;
|
|
317
|
+
}
|
|
318
|
+
/** Drop a trailing separator so `cwd = ?` and the `cwd LIKE ? + sep` subdir
|
|
319
|
+
* wildcard agree; a root path (`/`, `C:\`) keeps its separator. */
|
|
320
|
+
function stripTrailingSep(p) {
|
|
321
|
+
const stripped = p.replace(/[\\/]+$/, '');
|
|
322
|
+
return stripped.length > 0 && !/^[a-zA-Z]:$/.test(stripped) ? stripped : p;
|
|
323
|
+
}
|
|
324
|
+
/** Canonical 8-4-4-4-12 hex UUID (covers both v4 and the v7 ids newer harnesses mint). */
|
|
325
|
+
const UUID_36 = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/;
|
|
326
|
+
/** kimi and rush mint `session_` + a UUID. */
|
|
327
|
+
const SESSION_UUID_PREFIX = /^session_/;
|
|
328
|
+
/** opencode mints `ses_` + a 26-char ULID — NOT a UUID, so it needs its own shape. */
|
|
329
|
+
const SES_ULID = /^ses_[0-9a-z]{26}$/;
|
|
330
|
+
/**
|
|
331
|
+
* Whether a query is a session id in full, rather than an id prefix or a search
|
|
332
|
+
* phrase.
|
|
333
|
+
*
|
|
334
|
+
* Callers use this to decide that an id lookup is the ONLY admissible
|
|
335
|
+
* interpretation: a complete id is unique, so when it misses there is nothing
|
|
336
|
+
* left to widen to. Without the check, `sessions <uuid>` fell through to the
|
|
337
|
+
* FTS content search, which tokenizes the UUID and matches every transcript that
|
|
338
|
+
* merely mentions it — surfacing unrelated sessions as if they were id matches.
|
|
339
|
+
*
|
|
340
|
+
* The accepted shapes are the ones the index actually holds, measured over a
|
|
341
|
+
* 12,507-row index: a bare UUID (11,116 rows), `session_` + UUID (1,360 — kimi
|
|
342
|
+
* and rush), and `ses_` + ULID (15 — opencode). Deliberately NOT covered, so a
|
|
343
|
+
* miss keeps today's search behavior rather than gaining a wrong error: routine
|
|
344
|
+
* run ids (ISO timestamps, matched via `routineRunId` below) and cloud execution
|
|
345
|
+
* ids, whose charset is too permissive to distinguish from a search phrase.
|
|
346
|
+
*/
|
|
347
|
+
export function isCompleteSessionId(query) {
|
|
348
|
+
const q = query.trim().toLowerCase();
|
|
349
|
+
return UUID_36.test(q.replace(SESSION_UUID_PREFIX, '')) || SES_ULID.test(q);
|
|
177
350
|
}
|
|
178
351
|
/**
|
|
179
352
|
* Resolve a session by full or short ID. Accepts a pre-loaded session list
|
|
@@ -230,26 +403,39 @@ export function searchContentIndex(sessions, query) {
|
|
|
230
403
|
* growing transcript over and over. The cached row is good enough for a few
|
|
231
404
|
* seconds; once writes settle or the debounce expires, the file is parsed once.
|
|
232
405
|
*/
|
|
233
|
-
function filterChangedFiles(filePaths) {
|
|
234
|
-
const
|
|
235
|
-
const out = [];
|
|
236
|
-
const now = Date.now();
|
|
406
|
+
export function filterChangedFiles(filePaths) {
|
|
407
|
+
const entries = [];
|
|
237
408
|
for (const filePath of filePaths) {
|
|
238
409
|
const stat = safeStatSync(filePath);
|
|
239
410
|
if (!stat)
|
|
240
411
|
continue;
|
|
412
|
+
entries.push({ filePath, fileMtimeMs: stat.mtimeMs, fileSize: stat.size });
|
|
413
|
+
}
|
|
414
|
+
return filterChangedEntries(entries);
|
|
415
|
+
}
|
|
416
|
+
/**
|
|
417
|
+
* Ledger-compare pre-stat'd entries (from the walk's own stat) without a second
|
|
418
|
+
* stat. Same debounce and change-detection as filterChangedFiles; the raw
|
|
419
|
+
* mtime is floored here so warm files match the ledger exactly as the stat path
|
|
420
|
+
* does (Math.floor(stat.mtimeMs)).
|
|
421
|
+
*/
|
|
422
|
+
export function filterChangedEntries(entries) {
|
|
423
|
+
const ledger = getScanStampsForPaths(entries.map(e => e.filePath));
|
|
424
|
+
const out = [];
|
|
425
|
+
const now = Date.now();
|
|
426
|
+
for (const entry of entries) {
|
|
241
427
|
const scan = {
|
|
242
|
-
fileMtimeMs: Math.floor(
|
|
243
|
-
fileSize:
|
|
428
|
+
fileMtimeMs: Math.floor(entry.fileMtimeMs),
|
|
429
|
+
fileSize: entry.fileSize,
|
|
244
430
|
};
|
|
245
|
-
const prev = ledger.get(filePath);
|
|
431
|
+
const prev = ledger.get(entry.filePath);
|
|
246
432
|
if (prev && prev.fileMtimeMs === scan.fileMtimeMs && prev.fileSize === scan.fileSize) {
|
|
247
433
|
continue;
|
|
248
434
|
}
|
|
249
435
|
if (prev && shouldDeferRecentAppend(prev, scan, now)) {
|
|
250
436
|
continue;
|
|
251
437
|
}
|
|
252
|
-
out.push({ filePath, scan });
|
|
438
|
+
out.push({ filePath: entry.filePath, scan });
|
|
253
439
|
}
|
|
254
440
|
return out;
|
|
255
441
|
}
|
|
@@ -262,6 +448,96 @@ export function shouldDeferRecentAppend(prev, current, nowMs, debounceMs = ACTIV
|
|
|
262
448
|
return false;
|
|
263
449
|
return nowMs - prev.scannedAt < debounceMs;
|
|
264
450
|
}
|
|
451
|
+
/**
|
|
452
|
+
* Walk a set of leaf transcript directories and return the files that changed,
|
|
453
|
+
* skipping the per-file `stat` of directories whose (mtime, entry_count) matches
|
|
454
|
+
* the dir_ledger.
|
|
455
|
+
*
|
|
456
|
+
* Per leaf dir:
|
|
457
|
+
* - `stat` the dir once and `readdir` it (one cheap syscall) to get the entry
|
|
458
|
+
* count and the file list.
|
|
459
|
+
* - If the dir matches the dir_ledger (floored mtime AND entry_count), no file
|
|
460
|
+
* was created / deleted / renamed since we last walked it. We then stat ONLY
|
|
461
|
+
* the hot files (live-root files, or files scanned within HOT_FILE_WINDOW_MS)
|
|
462
|
+
* and run just those through the ledger compare — so an in-place append to a
|
|
463
|
+
* still-live session is still caught, while immutable backup/version dirs
|
|
464
|
+
* collapse to a single dir stat and zero per-file stats.
|
|
465
|
+
* - Else (changed dir, or no ledger row) we stat every file (today's full
|
|
466
|
+
* walk) and record the fresh dir stamp so the next run can short-circuit.
|
|
467
|
+
*
|
|
468
|
+
* The kill-switch (`AGENTS_SESSIONS_NO_DIR_LEDGER=1`) forces the full-walk branch
|
|
469
|
+
* for every dir and never consults or records the dir_ledger.
|
|
470
|
+
*/
|
|
471
|
+
export function collectChangedFilesInLeafDirs(leafDirs, ext) {
|
|
472
|
+
const disabled = dirLedgerDisabled();
|
|
473
|
+
const dirStamps = disabled ? new Map() : getDirLedgerForPaths(leafDirs.map(d => d.dirPath));
|
|
474
|
+
const now = Date.now();
|
|
475
|
+
// Files whose per-file stat we still need to ledger-compare this run.
|
|
476
|
+
const toCompare = [];
|
|
477
|
+
const allFiles = [];
|
|
478
|
+
const dirScansToRecord = [];
|
|
479
|
+
for (const { dirPath, isLiveRoot } of leafDirs) {
|
|
480
|
+
const dirStat = safeStatSync(dirPath);
|
|
481
|
+
if (!dirStat?.isDirectory())
|
|
482
|
+
continue;
|
|
483
|
+
let names;
|
|
484
|
+
try {
|
|
485
|
+
names = fs.readdirSync(dirPath).filter(f => f.endsWith(ext));
|
|
486
|
+
}
|
|
487
|
+
catch {
|
|
488
|
+
continue;
|
|
489
|
+
}
|
|
490
|
+
const files = names.map(f => path.join(dirPath, f));
|
|
491
|
+
for (const filePath of files)
|
|
492
|
+
allFiles.push({ filePath, isLiveRoot });
|
|
493
|
+
const dirMtimeMs = Math.floor(dirStat.mtimeMs);
|
|
494
|
+
const entryCount = names.length;
|
|
495
|
+
const prevDir = dirStamps.get(dirPath);
|
|
496
|
+
const dirUnchanged = !disabled && prevDir !== undefined && prevDir.dirMtimeMs === dirMtimeMs && prevDir.entryCount === entryCount;
|
|
497
|
+
if (dirUnchanged) {
|
|
498
|
+
// Contents did not change (no create/delete/rename). Stat only the hot
|
|
499
|
+
// files; the rest are served from the DB with no stat. An immutable backup
|
|
500
|
+
// dir (not a live root, nothing recently scanned) does zero per-file stats.
|
|
501
|
+
//
|
|
502
|
+
// A live-root dir treats every file as hot — that is the tree an agent
|
|
503
|
+
// appends to live, and an append does NOT bump the parent-dir mtime, so
|
|
504
|
+
// without this a growing session would be silently skipped. A non-live
|
|
505
|
+
// file is hot only if it was scanned within HOT_FILE_WINDOW_MS (bulk ledger
|
|
506
|
+
// lookup), covering a session under a version/backup path that is somehow
|
|
507
|
+
// still being written.
|
|
508
|
+
const stamps = isLiveRoot ? null : getScanStampsForPaths(files);
|
|
509
|
+
for (const filePath of files) {
|
|
510
|
+
let hot = isLiveRoot;
|
|
511
|
+
if (!hot && stamps) {
|
|
512
|
+
const s = stamps.get(filePath);
|
|
513
|
+
hot = s?.scannedAt !== undefined && now - s.scannedAt <= HOT_FILE_WINDOW_MS;
|
|
514
|
+
}
|
|
515
|
+
if (!hot)
|
|
516
|
+
continue;
|
|
517
|
+
const stat = safeStatSync(filePath);
|
|
518
|
+
if (!stat)
|
|
519
|
+
continue;
|
|
520
|
+
toCompare.push({ filePath, fileMtimeMs: stat.mtimeMs, fileSize: stat.size });
|
|
521
|
+
}
|
|
522
|
+
// No dir stamp to record — nothing about the dir changed.
|
|
523
|
+
}
|
|
524
|
+
else {
|
|
525
|
+
// Changed dir (or cold ledger): full per-file stat, exactly as today.
|
|
526
|
+
for (const filePath of files) {
|
|
527
|
+
const stat = safeStatSync(filePath);
|
|
528
|
+
if (!stat)
|
|
529
|
+
continue;
|
|
530
|
+
toCompare.push({ filePath, fileMtimeMs: stat.mtimeMs, fileSize: stat.size });
|
|
531
|
+
}
|
|
532
|
+
if (!disabled)
|
|
533
|
+
dirScansToRecord.push({ dirPath, dirMtimeMs, entryCount });
|
|
534
|
+
}
|
|
535
|
+
}
|
|
536
|
+
const changed = filterChangedEntries(toCompare);
|
|
537
|
+
if (dirScansToRecord.length > 0)
|
|
538
|
+
recordDirScans(dirScansToRecord);
|
|
539
|
+
return { changed, allFiles };
|
|
540
|
+
}
|
|
265
541
|
// ---------------------------------------------------------------------------
|
|
266
542
|
// Multi-version directory scanning
|
|
267
543
|
// ---------------------------------------------------------------------------
|
|
@@ -294,6 +570,17 @@ export function getAgentSessionDirs(agent, subdir) {
|
|
|
294
570
|
try {
|
|
295
571
|
for (const version of fs.readdirSync(versionsBase)) {
|
|
296
572
|
addDir(path.join(versionsBase, version, 'home', configDirName, subdir));
|
|
573
|
+
// Codex's managed home is not always where the version layout says. On macOS
|
|
574
|
+
// the versioned path overflows SUN_LEN (104 bytes) for codex's control
|
|
575
|
+
// socket, so the shim relocates the home to
|
|
576
|
+
// `<agentsUserDir>/.codex-homes/<version>/.codex` (lib/codex-home.ts). Every
|
|
577
|
+
// transcript an isolated codex writes lands there, and nothing scanned it —
|
|
578
|
+
// `agents sessions --roots` listed only the user's own ~/.codex, so a managed
|
|
579
|
+
// copy's own history was invisible. addDir skips what does not exist, so this
|
|
580
|
+
// is inert on Linux and for versions that never needed relocating.
|
|
581
|
+
if (agent === 'codex') {
|
|
582
|
+
addDir(path.join(shortCodexHome(getUserAgentsDir(), version), subdir));
|
|
583
|
+
}
|
|
297
584
|
}
|
|
298
585
|
}
|
|
299
586
|
catch { /* dir unreadable */ }
|
|
@@ -324,6 +611,7 @@ const SESSION_ROOT_SPECS = [
|
|
|
324
611
|
{ agent: 'antigravity', subdir: 'conversations' },
|
|
325
612
|
{ agent: 'droid', subdir: 'sessions' },
|
|
326
613
|
{ agent: 'kimi', subdir: 'sessions' },
|
|
614
|
+
{ agent: 'grok', subdir: 'sessions' },
|
|
327
615
|
];
|
|
328
616
|
function sessionRootSubdir(agent) {
|
|
329
617
|
return SESSION_ROOT_SPECS.find((spec) => spec.agent === agent)?.subdir ?? null;
|
|
@@ -402,7 +690,14 @@ async function readRoutineArchiveMeta(agent, filePath) {
|
|
|
402
690
|
return null;
|
|
403
691
|
if (agent === 'claude') {
|
|
404
692
|
const sessionId = path.basename(filePath).replace(/\.jsonl$/, '');
|
|
405
|
-
|
|
693
|
+
// Routine archives are finalized, immutable transcripts — no live append, so
|
|
694
|
+
// no continuation to resume. A FULL parse (undefined prior) is correct here;
|
|
695
|
+
// the returned continuation is unused by this archive path.
|
|
696
|
+
const stat = safeStatSync(filePath);
|
|
697
|
+
if (!stat)
|
|
698
|
+
return null;
|
|
699
|
+
const scanStamp = { fileMtimeMs: Math.floor(stat.mtimeMs), fileSize: stat.size };
|
|
700
|
+
const result = await readClaudeMeta(filePath, sessionId, scanStamp, undefined);
|
|
406
701
|
return result ? { ...result, meta: decorateRoutineSession(result.meta, info) } : null;
|
|
407
702
|
}
|
|
408
703
|
if (agent === 'codex') {
|
|
@@ -416,12 +711,13 @@ async function scanRoutineArchivesIncremental(agent, onProgress) {
|
|
|
416
711
|
if (!subdir)
|
|
417
712
|
return;
|
|
418
713
|
const ext = agent === 'gemini' ? '.json' : '.jsonl';
|
|
419
|
-
const
|
|
714
|
+
const prestat = [];
|
|
420
715
|
for (const sessionsDir of getRoutineArchiveSessionDirs(agent, subdir)) {
|
|
421
|
-
for (const
|
|
422
|
-
|
|
716
|
+
for (const f of walkForFilesWithStat(sessionsDir, ext, 100_000)) {
|
|
717
|
+
prestat.push({ filePath: f.path, fileMtimeMs: f.mtimeMs, fileSize: f.size });
|
|
718
|
+
}
|
|
423
719
|
}
|
|
424
|
-
const changed =
|
|
720
|
+
const changed = filterChangedEntries(prestat);
|
|
425
721
|
if (changed.length === 0)
|
|
426
722
|
return;
|
|
427
723
|
onProgress?.({ agent, parsed: 0, total: changed.length });
|
|
@@ -533,40 +829,64 @@ export function buildClaudeLabelMap() {
|
|
|
533
829
|
async function scanClaudeIncremental(onProgress) {
|
|
534
830
|
const account = getClaudeAccount();
|
|
535
831
|
const labelMap = buildClaudeLabelMap();
|
|
536
|
-
|
|
537
|
-
|
|
538
|
-
|
|
832
|
+
// Enumerate every leaf project dir across all Claude roots. The FIRST root
|
|
833
|
+
// returned by getAgentSessionDirs is the agent's live `~/.claude/projects` —
|
|
834
|
+
// the only tree Claude appends to in place, so its project dirs are live roots
|
|
835
|
+
// (every file hot). Version-home + backup roots are immutable: their dirs
|
|
836
|
+
// short-circuit to a single dir stat when unchanged.
|
|
837
|
+
const roots = getAgentSessionDirs('claude', 'projects');
|
|
838
|
+
const leafDirs = [];
|
|
839
|
+
const seenLeaf = new Set();
|
|
840
|
+
roots.forEach((projectsDir, rootIdx) => {
|
|
841
|
+
const isLiveRoot = rootIdx === 0;
|
|
539
842
|
let projectDirs;
|
|
540
843
|
try {
|
|
541
844
|
projectDirs = fs.readdirSync(projectsDir);
|
|
542
845
|
}
|
|
543
846
|
catch {
|
|
544
|
-
|
|
847
|
+
return;
|
|
545
848
|
}
|
|
546
849
|
for (const dirName of projectDirs) {
|
|
547
850
|
const dirPath = path.join(projectsDir, dirName);
|
|
548
|
-
const
|
|
549
|
-
if (
|
|
550
|
-
continue;
|
|
551
|
-
let files;
|
|
552
|
-
try {
|
|
553
|
-
files = fs.readdirSync(dirPath).filter(f => f.endsWith('.jsonl'));
|
|
554
|
-
}
|
|
555
|
-
catch {
|
|
851
|
+
const key = safeRealpathSync(dirPath) || dirPath;
|
|
852
|
+
if (seenLeaf.has(key))
|
|
556
853
|
continue;
|
|
557
|
-
|
|
558
|
-
|
|
559
|
-
const sessionId = file.replace('.jsonl', '');
|
|
560
|
-
if (seen.has(sessionId))
|
|
561
|
-
continue;
|
|
562
|
-
seen.add(sessionId);
|
|
563
|
-
filePaths.push(path.join(dirPath, file));
|
|
564
|
-
}
|
|
854
|
+
seenLeaf.add(key);
|
|
855
|
+
leafDirs.push({ dirPath, isLiveRoot });
|
|
565
856
|
}
|
|
566
|
-
}
|
|
567
|
-
const changed =
|
|
857
|
+
});
|
|
858
|
+
const { changed: changedAll, allFiles } = collectChangedFilesInLeafDirs(leafDirs, '.jsonl');
|
|
859
|
+
// Restore the pre-A-2 cross-root precedence: a session id present in multiple
|
|
860
|
+
// roots is ALWAYS served from its live path, never a frozen backup/version
|
|
861
|
+
// copy. Pre-A-2, dedup happened at enumeration time via a live-first `seen`
|
|
862
|
+
// set, so a non-live copy was never even stat'd when a live copy existed. This
|
|
863
|
+
// PR must not regress that to "whichever copy changed this run wins" — a cold
|
|
864
|
+
// (unchanged) live copy paired with a freshly-written backup snapshot would
|
|
865
|
+
// otherwise flip the row's file_path to the backup path.
|
|
866
|
+
//
|
|
867
|
+
// allFiles is every transcript file across all roots in live-first order, so
|
|
868
|
+
// the FIRST occurrence of each session id is its live (or highest-precedence)
|
|
869
|
+
// path — the durable winner, independent of which copy was flagged changed.
|
|
870
|
+
const sessionIdOf = (fp) => path.basename(fp).replace('.jsonl', '');
|
|
871
|
+
const winnerBySession = new Map();
|
|
872
|
+
for (const { filePath } of allFiles) {
|
|
873
|
+
const id = sessionIdOf(filePath);
|
|
874
|
+
if (!winnerBySession.has(id))
|
|
875
|
+
winnerBySession.set(id, filePath);
|
|
876
|
+
}
|
|
877
|
+
// Keep a changed entry only if it is its session's winner. A changed non-live
|
|
878
|
+
// copy is dropped whenever a live copy exists anywhere; the winning path is
|
|
879
|
+
// parsed only when it itself changed (a cold winner needs no re-parse — its DB
|
|
880
|
+
// row already points at the live path).
|
|
881
|
+
const changed = changedAll.filter(e => winnerBySession.get(sessionIdOf(e.filePath)) === e.filePath);
|
|
568
882
|
if (changed.length > 0) {
|
|
569
883
|
onProgress?.({ agent: 'claude', parsed: 0, total: changed.length });
|
|
884
|
+
// Bulk-fetch each changed file's prior resumable continuation. A file with a
|
|
885
|
+
// usable prior state + growth goes incremental (re-parse only the appended
|
|
886
|
+
// bytes); everything else does a FULL from-offset-0 parse. The decision + the
|
|
887
|
+
// parse both live in scanClaudeSessionResumable so full and incremental share
|
|
888
|
+
// one reducer and produce identical rows.
|
|
889
|
+
const priorStates = getParserStatesForPaths(changed.map(c => c.filePath));
|
|
570
890
|
const entries = [];
|
|
571
891
|
const touched = [];
|
|
572
892
|
let parsed = 0;
|
|
@@ -574,9 +894,16 @@ async function scanClaudeIncremental(onProgress) {
|
|
|
574
894
|
try {
|
|
575
895
|
const sessionId = path.basename(filePath).replace('.jsonl', '');
|
|
576
896
|
const label = labelMap.get(sessionId) ?? undefined;
|
|
577
|
-
const
|
|
897
|
+
const priorRow = priorStates.get(filePath);
|
|
898
|
+
const result = await readClaudeMeta(filePath, sessionId, scan, priorRow, account, label);
|
|
578
899
|
if (result) {
|
|
579
|
-
entries.push({
|
|
900
|
+
entries.push({
|
|
901
|
+
meta: result.meta,
|
|
902
|
+
content: result.content,
|
|
903
|
+
scan,
|
|
904
|
+
parserState: result.parserState,
|
|
905
|
+
contentText: result.contentText,
|
|
906
|
+
});
|
|
580
907
|
}
|
|
581
908
|
else {
|
|
582
909
|
touched.push({ filePath, scan });
|
|
@@ -596,16 +923,27 @@ async function scanClaudeIncremental(onProgress) {
|
|
|
596
923
|
if (labelMap.size > 0)
|
|
597
924
|
syncLabels(labelMap);
|
|
598
925
|
}
|
|
599
|
-
/**
|
|
600
|
-
|
|
601
|
-
|
|
926
|
+
/**
|
|
927
|
+
* Stream-parse a single Claude JSONL file to extract session metadata, resuming
|
|
928
|
+
* from the persisted continuation when the file merely grew (see
|
|
929
|
+
* {@link scanClaudeSessionResumable}). Returns the row's meta + FTS content plus
|
|
930
|
+
* the serialized continuation (parser_state + content_text) to persist for the
|
|
931
|
+
* next scan.
|
|
932
|
+
*/
|
|
933
|
+
async function readClaudeMeta(filePath, sessionId, scanStamp, priorRow, account, label) {
|
|
934
|
+
const prior = parsePriorClaudeState(priorRow);
|
|
935
|
+
const { scan, newState, mode } = await scanClaudeSessionResumable(filePath, prior, scanStamp.fileMtimeMs, scanStamp.fileSize, priorRow?.fileMtimeMs);
|
|
936
|
+
if (mode === 'incremental')
|
|
937
|
+
claudeIncrementalScanCount++;
|
|
938
|
+
else
|
|
939
|
+
claudeFullScanCount++;
|
|
602
940
|
const isTeamOrigin = scan.entrypoint === 'sdk-cli';
|
|
603
941
|
let meta;
|
|
604
942
|
if (scan.timestamp) {
|
|
605
943
|
const cwd = normalizeCwd(scan.cwd || '');
|
|
606
944
|
meta = {
|
|
607
945
|
id: sessionId,
|
|
608
|
-
shortId: sessionId
|
|
946
|
+
shortId: deriveShortId(sessionId),
|
|
609
947
|
agent: 'claude',
|
|
610
948
|
timestamp: scan.timestamp,
|
|
611
949
|
lastActivity: scan.lastActivity,
|
|
@@ -636,7 +974,7 @@ async function readClaudeMeta(filePath, sessionId, account, label) {
|
|
|
636
974
|
const stat = safeStatSync(filePath);
|
|
637
975
|
meta = {
|
|
638
976
|
id: sessionId,
|
|
639
|
-
shortId: sessionId
|
|
977
|
+
shortId: deriveShortId(sessionId),
|
|
640
978
|
agent: 'claude',
|
|
641
979
|
timestamp: stat ? stat.mtime.toISOString() : new Date().toISOString(),
|
|
642
980
|
lastActivity: scan.lastActivity,
|
|
@@ -659,7 +997,15 @@ async function readClaudeMeta(filePath, sessionId, account, label) {
|
|
|
659
997
|
plan: scan.plan,
|
|
660
998
|
};
|
|
661
999
|
}
|
|
662
|
-
return {
|
|
1000
|
+
return {
|
|
1001
|
+
meta,
|
|
1002
|
+
content: scan.contentText || '',
|
|
1003
|
+
// Persist the continuation so the next scan of this file can resume from the
|
|
1004
|
+
// offset instead of a full reparse. content_text is the same accumulated user
|
|
1005
|
+
// doc, cached so the resume can hydrate userTexts without re-reading the file.
|
|
1006
|
+
parserState: JSON.stringify(newState),
|
|
1007
|
+
contentText: newState.contentText,
|
|
1008
|
+
};
|
|
663
1009
|
}
|
|
664
1010
|
// ---------------------------------------------------------------------------
|
|
665
1011
|
// Codex account info
|
|
@@ -745,37 +1091,58 @@ async function scanCodexIncremental(onProgress) {
|
|
|
745
1091
|
// readCodexMeta when a changed session actually needs it — never eagerly here,
|
|
746
1092
|
// so a no-op scan (changed.length === 0) never touches the credential file.
|
|
747
1093
|
const currentVersion = await getCurrentAgentVersion('codex');
|
|
748
|
-
const
|
|
1094
|
+
const prestat = [];
|
|
749
1095
|
for (const sessionsDir of getAgentSessionDirs('codex', 'sessions')) {
|
|
750
|
-
// High limit:
|
|
751
|
-
|
|
752
|
-
|
|
1096
|
+
// High limit: the walk stats each file once here; parsing is gated by the
|
|
1097
|
+
// ledger match below, which reuses that stat instead of re-stat'ing.
|
|
1098
|
+
for (const f of walkForFilesWithStat(sessionsDir, '.jsonl', 100_000)) {
|
|
1099
|
+
prestat.push({ filePath: f.path, fileMtimeMs: f.mtimeMs, fileSize: f.size });
|
|
753
1100
|
}
|
|
754
1101
|
}
|
|
755
|
-
const changed =
|
|
1102
|
+
const changed = filterChangedEntries(prestat);
|
|
756
1103
|
// Codex keeps human-readable titles (`thread_name`) in `session_index.jsonl`,
|
|
757
|
-
// which updates independently of the rollout files
|
|
758
|
-
//
|
|
1104
|
+
// which updates independently of the rollout files. Stat each index against the
|
|
1105
|
+
// ledger *without reading it*; only read + re-apply titles when the index (or a
|
|
1106
|
+
// rollout) actually changed. On a fully unchanged scan this collapses to a
|
|
1107
|
+
// couple of stat() calls instead of a full read + a `syncTopics` DB pass.
|
|
1108
|
+
const titleIndex = diffCodexTitleIndexes();
|
|
1109
|
+
if (changed.length === 0 && !titleIndex.changed)
|
|
1110
|
+
return;
|
|
759
1111
|
const titles = readCodexThreadNames();
|
|
760
1112
|
if (changed.length === 0) {
|
|
1113
|
+
// No rollouts changed, but the title index did — apply the new titles.
|
|
761
1114
|
syncTopics(titles);
|
|
1115
|
+
recordScans(titleIndex.stamps);
|
|
762
1116
|
return;
|
|
763
1117
|
}
|
|
764
1118
|
onProgress?.({ agent: 'codex', parsed: 0, total: changed.length });
|
|
1119
|
+
// Bulk-fetch each changed rollout's prior resumable continuation. A file with
|
|
1120
|
+
// a usable prior state + growth goes incremental (re-parse only the appended
|
|
1121
|
+
// bytes); everything else does a FULL from-offset-0 parse. The decision + the
|
|
1122
|
+
// parse both live in scanCodexSessionResumable so full and incremental share
|
|
1123
|
+
// one reducer and produce identical rows.
|
|
1124
|
+
const priorStates = getParserStatesForPaths(changed.map(c => c.filePath));
|
|
765
1125
|
const entries = [];
|
|
766
1126
|
const touched = [];
|
|
767
1127
|
const seen = new Set();
|
|
768
1128
|
let parsed = 0;
|
|
769
1129
|
for (const { filePath, scan } of changed) {
|
|
770
1130
|
try {
|
|
771
|
-
const
|
|
1131
|
+
const priorRow = priorStates.get(filePath);
|
|
1132
|
+
const result = await readCodexMeta(filePath, getCodexAccount, currentVersion, scan, priorRow);
|
|
772
1133
|
if (result && !seen.has(result.meta.id)) {
|
|
773
1134
|
seen.add(result.meta.id);
|
|
774
1135
|
// Prefer the Codex-generated title over the first-prompt fallback.
|
|
775
1136
|
const title = titles.get(result.meta.id);
|
|
776
1137
|
if (title)
|
|
777
1138
|
result.meta.topic = title;
|
|
778
|
-
entries.push({
|
|
1139
|
+
entries.push({
|
|
1140
|
+
meta: result.meta,
|
|
1141
|
+
content: result.content,
|
|
1142
|
+
scan,
|
|
1143
|
+
parserState: result.parserState,
|
|
1144
|
+
contentText: result.contentText,
|
|
1145
|
+
});
|
|
779
1146
|
}
|
|
780
1147
|
else {
|
|
781
1148
|
touched.push({ filePath, scan });
|
|
@@ -789,9 +1156,12 @@ async function scanCodexIncremental(onProgress) {
|
|
|
789
1156
|
}
|
|
790
1157
|
upsertSessionsBatch(entries);
|
|
791
1158
|
recordScans(touched);
|
|
792
|
-
//
|
|
793
|
-
// last scan
|
|
794
|
-
|
|
1159
|
+
// Only when the title index changed can an *unchanged* rollout have gained a
|
|
1160
|
+
// title since the last scan; the inline titles applied above already cover
|
|
1161
|
+
// every changed session, so skip the extra sync when the index is untouched.
|
|
1162
|
+
if (titleIndex.changed)
|
|
1163
|
+
syncTopics(titles);
|
|
1164
|
+
recordScans(titleIndex.stamps);
|
|
795
1165
|
}
|
|
796
1166
|
/** Parse the lines of a Codex `session_index.jsonl` into a session id -> title map. */
|
|
797
1167
|
export function parseCodexThreadNameIndex(raw) {
|
|
@@ -812,6 +1182,32 @@ export function parseCodexThreadNameIndex(raw) {
|
|
|
812
1182
|
}
|
|
813
1183
|
return titles;
|
|
814
1184
|
}
|
|
1185
|
+
/**
|
|
1186
|
+
* Stat every Codex `session_index.jsonl` and diff it against the scan ledger
|
|
1187
|
+
* *without reading it*. Returns the fresh stamps (persisted only after a
|
|
1188
|
+
* successful title sync) and whether any index changed since the last scan — the
|
|
1189
|
+
* signal that lets a no-op scan skip the file read + `syncTopics` entirely.
|
|
1190
|
+
*
|
|
1191
|
+
* The index path is a sibling of `sessions/` (never inside it), so it is never
|
|
1192
|
+
* walked as a rollout and its ledger row can't collide with a transcript's.
|
|
1193
|
+
*/
|
|
1194
|
+
function diffCodexTitleIndexes() {
|
|
1195
|
+
const stamps = [];
|
|
1196
|
+
let changed = false;
|
|
1197
|
+
for (const sessionsDir of getAgentSessionDirs('codex', 'sessions')) {
|
|
1198
|
+
const indexPath = path.join(path.dirname(sessionsDir), 'session_index.jsonl');
|
|
1199
|
+
const stat = safeStatSync(indexPath);
|
|
1200
|
+
if (!stat)
|
|
1201
|
+
continue; // no index in this home
|
|
1202
|
+
const scan = { fileMtimeMs: Math.floor(stat.mtimeMs), fileSize: stat.size };
|
|
1203
|
+
const prev = getScanStampByPath(indexPath);
|
|
1204
|
+
if (!prev || prev.fileMtimeMs !== scan.fileMtimeMs || prev.fileSize !== scan.fileSize) {
|
|
1205
|
+
changed = true;
|
|
1206
|
+
}
|
|
1207
|
+
stamps.push({ filePath: indexPath, scan });
|
|
1208
|
+
}
|
|
1209
|
+
return { stamps, changed };
|
|
1210
|
+
}
|
|
815
1211
|
/**
|
|
816
1212
|
* Read Codex session titles across every Codex home (live + versioned). The
|
|
817
1213
|
* `session_index.jsonl` file sits beside each `sessions/` rollout tree.
|
|
@@ -839,15 +1235,36 @@ function readCodexThreadNames() {
|
|
|
839
1235
|
* performs is deferred until we know this file is a real session worth building
|
|
840
1236
|
* metadata for, and only then — never during the file walk / stat phase.
|
|
841
1237
|
*/
|
|
842
|
-
export async function readCodexMeta(filePath, resolveAccount, currentVersion) {
|
|
843
|
-
|
|
1238
|
+
export async function readCodexMeta(filePath, resolveAccount, currentVersion, scanStamp, priorRow) {
|
|
1239
|
+
// Resume from the persisted continuation when the file merely grew; otherwise
|
|
1240
|
+
// full-parse from byte 0. Both branches share one reducer, so an append yields
|
|
1241
|
+
// a row identical to a from-scratch reparse. When no stamp is supplied (a
|
|
1242
|
+
// caller outside the live scan path), fall back to a plain full parse with no
|
|
1243
|
+
// continuation to persist.
|
|
1244
|
+
let scan;
|
|
1245
|
+
let newState;
|
|
1246
|
+
let newOffset = 0;
|
|
1247
|
+
if (scanStamp) {
|
|
1248
|
+
const prior = parsePriorCodexState(priorRow);
|
|
1249
|
+
const result = await scanCodexSessionResumable(filePath, prior, scanStamp.fileMtimeMs, scanStamp.fileSize, priorRow?.fileMtimeMs);
|
|
1250
|
+
if (result.mode === 'incremental')
|
|
1251
|
+
codexIncrementalScanCount++;
|
|
1252
|
+
else
|
|
1253
|
+
codexFullScanCount++;
|
|
1254
|
+
scan = result.scan;
|
|
1255
|
+
newState = result.newState;
|
|
1256
|
+
newOffset = result.newOffset;
|
|
1257
|
+
}
|
|
1258
|
+
else {
|
|
1259
|
+
scan = await scanCodexSession(filePath);
|
|
1260
|
+
}
|
|
844
1261
|
const sessionId = scan.sessionId || '';
|
|
845
1262
|
if (!sessionId)
|
|
846
1263
|
return null;
|
|
847
1264
|
const cwd = normalizeCwd(scan.cwd || '');
|
|
848
1265
|
const meta = {
|
|
849
1266
|
id: sessionId,
|
|
850
|
-
shortId: sessionId
|
|
1267
|
+
shortId: deriveShortId(sessionId),
|
|
851
1268
|
agent: 'codex',
|
|
852
1269
|
// Codex `session_meta` only carries the start time; use file mtime when
|
|
853
1270
|
// it's newer so long-running sessions register as recently active.
|
|
@@ -872,7 +1289,15 @@ export async function readCodexMeta(filePath, resolveAccount, currentVersion) {
|
|
|
872
1289
|
createdTickets: scan.createdTickets,
|
|
873
1290
|
spawnedTeam: scan.spawnedTeam,
|
|
874
1291
|
};
|
|
875
|
-
return {
|
|
1292
|
+
return {
|
|
1293
|
+
meta,
|
|
1294
|
+
content: scan.contentText || '',
|
|
1295
|
+
// Persist the continuation so the next scan of this rollout resumes from the
|
|
1296
|
+
// offset instead of a full reparse; content_text caches the accumulated user
|
|
1297
|
+
// doc for the resume's hydrate. Absent when no stamp was supplied.
|
|
1298
|
+
parserState: newState ? JSON.stringify(newState) : undefined,
|
|
1299
|
+
contentText: newState?.contentText,
|
|
1300
|
+
};
|
|
876
1301
|
}
|
|
877
1302
|
/**
|
|
878
1303
|
* Codex writes `session_meta` (with the start timestamp) on the first line of a
|
|
@@ -901,33 +1326,34 @@ function pickLatestCodexTimestamp(metaTimestamp, filePath) {
|
|
|
901
1326
|
async function scanGeminiIncremental(onProgress) {
|
|
902
1327
|
const currentVersion = await getCurrentAgentVersion('gemini');
|
|
903
1328
|
const projectMap = buildGeminiProjectMap();
|
|
904
|
-
|
|
905
|
-
|
|
1329
|
+
// Each `<tmpDir>/<hashDir>/chats` is a leaf dir of Gemini transcripts. The
|
|
1330
|
+
// FIRST tmp root is the live `~/.gemini/tmp` — its chats dirs are live roots;
|
|
1331
|
+
// version-home + backup roots are immutable and short-circuit when unchanged.
|
|
1332
|
+
const tmpRoots = getAgentSessionDirs('gemini', 'tmp');
|
|
1333
|
+
const leafDirs = [];
|
|
1334
|
+
const seenLeaf = new Set();
|
|
1335
|
+
tmpRoots.forEach((tmpDir, rootIdx) => {
|
|
1336
|
+
const isLiveRoot = rootIdx === 0;
|
|
906
1337
|
let hashDirs;
|
|
907
1338
|
try {
|
|
908
1339
|
hashDirs = fs.readdirSync(tmpDir);
|
|
909
1340
|
}
|
|
910
1341
|
catch {
|
|
911
|
-
|
|
1342
|
+
return;
|
|
912
1343
|
}
|
|
913
1344
|
for (const hashDir of hashDirs) {
|
|
914
1345
|
const chatsDir = path.join(tmpDir, hashDir, 'chats');
|
|
915
1346
|
if (!fs.existsSync(chatsDir))
|
|
916
1347
|
continue;
|
|
917
|
-
|
|
918
|
-
|
|
919
|
-
chatFiles = fs.readdirSync(chatsDir).filter(f => f.endsWith('.json'));
|
|
920
|
-
}
|
|
921
|
-
catch {
|
|
1348
|
+
const key = safeRealpathSync(chatsDir) || chatsDir;
|
|
1349
|
+
if (seenLeaf.has(key))
|
|
922
1350
|
continue;
|
|
923
|
-
|
|
924
|
-
|
|
925
|
-
filePaths.push({ filePath: path.join(chatsDir, file), hashDir });
|
|
926
|
-
}
|
|
1351
|
+
seenLeaf.add(key);
|
|
1352
|
+
leafDirs.push({ dirPath: chatsDir, isLiveRoot });
|
|
927
1353
|
}
|
|
928
|
-
}
|
|
929
|
-
const
|
|
930
|
-
const changedByPath = new Map(
|
|
1354
|
+
});
|
|
1355
|
+
const { changed } = collectChangedFilesInLeafDirs(leafDirs, '.json');
|
|
1356
|
+
const changedByPath = new Map(changed.map(c => [c.filePath, c.scan]));
|
|
931
1357
|
if (changedByPath.size === 0)
|
|
932
1358
|
return;
|
|
933
1359
|
onProgress?.({ agent: 'gemini', parsed: 0, total: changedByPath.size });
|
|
@@ -935,10 +1361,9 @@ async function scanGeminiIncremental(onProgress) {
|
|
|
935
1361
|
const touched = [];
|
|
936
1362
|
const seen = new Set();
|
|
937
1363
|
let parsed = 0;
|
|
938
|
-
for (const { filePath,
|
|
939
|
-
|
|
940
|
-
|
|
941
|
-
continue;
|
|
1364
|
+
for (const { filePath, scan } of changed) {
|
|
1365
|
+
// The hashDir is the directory two levels up: <hashDir>/chats/<file>.json.
|
|
1366
|
+
const hashDir = path.basename(path.dirname(path.dirname(filePath)));
|
|
942
1367
|
try {
|
|
943
1368
|
const result = readGeminiMeta(filePath, hashDir, projectMap, currentVersion);
|
|
944
1369
|
if (result && !seen.has(result.meta.id)) {
|
|
@@ -1058,7 +1483,7 @@ function readGeminiMeta(filePath, hashDir, projectMap, currentVersion) {
|
|
|
1058
1483
|
: undefined;
|
|
1059
1484
|
const meta = {
|
|
1060
1485
|
id: sessionId,
|
|
1061
|
-
shortId: sessionId
|
|
1486
|
+
shortId: deriveShortId(sessionId),
|
|
1062
1487
|
agent: 'gemini',
|
|
1063
1488
|
timestamp: startTime || (stat ? stat.mtime.toISOString() : new Date().toISOString()),
|
|
1064
1489
|
lastActivity: lastTsMs !== undefined ? new Date(lastTsMs).toISOString() : undefined,
|
|
@@ -1204,7 +1629,7 @@ function readAntigravityMeta(filePath, currentVersion) {
|
|
|
1204
1629
|
const stat = safeStatSync(filePath);
|
|
1205
1630
|
const meta = {
|
|
1206
1631
|
id: sessionId,
|
|
1207
|
-
shortId: sessionId
|
|
1632
|
+
shortId: deriveShortId(sessionId),
|
|
1208
1633
|
agent: 'antigravity',
|
|
1209
1634
|
timestamp: stat ? stat.mtime.toISOString() : new Date().toISOString(),
|
|
1210
1635
|
project: normalizedCwd ? path.basename(normalizedCwd) : undefined,
|
|
@@ -1332,7 +1757,7 @@ async function scanOpenCodeIncremental() {
|
|
|
1332
1757
|
const topic = title || undefined;
|
|
1333
1758
|
const meta = {
|
|
1334
1759
|
id,
|
|
1335
|
-
shortId: id
|
|
1760
|
+
shortId: deriveShortId(id, /^ses_/),
|
|
1336
1761
|
agent: 'opencode',
|
|
1337
1762
|
timestamp,
|
|
1338
1763
|
lastActivity,
|
|
@@ -1402,7 +1827,7 @@ async function scanOpenClawIncremental() {
|
|
|
1402
1827
|
entries.push({
|
|
1403
1828
|
meta: {
|
|
1404
1829
|
id: `openclaw-${agentId}`,
|
|
1405
|
-
shortId: agentId
|
|
1830
|
+
shortId: deriveShortId(agentId),
|
|
1406
1831
|
agent: 'openclaw',
|
|
1407
1832
|
timestamp: new Date().toISOString(),
|
|
1408
1833
|
project: name,
|
|
@@ -1438,7 +1863,7 @@ async function scanOpenClawIncremental() {
|
|
|
1438
1863
|
entries.push({
|
|
1439
1864
|
meta: {
|
|
1440
1865
|
id: `openclaw-cron-${jobId}`,
|
|
1441
|
-
shortId: jobId
|
|
1866
|
+
shortId: deriveShortId(jobId),
|
|
1442
1867
|
agent: 'openclaw',
|
|
1443
1868
|
timestamp: new Date().toISOString(),
|
|
1444
1869
|
project: `${jobName} (${agentId || 'unknown'})`,
|
|
@@ -1512,7 +1937,7 @@ async function readRushMeta(filePath, sessionId) {
|
|
|
1512
1937
|
const stat = safeStatSync(filePath);
|
|
1513
1938
|
const timestamp = scan.timestamp
|
|
1514
1939
|
|| (stat ? stat.mtime.toISOString() : new Date().toISOString());
|
|
1515
|
-
const shortId = sessionId
|
|
1940
|
+
const shortId = deriveShortId(sessionId, /^session_/);
|
|
1516
1941
|
const meta = {
|
|
1517
1942
|
id: sessionId,
|
|
1518
1943
|
shortId,
|
|
@@ -1672,7 +2097,7 @@ function readHermesMeta(filePath) {
|
|
|
1672
2097
|
: typeof session.session_start === 'string'
|
|
1673
2098
|
? session.session_start
|
|
1674
2099
|
: stat ? stat.mtime.toISOString() : new Date().toISOString();
|
|
1675
|
-
const shortId = sessionId
|
|
2100
|
+
const shortId = deriveShortId(sessionId, /^api-/);
|
|
1676
2101
|
const model = typeof session.model === 'string' ? session.model : undefined;
|
|
1677
2102
|
const platform = typeof session.platform === 'string' ? session.platform : undefined;
|
|
1678
2103
|
const meta = {
|
|
@@ -1713,14 +2138,15 @@ function extractHermesMessageText(content) {
|
|
|
1713
2138
|
*/
|
|
1714
2139
|
async function scanDroidIncremental(onProgress) {
|
|
1715
2140
|
const currentVersion = await getCurrentAgentVersion('droid');
|
|
1716
|
-
const
|
|
2141
|
+
const prestat = [];
|
|
1717
2142
|
for (const sessionsDir of getAgentSessionDirs('droid', 'sessions')) {
|
|
1718
|
-
// High limit:
|
|
1719
|
-
|
|
1720
|
-
|
|
2143
|
+
// High limit: the walk stats each file once here; parsing is gated by the
|
|
2144
|
+
// ledger match below, which reuses that stat instead of re-stat'ing.
|
|
2145
|
+
for (const f of walkForFilesWithStat(sessionsDir, '.jsonl', 100_000)) {
|
|
2146
|
+
prestat.push({ filePath: f.path, fileMtimeMs: f.mtimeMs, fileSize: f.size });
|
|
1721
2147
|
}
|
|
1722
2148
|
}
|
|
1723
|
-
const changed =
|
|
2149
|
+
const changed = filterChangedEntries(prestat);
|
|
1724
2150
|
if (changed.length === 0)
|
|
1725
2151
|
return;
|
|
1726
2152
|
onProgress?.({ agent: 'droid', parsed: 0, total: changed.length });
|
|
@@ -1772,7 +2198,7 @@ async function readDroidMeta(filePath, currentVersion) {
|
|
|
1772
2198
|
const cwd = normalizeCwd(scan.cwd || '');
|
|
1773
2199
|
const meta = {
|
|
1774
2200
|
id: sessionId,
|
|
1775
|
-
shortId: sessionId
|
|
2201
|
+
shortId: deriveShortId(sessionId),
|
|
1776
2202
|
agent: 'droid',
|
|
1777
2203
|
timestamp: scan.timestamp || (stat ? stat.mtime.toISOString() : new Date().toISOString()),
|
|
1778
2204
|
lastActivity: scan.lastActivity,
|
|
@@ -1916,262 +2342,229 @@ function extractDroidMessageText(content) {
|
|
|
1916
2342
|
.join('\n')
|
|
1917
2343
|
.trim();
|
|
1918
2344
|
}
|
|
1919
|
-
/**
|
|
1920
|
-
export
|
|
1921
|
-
|
|
1922
|
-
|
|
1923
|
-
|
|
1924
|
-
|
|
1925
|
-
|
|
1926
|
-
|
|
1927
|
-
|
|
1928
|
-
|
|
1929
|
-
|
|
1930
|
-
|
|
1931
|
-
|
|
1932
|
-
|
|
1933
|
-
|
|
1934
|
-
|
|
1935
|
-
|
|
1936
|
-
|
|
1937
|
-
|
|
1938
|
-
|
|
1939
|
-
|
|
1940
|
-
|
|
1941
|
-
|
|
1942
|
-
|
|
1943
|
-
|
|
1944
|
-
|
|
1945
|
-
|
|
1946
|
-
|
|
1947
|
-
|
|
1948
|
-
|
|
1949
|
-
|
|
1950
|
-
|
|
1951
|
-
|
|
1952
|
-
|
|
1953
|
-
|
|
1954
|
-
|
|
1955
|
-
|
|
1956
|
-
//
|
|
1957
|
-
//
|
|
1958
|
-
|
|
1959
|
-
|
|
1960
|
-
|
|
1961
|
-
|
|
1962
|
-
|
|
1963
|
-
|
|
1964
|
-
|
|
1965
|
-
|
|
1966
|
-
|
|
1967
|
-
|
|
2345
|
+
/** Zero-value accumulator for a fresh (from-byte-0) Claude parse. */
|
|
2346
|
+
export function initClaudeParseState() {
|
|
2347
|
+
return {
|
|
2348
|
+
timestamp: undefined,
|
|
2349
|
+
cwd: undefined,
|
|
2350
|
+
gitBranch: undefined,
|
|
2351
|
+
version: undefined,
|
|
2352
|
+
topic: undefined,
|
|
2353
|
+
customTitle: undefined,
|
|
2354
|
+
aiTitle: undefined,
|
|
2355
|
+
entrypoint: undefined,
|
|
2356
|
+
messageCount: 0,
|
|
2357
|
+
tokenCount: 0,
|
|
2358
|
+
outputTokens: 0,
|
|
2359
|
+
sawTokenCount: false,
|
|
2360
|
+
costUsd: 0,
|
|
2361
|
+
sawCost: false,
|
|
2362
|
+
firstTsMs: undefined,
|
|
2363
|
+
lastTsMs: undefined,
|
|
2364
|
+
seenAssistantIds: new Set(),
|
|
2365
|
+
userTexts: [],
|
|
2366
|
+
sawPrCreate: false,
|
|
2367
|
+
prUrl: undefined,
|
|
2368
|
+
prNumber: undefined,
|
|
2369
|
+
createdTickets: new Set(),
|
|
2370
|
+
pendingTicketTools: new Set(),
|
|
2371
|
+
spawnedTeam: undefined,
|
|
2372
|
+
plan: undefined,
|
|
2373
|
+
};
|
|
2374
|
+
}
|
|
2375
|
+
/**
|
|
2376
|
+
* Fold one parsed transcript line into the accumulator. This is the exact loop
|
|
2377
|
+
* body {@link scanClaudeSession} used to run inline — extracted verbatim,
|
|
2378
|
+
* mutating `state.*` in place. `parsed` is the already-`JSON.parse`d line (the
|
|
2379
|
+
* malformed-line skip happens in the caller, as before).
|
|
2380
|
+
*/
|
|
2381
|
+
export function applyClaudeLine(state, parsed) {
|
|
2382
|
+
// entrypoint ships on the first envelope event (attachment/user/assistant)
|
|
2383
|
+
// and is the clean structural signal for "was this a team spawn?"
|
|
2384
|
+
if (!state.entrypoint && typeof parsed.entrypoint === 'string') {
|
|
2385
|
+
state.entrypoint = parsed.entrypoint;
|
|
2386
|
+
}
|
|
2387
|
+
// Produced-artifact signals, structurally (independent of the PR gate below):
|
|
2388
|
+
// - a Bash `agents teams create/add` command → the team it spawned
|
|
2389
|
+
// - a Linear create_issue / `gh issue create` tool_use → its result carries
|
|
2390
|
+
// the new ticket ref, read from the matching tool_result.
|
|
2391
|
+
if (parsed.type === 'assistant' && Array.isArray(parsed.message?.content)) {
|
|
2392
|
+
for (const b of parsed.message.content) {
|
|
2393
|
+
if (b?.type !== 'tool_use')
|
|
1968
2394
|
continue;
|
|
2395
|
+
if (!state.spawnedTeam && typeof b?.input?.command === 'string') {
|
|
2396
|
+
const team = detectSpawnedTeam(b.input.command);
|
|
2397
|
+
if (team)
|
|
2398
|
+
state.spawnedTeam = team;
|
|
1969
2399
|
}
|
|
1970
|
-
|
|
1971
|
-
|
|
1972
|
-
if (!entrypoint && typeof parsed.entrypoint === 'string') {
|
|
1973
|
-
entrypoint = parsed.entrypoint;
|
|
1974
|
-
}
|
|
1975
|
-
// Produced-artifact signals, structurally (independent of the PR gate below):
|
|
1976
|
-
// - a Bash `agents teams create/add` command → the team it spawned
|
|
1977
|
-
// - a Linear create_issue / `gh issue create` tool_use → its result carries
|
|
1978
|
-
// the new ticket ref, read from the matching tool_result.
|
|
1979
|
-
if (parsed.type === 'assistant' && Array.isArray(parsed.message?.content)) {
|
|
1980
|
-
for (const b of parsed.message.content) {
|
|
1981
|
-
if (b?.type !== 'tool_use')
|
|
1982
|
-
continue;
|
|
1983
|
-
if (!spawnedTeam && typeof b?.input?.command === 'string') {
|
|
1984
|
-
const team = detectSpawnedTeam(b.input.command);
|
|
1985
|
-
if (team)
|
|
1986
|
-
spawnedTeam = team;
|
|
1987
|
-
}
|
|
1988
|
-
if (typeof b?.id === 'string' && isTicketCreateTool(b?.name, b?.input?.command)) {
|
|
1989
|
-
pendingTicketTools.add(b.id);
|
|
1990
|
-
}
|
|
1991
|
-
// ExitPlanMode plan markdown — last one wins so a re-planned session
|
|
1992
|
-
// reports its most recent plan.
|
|
1993
|
-
if (b?.name === 'ExitPlanMode' && typeof b?.input?.plan === 'string') {
|
|
1994
|
-
const p = b.input.plan.trim();
|
|
1995
|
-
if (p)
|
|
1996
|
-
plan = b.input.plan;
|
|
1997
|
-
}
|
|
1998
|
-
}
|
|
1999
|
-
}
|
|
2000
|
-
if (pendingTicketTools.size > 0 && parsed.type === 'user' && Array.isArray(parsed.message?.content)) {
|
|
2001
|
-
for (const b of parsed.message.content) {
|
|
2002
|
-
if (b?.type !== 'tool_result' || typeof b?.tool_use_id !== 'string')
|
|
2003
|
-
continue;
|
|
2004
|
-
if (!pendingTicketTools.has(b.tool_use_id))
|
|
2005
|
-
continue;
|
|
2006
|
-
pendingTicketTools.delete(b.tool_use_id);
|
|
2007
|
-
const text = typeof b.content === 'string'
|
|
2008
|
-
? b.content
|
|
2009
|
-
: Array.isArray(b.content) ? b.content.map((c) => c?.text || '').join('\n') : '';
|
|
2010
|
-
const t = extractCreatedTicket(text);
|
|
2011
|
-
if (t)
|
|
2012
|
-
createdTickets.add(t);
|
|
2013
|
-
}
|
|
2014
|
-
}
|
|
2015
|
-
// PR signal, structurally: a Bash tool_use whose command is `gh pr create`
|
|
2016
|
-
// marks intent; the pull URL is then read from a tool_result's output.
|
|
2017
|
-
if (!prUrl) {
|
|
2018
|
-
if (!sawPrCreate && parsed.type === 'assistant' && Array.isArray(parsed.message?.content)) {
|
|
2019
|
-
for (const b of parsed.message.content) {
|
|
2020
|
-
if (b?.type === 'tool_use' && typeof b?.input?.command === 'string' && isPrCreateCommand(b.input.command)) {
|
|
2021
|
-
sawPrCreate = true;
|
|
2022
|
-
}
|
|
2023
|
-
}
|
|
2024
|
-
}
|
|
2025
|
-
if (sawPrCreate && parsed.type === 'user' && Array.isArray(parsed.message?.content)) {
|
|
2026
|
-
for (const b of parsed.message.content) {
|
|
2027
|
-
if (b?.type !== 'tool_result')
|
|
2028
|
-
continue;
|
|
2029
|
-
const text = typeof b.content === 'string'
|
|
2030
|
-
? b.content
|
|
2031
|
-
: Array.isArray(b.content) ? b.content.map((c) => c?.text || '').join('\n') : '';
|
|
2032
|
-
const pr = extractPrUrl(text);
|
|
2033
|
-
if (pr) {
|
|
2034
|
-
prUrl = pr.url;
|
|
2035
|
-
prNumber = pr.number;
|
|
2036
|
-
}
|
|
2037
|
-
}
|
|
2038
|
-
}
|
|
2039
|
-
}
|
|
2040
|
-
// Track duration across every timestamped event, not just the first.
|
|
2041
|
-
if (typeof parsed.timestamp === 'string') {
|
|
2042
|
-
const ms = new Date(parsed.timestamp).getTime();
|
|
2043
|
-
if (!Number.isNaN(ms)) {
|
|
2044
|
-
if (firstTsMs === undefined || ms < firstTsMs)
|
|
2045
|
-
firstTsMs = ms;
|
|
2046
|
-
if (lastTsMs === undefined || ms > lastTsMs)
|
|
2047
|
-
lastTsMs = ms;
|
|
2048
|
-
}
|
|
2400
|
+
if (typeof b?.id === 'string' && isTicketCreateTool(b?.name, b?.input?.command)) {
|
|
2401
|
+
state.pendingTicketTools.add(b.id);
|
|
2049
2402
|
}
|
|
2050
|
-
|
|
2051
|
-
|
|
2052
|
-
|
|
2053
|
-
|
|
2054
|
-
|
|
2403
|
+
// ExitPlanMode plan markdown — last one wins so a re-planned session
|
|
2404
|
+
// reports its most recent plan.
|
|
2405
|
+
if (b?.name === 'ExitPlanMode' && typeof b?.input?.plan === 'string') {
|
|
2406
|
+
const p = b.input.plan.trim();
|
|
2407
|
+
if (p)
|
|
2408
|
+
state.plan = b.input.plan;
|
|
2055
2409
|
}
|
|
2056
|
-
|
|
2057
|
-
|
|
2058
|
-
|
|
2059
|
-
|
|
2410
|
+
}
|
|
2411
|
+
}
|
|
2412
|
+
if (state.pendingTicketTools.size > 0 && parsed.type === 'user' && Array.isArray(parsed.message?.content)) {
|
|
2413
|
+
for (const b of parsed.message.content) {
|
|
2414
|
+
if (b?.type !== 'tool_result' || typeof b?.tool_use_id !== 'string')
|
|
2060
2415
|
continue;
|
|
2061
|
-
|
|
2062
|
-
if (parsed.type === 'ai-title') {
|
|
2063
|
-
const t = typeof parsed.aiTitle === 'string' ? parsed.aiTitle.trim() : '';
|
|
2064
|
-
if (t)
|
|
2065
|
-
aiTitle = t;
|
|
2416
|
+
if (!state.pendingTicketTools.has(b.tool_use_id))
|
|
2066
2417
|
continue;
|
|
2067
|
-
|
|
2068
|
-
|
|
2069
|
-
|
|
2070
|
-
|
|
2071
|
-
|
|
2072
|
-
|
|
2073
|
-
|
|
2074
|
-
|
|
2418
|
+
state.pendingTicketTools.delete(b.tool_use_id);
|
|
2419
|
+
const text = typeof b.content === 'string'
|
|
2420
|
+
? b.content
|
|
2421
|
+
: Array.isArray(b.content) ? b.content.map((c) => c?.text || '').join('\n') : '';
|
|
2422
|
+
const t = extractCreatedTicket(text);
|
|
2423
|
+
if (t)
|
|
2424
|
+
state.createdTickets.add(t);
|
|
2425
|
+
}
|
|
2426
|
+
}
|
|
2427
|
+
// PR signal, structurally: a Bash tool_use whose command is `gh pr create`
|
|
2428
|
+
// marks intent; the pull URL is then read from a tool_result's output.
|
|
2429
|
+
if (!state.prUrl) {
|
|
2430
|
+
if (!state.sawPrCreate && parsed.type === 'assistant' && Array.isArray(parsed.message?.content)) {
|
|
2431
|
+
for (const b of parsed.message.content) {
|
|
2432
|
+
if (b?.type === 'tool_use' && typeof b?.input?.command === 'string' && isPrCreateCommand(b.input.command)) {
|
|
2433
|
+
state.sawPrCreate = true;
|
|
2075
2434
|
}
|
|
2076
|
-
continue;
|
|
2077
2435
|
}
|
|
2078
|
-
|
|
2079
|
-
|
|
2080
|
-
const
|
|
2081
|
-
|
|
2082
|
-
|
|
2083
|
-
|
|
2084
|
-
|
|
2085
|
-
|
|
2086
|
-
|
|
2087
|
-
|
|
2088
|
-
|
|
2089
|
-
|
|
2090
|
-
const usageObj = parsed.message?.usage || parsed.usage;
|
|
2091
|
-
const usage = getClaudeUsageTotal(usageObj);
|
|
2092
|
-
if (usage !== null) {
|
|
2093
|
-
tokenCount += usage;
|
|
2094
|
-
sawTokenCount = true;
|
|
2095
|
-
}
|
|
2096
|
-
if (typeof usageObj?.output_tokens === 'number')
|
|
2097
|
-
outputTokens += usageObj.output_tokens;
|
|
2098
|
-
// Per-assistant-message cost: each event carries its own model, so we
|
|
2099
|
-
// multiply that event's raw token directions by that model's price.
|
|
2100
|
-
const model = parsed.message?.model;
|
|
2101
|
-
if (model && usageObj && typeof usageObj === 'object') {
|
|
2102
|
-
const eventCost = costOfUsage({
|
|
2103
|
-
model,
|
|
2104
|
-
inputTokens: usageObj.input_tokens,
|
|
2105
|
-
outputTokens: usageObj.output_tokens,
|
|
2106
|
-
cacheReadTokens: usageObj.cache_read_input_tokens,
|
|
2107
|
-
cacheCreationTokens: usageObj.cache_creation_input_tokens,
|
|
2108
|
-
});
|
|
2109
|
-
if (eventCost > 0) {
|
|
2110
|
-
costUsd += eventCost;
|
|
2111
|
-
sawCost = true;
|
|
2436
|
+
}
|
|
2437
|
+
if (state.sawPrCreate && parsed.type === 'user' && Array.isArray(parsed.message?.content)) {
|
|
2438
|
+
for (const b of parsed.message.content) {
|
|
2439
|
+
if (b?.type !== 'tool_result')
|
|
2440
|
+
continue;
|
|
2441
|
+
const text = typeof b.content === 'string'
|
|
2442
|
+
? b.content
|
|
2443
|
+
: Array.isArray(b.content) ? b.content.map((c) => c?.text || '').join('\n') : '';
|
|
2444
|
+
const pr = extractPrUrl(text);
|
|
2445
|
+
if (pr) {
|
|
2446
|
+
state.prUrl = pr.url;
|
|
2447
|
+
state.prNumber = pr.number;
|
|
2112
2448
|
}
|
|
2113
2449
|
}
|
|
2114
2450
|
}
|
|
2115
2451
|
}
|
|
2116
|
-
|
|
2117
|
-
|
|
2118
|
-
|
|
2452
|
+
// Track duration across every timestamped event, not just the first.
|
|
2453
|
+
if (typeof parsed.timestamp === 'string') {
|
|
2454
|
+
const ms = new Date(parsed.timestamp).getTime();
|
|
2455
|
+
if (!Number.isNaN(ms)) {
|
|
2456
|
+
if (state.firstTsMs === undefined || ms < state.firstTsMs)
|
|
2457
|
+
state.firstTsMs = ms;
|
|
2458
|
+
if (state.lastTsMs === undefined || ms > state.lastTsMs)
|
|
2459
|
+
state.lastTsMs = ms;
|
|
2460
|
+
}
|
|
2119
2461
|
}
|
|
2120
|
-
|
|
2121
|
-
|
|
2462
|
+
if (!state.timestamp && (parsed.type === 'user' || parsed.type === 'assistant') && parsed.timestamp) {
|
|
2463
|
+
state.timestamp = parsed.timestamp;
|
|
2464
|
+
state.cwd = parsed.cwd || '';
|
|
2465
|
+
state.gitBranch = parsed.gitBranch || undefined;
|
|
2466
|
+
state.version = parsed.version || undefined;
|
|
2467
|
+
}
|
|
2468
|
+
if (parsed.type === 'custom-title') {
|
|
2469
|
+
const t = typeof parsed.customTitle === 'string' ? parsed.customTitle.trim() : '';
|
|
2470
|
+
if (t)
|
|
2471
|
+
state.customTitle = t;
|
|
2472
|
+
return;
|
|
2473
|
+
}
|
|
2474
|
+
if (parsed.type === 'ai-title') {
|
|
2475
|
+
const t = typeof parsed.aiTitle === 'string' ? parsed.aiTitle.trim() : '';
|
|
2476
|
+
if (t)
|
|
2477
|
+
state.aiTitle = t;
|
|
2478
|
+
return;
|
|
2479
|
+
}
|
|
2480
|
+
if (parsed.type === 'user') {
|
|
2481
|
+
const text = extractClaudeUserText(parsed);
|
|
2482
|
+
if (text) {
|
|
2483
|
+
state.messageCount++;
|
|
2484
|
+
state.userTexts.push(text);
|
|
2485
|
+
if (!state.topic)
|
|
2486
|
+
state.topic = extractSessionTopic(text);
|
|
2487
|
+
}
|
|
2488
|
+
return;
|
|
2489
|
+
}
|
|
2490
|
+
if (parsed.type !== 'assistant')
|
|
2491
|
+
return;
|
|
2492
|
+
const assistantId = typeof parsed.message?.id === 'string'
|
|
2493
|
+
? parsed.message.id
|
|
2494
|
+
: typeof parsed.uuid === 'string'
|
|
2495
|
+
? parsed.uuid
|
|
2496
|
+
: undefined;
|
|
2497
|
+
const logicalId = assistantId || `${parsed.timestamp || ''}:${state.seenAssistantIds.size}`;
|
|
2498
|
+
if (state.seenAssistantIds.has(logicalId))
|
|
2499
|
+
return;
|
|
2500
|
+
state.seenAssistantIds.add(logicalId);
|
|
2501
|
+
state.messageCount++;
|
|
2502
|
+
const usageObj = parsed.message?.usage || parsed.usage;
|
|
2503
|
+
const usage = getClaudeUsageTotal(usageObj);
|
|
2504
|
+
if (usage !== null) {
|
|
2505
|
+
state.tokenCount += usage;
|
|
2506
|
+
state.sawTokenCount = true;
|
|
2507
|
+
}
|
|
2508
|
+
if (typeof usageObj?.output_tokens === 'number')
|
|
2509
|
+
state.outputTokens += usageObj.output_tokens;
|
|
2510
|
+
// Per-assistant-message cost: each event carries its own model, so we
|
|
2511
|
+
// multiply that event's raw token directions by that model's price.
|
|
2512
|
+
const model = parsed.message?.model;
|
|
2513
|
+
if (model && usageObj && typeof usageObj === 'object') {
|
|
2514
|
+
const eventCost = costOfUsage({
|
|
2515
|
+
model,
|
|
2516
|
+
inputTokens: usageObj.input_tokens,
|
|
2517
|
+
outputTokens: usageObj.output_tokens,
|
|
2518
|
+
cacheReadTokens: usageObj.cache_read_input_tokens,
|
|
2519
|
+
cacheCreationTokens: usageObj.cache_creation_input_tokens,
|
|
2520
|
+
});
|
|
2521
|
+
if (eventCost > 0) {
|
|
2522
|
+
state.costUsd += eventCost;
|
|
2523
|
+
state.sawCost = true;
|
|
2524
|
+
}
|
|
2525
|
+
}
|
|
2526
|
+
}
|
|
2527
|
+
/**
|
|
2528
|
+
* Build the {@link ClaudeSessionScan} return object from an accumulator. This is
|
|
2529
|
+
* the exact return-building {@link scanClaudeSession} used to run inline.
|
|
2530
|
+
*/
|
|
2531
|
+
export function finalizeClaudeScan(state) {
|
|
2532
|
+
const durationMs = state.firstTsMs !== undefined && state.lastTsMs !== undefined && state.lastTsMs > state.firstTsMs
|
|
2533
|
+
? state.lastTsMs - state.firstTsMs
|
|
2122
2534
|
: undefined;
|
|
2123
2535
|
// Prefer an explicit session title (user `/rename` > Claude auto-title) over
|
|
2124
2536
|
// the first-prompt topic.
|
|
2125
|
-
const resolvedTopic = customTitle || aiTitle || topic;
|
|
2126
|
-
const worktree = detectWorktree(cwd, gitBranch);
|
|
2127
|
-
const ticket = detectTicket(userTexts.join('\n') || undefined, gitBranch);
|
|
2537
|
+
const resolvedTopic = state.customTitle || state.aiTitle || state.topic;
|
|
2538
|
+
const worktree = detectWorktree(state.cwd, state.gitBranch);
|
|
2539
|
+
const ticket = detectTicket(state.userTexts.join('\n') || undefined, state.gitBranch);
|
|
2128
2540
|
return {
|
|
2129
|
-
timestamp,
|
|
2130
|
-
cwd,
|
|
2131
|
-
gitBranch,
|
|
2132
|
-
version,
|
|
2541
|
+
timestamp: state.timestamp,
|
|
2542
|
+
cwd: state.cwd,
|
|
2543
|
+
gitBranch: state.gitBranch,
|
|
2544
|
+
version: state.version,
|
|
2133
2545
|
topic: resolvedTopic,
|
|
2134
|
-
entrypoint,
|
|
2135
|
-
messageCount,
|
|
2136
|
-
tokenCount: sawTokenCount ? tokenCount : undefined,
|
|
2137
|
-
outputTokens: sawTokenCount ? outputTokens : undefined,
|
|
2138
|
-
costUsd: sawCost ? costUsd : undefined,
|
|
2546
|
+
entrypoint: state.entrypoint,
|
|
2547
|
+
messageCount: state.messageCount,
|
|
2548
|
+
tokenCount: state.sawTokenCount ? state.tokenCount : undefined,
|
|
2549
|
+
outputTokens: state.sawTokenCount ? state.outputTokens : undefined,
|
|
2550
|
+
costUsd: state.sawCost ? state.costUsd : undefined,
|
|
2139
2551
|
durationMs,
|
|
2140
|
-
lastActivity: lastTsMs !== undefined ? new Date(lastTsMs).toISOString() : undefined,
|
|
2141
|
-
contentText: userTexts.length > 0 ? userTexts.join('\n') : undefined,
|
|
2142
|
-
prUrl,
|
|
2143
|
-
prNumber,
|
|
2552
|
+
lastActivity: state.lastTsMs !== undefined ? new Date(state.lastTsMs).toISOString() : undefined,
|
|
2553
|
+
contentText: state.userTexts.length > 0 ? state.userTexts.join('\n') : undefined,
|
|
2554
|
+
prUrl: state.prUrl,
|
|
2555
|
+
prNumber: state.prNumber,
|
|
2144
2556
|
worktreeSlug: worktree?.slug,
|
|
2145
2557
|
ticketId: ticket?.id,
|
|
2146
|
-
createdTickets: createdTickets.size > 0 ? [...createdTickets] : undefined,
|
|
2147
|
-
spawnedTeam,
|
|
2148
|
-
plan,
|
|
2558
|
+
createdTickets: state.createdTickets.size > 0 ? [...state.createdTickets] : undefined,
|
|
2559
|
+
spawnedTeam: state.spawnedTeam,
|
|
2560
|
+
plan: state.plan,
|
|
2149
2561
|
};
|
|
2150
2562
|
}
|
|
2151
|
-
/** Stream a
|
|
2152
|
-
async function
|
|
2563
|
+
/** Stream a Claude JSONL file and extract scan-level metadata (timestamp, cwd, topic, tokens). */
|
|
2564
|
+
export async function scanClaudeSession(filePath) {
|
|
2153
2565
|
const stream = fs.createReadStream(filePath, { encoding: 'utf-8' });
|
|
2154
2566
|
const rl = readline.createInterface({ input: stream, crlfDelay: Infinity });
|
|
2155
|
-
|
|
2156
|
-
let timestamp;
|
|
2157
|
-
let cwd;
|
|
2158
|
-
let gitBranch;
|
|
2159
|
-
let version;
|
|
2160
|
-
let topic;
|
|
2161
|
-
let messageCount = 0;
|
|
2162
|
-
let tokenCount;
|
|
2163
|
-
let model;
|
|
2164
|
-
let lastTotalTokenUsage;
|
|
2165
|
-
let firstTsMs;
|
|
2166
|
-
let lastTsMs;
|
|
2167
|
-
const userTexts = [];
|
|
2168
|
-
let sawPrCreate = false;
|
|
2169
|
-
let prUrl;
|
|
2170
|
-
let prNumber;
|
|
2171
|
-
// Produced artifacts (mirror of the Claude scan): created tracker refs + spawned team.
|
|
2172
|
-
const createdTickets = new Set();
|
|
2173
|
-
const pendingTicketTools = new Set();
|
|
2174
|
-
let spawnedTeam;
|
|
2567
|
+
const state = initClaudeParseState();
|
|
2175
2568
|
try {
|
|
2176
2569
|
for await (const line of rl) {
|
|
2177
2570
|
if (!line.trim())
|
|
@@ -2183,140 +2576,707 @@ async function scanCodexSession(filePath) {
|
|
|
2183
2576
|
catch {
|
|
2184
2577
|
continue;
|
|
2185
2578
|
}
|
|
2186
|
-
|
|
2187
|
-
|
|
2188
|
-
|
|
2189
|
-
|
|
2190
|
-
|
|
2191
|
-
|
|
2192
|
-
|
|
2193
|
-
|
|
2194
|
-
|
|
2195
|
-
|
|
2196
|
-
|
|
2197
|
-
|
|
2198
|
-
|
|
2199
|
-
|
|
2200
|
-
|
|
2201
|
-
|
|
2202
|
-
|
|
2203
|
-
|
|
2204
|
-
|
|
2205
|
-
|
|
2206
|
-
|
|
2207
|
-
|
|
2208
|
-
|
|
2209
|
-
|
|
2210
|
-
|
|
2211
|
-
|
|
2212
|
-
|
|
2213
|
-
|
|
2214
|
-
|
|
2215
|
-
|
|
2216
|
-
|
|
2217
|
-
|
|
2218
|
-
|
|
2219
|
-
|
|
2220
|
-
|
|
2221
|
-
|
|
2222
|
-
|
|
2223
|
-
|
|
2224
|
-
|
|
2225
|
-
|
|
2226
|
-
|
|
2227
|
-
|
|
2228
|
-
|
|
2229
|
-
|
|
2230
|
-
|
|
2231
|
-
|
|
2232
|
-
|
|
2579
|
+
applyClaudeLine(state, parsed);
|
|
2580
|
+
}
|
|
2581
|
+
}
|
|
2582
|
+
finally {
|
|
2583
|
+
rl.close();
|
|
2584
|
+
stream.destroy();
|
|
2585
|
+
}
|
|
2586
|
+
return finalizeClaudeScan(state);
|
|
2587
|
+
}
|
|
2588
|
+
/** Cap on the FIFO window of recent assistant ids persisted in the continuation. */
|
|
2589
|
+
const SEEN_IDS_RECENT_CAP = 256;
|
|
2590
|
+
/**
|
|
2591
|
+
* Snapshot a live {@link ClaudeParseState} into its serializable form at
|
|
2592
|
+
* `offset` bytes consumed. Round-trips through {@link hydrateClaudeParseState}
|
|
2593
|
+
* so incremental replay equals a full parse.
|
|
2594
|
+
*/
|
|
2595
|
+
export function serializeClaudeParserState(state, offset) {
|
|
2596
|
+
const allIds = [...state.seenAssistantIds];
|
|
2597
|
+
const seenIdsRecent = allIds.length > SEEN_IDS_RECENT_CAP
|
|
2598
|
+
? allIds.slice(allIds.length - SEEN_IDS_RECENT_CAP)
|
|
2599
|
+
: allIds;
|
|
2600
|
+
const ticket = detectTicket(state.userTexts.join('\n') || undefined, state.gitBranch);
|
|
2601
|
+
return {
|
|
2602
|
+
v: 1,
|
|
2603
|
+
offset,
|
|
2604
|
+
timestamp: state.timestamp,
|
|
2605
|
+
cwd: state.cwd,
|
|
2606
|
+
gitBranch: state.gitBranch,
|
|
2607
|
+
version: state.version,
|
|
2608
|
+
entrypoint: state.entrypoint,
|
|
2609
|
+
firstTsMs: state.firstTsMs,
|
|
2610
|
+
topic: state.topic,
|
|
2611
|
+
customTitle: state.customTitle,
|
|
2612
|
+
aiTitle: state.aiTitle,
|
|
2613
|
+
plan: state.plan,
|
|
2614
|
+
lastTsMs: state.lastTsMs,
|
|
2615
|
+
messageCount: state.messageCount,
|
|
2616
|
+
tokenCount: state.tokenCount,
|
|
2617
|
+
outputTokens: state.outputTokens,
|
|
2618
|
+
sawTokenCount: state.sawTokenCount,
|
|
2619
|
+
sawCost: state.sawCost,
|
|
2620
|
+
costUsd: state.costUsd,
|
|
2621
|
+
seenIdsSize: state.seenAssistantIds.size,
|
|
2622
|
+
seenIdsRecent,
|
|
2623
|
+
sawPrCreate: state.sawPrCreate,
|
|
2624
|
+
prUrl: state.prUrl,
|
|
2625
|
+
prNumber: state.prNumber,
|
|
2626
|
+
pendingTicketTools: [...state.pendingTicketTools],
|
|
2627
|
+
createdTickets: [...state.createdTickets],
|
|
2628
|
+
spawnedTeam: state.spawnedTeam,
|
|
2629
|
+
// ticketId is derived at finalize time; persist it (and content_text) so a
|
|
2630
|
+
// consumer (B-2) can rebuild the row + FTS doc on append without re-reading
|
|
2631
|
+
// the whole file. worktreeSlug is re-derived from cwd/gitBranch, so it need
|
|
2632
|
+
// not be persisted.
|
|
2633
|
+
ticketId: ticket?.id,
|
|
2634
|
+
contentText: state.userTexts.length > 0 ? state.userTexts.join('\n') : undefined,
|
|
2635
|
+
};
|
|
2636
|
+
}
|
|
2637
|
+
/**
|
|
2638
|
+
* Rebuild a live {@link ClaudeParseState} from a persisted continuation so that
|
|
2639
|
+
* applying the appended lines yields the same accumulator a full parse would.
|
|
2640
|
+
*
|
|
2641
|
+
* `seenAssistantIds` is rehydrated from the recent-id FIFO window, then padded
|
|
2642
|
+
* with unique sentinel entries so its `.size` matches the true prior count
|
|
2643
|
+
* (`seenIdsSize`) — the fallback id `${ts}:${size}` must line up with the full
|
|
2644
|
+
* parse even when the window dropped older ids. Padding sentinels can never
|
|
2645
|
+
* collide with a real logical id (real ids are message ids/uuids or
|
|
2646
|
+
* `${ts}:${n}`; the sentinel prefix is not JSON-line-derived).
|
|
2647
|
+
*
|
|
2648
|
+
* `userTexts` is rehydrated as a single joined blob from `contentText`: only
|
|
2649
|
+
* `userTexts.join('\n')` (detectTicket + contentText) and `userTexts.length > 0`
|
|
2650
|
+
* are ever read downstream, and both are preserved by a one-element array
|
|
2651
|
+
* holding the joined content.
|
|
2652
|
+
*/
|
|
2653
|
+
export function hydrateClaudeParseState(prior) {
|
|
2654
|
+
const seen = new Set(prior.seenIdsRecent);
|
|
2655
|
+
// Pad to the true prior size so `seenAssistantIds.size` (which feeds the
|
|
2656
|
+
// fallback logical id) is exact even when older ids fell out of the window.
|
|
2657
|
+
let pad = 0;
|
|
2658
|
+
while (seen.size < prior.seenIdsSize) {
|
|
2659
|
+
seen.add(`pad:${pad++}`);
|
|
2660
|
+
}
|
|
2661
|
+
return {
|
|
2662
|
+
timestamp: prior.timestamp,
|
|
2663
|
+
cwd: prior.cwd,
|
|
2664
|
+
gitBranch: prior.gitBranch,
|
|
2665
|
+
version: prior.version,
|
|
2666
|
+
topic: prior.topic,
|
|
2667
|
+
customTitle: prior.customTitle,
|
|
2668
|
+
aiTitle: prior.aiTitle,
|
|
2669
|
+
entrypoint: prior.entrypoint,
|
|
2670
|
+
messageCount: prior.messageCount,
|
|
2671
|
+
tokenCount: prior.tokenCount,
|
|
2672
|
+
outputTokens: prior.outputTokens,
|
|
2673
|
+
sawTokenCount: prior.sawTokenCount,
|
|
2674
|
+
costUsd: prior.costUsd,
|
|
2675
|
+
sawCost: prior.sawCost,
|
|
2676
|
+
firstTsMs: prior.firstTsMs,
|
|
2677
|
+
lastTsMs: prior.lastTsMs,
|
|
2678
|
+
seenAssistantIds: seen,
|
|
2679
|
+
userTexts: prior.contentText !== undefined && prior.contentText.length > 0 ? [prior.contentText] : [],
|
|
2680
|
+
sawPrCreate: prior.sawPrCreate,
|
|
2681
|
+
prUrl: prior.prUrl,
|
|
2682
|
+
prNumber: prior.prNumber,
|
|
2683
|
+
createdTickets: new Set(prior.createdTickets),
|
|
2684
|
+
pendingTicketTools: new Set(prior.pendingTicketTools),
|
|
2685
|
+
spawnedTeam: prior.spawnedTeam,
|
|
2686
|
+
plan: prior.plan,
|
|
2687
|
+
};
|
|
2688
|
+
}
|
|
2689
|
+
/**
|
|
2690
|
+
* Resume a Claude parse from `fromOffset` bytes into the file, folding only the
|
|
2691
|
+
* newly-appended lines into `prior`. Returns the finalized scan, the next
|
|
2692
|
+
* serialized continuation, and the byte offset to resume from next time —
|
|
2693
|
+
* `newOffset` stops at the last `'\n'` seen so a half-written trailing record is
|
|
2694
|
+
* re-read (not lost) on the next append.
|
|
2695
|
+
*
|
|
2696
|
+
* NOT wired into the live scan path yet (that is B-2); {@link scanClaudeSession}
|
|
2697
|
+
* remains the only caller-facing entry point. This exists so the parity harness
|
|
2698
|
+
* can prove full === hydrate(state@k) + apply(k+1..n).
|
|
2699
|
+
*/
|
|
2700
|
+
export async function scanClaudeSessionIncremental(filePath, fromOffset, prior) {
|
|
2701
|
+
const state = hydrateClaudeParseState(prior);
|
|
2702
|
+
// Read the appended byte range and apply ONLY newline-terminated lines. The
|
|
2703
|
+
// applied lines and `newOffset` MUST stay consistent: readline (like the full
|
|
2704
|
+
// parse) would emit a trailing UNTERMINATED final line at EOF, but `newOffset`
|
|
2705
|
+
// stops before it — so the next pass, after that record's '\n' is flushed,
|
|
2706
|
+
// would re-read and re-apply the same line and double-count it (user events
|
|
2707
|
+
// have no dedup, unlike assistant `seenAssistantIds`). A record written
|
|
2708
|
+
// non-atomically — bytes first, then '\n' in a second write — is exactly this
|
|
2709
|
+
// case. So we slice at the last '\n' ourselves: everything up to and including
|
|
2710
|
+
// it is a run of complete lines we apply and commit; any tail after it
|
|
2711
|
+
// (syntactically broken OR complete-but-not-yet-terminated) is a still-being-
|
|
2712
|
+
// written record we DEFER to the next pass, once its '\n' lands.
|
|
2713
|
+
const chunks = [];
|
|
2714
|
+
const stream = fs.createReadStream(filePath, { start: fromOffset });
|
|
2715
|
+
try {
|
|
2716
|
+
for await (const chunk of stream) {
|
|
2717
|
+
chunks.push(typeof chunk === 'string' ? Buffer.from(chunk, 'utf-8') : chunk);
|
|
2718
|
+
}
|
|
2719
|
+
}
|
|
2720
|
+
finally {
|
|
2721
|
+
stream.destroy();
|
|
2722
|
+
}
|
|
2723
|
+
const appended = Buffer.concat(chunks);
|
|
2724
|
+
// Bytes up to AND INCLUDING the last '\n' are the committed, complete-line run.
|
|
2725
|
+
const lastNl = appended.lastIndexOf(0x0a);
|
|
2726
|
+
const consumedBytes = lastNl === -1 ? 0 : lastNl + 1;
|
|
2727
|
+
if (consumedBytes > 0) {
|
|
2728
|
+
// split('\n') on the committed run: the element after the final '\n' is ''
|
|
2729
|
+
// (skipped by the trim guard). A stray '\r' from CRLF is tolerated by the
|
|
2730
|
+
// JSON.parse below exactly as the full parse tolerates it.
|
|
2731
|
+
for (const line of appended.subarray(0, consumedBytes).toString('utf-8').split('\n')) {
|
|
2732
|
+
if (!line.trim())
|
|
2733
|
+
continue;
|
|
2734
|
+
let parsed;
|
|
2735
|
+
try {
|
|
2736
|
+
parsed = JSON.parse(line);
|
|
2233
2737
|
}
|
|
2234
|
-
|
|
2235
|
-
const payload = parsed.payload || {};
|
|
2236
|
-
sessionId = payload.id || sessionId;
|
|
2237
|
-
timestamp = payload.timestamp || parsed.timestamp || timestamp;
|
|
2238
|
-
cwd = payload.cwd || cwd;
|
|
2239
|
-
gitBranch = payload.git?.branch || gitBranch;
|
|
2240
|
-
version = payload.cli_version || payload.version || version;
|
|
2241
|
-
model = payload.model || model;
|
|
2738
|
+
catch {
|
|
2242
2739
|
continue;
|
|
2243
2740
|
}
|
|
2244
|
-
|
|
2245
|
-
|
|
2246
|
-
|
|
2247
|
-
|
|
2248
|
-
|
|
2249
|
-
|
|
2250
|
-
|
|
2251
|
-
|
|
2252
|
-
|
|
2253
|
-
|
|
2254
|
-
|
|
2255
|
-
|
|
2256
|
-
|
|
2741
|
+
applyClaudeLine(state, parsed);
|
|
2742
|
+
}
|
|
2743
|
+
}
|
|
2744
|
+
const newOffset = fromOffset + consumedBytes;
|
|
2745
|
+
return {
|
|
2746
|
+
scan: finalizeClaudeScan(state),
|
|
2747
|
+
newState: serializeClaudeParserState(state, newOffset),
|
|
2748
|
+
newOffset,
|
|
2749
|
+
};
|
|
2750
|
+
}
|
|
2751
|
+
/** Serialized zero-value continuation: a fresh accumulator at offset 0, used to drive a FULL parse from the start through the same resumable path. */
|
|
2752
|
+
function freshClaudeParserState() {
|
|
2753
|
+
return serializeClaudeParserState(initClaudeParseState(), 0);
|
|
2754
|
+
}
|
|
2755
|
+
/**
|
|
2756
|
+
* Decide full-vs-incremental for one Claude file and parse it uniformly, always
|
|
2757
|
+
* returning a finalized scan plus the continuation to persist. Both branches run
|
|
2758
|
+
* through the SAME reducer (via {@link scanClaudeSessionIncremental}), so the row
|
|
2759
|
+
* an append produces is identical to a from-scratch full reparse by construction
|
|
2760
|
+
* (the B-1 parity harness proves this at the function level).
|
|
2761
|
+
*
|
|
2762
|
+
* INCREMENTAL when a prior continuation exists AND the file grew past the
|
|
2763
|
+
* persisted offset AND its mtime did not go backwards — an in-place append.
|
|
2764
|
+
* FULL (from byte 0, fresh state) otherwise: cold start (no prior), truncation /
|
|
2765
|
+
* rewrite (size shrank to at or below the offset), or a clock rewind / restore
|
|
2766
|
+
* (mtime older than the last parse). A FULL parse still produces a continuation,
|
|
2767
|
+
* so the file's very next append can go incremental.
|
|
2768
|
+
*
|
|
2769
|
+
* `mode` is returned so the caller (and tests) can confirm which branch ran.
|
|
2770
|
+
*/
|
|
2771
|
+
async function scanClaudeSessionResumable(filePath, prior, currentFileMtimeMs, currentFileSize, priorFileMtimeMs) {
|
|
2772
|
+
// File size + mtime cannot distinguish an APPEND from an in-place rewrite or a
|
|
2773
|
+
// restore that dropped DIFFERENT, larger content at the same path: both grow
|
|
2774
|
+
// the file and move mtime forward. Resuming from the stored offset across that
|
|
2775
|
+
// boundary would fold the new file's bytes into an accumulator hydrated from
|
|
2776
|
+
// the OLD session, so the persisted row silently diverges from a full reparse.
|
|
2777
|
+
// So the metadata gate below only makes a file ELIGIBLE; before trusting the
|
|
2778
|
+
// offset we re-read the transcript's first user/assistant timestamp and require
|
|
2779
|
+
// it to still match the prior continuation's. An append keeps that identity
|
|
2780
|
+
// byte-for-byte; a rewrite/restore of a different session changes it. A
|
|
2781
|
+
// mismatch — or an identity we cannot derive — falls back to a FULL parse,
|
|
2782
|
+
// which is always correct. (A shrink is already handled: currentFileSize is not
|
|
2783
|
+
// > prior.offset, so it takes the FULL branch.)
|
|
2784
|
+
let canIncrement = false;
|
|
2785
|
+
if (prior !== null &&
|
|
2786
|
+
currentFileSize > prior.offset &&
|
|
2787
|
+
(priorFileMtimeMs === undefined || currentFileMtimeMs >= priorFileMtimeMs) &&
|
|
2788
|
+
prior.timestamp !== undefined) {
|
|
2789
|
+
canIncrement = (await claudeSessionIdentityAt(filePath)) === prior.timestamp;
|
|
2790
|
+
}
|
|
2791
|
+
if (canIncrement && prior !== null) {
|
|
2792
|
+
const result = await scanClaudeSessionIncremental(filePath, prior.offset, prior);
|
|
2793
|
+
return { ...result, mode: 'incremental' };
|
|
2794
|
+
}
|
|
2795
|
+
const result = await scanClaudeSessionIncremental(filePath, 0, freshClaudeParserState());
|
|
2796
|
+
return { ...result, mode: 'full' };
|
|
2797
|
+
}
|
|
2798
|
+
/**
|
|
2799
|
+
* Cheaply derive a Claude transcript's session identity — the first
|
|
2800
|
+
* user/assistant event `timestamp` — by streaming only the START of the file
|
|
2801
|
+
* (at most `maxBytes`) and stopping at the first such event. Used by
|
|
2802
|
+
* {@link scanClaudeSessionResumable} to confirm a grown file is still the SAME
|
|
2803
|
+
* session before resuming from a stored parse offset. Returns undefined when no
|
|
2804
|
+
* user/assistant event appears within the budget, which forces a FULL parse.
|
|
2805
|
+
*/
|
|
2806
|
+
async function claudeSessionIdentityAt(filePath, maxBytes = 1_048_576) {
|
|
2807
|
+
const state = initClaudeParseState();
|
|
2808
|
+
const stream = fs.createReadStream(filePath, { start: 0, end: maxBytes - 1, encoding: 'utf-8' });
|
|
2809
|
+
const rl = readline.createInterface({ input: stream, crlfDelay: Infinity });
|
|
2810
|
+
try {
|
|
2811
|
+
for await (const line of rl) {
|
|
2812
|
+
if (!line.trim())
|
|
2257
2813
|
continue;
|
|
2814
|
+
let parsed;
|
|
2815
|
+
try {
|
|
2816
|
+
parsed = JSON.parse(line);
|
|
2258
2817
|
}
|
|
2259
|
-
|
|
2260
|
-
|
|
2261
|
-
const total = getCodexTokenCount(totalUsage);
|
|
2262
|
-
if (total !== null)
|
|
2263
|
-
tokenCount = total;
|
|
2264
|
-
// token_count is cumulative — keep the latest snapshot and price it once
|
|
2265
|
-
// after the stream, so we don't double-count across intermediate events.
|
|
2266
|
-
if (totalUsage && typeof totalUsage === 'object')
|
|
2267
|
-
lastTotalTokenUsage = totalUsage;
|
|
2268
|
-
// Codex also stamps the model on the rate_limits/token_count payload on
|
|
2269
|
-
// some versions; prefer session_meta but fall back to it.
|
|
2270
|
-
if (!model && typeof parsed.payload.info?.model === 'string')
|
|
2271
|
-
model = parsed.payload.info.model;
|
|
2818
|
+
catch {
|
|
2819
|
+
continue;
|
|
2272
2820
|
}
|
|
2821
|
+
applyClaudeLine(state, parsed);
|
|
2822
|
+
if (state.timestamp !== undefined)
|
|
2823
|
+
break;
|
|
2273
2824
|
}
|
|
2274
2825
|
}
|
|
2275
2826
|
finally {
|
|
2276
2827
|
rl.close();
|
|
2277
2828
|
stream.destroy();
|
|
2278
2829
|
}
|
|
2830
|
+
return state.timestamp;
|
|
2831
|
+
}
|
|
2832
|
+
/**
|
|
2833
|
+
* Parse the prior continuation blob for a changed file into a usable
|
|
2834
|
+
* {@link ClaudeParserState}, or null when there is none / it is unusable. A blob
|
|
2835
|
+
* from a different serialization version is treated as absent so the file falls
|
|
2836
|
+
* back to a clean FULL parse rather than resuming against a stale shape.
|
|
2837
|
+
*/
|
|
2838
|
+
function parsePriorClaudeState(row) {
|
|
2839
|
+
if (!row?.parserState)
|
|
2840
|
+
return null;
|
|
2841
|
+
try {
|
|
2842
|
+
const parsed = JSON.parse(row.parserState);
|
|
2843
|
+
if (parsed?.v !== 1 || typeof parsed.offset !== 'number')
|
|
2844
|
+
return null;
|
|
2845
|
+
return parsed;
|
|
2846
|
+
}
|
|
2847
|
+
catch {
|
|
2848
|
+
return null;
|
|
2849
|
+
}
|
|
2850
|
+
}
|
|
2851
|
+
/** Test seam: how many times the incremental (append-resume) branch was taken since the last reset. */
|
|
2852
|
+
let claudeIncrementalScanCount = 0;
|
|
2853
|
+
/** Test seam: how many times a full (from-offset-0) Claude parse ran since the last reset. */
|
|
2854
|
+
let claudeFullScanCount = 0;
|
|
2855
|
+
/** Test seam: read the (incremental, full) Claude parse counters. */
|
|
2856
|
+
export function __claudeScanBranchCountsForTest() {
|
|
2857
|
+
return { incremental: claudeIncrementalScanCount, full: claudeFullScanCount };
|
|
2858
|
+
}
|
|
2859
|
+
/** Test seam: reset the Claude parse-branch counters to observe a scan from a clean slate. */
|
|
2860
|
+
export function __resetClaudeScanBranchCountsForTest() {
|
|
2861
|
+
claudeIncrementalScanCount = 0;
|
|
2862
|
+
claudeFullScanCount = 0;
|
|
2863
|
+
}
|
|
2864
|
+
/** Zero-value accumulator for a fresh (from-byte-0) Codex parse. */
|
|
2865
|
+
export function initCodexParseState() {
|
|
2866
|
+
return {
|
|
2867
|
+
sessionId: undefined,
|
|
2868
|
+
timestamp: undefined,
|
|
2869
|
+
cwd: undefined,
|
|
2870
|
+
gitBranch: undefined,
|
|
2871
|
+
version: undefined,
|
|
2872
|
+
model: undefined,
|
|
2873
|
+
topic: undefined,
|
|
2874
|
+
messageCount: 0,
|
|
2875
|
+
tokenCount: undefined,
|
|
2876
|
+
lastTotalTokenUsage: undefined,
|
|
2877
|
+
firstTsMs: undefined,
|
|
2878
|
+
lastTsMs: undefined,
|
|
2879
|
+
userTexts: [],
|
|
2880
|
+
sawPrCreate: false,
|
|
2881
|
+
prUrl: undefined,
|
|
2882
|
+
prNumber: undefined,
|
|
2883
|
+
createdTickets: new Set(),
|
|
2884
|
+
pendingTicketTools: new Set(),
|
|
2885
|
+
spawnedTeam: undefined,
|
|
2886
|
+
};
|
|
2887
|
+
}
|
|
2888
|
+
/**
|
|
2889
|
+
* Fold one parsed Codex line into the accumulator — the exact loop body
|
|
2890
|
+
* {@link scanCodexSession} used to run inline, extracted verbatim and mutating
|
|
2891
|
+
* `state.*` in place. `parsed` is the already-`JSON.parse`d line (the
|
|
2892
|
+
* malformed-line skip happens in the caller, as before).
|
|
2893
|
+
*/
|
|
2894
|
+
export function applyCodexLine(state, parsed) {
|
|
2895
|
+
// PR signal, structurally: a Codex `function_call` whose command is
|
|
2896
|
+
// `gh pr create`, then the pull URL from a `function_call_output`.
|
|
2897
|
+
if (parsed.type === 'response_item') {
|
|
2898
|
+
const p = parsed.payload || {};
|
|
2899
|
+
if (p.type === 'function_call') {
|
|
2900
|
+
let cmd = '';
|
|
2901
|
+
try {
|
|
2902
|
+
const args = typeof p.arguments === 'string' ? JSON.parse(p.arguments) : (p.arguments || {});
|
|
2903
|
+
cmd = String(args.command || args.cmd || '');
|
|
2904
|
+
}
|
|
2905
|
+
catch { /* non-JSON args */ }
|
|
2906
|
+
if (!state.prUrl && !state.sawPrCreate && isPrCreateCommand(cmd))
|
|
2907
|
+
state.sawPrCreate = true;
|
|
2908
|
+
if (!state.spawnedTeam) {
|
|
2909
|
+
const team = detectSpawnedTeam(cmd);
|
|
2910
|
+
if (team)
|
|
2911
|
+
state.spawnedTeam = team;
|
|
2912
|
+
}
|
|
2913
|
+
if (typeof p.call_id === 'string' && isTicketCreateTool(p.name, cmd)) {
|
|
2914
|
+
state.pendingTicketTools.add(p.call_id);
|
|
2915
|
+
}
|
|
2916
|
+
}
|
|
2917
|
+
if (p.type === 'function_call_output') {
|
|
2918
|
+
if (!state.prUrl && state.sawPrCreate) {
|
|
2919
|
+
const pr = extractPrUrl(String(p.output || ''));
|
|
2920
|
+
if (pr) {
|
|
2921
|
+
state.prUrl = pr.url;
|
|
2922
|
+
state.prNumber = pr.number;
|
|
2923
|
+
}
|
|
2924
|
+
}
|
|
2925
|
+
if (typeof p.call_id === 'string' && state.pendingTicketTools.has(p.call_id)) {
|
|
2926
|
+
state.pendingTicketTools.delete(p.call_id);
|
|
2927
|
+
const t = extractCreatedTicket(String(p.output || ''));
|
|
2928
|
+
if (t)
|
|
2929
|
+
state.createdTickets.add(t);
|
|
2930
|
+
}
|
|
2931
|
+
}
|
|
2932
|
+
}
|
|
2933
|
+
// Track duration across every timestamped event.
|
|
2934
|
+
if (typeof parsed.timestamp === 'string') {
|
|
2935
|
+
const ms = new Date(parsed.timestamp).getTime();
|
|
2936
|
+
if (!Number.isNaN(ms)) {
|
|
2937
|
+
if (state.firstTsMs === undefined || ms < state.firstTsMs)
|
|
2938
|
+
state.firstTsMs = ms;
|
|
2939
|
+
if (state.lastTsMs === undefined || ms > state.lastTsMs)
|
|
2940
|
+
state.lastTsMs = ms;
|
|
2941
|
+
}
|
|
2942
|
+
}
|
|
2943
|
+
if (parsed.type === 'session_meta') {
|
|
2944
|
+
const payload = parsed.payload || {};
|
|
2945
|
+
state.sessionId = payload.id || state.sessionId;
|
|
2946
|
+
state.timestamp = payload.timestamp || parsed.timestamp || state.timestamp;
|
|
2947
|
+
state.cwd = payload.cwd || state.cwd;
|
|
2948
|
+
state.gitBranch = payload.git?.branch || state.gitBranch;
|
|
2949
|
+
state.version = payload.cli_version || payload.version || state.version;
|
|
2950
|
+
state.model = payload.model || state.model;
|
|
2951
|
+
return;
|
|
2952
|
+
}
|
|
2953
|
+
if (parsed.type === 'response_item' && parsed.payload?.type === 'message') {
|
|
2954
|
+
const role = parsed.payload.role === 'user' || parsed.payload.role === 'developer'
|
|
2955
|
+
? 'user'
|
|
2956
|
+
: 'assistant';
|
|
2957
|
+
const text = extractCodexMessageText(parsed.payload.content, role);
|
|
2958
|
+
if (!text)
|
|
2959
|
+
return;
|
|
2960
|
+
state.messageCount++;
|
|
2961
|
+
if (role === 'user') {
|
|
2962
|
+
state.userTexts.push(text);
|
|
2963
|
+
if (!state.topic)
|
|
2964
|
+
state.topic = extractSessionTopic(text);
|
|
2965
|
+
}
|
|
2966
|
+
return;
|
|
2967
|
+
}
|
|
2968
|
+
if (parsed.type === 'event_msg' && parsed.payload?.type === 'token_count') {
|
|
2969
|
+
const totalUsage = parsed.payload.info?.total_token_usage;
|
|
2970
|
+
const total = getCodexTokenCount(totalUsage);
|
|
2971
|
+
if (total !== null)
|
|
2972
|
+
state.tokenCount = total;
|
|
2973
|
+
// token_count is cumulative — keep the latest snapshot and price it once
|
|
2974
|
+
// after the stream, so we don't double-count across intermediate events.
|
|
2975
|
+
if (totalUsage && typeof totalUsage === 'object')
|
|
2976
|
+
state.lastTotalTokenUsage = totalUsage;
|
|
2977
|
+
// Codex also stamps the model on the rate_limits/token_count payload on
|
|
2978
|
+
// some versions; prefer session_meta but fall back to it.
|
|
2979
|
+
if (!state.model && typeof parsed.payload.info?.model === 'string')
|
|
2980
|
+
state.model = parsed.payload.info.model;
|
|
2981
|
+
}
|
|
2982
|
+
}
|
|
2983
|
+
/**
|
|
2984
|
+
* Build the {@link CodexSessionScan} return object from an accumulator — the
|
|
2985
|
+
* exact return-building {@link scanCodexSession} used to run inline.
|
|
2986
|
+
*/
|
|
2987
|
+
export function finalizeCodexScan(state) {
|
|
2279
2988
|
// Price the final cumulative token snapshot once, against the session model.
|
|
2280
2989
|
let costUsd;
|
|
2281
|
-
if (model && lastTotalTokenUsage) {
|
|
2990
|
+
if (state.model && state.lastTotalTokenUsage) {
|
|
2282
2991
|
const c = costOfUsage({
|
|
2283
|
-
model,
|
|
2284
|
-
inputTokens: lastTotalTokenUsage.input_tokens,
|
|
2285
|
-
outputTokens: (lastTotalTokenUsage.output_tokens ?? 0) + (lastTotalTokenUsage.reasoning_output_tokens ?? 0),
|
|
2286
|
-
cacheReadTokens: lastTotalTokenUsage.cached_input_tokens,
|
|
2992
|
+
model: state.model,
|
|
2993
|
+
inputTokens: state.lastTotalTokenUsage.input_tokens,
|
|
2994
|
+
outputTokens: (state.lastTotalTokenUsage.output_tokens ?? 0) + (state.lastTotalTokenUsage.reasoning_output_tokens ?? 0),
|
|
2995
|
+
cacheReadTokens: state.lastTotalTokenUsage.cached_input_tokens,
|
|
2287
2996
|
});
|
|
2288
2997
|
if (c > 0)
|
|
2289
2998
|
costUsd = c;
|
|
2290
2999
|
}
|
|
2291
|
-
const durationMs = firstTsMs !== undefined && lastTsMs !== undefined && lastTsMs > firstTsMs
|
|
2292
|
-
? lastTsMs - firstTsMs
|
|
3000
|
+
const durationMs = state.firstTsMs !== undefined && state.lastTsMs !== undefined && state.lastTsMs > state.firstTsMs
|
|
3001
|
+
? state.lastTsMs - state.firstTsMs
|
|
2293
3002
|
: undefined;
|
|
2294
|
-
const worktree = detectWorktree(cwd, gitBranch);
|
|
2295
|
-
const ticket = detectTicket(userTexts.join('\n') || undefined, gitBranch);
|
|
3003
|
+
const worktree = detectWorktree(state.cwd, state.gitBranch);
|
|
3004
|
+
const ticket = detectTicket(state.userTexts.join('\n') || undefined, state.gitBranch);
|
|
2296
3005
|
return {
|
|
2297
|
-
sessionId,
|
|
2298
|
-
timestamp,
|
|
2299
|
-
cwd,
|
|
2300
|
-
gitBranch,
|
|
2301
|
-
version,
|
|
2302
|
-
topic,
|
|
2303
|
-
messageCount,
|
|
2304
|
-
tokenCount,
|
|
2305
|
-
outputTokens: lastTotalTokenUsage
|
|
2306
|
-
? (lastTotalTokenUsage.output_tokens ?? 0) + (lastTotalTokenUsage.reasoning_output_tokens ?? 0)
|
|
3006
|
+
sessionId: state.sessionId,
|
|
3007
|
+
timestamp: state.timestamp,
|
|
3008
|
+
cwd: state.cwd,
|
|
3009
|
+
gitBranch: state.gitBranch,
|
|
3010
|
+
version: state.version,
|
|
3011
|
+
topic: state.topic,
|
|
3012
|
+
messageCount: state.messageCount,
|
|
3013
|
+
tokenCount: state.tokenCount,
|
|
3014
|
+
outputTokens: state.lastTotalTokenUsage
|
|
3015
|
+
? (state.lastTotalTokenUsage.output_tokens ?? 0) + (state.lastTotalTokenUsage.reasoning_output_tokens ?? 0)
|
|
2307
3016
|
: undefined,
|
|
2308
3017
|
costUsd,
|
|
2309
3018
|
durationMs,
|
|
2310
|
-
lastActivity: lastTsMs !== undefined ? new Date(lastTsMs).toISOString() : undefined,
|
|
2311
|
-
contentText: userTexts.length > 0 ? userTexts.join('\n') : undefined,
|
|
2312
|
-
prUrl,
|
|
2313
|
-
prNumber,
|
|
3019
|
+
lastActivity: state.lastTsMs !== undefined ? new Date(state.lastTsMs).toISOString() : undefined,
|
|
3020
|
+
contentText: state.userTexts.length > 0 ? state.userTexts.join('\n') : undefined,
|
|
3021
|
+
prUrl: state.prUrl,
|
|
3022
|
+
prNumber: state.prNumber,
|
|
2314
3023
|
worktreeSlug: worktree?.slug,
|
|
2315
3024
|
ticketId: ticket?.id,
|
|
2316
|
-
createdTickets: createdTickets.size > 0 ? [...createdTickets] : undefined,
|
|
2317
|
-
spawnedTeam,
|
|
3025
|
+
createdTickets: state.createdTickets.size > 0 ? [...state.createdTickets] : undefined,
|
|
3026
|
+
spawnedTeam: state.spawnedTeam,
|
|
2318
3027
|
};
|
|
2319
3028
|
}
|
|
3029
|
+
/** Stream a Codex JSONL file and extract scan-level metadata (session ID, cwd, topic, tokens). */
|
|
3030
|
+
async function scanCodexSession(filePath) {
|
|
3031
|
+
const stream = fs.createReadStream(filePath, { encoding: 'utf-8' });
|
|
3032
|
+
const rl = readline.createInterface({ input: stream, crlfDelay: Infinity });
|
|
3033
|
+
const state = initCodexParseState();
|
|
3034
|
+
try {
|
|
3035
|
+
for await (const line of rl) {
|
|
3036
|
+
if (!line.trim())
|
|
3037
|
+
continue;
|
|
3038
|
+
let parsed;
|
|
3039
|
+
try {
|
|
3040
|
+
parsed = JSON.parse(line);
|
|
3041
|
+
}
|
|
3042
|
+
catch {
|
|
3043
|
+
continue;
|
|
3044
|
+
}
|
|
3045
|
+
applyCodexLine(state, parsed);
|
|
3046
|
+
}
|
|
3047
|
+
}
|
|
3048
|
+
finally {
|
|
3049
|
+
rl.close();
|
|
3050
|
+
stream.destroy();
|
|
3051
|
+
}
|
|
3052
|
+
return finalizeCodexScan(state);
|
|
3053
|
+
}
|
|
3054
|
+
/**
|
|
3055
|
+
* Snapshot a live {@link CodexParseState} into its serializable form at `offset`
|
|
3056
|
+
* bytes consumed. Round-trips through {@link hydrateCodexParseState} so
|
|
3057
|
+
* incremental replay equals a full parse.
|
|
3058
|
+
*/
|
|
3059
|
+
export function serializeCodexParserState(state, offset) {
|
|
3060
|
+
const ticket = detectTicket(state.userTexts.join('\n') || undefined, state.gitBranch);
|
|
3061
|
+
return {
|
|
3062
|
+
v: 1,
|
|
3063
|
+
offset,
|
|
3064
|
+
sessionId: state.sessionId,
|
|
3065
|
+
timestamp: state.timestamp,
|
|
3066
|
+
cwd: state.cwd,
|
|
3067
|
+
gitBranch: state.gitBranch,
|
|
3068
|
+
version: state.version,
|
|
3069
|
+
model: state.model,
|
|
3070
|
+
topic: state.topic,
|
|
3071
|
+
messageCount: state.messageCount,
|
|
3072
|
+
tokenCount: state.tokenCount,
|
|
3073
|
+
lastTotalTokenUsage: state.lastTotalTokenUsage,
|
|
3074
|
+
firstTsMs: state.firstTsMs,
|
|
3075
|
+
lastTsMs: state.lastTsMs,
|
|
3076
|
+
sawPrCreate: state.sawPrCreate,
|
|
3077
|
+
prUrl: state.prUrl,
|
|
3078
|
+
prNumber: state.prNumber,
|
|
3079
|
+
pendingTicketTools: [...state.pendingTicketTools],
|
|
3080
|
+
createdTickets: [...state.createdTickets],
|
|
3081
|
+
spawnedTeam: state.spawnedTeam,
|
|
3082
|
+
// ticketId is derived at finalize time; persist it (and content_text) so a
|
|
3083
|
+
// consumer can rebuild the row + FTS doc on append without re-reading the
|
|
3084
|
+
// whole file. worktreeSlug is re-derived from cwd/gitBranch, so it need not
|
|
3085
|
+
// be persisted.
|
|
3086
|
+
ticketId: ticket?.id,
|
|
3087
|
+
contentText: state.userTexts.length > 0 ? state.userTexts.join('\n') : undefined,
|
|
3088
|
+
};
|
|
3089
|
+
}
|
|
3090
|
+
/**
|
|
3091
|
+
* Rebuild a live {@link CodexParseState} from a persisted continuation so that
|
|
3092
|
+
* applying the appended lines yields the same accumulator a full parse would.
|
|
3093
|
+
*
|
|
3094
|
+
* `userTexts` is rehydrated as a single joined blob from `contentText`: only
|
|
3095
|
+
* `userTexts.join('\n')` (detectTicket + contentText) and `userTexts.length > 0`
|
|
3096
|
+
* are ever read downstream, and both are preserved by a one-element array
|
|
3097
|
+
* holding the joined content. Topic is first-wins and already persisted, so a
|
|
3098
|
+
* collapsed userTexts never changes it.
|
|
3099
|
+
*/
|
|
3100
|
+
export function hydrateCodexParseState(prior) {
|
|
3101
|
+
return {
|
|
3102
|
+
sessionId: prior.sessionId,
|
|
3103
|
+
timestamp: prior.timestamp,
|
|
3104
|
+
cwd: prior.cwd,
|
|
3105
|
+
gitBranch: prior.gitBranch,
|
|
3106
|
+
version: prior.version,
|
|
3107
|
+
model: prior.model,
|
|
3108
|
+
topic: prior.topic,
|
|
3109
|
+
messageCount: prior.messageCount,
|
|
3110
|
+
tokenCount: prior.tokenCount,
|
|
3111
|
+
lastTotalTokenUsage: prior.lastTotalTokenUsage,
|
|
3112
|
+
firstTsMs: prior.firstTsMs,
|
|
3113
|
+
lastTsMs: prior.lastTsMs,
|
|
3114
|
+
userTexts: prior.contentText !== undefined && prior.contentText.length > 0 ? [prior.contentText] : [],
|
|
3115
|
+
sawPrCreate: prior.sawPrCreate,
|
|
3116
|
+
prUrl: prior.prUrl,
|
|
3117
|
+
prNumber: prior.prNumber,
|
|
3118
|
+
createdTickets: new Set(prior.createdTickets),
|
|
3119
|
+
pendingTicketTools: new Set(prior.pendingTicketTools),
|
|
3120
|
+
spawnedTeam: prior.spawnedTeam,
|
|
3121
|
+
};
|
|
3122
|
+
}
|
|
3123
|
+
/**
|
|
3124
|
+
* Resume a Codex parse from `fromOffset` bytes into the file, folding only the
|
|
3125
|
+
* newly-appended lines into `prior`. Returns the finalized scan, the next
|
|
3126
|
+
* serialized continuation, and the byte offset to resume from next time.
|
|
3127
|
+
*
|
|
3128
|
+
* Same trailing-line discipline as {@link scanClaudeSessionIncremental}: apply
|
|
3129
|
+
* ONLY the run of newline-terminated lines (slice at the last `'\n'`), and set
|
|
3130
|
+
* `newOffset = fromOffset + consumedBytes`. Any tail after the last `'\n'` —
|
|
3131
|
+
* syntactically broken OR a complete-but-not-yet-terminated record — is DEFERRED
|
|
3132
|
+
* to the next pass once its `'\n'` lands. Codex `messageCount` is additive with
|
|
3133
|
+
* NO dedup, so re-reading a still-unterminated complete line would double-count
|
|
3134
|
+
* it; deferring prevents that (the bug class prix-cloud caught for Claude).
|
|
3135
|
+
*/
|
|
3136
|
+
export async function scanCodexSessionIncremental(filePath, fromOffset, prior) {
|
|
3137
|
+
const state = hydrateCodexParseState(prior);
|
|
3138
|
+
const chunks = [];
|
|
3139
|
+
const stream = fs.createReadStream(filePath, { start: fromOffset });
|
|
3140
|
+
try {
|
|
3141
|
+
for await (const chunk of stream) {
|
|
3142
|
+
chunks.push(typeof chunk === 'string' ? Buffer.from(chunk, 'utf-8') : chunk);
|
|
3143
|
+
}
|
|
3144
|
+
}
|
|
3145
|
+
finally {
|
|
3146
|
+
stream.destroy();
|
|
3147
|
+
}
|
|
3148
|
+
const appended = Buffer.concat(chunks);
|
|
3149
|
+
// Bytes up to AND INCLUDING the last '\n' are the committed, complete-line run.
|
|
3150
|
+
const lastNl = appended.lastIndexOf(0x0a);
|
|
3151
|
+
const consumedBytes = lastNl === -1 ? 0 : lastNl + 1;
|
|
3152
|
+
if (consumedBytes > 0) {
|
|
3153
|
+
for (const line of appended.subarray(0, consumedBytes).toString('utf-8').split('\n')) {
|
|
3154
|
+
if (!line.trim())
|
|
3155
|
+
continue;
|
|
3156
|
+
let parsed;
|
|
3157
|
+
try {
|
|
3158
|
+
parsed = JSON.parse(line);
|
|
3159
|
+
}
|
|
3160
|
+
catch {
|
|
3161
|
+
continue;
|
|
3162
|
+
}
|
|
3163
|
+
applyCodexLine(state, parsed);
|
|
3164
|
+
}
|
|
3165
|
+
}
|
|
3166
|
+
const newOffset = fromOffset + consumedBytes;
|
|
3167
|
+
return {
|
|
3168
|
+
scan: finalizeCodexScan(state),
|
|
3169
|
+
newState: serializeCodexParserState(state, newOffset),
|
|
3170
|
+
newOffset,
|
|
3171
|
+
};
|
|
3172
|
+
}
|
|
3173
|
+
/** Serialized zero-value continuation: a fresh accumulator at offset 0, used to drive a FULL parse from the start through the same resumable path. */
|
|
3174
|
+
function freshCodexParserState() {
|
|
3175
|
+
return serializeCodexParserState(initCodexParseState(), 0);
|
|
3176
|
+
}
|
|
3177
|
+
/**
|
|
3178
|
+
* Cheaply derive a Codex rollout's session identity — the `session_meta` id — by
|
|
3179
|
+
* streaming only the START of the file (at most `maxBytes`) and stopping once the
|
|
3180
|
+
* id is known. Mirrors {@link claudeSessionIdentityAt}: used by
|
|
3181
|
+
* {@link scanCodexSessionResumable} to confirm a grown file is still the SAME
|
|
3182
|
+
* session before resuming from a stored parse offset. Codex writes `session_meta`
|
|
3183
|
+
* (carrying the durable session UUID) on the first line of every rollout, so the
|
|
3184
|
+
* id is reached almost immediately. Returns undefined when no id appears within
|
|
3185
|
+
* the budget, which forces a FULL parse.
|
|
3186
|
+
*/
|
|
3187
|
+
async function codexSessionIdentityAt(filePath, maxBytes = 1_048_576) {
|
|
3188
|
+
const state = initCodexParseState();
|
|
3189
|
+
const stream = fs.createReadStream(filePath, { start: 0, end: maxBytes - 1, encoding: 'utf-8' });
|
|
3190
|
+
const rl = readline.createInterface({ input: stream, crlfDelay: Infinity });
|
|
3191
|
+
try {
|
|
3192
|
+
for await (const line of rl) {
|
|
3193
|
+
if (!line.trim())
|
|
3194
|
+
continue;
|
|
3195
|
+
let parsed;
|
|
3196
|
+
try {
|
|
3197
|
+
parsed = JSON.parse(line);
|
|
3198
|
+
}
|
|
3199
|
+
catch {
|
|
3200
|
+
continue;
|
|
3201
|
+
}
|
|
3202
|
+
applyCodexLine(state, parsed);
|
|
3203
|
+
if (state.sessionId !== undefined)
|
|
3204
|
+
break;
|
|
3205
|
+
}
|
|
3206
|
+
}
|
|
3207
|
+
finally {
|
|
3208
|
+
rl.close();
|
|
3209
|
+
stream.destroy();
|
|
3210
|
+
}
|
|
3211
|
+
return state.sessionId;
|
|
3212
|
+
}
|
|
3213
|
+
/**
|
|
3214
|
+
* Decide full-vs-incremental for one Codex rollout and parse it uniformly,
|
|
3215
|
+
* always returning a finalized scan plus the continuation to persist. Both
|
|
3216
|
+
* branches run through the SAME reducer (via {@link scanCodexSessionIncremental}),
|
|
3217
|
+
* so an append produces a row identical to a from-scratch full reparse by
|
|
3218
|
+
* construction. INCREMENTAL when a prior continuation exists AND the file grew
|
|
3219
|
+
* past the persisted offset AND its mtime did not go backwards; FULL (from byte
|
|
3220
|
+
* 0, fresh state) otherwise (cold start, truncation/rewrite, clock rewind).
|
|
3221
|
+
*/
|
|
3222
|
+
async function scanCodexSessionResumable(filePath, prior, currentFileMtimeMs, currentFileSize, priorFileMtimeMs) {
|
|
3223
|
+
// File size + mtime cannot distinguish an APPEND from an in-place rewrite or a
|
|
3224
|
+
// restore that dropped a DIFFERENT, larger rollout at the same path: both grow
|
|
3225
|
+
// the file and move mtime forward. Resuming from the stored offset across that
|
|
3226
|
+
// boundary would fold the new session's bytes into an accumulator hydrated from
|
|
3227
|
+
// the OLD session, so the persisted row silently diverges from a full reparse.
|
|
3228
|
+
// So the metadata gate below only makes a file ELIGIBLE; before trusting the
|
|
3229
|
+
// offset we re-read the rollout's `session_meta` id and require it to still
|
|
3230
|
+
// match the prior continuation's. An append keeps that id; a rewrite/restore of
|
|
3231
|
+
// a different session changes it. A mismatch — or an id we cannot derive —
|
|
3232
|
+
// falls back to a FULL parse, which is always correct. (A shrink is already
|
|
3233
|
+
// handled: currentFileSize is not > prior.offset, so it takes the FULL branch.)
|
|
3234
|
+
let canIncrement = false;
|
|
3235
|
+
if (prior !== null &&
|
|
3236
|
+
currentFileSize > prior.offset &&
|
|
3237
|
+
(priorFileMtimeMs === undefined || currentFileMtimeMs >= priorFileMtimeMs) &&
|
|
3238
|
+
prior.sessionId !== undefined) {
|
|
3239
|
+
canIncrement = (await codexSessionIdentityAt(filePath)) === prior.sessionId;
|
|
3240
|
+
}
|
|
3241
|
+
if (canIncrement && prior !== null) {
|
|
3242
|
+
const result = await scanCodexSessionIncremental(filePath, prior.offset, prior);
|
|
3243
|
+
return { ...result, mode: 'incremental' };
|
|
3244
|
+
}
|
|
3245
|
+
const result = await scanCodexSessionIncremental(filePath, 0, freshCodexParserState());
|
|
3246
|
+
return { ...result, mode: 'full' };
|
|
3247
|
+
}
|
|
3248
|
+
/**
|
|
3249
|
+
* Parse the prior continuation blob for a changed Codex file into a usable
|
|
3250
|
+
* {@link CodexParserState}, or null when there is none / it is unusable. A blob
|
|
3251
|
+
* from a different serialization version is treated as absent so the file falls
|
|
3252
|
+
* back to a clean FULL parse rather than resuming against a stale shape.
|
|
3253
|
+
*/
|
|
3254
|
+
function parsePriorCodexState(row) {
|
|
3255
|
+
if (!row?.parserState)
|
|
3256
|
+
return null;
|
|
3257
|
+
try {
|
|
3258
|
+
const parsed = JSON.parse(row.parserState);
|
|
3259
|
+
if (parsed?.v !== 1 || typeof parsed.offset !== 'number')
|
|
3260
|
+
return null;
|
|
3261
|
+
return parsed;
|
|
3262
|
+
}
|
|
3263
|
+
catch {
|
|
3264
|
+
return null;
|
|
3265
|
+
}
|
|
3266
|
+
}
|
|
3267
|
+
/** Test seam: how many times the incremental (append-resume) branch was taken since the last reset. */
|
|
3268
|
+
let codexIncrementalScanCount = 0;
|
|
3269
|
+
/** Test seam: how many times a full (from-offset-0) Codex parse ran since the last reset. */
|
|
3270
|
+
let codexFullScanCount = 0;
|
|
3271
|
+
/** Test seam: read the (incremental, full) Codex parse counters. */
|
|
3272
|
+
export function __codexScanBranchCountsForTest() {
|
|
3273
|
+
return { incremental: codexIncrementalScanCount, full: codexFullScanCount };
|
|
3274
|
+
}
|
|
3275
|
+
/** Test seam: reset the Codex parse-branch counters to observe a scan from a clean slate. */
|
|
3276
|
+
export function __resetCodexScanBranchCountsForTest() {
|
|
3277
|
+
codexIncrementalScanCount = 0;
|
|
3278
|
+
codexFullScanCount = 0;
|
|
3279
|
+
}
|
|
2320
3280
|
/** Resolve the working directory for an OpenClaw agent from its workspace config. */
|
|
2321
3281
|
function getOpenClawSessionCwd(agentId) {
|
|
2322
3282
|
const workspace = agentId ? getOpenClawWorkspaceMap().get(agentId) : undefined;
|
|
@@ -2576,16 +3536,20 @@ async function scanKimiIncremental(onProgress) {
|
|
|
2576
3536
|
if (changed.length === 0)
|
|
2577
3537
|
return;
|
|
2578
3538
|
onProgress?.({ agent: 'kimi', parsed: 0, total: changed.length });
|
|
3539
|
+
// Bulk-fetch each changed session's prior wire-parse continuation (offset +
|
|
3540
|
+
// counter bases). A session whose wire.jsonl grew resumes from the offset;
|
|
3541
|
+
// everything else (cold start, truncation) full-parses from byte 0.
|
|
3542
|
+
const priorStates = getParserStatesForPaths(changed.map(c => c.filePath));
|
|
2579
3543
|
const scanEntries = [];
|
|
2580
3544
|
const touched = [];
|
|
2581
3545
|
const seen = new Set();
|
|
2582
3546
|
let parsed = 0;
|
|
2583
3547
|
for (const { filePath, scan } of changed) {
|
|
2584
3548
|
try {
|
|
2585
|
-
const result = readKimiMeta(filePath);
|
|
3549
|
+
const result = readKimiMeta(filePath, priorStates.get(filePath));
|
|
2586
3550
|
if (result && !seen.has(result.meta.id)) {
|
|
2587
3551
|
seen.add(result.meta.id);
|
|
2588
|
-
scanEntries.push({ meta: result.meta, content: result.content, scan });
|
|
3552
|
+
scanEntries.push({ meta: result.meta, content: result.content, scan, parserState: result.parserState });
|
|
2589
3553
|
}
|
|
2590
3554
|
else {
|
|
2591
3555
|
touched.push({ filePath, scan });
|
|
@@ -2601,7 +3565,7 @@ async function scanKimiIncremental(onProgress) {
|
|
|
2601
3565
|
recordScans(touched);
|
|
2602
3566
|
}
|
|
2603
3567
|
/** Parse a single Kimi session state.json file to extract session metadata. */
|
|
2604
|
-
export function readKimiMeta(filePath) {
|
|
3568
|
+
export function readKimiMeta(filePath, priorRow) {
|
|
2605
3569
|
let state;
|
|
2606
3570
|
try {
|
|
2607
3571
|
state = JSON.parse(fs.readFileSync(filePath, 'utf-8'));
|
|
@@ -2626,7 +3590,7 @@ export function readKimiMeta(filePath) {
|
|
|
2626
3590
|
const stat = safeStatSync(filePath);
|
|
2627
3591
|
const timestamp = updatedAt || createdAt
|
|
2628
3592
|
|| (stat ? stat.mtime.toISOString() : new Date().toISOString());
|
|
2629
|
-
const shortId = sessionId
|
|
3593
|
+
const shortId = deriveShortId(sessionId, /^session_/);
|
|
2630
3594
|
// Try to infer project from session directory path
|
|
2631
3595
|
// ~/.kimi-code/sessions/<workdir_hash>/session_<uuid>/
|
|
2632
3596
|
const workDirName = path.basename(path.dirname(sessionDir));
|
|
@@ -2637,8 +3601,11 @@ export function readKimiMeta(filePath) {
|
|
|
2637
3601
|
project = parts.slice(0, -1).join('/');
|
|
2638
3602
|
}
|
|
2639
3603
|
}
|
|
2640
|
-
// Parse wire.jsonl
|
|
2641
|
-
|
|
3604
|
+
// Parse wire.jsonl incrementally: resume from the persisted offset + counter
|
|
3605
|
+
// bases when the wire grew, else full-parse from byte 0. The continuation is
|
|
3606
|
+
// persisted on this session's state.json ledger row.
|
|
3607
|
+
const prior = parsePriorKimiState(priorRow);
|
|
3608
|
+
const { messageCount, tokenCount, outputTokens, newState } = parseKimiWireMetricsIncremental(sessionDir, prior);
|
|
2642
3609
|
const meta = {
|
|
2643
3610
|
id: sessionId,
|
|
2644
3611
|
shortId,
|
|
@@ -2651,46 +3618,259 @@ export function readKimiMeta(filePath) {
|
|
|
2651
3618
|
tokenCount: tokenCount > 0 ? tokenCount : undefined,
|
|
2652
3619
|
outputTokens: outputTokens > 0 ? outputTokens : undefined,
|
|
2653
3620
|
};
|
|
2654
|
-
return { meta, content: lastPrompt || '' };
|
|
3621
|
+
return { meta, content: lastPrompt || '', parserState: JSON.stringify(newState) };
|
|
2655
3622
|
}
|
|
2656
|
-
/**
|
|
2657
|
-
|
|
2658
|
-
|
|
2659
|
-
|
|
2660
|
-
|
|
3623
|
+
/** Fold one parsed Kimi wire event into the additive counters, in place. */
|
|
3624
|
+
function applyKimiWireEvent(acc, event) {
|
|
3625
|
+
if (event.type === 'context.append_message') {
|
|
3626
|
+
acc.messageCount++;
|
|
3627
|
+
}
|
|
3628
|
+
else if (event.type === 'usage.record' && event.usage) {
|
|
3629
|
+
// Kimi usage structure: inputOther + output + inputCacheRead + inputCacheCreation
|
|
3630
|
+
const u = event.usage;
|
|
3631
|
+
acc.tokenCount += (u.inputOther || 0) + (u.output || 0) + (u.inputCacheRead || 0) + (u.inputCacheCreation || 0);
|
|
3632
|
+
acc.outputTokens += (u.output || 0);
|
|
3633
|
+
}
|
|
3634
|
+
}
|
|
3635
|
+
/**
|
|
3636
|
+
* Incrementally parse Kimi's wire.jsonl for message-count and token counters,
|
|
3637
|
+
* resuming from a persisted continuation instead of re-reading from byte 0 every
|
|
3638
|
+
* scan. Returns the finalized counters and the next {@link KimiParserState} to
|
|
3639
|
+
* persist (offset + the three counter bases).
|
|
3640
|
+
*
|
|
3641
|
+
* Same trailing-line discipline as {@link scanClaudeSessionIncremental}: read
|
|
3642
|
+
* only the appended byte range from `prior.offset`, apply ONLY the run of
|
|
3643
|
+
* newline-terminated lines (slice at the last `'\n'`), and advance the offset to
|
|
3644
|
+
* `prior.offset + consumedBytes`. A complete-but-not-yet-terminated last record
|
|
3645
|
+
* is DEFERRED to the next pass; because these counters are additive with no
|
|
3646
|
+
* dedup, re-reading such a line would double-count it. FULL parse from byte 0
|
|
3647
|
+
* (fresh counters) when there is no prior OR the file shrank below the stored
|
|
3648
|
+
* offset (truncation/rewrite).
|
|
3649
|
+
*/
|
|
3650
|
+
export function parseKimiWireMetricsIncremental(sessionDir, prior) {
|
|
2661
3651
|
const wirePath = path.join(sessionDir, 'agents', 'main', 'wire.jsonl');
|
|
2662
|
-
|
|
2663
|
-
|
|
2664
|
-
|
|
2665
|
-
|
|
2666
|
-
return { messageCount: 0, tokenCount: 0, outputTokens: 0 };
|
|
3652
|
+
const stat = safeStatSync(wirePath);
|
|
3653
|
+
if (!stat) {
|
|
3654
|
+
// No wire.jsonl (yet): zero counters, offset 0 so a later append is a clean
|
|
3655
|
+
// full parse.
|
|
3656
|
+
return { messageCount: 0, tokenCount: 0, outputTokens: 0, newState: { v: 1, offset: 0, messageCount: 0, tokenCount: 0, outputTokens: 0 } };
|
|
3657
|
+
}
|
|
3658
|
+
// INCREMENTAL only when a usable prior exists AND the file grew past its
|
|
3659
|
+
// offset; otherwise FULL from byte 0 with fresh counters (cold start OR the
|
|
3660
|
+
// file shrank to/below the offset — a truncation/rewrite).
|
|
3661
|
+
//
|
|
3662
|
+
// No session-identity re-check is needed here (unlike Claude's
|
|
3663
|
+
// claudeSessionIdentityAt / Codex's codexSessionIdentityAt, which guard against
|
|
3664
|
+
// an in-place rewrite dropping a DIFFERENT session at the same path). A Kimi
|
|
3665
|
+
// wire.jsonl is uniquely keyed by its session dir — `.../session_<uuid>/agents/
|
|
3666
|
+
// main/wire.jsonl` (see readKimiMeta: sessionId is `session_<uuid>` and must
|
|
3667
|
+
// start with `session_`) — and Kimi only ever APPENDS to that per-session log.
|
|
3668
|
+
// The path therefore cannot host a different session's transcript, so a
|
|
3669
|
+
// size-grew wire.jsonl is always the same session's append. (A truncation/
|
|
3670
|
+
// rewrite — the only way its bytes could diverge — already shrinks it to/below
|
|
3671
|
+
// the offset and takes the FULL branch above.)
|
|
3672
|
+
const canIncrement = prior !== null && stat.size > prior.offset;
|
|
3673
|
+
const fromOffset = canIncrement ? prior.offset : 0;
|
|
3674
|
+
const acc = canIncrement
|
|
3675
|
+
? { messageCount: prior.messageCount, tokenCount: prior.tokenCount, outputTokens: prior.outputTokens }
|
|
3676
|
+
: { messageCount: 0, tokenCount: 0, outputTokens: 0 };
|
|
3677
|
+
let consumedBytes = 0;
|
|
3678
|
+
let fd;
|
|
3679
|
+
try {
|
|
3680
|
+
// Read ONLY the appended byte range [fromOffset, stat.size) — not the whole
|
|
3681
|
+
// file. readSync from an explicit position keeps this function synchronous
|
|
3682
|
+
// (its callers are sync) while making the disk read + allocation scale with
|
|
3683
|
+
// the appended delta, not total file size, matching scanCodexSessionIncremental
|
|
3684
|
+
// / scanClaudeSessionIncremental. Bytes past the stat'd size are a concurrent
|
|
3685
|
+
// append and are deferred to the next scan.
|
|
3686
|
+
const bytesToRead = Math.max(0, stat.size - fromOffset);
|
|
3687
|
+
const appended = Buffer.allocUnsafe(bytesToRead);
|
|
3688
|
+
if (bytesToRead > 0) {
|
|
3689
|
+
fd = fs.openSync(wirePath, 'r');
|
|
3690
|
+
let read = 0;
|
|
3691
|
+
while (read < bytesToRead) {
|
|
3692
|
+
const n = fs.readSync(fd, appended, read, bytesToRead - read, fromOffset + read);
|
|
3693
|
+
if (n <= 0)
|
|
3694
|
+
break;
|
|
3695
|
+
read += n;
|
|
3696
|
+
}
|
|
3697
|
+
const chunk = read === bytesToRead ? appended : appended.subarray(0, read);
|
|
3698
|
+
// Bytes up to AND INCLUDING the last '\n' are the committed, complete-line run.
|
|
3699
|
+
const lastNl = chunk.lastIndexOf(0x0a);
|
|
3700
|
+
consumedBytes = lastNl === -1 ? 0 : lastNl + 1;
|
|
3701
|
+
if (consumedBytes > 0) {
|
|
3702
|
+
for (const line of chunk.subarray(0, consumedBytes).toString('utf-8').split('\n')) {
|
|
3703
|
+
if (!line.trim())
|
|
3704
|
+
continue;
|
|
3705
|
+
try {
|
|
3706
|
+
applyKimiWireEvent(acc, JSON.parse(line));
|
|
3707
|
+
}
|
|
3708
|
+
catch {
|
|
3709
|
+
// Malformed line, skip
|
|
3710
|
+
}
|
|
3711
|
+
}
|
|
3712
|
+
}
|
|
3713
|
+
}
|
|
3714
|
+
}
|
|
3715
|
+
catch {
|
|
3716
|
+
// If wire.jsonl can't be read, keep the accumulated counters (0s on a cold
|
|
3717
|
+
// parse) — graceful degradation, matching the pre-incremental behavior.
|
|
2667
3718
|
}
|
|
3719
|
+
finally {
|
|
3720
|
+
if (fd !== undefined) {
|
|
3721
|
+
try {
|
|
3722
|
+
fs.closeSync(fd);
|
|
3723
|
+
}
|
|
3724
|
+
catch { /* already closed / gone */ }
|
|
3725
|
+
}
|
|
3726
|
+
}
|
|
3727
|
+
return {
|
|
3728
|
+
messageCount: acc.messageCount,
|
|
3729
|
+
tokenCount: acc.tokenCount,
|
|
3730
|
+
outputTokens: acc.outputTokens,
|
|
3731
|
+
newState: { v: 1, offset: fromOffset + consumedBytes, messageCount: acc.messageCount, tokenCount: acc.tokenCount, outputTokens: acc.outputTokens },
|
|
3732
|
+
};
|
|
3733
|
+
}
|
|
3734
|
+
/**
|
|
3735
|
+
* Parse the prior continuation blob for a changed Kimi session into a usable
|
|
3736
|
+
* {@link KimiParserState}, or null when there is none / it is unusable. A blob
|
|
3737
|
+
* from a different serialization version is treated as absent so the wire parse
|
|
3738
|
+
* falls back to a clean FULL parse rather than resuming against a stale shape.
|
|
3739
|
+
*/
|
|
3740
|
+
function parsePriorKimiState(row) {
|
|
3741
|
+
if (!row?.parserState)
|
|
3742
|
+
return null;
|
|
2668
3743
|
try {
|
|
2669
|
-
const
|
|
2670
|
-
|
|
2671
|
-
|
|
3744
|
+
const parsed = JSON.parse(row.parserState);
|
|
3745
|
+
if (parsed?.v !== 1 || typeof parsed.offset !== 'number')
|
|
3746
|
+
return null;
|
|
3747
|
+
return parsed;
|
|
3748
|
+
}
|
|
3749
|
+
catch {
|
|
3750
|
+
return null;
|
|
3751
|
+
}
|
|
3752
|
+
}
|
|
3753
|
+
/**
|
|
3754
|
+
* Scan Grok sessions. Grok stores one directory per session under
|
|
3755
|
+
* ~/.grok/sessions/<url-encoded-cwd>/<uuid>/, each holding a summary.json with
|
|
3756
|
+
* structured metadata (id, cwd, title, timestamps, message count). Same
|
|
3757
|
+
* dir-per-session (L3) shape as Kimi, so it walks two levels and gates the
|
|
3758
|
+
* summary.json read through the scan ledger. Before this, Grok had a type slot
|
|
3759
|
+
* and a placeholder parser but no scanner, so `agents sessions` never indexed it.
|
|
3760
|
+
*/
|
|
3761
|
+
async function scanGrokIncremental(onProgress) {
|
|
3762
|
+
const currentVersion = await getCurrentAgentVersion('grok');
|
|
3763
|
+
const filePaths = [];
|
|
3764
|
+
for (const sessionsDir of getAgentSessionDirs('grok', 'sessions')) {
|
|
3765
|
+
if (!fs.existsSync(sessionsDir))
|
|
3766
|
+
continue;
|
|
3767
|
+
let cwdDirNames;
|
|
3768
|
+
try {
|
|
3769
|
+
cwdDirNames = fs.readdirSync(sessionsDir);
|
|
3770
|
+
}
|
|
3771
|
+
catch {
|
|
3772
|
+
continue;
|
|
3773
|
+
}
|
|
3774
|
+
for (const cwdDirName of cwdDirNames) {
|
|
3775
|
+
const cwdDir = path.join(sessionsDir, cwdDirName);
|
|
3776
|
+
const stat = safeStatSync(cwdDir);
|
|
3777
|
+
if (!stat?.isDirectory())
|
|
2672
3778
|
continue;
|
|
3779
|
+
let sessionNames;
|
|
2673
3780
|
try {
|
|
2674
|
-
|
|
2675
|
-
if (event.type === 'context.append_message') {
|
|
2676
|
-
messageCount++;
|
|
2677
|
-
}
|
|
2678
|
-
else if (event.type === 'usage.record' && event.usage) {
|
|
2679
|
-
// Kimi usage structure: inputOther + output + inputCacheRead + inputCacheCreation
|
|
2680
|
-
const u = event.usage;
|
|
2681
|
-
tokenCount += (u.inputOther || 0) + (u.output || 0) + (u.inputCacheRead || 0) + (u.inputCacheCreation || 0);
|
|
2682
|
-
outputTokens += (u.output || 0);
|
|
2683
|
-
}
|
|
3781
|
+
sessionNames = fs.readdirSync(cwdDir);
|
|
2684
3782
|
}
|
|
2685
3783
|
catch {
|
|
2686
|
-
|
|
3784
|
+
continue;
|
|
3785
|
+
}
|
|
3786
|
+
for (const sessionName of sessionNames) {
|
|
3787
|
+
const summaryPath = path.join(cwdDir, sessionName, 'summary.json');
|
|
3788
|
+
if (!fs.existsSync(summaryPath))
|
|
3789
|
+
continue;
|
|
3790
|
+
filePaths.push(summaryPath);
|
|
2687
3791
|
}
|
|
2688
3792
|
}
|
|
2689
3793
|
}
|
|
3794
|
+
const changed = filterChangedFiles(filePaths);
|
|
3795
|
+
if (changed.length === 0)
|
|
3796
|
+
return;
|
|
3797
|
+
onProgress?.({ agent: 'grok', parsed: 0, total: changed.length });
|
|
3798
|
+
const scanEntries = [];
|
|
3799
|
+
const touched = [];
|
|
3800
|
+
const seen = new Set();
|
|
3801
|
+
let parsed = 0;
|
|
3802
|
+
for (const { filePath, scan } of changed) {
|
|
3803
|
+
try {
|
|
3804
|
+
const result = readGrokMeta(filePath, currentVersion);
|
|
3805
|
+
if (result && !seen.has(result.meta.id)) {
|
|
3806
|
+
seen.add(result.meta.id);
|
|
3807
|
+
scanEntries.push({ meta: result.meta, content: result.content, scan });
|
|
3808
|
+
}
|
|
3809
|
+
else {
|
|
3810
|
+
touched.push({ filePath, scan });
|
|
3811
|
+
}
|
|
3812
|
+
}
|
|
3813
|
+
catch {
|
|
3814
|
+
touched.push({ filePath, scan });
|
|
3815
|
+
}
|
|
3816
|
+
parsed++;
|
|
3817
|
+
onProgress?.({ agent: 'grok', parsed, total: changed.length });
|
|
3818
|
+
}
|
|
3819
|
+
upsertSessionsBatch(scanEntries);
|
|
3820
|
+
recordScans(touched);
|
|
3821
|
+
}
|
|
3822
|
+
/** Parse a single Grok session summary.json into session metadata. */
|
|
3823
|
+
export function readGrokMeta(filePath, currentVersion) {
|
|
3824
|
+
let summary;
|
|
3825
|
+
try {
|
|
3826
|
+
summary = JSON.parse(fs.readFileSync(filePath, 'utf-8'));
|
|
3827
|
+
}
|
|
2690
3828
|
catch {
|
|
2691
|
-
|
|
3829
|
+
return null;
|
|
2692
3830
|
}
|
|
2693
|
-
|
|
3831
|
+
const sessionDir = path.dirname(filePath);
|
|
3832
|
+
// The uuid directory name is the canonical id; summary.info.id mirrors it.
|
|
3833
|
+
const sessionId = (typeof summary?.info?.id === 'string' && summary.info.id) || path.basename(sessionDir);
|
|
3834
|
+
if (!sessionId)
|
|
3835
|
+
return null;
|
|
3836
|
+
const cwd = normalizeCwd(typeof summary?.info?.cwd === 'string' ? summary.info.cwd : '');
|
|
3837
|
+
const topic = (typeof summary?.generated_title === 'string' && summary.generated_title.trim()) ||
|
|
3838
|
+
(typeof summary?.session_summary === 'string' && summary.session_summary.trim()) ||
|
|
3839
|
+
undefined;
|
|
3840
|
+
// created_at is the session start; last_active_at/updated_at is the latest
|
|
3841
|
+
// activity. Coerce timestamp to never-null (NOT NULL column) via the file mtime,
|
|
3842
|
+
// matching how the other dir-per-session parsers (Kimi) fall back.
|
|
3843
|
+
const createdAt = typeof summary?.created_at === 'string' ? summary.created_at : undefined;
|
|
3844
|
+
const lastActivity = (typeof summary?.last_active_at === 'string' && summary.last_active_at) ||
|
|
3845
|
+
(typeof summary?.updated_at === 'string' && summary.updated_at) ||
|
|
3846
|
+
undefined;
|
|
3847
|
+
const stat = safeStatSync(filePath);
|
|
3848
|
+
const timestamp = createdAt || lastActivity || (stat ? stat.mtime.toISOString() : new Date().toISOString());
|
|
3849
|
+
const messageCount = typeof summary?.num_chat_messages === 'number'
|
|
3850
|
+
? summary.num_chat_messages
|
|
3851
|
+
: typeof summary?.num_messages === 'number'
|
|
3852
|
+
? summary.num_messages
|
|
3853
|
+
: undefined;
|
|
3854
|
+
// Grok records its managed home in summary.grok_home
|
|
3855
|
+
// (…/versions/grok/<version>/home/.grok) — recover the version from it.
|
|
3856
|
+
let embeddedVersion;
|
|
3857
|
+
if (typeof summary?.grok_home === 'string') {
|
|
3858
|
+
embeddedVersion = summary.grok_home.match(/versions\/grok\/([^/]+)\//)?.[1];
|
|
3859
|
+
}
|
|
3860
|
+
const meta = {
|
|
3861
|
+
id: sessionId,
|
|
3862
|
+
shortId: deriveShortId(sessionId),
|
|
3863
|
+
agent: 'grok',
|
|
3864
|
+
timestamp,
|
|
3865
|
+
lastActivity,
|
|
3866
|
+
project: cwd ? path.basename(cwd) : undefined,
|
|
3867
|
+
cwd: cwd || undefined,
|
|
3868
|
+
filePath,
|
|
3869
|
+
version: resolveSessionVersion('grok', filePath, embeddedVersion, currentVersion),
|
|
3870
|
+
topic,
|
|
3871
|
+
messageCount,
|
|
3872
|
+
};
|
|
3873
|
+
return { meta, content: topic || '' };
|
|
2694
3874
|
}
|
|
2695
3875
|
/** Parse a time filter string (relative like '7d' or ISO timestamp) into epoch milliseconds. */
|
|
2696
3876
|
export function parseTimeFilter(input) {
|