@phnx-labs/agents-cli 1.22.25 → 1.22.26

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (82) hide show
  1. package/CHANGELOG.md +183 -0
  2. package/README.md +17 -2
  3. package/dist/bin/agents +0 -0
  4. package/dist/browser.js +14 -4
  5. package/dist/commands/apply.js +52 -8
  6. package/dist/commands/browser.js +35 -0
  7. package/dist/commands/doctor.js +8 -0
  8. package/dist/commands/insights.d.ts +25 -19
  9. package/dist/commands/insights.js +107 -33
  10. package/dist/commands/reconnect.d.ts +46 -0
  11. package/dist/commands/reconnect.js +109 -0
  12. package/dist/commands/routines.js +2 -2
  13. package/dist/commands/secrets.d.ts +2 -8
  14. package/dist/commands/secrets.js +29 -105
  15. package/dist/commands/sessions.js +4 -0
  16. package/dist/commands/setup-secrets.d.ts +1 -0
  17. package/dist/commands/setup-secrets.js +1 -1
  18. package/dist/commands/setup.d.ts +26 -3
  19. package/dist/commands/setup.js +105 -46
  20. package/dist/commands/teams.d.ts +6 -0
  21. package/dist/commands/teams.js +43 -0
  22. package/dist/commands/trends.d.ts +8 -0
  23. package/dist/commands/trends.js +10 -156
  24. package/dist/index.js +1 -1
  25. package/dist/lib/agents.d.ts +11 -0
  26. package/dist/lib/agents.js +29 -2
  27. package/dist/lib/analytics/dashboard.d.ts +10 -6
  28. package/dist/lib/analytics/dashboard.js +6 -4
  29. package/dist/lib/analytics/mix-commands.d.ts +53 -0
  30. package/dist/lib/analytics/mix-commands.js +229 -0
  31. package/dist/lib/analytics/recipes.d.ts +19 -14
  32. package/dist/lib/analytics/recipes.js +4 -2
  33. package/dist/lib/browser/ipc.d.ts +26 -0
  34. package/dist/lib/browser/ipc.js +139 -24
  35. package/dist/lib/browser/profiles.d.ts +11 -0
  36. package/dist/lib/browser/profiles.js +1 -1
  37. package/dist/lib/browser/stream.d.ts +14 -0
  38. package/dist/lib/browser/stream.js +71 -0
  39. package/dist/lib/channels/owner-sink.d.ts +27 -0
  40. package/dist/lib/channels/owner-sink.js +93 -0
  41. package/dist/lib/devices/doctor-findings.d.ts +7 -1
  42. package/dist/lib/devices/doctor-findings.js +33 -1
  43. package/dist/lib/fleet/apply.d.ts +59 -3
  44. package/dist/lib/fleet/apply.js +183 -6
  45. package/dist/lib/fleet/types.d.ts +21 -2
  46. package/dist/lib/hooks/cache.js +15 -0
  47. package/dist/lib/hosts/passthrough.d.ts +23 -0
  48. package/dist/lib/hosts/passthrough.js +45 -0
  49. package/dist/lib/hosts/ready.d.ts +2 -0
  50. package/dist/lib/hosts/ready.js +10 -1
  51. package/dist/lib/hosts/reconnect.d.ts +14 -12
  52. package/dist/lib/hosts/reconnect.js +41 -40
  53. package/dist/lib/menubar/MenubarHelper.app/Contents/CodeResources +0 -0
  54. package/dist/lib/menubar/MenubarHelper.app/Contents/MacOS/MenubarHelper +0 -0
  55. package/dist/lib/routines.js +14 -2
  56. package/dist/lib/runner.d.ts +0 -3
  57. package/dist/lib/runner.js +1 -14
  58. package/dist/lib/secrets/Agents CLI.app/Contents/CodeResources +0 -0
  59. package/dist/lib/secrets/Agents CLI.app/Contents/MacOS/Agents CLI +0 -0
  60. package/dist/lib/secrets/push.d.ts +94 -0
  61. package/dist/lib/secrets/push.js +145 -0
  62. package/dist/lib/secrets/reaper.d.ts +15 -1
  63. package/dist/lib/secrets/reaper.js +30 -3
  64. package/dist/lib/session/db.d.ts +21 -3
  65. package/dist/lib/session/db.js +221 -13
  66. package/dist/lib/session/discover.d.ts +1 -0
  67. package/dist/lib/session/discover.js +115 -19
  68. package/dist/lib/session/insights.d.ts +18 -0
  69. package/dist/lib/session/insights.js +143 -1
  70. package/dist/lib/session/tool-index.js +133 -22
  71. package/dist/lib/session/tool-store.d.ts +26 -2
  72. package/dist/lib/session/tool-store.js +36 -17
  73. package/dist/lib/ssh-exec.js +8 -2
  74. package/dist/lib/startup/command-registry.d.ts +1 -0
  75. package/dist/lib/startup/command-registry.js +4 -0
  76. package/dist/lib/teams/agents.d.ts +13 -0
  77. package/dist/lib/teams/agents.js +75 -7
  78. package/dist/lib/teams/placement-probe.d.ts +21 -0
  79. package/dist/lib/teams/placement-probe.js +135 -0
  80. package/dist/lib/teams/scheduler.d.ts +74 -1
  81. package/dist/lib/teams/scheduler.js +187 -10
  82. package/package.json +1 -1
@@ -0,0 +1,145 @@
1
+ /**
2
+ * Push one bundle's values to a remote host over SSH — the provisioning
3
+ * primitive behind `agents secrets export --host` and, from RUSH-1968, behind
4
+ * `agents fleet apply --provision-secrets`.
5
+ *
6
+ * This logic used to live inline in the `export --host` command action. It moved
7
+ * here because `lib/fleet/apply.ts` needs it and a lib MUST NOT import a command
8
+ * module — and because the absence of a callable primitive is part of why
9
+ * `fleet apply` never provisioned secrets at all, leaving an operator to
10
+ * hand-export the file store's master key across the fleet.
11
+ *
12
+ * Two rules shape the shape of this module:
13
+ *
14
+ * - **It never prints.** The lib layer stays `console.*`-free (SEC-14/SEC-16), so
15
+ * every outcome is returned as data and the CALLER renders it. That is also
16
+ * what lets `fleet apply` fold a push into its own per-device report instead of
17
+ * interleaving stray lines into it.
18
+ * - **Resolve once, push N times.** `resolveBundleForPush` is separate because
19
+ * reading a bundle can prompt (Touch ID); doing it per host would prompt per
20
+ * host. `export --host a,b,c` resolves once and pushes three times.
21
+ */
22
+ import { sshExec } from '../ssh-exec.js';
23
+ import { remoteShellFor, buildWindowsStdinImportCommand } from '../hosts/remote-cmd.js';
24
+ import { resolveRemoteOsSync } from '../hosts/remote-os.js';
25
+ import { remoteSecretsRaw, verifyRemoteKeychainPush, keychainWriteFailureMessage, buildRemoteFileImportCommand, } from './remote.js';
26
+ import { readAndResolveBundleEnv } from './bundles.js';
27
+ /**
28
+ * Serialize a resolved env map to `.env` lines that round-trip losslessly through
29
+ * `parseDotenv` on the remote: `KEY="VALUE"`. parseDotenv strips exactly one outer
30
+ * quote pair and takes the inner bytes verbatim (no unescaping), so any single-line
31
+ * value survives unchanged with no escaping. Newlines would break its line-based
32
+ * parse, so multi-line values are rejected rather than silently corrupted.
33
+ */
34
+ export function bundleEnvToDotenv(env) {
35
+ const lines = [];
36
+ for (const [k, v] of Object.entries(env)) {
37
+ if (/[\r\n]/.test(v)) {
38
+ throw new Error(`Key '${k}' has a multi-line value; the SSH .env transport can't carry newlines. ` +
39
+ `Set it directly on the remote with 'agents secrets add ${k} --value-stdin'.`);
40
+ }
41
+ lines.push(`${k}="${v}"`);
42
+ }
43
+ return lines.join('\n') + '\n';
44
+ }
45
+ /**
46
+ * Read and resolve a bundle once, for pushing to one or more hosts.
47
+ *
48
+ * `agentOnly` + `keyMode: 'storage'` match what `export --host` has always
49
+ * passed: storage-shaped values, and the headless guard that fails fast rather
50
+ * than popping Touch ID inside an automated run.
51
+ */
52
+ export function resolveBundleForPush(bundle, caller) {
53
+ const { env } = readAndResolveBundleEnv(bundle, { caller, keyMode: 'storage', agentOnly: true });
54
+ return { env, dotenv: bundleEnvToDotenv(env), keyCount: Object.keys(env).length };
55
+ }
56
+ function isPowershellTarget(host) {
57
+ return remoteShellFor(resolveRemoteOsSync(host.split('@').pop() ?? host)) === 'powershell';
58
+ }
59
+ /** Choose the transport for one push. Pure: registry read in, plan out. */
60
+ export function planPushTransport(resolved, bundle, host, opts) {
61
+ const powershell = isPowershellTarget(host);
62
+ if (opts.remoteBackend === 'file') {
63
+ // Both file-backend paths build a POSIX `bash -lc` command. Refuse a Windows
64
+ // target cleanly rather than emit broken PowerShell (fail loud at the
65
+ // boundary, never a silent wrong path).
66
+ if (powershell) {
67
+ return { kind: 'refuse', message: 'file backend export to a Windows target is not yet supported' };
68
+ }
69
+ const { remoteCmd, input } = buildRemoteFileImportCommand(bundle, resolved.dotenv, {
70
+ passphrase: opts.passphrase ?? '',
71
+ force: opts.force,
72
+ });
73
+ return { kind: 'ssh', remoteCmd, input };
74
+ }
75
+ if (powershell) {
76
+ // Keychain on a Windows target: the `agents.ps1` shim doesn't forward
77
+ // ssh-piped stdin to node, so `--from -` would hang. Bridge the piped .env
78
+ // through PowerShell into a temp file and import `--from <file>` (deleted
79
+ // afterwards). Same hardened ssh engine; the .env still only ever crosses
80
+ // the wire over ssh stdin.
81
+ return {
82
+ kind: 'ssh',
83
+ remoteCmd: buildWindowsStdinImportCommand(bundle, { force: opts.force }),
84
+ input: resolved.dotenv,
85
+ };
86
+ }
87
+ // Keychain on a POSIX target: OS-aware wrapping + the hardened ssh engine
88
+ // (BatchMode, ConnectTimeout, keepalive, control-socket reuse) via the same
89
+ // path the READ inverse (`remoteResolveEnv`) uses.
90
+ return {
91
+ kind: 'remote-secrets',
92
+ args: ['import', bundle, '--from', '-', ...(opts.force ? ['--force'] : [])],
93
+ input: resolved.dotenv,
94
+ };
95
+ }
96
+ /**
97
+ * Push an already-resolved bundle to ONE host.
98
+ *
99
+ * Drives the remote's own `agents secrets import --from -`, so the values land
100
+ * in the remote's chosen backend and the .env is read off ssh stdin rather than
101
+ * parsed by a remote shell. `import` auto-creates the bundle.
102
+ */
103
+ export function pushResolvedBundleToHost(resolved, bundle, host, opts) {
104
+ const fail = (message) => ({ ok: false, host, bundle, keyCount: resolved.keyCount, message });
105
+ const plan = planPushTransport(resolved, bundle, host, opts);
106
+ if (plan.kind === 'refuse')
107
+ return fail(plan.message);
108
+ const res = plan.kind === 'ssh'
109
+ ? sshExec(host, plan.remoteCmd, { input: plan.input })
110
+ : remoteSecretsRaw(host, plan.args, { input: plan.input, osLookupName: host });
111
+ if (res.code === null) {
112
+ return fail(res.stderr.trim() || (res.timedOut ? 'ssh timed out' : 'ssh failed'));
113
+ }
114
+ if (res.code !== 0) {
115
+ const msg = (res.stderr || res.stdout || '').trim();
116
+ return fail(`remote import failed (exit ${res.code})${msg ? `: ${msg}` : ''}`);
117
+ }
118
+ // A keychain-backed push to a macOS remote over headless SSH can land the
119
+ // bundle metadata but no READABLE value items: the remote login keychain is
120
+ // locked in the non-interactive SSH context, so Security accepts the write but
121
+ // the biometry-ACL'd item is unreadable — and the remote `import` still exits
122
+ // 0. Read it back the way a release will and FAIL LOUDLY, rather than leave a
123
+ // metadata-only bundle that breaks later with "stored item not found". The
124
+ // file backend is headless-readable by construction, so it is skipped.
125
+ if (opts.remoteBackend === 'keychain') {
126
+ const verdict = verifyRemoteKeychainPush(host, bundle, Object.keys(resolved.env), { osLookupName: host });
127
+ if (!verdict.ok) {
128
+ return fail(verdict.kind === 'locked-keychain'
129
+ ? keychainWriteFailureMessage(host, bundle, verdict.reason)
130
+ : `pushed '${bundle}' but could not verify it on the remote: ${verdict.reason}`);
131
+ }
132
+ }
133
+ const remoteMsg = (res.stdout || '').trim().split('\n').map((l) => l.trim()).filter(Boolean).pop();
134
+ return {
135
+ ok: true,
136
+ host,
137
+ bundle,
138
+ keyCount: resolved.keyCount,
139
+ message: remoteMsg || `${resolved.keyCount} key(s) exported`,
140
+ };
141
+ }
142
+ /** Resolve and push in one call — for a single host. */
143
+ export function pushBundleToHost(bundle, host, opts) {
144
+ return pushResolvedBundleToHost(resolveBundleForPush(bundle, opts.operation), bundle, host, opts);
145
+ }
@@ -27,7 +27,12 @@ export interface KeychainProcessSnapshot {
27
27
  * `null` means "could not capture" — the planner must fail closed.
28
28
  */
29
29
  startTime: string | null;
30
- /** True when this process's executable path matches the installed helper. */
30
+ /**
31
+ * True when this process is a REAP-ELIGIBLE helper invocation — the installed
32
+ * helper binary running a short-lived keychain verb. False for a non-helper
33
+ * process AND for the long-lived `watch-lock` watcher (see
34
+ * {@link isReapableHelperCommand}), which must never be reaped.
35
+ */
31
36
  isHelper: boolean;
32
37
  }
33
38
  /**
@@ -78,6 +83,15 @@ export declare function planKeychainReap(snapshots: KeychainProcessSnapshot[], n
78
83
  * Returns null for an unparseable value so the caller drops the row.
79
84
  */
80
85
  export declare function parseEtimeToSeconds(raw: string): number | null;
86
+ /**
87
+ * Whether a `ps` command line is a REAP-ELIGIBLE helper invocation: the installed
88
+ * helper binary running a short-lived keychain verb (get/has/list/set/delete/
89
+ * migrate-*) that a wedged `coreauthd` can hang. Returns false for a non-helper
90
+ * command AND for the deliberately long-lived `watch-lock` watcher — matching by
91
+ * the full argv (`ps … command=`), so a live-parent `watch-lock` child is never
92
+ * mistaken for a stuck read and killed. Pure; unit-tested.
93
+ */
94
+ export declare function isReapableHelperCommand(command: string, helperPath: string): boolean;
81
95
  /** Test seam: reset the persisted candidate state. */
82
96
  export declare function resetKeychainReaperCandidatesForTest(): void;
83
97
  /**
@@ -134,6 +134,31 @@ function parsePsLine(line) {
134
134
  return null;
135
135
  return { pid, ppid, elapsedSec, command };
136
136
  }
137
+ /**
138
+ * The one helper verb that is DELIBERATELY long-lived: the broker's auto-lock
139
+ * sleep/lock watcher (`spawn(getKeychainHelperPath(), ['watch-lock'], …)` in
140
+ * `agent.ts`). It lives for the broker's whole hold — potentially days — as a
141
+ * healthy child of the live broker/daemon, emitting LOCK/SLEEP lines that wipe
142
+ * the in-memory secret store on sleep. It is NOT a stuck keychain read, so the
143
+ * reaper must never target it: killing it silently disables auto-lock-on-sleep.
144
+ */
145
+ const HELPER_WATCH_LOCK_VERB = 'watch-lock';
146
+ /**
147
+ * Whether a `ps` command line is a REAP-ELIGIBLE helper invocation: the installed
148
+ * helper binary running a short-lived keychain verb (get/has/list/set/delete/
149
+ * migrate-*) that a wedged `coreauthd` can hang. Returns false for a non-helper
150
+ * command AND for the deliberately long-lived `watch-lock` watcher — matching by
151
+ * the full argv (`ps … command=`), so a live-parent `watch-lock` child is never
152
+ * mistaken for a stuck read and killed. Pure; unit-tested.
153
+ */
154
+ export function isReapableHelperCommand(command, helperPath) {
155
+ if (command === helperPath)
156
+ return true; // bare exec, no verb — never watch-lock
157
+ if (!command.startsWith(`${helperPath} `))
158
+ return false; // not our helper
159
+ const firstArg = command.slice(helperPath.length + 1).trimStart().split(/\s+/)[0];
160
+ return firstArg !== HELPER_WATCH_LOCK_VERB;
161
+ }
137
162
  /** Module-state for the two-sweep stuck-parent debounce. */
138
163
  let stuckParentCandidates = new Map();
139
164
  /** Test seam: reset the persisted candidate state. */
@@ -178,9 +203,11 @@ export function reapOrphanedKeychainProcesses() {
178
203
  if (!parsed)
179
204
  continue;
180
205
  const { pid, ppid, elapsedSec, command } = parsed;
181
- // Exact path-match: the helper invocation's command line begins with the
182
- // absolute helper path, followed by a space and its arguments (or nothing).
183
- const isHelper = command === helperPath || command.startsWith(`${helperPath} `);
206
+ // Reap-eligible = the helper binary running a short-lived keychain verb. The
207
+ // full-argv match excludes the deliberately long-lived `watch-lock` watcher,
208
+ // whose live-parent child would otherwise be killed as if it were stuck
209
+ // (RUSH-2232 — that silently disabled auto-lock-on-sleep).
210
+ const isHelper = isReapableHelperCommand(command, helperPath);
184
211
  rows.push({ pid, ppid, elapsedSec, isHelper, startTime: null });
185
212
  }
186
213
  const rowByPid = new Map(rows.map((r) => [r.pid, r]));
@@ -7,12 +7,12 @@
7
7
  * BM25 ranking, and label-first search for /rename'd sessions.
8
8
  */
9
9
  import Database from '../sqlite.js';
10
- import type { SessionAgentId, SessionMeta } from './types.js';
10
+ import type { SessionAgentId, SessionEvent, SessionMeta } from './types.js';
11
11
  import { type IndexedToolCall } from './tool-calls.js';
12
12
  /** Current schema version; bumped when migrations are added. Exported so tests
13
13
  * assert against the constant instead of hardcoding a number that every bump
14
14
  * then has to chase (docs/05-sessions.md calls the constant the source of truth). */
15
- export declare const SCHEMA_VERSION = 34;
15
+ export declare const SCHEMA_VERSION = 36;
16
16
  /**
17
17
  * Bump to force `agents sessions backfill resources` to re-derive every
18
18
  * session's skill/slash-command tallies on its next run (resource_scan_ledger
@@ -26,7 +26,7 @@ export declare const RESOURCE_INDEX_VERSION = 1;
26
26
  * re-derives on the next `agents insights` instead of silently reporting stale
27
27
  * numbers alongside fresh ones. Same role as RESOURCE_INDEX_VERSION.
28
28
  */
29
- export declare const INSIGHTS_EXTRACTOR_VERSION = 3;
29
+ export declare const INSIGHTS_EXTRACTOR_VERSION = 4;
30
30
  /** Raw row shape returned from the sessions table. */
31
31
  export interface SessionRow {
32
32
  id: string;
@@ -149,6 +149,23 @@ export interface FtsOptimizeResult {
149
149
  * the DB file; run VACUUM (with the daemon stopped) to return it to the OS.
150
150
  */
151
151
  export declare function optimizeSessionSearchIndex(): FtsOptimizeResult[];
152
+ /**
153
+ * Keep the FTS indexes from degrading on the normal scan path.
154
+ *
155
+ * `optimizeSessionSearchIndex` is the full, unbounded compaction behind
156
+ * `agents sessions optimize`. Leaving it as the ONLY compaction meant the index
157
+ * degraded until a human happened to run that command, which is how
158
+ * `tool_call_text_data` reached gigabytes for tens of MB of content. This is the
159
+ * automatic counterpart: bounded, threshold-gated, and safe to call after every
160
+ * batch of writes. Non-destructive — merging never changes what is searchable.
161
+ *
162
+ * Returns one result per table it actually merged (empty when every table is
163
+ * under the threshold, which is the common case on a warm index).
164
+ */
165
+ export declare function maintainSessionSearchIndex(db?: Database.Database, options?: {
166
+ segmentThreshold?: number;
167
+ mergePages?: number;
168
+ }): FtsOptimizeResult[];
152
169
  /**
153
170
  * Try to claim the right to run the incremental scan. Returns true if this
154
171
  * process should proceed with scanning, false if another live process is
@@ -245,6 +262,7 @@ export declare function upsertSessionsBatch(entries: Array<{
245
262
  scan?: ScanStamp;
246
263
  parserState?: string;
247
264
  contentText?: string;
265
+ events?: SessionEvent[];
248
266
  toolCalls?: IndexedToolCall[];
249
267
  toolScan?: ScanStamp;
250
268
  toolIndexMode?: 'replace' | 'append';
@@ -26,7 +26,7 @@ const DB_PATH = getSessionsDbPath();
26
26
  /** Current schema version; bumped when migrations are added. Exported so tests
27
27
  * assert against the constant instead of hardcoding a number that every bump
28
28
  * then has to chase (docs/05-sessions.md calls the constant the source of truth). */
29
- export const SCHEMA_VERSION = 34;
29
+ export const SCHEMA_VERSION = 36;
30
30
  /**
31
31
  * Bump to force `agents sessions backfill resources` to re-derive every
32
32
  * session's skill/slash-command tallies on its next run (resource_scan_ledger
@@ -204,6 +204,12 @@ CREATE TABLE IF NOT EXISTS tool_program_occurrences (
204
204
  CREATE INDEX IF NOT EXISTS idx_tool_program_occurrences_program
205
205
  ON tool_program_occurrences(program, call_key);
206
206
 
207
+ -- Derived search index over tool_calls. call_key is UNINDEXED -- it is carried
208
+ -- for display, NOT for lookup: an FTS5 table has no index on an ordinary column,
209
+ -- so DELETE ... WHERE call_key = ? scans the whole index once per call, which is
210
+ -- quadratic in a session's call count. Every write here therefore addresses a
211
+ -- row by rowid, mirroring the tool_calls.rowid of the call it describes, so a
212
+ -- delete is a single rowid seek (tool-store.ts persistToolCalls/deleteSessionCalls).
207
213
  CREATE VIRTUAL TABLE IF NOT EXISTS tool_call_text USING fts5(
208
214
  call_key UNINDEXED,
209
215
  tool,
@@ -224,7 +230,15 @@ CREATE TABLE IF NOT EXISTS tool_scan_ledger (
224
230
  extractor_version INTEGER NOT NULL,
225
231
  indexed_at INTEGER NOT NULL,
226
232
  call_count INTEGER NOT NULL,
227
- evidence_bytes INTEGER NOT NULL
233
+ evidence_bytes INTEGER NOT NULL,
234
+ -- Resume point for the incremental tool scan. parsed_offset is the byte
235
+ -- offset just past the last COMPLETE newline-terminated record consumed, and
236
+ -- parser_state is the serialized ToolCallCollector snapshot at that offset
237
+ -- (next ordinal + still-unresolved calls). Together they let the next scan of
238
+ -- a session that only grew read the appended bytes instead of the whole file.
239
+ -- NULL means "no resume point" — the next scan re-reads from byte 0.
240
+ parser_state TEXT,
241
+ parsed_offset INTEGER
228
242
  );
229
243
 
230
244
  -- Skill/slash-command usage per session (#12), computed from a session's
@@ -294,7 +308,7 @@ CREATE TABLE IF NOT EXISTS session_insights (
294
308
  * re-derives on the next `agents insights` instead of silently reporting stale
295
309
  * numbers alongside fresh ones. Same role as RESOURCE_INDEX_VERSION.
296
310
  */
297
- export const INSIGHTS_EXTRACTOR_VERSION = 3;
311
+ export const INSIGHTS_EXTRACTOR_VERSION = 4;
298
312
  let dbInstance = null;
299
313
  /**
300
314
  * Apply schema migrations from `fromVersion` → SCHEMA_VERSION. The new
@@ -810,6 +824,57 @@ function migrateSchema(db, fromVersion) {
810
824
  );
811
825
  `);
812
826
  }
827
+ if (fromVersion < 35) {
828
+ // v34 -> v35: the default listing sort was `ORDER BY IFNULL(last_activity,
829
+ // timestamp) DESC` — wrapping the column in IFNULL() makes SQLite unable to
830
+ // satisfy it from idx_sessions_last_activity, so every list/resume query did
831
+ // a full table sort instead of an index walk (RUSH-2211). Every upsert path
832
+ // already writes a non-NULL last_activity (resolveLastActivity falls back to
833
+ // `timestamp`, itself NOT NULL) — the only rows that can still be NULL here
834
+ // are ones written before the v8 migration that somehow slipped the backfill,
835
+ // or seeded directly by a test. Backfill them so the column is unconditionally
836
+ // NOT NULL, then querySessions can sort on the bare column and use the index.
837
+ db.exec(`UPDATE sessions SET last_activity = timestamp WHERE last_activity IS NULL`);
838
+ }
839
+ if (fromVersion < 36) {
840
+ // v35 -> v36: make the tool index incremental, and stop paying a full FTS
841
+ // scan per deleted call.
842
+ //
843
+ // (a) tool_scan_ledger gains a resume point (parser_state + parsed_offset).
844
+ // Existing rows get NULLs, which read as "no resume point": the next
845
+ // scan of each session re-reads it once from byte 0 and records a resume
846
+ // point, so every scan after that is incremental. No ledger is wiped.
847
+ //
848
+ // (b) tool_call_text is rebuilt so its rowid mirrors tool_calls.rowid. The
849
+ // old rows were inserted with FTS5-assigned rowids and are only
850
+ // addressable by the UNINDEXED call_key, i.e. a full index scan per
851
+ // delete. There is no ALTER for that, and the rowids cannot be repaired
852
+ // in place, so the table is dropped and repopulated from tool_calls --
853
+ // the same non-destructive derived-table rebuild v27 did (the source of
854
+ // truth is tool_calls, which is untouched). The rebuild also lands the
855
+ // content as one merged segment, which is the compaction
856
+ // optimizeSessionSearchIndex would otherwise have to do afterwards.
857
+ const ledgerCols = new Set(db.prepare(`PRAGMA table_info(tool_scan_ledger)`).all()
858
+ .map((column) => column.name));
859
+ if (!ledgerCols.has('parser_state'))
860
+ db.exec(`ALTER TABLE tool_scan_ledger ADD COLUMN parser_state TEXT`);
861
+ if (!ledgerCols.has('parsed_offset'))
862
+ db.exec(`ALTER TABLE tool_scan_ledger ADD COLUMN parsed_offset INTEGER`);
863
+ db.exec(`
864
+ DROP TABLE IF EXISTS tool_call_text;
865
+ CREATE VIRTUAL TABLE tool_call_text USING fts5(
866
+ call_key UNINDEXED,
867
+ tool,
868
+ input,
869
+ output,
870
+ error,
871
+ tokenize = 'trigram'
872
+ );
873
+ INSERT INTO tool_call_text (rowid, call_key, tool, input, output, error)
874
+ SELECT rowid, call_key, tool, input, coalesce(output, ''), coalesce(error, '')
875
+ FROM tool_calls;
876
+ `);
877
+ }
813
878
  }
814
879
  /**
815
880
  * Stamp `account_key` / `account_org` / `account` on every Claude row from its
@@ -988,6 +1053,49 @@ export function optimizeSessionSearchIndex() {
988
1053
  return { table, segmentsBefore, segmentsAfter: segments(table) };
989
1054
  });
990
1055
  }
1056
+ /**
1057
+ * Segment count above which a scan pays for a slice of merge work. Below it the
1058
+ * index is small enough that querying it is not the bottleneck and merging is
1059
+ * pure overhead on every scan.
1060
+ */
1061
+ const FTS_MAINTENANCE_SEGMENT_THRESHOLD = 512;
1062
+ /**
1063
+ * Page budget for one incremental merge. FTS5's `'merge'` command does at most
1064
+ * this much work and returns — it is not `'optimize'`, which merges the whole
1065
+ * index in one unbounded pass. That bound is why this can run on the scan path:
1066
+ * the cost per scan is fixed, and repeated scans converge the index instead of
1067
+ * one scan stalling on a multi-gigabyte compaction.
1068
+ */
1069
+ const FTS_MAINTENANCE_MERGE_PAGES = 64;
1070
+ /**
1071
+ * Keep the FTS indexes from degrading on the normal scan path.
1072
+ *
1073
+ * `optimizeSessionSearchIndex` is the full, unbounded compaction behind
1074
+ * `agents sessions optimize`. Leaving it as the ONLY compaction meant the index
1075
+ * degraded until a human happened to run that command, which is how
1076
+ * `tool_call_text_data` reached gigabytes for tens of MB of content. This is the
1077
+ * automatic counterpart: bounded, threshold-gated, and safe to call after every
1078
+ * batch of writes. Non-destructive — merging never changes what is searchable.
1079
+ *
1080
+ * Returns one result per table it actually merged (empty when every table is
1081
+ * under the threshold, which is the common case on a warm index).
1082
+ */
1083
+ export function maintainSessionSearchIndex(db = getDB(), options = {}) {
1084
+ const threshold = options.segmentThreshold ?? FTS_MAINTENANCE_SEGMENT_THRESHOLD;
1085
+ const pages = options.mergePages ?? FTS_MAINTENANCE_MERGE_PAGES;
1086
+ // Hardcoded literals — never interpolate caller input into an identifier.
1087
+ const tables = ['tool_call_text', 'session_text'];
1088
+ const segments = (table) => db.prepare(`SELECT count(*) AS n FROM ${table}_data`).get().n;
1089
+ const results = [];
1090
+ for (const table of tables) {
1091
+ const segmentsBefore = segments(table);
1092
+ if (segmentsBefore < threshold)
1093
+ continue;
1094
+ db.prepare(`INSERT INTO ${table}(${table}, rank) VALUES('merge', ?)`).run(pages);
1095
+ results.push({ table, segmentsBefore, segmentsAfter: segments(table) });
1096
+ }
1097
+ return results;
1098
+ }
991
1099
  // ---------------------------------------------------------------------------
992
1100
  // Scan coordinator — prevents concurrent full scans across processes
993
1101
  // ---------------------------------------------------------------------------
@@ -1627,9 +1735,10 @@ export function upsertSessionsBatch(entries) {
1627
1735
  const stat = fs.statSync(toolSourcePath);
1628
1736
  return { fileMtimeMs: stat.mtimeMs, fileSize: stat.size };
1629
1737
  })();
1630
- // Non-resumable harnesses already parse here for todos/recent dirs. Derive
1631
- // the tool index from those same in-memory events: no second file read.
1632
- const events = parseSession(entry.meta.filePath, entry.meta.agent);
1738
+ // Some non-resumable scanners already normalized the transcript while
1739
+ // deriving metadata. Reuse those events; scanners that only read summary
1740
+ // metadata fall back to exactly one normalized parse here.
1741
+ const events = entry.events ?? parseSession(entry.meta.filePath, entry.meta.agent);
1633
1742
  writeResourceUsage(entry.meta.id, events, entry.meta.cwd);
1634
1743
  return {
1635
1744
  ...entry,
@@ -1640,6 +1749,8 @@ export function upsertSessionsBatch(entries) {
1640
1749
  },
1641
1750
  toolCalls: toolCallsFromEvents(events),
1642
1751
  toolScan,
1752
+ // These are complete event arrays, not an appended tail. Append would
1753
+ // duplicate existing evidence even when persistToolCalls supports it.
1643
1754
  toolIndexMode: 'replace',
1644
1755
  };
1645
1756
  }
@@ -1773,12 +1884,18 @@ export function upsertSessionsBatch(entries) {
1773
1884
  if (!toolScan || !entry.toolCalls)
1774
1885
  continue;
1775
1886
  try {
1776
- persistToolCalls(db, entry.meta, entry.toolCalls, toolScan, entry.toolIndexMode ?? 'replace');
1887
+ persistToolCalls(db, entry.meta, entry.toolCalls, toolScan, { mode: entry.toolIndexMode ?? 'replace' });
1777
1888
  }
1778
1889
  catch {
1779
1890
  // Boundary is intentionally retryable via tool_scan_ledger.
1780
1891
  }
1781
1892
  }
1893
+ // Every batch appends FTS segments (session_text always, tool_call_text for the
1894
+ // harnesses indexed above). Pay a bounded slice of the merge here so the
1895
+ // scan path keeps its own index healthy instead of leaving all compaction to
1896
+ // the manual `agents sessions optimize` (RUSH-2208). Threshold-gated, so a
1897
+ // small index costs two counts and nothing else.
1898
+ maintainSessionSearchIndex(db);
1782
1899
  }
1783
1900
  /**
1784
1901
  * Sync labels for a set of sessions. For each id in the map, if the stored
@@ -2118,6 +2235,51 @@ function buildSessionWhere(options) {
2118
2235
  const clause = where.length > 0 ? `WHERE ${where.join(' AND ')}` : '';
2119
2236
  return { clause, params };
2120
2237
  }
2238
+ /**
2239
+ * Resolve which of the given file paths no longer exist, batching the check
2240
+ * per directory instead of one `fs.existsSync` stat syscall per file.
2241
+ * Transcript trees put many sessions in the same directory (one Claude
2242
+ * `~/.claude/projects/<slug>/` holds every session for that project), so
2243
+ * `readdirSync` once per directory and a Set membership test collapses what
2244
+ * used to be N stat syscalls into (number of distinct directories) readdir
2245
+ * syscalls — the same existence answer, far fewer syscalls on a large index
2246
+ * (RUSH-2211). Falls back to per-file existsSync only when the directory
2247
+ * itself can't be listed (permissions, race with a concurrent delete).
2248
+ */
2249
+ function findMissingFilePaths(filePaths) {
2250
+ const byDir = new Map();
2251
+ for (const p of filePaths) {
2252
+ const dir = path.dirname(p);
2253
+ let basenames = byDir.get(dir);
2254
+ if (!basenames) {
2255
+ basenames = new Set();
2256
+ byDir.set(dir, basenames);
2257
+ }
2258
+ basenames.add(path.basename(p));
2259
+ }
2260
+ const missing = new Set();
2261
+ for (const [dir, basenames] of byDir) {
2262
+ let entries;
2263
+ try {
2264
+ entries = new Set(fs.readdirSync(dir));
2265
+ }
2266
+ catch {
2267
+ // Directory itself is gone (or unreadable) — every file in it is missing.
2268
+ // Also covers the race where readdir loses to a concurrent delete: fall
2269
+ // back to a direct stat rather than assuming existence.
2270
+ for (const base of basenames) {
2271
+ if (!fs.existsSync(path.join(dir, base)))
2272
+ missing.add(path.join(dir, base));
2273
+ }
2274
+ continue;
2275
+ }
2276
+ for (const base of basenames) {
2277
+ if (!entries.has(base))
2278
+ missing.add(path.join(dir, base));
2279
+ }
2280
+ }
2281
+ return missing;
2282
+ }
2121
2283
  /** Query sessions from the database, applying filters and ordering by last-activity descending (default). */
2122
2284
  export function querySessions(options = {}) {
2123
2285
  const db = getDB();
@@ -2130,11 +2292,18 @@ export function querySessions(options = {}) {
2130
2292
  : '';
2131
2293
  // NULLs last so unpriced / duration-less rows never crowd out real data when
2132
2294
  // sorting by cost or duration. timestamp is never null (NOT NULL column).
2295
+ // Default sort is the bare `last_activity` column, not `IFNULL(last_activity,
2296
+ // timestamp)` — the v35 migration backfills every row so last_activity is
2297
+ // never NULL, and every upsert path (resolveLastActivity) keeps it that way
2298
+ // going forward. Wrapping the column in IFNULL() defeats
2299
+ // idx_sessions_last_activity (SQLite can't use an index on an expression that
2300
+ // isn't the bare column); the bare column lets the planner walk the index
2301
+ // instead of sorting the whole result set (RUSH-2211).
2133
2302
  const orderClause = options.sortBy === 'cost'
2134
2303
  ? 'ORDER BY cost_usd IS NULL, cost_usd DESC, timestamp DESC'
2135
2304
  : options.sortBy === 'duration'
2136
2305
  ? 'ORDER BY duration_ms IS NULL, duration_ms DESC, timestamp DESC'
2137
- : 'ORDER BY IFNULL(last_activity, timestamp) DESC, timestamp DESC';
2306
+ : 'ORDER BY last_activity DESC, timestamp DESC';
2138
2307
  const sql = `SELECT * FROM sessions ${clause} ${orderClause} ${limitClause}`;
2139
2308
  const rows = db.prepare(sql).all(...params);
2140
2309
  if (options.skipExistenceCheck) {
@@ -2147,7 +2316,8 @@ export function querySessions(options = {}) {
2147
2316
  // surfacing in the Factory UI if any code path forgets to rewrite (#136).
2148
2317
  // Synthetic rows (OpenClaw channels/cron — see scanOpenClawIncremental) carry
2149
2318
  // an empty file_path and are exempt; they're keyed by CLI output, not files.
2150
- const missing = rows.filter(r => r.file_path && !fs.existsSync(r.file_path));
2319
+ const missingPaths = findMissingFilePaths(rows.map(r => r.file_path).filter((p) => !!p));
2320
+ const missing = rows.filter(r => r.file_path && missingPaths.has(r.file_path));
2151
2321
  if (missing.length > 0) {
2152
2322
  const purge = db.transaction(() => {
2153
2323
  for (const row of missing)
@@ -2628,6 +2798,20 @@ export function buildFtsQuery(input) {
2628
2798
  const expr = terms.map(t => `${t}*`).join(' OR ');
2629
2799
  return { expr, terms };
2630
2800
  }
2801
+ /**
2802
+ * Build a `label:(...)` FTS5 column-filter MATCH expression for the label
2803
+ * tier. Unlike `buildFtsQuery` (2-char floor, tuned for full-content search),
2804
+ * this allows 1-char terms: label search is the interactive type-ahead path —
2805
+ * the query grows one keystroke at a time, so a single character has to be
2806
+ * indexable too. Terms are filtered to `[a-z0-9]` before being embedded in the
2807
+ * expression string, so there's no FTS5 syntax injection from user input.
2808
+ */
2809
+ function buildLabelFtsQuery(input) {
2810
+ const terms = input.toLowerCase().split(/[^a-z0-9]+/).filter(t => t.length >= 1);
2811
+ if (terms.length === 0)
2812
+ return '';
2813
+ return `label:(${terms.map(t => `${t}*`).join(' OR ')})`;
2814
+ }
2631
2815
  /**
2632
2816
  * Label-first search. Sessions whose custom label substring-matches the query
2633
2817
  * always rank ahead of FTS5 hits — this gives predictable behavior when a user
@@ -2655,10 +2839,34 @@ export function ftsSearch(input, limit = 200) {
2655
2839
  // its `label` — set by an agent title / `/rename`, or seeded at launch from
2656
2840
  // `agents run --name`. Typing it resolves the session ahead of any FTS content
2657
2841
  // hit.
2658
- const labelRows = db.prepare(`
2659
- SELECT id, label FROM sessions
2660
- WHERE label IS NOT NULL AND LOWER(label) LIKE ?
2661
- `).all(`%${lower}%`);
2842
+ //
2843
+ // Candidates come from the FTS5 `label` column, not a raw `LOWER(label) LIKE
2844
+ // '%q%'` scan of `sessions`: a leading wildcard can't use any index, so on a
2845
+ // large session table that was a full-table scan on every keystroke of
2846
+ // interactive search (RUSH-2211). `session_text.label` is kept 1:1 with
2847
+ // `sessions.label` by every upsert path (storedFtsLabel), so this is the
2848
+ // same data, indexed. Token-prefix matching seeks the FTS index instead of
2849
+ // scanning every row, at the cost of only matching at token boundaries — a
2850
+ // substring inside a single token (e.g. "ckf" inside "quickfix") no longer
2851
+ // matches, since FTS5 only indexes prefixes of whole tokens, not arbitrary
2852
+ // interior slices. (A slice spanning a token boundary, like "ix-b" inside
2853
+ // "fix-bug", still matches: "ix-b" tokenizes to "ix" + "b", and "b" is a
2854
+ // valid prefix of the "bug" token.) That's the accepted trade-off for an
2855
+ // indexable interactive path; the exact/prefix/contains scoring below still
2856
+ // runs in JS over the FTS candidate set, so ranking among real matches is
2857
+ // unchanged. Only a query with no indexable token (rare — e.g.
2858
+ // punctuation-only input) falls back to the direct scan rather than
2859
+ // silently dropping the tier.
2860
+ const labelMatchExpr = buildLabelFtsQuery(input);
2861
+ const labelRows = labelMatchExpr
2862
+ ? db.prepare(`
2863
+ SELECT session_id AS id, label FROM session_text
2864
+ WHERE session_text MATCH ?
2865
+ `).all(labelMatchExpr)
2866
+ : db.prepare(`
2867
+ SELECT id, label FROM sessions
2868
+ WHERE label IS NOT NULL AND LOWER(label) LIKE ?
2869
+ `).all(`%${lower}%`);
2662
2870
  let hasExactLabelMatch = false;
2663
2871
  for (const row of labelRows) {
2664
2872
  // Score the label by match quality (exact > prefix > contains).
@@ -690,6 +690,7 @@ export declare function __resetCodexScanBranchCountsForTest(): void;
690
690
  export declare function readCursorMeta(filePath: string, currentVersion?: string): {
691
691
  meta: SessionMeta;
692
692
  content: string;
693
+ events: SessionEvent[];
693
694
  } | null;
694
695
  /** Parse a single Kimi session state.json file to extract session metadata. */
695
696
  export declare function readKimiMeta(filePath: string, priorRow?: {