chat-recall 0.0.1 → 0.4.8

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (94) hide show
  1. package/LICENSE +93 -0
  2. package/dist/auto-update.d.ts +52 -0
  3. package/dist/auto-update.d.ts.map +1 -0
  4. package/dist/auto-update.js +157 -0
  5. package/dist/auto-update.js.map +1 -0
  6. package/dist/cli.d.ts +6 -0
  7. package/dist/cli.d.ts.map +1 -0
  8. package/dist/cli.js +26102 -0
  9. package/dist/cli.js.map +1 -0
  10. package/dist/client-events.d.ts +11 -0
  11. package/dist/client-events.d.ts.map +1 -0
  12. package/dist/client-events.js +77 -0
  13. package/dist/client-events.js.map +1 -0
  14. package/dist/collector-migrate.d.ts +27 -0
  15. package/dist/collector-migrate.d.ts.map +1 -0
  16. package/dist/collector-migrate.js +60 -0
  17. package/dist/collector-migrate.js.map +1 -0
  18. package/dist/device-auth.d.ts +37 -0
  19. package/dist/device-auth.d.ts.map +1 -0
  20. package/dist/device-auth.js +161 -0
  21. package/dist/device-auth.js.map +1 -0
  22. package/dist/escalate.d.ts +99 -0
  23. package/dist/escalate.d.ts.map +1 -0
  24. package/dist/escalate.js +219 -0
  25. package/dist/escalate.js.map +1 -0
  26. package/dist/http.d.ts +14 -0
  27. package/dist/http.d.ts.map +1 -0
  28. package/dist/http.js +49 -0
  29. package/dist/http.js.map +1 -0
  30. package/dist/install-skills.d.ts +51 -0
  31. package/dist/install-skills.d.ts.map +1 -0
  32. package/dist/install-skills.js +223 -0
  33. package/dist/install-skills.js.map +1 -0
  34. package/dist/intent-drain.d.ts +25 -0
  35. package/dist/intent-drain.d.ts.map +1 -0
  36. package/dist/intent-drain.js +189 -0
  37. package/dist/intent-drain.js.map +1 -0
  38. package/dist/keyring.d.ts +28 -0
  39. package/dist/keyring.d.ts.map +1 -0
  40. package/dist/keyring.js +163 -0
  41. package/dist/keyring.js.map +1 -0
  42. package/dist/mcp.d.ts +8 -0
  43. package/dist/mcp.d.ts.map +1 -0
  44. package/dist/mcp.js +23665 -0
  45. package/dist/mcp.js.map +1 -0
  46. package/dist/project-tasks.d.ts +54 -0
  47. package/dist/project-tasks.d.ts.map +1 -0
  48. package/dist/project-tasks.js +204 -0
  49. package/dist/project-tasks.js.map +1 -0
  50. package/dist/repair.d.ts +73 -0
  51. package/dist/repair.d.ts.map +1 -0
  52. package/dist/repair.js +267 -0
  53. package/dist/repair.js.map +1 -0
  54. package/dist/self-restart.d.ts +91 -0
  55. package/dist/self-restart.d.ts.map +1 -0
  56. package/dist/self-restart.js +137 -0
  57. package/dist/self-restart.js.map +1 -0
  58. package/dist/service-installer.d.ts +32 -0
  59. package/dist/service-installer.d.ts.map +1 -0
  60. package/dist/service-installer.js +231 -0
  61. package/dist/service-installer.js.map +1 -0
  62. package/dist/sync-client.d.ts +234 -0
  63. package/dist/sync-client.d.ts.map +1 -0
  64. package/dist/sync-client.js +1985 -0
  65. package/dist/sync-client.js.map +1 -0
  66. package/dist/sync-ledger.d.ts +131 -0
  67. package/dist/sync-ledger.d.ts.map +1 -0
  68. package/dist/sync-ledger.js +366 -0
  69. package/dist/sync-ledger.js.map +1 -0
  70. package/dist/update-notice.d.ts +16 -0
  71. package/dist/update-notice.d.ts.map +1 -0
  72. package/dist/update-notice.js +107 -0
  73. package/dist/update-notice.js.map +1 -0
  74. package/dist/verify-deep.d.ts +79 -0
  75. package/dist/verify-deep.d.ts.map +1 -0
  76. package/dist/verify-deep.js +97 -0
  77. package/dist/verify-deep.js.map +1 -0
  78. package/dist/verify-repair.d.ts +21 -0
  79. package/dist/verify-repair.d.ts.map +1 -0
  80. package/dist/verify-repair.js +58 -0
  81. package/dist/verify-repair.js.map +1 -0
  82. package/dist/watch.js +21550 -0
  83. package/hooks/chat_recall_escalate_hook.sh +51 -0
  84. package/hooks/chat_recall_resume_hook.sh +69 -0
  85. package/hooks/chat_recall_save_hook.sh +254 -0
  86. package/hooks/chat_recall_wakeup_hook.sh +39 -0
  87. package/package.json +38 -4
  88. package/skills/chat-recall/SKILL.md +76 -0
  89. package/skills/chat-recall-changes/SKILL.md +50 -0
  90. package/skills/chat-recall-find/SKILL.md +64 -0
  91. package/skills/chat-recall-memory/SKILL.md +55 -0
  92. package/skills/chat-recall-project/SKILL.md +75 -0
  93. package/skills/chat-recall-resume/SKILL.md +62 -0
  94. package/README.md +0 -7
@@ -0,0 +1,1985 @@
1
+ /**
2
+ * The one sync client (thin collector): walk local sessions AND every other
3
+ * memory source, computing EVERYTHING live from the raw session files —
4
+ * reading NOTHING from the local SQLite store. Redact secrets (ALWAYS —
5
+ * `force:true`, never gated on the index-time toggle), optionally hash
6
+ * project paths, and push to the server's `/api/sync`
7
+ * (packages/server/src/routes/sync.ts). Credentials live in a 0600 file,
8
+ * never in settings.json.
9
+ *
10
+ * Thin-collector contract: the collector owns NO derived state. It reads the
11
+ * raw transcripts (which it must read anyway to ship them) and derives
12
+ * telemetry, project ids, KG triples, diffs, outcomes, commits and markers on
13
+ * the spot. The server is the source of truth for everything stateful
14
+ * (tombstones, summaries, secret findings/rules/dismissals) — the collector
15
+ * neither reads nor writes those locally.
16
+ *
17
+ * What ships (all live-computed from the session files):
18
+ * conversations — per-turn redacted session transcripts + live telemetry meta
19
+ * items/links — every non-session source type (plan, task, claude_md,
20
+ * history, paste, diary, skill, mcp, command, agent,
21
+ * hook, plugin) with redacted chunks + relationships
22
+ * derived — compute rows (diff/outcome/commits/markers) computed live
23
+ * + outcome-badge row (toggle: upload.sessionMeta)
24
+ * kg — knowledge-graph entities/triples extracted live from the
25
+ * redacted text (toggle: upload.sessionMeta)
26
+ *
27
+ * What does NOT ship, by design: the full-conversation JSON blob
28
+ * (content_cache) and embeddings. The server reconstructs the conversation
29
+ * view from the per-turn chunks. Tombstones, AI summaries, and secret
30
+ * findings/dismissals/custom-rules are NOT shipped by the collector — the
31
+ * server owns them.
32
+ */
33
+ import { join, dirname, basename } from 'node:path';
34
+ import { readFileSync, writeFileSync, mkdirSync, mkdtempSync, rmSync, chmodSync, unlinkSync, readdirSync, statSync } from 'node:fs';
35
+ import { tmpdir, hostname } from 'node:os';
36
+ import { createHash } from 'node:crypto';
37
+ import { redactSecrets, scanTextForFindings, installServerRulePack, serverRulePackVersion } from '@chat-recall/engine/core/secret-redactor.js';
38
+ import { discoverSessionSources, installSourceExclusions, sourceId } from '@chat-recall/engine/core/source-discovery.js';
39
+ import { discoverHomes } from '@chat-recall/engine/core/home-discovery.js';
40
+ import { homeDecision, approveHome, declineHome } from '@chat-recall/engine/core/home-approval.js';
41
+ import { scanFileForSecrets, scanDirForSecrets, scanTenantRules, isSecretScannerAvailable } from '@chat-recall/engine/core/secret-scanner.js';
42
+ import { isInternalToolPrompt } from '@chat-recall/engine/core/internal-prompts.js';
43
+ import { dropFuzzyFindings } from '@chat-recall/engine/core/secret-precision.js';
44
+ import { loadSettings, saveSettings, isProjectSyncable } from '@chat-recall/engine/core/settings.js';
45
+ import { getDataDir } from '@chat-recall/engine/core/paths.js';
46
+ import { listAvailableBackends } from '@chat-recall/engine/core/tool-backend.js';
47
+ import { extractTurnsAny, replaySessionAny } from '@chat-recall/engine/core/session-multi-tool.js';
48
+ import { parseTranscript, trimTranscriptForSync, TRANSCRIPT_VERSION, gzipContainer, mapContainerText, buildRawContainer, containerSrcHash, parseTranscriptFromContainer, updateShadow } from '@chat-recall/engine/transcript/index.js';
49
+ import { parseClaudeTranscriptText } from '@chat-recall/engine/transcript/claude.js';
50
+ import { parseGeminiTranscriptText } from '@chat-recall/engine/transcript/gemini.js';
51
+ import { parseCodexTranscriptText } from '@chat-recall/engine/transcript/codex.js';
52
+ import { getBackendForId, getBackend } from '@chat-recall/engine/core/tool-backend.js';
53
+ import { computeOutcome } from '@chat-recall/engine/core/session-outcome.js';
54
+ import { fileURLToPath } from 'node:url';
55
+ let ownVersion = '0.0.0';
56
+ try {
57
+ const dir = dirname(fileURLToPath(import.meta.url));
58
+ ownVersion = JSON.parse(readFileSync(join(dir, '../package.json'), 'utf-8')).version || ownVersion;
59
+ }
60
+ catch {
61
+ try {
62
+ // @ts-ignore
63
+ if (typeof __CLI_VERSION__ === 'string')
64
+ ownVersion = __CLI_VERSION__;
65
+ }
66
+ catch { }
67
+ }
68
+ import { markPrompt, summarizeMarkers } from '@chat-recall/engine/core/session-sentiment.js';
69
+ import { parseSessionFile, readCwdFromJsonl } from '@chat-recall/engine/parsers/session.js';
70
+ import { resolveProjectId } from '@chat-recall/engine/core/project-resolver.js';
71
+ import { extractEntities } from '@chat-recall/engine/core/entity-extractor.js';
72
+ import { buildSourceRegistry } from '@chat-recall/engine/parsers/all-sources.js';
73
+ import { getSyncedRows, markSynced, getLedgerData, persistLedgerData, fieldNeedsScan, markFieldCoverage, fieldNeedsFullPass, markFieldFullPassDone, syncMode, markFullResync, loadItemVersions, saveItemVersions } from './sync-ledger.js';
74
+ import { extractorVersionForTool, extractorVersionForId, toolOfId, EXTRACTOR_VERSION } from '@chat-recall/engine/core/extractor-version.js';
75
+ import { SYNC_FIELDS } from '@chat-recall/engine/core/sync-fields.js';
76
+ import { acquireIndexLock } from '@chat-recall/engine/core/index-lock.js';
77
+ import { fetchWithTimeout } from './http.js';
78
+ import '@chat-recall/engine/core/backends/index.js'; // register the tool backends
79
+ // Lives under the data dir so CHAT_RECALL_DATA_DIR isolates credentials the
80
+ // same way it isolates the index (tests, multi-profile setups).
81
+ const credPath = () => join(getDataDir(), 'credentials.json');
82
+ /**
83
+ * Multi-target: logging in ADDS a server (or updates its token). Every
84
+ * sync pushes to ALL targets; the per-server ledger keeps coverage
85
+ * independent. File shape is {targets:[...]} with the legacy single-object
86
+ * form still readable.
87
+ */
88
+ export function saveCredentials(c) {
89
+ mkdirSync(dirname(credPath()), { recursive: true });
90
+ const targets = loadAllCredentials().filter((t) => t.serverUrl !== c.serverUrl);
91
+ // Newest login FIRST: single-target consumers (loadCredentials → targets[0],
92
+ // i.e. sync + every MCP read) follow the most recent `chat-recall login`.
93
+ // Appending here once meant a fresh SaaS login silently kept syncing to a
94
+ // stale localhost self-host target that happened to sit at position 0.
95
+ targets.unshift(c);
96
+ writeFileSync(credPath(), JSON.stringify({ targets }, null, 2));
97
+ try {
98
+ chmodSync(credPath(), 0o600);
99
+ }
100
+ catch { /* windows */ }
101
+ try {
102
+ const s = loadSettings();
103
+ s.sync.enabled = true;
104
+ s.sync.endpoint = targets.map((t) => t.serverUrl).join(',');
105
+ saveSettings(s);
106
+ }
107
+ catch { /* settings unwritable — manual `chat-recall sync` still works */ }
108
+ }
109
+ export function loadAllCredentials() {
110
+ try {
111
+ const parsed = JSON.parse(readFileSync(credPath(), 'utf-8'));
112
+ // token may be '' for a no-auth local self-host server (AUTH_PROVIDER=none) —
113
+ // such a target is valid and pushes without an Authorization header.
114
+ if (Array.isArray(parsed?.targets))
115
+ return parsed.targets.filter((t) => t?.serverUrl).map((t) => ({ serverUrl: t.serverUrl, token: t.token || '' }));
116
+ if (parsed?.serverUrl)
117
+ return [{ serverUrl: parsed.serverUrl, token: parsed.token || '' }]; // legacy single
118
+ }
119
+ catch { /* none */ }
120
+ return [];
121
+ }
122
+ // In-memory cache for tenant security configuration. Fetched at the start of
123
+ // each `sync` call, but for the watch daemon we refresh only every 5 minutes
124
+ // so repeated incremental syncs don't hammer the server.
125
+ const _securityConfigCache = new Map();
126
+ const SECURITY_CONFIG_TTL_MS = 5 * 60 * 1000;
127
+ async function fetchTenantSecurityConfig(cred) {
128
+ const base = cred.serverUrl.replace(/\/+$/, '');
129
+ const cached = _securityConfigCache.get(base);
130
+ if (cached && Date.now() - cached.fetchedAt < SECURITY_CONFIG_TTL_MS)
131
+ return cached.config;
132
+ const headers = { 'content-type': 'application/json' };
133
+ if (cred.token)
134
+ headers.authorization = `Bearer ${cred.token}`;
135
+ try {
136
+ const [rulesRes, settingsRes] = await Promise.all([
137
+ fetchWithTimeout(`${base}/api/secrets/rules`, { headers }),
138
+ fetchWithTimeout(`${base}/api/teams/security-config`, { headers }).catch(() => null),
139
+ ]);
140
+ let tenantRules = [];
141
+ let rulePackVersion = null;
142
+ if (rulesRes.ok) {
143
+ const body = await rulesRes.json().catch(() => ({}));
144
+ const enabled = (body.rules || []).filter((r) => r.enabled !== false);
145
+ tenantRules = enabled;
146
+ // The REDACTION set. Prefer the server's `pack` — it carries chat-recall's
147
+ // curated builtin rules as well as this tenant's `redact` ones, which is
148
+ // what lets coverage improve for every customer from a server deploy
149
+ // instead of a CLI release on each device. Fall back to deriving it from
150
+ // the tenant rules when talking to a server that predates `pack`.
151
+ //
152
+ // installServerRulePack is add-only (it can never weaken the compiled-in
153
+ // builtins) and drops individual rules that fail its safety checks; we log
154
+ // those, because a silently ignored rule reads as coverage the operator
155
+ // does not actually have.
156
+ const packSpec = body.pack?.rules
157
+ ? { version: body.pack.version, rules: body.pack.rules.map((r) => ({ name: r.name, regex: r.regex, flags: r.flags, redact: true, source: r.source })) }
158
+ : { version: body.version, rules: enabled.filter((r) => r.redact === true).map((r) => ({ name: r.name, regex: r.regex, redact: true, source: 'tenant' })) };
159
+ const pack = installServerRulePack(packSpec);
160
+ rulePackVersion = serverRulePackVersion();
161
+ for (const r of pack.rejected) {
162
+ console.error(`[sync] server redaction rule "${r.name}" ignored: ${r.reason}`);
163
+ }
164
+ if (pack.accepted > 0) {
165
+ const rev = body.pack?.revision ? `, rev ${body.pack.revision}` : '';
166
+ console.error(`[sync] ${pack.accepted} server redaction rule(s) active (pack ${rulePackVersion}${rev})`);
167
+ }
168
+ }
169
+ // Default verification ON. The server endpoint may not exist yet (older
170
+ // servers), so we fail open to "verified enabled" — this is the product default.
171
+ let verifySecrets = true;
172
+ if (settingsRes?.ok) {
173
+ const body = await settingsRes.json().catch(() => ({}));
174
+ if (typeof body.verifySecrets === 'boolean')
175
+ verifySecrets = body.verifySecrets;
176
+ }
177
+ const config = { tenantRules, verifySecrets, rulePackVersion };
178
+ _securityConfigCache.set(base, { fetchedAt: Date.now(), config });
179
+ return config;
180
+ }
181
+ catch {
182
+ // If the server is unreachable or the endpoints don't exist, fall back to
183
+ // no tenant rules and verification enabled. This keeps the collector
184
+ // working during transient outages and with older server images.
185
+ return { tenantRules: [], verifySecrets: true, rulePackVersion: null };
186
+ }
187
+ }
188
+ export function _resetTenantSecurityConfigCache() {
189
+ _securityConfigCache.clear();
190
+ }
191
+ const _syncConfigCache = new Map();
192
+ async function fetchTenantSyncConfig(cred) {
193
+ const base = cred.serverUrl.replace(/\/+$/, '');
194
+ const cached = _syncConfigCache.get(base);
195
+ if (cached && Date.now() - cached.fetchedAt < SECURITY_CONFIG_TTL_MS)
196
+ return cached.config;
197
+ const headers = { 'content-type': 'application/json' };
198
+ if (cred.token)
199
+ headers.authorization = `Bearer ${cred.token}`;
200
+ try {
201
+ // Report what this machine has BEFORE reading the selection back, so a
202
+ // freshly-discovered source shows up in the dashboard on the first sync
203
+ // rather than the second. Paths travel in this direction only.
204
+ try {
205
+ const sources = discoverSessionSources();
206
+ if (sources.length > 0) {
207
+ await fetchWithTimeout(`${base}/api/sync-config/sources`, {
208
+ method: 'POST', headers,
209
+ body: JSON.stringify({ device: deviceLabel(), sources }),
210
+ }).catch(() => null);
211
+ }
212
+ }
213
+ catch { /* reporting is advisory — never block a sync on it */ }
214
+ const res = await fetchWithTimeout(`${base}/api/sync-config`, { headers });
215
+ if (!res.ok)
216
+ return { excludeTools: [], excludeProjects: [], excludeSources: [], approveSources: [] };
217
+ const body = await res.json().catch(() => ({}));
218
+ const config = {
219
+ excludeTools: Array.isArray(body.excludeTools) ? body.excludeTools.filter((t) => typeof t === 'string') : [],
220
+ excludeProjects: Array.isArray(body.excludeProjects) ? body.excludeProjects.filter((p) => typeof p === 'string') : [],
221
+ // Ids only — installSourceExclusions drops anything else, so a server
222
+ // sending a path cannot make this collector read it.
223
+ excludeSources: Array.isArray(body.excludeSources) ? body.excludeSources.filter((x) => typeof x === 'string') : [],
224
+ approveSources: Array.isArray(body.approveSources) ? body.approveSources.filter((x) => typeof x === 'string') : [],
225
+ };
226
+ const applied = installSourceExclusions(config.excludeSources);
227
+ if (applied.excluded > 0) {
228
+ console.error(`[sync] ${applied.excluded} transcript source(s) excluded by tenant config`);
229
+ }
230
+ // Decisions made in the dashboard become LOCAL decisions here. The server
231
+ // sends ids, never paths: we approve/decline only a home this machine itself
232
+ // discovered, so the dashboard can answer the question but can never
233
+ // introduce a folder for us to read.
234
+ try {
235
+ const byId = new Map(discoverHomes({ includeRunning: false }).map((h) => [sourceId(h.path), h.path]));
236
+ let decided = 0;
237
+ for (const id of config.approveSources) {
238
+ const path = byId.get(id);
239
+ if (path && homeDecision(path) === 'pending') {
240
+ approveHome(path);
241
+ decided++;
242
+ }
243
+ }
244
+ for (const id of config.excludeSources) {
245
+ const path = byId.get(id);
246
+ const d = path ? homeDecision(path) : null;
247
+ if (path && d !== 'declined' && d !== 'primary') {
248
+ declineHome(path);
249
+ decided++;
250
+ }
251
+ }
252
+ if (decided > 0)
253
+ console.error(`[sync] applied ${decided} folder decision(s) from the dashboard`);
254
+ }
255
+ catch { /* advisory — a failure must not stop the sync */ }
256
+ _syncConfigCache.set(base, { fetchedAt: Date.now(), config });
257
+ return config;
258
+ }
259
+ catch {
260
+ return { excludeTools: [], excludeProjects: [], excludeSources: [], approveSources: [] };
261
+ }
262
+ }
263
+ /** Stable-ish machine label for the dashboard's per-device grouping. */
264
+ function deviceLabel() {
265
+ try {
266
+ return hostname();
267
+ }
268
+ catch {
269
+ return 'unknown';
270
+ }
271
+ }
272
+ export function _resetTenantSyncConfigCache() {
273
+ _syncConfigCache.clear();
274
+ }
275
+ /** Legacy single-target accessor — first target. */
276
+ /** The ACTIVE target = most recent login (saveCredentials unshifts). */
277
+ export function loadCredentials() {
278
+ return loadAllCredentials()[0] ?? null;
279
+ }
280
+ export function removeCredentials(serverUrl) {
281
+ const targets = loadAllCredentials();
282
+ const kept = targets.filter((t) => t.serverUrl !== serverUrl);
283
+ if (kept.length === targets.length)
284
+ return false;
285
+ writeFileSync(credPath(), JSON.stringify({ targets: kept }, null, 2));
286
+ return true;
287
+ }
288
+ /** Tokenize an absolute project path so the server can group by project
289
+ * without learning the developer's filesystem layout. */
290
+ const hashPath = (p) => (p ? 'p_' + createHash('sha256').update(p).digest('hex').slice(0, 12) : '');
291
+ /** Redact every string anywhere inside a JSON-serializable value. Used for
292
+ * derived payloads (diffs carry file content; outcomes carry prompt text)
293
+ * where field-by-field redaction would be fragile. */
294
+ function redactDeep(v, count) {
295
+ if (typeof v === 'string')
296
+ return redactSecrets(v, { force: true, count });
297
+ if (Array.isArray(v))
298
+ return v.map((x) => redactDeep(x, count));
299
+ if (v && typeof v === 'object') {
300
+ const o = {};
301
+ for (const [k, val] of Object.entries(v))
302
+ o[k] = redactDeep(val, count);
303
+ return o;
304
+ }
305
+ return v;
306
+ }
307
+ /** compute_cache kinds shipped to the server. Must stay in step with the
308
+ * kinds the server's conversation routes read (heavyCacheGet keys). */
309
+ const COMPUTE_KINDS = ['diff', 'outcome', 'commits', 'markers'];
310
+ /** Per-row ceiling for derived payloads — matches the local heavy-cache L2
311
+ * ceiling (metadata-cache.ts) so we never ship a row the server-side
312
+ * SQLite edition couldn't store anyway. */
313
+ const MAX_DERIVED_ROW_BYTES = 2 * 1024 * 1024;
314
+ /**
315
+ * Local/LAN sync targets need no WAN-style pacing — you own the box and there
316
+ * is no shared-tenant or rate-limiter to be polite to, so they get a higher
317
+ * in-flight cap and zero inter-batch sleep. Remote/SaaS hosts stay gentler.
318
+ */
319
+ function isLocalHost(serverUrl) {
320
+ let host = '';
321
+ try {
322
+ host = new URL(serverUrl).hostname.toLowerCase();
323
+ }
324
+ catch {
325
+ return false;
326
+ }
327
+ if (host === 'localhost' || host === '127.0.0.1' || host === '::1' || host === '0.0.0.0')
328
+ return true;
329
+ if (host.endsWith('.local') || host.endsWith('.lan'))
330
+ return true;
331
+ if (/^10\./.test(host))
332
+ return true; // 10.0.0.0/8
333
+ if (/^192\.168\./.test(host))
334
+ return true; // 192.168.0.0/16
335
+ if (/^172\.(1[6-9]|2\d|3[01])\./.test(host))
336
+ return true; // 172.16.0.0/12
337
+ return false;
338
+ }
339
+ export async function syncSessions(opts = {}) {
340
+ const targets = loadAllCredentials();
341
+ if (targets.length === 0)
342
+ throw new Error('Not logged in — run `chat-recall login <server-url> --token <token>`');
343
+ let agg = null;
344
+ const errors = [];
345
+ for (const cred of targets) {
346
+ try {
347
+ const r = await syncToTarget(cred, opts);
348
+ agg = agg ? {
349
+ uploaded: agg.uploaded + r.uploaded, skipped: agg.skipped + r.skipped, redactions: agg.redactions + r.redactions,
350
+ items: agg.items + r.items, links: agg.links + r.links, findings: agg.findings + r.findings,
351
+ derived: agg.derived + r.derived, kgEntities: agg.kgEntities + r.kgEntities, kgTriples: agg.kgTriples + r.kgTriples,
352
+ scanned: Math.max(agg.scanned, r.scanned), scanMs: Math.max(agg.scanMs, r.scanMs),
353
+ } : r;
354
+ }
355
+ catch (e) {
356
+ errors.push(`${cred.serverUrl}: ${e instanceof Error ? e.message : e}`);
357
+ }
358
+ }
359
+ if (!agg)
360
+ throw new Error(`sync failed for all targets:\n ${errors.join('\n ')}`);
361
+ if (errors.length > 0)
362
+ console.error(`[sync] ${errors.length} target(s) failed (others succeeded):\n ${errors.join('\n ')}`);
363
+ return agg;
364
+ }
365
+ /** POST one derived-field batch, retrying the ingest gate's 429 (honoring
366
+ * Retry-After) and transient 5xx. */
367
+ async function postFieldBatch(base, token, rows) {
368
+ const headers = { 'content-type': 'application/json' };
369
+ if (token)
370
+ headers.authorization = `Bearer ${token}`;
371
+ const sleep = (ms) => new Promise((r) => setTimeout(r, ms));
372
+ for (let attempt = 0;; attempt++) {
373
+ const res = await fetchWithTimeout(`${base}/api/sync`, { method: 'POST', headers, body: JSON.stringify({ fields: rows }) }, 60_000);
374
+ if (res.ok)
375
+ return;
376
+ const retryable = res.status === 429 || res.status >= 500;
377
+ if (!retryable || attempt >= 5)
378
+ throw new Error(`${base}: fields batch HTTP ${res.status} ${await res.text().catch(() => '')}`);
379
+ const ra = Number(res.headers.get('retry-after')) || 0;
380
+ await sleep(ra > 0 ? ra * 1000 : [500, 1500, 4000, 8000, 15000][Math.min(attempt, 4)]);
381
+ }
382
+ }
383
+ /**
384
+ * Reconcile every DERIVED FIELD (engine core/sync-fields.ts) for one target,
385
+ * pushing ONLY missing/stale fields — never the conversation.
386
+ *
387
+ * Per field: a one-time FULL pass (all sessions) runs when the field's version
388
+ * bumped or a re-scan was forced; otherwise only `convRefs` (the new/changed
389
+ * sessions this sync already listed) are checked. A session is (re)scanned only
390
+ * when the coverage ledger says the field is missing/stale — an absent-but-
391
+ * current field is skipped (the no-retry guarantee). Present values ship via the
392
+ * fields[] batch and are ledger-stamped after the server acks; absent results
393
+ * are stamped locally (nothing to ship) so they're never re-scanned.
394
+ */
395
+ async function reconcileFieldsForTarget(cred, fullRefs, opts = {}) {
396
+ const base = cred.serverUrl.replace(/\/+$/, '');
397
+ const ledger = getSyncedRows(base);
398
+ let pushed = 0, absent = 0, scanned = 0;
399
+ for (const field of SYNC_FIELDS) {
400
+ const needFull = !!opts.force || fieldNeedsFullPass(base, field);
401
+ const candidates = needFull ? fullRefs() : (opts.convRefs ?? fullRefs());
402
+ const send = [];
403
+ const mtimeById = new Map();
404
+ const absentIds = [];
405
+ for (const ref of candidates) {
406
+ if (!fieldNeedsScan(ledger.get(ref.prefixedId), field, ref.mtime, opts.force))
407
+ continue;
408
+ scanned++;
409
+ mtimeById.set(ref.prefixedId, ref.mtime);
410
+ const raw = field.scan(ref);
411
+ const value = raw && raw.trim()
412
+ ? redactSecrets(raw.trim(), { force: true, count: { redactions: 0 } }).slice(0, 200)
413
+ : null;
414
+ if (value !== null)
415
+ send.push({ session_id: ref.prefixedId, field: field.name, value });
416
+ else
417
+ absentIds.push(ref.prefixedId);
418
+ }
419
+ // Present values: ship in chunks, stamp coverage only after each ack so an
420
+ // interrupted run resumes cleanly.
421
+ for (let i = 0; i < send.length; i += 200) {
422
+ const chunk = send.slice(i, i + 200);
423
+ await postFieldBatch(base, cred.token, chunk);
424
+ markFieldCoverage(base, field, chunk.map((r) => ({ id: r.session_id, present: true, mtime: mtimeById.get(r.session_id) ?? 0 })));
425
+ pushed += chunk.length;
426
+ }
427
+ // Absent: nothing to ship; stamp locally so they're never re-scanned.
428
+ if (absentIds.length) {
429
+ markFieldCoverage(base, field, absentIds.map((id) => ({ id, present: false, mtime: mtimeById.get(id) ?? 0 })));
430
+ absent += absentIds.length;
431
+ }
432
+ // Full pass completed without throwing → record so it isn't repeated until
433
+ // the field version bumps or a re-scan is forced again.
434
+ if (needFull)
435
+ markFieldFullPassDone(base, field);
436
+ }
437
+ return { pushed, absent, scanned };
438
+ }
439
+ /**
440
+ * Manual entrypoint (CLI `reconcile`): reconcile derived fields for ALL targets.
441
+ * Normal `sync` already runs reconciliation each pass; this is the explicit
442
+ * on-demand backfill / forced re-scan (also what a UI "re-scan this field"
443
+ * action drives, via forceFieldRescan + a sync).
444
+ */
445
+ export async function reconcileFields(opts = {}) {
446
+ const targets = loadAllCredentials();
447
+ if (targets.length === 0)
448
+ throw new Error('Not logged in — run `chat-recall login <server-url> --token <token>`');
449
+ let cached = null;
450
+ const fullRefs = () => (cached ??= listAvailableBackends().flatMap((b) => { try {
451
+ return b.listSessions({});
452
+ }
453
+ catch {
454
+ return [];
455
+ } }));
456
+ let pushed = 0, absent = 0, scanned = 0;
457
+ const perTarget = {};
458
+ for (const cred of targets) {
459
+ const r = await reconcileFieldsForTarget(cred, fullRefs, { force: opts.force });
460
+ pushed += r.pushed;
461
+ absent += r.absent;
462
+ scanned += r.scanned;
463
+ perTarget[cred.serverUrl.replace(/\/+$/, '')] = { pushed: r.pushed, absent: r.absent };
464
+ }
465
+ return { sessions: fullRefs().length, scanned, pushed, absent, perTarget };
466
+ }
467
+ /**
468
+ * Reap batch-scan temp dirs leaked by a previous run.
469
+ *
470
+ * The scan below deletes its dir in a `finally` — which a SIGKILL or the OOM
471
+ * killer skips entirely, and this loop is exactly the one that has OOM-killed
472
+ * the daemon before. Each leaked dir holds one file per session: one of them
473
+ * was found at 3.4GB / 309k files, and together with other cruft it filled a
474
+ * 31GB tmpfs, at which point *everything* on the box that needed /tmp broke.
475
+ *
476
+ * So: before starting a scan, delete our own stale dirs. 6h is far longer than
477
+ * any real scan, so an in-flight dir from a concurrent process is never at risk.
478
+ */
479
+ export function reapStaleScanDirs(dir = tmpdir(), maxAgeMs = 6 * 3600_000, now = Date.now()) {
480
+ let reaped = 0;
481
+ let names;
482
+ try {
483
+ names = readdirSync(dir);
484
+ }
485
+ catch {
486
+ return 0;
487
+ }
488
+ for (const name of names) {
489
+ if (!name.startsWith('cr-batchscan-'))
490
+ continue;
491
+ const p = join(dir, name);
492
+ try {
493
+ if (now - statSync(p).mtimeMs < maxAgeMs)
494
+ continue;
495
+ rmSync(p, { recursive: true, force: true });
496
+ reaped++;
497
+ }
498
+ catch { /* raced with its owner, or not ours to remove */ }
499
+ }
500
+ return reaped;
501
+ }
502
+ /**
503
+ * Run the optional external detectors (gitleaks/trufflehog) over many sessions
504
+ * with a BOUNDED amount of pre-redaction text on disk at any one time.
505
+ *
506
+ * The detectors take a path, so their input has to be materialized — and that
507
+ * input is raw session text, i.e. real credentials in cleartext. Writing all of
508
+ * it at once is what left 3.4GB / 309k such files in /tmp when the daemon was
509
+ * OOM-killed mid-scan (the `finally` never ran). So we fill a temp dir up to
510
+ * `maxBytes`, scan it, delete it, and repeat: worst-case exposure — and
511
+ * worst-case leak if this process is killed — is one slice, not the whole walk.
512
+ *
513
+ * Cost of slicing is 2 detector spawns per extra slice, still far below the
514
+ * 2·N of per-session scanning.
515
+ *
516
+ * `text()` returning null (unexportable session) is skipped silently: it just
517
+ * gets no external findings, the builtin scan still covers it.
518
+ * `scan` is injectable so tests don't need the binaries installed.
519
+ */
520
+ export function batchScanExternal(items, opts = {}) {
521
+ const maxBytes = opts.maxBytes ?? BATCH_SCAN_MAX_BYTES;
522
+ const scan = opts.scan ?? scanDirForSecrets;
523
+ const findings = new Map();
524
+ let scanned = 0, slices = 0, scanMs = 0;
525
+ if (items.length === 0)
526
+ return { findings, scanned, slices, scanMs };
527
+ reapStaleScanDirs();
528
+ let scanDir = mkdtempSync(join(tmpdir(), 'cr-batchscan-'));
529
+ let safeToId = new Map();
530
+ let bytesInDir = 0;
531
+ /** Scan what's currently materialized, record findings, then delete the dir. */
532
+ const flushSlice = () => {
533
+ if (safeToId.size > 0) {
534
+ const t0 = performance.now();
535
+ for (const f of scan(scanDir, { verifyOnly: opts.verifyOnly })) {
536
+ const id = safeToId.get(basename(f.file).replace(/\.txt$/, ''));
537
+ if (!id)
538
+ continue;
539
+ const list = findings.get(id) ?? [];
540
+ list.push({ detector: f.detector, rule: f.rule, line: f.line, preview: f.preview, verified: f.verified });
541
+ findings.set(id, list);
542
+ }
543
+ scanMs += performance.now() - t0;
544
+ scanned += safeToId.size;
545
+ slices++;
546
+ }
547
+ try {
548
+ rmSync(scanDir, { recursive: true, force: true });
549
+ }
550
+ catch { /* best-effort */ }
551
+ safeToId = new Map();
552
+ bytesInDir = 0;
553
+ };
554
+ try {
555
+ for (const item of items) {
556
+ try {
557
+ const raw = item.text();
558
+ if (raw === null)
559
+ continue;
560
+ const safe = item.id.replace(/[^A-Za-z0-9_-]/g, '');
561
+ safeToId.set(safe, item.id);
562
+ writeFileSync(join(scanDir, `${safe}.txt`), raw);
563
+ bytesInDir += Buffer.byteLength(raw);
564
+ }
565
+ catch { /* unexportable session — no external findings for it */ }
566
+ // Bound on bytes ACTUALLY written, not an estimate. A single session
567
+ // larger than the cap still gets scanned — alone, in its own slice.
568
+ if (bytesInDir >= maxBytes) {
569
+ flushSlice();
570
+ scanDir = mkdtempSync(join(tmpdir(), 'cr-batchscan-'));
571
+ }
572
+ }
573
+ flushSlice();
574
+ }
575
+ finally {
576
+ // flushSlice removed the final dir already (force:true makes this a no-op);
577
+ // this is what cleans up when the loop above threw.
578
+ try {
579
+ rmSync(scanDir, { recursive: true, force: true });
580
+ }
581
+ catch { /* best-effort */ }
582
+ }
583
+ return { findings, scanned, slices, scanMs };
584
+ }
585
+ async function syncToTarget(cred, opts = {}) {
586
+ const sync = loadSettings().sync;
587
+ // Exclusions = local settings ∪ server-side tenant config (dashboard-edited).
588
+ const serverCfg = await fetchTenantSyncConfig(cred);
589
+ const excludeTools = new Set([...(sync?.excludeTools ?? []), ...serverCfg.excludeTools]);
590
+ const excludeProjects = [...(sync?.excludeProjects ?? []), ...serverCfg.excludeProjects];
591
+ // Upload category toggles (settings.sync.upload, all default true).
592
+ // Redaction of shipped text always runs regardless of toggles.
593
+ const upload = {
594
+ raw: sync?.upload?.raw !== false,
595
+ sessionMeta: sync?.upload?.sessionMeta !== false,
596
+ findings: sync?.upload?.findings !== false,
597
+ };
598
+ // Cleartext paths: explicit flag wins, else the persistent setting
599
+ // (sync.pathsCleartext — what the watch daemon uses).
600
+ const cleartext = opts.cleartextPaths ?? sync?.pathsCleartext === true;
601
+ const mapPath = (p) => (cleartext ? p : hashPath(p));
602
+ const excluded = (projectPath) => excludeProjects.some((x) => x && projectPath.includes(x));
603
+ // Opt-in selective sync (settings.sync.syncMode='only'): ship ONLY allowlisted
604
+ // projects. Allowlist = local ∪ tenant config; matched on the resolved project
605
+ // id (what the dashboard picker lists) or a path substring. 'all'/unset ships
606
+ // every non-excluded project (unchanged default).
607
+ const syncOnly = new Set([...(sync?.syncOnlyProjects ?? []), ...(serverCfg.syncOnlyProjects ?? [])]);
608
+ const includedProject = (projectPath) => {
609
+ if ((sync?.syncMode ?? 'all') !== 'only')
610
+ return true; // fast path: no resolve
611
+ const r = resolveProjectId(projectPath);
612
+ return isProjectSyncable(r.source === 'ignored' ? '' : r.id, projectPath, { syncMode: 'only', syncOnly });
613
+ };
614
+ // Ledger mode walks ALL sessions (the ledger does the skipping — that's
615
+ // what lets a previously-failed session retry no matter how old it is).
616
+ // Watermark mode keeps the cheap bounded walk for explicit --since runs.
617
+ // Version handshake: refuse loudly instead of silently degrading. An old
618
+ // server ignores payload fields it doesn't know — which looks like
619
+ // success and produces gutted data (lived experience, June 2026).
620
+ const MIN_SERVER_API = 2;
621
+ try {
622
+ const caps = await fetchWithTimeout(`${cred.serverUrl.replace(/\/$/, '')}/api/capabilities`).then((r) => r.json());
623
+ if ((caps.apiVersion ?? 0) < MIN_SERVER_API) {
624
+ throw new Error(`server too old: apiVersion ${caps.apiVersion ?? 'none'} < required ${MIN_SERVER_API} — update the server image before syncing`);
625
+ }
626
+ }
627
+ catch (e) {
628
+ if (e instanceof Error && e.message.startsWith('server too old'))
629
+ throw e;
630
+ throw new Error(`cannot verify server version (${e instanceof Error ? e.message : e}) — refusing to sync blind`);
631
+ }
632
+ // Fetch tenant-specific secret-scan configuration from the server. Rules are
633
+ // configured in the dashboard and must be applied to raw session text on the
634
+ // client (the server never sees unredacted secrets). Verification live-checks
635
+ // matched keys with their issuers (e.g. AWS, GitHub) — on by default, but a
636
+ // tenant admin can disable it.
637
+ const { tenantRules, verifySecrets } = await fetchTenantSecurityConfig(cred);
638
+ // Walk scope. 'full' (default) lists EVERY session — the ledger does the
639
+ // skipping, which is what lets an old failed session retry forever. 'changed'
640
+ // bounds the LISTING to recently-touched sessions (mtime window with overlap):
641
+ // the watch daemon's per-file-change ticks use it so a burst of keystrokes
642
+ // costs a walk over ~a handful of refs, not 30k+. Failed/old sessions still
643
+ // converge via the daemon's 15-min heartbeat + startup ticks, which stay
644
+ // 'full'. The ledger still drives skip/append/full per ref in BOTH modes.
645
+ const CHANGED_WALK_OVERLAP_MS = 15 * 60_000;
646
+ const changedWalk = opts.walk === 'changed';
647
+ const listSinceMs = changedWalk
648
+ ? Math.max(0, (opts.sinceMs ?? 0) - CHANGED_WALK_OVERLAP_MS)
649
+ : (opts.useLedger ? undefined : opts.sinceMs);
650
+ const refs = listAvailableBackends().flatMap((b) => {
651
+ try {
652
+ return b.listSessions({ sinceMs: listSinceMs });
653
+ }
654
+ catch {
655
+ return [];
656
+ }
657
+ });
658
+ // ── Upload plumbing, created up-front so the walks below can stream.
659
+ // Byte-aware batching: cap each POST at ~4MB of serialized payload (and a
660
+ // per-field item cap) so transcript-heavy runs can't blow the server's
661
+ // body limit. A single row larger than the cap still ships alone.
662
+ //
663
+ // Streaming matters: a 5k-session backfill with diff payloads would be
664
+ // gigabytes if accumulated before upload. Each batcher flushes as soon as
665
+ // its window fills, so memory stays bounded at ~one batch per category.
666
+ //
667
+ // Backpressure, not a blind sleep. Server-side ingest does real work per row
668
+ // (chunking, classification, FTS/vector inserts); an UNBOUNDED backfill
669
+ // browned out a node on 2026-06-11. The protection that matters is bounding
670
+ // how many batches hit the server AT ONCE — so we cap in-flight uploads (the
671
+ // cap is the rate limit) and let the server push back with 429/Retry-After,
672
+ // sleeping only when it actually tells us to. Local/LAN targets need none of
673
+ // this politeness (you own the box) → higher cap, zero pacing. An explicit
674
+ // --throttle (opts.throttleMs>0) forces the old serial+sleep mode for a
675
+ // known-fragile server.
676
+ const MAX_BATCH_BYTES = 4 * 1024 * 1024;
677
+ const sleep = (ms) => new Promise((r) => setTimeout(r, ms));
678
+ const explicitThrottle = typeof opts.throttleMs === 'number' && opts.throttleMs > 0;
679
+ const throttleMs = explicitThrottle ? opts.throttleMs : 0;
680
+ const maxInflight = explicitThrottle ? 1 : (isLocalHost(cred.serverUrl) ? 6 : 4);
681
+ // Per-attempt upload timeout. WITHOUT this, a half-open/slow connection makes
682
+ // `fetch` hang FOREVER (Node fetch has no default timeout) — observed as a
683
+ // sync stuck for 20-40min at idle CPU with zero progress. With it, a hung
684
+ // upload aborts, surfaces as an error, and goes through the retry path below.
685
+ const UPLOAD_TIMEOUT_MS = Number(process.env.CHAT_RECALL_UPLOAD_TIMEOUT_MS) || 90_000;
686
+ // Single POST with retry. Retryable: network/abort errors, HTTP 429 (obeying
687
+ // Retry-After — the server's own rate signal), and 5xx. Non-429 4xx (bad
688
+ // token, oversized body) is fatal: retrying can't fix it.
689
+ const post = async (body) => {
690
+ const payload = JSON.stringify(body);
691
+ const RETRY_DELAYS_MS = [2000, 8000, 30000];
692
+ for (let attempt = 0;; attempt++) {
693
+ const ac = new AbortController();
694
+ const timer = setTimeout(() => ac.abort(new Error(`upload timed out after ${UPLOAD_TIMEOUT_MS}ms`)), UPLOAD_TIMEOUT_MS);
695
+ try {
696
+ const res = await fetch(`${base}/api/sync`, {
697
+ method: 'POST',
698
+ // No token ⇒ no auth header: a no-auth local server (AUTH_PROVIDER=none)
699
+ // accepts it as the single 'default' tenant.
700
+ headers: cred.token
701
+ ? { 'content-type': 'application/json', authorization: `Bearer ${cred.token}` }
702
+ : { 'content-type': 'application/json' },
703
+ body: payload,
704
+ signal: ac.signal,
705
+ });
706
+ if (res.ok) {
707
+ const body = await res.json().catch(() => ({}));
708
+ if (body.cli && body.cli.version) {
709
+ const { planAutoUpdate, runAutoUpdate } = await import('./auto-update.js');
710
+ const authHeaders = cred.token ? { authorization: `Bearer ${cred.token}` } : {};
711
+ const plan = planAutoUpdate(base, { cli: body.cli }, ownVersion, process.env.CHAT_RECALL_AUTO_UPDATE);
712
+ if (plan.update) {
713
+ void runAutoUpdate(base, authHeaders, ownVersion).catch(() => { });
714
+ }
715
+ }
716
+ break;
717
+ }
718
+ const retryable = res.status === 429 || res.status >= 500;
719
+ const text = await res.text().catch(() => '');
720
+ // Auth rejection is a standing condition, not a glitch: the device
721
+ // token was revoked or the tenant is gone. Every future sync will fail
722
+ // the same way until the human reconnects — say exactly that.
723
+ if (res.status === 401 || res.status === 403) {
724
+ throw new Error(`server rejected this machine's device token (HTTP ${res.status} — revoked?). Reconnect with: chat-recall login ${base}`);
725
+ }
726
+ if (!retryable)
727
+ throw new Error(`sync failed: HTTP ${res.status} ${text}`);
728
+ if (attempt >= RETRY_DELAYS_MS.length)
729
+ throw new Error(`sync failed after ${attempt + 1} attempts: HTTP ${res.status} ${text}`);
730
+ // 429: obey Retry-After (seconds) when the server sends it; else fall
731
+ // back to the backoff schedule. This puts the pacing decision where it
732
+ // belongs — on the server that knows its own load.
733
+ const retryAfterS = Number(res.headers.get('retry-after'));
734
+ const waitMs = res.status === 429 && retryAfterS > 0 ? retryAfterS * 1000 : RETRY_DELAYS_MS[attempt];
735
+ console.error(`[sync] HTTP ${res.status} from ${base} — backing off ${Math.round(waitMs / 1000)}s (attempt ${attempt + 1})`);
736
+ await sleep(waitMs);
737
+ continue;
738
+ }
739
+ catch (err) {
740
+ const fatal = err instanceof Error && /HTTP 4\d\d|sync failed after/.test(err.message);
741
+ if (fatal || attempt >= RETRY_DELAYS_MS.length)
742
+ throw err;
743
+ console.error(`[sync] upload attempt ${attempt + 1} failed (${err instanceof Error ? err.message : err}) — retrying in ${RETRY_DELAYS_MS[attempt] / 1000}s`);
744
+ await sleep(RETRY_DELAYS_MS[attempt]);
745
+ continue;
746
+ }
747
+ finally {
748
+ clearTimeout(timer);
749
+ }
750
+ }
751
+ if (throttleMs > 0)
752
+ await sleep(throttleMs);
753
+ };
754
+ // Bounded-concurrency pool. Up to `maxInflight` POSTs run at once; awaiting a
755
+ // free slot in `submit` is what paces the producer (natural backpressure).
756
+ // A POST that fails after its own retries is captured into `poolError` and
757
+ // re-thrown at the next slot-wait / drain, so a fatal error still aborts the
758
+ // whole sync (matching the old serial behaviour).
759
+ const inflight = new Set();
760
+ let poolError = null;
761
+ const submit = async (body, after) => {
762
+ while (inflight.size >= maxInflight)
763
+ await Promise.race(inflight);
764
+ if (poolError)
765
+ throw poolError;
766
+ let task;
767
+ task = (async () => { await post(body); after(); })()
768
+ .catch((e) => { poolError = poolError ?? e; })
769
+ .finally(() => { inflight.delete(task); });
770
+ inflight.add(task);
771
+ };
772
+ const drainInflight = async () => {
773
+ await Promise.allSettled(inflight);
774
+ if (poolError)
775
+ throw poolError;
776
+ };
777
+ // Track every local session id we see this sync. After the walk, any
778
+ // session id recorded in the per-server ledger but missing locally is a
779
+ // deletion → ship a tombstone so the server removes it. This only covers
780
+ // sessions this device previously synced; deletions on other devices are
781
+ // handled by their own collectors.
782
+ const localSessionIds = new Set();
783
+ // session_id → content hash for the FULL conversations shipped this run, so
784
+ // the convBatch ack can stamp the ledger `h` alongside {m,o,s}. A later tick
785
+ // that re-classifies the session as FULL on an mtime-only change then matches
786
+ // this hash and skips the rebuild entirely.
787
+ const shippedHash = new Map();
788
+ // `flatten`: each add() is a GROUP of rows that must land in the same
789
+ // POST (e.g. one session's findings — the server replaces a session's
790
+ // findings wholesale, so splitting a session across two batches would
791
+ // lose the first half).
792
+ const makeBatcher = (field, maxItems, flatten = false, onFlush) => {
793
+ let batch = [];
794
+ let batchBytes = 0;
795
+ return {
796
+ sent: 0,
797
+ async add(row) {
798
+ const size = JSON.stringify(row).length;
799
+ if (batch.length > 0 && (batchBytes + size > MAX_BATCH_BYTES || batch.length >= maxItems))
800
+ await this.drain();
801
+ batch.push(row);
802
+ batchBytes += size;
803
+ },
804
+ async drain() {
805
+ if (batch.length === 0)
806
+ return;
807
+ const rows = flatten ? batch.flat() : batch;
808
+ batch = [];
809
+ batchBytes = 0;
810
+ // Hand off to the concurrency pool; `sent`/onFlush (e.g. markSynced)
811
+ // fire only after the server acks THIS batch, same as before.
812
+ await submit({ [field]: rows }, () => { this.sent += rows.length; onFlush?.(rows); });
813
+ },
814
+ };
815
+ };
816
+ const base = cred.serverUrl.replace(/\/$/, '');
817
+ const ledger = opts.useLedger ? getSyncedRows(base) : null;
818
+ const convBatch = makeBatcher('conversations', 25, false, (rows) => {
819
+ // Server acked this batch — record exactly these sessions at exactly
820
+ // the mtimes that were shipped. Failures never reach here, so an
821
+ // unsynced session stays unsynced and retries next tick.
822
+ //
823
+ // For append-only backends, also stamp the byte cursor (o=size, s=size) so
824
+ // the next tick can APPEND from here instead of re-doing a FULL sync. We
825
+ // read the CURRENT file size (not the size at ship time) — the file may
826
+ // have grown between ship and ack, but that's fine: the next tick's gate
827
+ // sees size > ack.s and APPENDs the new tail. If it shrank (rotation), the
828
+ // gate falls back to FULL.
829
+ try {
830
+ markSynced(base, rows.map((r) => {
831
+ // Record the SAME offset the conv shipped to the server (conv.from_offset
832
+ // = build-time file size), NOT a freshly-stat'd size — otherwise an
833
+ // actively-growing file's ledger offset would differ from the server's
834
+ // stored `o`, and every subsequent append would miss the continuity
835
+ // check and fall back to FULL. Consistency here is what lets the active
836
+ // session actually append.
837
+ const off = typeof r.from_offset === 'number' ? r.from_offset : 0;
838
+ const hash = shippedHash.get(r.session_id);
839
+ // acked: the server accepted the batch containing this conversation, so
840
+ // the cursor is a real delivery receipt.
841
+ return off > 0
842
+ ? { id: r.session_id, mtime: r.mtime, offset: off, size: off, hash, acked: true }
843
+ : { id: r.session_id, mtime: r.mtime, hash };
844
+ }));
845
+ }
846
+ catch { /* ledger is local bookkeeping — never fail an upload over it */ }
847
+ });
848
+ // ── Append batcher (tail-only sync) ──────────────────────────────
849
+ // See docs/SYNC-INCREMENTAL.md. Append conversations ship with append:true;
850
+ // the server appends chunks + merges the envelope. The server may respond
851
+ // with full_resync_needed:[ids] when it has no prior envelope for a session
852
+ // (data loss, first sync) — those sessions are NOT marked synced, so the
853
+ // next tick re-classifies them as FULL (offset 0 or absent → full).
854
+ //
855
+ // Append payloads are small (tail only), so we batch up to 50 per POST and
856
+ // inspect the response body for the full_resync_needed list. This path does
857
+ // NOT go through the shared `post`/`submit` pool — it needs the response
858
+ // body, which the shared path discards.
859
+ const appendResults = { uploaded: 0, fullResyncNeeded: new Set() };
860
+ const postAppend = async (convs) => {
861
+ const headers = { 'content-type': 'application/json' };
862
+ if (cred.token)
863
+ headers.authorization = `Bearer ${cred.token}`;
864
+ const ac = new AbortController();
865
+ const timer = setTimeout(() => ac.abort(new Error(`upload timed out after ${UPLOAD_TIMEOUT_MS}ms`)), UPLOAD_TIMEOUT_MS);
866
+ try {
867
+ const res = await fetch(`${base}/api/sync`, {
868
+ method: 'POST', headers,
869
+ body: JSON.stringify({ conversations: convs }),
870
+ signal: ac.signal,
871
+ });
872
+ if (!res.ok)
873
+ throw new Error(`append sync failed: HTTP ${res.status} ${await res.text().catch(() => '')}`);
874
+ const body = await res.json().catch(() => ({}));
875
+ if (body.cli && body.cli.version) {
876
+ const { planAutoUpdate, runAutoUpdate } = await import('./auto-update.js');
877
+ const authHeaders = cred.token ? { authorization: `Bearer ${cred.token}` } : {};
878
+ const plan = planAutoUpdate(base, { cli: body.cli }, ownVersion, process.env.CHAT_RECALL_AUTO_UPDATE);
879
+ if (plan.update) {
880
+ void runAutoUpdate(base, authHeaders, ownVersion).catch(() => { });
881
+ }
882
+ }
883
+ const fullNeeded = new Set(body.full_resync_needed ?? []);
884
+ // Mark synced ONLY the sessions the server accepted as appends. Sessions
885
+ // the server asked to full-resync stay unmarked → next tick = FULL.
886
+ const acked = convs.filter((c) => !fullNeeded.has(c.session_id));
887
+ if (acked.length > 0) {
888
+ try {
889
+ markSynced(base, acked.map((c) => ({
890
+ id: c.session_id,
891
+ mtime: c.mtime,
892
+ offset: c.from_offset,
893
+ size: c.from_offset, // size = new offset (file grew to here)
894
+ acked: true, // server merged this tail
895
+ })));
896
+ }
897
+ catch { /* ledger — never fail an upload over it */ }
898
+ }
899
+ appendResults.uploaded += acked.length;
900
+ for (const id of fullNeeded)
901
+ appendResults.fullResyncNeeded.add(id);
902
+ }
903
+ finally {
904
+ clearTimeout(timer);
905
+ }
906
+ };
907
+ let appendBatch = [];
908
+ const APPEND_BATCH_SIZE = 50;
909
+ const appendFlush = async () => {
910
+ if (appendBatch.length === 0)
911
+ return;
912
+ const batch = appendBatch;
913
+ appendBatch = [];
914
+ await postAppend(batch);
915
+ };
916
+ const appendAdd = async (conv) => {
917
+ appendBatch.push(conv);
918
+ if (appendBatch.length >= APPEND_BATCH_SIZE)
919
+ await appendFlush();
920
+ };
921
+ const itemsBatch = makeBatcher('items', 100);
922
+ const linksBatch = makeBatcher('links', 2000);
923
+ const derivedBatch = makeBatcher('derived', 50);
924
+ const kgEntityBatch = makeBatcher('kg_entities', 2000);
925
+ const kgTripleBatch = makeBatcher('kg_triples', 1000);
926
+ // Secret findings ship per-session as a GROUP (flatten=true) — the server
927
+ // replaces a session's findings wholesale, so they must land in one POST.
928
+ const findingsBatch = makeBatcher('findings', 200, true);
929
+ // External detectors (gitleaks/trufflehog) are OPT-IN and default OFF:
930
+ // isSecretScannerAvailable() returns false unless CHAT_RECALL_EXTERNAL_SCANNERS
931
+ // is set AND a binary is on PATH. Resolved once (each check spawns `which`).
932
+ // CHAT_RECALL_SCAN_SECRETS=0 opts out even when enabled (e.g. to keep a huge
933
+ // backfill fast); upload.findings=false disables shipping too.
934
+ // NOTE: this flag gates ONLY the external binaries. Builtin regex findings
935
+ // and tenant rules run for every user further down (buildConversationSync),
936
+ // and redaction of everything shipped is unconditional.
937
+ const scanSecrets = isSecretScannerAvailable() && upload.findings && process.env.CHAT_RECALL_SCAN_SECRETS !== '0';
938
+ const verifySecretsScan = verifySecrets;
939
+ // ── Live KG accumulator. Triples are extracted on the spot from each
940
+ // item's redacted text (no local knowledge-graph DB is read). Dedupe by
941
+ // subject|predicate|object|valid_from; entity types come from `is_a`
942
+ // triples seen in the same pass, defaulting to 'unknown' otherwise.
943
+ const kgSeen = new Set();
944
+ const kgEntityType = new Map();
945
+ const kgRows = [];
946
+ const accumulateKg = (triples, sourceSession) => {
947
+ for (const t of triples) {
948
+ const key = `${t.subject}|${t.predicate}|${t.object}|${t.validFrom ?? ''}`;
949
+ if (kgSeen.has(key))
950
+ continue;
951
+ kgSeen.add(key);
952
+ if (t.predicate === 'is_a' && !kgEntityType.has(t.subject))
953
+ kgEntityType.set(t.subject, t.object);
954
+ kgRows.push({
955
+ subject: t.subject, predicate: t.predicate, object: t.object,
956
+ valid_from: t.validFrom ?? null, valid_to: null,
957
+ confidence: t.confidence ?? 1.0, source_session: sourceSession,
958
+ });
959
+ }
960
+ };
961
+ let skipped = 0, redactions = 0, walked = 0;
962
+ let scanMs = 0, scanned = 0; // secret-scan metrics
963
+ const slice = refs.slice(0, opts.limit ?? refs.length);
964
+ // CHAT_RECALL_SYNC_TRACE=1: phase + progress markers with heap usage, to
965
+ // stderr. This is how the 2026-07-03 walk OOM was localized — keep it.
966
+ const trace = process.env.CHAT_RECALL_SYNC_TRACE === '1'
967
+ ? (msg) => { const m = process.memoryUsage(); console.error(`[sync-trace] ${msg} · heap=${(m.heapUsed / 1048576).toFixed(0)}MB rss=${(m.rss / 1048576).toFixed(0)}MB`); }
968
+ : null;
969
+ trace?.(`walk start: ${slice.length} ref(s) → ${base}`);
970
+ // ── Derived-field reconciliation (native title, …) — runs FIRST, BEFORE the
971
+ // heavy conversation walk, so it's never blocked by per-session git/replay
972
+ // compute (titles flow even when the conversation phase is slow). Pushes only
973
+ // {session_id, field, value}, never a conversation. Cheap pass over the
974
+ // sessions this sync listed; a one-time full pass per field when its version
975
+ // bumped or a re-scan was forced. Best-effort — a reconcile failure must not
976
+ // fail the conversation sync.
977
+ if (upload.sessionMeta) {
978
+ try {
979
+ await reconcileFieldsForTarget(cred, () => listAvailableBackends().flatMap((b) => { try {
980
+ return b.listSessions({});
981
+ }
982
+ catch {
983
+ return [];
984
+ } }), { convRefs: slice });
985
+ }
986
+ catch (e) {
987
+ console.error(`[sync] field reconcile (${base}): ${e instanceof Error ? e.message : e}`);
988
+ }
989
+ }
990
+ trace?.('field reconcile done');
991
+ // Shared gate: classify each session's sync mode for this tick. Used by
992
+ // both the batch pre-scan and the upload loop so they stay in lockstep.
993
+ // See docs/SYNC-INCREMENTAL.md for the skip/append/full decision.
994
+ const backendFor = (ref) => getBackendForId(ref.prefixedId);
995
+ const modeOf = (ref) => {
996
+ if (excludeTools.has(ref.toolId))
997
+ return 'skip';
998
+ if (excluded(ref.projectPath))
999
+ return 'skip';
1000
+ if (!includedProject(ref.projectPath))
1001
+ return 'skip';
1002
+ if (!ledger)
1003
+ return ref.mtime >= (opts.sinceMs ?? 0) ? 'full' : 'skip';
1004
+ const ack = ledger.get(ref.prefixedId);
1005
+ const backend = backendFor(ref);
1006
+ // A session that exists in more than one home cannot be tail-appended: the
1007
+ // size is the SUM across copies while a byte offset addresses one file, so
1008
+ // an append would ship bytes that do not match the recorded offset. Force
1009
+ // FULL, which ships the unioned container and is correct either way.
1010
+ const split = !!backend?.spansMultipleSources?.(ref.prefixedId);
1011
+ const isAO = !!backend?.isAppendOnly?.() && !split;
1012
+ const size = isAO ? (backend?.fileSize?.(ref.prefixedId) ?? 0) : 0;
1013
+ return syncMode(ack, ref.mtime, size, extractorVersionForTool(ref.toolId), isAO);
1014
+ };
1015
+ const willBuild = (ref) => modeOf(ref) !== 'skip';
1016
+ // ── Batch secret scan (optional external detectors) ────────────────
1017
+ // Only runs when CHAT_RECALL_EXTERNAL_SCANNERS=1 and gitleaks/trufflehog are
1018
+ // on PATH — `scanSecrets` above already folds that gate in, so by default
1019
+ // nothing here executes and no raw text is ever written to disk.
1020
+ //
1021
+ // gitleaks/trufflehog are directory-capable and report each finding's file.
1022
+ // Materialize to-be-FULL-synced sessions' raw text into a temp dir (filename
1023
+ // = sanitized session id), scan the dir ONCE per detector, then map findings
1024
+ // back to sessions by filename. That replaces 2·N cold process spawns (the
1025
+ // dominant per-session cost) with 2 per SLICE. null ⇒ no batch ran (scanning
1026
+ // off) and buildConversationSync falls back to its per-session path.
1027
+ //
1028
+ // SLICED, not one big dir: this text is PRE-redaction — real credentials in
1029
+ // cleartext on the customer's disk — and cleanup lives in a `finally` that a
1030
+ // SIGKILL/OOM skips (that is how 3.4GB / 309k files were left behind in
1031
+ // /tmp). Bounding the dir to BATCH_SCAN_MAX_BYTES bounds the worst-case
1032
+ // exposure and the worst-case leak to that many bytes, at a cost of 2 extra
1033
+ // spawns per additional slice.
1034
+ //
1035
+ // APPEND sessions are excluded — they skip external detectors (the head was
1036
+ // already scanned; the tail uses builtin regex only; the FULL re-sync when
1037
+ // the session closes covers the whole file).
1038
+ let externalFindings = null;
1039
+ if (scanSecrets) {
1040
+ externalFindings = new Map();
1041
+ // Oversized sessions are excluded: exportRawSession would materialize the
1042
+ // whole file (hundreds of MB — this loop is what OOM-killed the daemon).
1043
+ // Their bounded tail still gets the builtin scan in buildConversationSync.
1044
+ const toScan = slice.filter((ref) => modeOf(ref) === 'full' && sessionFileBytes(ref) <= FULL_BUILD_MAX_BYTES);
1045
+ trace?.(`batch secret scan start: ${toScan.length} session(s), ≤${(BATCH_SCAN_MAX_BYTES / 1048576).toFixed(0)}MB per slice`);
1046
+ const batch = batchScanExternal(toScan.map((ref) => ({
1047
+ id: ref.prefixedId,
1048
+ text: () => {
1049
+ const backend = getBackendForId(ref.prefixedId) ?? getBackend('claude');
1050
+ const exp = backend.exportRawSession(ref.prefixedId);
1051
+ const container = exp ? buildRawContainer(exp) : null;
1052
+ return container ? container.files.map((f) => f.text).join('\n') : null;
1053
+ },
1054
+ })), { verifyOnly: verifySecretsScan });
1055
+ externalFindings = batch.findings;
1056
+ scanned = batch.scanned;
1057
+ scanMs = batch.scanMs;
1058
+ trace?.(`batch secret scan done: ${scanned} session(s) in ${batch.slices} slice(s)`);
1059
+ }
1060
+ trace?.('conversation walk start');
1061
+ let traceN = 0;
1062
+ for (const ref of slice) {
1063
+ if (trace && ++traceN % 500 === 0)
1064
+ trace(`walk ${traceN}/${slice.length} · at ${ref.prefixedId}`);
1065
+ localSessionIds.add(ref.prefixedId);
1066
+ const mode = modeOf(ref);
1067
+ if (mode === 'skip') {
1068
+ skipped++;
1069
+ continue;
1070
+ }
1071
+ trace?.(`build (${mode}) ${ref.prefixedId} · ${(sessionFileBytes(ref) / 1048576).toFixed(1)}MB`);
1072
+ // ── APPEND: tail-only ship (no raw_b64, no telemetry, no derived) ──
1073
+ // `promotedForFindings` flips true when the tail carried a secret finding:
1074
+ // the server replaces findings wholesale per session, so a tail-only ship
1075
+ // would WIPE the head's findings. Instead of hiding the tail's secret until
1076
+ // some far-future FULL re-sync, we fall through to the FULL path NOW (which
1077
+ // re-scans head+tail and ships the complete set). Rare event — a secret
1078
+ // pasted into a long-lived, still-appending session.
1079
+ let promotedForFindings = false;
1080
+ if (mode === 'append' && ledger) {
1081
+ const ack = ledger.get(ref.prefixedId);
1082
+ try {
1083
+ const tail = await buildConversationTail(ref, ack.o ?? 0, { mapPath, tenantRules });
1084
+ if (tail && tail.findings.length > 0) {
1085
+ promotedForFindings = true; // fall through to FULL (do NOT continue)
1086
+ }
1087
+ else if (tail) {
1088
+ redactions += tail.redactions;
1089
+ await appendAdd(tail.conv);
1090
+ walked++;
1091
+ if (walked % 250 === 0)
1092
+ console.error(`[sync] ${walked}/${slice.length} sessions…`);
1093
+ continue;
1094
+ }
1095
+ else {
1096
+ // Nothing new to ship (tail empty/unparseable). Stamp the MTIME only.
1097
+ //
1098
+ // This used to re-stamp the cursor at the local fileSize() to stop the
1099
+ // gate retrying every tick. That was a delivery receipt for bytes the
1100
+ // server never received: once fileSize() started summing profile homes,
1101
+ // the cursor jumped to the union size and the session was sealed as
1102
+ // complete with a third of its records missing (7adc748c: 2004 of
1103
+ // 2698). mtime alone gives the same anti-thrash — syncMode SKIPs an
1104
+ // unchanged file on mtime — while leaving the cursor where the server
1105
+ // actually is, so the next growth re-attempts from the true offset.
1106
+ try {
1107
+ markSynced(base, [{ id: ref.prefixedId, mtime: ref.mtime }]);
1108
+ }
1109
+ catch { /* ledger — best-effort */ }
1110
+ skipped++;
1111
+ continue;
1112
+ }
1113
+ }
1114
+ catch (err) {
1115
+ console.error(`[sync] append tail failed for ${ref.prefixedId} (will retry as full next tick): ${err instanceof Error ? err.message : err}`);
1116
+ // Don't mark synced → next tick re-classifies. If the tail failed, a
1117
+ // FULL sync is the safe fallback (the gate sees an unchanged cursor).
1118
+ skipped++;
1119
+ continue;
1120
+ }
1121
+ }
1122
+ // ── FULL: the existing whole-conversation path ──
1123
+ // Canonical envelope (live): parse the transcript NOW with THE parser —
1124
+ // no local content cache is read or warmed. What ships is exactly what
1125
+ // the local dashboard would render for the same file.
1126
+ const mtime = Math.floor(ref.mtime) || 0;
1127
+ // Hand this session its slice of the batch-scan results (when scanning is
1128
+ // on). `?? []` means "batch ran, found nothing here" — distinct from the
1129
+ // `undefined` that triggers the per-session fallback inside the builder.
1130
+ // A session promoted from append (secret in the tail) wasn't in the batch
1131
+ // scan set — pass undefined so buildConversationSync runs the per-session
1132
+ // external detector over the whole file, yielding the COMPLETE finding set.
1133
+ const precomputedExternal = promotedForFindings
1134
+ ? undefined
1135
+ : (externalFindings ? (externalFindings.get(ref.prefixedId) ?? []) : undefined);
1136
+ // Only trust the ledger's content hash for the skip-gate when the row is at
1137
+ // the current extractor version — a version bump must force a real re-derive
1138
+ // even if the transcript bytes are unchanged.
1139
+ const ackRow = ledger?.get(ref.prefixedId);
1140
+ const priorContentHash = ackRow && ackRow.v >= extractorVersionForId(ref.prefixedId) ? ackRow.h : undefined;
1141
+ const built = await buildConversationSync(ref, mtime, { mapPath, includeRaw: upload.raw, includeMeta: upload.sessionMeta, scanSecrets, verifySecrets: verifySecretsScan, tenantRules, precomputedExternal, priorContentHash });
1142
+ if (built && 'unchanged' in built) {
1143
+ // Content byte-identical to the last FULL sync to this target (mtime bumped
1144
+ // / resume rewrite / OpenCode summary bump). Nothing to re-ship, re-extract
1145
+ // or git-replay. Re-stamp the ledger exactly as a real FULL ack would —
1146
+ // mtime, the (unchanged) hash, and the current byte cursor — so the NEXT
1147
+ // tick reaches a true SKIP and doesn't even re-export. Byte-identical
1148
+ // content ⇒ identical size, so the cursor we write equals the one already
1149
+ // there for append-only backends; for OpenCode fileSize() is absent (0).
1150
+ if (ledger) {
1151
+ try {
1152
+ // Content is byte-identical to the last FULL sync, so the server
1153
+ // already holds these bytes — that makes this a legitimate ack. But
1154
+ // only up to the cursor it already had: if the local size now EXCEEDS
1155
+ // it (e.g. fileSize() sums a second profile home that was never
1156
+ // shipped), advancing would again receipt undelivered bytes.
1157
+ const off = getBackendForId(ref.prefixedId)?.fileSize?.(ref.rawId) ?? 0;
1158
+ const priorCursor = ackRow?.s ?? 0;
1159
+ const safeToAck = off > 0 && off <= priorCursor;
1160
+ markSynced(base, [safeToAck
1161
+ ? { id: ref.prefixedId, mtime: ref.mtime, offset: off, size: off, hash: built.srcHash, acked: true }
1162
+ : { id: ref.prefixedId, mtime: ref.mtime, hash: built.srcHash }]);
1163
+ }
1164
+ catch { /* ledger is local bookkeeping — never fail a sync over it */ }
1165
+ }
1166
+ skipped++;
1167
+ continue;
1168
+ }
1169
+ if (!built) {
1170
+ // Nothing worth shipping for THIS content — empty, unparseable, or one
1171
+ // of chat-recall's own internal LLM invocations (roughly half of all
1172
+ // "sessions" on a busy machine). Every null verdict is a pure function
1173
+ // of file content, so ack it at this mtime: without the ack these
1174
+ // sessions re-classify as FULL on every full walk, forever — observed
1175
+ // as ~5k sessions re-exported and re-secret-scanned per 15-min
1176
+ // heartbeat (4GB+ heap peak, 12-minute walks, heap-OOM crash loops).
1177
+ // A real content change bumps mtime and re-evaluates.
1178
+ if (ledger) {
1179
+ try {
1180
+ markSynced(base, [{ id: ref.prefixedId, mtime: ref.mtime }]);
1181
+ }
1182
+ catch { /* ledger is local bookkeeping — never fail a sync over it */ }
1183
+ }
1184
+ skipped++;
1185
+ continue;
1186
+ }
1187
+ redactions += built.redactions;
1188
+ // Remember the content hash so the convBatch ack stamps ledger `h` (enables
1189
+ // the mtime-only skip on a later tick). Undefined for the oversized-tail path.
1190
+ if (built.srcHash)
1191
+ shippedHash.set(ref.prefixedId, built.srcHash);
1192
+ await convBatch.add(built.conv);
1193
+ // Ship this session's secret findings as one group (server replaces wholesale).
1194
+ if (built.findings.length > 0)
1195
+ await findingsBatch.add(built.findings);
1196
+ // Knowledge graph: extract triples live from the redacted conversation
1197
+ // text (no local KG DB is read). Accumulated + deduped, flushed at the end.
1198
+ if (upload.sessionMeta) {
1199
+ accumulateKg(extractEntities(built.kgText, { projectPath: built.projectPath, sourceType: 'session', sessionId: ref.prefixedId, validFrom: ref.mtime ? new Date(ref.mtime).toISOString().slice(0, 10) : undefined }), ref.prefixedId);
1200
+ }
1201
+ // Derived data — the CLI has the FS/git/transcript, the server doesn't.
1202
+ // Always computed live (diff/outcome/commits/markers + badge row); no
1203
+ // local compute/outcome cache is read or warmed. Oversized transcripts are
1204
+ // excluded: replay/outcome re-read the WHOLE file (readEvents), and a
1205
+ // 353MB session with 100MB+ single JSON lines OOMed the walk right after
1206
+ // its (bounded) conversation shipped.
1207
+ if (upload.sessionMeta && sessionFileBytes(ref) <= FULL_BUILD_MAX_BYTES) {
1208
+ try {
1209
+ const count = { redactions: 0 };
1210
+ await derivedBatch.add(collectDerived(ref, mtime, count));
1211
+ redactions += count.redactions;
1212
+ }
1213
+ catch (err) {
1214
+ // Derived is best-effort — the conversation still shipped — but it must
1215
+ // NOT be silent: a derived failure means this session shows empty
1216
+ // diffs/Activity on the server, and a SILENT swallow here is exactly why
1217
+ // a stale-data regression went unnoticed for days. Log it so it's
1218
+ // visible and diagnosable.
1219
+ console.error(`[sync] derived compute failed for ${ref.prefixedId} (diffs/Activity will be empty for it): ${err instanceof Error ? err.message : err}`);
1220
+ }
1221
+ }
1222
+ // Fork lineage link (collected at index time too; shipping here keeps
1223
+ // sync self-sufficient for sessions the indexer hasn't visited).
1224
+ try {
1225
+ const { detectForkPredecessor } = await import('@chat-recall/engine/transcript/raw.js');
1226
+ if (ref.fullPath) {
1227
+ const predecessor = detectForkPredecessor(ref.fullPath);
1228
+ if (predecessor) {
1229
+ await linksBatch.add({
1230
+ source_type: 'session', source_id: ref.prefixedId,
1231
+ target_type: 'session', target_id: predecessor,
1232
+ link_type: 'forked_from', confidence: 1.0,
1233
+ });
1234
+ }
1235
+ }
1236
+ }
1237
+ catch { /* best-effort */ }
1238
+ walked++;
1239
+ if (walked % 250 === 0)
1240
+ console.error(`[sync] ${walked}/${slice.length} sessions…`);
1241
+ }
1242
+ // ── Non-session sources: plan/task/claude_md/history/paste/diary/skill/
1243
+ // mcp/command/agent/hook/plugin. Same registry the indexer uses, so
1244
+ // sync coverage tracks index coverage automatically. Session-type
1245
+ // sources are skipped — sessions ship as conversations above.
1246
+ // (excludeTools is session-only: source plugins don't carry a tool id.)
1247
+ // Per-tool item-extractor versions (see sync-ledger). A tool whose recorded
1248
+ // version is below its current version re-ships ALL its items ONCE (mtime
1249
+ // gate bypassed), so an attribution/extraction fix self-heals — same idea as
1250
+ // sessions, at the tool level. Unrecorded tools seed at EXTRACTOR_VERSION
1251
+ // (base), so only tools bumped ABOVE base (e.g. agy) re-ship on first run —
1252
+ // not a blanket re-ship of every item.
1253
+ const itemVersions = loadItemVersions(base);
1254
+ const seenTools = new Set();
1255
+ const registry = buildSourceRegistry();
1256
+ for (const sourceType of registry.getRegisteredTypes()) {
1257
+ if (sourceType === 'session')
1258
+ continue;
1259
+ for (const source of registry.getAll(sourceType)) {
1260
+ let discovered;
1261
+ try {
1262
+ discovered = source.discover();
1263
+ }
1264
+ catch {
1265
+ continue;
1266
+ }
1267
+ try {
1268
+ for await (const item of discovered) {
1269
+ const itemTool = toolOfId(item.id);
1270
+ seenTools.add(itemTool);
1271
+ const versionStale = (itemVersions[itemTool] ?? EXTRACTOR_VERSION) < extractorVersionForId(item.id);
1272
+ if ((item.mtime || 0) < (opts.sinceMs ?? 0) && !versionStale)
1273
+ continue;
1274
+ if (excluded(item.projectPath || ''))
1275
+ continue;
1276
+ if (!includedProject(item.projectPath || ''))
1277
+ continue;
1278
+ const count = { redactions: 0 };
1279
+ let chunks = [];
1280
+ try {
1281
+ chunks = await source.parse(item);
1282
+ }
1283
+ catch { /* unparseable item still ships metadata */ }
1284
+ // Project id resolved live on THIS machine (git/FS available); the
1285
+ // server stores it verbatim. `ignored:` rules mean "no project".
1286
+ const resolved = resolveProjectId(item.projectPath || '');
1287
+ const projectId = resolved.source === 'ignored' ? '' : resolved.id;
1288
+ const redactedChunks = chunks
1289
+ .filter((c) => c.text?.trim())
1290
+ .map((c) => ({
1291
+ text: redactSecrets(c.text, { force: true, count }),
1292
+ chunk_type: c.chunkType || '',
1293
+ // Chunk titles are derived from message text and so can carry a
1294
+ // secret exactly like the body can. This was the one field on the
1295
+ // upload path that shipped raw.
1296
+ title: redactSecrets(c.title || '', { force: true, count }),
1297
+ }));
1298
+ await itemsBatch.add({
1299
+ id: item.id,
1300
+ source_type: item.sourceType,
1301
+ project_id: projectId || undefined,
1302
+ title: redactSecrets(item.title || '', { force: true, count }),
1303
+ project_path: mapPath(item.projectPath || ''),
1304
+ content_preview: redactSecrets(item.contentPreview || '', { force: true, count }),
1305
+ mtime: Math.floor(item.mtime) || 0,
1306
+ extra: item.extra ? redactDeep(item.extra, count) : undefined,
1307
+ chunks: redactedChunks,
1308
+ });
1309
+ // KG: extract triples live from this item's redacted chunk text.
1310
+ if (upload.sessionMeta && redactedChunks.length > 0) {
1311
+ accumulateKg(extractEntities(redactedChunks.map((c) => c.text).join('\n'), { projectPath: item.projectPath || '', sourceType: item.sourceType, validFrom: item.mtime ? new Date(Number(item.mtime)).toISOString().slice(0, 10) : undefined }), null);
1312
+ }
1313
+ try {
1314
+ for (const l of await source.extractLinks(item)) {
1315
+ await linksBatch.add({
1316
+ source_type: l.sourceType, source_id: l.sourceId,
1317
+ target_type: l.targetType, target_id: l.targetId,
1318
+ link_type: l.linkType, confidence: l.confidence,
1319
+ });
1320
+ }
1321
+ }
1322
+ catch { /* links are best-effort */ }
1323
+ redactions += count.redactions;
1324
+ }
1325
+ }
1326
+ catch (err) {
1327
+ // One broken source must not kill the sync — but a SYSTEMATICALLY failing
1328
+ // source (bad discover/parse) would otherwise silently ship zero items
1329
+ // for its type forever, with no signal. Log it (once per source per run).
1330
+ console.error(`[sync] source '${sourceType}' failed mid-walk (its items may be incomplete this run): ${err instanceof Error ? err.message : err}`);
1331
+ }
1332
+ }
1333
+ }
1334
+ // ── Knowledge graph (live). Triples were extracted on the spot from each
1335
+ // session/item's redacted text during the walks above — so what ships is
1336
+ // exactly the facts derivable from the sessions in THIS sync (which is
1337
+ // also the incremental-correct set: no manual-fact replay, no local KG
1338
+ // DB). Entities are derived from the triple subjects/objects, typed from
1339
+ // any `is_a` triple seen in the same pass.
1340
+ if (upload.sessionMeta && kgRows.length > 0) {
1341
+ const entityNames = new Set();
1342
+ for (const t of kgRows) {
1343
+ const count = { redactions: 0 };
1344
+ await kgTripleBatch.add(redactDeep(t, count));
1345
+ redactions += count.redactions;
1346
+ entityNames.add(t.subject);
1347
+ entityNames.add(t.object);
1348
+ }
1349
+ for (const name of entityNames) {
1350
+ const count = { redactions: 0 };
1351
+ await kgEntityBatch.add(redactDeep({ name, type: kgEntityType.get(name) ?? 'unknown', properties: {} }, count));
1352
+ redactions += count.redactions;
1353
+ }
1354
+ }
1355
+ // ── Drain every batcher (the walks above flushed full windows already;
1356
+ // this ships the tails). The append batch flushes BEFORE the inflight
1357
+ // barrier so its full_resync_needed set is populated before we process it.
1358
+ await appendFlush();
1359
+ // Server said full_resync_needed for these append sessions (no prior
1360
+ // envelope on the server). Wipe their ledger rows so the next tick
1361
+ // classifies them as FULL (never-synced) instead of retrying append or
1362
+ // skipping — which would leave the server without the data forever.
1363
+ for (const id of appendResults.fullResyncNeeded) {
1364
+ try {
1365
+ markFullResync(base, id);
1366
+ }
1367
+ catch { /* ledger — best-effort */ }
1368
+ }
1369
+ await convBatch.drain();
1370
+ await itemsBatch.drain();
1371
+ await linksBatch.drain();
1372
+ await derivedBatch.drain();
1373
+ await kgEntityBatch.drain();
1374
+ await kgTripleBatch.drain();
1375
+ await findingsBatch.drain();
1376
+ // Barrier: wait for every pooled upload to land (and surface any fatal
1377
+ // error) before tombstones/prune, which must run strictly after all data.
1378
+ await drainInflight();
1379
+ // Items shipped successfully → record each seen tool's current item-extractor
1380
+ // version, so a version-triggered re-ship (e.g. the agy attribution fix)
1381
+ // happens exactly ONCE per bump instead of every sync. Best-effort.
1382
+ if (seenTools.size > 0) {
1383
+ try {
1384
+ const rec = {};
1385
+ for (const t of seenTools)
1386
+ rec[t] = extractorVersionForTool(t);
1387
+ saveItemVersions(base, rec);
1388
+ }
1389
+ catch { /* local bookkeeping — best-effort */ }
1390
+ }
1391
+ // ── Tombstones for sessions this device previously synced but that no
1392
+ // longer exist locally (deleted transcript file, cleared cache, etc).
1393
+ // We only tombstone sessions we have a ledger record for — never data
1394
+ // from other devices. STRICTLY full-walk only: a changed-scope walk
1395
+ // lists only recent sessions, so "in ledger but not walked" means
1396
+ // "old and unchanged", NOT "deleted" — tombstoning there would wipe
1397
+ // every quiet session from the server.
1398
+ if (ledger && !changedWalk) {
1399
+ const tombstones = [];
1400
+ for (const [sessionId] of ledger) {
1401
+ if (!localSessionIds.has(sessionId)) {
1402
+ tombstones.push({ session_id: sessionId, deleted_at: Date.now() });
1403
+ }
1404
+ }
1405
+ if (tombstones.length > 0) {
1406
+ await post({ tombstones });
1407
+ // Remove tombstoned ids from the ledger so we don't keep shipping them.
1408
+ const data = getLedgerData(base);
1409
+ for (const t of tombstones)
1410
+ delete data[t.session_id];
1411
+ persistLedgerData(base, data);
1412
+ }
1413
+ }
1414
+ if (opts.prune)
1415
+ await post({ prune_empty_sessions: true });
1416
+ return {
1417
+ uploaded: convBatch.sent + appendResults.uploaded, skipped, redactions,
1418
+ items: itemsBatch.sent, links: linksBatch.sent, findings: findingsBatch.sent, derived: derivedBatch.sent,
1419
+ kgEntities: kgEntityBatch.sent, kgTriples: kgTripleBatch.sent,
1420
+ scanned, scanMs: Math.round(scanMs),
1421
+ };
1422
+ }
1423
+ /** Transcript bytes beyond which a FULL sync must not materialize the whole
1424
+ * file. The FULL pipeline copies the transcript ~5× (parse, trim+redact,
1425
+ * raw export, raw-text join, redacted container) — a 350MB session transiently
1426
+ * needs >1.5GB of heap, which OOM-killed the watch daemon in a restart loop
1427
+ * (ledger never acked → same file retried on every startup tick, forever). */
1428
+ export const FULL_BUILD_MAX_BYTES = Math.max(8, parseInt(process.env.CHAT_RECALL_FULL_BUILD_MAX_MB || '64', 10)) * 1024 * 1024;
1429
+ /** How much of an oversized transcript's tail still ships (newest content). */
1430
+ const FULL_BUILD_TAIL_BYTES = 16 * 1024 * 1024;
1431
+ /** Ceiling on PRE-REDACTION session text materialized on disk at any one moment
1432
+ * by the optional external-detector batch scan. The batch dir is flushed and
1433
+ * deleted every time it crosses this, so a SIGKILL/OOM mid-scan can strand at
1434
+ * most this many bytes of cleartext (the incident stranded 3.4GB), and the
1435
+ * window in which they exist is proportionally short. Raising it buys fewer
1436
+ * detector spawns; lowering it buys a smaller blast radius. */
1437
+ export const BATCH_SCAN_MAX_BYTES = Math.max(1, parseInt(process.env.CHAT_RECALL_BATCHSCAN_MAX_MB || '64', 10)) * 1024 * 1024;
1438
+ /** Bytes an append-only session occupies on disk (0 for non-AO backends). */
1439
+ export function sessionFileBytes(ref) {
1440
+ const backend = getBackendForId(ref.prefixedId);
1441
+ if (!backend?.isAppendOnly?.())
1442
+ return 0;
1443
+ return backend.fileSize?.(ref.prefixedId) ?? 0;
1444
+ }
1445
+ export async function buildConversationSync(ref, mtime, opts = {}) {
1446
+ const mapPath = opts.mapPath ?? ((p) => p);
1447
+ let includeRaw = opts.includeRaw !== false;
1448
+ let includeMeta = opts.includeMeta !== false;
1449
+ let srcHash;
1450
+ // Oversized-transcript guard: ship the newest FULL_BUILD_TAIL_BYTES as this
1451
+ // session's FULL sync instead of materializing the whole file. The envelope
1452
+ // is marked truncated, raw/meta (whole-file reads) are dropped, and
1453
+ // from_offset = file size — so the ledger advances and every later tick is
1454
+ // a normal bounded APPEND. Truncation only loses the transcript's OLD head.
1455
+ const fileBytes = sessionFileBytes(ref);
1456
+ const oversized = fileBytes > FULL_BUILD_MAX_BYTES;
1457
+ let tailText = '';
1458
+ let tailOffsetEnd = 0;
1459
+ // Parse the transcript live — this is the single canonical parse; the
1460
+ // envelope and the redacted text both come from it. When the shadow path
1461
+ // runs (normal-size sessions) it also yields the fullest raw container, which
1462
+ // the secret scan + raw archive below reuse instead of re-exporting.
1463
+ let transcript = null;
1464
+ let fullestContainer = null;
1465
+ if (oversized) {
1466
+ const backend = getBackendForId(ref.prefixedId);
1467
+ if (!backend?.readFromOffset)
1468
+ return null;
1469
+ try {
1470
+ const tail = await backend.readFromOffset(ref.prefixedId, fileBytes - FULL_BUILD_TAIL_BYTES);
1471
+ tailText = tail.text;
1472
+ tailOffsetEnd = tail.newOffset;
1473
+ const messages = parseTailMessages(ref.toolId, tailText);
1474
+ if (messages.length > 0)
1475
+ transcript = { messages, subagents: [] };
1476
+ }
1477
+ catch { /* unreadable tail */ }
1478
+ includeRaw = false; // exportRawSession would read the whole file
1479
+ includeMeta = false; // parseSessionFile would read the whole file
1480
+ // NB: oversized sessions are NOT shadowed — a full export would OOM the
1481
+ // walk (the very reason for the tail path). A session truncated FROM
1482
+ // oversized down to normal falls back to a fresh 'created' shadow; the
1483
+ // recovery command backfills its pre-truncation history from the archive.
1484
+ }
1485
+ else {
1486
+ // Export the raw transcript ONCE, fold it into the shadow archive, and
1487
+ // parse the FULLEST-known state. If an upstream tool truncated the file in
1488
+ // place (e.g. a Claude Code 2.1.20x `--resume` rewrite), the shadow still
1489
+ // holds the lost history, and the merged container — not the shrunken disk
1490
+ // file — is what we parse and ship. The server therefore never sees a
1491
+ // shrink, and every viewer tab keeps its full history.
1492
+ try {
1493
+ const backend = getBackendForId(ref.prefixedId) ?? getBackend('claude');
1494
+ const exp = backend.exportRawSession(ref.prefixedId);
1495
+ // Content-identity gate: hash the raw export (mtime-independent) BEFORE the
1496
+ // shadow merge / parse / redact / KG / git-replay. If it equals what this
1497
+ // target last FULL-synced, nothing downstream can differ — bail now. This
1498
+ // is what stops a resume-truncated or mtime-touched session from being
1499
+ // fully rebuilt on every tick (the export/read still happens — it's how we
1500
+ // know — but everything expensive after it is skipped).
1501
+ if (exp) {
1502
+ try {
1503
+ srcHash = containerSrcHash(buildRawContainer(exp));
1504
+ }
1505
+ catch {
1506
+ srcHash = undefined;
1507
+ }
1508
+ if (srcHash && opts.priorContentHash && srcHash === opts.priorContentHash) {
1509
+ return { unchanged: true, srcHash };
1510
+ }
1511
+ }
1512
+ const shadow = updateShadow(ref.rawId, exp);
1513
+ if (shadow.container) {
1514
+ fullestContainer = shadow.container;
1515
+ if (shadow.status === 'rewrite-merged' && shadow.recovered > 0) {
1516
+ console.error(`[sync] shadow recovered ${shadow.recovered} record(s) for ${ref.prefixedId} after an upstream rewrite/truncation`);
1517
+ }
1518
+ const t = parseTranscriptFromContainer(shadow.container);
1519
+ if (t.messages.length > 0 || t.subagents.length > 0)
1520
+ transcript = { messages: t.messages, subagents: t.subagents };
1521
+ }
1522
+ }
1523
+ catch { /* shadow/export failed — fall back to the live parse below */ }
1524
+ if (!transcript) {
1525
+ try {
1526
+ const t = await parseTranscript(ref.prefixedId);
1527
+ if (t && (t.messages.length > 0 || t.subagents.length > 0))
1528
+ transcript = t;
1529
+ }
1530
+ catch { /* unparseable */ }
1531
+ }
1532
+ }
1533
+ if (!transcript)
1534
+ return null;
1535
+ const textMessages = transcript.messages.filter((m) => m.role !== 'summary' && m.content?.trim());
1536
+ if (textMessages.length === 0 && !transcript.messages.some((m) => m.toolCalls?.length))
1537
+ return null;
1538
+ // Skip chat-recall's OWN LLM invocations that the AI CLI logged as sessions —
1539
+ // summary generation and health-check pings. On this machine ~half of all
1540
+ // "sessions" were these internal calls; indexing them buries the real
1541
+ // conversation list. Anchored to the first user message so a real chat that
1542
+ // merely quotes the prompt is not dropped.
1543
+ const firstUser = textMessages.find((m) => m.role === 'user');
1544
+ if (isInternalToolPrompt(firstUser?.content))
1545
+ return null;
1546
+ // Trim payload bodies (tool inputs/results, thinking — raw replay never
1547
+ // ships), then redact every string. Counts are preserved exactly.
1548
+ const count = { redactions: 0 };
1549
+ const envelope = redactDeep({ v: TRANSCRIPT_VERSION, ...trimTranscriptForSync(transcript) }, count);
1550
+ // Live telemetry: parse the raw session file with THE metadata parser
1551
+ // (tokens/cost/models/tools/duration/branch/files). Replaces the former
1552
+ // local-store `extra_json` read. The metadata parser is Claude-JSONL shaped;
1553
+ // for other tools the fields it can't read stay at their zero defaults,
1554
+ // which is the same behaviour as an un-indexed session.
1555
+ let projectPath = ref.projectPath;
1556
+ const meta = { messageCount: textMessages.length };
1557
+ // Flag tail-only FULL syncs so the UI can badge partial history.
1558
+ if (oversized)
1559
+ meta.truncated = true;
1560
+ // Single-prompt invocations (batch/bot runs) carry a flag so the UI can
1561
+ // badge them and lists can de-emphasize them.
1562
+ if (textMessages.filter((m) => m.role === 'user').length <= 1)
1563
+ meta.oneShot = true;
1564
+ // Real project path: the cwd read live from inside the transcript wins over
1565
+ // the decoded dir name (`chat-recall` → `chat/recall` mangling).
1566
+ try {
1567
+ const cwd = ref.fullPath ? readCwdFromJsonl(ref.fullPath) : '';
1568
+ if (cwd)
1569
+ projectPath = cwd;
1570
+ }
1571
+ catch { /* fall back to ref.projectPath */ }
1572
+ const resolved = resolveProjectId(projectPath);
1573
+ const projectId = resolved.source === 'ignored' ? '' : resolved.id;
1574
+ if (includeMeta && ref.fullPath) {
1575
+ try {
1576
+ const parsed = await parseSessionFile(ref.fullPath);
1577
+ const m = parsed.metadata;
1578
+ const live = {
1579
+ inputTokens: m.inputTokens, outputTokens: m.outputTokens,
1580
+ cacheReadTokens: m.cacheReadTokens, cacheCreationTokens: m.cacheCreationTokens,
1581
+ peakContextTokens: m.peakContextTokens, costUsd: m.costUsd, durationMs: m.durationMs,
1582
+ modelsUsed: m.modelsUsed, toolsUsed: m.toolsUsed, gitBranch: m.gitBranch,
1583
+ // filesModified drives the analytics Patterns panels (Hot Files,
1584
+ // Redundancy). Without it those panels are permanently empty on the
1585
+ // server — the field simply wasn't in the synced metadata before.
1586
+ filesModified: m.filesModified,
1587
+ };
1588
+ for (const [k, v] of Object.entries(live)) {
1589
+ if (v === undefined)
1590
+ continue;
1591
+ if (Array.isArray(v) && v.length === 0)
1592
+ continue; // empty models/tools = un-parseable for this tool
1593
+ meta[k] = v;
1594
+ }
1595
+ }
1596
+ catch { /* telemetry is best-effort — the conversation still ships */ }
1597
+ }
1598
+ // Materialize the raw session ONCE — uniform across ALL vendors: Claude/
1599
+ // Gemini/Codex file bytes AND the OpenCode row-dump are all UTF-8 text in the
1600
+ // container. Reused for BOTH the secret scan (pre-redaction) and the raw_b64
1601
+ // archive. (Scanning ref.fullPath was wrong for OpenCode — that's the DB.)
1602
+ // Oversized sessions never materialize the container — the builtin regex
1603
+ // scan runs over the bounded tail instead (same coverage the APPEND path
1604
+ // gives a growing session's new bytes).
1605
+ // Reuse the shadow's fullest container (already exported + merged above) so
1606
+ // the secret scan and raw archive cover the RECOVERED full history, not the
1607
+ // truncated disk file. Only re-export when the shadow path didn't produce one.
1608
+ let container = fullestContainer;
1609
+ if (!container && !oversized) {
1610
+ try {
1611
+ const backend = getBackendForId(ref.prefixedId) ?? getBackend('claude');
1612
+ const exp = backend.exportRawSession(ref.prefixedId);
1613
+ container = exp ? buildRawContainer(exp) : null;
1614
+ }
1615
+ catch { /* export unavailable — conversation still ships */ }
1616
+ }
1617
+ // Secret SCAN on the materialized raw text (where secrets actually are,
1618
+ // pre-redaction). MASKED findings (last-4) ship to the server's secret_findings.
1619
+ // 1. BUILT-IN regex engine (same rules as the redactor) — EVERY user, zero
1620
+ // deps. This is what gives vanilla SaaS users real secret detection.
1621
+ // 2. gitleaks/trufflehog — OPTIONAL deeper engines, only when installed.
1622
+ // 3. TENANT rules configured in the dashboard — fetched from the server and
1623
+ // applied to the raw text on the client so unredacted secrets stay local.
1624
+ const findings = [];
1625
+ let scanMs = 0;
1626
+ if (container) {
1627
+ const t0 = performance.now();
1628
+ const rawText = container.files.map((f) => f.text).join('\n');
1629
+ try {
1630
+ for (const f of scanTextForFindings(rawText)) {
1631
+ findings.push({ session_id: ref.prefixedId, detector: 'builtin', rule: f.rule, line: f.line, preview: f.preview });
1632
+ }
1633
+ }
1634
+ catch { /* best-effort */ }
1635
+ if (opts.precomputedExternal) {
1636
+ // Batch path: gitleaks/trufflehog already ran once over a directory of
1637
+ // all sessions; this session's slice is handed in. No per-session spawn.
1638
+ // Tenant rules are cheap in-memory regex, so still run them here.
1639
+ for (const f of opts.precomputedExternal) {
1640
+ const verifiedAt = f.verified ? new Date().toISOString() : null;
1641
+ findings.push({ session_id: ref.prefixedId, detector: f.detector, rule: f.rule, line: f.line, preview: f.preview, verified_at: verifiedAt });
1642
+ }
1643
+ if (opts.tenantRules && opts.tenantRules.length > 0) {
1644
+ try {
1645
+ for (const f of scanTenantRules(rawText, opts.tenantRules))
1646
+ findings.push({ session_id: ref.prefixedId, detector: f.detector, rule: f.rule, line: f.line, preview: f.preview });
1647
+ }
1648
+ catch { /* best-effort */ }
1649
+ }
1650
+ }
1651
+ else if (opts.scanSecrets) {
1652
+ // Per-session fallback (no batch map supplied — e.g. unit tests, callers
1653
+ // outside the sync loop). gitleaks/trufflehog take a path: write once.
1654
+ let tmp = null;
1655
+ try {
1656
+ tmp = join(tmpdir(), `cr-scan-${ref.prefixedId.replace(/[^A-Za-z0-9_-]/g, '')}-${mtime}.txt`);
1657
+ writeFileSync(tmp, rawText);
1658
+ for (const f of scanFileForSecrets(tmp, { verifyOnly: opts.verifySecrets, tenantRules: opts.tenantRules })) {
1659
+ const verifiedAt = f.verified ? new Date().toISOString() : null;
1660
+ findings.push({ session_id: ref.prefixedId, detector: f.detector, rule: f.rule, line: f.line, preview: f.preview, verified_at: verifiedAt });
1661
+ }
1662
+ }
1663
+ catch { /* detector error — never block the sync */ }
1664
+ finally {
1665
+ if (tmp) {
1666
+ try {
1667
+ unlinkSync(tmp);
1668
+ }
1669
+ catch { /* already gone */ }
1670
+ }
1671
+ }
1672
+ }
1673
+ else if (opts.tenantRules && opts.tenantRules.length > 0) {
1674
+ // Tenant rules run even when gitleaks/trufflehog are absent — they are
1675
+ // pure regex and complement the built-in scanner.
1676
+ try {
1677
+ for (const f of scanTenantRules(rawText, opts.tenantRules)) {
1678
+ findings.push({ session_id: ref.prefixedId, detector: f.detector, rule: f.rule, line: f.line, preview: f.preview });
1679
+ }
1680
+ }
1681
+ catch { /* best-effort */ }
1682
+ }
1683
+ scanMs = performance.now() - t0;
1684
+ }
1685
+ // Oversized sessions: builtin regex (+ tenant rules) over the bounded tail —
1686
+ // the container path above never ran for them.
1687
+ if (oversized && tailText) {
1688
+ const t0 = performance.now();
1689
+ try {
1690
+ for (const f of scanTextForFindings(tailText)) {
1691
+ findings.push({ session_id: ref.prefixedId, detector: 'builtin', rule: f.rule, line: f.line, preview: f.preview });
1692
+ }
1693
+ if (opts.tenantRules && opts.tenantRules.length > 0) {
1694
+ for (const f of scanTenantRules(tailText, opts.tenantRules)) {
1695
+ findings.push({ session_id: ref.prefixedId, detector: f.detector, rule: f.rule, line: f.line, preview: f.preview });
1696
+ }
1697
+ }
1698
+ }
1699
+ catch { /* best-effort */ }
1700
+ scanMs = performance.now() - t0;
1701
+ }
1702
+ // High-precision default: drop fuzzy/low-precision detector findings
1703
+ // (generic-api-key, Box, URI, …) before shipping so the Security view isn't
1704
+ // buried in false positives. Opt back in with CHAT_RECALL_INCLUDE_FUZZY=1.
1705
+ const keptFindings = dropFuzzyFindings(findings, (f) => ({ detector: f.detector, rule: f.rule }));
1706
+ findings.length = 0;
1707
+ findings.push(...keptFindings);
1708
+ // Raw archive: redact the SAME container so secrets never reach the server.
1709
+ let raw_b64;
1710
+ let raw_size;
1711
+ if (includeRaw && container) {
1712
+ try {
1713
+ const count2 = { redactions: 0 };
1714
+ const redacted = mapContainerText(container, (t) => redactSecrets(t, { force: true, count: count2 }));
1715
+ count.redactions += count2.redactions;
1716
+ const { gz, size } = gzipContainer(redacted);
1717
+ if (gz.length <= 8 * 1024 * 1024) {
1718
+ raw_b64 = gz.toString('base64');
1719
+ raw_size = size;
1720
+ }
1721
+ }
1722
+ catch { /* raw is additive — conversation still ships */ }
1723
+ }
1724
+ const envTexts = envelope.messages.filter((m) => m.content?.trim());
1725
+ const redactedText = envTexts.map((m) => m.content).join('\n');
1726
+ // Native tool title (Claude ai-title, OpenCode session.title) is NOT shipped
1727
+ // here — it's a derived field reconciled separately (sync-fields.ts), so a
1728
+ // title change never drags the conversation back through sync.
1729
+ return {
1730
+ conv: {
1731
+ session_id: ref.prefixedId,
1732
+ tool: ref.toolId,
1733
+ project_path: mapPath(projectPath),
1734
+ // Resolved on THIS machine (git/FS available) — the server stores it
1735
+ // verbatim; it cannot resolve paths it can't see.
1736
+ project_id: projectId || undefined,
1737
+ redacted_text: redactedText,
1738
+ envelope,
1739
+ raw_b64,
1740
+ raw_size,
1741
+ first_prompt: envTexts.find((m) => m.role === 'user')?.content?.slice(0, 200),
1742
+ // Byte offset this FULL sync is synced THROUGH (file size at build time,
1743
+ // or the end of the bounded tail for an oversized session). The server
1744
+ // stores it as the envelope's `o`; the ledger records the SAME value
1745
+ // (convBatch onFlush uses this, not a recomputed size) so the next
1746
+ // append's base_offset matches exactly — no spurious continuity misses on
1747
+ // an actively-growing session. undefined for non-append-only backends.
1748
+ from_offset: oversized ? tailOffsetEnd : getBackendForId(ref.prefixedId)?.fileSize?.(ref.rawId),
1749
+ meta,
1750
+ mtime,
1751
+ },
1752
+ kgText: redactedText,
1753
+ projectPath,
1754
+ redactions: count.redactions,
1755
+ findings,
1756
+ scanMs,
1757
+ srcHash,
1758
+ };
1759
+ }
1760
+ /**
1761
+ * Parse raw JSONL tail text into canonical messages for one tool. The tail is
1762
+ * the bytes after the last shipped offset — complete JSONL lines only (the
1763
+ * backend's readFromOffset snaps to the last newline). Returns [] when the
1764
+ * tail has no usable messages (blank, all-summary, internal-tool prompt).
1765
+ */
1766
+ function parseTailMessages(tool, text) {
1767
+ if (!text || !text.trim())
1768
+ return [];
1769
+ if (tool === 'claude')
1770
+ return parseClaudeTranscriptText(text);
1771
+ if (tool === 'gemini')
1772
+ return parseGeminiTranscriptText(text);
1773
+ if (tool === 'codex')
1774
+ return parseCodexTranscriptText(text);
1775
+ return [];
1776
+ }
1777
+ /**
1778
+ * Build the tail-only conversation payload for an APPEND sync (see
1779
+ * docs/SYNC-INCREMENTAL.md). Reads ONLY the new bytes from `fromOffset` via
1780
+ * the backend's `readFromOffset`, parses them into messages, and ships a
1781
+ * minimal append envelope — no raw_b64, no telemetry meta, no title/
1782
+ * first_prompt (all head-derived; the server keeps its prior values), no
1783
+ * derived (deferred to the FULL re-sync when the session goes quiet).
1784
+ *
1785
+ * Returns null when the tail is empty / unparseable / all-internal — the
1786
+ * caller treats that as "nothing to ship this tick" and leaves the ledger
1787
+ * cursor unchanged.
1788
+ */
1789
+ export async function buildConversationTail(ref, fromOffset, opts = {}) {
1790
+ const backend = getBackendForId(ref.prefixedId);
1791
+ if (!backend?.readFromOffset)
1792
+ return null;
1793
+ const mapPath = opts.mapPath ?? ((p) => p);
1794
+ const { text: tailText, newOffset } = await backend.readFromOffset(ref.prefixedId, fromOffset);
1795
+ if (!tailText.trim())
1796
+ return null;
1797
+ const messages = parseTailMessages(ref.toolId, tailText);
1798
+ if (messages.length === 0)
1799
+ return null;
1800
+ const textMessages = messages.filter((m) => m.role !== 'summary' && m.content?.trim());
1801
+ if (textMessages.length === 0 && !messages.some((m) => m.toolCalls?.length))
1802
+ return null;
1803
+ // Skip chat-recall's OWN internal prompts in the tail (same gate as the
1804
+ // full builder). A tail that is ONLY an internal prompt ships nothing.
1805
+ const firstUser = textMessages.find((m) => m.role === 'user');
1806
+ if (isInternalToolPrompt(firstUser?.content))
1807
+ return null;
1808
+ // Trim + redact the tail envelope only (not the whole transcript).
1809
+ const count = { redactions: 0 };
1810
+ const envelope = redactDeep({ v: TRANSCRIPT_VERSION, ...trimTranscriptForSync({ messages, subagents: [] }) }, count);
1811
+ const envTexts = envelope.messages.filter((m) => m.content?.trim());
1812
+ const redactedText = envTexts.map((m) => m.content).join('\n');
1813
+ // Secret scan on the tail text only (builtin regex + tenant rules — cheap,
1814
+ // in-memory). External detectors skip append ticks; the FULL re-sync covers
1815
+ // the whole file when the session closes.
1816
+ const findings = [];
1817
+ try {
1818
+ for (const f of scanTextForFindings(tailText)) {
1819
+ findings.push({ session_id: ref.prefixedId, detector: 'builtin', rule: f.rule, line: f.line, preview: f.preview });
1820
+ }
1821
+ }
1822
+ catch { /* best-effort */ }
1823
+ if (opts.tenantRules && opts.tenantRules.length > 0) {
1824
+ try {
1825
+ for (const f of scanTenantRules(tailText, opts.tenantRules))
1826
+ findings.push({ session_id: ref.prefixedId, detector: f.detector, rule: f.rule, line: f.line, preview: f.preview });
1827
+ }
1828
+ catch { /* best-effort */ }
1829
+ }
1830
+ const keptFindings = dropFuzzyFindings(findings, (f) => ({ detector: f.detector, rule: f.rule }));
1831
+ findings.length = 0;
1832
+ findings.push(...keptFindings);
1833
+ // No title / first_prompt / meta / raw_b64 — all head-derived or whole-file.
1834
+ // The server's append path updates ONLY mtime + appends chunks + merges the
1835
+ // envelope; it keeps its prior title/preview/extra/raw_b64 untouched.
1836
+ return {
1837
+ conv: {
1838
+ session_id: ref.prefixedId,
1839
+ tool: ref.toolId,
1840
+ project_path: mapPath(ref.projectPath),
1841
+ append: true,
1842
+ base_offset: fromOffset, // where this tail STARTS — server validates it
1843
+ // equals its stored synced-through offset.
1844
+ from_offset: newOffset, // where it ends → new synced-through offset.
1845
+ redacted_text: redactedText,
1846
+ envelope,
1847
+ mtime: Math.floor(ref.mtime) || 0,
1848
+ },
1849
+ redactions: count.redactions,
1850
+ newOffset,
1851
+ findings,
1852
+ };
1853
+ }
1854
+ /**
1855
+ * Per-session derived payload: the four compute kinds the server's deep-dive
1856
+ * routes read (diff/outcome/commits/markers) + the outcome-badge row. ALWAYS
1857
+ * computed live from the transcript/git (no local compute/outcome cache is
1858
+ * read or warmed — thin collector). Everything is deep-redacted — diffs carry
1859
+ * file content.
1860
+ */
1861
+ function collectDerived(ref, mtime, count) {
1862
+ const id = ref.prefixedId;
1863
+ const compute = [];
1864
+ let fullOutcome = null;
1865
+ for (const kind of COMPUTE_KINDS) {
1866
+ let data = null;
1867
+ try {
1868
+ if (kind === 'diff') {
1869
+ const replay = replaySessionAny(id);
1870
+ if (replay.found)
1871
+ data = replay;
1872
+ }
1873
+ else if (kind === 'outcome') {
1874
+ fullOutcome = computeOutcome(id);
1875
+ if (fullOutcome.found)
1876
+ data = fullOutcome;
1877
+ }
1878
+ else if (kind === 'commits') {
1879
+ // computeOutcome already ran git for the session window — reuse.
1880
+ if (!fullOutcome)
1881
+ fullOutcome = computeOutcome(id);
1882
+ if (fullOutcome.found)
1883
+ data = fullOutcome.commits;
1884
+ }
1885
+ else if (kind === 'markers') {
1886
+ // Analysis-only use of the turns extractor (R4) — markers need
1887
+ // per-prompt line/ts, not render fidelity.
1888
+ const turns = extractTurnsAny(id, { maxTurns: 50_000 });
1889
+ const prompts = turns.turns
1890
+ .filter((t) => t.kind === 'user' && t.text)
1891
+ .map((t) => ({ line: t.line, ts: t.ts, tsIso: t.tsIso, ...markPrompt(t.text) }));
1892
+ data = { sessionId: id, prompts, summary: summarizeMarkers(prompts) };
1893
+ }
1894
+ }
1895
+ catch {
1896
+ data = null;
1897
+ }
1898
+ if (data === null)
1899
+ continue;
1900
+ const redacted = redactDeep(data, count);
1901
+ if (JSON.stringify(redacted).length > MAX_DERIVED_ROW_BYTES)
1902
+ continue;
1903
+ compute.push({ kind, mtime, data: redacted });
1904
+ }
1905
+ // Outcome-badge row (session_outcome_cache) — what the list badges read.
1906
+ // Computed live from the outcome we already ran above.
1907
+ let outcomeRow = null;
1908
+ if (fullOutcome?.found) {
1909
+ outcomeRow = {
1910
+ sessionId: id,
1911
+ tool: ref.toolId,
1912
+ status: fullOutcome.status,
1913
+ reason: fullOutcome.reason,
1914
+ fileMtime: mtime,
1915
+ fileSize: 0,
1916
+ contentHash: '',
1917
+ fileCount: fullOutcome.fileCount,
1918
+ linesAdded: fullOutcome.totalLinesAdded,
1919
+ linesRemoved: fullOutcome.totalLinesRemoved,
1920
+ commits: fullOutcome.commits?.totalCommits ?? 0,
1921
+ isFull: true,
1922
+ classifiedAt: Date.now(),
1923
+ lastScannedOffset: 0,
1924
+ };
1925
+ }
1926
+ return {
1927
+ session_id: id,
1928
+ mtime,
1929
+ compute,
1930
+ outcome_row: outcomeRow ? redactDeep({ ...outcomeRow, sessionId: undefined }, count) : null,
1931
+ };
1932
+ }
1933
+ export function isSyncSkip(r) {
1934
+ return typeof r.skipped === 'string';
1935
+ }
1936
+ export async function syncIncremental(opts = {}) {
1937
+ const cred = loadCredentials();
1938
+ if (!cred)
1939
+ return { skipped: 'no-credentials' };
1940
+ let settings = loadSettings();
1941
+ // Master switch (settings.sync.enabled) — `chat-recall login` flips it on;
1942
+ // the user can turn background sync off without logging out. Legacy logins
1943
+ // (credentials.json written before login set the flag) have enabled=false
1944
+ // AND no endpoint recorded: treat that as consent (the only way to get
1945
+ // credentials is an explicit login) and migrate the settings once. An
1946
+ // endpoint WITH enabled=false means the user deliberately paused — respect it.
1947
+ if (!settings.sync.enabled) {
1948
+ if (settings.sync.endpoint)
1949
+ return { skipped: 'paused' }; // user paused
1950
+ settings.sync.enabled = true;
1951
+ settings.sync.endpoint = cred.serverUrl;
1952
+ saveSettings(settings);
1953
+ settings = loadSettings();
1954
+ }
1955
+ // SINGLE WRITER — see docs/SYNC.md. Every background sync (MCP ticks across
1956
+ // sessions, any file-watcher, manual incremental) serializes on ONE
1957
+ // cross-platform lock (O_EXCL lockfile; Linux/macOS/Windows). One writer of
1958
+ // the sync ledger at a time, ever. Can't get the lock ⇒ another writer is
1959
+ // mid-sync ⇒ skip this tick. Nothing is lost (the per-session ledger +
1960
+ // watermark make ticks idempotent; the next tick picks up the rest).
1961
+ const lock = acquireIndexLock({ kind: 'sync-incremental', staleAfterMs: 10 * 60_000 });
1962
+ if (!lock)
1963
+ return { skipped: 'lock-held' };
1964
+ try {
1965
+ const startedAt = Date.now();
1966
+ // Per-session ledger replaces the global watermark for sessions: a FULL
1967
+ // walk lists ALL sessions and the ledger skips acked ones — so a session
1968
+ // that failed once retries forever until it lands. scope:'changed'
1969
+ // (file-change ticks from the watch daemon) bounds the listing to the
1970
+ // recent-mtime window instead — a keystroke-burst tick walks a handful of
1971
+ // refs, not 30k+ — while heartbeat/startup ticks stay 'full' so the
1972
+ // retry-forever property holds. The watermark also bounds the
1973
+ // non-session items walk in both scopes.
1974
+ const result = await syncSessions({ useLedger: true, sinceMs: settings.sync.lastSyncAt, walk: opts.scope ?? 'full' });
1975
+ // Re-load before saving so we don't clobber settings written mid-sync.
1976
+ const fresh = loadSettings();
1977
+ fresh.sync.lastSyncAt = startedAt;
1978
+ saveSettings(fresh);
1979
+ return result;
1980
+ }
1981
+ finally {
1982
+ lock.release();
1983
+ }
1984
+ }
1985
+ //# sourceMappingURL=sync-client.js.map