chat-recall 0.0.1 → 0.4.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +93 -0
- package/dist/auto-update.d.ts +52 -0
- package/dist/auto-update.d.ts.map +1 -0
- package/dist/auto-update.js +157 -0
- package/dist/auto-update.js.map +1 -0
- package/dist/cli.d.ts +6 -0
- package/dist/cli.d.ts.map +1 -0
- package/dist/cli.js +26102 -0
- package/dist/cli.js.map +1 -0
- package/dist/client-events.d.ts +11 -0
- package/dist/client-events.d.ts.map +1 -0
- package/dist/client-events.js +77 -0
- package/dist/client-events.js.map +1 -0
- package/dist/collector-migrate.d.ts +27 -0
- package/dist/collector-migrate.d.ts.map +1 -0
- package/dist/collector-migrate.js +60 -0
- package/dist/collector-migrate.js.map +1 -0
- package/dist/device-auth.d.ts +37 -0
- package/dist/device-auth.d.ts.map +1 -0
- package/dist/device-auth.js +161 -0
- package/dist/device-auth.js.map +1 -0
- package/dist/escalate.d.ts +99 -0
- package/dist/escalate.d.ts.map +1 -0
- package/dist/escalate.js +219 -0
- package/dist/escalate.js.map +1 -0
- package/dist/http.d.ts +14 -0
- package/dist/http.d.ts.map +1 -0
- package/dist/http.js +49 -0
- package/dist/http.js.map +1 -0
- package/dist/install-skills.d.ts +51 -0
- package/dist/install-skills.d.ts.map +1 -0
- package/dist/install-skills.js +223 -0
- package/dist/install-skills.js.map +1 -0
- package/dist/intent-drain.d.ts +25 -0
- package/dist/intent-drain.d.ts.map +1 -0
- package/dist/intent-drain.js +189 -0
- package/dist/intent-drain.js.map +1 -0
- package/dist/keyring.d.ts +28 -0
- package/dist/keyring.d.ts.map +1 -0
- package/dist/keyring.js +163 -0
- package/dist/keyring.js.map +1 -0
- package/dist/mcp.d.ts +8 -0
- package/dist/mcp.d.ts.map +1 -0
- package/dist/mcp.js +23665 -0
- package/dist/mcp.js.map +1 -0
- package/dist/project-tasks.d.ts +54 -0
- package/dist/project-tasks.d.ts.map +1 -0
- package/dist/project-tasks.js +204 -0
- package/dist/project-tasks.js.map +1 -0
- package/dist/repair.d.ts +73 -0
- package/dist/repair.d.ts.map +1 -0
- package/dist/repair.js +267 -0
- package/dist/repair.js.map +1 -0
- package/dist/self-restart.d.ts +91 -0
- package/dist/self-restart.d.ts.map +1 -0
- package/dist/self-restart.js +137 -0
- package/dist/self-restart.js.map +1 -0
- package/dist/service-installer.d.ts +32 -0
- package/dist/service-installer.d.ts.map +1 -0
- package/dist/service-installer.js +231 -0
- package/dist/service-installer.js.map +1 -0
- package/dist/sync-client.d.ts +234 -0
- package/dist/sync-client.d.ts.map +1 -0
- package/dist/sync-client.js +1985 -0
- package/dist/sync-client.js.map +1 -0
- package/dist/sync-ledger.d.ts +131 -0
- package/dist/sync-ledger.d.ts.map +1 -0
- package/dist/sync-ledger.js +366 -0
- package/dist/sync-ledger.js.map +1 -0
- package/dist/update-notice.d.ts +16 -0
- package/dist/update-notice.d.ts.map +1 -0
- package/dist/update-notice.js +107 -0
- package/dist/update-notice.js.map +1 -0
- package/dist/verify-deep.d.ts +79 -0
- package/dist/verify-deep.d.ts.map +1 -0
- package/dist/verify-deep.js +97 -0
- package/dist/verify-deep.js.map +1 -0
- package/dist/verify-repair.d.ts +21 -0
- package/dist/verify-repair.d.ts.map +1 -0
- package/dist/verify-repair.js +58 -0
- package/dist/verify-repair.js.map +1 -0
- package/dist/watch.js +21550 -0
- package/hooks/chat_recall_escalate_hook.sh +51 -0
- package/hooks/chat_recall_resume_hook.sh +69 -0
- package/hooks/chat_recall_save_hook.sh +254 -0
- package/hooks/chat_recall_wakeup_hook.sh +39 -0
- package/package.json +38 -4
- package/skills/chat-recall/SKILL.md +76 -0
- package/skills/chat-recall-changes/SKILL.md +50 -0
- package/skills/chat-recall-find/SKILL.md +64 -0
- package/skills/chat-recall-memory/SKILL.md +55 -0
- package/skills/chat-recall-project/SKILL.md +75 -0
- package/skills/chat-recall-resume/SKILL.md +62 -0
- package/README.md +0 -7
|
@@ -0,0 +1,1985 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The one sync client (thin collector): walk local sessions AND every other
|
|
3
|
+
* memory source, computing EVERYTHING live from the raw session files —
|
|
4
|
+
* reading NOTHING from the local SQLite store. Redact secrets (ALWAYS —
|
|
5
|
+
* `force:true`, never gated on the index-time toggle), optionally hash
|
|
6
|
+
* project paths, and push to the server's `/api/sync`
|
|
7
|
+
* (packages/server/src/routes/sync.ts). Credentials live in a 0600 file,
|
|
8
|
+
* never in settings.json.
|
|
9
|
+
*
|
|
10
|
+
* Thin-collector contract: the collector owns NO derived state. It reads the
|
|
11
|
+
* raw transcripts (which it must read anyway to ship them) and derives
|
|
12
|
+
* telemetry, project ids, KG triples, diffs, outcomes, commits and markers on
|
|
13
|
+
* the spot. The server is the source of truth for everything stateful
|
|
14
|
+
* (tombstones, summaries, secret findings/rules/dismissals) — the collector
|
|
15
|
+
* neither reads nor writes those locally.
|
|
16
|
+
*
|
|
17
|
+
* What ships (all live-computed from the session files):
|
|
18
|
+
* conversations — per-turn redacted session transcripts + live telemetry meta
|
|
19
|
+
* items/links — every non-session source type (plan, task, claude_md,
|
|
20
|
+
* history, paste, diary, skill, mcp, command, agent,
|
|
21
|
+
* hook, plugin) with redacted chunks + relationships
|
|
22
|
+
* derived — compute rows (diff/outcome/commits/markers) computed live
|
|
23
|
+
* + outcome-badge row (toggle: upload.sessionMeta)
|
|
24
|
+
* kg — knowledge-graph entities/triples extracted live from the
|
|
25
|
+
* redacted text (toggle: upload.sessionMeta)
|
|
26
|
+
*
|
|
27
|
+
* What does NOT ship, by design: the full-conversation JSON blob
|
|
28
|
+
* (content_cache) and embeddings. The server reconstructs the conversation
|
|
29
|
+
* view from the per-turn chunks. Tombstones, AI summaries, and secret
|
|
30
|
+
* findings/dismissals/custom-rules are NOT shipped by the collector — the
|
|
31
|
+
* server owns them.
|
|
32
|
+
*/
|
|
33
|
+
import { join, dirname, basename } from 'node:path';
|
|
34
|
+
import { readFileSync, writeFileSync, mkdirSync, mkdtempSync, rmSync, chmodSync, unlinkSync, readdirSync, statSync } from 'node:fs';
|
|
35
|
+
import { tmpdir, hostname } from 'node:os';
|
|
36
|
+
import { createHash } from 'node:crypto';
|
|
37
|
+
import { redactSecrets, scanTextForFindings, installServerRulePack, serverRulePackVersion } from '@chat-recall/engine/core/secret-redactor.js';
|
|
38
|
+
import { discoverSessionSources, installSourceExclusions, sourceId } from '@chat-recall/engine/core/source-discovery.js';
|
|
39
|
+
import { discoverHomes } from '@chat-recall/engine/core/home-discovery.js';
|
|
40
|
+
import { homeDecision, approveHome, declineHome } from '@chat-recall/engine/core/home-approval.js';
|
|
41
|
+
import { scanFileForSecrets, scanDirForSecrets, scanTenantRules, isSecretScannerAvailable } from '@chat-recall/engine/core/secret-scanner.js';
|
|
42
|
+
import { isInternalToolPrompt } from '@chat-recall/engine/core/internal-prompts.js';
|
|
43
|
+
import { dropFuzzyFindings } from '@chat-recall/engine/core/secret-precision.js';
|
|
44
|
+
import { loadSettings, saveSettings, isProjectSyncable } from '@chat-recall/engine/core/settings.js';
|
|
45
|
+
import { getDataDir } from '@chat-recall/engine/core/paths.js';
|
|
46
|
+
import { listAvailableBackends } from '@chat-recall/engine/core/tool-backend.js';
|
|
47
|
+
import { extractTurnsAny, replaySessionAny } from '@chat-recall/engine/core/session-multi-tool.js';
|
|
48
|
+
import { parseTranscript, trimTranscriptForSync, TRANSCRIPT_VERSION, gzipContainer, mapContainerText, buildRawContainer, containerSrcHash, parseTranscriptFromContainer, updateShadow } from '@chat-recall/engine/transcript/index.js';
|
|
49
|
+
import { parseClaudeTranscriptText } from '@chat-recall/engine/transcript/claude.js';
|
|
50
|
+
import { parseGeminiTranscriptText } from '@chat-recall/engine/transcript/gemini.js';
|
|
51
|
+
import { parseCodexTranscriptText } from '@chat-recall/engine/transcript/codex.js';
|
|
52
|
+
import { getBackendForId, getBackend } from '@chat-recall/engine/core/tool-backend.js';
|
|
53
|
+
import { computeOutcome } from '@chat-recall/engine/core/session-outcome.js';
|
|
54
|
+
import { fileURLToPath } from 'node:url';
|
|
55
|
+
let ownVersion = '0.0.0';
|
|
56
|
+
try {
|
|
57
|
+
const dir = dirname(fileURLToPath(import.meta.url));
|
|
58
|
+
ownVersion = JSON.parse(readFileSync(join(dir, '../package.json'), 'utf-8')).version || ownVersion;
|
|
59
|
+
}
|
|
60
|
+
catch {
|
|
61
|
+
try {
|
|
62
|
+
// @ts-ignore
|
|
63
|
+
if (typeof __CLI_VERSION__ === 'string')
|
|
64
|
+
ownVersion = __CLI_VERSION__;
|
|
65
|
+
}
|
|
66
|
+
catch { }
|
|
67
|
+
}
|
|
68
|
+
import { markPrompt, summarizeMarkers } from '@chat-recall/engine/core/session-sentiment.js';
|
|
69
|
+
import { parseSessionFile, readCwdFromJsonl } from '@chat-recall/engine/parsers/session.js';
|
|
70
|
+
import { resolveProjectId } from '@chat-recall/engine/core/project-resolver.js';
|
|
71
|
+
import { extractEntities } from '@chat-recall/engine/core/entity-extractor.js';
|
|
72
|
+
import { buildSourceRegistry } from '@chat-recall/engine/parsers/all-sources.js';
|
|
73
|
+
import { getSyncedRows, markSynced, getLedgerData, persistLedgerData, fieldNeedsScan, markFieldCoverage, fieldNeedsFullPass, markFieldFullPassDone, syncMode, markFullResync, loadItemVersions, saveItemVersions } from './sync-ledger.js';
|
|
74
|
+
import { extractorVersionForTool, extractorVersionForId, toolOfId, EXTRACTOR_VERSION } from '@chat-recall/engine/core/extractor-version.js';
|
|
75
|
+
import { SYNC_FIELDS } from '@chat-recall/engine/core/sync-fields.js';
|
|
76
|
+
import { acquireIndexLock } from '@chat-recall/engine/core/index-lock.js';
|
|
77
|
+
import { fetchWithTimeout } from './http.js';
|
|
78
|
+
import '@chat-recall/engine/core/backends/index.js'; // register the tool backends
|
|
79
|
+
// Lives under the data dir so CHAT_RECALL_DATA_DIR isolates credentials the
|
|
80
|
+
// same way it isolates the index (tests, multi-profile setups).
|
|
81
|
+
const credPath = () => join(getDataDir(), 'credentials.json');
|
|
82
|
+
/**
|
|
83
|
+
* Multi-target: logging in ADDS a server (or updates its token). Every
|
|
84
|
+
* sync pushes to ALL targets; the per-server ledger keeps coverage
|
|
85
|
+
* independent. File shape is {targets:[...]} with the legacy single-object
|
|
86
|
+
* form still readable.
|
|
87
|
+
*/
|
|
88
|
+
export function saveCredentials(c) {
|
|
89
|
+
mkdirSync(dirname(credPath()), { recursive: true });
|
|
90
|
+
const targets = loadAllCredentials().filter((t) => t.serverUrl !== c.serverUrl);
|
|
91
|
+
// Newest login FIRST: single-target consumers (loadCredentials → targets[0],
|
|
92
|
+
// i.e. sync + every MCP read) follow the most recent `chat-recall login`.
|
|
93
|
+
// Appending here once meant a fresh SaaS login silently kept syncing to a
|
|
94
|
+
// stale localhost self-host target that happened to sit at position 0.
|
|
95
|
+
targets.unshift(c);
|
|
96
|
+
writeFileSync(credPath(), JSON.stringify({ targets }, null, 2));
|
|
97
|
+
try {
|
|
98
|
+
chmodSync(credPath(), 0o600);
|
|
99
|
+
}
|
|
100
|
+
catch { /* windows */ }
|
|
101
|
+
try {
|
|
102
|
+
const s = loadSettings();
|
|
103
|
+
s.sync.enabled = true;
|
|
104
|
+
s.sync.endpoint = targets.map((t) => t.serverUrl).join(',');
|
|
105
|
+
saveSettings(s);
|
|
106
|
+
}
|
|
107
|
+
catch { /* settings unwritable — manual `chat-recall sync` still works */ }
|
|
108
|
+
}
|
|
109
|
+
export function loadAllCredentials() {
|
|
110
|
+
try {
|
|
111
|
+
const parsed = JSON.parse(readFileSync(credPath(), 'utf-8'));
|
|
112
|
+
// token may be '' for a no-auth local self-host server (AUTH_PROVIDER=none) —
|
|
113
|
+
// such a target is valid and pushes without an Authorization header.
|
|
114
|
+
if (Array.isArray(parsed?.targets))
|
|
115
|
+
return parsed.targets.filter((t) => t?.serverUrl).map((t) => ({ serverUrl: t.serverUrl, token: t.token || '' }));
|
|
116
|
+
if (parsed?.serverUrl)
|
|
117
|
+
return [{ serverUrl: parsed.serverUrl, token: parsed.token || '' }]; // legacy single
|
|
118
|
+
}
|
|
119
|
+
catch { /* none */ }
|
|
120
|
+
return [];
|
|
121
|
+
}
|
|
122
|
+
// In-memory cache for tenant security configuration. Fetched at the start of
|
|
123
|
+
// each `sync` call, but for the watch daemon we refresh only every 5 minutes
|
|
124
|
+
// so repeated incremental syncs don't hammer the server.
|
|
125
|
+
const _securityConfigCache = new Map();
|
|
126
|
+
const SECURITY_CONFIG_TTL_MS = 5 * 60 * 1000;
|
|
127
|
+
async function fetchTenantSecurityConfig(cred) {
|
|
128
|
+
const base = cred.serverUrl.replace(/\/+$/, '');
|
|
129
|
+
const cached = _securityConfigCache.get(base);
|
|
130
|
+
if (cached && Date.now() - cached.fetchedAt < SECURITY_CONFIG_TTL_MS)
|
|
131
|
+
return cached.config;
|
|
132
|
+
const headers = { 'content-type': 'application/json' };
|
|
133
|
+
if (cred.token)
|
|
134
|
+
headers.authorization = `Bearer ${cred.token}`;
|
|
135
|
+
try {
|
|
136
|
+
const [rulesRes, settingsRes] = await Promise.all([
|
|
137
|
+
fetchWithTimeout(`${base}/api/secrets/rules`, { headers }),
|
|
138
|
+
fetchWithTimeout(`${base}/api/teams/security-config`, { headers }).catch(() => null),
|
|
139
|
+
]);
|
|
140
|
+
let tenantRules = [];
|
|
141
|
+
let rulePackVersion = null;
|
|
142
|
+
if (rulesRes.ok) {
|
|
143
|
+
const body = await rulesRes.json().catch(() => ({}));
|
|
144
|
+
const enabled = (body.rules || []).filter((r) => r.enabled !== false);
|
|
145
|
+
tenantRules = enabled;
|
|
146
|
+
// The REDACTION set. Prefer the server's `pack` — it carries chat-recall's
|
|
147
|
+
// curated builtin rules as well as this tenant's `redact` ones, which is
|
|
148
|
+
// what lets coverage improve for every customer from a server deploy
|
|
149
|
+
// instead of a CLI release on each device. Fall back to deriving it from
|
|
150
|
+
// the tenant rules when talking to a server that predates `pack`.
|
|
151
|
+
//
|
|
152
|
+
// installServerRulePack is add-only (it can never weaken the compiled-in
|
|
153
|
+
// builtins) and drops individual rules that fail its safety checks; we log
|
|
154
|
+
// those, because a silently ignored rule reads as coverage the operator
|
|
155
|
+
// does not actually have.
|
|
156
|
+
const packSpec = body.pack?.rules
|
|
157
|
+
? { version: body.pack.version, rules: body.pack.rules.map((r) => ({ name: r.name, regex: r.regex, flags: r.flags, redact: true, source: r.source })) }
|
|
158
|
+
: { version: body.version, rules: enabled.filter((r) => r.redact === true).map((r) => ({ name: r.name, regex: r.regex, redact: true, source: 'tenant' })) };
|
|
159
|
+
const pack = installServerRulePack(packSpec);
|
|
160
|
+
rulePackVersion = serverRulePackVersion();
|
|
161
|
+
for (const r of pack.rejected) {
|
|
162
|
+
console.error(`[sync] server redaction rule "${r.name}" ignored: ${r.reason}`);
|
|
163
|
+
}
|
|
164
|
+
if (pack.accepted > 0) {
|
|
165
|
+
const rev = body.pack?.revision ? `, rev ${body.pack.revision}` : '';
|
|
166
|
+
console.error(`[sync] ${pack.accepted} server redaction rule(s) active (pack ${rulePackVersion}${rev})`);
|
|
167
|
+
}
|
|
168
|
+
}
|
|
169
|
+
// Default verification ON. The server endpoint may not exist yet (older
|
|
170
|
+
// servers), so we fail open to "verified enabled" — this is the product default.
|
|
171
|
+
let verifySecrets = true;
|
|
172
|
+
if (settingsRes?.ok) {
|
|
173
|
+
const body = await settingsRes.json().catch(() => ({}));
|
|
174
|
+
if (typeof body.verifySecrets === 'boolean')
|
|
175
|
+
verifySecrets = body.verifySecrets;
|
|
176
|
+
}
|
|
177
|
+
const config = { tenantRules, verifySecrets, rulePackVersion };
|
|
178
|
+
_securityConfigCache.set(base, { fetchedAt: Date.now(), config });
|
|
179
|
+
return config;
|
|
180
|
+
}
|
|
181
|
+
catch {
|
|
182
|
+
// If the server is unreachable or the endpoints don't exist, fall back to
|
|
183
|
+
// no tenant rules and verification enabled. This keeps the collector
|
|
184
|
+
// working during transient outages and with older server images.
|
|
185
|
+
return { tenantRules: [], verifySecrets: true, rulePackVersion: null };
|
|
186
|
+
}
|
|
187
|
+
}
|
|
188
|
+
export function _resetTenantSecurityConfigCache() {
|
|
189
|
+
_securityConfigCache.clear();
|
|
190
|
+
}
|
|
191
|
+
const _syncConfigCache = new Map();
|
|
192
|
+
async function fetchTenantSyncConfig(cred) {
|
|
193
|
+
const base = cred.serverUrl.replace(/\/+$/, '');
|
|
194
|
+
const cached = _syncConfigCache.get(base);
|
|
195
|
+
if (cached && Date.now() - cached.fetchedAt < SECURITY_CONFIG_TTL_MS)
|
|
196
|
+
return cached.config;
|
|
197
|
+
const headers = { 'content-type': 'application/json' };
|
|
198
|
+
if (cred.token)
|
|
199
|
+
headers.authorization = `Bearer ${cred.token}`;
|
|
200
|
+
try {
|
|
201
|
+
// Report what this machine has BEFORE reading the selection back, so a
|
|
202
|
+
// freshly-discovered source shows up in the dashboard on the first sync
|
|
203
|
+
// rather than the second. Paths travel in this direction only.
|
|
204
|
+
try {
|
|
205
|
+
const sources = discoverSessionSources();
|
|
206
|
+
if (sources.length > 0) {
|
|
207
|
+
await fetchWithTimeout(`${base}/api/sync-config/sources`, {
|
|
208
|
+
method: 'POST', headers,
|
|
209
|
+
body: JSON.stringify({ device: deviceLabel(), sources }),
|
|
210
|
+
}).catch(() => null);
|
|
211
|
+
}
|
|
212
|
+
}
|
|
213
|
+
catch { /* reporting is advisory — never block a sync on it */ }
|
|
214
|
+
const res = await fetchWithTimeout(`${base}/api/sync-config`, { headers });
|
|
215
|
+
if (!res.ok)
|
|
216
|
+
return { excludeTools: [], excludeProjects: [], excludeSources: [], approveSources: [] };
|
|
217
|
+
const body = await res.json().catch(() => ({}));
|
|
218
|
+
const config = {
|
|
219
|
+
excludeTools: Array.isArray(body.excludeTools) ? body.excludeTools.filter((t) => typeof t === 'string') : [],
|
|
220
|
+
excludeProjects: Array.isArray(body.excludeProjects) ? body.excludeProjects.filter((p) => typeof p === 'string') : [],
|
|
221
|
+
// Ids only — installSourceExclusions drops anything else, so a server
|
|
222
|
+
// sending a path cannot make this collector read it.
|
|
223
|
+
excludeSources: Array.isArray(body.excludeSources) ? body.excludeSources.filter((x) => typeof x === 'string') : [],
|
|
224
|
+
approveSources: Array.isArray(body.approveSources) ? body.approveSources.filter((x) => typeof x === 'string') : [],
|
|
225
|
+
};
|
|
226
|
+
const applied = installSourceExclusions(config.excludeSources);
|
|
227
|
+
if (applied.excluded > 0) {
|
|
228
|
+
console.error(`[sync] ${applied.excluded} transcript source(s) excluded by tenant config`);
|
|
229
|
+
}
|
|
230
|
+
// Decisions made in the dashboard become LOCAL decisions here. The server
|
|
231
|
+
// sends ids, never paths: we approve/decline only a home this machine itself
|
|
232
|
+
// discovered, so the dashboard can answer the question but can never
|
|
233
|
+
// introduce a folder for us to read.
|
|
234
|
+
try {
|
|
235
|
+
const byId = new Map(discoverHomes({ includeRunning: false }).map((h) => [sourceId(h.path), h.path]));
|
|
236
|
+
let decided = 0;
|
|
237
|
+
for (const id of config.approveSources) {
|
|
238
|
+
const path = byId.get(id);
|
|
239
|
+
if (path && homeDecision(path) === 'pending') {
|
|
240
|
+
approveHome(path);
|
|
241
|
+
decided++;
|
|
242
|
+
}
|
|
243
|
+
}
|
|
244
|
+
for (const id of config.excludeSources) {
|
|
245
|
+
const path = byId.get(id);
|
|
246
|
+
const d = path ? homeDecision(path) : null;
|
|
247
|
+
if (path && d !== 'declined' && d !== 'primary') {
|
|
248
|
+
declineHome(path);
|
|
249
|
+
decided++;
|
|
250
|
+
}
|
|
251
|
+
}
|
|
252
|
+
if (decided > 0)
|
|
253
|
+
console.error(`[sync] applied ${decided} folder decision(s) from the dashboard`);
|
|
254
|
+
}
|
|
255
|
+
catch { /* advisory — a failure must not stop the sync */ }
|
|
256
|
+
_syncConfigCache.set(base, { fetchedAt: Date.now(), config });
|
|
257
|
+
return config;
|
|
258
|
+
}
|
|
259
|
+
catch {
|
|
260
|
+
return { excludeTools: [], excludeProjects: [], excludeSources: [], approveSources: [] };
|
|
261
|
+
}
|
|
262
|
+
}
|
|
263
|
+
/** Stable-ish machine label for the dashboard's per-device grouping. */
|
|
264
|
+
function deviceLabel() {
|
|
265
|
+
try {
|
|
266
|
+
return hostname();
|
|
267
|
+
}
|
|
268
|
+
catch {
|
|
269
|
+
return 'unknown';
|
|
270
|
+
}
|
|
271
|
+
}
|
|
272
|
+
export function _resetTenantSyncConfigCache() {
|
|
273
|
+
_syncConfigCache.clear();
|
|
274
|
+
}
|
|
275
|
+
/** Legacy single-target accessor — first target. */
|
|
276
|
+
/** The ACTIVE target = most recent login (saveCredentials unshifts). */
|
|
277
|
+
export function loadCredentials() {
|
|
278
|
+
return loadAllCredentials()[0] ?? null;
|
|
279
|
+
}
|
|
280
|
+
export function removeCredentials(serverUrl) {
|
|
281
|
+
const targets = loadAllCredentials();
|
|
282
|
+
const kept = targets.filter((t) => t.serverUrl !== serverUrl);
|
|
283
|
+
if (kept.length === targets.length)
|
|
284
|
+
return false;
|
|
285
|
+
writeFileSync(credPath(), JSON.stringify({ targets: kept }, null, 2));
|
|
286
|
+
return true;
|
|
287
|
+
}
|
|
288
|
+
/** Tokenize an absolute project path so the server can group by project
|
|
289
|
+
* without learning the developer's filesystem layout. */
|
|
290
|
+
const hashPath = (p) => (p ? 'p_' + createHash('sha256').update(p).digest('hex').slice(0, 12) : '');
|
|
291
|
+
/** Redact every string anywhere inside a JSON-serializable value. Used for
|
|
292
|
+
* derived payloads (diffs carry file content; outcomes carry prompt text)
|
|
293
|
+
* where field-by-field redaction would be fragile. */
|
|
294
|
+
function redactDeep(v, count) {
|
|
295
|
+
if (typeof v === 'string')
|
|
296
|
+
return redactSecrets(v, { force: true, count });
|
|
297
|
+
if (Array.isArray(v))
|
|
298
|
+
return v.map((x) => redactDeep(x, count));
|
|
299
|
+
if (v && typeof v === 'object') {
|
|
300
|
+
const o = {};
|
|
301
|
+
for (const [k, val] of Object.entries(v))
|
|
302
|
+
o[k] = redactDeep(val, count);
|
|
303
|
+
return o;
|
|
304
|
+
}
|
|
305
|
+
return v;
|
|
306
|
+
}
|
|
307
|
+
/** compute_cache kinds shipped to the server. Must stay in step with the
|
|
308
|
+
* kinds the server's conversation routes read (heavyCacheGet keys). */
|
|
309
|
+
const COMPUTE_KINDS = ['diff', 'outcome', 'commits', 'markers'];
|
|
310
|
+
/** Per-row ceiling for derived payloads — matches the local heavy-cache L2
|
|
311
|
+
* ceiling (metadata-cache.ts) so we never ship a row the server-side
|
|
312
|
+
* SQLite edition couldn't store anyway. */
|
|
313
|
+
const MAX_DERIVED_ROW_BYTES = 2 * 1024 * 1024;
|
|
314
|
+
/**
|
|
315
|
+
* Local/LAN sync targets need no WAN-style pacing — you own the box and there
|
|
316
|
+
* is no shared-tenant or rate-limiter to be polite to, so they get a higher
|
|
317
|
+
* in-flight cap and zero inter-batch sleep. Remote/SaaS hosts stay gentler.
|
|
318
|
+
*/
|
|
319
|
+
function isLocalHost(serverUrl) {
|
|
320
|
+
let host = '';
|
|
321
|
+
try {
|
|
322
|
+
host = new URL(serverUrl).hostname.toLowerCase();
|
|
323
|
+
}
|
|
324
|
+
catch {
|
|
325
|
+
return false;
|
|
326
|
+
}
|
|
327
|
+
if (host === 'localhost' || host === '127.0.0.1' || host === '::1' || host === '0.0.0.0')
|
|
328
|
+
return true;
|
|
329
|
+
if (host.endsWith('.local') || host.endsWith('.lan'))
|
|
330
|
+
return true;
|
|
331
|
+
if (/^10\./.test(host))
|
|
332
|
+
return true; // 10.0.0.0/8
|
|
333
|
+
if (/^192\.168\./.test(host))
|
|
334
|
+
return true; // 192.168.0.0/16
|
|
335
|
+
if (/^172\.(1[6-9]|2\d|3[01])\./.test(host))
|
|
336
|
+
return true; // 172.16.0.0/12
|
|
337
|
+
return false;
|
|
338
|
+
}
|
|
339
|
+
export async function syncSessions(opts = {}) {
|
|
340
|
+
const targets = loadAllCredentials();
|
|
341
|
+
if (targets.length === 0)
|
|
342
|
+
throw new Error('Not logged in — run `chat-recall login <server-url> --token <token>`');
|
|
343
|
+
let agg = null;
|
|
344
|
+
const errors = [];
|
|
345
|
+
for (const cred of targets) {
|
|
346
|
+
try {
|
|
347
|
+
const r = await syncToTarget(cred, opts);
|
|
348
|
+
agg = agg ? {
|
|
349
|
+
uploaded: agg.uploaded + r.uploaded, skipped: agg.skipped + r.skipped, redactions: agg.redactions + r.redactions,
|
|
350
|
+
items: agg.items + r.items, links: agg.links + r.links, findings: agg.findings + r.findings,
|
|
351
|
+
derived: agg.derived + r.derived, kgEntities: agg.kgEntities + r.kgEntities, kgTriples: agg.kgTriples + r.kgTriples,
|
|
352
|
+
scanned: Math.max(agg.scanned, r.scanned), scanMs: Math.max(agg.scanMs, r.scanMs),
|
|
353
|
+
} : r;
|
|
354
|
+
}
|
|
355
|
+
catch (e) {
|
|
356
|
+
errors.push(`${cred.serverUrl}: ${e instanceof Error ? e.message : e}`);
|
|
357
|
+
}
|
|
358
|
+
}
|
|
359
|
+
if (!agg)
|
|
360
|
+
throw new Error(`sync failed for all targets:\n ${errors.join('\n ')}`);
|
|
361
|
+
if (errors.length > 0)
|
|
362
|
+
console.error(`[sync] ${errors.length} target(s) failed (others succeeded):\n ${errors.join('\n ')}`);
|
|
363
|
+
return agg;
|
|
364
|
+
}
|
|
365
|
+
/** POST one derived-field batch, retrying the ingest gate's 429 (honoring
|
|
366
|
+
* Retry-After) and transient 5xx. */
|
|
367
|
+
async function postFieldBatch(base, token, rows) {
|
|
368
|
+
const headers = { 'content-type': 'application/json' };
|
|
369
|
+
if (token)
|
|
370
|
+
headers.authorization = `Bearer ${token}`;
|
|
371
|
+
const sleep = (ms) => new Promise((r) => setTimeout(r, ms));
|
|
372
|
+
for (let attempt = 0;; attempt++) {
|
|
373
|
+
const res = await fetchWithTimeout(`${base}/api/sync`, { method: 'POST', headers, body: JSON.stringify({ fields: rows }) }, 60_000);
|
|
374
|
+
if (res.ok)
|
|
375
|
+
return;
|
|
376
|
+
const retryable = res.status === 429 || res.status >= 500;
|
|
377
|
+
if (!retryable || attempt >= 5)
|
|
378
|
+
throw new Error(`${base}: fields batch HTTP ${res.status} ${await res.text().catch(() => '')}`);
|
|
379
|
+
const ra = Number(res.headers.get('retry-after')) || 0;
|
|
380
|
+
await sleep(ra > 0 ? ra * 1000 : [500, 1500, 4000, 8000, 15000][Math.min(attempt, 4)]);
|
|
381
|
+
}
|
|
382
|
+
}
|
|
383
|
+
/**
|
|
384
|
+
* Reconcile every DERIVED FIELD (engine core/sync-fields.ts) for one target,
|
|
385
|
+
* pushing ONLY missing/stale fields — never the conversation.
|
|
386
|
+
*
|
|
387
|
+
* Per field: a one-time FULL pass (all sessions) runs when the field's version
|
|
388
|
+
* bumped or a re-scan was forced; otherwise only `convRefs` (the new/changed
|
|
389
|
+
* sessions this sync already listed) are checked. A session is (re)scanned only
|
|
390
|
+
* when the coverage ledger says the field is missing/stale — an absent-but-
|
|
391
|
+
* current field is skipped (the no-retry guarantee). Present values ship via the
|
|
392
|
+
* fields[] batch and are ledger-stamped after the server acks; absent results
|
|
393
|
+
* are stamped locally (nothing to ship) so they're never re-scanned.
|
|
394
|
+
*/
|
|
395
|
+
async function reconcileFieldsForTarget(cred, fullRefs, opts = {}) {
|
|
396
|
+
const base = cred.serverUrl.replace(/\/+$/, '');
|
|
397
|
+
const ledger = getSyncedRows(base);
|
|
398
|
+
let pushed = 0, absent = 0, scanned = 0;
|
|
399
|
+
for (const field of SYNC_FIELDS) {
|
|
400
|
+
const needFull = !!opts.force || fieldNeedsFullPass(base, field);
|
|
401
|
+
const candidates = needFull ? fullRefs() : (opts.convRefs ?? fullRefs());
|
|
402
|
+
const send = [];
|
|
403
|
+
const mtimeById = new Map();
|
|
404
|
+
const absentIds = [];
|
|
405
|
+
for (const ref of candidates) {
|
|
406
|
+
if (!fieldNeedsScan(ledger.get(ref.prefixedId), field, ref.mtime, opts.force))
|
|
407
|
+
continue;
|
|
408
|
+
scanned++;
|
|
409
|
+
mtimeById.set(ref.prefixedId, ref.mtime);
|
|
410
|
+
const raw = field.scan(ref);
|
|
411
|
+
const value = raw && raw.trim()
|
|
412
|
+
? redactSecrets(raw.trim(), { force: true, count: { redactions: 0 } }).slice(0, 200)
|
|
413
|
+
: null;
|
|
414
|
+
if (value !== null)
|
|
415
|
+
send.push({ session_id: ref.prefixedId, field: field.name, value });
|
|
416
|
+
else
|
|
417
|
+
absentIds.push(ref.prefixedId);
|
|
418
|
+
}
|
|
419
|
+
// Present values: ship in chunks, stamp coverage only after each ack so an
|
|
420
|
+
// interrupted run resumes cleanly.
|
|
421
|
+
for (let i = 0; i < send.length; i += 200) {
|
|
422
|
+
const chunk = send.slice(i, i + 200);
|
|
423
|
+
await postFieldBatch(base, cred.token, chunk);
|
|
424
|
+
markFieldCoverage(base, field, chunk.map((r) => ({ id: r.session_id, present: true, mtime: mtimeById.get(r.session_id) ?? 0 })));
|
|
425
|
+
pushed += chunk.length;
|
|
426
|
+
}
|
|
427
|
+
// Absent: nothing to ship; stamp locally so they're never re-scanned.
|
|
428
|
+
if (absentIds.length) {
|
|
429
|
+
markFieldCoverage(base, field, absentIds.map((id) => ({ id, present: false, mtime: mtimeById.get(id) ?? 0 })));
|
|
430
|
+
absent += absentIds.length;
|
|
431
|
+
}
|
|
432
|
+
// Full pass completed without throwing → record so it isn't repeated until
|
|
433
|
+
// the field version bumps or a re-scan is forced again.
|
|
434
|
+
if (needFull)
|
|
435
|
+
markFieldFullPassDone(base, field);
|
|
436
|
+
}
|
|
437
|
+
return { pushed, absent, scanned };
|
|
438
|
+
}
|
|
439
|
+
/**
|
|
440
|
+
* Manual entrypoint (CLI `reconcile`): reconcile derived fields for ALL targets.
|
|
441
|
+
* Normal `sync` already runs reconciliation each pass; this is the explicit
|
|
442
|
+
* on-demand backfill / forced re-scan (also what a UI "re-scan this field"
|
|
443
|
+
* action drives, via forceFieldRescan + a sync).
|
|
444
|
+
*/
|
|
445
|
+
export async function reconcileFields(opts = {}) {
|
|
446
|
+
const targets = loadAllCredentials();
|
|
447
|
+
if (targets.length === 0)
|
|
448
|
+
throw new Error('Not logged in — run `chat-recall login <server-url> --token <token>`');
|
|
449
|
+
let cached = null;
|
|
450
|
+
const fullRefs = () => (cached ??= listAvailableBackends().flatMap((b) => { try {
|
|
451
|
+
return b.listSessions({});
|
|
452
|
+
}
|
|
453
|
+
catch {
|
|
454
|
+
return [];
|
|
455
|
+
} }));
|
|
456
|
+
let pushed = 0, absent = 0, scanned = 0;
|
|
457
|
+
const perTarget = {};
|
|
458
|
+
for (const cred of targets) {
|
|
459
|
+
const r = await reconcileFieldsForTarget(cred, fullRefs, { force: opts.force });
|
|
460
|
+
pushed += r.pushed;
|
|
461
|
+
absent += r.absent;
|
|
462
|
+
scanned += r.scanned;
|
|
463
|
+
perTarget[cred.serverUrl.replace(/\/+$/, '')] = { pushed: r.pushed, absent: r.absent };
|
|
464
|
+
}
|
|
465
|
+
return { sessions: fullRefs().length, scanned, pushed, absent, perTarget };
|
|
466
|
+
}
|
|
467
|
+
/**
|
|
468
|
+
* Reap batch-scan temp dirs leaked by a previous run.
|
|
469
|
+
*
|
|
470
|
+
* The scan below deletes its dir in a `finally` — which a SIGKILL or the OOM
|
|
471
|
+
* killer skips entirely, and this loop is exactly the one that has OOM-killed
|
|
472
|
+
* the daemon before. Each leaked dir holds one file per session: one of them
|
|
473
|
+
* was found at 3.4GB / 309k files, and together with other cruft it filled a
|
|
474
|
+
* 31GB tmpfs, at which point *everything* on the box that needed /tmp broke.
|
|
475
|
+
*
|
|
476
|
+
* So: before starting a scan, delete our own stale dirs. 6h is far longer than
|
|
477
|
+
* any real scan, so an in-flight dir from a concurrent process is never at risk.
|
|
478
|
+
*/
|
|
479
|
+
export function reapStaleScanDirs(dir = tmpdir(), maxAgeMs = 6 * 3600_000, now = Date.now()) {
|
|
480
|
+
let reaped = 0;
|
|
481
|
+
let names;
|
|
482
|
+
try {
|
|
483
|
+
names = readdirSync(dir);
|
|
484
|
+
}
|
|
485
|
+
catch {
|
|
486
|
+
return 0;
|
|
487
|
+
}
|
|
488
|
+
for (const name of names) {
|
|
489
|
+
if (!name.startsWith('cr-batchscan-'))
|
|
490
|
+
continue;
|
|
491
|
+
const p = join(dir, name);
|
|
492
|
+
try {
|
|
493
|
+
if (now - statSync(p).mtimeMs < maxAgeMs)
|
|
494
|
+
continue;
|
|
495
|
+
rmSync(p, { recursive: true, force: true });
|
|
496
|
+
reaped++;
|
|
497
|
+
}
|
|
498
|
+
catch { /* raced with its owner, or not ours to remove */ }
|
|
499
|
+
}
|
|
500
|
+
return reaped;
|
|
501
|
+
}
|
|
502
|
+
/**
|
|
503
|
+
* Run the optional external detectors (gitleaks/trufflehog) over many sessions
|
|
504
|
+
* with a BOUNDED amount of pre-redaction text on disk at any one time.
|
|
505
|
+
*
|
|
506
|
+
* The detectors take a path, so their input has to be materialized — and that
|
|
507
|
+
* input is raw session text, i.e. real credentials in cleartext. Writing all of
|
|
508
|
+
* it at once is what left 3.4GB / 309k such files in /tmp when the daemon was
|
|
509
|
+
* OOM-killed mid-scan (the `finally` never ran). So we fill a temp dir up to
|
|
510
|
+
* `maxBytes`, scan it, delete it, and repeat: worst-case exposure — and
|
|
511
|
+
* worst-case leak if this process is killed — is one slice, not the whole walk.
|
|
512
|
+
*
|
|
513
|
+
* Cost of slicing is 2 detector spawns per extra slice, still far below the
|
|
514
|
+
* 2·N of per-session scanning.
|
|
515
|
+
*
|
|
516
|
+
* `text()` returning null (unexportable session) is skipped silently: it just
|
|
517
|
+
* gets no external findings, the builtin scan still covers it.
|
|
518
|
+
* `scan` is injectable so tests don't need the binaries installed.
|
|
519
|
+
*/
|
|
520
|
+
export function batchScanExternal(items, opts = {}) {
|
|
521
|
+
const maxBytes = opts.maxBytes ?? BATCH_SCAN_MAX_BYTES;
|
|
522
|
+
const scan = opts.scan ?? scanDirForSecrets;
|
|
523
|
+
const findings = new Map();
|
|
524
|
+
let scanned = 0, slices = 0, scanMs = 0;
|
|
525
|
+
if (items.length === 0)
|
|
526
|
+
return { findings, scanned, slices, scanMs };
|
|
527
|
+
reapStaleScanDirs();
|
|
528
|
+
let scanDir = mkdtempSync(join(tmpdir(), 'cr-batchscan-'));
|
|
529
|
+
let safeToId = new Map();
|
|
530
|
+
let bytesInDir = 0;
|
|
531
|
+
/** Scan what's currently materialized, record findings, then delete the dir. */
|
|
532
|
+
const flushSlice = () => {
|
|
533
|
+
if (safeToId.size > 0) {
|
|
534
|
+
const t0 = performance.now();
|
|
535
|
+
for (const f of scan(scanDir, { verifyOnly: opts.verifyOnly })) {
|
|
536
|
+
const id = safeToId.get(basename(f.file).replace(/\.txt$/, ''));
|
|
537
|
+
if (!id)
|
|
538
|
+
continue;
|
|
539
|
+
const list = findings.get(id) ?? [];
|
|
540
|
+
list.push({ detector: f.detector, rule: f.rule, line: f.line, preview: f.preview, verified: f.verified });
|
|
541
|
+
findings.set(id, list);
|
|
542
|
+
}
|
|
543
|
+
scanMs += performance.now() - t0;
|
|
544
|
+
scanned += safeToId.size;
|
|
545
|
+
slices++;
|
|
546
|
+
}
|
|
547
|
+
try {
|
|
548
|
+
rmSync(scanDir, { recursive: true, force: true });
|
|
549
|
+
}
|
|
550
|
+
catch { /* best-effort */ }
|
|
551
|
+
safeToId = new Map();
|
|
552
|
+
bytesInDir = 0;
|
|
553
|
+
};
|
|
554
|
+
try {
|
|
555
|
+
for (const item of items) {
|
|
556
|
+
try {
|
|
557
|
+
const raw = item.text();
|
|
558
|
+
if (raw === null)
|
|
559
|
+
continue;
|
|
560
|
+
const safe = item.id.replace(/[^A-Za-z0-9_-]/g, '');
|
|
561
|
+
safeToId.set(safe, item.id);
|
|
562
|
+
writeFileSync(join(scanDir, `${safe}.txt`), raw);
|
|
563
|
+
bytesInDir += Buffer.byteLength(raw);
|
|
564
|
+
}
|
|
565
|
+
catch { /* unexportable session — no external findings for it */ }
|
|
566
|
+
// Bound on bytes ACTUALLY written, not an estimate. A single session
|
|
567
|
+
// larger than the cap still gets scanned — alone, in its own slice.
|
|
568
|
+
if (bytesInDir >= maxBytes) {
|
|
569
|
+
flushSlice();
|
|
570
|
+
scanDir = mkdtempSync(join(tmpdir(), 'cr-batchscan-'));
|
|
571
|
+
}
|
|
572
|
+
}
|
|
573
|
+
flushSlice();
|
|
574
|
+
}
|
|
575
|
+
finally {
|
|
576
|
+
// flushSlice removed the final dir already (force:true makes this a no-op);
|
|
577
|
+
// this is what cleans up when the loop above threw.
|
|
578
|
+
try {
|
|
579
|
+
rmSync(scanDir, { recursive: true, force: true });
|
|
580
|
+
}
|
|
581
|
+
catch { /* best-effort */ }
|
|
582
|
+
}
|
|
583
|
+
return { findings, scanned, slices, scanMs };
|
|
584
|
+
}
|
|
585
|
+
async function syncToTarget(cred, opts = {}) {
|
|
586
|
+
const sync = loadSettings().sync;
|
|
587
|
+
// Exclusions = local settings ∪ server-side tenant config (dashboard-edited).
|
|
588
|
+
const serverCfg = await fetchTenantSyncConfig(cred);
|
|
589
|
+
const excludeTools = new Set([...(sync?.excludeTools ?? []), ...serverCfg.excludeTools]);
|
|
590
|
+
const excludeProjects = [...(sync?.excludeProjects ?? []), ...serverCfg.excludeProjects];
|
|
591
|
+
// Upload category toggles (settings.sync.upload, all default true).
|
|
592
|
+
// Redaction of shipped text always runs regardless of toggles.
|
|
593
|
+
const upload = {
|
|
594
|
+
raw: sync?.upload?.raw !== false,
|
|
595
|
+
sessionMeta: sync?.upload?.sessionMeta !== false,
|
|
596
|
+
findings: sync?.upload?.findings !== false,
|
|
597
|
+
};
|
|
598
|
+
// Cleartext paths: explicit flag wins, else the persistent setting
|
|
599
|
+
// (sync.pathsCleartext — what the watch daemon uses).
|
|
600
|
+
const cleartext = opts.cleartextPaths ?? sync?.pathsCleartext === true;
|
|
601
|
+
const mapPath = (p) => (cleartext ? p : hashPath(p));
|
|
602
|
+
const excluded = (projectPath) => excludeProjects.some((x) => x && projectPath.includes(x));
|
|
603
|
+
// Opt-in selective sync (settings.sync.syncMode='only'): ship ONLY allowlisted
|
|
604
|
+
// projects. Allowlist = local ∪ tenant config; matched on the resolved project
|
|
605
|
+
// id (what the dashboard picker lists) or a path substring. 'all'/unset ships
|
|
606
|
+
// every non-excluded project (unchanged default).
|
|
607
|
+
const syncOnly = new Set([...(sync?.syncOnlyProjects ?? []), ...(serverCfg.syncOnlyProjects ?? [])]);
|
|
608
|
+
const includedProject = (projectPath) => {
|
|
609
|
+
if ((sync?.syncMode ?? 'all') !== 'only')
|
|
610
|
+
return true; // fast path: no resolve
|
|
611
|
+
const r = resolveProjectId(projectPath);
|
|
612
|
+
return isProjectSyncable(r.source === 'ignored' ? '' : r.id, projectPath, { syncMode: 'only', syncOnly });
|
|
613
|
+
};
|
|
614
|
+
// Ledger mode walks ALL sessions (the ledger does the skipping — that's
|
|
615
|
+
// what lets a previously-failed session retry no matter how old it is).
|
|
616
|
+
// Watermark mode keeps the cheap bounded walk for explicit --since runs.
|
|
617
|
+
// Version handshake: refuse loudly instead of silently degrading. An old
|
|
618
|
+
// server ignores payload fields it doesn't know — which looks like
|
|
619
|
+
// success and produces gutted data (lived experience, June 2026).
|
|
620
|
+
const MIN_SERVER_API = 2;
|
|
621
|
+
try {
|
|
622
|
+
const caps = await fetchWithTimeout(`${cred.serverUrl.replace(/\/$/, '')}/api/capabilities`).then((r) => r.json());
|
|
623
|
+
if ((caps.apiVersion ?? 0) < MIN_SERVER_API) {
|
|
624
|
+
throw new Error(`server too old: apiVersion ${caps.apiVersion ?? 'none'} < required ${MIN_SERVER_API} — update the server image before syncing`);
|
|
625
|
+
}
|
|
626
|
+
}
|
|
627
|
+
catch (e) {
|
|
628
|
+
if (e instanceof Error && e.message.startsWith('server too old'))
|
|
629
|
+
throw e;
|
|
630
|
+
throw new Error(`cannot verify server version (${e instanceof Error ? e.message : e}) — refusing to sync blind`);
|
|
631
|
+
}
|
|
632
|
+
// Fetch tenant-specific secret-scan configuration from the server. Rules are
|
|
633
|
+
// configured in the dashboard and must be applied to raw session text on the
|
|
634
|
+
// client (the server never sees unredacted secrets). Verification live-checks
|
|
635
|
+
// matched keys with their issuers (e.g. AWS, GitHub) — on by default, but a
|
|
636
|
+
// tenant admin can disable it.
|
|
637
|
+
const { tenantRules, verifySecrets } = await fetchTenantSecurityConfig(cred);
|
|
638
|
+
// Walk scope. 'full' (default) lists EVERY session — the ledger does the
|
|
639
|
+
// skipping, which is what lets an old failed session retry forever. 'changed'
|
|
640
|
+
// bounds the LISTING to recently-touched sessions (mtime window with overlap):
|
|
641
|
+
// the watch daemon's per-file-change ticks use it so a burst of keystrokes
|
|
642
|
+
// costs a walk over ~a handful of refs, not 30k+. Failed/old sessions still
|
|
643
|
+
// converge via the daemon's 15-min heartbeat + startup ticks, which stay
|
|
644
|
+
// 'full'. The ledger still drives skip/append/full per ref in BOTH modes.
|
|
645
|
+
const CHANGED_WALK_OVERLAP_MS = 15 * 60_000;
|
|
646
|
+
const changedWalk = opts.walk === 'changed';
|
|
647
|
+
const listSinceMs = changedWalk
|
|
648
|
+
? Math.max(0, (opts.sinceMs ?? 0) - CHANGED_WALK_OVERLAP_MS)
|
|
649
|
+
: (opts.useLedger ? undefined : opts.sinceMs);
|
|
650
|
+
const refs = listAvailableBackends().flatMap((b) => {
|
|
651
|
+
try {
|
|
652
|
+
return b.listSessions({ sinceMs: listSinceMs });
|
|
653
|
+
}
|
|
654
|
+
catch {
|
|
655
|
+
return [];
|
|
656
|
+
}
|
|
657
|
+
});
|
|
658
|
+
// ── Upload plumbing, created up-front so the walks below can stream.
|
|
659
|
+
// Byte-aware batching: cap each POST at ~4MB of serialized payload (and a
|
|
660
|
+
// per-field item cap) so transcript-heavy runs can't blow the server's
|
|
661
|
+
// body limit. A single row larger than the cap still ships alone.
|
|
662
|
+
//
|
|
663
|
+
// Streaming matters: a 5k-session backfill with diff payloads would be
|
|
664
|
+
// gigabytes if accumulated before upload. Each batcher flushes as soon as
|
|
665
|
+
// its window fills, so memory stays bounded at ~one batch per category.
|
|
666
|
+
//
|
|
667
|
+
// Backpressure, not a blind sleep. Server-side ingest does real work per row
|
|
668
|
+
// (chunking, classification, FTS/vector inserts); an UNBOUNDED backfill
|
|
669
|
+
// browned out a node on 2026-06-11. The protection that matters is bounding
|
|
670
|
+
// how many batches hit the server AT ONCE — so we cap in-flight uploads (the
|
|
671
|
+
// cap is the rate limit) and let the server push back with 429/Retry-After,
|
|
672
|
+
// sleeping only when it actually tells us to. Local/LAN targets need none of
|
|
673
|
+
// this politeness (you own the box) → higher cap, zero pacing. An explicit
|
|
674
|
+
// --throttle (opts.throttleMs>0) forces the old serial+sleep mode for a
|
|
675
|
+
// known-fragile server.
|
|
676
|
+
const MAX_BATCH_BYTES = 4 * 1024 * 1024;
|
|
677
|
+
const sleep = (ms) => new Promise((r) => setTimeout(r, ms));
|
|
678
|
+
const explicitThrottle = typeof opts.throttleMs === 'number' && opts.throttleMs > 0;
|
|
679
|
+
const throttleMs = explicitThrottle ? opts.throttleMs : 0;
|
|
680
|
+
const maxInflight = explicitThrottle ? 1 : (isLocalHost(cred.serverUrl) ? 6 : 4);
|
|
681
|
+
// Per-attempt upload timeout. WITHOUT this, a half-open/slow connection makes
|
|
682
|
+
// `fetch` hang FOREVER (Node fetch has no default timeout) — observed as a
|
|
683
|
+
// sync stuck for 20-40min at idle CPU with zero progress. With it, a hung
|
|
684
|
+
// upload aborts, surfaces as an error, and goes through the retry path below.
|
|
685
|
+
const UPLOAD_TIMEOUT_MS = Number(process.env.CHAT_RECALL_UPLOAD_TIMEOUT_MS) || 90_000;
|
|
686
|
+
// Single POST with retry. Retryable: network/abort errors, HTTP 429 (obeying
|
|
687
|
+
// Retry-After — the server's own rate signal), and 5xx. Non-429 4xx (bad
|
|
688
|
+
// token, oversized body) is fatal: retrying can't fix it.
|
|
689
|
+
const post = async (body) => {
|
|
690
|
+
const payload = JSON.stringify(body);
|
|
691
|
+
const RETRY_DELAYS_MS = [2000, 8000, 30000];
|
|
692
|
+
for (let attempt = 0;; attempt++) {
|
|
693
|
+
const ac = new AbortController();
|
|
694
|
+
const timer = setTimeout(() => ac.abort(new Error(`upload timed out after ${UPLOAD_TIMEOUT_MS}ms`)), UPLOAD_TIMEOUT_MS);
|
|
695
|
+
try {
|
|
696
|
+
const res = await fetch(`${base}/api/sync`, {
|
|
697
|
+
method: 'POST',
|
|
698
|
+
// No token ⇒ no auth header: a no-auth local server (AUTH_PROVIDER=none)
|
|
699
|
+
// accepts it as the single 'default' tenant.
|
|
700
|
+
headers: cred.token
|
|
701
|
+
? { 'content-type': 'application/json', authorization: `Bearer ${cred.token}` }
|
|
702
|
+
: { 'content-type': 'application/json' },
|
|
703
|
+
body: payload,
|
|
704
|
+
signal: ac.signal,
|
|
705
|
+
});
|
|
706
|
+
if (res.ok) {
|
|
707
|
+
const body = await res.json().catch(() => ({}));
|
|
708
|
+
if (body.cli && body.cli.version) {
|
|
709
|
+
const { planAutoUpdate, runAutoUpdate } = await import('./auto-update.js');
|
|
710
|
+
const authHeaders = cred.token ? { authorization: `Bearer ${cred.token}` } : {};
|
|
711
|
+
const plan = planAutoUpdate(base, { cli: body.cli }, ownVersion, process.env.CHAT_RECALL_AUTO_UPDATE);
|
|
712
|
+
if (plan.update) {
|
|
713
|
+
void runAutoUpdate(base, authHeaders, ownVersion).catch(() => { });
|
|
714
|
+
}
|
|
715
|
+
}
|
|
716
|
+
break;
|
|
717
|
+
}
|
|
718
|
+
const retryable = res.status === 429 || res.status >= 500;
|
|
719
|
+
const text = await res.text().catch(() => '');
|
|
720
|
+
// Auth rejection is a standing condition, not a glitch: the device
|
|
721
|
+
// token was revoked or the tenant is gone. Every future sync will fail
|
|
722
|
+
// the same way until the human reconnects — say exactly that.
|
|
723
|
+
if (res.status === 401 || res.status === 403) {
|
|
724
|
+
throw new Error(`server rejected this machine's device token (HTTP ${res.status} — revoked?). Reconnect with: chat-recall login ${base}`);
|
|
725
|
+
}
|
|
726
|
+
if (!retryable)
|
|
727
|
+
throw new Error(`sync failed: HTTP ${res.status} ${text}`);
|
|
728
|
+
if (attempt >= RETRY_DELAYS_MS.length)
|
|
729
|
+
throw new Error(`sync failed after ${attempt + 1} attempts: HTTP ${res.status} ${text}`);
|
|
730
|
+
// 429: obey Retry-After (seconds) when the server sends it; else fall
|
|
731
|
+
// back to the backoff schedule. This puts the pacing decision where it
|
|
732
|
+
// belongs — on the server that knows its own load.
|
|
733
|
+
const retryAfterS = Number(res.headers.get('retry-after'));
|
|
734
|
+
const waitMs = res.status === 429 && retryAfterS > 0 ? retryAfterS * 1000 : RETRY_DELAYS_MS[attempt];
|
|
735
|
+
console.error(`[sync] HTTP ${res.status} from ${base} — backing off ${Math.round(waitMs / 1000)}s (attempt ${attempt + 1})`);
|
|
736
|
+
await sleep(waitMs);
|
|
737
|
+
continue;
|
|
738
|
+
}
|
|
739
|
+
catch (err) {
|
|
740
|
+
const fatal = err instanceof Error && /HTTP 4\d\d|sync failed after/.test(err.message);
|
|
741
|
+
if (fatal || attempt >= RETRY_DELAYS_MS.length)
|
|
742
|
+
throw err;
|
|
743
|
+
console.error(`[sync] upload attempt ${attempt + 1} failed (${err instanceof Error ? err.message : err}) — retrying in ${RETRY_DELAYS_MS[attempt] / 1000}s`);
|
|
744
|
+
await sleep(RETRY_DELAYS_MS[attempt]);
|
|
745
|
+
continue;
|
|
746
|
+
}
|
|
747
|
+
finally {
|
|
748
|
+
clearTimeout(timer);
|
|
749
|
+
}
|
|
750
|
+
}
|
|
751
|
+
if (throttleMs > 0)
|
|
752
|
+
await sleep(throttleMs);
|
|
753
|
+
};
|
|
754
|
+
// Bounded-concurrency pool. Up to `maxInflight` POSTs run at once; awaiting a
|
|
755
|
+
// free slot in `submit` is what paces the producer (natural backpressure).
|
|
756
|
+
// A POST that fails after its own retries is captured into `poolError` and
|
|
757
|
+
// re-thrown at the next slot-wait / drain, so a fatal error still aborts the
|
|
758
|
+
// whole sync (matching the old serial behaviour).
|
|
759
|
+
const inflight = new Set();
|
|
760
|
+
let poolError = null;
|
|
761
|
+
const submit = async (body, after) => {
|
|
762
|
+
while (inflight.size >= maxInflight)
|
|
763
|
+
await Promise.race(inflight);
|
|
764
|
+
if (poolError)
|
|
765
|
+
throw poolError;
|
|
766
|
+
let task;
|
|
767
|
+
task = (async () => { await post(body); after(); })()
|
|
768
|
+
.catch((e) => { poolError = poolError ?? e; })
|
|
769
|
+
.finally(() => { inflight.delete(task); });
|
|
770
|
+
inflight.add(task);
|
|
771
|
+
};
|
|
772
|
+
const drainInflight = async () => {
|
|
773
|
+
await Promise.allSettled(inflight);
|
|
774
|
+
if (poolError)
|
|
775
|
+
throw poolError;
|
|
776
|
+
};
|
|
777
|
+
// Track every local session id we see this sync. After the walk, any
|
|
778
|
+
// session id recorded in the per-server ledger but missing locally is a
|
|
779
|
+
// deletion → ship a tombstone so the server removes it. This only covers
|
|
780
|
+
// sessions this device previously synced; deletions on other devices are
|
|
781
|
+
// handled by their own collectors.
|
|
782
|
+
const localSessionIds = new Set();
|
|
783
|
+
// session_id → content hash for the FULL conversations shipped this run, so
|
|
784
|
+
// the convBatch ack can stamp the ledger `h` alongside {m,o,s}. A later tick
|
|
785
|
+
// that re-classifies the session as FULL on an mtime-only change then matches
|
|
786
|
+
// this hash and skips the rebuild entirely.
|
|
787
|
+
const shippedHash = new Map();
|
|
788
|
+
// `flatten`: each add() is a GROUP of rows that must land in the same
|
|
789
|
+
// POST (e.g. one session's findings — the server replaces a session's
|
|
790
|
+
// findings wholesale, so splitting a session across two batches would
|
|
791
|
+
// lose the first half).
|
|
792
|
+
const makeBatcher = (field, maxItems, flatten = false, onFlush) => {
|
|
793
|
+
let batch = [];
|
|
794
|
+
let batchBytes = 0;
|
|
795
|
+
return {
|
|
796
|
+
sent: 0,
|
|
797
|
+
async add(row) {
|
|
798
|
+
const size = JSON.stringify(row).length;
|
|
799
|
+
if (batch.length > 0 && (batchBytes + size > MAX_BATCH_BYTES || batch.length >= maxItems))
|
|
800
|
+
await this.drain();
|
|
801
|
+
batch.push(row);
|
|
802
|
+
batchBytes += size;
|
|
803
|
+
},
|
|
804
|
+
async drain() {
|
|
805
|
+
if (batch.length === 0)
|
|
806
|
+
return;
|
|
807
|
+
const rows = flatten ? batch.flat() : batch;
|
|
808
|
+
batch = [];
|
|
809
|
+
batchBytes = 0;
|
|
810
|
+
// Hand off to the concurrency pool; `sent`/onFlush (e.g. markSynced)
|
|
811
|
+
// fire only after the server acks THIS batch, same as before.
|
|
812
|
+
await submit({ [field]: rows }, () => { this.sent += rows.length; onFlush?.(rows); });
|
|
813
|
+
},
|
|
814
|
+
};
|
|
815
|
+
};
|
|
816
|
+
const base = cred.serverUrl.replace(/\/$/, '');
|
|
817
|
+
const ledger = opts.useLedger ? getSyncedRows(base) : null;
|
|
818
|
+
const convBatch = makeBatcher('conversations', 25, false, (rows) => {
|
|
819
|
+
// Server acked this batch — record exactly these sessions at exactly
|
|
820
|
+
// the mtimes that were shipped. Failures never reach here, so an
|
|
821
|
+
// unsynced session stays unsynced and retries next tick.
|
|
822
|
+
//
|
|
823
|
+
// For append-only backends, also stamp the byte cursor (o=size, s=size) so
|
|
824
|
+
// the next tick can APPEND from here instead of re-doing a FULL sync. We
|
|
825
|
+
// read the CURRENT file size (not the size at ship time) — the file may
|
|
826
|
+
// have grown between ship and ack, but that's fine: the next tick's gate
|
|
827
|
+
// sees size > ack.s and APPENDs the new tail. If it shrank (rotation), the
|
|
828
|
+
// gate falls back to FULL.
|
|
829
|
+
try {
|
|
830
|
+
markSynced(base, rows.map((r) => {
|
|
831
|
+
// Record the SAME offset the conv shipped to the server (conv.from_offset
|
|
832
|
+
// = build-time file size), NOT a freshly-stat'd size — otherwise an
|
|
833
|
+
// actively-growing file's ledger offset would differ from the server's
|
|
834
|
+
// stored `o`, and every subsequent append would miss the continuity
|
|
835
|
+
// check and fall back to FULL. Consistency here is what lets the active
|
|
836
|
+
// session actually append.
|
|
837
|
+
const off = typeof r.from_offset === 'number' ? r.from_offset : 0;
|
|
838
|
+
const hash = shippedHash.get(r.session_id);
|
|
839
|
+
// acked: the server accepted the batch containing this conversation, so
|
|
840
|
+
// the cursor is a real delivery receipt.
|
|
841
|
+
return off > 0
|
|
842
|
+
? { id: r.session_id, mtime: r.mtime, offset: off, size: off, hash, acked: true }
|
|
843
|
+
: { id: r.session_id, mtime: r.mtime, hash };
|
|
844
|
+
}));
|
|
845
|
+
}
|
|
846
|
+
catch { /* ledger is local bookkeeping — never fail an upload over it */ }
|
|
847
|
+
});
|
|
848
|
+
// ── Append batcher (tail-only sync) ──────────────────────────────
|
|
849
|
+
// See docs/SYNC-INCREMENTAL.md. Append conversations ship with append:true;
|
|
850
|
+
// the server appends chunks + merges the envelope. The server may respond
|
|
851
|
+
// with full_resync_needed:[ids] when it has no prior envelope for a session
|
|
852
|
+
// (data loss, first sync) — those sessions are NOT marked synced, so the
|
|
853
|
+
// next tick re-classifies them as FULL (offset 0 or absent → full).
|
|
854
|
+
//
|
|
855
|
+
// Append payloads are small (tail only), so we batch up to 50 per POST and
|
|
856
|
+
// inspect the response body for the full_resync_needed list. This path does
|
|
857
|
+
// NOT go through the shared `post`/`submit` pool — it needs the response
|
|
858
|
+
// body, which the shared path discards.
|
|
859
|
+
const appendResults = { uploaded: 0, fullResyncNeeded: new Set() };
|
|
860
|
+
const postAppend = async (convs) => {
|
|
861
|
+
const headers = { 'content-type': 'application/json' };
|
|
862
|
+
if (cred.token)
|
|
863
|
+
headers.authorization = `Bearer ${cred.token}`;
|
|
864
|
+
const ac = new AbortController();
|
|
865
|
+
const timer = setTimeout(() => ac.abort(new Error(`upload timed out after ${UPLOAD_TIMEOUT_MS}ms`)), UPLOAD_TIMEOUT_MS);
|
|
866
|
+
try {
|
|
867
|
+
const res = await fetch(`${base}/api/sync`, {
|
|
868
|
+
method: 'POST', headers,
|
|
869
|
+
body: JSON.stringify({ conversations: convs }),
|
|
870
|
+
signal: ac.signal,
|
|
871
|
+
});
|
|
872
|
+
if (!res.ok)
|
|
873
|
+
throw new Error(`append sync failed: HTTP ${res.status} ${await res.text().catch(() => '')}`);
|
|
874
|
+
const body = await res.json().catch(() => ({}));
|
|
875
|
+
if (body.cli && body.cli.version) {
|
|
876
|
+
const { planAutoUpdate, runAutoUpdate } = await import('./auto-update.js');
|
|
877
|
+
const authHeaders = cred.token ? { authorization: `Bearer ${cred.token}` } : {};
|
|
878
|
+
const plan = planAutoUpdate(base, { cli: body.cli }, ownVersion, process.env.CHAT_RECALL_AUTO_UPDATE);
|
|
879
|
+
if (plan.update) {
|
|
880
|
+
void runAutoUpdate(base, authHeaders, ownVersion).catch(() => { });
|
|
881
|
+
}
|
|
882
|
+
}
|
|
883
|
+
const fullNeeded = new Set(body.full_resync_needed ?? []);
|
|
884
|
+
// Mark synced ONLY the sessions the server accepted as appends. Sessions
|
|
885
|
+
// the server asked to full-resync stay unmarked → next tick = FULL.
|
|
886
|
+
const acked = convs.filter((c) => !fullNeeded.has(c.session_id));
|
|
887
|
+
if (acked.length > 0) {
|
|
888
|
+
try {
|
|
889
|
+
markSynced(base, acked.map((c) => ({
|
|
890
|
+
id: c.session_id,
|
|
891
|
+
mtime: c.mtime,
|
|
892
|
+
offset: c.from_offset,
|
|
893
|
+
size: c.from_offset, // size = new offset (file grew to here)
|
|
894
|
+
acked: true, // server merged this tail
|
|
895
|
+
})));
|
|
896
|
+
}
|
|
897
|
+
catch { /* ledger — never fail an upload over it */ }
|
|
898
|
+
}
|
|
899
|
+
appendResults.uploaded += acked.length;
|
|
900
|
+
for (const id of fullNeeded)
|
|
901
|
+
appendResults.fullResyncNeeded.add(id);
|
|
902
|
+
}
|
|
903
|
+
finally {
|
|
904
|
+
clearTimeout(timer);
|
|
905
|
+
}
|
|
906
|
+
};
|
|
907
|
+
let appendBatch = [];
|
|
908
|
+
const APPEND_BATCH_SIZE = 50;
|
|
909
|
+
const appendFlush = async () => {
|
|
910
|
+
if (appendBatch.length === 0)
|
|
911
|
+
return;
|
|
912
|
+
const batch = appendBatch;
|
|
913
|
+
appendBatch = [];
|
|
914
|
+
await postAppend(batch);
|
|
915
|
+
};
|
|
916
|
+
const appendAdd = async (conv) => {
|
|
917
|
+
appendBatch.push(conv);
|
|
918
|
+
if (appendBatch.length >= APPEND_BATCH_SIZE)
|
|
919
|
+
await appendFlush();
|
|
920
|
+
};
|
|
921
|
+
const itemsBatch = makeBatcher('items', 100);
|
|
922
|
+
const linksBatch = makeBatcher('links', 2000);
|
|
923
|
+
const derivedBatch = makeBatcher('derived', 50);
|
|
924
|
+
const kgEntityBatch = makeBatcher('kg_entities', 2000);
|
|
925
|
+
const kgTripleBatch = makeBatcher('kg_triples', 1000);
|
|
926
|
+
// Secret findings ship per-session as a GROUP (flatten=true) — the server
|
|
927
|
+
// replaces a session's findings wholesale, so they must land in one POST.
|
|
928
|
+
const findingsBatch = makeBatcher('findings', 200, true);
|
|
929
|
+
// External detectors (gitleaks/trufflehog) are OPT-IN and default OFF:
|
|
930
|
+
// isSecretScannerAvailable() returns false unless CHAT_RECALL_EXTERNAL_SCANNERS
|
|
931
|
+
// is set AND a binary is on PATH. Resolved once (each check spawns `which`).
|
|
932
|
+
// CHAT_RECALL_SCAN_SECRETS=0 opts out even when enabled (e.g. to keep a huge
|
|
933
|
+
// backfill fast); upload.findings=false disables shipping too.
|
|
934
|
+
// NOTE: this flag gates ONLY the external binaries. Builtin regex findings
|
|
935
|
+
// and tenant rules run for every user further down (buildConversationSync),
|
|
936
|
+
// and redaction of everything shipped is unconditional.
|
|
937
|
+
const scanSecrets = isSecretScannerAvailable() && upload.findings && process.env.CHAT_RECALL_SCAN_SECRETS !== '0';
|
|
938
|
+
const verifySecretsScan = verifySecrets;
|
|
939
|
+
// ── Live KG accumulator. Triples are extracted on the spot from each
|
|
940
|
+
// item's redacted text (no local knowledge-graph DB is read). Dedupe by
|
|
941
|
+
// subject|predicate|object|valid_from; entity types come from `is_a`
|
|
942
|
+
// triples seen in the same pass, defaulting to 'unknown' otherwise.
|
|
943
|
+
const kgSeen = new Set();
|
|
944
|
+
const kgEntityType = new Map();
|
|
945
|
+
const kgRows = [];
|
|
946
|
+
const accumulateKg = (triples, sourceSession) => {
|
|
947
|
+
for (const t of triples) {
|
|
948
|
+
const key = `${t.subject}|${t.predicate}|${t.object}|${t.validFrom ?? ''}`;
|
|
949
|
+
if (kgSeen.has(key))
|
|
950
|
+
continue;
|
|
951
|
+
kgSeen.add(key);
|
|
952
|
+
if (t.predicate === 'is_a' && !kgEntityType.has(t.subject))
|
|
953
|
+
kgEntityType.set(t.subject, t.object);
|
|
954
|
+
kgRows.push({
|
|
955
|
+
subject: t.subject, predicate: t.predicate, object: t.object,
|
|
956
|
+
valid_from: t.validFrom ?? null, valid_to: null,
|
|
957
|
+
confidence: t.confidence ?? 1.0, source_session: sourceSession,
|
|
958
|
+
});
|
|
959
|
+
}
|
|
960
|
+
};
|
|
961
|
+
let skipped = 0, redactions = 0, walked = 0;
|
|
962
|
+
let scanMs = 0, scanned = 0; // secret-scan metrics
|
|
963
|
+
const slice = refs.slice(0, opts.limit ?? refs.length);
|
|
964
|
+
// CHAT_RECALL_SYNC_TRACE=1: phase + progress markers with heap usage, to
|
|
965
|
+
// stderr. This is how the 2026-07-03 walk OOM was localized — keep it.
|
|
966
|
+
const trace = process.env.CHAT_RECALL_SYNC_TRACE === '1'
|
|
967
|
+
? (msg) => { const m = process.memoryUsage(); console.error(`[sync-trace] ${msg} · heap=${(m.heapUsed / 1048576).toFixed(0)}MB rss=${(m.rss / 1048576).toFixed(0)}MB`); }
|
|
968
|
+
: null;
|
|
969
|
+
trace?.(`walk start: ${slice.length} ref(s) → ${base}`);
|
|
970
|
+
// ── Derived-field reconciliation (native title, …) — runs FIRST, BEFORE the
|
|
971
|
+
// heavy conversation walk, so it's never blocked by per-session git/replay
|
|
972
|
+
// compute (titles flow even when the conversation phase is slow). Pushes only
|
|
973
|
+
// {session_id, field, value}, never a conversation. Cheap pass over the
|
|
974
|
+
// sessions this sync listed; a one-time full pass per field when its version
|
|
975
|
+
// bumped or a re-scan was forced. Best-effort — a reconcile failure must not
|
|
976
|
+
// fail the conversation sync.
|
|
977
|
+
if (upload.sessionMeta) {
|
|
978
|
+
try {
|
|
979
|
+
await reconcileFieldsForTarget(cred, () => listAvailableBackends().flatMap((b) => { try {
|
|
980
|
+
return b.listSessions({});
|
|
981
|
+
}
|
|
982
|
+
catch {
|
|
983
|
+
return [];
|
|
984
|
+
} }), { convRefs: slice });
|
|
985
|
+
}
|
|
986
|
+
catch (e) {
|
|
987
|
+
console.error(`[sync] field reconcile (${base}): ${e instanceof Error ? e.message : e}`);
|
|
988
|
+
}
|
|
989
|
+
}
|
|
990
|
+
trace?.('field reconcile done');
|
|
991
|
+
// Shared gate: classify each session's sync mode for this tick. Used by
|
|
992
|
+
// both the batch pre-scan and the upload loop so they stay in lockstep.
|
|
993
|
+
// See docs/SYNC-INCREMENTAL.md for the skip/append/full decision.
|
|
994
|
+
const backendFor = (ref) => getBackendForId(ref.prefixedId);
|
|
995
|
+
const modeOf = (ref) => {
|
|
996
|
+
if (excludeTools.has(ref.toolId))
|
|
997
|
+
return 'skip';
|
|
998
|
+
if (excluded(ref.projectPath))
|
|
999
|
+
return 'skip';
|
|
1000
|
+
if (!includedProject(ref.projectPath))
|
|
1001
|
+
return 'skip';
|
|
1002
|
+
if (!ledger)
|
|
1003
|
+
return ref.mtime >= (opts.sinceMs ?? 0) ? 'full' : 'skip';
|
|
1004
|
+
const ack = ledger.get(ref.prefixedId);
|
|
1005
|
+
const backend = backendFor(ref);
|
|
1006
|
+
// A session that exists in more than one home cannot be tail-appended: the
|
|
1007
|
+
// size is the SUM across copies while a byte offset addresses one file, so
|
|
1008
|
+
// an append would ship bytes that do not match the recorded offset. Force
|
|
1009
|
+
// FULL, which ships the unioned container and is correct either way.
|
|
1010
|
+
const split = !!backend?.spansMultipleSources?.(ref.prefixedId);
|
|
1011
|
+
const isAO = !!backend?.isAppendOnly?.() && !split;
|
|
1012
|
+
const size = isAO ? (backend?.fileSize?.(ref.prefixedId) ?? 0) : 0;
|
|
1013
|
+
return syncMode(ack, ref.mtime, size, extractorVersionForTool(ref.toolId), isAO);
|
|
1014
|
+
};
|
|
1015
|
+
const willBuild = (ref) => modeOf(ref) !== 'skip';
|
|
1016
|
+
// ── Batch secret scan (optional external detectors) ────────────────
|
|
1017
|
+
// Only runs when CHAT_RECALL_EXTERNAL_SCANNERS=1 and gitleaks/trufflehog are
|
|
1018
|
+
// on PATH — `scanSecrets` above already folds that gate in, so by default
|
|
1019
|
+
// nothing here executes and no raw text is ever written to disk.
|
|
1020
|
+
//
|
|
1021
|
+
// gitleaks/trufflehog are directory-capable and report each finding's file.
|
|
1022
|
+
// Materialize to-be-FULL-synced sessions' raw text into a temp dir (filename
|
|
1023
|
+
// = sanitized session id), scan the dir ONCE per detector, then map findings
|
|
1024
|
+
// back to sessions by filename. That replaces 2·N cold process spawns (the
|
|
1025
|
+
// dominant per-session cost) with 2 per SLICE. null ⇒ no batch ran (scanning
|
|
1026
|
+
// off) and buildConversationSync falls back to its per-session path.
|
|
1027
|
+
//
|
|
1028
|
+
// SLICED, not one big dir: this text is PRE-redaction — real credentials in
|
|
1029
|
+
// cleartext on the customer's disk — and cleanup lives in a `finally` that a
|
|
1030
|
+
// SIGKILL/OOM skips (that is how 3.4GB / 309k files were left behind in
|
|
1031
|
+
// /tmp). Bounding the dir to BATCH_SCAN_MAX_BYTES bounds the worst-case
|
|
1032
|
+
// exposure and the worst-case leak to that many bytes, at a cost of 2 extra
|
|
1033
|
+
// spawns per additional slice.
|
|
1034
|
+
//
|
|
1035
|
+
// APPEND sessions are excluded — they skip external detectors (the head was
|
|
1036
|
+
// already scanned; the tail uses builtin regex only; the FULL re-sync when
|
|
1037
|
+
// the session closes covers the whole file).
|
|
1038
|
+
let externalFindings = null;
|
|
1039
|
+
if (scanSecrets) {
|
|
1040
|
+
externalFindings = new Map();
|
|
1041
|
+
// Oversized sessions are excluded: exportRawSession would materialize the
|
|
1042
|
+
// whole file (hundreds of MB — this loop is what OOM-killed the daemon).
|
|
1043
|
+
// Their bounded tail still gets the builtin scan in buildConversationSync.
|
|
1044
|
+
const toScan = slice.filter((ref) => modeOf(ref) === 'full' && sessionFileBytes(ref) <= FULL_BUILD_MAX_BYTES);
|
|
1045
|
+
trace?.(`batch secret scan start: ${toScan.length} session(s), ≤${(BATCH_SCAN_MAX_BYTES / 1048576).toFixed(0)}MB per slice`);
|
|
1046
|
+
const batch = batchScanExternal(toScan.map((ref) => ({
|
|
1047
|
+
id: ref.prefixedId,
|
|
1048
|
+
text: () => {
|
|
1049
|
+
const backend = getBackendForId(ref.prefixedId) ?? getBackend('claude');
|
|
1050
|
+
const exp = backend.exportRawSession(ref.prefixedId);
|
|
1051
|
+
const container = exp ? buildRawContainer(exp) : null;
|
|
1052
|
+
return container ? container.files.map((f) => f.text).join('\n') : null;
|
|
1053
|
+
},
|
|
1054
|
+
})), { verifyOnly: verifySecretsScan });
|
|
1055
|
+
externalFindings = batch.findings;
|
|
1056
|
+
scanned = batch.scanned;
|
|
1057
|
+
scanMs = batch.scanMs;
|
|
1058
|
+
trace?.(`batch secret scan done: ${scanned} session(s) in ${batch.slices} slice(s)`);
|
|
1059
|
+
}
|
|
1060
|
+
trace?.('conversation walk start');
|
|
1061
|
+
let traceN = 0;
|
|
1062
|
+
for (const ref of slice) {
|
|
1063
|
+
if (trace && ++traceN % 500 === 0)
|
|
1064
|
+
trace(`walk ${traceN}/${slice.length} · at ${ref.prefixedId}`);
|
|
1065
|
+
localSessionIds.add(ref.prefixedId);
|
|
1066
|
+
const mode = modeOf(ref);
|
|
1067
|
+
if (mode === 'skip') {
|
|
1068
|
+
skipped++;
|
|
1069
|
+
continue;
|
|
1070
|
+
}
|
|
1071
|
+
trace?.(`build (${mode}) ${ref.prefixedId} · ${(sessionFileBytes(ref) / 1048576).toFixed(1)}MB`);
|
|
1072
|
+
// ── APPEND: tail-only ship (no raw_b64, no telemetry, no derived) ──
|
|
1073
|
+
// `promotedForFindings` flips true when the tail carried a secret finding:
|
|
1074
|
+
// the server replaces findings wholesale per session, so a tail-only ship
|
|
1075
|
+
// would WIPE the head's findings. Instead of hiding the tail's secret until
|
|
1076
|
+
// some far-future FULL re-sync, we fall through to the FULL path NOW (which
|
|
1077
|
+
// re-scans head+tail and ships the complete set). Rare event — a secret
|
|
1078
|
+
// pasted into a long-lived, still-appending session.
|
|
1079
|
+
let promotedForFindings = false;
|
|
1080
|
+
if (mode === 'append' && ledger) {
|
|
1081
|
+
const ack = ledger.get(ref.prefixedId);
|
|
1082
|
+
try {
|
|
1083
|
+
const tail = await buildConversationTail(ref, ack.o ?? 0, { mapPath, tenantRules });
|
|
1084
|
+
if (tail && tail.findings.length > 0) {
|
|
1085
|
+
promotedForFindings = true; // fall through to FULL (do NOT continue)
|
|
1086
|
+
}
|
|
1087
|
+
else if (tail) {
|
|
1088
|
+
redactions += tail.redactions;
|
|
1089
|
+
await appendAdd(tail.conv);
|
|
1090
|
+
walked++;
|
|
1091
|
+
if (walked % 250 === 0)
|
|
1092
|
+
console.error(`[sync] ${walked}/${slice.length} sessions…`);
|
|
1093
|
+
continue;
|
|
1094
|
+
}
|
|
1095
|
+
else {
|
|
1096
|
+
// Nothing new to ship (tail empty/unparseable). Stamp the MTIME only.
|
|
1097
|
+
//
|
|
1098
|
+
// This used to re-stamp the cursor at the local fileSize() to stop the
|
|
1099
|
+
// gate retrying every tick. That was a delivery receipt for bytes the
|
|
1100
|
+
// server never received: once fileSize() started summing profile homes,
|
|
1101
|
+
// the cursor jumped to the union size and the session was sealed as
|
|
1102
|
+
// complete with a third of its records missing (7adc748c: 2004 of
|
|
1103
|
+
// 2698). mtime alone gives the same anti-thrash — syncMode SKIPs an
|
|
1104
|
+
// unchanged file on mtime — while leaving the cursor where the server
|
|
1105
|
+
// actually is, so the next growth re-attempts from the true offset.
|
|
1106
|
+
try {
|
|
1107
|
+
markSynced(base, [{ id: ref.prefixedId, mtime: ref.mtime }]);
|
|
1108
|
+
}
|
|
1109
|
+
catch { /* ledger — best-effort */ }
|
|
1110
|
+
skipped++;
|
|
1111
|
+
continue;
|
|
1112
|
+
}
|
|
1113
|
+
}
|
|
1114
|
+
catch (err) {
|
|
1115
|
+
console.error(`[sync] append tail failed for ${ref.prefixedId} (will retry as full next tick): ${err instanceof Error ? err.message : err}`);
|
|
1116
|
+
// Don't mark synced → next tick re-classifies. If the tail failed, a
|
|
1117
|
+
// FULL sync is the safe fallback (the gate sees an unchanged cursor).
|
|
1118
|
+
skipped++;
|
|
1119
|
+
continue;
|
|
1120
|
+
}
|
|
1121
|
+
}
|
|
1122
|
+
// ── FULL: the existing whole-conversation path ──
|
|
1123
|
+
// Canonical envelope (live): parse the transcript NOW with THE parser —
|
|
1124
|
+
// no local content cache is read or warmed. What ships is exactly what
|
|
1125
|
+
// the local dashboard would render for the same file.
|
|
1126
|
+
const mtime = Math.floor(ref.mtime) || 0;
|
|
1127
|
+
// Hand this session its slice of the batch-scan results (when scanning is
|
|
1128
|
+
// on). `?? []` means "batch ran, found nothing here" — distinct from the
|
|
1129
|
+
// `undefined` that triggers the per-session fallback inside the builder.
|
|
1130
|
+
// A session promoted from append (secret in the tail) wasn't in the batch
|
|
1131
|
+
// scan set — pass undefined so buildConversationSync runs the per-session
|
|
1132
|
+
// external detector over the whole file, yielding the COMPLETE finding set.
|
|
1133
|
+
const precomputedExternal = promotedForFindings
|
|
1134
|
+
? undefined
|
|
1135
|
+
: (externalFindings ? (externalFindings.get(ref.prefixedId) ?? []) : undefined);
|
|
1136
|
+
// Only trust the ledger's content hash for the skip-gate when the row is at
|
|
1137
|
+
// the current extractor version — a version bump must force a real re-derive
|
|
1138
|
+
// even if the transcript bytes are unchanged.
|
|
1139
|
+
const ackRow = ledger?.get(ref.prefixedId);
|
|
1140
|
+
const priorContentHash = ackRow && ackRow.v >= extractorVersionForId(ref.prefixedId) ? ackRow.h : undefined;
|
|
1141
|
+
const built = await buildConversationSync(ref, mtime, { mapPath, includeRaw: upload.raw, includeMeta: upload.sessionMeta, scanSecrets, verifySecrets: verifySecretsScan, tenantRules, precomputedExternal, priorContentHash });
|
|
1142
|
+
if (built && 'unchanged' in built) {
|
|
1143
|
+
// Content byte-identical to the last FULL sync to this target (mtime bumped
|
|
1144
|
+
// / resume rewrite / OpenCode summary bump). Nothing to re-ship, re-extract
|
|
1145
|
+
// or git-replay. Re-stamp the ledger exactly as a real FULL ack would —
|
|
1146
|
+
// mtime, the (unchanged) hash, and the current byte cursor — so the NEXT
|
|
1147
|
+
// tick reaches a true SKIP and doesn't even re-export. Byte-identical
|
|
1148
|
+
// content ⇒ identical size, so the cursor we write equals the one already
|
|
1149
|
+
// there for append-only backends; for OpenCode fileSize() is absent (0).
|
|
1150
|
+
if (ledger) {
|
|
1151
|
+
try {
|
|
1152
|
+
// Content is byte-identical to the last FULL sync, so the server
|
|
1153
|
+
// already holds these bytes — that makes this a legitimate ack. But
|
|
1154
|
+
// only up to the cursor it already had: if the local size now EXCEEDS
|
|
1155
|
+
// it (e.g. fileSize() sums a second profile home that was never
|
|
1156
|
+
// shipped), advancing would again receipt undelivered bytes.
|
|
1157
|
+
const off = getBackendForId(ref.prefixedId)?.fileSize?.(ref.rawId) ?? 0;
|
|
1158
|
+
const priorCursor = ackRow?.s ?? 0;
|
|
1159
|
+
const safeToAck = off > 0 && off <= priorCursor;
|
|
1160
|
+
markSynced(base, [safeToAck
|
|
1161
|
+
? { id: ref.prefixedId, mtime: ref.mtime, offset: off, size: off, hash: built.srcHash, acked: true }
|
|
1162
|
+
: { id: ref.prefixedId, mtime: ref.mtime, hash: built.srcHash }]);
|
|
1163
|
+
}
|
|
1164
|
+
catch { /* ledger is local bookkeeping — never fail a sync over it */ }
|
|
1165
|
+
}
|
|
1166
|
+
skipped++;
|
|
1167
|
+
continue;
|
|
1168
|
+
}
|
|
1169
|
+
if (!built) {
|
|
1170
|
+
// Nothing worth shipping for THIS content — empty, unparseable, or one
|
|
1171
|
+
// of chat-recall's own internal LLM invocations (roughly half of all
|
|
1172
|
+
// "sessions" on a busy machine). Every null verdict is a pure function
|
|
1173
|
+
// of file content, so ack it at this mtime: without the ack these
|
|
1174
|
+
// sessions re-classify as FULL on every full walk, forever — observed
|
|
1175
|
+
// as ~5k sessions re-exported and re-secret-scanned per 15-min
|
|
1176
|
+
// heartbeat (4GB+ heap peak, 12-minute walks, heap-OOM crash loops).
|
|
1177
|
+
// A real content change bumps mtime and re-evaluates.
|
|
1178
|
+
if (ledger) {
|
|
1179
|
+
try {
|
|
1180
|
+
markSynced(base, [{ id: ref.prefixedId, mtime: ref.mtime }]);
|
|
1181
|
+
}
|
|
1182
|
+
catch { /* ledger is local bookkeeping — never fail a sync over it */ }
|
|
1183
|
+
}
|
|
1184
|
+
skipped++;
|
|
1185
|
+
continue;
|
|
1186
|
+
}
|
|
1187
|
+
redactions += built.redactions;
|
|
1188
|
+
// Remember the content hash so the convBatch ack stamps ledger `h` (enables
|
|
1189
|
+
// the mtime-only skip on a later tick). Undefined for the oversized-tail path.
|
|
1190
|
+
if (built.srcHash)
|
|
1191
|
+
shippedHash.set(ref.prefixedId, built.srcHash);
|
|
1192
|
+
await convBatch.add(built.conv);
|
|
1193
|
+
// Ship this session's secret findings as one group (server replaces wholesale).
|
|
1194
|
+
if (built.findings.length > 0)
|
|
1195
|
+
await findingsBatch.add(built.findings);
|
|
1196
|
+
// Knowledge graph: extract triples live from the redacted conversation
|
|
1197
|
+
// text (no local KG DB is read). Accumulated + deduped, flushed at the end.
|
|
1198
|
+
if (upload.sessionMeta) {
|
|
1199
|
+
accumulateKg(extractEntities(built.kgText, { projectPath: built.projectPath, sourceType: 'session', sessionId: ref.prefixedId, validFrom: ref.mtime ? new Date(ref.mtime).toISOString().slice(0, 10) : undefined }), ref.prefixedId);
|
|
1200
|
+
}
|
|
1201
|
+
// Derived data — the CLI has the FS/git/transcript, the server doesn't.
|
|
1202
|
+
// Always computed live (diff/outcome/commits/markers + badge row); no
|
|
1203
|
+
// local compute/outcome cache is read or warmed. Oversized transcripts are
|
|
1204
|
+
// excluded: replay/outcome re-read the WHOLE file (readEvents), and a
|
|
1205
|
+
// 353MB session with 100MB+ single JSON lines OOMed the walk right after
|
|
1206
|
+
// its (bounded) conversation shipped.
|
|
1207
|
+
if (upload.sessionMeta && sessionFileBytes(ref) <= FULL_BUILD_MAX_BYTES) {
|
|
1208
|
+
try {
|
|
1209
|
+
const count = { redactions: 0 };
|
|
1210
|
+
await derivedBatch.add(collectDerived(ref, mtime, count));
|
|
1211
|
+
redactions += count.redactions;
|
|
1212
|
+
}
|
|
1213
|
+
catch (err) {
|
|
1214
|
+
// Derived is best-effort — the conversation still shipped — but it must
|
|
1215
|
+
// NOT be silent: a derived failure means this session shows empty
|
|
1216
|
+
// diffs/Activity on the server, and a SILENT swallow here is exactly why
|
|
1217
|
+
// a stale-data regression went unnoticed for days. Log it so it's
|
|
1218
|
+
// visible and diagnosable.
|
|
1219
|
+
console.error(`[sync] derived compute failed for ${ref.prefixedId} (diffs/Activity will be empty for it): ${err instanceof Error ? err.message : err}`);
|
|
1220
|
+
}
|
|
1221
|
+
}
|
|
1222
|
+
// Fork lineage link (collected at index time too; shipping here keeps
|
|
1223
|
+
// sync self-sufficient for sessions the indexer hasn't visited).
|
|
1224
|
+
try {
|
|
1225
|
+
const { detectForkPredecessor } = await import('@chat-recall/engine/transcript/raw.js');
|
|
1226
|
+
if (ref.fullPath) {
|
|
1227
|
+
const predecessor = detectForkPredecessor(ref.fullPath);
|
|
1228
|
+
if (predecessor) {
|
|
1229
|
+
await linksBatch.add({
|
|
1230
|
+
source_type: 'session', source_id: ref.prefixedId,
|
|
1231
|
+
target_type: 'session', target_id: predecessor,
|
|
1232
|
+
link_type: 'forked_from', confidence: 1.0,
|
|
1233
|
+
});
|
|
1234
|
+
}
|
|
1235
|
+
}
|
|
1236
|
+
}
|
|
1237
|
+
catch { /* best-effort */ }
|
|
1238
|
+
walked++;
|
|
1239
|
+
if (walked % 250 === 0)
|
|
1240
|
+
console.error(`[sync] ${walked}/${slice.length} sessions…`);
|
|
1241
|
+
}
|
|
1242
|
+
// ── Non-session sources: plan/task/claude_md/history/paste/diary/skill/
|
|
1243
|
+
// mcp/command/agent/hook/plugin. Same registry the indexer uses, so
|
|
1244
|
+
// sync coverage tracks index coverage automatically. Session-type
|
|
1245
|
+
// sources are skipped — sessions ship as conversations above.
|
|
1246
|
+
// (excludeTools is session-only: source plugins don't carry a tool id.)
|
|
1247
|
+
// Per-tool item-extractor versions (see sync-ledger). A tool whose recorded
|
|
1248
|
+
// version is below its current version re-ships ALL its items ONCE (mtime
|
|
1249
|
+
// gate bypassed), so an attribution/extraction fix self-heals — same idea as
|
|
1250
|
+
// sessions, at the tool level. Unrecorded tools seed at EXTRACTOR_VERSION
|
|
1251
|
+
// (base), so only tools bumped ABOVE base (e.g. agy) re-ship on first run —
|
|
1252
|
+
// not a blanket re-ship of every item.
|
|
1253
|
+
const itemVersions = loadItemVersions(base);
|
|
1254
|
+
const seenTools = new Set();
|
|
1255
|
+
const registry = buildSourceRegistry();
|
|
1256
|
+
for (const sourceType of registry.getRegisteredTypes()) {
|
|
1257
|
+
if (sourceType === 'session')
|
|
1258
|
+
continue;
|
|
1259
|
+
for (const source of registry.getAll(sourceType)) {
|
|
1260
|
+
let discovered;
|
|
1261
|
+
try {
|
|
1262
|
+
discovered = source.discover();
|
|
1263
|
+
}
|
|
1264
|
+
catch {
|
|
1265
|
+
continue;
|
|
1266
|
+
}
|
|
1267
|
+
try {
|
|
1268
|
+
for await (const item of discovered) {
|
|
1269
|
+
const itemTool = toolOfId(item.id);
|
|
1270
|
+
seenTools.add(itemTool);
|
|
1271
|
+
const versionStale = (itemVersions[itemTool] ?? EXTRACTOR_VERSION) < extractorVersionForId(item.id);
|
|
1272
|
+
if ((item.mtime || 0) < (opts.sinceMs ?? 0) && !versionStale)
|
|
1273
|
+
continue;
|
|
1274
|
+
if (excluded(item.projectPath || ''))
|
|
1275
|
+
continue;
|
|
1276
|
+
if (!includedProject(item.projectPath || ''))
|
|
1277
|
+
continue;
|
|
1278
|
+
const count = { redactions: 0 };
|
|
1279
|
+
let chunks = [];
|
|
1280
|
+
try {
|
|
1281
|
+
chunks = await source.parse(item);
|
|
1282
|
+
}
|
|
1283
|
+
catch { /* unparseable item still ships metadata */ }
|
|
1284
|
+
// Project id resolved live on THIS machine (git/FS available); the
|
|
1285
|
+
// server stores it verbatim. `ignored:` rules mean "no project".
|
|
1286
|
+
const resolved = resolveProjectId(item.projectPath || '');
|
|
1287
|
+
const projectId = resolved.source === 'ignored' ? '' : resolved.id;
|
|
1288
|
+
const redactedChunks = chunks
|
|
1289
|
+
.filter((c) => c.text?.trim())
|
|
1290
|
+
.map((c) => ({
|
|
1291
|
+
text: redactSecrets(c.text, { force: true, count }),
|
|
1292
|
+
chunk_type: c.chunkType || '',
|
|
1293
|
+
// Chunk titles are derived from message text and so can carry a
|
|
1294
|
+
// secret exactly like the body can. This was the one field on the
|
|
1295
|
+
// upload path that shipped raw.
|
|
1296
|
+
title: redactSecrets(c.title || '', { force: true, count }),
|
|
1297
|
+
}));
|
|
1298
|
+
await itemsBatch.add({
|
|
1299
|
+
id: item.id,
|
|
1300
|
+
source_type: item.sourceType,
|
|
1301
|
+
project_id: projectId || undefined,
|
|
1302
|
+
title: redactSecrets(item.title || '', { force: true, count }),
|
|
1303
|
+
project_path: mapPath(item.projectPath || ''),
|
|
1304
|
+
content_preview: redactSecrets(item.contentPreview || '', { force: true, count }),
|
|
1305
|
+
mtime: Math.floor(item.mtime) || 0,
|
|
1306
|
+
extra: item.extra ? redactDeep(item.extra, count) : undefined,
|
|
1307
|
+
chunks: redactedChunks,
|
|
1308
|
+
});
|
|
1309
|
+
// KG: extract triples live from this item's redacted chunk text.
|
|
1310
|
+
if (upload.sessionMeta && redactedChunks.length > 0) {
|
|
1311
|
+
accumulateKg(extractEntities(redactedChunks.map((c) => c.text).join('\n'), { projectPath: item.projectPath || '', sourceType: item.sourceType, validFrom: item.mtime ? new Date(Number(item.mtime)).toISOString().slice(0, 10) : undefined }), null);
|
|
1312
|
+
}
|
|
1313
|
+
try {
|
|
1314
|
+
for (const l of await source.extractLinks(item)) {
|
|
1315
|
+
await linksBatch.add({
|
|
1316
|
+
source_type: l.sourceType, source_id: l.sourceId,
|
|
1317
|
+
target_type: l.targetType, target_id: l.targetId,
|
|
1318
|
+
link_type: l.linkType, confidence: l.confidence,
|
|
1319
|
+
});
|
|
1320
|
+
}
|
|
1321
|
+
}
|
|
1322
|
+
catch { /* links are best-effort */ }
|
|
1323
|
+
redactions += count.redactions;
|
|
1324
|
+
}
|
|
1325
|
+
}
|
|
1326
|
+
catch (err) {
|
|
1327
|
+
// One broken source must not kill the sync — but a SYSTEMATICALLY failing
|
|
1328
|
+
// source (bad discover/parse) would otherwise silently ship zero items
|
|
1329
|
+
// for its type forever, with no signal. Log it (once per source per run).
|
|
1330
|
+
console.error(`[sync] source '${sourceType}' failed mid-walk (its items may be incomplete this run): ${err instanceof Error ? err.message : err}`);
|
|
1331
|
+
}
|
|
1332
|
+
}
|
|
1333
|
+
}
|
|
1334
|
+
// ── Knowledge graph (live). Triples were extracted on the spot from each
|
|
1335
|
+
// session/item's redacted text during the walks above — so what ships is
|
|
1336
|
+
// exactly the facts derivable from the sessions in THIS sync (which is
|
|
1337
|
+
// also the incremental-correct set: no manual-fact replay, no local KG
|
|
1338
|
+
// DB). Entities are derived from the triple subjects/objects, typed from
|
|
1339
|
+
// any `is_a` triple seen in the same pass.
|
|
1340
|
+
if (upload.sessionMeta && kgRows.length > 0) {
|
|
1341
|
+
const entityNames = new Set();
|
|
1342
|
+
for (const t of kgRows) {
|
|
1343
|
+
const count = { redactions: 0 };
|
|
1344
|
+
await kgTripleBatch.add(redactDeep(t, count));
|
|
1345
|
+
redactions += count.redactions;
|
|
1346
|
+
entityNames.add(t.subject);
|
|
1347
|
+
entityNames.add(t.object);
|
|
1348
|
+
}
|
|
1349
|
+
for (const name of entityNames) {
|
|
1350
|
+
const count = { redactions: 0 };
|
|
1351
|
+
await kgEntityBatch.add(redactDeep({ name, type: kgEntityType.get(name) ?? 'unknown', properties: {} }, count));
|
|
1352
|
+
redactions += count.redactions;
|
|
1353
|
+
}
|
|
1354
|
+
}
|
|
1355
|
+
// ── Drain every batcher (the walks above flushed full windows already;
|
|
1356
|
+
// this ships the tails). The append batch flushes BEFORE the inflight
|
|
1357
|
+
// barrier so its full_resync_needed set is populated before we process it.
|
|
1358
|
+
await appendFlush();
|
|
1359
|
+
// Server said full_resync_needed for these append sessions (no prior
|
|
1360
|
+
// envelope on the server). Wipe their ledger rows so the next tick
|
|
1361
|
+
// classifies them as FULL (never-synced) instead of retrying append or
|
|
1362
|
+
// skipping — which would leave the server without the data forever.
|
|
1363
|
+
for (const id of appendResults.fullResyncNeeded) {
|
|
1364
|
+
try {
|
|
1365
|
+
markFullResync(base, id);
|
|
1366
|
+
}
|
|
1367
|
+
catch { /* ledger — best-effort */ }
|
|
1368
|
+
}
|
|
1369
|
+
await convBatch.drain();
|
|
1370
|
+
await itemsBatch.drain();
|
|
1371
|
+
await linksBatch.drain();
|
|
1372
|
+
await derivedBatch.drain();
|
|
1373
|
+
await kgEntityBatch.drain();
|
|
1374
|
+
await kgTripleBatch.drain();
|
|
1375
|
+
await findingsBatch.drain();
|
|
1376
|
+
// Barrier: wait for every pooled upload to land (and surface any fatal
|
|
1377
|
+
// error) before tombstones/prune, which must run strictly after all data.
|
|
1378
|
+
await drainInflight();
|
|
1379
|
+
// Items shipped successfully → record each seen tool's current item-extractor
|
|
1380
|
+
// version, so a version-triggered re-ship (e.g. the agy attribution fix)
|
|
1381
|
+
// happens exactly ONCE per bump instead of every sync. Best-effort.
|
|
1382
|
+
if (seenTools.size > 0) {
|
|
1383
|
+
try {
|
|
1384
|
+
const rec = {};
|
|
1385
|
+
for (const t of seenTools)
|
|
1386
|
+
rec[t] = extractorVersionForTool(t);
|
|
1387
|
+
saveItemVersions(base, rec);
|
|
1388
|
+
}
|
|
1389
|
+
catch { /* local bookkeeping — best-effort */ }
|
|
1390
|
+
}
|
|
1391
|
+
// ── Tombstones for sessions this device previously synced but that no
|
|
1392
|
+
// longer exist locally (deleted transcript file, cleared cache, etc).
|
|
1393
|
+
// We only tombstone sessions we have a ledger record for — never data
|
|
1394
|
+
// from other devices. STRICTLY full-walk only: a changed-scope walk
|
|
1395
|
+
// lists only recent sessions, so "in ledger but not walked" means
|
|
1396
|
+
// "old and unchanged", NOT "deleted" — tombstoning there would wipe
|
|
1397
|
+
// every quiet session from the server.
|
|
1398
|
+
if (ledger && !changedWalk) {
|
|
1399
|
+
const tombstones = [];
|
|
1400
|
+
for (const [sessionId] of ledger) {
|
|
1401
|
+
if (!localSessionIds.has(sessionId)) {
|
|
1402
|
+
tombstones.push({ session_id: sessionId, deleted_at: Date.now() });
|
|
1403
|
+
}
|
|
1404
|
+
}
|
|
1405
|
+
if (tombstones.length > 0) {
|
|
1406
|
+
await post({ tombstones });
|
|
1407
|
+
// Remove tombstoned ids from the ledger so we don't keep shipping them.
|
|
1408
|
+
const data = getLedgerData(base);
|
|
1409
|
+
for (const t of tombstones)
|
|
1410
|
+
delete data[t.session_id];
|
|
1411
|
+
persistLedgerData(base, data);
|
|
1412
|
+
}
|
|
1413
|
+
}
|
|
1414
|
+
if (opts.prune)
|
|
1415
|
+
await post({ prune_empty_sessions: true });
|
|
1416
|
+
return {
|
|
1417
|
+
uploaded: convBatch.sent + appendResults.uploaded, skipped, redactions,
|
|
1418
|
+
items: itemsBatch.sent, links: linksBatch.sent, findings: findingsBatch.sent, derived: derivedBatch.sent,
|
|
1419
|
+
kgEntities: kgEntityBatch.sent, kgTriples: kgTripleBatch.sent,
|
|
1420
|
+
scanned, scanMs: Math.round(scanMs),
|
|
1421
|
+
};
|
|
1422
|
+
}
|
|
1423
|
+
/** Transcript bytes beyond which a FULL sync must not materialize the whole
|
|
1424
|
+
* file. The FULL pipeline copies the transcript ~5× (parse, trim+redact,
|
|
1425
|
+
* raw export, raw-text join, redacted container) — a 350MB session transiently
|
|
1426
|
+
* needs >1.5GB of heap, which OOM-killed the watch daemon in a restart loop
|
|
1427
|
+
* (ledger never acked → same file retried on every startup tick, forever). */
|
|
1428
|
+
export const FULL_BUILD_MAX_BYTES = Math.max(8, parseInt(process.env.CHAT_RECALL_FULL_BUILD_MAX_MB || '64', 10)) * 1024 * 1024;
|
|
1429
|
+
/** How much of an oversized transcript's tail still ships (newest content). */
|
|
1430
|
+
const FULL_BUILD_TAIL_BYTES = 16 * 1024 * 1024;
|
|
1431
|
+
/** Ceiling on PRE-REDACTION session text materialized on disk at any one moment
|
|
1432
|
+
* by the optional external-detector batch scan. The batch dir is flushed and
|
|
1433
|
+
* deleted every time it crosses this, so a SIGKILL/OOM mid-scan can strand at
|
|
1434
|
+
* most this many bytes of cleartext (the incident stranded 3.4GB), and the
|
|
1435
|
+
* window in which they exist is proportionally short. Raising it buys fewer
|
|
1436
|
+
* detector spawns; lowering it buys a smaller blast radius. */
|
|
1437
|
+
export const BATCH_SCAN_MAX_BYTES = Math.max(1, parseInt(process.env.CHAT_RECALL_BATCHSCAN_MAX_MB || '64', 10)) * 1024 * 1024;
|
|
1438
|
+
/** Bytes an append-only session occupies on disk (0 for non-AO backends). */
|
|
1439
|
+
export function sessionFileBytes(ref) {
|
|
1440
|
+
const backend = getBackendForId(ref.prefixedId);
|
|
1441
|
+
if (!backend?.isAppendOnly?.())
|
|
1442
|
+
return 0;
|
|
1443
|
+
return backend.fileSize?.(ref.prefixedId) ?? 0;
|
|
1444
|
+
}
|
|
1445
|
+
export async function buildConversationSync(ref, mtime, opts = {}) {
|
|
1446
|
+
const mapPath = opts.mapPath ?? ((p) => p);
|
|
1447
|
+
let includeRaw = opts.includeRaw !== false;
|
|
1448
|
+
let includeMeta = opts.includeMeta !== false;
|
|
1449
|
+
let srcHash;
|
|
1450
|
+
// Oversized-transcript guard: ship the newest FULL_BUILD_TAIL_BYTES as this
|
|
1451
|
+
// session's FULL sync instead of materializing the whole file. The envelope
|
|
1452
|
+
// is marked truncated, raw/meta (whole-file reads) are dropped, and
|
|
1453
|
+
// from_offset = file size — so the ledger advances and every later tick is
|
|
1454
|
+
// a normal bounded APPEND. Truncation only loses the transcript's OLD head.
|
|
1455
|
+
const fileBytes = sessionFileBytes(ref);
|
|
1456
|
+
const oversized = fileBytes > FULL_BUILD_MAX_BYTES;
|
|
1457
|
+
let tailText = '';
|
|
1458
|
+
let tailOffsetEnd = 0;
|
|
1459
|
+
// Parse the transcript live — this is the single canonical parse; the
|
|
1460
|
+
// envelope and the redacted text both come from it. When the shadow path
|
|
1461
|
+
// runs (normal-size sessions) it also yields the fullest raw container, which
|
|
1462
|
+
// the secret scan + raw archive below reuse instead of re-exporting.
|
|
1463
|
+
let transcript = null;
|
|
1464
|
+
let fullestContainer = null;
|
|
1465
|
+
if (oversized) {
|
|
1466
|
+
const backend = getBackendForId(ref.prefixedId);
|
|
1467
|
+
if (!backend?.readFromOffset)
|
|
1468
|
+
return null;
|
|
1469
|
+
try {
|
|
1470
|
+
const tail = await backend.readFromOffset(ref.prefixedId, fileBytes - FULL_BUILD_TAIL_BYTES);
|
|
1471
|
+
tailText = tail.text;
|
|
1472
|
+
tailOffsetEnd = tail.newOffset;
|
|
1473
|
+
const messages = parseTailMessages(ref.toolId, tailText);
|
|
1474
|
+
if (messages.length > 0)
|
|
1475
|
+
transcript = { messages, subagents: [] };
|
|
1476
|
+
}
|
|
1477
|
+
catch { /* unreadable tail */ }
|
|
1478
|
+
includeRaw = false; // exportRawSession would read the whole file
|
|
1479
|
+
includeMeta = false; // parseSessionFile would read the whole file
|
|
1480
|
+
// NB: oversized sessions are NOT shadowed — a full export would OOM the
|
|
1481
|
+
// walk (the very reason for the tail path). A session truncated FROM
|
|
1482
|
+
// oversized down to normal falls back to a fresh 'created' shadow; the
|
|
1483
|
+
// recovery command backfills its pre-truncation history from the archive.
|
|
1484
|
+
}
|
|
1485
|
+
else {
|
|
1486
|
+
// Export the raw transcript ONCE, fold it into the shadow archive, and
|
|
1487
|
+
// parse the FULLEST-known state. If an upstream tool truncated the file in
|
|
1488
|
+
// place (e.g. a Claude Code 2.1.20x `--resume` rewrite), the shadow still
|
|
1489
|
+
// holds the lost history, and the merged container — not the shrunken disk
|
|
1490
|
+
// file — is what we parse and ship. The server therefore never sees a
|
|
1491
|
+
// shrink, and every viewer tab keeps its full history.
|
|
1492
|
+
try {
|
|
1493
|
+
const backend = getBackendForId(ref.prefixedId) ?? getBackend('claude');
|
|
1494
|
+
const exp = backend.exportRawSession(ref.prefixedId);
|
|
1495
|
+
// Content-identity gate: hash the raw export (mtime-independent) BEFORE the
|
|
1496
|
+
// shadow merge / parse / redact / KG / git-replay. If it equals what this
|
|
1497
|
+
// target last FULL-synced, nothing downstream can differ — bail now. This
|
|
1498
|
+
// is what stops a resume-truncated or mtime-touched session from being
|
|
1499
|
+
// fully rebuilt on every tick (the export/read still happens — it's how we
|
|
1500
|
+
// know — but everything expensive after it is skipped).
|
|
1501
|
+
if (exp) {
|
|
1502
|
+
try {
|
|
1503
|
+
srcHash = containerSrcHash(buildRawContainer(exp));
|
|
1504
|
+
}
|
|
1505
|
+
catch {
|
|
1506
|
+
srcHash = undefined;
|
|
1507
|
+
}
|
|
1508
|
+
if (srcHash && opts.priorContentHash && srcHash === opts.priorContentHash) {
|
|
1509
|
+
return { unchanged: true, srcHash };
|
|
1510
|
+
}
|
|
1511
|
+
}
|
|
1512
|
+
const shadow = updateShadow(ref.rawId, exp);
|
|
1513
|
+
if (shadow.container) {
|
|
1514
|
+
fullestContainer = shadow.container;
|
|
1515
|
+
if (shadow.status === 'rewrite-merged' && shadow.recovered > 0) {
|
|
1516
|
+
console.error(`[sync] shadow recovered ${shadow.recovered} record(s) for ${ref.prefixedId} after an upstream rewrite/truncation`);
|
|
1517
|
+
}
|
|
1518
|
+
const t = parseTranscriptFromContainer(shadow.container);
|
|
1519
|
+
if (t.messages.length > 0 || t.subagents.length > 0)
|
|
1520
|
+
transcript = { messages: t.messages, subagents: t.subagents };
|
|
1521
|
+
}
|
|
1522
|
+
}
|
|
1523
|
+
catch { /* shadow/export failed — fall back to the live parse below */ }
|
|
1524
|
+
if (!transcript) {
|
|
1525
|
+
try {
|
|
1526
|
+
const t = await parseTranscript(ref.prefixedId);
|
|
1527
|
+
if (t && (t.messages.length > 0 || t.subagents.length > 0))
|
|
1528
|
+
transcript = t;
|
|
1529
|
+
}
|
|
1530
|
+
catch { /* unparseable */ }
|
|
1531
|
+
}
|
|
1532
|
+
}
|
|
1533
|
+
if (!transcript)
|
|
1534
|
+
return null;
|
|
1535
|
+
const textMessages = transcript.messages.filter((m) => m.role !== 'summary' && m.content?.trim());
|
|
1536
|
+
if (textMessages.length === 0 && !transcript.messages.some((m) => m.toolCalls?.length))
|
|
1537
|
+
return null;
|
|
1538
|
+
// Skip chat-recall's OWN LLM invocations that the AI CLI logged as sessions —
|
|
1539
|
+
// summary generation and health-check pings. On this machine ~half of all
|
|
1540
|
+
// "sessions" were these internal calls; indexing them buries the real
|
|
1541
|
+
// conversation list. Anchored to the first user message so a real chat that
|
|
1542
|
+
// merely quotes the prompt is not dropped.
|
|
1543
|
+
const firstUser = textMessages.find((m) => m.role === 'user');
|
|
1544
|
+
if (isInternalToolPrompt(firstUser?.content))
|
|
1545
|
+
return null;
|
|
1546
|
+
// Trim payload bodies (tool inputs/results, thinking — raw replay never
|
|
1547
|
+
// ships), then redact every string. Counts are preserved exactly.
|
|
1548
|
+
const count = { redactions: 0 };
|
|
1549
|
+
const envelope = redactDeep({ v: TRANSCRIPT_VERSION, ...trimTranscriptForSync(transcript) }, count);
|
|
1550
|
+
// Live telemetry: parse the raw session file with THE metadata parser
|
|
1551
|
+
// (tokens/cost/models/tools/duration/branch/files). Replaces the former
|
|
1552
|
+
// local-store `extra_json` read. The metadata parser is Claude-JSONL shaped;
|
|
1553
|
+
// for other tools the fields it can't read stay at their zero defaults,
|
|
1554
|
+
// which is the same behaviour as an un-indexed session.
|
|
1555
|
+
let projectPath = ref.projectPath;
|
|
1556
|
+
const meta = { messageCount: textMessages.length };
|
|
1557
|
+
// Flag tail-only FULL syncs so the UI can badge partial history.
|
|
1558
|
+
if (oversized)
|
|
1559
|
+
meta.truncated = true;
|
|
1560
|
+
// Single-prompt invocations (batch/bot runs) carry a flag so the UI can
|
|
1561
|
+
// badge them and lists can de-emphasize them.
|
|
1562
|
+
if (textMessages.filter((m) => m.role === 'user').length <= 1)
|
|
1563
|
+
meta.oneShot = true;
|
|
1564
|
+
// Real project path: the cwd read live from inside the transcript wins over
|
|
1565
|
+
// the decoded dir name (`chat-recall` → `chat/recall` mangling).
|
|
1566
|
+
try {
|
|
1567
|
+
const cwd = ref.fullPath ? readCwdFromJsonl(ref.fullPath) : '';
|
|
1568
|
+
if (cwd)
|
|
1569
|
+
projectPath = cwd;
|
|
1570
|
+
}
|
|
1571
|
+
catch { /* fall back to ref.projectPath */ }
|
|
1572
|
+
const resolved = resolveProjectId(projectPath);
|
|
1573
|
+
const projectId = resolved.source === 'ignored' ? '' : resolved.id;
|
|
1574
|
+
if (includeMeta && ref.fullPath) {
|
|
1575
|
+
try {
|
|
1576
|
+
const parsed = await parseSessionFile(ref.fullPath);
|
|
1577
|
+
const m = parsed.metadata;
|
|
1578
|
+
const live = {
|
|
1579
|
+
inputTokens: m.inputTokens, outputTokens: m.outputTokens,
|
|
1580
|
+
cacheReadTokens: m.cacheReadTokens, cacheCreationTokens: m.cacheCreationTokens,
|
|
1581
|
+
peakContextTokens: m.peakContextTokens, costUsd: m.costUsd, durationMs: m.durationMs,
|
|
1582
|
+
modelsUsed: m.modelsUsed, toolsUsed: m.toolsUsed, gitBranch: m.gitBranch,
|
|
1583
|
+
// filesModified drives the analytics Patterns panels (Hot Files,
|
|
1584
|
+
// Redundancy). Without it those panels are permanently empty on the
|
|
1585
|
+
// server — the field simply wasn't in the synced metadata before.
|
|
1586
|
+
filesModified: m.filesModified,
|
|
1587
|
+
};
|
|
1588
|
+
for (const [k, v] of Object.entries(live)) {
|
|
1589
|
+
if (v === undefined)
|
|
1590
|
+
continue;
|
|
1591
|
+
if (Array.isArray(v) && v.length === 0)
|
|
1592
|
+
continue; // empty models/tools = un-parseable for this tool
|
|
1593
|
+
meta[k] = v;
|
|
1594
|
+
}
|
|
1595
|
+
}
|
|
1596
|
+
catch { /* telemetry is best-effort — the conversation still ships */ }
|
|
1597
|
+
}
|
|
1598
|
+
// Materialize the raw session ONCE — uniform across ALL vendors: Claude/
|
|
1599
|
+
// Gemini/Codex file bytes AND the OpenCode row-dump are all UTF-8 text in the
|
|
1600
|
+
// container. Reused for BOTH the secret scan (pre-redaction) and the raw_b64
|
|
1601
|
+
// archive. (Scanning ref.fullPath was wrong for OpenCode — that's the DB.)
|
|
1602
|
+
// Oversized sessions never materialize the container — the builtin regex
|
|
1603
|
+
// scan runs over the bounded tail instead (same coverage the APPEND path
|
|
1604
|
+
// gives a growing session's new bytes).
|
|
1605
|
+
// Reuse the shadow's fullest container (already exported + merged above) so
|
|
1606
|
+
// the secret scan and raw archive cover the RECOVERED full history, not the
|
|
1607
|
+
// truncated disk file. Only re-export when the shadow path didn't produce one.
|
|
1608
|
+
let container = fullestContainer;
|
|
1609
|
+
if (!container && !oversized) {
|
|
1610
|
+
try {
|
|
1611
|
+
const backend = getBackendForId(ref.prefixedId) ?? getBackend('claude');
|
|
1612
|
+
const exp = backend.exportRawSession(ref.prefixedId);
|
|
1613
|
+
container = exp ? buildRawContainer(exp) : null;
|
|
1614
|
+
}
|
|
1615
|
+
catch { /* export unavailable — conversation still ships */ }
|
|
1616
|
+
}
|
|
1617
|
+
// Secret SCAN on the materialized raw text (where secrets actually are,
|
|
1618
|
+
// pre-redaction). MASKED findings (last-4) ship to the server's secret_findings.
|
|
1619
|
+
// 1. BUILT-IN regex engine (same rules as the redactor) — EVERY user, zero
|
|
1620
|
+
// deps. This is what gives vanilla SaaS users real secret detection.
|
|
1621
|
+
// 2. gitleaks/trufflehog — OPTIONAL deeper engines, only when installed.
|
|
1622
|
+
// 3. TENANT rules configured in the dashboard — fetched from the server and
|
|
1623
|
+
// applied to the raw text on the client so unredacted secrets stay local.
|
|
1624
|
+
const findings = [];
|
|
1625
|
+
let scanMs = 0;
|
|
1626
|
+
if (container) {
|
|
1627
|
+
const t0 = performance.now();
|
|
1628
|
+
const rawText = container.files.map((f) => f.text).join('\n');
|
|
1629
|
+
try {
|
|
1630
|
+
for (const f of scanTextForFindings(rawText)) {
|
|
1631
|
+
findings.push({ session_id: ref.prefixedId, detector: 'builtin', rule: f.rule, line: f.line, preview: f.preview });
|
|
1632
|
+
}
|
|
1633
|
+
}
|
|
1634
|
+
catch { /* best-effort */ }
|
|
1635
|
+
if (opts.precomputedExternal) {
|
|
1636
|
+
// Batch path: gitleaks/trufflehog already ran once over a directory of
|
|
1637
|
+
// all sessions; this session's slice is handed in. No per-session spawn.
|
|
1638
|
+
// Tenant rules are cheap in-memory regex, so still run them here.
|
|
1639
|
+
for (const f of opts.precomputedExternal) {
|
|
1640
|
+
const verifiedAt = f.verified ? new Date().toISOString() : null;
|
|
1641
|
+
findings.push({ session_id: ref.prefixedId, detector: f.detector, rule: f.rule, line: f.line, preview: f.preview, verified_at: verifiedAt });
|
|
1642
|
+
}
|
|
1643
|
+
if (opts.tenantRules && opts.tenantRules.length > 0) {
|
|
1644
|
+
try {
|
|
1645
|
+
for (const f of scanTenantRules(rawText, opts.tenantRules))
|
|
1646
|
+
findings.push({ session_id: ref.prefixedId, detector: f.detector, rule: f.rule, line: f.line, preview: f.preview });
|
|
1647
|
+
}
|
|
1648
|
+
catch { /* best-effort */ }
|
|
1649
|
+
}
|
|
1650
|
+
}
|
|
1651
|
+
else if (opts.scanSecrets) {
|
|
1652
|
+
// Per-session fallback (no batch map supplied — e.g. unit tests, callers
|
|
1653
|
+
// outside the sync loop). gitleaks/trufflehog take a path: write once.
|
|
1654
|
+
let tmp = null;
|
|
1655
|
+
try {
|
|
1656
|
+
tmp = join(tmpdir(), `cr-scan-${ref.prefixedId.replace(/[^A-Za-z0-9_-]/g, '')}-${mtime}.txt`);
|
|
1657
|
+
writeFileSync(tmp, rawText);
|
|
1658
|
+
for (const f of scanFileForSecrets(tmp, { verifyOnly: opts.verifySecrets, tenantRules: opts.tenantRules })) {
|
|
1659
|
+
const verifiedAt = f.verified ? new Date().toISOString() : null;
|
|
1660
|
+
findings.push({ session_id: ref.prefixedId, detector: f.detector, rule: f.rule, line: f.line, preview: f.preview, verified_at: verifiedAt });
|
|
1661
|
+
}
|
|
1662
|
+
}
|
|
1663
|
+
catch { /* detector error — never block the sync */ }
|
|
1664
|
+
finally {
|
|
1665
|
+
if (tmp) {
|
|
1666
|
+
try {
|
|
1667
|
+
unlinkSync(tmp);
|
|
1668
|
+
}
|
|
1669
|
+
catch { /* already gone */ }
|
|
1670
|
+
}
|
|
1671
|
+
}
|
|
1672
|
+
}
|
|
1673
|
+
else if (opts.tenantRules && opts.tenantRules.length > 0) {
|
|
1674
|
+
// Tenant rules run even when gitleaks/trufflehog are absent — they are
|
|
1675
|
+
// pure regex and complement the built-in scanner.
|
|
1676
|
+
try {
|
|
1677
|
+
for (const f of scanTenantRules(rawText, opts.tenantRules)) {
|
|
1678
|
+
findings.push({ session_id: ref.prefixedId, detector: f.detector, rule: f.rule, line: f.line, preview: f.preview });
|
|
1679
|
+
}
|
|
1680
|
+
}
|
|
1681
|
+
catch { /* best-effort */ }
|
|
1682
|
+
}
|
|
1683
|
+
scanMs = performance.now() - t0;
|
|
1684
|
+
}
|
|
1685
|
+
// Oversized sessions: builtin regex (+ tenant rules) over the bounded tail —
|
|
1686
|
+
// the container path above never ran for them.
|
|
1687
|
+
if (oversized && tailText) {
|
|
1688
|
+
const t0 = performance.now();
|
|
1689
|
+
try {
|
|
1690
|
+
for (const f of scanTextForFindings(tailText)) {
|
|
1691
|
+
findings.push({ session_id: ref.prefixedId, detector: 'builtin', rule: f.rule, line: f.line, preview: f.preview });
|
|
1692
|
+
}
|
|
1693
|
+
if (opts.tenantRules && opts.tenantRules.length > 0) {
|
|
1694
|
+
for (const f of scanTenantRules(tailText, opts.tenantRules)) {
|
|
1695
|
+
findings.push({ session_id: ref.prefixedId, detector: f.detector, rule: f.rule, line: f.line, preview: f.preview });
|
|
1696
|
+
}
|
|
1697
|
+
}
|
|
1698
|
+
}
|
|
1699
|
+
catch { /* best-effort */ }
|
|
1700
|
+
scanMs = performance.now() - t0;
|
|
1701
|
+
}
|
|
1702
|
+
// High-precision default: drop fuzzy/low-precision detector findings
|
|
1703
|
+
// (generic-api-key, Box, URI, …) before shipping so the Security view isn't
|
|
1704
|
+
// buried in false positives. Opt back in with CHAT_RECALL_INCLUDE_FUZZY=1.
|
|
1705
|
+
const keptFindings = dropFuzzyFindings(findings, (f) => ({ detector: f.detector, rule: f.rule }));
|
|
1706
|
+
findings.length = 0;
|
|
1707
|
+
findings.push(...keptFindings);
|
|
1708
|
+
// Raw archive: redact the SAME container so secrets never reach the server.
|
|
1709
|
+
let raw_b64;
|
|
1710
|
+
let raw_size;
|
|
1711
|
+
if (includeRaw && container) {
|
|
1712
|
+
try {
|
|
1713
|
+
const count2 = { redactions: 0 };
|
|
1714
|
+
const redacted = mapContainerText(container, (t) => redactSecrets(t, { force: true, count: count2 }));
|
|
1715
|
+
count.redactions += count2.redactions;
|
|
1716
|
+
const { gz, size } = gzipContainer(redacted);
|
|
1717
|
+
if (gz.length <= 8 * 1024 * 1024) {
|
|
1718
|
+
raw_b64 = gz.toString('base64');
|
|
1719
|
+
raw_size = size;
|
|
1720
|
+
}
|
|
1721
|
+
}
|
|
1722
|
+
catch { /* raw is additive — conversation still ships */ }
|
|
1723
|
+
}
|
|
1724
|
+
const envTexts = envelope.messages.filter((m) => m.content?.trim());
|
|
1725
|
+
const redactedText = envTexts.map((m) => m.content).join('\n');
|
|
1726
|
+
// Native tool title (Claude ai-title, OpenCode session.title) is NOT shipped
|
|
1727
|
+
// here — it's a derived field reconciled separately (sync-fields.ts), so a
|
|
1728
|
+
// title change never drags the conversation back through sync.
|
|
1729
|
+
return {
|
|
1730
|
+
conv: {
|
|
1731
|
+
session_id: ref.prefixedId,
|
|
1732
|
+
tool: ref.toolId,
|
|
1733
|
+
project_path: mapPath(projectPath),
|
|
1734
|
+
// Resolved on THIS machine (git/FS available) — the server stores it
|
|
1735
|
+
// verbatim; it cannot resolve paths it can't see.
|
|
1736
|
+
project_id: projectId || undefined,
|
|
1737
|
+
redacted_text: redactedText,
|
|
1738
|
+
envelope,
|
|
1739
|
+
raw_b64,
|
|
1740
|
+
raw_size,
|
|
1741
|
+
first_prompt: envTexts.find((m) => m.role === 'user')?.content?.slice(0, 200),
|
|
1742
|
+
// Byte offset this FULL sync is synced THROUGH (file size at build time,
|
|
1743
|
+
// or the end of the bounded tail for an oversized session). The server
|
|
1744
|
+
// stores it as the envelope's `o`; the ledger records the SAME value
|
|
1745
|
+
// (convBatch onFlush uses this, not a recomputed size) so the next
|
|
1746
|
+
// append's base_offset matches exactly — no spurious continuity misses on
|
|
1747
|
+
// an actively-growing session. undefined for non-append-only backends.
|
|
1748
|
+
from_offset: oversized ? tailOffsetEnd : getBackendForId(ref.prefixedId)?.fileSize?.(ref.rawId),
|
|
1749
|
+
meta,
|
|
1750
|
+
mtime,
|
|
1751
|
+
},
|
|
1752
|
+
kgText: redactedText,
|
|
1753
|
+
projectPath,
|
|
1754
|
+
redactions: count.redactions,
|
|
1755
|
+
findings,
|
|
1756
|
+
scanMs,
|
|
1757
|
+
srcHash,
|
|
1758
|
+
};
|
|
1759
|
+
}
|
|
1760
|
+
/**
|
|
1761
|
+
* Parse raw JSONL tail text into canonical messages for one tool. The tail is
|
|
1762
|
+
* the bytes after the last shipped offset — complete JSONL lines only (the
|
|
1763
|
+
* backend's readFromOffset snaps to the last newline). Returns [] when the
|
|
1764
|
+
* tail has no usable messages (blank, all-summary, internal-tool prompt).
|
|
1765
|
+
*/
|
|
1766
|
+
function parseTailMessages(tool, text) {
|
|
1767
|
+
if (!text || !text.trim())
|
|
1768
|
+
return [];
|
|
1769
|
+
if (tool === 'claude')
|
|
1770
|
+
return parseClaudeTranscriptText(text);
|
|
1771
|
+
if (tool === 'gemini')
|
|
1772
|
+
return parseGeminiTranscriptText(text);
|
|
1773
|
+
if (tool === 'codex')
|
|
1774
|
+
return parseCodexTranscriptText(text);
|
|
1775
|
+
return [];
|
|
1776
|
+
}
|
|
1777
|
+
/**
|
|
1778
|
+
* Build the tail-only conversation payload for an APPEND sync (see
|
|
1779
|
+
* docs/SYNC-INCREMENTAL.md). Reads ONLY the new bytes from `fromOffset` via
|
|
1780
|
+
* the backend's `readFromOffset`, parses them into messages, and ships a
|
|
1781
|
+
* minimal append envelope — no raw_b64, no telemetry meta, no title/
|
|
1782
|
+
* first_prompt (all head-derived; the server keeps its prior values), no
|
|
1783
|
+
* derived (deferred to the FULL re-sync when the session goes quiet).
|
|
1784
|
+
*
|
|
1785
|
+
* Returns null when the tail is empty / unparseable / all-internal — the
|
|
1786
|
+
* caller treats that as "nothing to ship this tick" and leaves the ledger
|
|
1787
|
+
* cursor unchanged.
|
|
1788
|
+
*/
|
|
1789
|
+
export async function buildConversationTail(ref, fromOffset, opts = {}) {
|
|
1790
|
+
const backend = getBackendForId(ref.prefixedId);
|
|
1791
|
+
if (!backend?.readFromOffset)
|
|
1792
|
+
return null;
|
|
1793
|
+
const mapPath = opts.mapPath ?? ((p) => p);
|
|
1794
|
+
const { text: tailText, newOffset } = await backend.readFromOffset(ref.prefixedId, fromOffset);
|
|
1795
|
+
if (!tailText.trim())
|
|
1796
|
+
return null;
|
|
1797
|
+
const messages = parseTailMessages(ref.toolId, tailText);
|
|
1798
|
+
if (messages.length === 0)
|
|
1799
|
+
return null;
|
|
1800
|
+
const textMessages = messages.filter((m) => m.role !== 'summary' && m.content?.trim());
|
|
1801
|
+
if (textMessages.length === 0 && !messages.some((m) => m.toolCalls?.length))
|
|
1802
|
+
return null;
|
|
1803
|
+
// Skip chat-recall's OWN internal prompts in the tail (same gate as the
|
|
1804
|
+
// full builder). A tail that is ONLY an internal prompt ships nothing.
|
|
1805
|
+
const firstUser = textMessages.find((m) => m.role === 'user');
|
|
1806
|
+
if (isInternalToolPrompt(firstUser?.content))
|
|
1807
|
+
return null;
|
|
1808
|
+
// Trim + redact the tail envelope only (not the whole transcript).
|
|
1809
|
+
const count = { redactions: 0 };
|
|
1810
|
+
const envelope = redactDeep({ v: TRANSCRIPT_VERSION, ...trimTranscriptForSync({ messages, subagents: [] }) }, count);
|
|
1811
|
+
const envTexts = envelope.messages.filter((m) => m.content?.trim());
|
|
1812
|
+
const redactedText = envTexts.map((m) => m.content).join('\n');
|
|
1813
|
+
// Secret scan on the tail text only (builtin regex + tenant rules — cheap,
|
|
1814
|
+
// in-memory). External detectors skip append ticks; the FULL re-sync covers
|
|
1815
|
+
// the whole file when the session closes.
|
|
1816
|
+
const findings = [];
|
|
1817
|
+
try {
|
|
1818
|
+
for (const f of scanTextForFindings(tailText)) {
|
|
1819
|
+
findings.push({ session_id: ref.prefixedId, detector: 'builtin', rule: f.rule, line: f.line, preview: f.preview });
|
|
1820
|
+
}
|
|
1821
|
+
}
|
|
1822
|
+
catch { /* best-effort */ }
|
|
1823
|
+
if (opts.tenantRules && opts.tenantRules.length > 0) {
|
|
1824
|
+
try {
|
|
1825
|
+
for (const f of scanTenantRules(tailText, opts.tenantRules))
|
|
1826
|
+
findings.push({ session_id: ref.prefixedId, detector: f.detector, rule: f.rule, line: f.line, preview: f.preview });
|
|
1827
|
+
}
|
|
1828
|
+
catch { /* best-effort */ }
|
|
1829
|
+
}
|
|
1830
|
+
const keptFindings = dropFuzzyFindings(findings, (f) => ({ detector: f.detector, rule: f.rule }));
|
|
1831
|
+
findings.length = 0;
|
|
1832
|
+
findings.push(...keptFindings);
|
|
1833
|
+
// No title / first_prompt / meta / raw_b64 — all head-derived or whole-file.
|
|
1834
|
+
// The server's append path updates ONLY mtime + appends chunks + merges the
|
|
1835
|
+
// envelope; it keeps its prior title/preview/extra/raw_b64 untouched.
|
|
1836
|
+
return {
|
|
1837
|
+
conv: {
|
|
1838
|
+
session_id: ref.prefixedId,
|
|
1839
|
+
tool: ref.toolId,
|
|
1840
|
+
project_path: mapPath(ref.projectPath),
|
|
1841
|
+
append: true,
|
|
1842
|
+
base_offset: fromOffset, // where this tail STARTS — server validates it
|
|
1843
|
+
// equals its stored synced-through offset.
|
|
1844
|
+
from_offset: newOffset, // where it ends → new synced-through offset.
|
|
1845
|
+
redacted_text: redactedText,
|
|
1846
|
+
envelope,
|
|
1847
|
+
mtime: Math.floor(ref.mtime) || 0,
|
|
1848
|
+
},
|
|
1849
|
+
redactions: count.redactions,
|
|
1850
|
+
newOffset,
|
|
1851
|
+
findings,
|
|
1852
|
+
};
|
|
1853
|
+
}
|
|
1854
|
+
/**
|
|
1855
|
+
* Per-session derived payload: the four compute kinds the server's deep-dive
|
|
1856
|
+
* routes read (diff/outcome/commits/markers) + the outcome-badge row. ALWAYS
|
|
1857
|
+
* computed live from the transcript/git (no local compute/outcome cache is
|
|
1858
|
+
* read or warmed — thin collector). Everything is deep-redacted — diffs carry
|
|
1859
|
+
* file content.
|
|
1860
|
+
*/
|
|
1861
|
+
function collectDerived(ref, mtime, count) {
|
|
1862
|
+
const id = ref.prefixedId;
|
|
1863
|
+
const compute = [];
|
|
1864
|
+
let fullOutcome = null;
|
|
1865
|
+
for (const kind of COMPUTE_KINDS) {
|
|
1866
|
+
let data = null;
|
|
1867
|
+
try {
|
|
1868
|
+
if (kind === 'diff') {
|
|
1869
|
+
const replay = replaySessionAny(id);
|
|
1870
|
+
if (replay.found)
|
|
1871
|
+
data = replay;
|
|
1872
|
+
}
|
|
1873
|
+
else if (kind === 'outcome') {
|
|
1874
|
+
fullOutcome = computeOutcome(id);
|
|
1875
|
+
if (fullOutcome.found)
|
|
1876
|
+
data = fullOutcome;
|
|
1877
|
+
}
|
|
1878
|
+
else if (kind === 'commits') {
|
|
1879
|
+
// computeOutcome already ran git for the session window — reuse.
|
|
1880
|
+
if (!fullOutcome)
|
|
1881
|
+
fullOutcome = computeOutcome(id);
|
|
1882
|
+
if (fullOutcome.found)
|
|
1883
|
+
data = fullOutcome.commits;
|
|
1884
|
+
}
|
|
1885
|
+
else if (kind === 'markers') {
|
|
1886
|
+
// Analysis-only use of the turns extractor (R4) — markers need
|
|
1887
|
+
// per-prompt line/ts, not render fidelity.
|
|
1888
|
+
const turns = extractTurnsAny(id, { maxTurns: 50_000 });
|
|
1889
|
+
const prompts = turns.turns
|
|
1890
|
+
.filter((t) => t.kind === 'user' && t.text)
|
|
1891
|
+
.map((t) => ({ line: t.line, ts: t.ts, tsIso: t.tsIso, ...markPrompt(t.text) }));
|
|
1892
|
+
data = { sessionId: id, prompts, summary: summarizeMarkers(prompts) };
|
|
1893
|
+
}
|
|
1894
|
+
}
|
|
1895
|
+
catch {
|
|
1896
|
+
data = null;
|
|
1897
|
+
}
|
|
1898
|
+
if (data === null)
|
|
1899
|
+
continue;
|
|
1900
|
+
const redacted = redactDeep(data, count);
|
|
1901
|
+
if (JSON.stringify(redacted).length > MAX_DERIVED_ROW_BYTES)
|
|
1902
|
+
continue;
|
|
1903
|
+
compute.push({ kind, mtime, data: redacted });
|
|
1904
|
+
}
|
|
1905
|
+
// Outcome-badge row (session_outcome_cache) — what the list badges read.
|
|
1906
|
+
// Computed live from the outcome we already ran above.
|
|
1907
|
+
let outcomeRow = null;
|
|
1908
|
+
if (fullOutcome?.found) {
|
|
1909
|
+
outcomeRow = {
|
|
1910
|
+
sessionId: id,
|
|
1911
|
+
tool: ref.toolId,
|
|
1912
|
+
status: fullOutcome.status,
|
|
1913
|
+
reason: fullOutcome.reason,
|
|
1914
|
+
fileMtime: mtime,
|
|
1915
|
+
fileSize: 0,
|
|
1916
|
+
contentHash: '',
|
|
1917
|
+
fileCount: fullOutcome.fileCount,
|
|
1918
|
+
linesAdded: fullOutcome.totalLinesAdded,
|
|
1919
|
+
linesRemoved: fullOutcome.totalLinesRemoved,
|
|
1920
|
+
commits: fullOutcome.commits?.totalCommits ?? 0,
|
|
1921
|
+
isFull: true,
|
|
1922
|
+
classifiedAt: Date.now(),
|
|
1923
|
+
lastScannedOffset: 0,
|
|
1924
|
+
};
|
|
1925
|
+
}
|
|
1926
|
+
return {
|
|
1927
|
+
session_id: id,
|
|
1928
|
+
mtime,
|
|
1929
|
+
compute,
|
|
1930
|
+
outcome_row: outcomeRow ? redactDeep({ ...outcomeRow, sessionId: undefined }, count) : null,
|
|
1931
|
+
};
|
|
1932
|
+
}
|
|
1933
|
+
export function isSyncSkip(r) {
|
|
1934
|
+
return typeof r.skipped === 'string';
|
|
1935
|
+
}
|
|
1936
|
+
export async function syncIncremental(opts = {}) {
|
|
1937
|
+
const cred = loadCredentials();
|
|
1938
|
+
if (!cred)
|
|
1939
|
+
return { skipped: 'no-credentials' };
|
|
1940
|
+
let settings = loadSettings();
|
|
1941
|
+
// Master switch (settings.sync.enabled) — `chat-recall login` flips it on;
|
|
1942
|
+
// the user can turn background sync off without logging out. Legacy logins
|
|
1943
|
+
// (credentials.json written before login set the flag) have enabled=false
|
|
1944
|
+
// AND no endpoint recorded: treat that as consent (the only way to get
|
|
1945
|
+
// credentials is an explicit login) and migrate the settings once. An
|
|
1946
|
+
// endpoint WITH enabled=false means the user deliberately paused — respect it.
|
|
1947
|
+
if (!settings.sync.enabled) {
|
|
1948
|
+
if (settings.sync.endpoint)
|
|
1949
|
+
return { skipped: 'paused' }; // user paused
|
|
1950
|
+
settings.sync.enabled = true;
|
|
1951
|
+
settings.sync.endpoint = cred.serverUrl;
|
|
1952
|
+
saveSettings(settings);
|
|
1953
|
+
settings = loadSettings();
|
|
1954
|
+
}
|
|
1955
|
+
// SINGLE WRITER — see docs/SYNC.md. Every background sync (MCP ticks across
|
|
1956
|
+
// sessions, any file-watcher, manual incremental) serializes on ONE
|
|
1957
|
+
// cross-platform lock (O_EXCL lockfile; Linux/macOS/Windows). One writer of
|
|
1958
|
+
// the sync ledger at a time, ever. Can't get the lock ⇒ another writer is
|
|
1959
|
+
// mid-sync ⇒ skip this tick. Nothing is lost (the per-session ledger +
|
|
1960
|
+
// watermark make ticks idempotent; the next tick picks up the rest).
|
|
1961
|
+
const lock = acquireIndexLock({ kind: 'sync-incremental', staleAfterMs: 10 * 60_000 });
|
|
1962
|
+
if (!lock)
|
|
1963
|
+
return { skipped: 'lock-held' };
|
|
1964
|
+
try {
|
|
1965
|
+
const startedAt = Date.now();
|
|
1966
|
+
// Per-session ledger replaces the global watermark for sessions: a FULL
|
|
1967
|
+
// walk lists ALL sessions and the ledger skips acked ones — so a session
|
|
1968
|
+
// that failed once retries forever until it lands. scope:'changed'
|
|
1969
|
+
// (file-change ticks from the watch daemon) bounds the listing to the
|
|
1970
|
+
// recent-mtime window instead — a keystroke-burst tick walks a handful of
|
|
1971
|
+
// refs, not 30k+ — while heartbeat/startup ticks stay 'full' so the
|
|
1972
|
+
// retry-forever property holds. The watermark also bounds the
|
|
1973
|
+
// non-session items walk in both scopes.
|
|
1974
|
+
const result = await syncSessions({ useLedger: true, sinceMs: settings.sync.lastSyncAt, walk: opts.scope ?? 'full' });
|
|
1975
|
+
// Re-load before saving so we don't clobber settings written mid-sync.
|
|
1976
|
+
const fresh = loadSettings();
|
|
1977
|
+
fresh.sync.lastSyncAt = startedAt;
|
|
1978
|
+
saveSettings(fresh);
|
|
1979
|
+
return result;
|
|
1980
|
+
}
|
|
1981
|
+
finally {
|
|
1982
|
+
lock.release();
|
|
1983
|
+
}
|
|
1984
|
+
}
|
|
1985
|
+
//# sourceMappingURL=sync-client.js.map
|