chat-recall 0.5.6 → 0.5.7

package/dist/sync-client.js

@@ -45,13 +45,11 @@ import { dropFuzzyFindings } from '@chat-recall/engine/core/secret-precision.js'
 import { loadSettings, saveSettings, isProjectSyncable } from '@chat-recall/engine/core/settings.js';
 import { getDataDir } from '@chat-recall/engine/core/paths.js';
 import { listAvailableBackends } from '@chat-recall/engine/core/tool-backend.js';
-import { extractTurnsAny, replaySessionAny } from '@chat-recall/engine/core/session-multi-tool.js';
 import { parseTranscript, trimTranscriptForSync, TRANSCRIPT_VERSION, gzipContainer, mapContainerText, buildRawContainer, containerSrcHash, parseTranscriptFromContainer, updateShadow } from '@chat-recall/engine/transcript/index.js';
 import { parseClaudeTranscriptText } from '@chat-recall/engine/transcript/claude.js';
 import { parseGeminiTranscriptText } from '@chat-recall/engine/transcript/gemini.js';
 import { parseCodexTranscriptText } from '@chat-recall/engine/transcript/codex.js';
 import { getBackendForId, getBackend } from '@chat-recall/engine/core/tool-backend.js';
-import { computeOutcome } from '@chat-recall/engine/core/session-outcome.js';
 import { fileURLToPath } from 'node:url';
 let ownVersion = '0.0.0';
 try {
@@ -66,7 +64,6 @@ catch {
     }
     catch { }
 }
-import { markPrompt, summarizeMarkers } from '@chat-recall/engine/core/session-sentiment.js';
 import { parseSessionFile, readCwdFromJsonl } from '@chat-recall/engine/parsers/session.js';
 import { resolveProjectId } from '@chat-recall/engine/core/project-resolver.js';
 import { extractEntities } from '@chat-recall/engine/core/entity-extractor.js';
@@ -76,7 +73,61 @@ import { extractorVersionForTool, extractorVersionForId, toolOfId, EXTRACTOR_VER
 import { SYNC_FIELDS } from '@chat-recall/engine/core/sync-fields.js';
 import { acquireIndexLock } from '@chat-recall/engine/core/index-lock.js';
 import { fetchWithTimeout } from './http.js';
-import { withSessionReadCache } from '@chat-recall/engine/core/live-session-scan.js';
+import { gzipSync } from 'zlib';
+import { redactDeep } from './redact-deep.js';
+import { collectDerivedRows } from './derived.js';
+import { scanPool } from './scan-pool.js';
+import { isPrivateHost, transportRisk, assertTransportSafe } from './transport-safety.js';
+import { breakerState, breakerNotice, noteTargetSuccess, noteTargetFailure } from './target-breaker.js';
+import { setTelemetryEligible } from './telemetry-consent.js';
+import { record, flush } from './telemetry.js';
+/**
+ * The server rule pack as it arrived, kept for the scan workers.
+ *
+ * Workers run in their own isolate with their own copy of the redactor, so they
+ * need the patterns as data. The installed form is compiled RegExp objects,
+ * which cannot cross a postMessage.
+ */
+/**
+ * Walk progress for the next upload to carry.
+ *
+ * Module-scope because the producer (the walk) and the consumer (the batch
+ * poster) are far apart, and threading it through every batcher signature to
+ * deliver a cosmetic field would be worse than a single slot.
+ */
+/**
+ * Reduce an error message to a bounded CLASS.
+ *
+ * Telemetry must never carry a raw error string: server errors quote paths,
+ * payload fragments and occasionally tokens. An enum is enough to answer "what
+ * is breaking for customers", and cannot leak.
+ */
+function classifyError(msg) {
+    const m = msg.toLowerCase();
+    if (m.includes('429'))
+        return 'rate_limited';
+    if (m.includes('402'))
+        return 'payment_required';
+    if (m.includes('401') || m.includes('403') || m.includes('token'))
+        return 'auth';
+    if (m.includes('timed out') || m.includes('etimedout'))
+        return 'timeout';
+    if (m.includes('econnrefused'))
+        return 'refused';
+    if (m.includes('enotfound') || m.includes('dns'))
+        return 'dns';
+    if (m.includes('fetch failed') || m.includes('econnreset'))
+        return 'network';
+    if (m.includes('plain http') || m.includes('https'))
+        return 'insecure_transport';
+    if (/5\d\d/.test(m))
+        return 'server_error';
+    return 'other';
+}
+let walkProgressForUpload = null;
+let lastServerPackSpec = null;
+import { reportWalkProgress, readCollectorHealth, updateCollectorHealth } from '@chat-recall/engine/core/collector-health.js';
+import { withSessionScanScope } from '@chat-recall/engine/core/live-session-scan.js';
 import '@chat-recall/engine/core/backends/index.js'; // register the tool backends
 // Lives under the data dir so CHAT_RECALL_DATA_DIR isolates credentials the
 // same way it isolates the index (tests, multi-profile setups).
@@ -88,6 +139,10 @@ const credPath = () => join(getDataDir(), 'credentials.json');
  * form still readable.
  */
 export function saveCredentials(c) {
+    // REFUSE AT THE DOOR. Every login path funnels through here, so this is the
+    // one place that can stop a cleartext target being persisted at all — better
+    // than warning at sync time, by which point the user believes they are set up.
+    assertTransportSafe(c.serverUrl);
     mkdirSync(dirname(credPath()), { recursive: true });
     const targets = loadAllCredentials().filter((t) => t.serverUrl !== c.serverUrl);
     // Newest login FIRST: single-target consumers (loadCredentials → targets[0],
@@ -164,6 +219,10 @@ async function fetchTenantSecurityConfig(cred) {
             const packSpec = body.pack?.rules
                 ? { version: body.pack.version, rules: body.pack.rules.map((r) => ({ name: r.name, regex: r.regex, flags: r.flags, redact: true, source: r.source })) }
                 : { version: body.version, rules: enabled.filter((r) => r.redact === true).map((r) => ({ name: r.name, regex: r.regex, redact: true, source: 'tenant' })) };
+            // Keep the SPEC, not just the compiled rules: a worker needs the raw
+            // patterns to install them in its own isolate, and the compiled form
+            // holds RegExp objects that do not survive a postMessage.
+            lastServerPackSpec = { version: packSpec.version || '', rules: packSpec.rules };
             const pack = installServerRulePack(packSpec);
             rulePackVersion = serverRulePackVersion();
             for (const r of pack.rejected) {
@@ -181,6 +240,19 @@ async function fetchTenantSecurityConfig(cred) {
             const body = await settingsRes.json().catch(() => ({}));
             if (typeof body.verifySecrets === 'boolean')
                 verifySecrets = body.verifySecrets;
+            // Learned HERE because this request happens on every sync. Reading it from
+            // the /api/sync response meant a fully-synced machine — which uploads
+            // nothing, so gets no response — could never learn its own eligibility.
+            if (typeof body.telemetry === 'boolean') {
+                setTelemetryEligible(base, body.telemetry);
+                try {
+                    const prior = readCollectorHealth()?.telemetryEligible ?? {};
+                    updateCollectorHealth({
+                        telemetryEligible: { ...prior, [base]: { allowed: body.telemetry, at: Date.now() } },
+                    });
+                }
+                catch { /* cosmetic — never fail a sync over it */ }
+            }
         }
         const config = { tenantRules, verifySecrets, rulePackVersion };
         _securityConfigCache.set(base, { fetchedAt: Date.now(), config });
@@ -296,22 +368,6 @@ export function removeCredentials(serverUrl) {
 /** Tokenize an absolute project path so the server can group by project
  *  without learning the developer's filesystem layout. */
 const hashPath = (p) => (p ? 'p_' + createHash('sha256').update(p).digest('hex').slice(0, 12) : '');
-/** Redact every string anywhere inside a JSON-serializable value. Used for
- *  derived payloads (diffs carry file content; outcomes carry prompt text)
- *  where field-by-field redaction would be fragile. */
-function redactDeep(v, count) {
-    if (typeof v === 'string')
-        return redactSecrets(v, { force: true, count });
-    if (Array.isArray(v))
-        return v.map((x) => redactDeep(x, count));
-    if (v && typeof v === 'object') {
-        const o = {};
-        for (const [k, val] of Object.entries(v))
-            o[k] = redactDeep(val, count);
-        return o;
-    }
-    return v;
-}
 /** compute_cache kinds shipped to the server. Must stay in step with the
  *  kinds the server's conversation routes read (heavyCacheGet keys). */
 const COMPUTE_KINDS = ['diff', 'outcome', 'commits', 'markers'];
@@ -324,26 +380,14 @@ const MAX_DERIVED_ROW_BYTES = 2 * 1024 * 1024;
  * is no shared-tenant or rate-limiter to be polite to, so they get a higher
  * in-flight cap and zero inter-batch sleep. Remote/SaaS hosts stay gentler.
  */
-function isLocalHost(serverUrl) {
-    let host = '';
-    try {
-        host = new URL(serverUrl).hostname.toLowerCase();
-    }
-    catch {
-        return false;
-    }
-    if (host === 'localhost' || host === '127.0.0.1' || host === '::1' || host === '0.0.0.0')
-        return true;
-    if (host.endsWith('.local') || host.endsWith('.lan'))
-        return true;
-    if (/^10\./.test(host))
-        return true; // 10.0.0.0/8
-    if (/^192\.168\./.test(host))
-        return true; // 192.168.0.0/16
-    if (/^172\.(1[6-9]|2\d|3[01])\./.test(host))
-        return true; // 172.16.0.0/12
-    return false;
-}
+/**
+ * "You own this box", for upload pacing.
+ *
+ * One definition, shared with the transport gate — these were two copies of the
+ * same private-address list, and the moment they disagree either politeness or
+ * SECURITY is wrong. transport-safety.ts owns the list.
+ */
+const isLocalHost = isPrivateHost;
 export async function syncSessions(opts = {}) {
     const targets = loadAllCredentials();
     if (targets.length === 0)
@@ -356,9 +400,24 @@ export async function syncSessions(opts = {}) {
     catch { /* never block a sync on housekeeping */ }
     let agg = null;
     const errors = [];
+    const skippedTargets = [];
     for (const cred of targets) {
+        // CIRCUIT BREAKER. Targets are walked in SEQUENCE, so a dead one does not
+        // just fail — it delays every ship to the healthy targets behind it, and
+        // delays the walk's completion, which the ledger, the health file and the
+        // progress report all wait on. A LAN box that is off should cost one log
+        // line, not a full retry storm per session, every walk, forever.
+        const verdict = breakerState(cred.serverUrl);
+        if (verdict.open) {
+            const notice = breakerNotice(cred.serverUrl, verdict);
+            if (notice)
+                console.error(`[sync] ${notice}`);
+            skippedTargets.push(cred.serverUrl);
+            continue;
+        }
         try {
             const r = await syncToTarget(cred, opts);
+            noteTargetSuccess(cred.serverUrl);
             agg = agg ? {
                 uploaded: agg.uploaded + r.uploaded, skipped: agg.skipped + r.skipped, redactions: agg.redactions + r.redactions,
                 items: agg.items + r.items, links: agg.links + r.links, findings: agg.findings + r.findings,
@@ -367,14 +426,59 @@ export async function syncSessions(opts = {}) {
             } : r;
         }
         catch (e) {
-            errors.push(`${cred.serverUrl}: ${e instanceof Error ? e.message : e}`);
+            const msg = e instanceof Error ? e.message : String(e);
+            errors.push(`${cred.serverUrl}: ${msg}`);
+            // Report the NEXT attempt in the same breath as the failure, so the log
+            // says what will happen rather than leaving the user to infer it.
+            const next = noteTargetFailure(cred.serverUrl, msg);
+            const notice = breakerNotice(cred.serverUrl, next);
+            if (notice)
+                console.error(`[sync] ${notice}`);
+            // The error CLASS, never the message: a server error string can quote a
+            // path or a payload, and this channel must not carry either.
+            record({
+                kind: next.open ? 'breaker_trip' : 'target_failure',
+                failures: next.failures,
+                retryInMs: next.retryInMs,
+                errorClass: classifyError(msg),
+                local: isLocalHost(cred.serverUrl) ? 1 : 0,
+            });
         }
     }
+    // A walk that shipped nothing because every target is in cooldown is NOT a
+    // success, and must not be reported as one — the caller marks target health
+    // from this, and "0 pushed, no error" would read as a healthy quiet walk.
+    if (!agg && skippedTargets.length > 0 && errors.length === 0) {
+        throw new Error(`every sync target is in failure cooldown: ${skippedTargets.join(', ')}. `
+            + 'They will be retried automatically; run `chat-recall doctor` for the last error.');
+    }
     // Ledger writes are coalesced during the walk (see persist() in sync-ledger).
     // This is the durability point: everything acked above reaches disk before we
     // report success, so a crash after this cannot re-ship what the server has.
     // In the finally, because a partial walk's acks are still worth keeping.
     flushLedger();
+    // OPERATIONAL TELEMETRY, gated on consent + plan (telemetry-consent.ts). These
+    // are the numbers that were previously only ever measured on one developer
+    // machine, which is why "how long is a first sync for a 5,000-session
+    // customer?" had no answer. Numbers only — the payload cannot carry a path,
+    // project or session id, structurally.
+    if (agg) {
+        record({
+            kind: 'sync_walk',
+            targets: targets.length,
+            skippedTargets: skippedTargets.length,
+            failedTargets: errors.length,
+            uploaded: agg.uploaded,
+            skipped: agg.skipped,
+            redactions: agg.redactions,
+            scanned: agg.scanned,
+            scanMs: Math.round(agg.scanMs),
+            rssPeakMb: Math.round(process.memoryUsage().rss / 1048576),
+        });
+    }
+    // After the walk: the connection is warm and the numbers are final. Never
+    // awaited into the caller's error path — telemetry must not fail a sync.
+    void flush().catch(() => { });
     if (!agg)
         throw new Error(`sync failed for all targets:\n  ${errors.join('\n  ')}`);
     if (errors.length > 0)
@@ -610,7 +714,38 @@ export function batchScanExternal(items, opts = {}) {
  */
 const CAPS_TTL_MS = 60 * 60_000;
 const capsOk = new Map(); // serverUrl -> epoch ms verified
-export function _resetCapsCacheForTests() { capsOk.clear(); }
+/** Ingest concurrency the server says it will accept, per serverUrl. */
+const advertisedIngestConc = new Map();
+/** Servers that accept a gzipped request body. */
+const advertisedGzipBody = new Set();
+export function _resetCapsCacheForTests() {
+    capsOk.clear();
+    advertisedIngestConc.clear();
+    advertisedGzipBody.clear();
+}
+/**
+ * Learn the tenant-scoped ingest allowance from an authenticated response.
+ *
+ * `verifyServerApi` sets the same map from /api/capabilities, which is pre-auth
+ * and therefore can only report the CLASS ceiling. This overrides it with the
+ * number that actually applies to this tenant.
+ */
+export function setServerIngestConcurrency(serverUrl, n) {
+    advertisedIngestConc.set(serverUrl.replace(/\/$/, ''), n);
+}
+/** Does this server accept `Content-Encoding: gzip` on a request body? */
+export function serverAcceptsGzipBody(serverUrl) {
+    return advertisedGzipBody.has(serverUrl.replace(/\/$/, ''));
+}
+/**
+ * How many uploads may be in flight at once, according to the SERVER.
+ *
+ * Returns null when the server does not advertise it (older image) — the caller
+ * then keeps its own default.
+ */
+export function serverIngestConcurrency(serverUrl) {
+    return advertisedIngestConc.get(serverUrl.replace(/\/$/, '')) ?? null;
+}
 async function verifyServerApi(serverUrl, minApi) {
     const base = serverUrl.replace(/\/$/, '');
     const seen = capsOk.get(base);
@@ -620,10 +755,45 @@ async function verifyServerApi(serverUrl, minApi) {
     if ((caps.apiVersion ?? 0) < minApi) {
         throw new Error(`server too old: apiVersion ${caps.apiVersion ?? 'none'} < required ${minApi} — update the server image before syncing`);
     }
+    // Learn the server's in-flight budget instead of assuming one. The old
+    // hardcoded 4 was exactly twice the server's per-tenant cap of 2, so every
+    // walk manufactured its own 429 storm — 3,526 of them in one measured log.
+    // A number the server owns can be tuned without shipping a CLI release.
+    const adv = caps.limits?.ingestConcurrencyPerTenant;
+    if (typeof adv === 'number' && Number.isFinite(adv) && adv >= 1) {
+        advertisedIngestConc.set(base, Math.floor(adv));
+    }
+    if (caps.limits?.acceptsGzipBody === true)
+        advertisedGzipBody.add(base);
+    else
+        advertisedGzipBody.delete(base);
     capsOk.set(base, Date.now());
     return true;
 }
+/**
+ * One walk = one scan scope.
+ *
+ * Locating a session used to probe every project directory in every profile
+ * home, PER SESSION — and the walk asks twice per session
+ * (`spansMultipleSources`, then `fileSize`). Measured on a 15,724-session
+ * corpus: 62.8 seconds of filesystem syscalls to discover that 2 sessions had
+ * changed. With the scope open the directories are listed once instead: 196 ms.
+ *
+ * The scope is the honest place for the cache. A walk snapshots its session
+ * list up front, so within it "where does this session live" genuinely cannot
+ * change; outside it every lookup still reads the filesystem.
+ */
 async function syncToTarget(cred, opts = {}) {
+    return withSessionScanScope(() => syncToTargetInScope(cred, opts));
+}
+async function syncToTargetInScope(cred, opts = {}) {
+    // Checked again here, not only at login: credentials.json is a plain file a
+    // user can edit, an env var can supply a target that never saw the login gate
+    // (see credentials-env.ts), and a target saved by an older version predates
+    // the check entirely. The upload path is the last place to say no.
+    const risk = transportRisk(cred.serverUrl);
+    if (risk)
+        throw new Error(risk);
     const sync = loadSettings().sync;
     // Exclusions = local settings ∪ server-side tenant config (dashboard-edited).
     const serverCfg = await fetchTenantSyncConfig(cred);
@@ -720,7 +890,21 @@ async function syncToTarget(cred, opts = {}) {
     const sleep = (ms) => new Promise((r) => setTimeout(r, ms));
     const explicitThrottle = typeof opts.throttleMs === 'number' && opts.throttleMs > 0;
     const throttleMs = explicitThrottle ? opts.throttleMs : 0;
-    const maxInflight = explicitThrottle ? 1 : (isLocalHost(cred.serverUrl) ? 6 : 4);
+    // In-flight budget: the server's advertised per-tenant ingest concurrency when
+    // it publishes one, else the historical default. Never exceed what the server
+    // will accept — an extra in-flight batch does not go faster, it comes back
+    // 429 and then sleeps out its Retry-After.
+    //
+    // A local target is yours: no politeness needed, and no advertised cap to
+    // respect beyond its own.
+    const LOCAL_INFLIGHT = 6;
+    const REMOTE_INFLIGHT_FALLBACK = 4;
+    const advertised = serverIngestConcurrency(cred.serverUrl);
+    const maxInflight = explicitThrottle
+        ? 1
+        : isLocalHost(cred.serverUrl)
+            ? Math.max(LOCAL_INFLIGHT, advertised ?? 0)
+            : (advertised ?? REMOTE_INFLIGHT_FALLBACK);
     // Per-attempt upload timeout. WITHOUT this, a half-open/slow connection makes
     // `fetch` hang FOREVER (Node fetch has no default timeout) — observed as a
     // sync stuck for 20-40min at idle CPU with zero progress. With it, a hung
@@ -729,8 +913,41 @@ async function syncToTarget(cred, opts = {}) {
     // Single POST with retry. Retryable: network/abort errors, HTTP 429 (obeying
     // Retry-After — the server's own rate signal), and 5xx. Non-429 4xx (bad
     // token, oversized body) is fatal: retrying can't fix it.
+    // COMPRESS THE BODY. Nothing compressed it before — every sync shipped raw
+    // JSON. Measured over five real sessions, 11.47 MB of envelope became 6.69 MB:
+    // 42% off the wire for one header and one gzipSync.
+    //
+    // Why gzip and not a binary/multipart redesign. The payload is ~64% `raw_b64`
+    // — a gzipped transcript, base64'd into JSON, which inflates it ~33%. Sending
+    // those bytes binary alongside a compressed metadata part measures 6.66 MB:
+    // the SAME bytes as just gzipping the whole thing, because gzip recovers most
+    // of what base64 wasted. Binary buys CPU (115 ms vs 318 ms), not bandwidth,
+    // and costs a new wire format on both sides. Brotli reaches 6.07 MB — another
+    // 5% — and is where to go if bandwidth ever matters more than simplicity.
+    //
+    // Gated on the server advertising it: body-parser inflates transparently and
+    // its size limit still applies to the DECOMPRESSED body, so the 32 MB ceiling
+    // on /api/sync is unchanged.
+    const gzipBody = serverAcceptsGzipBody(cred.serverUrl);
     const post = async (body) => {
-        const payload = JSON.stringify(body);
+        // SERIALISE INTO A BUFFER AND KEEP ONLY THAT. Retries hold the payload for
+        // up to four attempts × a 90s timeout, so anything still referenced here is
+        // resident for minutes. The JSON string is scoped to this block so it is
+        // collectable the moment the Buffer exists — previously both the string and
+        // (with gzip on) the compressed copy stayed reachable for the whole retry
+        // chain, i.e. ~2.3× the payload held for as long as the server was slow.
+        const payload = (() => {
+            // Ride the progress report along on whatever batch is already going out.
+            // The UI had only an AGE to show ("25m behind"), which during a first sync
+            // is indistinguishable from broken. No extra request, no new endpoint —
+            // the server stashes it and serves it on the status the UI already polls.
+            const withProgress = walkProgressForUpload
+                ? { ...body, progress: walkProgressForUpload }
+                : body;
+            const json = JSON.stringify(withProgress);
+            const raw = Buffer.from(json, 'utf8');
+            return gzipBody ? gzipSync(raw, { level: 6 }) : raw;
+        })();
         const RETRY_DELAYS_MS = [2000, 8000, 30000];
         for (let attempt = 0;; attempt++) {
             const ac = new AbortController();
@@ -740,14 +957,38 @@ async function syncToTarget(cred, opts = {}) {
                     method: 'POST',
                     // No token ⇒ no auth header: a no-auth local server (AUTH_PROVIDER=none)
                     // accepts it as the single 'default' tenant.
-                    headers: cred.token
-                        ? { 'content-type': 'application/json', authorization: `Bearer ${cred.token}` }
-                        : { 'content-type': 'application/json' },
+                    headers: {
+                        'content-type': 'application/json',
+                        ...(gzipBody ? { 'content-encoding': 'gzip' } : {}),
+                        ...(cred.token ? { authorization: `Bearer ${cred.token}` } : {}),
+                    },
                     body: payload,
                     signal: ac.signal,
                 });
                 if (res.ok) {
                     const body = await res.json().catch(() => ({}));
+                    // The server owns the plan answer; the user's opt-out is checked
+                    // separately and wins over it.
+                    if (typeof body.telemetry === 'boolean') {
+                        setTelemetryEligible(base, body.telemetry);
+                        // Persisted so `doctor` — a separate process that never syncs — can
+                        // tell the user the truth about what leaves their machine.
+                        try {
+                            const prior = readCollectorHealth()?.telemetryEligible ?? {};
+                            updateCollectorHealth({
+                                telemetryEligible: { ...prior, [base]: { allowed: body.telemetry, at: Date.now() } },
+                            });
+                        }
+                        catch { /* cosmetic — never fail a sync over it */ }
+                    }
+                    // The AUTHENTICATED per-tenant allowance beats the pre-auth class
+                    // ceiling from /api/capabilities: only this response knows the plan,
+                    // and the gate scales the ceiling down per tenant. Learning it here is
+                    // what stops a free tenant asking for 6 slots it will never get.
+                    const adv = body.limits?.ingestConcurrencyPerTenant;
+                    if (typeof adv === 'number' && Number.isFinite(adv) && adv >= 1) {
+                        setServerIngestConcurrency(base, Math.floor(adv));
+                    }
                     if (body.cli && body.cli.version) {
                         const { planAutoUpdate, runAutoUpdate } = await import('./auto-update.js');
                         const authHeaders = cred.token ? { authorization: `Bearer ${cred.token}` } : {};
@@ -823,15 +1064,29 @@ async function syncToTarget(cred, opts = {}) {
     // whole sync (matching the old serial behaviour).
     const inflight = new Set();
     let poolError = null;
-    const submit = async (body, after) => {
-        while (inflight.size >= maxInflight)
+    // BOUND THE BYTES, not only the request count. `maxInflight` alone permits
+    // maxInflight × the largest batch resident at once, and a batch is capped at
+    // 4 MB only for the row categories that go through a batcher — a single
+    // oversized conversation ships ALONE and can be far larger. Counting requests
+    // said "4 in flight" whether that was 4 KB or 40 MB.
+    //
+    // An item bigger than the whole budget still ships, alone, once the pipe is
+    // empty: refusing it would silently drop a session, and waiting forever would
+    // deadlock the walk.
+    const MAX_INFLIGHT_BYTES = (Number(process.env.CHAT_RECALL_MAX_INFLIGHT_MB) || 32) * 1024 * 1024;
+    let inflightBytes = 0;
+    const submit = async (body, after, bytesHint = 0) => {
+        while (inflight.size > 0
+            && (inflight.size >= maxInflight || inflightBytes + bytesHint > MAX_INFLIGHT_BYTES)) {
             await Promise.race(inflight);
+        }
         if (poolError)
             throw poolError;
+        inflightBytes += bytesHint;
         let task;
         task = (async () => { await post(body); after(); })()
             .catch((e) => { poolError = poolError ?? e; })
-            .finally(() => { inflight.delete(task); });
+            .finally(() => { inflight.delete(task); inflightBytes -= bytesHint; });
         inflight.add(task);
     };
     const drainInflight = async () => {
@@ -860,7 +1115,12 @@ async function syncToTarget(cred, opts = {}) {
         return {
             sent: 0,
             async add(row) {
-                const size = JSON.stringify(row).length;
+                // BYTES, not UTF-16 code units. `.length` on the serialised string
+                // undercounts exactly the content that blows the budget: any non-ASCII
+                // character is 2–4 bytes on the wire but 1–2 units here, so a batch of
+                // emoji-heavy or CJK transcript could be ~3× the 4 MB cap it was
+                // measured against and get rejected by the server's body limit.
+                const size = Buffer.byteLength(JSON.stringify(row));
                 if (batch.length > 0 && (batchBytes + size > MAX_BATCH_BYTES || batch.length >= maxItems))
                     await this.drain();
                 batch.push(row);
@@ -870,11 +1130,14 @@ async function syncToTarget(cred, opts = {}) {
                 if (batch.length === 0)
                     return;
                 const rows = flatten ? batch.flat() : batch;
+                const bytes = batchBytes;
                 batch = [];
                 batchBytes = 0;
                 // Hand off to the concurrency pool; `sent`/onFlush (e.g. markSynced)
-                // fire only after the server acks THIS batch, same as before.
-                await submit({ [field]: rows }, () => { this.sent += rows.length; onFlush?.(rows); });
+                // fire only after the server acks THIS batch, same as before. The byte
+                // total is already known here, so the pool can bound memory by size
+                // instead of guessing from the request count.
+                await submit({ [field]: rows }, () => { this.sent += rows.length; onFlush?.(rows); }, bytes);
             },
         };
     };
@@ -1144,8 +1407,68 @@ async function syncToTarget(cred, opts = {}) {
         trace?.(`batch secret scan done: ${scanned} session(s) in ${batch.slices} slice(s)`);
     }
     trace?.('conversation walk start');
+    // Say that a walk STARTED, before anything is done. A first sync on a large
+    // corpus runs for a long time, and "0 of 15,724" is the difference between a
+    // user waiting and a user assuming it is broken.
+    const walkStartedAt = Date.now();
+    reportWalkProgress({ done: 0, total: slice.length, startedAt: walkStartedAt, complete: false });
+    walkProgressForUpload = { done: 0, total: slice.length, complete: false };
     let traceN = 0;
+    // ── KEEP THE EVENT LOOP ALIVE ───────────────────────────────────────────
+    //
+    // THIS IS WHY THE COLLECTOR LOOKED DEAD. Every `await` in this loop resolves
+    // as a MICROTASK, and Node drains the entire microtask queue before it
+    // advances to the timer phase. So a backlog of sessions — each one followed by
+    // a multi-second synchronous secret scan — runs as ONE unbroken chain, and for
+    // its whole duration nothing else in the process happens:
+    //
+    //   · the 60-second heartbeat never fires, so the log goes silent
+    //   · collector-health.json freezes, so every reader says "not running"
+    //   · SIGCHLD is not serviced, so a finished codeindex child sits <defunct>
+    //   · SIGTERM is not serviced, so systemd waits 90s and then SIGKILLs
+    //
+    // Measured: 12 sessions, 388 MB, 24.2 seconds of walking, ZERO of 24 expected
+    // heartbeats. With `setImmediate` between sessions — the check phase runs
+    // AFTER the timer phase — the same work fired 11. The daemon was never hung;
+    // it simply never yielded, and "never yields for nine minutes" is
+    // indistinguishable from dead to everything watching it.
+    //
+    // Once per session, unconditionally. A first version gated this behind a
+    // 50 ms budget to avoid "wasting" loop turns on the ~15,700 refs that skip in
+    // microseconds — then measured it: 15,727 unconditional yields cost 37.9 ms
+    // total, 2.4 µs each, against a walk that takes over a minute. The budget
+    // bought nothing and cost an arbitrary constant plus a clock read in the hot
+    // path, so it is gone. One yield per iteration is deterministic and needs no
+    // tuning.
+    //
+    // NOT THE WHOLE ANSWER, and worth being clear about: this makes the process
+    // RESPONSIVE while it works, it does not make the work faster. The scan is
+    // pure, CPU-bound and uses one of this machine's twelve cores. Moving it to
+    // worker_threads is the real fix — it would keep the main thread completely
+    // free AND parallelise across cores. That is a much larger change; yielding is
+    // the correct minimal one, and it is what turns "the collector is dead" back
+    // into "the collector is busy".
+    let considered = 0;
     for (const ref of slice) {
+        await new Promise((resolve) => setImmediate(resolve));
+        // PROGRESS COUNTS SESSIONS CONSIDERED, not sessions shipped.
+        //
+        // Reporting `walked` (the shipped count) looked right and was useless: on an
+        // already-synced corpus the ledger skips ~15,700 of 15,727 sessions, so the
+        // report sat at "0 of 15,727 (0%)" for the whole walk and then vanished.
+        // What a waiting user needs to know is how much of the corpus has been
+        // LOOKED AT, which is this counter and matches the denominator.
+        considered++;
+        // complete is `considered === slice.length`, not a constant false.
+        //
+        // The post-loop report below sets complete:true, but it lands after the last
+        // upload has gone out, so it only ships on the NEXT post — and a walk with
+        // nothing new to upload never makes one. The server was therefore left
+        // holding done === total with complete:false forever, and the UI read that
+        // as "syncing 99%" until something else happened to sync.
+        const walkDone = considered >= slice.length;
+        reportWalkProgress({ done: considered, total: slice.length, startedAt: walkStartedAt, complete: walkDone });
+        walkProgressForUpload = { done: considered, total: slice.length, complete: walkDone };
         if (trace && ++traceN % 500 === 0)
             trace(`walk ${traceN}/${slice.length} · at ${ref.prefixedId}`);
         localSessionIds.add(ref.prefixedId);
@@ -1293,8 +1616,32 @@ async function syncToTarget(cred, opts = {}) {
         if (upload.sessionMeta && sessionFileBytes(ref) <= FULL_BUILD_MAX_BYTES) {
             try {
                 const count = { redactions: 0 };
-                await derivedBatch.add(collectDerived(ref, mtime, count));
-                redactions += count.redactions;
+                // OFF THE MAIN THREAD. This is the heaviest work in the walk: replay,
+                // outcome and turns each read the whole transcript and shell out to git,
+                // measured at ~5 seconds per session against 78ms for the secret scan.
+                // It was the entire remaining tail of a walk — 15,500 sessions skip in
+                // seconds, then the couple of hundred that need rebuilding take minutes,
+                // all of it on the thread that has to answer signals.
+                //
+                // Only an id crosses the boundary; the worker reads the transcript
+                // itself. Null means the pool could not take it, and the inline path —
+                // the code that shipped before this — runs instead.
+                const offloaded = await scanPool().runDerived({
+                    prefixedId: ref.prefixedId, toolId: ref.toolId, mtime,
+                    maxRowBytes: MAX_DERIVED_ROW_BYTES,
+                    pack: lastServerPackSpec && scanPool().needsPack(lastServerPackSpec.version)
+                        ? { version: lastServerPackSpec.version, rules: lastServerPackSpec.rules }
+                        : undefined,
+                    packVersion: lastServerPackSpec?.version ?? '',
+                });
+                if (offloaded?.rows) {
+                    redactions += offloaded.redactions;
+                    await derivedBatch.add(offloaded.rows);
+                }
+                else {
+                    await derivedBatch.add(collectDerivedRows(ref, mtime, count, MAX_DERIVED_ROW_BYTES));
+                    redactions += count.redactions;
+                }
             }
             catch (err) {
                 // Derived is best-effort — the conversation still shipped — but it must
@@ -1495,6 +1842,12 @@ async function syncToTarget(cred, opts = {}) {
     }
     if (opts.prune)
         await post({ prune_empty_sessions: true });
+    // The walk finished. Marking it complete is what stops every later reader
+    // showing a frozen "8,400 of 15,724" forever — and it is also how a reader
+    // tells a finished walk from a daemon that died mid-walk, which is the state
+    // an in-progress value left behind means.
+    reportWalkProgress({ done: slice.length, total: slice.length, startedAt: walkStartedAt, complete: true });
+    walkProgressForUpload = { done: slice.length, total: slice.length, complete: true };
     return {
         uploaded: convBatch.sent + appendResults.uploaded, skipped, redactions,
         items: itemsBatch.sent, links: linksBatch.sent, findings: findingsBatch.sent, derived: derivedBatch.sent,
@@ -1580,18 +1933,53 @@ export async function buildConversationSync(ref, mtime, opts = {}) {
             // is what stops a resume-truncated or mtime-touched session from being
             // fully rebuilt on every tick (the export/read still happens — it's how we
             // know — but everything expensive after it is skipped).
+            // Build the container ONCE. The hash is needed before the shadow is
+            // touched (a content match short-circuits the whole session), and
+            // updateShadow used to build a second identical copy from the same
+            // export — a wasted copy of every byte on a 47 MB transcript.
+            let built;
             if (exp) {
                 try {
-                    srcHash = containerSrcHash(buildRawContainer(exp));
+                    built = buildRawContainer(exp);
+                    srcHash = containerSrcHash(built);
                 }
                 catch {
+                    built = undefined;
                     srcHash = undefined;
                 }
                 if (srcHash && opts.priorContentHash && srcHash === opts.priorContentHash) {
                     return { unchanged: true, srcHash };
                 }
+                // THE SIZE GUARD, MEASURED ON WHAT WE ACTUALLY HAVE.
+                //
+                // `oversized` above is decided from sessionFileBytes(), which returns 0
+                // for any backend that is not append-only — OpenCode implements neither
+                // fileSize() nor isAppendOnly(), because its sessions live in SQLite and
+                // have no file at all. So for those the ceiling never fired and a huge
+                // session took the FULL path: replay and computeOutcome each re-read the
+                // whole thing, and a 353 MB session with 100 MB JSON lines OOMed the walk.
+                //
+                // The export has already happened by here, and it is bounded by the
+                // session's real size. What this protects is everything AFTER it — the
+                // scan, the redact, the git replay, the derived compute — which is where
+                // the memory actually went. Measuring the container closes the guard for
+                // every backend, including any whose fileSize() under-reports.
+                const containerBytes = built
+                    ? built.files.reduce((n, f) => n + Buffer.byteLength(f.text), 0)
+                    : 0;
+                if (containerBytes > FULL_BUILD_MAX_BYTES) {
+                    console.error(`[sync] ${ref.prefixedId} is ${(containerBytes / 1048576).toFixed(0)}MB `
+                        + `(over the ${(FULL_BUILD_MAX_BYTES / 1048576).toFixed(0)}MB ceiling) — shipping without `
+                        + 'the raw archive and derived data to keep the walk in memory');
+                    // Size and tool only. Knowing that customers hit this, and how big
+                    // their sessions get, is how the ceiling gets set from evidence rather
+                    // than from one developer's corpus.
+                    record({ kind: 'oversized_session', mb: Math.round(containerBytes / 1048576), tool: ref.toolId });
+                    includeRaw = false;
+                    includeMeta = false;
+                }
             }
-            const shadow = updateShadow(ref.rawId, exp);
+            const shadow = updateShadow(ref.rawId, exp, built);
             if (shadow.container) {
                 fullestContainer = shadow.container;
                 if (shadow.status === 'rewrite-merged' && shadow.recovered > 0) {
@@ -1705,11 +2093,42 @@ export async function buildConversationSync(ref, mtime, opts = {}) {
     //      applied to the raw text on the client so unredacted secrets stay local.
     const findings = [];
     let scanMs = 0;
+    // OFF THE MAIN THREAD. Scan + redact + gzip + base64 are ~98% of the CPU this
+    // function burns (measured: 31.2 s over 200 MB of real sessions) and all four
+    // touch the same text, so one worker task carries the text in and returns only
+    // small results — findings, a redaction count and the compressed archive. The
+    // 30 MB string never comes back, which is what keeps the transfer at 1.8% of
+    // the work moved.
+    //
+    // Null means "not offloaded" for any reason at all (pool disabled, no worker
+    // bundle, worker died, task threw). Every use below falls back to the inline
+    // code that shipped before this existed.
+    let offloaded = null;
+    if (container) {
+        const pool = scanPool();
+        if (pool.enabled) {
+            const packVersion = lastServerPackSpec?.version ?? '';
+            const sendPack = lastServerPackSpec !== null && pool.needsPack(packVersion);
+            offloaded = await pool.run({
+                files: container.files.map((f) => ({ name: f.name, text: f.text })),
+                container: { v: 1, tool: container.tool, mtime: container.mtime, srcHash: container.srcHash },
+                includeRaw,
+                maxRawBytes: 8 * 1024 * 1024,
+                pack: sendPack && lastServerPackSpec
+                    ? { version: packVersion, rules: lastServerPackSpec.rules }
+                    : undefined,
+                packVersion,
+            });
+            if (offloaded && sendPack)
+                pool.markPackSent(packVersion);
+        }
+    }
     if (container) {
         const t0 = performance.now();
         const rawText = container.files.map((f) => f.text).join('\n');
         try {
-            for (const f of scanTextForFindings(rawText)) {
+            const builtin = offloaded ? offloaded.findings : scanTextForFindings(rawText);
+            for (const f of builtin) {
                 findings.push({ session_id: ref.prefixedId, detector: 'builtin', rule: f.rule, line: f.line, preview: f.preview });
             }
         }
@@ -1791,17 +2210,27 @@ export async function buildConversationSync(ref, mtime, opts = {}) {
     let raw_b64;
     let raw_size;
     if (includeRaw && container) {
-        try {
-            const count2 = { redactions: 0 };
-            const redacted = mapContainerText(container, (t) => redactSecrets(t, { force: true, count: count2 }));
-            count.redactions += count2.redactions;
-            const { gz, size } = gzipContainer(redacted);
-            if (gz.length <= 8 * 1024 * 1024) {
-                raw_b64 = gz.toString('base64');
-                raw_size = size;
+        if (offloaded) {
+            // The worker already redacted, gzipped and encoded it. An absent rawB64
+            // means the archive exceeded the 8 MB ceiling — the same decision the
+            // inline path makes, taken in the worker so the bytes never travel.
+            count.redactions += offloaded.redactions;
+            raw_b64 = offloaded.rawB64;
+            raw_size = offloaded.rawSize;
+        }
+        else {
+            try {
+                const count2 = { redactions: 0 };
+                const redacted = mapContainerText(container, (t) => redactSecrets(t, { force: true, count: count2 }));
+                count.redactions += count2.redactions;
+                const { gz, size } = gzipContainer(redacted);
+                if (gz.length <= 8 * 1024 * 1024) {
+                    raw_b64 = gz.toString('base64');
+                    raw_size = size;
+                }
             }
+            catch { /* raw is additive — conversation still ships */ }
         }
-        catch { /* raw is additive — conversation still ships */ }
     }
     const envTexts = envelope.messages.filter((m) => m.content?.trim());
     const redactedText = envTexts.map((m) => m.content).join('\n');
@@ -1933,90 +2362,6 @@ export async function buildConversationTail(ref, fromOffset, opts = {}) {
         findings,
     };
 }
-/**
- * Per-session derived payload: the four compute kinds the server's deep-dive
- * routes read (diff/outcome/commits/markers) + the outcome-badge row. ALWAYS
- * computed live from the transcript/git (no local compute/outcome cache is
- * read or warmed — thin collector). Everything is deep-redacted — diffs carry
- * file content.
- */
-function collectDerived(ref, mtime, count) {
-    const id = ref.prefixedId;
-    const compute = [];
-    let fullOutcome = null;
-    // One read of this transcript instead of four. replay, computeOutcome (twice
-    // internally) and extractTurns each re-read the whole file; the scope holds
-    // exactly one session's text and drops it on the way out.
-    return withSessionReadCache(() => {
-        for (const kind of COMPUTE_KINDS) {
-            let data = null;
-            try {
-                if (kind === 'diff') {
-                    const replay = replaySessionAny(id);
-                    if (replay.found)
-                        data = replay;
-                }
-                else if (kind === 'outcome') {
-                    fullOutcome = computeOutcome(id);
-                    if (fullOutcome.found)
-                        data = fullOutcome;
-                }
-                else if (kind === 'commits') {
-                    // computeOutcome already ran git for the session window — reuse.
-                    if (!fullOutcome)
-                        fullOutcome = computeOutcome(id);
-                    if (fullOutcome.found)
-                        data = fullOutcome.commits;
-                }
-                else if (kind === 'markers') {
-                    // Analysis-only use of the turns extractor (R4) — markers need
-                    // per-prompt line/ts, not render fidelity.
-                    const turns = extractTurnsAny(id, { maxTurns: 50_000 });
-                    const prompts = turns.turns
-                        .filter((t) => t.kind === 'user' && t.text)
-                        .map((t) => ({ line: t.line, ts: t.ts, tsIso: t.tsIso, ...markPrompt(t.text) }));
-                    data = { sessionId: id, prompts, summary: summarizeMarkers(prompts) };
-                }
-            }
-            catch {
-                data = null;
-            }
-            if (data === null)
-                continue;
-            const redacted = redactDeep(data, count);
-            if (JSON.stringify(redacted).length > MAX_DERIVED_ROW_BYTES)
-                continue;
-            compute.push({ kind, mtime, data: redacted });
-        }
-        // Outcome-badge row (session_outcome_cache) — what the list badges read.
-        // Computed live from the outcome we already ran above.
-        let outcomeRow = null;
-        if (fullOutcome?.found) {
-            outcomeRow = {
-                sessionId: id,
-                tool: ref.toolId,
-                status: fullOutcome.status,
-                reason: fullOutcome.reason,
-                fileMtime: mtime,
-                fileSize: 0,
-                contentHash: '',
-                fileCount: fullOutcome.fileCount,
-                linesAdded: fullOutcome.totalLinesAdded,
-                linesRemoved: fullOutcome.totalLinesRemoved,
-                commits: fullOutcome.commits?.totalCommits ?? 0,
-                isFull: true,
-                classifiedAt: Date.now(),
-                lastScannedOffset: 0,
-            };
-        }
-        return {
-            session_id: id,
-            mtime,
-            compute,
-            outcome_row: outcomeRow ? redactDeep({ ...outcomeRow, sessionId: undefined }, count) : null,
-        };
-    });
-}
 export function isSyncSkip(r) {
     return typeof r.skipped === 'string';
 }