@gamaze/hicortex 0.13.1 → 0.13.2

package/dist/nightly.js

@@ -58,7 +58,6 @@ const db_js_1 = require("./db.js");
 const llm_js_1 = require("./llm.js");
 const embedder_js_1 = require("./embedder.js");
 const storage = __importStar(require("./storage.js"));
-const distiller_js_1 = require("./distiller.js");
 const consolidate_js_1 = require("./consolidate.js");
 const domain_classify_js_1 = require("./domain-classify.js");
 const nofit_js_1 = require("./nofit.js");
@@ -68,6 +67,8 @@ const pi_transcript_reader_js_1 = require("./pi-transcript-reader.js");
 const oc_transcript_reader_js_1 = require("./oc-transcript-reader.js");
 const features_js_1 = require("./features.js");
 const state_js_1 = require("./state.js");
+const capture_cursors_js_1 = require("./capture-cursors.js");
+const capture_js_1 = require("./capture.js");
 const telemetry_js_1 = require("./telemetry.js");
 const HICORTEX_HOME = (0, paths_js_1.hicortexHome)();
 function readNightlyConfig(stateDir) {
@@ -97,12 +98,81 @@ function readLastRun(stateDir = HICORTEX_HOME) {
     const d = new Date(ts);
     return isNaN(d.getTime()) ? new Date(0) : d;
 }
+/**
+ * Discovery watermark. Normally the last-nightly timestamp; with
+ * `--recapture-window <days>` (#189 Tier-2 recovery) the window may only
+ * WIDEN — since = min(lastNightly, now−N days). Taking the earlier of the two
+ * means a machine that was offline longer than N days still re-discovers every
+ * session it missed; using now−N unconditionally would NARROW the window and
+ * skip (then, via writeLastRun, permanently lose) the 8-to-N-day-old sessions
+ * (#189 review, fix 3). Per-session cursors keep the wide re-scan cheap: an
+ * already-captured session yields an empty delta.
+ */
+function computeSince(stateDir, recaptureWindowDays) {
+    const lastRun = readLastRun(stateDir);
+    if (recaptureWindowDays && recaptureWindowDays > 0) {
+        const windowStart = new Date(Date.now() - recaptureWindowDays * 24 * 60 * 60 * 1000);
+        return windowStart < lastRun ? windowStart : lastRun;
+    }
+    return lastRun;
+}
+/** POST /distill transport for server mode — localhost, no auth (localhost bypasses). */
+function makeLocalPost(port) {
+    return async (body) => {
+        const resp = await fetch(`http://127.0.0.1:${port}/distill`, {
+            method: "POST",
+            headers: { "Content-Type": "application/json" },
+            body: JSON.stringify(body),
+            // Synchronous 35B distillation of a large segment can take minutes.
+            signal: AbortSignal.timeout(20 * 60 * 1000),
+        });
+        return normalizePostResult(resp);
+    };
+}
+/** POST /distill transport for client mode — remote URL + optional bearer token. */
+function makeRemotePost(serverUrl, authToken) {
+    return async (body) => {
+        const resp = await fetch(`${serverUrl}/distill`, {
+            method: "POST",
+            headers: {
+                "Content-Type": "application/json",
+                ...(authToken ? { Authorization: `Bearer ${authToken}` } : {}),
+            },
+            body: JSON.stringify(body),
+            signal: AbortSignal.timeout(20 * 60 * 1000),
+        });
+        return normalizePostResult(resp);
+    };
+}
+async function normalizePostResult(resp) {
+    if (resp.status === 201) {
+        const data = (await resp.json().catch(() => ({})));
+        return { status: 201, distilled: data.distilled ?? 0, dropped: data.dropped ?? [] };
+    }
+    if (resp.status === 200) {
+        const data = (await resp.json().catch(() => ({})));
+        return { status: 200, skipped: Boolean(data.skipped) };
+    }
+    const data = (await resp.json().catch(() => ({})));
+    return { status: resp.status, error: data.error ?? "unknown error" };
+}
 function writeLastRun(stateDir = HICORTEX_HOME) {
     (0, state_js_1.updateState)((s) => {
         s.lastNightly = new Date().toISOString();
         return s;
     }, stateDir);
 }
+/**
+ * How long a full nightly waits for the capture lock before giving up and
+ * running consolidation without capturing (fix 10). Comfortably longer than a
+ * typical daytime --capture-only run; short enough not to stall the box.
+ * Overridable via HICORTEX_CAPTURE_LOCK_WAIT_MS (tests only).
+ */
+const CAPTURE_LOCK_WAIT_MS = 30 * 60 * 1000;
+function captureLockWaitMs() {
+    const env = Number(process.env.HICORTEX_CAPTURE_LOCK_WAIT_MS);
+    return Number.isFinite(env) && env >= 0 ? env : CAPTURE_LOCK_WAIT_MS;
+}
 const NIGHTLY_LOG_MAX_BYTES = 1024 * 1024; // 1 MB — years of normal runs
 /**
  * Keep ~/.hicortex/nightly.log bounded. The launchd plist and systemd unit
@@ -125,6 +195,7 @@ async function runNightly(options = {}) {
     const dryRun = options.dryRun ?? false;
     const captureOnly = options.captureOnly ?? false;
     const stateDir = options.stateDir ?? HICORTEX_HOME;
+    const recaptureWindowDays = options.recaptureWindowDays;
     rotateNightlyLog(stateDir);
     // One-time migration of legacy state files (no-op if state.json exists)
     (0, state_js_1.migrateLegacyState)(stateDir);
@@ -133,7 +204,7 @@ async function runNightly(options = {}) {
     if (savedConfig?.mode === "client") {
         // --capture-only is accepted in client mode but irrelevant: client nightly
         // is already capture-only (no consolidation step).
-        await runClientNightly(savedConfig, dryRun);
+        await runClientNightly(savedConfig, dryRun, stateDir, recaptureWindowDays);
         return;
     }
     const dbPath = (0, db_js_1.resolveDbPath)(options.dbPath);
@@ -160,97 +231,99 @@ async function runNightly(options = {}) {
         }
         const llmConfig = resolved.config;
         const llm = llmConfig ? new llm_js_1.LlmClient(llmConfig) : null;
-        // Step 1: Read new transcripts (CC + Hermes + Pi + OpenClaw)
-        const since = readLastRun();
-        console.log(`[hicortex] Reading transcripts since ${since.toISOString()}`);
-        const ccBatches = (0, transcript_reader_js_1.readCcTranscripts)(since);
-        const hermesBatches = (0, hermes_transcript_reader_js_1.readHermesSessions)(since);
-        // Pi is a supported harness in the product (readPiTranscripts no-ops when
-        // ~/.pi/agent/sessions is absent). Retired only on specific deployments by
-        // simply having no Pi session files — not removed from the pipeline.
-        const piBatches = (0, pi_transcript_reader_js_1.readPiTranscripts)(since);
-        // OpenClaw persists sessions in the Pi v3 format at ~/.openclaw/agents/;
-        // no-ops when OC isn't installed.
-        const ocBatches = (0, oc_transcript_reader_js_1.readOcTranscripts)(since);
-        const batches = [...ccBatches, ...hermesBatches, ...piBatches, ...ocBatches];
-        if (ccBatches.length > 0)
-            console.log(`[hicortex] Found ${ccBatches.length} CC session(s)`);
-        if (hermesBatches.length > 0)
-            console.log(`[hicortex] Found ${hermesBatches.length} Hermes session(s)`);
-        if (piBatches.length > 0)
-            console.log(`[hicortex] Found ${piBatches.length} Pi session(s)`);
-        if (ocBatches.length > 0)
-            console.log(`[hicortex] Found ${ocBatches.length} OpenClaw session(s)`);
-        console.log(`[hicortex] Total: ${batches.length} new session(s)`);
-        if (batches.length === 0 && !dryRun) {
-            // Still run consolidation (unless capture-only) — there may be unscored memories from OC.
-            console.log(captureOnly
-                ? `[hicortex] No new transcripts. Nothing to capture.`
-                : `[hicortex] No new transcripts. Running consolidation only.`);
-        }
-        // Step 2: Denoise and POST each session to the local daemon via /distill.
-        // The dedup check and distillation quality (35B) are the server's concern.
+        // Steps 1+2: single-flight capture. The lock guards ONLY the capture phase
+        // (fix 10). We acquire it BEFORE reading the cursor store so a run that
+        // waited out another cannot act on a stale snapshot and clobber its cursor
+        // advances (fix 6). Batch-kind counts are hoisted for the telemetry at the
+        // end (they stay 0 if capture is skipped).
+        let ccBatches = [];
+        let hermesBatches = [];
+        let piBatches = [];
+        let ocBatches = [];
+        let batches = [];
         let memoriesIngested = 0;
         let hadTransientFailure = false;
-        for (const batch of batches) {
-            const transcript = (0, distiller_js_1.extractConversationText)(batch.entries);
-            if (transcript.length < 200) {
-                console.log(`[hicortex]   Skip ${batch.sessionId.slice(0, 8)} (${batch.projectName}): too short`);
-                continue;
-            }
-            console.log(`[hicortex]   Capturing ${batch.sessionId.slice(0, 8)} (${batch.projectName}, ${batch.date})`);
-            if (dryRun) {
-                console.log(`[hicortex]     [dry-run] Would POST ${transcript.length} chars to /distill`);
-                continue;
+        // Full nightly waits out a transient --capture-only overlap (each segment
+        // POST can block up to 20 min); capture-only fails fast. dry-run writes
+        // nothing so it needs no lock.
+        const lockWaitMs = captureLockWaitMs();
+        const releaseLock = dryRun
+            ? (() => { })
+            : await (0, capture_js_1.acquireCaptureLock)(stateDir, captureOnly ? 0 : lockWaitMs);
+        if (!releaseLock) {
+            if (captureOnly) {
+                console.warn("[hicortex] Another capture run holds the lock — skipping this capture-only run.");
+                return;
             }
+            // Full nightly couldn't get the lock even after waiting: skip capture and
+            // hold the watermark, but STILL run consolidation + telemetry below so an
+            // overlapping capture-only run never silently starves consolidation (fix 10).
+            console.warn(`[hicortex] Capture lock still held after waiting ${Math.round(lockWaitMs / 60000)} min — ` +
+                `skipping capture this run (watermark held), consolidation still runs.`);
+            hadTransientFailure = true;
+        }
+        else {
             try {
-                const resp = await fetch(`http://127.0.0.1:${port}/distill`, {
-                    method: "POST",
-                    headers: { "Content-Type": "application/json" },
-                    body: JSON.stringify({
-                        text: transcript,
-                        source_agent: batch.sourceAgent ?? `claude-code/${batch.projectName}`,
-                        project: batch.projectName,
-                        session_id: batch.sessionId,
-                        session_date: batch.date,
-                        privacy: "WORK",
-                    }),
-                    // Synchronous 35B distillation of a large session can take minutes.
-                    signal: AbortSignal.timeout(20 * 60 * 1000),
-                });
-                if (resp.status === 200) {
-                    const data = await resp.json();
-                    if (data.skipped) {
-                        console.log(`[hicortex]   Skip ${batch.sessionId.slice(0, 8)} (${batch.projectName}): already ingested`);
-                    }
-                }
-                else if (resp.status === 201) {
-                    const data = await resp.json();
-                    memoriesIngested += data.distilled ?? 0;
-                    console.log(`[hicortex]     → ${data.distilled ?? 0} memories extracted`);
-                    // Durable audit trail (#156): the server truncates each dropped entry.
-                    for (const d of data.dropped ?? []) {
-                        console.log(`[hicortex]     Substance gate: dropped "${d}"`);
-                    }
-                }
-                else if (resp.status === 429) {
-                    const data = await resp.json();
-                    console.log(`[hicortex]   Memory limit reached: ${data.error}. Stopping capture.`);
-                    break;
+                // Step 1: Read new transcripts (CC + Hermes + Pi + OpenClaw). Discovery
+                // is whole-session by mtime/ended_at; per-session cursors slice each
+                // discovered session down to its unseen delta (#189).
+                const since = computeSince(stateDir, recaptureWindowDays);
+                if (recaptureWindowDays) {
+                    console.log(`[hicortex] --recapture-window ${recaptureWindowDays}d: reading transcripts since ${since.toISOString()}`);
                 }
                 else {
-                    const data = await resp.json().catch(() => ({}));
-                    console.error(`[hicortex]     /distill returned ${resp.status}: ${data.error ?? "unknown error"} — will retry next run`);
-                    hadTransientFailure = true;
+                    console.log(`[hicortex] Reading transcripts since ${since.toISOString()}`);
+                }
+                const cursorStore = (0, capture_cursors_js_1.openCursorStore)(stateDir);
+                const cursorMap = cursorStore.map();
+                ccBatches = (0, transcript_reader_js_1.readCcTranscripts)(since, undefined, cursorMap);
+                hermesBatches = (0, hermes_transcript_reader_js_1.readHermesSessions)(since, undefined, cursorMap);
+                // Pi is a supported harness (readPiTranscripts no-ops when
+                // ~/.pi/agent/sessions is absent). Retired only on specific deployments
+                // by simply having no Pi session files — not removed from the pipeline.
+                piBatches = (0, pi_transcript_reader_js_1.readPiTranscripts)(since, undefined, cursorMap);
+                // OpenClaw persists sessions in the Pi v3 format at ~/.openclaw/agents/;
+                // no-ops when OC isn't installed.
+                ocBatches = (0, oc_transcript_reader_js_1.readOcTranscripts)(since, undefined, cursorMap);
+                batches = [...ccBatches, ...hermesBatches, ...piBatches, ...ocBatches];
+                if (ccBatches.length > 0)
+                    console.log(`[hicortex] Found ${ccBatches.length} CC session(s)`);
+                if (hermesBatches.length > 0)
+                    console.log(`[hicortex] Found ${hermesBatches.length} Hermes session(s)`);
+                if (piBatches.length > 0)
+                    console.log(`[hicortex] Found ${piBatches.length} Pi session(s)`);
+                if (ocBatches.length > 0)
+                    console.log(`[hicortex] Found ${ocBatches.length} OpenClaw session(s)`);
+                console.log(`[hicortex] Total: ${batches.length} new session(s)`);
+                if (batches.length === 0 && !dryRun) {
+                    console.log(captureOnly
+                        ? `[hicortex] No new transcripts. Nothing to capture.`
+                        : `[hicortex] No new transcripts. Running consolidation only.`);
                 }
+                // Step 2: pack each session's delta into ≤60K segments and POST to the
+                // local daemon via /distill; cursors advance on confirmed success.
+                const result = await (0, capture_js_1.captureBatches)(batches, {
+                    post: makeLocalPost(port),
+                    cursorStore,
+                    dryRun,
+                });
+                memoriesIngested = result.memoriesIngested;
+                // A 429/401 stop must hold the watermark too (fix 1): the loop abandoned
+                // the remaining sessions, and mtime discovery would never re-find them.
+                hadTransientFailure = result.hadTransientFailure || result.stopped !== undefined;
+            }
+            finally {
+                releaseLock();
             }
-            catch (err) {
-                const msg = err instanceof Error ? err.message : String(err);
-                console.error(`[hicortex]     Capture failed: ${msg} — will retry next run`);
-                hadTransientFailure = true;
+            console.log(`[hicortex] Capture complete: ${memoriesIngested} new memories`);
+            // Prune aged-out cursors (90d) — only on a clean run so a transient
+            // failure doesn't drop a still-needed cursor.
+            if (!dryRun && !hadTransientFailure) {
+                const pruned = (0, capture_cursors_js_1.pruneCursors)(stateDir);
+                if (pruned > 0)
+                    console.log(`[hicortex] Pruned ${pruned} stale capture cursor(s)`);
             }
         }
-        console.log(`[hicortex] Capture complete: ${memoriesIngested} new memories`);
         // Step 3: Consolidation — skipped in capture-only mode, dry-run, or no LLM.
         // Runs even if capture had transient failures (opens DB directly, independent
         // of the HTTP capture path). Full nightly only — capture-only runs are
@@ -327,7 +400,7 @@ async function runNightly(options = {}) {
                     `They will be retried on the next run.`);
             }
             else {
-                writeLastRun();
+                writeLastRun(stateDir);
             }
         }
         console.log(`[hicortex] Nightly pipeline complete.`);
@@ -360,7 +433,7 @@ async function runNightly(options = {}) {
 // ---------------------------------------------------------------------------
 // Client Mode Nightly — denoise locally, POST to remote server's /distill
 // ---------------------------------------------------------------------------
-async function runClientNightly(config, dryRun) {
+async function runClientNightly(config, dryRun, stateDir = HICORTEX_HOME, recaptureWindowDays) {
     const serverUrl = config.serverUrl.replace(/\/+$/, "");
     const authToken = config.authToken;
     console.log(`[hicortex] Client nightly starting${dryRun ? " (dry run)" : ""}`);
@@ -379,109 +452,84 @@ async function runClientNightly(config, dryRun) {
         return; // Don't update last-run so we retry
     }
     // No local LLM needed — distillation happens on the server.
-    // Read new transcripts (CC + Hermes + Pi + OpenClaw). Client reads local
-    // logs, denoises, and POSTs the denoised text to the server's /distill
-    // endpoint. All readers no-op when their harness isn't installed.
-    const since = readLastRun();
-    console.log(`[hicortex] Reading transcripts since ${since.toISOString()}`);
-    const ccBatches = (0, transcript_reader_js_1.readCcTranscripts)(since);
-    const hermesBatches = (0, hermes_transcript_reader_js_1.readHermesSessions)(since);
-    const piBatches = (0, pi_transcript_reader_js_1.readPiTranscripts)(since);
-    const ocBatches = (0, oc_transcript_reader_js_1.readOcTranscripts)(since);
-    const batches = [...ccBatches, ...hermesBatches, ...piBatches, ...ocBatches];
-    if (ccBatches.length > 0)
-        console.log(`[hicortex] Found ${ccBatches.length} CC session(s)`);
-    if (hermesBatches.length > 0)
-        console.log(`[hicortex] Found ${hermesBatches.length} Hermes session(s)`);
-    if (piBatches.length > 0)
-        console.log(`[hicortex] Found ${piBatches.length} Pi session(s)`);
-    if (ocBatches.length > 0)
-        console.log(`[hicortex] Found ${ocBatches.length} OpenClaw session(s)`);
-    console.log(`[hicortex] Total: ${batches.length} new session(s)`);
-    if (batches.length === 0) {
-        console.log(`[hicortex] Nothing to capture.`);
-        if (!dryRun)
-            writeLastRun();
+    // Single-flight (A5): acquire the capture lock BEFORE reading the cursor
+    // store so a run that waited out another can't act on a stale snapshot and
+    // clobber its advances (fix 6). The client is capture-only; on contention we
+    // skip and hold the watermark (no writeLastRun → retried next run). dry-run
+    // writes nothing so it needs no lock.
+    const releaseLock = dryRun ? (() => { }) : await (0, capture_js_1.acquireCaptureLock)(stateDir);
+    if (!releaseLock) {
+        console.warn("[hicortex] Another capture run holds the lock — skipping this run (watermark held).");
         return;
     }
-    let hadTransientFailure = false;
+    let ccBatches = [];
+    let hermesBatches = [];
+    let piBatches = [];
+    let ocBatches = [];
+    let batches = [];
     let memoriesIngested = 0;
     let sessionsSent = 0;
-    for (const batch of batches) {
-        const transcript = (0, distiller_js_1.extractConversationText)(batch.entries);
-        if (transcript.length < 200) {
-            console.log(`[hicortex]   Skip ${batch.sessionId.slice(0, 8)} (${batch.projectName}): too short`);
-            continue;
+    let hadTransientFailure = false;
+    try {
+        // Read new transcripts (CC + Hermes + Pi + OpenClaw). Client reads local
+        // logs, denoises, and POSTs the denoised text to the server's /distill
+        // endpoint. All readers no-op when their harness isn't installed. Per-session
+        // cursors slice each discovered session to its unseen delta (#189).
+        const since = computeSince(stateDir, recaptureWindowDays);
+        if (recaptureWindowDays) {
+            console.log(`[hicortex] --recapture-window ${recaptureWindowDays}d: reading transcripts since ${since.toISOString()}`);
         }
-        console.log(`[hicortex]   Capturing ${batch.sessionId.slice(0, 8)} (${batch.projectName}, ${batch.date})`);
-        if (dryRun) {
-            console.log(`[hicortex]     [dry-run] Would POST ${transcript.length} chars to ${serverUrl}/distill`);
-            continue;
+        else {
+            console.log(`[hicortex] Reading transcripts since ${since.toISOString()}`);
         }
-        try {
-            const resp = await fetch(`${serverUrl}/distill`, {
-                method: "POST",
-                headers: {
-                    "Content-Type": "application/json",
-                    ...(authToken ? { "Authorization": `Bearer ${authToken}` } : {}),
-                },
-                body: JSON.stringify({
-                    text: transcript,
-                    source_agent: batch.sourceAgent ?? `claude-code/${batch.projectName}`,
-                    project: batch.projectName,
-                    session_id: batch.sessionId,
-                    session_date: batch.date,
-                    privacy: "WORK",
-                }),
-                // Synchronous 35B distillation of a large session can take minutes.
-                signal: AbortSignal.timeout(20 * 60 * 1000),
-            });
-            if (resp.status === 200) {
-                const data = await resp.json();
-                if (data.skipped) {
-                    console.log(`[hicortex]   Skip ${batch.sessionId.slice(0, 8)}: already ingested on server`);
-                }
-            }
-            else if (resp.status === 201) {
-                const data = await resp.json();
-                const count = data.distilled ?? 0;
-                memoriesIngested += count;
-                sessionsSent++;
-                console.log(`[hicortex]     → ${count} memories sent to server`);
-                // Durable audit trail (#156): the server truncates each dropped entry.
-                for (const d of data.dropped ?? []) {
-                    console.log(`[hicortex]     Substance gate: dropped "${d}"`);
-                }
-            }
-            else if (resp.status === 401) {
-                console.error(`[hicortex]     Auth failed. Check authToken in ~/.hicortex/config.json`);
-                return; // No point retrying with wrong credentials
-            }
-            else if (resp.status === 429) {
-                const data = await resp.json().catch(() => ({}));
-                console.log(`[hicortex]   Server memory limit reached: ${data.error}`);
-                return;
-            }
-            else {
-                const data = await resp.json().catch(() => ({}));
-                console.error(`[hicortex]     /distill returned ${resp.status}: ${data.error ?? "unknown error"} — will retry next run`);
-                hadTransientFailure = true;
-            }
+        const cursorStore = (0, capture_cursors_js_1.openCursorStore)(stateDir);
+        const cursorMap = cursorStore.map();
+        ccBatches = (0, transcript_reader_js_1.readCcTranscripts)(since, undefined, cursorMap);
+        hermesBatches = (0, hermes_transcript_reader_js_1.readHermesSessions)(since, undefined, cursorMap);
+        piBatches = (0, pi_transcript_reader_js_1.readPiTranscripts)(since, undefined, cursorMap);
+        ocBatches = (0, oc_transcript_reader_js_1.readOcTranscripts)(since, undefined, cursorMap);
+        batches = [...ccBatches, ...hermesBatches, ...piBatches, ...ocBatches];
+        if (ccBatches.length > 0)
+            console.log(`[hicortex] Found ${ccBatches.length} CC session(s)`);
+        if (hermesBatches.length > 0)
+            console.log(`[hicortex] Found ${hermesBatches.length} Hermes session(s)`);
+        if (piBatches.length > 0)
+            console.log(`[hicortex] Found ${piBatches.length} Pi session(s)`);
+        if (ocBatches.length > 0)
+            console.log(`[hicortex] Found ${ocBatches.length} OpenClaw session(s)`);
+        console.log(`[hicortex] Total: ${batches.length} new session(s)`);
+        if (batches.length === 0) {
+            console.log(`[hicortex] Nothing to capture.`);
         }
-        catch (err) {
-            console.error(`[hicortex]     Capture failed: ${err instanceof Error ? err.message : String(err)} — will retry next run`);
-            hadTransientFailure = true;
+        else {
+            const result = await (0, capture_js_1.captureBatches)(batches, {
+                post: makeRemotePost(serverUrl, authToken),
+                cursorStore,
+                dryRun,
+            });
+            memoriesIngested = result.memoriesIngested;
+            sessionsSent = result.sessionsSent;
+            // A 401 (bad credentials) or 429 (server cap) stop must hold the watermark
+            // too (fix 1): the loop abandoned the remaining sessions.
+            hadTransientFailure = result.hadTransientFailure || result.stopped !== undefined;
         }
     }
-    // Only advance lastRun if every session was processed without a transient
-    // failure. Otherwise failed sessions would be permanently lost.
+    finally {
+        releaseLock();
+    }
+    // Advance lastRun only on a fully clean run (no transient failure and no
+    // terminal stop). An empty scan is clean → advance. Cursors already recorded
+    // whatever succeeded regardless.
     if (!dryRun) {
         if (hadTransientFailure) {
-            console.warn(`[hicortex] Not advancing lastRun — one or more sessions failed. ` +
-                `They will be retried on the next run.`);
+            console.warn(`[hicortex] Not advancing lastRun — capture failed or was stopped. ` +
+                `Will retry on the next run.`);
         }
         else {
-            writeLastRun();
+            writeLastRun(stateDir);
+            const pruned = (0, capture_cursors_js_1.pruneCursors)(stateDir);
+            if (pruned > 0)
+                console.log(`[hicortex] Pruned ${pruned} stale capture cursor(s)`);
         }
     }
     console.log(`[hicortex] Client nightly complete: ${memoriesIngested} memories from ${sessionsSent} sessions → ${serverUrl}`);
@@ -495,7 +543,7 @@ async function runClientNightly(config, dryRun) {
         ].filter(Boolean);
         const agentType = kinds.length > 1 ? "mixed" : (kinds[0] ?? "cc");
         await (0, telemetry_js_1.sendTelemetry)({
-            id: (0, telemetry_js_1.getTelemetryId)(HICORTEX_HOME),
+            id: (0, telemetry_js_1.getTelemetryId)(stateDir),
             v: VERSION,
             mode: "client",
             agent: agentType,

package/dist/oc-transcript-reader.d.ts

@@ -10,11 +10,12 @@
  * Known limitation: rotated files (`*.jsonl.reset.<ts>`) are not read — only
  * live `*.jsonl` files. Server-side session dedup keeps re-reads idempotent.
  */
-import { type TranscriptBatch } from "./pi-transcript-reader.js";
+import { type TranscriptBatch, type CursorMap } from "./pi-transcript-reader.js";
 /**
  * Read OpenClaw session transcripts modified after `since`.
  *
  * @param since Only return sessions with mtime > this date
  * @param agentsDir Override the OC agents directory (default: ~/.openclaw/agents/)
+ * @param cursors Per-session capture cursors (#189); keyed `oc:<agentId>:<sid>`
  */
-export declare function readOcTranscripts(since: Date, agentsDir?: string): TranscriptBatch[];
+export declare function readOcTranscripts(since: Date, agentsDir?: string, cursors?: CursorMap): TranscriptBatch[];

package/dist/oc-transcript-reader.js

@@ -23,8 +23,9 @@ const DEFAULT_OC_AGENTS_DIR = (0, node_path_1.join)((0, node_os_1.homedir)(), ".
  *
  * @param since Only return sessions with mtime > this date
  * @param agentsDir Override the OC agents directory (default: ~/.openclaw/agents/)
+ * @param cursors Per-session capture cursors (#189); keyed `oc:<agentId>:<sid>`
  */
-function readOcTranscripts(since, agentsDir = DEFAULT_OC_AGENTS_DIR) {
+function readOcTranscripts(since, agentsDir = DEFAULT_OC_AGENTS_DIR, cursors = {}) {
     let agentIds;
     try {
         agentIds = (0, node_fs_1.readdirSync)(agentsDir);
@@ -44,8 +45,9 @@ function readOcTranscripts(since, agentsDir = DEFAULT_OC_AGENTS_DIR) {
             continue;
         }
         // agents/<agentId>/ contains a `sessions/` child with *.jsonl — exactly
-        // the <root>/<projectDir>/*.jsonl shape readPiTranscripts walks.
-        for (const batch of (0, pi_transcript_reader_js_1.readPiTranscripts)(since, agentPath)) {
+        // the <root>/<projectDir>/*.jsonl shape readPiTranscripts walks. The
+        // key prefix namespaces cursors per agent (oc:<agentId>:<sessionId>).
+        for (const batch of (0, pi_transcript_reader_js_1.readPiTranscripts)(since, agentPath, cursors, `oc:${agentId}`)) {
             batches.push({
                 ...batch,
                 // The Pi walk labels the project from the cwd or the "sessions" dir

package/dist/pi-transcript-reader.d.ts

@@ -26,13 +26,8 @@
  * becomes --home-agents-Agents-raider--. The session header's `cwd` field
  * is the canonical path; the directory name is a filesystem-safe encoding.
  */
-export interface TranscriptBatch {
-    sessionId: string;
-    projectName: string;
-    date: string;
-    entries: unknown[];
-    sourceAgent?: string;
-}
+import type { TranscriptBatch, CursorMap } from "./transcript-reader.js";
+export type { TranscriptBatch, CursorMap };
 /**
  * Read Pi session transcripts modified after `since`.
  *
@@ -41,5 +36,7 @@ export interface TranscriptBatch {
  *
  * @param since Only return sessions with mtime > this date
  * @param sessionsDir Override the session directory (default: ~/.pi/agent/sessions/)
+ * @param cursors Per-session capture cursors (#189); default empty = whole file
+ * @param keyPrefix Cursor-key namespace ("pi" here; OC passes "oc:<agentId>")
  */
-export declare function readPiTranscripts(since: Date, sessionsDir?: string): TranscriptBatch[];
+export declare function readPiTranscripts(since: Date, sessionsDir?: string, cursors?: CursorMap, keyPrefix?: string): TranscriptBatch[];

package/dist/pi-transcript-reader.js

@@ -41,8 +41,10 @@ const DEFAULT_PI_SESSIONS_DIR = (0, node_path_1.join)((0, node_os_1.homedir)(),
  *
  * @param since Only return sessions with mtime > this date
  * @param sessionsDir Override the session directory (default: ~/.pi/agent/sessions/)
+ * @param cursors Per-session capture cursors (#189); default empty = whole file
+ * @param keyPrefix Cursor-key namespace ("pi" here; OC passes "oc:<agentId>")
  */
-function readPiTranscripts(since, sessionsDir = DEFAULT_PI_SESSIONS_DIR) {
+function readPiTranscripts(since, sessionsDir = DEFAULT_PI_SESSIONS_DIR, cursors = {}, keyPrefix = "pi") {
     const batches = [];
     let projectDirs;
     try {
@@ -80,6 +82,7 @@ function readPiTranscripts(since, sessionsDir = DEFAULT_PI_SESSIONS_DIR) {
                 const raw = (0, node_fs_1.readFileSync)(filePath, "utf-8");
                 const lines = raw.split("\n").filter((l) => l.trim());
                 const entries = [];
+                const timestamps = [];
                 let sessionId = "";
                 let sessionCwd = "";
                 let sessionDate = "";
@@ -87,6 +90,7 @@ function readPiTranscripts(since, sessionsDir = DEFAULT_PI_SESSIONS_DIR) {
                     try {
                         const entry = JSON.parse(line);
                         entries.push(entry);
+                        timestamps.push(typeof entry.timestamp === "string" ? entry.timestamp : "");
                         // Extract metadata from the session header
                         if (entry.type === "session") {
                             sessionId = entry.id ?? "";
@@ -111,14 +115,38 @@ function readPiTranscripts(since, sessionsDir = DEFAULT_PI_SESSIONS_DIR) {
                 if (!sessionDate) {
                     sessionDate = extractDateFromFilename(file) ?? "";
                 }
-                if (entries.length > 0) {
-                    batches.push({
-                        sessionId,
-                        projectName,
-                        date: sessionDate,
-                        entries,
-                    });
+                if (entries.length === 0)
+                    continue;
+                // Incremental slice (#189): append-only JSONL v3, same discipline as CC.
+                const cursorKey = `${keyPrefix}:${sessionId}`;
+                const pos = cursors[cursorKey] ?? { cursor: 0, gen: 0 };
+                let start = pos.cursor;
+                let gen = pos.gen;
+                if (start > entries.length) {
+                    // shrink guard (truncation/rotation) — reset + bump generation (fix 8)
+                    start = 0;
+                    gen = pos.gen + 1;
+                }
+                const delta = entries.slice(start);
+                if (delta.length === 0)
+                    continue; // cursor already covers the file
+                const entryCursors = delta.map((_, i) => start + i + 1);
+                // Prefer the last timestamped entry in the delta for per-night dating.
+                let deltaDate = "";
+                for (let i = start; i < entries.length; i++) {
+                    if (timestamps[i])
+                        deltaDate = timestamps[i].slice(0, 10);
                 }
+                batches.push({
+                    sessionId,
+                    projectName,
+                    date: deltaDate || sessionDate || new Date().toISOString().slice(0, 10),
+                    entries: delta,
+                    cursorKey,
+                    startCursor: start,
+                    generation: gen,
+                    entryCursors,
+                });
             }
             catch {
                 // File read or parse failed — skip