@gamaze/hicortex 0.13.1 → 0.13.3

package/dist/mcp-server.js

@@ -681,11 +681,27 @@ async function startServer(options = {}) {
             res.status(400).json({ error: "Provide either 'text' (string) or 'messages' (array)" });
             return;
         }
+        // Escape LIKE wildcards — Hermes ids contain "_" (e.g. 20260701_045744_...).
+        const escapeLike = (s) => s.replace(/[\\%_]/g, (m) => "\\" + m);
+        // Segment-exact dedup (#189): an incremental capture POST carries
+        // segment_id "<start>-<end>[.pN]". Skip iff THIS exact segment's chunks are
+        // already stored (keys "<sid>#<segment_id>#<i>"). This is what lets a failed
+        // segment be safely retried with the same id, and a legacy session-level row
+        // (key "<sid>#<i>", no segment) does NOT match — so the #189 recovery
+        // re-ingest is never blocked by night-1's whole-session rows.
+        if (session_id && segment_id) {
+            const likePrefix = `${escapeLike(session_id)}#${escapeLike(segment_id)}#%`;
+            const existing = db.prepare("SELECT COUNT(*) as c FROM memories WHERE source_session LIKE ? ESCAPE '\\'").get(likePrefix);
+            if (existing.c > 0) {
+                res.status(200).json({ skipped: true, existing_count: existing.c });
+                return;
+            }
+        }
         // Session-level dedup: when session_id is present and this is a whole-session
-        // POST (no segment_id), skip if any chunk of this session is already stored.
+        // POST (no segment_id — legacy ≤0.13.1 clients), skip if any chunk of this
+        // session is already stored. Unchanged: legacy clients keep exact behaviour.
         if (session_id && !segment_id) {
-            // Escape LIKE wildcards — Hermes ids contain "_" (e.g. 20260701_045744_...).
-            const likePrefix = `${session_id.replace(/[\\%_]/g, (m) => "\\" + m)}#%`;
+            const likePrefix = `${escapeLike(session_id)}#%`;
             const existing = db.prepare("SELECT COUNT(*) as c FROM memories WHERE source_session = ? OR source_session LIKE ? ESCAPE '\\'").get(session_id, likePrefix);
             if (existing.c > 0) {
                 res.status(200).json({ skipped: true, existing_count: existing.c });
@@ -722,24 +738,38 @@ async function startServer(options = {}) {
             // server-side per-entry console.log in distillChunk stays as well.
             const dropped = [];
             const entries = await (0, distiller_js_1.distillSession)(llm, conversationText, project ?? "unknown", date, chunkSize, dropped);
-            const ids = [];
+            // Phase 1 — embed every chunk up front (async). If ANY embed fails we
+            // never reach the insert, so nothing is stored.
+            const createdAt = new Date(date).toISOString();
+            const toStore = [];
             for (let i = 0; i < entries.length; i++) {
                 const entry = entries[i];
                 if (typeof entry !== "string" || !entry.trim())
                     continue;
-                const embedding = await (0, embedder_js_1.embed)(entry);
-                const id = storage.insertMemory(db, entry, embedding, {
-                    sourceAgent: source_agent ?? "unknown",
-                    // Per-chunk key: "<session_id>#<i>". The prefix matches the nightly
-                    // dedup check above, so a re-run of the same session is fully idempotent.
-                    sourceSession: sourcePrefix ? `${sourcePrefix}#${i}` : undefined,
-                    project: project ?? undefined,
-                    memoryType: "episode",
-                    privacy: privacy ?? "WORK",
-                    createdAt: new Date(date).toISOString(),
-                });
-                ids.push(id);
+                toStore.push({ entry, embedding: await (0, embedder_js_1.embed)(entry), i });
             }
+            // Phase 2 — insert all chunks in ONE transaction (fix 4). A segment's
+            // chunks are all-or-nothing: any insert failure rolls back the whole set
+            // and returns 500, so the content-blind segment-exact dedup never sees a
+            // half-stored segment and the retry re-distills cleanly. (Applies to the
+            // legacy whole-session path too — same loop.)
+            const insertAll = db.transaction(() => {
+                const out = [];
+                for (const { entry, embedding, i } of toStore) {
+                    out.push(storage.insertMemory(db, entry, embedding, {
+                        sourceAgent: source_agent ?? "unknown",
+                        // Per-chunk key: "<session_id>[#<segment_id>]#<i>". The prefix
+                        // matches the dedup checks above, so a re-run is idempotent.
+                        sourceSession: sourcePrefix ? `${sourcePrefix}#${i}` : undefined,
+                        project: project ?? undefined,
+                        memoryType: "episode",
+                        privacy: privacy ?? "WORK",
+                        createdAt,
+                    }));
+                }
+                return out;
+            });
+            const ids = insertAll();
             res.status(201).json({
                 ids,
                 distilled: ids.length,

package/dist/nightly.d.ts

@@ -15,4 +15,6 @@ export declare function runNightly(options?: {
     captureOnly?: boolean;
     dbPath?: string;
     stateDir?: string;
+    /** #189 Tier-2 recovery: override discovery to now−N days for one run. */
+    recaptureWindowDays?: number;
 }): Promise<void>;

package/dist/nightly.js

@@ -58,7 +58,6 @@ const db_js_1 = require("./db.js");
 const llm_js_1 = require("./llm.js");
 const embedder_js_1 = require("./embedder.js");
 const storage = __importStar(require("./storage.js"));
-const distiller_js_1 = require("./distiller.js");
 const consolidate_js_1 = require("./consolidate.js");
 const domain_classify_js_1 = require("./domain-classify.js");
 const nofit_js_1 = require("./nofit.js");
@@ -68,6 +67,8 @@ const pi_transcript_reader_js_1 = require("./pi-transcript-reader.js");
 const oc_transcript_reader_js_1 = require("./oc-transcript-reader.js");
 const features_js_1 = require("./features.js");
 const state_js_1 = require("./state.js");
+const capture_cursors_js_1 = require("./capture-cursors.js");
+const capture_js_1 = require("./capture.js");
 const telemetry_js_1 = require("./telemetry.js");
 const HICORTEX_HOME = (0, paths_js_1.hicortexHome)();
 function readNightlyConfig(stateDir) {
@@ -97,12 +98,81 @@ function readLastRun(stateDir = HICORTEX_HOME) {
     const d = new Date(ts);
     return isNaN(d.getTime()) ? new Date(0) : d;
 }
+/**
+ * Discovery watermark. Normally the last-nightly timestamp; with
+ * `--recapture-window <days>` (#189 Tier-2 recovery) the window may only
+ * WIDEN — since = min(lastNightly, now−N days). Taking the earlier of the two
+ * means a machine that was offline longer than N days still re-discovers every
+ * session it missed; using now−N unconditionally would NARROW the window and
+ * skip (then, via writeLastRun, permanently lose) the 8-to-N-day-old sessions
+ * (#189 review, fix 3). Per-session cursors keep the wide re-scan cheap: an
+ * already-captured session yields an empty delta.
+ */
+function computeSince(stateDir, recaptureWindowDays) {
+    const lastRun = readLastRun(stateDir);
+    if (recaptureWindowDays && recaptureWindowDays > 0) {
+        const windowStart = new Date(Date.now() - recaptureWindowDays * 24 * 60 * 60 * 1000);
+        return windowStart < lastRun ? windowStart : lastRun;
+    }
+    return lastRun;
+}
+/** POST /distill transport for server mode — localhost, no auth (localhost bypasses). */
+function makeLocalPost(port) {
+    return async (body) => {
+        const resp = await fetch(`http://127.0.0.1:${port}/distill`, {
+            method: "POST",
+            headers: { "Content-Type": "application/json" },
+            body: JSON.stringify(body),
+            // Synchronous 35B distillation of a large segment can take minutes.
+            signal: AbortSignal.timeout(20 * 60 * 1000),
+        });
+        return normalizePostResult(resp);
+    };
+}
+/** POST /distill transport for client mode — remote URL + optional bearer token. */
+function makeRemotePost(serverUrl, authToken) {
+    return async (body) => {
+        const resp = await fetch(`${serverUrl}/distill`, {
+            method: "POST",
+            headers: {
+                "Content-Type": "application/json",
+                ...(authToken ? { Authorization: `Bearer ${authToken}` } : {}),
+            },
+            body: JSON.stringify(body),
+            signal: AbortSignal.timeout(20 * 60 * 1000),
+        });
+        return normalizePostResult(resp);
+    };
+}
+async function normalizePostResult(resp) {
+    if (resp.status === 201) {
+        const data = (await resp.json().catch(() => ({})));
+        return { status: 201, distilled: data.distilled ?? 0, dropped: data.dropped ?? [] };
+    }
+    if (resp.status === 200) {
+        const data = (await resp.json().catch(() => ({})));
+        return { status: 200, skipped: Boolean(data.skipped) };
+    }
+    const data = (await resp.json().catch(() => ({})));
+    return { status: resp.status, error: data.error ?? "unknown error" };
+}
 function writeLastRun(stateDir = HICORTEX_HOME) {
     (0, state_js_1.updateState)((s) => {
         s.lastNightly = new Date().toISOString();
         return s;
     }, stateDir);
 }
+/**
+ * How long a full nightly waits for the capture lock before giving up and
+ * running consolidation without capturing (fix 10). Comfortably longer than a
+ * typical daytime --capture-only run; short enough not to stall the box.
+ * Overridable via HICORTEX_CAPTURE_LOCK_WAIT_MS (tests only).
+ */
+const CAPTURE_LOCK_WAIT_MS = 30 * 60 * 1000;
+function captureLockWaitMs() {
+    const env = Number(process.env.HICORTEX_CAPTURE_LOCK_WAIT_MS);
+    return Number.isFinite(env) && env >= 0 ? env : CAPTURE_LOCK_WAIT_MS;
+}
 const NIGHTLY_LOG_MAX_BYTES = 1024 * 1024; // 1 MB — years of normal runs
 /**
  * Keep ~/.hicortex/nightly.log bounded. The launchd plist and systemd unit
@@ -125,6 +195,7 @@ async function runNightly(options = {}) {
     const dryRun = options.dryRun ?? false;
     const captureOnly = options.captureOnly ?? false;
     const stateDir = options.stateDir ?? HICORTEX_HOME;
+    const recaptureWindowDays = options.recaptureWindowDays;
     rotateNightlyLog(stateDir);
     // One-time migration of legacy state files (no-op if state.json exists)
     (0, state_js_1.migrateLegacyState)(stateDir);
@@ -133,7 +204,7 @@ async function runNightly(options = {}) {
     if (savedConfig?.mode === "client") {
         // --capture-only is accepted in client mode but irrelevant: client nightly
         // is already capture-only (no consolidation step).
-        await runClientNightly(savedConfig, dryRun);
+        await runClientNightly(savedConfig, dryRun, stateDir, recaptureWindowDays);
         return;
     }
     const dbPath = (0, db_js_1.resolveDbPath)(options.dbPath);
@@ -160,97 +231,99 @@ async function runNightly(options = {}) {
         }
         const llmConfig = resolved.config;
         const llm = llmConfig ? new llm_js_1.LlmClient(llmConfig) : null;
-        // Step 1: Read new transcripts (CC + Hermes + Pi + OpenClaw)
-        const since = readLastRun();
-        console.log(`[hicortex] Reading transcripts since ${since.toISOString()}`);
-        const ccBatches = (0, transcript_reader_js_1.readCcTranscripts)(since);
-        const hermesBatches = (0, hermes_transcript_reader_js_1.readHermesSessions)(since);
-        // Pi is a supported harness in the product (readPiTranscripts no-ops when
-        // ~/.pi/agent/sessions is absent). Retired only on specific deployments by
-        // simply having no Pi session files — not removed from the pipeline.
-        const piBatches = (0, pi_transcript_reader_js_1.readPiTranscripts)(since);
-        // OpenClaw persists sessions in the Pi v3 format at ~/.openclaw/agents/;
-        // no-ops when OC isn't installed.
-        const ocBatches = (0, oc_transcript_reader_js_1.readOcTranscripts)(since);
-        const batches = [...ccBatches, ...hermesBatches, ...piBatches, ...ocBatches];
-        if (ccBatches.length > 0)
-            console.log(`[hicortex] Found ${ccBatches.length} CC session(s)`);
-        if (hermesBatches.length > 0)
-            console.log(`[hicortex] Found ${hermesBatches.length} Hermes session(s)`);
-        if (piBatches.length > 0)
-            console.log(`[hicortex] Found ${piBatches.length} Pi session(s)`);
-        if (ocBatches.length > 0)
-            console.log(`[hicortex] Found ${ocBatches.length} OpenClaw session(s)`);
-        console.log(`[hicortex] Total: ${batches.length} new session(s)`);
-        if (batches.length === 0 && !dryRun) {
-            // Still run consolidation (unless capture-only) — there may be unscored memories from OC.
-            console.log(captureOnly
-                ? `[hicortex] No new transcripts. Nothing to capture.`
-                : `[hicortex] No new transcripts. Running consolidation only.`);
-        }
-        // Step 2: Denoise and POST each session to the local daemon via /distill.
-        // The dedup check and distillation quality (35B) are the server's concern.
+        // Steps 1+2: single-flight capture. The lock guards ONLY the capture phase
+        // (fix 10). We acquire it BEFORE reading the cursor store so a run that
+        // waited out another cannot act on a stale snapshot and clobber its cursor
+        // advances (fix 6). Batch-kind counts are hoisted for the telemetry at the
+        // end (they stay 0 if capture is skipped).
+        let ccBatches = [];
+        let hermesBatches = [];
+        let piBatches = [];
+        let ocBatches = [];
+        let batches = [];
         let memoriesIngested = 0;
         let hadTransientFailure = false;
-        for (const batch of batches) {
-            const transcript = (0, distiller_js_1.extractConversationText)(batch.entries);
-            if (transcript.length < 200) {
-                console.log(`[hicortex]   Skip ${batch.sessionId.slice(0, 8)} (${batch.projectName}): too short`);
-                continue;
-            }
-            console.log(`[hicortex]   Capturing ${batch.sessionId.slice(0, 8)} (${batch.projectName}, ${batch.date})`);
-            if (dryRun) {
-                console.log(`[hicortex]     [dry-run] Would POST ${transcript.length} chars to /distill`);
-                continue;
+        // Full nightly waits out a transient --capture-only overlap (each segment
+        // POST can block up to 20 min); capture-only fails fast. dry-run writes
+        // nothing so it needs no lock.
+        const lockWaitMs = captureLockWaitMs();
+        const releaseLock = dryRun
+            ? (() => { })
+            : await (0, capture_js_1.acquireCaptureLock)(stateDir, captureOnly ? 0 : lockWaitMs);
+        if (!releaseLock) {
+            if (captureOnly) {
+                console.warn("[hicortex] Another capture run holds the lock — skipping this capture-only run.");
+                return;
             }
+            // Full nightly couldn't get the lock even after waiting: skip capture and
+            // hold the watermark, but STILL run consolidation + telemetry below so an
+            // overlapping capture-only run never silently starves consolidation (fix 10).
+            console.warn(`[hicortex] Capture lock still held after waiting ${Math.round(lockWaitMs / 60000)} min — ` +
+                `skipping capture this run (watermark held), consolidation still runs.`);
+            hadTransientFailure = true;
+        }
+        else {
             try {
-                const resp = await fetch(`http://127.0.0.1:${port}/distill`, {
-                    method: "POST",
-                    headers: { "Content-Type": "application/json" },
-                    body: JSON.stringify({
-                        text: transcript,
-                        source_agent: batch.sourceAgent ?? `claude-code/${batch.projectName}`,
-                        project: batch.projectName,
-                        session_id: batch.sessionId,
-                        session_date: batch.date,
-                        privacy: "WORK",
-                    }),
-                    // Synchronous 35B distillation of a large session can take minutes.
-                    signal: AbortSignal.timeout(20 * 60 * 1000),
-                });
-                if (resp.status === 200) {
-                    const data = await resp.json();
-                    if (data.skipped) {
-                        console.log(`[hicortex]   Skip ${batch.sessionId.slice(0, 8)} (${batch.projectName}): already ingested`);
-                    }
-                }
-                else if (resp.status === 201) {
-                    const data = await resp.json();
-                    memoriesIngested += data.distilled ?? 0;
-                    console.log(`[hicortex]     → ${data.distilled ?? 0} memories extracted`);
-                    // Durable audit trail (#156): the server truncates each dropped entry.
-                    for (const d of data.dropped ?? []) {
-                        console.log(`[hicortex]     Substance gate: dropped "${d}"`);
-                    }
-                }
-                else if (resp.status === 429) {
-                    const data = await resp.json();
-                    console.log(`[hicortex]   Memory limit reached: ${data.error}. Stopping capture.`);
-                    break;
+                // Step 1: Read new transcripts (CC + Hermes + Pi + OpenClaw). Discovery
+                // is whole-session by mtime/ended_at; per-session cursors slice each
+                // discovered session down to its unseen delta (#189).
+                const since = computeSince(stateDir, recaptureWindowDays);
+                if (recaptureWindowDays) {
+                    console.log(`[hicortex] --recapture-window ${recaptureWindowDays}d: reading transcripts since ${since.toISOString()}`);
                 }
                 else {
-                    const data = await resp.json().catch(() => ({}));
-                    console.error(`[hicortex]     /distill returned ${resp.status}: ${data.error ?? "unknown error"} — will retry next run`);
-                    hadTransientFailure = true;
+                    console.log(`[hicortex] Reading transcripts since ${since.toISOString()}`);
+                }
+                const cursorStore = (0, capture_cursors_js_1.openCursorStore)(stateDir);
+                const cursorMap = cursorStore.map();
+                ccBatches = (0, transcript_reader_js_1.readCcTranscripts)(since, undefined, cursorMap);
+                hermesBatches = (0, hermes_transcript_reader_js_1.readHermesSessions)(since, undefined, cursorMap);
+                // Pi is a supported harness (readPiTranscripts no-ops when
+                // ~/.pi/agent/sessions is absent). Retired only on specific deployments
+                // by simply having no Pi session files — not removed from the pipeline.
+                piBatches = (0, pi_transcript_reader_js_1.readPiTranscripts)(since, undefined, cursorMap);
+                // OpenClaw persists sessions in the Pi v3 format at ~/.openclaw/agents/;
+                // no-ops when OC isn't installed.
+                ocBatches = (0, oc_transcript_reader_js_1.readOcTranscripts)(since, undefined, cursorMap);
+                batches = [...ccBatches, ...hermesBatches, ...piBatches, ...ocBatches];
+                if (ccBatches.length > 0)
+                    console.log(`[hicortex] Found ${ccBatches.length} CC session(s)`);
+                if (hermesBatches.length > 0)
+                    console.log(`[hicortex] Found ${hermesBatches.length} Hermes session(s)`);
+                if (piBatches.length > 0)
+                    console.log(`[hicortex] Found ${piBatches.length} Pi session(s)`);
+                if (ocBatches.length > 0)
+                    console.log(`[hicortex] Found ${ocBatches.length} OpenClaw session(s)`);
+                console.log(`[hicortex] Total: ${batches.length} new session(s)`);
+                if (batches.length === 0 && !dryRun) {
+                    console.log(captureOnly
+                        ? `[hicortex] No new transcripts. Nothing to capture.`
+                        : `[hicortex] No new transcripts. Running consolidation only.`);
                 }
+                // Step 2: pack each session's delta into ≤60K segments and POST to the
+                // local daemon via /distill; cursors advance on confirmed success.
+                const result = await (0, capture_js_1.captureBatches)(batches, {
+                    post: makeLocalPost(port),
+                    cursorStore,
+                    dryRun,
+                });
+                memoriesIngested = result.memoriesIngested;
+                // A 429/401 stop must hold the watermark too (fix 1): the loop abandoned
+                // the remaining sessions, and mtime discovery would never re-find them.
+                hadTransientFailure = result.hadTransientFailure || result.stopped !== undefined;
+            }
+            finally {
+                releaseLock();
             }
-            catch (err) {
-                const msg = err instanceof Error ? err.message : String(err);
-                console.error(`[hicortex]     Capture failed: ${msg} — will retry next run`);
-                hadTransientFailure = true;
+            console.log(`[hicortex] Capture complete: ${memoriesIngested} new memories`);
+            // Prune aged-out cursors (90d) — only on a clean run so a transient
+            // failure doesn't drop a still-needed cursor.
+            if (!dryRun && !hadTransientFailure) {
+                const pruned = (0, capture_cursors_js_1.pruneCursors)(stateDir);
+                if (pruned > 0)
+                    console.log(`[hicortex] Pruned ${pruned} stale capture cursor(s)`);
             }
         }
-        console.log(`[hicortex] Capture complete: ${memoriesIngested} new memories`);
         // Step 3: Consolidation — skipped in capture-only mode, dry-run, or no LLM.
         // Runs even if capture had transient failures (opens DB directly, independent
         // of the HTTP capture path). Full nightly only — capture-only runs are
@@ -327,7 +400,7 @@ async function runNightly(options = {}) {
                     `They will be retried on the next run.`);
             }
             else {
-                writeLastRun();
+                writeLastRun(stateDir);
             }
         }
         console.log(`[hicortex] Nightly pipeline complete.`);
@@ -360,7 +433,7 @@ async function runNightly(options = {}) {
 // ---------------------------------------------------------------------------
 // Client Mode Nightly — denoise locally, POST to remote server's /distill
 // ---------------------------------------------------------------------------
-async function runClientNightly(config, dryRun) {
+async function runClientNightly(config, dryRun, stateDir = HICORTEX_HOME, recaptureWindowDays) {
     const serverUrl = config.serverUrl.replace(/\/+$/, "");
     const authToken = config.authToken;
     console.log(`[hicortex] Client nightly starting${dryRun ? " (dry run)" : ""}`);
@@ -379,109 +452,84 @@ async function runClientNightly(config, dryRun) {
         return; // Don't update last-run so we retry
     }
     // No local LLM needed — distillation happens on the server.
-    // Read new transcripts (CC + Hermes + Pi + OpenClaw). Client reads local
-    // logs, denoises, and POSTs the denoised text to the server's /distill
-    // endpoint. All readers no-op when their harness isn't installed.
-    const since = readLastRun();
-    console.log(`[hicortex] Reading transcripts since ${since.toISOString()}`);
-    const ccBatches = (0, transcript_reader_js_1.readCcTranscripts)(since);
-    const hermesBatches = (0, hermes_transcript_reader_js_1.readHermesSessions)(since);
-    const piBatches = (0, pi_transcript_reader_js_1.readPiTranscripts)(since);
-    const ocBatches = (0, oc_transcript_reader_js_1.readOcTranscripts)(since);
-    const batches = [...ccBatches, ...hermesBatches, ...piBatches, ...ocBatches];
-    if (ccBatches.length > 0)
-        console.log(`[hicortex] Found ${ccBatches.length} CC session(s)`);
-    if (hermesBatches.length > 0)
-        console.log(`[hicortex] Found ${hermesBatches.length} Hermes session(s)`);
-    if (piBatches.length > 0)
-        console.log(`[hicortex] Found ${piBatches.length} Pi session(s)`);
-    if (ocBatches.length > 0)
-        console.log(`[hicortex] Found ${ocBatches.length} OpenClaw session(s)`);
-    console.log(`[hicortex] Total: ${batches.length} new session(s)`);
-    if (batches.length === 0) {
-        console.log(`[hicortex] Nothing to capture.`);
-        if (!dryRun)
-            writeLastRun();
+    // Single-flight (A5): acquire the capture lock BEFORE reading the cursor
+    // store so a run that waited out another can't act on a stale snapshot and
+    // clobber its advances (fix 6). The client is capture-only; on contention we
+    // skip and hold the watermark (no writeLastRun → retried next run). dry-run
+    // writes nothing so it needs no lock.
+    const releaseLock = dryRun ? (() => { }) : await (0, capture_js_1.acquireCaptureLock)(stateDir);
+    if (!releaseLock) {
+        console.warn("[hicortex] Another capture run holds the lock — skipping this run (watermark held).");
         return;
     }
-    let hadTransientFailure = false;
+    let ccBatches = [];
+    let hermesBatches = [];
+    let piBatches = [];
+    let ocBatches = [];
+    let batches = [];
     let memoriesIngested = 0;
     let sessionsSent = 0;
-    for (const batch of batches) {
-        const transcript = (0, distiller_js_1.extractConversationText)(batch.entries);
-        if (transcript.length < 200) {
-            console.log(`[hicortex]   Skip ${batch.sessionId.slice(0, 8)} (${batch.projectName}): too short`);
-            continue;
+    let hadTransientFailure = false;
+    try {
+        // Read new transcripts (CC + Hermes + Pi + OpenClaw). Client reads local
+        // logs, denoises, and POSTs the denoised text to the server's /distill
+        // endpoint. All readers no-op when their harness isn't installed. Per-session
+        // cursors slice each discovered session to its unseen delta (#189).
+        const since = computeSince(stateDir, recaptureWindowDays);
+        if (recaptureWindowDays) {
+            console.log(`[hicortex] --recapture-window ${recaptureWindowDays}d: reading transcripts since ${since.toISOString()}`);
         }
-        console.log(`[hicortex]   Capturing ${batch.sessionId.slice(0, 8)} (${batch.projectName}, ${batch.date})`);
-        if (dryRun) {
-            console.log(`[hicortex]     [dry-run] Would POST ${transcript.length} chars to ${serverUrl}/distill`);
-            continue;
+        else {
+            console.log(`[hicortex] Reading transcripts since ${since.toISOString()}`);
         }
-        try {
-            const resp = await fetch(`${serverUrl}/distill`, {
-                method: "POST",
-                headers: {
-                    "Content-Type": "application/json",
-                    ...(authToken ? { "Authorization": `Bearer ${authToken}` } : {}),
-                },
-                body: JSON.stringify({
-                    text: transcript,
-                    source_agent: batch.sourceAgent ?? `claude-code/${batch.projectName}`,
-                    project: batch.projectName,
-                    session_id: batch.sessionId,
-                    session_date: batch.date,
-                    privacy: "WORK",
-                }),
-                // Synchronous 35B distillation of a large session can take minutes.
-                signal: AbortSignal.timeout(20 * 60 * 1000),
-            });
-            if (resp.status === 200) {
-                const data = await resp.json();
-                if (data.skipped) {
-                    console.log(`[hicortex]   Skip ${batch.sessionId.slice(0, 8)}: already ingested on server`);
-                }
-            }
-            else if (resp.status === 201) {
-                const data = await resp.json();
-                const count = data.distilled ?? 0;
-                memoriesIngested += count;
-                sessionsSent++;
-                console.log(`[hicortex]     → ${count} memories sent to server`);
-                // Durable audit trail (#156): the server truncates each dropped entry.
-                for (const d of data.dropped ?? []) {
-                    console.log(`[hicortex]     Substance gate: dropped "${d}"`);
-                }
-            }
-            else if (resp.status === 401) {
-                console.error(`[hicortex]     Auth failed. Check authToken in ~/.hicortex/config.json`);
-                return; // No point retrying with wrong credentials
-            }
-            else if (resp.status === 429) {
-                const data = await resp.json().catch(() => ({}));
-                console.log(`[hicortex]   Server memory limit reached: ${data.error}`);
-                return;
-            }
-            else {
-                const data = await resp.json().catch(() => ({}));
-                console.error(`[hicortex]     /distill returned ${resp.status}: ${data.error ?? "unknown error"} — will retry next run`);
-                hadTransientFailure = true;
-            }
+        const cursorStore = (0, capture_cursors_js_1.openCursorStore)(stateDir);
+        const cursorMap = cursorStore.map();
+        ccBatches = (0, transcript_reader_js_1.readCcTranscripts)(since, undefined, cursorMap);
+        hermesBatches = (0, hermes_transcript_reader_js_1.readHermesSessions)(since, undefined, cursorMap);
+        piBatches = (0, pi_transcript_reader_js_1.readPiTranscripts)(since, undefined, cursorMap);
+        ocBatches = (0, oc_transcript_reader_js_1.readOcTranscripts)(since, undefined, cursorMap);
+        batches = [...ccBatches, ...hermesBatches, ...piBatches, ...ocBatches];
+        if (ccBatches.length > 0)
+            console.log(`[hicortex] Found ${ccBatches.length} CC session(s)`);
+        if (hermesBatches.length > 0)
+            console.log(`[hicortex] Found ${hermesBatches.length} Hermes session(s)`);
+        if (piBatches.length > 0)
+            console.log(`[hicortex] Found ${piBatches.length} Pi session(s)`);
+        if (ocBatches.length > 0)
+            console.log(`[hicortex] Found ${ocBatches.length} OpenClaw session(s)`);
+        console.log(`[hicortex] Total: ${batches.length} new session(s)`);
+        if (batches.length === 0) {
+            console.log(`[hicortex] Nothing to capture.`);
         }
-        catch (err) {
-            console.error(`[hicortex]     Capture failed: ${err instanceof Error ? err.message : String(err)} — will retry next run`);
-            hadTransientFailure = true;
+        else {
+            const result = await (0, capture_js_1.captureBatches)(batches, {
+                post: makeRemotePost(serverUrl, authToken),
+                cursorStore,
+                dryRun,
+            });
+            memoriesIngested = result.memoriesIngested;
+            sessionsSent = result.sessionsSent;
+            // A 401 (bad credentials) or 429 (server cap) stop must hold the watermark
+            // too (fix 1): the loop abandoned the remaining sessions.
+            hadTransientFailure = result.hadTransientFailure || result.stopped !== undefined;
         }
     }
-    // Only advance lastRun if every session was processed without a transient
-    // failure. Otherwise failed sessions would be permanently lost.
+    finally {
+        releaseLock();
+    }
+    // Advance lastRun only on a fully clean run (no transient failure and no
+    // terminal stop). An empty scan is clean → advance. Cursors already recorded
+    // whatever succeeded regardless.
     if (!dryRun) {
         if (hadTransientFailure) {
-            console.warn(`[hicortex] Not advancing lastRun — one or more sessions failed. ` +
-                `They will be retried on the next run.`);
+            console.warn(`[hicortex] Not advancing lastRun — capture failed or was stopped. ` +
+                `Will retry on the next run.`);
         }
         else {
-            writeLastRun();
+            writeLastRun(stateDir);
+            const pruned = (0, capture_cursors_js_1.pruneCursors)(stateDir);
+            if (pruned > 0)
+                console.log(`[hicortex] Pruned ${pruned} stale capture cursor(s)`);
         }
     }
     console.log(`[hicortex] Client nightly complete: ${memoriesIngested} memories from ${sessionsSent} sessions → ${serverUrl}`);
@@ -495,7 +543,7 @@ async function runClientNightly(config, dryRun) {
         ].filter(Boolean);
         const agentType = kinds.length > 1 ? "mixed" : (kinds[0] ?? "cc");
         await (0, telemetry_js_1.sendTelemetry)({
-            id: (0, telemetry_js_1.getTelemetryId)(HICORTEX_HOME),
+            id: (0, telemetry_js_1.getTelemetryId)(stateDir),
             v: VERSION,
             mode: "client",
             agent: agentType,

package/dist/oc-transcript-reader.d.ts

@@ -10,11 +10,12 @@
  * Known limitation: rotated files (`*.jsonl.reset.<ts>`) are not read — only
  * live `*.jsonl` files. Server-side session dedup keeps re-reads idempotent.
  */
-import { type TranscriptBatch } from "./pi-transcript-reader.js";
+import { type TranscriptBatch, type CursorMap } from "./pi-transcript-reader.js";
 /**
  * Read OpenClaw session transcripts modified after `since`.
  *
  * @param since Only return sessions with mtime > this date
  * @param agentsDir Override the OC agents directory (default: ~/.openclaw/agents/)
+ * @param cursors Per-session capture cursors (#189); keyed `oc:<agentId>:<sid>`
  */
-export declare function readOcTranscripts(since: Date, agentsDir?: string): TranscriptBatch[];
+export declare function readOcTranscripts(since: Date, agentsDir?: string, cursors?: CursorMap): TranscriptBatch[];

package/dist/oc-transcript-reader.js

@@ -23,8 +23,9 @@ const DEFAULT_OC_AGENTS_DIR = (0, node_path_1.join)((0, node_os_1.homedir)(), ".
  *
  * @param since Only return sessions with mtime > this date
  * @param agentsDir Override the OC agents directory (default: ~/.openclaw/agents/)
+ * @param cursors Per-session capture cursors (#189); keyed `oc:<agentId>:<sid>`
  */
-function readOcTranscripts(since, agentsDir = DEFAULT_OC_AGENTS_DIR) {
+function readOcTranscripts(since, agentsDir = DEFAULT_OC_AGENTS_DIR, cursors = {}) {
     let agentIds;
     try {
         agentIds = (0, node_fs_1.readdirSync)(agentsDir);
@@ -44,8 +45,9 @@ function readOcTranscripts(since, agentsDir = DEFAULT_OC_AGENTS_DIR) {
             continue;
         }
         // agents/<agentId>/ contains a `sessions/` child with *.jsonl — exactly
-        // the <root>/<projectDir>/*.jsonl shape readPiTranscripts walks.
-        for (const batch of (0, pi_transcript_reader_js_1.readPiTranscripts)(since, agentPath)) {
+        // the <root>/<projectDir>/*.jsonl shape readPiTranscripts walks. The
+        // key prefix namespaces cursors per agent (oc:<agentId>:<sessionId>).
+        for (const batch of (0, pi_transcript_reader_js_1.readPiTranscripts)(since, agentPath, cursors, `oc:${agentId}`)) {
             batches.push({
                 ...batch,
                 // The Pi walk labels the project from the cwd or the "sessions" dir