@gamaze/hicortex 0.13.1 → 0.13.2

package/dist/capture.js

@@ -0,0 +1,386 @@
+"use strict";
+/**
+ * Incremental, cursor-aware capture loop (#189).
+ *
+ * Extracted from the two near-identical loops that lived in nightly.ts (server
+ * and client mode). Both now share this logic: pack each session's delta into
+ * ordered segments below the server's distill cap, POST them in order with a
+ * deterministic `segment_id`, and advance the per-session cursor ONLY after
+ * server-confirmed success — so a multi-day session grows across nights with no
+ * loss and no silent truncation.
+ *
+ * The POST transport is injected (`post`) so the mode-specific bits (localhost
+ * vs remote URL, Authorization header, timeout) stay in nightly.ts and the
+ * multi-night simulation can run as a pure unit test with no HTTP listener.
+ */
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.MIN_SEGMENT_CHARS = exports.SEGMENT_MAX_CHARS = void 0;
+exports.hardSplitText = hardSplitText;
+exports.packSegments = packSegments;
+exports.captureBatches = captureBatches;
+exports.acquireCaptureLock = acquireCaptureLock;
+const node_fs_1 = require("node:fs");
+const node_path_1 = require("node:path");
+const distiller_js_1 = require("./distiller.js");
+/**
+ * Max denoised chars per segment. Kept below the server's 80K distill cap
+ * (distiller.ts MAX_TRANSCRIPT_CHARS) with ~20K headroom so NO capture path can
+ * hit the silent truncation. LOAD-BEARING for #189 recovery: a re-ingested
+ * week-long session is re-sliced into ≤60K segments here instead of being
+ * truncated at 80K server-side. (Judgment constant — tunable later.)
+ */
+exports.SEGMENT_MAX_CHARS = 60_000;
+/**
+ * Minimum denoised chars for a FRESH whole session (startCursor 0) to be worth
+ * capturing — mirrors the long-standing pre-#189 200-char degenerate-session
+ * gate. It is applied ONLY to a whole-session capture that denoises to a single
+ * sub-200 segment. A delta beyond cursor 0 is always sent, however small: a
+ * session's concluding tail must never be held back, because once the session
+ * stops growing its mtime never re-crosses the watermark and the tail would be
+ * lost forever (#189 review, fix 5).
+ */
+exports.MIN_SEGMENT_CHARS = 200;
+/** Chars added by the "\n\n" joiner extractConversationText places between entries. */
+const JOINER_CHARS = 2;
+/**
+ * Split an already-denoised string into ≤maxChars pieces (A2 hard-split).
+ * Prefers paragraph, then line, then hard boundaries — mirrors the distiller's
+ * own splitIntoChunks, but WITHOUT its <200-char drop (every piece must survive,
+ * dup-over-loss).
+ */
+function hardSplitText(text, maxChars = exports.SEGMENT_MAX_CHARS) {
+    if (text.length <= maxChars)
+        return [text];
+    const pieces = [];
+    let remaining = text;
+    while (remaining.length > maxChars) {
+        let splitAt = remaining.lastIndexOf("\n\n", maxChars);
+        if (splitAt < maxChars * 0.5)
+            splitAt = remaining.lastIndexOf("\n", maxChars);
+        if (splitAt < maxChars * 0.3)
+            splitAt = maxChars;
+        pieces.push(remaining.slice(0, splitAt).trim());
+        remaining = remaining.slice(splitAt).trim();
+    }
+    if (remaining.length > 0)
+        pieces.push(remaining);
+    return pieces;
+}
+/**
+ * Pack a session's delta entries into ordered ≤maxChars segments.
+ *
+ * Sizing uses per-entry denoise lengths plus the "\n\n" joiners (A8) so the
+ * estimate matches what the server receives; the actual body is a re-denoise of
+ * the grouped entries (extractConversationText) so cleaning/redaction stay
+ * coherent. A single entry larger than maxChars is emitted as its own run of
+ * hard-split pieces (A2).
+ */
+function packSegments(entries, startCursor, entryCursors, maxChars = exports.SEGMENT_MAX_CHARS) {
+    const segments = [];
+    // Boundary cursor before entry i (startCursor for i=0, else entryCursors[i-1]).
+    const boundaryBefore = (i) => (i === 0 ? startCursor : entryCursors[i - 1]);
+    let groupStartIdx = -1;
+    let groupSize = 0;
+    const flushGroup = (endIdxExclusive) => {
+        if (groupStartIdx < 0)
+            return;
+        const groupEntries = entries.slice(groupStartIdx, endIdxExclusive);
+        segments.push({
+            text: (0, distiller_js_1.extractConversationText)(groupEntries),
+            segStart: boundaryBefore(groupStartIdx),
+            segEnd: entryCursors[endIdxExclusive - 1],
+            idSuffix: "",
+        });
+        groupStartIdx = -1;
+        groupSize = 0;
+    };
+    for (let i = 0; i < entries.length; i++) {
+        const entryText = (0, distiller_js_1.extractConversationText)([entries[i]]);
+        const entryLen = entryText.length;
+        if (entryLen > maxChars) {
+            // Oversized single entry — flush the pending group, then hard-split it
+            // into its own segments so no piece can reach the server's 80K cap.
+            // Defensive: extractTextFromContent currently caps a single message at
+            // ~20K denoised chars, so this branch does not fire at the 60K default —
+            // it guarantees the ≤maxChars invariant regardless of the denoiser (A2).
+            flushGroup(i);
+            const pieces = hardSplitText(entryText, maxChars);
+            pieces.forEach((piece, p) => {
+                segments.push({
+                    text: piece,
+                    segStart: boundaryBefore(i),
+                    segEnd: entryCursors[i],
+                    idSuffix: `.p${p}`,
+                });
+            });
+            continue;
+        }
+        // Would adding this entry (plus its joiner) overflow the current group?
+        const addition = (groupSize > 0 ? JOINER_CHARS : 0) + entryLen;
+        if (groupSize > 0 && groupSize + addition > maxChars) {
+            flushGroup(i);
+        }
+        if (groupStartIdx < 0)
+            groupStartIdx = i;
+        groupSize += addition;
+    }
+    flushGroup(entries.length);
+    return segments;
+}
+/**
+ * Capture a list of session delta batches: pack, POST in order, advance
+ * per-session cursors on success. Segments of one session POST in order; the
+ * first hard failure stops THAT session (cursor holds at the last confirmed
+ * boundary) while other sessions continue. A 429/401 stops the whole loop.
+ */
+async function captureBatches(batches, opts) {
+    const { post, cursorStore, dryRun = false, segmentMaxChars = exports.SEGMENT_MAX_CHARS } = opts;
+    let memoriesIngested = 0;
+    let sessionsSent = 0;
+    let hadTransientFailure = false;
+    let stopped;
+    for (const batch of batches) {
+        const short = batch.sessionId.slice(0, 8);
+        const segments = packSegments(batch.entries, batch.startCursor, batch.entryCursors, segmentMaxChars);
+        if (segments.length === 0)
+            continue;
+        // Fresh whole-session degenerate floor (fix 5): only a startCursor-0 capture
+        // that collapses to a single sub-200 segment is dropped (pre-#189 behaviour
+        // for degenerate sessions). Any real delta — including a small concluding
+        // tail — is sent below.
+        if (batch.startCursor === 0 && segments.length === 1 && segments[0].text.length < exports.MIN_SEGMENT_CHARS) {
+            if (!dryRun)
+                console.log(`[hicortex]   Skip ${short} (${batch.projectName}): too short`);
+            continue;
+        }
+        if (dryRun) {
+            const total = segments.reduce((n, s) => n + s.text.length, 0);
+            console.log(`[hicortex]   [dry-run] ${short} (${batch.projectName}): ${segments.length} segment(s), ${total} chars`);
+            continue;
+        }
+        console.log(`[hicortex]   Capturing ${short} (${batch.projectName}, ${batch.date})`);
+        // Segment ids carry the shrink generation so post-reset ids can't collide
+        // with pre-reset ones on the server's content-blind dedup (fix 8). gen 0
+        // has no prefix — keeps ids byte-identical to first-cut and to any already
+        // stored on an older server.
+        const genPrefix = batch.generation > 0 ? `g${batch.generation}.` : "";
+        // Cursor value the last confirmed boundary reached — advanced once, at
+        // session end (A4), so per-session file writes stay bounded.
+        let lastConfirmedEnd = batch.startCursor;
+        let sessionPosted = false;
+        for (let s = 0; s < segments.length; s++) {
+            const seg = segments[s];
+            // A segment advances the cursor to its segEnd only when it is the LAST
+            // segment ending at that boundary. Hard-split pieces (.p0,.p1,…) of one
+            // entry share the same segEnd; confirming an earlier piece must NOT move
+            // the cursor past the entry while a later piece is still unsent (fix 11).
+            const advancesBoundary = s === segments.length - 1 || segments[s + 1].segStart >= seg.segEnd;
+            // Pure-noise slice (all entries filtered to nothing) — nothing to store
+            // and the server rejects an empty body. Advance past it (content is gone
+            // either way) rather than POST.
+            if (seg.text.length === 0) {
+                if (advancesBoundary)
+                    lastConfirmedEnd = seg.segEnd;
+                continue;
+            }
+            const body = {
+                text: seg.text,
+                source_agent: batch.sourceAgent ?? `claude-code/${batch.projectName}`,
+                project: batch.projectName,
+                session_id: batch.sessionId,
+                segment_id: `${genPrefix}${seg.segStart}-${seg.segEnd}${seg.idSuffix}`,
+                session_date: batch.date,
+                privacy: "WORK",
+            };
+            let result;
+            try {
+                result = await post(body);
+            }
+            catch (err) {
+                console.error(`[hicortex]     Capture failed: ${err instanceof Error ? err.message : String(err)} — will retry next run`);
+                hadTransientFailure = true;
+                break;
+            }
+            if (result.status === 201) {
+                memoriesIngested += result.distilled ?? 0;
+                sessionPosted = true;
+                if (advancesBoundary)
+                    lastConfirmedEnd = seg.segEnd;
+                console.log(`[hicortex]     → ${result.distilled ?? 0} memories (segment ${body.segment_id})`);
+                for (const d of result.dropped ?? []) {
+                    console.log(`[hicortex]     Substance gate: dropped "${d}"`);
+                }
+            }
+            else if (result.status === 200) {
+                // Already ingested (segment-exact or legacy session dedup) — treat as
+                // confirmed and advance past it (only at a boundary, per fix 11).
+                if (advancesBoundary)
+                    lastConfirmedEnd = seg.segEnd;
+                if (result.skipped)
+                    console.log(`[hicortex]     Segment ${body.segment_id} already ingested`);
+            }
+            else if (result.status === 429) {
+                console.log(`[hicortex]   Memory limit reached: ${result.error}. Stopping capture.`);
+                stopped = "limit";
+                break;
+            }
+            else if (result.status === 401) {
+                console.error(`[hicortex]     Auth failed. Check authToken in ~/.hicortex/config.json`);
+                stopped = "auth";
+                break;
+            }
+            else {
+                console.error(`[hicortex]     /distill returned ${result.status}: ${result.error ?? "unknown error"} — will retry next run`);
+                hadTransientFailure = true;
+                break;
+            }
+        }
+        if (sessionPosted)
+            sessionsSent++;
+        // Advance the session cursor once, to the last confirmed boundary. Holds at
+        // startCursor when nothing was confirmed (whole delta failed or noise-only).
+        // A mid-session failure only breaks the segment loop above — this still
+        // records the boundaries that DID confirm, and other sessions continue.
+        if (lastConfirmedEnd > batch.startCursor) {
+            try {
+                cursorStore.advance(batch.cursorKey, lastConfirmedEnd, batch.generation);
+            }
+            catch (err) {
+                // Persisting the cursor failed (disk full / perms). Surface it as a
+                // transient failure so the watermark holds and we retry — never a
+                // silent warn-and-continue that re-captures forever (fix 7).
+                console.error(`[hicortex] Failed to persist capture cursor for ${batch.cursorKey}: ${err instanceof Error ? err.message : String(err)} — holding watermark`);
+                hadTransientFailure = true;
+                break;
+            }
+        }
+        if (stopped)
+            break;
+    }
+    return { memoriesIngested, sessionsSent, hadTransientFailure, stopped };
+}
+// ---------------------------------------------------------------------------
+// Single-flight guard (A5)
+// ---------------------------------------------------------------------------
+const LOCK_FILE = "capture.lock";
+/**
+ * A lock older than this is considered stale REGARDLESS of the recorded pid.
+ * Guards the EPERM case: a recycled pid owned by a long-lived root/other-user
+ * process would otherwise read as "alive forever" and wedge capture silently
+ * (#189 review, fix 2). 24h > the longest plausible distill run (20-min POST
+ * timeout × sessions).
+ */
+const LOCK_TTL_MS = 24 * 60 * 60 * 1000;
+const LOCK_POLL_MS = 2000;
+const sleep = (ms) => new Promise((r) => setTimeout(r, ms));
+/**
+ * Acquire an exclusive capture lock for `stateDir`. Returns a release function,
+ * or null if another LIVE, non-stale run holds it after waiting up to `waitMs`.
+ *
+ * Staleness = dead pid OR lockfile mtime older than LOCK_TTL_MS. A stale lock is
+ * reclaimed (with a re-verify + O_EXCL re-race to narrow the TOCTOU window,
+ * fix 12). `waitMs` lets the full nightly wait out a transient `--capture-only`
+ * overlap instead of dropping the night's capture (fix 10); pass 0 to fail fast.
+ *
+ * This stops a `nightly` and a `nightly --capture-only` (an encouraged workflow)
+ * from running the capture loop concurrently, which would race cursor writes and
+ * emit divergent segment boundaries → real duplication.
+ */
+async function acquireCaptureLock(stateDir, waitMs = 0) {
+    const lockPath = (0, node_path_1.join)(stateDir, LOCK_FILE);
+    try {
+        (0, node_fs_1.mkdirSync)(stateDir, { recursive: true });
+    }
+    catch {
+        /* best effort */
+    }
+    const deadline = Date.now() + waitMs;
+    for (;;) {
+        const release = tryAcquireOnce(lockPath);
+        if (release)
+            return release;
+        if (Date.now() >= deadline)
+            return null;
+        await sleep(Math.max(1, Math.min(LOCK_POLL_MS, deadline - Date.now())));
+    }
+}
+/** One acquire attempt: create-if-free, else reclaim-if-stale. */
+function tryAcquireOnce(lockPath) {
+    const release = () => {
+        try {
+            (0, node_fs_1.unlinkSync)(lockPath);
+        }
+        catch {
+            /* already gone */
+        }
+    };
+    const create = () => {
+        try {
+            const fd = (0, node_fs_1.openSync)(lockPath, "wx"); // O_CREAT | O_EXCL
+            (0, node_fs_1.writeSync)(fd, String(process.pid));
+            (0, node_fs_1.closeSync)(fd);
+            return true;
+        }
+        catch (err) {
+            if (err.code === "EEXIST")
+                return false;
+            throw err;
+        }
+    };
+    try {
+        if (create())
+            return release;
+        // Lock exists — read the holder pid and decide staleness.
+        const holderPid = readLockPid(lockPath);
+        if (!isLockStale(lockPath, holderPid))
+            return null; // live, recent → held
+        // Stale. Re-verify the file still carries the SAME pid we judged (another
+        // reclaimer may have taken it since), then unlink and re-race the O_EXCL
+        // create. A fresh holder → our create loses (EEXIST) → null (fix 12).
+        if (readLockPid(lockPath) !== holderPid)
+            return null;
+        try {
+            (0, node_fs_1.unlinkSync)(lockPath);
+        }
+        catch {
+            /* raced with another reclaimer */
+        }
+        return create() ? release : null;
+    }
+    catch {
+        // Filesystem refused the lock op entirely — don't wedge capture; proceed
+        // without the guard (behaviour before A5).
+        return release;
+    }
+}
+/** Read the recorded pid, or 0 if unreadable/absent. */
+function readLockPid(lockPath) {
+    try {
+        const pid = parseInt((0, node_fs_1.readFileSync)(lockPath, "utf-8").trim(), 10);
+        return Number.isFinite(pid) ? pid : 0;
+    }
+    catch {
+        return 0;
+    }
+}
+/** Stale = no/dead pid, OR the lockfile is older than the TTL (fix 2). */
+function isLockStale(lockPath, holderPid) {
+    if (!holderPid || !isProcessAlive(holderPid))
+        return true;
+    try {
+        return Date.now() - (0, node_fs_1.statSync)(lockPath).mtimeMs > LOCK_TTL_MS;
+    }
+    catch {
+        return true; // can't stat → treat as stale so we don't wedge forever
+    }
+}
+function isProcessAlive(pid) {
+    try {
+        process.kill(pid, 0);
+        return true;
+    }
+    catch (err) {
+        // ESRCH = no such process; EPERM = exists but not ours (still alive).
+        return err.code === "EPERM";
+    }
+}

package/dist/cli.js

@@ -62,8 +62,19 @@ switch (command) {
         else {
             const dryRun = args.includes("--dry-run");
             const captureOnly = args.includes("--capture-only");
+            // #189 Tier-2 recovery: re-discover sessions that went quiet before the
+            // upgrade by widening the discovery window to now−N days for one run.
+            let recaptureWindowDays;
+            const rwIdx = args.indexOf("--recapture-window");
+            if (rwIdx !== -1) {
+                recaptureWindowDays = parseInt(args[rwIdx + 1], 10);
+                if (isNaN(recaptureWindowDays) || recaptureWindowDays <= 0) {
+                    console.error("[hicortex] nightly: --recapture-window requires a positive integer (days)");
+                    process.exit(1);
+                }
+            }
             import("./nightly.js").then(({ runNightly }) => {
-                runNightly({ dryRun, captureOnly }).catch((err) => {
+                runNightly({ dryRun, captureOnly, recaptureWindowDays }).catch((err) => {
                     console.error("[hicortex] Nightly pipeline failed:", err);
                     process.exit(1);
                 });
@@ -194,6 +205,7 @@ Options:
   server --host <h>    Host (default: 127.0.0.1)
   nightly --dry-run         Preview without changes
   nightly --capture-only    Capture only, skip consolidation (safe to run multiple times/day)
+  nightly --recapture-window <days>   Re-discover sessions quiet since <days> ago (one-shot #189 recovery)
   nightly --status          Show nightly pipeline health
   relink --dry-run          Discovery + counts only, zero writes, cursor untouched
   relink --batch <n>        Memories per batch (default: 200)

package/dist/hermes-transcript-reader.d.ts

@@ -19,9 +19,13 @@
  * distillation and keeps per-session dedup clean (chunks are stored as
  * `<sessionId>#<chunkIndex>`; see nightly.ts).
  */
-import type { TranscriptBatch } from "./transcript-reader.js";
+import type { TranscriptBatch, CursorMap } from "./transcript-reader.js";
 /**
  * Read Hermes sessions that ended since `since`, across all profiles.
  * Returns one batch per session, parallel to readCcTranscripts().
+ *
+ * @param cursors Per-session capture cursors (#189), keyed `hermes:<profile>:<sid>`.
+ *   The cursor value is the max `messages.id` already captured; a resumed +
+ *   re-ended session yields only the new rows (`id > cursor`).
  */
-export declare function readHermesSessions(since: Date, hermesHome?: string): TranscriptBatch[];
+export declare function readHermesSessions(since: Date, hermesHome?: string, cursors?: CursorMap): TranscriptBatch[];

package/dist/hermes-transcript-reader.js

@@ -47,8 +47,12 @@ const NOISE_ROLES = new Set(["tool", "session_meta"]);
 /**
  * Read Hermes sessions that ended since `since`, across all profiles.
  * Returns one batch per session, parallel to readCcTranscripts().
+ *
+ * @param cursors Per-session capture cursors (#189), keyed `hermes:<profile>:<sid>`.
+ *   The cursor value is the max `messages.id` already captured; a resumed +
+ *   re-ended session yields only the new rows (`id > cursor`).
  */
-function readHermesSessions(since, hermesHome = HERMES_HOME) {
+function readHermesSessions(since, hermesHome = HERMES_HOME, cursors = {}) {
     const batches = [];
     const sinceEpoch = since.getTime() / 1000; // Hermes timestamps are unix seconds (REAL)
     for (const { profile, dbPath } of discoverProfileDbs(hermesHome)) {
@@ -63,15 +67,40 @@ function readHermesSessions(since, hermesHome = HERMES_HOME) {
             const sessions = db
                 .prepare("SELECT id, ended_at, source FROM sessions WHERE ended_at IS NOT NULL AND ended_at > ? ORDER BY ended_at")
                 .all(sinceEpoch);
-            const msgStmt = db.prepare("SELECT role, content, tool_name, timestamp FROM messages WHERE session_id = ? ORDER BY timestamp, id");
+            // Cursor is a message id (INTEGER PRIMARY KEY AUTOINCREMENT — strictly
+            // increasing, never reused), so `id > ?` returns exactly the rows added
+            // since last capture. ORDER BY id (NOT timestamp): id is the capture
+            // boundary, so ordering rows by id makes entryCursors monotonic and the
+            // last row's id the true max consumed — the segment boundary the packer
+            // advances to is then genuinely the largest id, never skipping a
+            // lower-id-but-later-timestamp row. Verified safe: id order == timestamp
+            // order in production (A1: 0 divergences / 4413 rows), so text ordering is
+            // unchanged in practice.
+            const msgStmt = db.prepare("SELECT id, role, content, tool_name, timestamp FROM messages WHERE session_id = ? AND id > ? ORDER BY id");
+            // Highest id in the session — used only for the shrink guard below.
+            const maxIdStmt = db.prepare("SELECT MAX(id) as m FROM messages WHERE session_id = ?");
             for (const s of sessions) {
                 // Skip automated (non-primary) sessions — cron runs are not
                 // conversations and would pollute memory. Checked before pulling
                 // messages so we don't even read them.
                 if (NON_PRIMARY_SOURCES.has(s.source))
                     continue;
-                const rows = msgStmt.all(s.id);
-                // Skip only genuinely empty sessions. Do NOT gate on message count —
+                const cursorKey = `hermes:${profile}:${s.id}`;
+                const pos = cursors[cursorKey] ?? { cursor: 0, gen: 0 };
+                let startCursor = pos.cursor;
+                let gen = pos.gen;
+                // Shrink guard: if the stored cursor exceeds the session's max id (DB
+                // reset/restore), re-read from 0 and bump the generation (fix 8). Cheap
+                // MAX(id) probe; the common path (cursor <= max) leaves it untouched.
+                if (startCursor > 0) {
+                    const maxId = maxIdStmt.get(s.id).m ?? 0;
+                    if (startCursor > maxId) {
+                        startCursor = 0;
+                        gen = pos.gen + 1;
+                    }
+                }
+                const rows = msgStmt.all(s.id, startCursor);
+                // Skip only genuinely empty deltas. Do NOT gate on message count —
                 // a short 2-message exchange can carry a real decision. Meaningful-
                 // content is gated downstream by the post-denoise 200-char check in
                 // nightly.ts, so short-but-dense sessions aren't dropped here.
@@ -84,6 +113,10 @@ function readHermesSessions(since, hermesHome = HERMES_HOME) {
                     role: NOISE_ROLES.has(r.role) ? "tool_result" : r.role,
                     content: r.content ?? "",
                 }));
+                // entryCursors are the row ids, monotonic under ORDER BY id — the last
+                // is the max consumed id, so segment boundaries and the final advance
+                // land exactly on it.
+                const entryCursors = rows.map((r) => r.id);
                 const endTs = s.ended_at ?? rows[rows.length - 1].timestamp;
                 batches.push({
                     sessionId: s.id,
@@ -91,6 +124,10 @@ function readHermesSessions(since, hermesHome = HERMES_HOME) {
                     sourceAgent: `hermes/${profile}`,
                     date: new Date(endTs * 1000).toISOString().slice(0, 10),
                     entries,
+                    cursorKey,
+                    startCursor,
+                    generation: gen,
+                    entryCursors,
                 });
             }
         }

package/dist/mcp-server.js

@@ -681,11 +681,27 @@ async function startServer(options = {}) {
             res.status(400).json({ error: "Provide either 'text' (string) or 'messages' (array)" });
             return;
         }
+        // Escape LIKE wildcards — Hermes ids contain "_" (e.g. 20260701_045744_...).
+        const escapeLike = (s) => s.replace(/[\\%_]/g, (m) => "\\" + m);
+        // Segment-exact dedup (#189): an incremental capture POST carries
+        // segment_id "<start>-<end>[.pN]". Skip iff THIS exact segment's chunks are
+        // already stored (keys "<sid>#<segment_id>#<i>"). This is what lets a failed
+        // segment be safely retried with the same id, and a legacy session-level row
+        // (key "<sid>#<i>", no segment) does NOT match — so the #189 recovery
+        // re-ingest is never blocked by night-1's whole-session rows.
+        if (session_id && segment_id) {
+            const likePrefix = `${escapeLike(session_id)}#${escapeLike(segment_id)}#%`;
+            const existing = db.prepare("SELECT COUNT(*) as c FROM memories WHERE source_session LIKE ? ESCAPE '\\'").get(likePrefix);
+            if (existing.c > 0) {
+                res.status(200).json({ skipped: true, existing_count: existing.c });
+                return;
+            }
+        }
         // Session-level dedup: when session_id is present and this is a whole-session
-        // POST (no segment_id), skip if any chunk of this session is already stored.
+        // POST (no segment_id — legacy ≤0.13.1 clients), skip if any chunk of this
+        // session is already stored. Unchanged: legacy clients keep exact behaviour.
         if (session_id && !segment_id) {
-            // Escape LIKE wildcards — Hermes ids contain "_" (e.g. 20260701_045744_...).
-            const likePrefix = `${session_id.replace(/[\\%_]/g, (m) => "\\" + m)}#%`;
+            const likePrefix = `${escapeLike(session_id)}#%`;
             const existing = db.prepare("SELECT COUNT(*) as c FROM memories WHERE source_session = ? OR source_session LIKE ? ESCAPE '\\'").get(session_id, likePrefix);
             if (existing.c > 0) {
                 res.status(200).json({ skipped: true, existing_count: existing.c });
@@ -722,24 +738,38 @@ async function startServer(options = {}) {
             // server-side per-entry console.log in distillChunk stays as well.
             const dropped = [];
             const entries = await (0, distiller_js_1.distillSession)(llm, conversationText, project ?? "unknown", date, chunkSize, dropped);
-            const ids = [];
+            // Phase 1 — embed every chunk up front (async). If ANY embed fails we
+            // never reach the insert, so nothing is stored.
+            const createdAt = new Date(date).toISOString();
+            const toStore = [];
             for (let i = 0; i < entries.length; i++) {
                 const entry = entries[i];
                 if (typeof entry !== "string" || !entry.trim())
                     continue;
-                const embedding = await (0, embedder_js_1.embed)(entry);
-                const id = storage.insertMemory(db, entry, embedding, {
-                    sourceAgent: source_agent ?? "unknown",
-                    // Per-chunk key: "<session_id>#<i>". The prefix matches the nightly
-                    // dedup check above, so a re-run of the same session is fully idempotent.
-                    sourceSession: sourcePrefix ? `${sourcePrefix}#${i}` : undefined,
-                    project: project ?? undefined,
-                    memoryType: "episode",
-                    privacy: privacy ?? "WORK",
-                    createdAt: new Date(date).toISOString(),
-                });
-                ids.push(id);
+                toStore.push({ entry, embedding: await (0, embedder_js_1.embed)(entry), i });
             }
+            // Phase 2 — insert all chunks in ONE transaction (fix 4). A segment's
+            // chunks are all-or-nothing: any insert failure rolls back the whole set
+            // and returns 500, so the content-blind segment-exact dedup never sees a
+            // half-stored segment and the retry re-distills cleanly. (Applies to the
+            // legacy whole-session path too — same loop.)
+            const insertAll = db.transaction(() => {
+                const out = [];
+                for (const { entry, embedding, i } of toStore) {
+                    out.push(storage.insertMemory(db, entry, embedding, {
+                        sourceAgent: source_agent ?? "unknown",
+                        // Per-chunk key: "<session_id>[#<segment_id>]#<i>". The prefix
+                        // matches the dedup checks above, so a re-run is idempotent.
+                        sourceSession: sourcePrefix ? `${sourcePrefix}#${i}` : undefined,
+                        project: project ?? undefined,
+                        memoryType: "episode",
+                        privacy: privacy ?? "WORK",
+                        createdAt,
+                    }));
+                }
+                return out;
+            });
+            const ids = insertAll();
             res.status(201).json({
                 ids,
                 distilled: ids.length,

package/dist/nightly.d.ts

@@ -15,4 +15,6 @@ export declare function runNightly(options?: {
     captureOnly?: boolean;
     dbPath?: string;
     stateDir?: string;
+    /** #189 Tier-2 recovery: override discovery to now−N days for one run. */
+    recaptureWindowDays?: number;
 }): Promise<void>;