@gamaze/hicortex 0.13.1 → 0.13.2

package/dist/capture-cursors.d.ts

@@ -0,0 +1,73 @@
+/**
+ * Per-session capture cursors (#189).
+ *
+ * The client is the source of truth for "how much of each session has already
+ * been captured". A cursor is advanced ONLY after the server confirms the
+ * corresponding segment(s) were ingested, so a crash between POST and cursor
+ * write can at worst cause a bounded, idempotent re-send — never silent loss.
+ *
+ * Storage: a SEPARATE small file `<hicortex-home>/capture-cursors.json`
+ * (NOT state.json — that file carries the large moduleIndex, and per-session
+ * whole-file rewrites there would be an avoidable corruption/IO surface). Same
+ * temp+rename atomic write discipline as state.ts.
+ *
+ * Cursor unit is reader-defined:
+ *   - JSONL readers (CC/Pi/OC): count of successfully-PARSED entries consumed.
+ *   - Hermes: max `messages.id` consumed (INTEGER PRIMARY KEY AUTOINCREMENT —
+ *     never reused, strictly increasing).
+ *
+ * `gen` (generation) is bumped by a reader's shrink-guard when a source file is
+ * truncated/rotated below the stored cursor. It is woven into the segment id
+ * (`g<gen>.<start>-<end>`) so post-reset segments can NEVER collide with a
+ * pre-reset id on the server's content-blind segment-exact dedup — a collision
+ * there would be silent LOSS, not the intended dup-over-loss (#189 review, fix 8).
+ *
+ * Keys are `<prefix>:<sessionId>`:
+ *   cc:<sid>  pi:<sid>  oc:<agentId>:<sid>  hermes:<profile>:<sid>
+ */
+export interface CursorEntry {
+    /** Reader-defined cursor value (entry count for JSONL, max id for Hermes). */
+    cursor: number;
+    /** Shrink-guard generation — woven into segment ids to avoid post-reset id collisions. */
+    gen: number;
+    /** ISO timestamp of the last advance — used for 90-day pruning. */
+    updated: string;
+}
+export type CursorFile = Record<string, CursorEntry>;
+/** Snapshot of a session's captured position, passed to the readers. */
+export interface CursorPosition {
+    cursor: number;
+    gen: number;
+}
+/** Map of cursor key → captured position. */
+export type CursorMap = Record<string, CursorPosition>;
+/** Load the cursor file. Missing/corrupt → empty (every key defaults to 0). */
+export declare function loadCursors(stateDir?: string): CursorFile;
+/**
+ * Atomically persist the cursor file (write-temp + rename). THROWS on failure —
+ * a persistent inability to record cursors must surface as a transient failure
+ * (hold the watermark), never a silent warn-and-continue that re-captures the
+ * same content every night (#189 review, fix 7).
+ */
+export declare function saveCursors(file: CursorFile, stateDir?: string): void;
+/**
+ * A read-once, write-on-advance cursor store. Single-flight (see
+ * acquireCaptureLock in capture.ts) guarantees no concurrent writer, so an
+ * in-memory copy flushed atomically on each advance is safe.
+ */
+export interface CursorStore {
+    /** Current position for `key` ({cursor:0, gen:0} when never captured). */
+    get(key: string): CursorPosition;
+    /** Advance `key` to `cursor` at `gen` and persist. Throws if the write fails. */
+    advance(key: string, cursor: number, gen: number): void;
+    /** Snapshot of key → position for passing to readers. */
+    map(): CursorMap;
+}
+/** Open a file-backed cursor store for `stateDir`. */
+export declare function openCursorStore(stateDir?: string): CursorStore;
+/**
+ * Drop cursor entries older than `days` (default 90). Best-effort: a prune-write
+ * failure is logged, not thrown (prune runs only after a clean nightly and must
+ * not fail the run). Returns the number pruned.
+ */
+export declare function pruneCursors(stateDir?: string, days?: number): number;

package/dist/capture-cursors.js

@@ -0,0 +1,133 @@
+"use strict";
+/**
+ * Per-session capture cursors (#189).
+ *
+ * The client is the source of truth for "how much of each session has already
+ * been captured". A cursor is advanced ONLY after the server confirms the
+ * corresponding segment(s) were ingested, so a crash between POST and cursor
+ * write can at worst cause a bounded, idempotent re-send — never silent loss.
+ *
+ * Storage: a SEPARATE small file `<hicortex-home>/capture-cursors.json`
+ * (NOT state.json — that file carries the large moduleIndex, and per-session
+ * whole-file rewrites there would be an avoidable corruption/IO surface). Same
+ * temp+rename atomic write discipline as state.ts.
+ *
+ * Cursor unit is reader-defined:
+ *   - JSONL readers (CC/Pi/OC): count of successfully-PARSED entries consumed.
+ *   - Hermes: max `messages.id` consumed (INTEGER PRIMARY KEY AUTOINCREMENT —
+ *     never reused, strictly increasing).
+ *
+ * `gen` (generation) is bumped by a reader's shrink-guard when a source file is
+ * truncated/rotated below the stored cursor. It is woven into the segment id
+ * (`g<gen>.<start>-<end>`) so post-reset segments can NEVER collide with a
+ * pre-reset id on the server's content-blind segment-exact dedup — a collision
+ * there would be silent LOSS, not the intended dup-over-loss (#189 review, fix 8).
+ *
+ * Keys are `<prefix>:<sessionId>`:
+ *   cc:<sid>  pi:<sid>  oc:<agentId>:<sid>  hermes:<profile>:<sid>
+ */
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.loadCursors = loadCursors;
+exports.saveCursors = saveCursors;
+exports.openCursorStore = openCursorStore;
+exports.pruneCursors = pruneCursors;
+const paths_js_1 = require("./paths.js");
+const node_fs_1 = require("node:fs");
+const node_path_1 = require("node:path");
+const HICORTEX_HOME = (0, paths_js_1.hicortexHome)();
+const CURSORS_FILE = "capture-cursors.json";
+/** Coerce a stored cursor value to a valid non-negative integer (fix 13). */
+function sanitizeCursor(raw, key) {
+    if (typeof raw === "number" && Number.isFinite(raw) && raw >= 0) {
+        return Math.floor(raw);
+    }
+    console.warn(`[hicortex] capture-cursors: ignoring invalid cursor for ${key} (${JSON.stringify(raw)}) — treating as 0`);
+    return 0;
+}
+function sanitizeGen(raw) {
+    return typeof raw === "number" && Number.isFinite(raw) && raw >= 0 ? Math.floor(raw) : 0;
+}
+/** Load the cursor file. Missing/corrupt → empty (every key defaults to 0). */
+function loadCursors(stateDir = HICORTEX_HOME) {
+    let parsed;
+    try {
+        parsed = JSON.parse((0, node_fs_1.readFileSync)((0, node_path_1.join)(stateDir, CURSORS_FILE), "utf-8"));
+    }
+    catch {
+        return {};
+    }
+    if (!parsed || typeof parsed !== "object")
+        return {};
+    const out = {};
+    for (const [key, val] of Object.entries(parsed)) {
+        if (!val || typeof val !== "object")
+            continue;
+        const v = val;
+        out[key] = {
+            cursor: sanitizeCursor(v.cursor, key),
+            gen: sanitizeGen(v.gen),
+            updated: typeof v.updated === "string" ? v.updated : new Date().toISOString(),
+        };
+    }
+    return out;
+}
+/**
+ * Atomically persist the cursor file (write-temp + rename). THROWS on failure —
+ * a persistent inability to record cursors must surface as a transient failure
+ * (hold the watermark), never a silent warn-and-continue that re-captures the
+ * same content every night (#189 review, fix 7).
+ */
+function saveCursors(file, stateDir = HICORTEX_HOME) {
+    (0, node_fs_1.mkdirSync)(stateDir, { recursive: true });
+    const path = (0, node_path_1.join)(stateDir, CURSORS_FILE);
+    const tmp = `${path}.tmp`;
+    (0, node_fs_1.writeFileSync)(tmp, JSON.stringify(file, null, 2));
+    (0, node_fs_1.renameSync)(tmp, path);
+}
+/** Open a file-backed cursor store for `stateDir`. */
+function openCursorStore(stateDir = HICORTEX_HOME) {
+    const file = loadCursors(stateDir);
+    return {
+        get(key) {
+            const e = file[key];
+            return { cursor: e?.cursor ?? 0, gen: e?.gen ?? 0 };
+        },
+        advance(key, cursor, gen) {
+            file[key] = { cursor, gen, updated: new Date().toISOString() };
+            saveCursors(file, stateDir); // throws on failure — caller holds the watermark
+        },
+        map() {
+            const out = {};
+            for (const [k, v] of Object.entries(file))
+                out[k] = { cursor: v.cursor, gen: v.gen };
+            return out;
+        },
+    };
+}
+/**
+ * Drop cursor entries older than `days` (default 90). Best-effort: a prune-write
+ * failure is logged, not thrown (prune runs only after a clean nightly and must
+ * not fail the run). Returns the number pruned.
+ */
+function pruneCursors(stateDir = HICORTEX_HOME, days = 90) {
+    const file = loadCursors(stateDir);
+    const cutoff = Date.now() - days * 24 * 60 * 60 * 1000;
+    let pruned = 0;
+    for (const [key, entry] of Object.entries(file)) {
+        const t = Date.parse(entry.updated);
+        if (!Number.isNaN(t) && t < cutoff) {
+            delete file[key];
+            pruned++;
+        }
+    }
+    if (pruned > 0) {
+        try {
+            saveCursors(file, stateDir);
+        }
+        catch (err) {
+            console.warn(`[hicortex] capture-cursors prune write failed: ${err instanceof Error ? err.message : String(err)}`);
+            return 0;
+        }
+    }
+    return pruned;
+}

package/dist/capture.d.ts

@@ -0,0 +1,124 @@
+/**
+ * Incremental, cursor-aware capture loop (#189).
+ *
+ * Extracted from the two near-identical loops that lived in nightly.ts (server
+ * and client mode). Both now share this logic: pack each session's delta into
+ * ordered segments below the server's distill cap, POST them in order with a
+ * deterministic `segment_id`, and advance the per-session cursor ONLY after
+ * server-confirmed success — so a multi-day session grows across nights with no
+ * loss and no silent truncation.
+ *
+ * The POST transport is injected (`post`) so the mode-specific bits (localhost
+ * vs remote URL, Authorization header, timeout) stay in nightly.ts and the
+ * multi-night simulation can run as a pure unit test with no HTTP listener.
+ */
+import type { TranscriptBatch } from "./transcript-reader.js";
+import type { CursorStore } from "./capture-cursors.js";
+/**
+ * Max denoised chars per segment. Kept below the server's 80K distill cap
+ * (distiller.ts MAX_TRANSCRIPT_CHARS) with ~20K headroom so NO capture path can
+ * hit the silent truncation. LOAD-BEARING for #189 recovery: a re-ingested
+ * week-long session is re-sliced into ≤60K segments here instead of being
+ * truncated at 80K server-side. (Judgment constant — tunable later.)
+ */
+export declare const SEGMENT_MAX_CHARS = 60000;
+/**
+ * Minimum denoised chars for a FRESH whole session (startCursor 0) to be worth
+ * capturing — mirrors the long-standing pre-#189 200-char degenerate-session
+ * gate. It is applied ONLY to a whole-session capture that denoises to a single
+ * sub-200 segment. A delta beyond cursor 0 is always sent, however small: a
+ * session's concluding tail must never be held back, because once the session
+ * stops growing its mtime never re-crosses the watermark and the tail would be
+ * lost forever (#189 review, fix 5).
+ */
+export declare const MIN_SEGMENT_CHARS = 200;
+/** One packed, ready-to-POST segment of a session's delta. */
+export interface Segment {
+    /** Denoised text body of the POST. */
+    text: string;
+    /** Cursor value the segment starts at. */
+    segStart: number;
+    /** Cursor value the segment ends at. */
+    segEnd: number;
+    /**
+     * Disambiguator for hard-split pieces of a single oversized entry (A2). Empty
+     * for normal segments; ".p0", ".p1", … when one entry is split mid-text.
+     * Keeps the server's `<sid>#<segment_id>#<i>` keys distinct so no piece's
+     * memories collide on the UNIQUE index.
+     */
+    idSuffix: string;
+}
+/** The wire body for POST /distill. */
+export interface DistillBody {
+    text: string;
+    source_agent: string;
+    project: string;
+    session_id: string;
+    segment_id: string;
+    session_date: string;
+    privacy: string;
+}
+/** Normalized POST result the caller's transport returns. */
+export interface PostResult {
+    status: number;
+    distilled?: number;
+    dropped?: string[];
+    skipped?: boolean;
+    error?: string;
+}
+export type PostFn = (body: DistillBody) => Promise<PostResult>;
+export interface CaptureOptions {
+    post: PostFn;
+    cursorStore: CursorStore;
+    dryRun?: boolean;
+    /** Segment size cap; defaults to SEGMENT_MAX_CHARS. Lowered in tests. */
+    segmentMaxChars?: number;
+}
+export interface CaptureResult {
+    memoriesIngested: number;
+    sessionsSent: number;
+    hadTransientFailure: boolean;
+    /**
+     * Set when the loop stopped early on a terminal server response: "limit"
+     * (429 memory cap) or "auth" (401). The caller decides watermark handling.
+     */
+    stopped?: "limit" | "auth";
+}
+/**
+ * Split an already-denoised string into ≤maxChars pieces (A2 hard-split).
+ * Prefers paragraph, then line, then hard boundaries — mirrors the distiller's
+ * own splitIntoChunks, but WITHOUT its <200-char drop (every piece must survive,
+ * dup-over-loss).
+ */
+export declare function hardSplitText(text: string, maxChars?: number): string[];
+/**
+ * Pack a session's delta entries into ordered ≤maxChars segments.
+ *
+ * Sizing uses per-entry denoise lengths plus the "\n\n" joiners (A8) so the
+ * estimate matches what the server receives; the actual body is a re-denoise of
+ * the grouped entries (extractConversationText) so cleaning/redaction stay
+ * coherent. A single entry larger than maxChars is emitted as its own run of
+ * hard-split pieces (A2).
+ */
+export declare function packSegments(entries: unknown[], startCursor: number, entryCursors: number[], maxChars?: number): Segment[];
+/**
+ * Capture a list of session delta batches: pack, POST in order, advance
+ * per-session cursors on success. Segments of one session POST in order; the
+ * first hard failure stops THAT session (cursor holds at the last confirmed
+ * boundary) while other sessions continue. A 429/401 stops the whole loop.
+ */
+export declare function captureBatches(batches: TranscriptBatch[], opts: CaptureOptions): Promise<CaptureResult>;
+/**
+ * Acquire an exclusive capture lock for `stateDir`. Returns a release function,
+ * or null if another LIVE, non-stale run holds it after waiting up to `waitMs`.
+ *
+ * Staleness = dead pid OR lockfile mtime older than LOCK_TTL_MS. A stale lock is
+ * reclaimed (with a re-verify + O_EXCL re-race to narrow the TOCTOU window,
+ * fix 12). `waitMs` lets the full nightly wait out a transient `--capture-only`
+ * overlap instead of dropping the night's capture (fix 10); pass 0 to fail fast.
+ *
+ * This stops a `nightly` and a `nightly --capture-only` (an encouraged workflow)
+ * from running the capture loop concurrently, which would race cursor writes and
+ * emit divergent segment boundaries → real duplication.
+ */
+export declare function acquireCaptureLock(stateDir: string, waitMs?: number): Promise<(() => void) | null>;