@gamaze/hicortex 0.13.0 → 0.13.2

package/dist/hermes-transcript-reader.js

@@ -47,8 +47,12 @@ const NOISE_ROLES = new Set(["tool", "session_meta"]);
 /**
  * Read Hermes sessions that ended since `since`, across all profiles.
  * Returns one batch per session, parallel to readCcTranscripts().
+ *
+ * @param cursors Per-session capture cursors (#189), keyed `hermes:<profile>:<sid>`.
+ *   The cursor value is the max `messages.id` already captured; a resumed +
+ *   re-ended session yields only the new rows (`id > cursor`).
  */
-function readHermesSessions(since, hermesHome = HERMES_HOME) {
+function readHermesSessions(since, hermesHome = HERMES_HOME, cursors = {}) {
     const batches = [];
     const sinceEpoch = since.getTime() / 1000; // Hermes timestamps are unix seconds (REAL)
     for (const { profile, dbPath } of discoverProfileDbs(hermesHome)) {
@@ -63,15 +67,40 @@ function readHermesSessions(since, hermesHome = HERMES_HOME) {
             const sessions = db
                 .prepare("SELECT id, ended_at, source FROM sessions WHERE ended_at IS NOT NULL AND ended_at > ? ORDER BY ended_at")
                 .all(sinceEpoch);
-            const msgStmt = db.prepare("SELECT role, content, tool_name, timestamp FROM messages WHERE session_id = ? ORDER BY timestamp, id");
+            // Cursor is a message id (INTEGER PRIMARY KEY AUTOINCREMENT — strictly
+            // increasing, never reused), so `id > ?` returns exactly the rows added
+            // since last capture. ORDER BY id (NOT timestamp): id is the capture
+            // boundary, so ordering rows by id makes entryCursors monotonic and the
+            // last row's id the true max consumed — the segment boundary the packer
+            // advances to is then genuinely the largest id, never skipping a
+            // lower-id-but-later-timestamp row. Verified safe: id order == timestamp
+            // order in production (A1: 0 divergences / 4413 rows), so text ordering is
+            // unchanged in practice.
+            const msgStmt = db.prepare("SELECT id, role, content, tool_name, timestamp FROM messages WHERE session_id = ? AND id > ? ORDER BY id");
+            // Highest id in the session — used only for the shrink guard below.
+            const maxIdStmt = db.prepare("SELECT MAX(id) as m FROM messages WHERE session_id = ?");
             for (const s of sessions) {
                 // Skip automated (non-primary) sessions — cron runs are not
                 // conversations and would pollute memory. Checked before pulling
                 // messages so we don't even read them.
                 if (NON_PRIMARY_SOURCES.has(s.source))
                     continue;
-                const rows = msgStmt.all(s.id);
-                // Skip only genuinely empty sessions. Do NOT gate on message count —
+                const cursorKey = `hermes:${profile}:${s.id}`;
+                const pos = cursors[cursorKey] ?? { cursor: 0, gen: 0 };
+                let startCursor = pos.cursor;
+                let gen = pos.gen;
+                // Shrink guard: if the stored cursor exceeds the session's max id (DB
+                // reset/restore), re-read from 0 and bump the generation (fix 8). Cheap
+                // MAX(id) probe; the common path (cursor <= max) leaves it untouched.
+                if (startCursor > 0) {
+                    const maxId = maxIdStmt.get(s.id).m ?? 0;
+                    if (startCursor > maxId) {
+                        startCursor = 0;
+                        gen = pos.gen + 1;
+                    }
+                }
+                const rows = msgStmt.all(s.id, startCursor);
+                // Skip only genuinely empty deltas. Do NOT gate on message count —
                 // a short 2-message exchange can carry a real decision. Meaningful-
                 // content is gated downstream by the post-denoise 200-char check in
                 // nightly.ts, so short-but-dense sessions aren't dropped here.
@@ -84,6 +113,10 @@ function readHermesSessions(since, hermesHome = HERMES_HOME) {
                     role: NOISE_ROLES.has(r.role) ? "tool_result" : r.role,
                     content: r.content ?? "",
                 }));
+                // entryCursors are the row ids, monotonic under ORDER BY id — the last
+                // is the max consumed id, so segment boundaries and the final advance
+                // land exactly on it.
+                const entryCursors = rows.map((r) => r.id);
                 const endTs = s.ended_at ?? rows[rows.length - 1].timestamp;
                 batches.push({
                     sessionId: s.id,
@@ -91,6 +124,10 @@ function readHermesSessions(since, hermesHome = HERMES_HOME) {
                     sourceAgent: `hermes/${profile}`,
                     date: new Date(endTs * 1000).toISOString().slice(0, 10),
                     entries,
+                    cursorKey,
+                    startCursor,
+                    generation: gen,
+                    entryCursors,
                 });
             }
         }

package/dist/init.d.ts

@@ -29,6 +29,14 @@ export declare function parseMcpListStatus(mcpListOutput: string): "connected" |
  * Exported for testability.
  */
 export declare function parseEnvFile(content: string): Record<string, string>;
+/**
+ * True when an LLM is already persisted and `init` must NOT re-run provider
+ * selection: a named/flat backend, a flat baseUrl+apiKey pair, OR a nested-only
+ * `models.score` (model or baseUrl). The last clause (0.13.1) stops init from
+ * walking a nested-only config back through selection and writing flat keys that
+ * a `models.score` would then silently shadow (nested > flat).
+ */
+export declare function isLlmConfigured(config: Record<string, unknown>): boolean;
 /**
  * Generate a random auth token in the format hctx-<32 hex chars>.
  * Exported for testability.

package/dist/init.js

@@ -20,6 +20,7 @@ Object.defineProperty(exports, "__esModule", { value: true });
 exports.GENERIC_DEFAULT_DOMAINS = void 0;
 exports.parseMcpListStatus = parseMcpListStatus;
 exports.parseEnvFile = parseEnvFile;
+exports.isLlmConfigured = isLlmConfigured;
 exports.generateAuthToken = generateAuthToken;
 exports.persistAuthToken = persistAuthToken;
 exports.decideAgentName = decideAgentName;
@@ -493,6 +494,18 @@ function mergeByKey(candidates) {
     }
     return [...seen.values()];
 }
+/**
+ * True when an LLM is already persisted and `init` must NOT re-run provider
+ * selection: a named/flat backend, a flat baseUrl+apiKey pair, OR a nested-only
+ * `models.score` (model or baseUrl). The last clause (0.13.1) stops init from
+ * walking a nested-only config back through selection and writing flat keys that
+ * a `models.score` would then silently shadow (nested > flat).
+ */
+function isLlmConfigured(config) {
+    const modelsScore = config.models?.score;
+    const hasModelsScore = Boolean(modelsScore?.model || modelsScore?.baseUrl);
+    return Boolean(config.llmBackend || (config.llmApiKey && config.llmBaseUrl) || hasModelsScore);
+}
 /**
  * Detect or ask for LLM config and persist to ~/.hicortex/config.json.
  * The daemon can't inherit shell env vars, so we persist here.
@@ -508,8 +521,8 @@ async function persistLlmConfig() {
         config = JSON.parse((0, node_fs_1.readFileSync)(configPath, "utf-8"));
     }
     catch { /* new file */ }
-    // Don't overwrite if LLM config already persisted
-    if (config.llmBackend || (config.llmApiKey && config.llmBaseUrl)) {
+    // Don't overwrite if LLM config already persisted (incl. a nested-only config).
+    if (isLlmConfigured(config)) {
         console.log(`  ✓ LLM config already configured`);
         return;
     }

package/dist/llm.d.ts

@@ -68,6 +68,25 @@ export declare function resolveExplicitLlmConfig(overrides?: {
  * the transition for any lingering call sites — remove after 0.10.0 ships.
  */
 export declare const resolveLlmConfigForCC: typeof resolveExplicitLlmConfig;
+export type { ModelTierOverride } from "./types.js";
+/**
+ * Normalize a nested `models: { <tier>: {model,baseUrl,apiKey,provider} }` block
+ * onto the flat `llm*` / `distill*` / `reflect*` / `classify*` keys the resolver
+ * already consumes. Nested overrides WIN over any flat key of the same name; every
+ * non-mapped key (llmBackend, licenseKey, distillFallback, contextClients, …)
+ * is preserved via spread. Pure: returns the SAME reference when there is no
+ * `models` key, so this is a provable no-op for every existing install.
+ *
+ * Robust to a malformed config.json: a config that parses to a scalar, array,
+ * or null is returned untouched (matching the pre-0.13.1 optional-chaining
+ * tolerance — this function must never throw at server/nightly boot).
+ *
+ * Fail-explicit (warn + skip, never throw): an invalid `models` value, an
+ * unknown tier name, a non-object tier value, a non-string field value, a tier
+ * apiKey/provider set without a baseUrl (they are baseUrl-gated downstream), and
+ * a dead score apiKey/provider under an ollama base.
+ */
+export declare function applyModelsBlock(saved: Record<string, unknown> | null): Record<string, unknown> | null;
 /**
  * Resolve an LlmConfig from a saved ~/.hicortex/config.json object.
  *
@@ -79,8 +98,12 @@ export declare const resolveLlmConfigForCC: typeof resolveExplicitLlmConfig;
  *
  * Returns `reason: "claude_binary_missing"` when claude-cli is configured but
  * the binary can't be found, so callers can log a context-specific message.
+ *
+ * `findBinary` is injectable (defaults to the real `findClaudeBinary`) so the
+ * claude-cli branch — including the missing-binary passthrough — can be pinned
+ * deterministically in tests without depending on the host filesystem.
  */
-export declare function resolveSavedLlmConfig(savedConfig: Record<string, unknown> | null): {
+export declare function resolveSavedLlmConfig(savedConfig: Record<string, unknown> | null, findBinary?: () => string | null): {
     config: LlmConfig | null;
     reason?: "claude_binary_missing";
 };

package/dist/llm.js

@@ -17,6 +17,7 @@
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.LlmClient = exports.RateLimitError = exports.resolveLlmConfigForCC = void 0;
 exports.resolveExplicitLlmConfig = resolveExplicitLlmConfig;
+exports.applyModelsBlock = applyModelsBlock;
 exports.resolveSavedLlmConfig = resolveSavedLlmConfig;
 exports.resolveClassifyProbeTarget = resolveClassifyProbeTarget;
 exports.findClaudeBinary = findClaudeBinary;
@@ -70,6 +71,117 @@ function resolveExplicitLlmConfig(overrides) {
  * the transition for any lingering call sites — remove after 0.10.0 ships.
  */
 exports.resolveLlmConfigForCC = resolveExplicitLlmConfig;
+/**
+ * Map from a `models.<tier>` name to the flat config keys it feeds. The base
+ * tier is `score` — score IS the base model today (completeFast reads
+ * config.model), so it lands on the llm* keys and its `provider` is ignored
+ * (the base provider comes from llmBackend / detectProvider, not config).
+ * Tiers with a `provider` key (distill/reflect/classify) apply their apiKey +
+ * provider through a baseUrl-gated overlay downstream; `score` (no provider
+ * key) rides the base resolution.
+ */
+const MODELS_TIER_KEYS = {
+    score: { model: "llmModel", baseUrl: "llmBaseUrl", apiKey: "llmApiKey" },
+    distill: { model: "distillModel", baseUrl: "distillBaseUrl", apiKey: "distillApiKey", provider: "distillProvider" },
+    reflect: { model: "reflectModel", baseUrl: "reflectBaseUrl", apiKey: "reflectApiKey", provider: "reflectProvider" },
+    classify: { model: "classifyModel", baseUrl: "classifyBaseUrl", apiKey: "classifyApiKey", provider: "classifyProvider" },
+};
+/**
+ * Normalize a nested `models: { <tier>: {model,baseUrl,apiKey,provider} }` block
+ * onto the flat `llm*` / `distill*` / `reflect*` / `classify*` keys the resolver
+ * already consumes. Nested overrides WIN over any flat key of the same name; every
+ * non-mapped key (llmBackend, licenseKey, distillFallback, contextClients, …)
+ * is preserved via spread. Pure: returns the SAME reference when there is no
+ * `models` key, so this is a provable no-op for every existing install.
+ *
+ * Robust to a malformed config.json: a config that parses to a scalar, array,
+ * or null is returned untouched (matching the pre-0.13.1 optional-chaining
+ * tolerance — this function must never throw at server/nightly boot).
+ *
+ * Fail-explicit (warn + skip, never throw): an invalid `models` value, an
+ * unknown tier name, a non-object tier value, a non-string field value, a tier
+ * apiKey/provider set without a baseUrl (they are baseUrl-gated downstream), and
+ * a dead score apiKey/provider under an ollama base.
+ */
+function applyModelsBlock(saved) {
+    // Guard the container itself first — `"models" in saved` throws a TypeError on
+    // a truthy non-object (config.json = `true`/`5`/`"x"`); such configs must pass
+    // through so the boot path degrades to recall-only exactly as before.
+    if (typeof saved !== "object" || saved === null || Array.isArray(saved))
+        return saved;
+    if (!("models" in saved))
+        return saved;
+    const models = saved.models;
+    if (typeof models !== "object" || models === null || Array.isArray(models)) {
+        console.warn(`[hicortex] Ignoring invalid "models" config: expected an object of per-tier overrides, got ${Array.isArray(models) ? "array" : models === null ? "null" : typeof models}`);
+        return saved;
+    }
+    const ollamaBase = saved.llmBackend === "ollama";
+    const mapped = {};
+    for (const [tier, value] of Object.entries(models)) {
+        const keys = MODELS_TIER_KEYS[tier];
+        if (!keys) {
+            console.warn(`[hicortex] Ignoring unknown "models" tier "${tier}" (expected: score, distill, reflect, classify)`);
+            continue;
+        }
+        if (typeof value !== "object" || value === null || Array.isArray(value)) {
+            console.warn(`[hicortex] Ignoring invalid "models.${tier}" override: expected an object with model/baseUrl/apiKey/provider`);
+            continue;
+        }
+        const o = value;
+        // Per-field string validation: a non-string value would map verbatim and
+        // fail opaquely downstream (e.g. baseUrl: 11434), so drop it with a warning.
+        const strField = (name) => {
+            const v = o[name];
+            if (v === undefined)
+                return undefined;
+            if (typeof v !== "string") {
+                console.warn(`[hicortex] Ignoring non-string "models.${tier}.${name}" (expected a string)`);
+                return undefined;
+            }
+            return v;
+        };
+        const model = strField("model");
+        const baseUrl = strField("baseUrl");
+        const apiKey = strField("apiKey");
+        const provider = strField("provider");
+        if (model !== undefined)
+            mapped[keys.model] = model;
+        if (baseUrl !== undefined)
+            mapped[keys.baseUrl] = baseUrl;
+        if (keys.provider) {
+            // Overlay tier (distill/reflect/classify): the downstream overlay only
+            // consumes apiKey/provider when the tier ALSO sets its own baseUrl.
+            // Without one, they would silently bill to the base key — so warn + drop.
+            if ((apiKey !== undefined || provider !== undefined) && baseUrl === undefined) {
+                console.warn(`[hicortex] Ignoring "models.${tier}" apiKey/provider without a baseUrl: they only take effect when the tier sets its own baseUrl`);
+            }
+            else {
+                if (apiKey !== undefined)
+                    mapped[keys.apiKey] = apiKey;
+                if (provider !== undefined)
+                    mapped[keys.provider] = provider;
+            }
+        }
+        else {
+            // score = base tier: no separate provider key, and apiKey rides llmApiKey.
+            if (provider !== undefined) {
+                console.warn(`[hicortex] Ignoring "models.score.provider": the base provider comes from llmBackend (or is auto-detected from the endpoint)`);
+            }
+            if (apiKey !== undefined) {
+                if (ollamaBase) {
+                    // The ollama base path hardcodes an empty api key and never reads
+                    // llmApiKey, so score.apiKey is dead there.
+                    console.warn(`[hicortex] Ignoring "models.score.apiKey": the base ollama path sends no api key`);
+                }
+                else {
+                    mapped[keys.apiKey] = apiKey;
+                }
+            }
+        }
+    }
+    return { ...saved, ...mapped };
+}
 /**
  * Resolve an LlmConfig from a saved ~/.hicortex/config.json object.
  *
@@ -81,11 +193,16 @@ exports.resolveLlmConfigForCC = resolveExplicitLlmConfig;
  *
  * Returns `reason: "claude_binary_missing"` when claude-cli is configured but
  * the binary can't be found, so callers can log a context-specific message.
+ *
+ * `findBinary` is injectable (defaults to the real `findClaudeBinary`) so the
+ * claude-cli branch — including the missing-binary passthrough — can be pinned
+ * deterministically in tests without depending on the host filesystem.
  */
-function resolveSavedLlmConfig(savedConfig) {
+function resolveSavedLlmConfig(savedConfig, findBinary = findClaudeBinary) {
+    savedConfig = applyModelsBlock(savedConfig);
     let llmConfig = null;
     if (savedConfig?.llmBackend === "claude-cli") {
-        const claudePath = findClaudeBinary();
+        const claudePath = findBinary();
         if (claudePath) {
             llmConfig = claudeCliConfig(claudePath);
         }

package/dist/mcp-server.js

@@ -332,7 +332,7 @@ async function startServer(options = {}) {
     // Named backends (claude-cli, ollama) → immediate config; everything else
     // goes through resolveExplicitLlmConfig which requires a user-chosen provider.
     // If nothing is configured: start recall-only with an unmissable warning.
-    const savedConfig = readConfigFile(stateDir);
+    const savedConfig = (0, llm_js_1.applyModelsBlock)(readConfigFile(stateDir));
     if (savedConfig?.llmBackend === "claude-cli") {
         const claudePath = (0, llm_js_1.findClaudeBinary)();
         if (claudePath) {
@@ -681,11 +681,27 @@ async function startServer(options = {}) {
             res.status(400).json({ error: "Provide either 'text' (string) or 'messages' (array)" });
             return;
         }
+        // Escape LIKE wildcards — Hermes ids contain "_" (e.g. 20260701_045744_...).
+        const escapeLike = (s) => s.replace(/[\\%_]/g, (m) => "\\" + m);
+        // Segment-exact dedup (#189): an incremental capture POST carries
+        // segment_id "<start>-<end>[.pN]". Skip iff THIS exact segment's chunks are
+        // already stored (keys "<sid>#<segment_id>#<i>"). This is what lets a failed
+        // segment be safely retried with the same id, and a legacy session-level row
+        // (key "<sid>#<i>", no segment) does NOT match — so the #189 recovery
+        // re-ingest is never blocked by night-1's whole-session rows.
+        if (session_id && segment_id) {
+            const likePrefix = `${escapeLike(session_id)}#${escapeLike(segment_id)}#%`;
+            const existing = db.prepare("SELECT COUNT(*) as c FROM memories WHERE source_session LIKE ? ESCAPE '\\'").get(likePrefix);
+            if (existing.c > 0) {
+                res.status(200).json({ skipped: true, existing_count: existing.c });
+                return;
+            }
+        }
         // Session-level dedup: when session_id is present and this is a whole-session
-        // POST (no segment_id), skip if any chunk of this session is already stored.
+        // POST (no segment_id — legacy ≤0.13.1 clients), skip if any chunk of this
+        // session is already stored. Unchanged: legacy clients keep exact behaviour.
         if (session_id && !segment_id) {
-            // Escape LIKE wildcards — Hermes ids contain "_" (e.g. 20260701_045744_...).
-            const likePrefix = `${session_id.replace(/[\\%_]/g, (m) => "\\" + m)}#%`;
+            const likePrefix = `${escapeLike(session_id)}#%`;
             const existing = db.prepare("SELECT COUNT(*) as c FROM memories WHERE source_session = ? OR source_session LIKE ? ESCAPE '\\'").get(session_id, likePrefix);
             if (existing.c > 0) {
                 res.status(200).json({ skipped: true, existing_count: existing.c });
@@ -717,26 +733,48 @@ async function startServer(options = {}) {
             ? `${session_id}${segment_id ? `#${segment_id}` : ""}`
             : undefined;
         try {
-            const entries = await (0, distiller_js_1.distillSession)(llm, conversationText, project ?? "unknown", date, chunkSize);
-            const ids = [];
+            // Collect gate-dropped entries so they can ride back in the response and
+            // land in the caller's file-persisted nightly log (#156 audit trail); the
+            // server-side per-entry console.log in distillChunk stays as well.
+            const dropped = [];
+            const entries = await (0, distiller_js_1.distillSession)(llm, conversationText, project ?? "unknown", date, chunkSize, dropped);
+            // Phase 1 — embed every chunk up front (async). If ANY embed fails we
+            // never reach the insert, so nothing is stored.
+            const createdAt = new Date(date).toISOString();
+            const toStore = [];
             for (let i = 0; i < entries.length; i++) {
                 const entry = entries[i];
                 if (typeof entry !== "string" || !entry.trim())
                     continue;
-                const embedding = await (0, embedder_js_1.embed)(entry);
-                const id = storage.insertMemory(db, entry, embedding, {
-                    sourceAgent: source_agent ?? "unknown",
-                    // Per-chunk key: "<session_id>#<i>". The prefix matches the nightly
-                    // dedup check above, so a re-run of the same session is fully idempotent.
-                    sourceSession: sourcePrefix ? `${sourcePrefix}#${i}` : undefined,
-                    project: project ?? undefined,
-                    memoryType: "episode",
-                    privacy: privacy ?? "WORK",
-                    createdAt: new Date(date).toISOString(),
-                });
-                ids.push(id);
+                toStore.push({ entry, embedding: await (0, embedder_js_1.embed)(entry), i });
             }
-            res.status(201).json({ ids, distilled: ids.length });
+            // Phase 2 — insert all chunks in ONE transaction (fix 4). A segment's
+            // chunks are all-or-nothing: any insert failure rolls back the whole set
+            // and returns 500, so the content-blind segment-exact dedup never sees a
+            // half-stored segment and the retry re-distills cleanly. (Applies to the
+            // legacy whole-session path too — same loop.)
+            const insertAll = db.transaction(() => {
+                const out = [];
+                for (const { entry, embedding, i } of toStore) {
+                    out.push(storage.insertMemory(db, entry, embedding, {
+                        sourceAgent: source_agent ?? "unknown",
+                        // Per-chunk key: "<session_id>[#<segment_id>]#<i>". The prefix
+                        // matches the dedup checks above, so a re-run is idempotent.
+                        sourceSession: sourcePrefix ? `${sourcePrefix}#${i}` : undefined,
+                        project: project ?? undefined,
+                        memoryType: "episode",
+                        privacy: privacy ?? "WORK",
+                        createdAt,
+                    }));
+                }
+                return out;
+            });
+            const ids = insertAll();
+            res.status(201).json({
+                ids,
+                distilled: ids.length,
+                dropped: dropped.map((d) => (d.length > 120 ? `${d.slice(0, 120)}…` : d)),
+            });
         }
         catch (err) {
             res.status(500).json({ error: "Distillation failed", message: err instanceof Error ? err.message : String(err) });

package/dist/nightly-status.js

@@ -18,6 +18,7 @@ const node_os_1 = require("node:os");
 const node_child_process_1 = require("node:child_process");
 const db_js_1 = require("./db.js");
 const state_js_1 = require("./state.js");
+const llm_js_1 = require("./llm.js");
 const HICORTEX_HOME = (0, paths_js_1.hicortexHome)();
 const CONFIG_PATH = (0, node_path_1.join)(HICORTEX_HOME, "config.json");
 async function showNightlyStatus() {
@@ -36,7 +37,9 @@ async function showNightlyStatus() {
     }
     // LLM config
     try {
-        const config = JSON.parse((0, node_fs_1.readFileSync)(CONFIG_PATH, "utf-8"));
+        // No `?? {}` coercion: a null/invalid parse must fall through to the catch
+        // below (as it did pre-0.13.1) rather than print a fabricated-healthy status.
+        const config = (0, llm_js_1.applyModelsBlock)(JSON.parse((0, node_fs_1.readFileSync)(CONFIG_PATH, "utf-8")));
         const backend = config.llmBackend ?? "auto-detect";
         const model = config.llmModel ?? "default";
         const mode = config.mode === "client" ? "client → " + (config.serverUrl ?? "?") : "server (local)";

package/dist/nightly.d.ts

@@ -15,4 +15,6 @@ export declare function runNightly(options?: {
     captureOnly?: boolean;
     dbPath?: string;
     stateDir?: string;
+    /** #189 Tier-2 recovery: override discovery to now−N days for one run. */
+    recaptureWindowDays?: number;
 }): Promise<void>;