@gamaze/hicortex 0.4.5 → 0.4.6

package/dist/distiller.js

@@ -205,15 +205,24 @@ async function distillSession(llm, conversation, projectName, date, chunkSizeCha
     }
     // Use provided chunk size or default to no chunking
     const chunkSize = chunkSizeChars ?? MAX_TRANSCRIPT_CHARS;
-    // If transcript fits in one chunk, distill directly
+    // If transcript fits in one chunk, distill directly (errors propagate)
     if (transcript.length <= chunkSize) {
         return distillChunk(llm, transcript, projectName, date);
     }
-    // Chunk large transcripts and distill each segment
+    // Chunk large transcripts and distill each segment.
+    //
+    // Partial success policy:
+    //   - If SOME chunks succeed and SOME fail, return the partial results and
+    //     log a warning. The caller gets *something* and can decide whether
+    //     to count this as success.
+    //   - If ALL chunks fail, throw — no useful output, and the caller needs
+    //     to know this session hit a transient error.
     const chunks = splitIntoChunks(transcript, chunkSize);
     console.log(`[hicortex]     Chunking ${transcript.length} chars into ${chunks.length} segments`);
     const allEntries = [];
     const seen = new Set();
+    let chunkFailures = 0;
+    let lastError = null;
     for (let i = 0; i < chunks.length; i++) {
         console.log(`[hicortex]     Chunk ${i + 1}/${chunks.length} (${chunks[i].length} chars)`);
         try {
@@ -230,30 +239,45 @@ async function distillSession(llm, conversation, projectName, date, chunkSizeCha
         catch (err) {
             const msg = err instanceof Error ? err.message : String(err);
             console.error(`[hicortex]     Chunk ${i + 1} failed: ${msg}`);
-            // Continue with remaining chunks — partial extraction is better than none
+            chunkFailures++;
+            lastError = err instanceof Error ? err : new Error(msg);
         }
     }
+    // If every chunk failed, the session wasn't actually processed. Throw so
+    // the nightly pipeline knows to retry this session next run.
+    if (chunkFailures === chunks.length) {
+        throw lastError ?? new Error("All distillation chunks failed");
+    }
+    if (chunkFailures > 0) {
+        console.warn(`[hicortex]     Partial distillation: ${chunks.length - chunkFailures}/${chunks.length} chunks succeeded`);
+    }
     return allEntries;
 }
 /**
  * Distill a single chunk of conversation text.
+ *
+ * Behaviour contract:
+ *   - Returns `[]` for legitimate empty results (NO_EXTRACT, empty LLM response,
+ *     transcript produced no entries). These are terminal states — the chunk was
+ *     processed successfully, there's just nothing worth keeping.
+ *   - Throws for transient errors (LLM unreachable, HTTP 4xx/5xx, timeout, model
+ *     not found, rate limit). These MUST propagate so the nightly pipeline can
+ *     distinguish "nothing to extract" from "try again later" and avoid
+ *     advancing the last-run watermark past sessions it never actually processed.
  */
 async function distillChunk(llm, transcript, projectName, date) {
     const prompt = (0, prompts_js_1.distillation)(projectName, date, transcript);
-    try {
-        const result = await llm.completeDistill(prompt);
-        if (!result)
-            return [];
-        if (result === "NO_EXTRACT" || result.slice(0, 20).includes("NO_EXTRACT")) {
-            return [];
-        }
-        return parseDistilledEntries(result);
-    }
-    catch (err) {
-        const msg = err instanceof Error ? err.message : String(err);
-        console.error(`[hicortex] Distillation LLM error: ${msg}`);
+    // NOTE: Intentionally no try/catch here. Transient LLM errors (network
+    // failures, 4xx/5xx, model-not-found, timeouts) propagate up to the caller
+    // so the nightly pipeline can treat them as "retry later" instead of
+    // "processed successfully with zero extractions".
+    const result = await llm.completeDistill(prompt);
+    if (!result)
+        return [];
+    if (result === "NO_EXTRACT" || result.slice(0, 20).includes("NO_EXTRACT")) {
         return [];
     }
+    return parseDistilledEntries(result);
 }
 /**
  * Split transcript text into chunks at natural boundaries (double newlines).

package/dist/llm.d.ts

@@ -69,6 +69,25 @@ export declare function claudeCliConfig(claudePath: string): LlmConfig;
  * Returns the model name if available, null otherwise.
  */
 export declare function probeOllama(baseUrl?: string): Promise<string | null>;
+/**
+ * Pre-flight health check for a specific Ollama endpoint + model.
+ * Returns { ok, reason } so callers can log a clear abort message.
+ *
+ *   - `ok: true` — endpoint reachable AND the requested model appears in
+ *     `/api/tags`. Safe to proceed with a batch distillation run.
+ *   - `ok: false, reason: "unreachable"` — network failure or non-2xx.
+ *   - `ok: false, reason: "model_missing"` — endpoint is up but the
+ *     model isn't listed (the exact case that caused data loss when
+ *     mhac-pro's Ollama didn't have the distill model loaded).
+ *
+ * Matches on exact name OR name prefix ("qwen3.5:35b" matches "qwen3.5:35b-a3b").
+ */
+export declare function probeOllamaModel(baseUrl: string, modelName: string): Promise<{
+    ok: true;
+} | {
+    ok: false;
+    reason: "unreachable" | "model_missing";
+}>;
 /**
  * For batch operations (nightly pipeline), prefer Ollama when available.
  * Claude CLI has strict rate limits that kill batch distillation.

package/dist/llm.js

@@ -26,6 +26,7 @@ exports.resolveLlmConfigForCC = resolveLlmConfigForCC;
 exports.findClaudeBinary = findClaudeBinary;
 exports.claudeCliConfig = claudeCliConfig;
 exports.probeOllama = probeOllama;
+exports.probeOllamaModel = probeOllamaModel;
 exports.preferOllamaForBatch = preferOllamaForBatch;
 const node_fs_1 = require("node:fs");
 const node_path_1 = require("node:path");
@@ -367,6 +368,35 @@ async function probeOllama(baseUrl = "http://localhost:11434") {
         return null;
     }
 }
+/**
+ * Pre-flight health check for a specific Ollama endpoint + model.
+ * Returns { ok, reason } so callers can log a clear abort message.
+ *
+ *   - `ok: true` — endpoint reachable AND the requested model appears in
+ *     `/api/tags`. Safe to proceed with a batch distillation run.
+ *   - `ok: false, reason: "unreachable"` — network failure or non-2xx.
+ *   - `ok: false, reason: "model_missing"` — endpoint is up but the
+ *     model isn't listed (the exact case that caused data loss when
+ *     mhac-pro's Ollama didn't have the distill model loaded).
+ *
+ * Matches on exact name OR name prefix ("qwen3.5:35b" matches "qwen3.5:35b-a3b").
+ */
+async function probeOllamaModel(baseUrl, modelName) {
+    try {
+        const resp = await fetch(`${baseUrl.replace(/\/$/, "")}/api/tags`, {
+            signal: AbortSignal.timeout(5000),
+        });
+        if (!resp.ok)
+            return { ok: false, reason: "unreachable" };
+        const data = (await resp.json());
+        const models = data.models ?? [];
+        const found = models.some((m) => m.name === modelName || m.name.startsWith(modelName + ":"));
+        return found ? { ok: true } : { ok: false, reason: "model_missing" };
+    }
+    catch {
+        return { ok: false, reason: "unreachable" };
+    }
+}
 /**
  * For batch operations (nightly pipeline), prefer Ollama when available.
  * Claude CLI has strict rate limits that kill batch distillation.

package/dist/nightly.js

@@ -182,6 +182,24 @@ async function runNightly(options = {}) {
         }
         // Step 2: Distill each session
         let memoriesIngested = 0;
+        let hadTransientFailure = false;
+        // Pre-flight health check for a remote distill endpoint.
+        // If the distill provider is Ollama on a remote host and that host (or the
+        // required model) is unreachable, abort BEFORE touching any sessions —
+        // prevents the data-loss bug where lastRun advances past sessions that
+        // were never actually processed.
+        if (batches.length > 0 && llmConfig.distillBaseUrl && (llmConfig.distillProvider ?? llmConfig.provider) === "ollama") {
+            const distillModel = llmConfig.distillModel ?? llmConfig.model;
+            const health = await (0, llm_js_1.probeOllamaModel)(llmConfig.distillBaseUrl, distillModel);
+            if (!health.ok) {
+                const reason = health.reason === "unreachable"
+                    ? `distill endpoint unreachable (${llmConfig.distillBaseUrl})`
+                    : `distill model not loaded (${distillModel} missing on ${llmConfig.distillBaseUrl})`;
+                console.error(`[hicortex] ABORT: ${reason} — will retry next run, lastRun unchanged`);
+                hadTransientFailure = true;
+                batches.length = 0; // Skip the distillation loop entirely
+            }
+        }
         // Detect safe chunk size based on model context window
         const chunkSize = await (0, distiller_js_1.detectChunkSize)(llmConfig.provider, llmConfig.distillModel ?? llmConfig.model, llmConfig.baseUrl);
         for (const batch of batches) {
@@ -190,6 +208,20 @@ async function runNightly(options = {}) {
                 console.log(`[hicortex]   Skip ${batch.sessionId.slice(0, 8)} (${batch.projectName}): too short`);
                 continue;
             }
+            // Server-mode per-session dedup: skip sessions already in the DB.
+            // Client mode gets this for free via the server's /ingest endpoint;
+            // server mode writes directly via storage.insertMemory and needs
+            // an explicit check. This makes retries of previously-failed runs
+            // idempotent.
+            if (!dryRun) {
+                const existing = db
+                    .prepare("SELECT COUNT(*) as c FROM memories WHERE source_session = ?")
+                    .get(batch.sessionId);
+                if (existing.c > 0) {
+                    console.log(`[hicortex]   Skip ${batch.sessionId.slice(0, 8)} (${batch.projectName}): already ingested`);
+                    continue;
+                }
+            }
             console.log(`[hicortex]   Distilling ${batch.sessionId.slice(0, 8)} (${batch.projectName}, ${batch.date})`);
             if (dryRun) {
                 console.log(`[hicortex]     [dry-run] Would distill ${transcript.length} chars`);
@@ -224,7 +256,8 @@ async function runNightly(options = {}) {
             }
             catch (err) {
                 const msg = err instanceof Error ? err.message : String(err);
-                console.error(`[hicortex]     Distillation failed: ${msg}`);
+                console.error(`[hicortex]     Distillation failed: ${msg} — will retry next run`);
+                hadTransientFailure = true;
             }
         }
         console.log(`[hicortex] Distillation complete: ${memoriesIngested} new memories`);
@@ -241,8 +274,17 @@ async function runNightly(options = {}) {
             console.log(`[hicortex] CLAUDE.md updated: ${injection.lessonsCount} lessons at ${injection.path}`);
         }
         // Step 5: Update last-run timestamp
+        // CRITICAL: only advance lastRun if every session was processed without
+        // a transient failure. Otherwise failed sessions would be permanently
+        // lost — they'd be older than the new lastRun and never retried.
         if (!dryRun) {
-            writeLastRun();
+            if (hadTransientFailure) {
+                console.warn(`[hicortex] Not advancing lastRun — one or more sessions failed. ` +
+                    `They will be retried on the next run.`);
+            }
+            else {
+                writeLastRun();
+            }
         }
         console.log(`[hicortex] Nightly pipeline complete.`);
     }
@@ -329,6 +371,22 @@ async function runClientNightly(config, dryRun) {
             writeLastRun();
         return;
     }
+    // Pre-flight health check for a remote distill endpoint (client mode).
+    // If the distill provider is Ollama on a remote host and the required model
+    // isn't loaded, abort BEFORE touching any sessions — same data-loss fix
+    // as server mode.
+    let hadTransientFailure = false;
+    if (llmConfig.distillBaseUrl && (llmConfig.distillProvider ?? llmConfig.provider) === "ollama") {
+        const distillModel = llmConfig.distillModel ?? llmConfig.model;
+        const health = await (0, llm_js_1.probeOllamaModel)(llmConfig.distillBaseUrl, distillModel);
+        if (!health.ok) {
+            const reason = health.reason === "unreachable"
+                ? `distill endpoint unreachable (${llmConfig.distillBaseUrl})`
+                : `distill model not loaded (${distillModel} missing on ${llmConfig.distillBaseUrl})`;
+            console.error(`[hicortex] ABORT: ${reason} — will retry next run, lastRun unchanged`);
+            return; // Don't touch lastRun; next trigger retries the same sessions
+        }
+    }
     // Distill each session and POST to server
     let memoriesIngested = 0;
     let sessionsSent = 0;
@@ -388,6 +446,7 @@ async function runClientNightly(config, dryRun) {
                 }
                 else {
                     console.error(`[hicortex]     Ingest failed (${resp.status}): ${result.error}`);
+                    hadTransientFailure = true;
                 }
             }
             if (sessionCount > 0) {
@@ -396,7 +455,8 @@ async function runClientNightly(config, dryRun) {
             }
         }
         catch (err) {
-            console.error(`[hicortex]     Failed: ${err instanceof Error ? err.message : String(err)}`);
+            console.error(`[hicortex]     Distillation failed: ${err instanceof Error ? err.message : String(err)} — will retry next run`);
+            hadTransientFailure = true;
         }
     }
     // Inject lessons from server into CLAUDE.md
@@ -408,8 +468,17 @@ async function runClientNightly(config, dryRun) {
             console.error(`[hicortex] CLAUDE.md injection failed: ${err instanceof Error ? err.message : String(err)}`);
         }
     }
-    if (!dryRun)
-        writeLastRun();
+    // Only advance lastRun if every session was processed without a transient
+    // failure. Otherwise failed sessions would be permanently lost.
+    if (!dryRun) {
+        if (hadTransientFailure) {
+            console.warn(`[hicortex] Not advancing lastRun — one or more sessions failed. ` +
+                `They will be retried on the next run.`);
+        }
+        else {
+            writeLastRun();
+        }
+    }
     console.log(`[hicortex] Client nightly complete: ${memoriesIngested} memories from ${sessionsSent} sessions → ${serverUrl}`);
 }
 /**

package/openclaw.plugin.json

@@ -2,7 +2,7 @@
   "id": "hicortex",
   "name": "Hicortex — Long-term Memory That Learns",
   "description": "Your agents remember past decisions, avoid repeated mistakes, and get smarter every day. Nightly reflection generates actionable lessons that automatically update agent behavior.",
-  "version": "0.4.5",
+  "version": "0.4.6",
   "kind": "lifecycle",
   "skills": ["./skills/hicortex-memory", "./skills/hicortex-learn", "./skills/hicortex-activate"],
   "configSchema": {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@gamaze/hicortex",
-  "version": "0.4.5",
+  "version": "0.4.6",
   "description": "Human-like memory for self-improving AI agents. Automatic capturing, nightly reflection, and cross-agent learning. Works with Claude Code and OpenClaw.",
   "main": "dist/index.js",
   "bin": {