@gamaze/hicortex 0.19.4 → 0.19.5

package/dist/distiller.d.ts

@@ -38,10 +38,24 @@ export declare function extractConversationText(messages: unknown[], redactionCo
  * `droppedOut`, when provided, is filled with every entry the substance gate
  * discarded (full text). Callers use it to build a durable audit trail (#156);
  * omitting it leaves gate behaviour unchanged.
+ *
+ * `segmentLabel` (optional) identifies the caller's segment in the #339
+ * over-firing warning (e.g. the capture pipeline's segment_id). Purely for
+ * log correlation — omitting it falls back to "chunk".
  */
 export declare function distillSession(llm: LlmClient, conversation: string, projectName: string, date: string, chunkSizeChars?: number, droppedOut?: string[], 
 /** Called with each chunk's token usage (#5 budget metering). Optional. */
-onUsage?: (usage: LlmUsage) => void): Promise<DistilledEntry[]>;
+onUsage?: (usage: LlmUsage) => void, 
+/** Segment identifier for the #339 NO_EXTRACT warning. Optional. */
+segmentLabel?: string): Promise<DistilledEntry[]>;
+/**
+ * The NO_EXTRACT check distillChunk applies to an LLM response. EXPORTED and
+ * shared (not copy-pasted) with scripts/distill-ab-check/, whose counts must
+ * classify empty verdicts exactly as production does (#339 CR finding 3).
+ * Tolerant by design: a literal NO_EXTRACT anywhere in the first 20 chars
+ * counts (models prepend stray whitespace or a short phrase).
+ */
+export declare function isNoExtractResponse(result: string): boolean;
 /**
  * Reject ONLY structurally-empty distiller fragments before they become
  * memories (#156). The distiller occasionally emits leftovers that parse into
@@ -76,3 +90,34 @@ export interface DistilledEntry {
     content: string;
     memoryType: "experience" | "knowledge" | "decisions";
 }
+/**
+ * Map a single-letter type tag to the stored memory_type. Unknown/absent →
+ * experience (the pre-#216 default). `[L]` is explicitly rejected →
+ * experience: the distiller must NEVER emit learnings (that's the reflection
+ * stage's job), so a model that emits `[L]` is wrong and we do not propagate
+ * it as a learning.
+ *
+ * The single-letter tags ([E]/[K]/[D]) are unchanged from the raw-enum era —
+ * the model is taught these as "EXPERIENCE/KNOWLEDGE/DECISIONS" concepts in prompts.ts
+ * (ordinary English the model understands), and only the resulting STORED
+ * value changed in #264 (episode→experience, fact→knowledge, decision→
+ * decisions). The tag letters stay stable so neither the prompt nor the
+ * parser needs to change; only this mapping table moves.
+ *
+ * EXPORTED (with parseDistilledEntries) for scripts/distill-ab-check/ (#339 CR
+ * finding 3): the A/B harness computes its counts from each variant build's own
+ * parser instead of a copy-pasted mirror, so harness numbers are by construction
+ * the numbers that build's production would store. tests/distill-ab-parser-contract.test.ts
+ * pins the src and dist parsers against the same corpus.
+ */
+export declare function typeFromTag(letter: string | undefined): DistilledEntry["memoryType"];
+/**
+ * Parse distilled markdown into individual memory entries with type tags.
+ * Each bullet becomes a separate memory. The leading `[E]`/`[F]`/`[D]` type
+ * tag is extracted (→ memoryType), stripped from the stored content, and
+ * passed to `insertMemory` via the `memoryType` option (#216). Bullets with
+ * no tag default to "experience" (backward compatible with pre-#216 distiller
+ * output that never carried a tag). EXPORTED for the A/B harness — see
+ * typeFromTag's comment (#339 CR finding 3).
+ */
+export declare function parseDistilledEntries(markdown: string): DistilledEntry[];

package/dist/distiller.js

@@ -8,11 +8,28 @@ Object.defineProperty(exports, "__esModule", { value: true });
 exports.detectChunkSize = detectChunkSize;
 exports.extractConversationText = extractConversationText;
 exports.distillSession = distillSession;
+exports.isNoExtractResponse = isNoExtractResponse;
 exports.hasMinimalSubstance = hasMinimalSubstance;
+exports.typeFromTag = typeFromTag;
+exports.parseDistilledEntries = parseDistilledEntries;
 const prompts_js_1 = require("./prompts.js");
 const redact_js_1 = require("./redact.js");
 const MAX_TRANSCRIPT_CHARS = 80_000;
 const MIN_CONVERSATION_CHARS = 200;
+// #339 (2026-08-24 postmortem): NO_EXTRACT over-firing visibility threshold.
+// Real summary-led segments that the model wrongly abandoned ran 38-64K
+// denoised chars; genuine pure-status noise is ~2.6K. A segment larger than
+// this whose every chunk returns an empty LLM verdict is far more likely the
+// model pattern-matching a bookkeeping-heavy OPENING to the ephemera gate than
+// a legitimately empty segment — so it gets a warning line. Warning ONLY: no
+// auto-retry (cost); the goal is that silent segment loss shows up in the
+// nightly log instead of in a weeks-later eval.
+//
+// The threshold compares the PRE-CHUNKING conversation length, never the chunk
+// length: default ollama chunking (numCtx 8192 → ~19.6K chars) and the
+// small-model speed cap (20K) both keep every chunk at or below this number,
+// so a chunk-level check would be unreachable exactly where the incident lived.
+const NO_EXTRACT_WARN_MIN_CHARS = 20_000;
 // Chunk size limits by model parameter count (for local/CPU inference)
 // Small models are slow on CPU — cap input size to keep inference under ~60s
 const SMALL_MODEL_PARAMS = 8_000_000_000; // 8B — threshold for "small"
@@ -225,10 +242,16 @@ function extractConversationText(messages, redactionConfig) {
  * `droppedOut`, when provided, is filled with every entry the substance gate
  * discarded (full text). Callers use it to build a durable audit trail (#156);
  * omitting it leaves gate behaviour unchanged.
+ *
+ * `segmentLabel` (optional) identifies the caller's segment in the #339
+ * over-firing warning (e.g. the capture pipeline's segment_id). Purely for
+ * log correlation — omitting it falls back to "chunk".
  */
 async function distillSession(llm, conversation, projectName, date, chunkSizeChars, droppedOut, 
 /** Called with each chunk's token usage (#5 budget metering). Optional. */
-onUsage) {
+onUsage, 
+/** Segment identifier for the #339 NO_EXTRACT warning. Optional. */
+segmentLabel) {
     if (conversation.length < MIN_CONVERSATION_CHARS) {
         return [];
     }
@@ -241,9 +264,11 @@ onUsage) {
     const chunkSize = chunkSizeChars ?? MAX_TRANSCRIPT_CHARS;
     // If transcript fits in one chunk, distill directly (errors propagate)
     if (transcript.length <= chunkSize) {
-        const { entries, dropped } = await distillChunk(llm, transcript, projectName, date, onUsage);
+        const { entries, dropped, emptyVerdict } = await distillChunk(llm, transcript, projectName, date, onUsage);
         if (droppedOut)
             droppedOut.push(...dropped);
+        if (emptyVerdict)
+            warnSuspiciousEmptySegment(segmentLabel, conversation.length);
         return entries;
     }
     // Chunk large transcripts and distill each segment.
@@ -259,11 +284,14 @@ onUsage) {
     const allEntries = [];
     const seen = new Set();
     let chunkFailures = 0;
+    let emptyVerdicts = 0;
     let lastError = null;
     for (let i = 0; i < chunks.length; i++) {
         console.log(`[hicortex]     Chunk ${i + 1}/${chunks.length} (${chunks[i].length} chars)`);
         try {
-            const { entries, dropped } = await distillChunk(llm, chunks[i], projectName, date, onUsage);
+            const { entries, dropped, emptyVerdict } = await distillChunk(llm, chunks[i], projectName, date, onUsage);
+            if (emptyVerdict)
+                emptyVerdicts++;
             if (droppedOut)
                 droppedOut.push(...dropped);
             for (const entry of entries) {
@@ -292,8 +320,38 @@ onUsage) {
     if (chunkFailures > 0) {
         console.warn(`[hicortex]     Partial distillation: ${chunks.length - chunkFailures}/${chunks.length} chunks succeeded`);
     }
+    // #339 segment-level net (CR finding 1): fire when the WHOLE segment produced
+    // zero memories and every processed chunk returned an empty LLM verdict. Fires
+    // only with zero chunk failures — failed chunks are already loudly visible,
+    // and "all failed" throws above. The size gate lives INSIDE
+    // warnSuspiciousEmptySegment (strict >, pre-chunking conversation length).
+    if (allEntries.length === 0 &&
+        chunkFailures === 0 &&
+        emptyVerdicts === chunks.length) {
+        warnSuspiciousEmptySegment(segmentLabel, conversation.length);
+    }
     return allEntries;
 }
+/**
+ * #339 over-firing visibility net: an empty result this large is the silent-loss
+ * signature (real summary-led segments are 38-64K chars; legitimate pure-status
+ * noise is ~2.6K). Warning-only, content-free — segment id + size, nothing from
+ * the transcript. The empty SUCCESS semantics are unchanged (no throw, no
+ * retry): the cursor advances, but the loss is now VISIBLE in the nightly log
+ * instead of surfacing weeks later in an eval.
+ *
+ * The size check lives here, not at call sites, so no path can skip it. Strict
+ * comparison: exactly NO_EXTRACT_WARN_MIN_CHARS chars is a legitimate small
+ * segment and stays silent.
+ */
+function warnSuspiciousEmptySegment(segmentLabel, segmentChars) {
+    if (segmentChars <= NO_EXTRACT_WARN_MIN_CHARS)
+        return;
+    console.warn(`[hicortex]     Suspicious empty distillation: zero memories for a ${segmentChars}-char ` +
+        `${segmentLabel ? `segment ${segmentLabel}` : "segment"} ` +
+        `(every chunk returned NO_EXTRACT or nothing parseable) — segments this large almost always ` +
+        `contain extractable material; if this repeats, suspect gate over-firing (logged only, not retried)`);
+}
 /**
  * Distill a single chunk of conversation text.
  *
@@ -309,6 +367,13 @@ onUsage) {
  *
  * `dropped` carries entries the substance gate rejected (full text) so the
  * caller can surface them in a durable audit trail (#156).
+ *
+ * `emptyVerdict` is true when the chunk was processed successfully but the LLM's
+ * verdict contained nothing extractable: bare NO_EXTRACT, an empty response, or
+ * text that parses to zero bullets (prose — the silent twin of NO_EXTRACT,
+ * #339 CR finding 2). The caller aggregates these for the segment-level
+ * over-firing warning; entries extracted then dropped by the substance gate do
+ * NOT count (their drops are already logged).
  */
 async function distillChunk(llm, transcript, projectName, date, onUsage) {
     const prompt = (0, prompts_js_1.distillation)(projectName, date, transcript);
@@ -323,10 +388,8 @@ async function distillChunk(llm, transcript, projectName, date, onUsage) {
     // trip a budget.
     if (usage && onUsage)
         onUsage(usage);
-    if (!result)
-        return { entries: [], dropped: [] };
-    if (result === "NO_EXTRACT" || result.slice(0, 20).includes("NO_EXTRACT")) {
-        return { entries: [], dropped: [] };
+    if (!result || isNoExtractResponse(result)) {
+        return { entries: [], dropped: [], emptyVerdict: true };
     }
     const parsed = parseDistilledEntries(result);
     // Smoke alarm (PR #218 review): the prompt enforces topic-first, but models
@@ -356,7 +419,20 @@ async function distillChunk(llm, transcript, projectName, date, onUsage) {
         }
         console.log(`[hicortex]     Substance gate: dropped ${dropped.length}/${parsed.length} content-free fragment(s)`);
     }
-    return { entries, dropped };
+    // Parsed-zero bypass (#339 CR finding 2): a non-empty response with no
+    // NO_EXTRACT token that still parses to zero bullets is the silent twin of
+    // NO_EXTRACT — an empty verdict for the over-firing net.
+    return { entries, dropped, emptyVerdict: parsed.length === 0 };
+}
+/**
+ * The NO_EXTRACT check distillChunk applies to an LLM response. EXPORTED and
+ * shared (not copy-pasted) with scripts/distill-ab-check/, whose counts must
+ * classify empty verdicts exactly as production does (#339 CR finding 3).
+ * Tolerant by design: a literal NO_EXTRACT anywhere in the first 20 chars
+ * counts (models prepend stray whitespace or a short phrase).
+ */
+function isNoExtractResponse(result) {
+    return result === "NO_EXTRACT" || result.slice(0, 20).includes("NO_EXTRACT");
 }
 /**
  * Split transcript text into chunks at natural boundaries (double newlines).
@@ -445,6 +521,12 @@ function hasMinimalSubstance(entry) {
  * value changed in #264 (episode→experience, fact→knowledge, decision→
  * decisions). The tag letters stay stable so neither the prompt nor the
  * parser needs to change; only this mapping table moves.
+ *
+ * EXPORTED (with parseDistilledEntries) for scripts/distill-ab-check/ (#339 CR
+ * finding 3): the A/B harness computes its counts from each variant build's own
+ * parser instead of a copy-pasted mirror, so harness numbers are by construction
+ * the numbers that build's production would store. tests/distill-ab-parser-contract.test.ts
+ * pins the src and dist parsers against the same corpus.
  */
 function typeFromTag(letter) {
     switch (letter) {
@@ -467,7 +549,8 @@ function typeFromTag(letter) {
  * tag is extracted (→ memoryType), stripped from the stored content, and
  * passed to `insertMemory` via the `memoryType` option (#216). Bullets with
  * no tag default to "experience" (backward compatible with pre-#216 distiller
- * output that never carried a tag).
+ * output that never carried a tag). EXPORTED for the A/B harness — see
+ * typeFromTag's comment (#339 CR finding 3).
  */
 function parseDistilledEntries(markdown) {
     const entries = [];

package/dist/mcp-server.js

@@ -1198,7 +1198,14 @@ async function startServer(options = {}) {
                 distillUsage.prompt += u.prompt_tokens ?? 0;
                 distillUsage.completion += u.completion_tokens ?? 0;
                 distillUsage.total += u.total_tokens ?? 0;
-            });
+            }, 
+            // #339: identify this POST in the NO_EXTRACT over-firing warning —
+            // segment_id (incremental capture), else session_id (legacy), else none.
+            typeof segment_id === "string" && segment_id
+                ? segment_id
+                : typeof session_id === "string" && session_id
+                    ? session_id
+                    : undefined);
             // Phase 1 — embed every chunk up front (async). If ANY embed fails we
             // never reach the insert, so nothing is stored.
             const createdAt = new Date(date).toISOString();

package/dist/prompts.d.ts

@@ -19,20 +19,23 @@ export declare function reflection(memoriesBlock: string, recentLessons?: string
 /**
  * Distillation prompt. Extracts knowledge from a session transcript.
  *
- * LAYOUT (#329 item 6): the ~6.3KB of static instructions come FIRST and the
- * transcript LAST, so every distill call shares a byte-identical instruction
- * prefix and provider-side prompt prefix caching can actually hit (per-session
- * calls with the same project/date share everything up to the transcript; a
- * multi-segment session — the common capture shape — re-uses the cached prefix
- * for every segment after the first). The static block is the transcript-first
- * block MOVED plus ONE deliberate addition in the same change: the #329 item-5
- * [D]-override sentence in the NEVER-RECORD section ("a version bump, merge,
- * or count is NEVER [D]"). Anyone diffing distill behavior across this change
- * must baseline against BOTH the reorder and that wording addition. NOTE the
- * prefix is only
- * fully shared while project/date agree: "# Session Memory: ${date} -
- * ${projectName}" and the (${date}) format examples interpolate inside the
- * static block by design (the model needs the real date in its output format).
+ * LAYOUT (REVERTED 2026-08-24): transcript BEFORE the static instruction
+ * block — the pre-0.19.4 order. The #329 item-6 reorder (static-first, for
+ * provider prefix caching) was REVERTED after a deterministic A/B on real
+ * segments: with instructions first, the model over-fires NO_EXTRACT on
+ * summary-led and long mixed sessions (a real coding segment: 15 memories →
+ * 0; a real Hermes session: rich → 0; isolation proved the LAYOUT caused it,
+ * not the item-5 sentence, which is KEPT). Silent shape-dependent segment
+ * loss beats any caching win. Re-attempting instructions-first requires a
+ * gate fix that passes the A/B matrix harness first.
+ *
+ * #339 gate hardening (same day): the NO_EXTRACT rule now carries an explicit
+ * whole-transcript guard + counter-example (a summary-led session that
+ * contains later decisions MUST be extracted) — the over-firing mechanism was
+ * the model pattern-matching a bookkeeping-heavy OPENING to the ephemera gate
+ * and abandoning the whole segment. Companion visibility net (warning on
+ * large empty results) lives in distiller.ts; the release-gate harness is
+ * scripts/distill-ab-check/.
  */
 export declare function distillation(projectName: string, date: string, transcript: string): string;
 /**

package/dist/prompts.js

@@ -107,25 +107,31 @@ Respond with a JSON array. Empty array [] is a valid response.`;
 /**
  * Distillation prompt. Extracts knowledge from a session transcript.
  *
- * LAYOUT (#329 item 6): the ~6.3KB of static instructions come FIRST and the
- * transcript LAST, so every distill call shares a byte-identical instruction
- * prefix and provider-side prompt prefix caching can actually hit (per-session
- * calls with the same project/date share everything up to the transcript; a
- * multi-segment session — the common capture shape — re-uses the cached prefix
- * for every segment after the first). The static block is the transcript-first
- * block MOVED plus ONE deliberate addition in the same change: the #329 item-5
- * [D]-override sentence in the NEVER-RECORD section ("a version bump, merge,
- * or count is NEVER [D]"). Anyone diffing distill behavior across this change
- * must baseline against BOTH the reorder and that wording addition. NOTE the
- * prefix is only
- * fully shared while project/date agree: "# Session Memory: ${date} -
- * ${projectName}" and the (${date}) format examples interpolate inside the
- * static block by design (the model needs the real date in its output format).
+ * LAYOUT (REVERTED 2026-08-24): transcript BEFORE the static instruction
+ * block — the pre-0.19.4 order. The #329 item-6 reorder (static-first, for
+ * provider prefix caching) was REVERTED after a deterministic A/B on real
+ * segments: with instructions first, the model over-fires NO_EXTRACT on
+ * summary-led and long mixed sessions (a real coding segment: 15 memories →
+ * 0; a real Hermes session: rich → 0; isolation proved the LAYOUT caused it,
+ * not the item-5 sentence, which is KEPT). Silent shape-dependent segment
+ * loss beats any caching win. Re-attempting instructions-first requires a
+ * gate fix that passes the A/B matrix harness first.
+ *
+ * #339 gate hardening (same day): the NO_EXTRACT rule now carries an explicit
+ * whole-transcript guard + counter-example (a summary-led session that
+ * contains later decisions MUST be extracted) — the over-firing mechanism was
+ * the model pattern-matching a bookkeeping-heavy OPENING to the ephemera gate
+ * and abandoning the whole segment. Companion visibility net (warning on
+ * large empty results) lives in distiller.ts; the release-gate harness is
+ * scripts/distill-ab-check/.
  */
 function distillation(projectName, date, transcript) {
     return `You are a memory extraction agent. Analyze this AI session transcript and extract
 knowledge worth remembering long-term.
 
+SESSION TRANSCRIPT (project: ${projectName}, date: ${date}):
+${transcript}
+
 EXTRACT into this markdown format:
 
 # Session Memory: ${date} - ${projectName}
@@ -213,6 +219,23 @@ user-confirmed standardization it embodies qualifies.
 If EVERY item in the transcript is never-record ephemera, output ONLY:
 "NO_EXTRACT" — zero memories is the correct result for a pure-status segment.
 
+NO_EXTRACT guard (a verdict on the WHOLE transcript, never on its opening):
+"NO_EXTRACT" requires that NO durable decision, knowledge, or correction
+appears ANYWHERE in the transcript — including after long bookkeeping
+stretches. The opening is not evidence about the rest: real sessions often
+OPEN with bookkeeping (a compaction summary, a task notification, a status
+recap) and CONTAIN extractable material later. Read to the END of the
+transcript before deciding; NO_EXTRACT on a long, mixed session is almost
+always a mistake — when in doubt, extract the durable items.
+Counter-example (MUST be extracted, never NO_EXTRACT): a session opens with
+"Session summary: continuing the API migration; prior PR merged, tests
+green" but later the user confirms "standardize on the queue-based worker —
+make it the documented default" and corrects the assistant: "no, don't gate
+retries behind a flag — remove the flag entirely". That session yields at
+least a [D] standardization and an [E] correction; the summary opening
+changes nothing. Emitting NO_EXTRACT there would lose the only record of
+both.
+
 RULES:
 - Extract MAX 20 items total (quality over quantity)
 - Use EXACT names/versions/paths/numbers as they appear in the transcript —
@@ -231,9 +254,6 @@ RULES:
   "[Strong Negative] User rejected per-agent billing"). The subject always comes first.
 - Omit any section that has zero items (don't include empty sections)
 - If nothing worth extracting, output ONLY: "NO_EXTRACT"
-
-SESSION TRANSCRIPT (project: ${projectName}, date: ${date}):
-${transcript}
 `;
 }
 /**

package/openclaw.plugin.json

@@ -2,7 +2,7 @@
   "id": "hicortex",
   "name": "Hicortex — Long-term Memory That Learns",
   "description": "Your agents remember past decisions, avoid repeated mistakes, and get smarter every day. Nightly reflection generates actionable lessons that automatically update agent behavior.",
-  "version": "0.19.4",
+  "version": "0.19.5",
   "kind": "lifecycle",
   "skills": ["./skills/hicortex-memory"],
   "configSchema": {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@gamaze/hicortex",
-  "version": "0.19.4",
+  "version": "0.19.5",
   "description": "Persistent agent identity for AI agents \u2014 a hand-edited identity layer, nightly-distilled experience, and lessons injected every session, shared across your whole fleet. Works with Hermes, OpenClaw, Claude Code, and Pi.",
   "main": "dist/index.js",
   "bin": {