@gamaze/hicortex 0.15.3 → 0.16.1

package/README.md

@@ -122,7 +122,7 @@ The run is resumable — interrupt it any time and it continues where it stopped
 - **hicortex_update** — Fix incorrect memories (re-embeds on content change)
 - **hicortex_delete** — Remove memories with cascade cleanup
 
-Skills: `/learn` to save explicit learnings.
+Explicit learnings: call `hicortex_ingest` directly (capture is otherwise automatic, nightly).
 
 ## Context Layer
 
@@ -202,6 +202,7 @@ Config at `~/.hicortex/config.json`. Created by `init`. Key options:
 | `models` | Optional nested per-stage model overrides (`score`/`distill`/`reflect`/`classify`) — see [Advanced: per-stage models](#advanced-per-stage-models) |
 | `distillFallback` | `"strict"` (default) — abort on remote distill failure, retry next run; `"local"` — fall back to base model (lower quality, 0.9.0 behaviour) |
 | `authToken` | Bearer token for endpoint auth. Generated on first `init` in server mode. Find the active token with `hicortex status` or in `~/.hicortex/config.json`. |
+| `corsAllowedOrigins` | Browser origins allowed to read cross-origin responses, e.g. `["https://ui.example.com"]`. **Empty by default** — the server sends no `Access-Control-Allow-Origin` and never `Allow-Credentials`, so no external web page can read its data. The bundled `/viz` and `/context/ui` pages are same-origin and need no entry. |
 | `licenseKey` | Commercial license key (optional; for display in `hicortex status`) |
 | `domains` | Your memory domain list (`[{name, description}]`). Scaffolded by `init`; edit freely — see [Memory Domains & Tags](#memory-domains--tags) |
 | `weakPrimaryFloor` | Minimum similarity for a no-fit memory to keep a weak domain association (default: 0.45) |
@@ -225,6 +226,7 @@ Config at `~/.hicortex/config.json`. Created by `init`. Key options:
 | `recallMinSimilarity` | Relevance floor for index entries (default: 0.55; text-search matches always pass) |
 | `recallReshowTurns` | Turns before an already-shown memory may reappear in the same session (default: 30) |
 | `recallMinPromptChars` | Prompts shorter than this skip the recall index (default: 20) |
+| `recallTitleChars` | Chars of each memory's first line shown in an index entry (default: 150, range 40–400). Raised from 100 on 2026-08-02: with topic-first memory titles, 150 chars carries the subject *and* its claim, where 100 cut the claim mid-sentence. Costs roughly +74 tokens per 6-line block |
 | `sessionIntentWeight` | Blend weight of the session-intent rolling centroid in the recall search vector: `query = (1-w)·prompt + w·centroid` (default: 0.33; set 0 to disable — pure-prompt recall, the kill-switch). The first turn of a session searches with pure prompt and seeds the centroid; subsequent turns blend so recall follows the session's intent instead of being query-literal. The EMA rate (0.4) is a shipped constant, not configurable |
 | `dedupMergeThreshold` | Minimum cosine similarity for `hicortex dedup` to cluster memories as near-duplicates (default: 0.92) |
 | `supersessionMinSimilarity` | Minimum cosine similarity for a nightly supersession candidate pair (default: 0.80) |

package/dist/consolidate.d.ts

@@ -162,9 +162,10 @@ export declare function buildSupersessionPrompt(oldContent: string, newContent:
 export declare function parseSupersessionReply(reply: string): boolean | null;
 /**
  * Nightly supersession-detection stage. Scans memories/rowid > cursor whose
- * shape suggests a decision/correction, checks each against its older
- * same-shape neighbors, and links confirmed supersessions. Dry-run performs
- * discovery + the free idempotency check only — no LLM calls, no writes, no
+ * shape is supersedable (decision/correction/fact/state — isSupersedableShape),
+ * checks each against its older same-shape neighbors, and links confirmed
+ * supersessions. Dry-run performs discovery + the free idempotency check only —
+ * no LLM calls, no writes, no
  * cursor persistence (mirrors stageImportance/stageContentDomains's dry-run
  * convention of never spending budget on a preview).
  *

package/dist/consolidate.js

@@ -307,7 +307,10 @@ async function stageReflection(db, memories, llm, budget, embedFn, dryRun) {
             const severity = String(lo.severity ?? "important");
             const confidence = String(lo.confidence ?? "medium");
             const sourcePattern = String(lo.source_pattern ?? "");
-            let content = `## Lesson: ${lessonText}\n\n`;
+            // No `## Lesson:` prefix: memory_type='lesson' carries the type, and the
+            // text is the topic-first first line (display reads the first line, not a
+            // header parse — see lessons-context.ts / index.ts).
+            let content = `${lessonText}\n\n`;
             content += `**Type:** ${lessonType}\n`;
             content += `**Severity:** ${severity}\n`;
             content += `**Confidence:** ${confidence}\n`;
@@ -857,11 +860,21 @@ const SUPERSESSION_NEIGHBOR_POOL = 15;
 const SUPERSESSION_NEIGHBOR_TOP_K = 5;
 /** Candidate rows read per SQL page (call budget stops the loop well before this in practice). */
 const SUPERSESSION_BATCH_SIZE = 500;
-/** A memory whose content/type marks it as a decision or correction. */
-function isDecisionShape(mem) {
+/**
+ * A memory whose content/type marks it as a SUPERSEDABLE claim — one a newer
+ * memory about the same subject can replace. Decisions and corrections were the
+ * original scope; plain facts and project-state updates were added because an
+ * updated fact ("distillModel is X" → later "is Y") otherwise never gets a
+ * superseded_by link and both versions compete in recall forever. Ordinary
+ * episodic chatter and problem/solution history stay excluded: they record
+ * events, not mutable state, so there is nothing to supersede.
+ */
+function isSupersedableShape(mem) {
     return (mem.memory_type === "decision" ||
         mem.content.includes("[Decisions Made]") ||
-        mem.content.includes("[Corrections & Rejections]"));
+        mem.content.includes("[Corrections & Rejections]") ||
+        mem.content.includes("[Facts Learned]") ||
+        mem.content.includes("[Project State Changes]"));
 }
 /** True when a `superseded_by` link already exists between the pair, either direction. */
 function alreadySupersedeLinked(db, oldId, newId) {
@@ -881,9 +894,12 @@ function buildSupersessionPrompt(oldContent, newContent) {
     return (`You are checking whether a NEWER memory supersedes an OLDER one in an AI agent's long-term memory.\n\n` +
         `OLDER MEMORY:\n${trunc(oldContent)}\n\n` +
         `NEWER MEMORY:\n${trunc(newContent)}\n\n` +
-        `Does the NEWER memory reverse, replace, or invalidate the OLDER one — e.g. a later decision overturns an ` +
-        `earlier one, or a correction retracts a prior claim? Two memories that are merely related, or that both ` +
-        `still hold true, are NOT a supersession.\n` +
+        `Does the NEWER memory reverse, replace, update, or invalidate the OLDER one — e.g. a later decision ` +
+        `overturns an earlier one, a correction retracts a prior claim, or a later fact updates the SAME subject's ` +
+        `value/status that has since changed (e.g. "model is X" → "model is Y")? Reply true ONLY for a genuine ` +
+        `replacement of the same fact/decision. Two memories that are merely related, or that can both still be ` +
+        `true — even about the same project or entity (different facts, an addition, an elaboration) — are NOT a ` +
+        `supersession.\n` +
         `Reply with ONLY a JSON object, no prose: {"superseded": true} or {"superseded": false}.`);
 }
 /**
@@ -932,16 +948,17 @@ async function findOlderNeighbors(db, candidate, embedFn, minSimilarity) {
     return storage
         .vectorSearch(db, embedding, SUPERSESSION_NEIGHBOR_POOL, [candidate.id])
         .filter((n) => n.created_at < candidate.created_at &&
-        isDecisionShape(n) &&
+        isSupersedableShape(n) &&
         (0, retrieval_js_1.l2ToCosine)(n.distance) >= minSimilarity)
         .sort((a, b) => (0, retrieval_js_1.l2ToCosine)(b.distance) - (0, retrieval_js_1.l2ToCosine)(a.distance))
         .slice(0, SUPERSESSION_NEIGHBOR_TOP_K);
 }
 /**
  * Nightly supersession-detection stage. Scans memories/rowid > cursor whose
- * shape suggests a decision/correction, checks each against its older
- * same-shape neighbors, and links confirmed supersessions. Dry-run performs
- * discovery + the free idempotency check only — no LLM calls, no writes, no
+ * shape is supersedable (decision/correction/fact/state — isSupersedableShape),
+ * checks each against its older same-shape neighbors, and links confirmed
+ * supersessions. Dry-run performs discovery + the free idempotency check only —
+ * no LLM calls, no writes, no
  * cursor persistence (mirrors stageImportance/stageContentDomains's dry-run
  * convention of never spending budget on a preview).
  *
@@ -965,9 +982,16 @@ async function stageSupersession(db, llm, budget, embedFn, dryRun, stateDir, opt
     const maxCalls = validNumber(options.maxCalls, exports.DEFAULT_SUPERSESSION_MAX_CALLS, (n) => n >= 0);
     const startCursor = (0, state_js_1.loadState)(stateDir).supersessionCursor ?? 0;
     const rows = db
-        .prepare(`SELECT rowid AS __rowid, * FROM memories
+        .prepare(
+    // Candidate shape must mirror isSupersedableShape() exactly — keep the two
+    // in lockstep (an inline SQL copy, so drift here silently narrows scope).
+    `SELECT rowid AS __rowid, * FROM memories
        WHERE rowid > ?
-         AND (memory_type = 'decision' OR content LIKE '%[Decisions Made]%' OR content LIKE '%[Corrections & Rejections]%')
+         AND (memory_type = 'decision'
+              OR content LIKE '%[Decisions Made]%'
+              OR content LIKE '%[Corrections & Rejections]%'
+              OR content LIKE '%[Facts Learned]%'
+              OR content LIKE '%[Project State Changes]%')
        ORDER BY rowid ASC LIMIT ?`)
         .all(startCursor, SUPERSESSION_BATCH_SIZE);
     let scanned = 0;

package/dist/db.js

@@ -126,20 +126,33 @@ CREATE INDEX IF NOT EXISTS idx_memories_created ON memories(created_at);
 CREATE INDEX IF NOT EXISTS idx_links_source ON memory_links(source_id);
 CREATE INDEX IF NOT EXISTS idx_links_target ON memory_links(target_id);
 `;
+// Fielded FTS5 (#205, migration v10 "fts_fielded"): three columns so bm25()
+// can weight matches per field (body / project / domain). Column order matters
+// — the weights passed to `bm25(memories_fts, w_body, w_project, w_domain)` in
+// storage.searchFts are positional on this declaration. `domain` reads the
+// derived PRIMARY `memories.domain` (the argmax-weight tag from the classifier,
+// set nightly), NOT a memory_tags join — that was judged too fiddly for this
+// phase (a multi-table FTS trigger is fragile and rebuilds on every tag edit).
+// `content_rowid='rowid'` is preserved for lockstep with the legacy schema.
 const FTS_SCHEMA = `
 CREATE VIRTUAL TABLE IF NOT EXISTS memories_fts USING fts5(
     content,
+    project,
+    domain,
     content_rowid='rowid'
 );
 
 CREATE TRIGGER IF NOT EXISTS memories_fts_insert AFTER INSERT ON memories
 BEGIN
-    INSERT INTO memories_fts (rowid, content) VALUES (NEW.rowid, NEW.content);
+    INSERT INTO memories_fts (rowid, content, project, domain)
+    VALUES (NEW.rowid, NEW.content, NEW.project, NEW.domain);
 END;
 
-CREATE TRIGGER IF NOT EXISTS memories_fts_update AFTER UPDATE OF content ON memories
+CREATE TRIGGER IF NOT EXISTS memories_fts_update AFTER UPDATE OF content, project, domain ON memories
 BEGIN
-    UPDATE memories_fts SET content = NEW.content WHERE rowid = NEW.rowid;
+    UPDATE memories_fts
+       SET content = NEW.content, project = NEW.project, domain = NEW.domain
+     WHERE rowid = NEW.rowid;
 END;
 
 CREATE TRIGGER IF NOT EXISTS memories_fts_delete AFTER DELETE ON memories
@@ -361,6 +374,73 @@ const MIGRATIONS = [
             db.exec("CREATE INDEX IF NOT EXISTS idx_dedup_log_source_session ON dedup_log(source_session)");
         },
     },
+    {
+        version: 10,
+        name: "fts_fielded",
+        up: (db) => {
+            // #205 fielded BM25F. FTS5 cannot be ALTERed (see the virtual-table rule
+            // above), so the single-column `memories_fts(content)` from pre-v10 must
+            // be dropped + recreated multi-column (`content, project, domain`) and
+            // rebuilt from the canonical `memories` rows. The runner wraps up() in a
+            // single transaction (db.ts:216) — that transaction IS the crash window:
+            // a power loss mid-migration rolls back, leaving the OLD single-column
+            // FTS intact (recall falls back to vector-only until the next open;
+            // never silently empty). The nightly capture lock does NOT cover
+            // initDb migrations, so the tx guard is the only safety net.
+            //
+            // Idempotent by construction: DROP IF EXISTS + CREATE + rebuild. On a
+            // fresh DB (where FTS_SCHEMA already created the multi-column form) this
+            // is a 0-row rebuild — wasted but harmless. On a legacy single-column DB
+            // it converts in place. Re-running against an already-migrated DB is a
+            // no-op shape + a refill of the same rows.
+            //
+            // Triggers are dropped + recreated to pick up the new column list (the
+            // pre-v10 update trigger fired only on `UPDATE OF content`; the new one
+            // also fires on project/domain updates so a tag reclassification lands
+            // in FTS without a content edit).
+            db.exec("DROP TRIGGER IF EXISTS memories_fts_insert");
+            db.exec("DROP TRIGGER IF EXISTS memories_fts_update");
+            db.exec("DROP TRIGGER IF EXISTS memories_fts_delete");
+            db.exec("DROP TABLE IF EXISTS memories_fts");
+            db.exec(`
+        CREATE VIRTUAL TABLE memories_fts USING fts5(
+          content,
+          project,
+          domain,
+          content_rowid='rowid'
+        )
+      `);
+            // COALESCE on project/domain because FTS5 stores NULL as no-tokens,
+            // which is what we want for unscoped memories (NULL domain = not yet
+            // classified; NULL project = no cwd-derived label). content is NOT NULL
+            // by the insert contract.
+            db.exec(`
+        INSERT INTO memories_fts (rowid, content, project, domain)
+        SELECT rowid, content, COALESCE(project, ''), COALESCE(domain, '') FROM memories
+      `);
+            db.exec(`
+        CREATE TRIGGER memories_fts_insert AFTER INSERT ON memories
+        BEGIN
+          INSERT INTO memories_fts (rowid, content, project, domain)
+          VALUES (NEW.rowid, NEW.content, NEW.project, NEW.domain);
+        END
+      `);
+            db.exec(`
+        CREATE TRIGGER memories_fts_update AFTER UPDATE OF content, project, domain ON memories
+        BEGIN
+          UPDATE memories_fts
+             SET content = NEW.content, project = NEW.project, domain = NEW.domain
+           WHERE rowid = NEW.rowid;
+        END
+      `);
+            db.exec(`
+        CREATE TRIGGER memories_fts_delete AFTER DELETE ON memories
+        BEGIN
+          DELETE FROM memories_fts WHERE rowid = OLD.rowid;
+        END
+      `);
+        },
+    },
 ];
 /**
  * Run all pending migrations against the database.

package/dist/distiller.js

@@ -314,6 +314,14 @@ async function distillChunk(llm, transcript, projectName, date) {
         return { entries: [], dropped: [] };
     }
     const parsed = parseDistilledEntries(result);
+    // Smoke alarm (PR #218 review): the prompt enforces topic-first, but models
+    // sometimes ignore constraints (cf. the prior max-15-bullet failure). Count
+    // entries that still look actor-led or bracket-led so a format regression
+    // shows in nightly logs, not months later in the next eval. Non-blocking.
+    const offTopic = parsed.filter((e) => /^\s*(user|ai|the user|assistant)\b/i.test(e) || /^\s*\[/.test(e)).length;
+    if (parsed.length > 0 && offTopic > 0) {
+        console.log(`[hicortex]     topic-first check: ${offTopic}/${parsed.length} entries look actor/bracket-led (prompt may be ignored)`);
+    }
     const entries = [];
     const dropped = [];
     for (const entry of parsed) {
@@ -409,23 +417,22 @@ function hasMinimalSubstance(entry) {
 function parseDistilledEntries(markdown) {
     const entries = [];
     const lines = markdown.split("\n");
-    let currentSection = "";
     for (const line of lines) {
         const trimmed = line.trim();
-        // Section headers
-        if (trimmed.startsWith("### ")) {
-            currentSection = trimmed.slice(4).trim();
+        // Skip all markdown headers (session title, classification, section
+        // headings). Sections are NOT prefixed onto entries: each bullet already
+        // starts with its [SUBJECT] (topic-first, enforced by prompts.ts), and
+        // prepending "[Section]" re-introduced the category-first prefix the
+        // 2026-08-02 corpus rewrite removed. The section label is unused
+        // downstream (distilled memories all store memory_type='episode').
+        if (trimmed.startsWith("# ") ||
+            trimmed.startsWith("## ") ||
+            trimmed.startsWith("### ")) {
             continue;
         }
-        // Skip top-level headers and classification
-        if (trimmed.startsWith("# ") || trimmed.startsWith("## "))
-            continue;
-        // Bullet items are individual memories
+        // Bullet items are individual, already topic-first memories.
         if (trimmed.startsWith("- ") && trimmed.length > 5) {
-            const entry = currentSection
-                ? `[${currentSection}] ${trimmed.slice(2)}`
-                : trimmed.slice(2);
-            entries.push(entry);
+            entries.push(trimmed.slice(2));
         }
     }
     return entries;

package/dist/eval/recall-sweep.d.ts

@@ -54,6 +54,29 @@
  * badly post-shift. Report written to data/eval-recall-sweep/report.md and
  * printed to stdout.
  *
+ * ## Section 4 — SCOPE dimension (#203 soft project affinity)
+ *
+ * A THIRD family on a SEPARATE project-labeled corpus (the blend-sweep corpus
+ * above has no project labels). Goal: prove #203's soft project affinity drops
+ * cross-scope noise (the boat/battery case — a "hardware" query surfacing
+ * "marine" memories on a shared token like "battery") below the cap WITHOUT a
+ * hard filter, and WITHOUT losing same-scope recall.
+ *
+ *   Corpus: 6 hardware + 6 marine memories. 3 marine memories are
+ *   CONTAMINATION SEEDS (share a token with the hardware queries); 3 are
+ *   marine-only fillers (control).
+ *
+ *   Each hardware query runs TWICE on the same static DB:
+ *     scope OFF — no `project` sent (byte-identical to pre-#203).
+ *     scope ON  — `project: "hardware"` (computeScore adds +projectAffinity 0.15
+ *                 to hardware memories; marine gets 0; no hard filter).
+ *
+ *   Metrics: contamination@5 (marine in top-5 / 5 — LOWER is better),
+ *   recall@5 (gold hardware memory surfaced), sameScope@5 (hardware in top-5).
+ *   Scope is orthogonal to the blend, so the blend weight is held at 0 here —
+ *   `project` is the ONLY variable. Same invariants (noStrengthen, real
+ *   embedder, uniform metadata, neverCalledEmbed self-check).
+ *
  * Run:  npm run eval:recall-sweep   (==  node dist/eval/recall-sweep.js)
  */
 export {};