@gamaze/hicortex 0.15.2 → 0.16.0

package/README.md

@@ -122,7 +122,7 @@ The run is resumable — interrupt it any time and it continues where it stopped
 - **hicortex_update** — Fix incorrect memories (re-embeds on content change)
 - **hicortex_delete** — Remove memories with cascade cleanup
 
-Skills: `/learn` to save explicit learnings.
+Explicit learnings: call `hicortex_ingest` directly (capture is otherwise automatic, nightly).
 
 ## Context Layer
 
@@ -202,6 +202,7 @@ Config at `~/.hicortex/config.json`. Created by `init`. Key options:
 | `models` | Optional nested per-stage model overrides (`score`/`distill`/`reflect`/`classify`) — see [Advanced: per-stage models](#advanced-per-stage-models) |
 | `distillFallback` | `"strict"` (default) — abort on remote distill failure, retry next run; `"local"` — fall back to base model (lower quality, 0.9.0 behaviour) |
 | `authToken` | Bearer token for endpoint auth. Generated on first `init` in server mode. Find the active token with `hicortex status` or in `~/.hicortex/config.json`. |
+| `corsAllowedOrigins` | Browser origins allowed to read cross-origin responses, e.g. `["https://ui.example.com"]`. **Empty by default** — the server sends no `Access-Control-Allow-Origin` and never `Allow-Credentials`, so no external web page can read its data. The bundled `/viz` and `/context/ui` pages are same-origin and need no entry. |
 | `licenseKey` | Commercial license key (optional; for display in `hicortex status`) |
 | `domains` | Your memory domain list (`[{name, description}]`). Scaffolded by `init`; edit freely — see [Memory Domains & Tags](#memory-domains--tags) |
 | `weakPrimaryFloor` | Minimum similarity for a no-fit memory to keep a weak domain association (default: 0.45) |
@@ -225,6 +226,7 @@ Config at `~/.hicortex/config.json`. Created by `init`. Key options:
 | `recallMinSimilarity` | Relevance floor for index entries (default: 0.55; text-search matches always pass) |
 | `recallReshowTurns` | Turns before an already-shown memory may reappear in the same session (default: 30) |
 | `recallMinPromptChars` | Prompts shorter than this skip the recall index (default: 20) |
+| `sessionIntentWeight` | Blend weight of the session-intent rolling centroid in the recall search vector: `query = (1-w)·prompt + w·centroid` (default: 0.33; set 0 to disable — pure-prompt recall, the kill-switch). The first turn of a session searches with pure prompt and seeds the centroid; subsequent turns blend so recall follows the session's intent instead of being query-literal. The EMA rate (0.4) is a shipped constant, not configurable |
 | `dedupMergeThreshold` | Minimum cosine similarity for `hicortex dedup` to cluster memories as near-duplicates (default: 0.92) |
 | `supersessionMinSimilarity` | Minimum cosine similarity for a nightly supersession candidate pair (default: 0.80) |
 | `supersessionMaxCalls` | Max classify-tier LLM calls the nightly's supersession stage spends per run (default: 30) |

package/dist/consolidate.d.ts

@@ -162,9 +162,10 @@ export declare function buildSupersessionPrompt(oldContent: string, newContent:
 export declare function parseSupersessionReply(reply: string): boolean | null;
 /**
  * Nightly supersession-detection stage. Scans memories/rowid > cursor whose
- * shape suggests a decision/correction, checks each against its older
- * same-shape neighbors, and links confirmed supersessions. Dry-run performs
- * discovery + the free idempotency check only — no LLM calls, no writes, no
+ * shape is supersedable (decision/correction/fact/state — isSupersedableShape),
+ * checks each against its older same-shape neighbors, and links confirmed
+ * supersessions. Dry-run performs discovery + the free idempotency check only —
+ * no LLM calls, no writes, no
  * cursor persistence (mirrors stageImportance/stageContentDomains's dry-run
  * convention of never spending budget on a preview).
  *

package/dist/consolidate.js

@@ -857,11 +857,21 @@ const SUPERSESSION_NEIGHBOR_POOL = 15;
 const SUPERSESSION_NEIGHBOR_TOP_K = 5;
 /** Candidate rows read per SQL page (call budget stops the loop well before this in practice). */
 const SUPERSESSION_BATCH_SIZE = 500;
-/** A memory whose content/type marks it as a decision or correction. */
-function isDecisionShape(mem) {
+/**
+ * A memory whose content/type marks it as a SUPERSEDABLE claim — one a newer
+ * memory about the same subject can replace. Decisions and corrections were the
+ * original scope; plain facts and project-state updates were added because an
+ * updated fact ("distillModel is X" → later "is Y") otherwise never gets a
+ * superseded_by link and both versions compete in recall forever. Ordinary
+ * episodic chatter and problem/solution history stay excluded: they record
+ * events, not mutable state, so there is nothing to supersede.
+ */
+function isSupersedableShape(mem) {
     return (mem.memory_type === "decision" ||
         mem.content.includes("[Decisions Made]") ||
-        mem.content.includes("[Corrections & Rejections]"));
+        mem.content.includes("[Corrections & Rejections]") ||
+        mem.content.includes("[Facts Learned]") ||
+        mem.content.includes("[Project State Changes]"));
 }
 /** True when a `superseded_by` link already exists between the pair, either direction. */
 function alreadySupersedeLinked(db, oldId, newId) {
@@ -881,9 +891,12 @@ function buildSupersessionPrompt(oldContent, newContent) {
     return (`You are checking whether a NEWER memory supersedes an OLDER one in an AI agent's long-term memory.\n\n` +
         `OLDER MEMORY:\n${trunc(oldContent)}\n\n` +
         `NEWER MEMORY:\n${trunc(newContent)}\n\n` +
-        `Does the NEWER memory reverse, replace, or invalidate the OLDER one — e.g. a later decision overturns an ` +
-        `earlier one, or a correction retracts a prior claim? Two memories that are merely related, or that both ` +
-        `still hold true, are NOT a supersession.\n` +
+        `Does the NEWER memory reverse, replace, update, or invalidate the OLDER one — e.g. a later decision ` +
+        `overturns an earlier one, a correction retracts a prior claim, or a later fact updates the SAME subject's ` +
+        `value/status that has since changed (e.g. "model is X" → "model is Y")? Reply true ONLY for a genuine ` +
+        `replacement of the same fact/decision. Two memories that are merely related, or that can both still be ` +
+        `true — even about the same project or entity (different facts, an addition, an elaboration) — are NOT a ` +
+        `supersession.\n` +
         `Reply with ONLY a JSON object, no prose: {"superseded": true} or {"superseded": false}.`);
 }
 /**
@@ -932,16 +945,17 @@ async function findOlderNeighbors(db, candidate, embedFn, minSimilarity) {
     return storage
         .vectorSearch(db, embedding, SUPERSESSION_NEIGHBOR_POOL, [candidate.id])
         .filter((n) => n.created_at < candidate.created_at &&
-        isDecisionShape(n) &&
+        isSupersedableShape(n) &&
         (0, retrieval_js_1.l2ToCosine)(n.distance) >= minSimilarity)
         .sort((a, b) => (0, retrieval_js_1.l2ToCosine)(b.distance) - (0, retrieval_js_1.l2ToCosine)(a.distance))
         .slice(0, SUPERSESSION_NEIGHBOR_TOP_K);
 }
 /**
  * Nightly supersession-detection stage. Scans memories/rowid > cursor whose
- * shape suggests a decision/correction, checks each against its older
- * same-shape neighbors, and links confirmed supersessions. Dry-run performs
- * discovery + the free idempotency check only — no LLM calls, no writes, no
+ * shape is supersedable (decision/correction/fact/state — isSupersedableShape),
+ * checks each against its older same-shape neighbors, and links confirmed
+ * supersessions. Dry-run performs discovery + the free idempotency check only —
+ * no LLM calls, no writes, no
  * cursor persistence (mirrors stageImportance/stageContentDomains's dry-run
  * convention of never spending budget on a preview).
  *
@@ -965,9 +979,16 @@ async function stageSupersession(db, llm, budget, embedFn, dryRun, stateDir, opt
     const maxCalls = validNumber(options.maxCalls, exports.DEFAULT_SUPERSESSION_MAX_CALLS, (n) => n >= 0);
     const startCursor = (0, state_js_1.loadState)(stateDir).supersessionCursor ?? 0;
     const rows = db
-        .prepare(`SELECT rowid AS __rowid, * FROM memories
+        .prepare(
+    // Candidate shape must mirror isSupersedableShape() exactly — keep the two
+    // in lockstep (an inline SQL copy, so drift here silently narrows scope).
+    `SELECT rowid AS __rowid, * FROM memories
        WHERE rowid > ?
-         AND (memory_type = 'decision' OR content LIKE '%[Decisions Made]%' OR content LIKE '%[Corrections & Rejections]%')
+         AND (memory_type = 'decision'
+              OR content LIKE '%[Decisions Made]%'
+              OR content LIKE '%[Corrections & Rejections]%'
+              OR content LIKE '%[Facts Learned]%'
+              OR content LIKE '%[Project State Changes]%')
        ORDER BY rowid ASC LIMIT ?`)
         .all(startCursor, SUPERSESSION_BATCH_SIZE);
     let scanned = 0;

package/dist/db.js

@@ -126,20 +126,33 @@ CREATE INDEX IF NOT EXISTS idx_memories_created ON memories(created_at);
 CREATE INDEX IF NOT EXISTS idx_links_source ON memory_links(source_id);
 CREATE INDEX IF NOT EXISTS idx_links_target ON memory_links(target_id);
 `;
+// Fielded FTS5 (#205, migration v10 "fts_fielded"): three columns so bm25()
+// can weight matches per field (body / project / domain). Column order matters
+// — the weights passed to `bm25(memories_fts, w_body, w_project, w_domain)` in
+// storage.searchFts are positional on this declaration. `domain` reads the
+// derived PRIMARY `memories.domain` (the argmax-weight tag from the classifier,
+// set nightly), NOT a memory_tags join — that was judged too fiddly for this
+// phase (a multi-table FTS trigger is fragile and rebuilds on every tag edit).
+// `content_rowid='rowid'` is preserved for lockstep with the legacy schema.
 const FTS_SCHEMA = `
 CREATE VIRTUAL TABLE IF NOT EXISTS memories_fts USING fts5(
     content,
+    project,
+    domain,
     content_rowid='rowid'
 );
 
 CREATE TRIGGER IF NOT EXISTS memories_fts_insert AFTER INSERT ON memories
 BEGIN
-    INSERT INTO memories_fts (rowid, content) VALUES (NEW.rowid, NEW.content);
+    INSERT INTO memories_fts (rowid, content, project, domain)
+    VALUES (NEW.rowid, NEW.content, NEW.project, NEW.domain);
 END;
 
-CREATE TRIGGER IF NOT EXISTS memories_fts_update AFTER UPDATE OF content ON memories
+CREATE TRIGGER IF NOT EXISTS memories_fts_update AFTER UPDATE OF content, project, domain ON memories
 BEGIN
-    UPDATE memories_fts SET content = NEW.content WHERE rowid = NEW.rowid;
+    UPDATE memories_fts
+       SET content = NEW.content, project = NEW.project, domain = NEW.domain
+     WHERE rowid = NEW.rowid;
 END;
 
 CREATE TRIGGER IF NOT EXISTS memories_fts_delete AFTER DELETE ON memories
@@ -361,6 +374,73 @@ const MIGRATIONS = [
             db.exec("CREATE INDEX IF NOT EXISTS idx_dedup_log_source_session ON dedup_log(source_session)");
         },
     },
+    {
+        version: 10,
+        name: "fts_fielded",
+        up: (db) => {
+            // #205 fielded BM25F. FTS5 cannot be ALTERed (see the virtual-table rule
+            // above), so the single-column `memories_fts(content)` from pre-v10 must
+            // be dropped + recreated multi-column (`content, project, domain`) and
+            // rebuilt from the canonical `memories` rows. The runner wraps up() in a
+            // single transaction (db.ts:216) — that transaction IS the crash window:
+            // a power loss mid-migration rolls back, leaving the OLD single-column
+            // FTS intact (recall falls back to vector-only until the next open;
+            // never silently empty). The nightly capture lock does NOT cover
+            // initDb migrations, so the tx guard is the only safety net.
+            //
+            // Idempotent by construction: DROP IF EXISTS + CREATE + rebuild. On a
+            // fresh DB (where FTS_SCHEMA already created the multi-column form) this
+            // is a 0-row rebuild — wasted but harmless. On a legacy single-column DB
+            // it converts in place. Re-running against an already-migrated DB is a
+            // no-op shape + a refill of the same rows.
+            //
+            // Triggers are dropped + recreated to pick up the new column list (the
+            // pre-v10 update trigger fired only on `UPDATE OF content`; the new one
+            // also fires on project/domain updates so a tag reclassification lands
+            // in FTS without a content edit).
+            db.exec("DROP TRIGGER IF EXISTS memories_fts_insert");
+            db.exec("DROP TRIGGER IF EXISTS memories_fts_update");
+            db.exec("DROP TRIGGER IF EXISTS memories_fts_delete");
+            db.exec("DROP TABLE IF EXISTS memories_fts");
+            db.exec(`
+        CREATE VIRTUAL TABLE memories_fts USING fts5(
+          content,
+          project,
+          domain,
+          content_rowid='rowid'
+        )
+      `);
+            // COALESCE on project/domain because FTS5 stores NULL as no-tokens,
+            // which is what we want for unscoped memories (NULL domain = not yet
+            // classified; NULL project = no cwd-derived label). content is NOT NULL
+            // by the insert contract.
+            db.exec(`
+        INSERT INTO memories_fts (rowid, content, project, domain)
+        SELECT rowid, content, COALESCE(project, ''), COALESCE(domain, '') FROM memories
+      `);
+            db.exec(`
+        CREATE TRIGGER memories_fts_insert AFTER INSERT ON memories
+        BEGIN
+          INSERT INTO memories_fts (rowid, content, project, domain)
+          VALUES (NEW.rowid, NEW.content, NEW.project, NEW.domain);
+        END
+      `);
+            db.exec(`
+        CREATE TRIGGER memories_fts_update AFTER UPDATE OF content, project, domain ON memories
+        BEGIN
+          UPDATE memories_fts
+             SET content = NEW.content, project = NEW.project, domain = NEW.domain
+           WHERE rowid = NEW.rowid;
+        END
+      `);
+            db.exec(`
+        CREATE TRIGGER memories_fts_delete AFTER DELETE ON memories
+        BEGIN
+          DELETE FROM memories_fts WHERE rowid = OLD.rowid;
+        END
+      `);
+        },
+    },
 ];
 /**
  * Run all pending migrations against the database.

package/dist/eval/recall-sweep.d.ts

@@ -0,0 +1,82 @@
+#!/usr/bin/env node
+/**
+ * Recall@k sweep eval for session-intent keying (#192, commit 0867d6c).
+ *
+ * SHIP GATE: does blending the prompt with the per-session EMA centroid
+ * (retrieval.blendQueryVector) improve recall across rephrased turns, AND
+ * what does it COST when a session shifts topics mid-stream? Picks the
+ * default `sessionIntentWeight`.
+ *
+ * Two scenario families, same corpus:
+ *
+ *   FOCUSED — 6 sessions (one per topic), 4 turns each of GENUINE REPHRASES
+ *   of one intent. Tests the UPSIDE: does the centroid pull drifting
+ *   rephrases back on-topic? (recall@5 / p@5 should rise with w.)
+ *
+ *   SHIFT — 5 sessions, each 2 turns on topic A then 2 turns on topic B (a
+ *   clean mid-session topic change). Tests the DOWNSIDE that focused-only
+ *   can't see: a higher weight leaves the centroid partly A right after the
+ *   shift, so the first turn(s) on B may LAG (lower recall@5 for B). The
+ *   load-bearing read-out is post-shift recovery at turn-3 (first B turn)
+ *   and turn-4, plus turns-to-recover.
+ *
+ * Method (both families):
+ *   - Synthetic corpus of 30 memories across 6 well-separated topics (5 each),
+ *     embedded with the REAL bge-small-en-v1.5 model (the production embedder).
+ *   - For each weight w in {0.0, 0.2, 0.4, 0.5, 0.6, 0.8}: fresh
+ *     SessionRecallRegistry per session; per turn, embed the prompt ONCE,
+ *     blend with the live centroid, pass the blended vector to retrieve() via
+ *     queryEmbedding, record recall@5 + p@5 for the turn's CURRENT topic,
+ *     then fold the prompt into the centroid (EMA α=0.4).
+ *
+ * Fairness controls (load-bearing — the eval is useless if these slip):
+ *   - `noStrengthen: true` on every retrieve(): the DB stays STATIC across all
+ *     retrieve calls. Strengthening would mutate effective_strength/access_count
+ *     between runs and contaminate cross-weight comparisons.
+ *   - Uniform memory metadata: every memory has base_strength=0.5, created_at
+ *     ≈ now, access_count=0, and NO links. So effective_strength, recency,
+ *     connections, and the freshness boost are all uniform → the ONLY
+ *     discriminator is vector cosine + RRF rank. That isolates the blend.
+ *   - Cold-exposure slots are a no-op here (all candidates equally cold),
+ *     so the top-k is the plain score order.
+ *   - The centroid update is weight-INDEPENDENT (EMA of prompts at α=0.4), so
+ *     the per-turn centroid is identical across all weights for a given
+ *     session; only the blend differs. Embeddings are precomputed once.
+ *
+ * Metrics (per turn, for the turn's CURRENT topic):
+ *   - recall@5  — did >=1 same-topic memory surface in the top-5? (coarse; the
+ *     requested ship-gate metric — saturates on a small corpus.)
+ *   - p@5       — on-topic count in the top-5 (0..5, finer).
+ *   - recall@1  — was the SINGLE top result on-topic? (finest; focused only.)
+ *
+ * Honest by construction: it prints whatever the numbers are, including
+ * w=0 winning, the blend hurting a focused scenario, or high weights lagging
+ * badly post-shift. Report written to data/eval-recall-sweep/report.md and
+ * printed to stdout.
+ *
+ * ## Section 4 — SCOPE dimension (#203 soft project affinity)
+ *
+ * A THIRD family on a SEPARATE project-labeled corpus (the blend-sweep corpus
+ * above has no project labels). Goal: prove #203's soft project affinity drops
+ * cross-scope noise (the boat/battery case — a "hardware" query surfacing
+ * "marine" memories on a shared token like "battery") below the cap WITHOUT a
+ * hard filter, and WITHOUT losing same-scope recall.
+ *
+ *   Corpus: 6 hardware + 6 marine memories. 3 marine memories are
+ *   CONTAMINATION SEEDS (share a token with the hardware queries); 3 are
+ *   marine-only fillers (control).
+ *
+ *   Each hardware query runs TWICE on the same static DB:
+ *     scope OFF — no `project` sent (byte-identical to pre-#203).
+ *     scope ON  — `project: "hardware"` (computeScore adds +projectAffinity 0.15
+ *                 to hardware memories; marine gets 0; no hard filter).
+ *
+ *   Metrics: contamination@5 (marine in top-5 / 5 — LOWER is better),
+ *   recall@5 (gold hardware memory surfaced), sameScope@5 (hardware in top-5).
+ *   Scope is orthogonal to the blend, so the blend weight is held at 0 here —
+ *   `project` is the ONLY variable. Same invariants (noStrengthen, real
+ *   embedder, uniform metadata, neverCalledEmbed self-check).
+ *
+ * Run:  npm run eval:recall-sweep   (==  node dist/eval/recall-sweep.js)
+ */
+export {};