@gamaze/hicortex 0.22.0 → 0.22.2

package/dist/recall-precision.js

@@ -0,0 +1,381 @@
+"use strict";
+/**
+ * Memory Precision (#476) — the event store behind the console's Memory
+ * Precision card, and the two-level context model made visible:
+ *
+ *   - Level 1 (pushed index): what every prompt receives unasked. One
+ *     `recall_pushes` row per NON-skipped /recall-index call + one
+ *     `recall_events` kind='shown' row per pushed line, carrying the RAW
+ *     cosines (similarity to the prompt, redundancy vs standing context) —
+ *     stored threshold-free, verdicts computed at render so recalibration
+ *     never rewrites history.
+ *   - Level 2 (recall depth): what the agent fetches when needed. One
+ *     `recall_events` kind='fetch' row per handleMemoryGet — the ONE funnel
+ *     the REST GET /memory and MCP hicortex_get paths share.
+ *
+ * Layering (the recall-index.ts / capture-health.ts convention): all write +
+ * aggregation logic lives here as pure functions over the db handle so tests
+ * exercise them without HTTP; mcp-server.ts only wires the record calls into
+ * the /recall-index and fetch funnels, and dashboard.ts spreads the window
+ * aggregation into /dashboard/data.
+ *
+ * FAIL-SOFT LAW: recording is telemetry. Any error here is caught by the
+ * CALLER (handleRecallIndex / handleMemoryGet) and swallowed — the recall
+ * response (status 200 + block) is never affected, and the exposure signal
+ * (touchMemoriesShown) never depends on it (the handler falls back to the
+ * plain exposure write when recording is absent or fails).
+ *
+ * The perf law (bounded hot-path cost): the recorder computes cosines with
+ * the request's ALREADY-MEMOIZED prompt embedding (createRecallRetrieveFn's
+ * embedPrompt — zero extra embeds) against STORED memory vectors, and writes
+ * one transaction of ≤ maxItems+1 INSERTs — microseconds inside an endpoint
+ * that already writes and spends ~100-300 ms embedding and searching.
+ */
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.recordRecallPush = recordRecallPush;
+exports.recordRecallFetch = recordRecallFetch;
+exports.pruneRecallPrecision = pruneRecallPrecision;
+exports.identityServedToAllClients = identityServedToAllClients;
+exports.createStandingContextBasis = createStandingContextBasis;
+exports.readMemoryPrecision = readMemoryPrecision;
+const storage = __importStar(require("./storage.js"));
+const retrieval_js_1 = require("./retrieval.js");
+const embedder_js_1 = require("./embedder.js");
+const identity_store_js_1 = require("./identity-store.js");
+const recall_index_js_1 = require("./recall-index.js");
+const calibration_js_1 = require("./calibration.js");
+/**
+ * Record one /recall-index push: the push row + (when lines were shown) the
+ * per-line kind='shown' event rows, in ONE transaction WITH the existing
+ * exposure write (storage.touchMemoriesShown — the spec's "same transaction";
+ * better-sqlite3 nests the inner transaction as a savepoint, so a recording
+ * failure rolls back the whole unit and the CALLER's fallback re-runs the
+ * exposure touch alone — shown_count never depends on telemetry).
+ *
+ * similarity = cosine(memory embedding, PURE prompt embedding), computed
+ * UNIFORMLY via cosineBetweenVectors against the stored vector — FTS-sourced
+ * picks (which bypass the cosine floor and carry similarity:null in
+ * MemorySearchResult) get a measured value here too. redundancy = MAX cosine
+ * against the standing-context basis vectors (NULL when the basis is empty —
+ * embed failure degrades to "unmeasured", never a guessed 0). Both stored
+ * raw; no thresholds touch this row.
+ */
+function recordRecallPush(db, entry, basis = []) {
+    const ts = entry.ts ?? new Date().toISOString();
+    // UTC YYYY-MM-DD of ts — day buckets are UTC everywhere (dashboard
+    // conventions; SQLite date('now') is UTC too).
+    const day = ts.slice(0, 10);
+    const excerpt = entry.prompt.slice(0, calibration_js_1.MEMORY_PRECISION_PROMPT_EXCERPT_CHARS);
+    const insertEvent = db.prepare(`INSERT INTO recall_events (ts, day, push_id, memory_id, similarity, redundancy, kind)
+     VALUES (?, ?, ?, ?, ?, ?, 'shown')`);
+    const tx = db.transaction(() => {
+        if (entry.ids.length > 0) {
+            // The exposure write rides INSIDE the recording transaction (spec: one
+            // atomic unit). Nested db.transaction → savepoint.
+            storage.touchMemoriesShown(db, entry.ids, ts);
+        }
+        const info = db
+            .prepare("INSERT INTO recall_pushes (ts, day, session_id, prompt_excerpt) VALUES (?, ?, ?, ?)")
+            .run(ts, day, entry.sessionId, excerpt);
+        const pushId = Number(info.lastInsertRowid);
+        if (entry.ids.length === 0)
+            return; // silent turn — push row only
+        const promptEmb = entry.promptEmbedding;
+        if (!promptEmb) {
+            // Contract violation (ids without an embedding): record the lines with
+            // NULL measures rather than guessing — honesty over fabrication.
+            for (const id of entry.ids)
+                insertEvent.run(ts, day, pushId, id, null, null);
+            return;
+        }
+        for (const id of entry.ids) {
+            const memEmb = storage.getStoredEmbedding(db, id);
+            const similarity = memEmb ? (0, retrieval_js_1.cosineBetweenVectors)(memEmb, promptEmb) : null;
+            let redundancy = null;
+            if (memEmb && basis.length > 0) {
+                for (const b of basis) {
+                    const c = (0, retrieval_js_1.cosineBetweenVectors)(memEmb, b);
+                    if (redundancy === null || c > redundancy)
+                        redundancy = c;
+                }
+            }
+            insertEvent.run(ts, day, pushId, id, similarity, redundancy);
+        }
+    });
+    tx();
+}
+/**
+ * Record one explicit fetch (handleMemoryGet — the funnel both REST GET
+ * /memory and MCP hicortex_get route through, so the row is written exactly
+ * once per fetch). similarity/redundancy are NULL on fetch rows: Level 2 is
+ * the use signal, not a relevance measure. push_id NULL — a fetch stands
+ * alone. Fail-soft at the CALLER (handleMemoryGet wraps this).
+ */
+function recordRecallFetch(db, memoryId, ts) {
+    const nowIso = ts ?? new Date().toISOString();
+    db.prepare(`INSERT INTO recall_events (ts, day, push_id, memory_id, similarity, redundancy, kind)
+     VALUES (?, ?, NULL, ?, NULL, NULL, 'fetch')`).run(nowIso, nowIso.slice(0, 10), memoryId);
+}
+/**
+ * Nightly retention prune (zero-LLM): drop both event tables' rows outside
+ * the rolling window whose length IS the retention constant (the
+ * CAPTURE_HEALTH_WINDOW_DAYS single-constant law — the card can never claim a
+ * window the store no longer has rows for). Inclusive cutoff: the last N
+ * calendar days with today counted stay (day >= today−(N−1)), the complement
+ * is deleted — retained rows == window rows.
+ */
+function pruneRecallPrecision(db) {
+    db.exec(`DELETE FROM recall_pushes WHERE day < date('now', '-${calibration_js_1.MEMORY_PRECISION_WINDOW_DAYS - 1} days')`);
+    db.exec(`DELETE FROM recall_events WHERE day < date('now', '-${calibration_js_1.MEMORY_PRECISION_WINDOW_DAYS - 1} days')`);
+}
+// ---------------------------------------------------------------------------
+// Standing-context basis — the redundancy reference (what every session
+// ALREADY knows before the index pushes anything)
+// ---------------------------------------------------------------------------
+/** Cache TTL for the embedded basis (module constant, not a tuning value: it
+ *  bounds how stale the redundancy reference may be — 24h mirrors the daily
+ *  lesson/identity cadence the /learnings + /identity surfaces serve). */
+const BASIS_TTL_MS = 24 * 60 * 60 * 1000;
+/**
+ * True when the identity scope is served to EVERY known client — the only
+ * case where identity sections belong in the corpus-level standing-context
+ * basis. /recall-index carries no client type, so a SCOPED identity config
+ * (any subset) excludes identity rather than guessing which sections the
+ * calling session sees. `clients` is the boot-resolved
+ * resolveIdentityClientsConfig output; the absent-config default ["cc"] is
+ * NOT all → identity excluded.
+ */
+function identityServedToAllClients(clients) {
+    return identity_store_js_1.KNOWN_IDENTITY_CLIENTS.every((c) => clients.includes(c));
+}
+/**
+ * The standing-context basis: the top lessons in /learnings order
+ * (storage.getLessons(db, 30) — exactly what the /learnings handler serves
+ * and every client's session-start hook injects) PLUS identity sections ONLY
+ * when identityServedToAllClients (see above). Embedded with the LOCAL
+ * embedder — no LLM anywhere in this feature. Cached for 24h (single-flight:
+ * concurrent callers share the embedding pass); an embed failure degrades to
+ * an EMPTY cached basis (redundancy NULL — "unmeasured", never guessed) and
+ * is retried after the TTL.
+ */
+function createStandingContextBasis(config) {
+    const embedFn = config.embedFn ?? embedder_js_1.embedBatch;
+    const now = config.now ?? Date.now;
+    const ttlMs = config.ttlMs ?? BASIS_TTL_MS;
+    let cache = null;
+    let inFlight = null;
+    const build = async (db) => {
+        const texts = [];
+        // Lessons: the /learnings set + order (last 30 days, created_at DESC).
+        // Read errors degrade to an empty lesson half — the basis is a reference,
+        // not a ledger.
+        try {
+            for (const lesson of storage.getLessons(db, 30))
+                texts.push(lesson.content);
+        }
+        catch (err) {
+            console.warn(`[hicortex] standing-context basis: lesson read failed (degraded): ` +
+                `${err instanceof Error ? err.message : String(err)}`);
+        }
+        // Identity: only when the scope is served to EVERY client — never guessed.
+        if (identityServedToAllClients(config.clients())) {
+            try {
+                const { sections } = (0, identity_store_js_1.readSections)(config.identityDir());
+                for (const content of Object.values(sections))
+                    texts.push(content);
+            }
+            catch (err) {
+                console.warn(`[hicortex] standing-context basis: identity read failed (degraded): ` +
+                    `${err instanceof Error ? err.message : String(err)}`);
+            }
+        }
+        return texts.length > 0 ? await embedFn(texts) : [];
+    };
+    const provider = async (db) => {
+        if (cache && now() < cache.expiresAt)
+            return cache.vectors;
+        if (!inFlight) {
+            inFlight = build(db)
+                .catch((err) => {
+                // Embed failure → EMPTY basis cached for the TTL: redundancy reads
+                // NULL (unmeasured) on every push in the period, never a guessed 0.
+                console.warn(`[hicortex] standing-context basis: embed failed (redundancy NULL this TTL): ` +
+                    `${err instanceof Error ? err.message : String(err)}`);
+                return [];
+            })
+                .then((vectors) => {
+                cache = { vectors, expiresAt: now() + ttlMs };
+                return vectors;
+            })
+                .finally(() => {
+                inFlight = null;
+            });
+        }
+        return inFlight;
+    };
+    const basis = provider;
+    basis.resetForTests = () => {
+        cache = null;
+    };
+    return basis;
+}
+const DAY_MS = 86_400_000;
+const round4 = (v) => Math.round(v * 10000) / 10000;
+/** Range string ("7d"|"30d"|…) → days; null when unparseable/"all" (the
+ *  caller — handleDashboardData — has already validated against
+ *  VALID_RANGES; anything unexpected degrades to the retention clamp). */
+function rangeToDays(range) {
+    if (range === "all")
+        return null;
+    const n = parseInt(range, 10);
+    return Number.isFinite(n) && n > 0 ? n : null;
+}
+/**
+ * The window aggregation behind /dashboard/data's memory_precision block
+ * (the readCaptureHealthWindow pattern). Effective window = min(range,
+ * MEMORY_PRECISION_WINDOW_DAYS) for every range incl. 180d/all (the #452
+ * selector's wide options clamp to what the retention honestly holds).
+ *
+ * Level 1 groups the stored event rows over the window (day >= today−(N−1),
+ * the inclusive capture-health convention); Level 2 is the corpus-wide
+ * adoption delta — live sums minus the NEWEST non-null `adoption` in
+ * dashboard_snapshots at/before now−N days (cumulative counters telescope:
+ * a missing nightly widens the effective window rather than corrupting it;
+ * two snapshots on one day are harmless — the newest wins; no pre-window
+ * baseline → null, undefined rather than zero). Divergence joins the
+ * window's per-memory shown/fetch counts against LIVE (non-absorbed)
+ * memories and renders the top-3 through the shared index-line path.
+ */
+function readMemoryPrecision(db, range, liveAdoption, opts) {
+    const now = opts?.now ?? new Date();
+    const redundantAbove = opts?.redundantAbove ?? calibration_js_1.MEMORY_PRECISION_REDUNDANT_ABOVE;
+    const windowDays = Math.min(rangeToDays(range) ?? calibration_js_1.MEMORY_PRECISION_WINDOW_DAYS, calibration_js_1.MEMORY_PRECISION_WINDOW_DAYS);
+    // Inclusive window start (UTC YYYY-MM-DD): today counts as day 1.
+    const cutoffDay = new Date(now.getTime() - (windowDays - 1) * DAY_MS).toISOString().slice(0, 10);
+    // --- Level 1: group the window's shown events (raw values, no thresholds
+    // in SQL beyond the redundant-share counter — the verdict stays at render). ---
+    const agg = db.prepare(`SELECT COUNT(*) AS lines,
+            SUM(CASE WHEN similarity IS NOT NULL THEN similarity ELSE 0 END) AS sim_sum,
+            SUM(CASE WHEN similarity IS NOT NULL THEN 1 ELSE 0 END) AS sim_n,
+            SUM(CASE WHEN redundancy IS NOT NULL AND redundancy >= ? THEN 1 ELSE 0 END) AS red_n,
+            SUM(CASE WHEN redundancy IS NOT NULL THEN 1 ELSE 0 END) AS red_measured
+       FROM recall_events
+      WHERE day >= ? AND kind = 'shown'`).get(redundantAbove, cutoffDay);
+    const pushes = db.prepare("SELECT COUNT(*) AS c FROM recall_pushes WHERE day >= ?").get(cutoffDay).c;
+    // --- Level 2: corpus adoption delta over the window (snapshot series). ---
+    const baselineCutoff = new Date(now.getTime() - windowDays * DAY_MS).toISOString();
+    const snapRows = db
+        .prepare("SELECT run_at, metrics FROM dashboard_snapshots WHERE run_at <= ? ORDER BY run_at DESC")
+        .all(baselineCutoff);
+    let baseline = null;
+    for (const r of snapRows) {
+        // Backfilled rows OMIT adoption (point-in-time, not reconstructable) —
+        // walk to the newest row that actually carries it.
+        try {
+            const m = JSON.parse(r.metrics);
+            if (m.adoption &&
+                typeof m.adoption.shown_sum === "number" &&
+                typeof m.adoption.used_sum === "number") {
+                baseline = { shown_sum: m.adoption.shown_sum, used_sum: m.adoption.used_sum };
+                break;
+            }
+        }
+        catch {
+            // A malformed metrics blob is skipped, not fatal — the walk continues
+            // to the next-older candidate.
+        }
+    }
+    const shown = baseline ? liveAdoption.shown_sum - baseline.shown_sum : null;
+    const used = baseline ? liveAdoption.used_sum - baseline.used_sum : null;
+    // Divide guard + honesty: no baseline → null (undefined, never 0); a window
+    // with zero (or negative — a restored DB) shown delta has no ratio.
+    const usesPerShowing = shown !== null && shown > 0 && used !== null ? round4(used / shown) : null;
+    // --- Divergence: per-memory window counters, gated to live rows. ---
+    const perMemory = db
+        .prepare(`SELECT memory_id,
+              SUM(CASE WHEN kind = 'shown' THEN 1 ELSE 0 END) AS shown,
+              SUM(CASE WHEN kind = 'fetch' THEN 1 ELSE 0 END) AS fetches
+         FROM recall_events
+        WHERE day >= ?
+        GROUP BY memory_id`)
+        .all(cutoffDay);
+    const divergingIds = perMemory
+        .filter((r) => r.shown >= calibration_js_1.MEMORY_PRECISION_DIVERGENCE_MIN_SHOWN && r.fetches === 0)
+        .map((r) => ({ id: r.memory_id, shown: r.shown }));
+    const memStmt = db.prepare(`SELECT id, content, created_at, domain, project, source_agent, memory_type
+       FROM memories WHERE id = ? AND COALESCE(status, '') != 'absorbed'`);
+    const diverging = [];
+    for (const d of divergingIds) {
+        const row = memStmt.get(d.id);
+        if (!row)
+            continue; // absorbed/deleted since — not a LIVE divergence
+        diverging.push({
+            id: row.id,
+            shown: d.shown,
+            // The SHARED index-line path (what agents see in the pushed index) —
+            // formatIndexLine at the default title cap, same as the digest.
+            line: (0, recall_index_js_1.formatIndexLine)({
+                id: row.id,
+                content: row.content,
+                score: 0,
+                effective_strength: 0,
+                access_count: 0,
+                memory_type: row.memory_type,
+                project: row.project,
+                domain: row.domain,
+                source_agent: row.source_agent,
+                created_at: row.created_at,
+                connections: 0,
+            }, calibration_js_1.RECALL_TITLE_CHARS),
+        });
+    }
+    diverging.sort((a, b) => b.shown - a.shown);
+    return {
+        window_days: windowDays,
+        level1: {
+            pushes,
+            lines: agg.lines,
+            mean_similarity: agg.sim_n > 0 ? round4((agg.sim_sum ?? 0) / agg.sim_n) : null,
+            redundant_share: agg.red_measured > 0 ? round4(agg.red_n / agg.red_measured) : null,
+            thresholds: {
+                redundant_above: redundantAbove,
+                divergence_min_shown: calibration_js_1.MEMORY_PRECISION_DIVERGENCE_MIN_SHOWN,
+            },
+        },
+        level2: { shown, used, uses_per_showing: usesPerShowing },
+        divergence: { count: diverging.length, top: diverging.slice(0, 3) },
+    };
+}

package/dist/retrieval.d.ts

@@ -55,14 +55,26 @@ export interface ScoringWeights {
     similarity: number;
     strength: number;
     connections: number;
+    /**
+     * #449 log-saturation degree K for the connections term: the credit is
+     * min(1, log1p(k)/log1p(K)) — full at k = K, exactly +0 at k = 0. A count,
+     * not a weight: validated ≥ 1 (the rrfK non-weight seam precedent); no
+     * upper bound (the eval sweep widens K past the planted fixture degrees).
+     */
+    connectionsSaturation: number;
     recency: number;
-    freshnessBoostDays: number;
-    freshnessBoostWeight: number;
+    /** #430 merged time curve: head amplitude at age 0 — the merged freshness
+     *  job. A deliberate overshoot (max total 1.15 pre-clamp), NOT a fifth
+     *  blend weight; the four blend weights still sum to 1.0. */
+    recencyHead: number;
+    /** #430 merged time curve: head window / join age in days. */
+    recencyHeadDays: number;
     supersededDemotion: number;
-    /** #203 soft affinity boost on exact project match. */
-    projectAffinity: number;
-    /** #203 soft affinity boost multiplier on max overlapping domain-tag weight. */
-    domainAffinity: number;
+    /** #430 merged scope affinity: ONE term — max() of the project-match
+     *  indicator (1 on exact match) and the max overlapping domain-tag weight,
+     *  × this weight (born from #203's two additive boosts at their shared
+     *  value). 0 (via the seam) disables the whole term. */
+    scopeAffinity: number;
     /** #205 RRF k parameter (1/(k+rank+1)). Larger ⇒ shallower rank curve. */
     rrfK: number;
     /** #205 composite-score share of the final blend (RRF gets the remainder). */
@@ -232,12 +244,13 @@ export declare function effectiveStrength(baseStrength: number, lastAccessed: st
  * Return a composite relevance score in [0, 1] for a candidate memory.
  * Exported for exact-value tests of the similarity component (#145).
  *
- * #203 soft affinity (options.scope + options.tagWeights): two additive,
- * graded, zero-boost-neutral terms — project affinity (exact project match)
- * and domain affinity (max overlapping memory_tags.weight × scope). Both are
- * 0 when the scope is absent (byte-identical to pre-#203) and NEVER negative
- * (a foreign memory adds 0, never a penalty — penalties re-introduce
- * soft-exclusion). See `AffinityScope`.
+ * Scope affinity (options.scope + options.tagWeights; #203, merged #430):
+ * ONE additive, graded, zero-boost-neutral term — max() of the project-match
+ * indicator (exact match ⇒ 1) and the max overlapping memory_tags.weight,
+ * × the scopeAffinity weight. It is 0 when the scope is absent
+ * (byte-identical to pre-#203) and NEVER negative (a foreign memory adds 0,
+ * never a penalty — penalties re-introduce soft-exclusion). See
+ * `AffinityScope`.
  */
 export interface AffinityScope {
     /** Exact-match project from the client (CC/OC cwd-derived; /search project). */
@@ -246,12 +259,13 @@ export interface AffinityScope {
      *  vocabulary as memory_tags (config `domains`). */
     missionDomains?: string[];
 }
-export declare function computeScore(memory: Memory, distance: number, connectionCount: number, maxConnections: number, now: Date, options?: {
+export declare function computeScore(memory: Memory, distance: number, connectionCount: number, now: Date, options?: {
     superseded?: boolean;
-    /** #203: when present, project/domain affinity boosts are applied. */
+    /** #203/#430: when present, the scope-affinity boost is applied. */
     scope?: AffinityScope;
     /** Candidate's graded domain tags (memory_tags rows). Loaded batched for
-     *  the whole candidate set in retrieve(); used for domain affinity. */
+     *  the whole candidate set in retrieve(); used for the scope-affinity
+     *  term's domain signal. */
     tagWeights?: Array<{
         tag: string;
         weight: number | null;
@@ -308,6 +322,11 @@ export declare function retrieve(db: Database.Database, embedFn: EmbedFn, query:
     ftsCandidates?: (fetchLimit: number, sourceAgent?: string) => Array<Memory & {
         rank: number;
     }>;
+    /** #458 eval clock pin — the instant the decay/recency terms score
+     *  against. Eval-only: no production caller passes it, and unset means
+     *  the live clock (byte-identical to pre-#458). Mirrors the
+     *  injectable-clock idiom of run-deadline.ts; see eval/eval-clock.ts. */
+    now?: Date;
 }): Promise<MemorySearchResult[]>;
 /**
  * Get recent context, optionally filtered by project.