@gamaze/hicortex 0.20.7 → 0.20.10

package/dist/eval/ranking-eval.js

@@ -0,0 +1,391 @@
+#!/usr/bin/env node
+"use strict";
+/**
+ * Search-trust ranking gate (#425) — the AC1/AC2/AC3 before/after photo.
+ *
+ *   npm run eval:ranking [--snapshot <db>] [--photo <out.json>]
+ *                        [--compare <baseline.json>] [--scoring '<json>']
+ *
+ *   --snapshot <db>    enables case 3: the deterministic real-query battery
+ *                      against a READONLY snapshot (openSnapshot — never
+ *                      initDb; the snapshot is never touched). Also reports
+ *                      where the real "Sirnäs" memory ranks today (the
+ *                      measured red photo pre-fix, AC10 pre-condition).
+ *   --photo <out.json> where to write the machine photo (defaults to
+ *                      data/ranking-eval-photo.json). ALWAYS written, even
+ *                      on a red run — the photo IS the measurement.
+ *   --compare <json>   gate against a previously recorded photo: case 2's
+ *                      returned list must be byte-identical, and the battery
+ *                      must hold top-1 stability >= 90% with no query losing
+ *                      a both-channel exact match from its top-3.
+ *   --scoring '<json>' a JSON object of ScoringWeights overrides passed to
+ *                      configureScoring (the eval/test seam, #408) — the
+ *                      sweep knob. Production runs omit it.
+ *
+ * Cases (fixtures: ranking-fixtures.ts, real bge-small-en-v1.5 embedder):
+ *   1. sirnas_exact_match — planted corpus in a THROWAWAY writable DB;
+ *      declared expectation: the proper-noun exact-match row returns #1.
+ *      Exit code reflects the verdict (a pre-fix run is EXPECTED to exit
+ *      non-zero — that red photo is the before picture).
+ *   2. no_match_control — same corpus, a "mamma"-class query with zero FTS
+ *      hits; records the full returned list for byte-stability comparison.
+ *   3. real-query battery (needs --snapshot) — ~30 queries derived
+ *      deterministically from the snapshot corpus (top/mid/rare-frequency
+ *      distinctive terms + proper nouns) + the fixed "Sirnäs" query; top-8
+ *      ids per query recorded to the photo.
+ *
+ * Honesty invariants: similarities are MEASURED (real embedder, embed-once,
+ * queryEmbedding reuse), noStrengthen on every retrieve() (the probe never
+ * perturbs the store), readonly snapshot access, and the planted corpora
+ * live in mkdtemp dirs removed on exit. Never touches ~/.hicortex.
+ */
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
+Object.defineProperty(exports, "__esModule", { value: true });
+const node_fs_1 = require("node:fs");
+const node_os_1 = require("node:os");
+const node_path_1 = require("node:path");
+const embedder_js_1 = require("../embedder.js");
+const db_js_1 = require("../db.js");
+const storage = __importStar(require("../storage.js"));
+const retrieval_js_1 = require("../retrieval.js");
+const eval_db_js_1 = require("./eval-db.js");
+const ranking_fixtures_js_1 = require("./ranking-fixtures.js");
+const ranking_battery_js_1 = require("./ranking-battery.js");
+const DEFAULT_PHOTO_PATH = (0, node_path_1.join)(process.cwd(), "data", "ranking-eval-photo.json");
+/** Battery gate thresholds (sweep protocol, issue #425 commit-3 gates). */
+const BATTERY_TOP1_STABILITY_MIN = 0.9;
+/** Plant one fixture corpus into a writable DB with its hardened metadata. */
+async function plantRankingFixture(db, fixture) {
+    const idByKey = new Map();
+    const now = Date.now();
+    for (const row of fixture.rows) {
+        const embedding = await (0, embedder_js_1.embed)(row.content);
+        const id = storage.insertMemory(db, row.content, embedding, {
+            sourceAgent: ranking_fixtures_js_1.RANKING_SOURCE_AGENT,
+            project: ranking_fixtures_js_1.RANKING_PROJECT,
+            memoryType: row.memoryType,
+            createdAt: row.createdAt,
+        });
+        const lastAccessed = new Date(now - row.lastAccessedDaysAgo * 86_400_000).toISOString();
+        storage.updateMemory(db, id, {
+            base_strength: row.baseStrength,
+            access_count: row.accessCount,
+            last_accessed: lastAccessed,
+        });
+        idByKey.set(row.key, id);
+    }
+    for (const row of fixture.rows) {
+        for (const to of row.linksTo) {
+            const sourceId = idByKey.get(row.key);
+            const targetId = idByKey.get(to);
+            if (sourceId && targetId)
+                storage.addLink(db, sourceId, targetId, "relates_to", 0.7);
+        }
+    }
+    return idByKey;
+}
+/**
+ * Run one planted case: full-corpus retrieve (limit = rows + headroom, so
+ * the cold-exposure swap is inert — scored.length <= limit — and the
+ * returned order IS the score order), results mapped back to fixture keys.
+ */
+async function measurePlantedCase(db, fixture, idByKey) {
+    const keyById = new Map([...idByKey.entries()].map(([k, v]) => [v, k]));
+    const queryEmb = await (0, embedder_js_1.embed)(fixture.query);
+    const ftsRows = storage.searchFts(db, fixture.query, 100, undefined);
+    const results = await (0, retrieval_js_1.retrieve)(db, embedder_js_1.embed, fixture.query, {
+        limit: fixture.rows.length + 8,
+        noStrengthen: true,
+        queryEmbedding: queryEmb,
+    });
+    const measured = results.map((r, i) => ({
+        key: keyById.get(r.id) ?? r.id,
+        rank: i + 1,
+        score: Math.round(r.score * 1e6) / 1e6,
+        similarity: r.similarity ?? null,
+        source: r.source ?? null,
+        effectiveStrength: Math.round(r.effective_strength * 1e6) / 1e6,
+        baseStrength: fixture.rows.find((x) => x.key === keyById.get(r.id))?.baseStrength ?? -1,
+        accessCount: r.access_count,
+    }));
+    const pass = fixture.expectedTop1
+        ? measured.length > 0 && measured[0].key === fixture.expectedTop1
+        : null;
+    const gap = measured.length >= 2
+        ? Math.round((measured[0].score - measured[1].score) * 1e6) / 1e6
+        : null;
+    return {
+        pass,
+        query: fixture.query,
+        ftsHitRows: ftsRows.length,
+        results: measured,
+        top1Top2ScoreGap: gap,
+    };
+}
+async function runBattery(db) {
+    const rows = db
+        .prepare("SELECT id, content FROM memories WHERE COALESCE(status, '') != 'absorbed'")
+        .all();
+    const queries = (0, ranking_battery_js_1.deriveBatteryQueries)(rows);
+    const out = [];
+    let sirnasRank = null;
+    for (const { q, band } of queries) {
+        const queryEmb = await (0, embedder_js_1.embed)(q);
+        const results = await (0, retrieval_js_1.retrieve)(db, embedder_js_1.embed, q, {
+            limit: 8,
+            noStrengthen: true,
+            queryEmbedding: queryEmb,
+        });
+        const top8 = results.map((r) => r.id);
+        // Per-result provenance for the sweep analysis: the channel that
+        // produced the row + (for term queries) whether the row's content
+        // carries the query term — lets the gate distinguish "the D3 property
+        // fired" (a both-channel exact match displaced a single-channel top-1)
+        // from a real regression.
+        const lower = q.toLowerCase();
+        const meta = results.map((r) => ({
+            id: r.id,
+            source: r.source ?? null,
+            term: q === ranking_battery_js_1.SIRNAS_QUERY
+                ? null
+                : (rows.find((x) => x.id === r.id)?.content ?? "").toLowerCase().includes(lower),
+        }));
+        out.push({ q, band, top8, meta });
+        if (q === ranking_battery_js_1.SIRNAS_QUERY) {
+            const idx = top8.indexOf(ranking_fixtures_js_1.REAL_SIRNAS_MEMORY_ID);
+            sirnasRank = idx >= 0 ? idx + 1 : null;
+        }
+    }
+    return { queries: out, sirnasRank };
+}
+// ---------------------------------------------------------------------------
+// Report rendering
+// ---------------------------------------------------------------------------
+function renderReport(args) {
+    const L = [];
+    const { case1, case2, battery } = args;
+    L.push("# Search-trust ranking gate (#425)\n");
+    L.push(`Generated: ${new Date().toISOString()}  \nScoring: ${JSON.stringify(args.scoring)}\n`);
+    L.push("## Case 1 — sirnas_exact_match (AC1)\n");
+    L.push(`Query "${case1.query}" — declared top-1: ${ranking_fixtures_js_1.SIRNAS_FIXTURE.expectedTop1} — ` +
+        `**${case1.pass ? "PASS" : "FAIL"}**${case1.top1Top2ScoreGap !== null ? ` (top1-top2 score gap ${case1.top1Top2ScoreGap.toFixed(4)})` : ""}\n`);
+    L.push("| rank | key | similarity | source | score | effStr | base | access |");
+    L.push("|---|---|---|---|---|---|---|---|");
+    for (const r of case1.results.slice(0, 6)) {
+        L.push(`| ${r.rank} | ${r.key} | ${r.similarity?.toFixed(4) ?? "-"} | ${r.source ?? "-"} | ` +
+            `${r.score.toFixed(4)} | ${r.effectiveStrength.toFixed(4)} | ${r.baseStrength} | ${r.accessCount} |`);
+    }
+    L.push("");
+    L.push("## Case 2 — no_match_control (AC2)\n");
+    L.push(`Query "${case2.query}" — FTS hits: ${case2.ftsHitRows} (must be 0 — no both-channel ` +
+        `candidates can exist)` +
+        (args.case2Stable === null ? "" : ` — vs baseline: **${args.case2Stable ? "identical" : "DRIFTED"}**`) +
+        "\n");
+    L.push("Returned order: " + case2.ids.map((k) => k).join(" → ") + "\n");
+    if (battery) {
+        L.push("## Case 3 — real-query battery (AC3 no-regression photo)\n");
+        L.push(`Queries: ${battery.queries.length}; Sirnäs rank for "${ranking_battery_js_1.SIRNAS_QUERY}": ` +
+            (battery.sirnasRank === null ? "not in top-8" : `#${battery.sirnasRank}`) + "\n");
+        if (args.comparison) {
+            L.push(`vs baseline: top-1 stability ${(args.comparison.top1Stability * 100).toFixed(1)}% ` +
+                `(gate >= ${(BATTERY_TOP1_STABILITY_MIN * 100).toFixed(0)}%; ` +
+                `excluding intended D3 promotions ` +
+                `${(args.comparison.top1StabilityExPromotions * 100).toFixed(1)}%) — ` +
+                `${args.comparison.changedTop1.length} changed top-1` +
+                (args.comparison.changedTop1.length > 0
+                    ? ` [${args.comparison.changedTop1.join(", ")}]`
+                    : "") +
+                `; queries that LOST their both-channel exact match from top-3: ` +
+                `${args.comparison.lostBothChannelExact.length}` +
+                (args.comparison.lostBothChannelExact.length > 0
+                    ? ` [${args.comparison.lostBothChannelExact.join(", ")}]`
+                    : "") +
+                `; per-id exact-match displacements from top-3 (reported, not gated): ` +
+                `${args.comparison.lostExactMatches.length}` +
+                "\n");
+        }
+        L.push("| band | query | top-1 id |");
+        L.push("|---|---|---|");
+        for (const b of battery.queries) {
+            L.push(`| ${b.band} | ${b.q} | ${b.top8[0]?.slice(0, 8) ?? "-"} |`);
+        }
+        L.push("");
+    }
+    return L.join("\n");
+}
+// ---------------------------------------------------------------------------
+// main
+// ---------------------------------------------------------------------------
+function usage() {
+    console.error("Usage: npm run eval:ranking -- [--snapshot <db>] [--photo <out.json>] [--compare <baseline.json>] [--scoring '<json>']");
+    process.exit(1);
+}
+async function main() {
+    const argv = process.argv.slice(2);
+    const flag = (name) => {
+        const i = argv.indexOf(name);
+        return i !== -1 && argv[i + 1] ? argv[i + 1] : undefined;
+    };
+    const snapshotPath = flag("--snapshot");
+    const photoPath = flag("--photo") ?? DEFAULT_PHOTO_PATH;
+    const comparePath = flag("--compare");
+    const scoringRaw = flag("--scoring");
+    if (argv.includes("--help") || argv.includes("-h"))
+        usage();
+    if (scoringRaw) {
+        try {
+            const parsed = JSON.parse(scoringRaw);
+            (0, retrieval_js_1.configureScoring)(parsed);
+        }
+        catch (err) {
+            console.error(`[ranking-eval] --scoring is not valid JSON: ${err instanceof Error ? err.message : err}`);
+            process.exit(1);
+        }
+    }
+    for (const f of [ranking_fixtures_js_1.SIRNAS_FIXTURE, ranking_fixtures_js_1.NO_MATCH_FIXTURE]) {
+        const problems = (0, ranking_fixtures_js_1.checkFixtureInvariants)(f);
+        if (problems.length > 0) {
+            console.error(`[ranking-eval] fixture ${f.key} invariant violations: ${problems.join("; ")}`);
+            process.exit(1);
+        }
+    }
+    console.log("[ranking-eval] loading real bge-small-en-v1.5 embedder...");
+    const t0 = Date.now();
+    // Cases 1 + 2 share one planted corpus DB (same rows; two queries).
+    const workDir = (0, node_fs_1.mkdtempSync)((0, node_path_1.join)((0, node_os_1.tmpdir)(), "hicortex-ranking-eval-"));
+    let case1;
+    let case2;
+    try {
+        const db = (0, db_js_1.initDb)((0, node_path_1.join)(workDir, "planted.db"));
+        try {
+            console.log("[ranking-eval] planting corpus (real embedder)...");
+            const idByKey = await plantRankingFixture(db, ranking_fixtures_js_1.SIRNAS_FIXTURE);
+            case1 = await measurePlantedCase(db, ranking_fixtures_js_1.SIRNAS_FIXTURE, idByKey);
+            const case2Measured = await measurePlantedCase(db, ranking_fixtures_js_1.NO_MATCH_FIXTURE, idByKey);
+            // The returned list keyed by fixture key (readable photos; ids would
+            // churn across replanting).
+            case2 = { ...case2Measured, ids: case2Measured.results.map((r) => r.key) };
+        }
+        finally {
+            db.close();
+        }
+    }
+    finally {
+        (0, node_fs_1.rmSync)(workDir, { recursive: true, force: true });
+    }
+    let battery = null;
+    let comparison = null;
+    let case2Stable = null;
+    // Content lookup for the exact-match gate (the SAME corpus both photos
+    // were recorded against — the snapshot, when comparing).
+    const contentById = new Map();
+    if (snapshotPath) {
+        console.log(`[ranking-eval] battery against readonly snapshot: ${snapshotPath}`);
+        const sdb = (0, eval_db_js_1.openSnapshot)(snapshotPath);
+        try {
+            battery = await runBattery(sdb);
+            for (const r of sdb
+                .prepare("SELECT id, content FROM memories WHERE COALESCE(status, '') != 'absorbed'")
+                .all()) {
+                contentById.set(r.id, r.content);
+            }
+        }
+        finally {
+            sdb.close();
+        }
+    }
+    if (comparePath) {
+        let baseline;
+        try {
+            baseline = JSON.parse((0, node_fs_1.readFileSync)(comparePath, "utf-8"));
+        }
+        catch (err) {
+            console.error(`[ranking-eval] cannot read baseline photo ${comparePath}: ${err instanceof Error ? err.message : err}`);
+            process.exit(1);
+        }
+        if (!baseline.case2 || !baseline.battery || !battery) {
+            console.error("[ranking-eval] baseline photo lacks case2/battery — record one with --photo on the same snapshot first");
+            process.exit(1);
+        }
+        case2Stable = (0, ranking_battery_js_1.compareCase2)(baseline.case2.ids, case2.ids);
+        comparison = (0, ranking_battery_js_1.batteryComparison)(baseline.battery, battery, contentById);
+    }
+    const photo = {
+        generatedAt: new Date().toISOString(),
+        scoring: (0, retrieval_js_1.getScoringWeights)(),
+        case1: {
+            pass: case1.pass,
+            expectedTop1: ranking_fixtures_js_1.SIRNAS_FIXTURE.expectedTop1,
+            top1Top2ScoreGap: case1.top1Top2ScoreGap,
+            results: case1.results,
+        },
+        case2: { ftsHitRows: case2.ftsHitRows, ids: case2.ids },
+        battery,
+        comparison,
+        case2Stable,
+    };
+    (0, node_fs_1.mkdirSync)((0, node_path_1.dirname)(photoPath), { recursive: true });
+    (0, node_fs_1.writeFileSync)(photoPath, JSON.stringify(photo, null, 2), "utf-8");
+    console.log(`[ranking-eval] photo written: ${photoPath}`);
+    const report = renderReport({
+        case1,
+        case2,
+        battery,
+        comparison,
+        case2Stable,
+        scoring: { ...photo.scoring },
+    });
+    console.log(report);
+    const reportPath = photoPath.endsWith(".json")
+        ? photoPath.slice(0, -5) + ".md"
+        : photoPath + ".md";
+    (0, node_fs_1.writeFileSync)(reportPath, report, "utf-8");
+    const batteryOk = comparison === null
+        ? true
+        : comparison.top1StabilityExPromotions >= BATTERY_TOP1_STABILITY_MIN &&
+            comparison.lostBothChannelExact.length === 0;
+    const case2Ok = case2Stable === null ? true : case2Stable;
+    const ok = case1.pass === true && case2Ok && batteryOk;
+    console.log(`[ranking-eval] ${ok ? "GATE PASS" : "GATE FAIL"} (case1 ${case1.pass ? "pass" : "FAIL"}, ` +
+        `case2 ${case2Ok ? "ok" : "DRIFT"}, battery ${batteryOk ? "ok" : "DRIFT"} ` +
+        `[top-1 ${((comparison?.top1Stability ?? 1) * 100).toFixed(1)}% raw / ` +
+        `${((comparison?.top1StabilityExPromotions ?? 1) * 100).toFixed(1)}% ex-promotions, ` +
+        `lost both-channel exact ${comparison?.lostBothChannelExact.length ?? 0}]) in ${Date.now() - t0}ms`);
+    process.exitCode = ok ? 0 : 1;
+}
+main().catch((err) => {
+    console.error("[ranking-eval] FAILED:", err instanceof Error ? err.stack : String(err));
+    process.exitCode = 1;
+});

package/dist/eval/ranking-fixtures.d.ts

@@ -0,0 +1,77 @@
+/**
+ * Planted ranking fixtures for the search-trust calibration gate (#425).
+ *
+ * Two fixture cases, both shaped like the field failure that opened the
+ * issue (the owner's live console test, 2026-09-13): a user searches a
+ * DISTINCTIVE PROPER NOUN they know exists; the one memory carrying that
+ * token has the best similarity of all candidates AND the literal FTS hit,
+ * yet ranks behind hardened but less-relevant memories that win on
+ * effective strength + graph connections.
+ *
+ *   (a) sirnas_exact_match — the AC1 fixture: a low-strength, unlinked,
+ *       never-accessed memory carrying an invented proper noun against
+ *       base-≥0.9 rivals with links among themselves, access history, and
+ *       older creation dates. The rivals share the query's DOMAIN
+ *       vocabulary (harbours, docks, boats) but never the proper noun.
+ *       Declared expectation: the proper-noun row ranks #1.
+ *   (b) no_match_control — the AC2 fixture: a "mamma"-class query with no
+ *       relevant match and NO FTS hits anywhere in the corpus. Whatever
+ *       nearest-neighbor junk vector search returns must be byte-stable
+ *       under the ranking work (no both-channel candidates exist, so the
+ *       boost can never fire — the case pins that the no-regression
+ *       guarantee is structural, not lucky).
+ *
+ * Honesty rules (the planted-pairs #393 A discipline):
+ *  - The texts are synthetic and generic (this package publishes to npm —
+ *    no real infrastructure, people, or fleet detail), but SHAPED like the
+ *    field failure: the proper noun is invented (it must appear nowhere
+ *    else — verified by an invariant test), the rivals are hardened with
+ *    metadata the production pipeline itself writes (base strength, access
+ *    hardening, link counts).
+ *  - Similarities are MEASURED at run time with the real bge-small-en-v1.5
+ *    embedder (ranking-eval.ts) — never asserted into existence. The
+ *    declared expectation is about ORDER under the shipped ranking, and
+ *    the report prints the measured similarity/score of every top row.
+ *  - Strength metadata is written through the production helpers
+ *    (insertMemory + updateMemory + addLink), not hand-rolled SQL, so the
+ *    fixture exercises the same rows retrieval() will see.
+ */
+export interface RankingRow {
+    /** Stable identifier used by expectations, links, and the report. */
+    key: string;
+    content: string;
+    /** ISO timestamp. */
+    createdAt: string;
+    /** knowledge | experience | decision. */
+    memoryType: string;
+    /** Birth base_strength — the value stageImportance would have written. */
+    baseStrength: number;
+    /** Access history (hardening + the decay clock). */
+    accessCount: number;
+    /** Days before the run's `now` of the last access (0 = touched today). */
+    lastAccessedDaysAgo: number;
+    /** Keys this row links to (links are created after all inserts). */
+    linksTo: string[];
+}
+export interface RankingFixture {
+    /** Case identifier (report + photo keys). */
+    key: string;
+    description: string;
+    rows: RankingRow[];
+    query: string;
+    /**
+     * The token that makes the query distinctive — invariant-tested to appear
+     * in exactly one row (the target) and in no rival/filler content.
+     */
+    distinctiveToken: string;
+    /** Declared expectation: this row's key must be returned at position 1. */
+    expectedTop1: string;
+}
+/** All planted rows share project + source_agent (metadata never refuses). */
+export declare const RANKING_PROJECT = "ranking-eval";
+export declare const RANKING_SOURCE_AGENT = "ranking-eval";
+export declare const SIRNAS_FIXTURE: RankingFixture;
+export declare const NO_MATCH_FIXTURE: RankingFixture;
+/** The battery runs against a real snapshot copy (ranking-eval.ts case 3). */
+export declare const REAL_SIRNAS_MEMORY_ID = "12980c9d-0eba-4963-95eb-0548bc2aa93e";
+export declare function checkFixtureInvariants(f: RankingFixture): string[];