@gamaze/hicortex 0.20.7 → 0.20.10

package/dist/eval/ranking-battery.d.ts

@@ -0,0 +1,78 @@
+/**
+ * The real-query battery for the #425 ranking gate — the DETERMINISTIC,
+ * dependency-free half (kept out of ranking-eval.ts so vitest can import
+ * it without pulling the ONNX embedder; same split as planted-fixtures vs
+ * planted-eval).
+ *
+ * The battery is a stable set of ~30 queries derived from the SNAPSHOT
+ * CORPUS ITSELF: fixed df windows over content tokens, fixed tie-breaks —
+ * same snapshot, same queries, every run. Bands: 10 high-df (50-400 docs),
+ * 10 mid-df (10-50), 5 rare-df (3-10), 4 mid-sentence capitalized proper
+ * nouns (df 2-8), plus the fixed "Sirnäs" query (the owner's live case).
+ *
+ * Also carries the photo-comparison gates the sweep protocol defines:
+ *   - case 2 byte-stability (no-match queries must not drift),
+ *   - battery top-1 stability >= 90%,
+ *   - no query losing a both-channel exact match from its top-3.
+ */
+export declare const SIRNAS_QUERY = "Sirn\u00E4s";
+/** Compact English stopword block (battery derivation only, not product). */
+export declare const STOPWORDS: Set<string>;
+export interface BatteryQuery {
+    q: string;
+    band: "proper-noun-fixed" | "high-df" | "mid-df" | "rare-df" | "proper-noun";
+}
+export interface BatteryResult {
+    q: string;
+    band: string;
+    top8: string[];
+    /** Per-result provenance (ranking-eval fills it; the gates read it). */
+    meta?: Array<{
+        id: string;
+        source: string | null;
+        term: boolean | null;
+    }>;
+}
+export interface BatteryPhoto {
+    queries: BatteryResult[];
+    /** Rank (1-based) of the real Sirnäs memory for the fixed query; null = not in top-8. */
+    sirnasRank: number | null;
+}
+/**
+ * Derive the battery from live corpus contents — DETERMINISTIC given the
+ * snapshot: fixed stopword list, fixed df windows, fixed tie-breaks
+ * (df desc, then token asc).
+ */
+export declare function deriveBatteryQueries(rows: Array<{
+    id: string;
+    content: string;
+}>): BatteryQuery[];
+/** Case 2 gate: the FULL returned list must be byte-identical (ids + order). */
+export declare function compareCase2(baseline: string[], current: string[]): boolean;
+export interface BatteryComparison {
+    top1Stability: number;
+    /** Stability with intended D3 promotions excluded: old top-1 was NOT a
+     *  both-channel exact match and the new top-1 IS one (the boost doing its
+     *  job on real queries — reported, not gated). */
+    top1StabilityExPromotions: number;
+    changedTop1: string[];
+    /** Queries whose top-3 LOST every both-channel exact match (the gate:
+     *  must be empty). */
+    lostBothChannelExact: string[];
+    /** Per-id exact-match (term-carrying) rows displaced from top-3 — reported
+     *  for the sweep record, not gated (multiple genuine matches make per-id
+     *  membership arbitrary). */
+    lostExactMatches: string[];
+}
+/**
+ * Battery gates vs a recorded baseline photo. `contentById` maps memory id →
+ * content for the corpus BOTH photos were recorded against.
+ *
+ * Gates: (a) any query whose baseline top-3 held a BOTH-CHANNEL exact match
+ * (vector+FTS agreeing on a term-carrying row) must still hold one — the
+ * genuine-match guarantee never regresses; (b) top-1 stability. Intended D3
+ * promotions (a both-channel exact match taking top-1 from a non-exact or
+ * single-channel row) are counted separately — they are the feature firing,
+ * and the sweep record shows both numbers.
+ */
+export declare function batteryComparison(baseline: BatteryPhoto, current: BatteryPhoto, contentById: Map<string, string>): BatteryComparison;

package/dist/eval/ranking-battery.js

@@ -0,0 +1,181 @@
+"use strict";
+/**
+ * The real-query battery for the #425 ranking gate — the DETERMINISTIC,
+ * dependency-free half (kept out of ranking-eval.ts so vitest can import
+ * it without pulling the ONNX embedder; same split as planted-fixtures vs
+ * planted-eval).
+ *
+ * The battery is a stable set of ~30 queries derived from the SNAPSHOT
+ * CORPUS ITSELF: fixed df windows over content tokens, fixed tie-breaks —
+ * same snapshot, same queries, every run. Bands: 10 high-df (50-400 docs),
+ * 10 mid-df (10-50), 5 rare-df (3-10), 4 mid-sentence capitalized proper
+ * nouns (df 2-8), plus the fixed "Sirnäs" query (the owner's live case).
+ *
+ * Also carries the photo-comparison gates the sweep protocol defines:
+ *   - case 2 byte-stability (no-match queries must not drift),
+ *   - battery top-1 stability >= 90%,
+ *   - no query losing a both-channel exact match from its top-3.
+ */
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.STOPWORDS = exports.SIRNAS_QUERY = void 0;
+exports.deriveBatteryQueries = deriveBatteryQueries;
+exports.compareCase2 = compareCase2;
+exports.batteryComparison = batteryComparison;
+exports.SIRNAS_QUERY = "Sirnäs";
+/** Compact English stopword block (battery derivation only, not product). */
+exports.STOPWORDS = new Set(("the a an and or but if while with without for from into onto about after before during under over " +
+    "this that these those there here when where which who whom whose what why how all any both each " +
+    "few more most other some such no nor not only own same so than too very can will just should now " +
+    "was were been being are was is am be have has had having do does did doing would could might must " +
+    "also because through between against above below off again further then once its it their they " +
+    "them we you your our i me my he she his her him us of as at by in on to per via due using use " +
+    "used using make made get gets got set sets let run ran running work works working need needs " +
+    "added add adds change changed new next last first second one two three session sessions " +
+    "memory memories project projects user users system systems thing things stuff lot bit way ways " +
+    "time times day days week weeks month months year years")
+    .split(/\s+/));
+/**
+ * Derive the battery from live corpus contents — DETERMINISTIC given the
+ * snapshot: fixed stopword list, fixed df windows, fixed tie-breaks
+ * (df desc, then token asc).
+ */
+function deriveBatteryQueries(rows) {
+    const docs = rows.map((r) => r.content.toLowerCase());
+    const df = new Map();
+    for (const doc of docs) {
+        const seen = new Set();
+        for (const m of doc.matchAll(/[\p{L}\p{N}]{5,}/gu)) {
+            const t = m[0];
+            if (exports.STOPWORDS.has(t) || seen.has(t))
+                continue;
+            seen.add(t);
+            df.set(t, (df.get(t) ?? 0) + 1);
+        }
+    }
+    const inWindow = (lo, hi) => [...df.entries()]
+        .filter(([t, d]) => d >= lo && d < hi && !exports.STOPWORDS.has(t))
+        .sort((a, b) => (b[1] - a[1] !== 0 ? b[1] - a[1] : a[0].localeCompare(b[0])))
+        .map(([t, d]) => [t, d]);
+    const queries = [
+        { q: exports.SIRNAS_QUERY, band: "proper-noun-fixed" },
+    ];
+    for (const [t] of inWindow(50, 401).slice(0, 10))
+        queries.push({ q: t, band: "high-df" });
+    for (const [t] of inWindow(10, 50).slice(0, 10))
+        queries.push({ q: t, band: "mid-df" });
+    for (const [t] of inWindow(3, 10).slice(0, 5))
+        queries.push({ q: t, band: "rare-df" });
+    // Proper nouns: capitalized, appearing mid-sentence (not the first word
+    // after a sentence break — cheap deterministic heuristic), df 2-8.
+    // "Sirnäs" itself may or may not surface here; the fixed query above
+    // guarantees coverage.
+    const capDf = new Map();
+    for (const content of rows.map((r) => r.content)) {
+        const seen = new Set();
+        for (const m of content.matchAll(/[^\s.!?;:]\s+([\p{Lu}][\p{L}\p{N}\p{M}-]{3,})/gu)) {
+            const t = m[1];
+            if (!exports.STOPWORDS.has(t.toLowerCase()))
+                seen.add(t);
+        }
+        for (const t of seen)
+            capDf.set(t, (capDf.get(t) ?? 0) + 1);
+    }
+    const properNouns = [...capDf.entries()]
+        .filter(([t, d]) => d >= 2 && d <= 8)
+        .sort((a, b) => (b[1] - a[1] !== 0 ? b[1] - a[1] : a[0].localeCompare(b[0])))
+        .map(([t]) => t);
+    for (const t of properNouns.slice(0, 4))
+        queries.push({ q: t, band: "proper-noun" });
+    return queries;
+}
+/** Case 2 gate: the FULL returned list must be byte-identical (ids + order). */
+function compareCase2(baseline, current) {
+    return (baseline.length === current.length &&
+        baseline.every((id, i) => id === current[i]));
+}
+/** A both-channel row carrying the query term — the genuine-match signature. */
+function isBothChannelExact(m, id, term, contentById) {
+    if (m)
+        return m.source === "both" && m.term === true;
+    // Baseline photos recorded before `meta` existed: fall back to content
+    // containment; conservatively treat source as unknown (not "both").
+    const content = contentById.get(id);
+    return Boolean(content && content.toLowerCase().includes(term));
+}
+/**
+ * Battery gates vs a recorded baseline photo. `contentById` maps memory id →
+ * content for the corpus BOTH photos were recorded against.
+ *
+ * Gates: (a) any query whose baseline top-3 held a BOTH-CHANNEL exact match
+ * (vector+FTS agreeing on a term-carrying row) must still hold one — the
+ * genuine-match guarantee never regresses; (b) top-1 stability. Intended D3
+ * promotions (a both-channel exact match taking top-1 from a non-exact or
+ * single-channel row) are counted separately — they are the feature firing,
+ * and the sweep record shows both numbers.
+ */
+function batteryComparison(baseline, current, contentById) {
+    const baseByQ = new Map(baseline.queries.map((b) => [b.q, b]));
+    const baseMeta = (b, rank) => b.meta?.[rank];
+    let stable = 0;
+    let stableExPromotions = 0;
+    let compared = 0;
+    const changedTop1 = [];
+    const lostBothChannelExact = [];
+    const lostExactMatches = [];
+    for (const cur of current.queries) {
+        const base = baseByQ.get(cur.q);
+        if (!base)
+            continue;
+        compared++;
+        if (cur.top8[0] === base.top8[0]) {
+            stable++;
+            stableExPromotions++;
+            continue;
+        }
+        changedTop1.push(cur.q);
+        const term = cur.q.toLowerCase();
+        const oldExact = baseMeta(base, 0) !== undefined
+            ? isBothChannelExact(baseMeta(base, 0), base.top8[0], term, contentById)
+            : null;
+        const newExact = cur.meta?.[0] !== undefined
+            ? isBothChannelExact(cur.meta[0], cur.top8[0], term, contentById)
+            : null;
+        // An intended promotion: the new top-1 is a both-channel exact match the
+        // old top-1 was not. (Null = provenance unknown — pre-meta baseline —
+        // counts as a plain change.)
+        if (newExact === true && oldExact === false)
+            stableExPromotions++;
+    }
+    for (const cur of current.queries) {
+        const base = baseByQ.get(cur.q);
+        if (!base || cur.q === exports.SIRNAS_QUERY)
+            continue;
+        const term = cur.q.toLowerCase();
+        const curTop3 = new Set(cur.top8.slice(0, 3));
+        let baseHadBothChannelExact = false;
+        for (let i = 0; i < Math.min(3, base.top8.length); i++) {
+            const id = base.top8[i];
+            const content = contentById.get(id);
+            const carries = Boolean(content && content.toLowerCase().includes(term));
+            if (carries && !curTop3.has(id))
+                lostExactMatches.push(`${cur.q}: ${id.slice(0, 8)}`);
+            const m = base.meta?.[i];
+            if (m ? m.source === "both" && m.term === true : false)
+                baseHadBothChannelExact = true;
+        }
+        if (baseHadBothChannelExact) {
+            const stillHas = cur.top8
+                .slice(0, 3)
+                .some((id, i) => cur.meta?.[i]?.source === "both" && cur.meta?.[i]?.term === true);
+            if (!stillHas)
+                lostBothChannelExact.push(cur.q);
+        }
+    }
+    return {
+        top1Stability: compared > 0 ? stable / compared : 1,
+        top1StabilityExPromotions: compared > 0 ? stableExPromotions / compared : 1,
+        changedTop1,
+        lostBothChannelExact,
+        lostExactMatches,
+    };
+}

package/dist/eval/ranking-eval.d.ts

@@ -0,0 +1,41 @@
+#!/usr/bin/env node
+/**
+ * Search-trust ranking gate (#425) — the AC1/AC2/AC3 before/after photo.
+ *
+ *   npm run eval:ranking [--snapshot <db>] [--photo <out.json>]
+ *                        [--compare <baseline.json>] [--scoring '<json>']
+ *
+ *   --snapshot <db>    enables case 3: the deterministic real-query battery
+ *                      against a READONLY snapshot (openSnapshot — never
+ *                      initDb; the snapshot is never touched). Also reports
+ *                      where the real "Sirnäs" memory ranks today (the
+ *                      measured red photo pre-fix, AC10 pre-condition).
+ *   --photo <out.json> where to write the machine photo (defaults to
+ *                      data/ranking-eval-photo.json). ALWAYS written, even
+ *                      on a red run — the photo IS the measurement.
+ *   --compare <json>   gate against a previously recorded photo: case 2's
+ *                      returned list must be byte-identical, and the battery
+ *                      must hold top-1 stability >= 90% with no query losing
+ *                      a both-channel exact match from its top-3.
+ *   --scoring '<json>' a JSON object of ScoringWeights overrides passed to
+ *                      configureScoring (the eval/test seam, #408) — the
+ *                      sweep knob. Production runs omit it.
+ *
+ * Cases (fixtures: ranking-fixtures.ts, real bge-small-en-v1.5 embedder):
+ *   1. sirnas_exact_match — planted corpus in a THROWAWAY writable DB;
+ *      declared expectation: the proper-noun exact-match row returns #1.
+ *      Exit code reflects the verdict (a pre-fix run is EXPECTED to exit
+ *      non-zero — that red photo is the before picture).
+ *   2. no_match_control — same corpus, a "mamma"-class query with zero FTS
+ *      hits; records the full returned list for byte-stability comparison.
+ *   3. real-query battery (needs --snapshot) — ~30 queries derived
+ *      deterministically from the snapshot corpus (top/mid/rare-frequency
+ *      distinctive terms + proper nouns) + the fixed "Sirnäs" query; top-8
+ *      ids per query recorded to the photo.
+ *
+ * Honesty invariants: similarities are MEASURED (real embedder, embed-once,
+ * queryEmbedding reuse), noStrengthen on every retrieve() (the probe never
+ * perturbs the store), readonly snapshot access, and the planted corpora
+ * live in mkdtemp dirs removed on exit. Never touches ~/.hicortex.
+ */
+export {};