@gamaze/hicortex 0.15.3 → 0.16.1

package/dist/eval/recall-sweep.js

@@ -55,6 +55,29 @@
  * badly post-shift. Report written to data/eval-recall-sweep/report.md and
  * printed to stdout.
  *
+ * ## Section 4 — SCOPE dimension (#203 soft project affinity)
+ *
+ * A THIRD family on a SEPARATE project-labeled corpus (the blend-sweep corpus
+ * above has no project labels). Goal: prove #203's soft project affinity drops
+ * cross-scope noise (the boat/battery case — a "hardware" query surfacing
+ * "marine" memories on a shared token like "battery") below the cap WITHOUT a
+ * hard filter, and WITHOUT losing same-scope recall.
+ *
+ *   Corpus: 6 hardware + 6 marine memories. 3 marine memories are
+ *   CONTAMINATION SEEDS (share a token with the hardware queries); 3 are
+ *   marine-only fillers (control).
+ *
+ *   Each hardware query runs TWICE on the same static DB:
+ *     scope OFF — no `project` sent (byte-identical to pre-#203).
+ *     scope ON  — `project: "hardware"` (computeScore adds +projectAffinity 0.15
+ *                 to hardware memories; marine gets 0; no hard filter).
+ *
+ *   Metrics: contamination@5 (marine in top-5 / 5 — LOWER is better),
+ *   recall@5 (gold hardware memory surfaced), sameScope@5 (hardware in top-5).
+ *   Scope is orthogonal to the blend, so the blend weight is held at 0 here —
+ *   `project` is the ONLY variable. Same invariants (noStrengthen, real
+ *   embedder, uniform metadata, neverCalledEmbed self-check).
+ *
  * Run:  npm run eval:recall-sweep   (==  node dist/eval/recall-sweep.js)
  */
 var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
@@ -301,6 +324,81 @@ const SHIFT_PAIRS = [
     ["deploy", "testing"],
 ];
 const WEIGHTS = [0.0, 0.2, 0.4, 0.5, 0.6, 0.8];
+const SCOPE_CORPUS = [
+    // --- hardware (6): the same-scope / gold memories ---
+    {
+        project: "hardware",
+        text: "Lithium-ion battery pack drains 3% per day on standby. Traced to a leaky protection-circuit MOSFET that draws 50mA while the device is powered off.",
+    },
+    {
+        project: "hardware",
+        text: "5V power supply brown-out: the rail drooped to 4.2V under peak load. The bulk capacitor was undersized; added a second decoupling stage.",
+    },
+    {
+        project: "hardware",
+        text: "Charge controller IC never enters constant-voltage mode — the battery overcharges because the CC/CV transition threshold was misconfigured in firmware.",
+    },
+    {
+        project: "hardware",
+        text: "ADC battery voltage reading was 40% off: the resistor divider ratio was wrong, so the fuel gauge reported a full pack at 60%. Recomputed and recalibrated.",
+    },
+    {
+        project: "hardware",
+        text: "Battery thermal runaway during charging: the NTC thermistor lead was cold-soldered, disabling temperature compensation. Reflowed the joint.",
+    },
+    {
+        project: "hardware",
+        text: "Current-sense shunt drifted under load: the 10 milli-ohm resistor heated and its temperature coefficient skewed the reading. Switched to a low-TCR manganin part.",
+    },
+    // --- marine SEEDS (3): share a token (battery / drain / voltage / charge /
+    // temperature) with the hardware queries — a naive keyword/vector search
+    // surfaces these for a hardware query. ---
+    {
+        project: "marine",
+        seed: true,
+        text: "Boat battery drained overnight at anchor: the bilge pump ran continuously because the float switch stuck closed. Replaced the switch.",
+    },
+    {
+        project: "marine",
+        seed: true,
+        text: "Marine house-bank voltage drop: the battery isolator had corroded terminals, dropping 0.8V under load. Cleaned and greased the lugs.",
+    },
+    {
+        project: "marine",
+        seed: true,
+        text: "Trolling-motor battery won't reach full charge: the on-board charger undercharges the deep-cycle bank because the temperature-compensation sense lead is on the wrong terminal.",
+    },
+    // --- marine FILLERS (3): no token overlap with any hardware query. Control
+    // row — these should never surface in either mode, and if they do it is a
+    // ranking bug, not a scope issue. ---
+    {
+        project: "marine",
+        text: "Jib furler jammed: the halyard wrapped on the swivel. Fitted a halyard deflector and re-led the furling line.",
+    },
+    {
+        project: "marine",
+        text: "Chartplotter took eight minutes to a GPS fix: the antenna was under the fiberglass hard-top. Relocated it to the T-top rail.",
+    },
+    {
+        project: "marine",
+        text: "Ablative antifouling wore thin below the waterline after 18 months. Hauled the boat, barrier-coated, and repainted.",
+    },
+];
+/**
+ * 5 hardware-scoped queries. Q1-Q4 each have a marine contamination seed
+ * (varying strength: Q1 near-identical, Q3 antonym-semantic, Q4 token-exact);
+ * Q5 is the CONTROL — no marine memory shares its tokens, so contamination
+ * should be ~0 in both modes (proves the metric isn't trivially inflated).
+ */
+const SCOPE_QUERIES = [
+    { prompt: "battery drain overnight", goldIndices: [0], expect: "strong (M6 near-identical)" },
+    { prompt: "power supply voltage drop", goldIndices: [1], expect: "moderate (M7 voltage)" },
+    { prompt: "battery overcharging", goldIndices: [2], expect: "weak (M8 antonym)" },
+    { prompt: "battery temperature compensation", goldIndices: [4], expect: "strong (M8 token)" },
+    { prompt: "current shunt drift", goldIndices: [5], expect: "control (none)" },
+];
+const SCOPE_K = 5;
+const SCOPE_PROJECT = "hardware";
 function buildSessions() {
     const focused = TOPICS.map((t) => ({
         kind: "focused",
@@ -340,6 +438,32 @@ async function buildCorpusDb(dbPath) {
     }
     return { db, idToTopic };
 }
+/**
+ * Build the SCOPE corpus DB (separate from the blend-sweep corpus — the scope
+ * corpus carries project labels the sweep corpus does not). Same honesty
+ * controls: real embedder, uniform metadata (base_strength=0.5, created_at~now,
+ * no links). The ONLY addition vs the sweep corpus is `project`, which is the
+ * #203 scope label computeScore reads.
+ *
+ * Returns `ids` in corpus order so SCOPE_QUERIES goldIndices map to ids.
+ */
+async function buildScopeDb(dbPath) {
+    const db = (0, db_js_1.initDb)(dbPath);
+    const idToProject = new Map();
+    const ids = [];
+    for (const mem of SCOPE_CORPUS) {
+        const vec = await (0, embedder_js_1.embed)(mem.text);
+        const id = storage.insertMemory(db, mem.text, vec, {
+            sourceAgent: "eval-scope",
+            memoryType: "episode",
+            baseStrength: 0.5, // uniform — strength is NOT a discriminator here
+            project: mem.project, // #203 scope label
+        });
+        idToProject.set(id, mem.project);
+        ids.push(id);
+    }
+    return { db, idToProject, ids };
+}
 /** Embed every distinct prompt once; return a cache keyed by prompt text. */
 async function embedAllPrompts(sessions) {
     const cache = new Map();
@@ -649,6 +773,147 @@ function renderReport(focused, shifts, records, sessions, meta) {
     L.push("");
     return L.join("\n");
 }
+/**
+ * Run the scope sweep: each hardware query is run TWICE on the same static DB —
+ * scope OFF (no `project` sent, byte-identical to pre-#203) and scope ON
+ * (`project: "hardware"`). The #203 affinity boosts hardware memories; marine
+ * memories get nothing. No hard filter anywhere.
+ *
+ * Scope is ORTHOGONAL to the session-intent blend, so the blend weight is held
+ * at 0 (pure prompt embedding) — the ONLY variable is whether `project` is in
+ * options. `queryEmbedding` is supplied (pure prompt), so retrieve() must never
+ * call the embedFn; the `neverCalledEmbed` self-check enforces that.
+ */
+async function runScopeSweep(db, idToProject, ids) {
+    // Precompute query embeddings once — pure prompt, no centroid (scope is the
+    // only variable; blend held at 0 to isolate it).
+    const queryEmb = new Map();
+    for (const q of SCOPE_QUERIES) {
+        if (!queryEmb.has(q.prompt))
+            queryEmb.set(q.prompt, await (0, embedder_js_1.embed)(q.prompt));
+    }
+    const neverCalledEmbed = async () => {
+        throw new Error("recall-sweep (scope): retrieve() called the embedFn — queryEmbedding was not honored. Eval aborted (results would be invalid).");
+    };
+    const records = [];
+    for (let qi = 0; qi < SCOPE_QUERIES.length; qi++) {
+        const q = SCOPE_QUERIES[qi];
+        const gold = new Set(q.goldIndices.map((i) => ids[i]));
+        const promptVec = queryEmb.get(q.prompt);
+        for (const mode of ["off", "on"]) {
+            const results = await (0, retrieval_js_1.retrieve)(db, neverCalledEmbed, q.prompt, {
+                limit: SCOPE_K,
+                queryEmbedding: promptVec,
+                noStrengthen: true,
+                // OFF: omit project entirely (byte-identical to pre-#203).
+                // ON:  send project — computeScore adds +projectAffinity (0.15) to
+                //      every hardware memory; marine memories get 0.
+                ...(mode === "on" ? { project: SCOPE_PROJECT } : {}),
+            });
+            const topIds = results.map((r) => r.id);
+            const marineCount = topIds.filter((id) => idToProject.get(id) === "marine").length;
+            const hardwareCount = topIds.filter((id) => idToProject.get(id) === "hardware").length;
+            const goldCount = topIds.filter((id) => gold.has(id)).length;
+            records.push({
+                queryLabel: `Q${qi + 1}`,
+                prompt: q.prompt,
+                mode,
+                recallAt5: goldCount > 0 ? 1 : 0,
+                goldP5: goldCount,
+                sameScopeP5: hardwareCount,
+                contaminationAt5: marineCount / SCOPE_K,
+                topIds,
+            });
+        }
+    }
+    return records;
+}
+function meanScope(rs, field) {
+    if (rs.length === 0)
+        return 0;
+    return rs.reduce((s, r) => s + r[field], 0) / rs.length;
+}
+function renderScopeSection(records) {
+    const L = [];
+    L.push("## 4. Scope contamination (#203 soft project affinity)\n");
+    L.push(`Separate synthetic corpus (project-labeled): ${SCOPE_CORPUS.filter((m) => m.project === "hardware").length} hardware + ` +
+        `${SCOPE_CORPUS.filter((m) => m.project === "marine").length} marine memories. ` +
+        `${SCOPE_CORPUS.filter((m) => m.project === "marine" && m.seed).length} marine memories are CONTAMINATION SEEDS ` +
+        `(share a token — battery / drain / voltage / charge / temperature — with the hardware queries); ` +
+        `${SCOPE_CORPUS.filter((m) => m.project === "marine" && !m.seed).length} are marine-only fillers (no token overlap — control). ` +
+        `Each hardware query runs TWICE: scope OFF (no \`project\` sent — byte-identical to pre-#203) and scope ON (\`project: "${SCOPE_PROJECT}"\`). ` +
+        `The #203 affinity adds +projectAffinity (0.15) to hardware memories in computeScore; marine memories get 0. No hard filter anywhere.\n`);
+    L.push("_Same invariants as the blend sweep: static DB (`noStrengthen: true`), real bge-small-en-v1.5 embedder, " +
+        "uniform metadata (base_strength=0.5, created_at~now, no links). Blend weight held at 0 (pure prompt) — " +
+        "scope is orthogonal to session-intent keying. The `neverCalledEmbed` self-check still passes._\n");
+    // ---- Per-query table ----
+    L.push("### Per-query results (scope OFF vs ON)\n");
+    L.push("contamination@5 = marine memories in top-5 / 5 (**LOWER is better**). recall@5 = the gold hardware memory surfaced (binary). " +
+        "sameScope@5 = hardware memories in top-5 (incl. non-gold). goldP@5 = gold count in top-5.\n");
+    L.push("| # | query | expected | mode | contamination@5 | recall@5 | sameScope@5 | goldP@5 |");
+    L.push("|---|---|---|---|---|---|---|---|");
+    for (let qi = 0; qi < SCOPE_QUERIES.length; qi++) {
+        const off = records.find((r) => r.queryLabel === `Q${qi + 1}` && r.mode === "off");
+        const on = records.find((r) => r.queryLabel === `Q${qi + 1}` && r.mode === "on");
+        const q = SCOPE_QUERIES[qi];
+        L.push(`| Q${qi + 1} | \`${q.prompt}\` | ${q.expect} | OFF | ${off.contaminationAt5.toFixed(2)} | ${off.recallAt5} | ${off.sameScopeP5} | ${off.goldP5} |`);
+        L.push(`| | | | **ON** | **${on.contaminationAt5.toFixed(2)}** | **${on.recallAt5}** | **${on.sameScopeP5}** | **${on.goldP5}** |`);
+    }
+    L.push("");
+    // ---- Aggregate ----
+    const offRecs = records.filter((r) => r.mode === "off");
+    const onRecs = records.filter((r) => r.mode === "on");
+    const contamOff = meanScope(offRecs, "contaminationAt5");
+    const contamOn = meanScope(onRecs, "contaminationAt5");
+    const recallOff = meanScope(offRecs, "recallAt5");
+    const recallOn = meanScope(onRecs, "recallAt5");
+    const sameOff = meanScope(offRecs, "sameScopeP5");
+    const sameOn = meanScope(onRecs, "sameScopeP5");
+    L.push("### Aggregate (mean over all 5 queries)\n");
+    L.push("| mode | contamination@5 | recall@5 (gold) | sameScope@5 (hardware) |");
+    L.push("|---|---|---|---|");
+    L.push(`| OFF (no project) | ${contamOff.toFixed(3)} | ${recallOff.toFixed(3)} | ${sameOff.toFixed(2)} |`);
+    L.push(`| ON (project=${SCOPE_PROJECT}) | ${contamOn.toFixed(3)} | ${recallOn.toFixed(3)} | ${sameOn.toFixed(2)} |`);
+    L.push("");
+    // ---- Verdict ----
+    const contamDrop = contamOff - contamOn;
+    const recallRegress = recallOn < recallOff - 1e-9;
+    const contamMaterial = contamDrop > 0.01; // any measurable mean drop = material
+    const pass = contamMaterial && !recallRegress;
+    L.push("### Scope verdict (PASS criterion: contamination drops materially AND same-scope recall does not regress)\n");
+    L.push(`- contamination@5: OFF ${contamOff.toFixed(3)} → ON ${contamOn.toFixed(3)} ` +
+        `(${contamDrop >= 0 ? "−" : "+"}${Math.abs(contamDrop).toFixed(3)}, ${pct(Math.abs(contamDrop))} absolute)`);
+    L.push(`- recall@5 (gold surfaced): OFF ${recallOff.toFixed(3)} → ON ${recallOn.toFixed(3)} ` +
+        `${recallRegress ? "**(REGRESSED — gold hardware memory lost)**" : "(no regression)"}`);
+    L.push(`- sameScope@5 (hardware in top-5): OFF ${sameOff.toFixed(2)} → ON ${sameOn.toFixed(2)} ` +
+        `(${sameOn >= sameOff ? "+" : ""}${(sameOn - sameOff).toFixed(2)})`);
+    let verdict;
+    if (pass) {
+        const magnitude = contamDrop > 0.15
+            ? "strongly"
+            : contamDrop > 0.05
+                ? "materially"
+                : "marginally";
+        verdict =
+            `**PASS** — soft project affinity ${magnitude} drops cross-scope contamination (−${pct(contamDrop)} absolute) ` +
+                `WITHOUT losing same-scope recall. The #203 change is sufficient on this corpus; no hard filter needed.`;
+    }
+    else if (!contamMaterial) {
+        verdict =
+            `**FAIL (insufficient)** — contamination did NOT drop materially (−${pct(contamDrop)} absolute). ` +
+                `This is a real finding: #203's soft affinity alone is INSUFFICIENT here, and the BM25/FTS-driven cases ` +
+                `the architect review predicted for #205 (field-weighted BM25F) likely persist — the marine seed wins on ` +
+                `raw token overlap that the additive +0.15 cannot overcome.`;
+    }
+    else {
+        verdict =
+            `**FAIL (recall regression)** — contamination dropped (−${pct(contamDrop)} absolute) but a gold hardware ` +
+                `memory was LOST when scope engaged (recall@5 ${recallOff.toFixed(3)} → ${recallOn.toFixed(3)}). The ` +
+                `affinity is somehow suppressing a same-scope gold result, which should not happen (it is additive only).`;
+    }
+    L.push(`- ${verdict}\n`);
+    return L.join("\n");
+}
 // ---------------------------------------------------------------------------
 // main
 // ---------------------------------------------------------------------------
@@ -665,14 +930,18 @@ async function main() {
     const tmpDir = (0, node_path_1.join)((0, node_os_1.tmpdir)(), `hicortex-recall-sweep-${(0, node_crypto_1.randomUUID)().slice(0, 8)}`);
     (0, node_fs_1.mkdirSync)(tmpDir, { recursive: true });
     const dbPath = (0, node_path_1.join)(tmpDir, "sweep.db");
+    const scopeDbPath = (0, node_path_1.join)(tmpDir, "scope.db");
     const reportDir = (0, node_path_1.join)(process.cwd(), "data", "eval-recall-sweep");
     console.log(`[recall-sweep] temp DB: ${dbPath}`);
+    console.log(`[recall-sweep] scope DB: ${scopeDbPath}`);
     console.log(`[recall-sweep] report dir: ${reportDir}`);
     console.log(`[recall-sweep] ${sessions.filter((s) => s.kind === "focused").length} focused + ` +
         `${sessions.filter((s) => s.kind === "shift").length} shift sessions; ` +
         `${WEIGHTS.length} weights = ${sessions.length * WEIGHTS.length} sweeps x 4 turns = ` +
-        `${sessions.length * WEIGHTS.length * 4} retrieves`);
+        `${sessions.length * WEIGHTS.length * 4} retrieves; ` +
+        `+ scope sweep = ${SCOPE_QUERIES.length} queries x 2 modes = ${SCOPE_QUERIES.length * 2} retrieves`);
     let db = null;
+    let scopeDb = null;
     try {
         console.log("[recall-sweep] building corpus (embedding 30 memories)...");
         const t0 = Date.now();
@@ -685,9 +954,20 @@ async function main() {
         console.log(`[recall-sweep] sweep done in ${Date.now() - t1}ms (${records.length} turn records)`);
         const focused = summarizeFocused(records, sessions);
         const shifts = summarizeShift(records);
-        const report = renderReport(focused, shifts, records, sessions, {
+        let report = renderReport(focused, shifts, records, sessions, {
             memoryCount: CORPUS.length,
         });
+        // ---- SCOPE sweep (#203) ----
+        console.log(`[recall-sweep] building scope corpus (embedding ${SCOPE_CORPUS.length} project-labeled memories)...`);
+        const t2 = Date.now();
+        const scopeBuilt = await buildScopeDb(scopeDbPath);
+        scopeDb = scopeBuilt.db;
+        console.log(`[recall-sweep] scope corpus ready in ${Date.now() - t2}ms (${scopeBuilt.ids.length} memories)`);
+        console.log("[recall-sweep] running scope sweep (OFF vs ON)...");
+        const t3 = Date.now();
+        const scopeRecords = await runScopeSweep(scopeDb, scopeBuilt.idToProject, scopeBuilt.ids);
+        console.log(`[recall-sweep] scope sweep done in ${Date.now() - t3}ms (${scopeRecords.length} turn records)`);
+        report += "\n\n" + renderScopeSection(scopeRecords);
         (0, node_fs_1.mkdirSync)(reportDir, { recursive: true });
         const reportPath = (0, node_path_1.join)(reportDir, "report.md");
         (0, node_fs_1.writeFileSync)(reportPath, report, "utf-8");
@@ -701,6 +981,12 @@ async function main() {
             }
             catch { /* already closed */ }
         }
+        if (scopeDb) {
+            try {
+                scopeDb.close();
+            }
+            catch { /* already closed */ }
+        }
         if ((0, node_fs_1.existsSync)(tmpDir)) {
             try {
                 (0, node_fs_1.rmSync)(tmpDir, { recursive: true, force: true });

package/dist/eval/relevance-eval.d.ts

@@ -0,0 +1,64 @@
+#!/usr/bin/env node
+/**
+ * Real-query relevance + SNIPPET eval — recall QUALITY on real agent prompts.
+ *
+ * v2 (spec `specs/2026-08-02-relevance-eval.md`) extends the v1 selection-only
+ * eval with the SNIPPET layer: v1 asked "did retrieve() surface the right
+ * memories?" (judge sees up to 2000 chars). Production shows the agent only a
+ * ~100-char one-liner (`recall-index.ts#memoryTitle`), so a memory can be
+ * genuinely relevant while its rendered line is useless — v1 scored that as a
+ * win. v2 grades BOTH: `full_verdict` (selection quality, judge sees full
+ * content) and `line_verdict` (snippet quality, judge sees ONLY the rendered
+ * production one-liner — imported from `recall-index.ts`, never reimplemented).
+ *
+ * v2 additions (spec §4, §5, §6, §6b) layered onto the v1 base (prompt
+ * sampling, readonly snapshot handling, embed-once + neverCalledEmbed,
+ * lenient JSON parse, distribution/CI reporting):
+ *   1. Dual verdict per surfaced memory — TWO separate, blind judge calls.
+ *   2. Snippet-length sweep (100/200/300/title+1st-sentence) on a fixed
+ *      40-prompt subset (8 per source).
+ *   3. Similarity-floor + retrieval-source analysis (near-free — logged, not
+ *      re-judged).
+ *   4. Token-cost estimate (char/4) per K and per snippet-length variant.
+ *   5. ~20 rendered ACTUAL production blocks dumped into the report.
+ *   6. Redundancy — one set-level judge call per (prompt × mode) over the
+ *      production 6.
+ *   7. Rate limiting + resumability (§6b, MANDATORY): serial calls,
+ *      `--judge-delay-ms` (default 2000), exponential backoff with jitter on
+ *      429/5xx/timeout (5→10→20→40→80s, max 5 retries, respects
+ *      `Retry-After`), checkpoint-per-call to a `.jsonl` sidecar, `--resume`,
+ *      progress logging, 10%-error-rate abort, `--max-calls` budget guard
+ *      (default 900).
+ *
+ * Prompt corpus (spec §2, owner decision §11.1): EVEN split, 20 prompts per
+ * source × 5 sources — Hermes (lenny, raider, nano) + CC (the DevOps
+ * `infrastructure` project, the `aironic-marine` project). Saved to
+ * `data/prompts.json`, stable/reused verbatim once a valid v2 set exists.
+ *
+ * Judge: GLM-5.2 via z.ai — the INSTRUMENT only. It never picks candidates;
+ * retrieve() (LLM-free) does. A dedicated raw HTTP caller (NOT `LlmClient`) is
+ * used here on purpose: `LlmClient.completeReflect` bakes in a
+ * nightly-tolerant retry policy (30s/60s/120s, unlimited rate-limit patience)
+ * that conflicts with §6b's specific real-time batch policy (5/10/20/40/80s +
+ * jitter, 5 retries, a hard call budget). Implemented directly here rather
+ * than adding a second retry mode to `llm.ts` (out of scope for this eval,
+ * and another agent is concurrently working elsewhere in this repo).
+ *
+ * Honesty invariants (non-negotiable — mirror recall-sweep.ts + spec §7):
+ *   - Snapshot opened READONLY via openSnapshot — never initDb.
+ *   - noStrengthen: true on every retrieve() call.
+ *   - Real bge-small-en-v1.5 embedder, embed-once + queryEmbedding reuse.
+ *   - neverCalledEmbed self-check ABORTS the run if retrieve() ignores
+ *     queryEmbedding (would invalidate every measured number).
+ *   - GLM-5.2 is the JUDGE only, real prompts, no synthetic queries.
+ *   - Production renderer (`formatIndexLine`/`memoryTitle`) imported from
+ *     `recall-index.ts`, never reimplemented.
+ *   - judge_error batches/units excluded from every denominator, reported
+ *     separately.
+ *
+ * Run:
+ *   npm run eval:relevance -- <snapshot.db> [prompts.json] [report.md] \
+ *       [--judge-delay-ms=2000] [--max-calls=900] [--resume] \
+ *       [--verdicts-json=path] [--verdicts-jsonl=path]
+ */
+export {};