@cohortapp/agent-sdk 2.16.0 → 2.18.4

package/lib/rag/rag.test.mjs

@@ -1,505 +0,0 @@
-/**
- * Maestro — RAG indexer + search tests (audit H10).
- *
- * Runs fully offline against a temp better-sqlite3 DB. Embeddings are
- * injected via the additive `_embedTexts` (indexer) / `_embedQuery` (search)
- * seams so no network call is ever made.
- *
- * Coverage:
- *   - FTS search ordering + result shape
- *   - L18: AND-first multi-term ranking (both-term doc beats one-term doc),
- *          with OR fallback when the strict conjunction has zero hits
- *   - L19: relevance is a faithful [0,1] from the NEGATIVE bm25() score,
- *          correctly ordered, and a zero-score row does not blow up
- *   - M12: a model/dim-mismatched embedding is NOT returned by semantic search
- *   - M12: re-index under a new model purges stale-dim embeddings
- *   - M13: an interrupted embed is completed on the next NON-force run
- *   - H11: the scoring SELECT does not pull body for all rows (bodies are
- *          materialized only for the returned topN), and a large candidate
- *          set is capped by the documented ceiling.
- */
-
-import { test } from "node:test";
-import assert from "node:assert/strict";
-import { mkdtempSync, mkdirSync, writeFileSync, rmSync } from "node:fs";
-import { tmpdir } from "node:os";
-import { join } from "node:path";
-
-import { buildIndex } from "./indexer.mjs";
-import { search, DEFAULT_MAX_SEMANTIC_CANDIDATES, ftsRelevance } from "./search.mjs";
-import { serializeVector, l2Normalize } from "./embed.mjs";
-import DatabaseCtor from "better-sqlite3";
-
-// ---------------------------------------------------------------------------
-// Helpers
-// ---------------------------------------------------------------------------
-
-function makeAgentRoot(files) {
-  const root = mkdtempSync(join(tmpdir(), "maestro-rag-"));
-  for (const [rel, content] of Object.entries(files)) {
-    const abs = join(root, rel);
-    mkdirSync(join(abs, ".."), { recursive: true });
-    writeFileSync(abs, content, "utf-8");
-  }
-  return root;
-}
-
-function dbPathFor(root) {
-  return join(root, "state/rag/index/agent.db");
-}
-
-/**
- * Deterministic fake embedder. Maps each text to a fixed-dim unit vector
- * whose direction is keyed by which "concept" tokens it contains, so that
- * texts sharing a concept are similar and the query lands nearest the
- * intended doc. Returns the { vectors, model } shape of embed.mjs.
- */
-function fakeEmbedder(model, dim) {
-  // Concept axes: each concept gets one hot dimension.
-  const CONCEPTS = ["alpha", "beta", "gamma"];
-  return async function _embedTexts(texts) {
-    const vectors = texts.map((t) => {
-      const v = new Float32Array(dim);
-      const lower = String(t || "").toLowerCase();
-      let any = false;
-      for (let i = 0; i < CONCEPTS.length && i < dim; i++) {
-        if (lower.includes(CONCEPTS[i])) { v[i] = 1; any = true; }
-      }
-      if (!any) v[dim - 1] = 1; // park empty/probe text on a junk axis
-      return l2Normalize(v);
-    });
-    return { vectors, model, usage: { total_tokens: 0 } };
-  };
-}
-
-function fakeQuery(model, dim) {
-  const embedTexts = fakeEmbedder(model, dim);
-  return async function _embedQuery(q) {
-    const { vectors } = await embedTexts([q]);
-    return { vector: vectors[0], model };
-  };
-}
-
-const SAMPLE = {
-  "memory/alpha.md": "# Alpha decision\n\nWe decided alpha is the strategy for Q3. alpha alpha.",
-  "memory/beta.md": "# Beta plan\n\nThe beta rollout covers beta markets. beta beta.",
-  "knowledge/gamma.md": "# Gamma notes\n\nGamma research on gamma topics. gamma gamma.",
-};
-
-// ---------------------------------------------------------------------------
-// FTS path — ordering + result shape
-// ---------------------------------------------------------------------------
-
-test("FTS search returns the matching doc with the documented result shape", async () => {
-  const root = makeAgentRoot(SAMPLE);
-  try {
-    const r = await buildIndex({ agentRoot: root, embeddings: false });
-    assert.equal(r.added, 3, "all three fixtures indexed");
-    assert.ok(r.chunks >= 3, "at least one chunk per fixture");
-
-    const hits = await search({ query: "alpha", agentRoot: root, mode: "fts", topN: 5 });
-    assert.ok(hits.length >= 1, "alpha query returns at least one hit");
-    assert.equal(hits[0].source_path, "memory/alpha.md", "best hit is the alpha doc");
-
-    const h = hits[0];
-    for (const k of ["source_path", "kind", "title", "heading", "snippet", "score", "relevance", "retrieval"]) {
-      assert.ok(k in h, `result has '${k}'`);
-    }
-    assert.equal(h.retrieval, "fts");
-    assert.equal(typeof h.relevance, "number");
-    assert.ok(h.relevance > 0 && h.relevance <= 1, "relevance in (0,1]");
-    assert.equal(typeof h.snippet, "string");
-  } finally {
-    rmSync(root, { recursive: true, force: true });
-  }
-});
-
-test("FTS results are ordered by relevance (top hit first)", async () => {
-  const root = makeAgentRoot(SAMPLE);
-  try {
-    await buildIndex({ agentRoot: root, embeddings: false });
-    const hits = await search({ query: "gamma", agentRoot: root, mode: "fts", topN: 5 });
-    assert.ok(hits.length >= 1);
-    assert.equal(hits[0].source_path, "knowledge/gamma.md");
-    for (let i = 1; i < hits.length; i++) {
-      assert.ok(hits[i].relevance <= hits[i - 1].relevance + 1e-9, "non-increasing relevance");
-    }
-  } finally {
-    rmSync(root, { recursive: true, force: true });
-  }
-});
-
-// ---------------------------------------------------------------------------
-// L18 — AND-first multi-term ranking (precision) with OR fallback
-// ---------------------------------------------------------------------------
-
-// Corpus where the AND/OR distinction matters: one doc has BOTH query terms,
-// one has only the first, one has only the second.
-const TWO_TERM = {
-  "memory/both.md": "# Both\n\nThe orchard manifesto covers orchard and manifesto together. orchard manifesto.",
-  "memory/orchard.md": "# Orchard only\n\nThis note is about the orchard. orchard orchard orchard orchard.",
-  "memory/manifesto.md": "# Manifesto only\n\nThis note is about the manifesto. manifesto manifesto manifesto.",
-};
-
-test("L18: a two-term query ranks the doc containing BOTH terms first (AND-first)", async () => {
-  const root = makeAgentRoot(TWO_TERM);
-  try {
-    await buildIndex({ agentRoot: root, embeddings: false });
-    const hits = await search({ query: "orchard manifesto", agentRoot: root, mode: "fts", topN: 5 });
-    assert.ok(hits.length >= 1, "two-term query returns hits");
-    // AND-first means only the both-terms doc satisfies the strict conjunction.
-    assert.equal(hits[0].source_path, "memory/both.md", "doc with BOTH terms ranks first");
-    // The one-term-only docs must rank no higher than the both-terms doc.
-    const both = hits.find((h) => h.source_path === "memory/both.md");
-    for (const h of hits) {
-      if (h.source_path === "memory/both.md") continue;
-      assert.ok(
-        both.relevance >= h.relevance,
-        `both-terms doc (${both.relevance}) >= ${h.source_path} (${h.relevance})`,
-      );
-    }
-  } finally {
-    rmSync(root, { recursive: true, force: true });
-  }
-});
-
-test("L18: AND-first falls back to OR when no doc contains every term", async () => {
-  const root = makeAgentRoot(TWO_TERM);
-  try {
-    await buildIndex({ agentRoot: root, embeddings: false });
-    // "orchard" exists; "zucchini" exists in no doc. Strict AND → zero hits, so
-    // the OR fallback must still surface the orchard docs rather than nothing.
-    const hits = await search({ query: "orchard zucchini", agentRoot: root, mode: "fts", topN: 5 });
-    assert.ok(hits.length >= 1, "OR fallback returns the orchard docs");
-    assert.ok(
-      hits.some((h) => h.source_path === "memory/orchard.md" || h.source_path === "memory/both.md"),
-      "an orchard-bearing doc is recovered by the OR fallback",
-    );
-  } finally {
-    rmSync(root, { recursive: true, force: true });
-  }
-});
-
-// ---------------------------------------------------------------------------
-// L19 — faithful [0,1] relevance from the NEGATIVE bm25() score; zero-safe
-// ---------------------------------------------------------------------------
-
-test("L19: FTS relevance is within [0,1], top hit is 1.0, and ordering matches rank", async () => {
-  const root = makeAgentRoot(TWO_TERM);
-  try {
-    await buildIndex({ agentRoot: root, embeddings: false });
-    const hits = await search({ query: "orchard manifesto", agentRoot: root, mode: "fts", topN: 5 });
-    assert.ok(hits.length >= 1);
-    for (const h of hits) {
-      assert.equal(typeof h.relevance, "number");
-      assert.ok(Number.isFinite(h.relevance), `relevance is finite (got ${h.relevance})`);
-      assert.ok(h.relevance >= 0 && h.relevance <= 1, `relevance in [0,1] (got ${h.relevance})`);
-      // bm25() is negative (lower = better); score must never be positive here.
-      assert.ok(h.score <= 0, `bm25 score is non-positive (got ${h.score})`);
-    }
-    assert.ok(Math.abs(hits[0].relevance - 1) < 1e-9, "top hit relevance is 1.0");
-    for (let i = 1; i < hits.length; i++) {
-      assert.ok(
-        hits[i].relevance <= hits[i - 1].relevance + 1e-9,
-        "relevance is non-increasing with rank",
-      );
-    }
-  } finally {
-    rmSync(root, { recursive: true, force: true });
-  }
-});
-
-test("L19: ftsRelevance transform is zero-safe and never Inf/NaN/out-of-range", () => {
-  // Direct unit test of the exported transform — the case the old code got
-  // wrong. bm25() is negative; 0 means "no overlap".
-  //   - top hit (score === best, both negative) → exactly 1.0
-  //   - a weaker (less-negative) score decays toward 0 and stays in [0,1]
-  //   - a row score of 0 → 0 (the old `best / 0` produced Infinity here)
-  //   - a best score of 0 (degenerate set) → 0, never a division blow-up
-  //   - non-finite inputs → 0, never NaN
-  assert.ok(Math.abs(ftsRelevance(-5, -5) - 1) < 1e-9, "top hit (score===best) is 1.0");
-  const weak = ftsRelevance(-1, -5);
-  assert.ok(weak > 0 && weak < 1, `weaker score decays into (0,1) (got ${weak})`);
-  assert.ok(Math.abs(weak - 0.2) < 1e-9, "ratio best/score is faithful (-5/-1 = 0.2)");
-  assert.equal(ftsRelevance(0, -5), 0, "row score 0 maps to 0 (no Infinity)");
-  assert.equal(ftsRelevance(-5, 0), 0, "best score 0 maps to 0 (no division blow-up)");
-  assert.equal(ftsRelevance(0, 0), 0, "all-zero maps to 0");
-  assert.equal(ftsRelevance(Infinity, -5), 0, "non-finite score → 0");
-  assert.equal(ftsRelevance(-5, NaN), 0, "non-finite best → 0");
-  for (const v of [ftsRelevance(-5, -5), weak, ftsRelevance(0, -5), ftsRelevance(-5, 0)]) {
-    assert.ok(Number.isFinite(v) && v >= 0 && v <= 1, `relevance ${v} is finite & in [0,1]`);
-  }
-});
-
-test("L19: a degenerate single-doc FTS result yields a finite, in-range relevance", async () => {
-  // End-to-end sanity: even the simplest corpus must never emit Inf/NaN
-  // relevance (the failure mode of the old `bestScore / r.score` ratio).
-  const root = makeAgentRoot({
-    "memory/solo.md": "# Solo\n\nwidget widget widget widget widget widget.",
-  });
-  try {
-    await buildIndex({ agentRoot: root, embeddings: false });
-    const hits = await search({ query: "widget", agentRoot: root, mode: "fts", topN: 5 });
-    assert.ok(hits.length >= 1, "single-doc corpus returns the doc");
-    for (const h of hits) {
-      assert.ok(Number.isFinite(h.relevance), "relevance finite on single-row result");
-      assert.ok(h.relevance >= 0 && h.relevance <= 1, "relevance in [0,1] on single-row result");
-    }
-    // The sole hit is the best hit → relevance pinned to 1.0 (best/best),
-    // never Infinity (which the old `bestScore / r.score` guard could emit).
-    assert.ok(Math.abs(hits[0].relevance - 1) < 1e-9, "sole/best hit relevance is exactly 1.0");
-  } finally {
-    rmSync(root, { recursive: true, force: true });
-  }
-});
-
-// ---------------------------------------------------------------------------
-// Semantic path — happy ordering via injected offline embeddings
-// ---------------------------------------------------------------------------
-
-test("semantic search ranks the concept-matching doc first (offline)", async () => {
-  const DIM = 8;
-  const MODEL = "fake-small";
-  const root = makeAgentRoot(SAMPLE);
-  try {
-    const r = await buildIndex({
-      agentRoot: root,
-      embeddings: true,
-      embedModel: MODEL,
-      _embedTexts: fakeEmbedder(MODEL, DIM),
-    });
-    assert.ok(r.embeds >= 3, "embeddings generated for all chunks");
-
-    const hits = await search({
-      query: "tell me about beta",
-      agentRoot: root,
-      mode: "semantic",
-      topN: 5,
-      embedModel: MODEL,
-      _embedQuery: fakeQuery(MODEL, DIM),
-    });
-    assert.ok(hits.length >= 1, "semantic returns hits");
-    assert.equal(hits[0].source_path, "memory/beta.md", "beta doc ranks first");
-    assert.equal(hits[0].retrieval, "semantic");
-    assert.ok(typeof hits[0].snippet === "string" && hits[0].snippet.length > 0, "snippet materialized for topN");
-  } finally {
-    rmSync(root, { recursive: true, force: true });
-  }
-});
-
-// ---------------------------------------------------------------------------
-// M12 — model/dim mismatch must NOT be returned
-// ---------------------------------------------------------------------------
-
-test("M12: a mismatched-dim embedding is excluded from semantic results", async () => {
-  const DIM = 8;
-  const MODEL = "fake-small";
-  const root = makeAgentRoot(SAMPLE);
-  try {
-    await buildIndex({
-      agentRoot: root,
-      embeddings: true,
-      embedModel: MODEL,
-      _embedTexts: fakeEmbedder(MODEL, DIM),
-    });
-
-    // Corrupt one chunk's embedding to a different (model, dim): point the
-    // gamma chunk at a vector that, if naively scored, would be a strong
-    // match for a "gamma" query — but it has the wrong dim, so cosine would
-    // be 0 (length mismatch) and the M12 SQL filter excludes it outright.
-    const db = new DatabaseCtor(dbPathFor(root));
-    try {
-      const gammaRow = db.prepare(
-        "SELECT rowid FROM chunks WHERE source_path = 'knowledge/gamma.md' LIMIT 1"
-      ).get();
-      assert.ok(gammaRow, "gamma chunk exists");
-      const wrong = l2Normalize(new Float32Array([0, 0, 1, 0])); // dim 4, not 8
-      db.prepare(
-        "UPDATE embeddings SET model = ?, dim = ?, vector = ? WHERE chunk_rowid = ?"
-      ).run("other-model", 4, serializeVector(wrong), gammaRow.rowid);
-    } finally {
-      db.close();
-    }
-
-    const hits = await search({
-      query: "gamma",
-      agentRoot: root,
-      mode: "semantic",
-      topN: 5,
-      embedModel: MODEL,
-      _embedQuery: fakeQuery(MODEL, DIM),
-    });
-    // The gamma doc had its embedding moved to a foreign (model, dim) so it
-    // must NOT appear in results filtered to the query's (MODEL, DIM).
-    assert.ok(
-      !hits.some((h) => h.source_path === "knowledge/gamma.md"),
-      "mismatched-dim gamma row is excluded"
-    );
-  } finally {
-    rmSync(root, { recursive: true, force: true });
-  }
-});
-
-test("M12: re-index under a new model purges stale-dim embeddings", async () => {
-  const root = makeAgentRoot(SAMPLE);
-  try {
-    await buildIndex({
-      agentRoot: root,
-      embeddings: true,
-      embedModel: "fake-small",
-      _embedTexts: fakeEmbedder("fake-small", 8),
-    });
-
-    // Re-index forcing a fresh embed under a NEW model + dim. Force re-chunks
-    // so newChunks is non-empty, but the purge must also clear old-dim rows.
-    const r2 = await buildIndex({
-      agentRoot: root,
-      force: true,
-      embeddings: true,
-      embedModel: "fake-large",
-      _embedTexts: fakeEmbedder("fake-large", 16),
-    });
-    assert.ok(r2.purged >= 0, "purged field present");
-
-    const db = new DatabaseCtor(dbPathFor(root), { readonly: true });
-    try {
-      const models = db.prepare("SELECT DISTINCT model, dim FROM embeddings").all();
-      assert.equal(models.length, 1, "only one (model, dim) survives");
-      assert.equal(models[0].model, "fake-large");
-      assert.equal(models[0].dim, 16);
-    } finally {
-      db.close();
-    }
-  } finally {
-    rmSync(root, { recursive: true, force: true });
-  }
-});
-
-// ---------------------------------------------------------------------------
-// M13 — interrupted embed completed on next non-force run
-// ---------------------------------------------------------------------------
-
-test("M13: missing embeddings are backfilled on the next non-force run", async () => {
-  const DIM = 8;
-  const MODEL = "fake-small";
-  const root = makeAgentRoot(SAMPLE);
-  try {
-    const r1 = await buildIndex({
-      agentRoot: root,
-      embeddings: true,
-      embedModel: MODEL,
-      _embedTexts: fakeEmbedder(MODEL, DIM),
-    });
-    const chunkCount = r1.chunks;
-    assert.ok(chunkCount >= 3);
-
-    // Simulate a crash mid-embed: delete a subset of embeddings while leaving
-    // files.mtime untouched (the file is "indexed" but partly unembedded).
-    const db = new DatabaseCtor(dbPathFor(root));
-    let deleted;
-    try {
-      deleted = db.prepare(
-        "DELETE FROM embeddings WHERE chunk_rowid IN " +
-        "(SELECT rowid FROM chunks WHERE source_path = 'memory/beta.md')"
-      ).run().changes;
-      assert.ok(deleted >= 1, "deleted at least one embedding to simulate interruption");
-      const before = db.prepare("SELECT COUNT(*) AS n FROM embeddings").get().n;
-      assert.equal(before, chunkCount - deleted);
-    } finally {
-      db.close();
-    }
-
-    // Re-run WITHOUT force. The unchanged-file skip would normally skip beta,
-    // but the missing-embedding sweep (M13) must re-embed the stranded chunks.
-    const r2 = await buildIndex({
-      agentRoot: root,
-      embeddings: true,
-      embedModel: MODEL,
-      _embedTexts: fakeEmbedder(MODEL, DIM),
-    });
-    assert.equal(r2.added, 0, "no new files");
-    assert.equal(r2.updated, 0, "no changed files (mtime unchanged)");
-    assert.equal(r2.embeds, deleted, "exactly the missing embeddings were re-embedded");
-
-    const db2 = new DatabaseCtor(dbPathFor(root), { readonly: true });
-    try {
-      const after = db2.prepare("SELECT COUNT(*) AS n FROM embeddings").get().n;
-      assert.equal(after, chunkCount, "every chunk has an embedding again");
-    } finally {
-      db2.close();
-    }
-
-    // And the beta doc is searchable again semantically.
-    const hits = await search({
-      query: "beta", agentRoot: root, mode: "semantic", topN: 5,
-      embedModel: MODEL, _embedQuery: fakeQuery(MODEL, DIM),
-    });
-    assert.ok(hits.some((h) => h.source_path === "memory/beta.md"), "beta re-embedded + retrievable");
-  } finally {
-    rmSync(root, { recursive: true, force: true });
-  }
-});
-
-// ---------------------------------------------------------------------------
-// H11 — body not pulled for all rows; large candidate set capped
-// ---------------------------------------------------------------------------
-
-test("H11: bodies are materialized only for the returned topN", async () => {
-  const DIM = 8;
-  const MODEL = "fake-small";
-  const root = makeAgentRoot(SAMPLE);
-  try {
-    await buildIndex({
-      agentRoot: root,
-      embeddings: true,
-      embedModel: MODEL,
-      _embedTexts: fakeEmbedder(MODEL, DIM),
-    });
-
-    const topN = 1;
-    const hits = await search({
-      query: "alpha", agentRoot: root, mode: "semantic", topN,
-      embedModel: MODEL, _embedQuery: fakeQuery(MODEL, DIM),
-    });
-    assert.equal(hits.length, topN, "exactly topN results");
-    // The single returned hit carries a non-empty snippet (its body WAS
-    // fetched), proving snippets are built from the per-topN body fetch and
-    // not from a body column on the scoring scan.
-    assert.ok(hits[0].snippet.length > 0, "returned hit has a snippet from its body");
-  } finally {
-    rmSync(root, { recursive: true, force: true });
-  }
-});
-
-test("H11: candidate set above the ceiling is capped, not OOM", async () => {
-  const DIM = 8;
-  const MODEL = "fake-small";
-  const root = makeAgentRoot(SAMPLE);
-  const prev = process.env.RAG_MAX_SEMANTIC_CANDIDATES;
-  try {
-    await buildIndex({
-      agentRoot: root,
-      embeddings: true,
-      embedModel: MODEL,
-      _embedTexts: fakeEmbedder(MODEL, DIM),
-    });
-
-    // Force the ceiling below our (tiny) corpus so the cap path is exercised.
-    process.env.RAG_MAX_SEMANTIC_CANDIDATES = "1";
-    const hits = await search({
-      query: "alpha", agentRoot: root, mode: "semantic", topN: 5,
-      embedModel: MODEL, _embedQuery: fakeQuery(MODEL, DIM),
-    });
-    // Cap == 1, so at most one candidate is scanned → at most one result.
-    assert.ok(hits.length <= 1, "scan capped to the ceiling");
-  } finally {
-    if (prev === undefined) delete process.env.RAG_MAX_SEMANTIC_CANDIDATES;
-    else process.env.RAG_MAX_SEMANTIC_CANDIDATES = prev;
-    rmSync(root, { recursive: true, force: true });
-  }
-});
-
-test("DEFAULT_MAX_SEMANTIC_CANDIDATES is exported and sane", () => {
-  assert.equal(typeof DEFAULT_MAX_SEMANTIC_CANDIDATES, "number");
-  assert.ok(DEFAULT_MAX_SEMANTIC_CANDIDATES > 0);
-});