llmnav 0.5.1

package/src/declaration.js

@@ -0,0 +1,334 @@
+import path from "node:path";
+import { lineAtOffset, normalizeNewlines, sha256 } from "./util.js";
+
+const DECLARATION_PATTERNS = {
+  javascript: [
+    { kind: "function", pattern: /^(?:export\s+)?(?:default\s+)?(?:async\s+)?function\s+([A-Za-z_$][\w$]*)\b/u },
+    { kind: "class", pattern: /^(?:export\s+)?(?:default\s+)?class\s+([A-Za-z_$][\w$]*)\b/u },
+    { kind: "interface", pattern: /^(?:export\s+)?interface\s+([A-Za-z_$][\w$]*)\b/u },
+    { kind: "type", pattern: /^(?:export\s+)?type\s+([A-Za-z_$][\w$]*)\b/u },
+    { kind: "variable", pattern: /^(?:export\s+)?(?:declare\s+)?(?:const|let|var)\s+([A-Za-z_$][\w$]*)\b/u },
+    { kind: "enum", pattern: /^(?:export\s+)?(?:const\s+)?enum\s+([A-Za-z_$][\w$]*)\b/u },
+  ],
+  go: [
+    { kind: "function", pattern: /^func\s+(?:\([^)]*\)\s*)?([A-Za-z_][A-Za-z0-9_]*)\b/u },
+    { kind: "type", pattern: /^type\s+([A-Za-z_][A-Za-z0-9_]*)\b/u },
+    { kind: "variable", pattern: /^(?:var|const)\s+([A-Za-z_][A-Za-z0-9_]*)\b/u },
+  ],
+  rust: [
+    { kind: "function", pattern: /^(?:pub(?:\([^)]*\))?\s+)?(?:async\s+)?fn\s+([A-Za-z_][A-Za-z0-9_]*)\b/u },
+    { kind: "struct", pattern: /^(?:pub(?:\([^)]*\))?\s+)?struct\s+([A-Za-z_][A-Za-z0-9_]*)\b/u },
+    { kind: "enum", pattern: /^(?:pub(?:\([^)]*\))?\s+)?enum\s+([A-Za-z_][A-Za-z0-9_]*)\b/u },
+    { kind: "trait", pattern: /^(?:pub(?:\([^)]*\))?\s+)?trait\s+([A-Za-z_][A-Za-z0-9_]*)\b/u },
+    { kind: "impl", pattern: /^impl(?:<[^>]+>)?\s+([^\s{]+)\b/u },
+  ],
+  python: [
+    { kind: "function", pattern: /^(?:async\s+)?def\s+([A-Za-z_][A-Za-z0-9_]*)\b/u },
+    { kind: "class", pattern: /^class\s+([A-Za-z_][A-Za-z0-9_]*)\b/u },
+  ],
+  generic: [
+    { kind: "class", pattern: /^(?:public\s+|private\s+|protected\s+|internal\s+|static\s+|final\s+|abstract\s+)*(?:class|interface|enum|struct|trait)\s+([A-Za-z_][A-Za-z0-9_]*)\b/u },
+    { kind: "function", pattern: /^(?:public\s+|private\s+|protected\s+|internal\s+|static\s+|final\s+|async\s+)*(?:[A-Za-z_][\w<>?\[\],.]*\s+)+([A-Za-z_][A-Za-z0-9_]*)\s*\(/u },
+  ],
+};
+
+export function findAttachedDeclaration(source, block, filePath) {
+  if (block.scope !== "symbol") return null;
+  const window = source.slice(block.end, block.end + 3000);
+  const skipped = skipTrivia(window);
+  const candidate = window.slice(skipped);
+  const extension = path.extname(filePath).toLowerCase();
+  const family = languageFamily(extension);
+  const language = languageName(extension);
+  const patterns = DECLARATION_PATTERNS[family] ?? DECLARATION_PATTERNS.generic;
+  const firstLines = normalizeNewlines(candidate).split("\n").slice(0, 12).join("\n");
+  const collapsed = firstLines.replace(/\s+/gu, " ").trim();
+
+  for (const definition of patterns) {
+    const match = candidate.match(definition.pattern);
+    if (!match) continue;
+    const declarationOffset = block.end + skipped;
+    const signature = extractSignature(collapsed);
+    const exported = isExportedDeclaration(language, match[1], signature);
+    const endOffset = findDeclarationEnd(source, declarationOffset, family);
+    return {
+      symbol: match[1],
+      kind: definition.kind,
+      line: lineAtOffset(source, declarationOffset),
+      signature,
+      language,
+      exported,
+      visibility: exported ? "public" : language === "typescript" || language === "javascript" ? "module" : "private",
+      receiver: language === "go" ? extractGoReceiver(candidate) : null,
+      offset: declarationOffset,
+      endOffset,
+      bodyHash: sha256(normalizeNewlines(source.slice(declarationOffset, endOffset)).trimEnd()),
+    };
+  }
+  return null;
+}
+
+function skipTrivia(value) {
+  let remaining = value;
+  let consumed = 0;
+  for (;;) {
+    const before = remaining;
+    const whitespace = remaining.match(/^\s+/u)?.[0] ?? "";
+    consumed += whitespace.length;
+    remaining = remaining.slice(whitespace.length);
+
+    const blockComment = remaining.match(/^\/\*(?!\s*llmnav\/)[\s\S]*?\*\//u)?.[0];
+    if (blockComment) {
+      consumed += blockComment.length;
+      remaining = remaining.slice(blockComment.length);
+      continue;
+    }
+
+    const lineComment = remaining.match(/^(?:\/\/|#|--)\s*(?!llmnav\/).*?(?:\r?\n|$)/u)?.[0];
+    if (lineComment) {
+      consumed += lineComment.length;
+      remaining = remaining.slice(lineComment.length);
+      continue;
+    }
+
+    const attribute = remaining.match(/^(?:@[A-Za-z_$][\w$]*(?:\([^\n]*\))?|#\[[^\]]+\])\s*(?:\r?\n)?/u)?.[0];
+    if (attribute) {
+      consumed += attribute.length;
+      remaining = remaining.slice(attribute.length);
+      continue;
+    }
+
+    if (remaining === before) break;
+  }
+  return consumed;
+}
+
+function extractSignature(collapsed) {
+  if (!collapsed) return "";
+  const boundaries = [collapsed.indexOf("{"), collapsed.indexOf("=>")].filter((index) => index >= 0);
+  const end = boundaries.length > 0 ? Math.min(...boundaries) : Math.min(collapsed.length, 500);
+  return collapsed.slice(0, end).trim().replace(/[;:]$/u, "");
+}
+
+function languageFamily(extension) {
+  if ([".ts", ".tsx", ".js", ".jsx", ".mjs", ".cjs", ".svelte", ".astro", ".vue"].includes(extension)) {
+    return "javascript";
+  }
+  if (extension === ".go") return "go";
+  if (extension === ".rs") return "rust";
+  if (extension === ".py") return "python";
+  return "generic";
+}
+
+function languageName(extension) {
+  if ([".ts", ".tsx", ".mts", ".cts"].includes(extension)) return "typescript";
+  if ([".js", ".jsx", ".mjs", ".cjs", ".svelte", ".astro", ".vue"].includes(extension)) return "javascript";
+  if (extension === ".go") return "go";
+  if (extension === ".rs") return "rust";
+  if (extension === ".py") return "python";
+  return "generic";
+}
+
+function isExportedDeclaration(language, symbol, signature) {
+  if (language === "typescript" || language === "javascript") return /^export\s+/u.test(signature);
+  if (language === "go") return /^[A-Z]/u.test(symbol);
+  if (language === "rust") return /^pub(?:\([^)]*\))?\s+/u.test(signature);
+  if (language === "python") return !symbol.startsWith("_");
+  return /^(?:public|export)\s+/u.test(signature);
+}
+
+function extractGoReceiver(candidate) {
+  const match = candidate.match(/^func\s+\(([^)]*)\)\s*/u);
+  if (!match) return null;
+  const parts = match[1].trim().split(/\s+/u);
+  return (parts.at(-1) ?? "").replace(/^\*+/u, "") || null;
+}
+
+function findDeclarationEnd(source, start, family) {
+  if (family === "python") return findPythonDeclarationEnd(source, start);
+  let state = "normal";
+  let escaped = false;
+  let regexCharacterClass = false;
+  let braces = 0;
+  let parentheses = 0;
+  let brackets = 0;
+  let openedBody = false;
+
+  for (let index = start; index < source.length; index += 1) {
+    const character = source[index];
+    const next = source[index + 1];
+    if (state === "line-comment") {
+      if (character === "\n") state = "normal";
+      continue;
+    }
+    if (state === "block-comment") {
+      if (character === "*" && next === "/") {
+        state = "normal";
+        index += 1;
+      }
+      continue;
+    }
+    if (state !== "normal") {
+      if (state === "regex") {
+        if (escaped) escaped = false;
+        else if (character === "\\") escaped = true;
+        else if (character === "[") regexCharacterClass = true;
+        else if (character === "]") regexCharacterClass = false;
+        else if (character === "/" && !regexCharacterClass) state = "normal";
+        continue;
+      }
+      if (escaped) {
+        escaped = false;
+      } else if (character === "\\") {
+        escaped = true;
+      } else if (character === state) {
+        state = "normal";
+      }
+      continue;
+    }
+    if (character === "/" && next === "/") {
+      state = "line-comment";
+      index += 1;
+      continue;
+    }
+    if (character === "/" && next === "*") {
+      state = "block-comment";
+      index += 1;
+      continue;
+    }
+    if (family === "javascript" && character === "/" && canStartJavaScriptRegex(source, start, index)) {
+      state = "regex";
+      regexCharacterClass = false;
+      continue;
+    }
+    if (character === '"' || character === "'" || character === "`") {
+      if (family === "rust" && character === "'" && !looksLikeRustCharacterLiteral(source, index)) continue;
+      state = character;
+      continue;
+    }
+    if (character === "(") parentheses += 1;
+    if (character === ")") parentheses = Math.max(0, parentheses - 1);
+    if (character === "[") brackets += 1;
+    if (character === "]") brackets = Math.max(0, brackets - 1);
+    if (character === "{") {
+      braces += 1;
+      openedBody = true;
+    } else if (character === "}" && openedBody) {
+      braces -= 1;
+      if (braces === 0) return index + 1;
+    } else if (character === ";" && !openedBody && parentheses === 0 && brackets === 0) {
+      return index + 1;
+    } else if (character === "\n" && !openedBody && parentheses === 0 && brackets === 0) {
+      const current = source.slice(start, index).trimEnd();
+      const nextCharacter = source.slice(index + 1).match(/^\s*(.)/u)?.[1] ?? "";
+      if (nextCharacter !== "{" && !/(?:=>|[=|&,([{])$/u.test(current)) return index;
+    }
+  }
+  return source.length;
+}
+
+function findPythonDeclarationEnd(source, start) {
+  const declarationLineStart = source.lastIndexOf("\n", start - 1) + 1;
+  const baseIndent = indentationWidth(source.slice(declarationLineStart, start));
+  let cursor = source.indexOf("\n", start);
+  if (cursor < 0) return source.length;
+  cursor += 1;
+  let bodyStarted = false;
+  while (cursor < source.length) {
+    const lineEnd = source.indexOf("\n", cursor);
+    const end = lineEnd < 0 ? source.length : lineEnd + 1;
+    const line = source.slice(cursor, lineEnd < 0 ? source.length : lineEnd);
+    if (/^\s*(?:#.*)?$/u.test(line)) {
+      cursor = end;
+      continue;
+    }
+    const indent = indentationWidth(line.match(/^[ \t]*/u)?.[0] ?? "");
+    if (indent > baseIndent) bodyStarted = true;
+    else if (bodyStarted) return cursor;
+    cursor = end;
+  }
+  return source.length;
+}
+
+function indentationWidth(value) {
+  let width = 0;
+  for (const character of value) width += character === "\t" ? 8 - (width % 8) : 1;
+  return width;
+}
+
+function canStartJavaScriptRegex(source, start, offset) {
+  const prefix = source.slice(start, offset).trimEnd();
+  if (!prefix) return true;
+  const previous = prefix.at(-1);
+  if (/[=(:,!&|?{};\[]/u.test(previous)) return true;
+  return /(?:^|\W)(?:case|delete|do|else|in|instanceof|new|return|throw|typeof|void|yield)\s*$/u.test(prefix);
+}
+
+function looksLikeRustCharacterLiteral(source, offset) {
+  return /^'(?:\\.|[^'\\\r\n])'/u.test(source.slice(offset));
+}
+
+export function extractImports(source, filePath) {
+  const extension = path.extname(filePath).toLowerCase();
+  const results = new Set();
+  const normalized = normalizeNewlines(source);
+  if ([".ts", ".tsx", ".js", ".jsx", ".mjs", ".cjs", ".svelte", ".astro", ".vue"].includes(extension)) {
+    const patterns = [
+      /^[ \t]*import\s+(?:[^;"']*?\s+from\s+)?["']([^"']+)["']/gmu,
+      /^[ \t]*export\s+[^;"']*?\s+from\s+["']([^"']+)["']/gmu,
+      /\b(?:require|import)\s*\(\s*["']([^"']+)["']\s*\)/gu,
+    ];
+    for (const pattern of patterns) {
+      for (const match of normalized.matchAll(pattern)) {
+        if (match[1] && isJavaScriptCodeOffset(normalized, match.index ?? 0)) results.add(match[1]);
+      }
+    }
+  } else if (extension === ".go") {
+    for (const match of normalized.matchAll(/^[ \t]*(?:import\s+)?(?:[A-Za-z_][\w]*\s+)?"([^"]+)"/gmu)) {
+      results.add(match[1]);
+    }
+  } else if (extension === ".rs") {
+    for (const match of normalized.matchAll(/^\s*(?:use|mod)\s+([^;]+);/gmu)) results.add(match[1].trim());
+  } else if (extension === ".py") {
+    for (const match of normalized.matchAll(/^\s*(?:from\s+([\w.]+)\s+import|import\s+([\w.]+))/gmu)) {
+      results.add(match[1] ?? match[2]);
+    }
+  }
+  return [...results].sort();
+}
+
+function isJavaScriptCodeOffset(source, targetOffset) {
+  let state = "normal";
+  let escaped = false;
+  for (let index = 0; index < targetOffset; index += 1) {
+    const character = source[index];
+    const next = source[index + 1];
+    if (state === "line-comment") {
+      if (character === "\n") state = "normal";
+      continue;
+    }
+    if (state === "block-comment") {
+      if (character === "*" && next === "/") {
+        state = "normal";
+        index += 1;
+      }
+      continue;
+    }
+    if (state !== "normal") {
+      if (escaped) escaped = false;
+      else if (character === "\\") escaped = true;
+      else if (character === state) state = "normal";
+      continue;
+    }
+    if (character === "/" && next === "/") {
+      state = "line-comment";
+      index += 1;
+    } else if (character === "/" && next === "*") {
+      state = "block-comment";
+      index += 1;
+    } else if (character === '"' || character === "'" || character === "`") {
+      state = character;
+    }
+  }
+  return state === "normal";
+}

package/src/doctor.js

@@ -0,0 +1,124 @@
+import path from "node:path";
+import { loadConfig } from "./config.js";
+import { generateProject } from "./generator.js";
+import { usableFileState } from "./incremental.js";
+import { verifySearchIndex } from "./inverted-index.js";
+import { recoverGenerationTransaction } from "./transaction.js";
+import { readJson, readJsonSafe, readText, sha256 } from "./util.js";
+
+export async function doctorProject(root) {
+  const checks = [];
+  let config;
+  try {
+    ({ config } = await loadConfig(root));
+    checks.push(pass("config", ".llmnav/config.json is valid"));
+  } catch (error) {
+    checks.push(fail("config", error instanceof Error ? error.message : String(error)));
+    return { ok: false, checks };
+  }
+
+  try {
+    const recovery = await recoverGenerationTransaction(root, {
+      cacheDirectory: config.generation.cacheDirectory,
+    });
+    checks.push(
+      recovery.recovered
+        ? pass("transaction", `recovered interrupted generation using ${recovery.action}`)
+        : pass("transaction", "no interrupted generation is pending"),
+    );
+  } catch (error) {
+    checks.push(fail("transaction", error instanceof Error ? error.message : String(error)));
+  }
+
+  const major = Number.parseInt(process.versions.node.split(".")[0], 10);
+  checks.push(major >= 22 ? pass("node", `Node.js ${process.versions.node}`) : fail("node", "Node.js 22 or newer is required"));
+
+  for (const [name, relativePath] of [
+    ["agent", ".llmnav/AGENT_INSTRUCTIONS.md"],
+    ["registry", ".llmnav/ids.jsonl"],
+    ["order", ".llmnav/order.lock"],
+    ["lexicon", ".llmnav/lexicon.json"],
+    ["index", `${config.generation.cacheDirectory}/index.json`],
+    ["search-index", `${config.generation.cacheDirectory}/search-index.json`],
+    ["file-state", `${config.generation.cacheDirectory}/file-state.json`],
+  ]) {
+    const exists = (await readText(path.join(root, relativePath), null)) !== null;
+    checks.push(exists ? pass(name, `${relativePath} exists`) : fail(name, `${relativePath} is missing`));
+  }
+
+  const cacheRoot = path.join(root, config.generation.cacheDirectory);
+  const indexRead = await inspectJson(path.join(cacheRoot, "index.json"));
+  const searchRead = await inspectJson(path.join(cacheRoot, "search-index.json"));
+  const fileStateRead = await inspectJson(path.join(cacheRoot, "file-state.json"));
+  const index = indexRead.value;
+  const searchIndex = searchRead.value;
+  const fileState = fileStateRead.value;
+  if (indexRead.error) checks.push(fail("index-integrity", indexRead.error));
+  if (searchRead.error) checks.push(fail("search-index-integrity", searchRead.error));
+  else if (index && searchIndex) {
+    checks.push(
+      verifySearchIndex(index, searchIndex)
+        ? pass("search-index-integrity", "search-index.json matches index.json")
+        : fail("search-index-integrity", "search-index.json does not match index.json"),
+    );
+  }
+  if (fileStateRead.error) checks.push(fail("file-state-integrity", fileStateRead.error));
+  else if (fileState) {
+    checks.push(
+      usableFileState(fileState)
+        ? pass("file-state-integrity", "file-state.json uses the supported schema")
+        : fail("file-state-integrity", "file-state.json uses an unsupported or malformed schema"),
+    );
+  }
+
+  const manifestPath = path.join(cacheRoot, "manifest.json");
+  const manifestRead = await inspectJson(manifestPath);
+  const manifest = manifestRead.value;
+  if (manifestRead.error) checks.push(fail("manifest", manifestRead.error));
+  if (!manifest && !manifestRead.error) {
+    checks.push(fail("manifest", `${config.generation.cacheDirectory}/manifest.json is missing`));
+  } else if (manifest) {
+    let valid = true;
+    for (const [relativePath, expectedHash] of Object.entries(manifest.files ?? {})) {
+      const content = await readText(path.join(root, relativePath), null);
+      if (content === null || sha256(content) !== expectedHash) {
+        valid = false;
+        checks.push(fail("manifest", `${relativePath} does not match its manifest hash`));
+      }
+    }
+    if (valid) checks.push(pass("manifest", "generated cache hashes are valid"));
+  }
+
+  const generated = await generateProject(root, { check: true, incremental: false });
+  checks.push(
+    generated.ok
+      ? pass("generated", "generated files match a full source rebuild")
+      : fail("generated", `regenerate ${generated.changedFiles.join(", ") || "after fixing diagnostics"}`),
+  );
+
+  const packageJson = await readJsonSafe(path.join(root, "package.json"), null);
+  if (packageJson?.name === "llmnav" && JSON.stringify(packageJson).includes("github.com/OWNER/")) {
+    checks.push(fail("release", "replace OWNER in package.json before publishing"));
+  }
+
+  return { ok: checks.every((check) => check.ok), checks };
+}
+
+async function inspectJson(filePath) {
+  try {
+    return { value: await readJson(filePath, null), error: null };
+  } catch (error) {
+    return {
+      value: null,
+      error: `${path.basename(filePath)} is malformed: ${error instanceof Error ? error.message : String(error)}`,
+    };
+  }
+}
+
+function pass(name, message) {
+  return { name, ok: true, message };
+}
+
+function fail(name, message) {
+  return { name, ok: false, message };
+}

package/src/editor.js

@@ -0,0 +1,107 @@
+/* llmnav/1 module
+id=llmnav.diagnostics.editor
+role=Serialize deterministic editor ranges and publish bounded editor task integrations.
+owns=editor diagnostic schema|zero-based ranges|VS Code task configuration
+excludes=editor extension runtime|absolute workspace URIs|source mutation
+search=editor diagnostics|VS Code problem matcher|problems panel|diagnostic range
+rel=workflow>llmnav.rules.validate
+stability=contract
+*/
+
+import { compareText, stableStringify, toPosix } from "./util.js";
+
+export const EDITOR_DIAGNOSTIC_SCHEMA_VERSION = 1;
+export const EDITOR_INTEGRATION_SCHEMA_VERSION = 1;
+
+export function diagnosticsToEditor(diagnostics) {
+  const byPath = new Map();
+  const sorted = [...diagnostics].sort(compareDiagnostics);
+  for (const diagnostic of sorted) {
+    const documentPath = toPosix(diagnostic.file);
+    const items = byPath.get(documentPath) ?? [];
+    const line = Math.max(0, Number(diagnostic.line ?? 1) - 1);
+    const character = Math.max(0, Number(diagnostic.column ?? 1) - 1);
+    items.push({
+      range: {
+        start: { line, character },
+        end: { line, character: character + 1 },
+      },
+      severity: severityNumber(diagnostic.severity),
+      level: diagnostic.severity,
+      code: diagnostic.code,
+      source: "llmnav",
+      message: diagnostic.message,
+    });
+    byPath.set(documentPath, items);
+  }
+  return {
+    schemaVersion: EDITOR_DIAGNOSTIC_SCHEMA_VERSION,
+    source: "llmnav",
+    coordinateBase: 0,
+    counts: countLevels(sorted),
+    documents: [...byPath.entries()]
+      .sort(([left], [right]) => compareText(left, right))
+      .map(([path, items]) => ({ path, diagnostics: items })),
+  };
+}
+
+export function renderEditorDiagnostics(diagnostics) {
+  return stableStringify(diagnosticsToEditor(diagnostics));
+}
+
+export function getEditorIntegration(name) {
+  if (name !== "vscode") throw editorUsageError(`Unknown editor integration ${JSON.stringify(name)}.`);
+  return {
+    schemaVersion: EDITOR_INTEGRATION_SCHEMA_VERSION,
+    editor: "vscode",
+    target: ".vscode/tasks.json",
+    config: {
+      version: "2.0.0",
+      tasks: [{
+        label: "LLMNav: check",
+        type: "shell",
+        command: "npm",
+        args: ["exec", "--", "llmnav", "check"],
+        group: { kind: "test", isDefault: false },
+        problemMatcher: {
+          owner: "llmnav",
+          fileLocation: ["relative", "${workspaceFolder}"],
+          source: "llmnav",
+          pattern: {
+            regexp: "^(.+):(\\d+):(\\d+) (error|warning|info) (LNV\\d+) (.+)$",
+            file: 1,
+            line: 2,
+            column: 3,
+            severity: 4,
+            code: 5,
+            message: 6,
+          },
+        },
+        presentation: { reveal: "silent", panel: "dedicated" },
+      }],
+    },
+  };
+}
+
+function severityNumber(level) {
+  if (level === "error") return 1;
+  if (level === "warning") return 2;
+  return 3;
+}
+
+function countLevels(diagnostics) {
+  const counts = { error: 0, warning: 0, info: 0 };
+  for (const item of diagnostics) counts[item.severity] = (counts[item.severity] ?? 0) + 1;
+  return counts;
+}
+
+function compareDiagnostics(left, right) {
+  return compareText(left.file, right.file) || left.line - right.line || left.column - right.column ||
+    compareText(left.code, right.code) || compareText(left.message, right.message);
+}
+
+function editorUsageError(message) {
+  const error = new Error(message);
+  error.exitCode = 2;
+  return error;
+}

package/src/evaluation.js

@@ -0,0 +1,67 @@
+/* llmnav/1 module
+id=llmnav.eval.measure
+role=Measure whether repository task queries retrieve expected semantic IDs within configured rank gates.
+owns=search regression tests|recall metrics|evaluation gates
+excludes=benchmark task execution|model quality scoring
+search=search recall|navigation benchmark|query regression
+rel=workflow>llmnav.search.query
+stability=architecture
+*/
+
+import path from "node:path";
+import { loadConfig } from "./config.js";
+import { loadSearchData, queryPreparedIndex } from "./search.js";
+import { assertNoSymlinkTraversal, parseJsonLines, readText } from "./util.js";
+
+export async function evaluateProject(root, options = {}) {
+  const { config } = await loadConfig(root);
+  const { index, lexicon, searchIndex } = await loadSearchData(root);
+  const queryPath = path.resolve(root, options.file ?? config.evaluation.queryFile);
+  await assertNoSymlinkTraversal(root, queryPath, "evaluation query file");
+  const parsed = parseJsonLines(await readText(queryPath, ""), queryPath);
+  if (parsed.errors.length > 0) {
+    return { ok: false, errors: parsed.errors, cases: [], metrics: emptyMetrics() };
+  }
+
+  const cases = [];
+  for (const record of parsed.records) {
+    if (!record || typeof record.query !== "string" || !Array.isArray(record.expected) || record.expected.length === 0) {
+      return {
+        ok: false,
+        errors: [`${queryPath}: each record requires query:string and expected:string[]`],
+        cases,
+        metrics: emptyMetrics(),
+      };
+    }
+    const results = queryPreparedIndex(index, searchIndex, record.query, { top: Math.max(options.top ?? 5, 5), lexicon });
+    const ids = results.map((result) => result.id);
+    const firstRank = ids.findIndex((id) => record.expected.includes(id));
+    cases.push({
+      query: record.query,
+      expected: record.expected,
+      actual: ids,
+      rank: firstRank < 0 ? null : firstRank + 1,
+      passAt1: firstRank === 0,
+      passAt5: firstRank >= 0 && firstRank < 5,
+    });
+  }
+
+  const total = cases.length;
+  const metrics = total === 0
+    ? emptyMetrics()
+    : {
+        total,
+        recallAt1: cases.filter((item) => item.passAt1).length / total,
+        recallAt5: cases.filter((item) => item.passAt5).length / total,
+        meanReciprocalRank: cases.reduce((sum, item) => sum + (item.rank ? 1 / item.rank : 0), 0) / total,
+      };
+  const ok =
+    total === 0 ||
+    (metrics.recallAt1 >= config.evaluation.minimumRecallAt1 &&
+      metrics.recallAt5 >= config.evaluation.minimumRecallAt5);
+  return { ok, errors: [], cases, metrics, thresholds: config.evaluation };
+}
+
+function emptyMetrics() {
+  return { total: 0, recallAt1: 0, recallAt5: 0, meanReciprocalRank: 0 };
+}