@design-ai/cli 4.56.0 → 4.57.0

package/cli/lib/recall.mjs

@@ -0,0 +1,193 @@
+// Corpus knowledge recall (retrieval-augmentation) for `design-ai prompt`/`pack`
+// --with-recall. Injects the most brief-relevant CORPUS KNOWLEDGE FILES, ranked by
+// the SAME shipped deterministic lexical (BM25-style) scorer that powers
+// `design-ai search --ranked` and learning selection (docs/AI-LEARNING-PHASE2.md,
+// Phase A). Sibling of learning selection: learning recalls LOCAL preferences,
+// recall recalls SHIPPED corpus knowledge. Zero-dependency and deterministic.
+//
+// NO embeddings in this path: the pack/prompt recall stays lexical-only. Embedding
+// rerank remains a search-only opt-in for now (docs/AI-LEARNING-PHASE2.md, Phase B);
+// there is deliberately no --embeddings recall variant.
+
+import { readFileSync } from "node:fs";
+import path from "node:path";
+
+import { rankedSearchCorpus, rankedPreview } from "./search-ranked.mjs";
+import { collectCorpusDocuments } from "./retrieval-index.mjs";
+import { loadLearningProfile } from "./learn-profile.mjs";
+import { selectLearningEntrySet } from "./learn-select.mjs";
+import { defaultLearningFile } from "./learn-shared.mjs";
+import { DESIGN_AI_HOME } from "./paths.mjs";
+import { DEFAULT_SEARCH_DIRS } from "./search.mjs";
+
+export const DEFAULT_RECALL_LIMIT = 5;
+
+// Mirrors parseLearningLimit but bounded 1-20 (recall injects whole corpus files, so a
+// tighter cap than learning's 1-100 keeps the recall section budget-friendly).
+export function parseRecallLimit(rawLimit) {
+  const limit = Number(rawLimit);
+  if (!Number.isInteger(limit) || limit < 1 || limit > 20) {
+    throw new Error("--recall-limit expects an integer from 1 to 20");
+  }
+  return limit;
+}
+
+// Cap each recalled file's rendered block so the total recall section stays bounded.
+// The pack's takeUtf8(maxBytes) truncation still applies to the WHOLE pack as-is, so
+// recall respects the byte budget by being subject to the same cap; this per-file cap
+// keeps any single file from crowding out the rest of the recall section.
+const RECALL_BLOCK_CHAR_CAP = 400;
+
+// First Markdown `# ` heading, else the first non-empty line — a short, deterministic
+// citation label for the recalled file.
+function firstHeadingOrLine(text) {
+  const lines = String(text || "").split("\n");
+  for (const line of lines) {
+    const trimmed = line.trim();
+    if (trimmed.startsWith("# ")) return trimmed.slice(2).trim();
+  }
+  for (const line of lines) {
+    const trimmed = line.trim();
+    if (trimmed.length > 0) return trimmed;
+  }
+  return "";
+}
+
+function readCorpusText(designAiPath, relPath) {
+  try {
+    return readFileSync(path.join(designAiPath, relPath), "utf8");
+  } catch {
+    return "";
+  }
+}
+
+// Bounded per-file block: relPath citation + heading/first line + matched excerpt.
+// Capped to ~RECALL_BLOCK_CHAR_CAP chars so no single file dominates the section.
+function renderRecallBlock({ id, text, matchedTokens }) {
+  const heading = firstHeadingOrLine(text);
+  const excerpt = rankedPreview(text, matchedTokens);
+  const parts = [`- ${id}`];
+  if (heading) parts.push(`  - ${heading}`);
+  if (excerpt) parts.push(`  - ${excerpt}`);
+  const block = parts.join("\n");
+  return block.length <= RECALL_BLOCK_CHAR_CAP
+    ? block
+    : `${block.slice(0, RECALL_BLOCK_CHAR_CAP)}...`;
+}
+
+function renderRecallMarkdown(selected, textById) {
+  const lines = ["## Recalled design knowledge", ""];
+  if (selected.length === 0) {
+    lines.push("No corpus knowledge files matched this brief.");
+    return lines.join("\n");
+  }
+
+  lines.push("Cite these shipped corpus knowledge files when they inform a design decision.");
+  lines.push("");
+  for (const item of selected) {
+    lines.push(renderRecallBlock({
+      id: item.id,
+      text: textById.get(item.id) || "",
+      matchedTokens: item.matchedTokens,
+    }));
+  }
+  return lines.join("\n");
+}
+
+// Returns a recall context mirroring learning's selection-item shape (id = corpus
+// relPath). REUSES rankedSearchCorpus for scoring — no reimplemented ranking. The
+// ranker already orders fully by score desc, id asc, so the result is deterministic.
+// Empty brief or no lexical hits -> selectedCount 0 and a "no corpus knowledge"
+// markdown line.
+export function buildRecallContext({
+  brief = "",
+  recallLimit = DEFAULT_RECALL_LIMIT,
+  designAiPath = DESIGN_AI_HOME,
+  dirs = DEFAULT_SEARCH_DIRS,
+} = {}) {
+  const query = String(brief || "").trim();
+  const candidateCount = collectCorpusDocuments({ designAiPath, dirs }).length;
+
+  const hits = query
+    ? rankedSearchCorpus({ query, designAiPath, dirs, limit: recallLimit }).hits
+    : [];
+
+  const selected = hits.map((hit) => ({
+    id: hit.relPath,
+    score: hit.score,
+    matchedTokens: hit.matchedTokens,
+  }));
+
+  const textById = new Map(
+    selected.map((item) => [item.id, readCorpusText(designAiPath, item.id)]),
+  );
+
+  return {
+    query,
+    mode: "lexical",
+    candidateCount,
+    selectedCount: selected.length,
+    selected,
+    markdown: renderRecallMarkdown(selected, textById),
+  };
+}
+
+// Read-side companion to `pack --with-recall`: a unified "what does design-ai
+// recall for this query" view. Combines (1) the top corpus knowledge files ranked
+// by rankedSearchCorpus with (2) the top local learning-profile entries ranked by
+// selectLearningEntrySet — BOTH using the same shipped deterministic lexical scorer.
+// Read-only: never writes the profile or any file. Empty query -> zero hits on both
+// sides. `limit` applies to BOTH lists; `category` scopes ONLY the learning list.
+export function buildLearnRecall({
+  query = "",
+  limit = DEFAULT_RECALL_LIMIT,
+  category = "",
+  designAiPath = DESIGN_AI_HOME,
+  learningFilePath = defaultLearningFile(),
+  dirs = DEFAULT_SEARCH_DIRS,
+} = {}) {
+  const normalizedQuery = String(query || "").trim();
+
+  const corpusCandidateCount = collectCorpusDocuments({ designAiPath, dirs }).length;
+  const corpusHits = normalizedQuery
+    ? rankedSearchCorpus({ query: normalizedQuery, designAiPath, dirs, limit }).hits
+    : [];
+  const corpusSelected = corpusHits.map((hit) => ({
+    id: hit.relPath,
+    score: hit.score,
+    matchedTokens: hit.matchedTokens,
+  }));
+
+  const profile = loadLearningProfile(learningFilePath);
+  const textById = new Map(profile.entries.map((entry) => [entry.id, entry.text || ""]));
+  const { selection } = selectLearningEntrySet(profile, {
+    query: normalizedQuery,
+    limit,
+    category,
+    includeFallback: false,
+  });
+  const learningSelected = normalizedQuery
+    ? selection.selected.map((item) => ({
+      id: item.id,
+      category: item.category,
+      score: item.score,
+      matchedTokens: item.matchedTokens,
+      text: textById.get(item.id) || "",
+    }))
+    : [];
+
+  return {
+    query: normalizedQuery,
+    corpus: {
+      candidateCount: corpusCandidateCount,
+      selectedCount: corpusSelected.length,
+      selected: corpusSelected,
+    },
+    learning: {
+      mode: selection.mode,
+      candidateCount: selection.candidateCount,
+      selectedCount: learningSelected.length,
+      selected: learningSelected,
+    },
+  };
+}

package/cli/lib/search-ranked.mjs

@@ -0,0 +1,170 @@
+// Ranked (BM25-style) corpus search for `design-ai search --ranked`, plus the
+// opt-in Phase B embedding rerank (docs/AI-LEARNING-PHASE2.md, Phase B).
+// Deterministic: ranks a live corpus scan, then reports retrieval-index staleness
+// instead of ever serving stale index text (docs/AI-LEARNING-PHASE2.md, Phase A).
+
+import { readFileSync } from "node:fs";
+import path from "node:path";
+
+import { embeddingIndexFile, loadEmbeddingIndexFile, vectorsById as embeddingVectorsById } from "./embedding-index.mjs";
+import { embedDocuments } from "./embedding-provider.mjs";
+import { cosineRerank } from "./embedding-rerank.mjs";
+import { buildLexicalStats, rankLexical } from "./lexical.mjs";
+import { PACKAGE_ROOT } from "./paths.mjs";
+import {
+  collectCorpusDocuments,
+  computeCorpusDigest,
+  computeLearningDigest,
+  corpusIndexFile,
+  defaultIndexDir,
+  loadIndexFile,
+} from "./retrieval-index.mjs";
+import { defaultLearningFile } from "./learn-shared.mjs";
+import { DEFAULT_SEARCH_DIRS } from "./search.mjs";
+
+const RANKED_PREVIEW_LEN = 120;
+
+// N = max(limit*5, 25): the number of top lexical candidates handed to the embedding
+// reranker. Documented constant (docs/AI-LEARNING-PHASE2.md, Phase B CLI wiring).
+export function embeddingCandidateCount(limit) {
+  return Math.max(limit * 5, 25);
+}
+
+export function rankedPreview(text, matchedTokens) {
+  const lines = text.split("\n");
+  for (const token of matchedTokens) {
+    for (const line of lines) {
+      const trimmed = line.trim();
+      if (trimmed && trimmed.toLowerCase().includes(token)) {
+        return trimmed.length <= RANKED_PREVIEW_LEN
+          ? trimmed
+          : `${trimmed.slice(0, RANKED_PREVIEW_LEN)}...`;
+      }
+    }
+  }
+  const firstLine = lines.map((line) => line.trim()).find((line) => line.length > 0) || "";
+  return firstLine.length <= RANKED_PREVIEW_LEN ? firstLine : `${firstLine.slice(0, RANKED_PREVIEW_LEN)}...`;
+}
+
+function corpusIndexNotice({ indexDir, currentDigest }) {
+  const loaded = loadIndexFile(corpusIndexFile(indexDir));
+  if (!loaded.present) {
+    return "no corpus index built yet; ranked results come from a live corpus scan (design-ai index --build)";
+  }
+  if (loaded.error) {
+    return `corpus index is unreadable (${loaded.error}); ranked results come from a live corpus scan (design-ai index --build)`;
+  }
+  if ((loaded.payload.source?.corpusDigest || "") !== currentDigest) {
+    return "corpus index is stale; ranked results come from a live corpus scan (design-ai index --build)";
+  }
+  return "";
+}
+
+export function rankedSearchCorpus({
+  query,
+  designAiPath = PACKAGE_ROOT,
+  dirs = DEFAULT_SEARCH_DIRS,
+  limit = 20,
+  indexDir = defaultIndexDir(),
+} = {}) {
+  const documents = collectCorpusDocuments({ designAiPath, dirs });
+  const stats = buildLexicalStats(documents.map(({ id, text }) => ({ id, text })));
+  const textById = new Map(documents.map((doc) => [doc.id, doc.text]));
+
+  const hits = rankLexical(query, stats, { limit }).map((hit) => ({
+    relPath: hit.id,
+    file: path.join(designAiPath, hit.id),
+    score: hit.score,
+    matchedTokens: hit.matchedTokens,
+    preview: rankedPreview(textById.get(hit.id) || "", hit.matchedTokens),
+  }));
+
+  return {
+    hits,
+    notice: corpusIndexNotice({ indexDir, currentDigest: computeCorpusDigest(documents) }),
+  };
+}
+
+function embeddingSidecarFreshness({ indexDir, designAiPath, learningFile, corpusDigest }) {
+  const file = embeddingIndexFile(indexDir);
+  const loaded = loadEmbeddingIndexFile(file);
+  if (!loaded.present) return { ok: false, reason: "no embedding index built yet (design-ai index --build --embeddings)" };
+  if (loaded.error) return { ok: false, reason: `embedding index is unreadable (${loaded.error})` };
+
+  const source = loaded.payload.source || {};
+  const sourceMatch =
+    source.designAiPath === path.resolve(designAiPath) && source.learningFile === path.resolve(learningFile);
+  if (!sourceMatch) {
+    return { ok: false, reason: "embedding index was built from a different checkout or learning file" };
+  }
+
+  const { digest: currentLearningDigest } = computeLearningDigest(learningFile);
+  const digestMatch = source.corpusDigest === corpusDigest && source.learningDigest === currentLearningDigest;
+  if (!digestMatch) {
+    return { ok: false, reason: "embedding index is stale (design-ai index --build --embeddings)" };
+  }
+
+  return { ok: true, payload: loaded.payload };
+}
+
+// Opt-in Phase B rerank: take the top-N lexical candidates, embed the QUERY via the
+// provider, cosine-rerank against the stored sidecar vectors, return the top `limit`.
+// ANY failure (no provider, no sidecar, stale sidecar, provider error) is reported
+// via `fallback: true` + a specific `notice` — callers must fall back to the lexical
+// path and exit 0; a stale sidecar must never silently serve embedding results.
+export function embeddingRerankSearch({
+  query,
+  provider,
+  designAiPath = PACKAGE_ROOT,
+  dirs = DEFAULT_SEARCH_DIRS,
+  limit = 20,
+  indexDir = defaultIndexDir(),
+  learningFile = defaultLearningFile(),
+  spawnRunner,
+} = {}) {
+  if (!provider || typeof provider.command !== "string" || provider.command.trim() === "") {
+    return { fallback: true, notice: "no embedding provider configured (pass --provider or configure ~/.design-ai/config.json)" };
+  }
+
+  // Freshness is checked against the FULL corpus (the same dirs `index --build
+  // --embeddings` always embeds), independent of this search's --dir filter — a
+  // scoped `--dir knowledge` search must not be compared against a digest of only
+  // the knowledge dir, or every scoped search would spuriously report the sidecar
+  // stale. Candidate generation below still respects the caller's `dirs` filter.
+  const fullCorpusDocuments = collectCorpusDocuments({ designAiPath, dirs: DEFAULT_SEARCH_DIRS });
+  const corpusDigest = computeCorpusDigest(fullCorpusDocuments);
+  const freshness = embeddingSidecarFreshness({ indexDir, designAiPath, learningFile, corpusDigest });
+  if (!freshness.ok) return { fallback: true, notice: freshness.reason };
+
+  const documents = dirs === DEFAULT_SEARCH_DIRS ? fullCorpusDocuments : collectCorpusDocuments({ designAiPath, dirs });
+  const candidateCount = embeddingCandidateCount(limit);
+  const stats = buildLexicalStats(documents.map(({ id, text }) => ({ id, text })));
+  const textById = new Map(documents.map((doc) => [doc.id, doc.text]));
+  const candidates = rankLexical(query, stats, { limit: candidateCount });
+  if (candidates.length === 0) {
+    return { fallback: true, notice: "no lexical candidates matched the query; nothing to rerank" };
+  }
+
+  const embedArgs = { provider, documents: [{ id: "__query__", text: query }] };
+  if (spawnRunner) embedArgs.spawnRunner = spawnRunner;
+  const embedded = embedDocuments(embedArgs);
+  if (!embedded.ok) return { fallback: true, notice: `embedding provider failed: ${embedded.error}` };
+
+  const queryVector = embedded.vectorsById.get("__query__");
+  const vectors = embeddingVectorsById(freshness.payload);
+  const reranked = cosineRerank({ queryVector, candidates, vectorsById: vectors, limit });
+
+  const hits = reranked.map((hit) => ({
+    relPath: hit.id,
+    file: path.join(designAiPath, hit.id),
+    // `score` is a cosine similarity for embedded candidates; a candidate the
+    // provider had no vector for (placed after all embedded ones, see
+    // embedding-rerank.mjs) keeps its lexical BM25 score here as a documented,
+    // deterministic fallback value rather than an undefined/null field.
+    score: hit.score,
+    matchedTokens: hit.matchedTokens,
+    preview: rankedPreview(textById.get(hit.id) || "", hit.matchedTokens),
+  }));
+
+  return { fallback: false, hits };
+}

package/cli/lib/search.mjs

@@ -20,7 +20,7 @@ export const DEFAULT_SEARCH_DIRS = [
 
 const PREVIEW_LEN = 120;
 const PREVIEW_BEFORE = 50;
-const SEARCH_OPTIONS = ["-h", "--help", "--json", "--limit", "--dir"];
+const SEARCH_OPTIONS = ["-h", "--help", "--json", "--limit", "--dir", "--ranked", "--embeddings", "--provider"];
 
 function exists(p) {
   try {
@@ -115,6 +115,9 @@ export function parseSearchArgs(args) {
     dirs: [],
     limit: 20,
     json: false,
+    ranked: false,
+    embeddings: false,
+    provider: "",
     help: false,
   };
 
@@ -124,6 +127,13 @@ export function parseSearchArgs(args) {
       out.help = true;
     } else if (arg === "--json") {
       out.json = true;
+    } else if (arg === "--ranked") {
+      out.ranked = true;
+    } else if (arg === "--embeddings") {
+      out.embeddings = true;
+    } else if (arg === "--provider") {
+      out.provider = args[i + 1] || "";
+      i += 1;
     } else if (arg === "--limit") {
       const next = args[i + 1];
       const limit = Number(next);