@design-ai/cli 4.56.0 → 4.57.0

package/cli/lib/lexical.mjs

@@ -0,0 +1,137 @@
+// Shared deterministic lexical (BM25-style) scoring for corpus and learning retrieval.
+// Phase A of docs/AI-LEARNING-PHASE2.md: zero-dependency, no randomness, no time-dependent scoring.
+
+export const BM25_K1 = 1.2;
+export const BM25_B = 0.75;
+
+// Hangul syllable block (U+AC00–U+D7A3). Korean surface forms are written without
+// spaces between stem and particle (버튼을, 접근성이), so a whitespace-delimited token
+// is an atomic surface form. To let a stem query match particle-attached documents
+// (and vice versa), we deterministically expand every Hangul run of length >= 2 into
+// its overlapping character bigrams and emit them ALONGSIDE the original token.
+//   버튼을 -> keep 버튼을, add 버튼, 튼을
+//   접근성 -> keep 접근성, add 접근, 근성
+//   버튼   -> the 2-char run's only bigram is itself; kept via dedupe
+// Latin/number tokens and mixed tokens' non-Hangul parts are unchanged. The rule is
+// pure and deterministic (no locale, no time, no randomness); order is normalized by
+// the dedupe+sort in lexicalQueryTokens and by termFrequencies for stats.
+const HANGUL_RUN = /[가-힣]{2,}/gu;
+
+function hangulBigrams(token) {
+  const bigrams = [];
+  for (const run of token.match(HANGUL_RUN) || []) {
+    for (let i = 0; i + 2 <= run.length; i += 1) {
+      bigrams.push(run.slice(i, i + 2));
+    }
+  }
+  return bigrams;
+}
+
+// Same base token model as learn-select.mjs: Unicode letters/numbers, lowercased,
+// length >= 2. Hangul tokens additionally emit character bigrams (see HANGUL_RUN).
+export function lexicalTokens(text) {
+  const base = (String(text || "")
+    .toLowerCase()
+    .match(/[\p{L}\p{N}]+/gu) || [])
+    .filter((token) => token.length >= 2);
+
+  const expanded = [];
+  for (const token of base) {
+    expanded.push(token);
+    for (const bigram of hangulBigrams(token)) {
+      // A 2-char Hangul run's only bigram is the token itself; do not re-emit it,
+      // which would double-count its term frequency and distort BM25.
+      if (bigram !== token) expanded.push(bigram);
+    }
+  }
+  return expanded;
+}
+
+export function lexicalQueryTokens(query) {
+  return Array.from(new Set(lexicalTokens(query))).sort();
+}
+
+function termFrequencies(tokens) {
+  const terms = {};
+  for (const token of tokens) {
+    terms[token] = (terms[token] || 0) + 1;
+  }
+  const sorted = {};
+  for (const term of Object.keys(terms).sort()) {
+    sorted[term] = terms[term];
+  }
+  return sorted;
+}
+
+// documents: [{ id, text }] -> deterministic index stats with sorted ids and term keys.
+export function buildLexicalStats(documents) {
+  const indexed = [...documents]
+    .sort((a, b) => (a.id < b.id ? -1 : a.id > b.id ? 1 : 0))
+    .map((doc) => {
+      const tokens = lexicalTokens(doc.text);
+      return { id: doc.id, length: tokens.length, terms: termFrequencies(tokens) };
+    });
+
+  const docFrequency = {};
+  for (const doc of indexed) {
+    for (const term of Object.keys(doc.terms)) {
+      docFrequency[term] = (docFrequency[term] || 0) + 1;
+    }
+  }
+  const sortedDocFrequency = {};
+  for (const term of Object.keys(docFrequency).sort()) {
+    sortedDocFrequency[term] = docFrequency[term];
+  }
+
+  const totalLength = indexed.reduce((sum, doc) => sum + doc.length, 0);
+  return {
+    documentCount: indexed.length,
+    termCount: Object.keys(sortedDocFrequency).length,
+    avgDocLength: indexed.length > 0 ? totalLength / indexed.length : 0,
+    docFrequency: sortedDocFrequency,
+    documents: indexed,
+  };
+}
+
+function inverseDocumentFrequency(documentCount, docFrequency) {
+  return Math.log(1 + (documentCount - docFrequency + 0.5) / (docFrequency + 0.5));
+}
+
+export function bm25Score(queryTokens, doc, stats) {
+  if (stats.documentCount === 0 || doc.length === 0) return { score: 0, matchedTokens: [] };
+
+  let score = 0;
+  const matchedTokens = [];
+  for (const token of queryTokens) {
+    const tf = doc.terms[token] || 0;
+    if (tf === 0) continue;
+    const df = stats.docFrequency[token] || 0;
+    const idf = inverseDocumentFrequency(stats.documentCount, df);
+    const lengthNorm = 1 - BM25_B + BM25_B * (doc.length / (stats.avgDocLength || 1));
+    score += idf * ((tf * (BM25_K1 + 1)) / (tf + BM25_K1 * lengthNorm));
+    matchedTokens.push(token);
+  }
+  return { score, matchedTokens };
+}
+
+// Fully ordered ranking: score desc, then id asc. Zero-score documents are excluded.
+export function rankLexical(query, stats, { limit = 20 } = {}) {
+  const queryTokens = lexicalQueryTokens(query);
+  if (queryTokens.length === 0) return [];
+
+  const ranked = [];
+  for (const doc of stats.documents) {
+    const { score, matchedTokens } = bm25Score(queryTokens, doc, stats);
+    if (score > 0) {
+      ranked.push({ id: doc.id, score, matchedTokens });
+    }
+  }
+  ranked.sort((a, b) => {
+    if (b.score !== a.score) return b.score - a.score;
+    return a.id < b.id ? -1 : a.id > b.id ? 1 : 0;
+  });
+  return ranked.slice(0, limit).map((hit) => ({
+    ...hit,
+    score: Number(hit.score.toFixed(6)),
+  }));
+}

package/cli/lib/local-config.mjs

@@ -0,0 +1,110 @@
+// Local, user-authored configuration for optional Phase B features
+// (docs/AI-LEARNING-PHASE2.md, "Phase A implementation review", decision 5 / FU-4).
+//
+// design-ai never writes this file. It is durable, per-machine, user-authored
+// configuration — the CLI only reads it. Presence of a configured provider is
+// NOT enough to enable embeddings: every consumer still requires the explicit
+// per-invocation --embeddings flag (config supplies the provider, the flag arms it).
+
+import { existsSync, readFileSync } from "node:fs";
+import { homedir } from "node:os";
+import path from "node:path";
+
+export function defaultConfigFile() {
+  return process.env.DESIGN_AI_CONFIG_FILE || path.join(homedir(), ".design-ai", "config.json");
+}
+
+const TOP_LEVEL_KEYS = new Set(["version", "embeddings"]);
+const EMBEDDINGS_KEYS = new Set(["provider", "modelLabel"]);
+const PROVIDER_KEYS = new Set(["command", "args"]);
+
+function fail(message) {
+  return { present: true, config: null, error: message };
+}
+
+function isPlainObject(value) {
+  return Boolean(value) && typeof value === "object" && !Array.isArray(value);
+}
+
+function validateProvider(provider) {
+  if (!isPlainObject(provider)) return "embeddings.provider must be an object";
+
+  const unknown = Object.keys(provider).filter((key) => !PROVIDER_KEYS.has(key));
+  if (unknown.length > 0) return `embeddings.provider has unknown key(s): ${unknown.join(", ")}`;
+
+  if (typeof provider.command !== "string" || provider.command.trim() === "") {
+    return "embeddings.provider.command is required and must be a non-empty string";
+  }
+
+  if (provider.args !== undefined) {
+    if (!Array.isArray(provider.args) || !provider.args.every((arg) => typeof arg === "string")) {
+      return "embeddings.provider.args must be an array of strings";
+    }
+  }
+
+  return "";
+}
+
+function validateEmbeddings(embeddings) {
+  if (!isPlainObject(embeddings)) return "embeddings must be an object";
+
+  const unknown = Object.keys(embeddings).filter((key) => !EMBEDDINGS_KEYS.has(key));
+  if (unknown.length > 0) return `embeddings has unknown key(s): ${unknown.join(", ")}`;
+
+  if (embeddings.provider === undefined) return "embeddings.provider is required";
+  const providerError = validateProvider(embeddings.provider);
+  if (providerError) return providerError;
+
+  if (embeddings.modelLabel !== undefined && typeof embeddings.modelLabel !== "string") {
+    return "embeddings.modelLabel must be a string";
+  }
+
+  return "";
+}
+
+function validateConfig(config) {
+  if (!isPlainObject(config)) return "config must be a JSON object";
+
+  const unknown = Object.keys(config).filter((key) => !TOP_LEVEL_KEYS.has(key));
+  if (unknown.length > 0) return `config has unknown top-level key(s): ${unknown.join(", ")}`;
+
+  if (config.version !== 1) return "config.version must be 1";
+
+  if (config.embeddings !== undefined) {
+    const embeddingsError = validateEmbeddings(config.embeddings);
+    if (embeddingsError) return embeddingsError;
+  }
+
+  return "";
+}
+
+// Never writes; design-ai only reads this user-authored file.
+export function loadLocalConfig(filePath = defaultConfigFile()) {
+  if (!existsSync(filePath)) return { present: false, config: null, error: "" };
+
+  let raw;
+  try {
+    raw = readFileSync(filePath, "utf8");
+  } catch (error) {
+    return fail(`unreadable config file: ${error.message}`);
+  }
+
+  let parsed;
+  try {
+    parsed = JSON.parse(raw);
+  } catch (error) {
+    return fail(`config file is not valid JSON: ${error.message}`);
+  }
+
+  const error = validateConfig(parsed);
+  if (error) return fail(error);
+
+  return { present: true, config: parsed, error: "" };
+}
+
+// Convenience accessor: the configured embedding provider, or null when absent
+// or the config is malformed/missing. Callers still must gate on --embeddings.
+export function configuredEmbeddingProvider(filePath = defaultConfigFile()) {
+  const { config } = loadLocalConfig(filePath);
+  return config?.embeddings?.provider || null;
+}

package/cli/lib/mcp-server.mjs

@@ -50,7 +50,7 @@ export const MCP_TOOLS = [
   {
     name: "design_ai_prompt",
     title: "Generate an agent prompt",
-    description: "Generate a ready-to-use design-ai agent prompt from a brief. Read-only unless withLearning is true, which records local usage metadata.",
+    description: "Generate a ready-to-use design-ai agent prompt from a brief. Read-only unless withLearning is true, which records local usage metadata. Supports opt-in recall augmentation via withRecall, which enriches the output with brief-relevant shipped corpus knowledge.",
     inputSchema: {
       type: "object",
       properties: {
@@ -59,6 +59,8 @@ export const MCP_TOOLS = [
         withLearning: optionalBoolean("Include local learning preferences and record local usage metadata."),
         learningCategory: optionalString("Optional learning category when withLearning is true."),
         learningLimit: optionalInteger({ description: "Maximum learning entries, 1-100.", minimum: 1, maximum: 100 }),
+        withRecall: optionalBoolean("Opt-in recall augmentation: enriches the output with brief-relevant shipped corpus knowledge ranked by the deterministic lexical scorer. Requires no index and makes no network call."),
+        recallLimit: optionalInteger({ description: "Maximum recall passages, 1-20.", minimum: 1, maximum: 20 }),
         json: optionalBoolean("Return machine-readable prompt plan JSON instead of Markdown."),
       },
       required: ["brief"],
@@ -68,7 +70,7 @@ export const MCP_TOOLS = [
   {
     name: "design_ai_pack",
     title: "Generate prompt plus bounded context",
-    description: "Generate a prompt pack with relevant design-ai context files. Read-only unless withLearning is true, which records local usage metadata.",
+    description: "Generate a prompt pack with relevant design-ai context files. Read-only unless withLearning is true, which records local usage metadata. Supports opt-in recall augmentation via withRecall, which enriches the output with brief-relevant shipped corpus knowledge.",
     inputSchema: {
       type: "object",
       properties: {
@@ -78,6 +80,8 @@ export const MCP_TOOLS = [
         withLearning: optionalBoolean("Include local learning preferences and record local usage metadata."),
         learningCategory: optionalString("Optional learning category when withLearning is true."),
         learningLimit: optionalInteger({ description: "Maximum learning entries, 1-100.", minimum: 1, maximum: 100 }),
+        withRecall: optionalBoolean("Opt-in recall augmentation: enriches the output with brief-relevant shipped corpus knowledge ranked by the deterministic lexical scorer. Requires no index and makes no network call."),
+        recallLimit: optionalInteger({ description: "Maximum recall passages, 1-20.", minimum: 1, maximum: 20 }),
         json: optionalBoolean("Return machine-readable pack JSON instead of Markdown."),
       },
       required: ["brief"],
@@ -87,13 +91,14 @@ export const MCP_TOOLS = [
   {
     name: "design_ai_search",
     title: "Search design-ai corpus",
-    description: "Search knowledge, examples, skills, docs, agents, or commands. Read-only.",
+    description: "Search knowledge, examples, skills, docs, agents, or commands. Read-only. Supports an opt-in ranked mode for deterministic BM25-style results.",
     inputSchema: {
       type: "object",
       properties: {
         query: { type: "string", minLength: 2, description: "Search query." },
         dir: optionalString("Optional corpus directory: knowledge, examples, skills, docs, agents, commands."),
         limit: optionalInteger({ description: "Maximum hits, 1-500.", minimum: 1, maximum: 500 }),
+        ranked: optionalBoolean("Opt-in deterministic BM25-style ranked results. Requires no index and never builds one; falls back to a live corpus scan when no index is present."),
       },
       required: ["query"],
       additionalProperties: false,
@@ -281,6 +286,8 @@ export function buildCliInvocation(toolName, input = {}) {
     maybeBool(args, "--with-learning", input.withLearning);
     maybePush(args, "--learning-category", input.learningCategory);
     maybePush(args, "--learning-limit", input.learningLimit);
+    maybeBool(args, "--with-recall", input.withRecall);
+    maybePush(args, "--recall-limit", input.recallLimit);
     maybeBool(args, "--json", input.json);
     return { args, stdin };
   }
@@ -292,6 +299,8 @@ export function buildCliInvocation(toolName, input = {}) {
     maybeBool(args, "--with-learning", input.withLearning);
     maybePush(args, "--learning-category", input.learningCategory);
     maybePush(args, "--learning-limit", input.learningLimit);
+    maybeBool(args, "--with-recall", input.withRecall);
+    maybePush(args, "--recall-limit", input.recallLimit);
     maybeBool(args, "--json", input.json);
     return { args, stdin };
   }
@@ -300,6 +309,7 @@ export function buildCliInvocation(toolName, input = {}) {
     args.push("search", assertString(input.query, "query"), "--json");
     maybePush(args, "--dir", input.dir);
     maybePush(args, "--limit", input.limit);
+    maybeBool(args, "--ranked", input.ranked);
     return { args, stdin };
   }
 

package/cli/lib/pack.mjs

@@ -4,6 +4,7 @@ import { readFileSync } from "node:fs";
 
 import { parseBriefSourceFlag } from "./brief.mjs";
 import { normalizeCategory, parseLearningLimit } from "./learn.mjs";
+import { parseRecallLimit } from "./recall.mjs";
 import { SYMLINK_PREFIX } from "./paths.mjs";
 import { parseOutputFlags } from "./output.mjs";
 import { readRouteManifestVersion } from "./route.mjs";
@@ -29,6 +30,8 @@ const PACK_OPTIONS = [
   "--with-learning",
   "--learning-category",
   "--learning-limit",
+  "--with-recall",
+  "--recall-limit",
 ];
 
 export function parsePackArgs(args) {
@@ -44,6 +47,8 @@ export function parsePackArgs(args) {
     withLearning: false,
     learningCategory: "",
     learningLimit: 0,
+    withRecall: false,
+    recallLimit: 0,
     evalTemplate: false,
     eval: false,
     strict: false,
@@ -58,6 +63,17 @@ export function parsePackArgs(args) {
       out.json = true;
     } else if (arg === "--with-learning") {
       out.withLearning = true;
+    } else if (arg === "--with-recall") {
+      out.withRecall = true;
+    } else if (arg === "--recall-limit") {
+      const limit = args[i + 1];
+      if (!limit || limit.startsWith("--")) throw new Error("--recall-limit expects an integer from 1 to 20");
+      try {
+        out.recallLimit = parseRecallLimit(limit);
+      } catch {
+        throw new Error("--recall-limit expects an integer from 1 to 20");
+      }
+      i += 1;
     } else if (arg === "--eval-template") {
       out.evalTemplate = true;
     } else if (arg === "--eval") {
@@ -108,20 +124,23 @@ export function parsePackArgs(args) {
   if ((out.learningCategory || out.learningLimit) && !out.withLearning) {
     throw new Error("--learning-category and --learning-limit require --with-learning");
   }
+  if (out.recallLimit && !out.withRecall) {
+    throw new Error("--recall-limit requires --with-recall");
+  }
   if (out.eval && out.evalTemplate) {
     throw new Error("Choose either --eval-template or --eval, not both");
   }
   if (out.strict && !out.eval) {
     throw new Error("--strict can only be used with --eval");
   }
-  if (out.evalTemplate && (out.briefParts.length > 0 || out.fromFile || out.stdin || out.routeId || out.withLearning)) {
-    throw new Error("--eval-template cannot be combined with a brief, --from-file, --stdin, --route, or --with-learning");
+  if (out.evalTemplate && (out.briefParts.length > 0 || out.fromFile || out.stdin || out.routeId || out.withLearning || out.withRecall)) {
+    throw new Error("--eval-template cannot be combined with a brief, --from-file, --stdin, --route, --with-learning, or --with-recall");
   }
   if (out.eval && (!out.fromFile && !out.stdin)) {
     throw new Error("--eval requires --from-file or --stdin");
   }
-  if (out.eval && (out.briefParts.length > 0 || out.routeId || out.withLearning)) {
-    throw new Error("--eval cannot be combined with an inline brief, --route, or --with-learning");
+  if (out.eval && (out.briefParts.length > 0 || out.routeId || out.withLearning || out.withRecall)) {
+    throw new Error("--eval cannot be combined with an inline brief, --route, --with-learning, or --with-recall");
   }
 
   return {
@@ -200,6 +219,8 @@ export function buildPromptPack({
   learningFilePath = "",
   learningCategory = "",
   learningLimit = 0,
+  withRecall = false,
+  recallLimit = 0,
 }) {
   const plan = buildPromptPlan({
     brief,
@@ -210,6 +231,8 @@ export function buildPromptPack({
     learningFilePath,
     learningCategory,
     learningLimit,
+    withRecall,
+    recallLimit,
   });
   const files = [];
   let usedBytes = 0;
@@ -572,6 +595,10 @@ export function renderPromptPack({ plan, files, summary, warnings = [] }) {
   lines.push("");
   lines.push("## Prompt");
   lines.push("");
+  // plan.prompt already carries the --with-recall "## Recalled design knowledge"
+  // section (rendered by renderPrompt). Recall respects the byte budget by being
+  // subject to the same whole-pack takeUtf8(maxBytes) truncation as everything else —
+  // there is no recall-specific budget carve-out.
   lines.push(plan.prompt);
   lines.push("");
   lines.push("## Context Files");

package/cli/lib/prompt.mjs

@@ -6,6 +6,7 @@ import { existsSync } from "node:fs";
 import { parseBriefSourceFlag } from "./brief.mjs";
 import { listExamples } from "./examples.mjs";
 import { buildLearningContext, normalizeCategory, parseLearningLimit } from "./learn.mjs";
+import { buildRecallContext, DEFAULT_RECALL_LIMIT, parseRecallLimit } from "./recall.mjs";
 import { SYMLINK_PREFIX } from "./paths.mjs";
 import { parseOutputFlags } from "./output.mjs";
 import { readRouteManifestVersion, routeBrief, routeById } from "./route.mjs";
@@ -26,6 +27,8 @@ const PROMPT_OPTIONS = [
   "--with-learning",
   "--learning-category",
   "--learning-limit",
+  "--with-recall",
+  "--recall-limit",
 ];
 const PROMPT_EVAL_VERSION = 1;
 const PROMPT_EVAL_DEFAULT_LIMIT = 12;
@@ -42,6 +45,8 @@ export function parsePromptArgs(args) {
     withLearning: false,
     learningCategory: "",
     learningLimit: 0,
+    withRecall: false,
+    recallLimit: 0,
     evalTemplate: false,
     eval: false,
     strict: false,
@@ -57,6 +62,17 @@ export function parsePromptArgs(args) {
       out.json = true;
     } else if (arg === "--with-learning") {
       out.withLearning = true;
+    } else if (arg === "--with-recall") {
+      out.withRecall = true;
+    } else if (arg === "--recall-limit") {
+      const limit = args[i + 1];
+      if (!limit || limit.startsWith("--")) throw new Error("--recall-limit expects an integer from 1 to 20");
+      try {
+        out.recallLimit = parseRecallLimit(limit);
+      } catch {
+        throw new Error("--recall-limit expects an integer from 1 to 20");
+      }
+      i += 1;
     } else if (arg === "--eval-template") {
       out.evalTemplate = true;
     } else if (arg === "--eval") {
@@ -96,20 +112,23 @@ export function parsePromptArgs(args) {
   if ((out.learningCategory || out.learningLimit) && !out.withLearning) {
     throw new Error("--learning-category and --learning-limit require --with-learning");
   }
+  if (out.recallLimit && !out.withRecall) {
+    throw new Error("--recall-limit requires --with-recall");
+  }
   if (out.eval && out.evalTemplate) {
     throw new Error("Choose either --eval-template or --eval, not both");
   }
   if (out.strict && !out.eval) {
     throw new Error("--strict can only be used with --eval");
   }
-  if (out.evalTemplate && (out.briefParts.length > 0 || out.fromFile || out.stdin || out.routeId || out.withLearning)) {
-    throw new Error("--eval-template cannot be combined with a brief, --from-file, --stdin, --route, or --with-learning");
+  if (out.evalTemplate && (out.briefParts.length > 0 || out.fromFile || out.stdin || out.routeId || out.withLearning || out.withRecall)) {
+    throw new Error("--eval-template cannot be combined with a brief, --from-file, --stdin, --route, --with-learning, or --with-recall");
   }
   if (out.eval && (!out.fromFile && !out.stdin)) {
     throw new Error("--eval requires --from-file or --stdin");
   }
-  if (out.eval && (out.briefParts.length > 0 || out.routeId || out.withLearning)) {
-    throw new Error("--eval cannot be combined with an inline brief, --route, or --with-learning");
+  if (out.eval && (out.briefParts.length > 0 || out.routeId || out.withLearning || out.withRecall)) {
+    throw new Error("--eval cannot be combined with an inline brief, --route, --with-learning, or --with-recall");
   }
 
   return {
@@ -209,6 +228,8 @@ export function buildPromptPlan({
   learningFilePath = "",
   learningCategory = "",
   learningLimit = 0,
+  withRecall = false,
+  recallLimit = 0,
 }) {
   const route = routeId
     ? routeById({ routeId, sourceRoot })
@@ -243,6 +264,13 @@ export function buildPromptPlan({
       query: brief,
     })
     : null;
+  const recallContext = withRecall
+    ? buildRecallContext({
+      brief,
+      recallLimit: recallLimit || DEFAULT_RECALL_LIMIT,
+      designAiPath: sourceRoot,
+    })
+    : null;
 
   return {
     brief,
@@ -254,6 +282,7 @@ export function buildPromptPlan({
     checklist,
     qualityCommand,
     ...(learningContext ? { learningContext } : {}),
+    ...(recallContext ? { recall: recallContext } : {}),
     prompt: renderPrompt({
       brief,
       route,
@@ -263,6 +292,7 @@ export function buildPromptPlan({
       checklist,
       qualityCommand,
       learningContext,
+      recallContext,
     }),
   };
 }
@@ -526,6 +556,7 @@ export function renderPrompt({
   checklist = checklistForRoute(route),
   qualityCommand = qualityCommandForRoute(route.id),
   learningContext = null,
+  recallContext = null,
 }) {
   const lines = [];
   lines.push("# design-ai task prompt");
@@ -571,6 +602,13 @@ export function renderPrompt({
     lines.push("");
   }
 
+  if (recallContext) {
+    lines.push("Recalled corpus knowledge:");
+    lines.push("");
+    lines.push(recallContext.markdown);
+    lines.push("");
+  }
+
   lines.push("Before producing the artifact, read these files in order:");
   for (const file of filesToRead) {
     lines.push(`- ${file}`);