@ktpartners/dgs-platform 2.6.2

package/deliver-great-systems/bin/lib/search.cjs

@@ -0,0 +1,570 @@
+/**
+ * Search — Fuzzy keyword search across ideas, specs, docs, and projects
+ *
+ * Provides the core search engine for cross-content discovery:
+ * fuzzy matching, content scanning, scope/tag filtering, result grouping.
+ * Used by the /dgs:search command and write-spec auto-search.
+ * Search is read-only — no files are created or modified.
+ */
+
+const fs = require('fs');
+const path = require('path');
+const { safeReadFile, output, error } = require('./core.cjs');
+const { getPlanningRoot } = require('./paths.cjs');
+
+// ─── Fuzzy Matching Engine ──────────────────────────────────────────────────
+
+/**
+ * Compute Levenshtein distance between two strings.
+ * Standard dynamic programming implementation.
+ *
+ * @param {string} a - First string
+ * @param {string} b - Second string
+ * @returns {number} Edit distance
+ */
+function levenshteinDistance(a, b) {
+  const m = a.length;
+  const n = b.length;
+
+  // Quick exits
+  if (m === 0) return n;
+  if (n === 0) return m;
+
+  // Use single-row optimization
+  let prev = new Array(n + 1);
+  let curr = new Array(n + 1);
+
+  for (let j = 0; j <= n; j++) {
+    prev[j] = j;
+  }
+
+  for (let i = 1; i <= m; i++) {
+    curr[0] = i;
+    for (let j = 1; j <= n; j++) {
+      const cost = a[i - 1] === b[j - 1] ? 0 : 1;
+      curr[j] = Math.min(
+        prev[j] + 1,       // deletion
+        curr[j - 1] + 1,   // insertion
+        prev[j - 1] + cost  // substitution
+      );
+    }
+    [prev, curr] = [curr, prev];
+  }
+
+  return prev[n];
+}
+
+/**
+ * Check if a query word fuzzy-matches a content word.
+ * Uses Levenshtein distance with adaptive threshold:
+ * - Words <= 4 chars: distance <= 1
+ * - Longer words: distance <= 2
+ * Also checks substring containment for partial matches.
+ *
+ * @param {string} text - Content word (lowercase)
+ * @param {string} word - Query word (lowercase)
+ * @returns {boolean}
+ */
+function fuzzyMatch(text, word) {
+  // Exact match
+  if (text === word) return true;
+
+  // Substring containment (text contains the query word)
+  if (text.includes(word)) return true;
+
+  // Levenshtein distance with adaptive threshold
+  const threshold = word.length <= 4 ? 1 : 2;
+  const dist = levenshteinDistance(text, word);
+  return dist <= threshold;
+}
+
+/**
+ * Check if ALL query words appear in the content (AND logic).
+ * Each query word is checked via fuzzyMatch against every word in the content.
+ * Case-insensitive.
+ *
+ * @param {string} content - Full text content to search
+ * @param {string[]} queryWords - Array of lowercase query words
+ * @returns {boolean}
+ */
+function matchesQuery(content, queryWords) {
+  if (!content || queryWords.length === 0) return false;
+
+  const contentLower = content.toLowerCase();
+  const contentWords = contentLower.split(/\s+/).filter(w => w.length > 0);
+
+  for (const qWord of queryWords) {
+    let found = false;
+    for (const cWord of contentWords) {
+      if (fuzzyMatch(cWord, qWord)) {
+        found = true;
+        break;
+      }
+    }
+    if (!found) return false;
+  }
+
+  return true;
+}
+
+// ─── Frontmatter Parsing ────────────────────────────────────────────────────
+
+/**
+ * Parse simple YAML frontmatter from file content.
+ * Reusable across content types.
+ *
+ * @param {string} content - File content
+ * @returns {{ frontmatter: object, body: string }}
+ */
+function parseFrontmatter(content) {
+  const match = content.match(/^---\n([\s\S]+?)\n---\n?([\s\S]*)/);
+  if (!match) return { frontmatter: {}, body: content };
+
+  const yaml = match[1];
+  const body = match[2] || '';
+  const frontmatter = {};
+
+  const lines = yaml.split('\n');
+  for (const line of lines) {
+    if (line.trim() === '') continue;
+
+    // Handle inline array values
+    const arrayMatch = line.match(/^([\w_]+):\s*\[([^\]]*)\]/);
+    if (arrayMatch) {
+      const key = arrayMatch[1];
+      const values = arrayMatch[2]
+        .split(',')
+        .map(v => v.trim().replace(/^["']|["']$/g, ''))
+        .filter(v => v !== '');
+      frontmatter[key] = values;
+      continue;
+    }
+
+    // Handle simple key: value
+    const kvMatch = line.match(/^([\w_]+):\s*(.*)/);
+    if (kvMatch) {
+      const key = kvMatch[1];
+      let value = kvMatch[2].trim();
+      value = value.replace(/^["']|["']$/g, '');
+      if (key === 'id' && /^\d+$/.test(value)) {
+        frontmatter[key] = parseInt(value, 10);
+      } else {
+        frontmatter[key] = value;
+      }
+    }
+  }
+
+  return { frontmatter, body };
+}
+
+// ─── Content Scanners ───────────────────────────────────────────────────────
+
+/**
+ * Scan ideas directories for searchable idea files.
+ * Scans .planning/ideas/{pending,done,rejected}/ for .md files.
+ *
+ * @param {string} cwd - Working directory
+ * @param {object} options - { include_rejected, tags }
+ * @returns {Array<{ type: string, id: number, title: string, filePath: string, state: string, tags: string[], author: string, content: string }>}
+ */
+function scanIdeas(cwd, options) {
+  const results = [];
+  const planRoot = getPlanningRoot(cwd);
+  const planRootRel = path.relative(cwd, planRoot) || '.';
+  const states = options.include_rejected
+    ? ['pending', 'done', 'rejected']
+    : ['pending', 'done'];
+
+  for (const state of states) {
+    const dir = path.join(planRoot, 'ideas', state);
+    let files;
+    try {
+      files = fs.readdirSync(dir).filter(f => f.endsWith('.md'));
+    } catch {
+      continue;
+    }
+
+    for (const file of files) {
+      const filePath = path.join(dir, file);
+      const content = safeReadFile(filePath);
+      if (!content) continue;
+
+      const { frontmatter, body } = parseFrontmatter(content);
+      const tags = frontmatter.tags || [];
+
+      // Tag filter: if tags option set, idea must have at least one matching tag (OR logic)
+      if (options.tags) {
+        const filterTags = options.tags.split(',').map(t => t.trim().toLowerCase());
+        const ideaTags = tags.map(t => t.toLowerCase());
+        const hasMatch = filterTags.some(ft => ideaTags.includes(ft));
+        if (!hasMatch) continue;
+      }
+
+      results.push({
+        type: 'idea',
+        id: frontmatter.id,
+        title: frontmatter.title || file,
+        filePath: path.join(planRootRel, 'ideas', state, file),
+        state,
+        tags,
+        author: frontmatter.author || '',
+        content: (frontmatter.title || '') + ' ' + body,
+      });
+    }
+  }
+
+  return results;
+}
+
+/**
+ * Scan specs directory for searchable spec files.
+ * Scans .planning/specs/ for .md files.
+ *
+ * @param {string} cwd - Working directory
+ * @returns {Array<{ type: string, id: string, title: string, filePath: string, status: string, author: string, content: string }>}
+ */
+function scanSpecs(cwd) {
+  const results = [];
+  const planRoot = getPlanningRoot(cwd);
+  const planRootRel = path.relative(cwd, planRoot) || '.';
+  const dir = path.join(planRoot, 'specs');
+  let files;
+  try {
+    files = fs.readdirSync(dir).filter(f => f.endsWith('.md'));
+  } catch {
+    return results;
+  }
+
+  for (const file of files) {
+    const filePath = path.join(dir, file);
+    const content = safeReadFile(filePath);
+    if (!content) continue;
+
+    const { frontmatter, body } = parseFrontmatter(content);
+
+    results.push({
+      type: 'spec',
+      id: frontmatter.id || file,
+      title: frontmatter.title || file,
+      filePath: path.join(planRootRel, 'specs', file),
+      status: frontmatter.status || 'draft',
+      author: frontmatter.author || '',
+      content: (frontmatter.title || '') + ' ' + body,
+    });
+  }
+
+  return results;
+}
+
+/**
+ * Scan docs directories for searchable documents.
+ * Scans .planning/docs/ (product), .planning/ideas/star/docs/ (idea-scoped),
+ * .planning/specs/star/docs/ (spec-scoped).
+ * Uses .extracted.txt sidecars when available.
+ *
+ * @param {string} cwd - Working directory
+ * @returns {Array<{ type: string, title: string, filePath: string, scope: string, content: string }>}
+ */
+function scanDocs(cwd) {
+  const results = [];
+
+  /**
+   * Scan a single docs directory and add results.
+   */
+  function scanDocsDir(docsDir, scope, relBase) {
+    let files;
+    try {
+      files = fs.readdirSync(docsDir).filter(f =>
+        f !== 'INDEX.md' &&
+        !f.endsWith('.extracted.txt') &&
+        !f.endsWith('.summary.txt') &&
+        !f.startsWith('.')
+      );
+    } catch {
+      return;
+    }
+
+    // Load .names.json for original names
+    let nameMap = {};
+    const namesPath = path.join(docsDir, '.names.json');
+    const namesContent = safeReadFile(namesPath);
+    if (namesContent) {
+      try { nameMap = JSON.parse(namesContent); } catch { /* ignore */ }
+    }
+
+    for (const file of files) {
+      const filePath = path.join(docsDir, file);
+      const relPath = path.join(relBase, file);
+      const title = nameMap[file] || file;
+
+      // Check for extracted text sidecar
+      const extractedPath = filePath + '.extracted.txt';
+      let searchContent;
+
+      if (fs.existsSync(extractedPath)) {
+        searchContent = safeReadFile(extractedPath) || '';
+      } else {
+        // Only use file content for text-like files
+        const ext = path.extname(file).toLowerCase();
+        if (ext === '.md' || ext === '.txt') {
+          searchContent = safeReadFile(filePath) || '';
+        } else {
+          searchContent = title; // Fall back to filename/title only
+        }
+      }
+
+      results.push({
+        type: 'doc',
+        title,
+        filePath: relPath,
+        scope,
+        content: title + ' ' + searchContent,
+      });
+    }
+  }
+
+  const planRoot = getPlanningRoot(cwd);
+  const planRootRel = path.relative(cwd, planRoot) || '.';
+
+  // Product docs
+  const productDocsDir = path.join(planRoot, 'docs');
+  scanDocsDir(productDocsDir, 'product', path.join(planRootRel, 'docs'));
+
+  // Idea-scoped docs
+  const ideaStates = ['pending', 'done', 'rejected'];
+  for (const state of ideaStates) {
+    const stateDir = path.join(planRoot, 'ideas', state);
+    let ideaDirs;
+    try {
+      ideaDirs = fs.readdirSync(stateDir, { withFileTypes: true })
+        .filter(e => e.isDirectory())
+        .map(e => e.name);
+    } catch {
+      continue;
+    }
+    for (const ideaSlug of ideaDirs) {
+      const docsDir = path.join(stateDir, ideaSlug, 'docs');
+      scanDocsDir(docsDir, `idea:${ideaSlug}`, path.join(planRootRel, 'ideas', state, ideaSlug, 'docs'));
+    }
+  }
+
+  // Spec-scoped docs
+  const specsDir = path.join(planRoot, 'specs');
+  let specDirs;
+  try {
+    specDirs = fs.readdirSync(specsDir, { withFileTypes: true })
+      .filter(e => e.isDirectory())
+      .map(e => e.name);
+  } catch {
+    specDirs = [];
+  }
+  for (const specSlug of specDirs) {
+    const docsDir = path.join(specsDir, specSlug, 'docs');
+    scanDocsDir(docsDir, `spec:${specSlug}`, path.join(planRootRel, 'specs', specSlug, 'docs'));
+  }
+
+  return results;
+}
+
+/**
+ * Scan for project directories (those containing STATE.md).
+ * Reads STATE.md, ROADMAP.md, and REQUIREMENTS.md as searchable content.
+ *
+ * @param {string} cwd - Working directory
+ * @returns {Array<{ type: string, title: string, filePath: string, content: string }>}
+ */
+function scanProjects(cwd) {
+  const results = [];
+  const planningDir = getPlanningRoot(cwd);
+  const planRootRel = path.relative(cwd, planningDir) || '.';
+  let entries;
+  try {
+    entries = fs.readdirSync(planningDir, { withFileTypes: true })
+      .filter(e => e.isDirectory());
+  } catch {
+    return results;
+  }
+
+  for (const entry of entries) {
+    const dirPath = path.join(planningDir, entry.name);
+    const statePath = path.join(dirPath, 'STATE.md');
+
+    if (!fs.existsSync(statePath)) continue;
+
+    let content = '';
+
+    // Read STATE.md
+    const stateContent = safeReadFile(statePath);
+    if (stateContent) content += stateContent + '\n';
+
+    // Read ROADMAP.md if exists
+    const roadmapPath = path.join(dirPath, 'ROADMAP.md');
+    const roadmapContent = safeReadFile(roadmapPath);
+    if (roadmapContent) content += roadmapContent + '\n';
+
+    // Read REQUIREMENTS.md if exists
+    const reqsPath = path.join(dirPath, 'REQUIREMENTS.md');
+    const reqsContent = safeReadFile(reqsPath);
+    if (reqsContent) content += reqsContent + '\n';
+
+    results.push({
+      type: 'project',
+      title: entry.name,
+      filePath: path.join(planRootRel, entry.name),
+      content: entry.name + ' ' + content,
+    });
+  }
+
+  return results;
+}
+
+// ─── Context Snippet Extraction ─────────────────────────────────────────────
+
+/**
+ * Extract a context snippet from content around the first matching query word.
+ * Returns the match line plus 1 line above and 1 line below (3-line snippet).
+ *
+ * @param {string} content - Full text content
+ * @param {string[]} queryWords - Lowercase query words
+ * @returns {string} Context snippet (trimmed)
+ */
+function extractSnippet(content, queryWords) {
+  if (!content) return '';
+
+  const lines = content.split('\n');
+
+  for (let i = 0; i < lines.length; i++) {
+    const lineLower = lines[i].toLowerCase();
+    const lineWords = lineLower.split(/\s+/);
+
+    for (const qWord of queryWords) {
+      for (const lWord of lineWords) {
+        if (fuzzyMatch(lWord, qWord)) {
+          const start = Math.max(0, i - 1);
+          const end = Math.min(lines.length - 1, i + 1);
+          const snippet = lines.slice(start, end + 1).join('\n').trim();
+          // Trim to reasonable length
+          return snippet.length > 300 ? snippet.substring(0, 300) + '...' : snippet;
+        }
+      }
+    }
+  }
+
+  // Fallback: first 3 lines
+  return lines.slice(0, 3).join('\n').trim().substring(0, 300);
+}
+
+// ─── Main Search Function ───────────────────────────────────────────────────
+
+/**
+ * Search across ideas, specs, docs, and projects.
+ *
+ * @param {string} cwd - Working directory
+ * @param {string} query - Search query string
+ * @param {object} options - { ideas_only, specs_only, docs_only, include_rejected, tags, max_per_type }
+ * @param {boolean} raw - Raw output mode
+ */
+function cmdSearch(cwd, query, options, raw) {
+  if (!query || query.trim() === '') {
+    error('Search query required');
+  }
+
+  const queryWords = query.toLowerCase().split(/\s+/).filter(w => w.length > 0);
+  if (queryWords.length === 0) {
+    error('Search query required');
+  }
+
+  const opts = options || {};
+  const maxPerType = parseInt(opts.max_per_type, 10) || 5;
+
+  // Determine which types to scan based on scope filters
+  const hasTypeFilter = opts.ideas_only || opts.specs_only || opts.docs_only;
+  const scanTypes = {
+    ideas: hasTypeFilter ? !!opts.ideas_only : true,
+    specs: hasTypeFilter ? !!opts.specs_only : true,
+    docs: hasTypeFilter ? !!opts.docs_only : true,
+    projects: hasTypeFilter ? false : true, // projects only when no type filter
+  };
+
+  const groups = [];
+  let totalMatches = 0;
+
+  // Fixed order: ideas, specs, docs, projects
+  const typeOrder = ['ideas', 'specs', 'docs', 'projects'];
+
+  for (const typeName of typeOrder) {
+    if (!scanTypes[typeName]) continue;
+
+    let items;
+    if (typeName === 'ideas') {
+      items = scanIdeas(cwd, {
+        include_rejected: opts.include_rejected,
+        tags: opts.tags,
+      });
+    } else if (typeName === 'specs') {
+      items = scanSpecs(cwd);
+    } else if (typeName === 'docs') {
+      items = scanDocs(cwd);
+    } else if (typeName === 'projects') {
+      items = scanProjects(cwd);
+    }
+
+    // Filter to matching items
+    const matches = items.filter(item => matchesQuery(item.content, queryWords));
+    const total = matches.length;
+
+    if (total === 0) continue;
+
+    const shown = Math.min(total, maxPerType);
+    const overflow = total - shown;
+
+    const results = matches.slice(0, maxPerType).map(item => {
+      const result = {
+        title: item.title,
+        path: item.filePath,
+        snippet: extractSnippet(item.content, queryWords),
+      };
+
+      // Add type-specific fields
+      if (typeName === 'ideas') {
+        result.id = item.id;
+        result.state = item.state;
+        result.tags = item.tags;
+        result.author = item.author;
+      } else if (typeName === 'specs') {
+        result.id = item.id;
+        result.status = item.status;
+        result.author = item.author;
+      } else if (typeName === 'docs') {
+        result.scope = item.scope;
+      }
+
+      return result;
+    });
+
+    groups.push({
+      type: typeName,
+      results,
+      total,
+      shown,
+      overflow,
+    });
+
+    totalMatches += total;
+  }
+
+  output({
+    query,
+    groups,
+    total_matches: totalMatches,
+  }, raw);
+}
+
+// ─── Exports ──────────────────────────────────────────────────────────────────
+
+module.exports = {
+  cmdSearch,
+  fuzzyMatch,
+  levenshteinDistance,
+  matchesQuery,
+};