agent-method 1.5.12

package/lib/cli/update-docs.js

@@ -0,0 +1,574 @@
+/** wwa update-docs — term-based documentation index and update tool. */
+
+import { readFileSync, existsSync, readdirSync, statSync, mkdirSync } from "node:fs";
+import { resolve, join, relative } from "node:path";
+import { safeWriteFile } from "./helpers.js";
+import {
+  resolveTokensPath,
+  resolveOutputDir,
+  loadBoundaries,
+} from "../boundaries.js";
+
+export function register(program) {
+  program
+    .command("update-docs [directory]")
+    .description(
+      "Build or query a term-to-file index for documentation updates"
+    )
+    .option("--scan", "Scan docs/ and build doc-terms-index.yaml")
+    .option("--query <term>", "Look up which files reference a term")
+    .option("--category <cat>", "Filter to a category (cli_commands, feature_ids, workflow_ids, directive_ids, validation_ids, versions, concepts, counts)")
+    .option("--stale", "Show terms where doc-tokens.yaml value differs from docs content")
+    .option("--json", "Output as JSON")
+    .action(async (directory, opts) => {
+      directory = directory || ".";
+      const d = resolve(directory);
+
+      if (opts.scan) {
+        await scanAndBuildIndex(d, opts);
+        return;
+      }
+
+      // Load existing index
+      const outputDir = await resolveOutputDir(d);
+      const indexPath = join(outputDir, "doc-terms-index.yaml");
+      if (!existsSync(indexPath)) {
+        console.log("\n  No doc-terms-index.yaml found.");
+        console.log("  Run `wwa update-docs --scan` to build the term index.\n");
+        return;
+      }
+
+      const yaml = (await import("js-yaml")).default;
+      const index = yaml.load(readFileSync(indexPath, "utf-8"));
+
+      if (opts.query) {
+        await queryIndex(index, opts.query, opts);
+        return;
+      }
+
+      if (opts.stale) {
+        await showStale(d, index, opts);
+        return;
+      }
+
+      // Default: show summary
+      printIndexSummary(index, opts);
+    });
+}
+
+// ---------------------------------------------------------------------------
+// Scan all docs/ and build the term-to-file reverse index
+// ---------------------------------------------------------------------------
+
+const MAX_CONTEXT_LENGTH = 120;
+
+function snippet(line) {
+  const trimmed = line.trim();
+  if (trimmed.length <= MAX_CONTEXT_LENGTH) return trimmed;
+  return trimmed.slice(0, MAX_CONTEXT_LENGTH) + "...";
+}
+
+const TERM_PATTERNS = {
+  cli_commands: {
+    description: "CLI command references (wwa <cmd>)",
+    pattern: /\bwwa\s+[a-z][\w-]*/g,
+  },
+  feature_ids: {
+    description: "Feature identifiers (CTX-01, QRY-02, HAI-05, SCAN-03, etc.)",
+    pattern: /\b(?:CTX|QRY|STT|TSK|HAI|TPL|EXP|SCAN)-\d{2}\b/g,
+  },
+  workflow_ids: {
+    description: "Workflow identifiers (WF-01 through WF-08)",
+    pattern: /\bWF-\d{2}\b/g,
+  },
+  directive_ids: {
+    description: "Protocol directive references (P-01 through P-10)",
+    pattern: /\bP-\d{2}\b/g,
+  },
+  validation_ids: {
+    description: "Validation check identifiers (V-01 through V-12)",
+    pattern: /\bV-\d{2}\b/g,
+  },
+  versions: {
+    description: "Methodology version references",
+    pattern: /\bv1\.\d+(?:\.\d+)?\b/g,
+  },
+  concepts: {
+    description: "Core methodology concepts",
+    patterns: [
+      "context pairing",
+      "dependency cascade",
+      "scoping rules",
+      "intelligence layer",
+      "interaction level",
+      "entry point",
+      "cold start",
+      "specialist context",
+      "scale management",
+      "session protocol",
+      "cascade table",
+      "integration profile",
+    ],
+  },
+  counts: {
+    description: "Numeric values tied to methodology metrics",
+    matchers: [
+      { pattern: /\b(\d+)\s+features?\b/g, token: "feature_count" },
+      { pattern: /\b(\d+)\s+domains?\b/g, token: "domain_count" },
+      { pattern: /\b(\d+)\s+directives?\b/g, token: "directive_count" },
+      { pattern: /\b(\d+)\s+workflows?\b/g, token: "workflow_count" },
+      { pattern: /\b(\d+)\s+query.?patterns?\b/g, token: "query_pattern_count" },
+      { pattern: /\b(\d+)\s+(?:total\s+)?commands?\b/g, token: "total_cmd_count" },
+    ],
+  },
+};
+
+async function scanAndBuildIndex(dir, opts) {
+  const bounds = await loadBoundaries(dir);
+  const scanDirs = bounds.scan?.include || ["docs/", ".context/", "templates/"];
+
+  // Collect all .md files in scan directories
+  const files = [];
+  for (const scanDir of scanDirs) {
+    const absDir = join(dir, scanDir);
+    if (existsSync(absDir)) {
+      collectMdFiles(absDir, dir, files);
+    }
+  }
+
+  console.log(`\n  Scanning ${files.length} documentation files...`);
+
+  // Load doc-tokens.yaml for cross-referencing token values and names
+  let tokens = {};
+  let names = {};
+  try {
+    const tokensPath = await resolveTokensPath(dir);
+    if (existsSync(tokensPath)) {
+      const yaml = (await import("js-yaml")).default;
+      const parsed = yaml.load(readFileSync(tokensPath, "utf-8"));
+      tokens = parsed?.tokens || {};
+      names = parsed?.names || {};
+    }
+  } catch { /* skip */ }
+
+  const index = {
+    generated: new Date().toISOString().slice(0, 10),
+    source_directory: scanDirs.join(", "),
+    files_scanned: files.length,
+    categories: {},
+  };
+
+  // Scan each file for each term category
+  for (const [catKey, catDef] of Object.entries(TERM_PATTERNS)) {
+    if (catKey === "counts") {
+      // Counts use a different structure — metrics grouped by token key
+      index.categories.counts = buildCountsCategory(catDef, files, dir, tokens);
+      continue;
+    }
+
+    const terms = {};
+
+    for (const filePath of files) {
+      const content = readFileSync(join(dir, filePath), "utf-8");
+      const lines = content.split("\n");
+
+      if (catKey === "concepts") {
+        // String-based matching for concept terms
+        for (const concept of catDef.patterns) {
+          const lower = concept.toLowerCase();
+          for (let i = 0; i < lines.length; i++) {
+            if (lines[i].toLowerCase().includes(lower)) {
+              addUsage(terms, concept, filePath, i + 1, snippet(lines[i]));
+            }
+          }
+        }
+      } else {
+        // Regex-based matching
+        for (let i = 0; i < lines.length; i++) {
+          const re = new RegExp(catDef.pattern.source, catDef.pattern.flags);
+          let m;
+          while ((m = re.exec(lines[i])) !== null) {
+            addUsage(terms, m[0], filePath, i + 1, snippet(lines[i]));
+          }
+        }
+      }
+    }
+
+    // Compute total_references and sort
+    const sorted = {};
+    for (const key of Object.keys(terms).sort()) {
+      const t = terms[key];
+      t.total_references = t.referenced_in.reduce((s, f) => s + f.occurrences, 0);
+      t.referenced_in.sort((a, b) => b.occurrences - a.occurrences);
+      sorted[key] = t;
+    }
+
+    // Enrich with canonical names from doc-tokens.yaml
+    if (catKey === "workflow_ids" && names.guided_workflows) {
+      enrichFromNames(sorted, names.guided_workflows);
+    } else if (catKey === "feature_ids" && names.feature_domains) {
+      enrichFeatureIds(sorted, names.feature_domains);
+    } else if (catKey === "directive_ids" && names.protocol_directives) {
+      enrichFromNames(sorted, names.protocol_directives);
+    }
+
+    if (Object.keys(sorted).length > 0) {
+      index.categories[catKey] = {
+        description: catDef.description,
+        term_count: Object.keys(sorted).length,
+        terms: sorted,
+      };
+    }
+  }
+
+  // Compute totals
+  let totalTerms = 0;
+  for (const cat of Object.values(index.categories)) {
+    totalTerms += cat.term_count || Object.keys(cat.metrics || {}).length;
+  }
+  index.total_terms = totalTerms;
+
+  // Write the index
+  const outputDir = await resolveOutputDir(dir);
+  if (!existsSync(outputDir)) mkdirSync(outputDir, { recursive: true });
+  const indexPath = join(outputDir, "doc-terms-index.yaml");
+
+  const yaml = (await import("js-yaml")).default;
+  const header = [
+    "# Documentation Term Index — reverse lookup for documentation updates",
+    "# Generated by: wwa update-docs --scan",
+    "# Used by: wwa update-docs --query <term>",
+    "#",
+    "# Schema: categories → terms → referenced_in (files) → usages (line + context)",
+    "# When a value changes, this index tells you exactly which files need updating.",
+    "#",
+    `# Last generated: ${index.generated}`,
+    "",
+  ].join("\n");
+
+  const output = yaml.dump(index, {
+    lineWidth: 120,
+    noRefs: true,
+    quotingType: '"',
+    sortKeys: false,
+  });
+  safeWriteFile(indexPath, header + output, "utf-8");
+
+  if (opts.json) {
+    console.log(JSON.stringify(index, null, 2));
+  } else {
+    console.log(`  Generated: ${relative(dir, indexPath)}`);
+    console.log(`  Files scanned: ${files.length}`);
+    console.log(`  Terms indexed: ${totalTerms}`);
+    console.log(`  Categories: ${Object.keys(index.categories).length}\n`);
+    for (const [key, cat] of Object.entries(index.categories)) {
+      if (cat.metrics) {
+        console.log(`    ${key}: ${Object.keys(cat.metrics).length} metrics — ${cat.description}`);
+      } else {
+        console.log(`    ${key}: ${cat.term_count} terms — ${cat.description}`);
+      }
+    }
+    console.log("");
+  }
+}
+
+// ---------------------------------------------------------------------------
+// Shared helper: add a usage to the terms map
+// ---------------------------------------------------------------------------
+
+function addUsage(terms, termKey, filePath, lineNum, context) {
+  if (!terms[termKey]) {
+    terms[termKey] = { referenced_in: [] };
+  }
+  const existing = terms[termKey].referenced_in.find((f) => f.file === filePath);
+  if (existing) {
+    existing.usages.push({ line: lineNum, context });
+    existing.occurrences++;
+  } else {
+    terms[termKey].referenced_in.push({
+      file: filePath,
+      occurrences: 1,
+      usages: [{ line: lineNum, context }],
+    });
+  }
+}
+
+// ---------------------------------------------------------------------------
+// Build counts category with metrics-grouped structure
+// ---------------------------------------------------------------------------
+
+function buildCountsCategory(catDef, files, dir, tokens) {
+  // Collect raw matches grouped by token key
+  const metricsRaw = {}; // token_key → { value → [{ file, line, context }] }
+
+  for (const filePath of files) {
+    const content = readFileSync(join(dir, filePath), "utf-8");
+    const lines = content.split("\n");
+
+    for (const matcher of catDef.matchers) {
+      for (let i = 0; i < lines.length; i++) {
+        const re = new RegExp(matcher.pattern.source, matcher.pattern.flags);
+        let m;
+        while ((m = re.exec(lines[i])) !== null) {
+          const value = parseInt(m[1], 10);
+          if (!metricsRaw[matcher.token]) metricsRaw[matcher.token] = {};
+          if (!metricsRaw[matcher.token][value]) metricsRaw[matcher.token][value] = [];
+          metricsRaw[matcher.token][value].push({
+            file: filePath,
+            line: i + 1,
+            context: snippet(lines[i]),
+          });
+        }
+      }
+    }
+  }
+
+  // Build structured metrics
+  const metrics = {};
+  for (const tokenKey of Object.keys(metricsRaw).sort()) {
+    const expected = tokens[tokenKey] ?? null;
+    const valuesFound = [];
+
+    for (const [valueStr, usages] of Object.entries(metricsRaw[tokenKey])) {
+      const value = parseInt(valueStr, 10);
+      // Group usages by file
+      const byFile = {};
+      for (const u of usages) {
+        if (!byFile[u.file]) byFile[u.file] = { file: u.file, occurrences: 0, usages: [] };
+        byFile[u.file].usages.push({ line: u.line, context: u.context });
+        byFile[u.file].occurrences++;
+      }
+
+      valuesFound.push({
+        value,
+        status: expected !== null && value === expected ? "current" : expected !== null ? "stale" : "unverified",
+        total_references: usages.length,
+        referenced_in: Object.values(byFile).sort((a, b) => b.occurrences - a.occurrences),
+      });
+    }
+
+    // Sort: current values first, then by value descending
+    valuesFound.sort((a, b) => {
+      if (a.status === "current" && b.status !== "current") return -1;
+      if (a.status !== "current" && b.status === "current") return 1;
+      return b.value - a.value;
+    });
+
+    metrics[tokenKey] = { expected, values_found: valuesFound };
+  }
+
+  return {
+    description: catDef.description,
+    term_count: Object.keys(metrics).length,
+    metrics,
+  };
+}
+
+// ---------------------------------------------------------------------------
+// Enrichment: add canonical_name from doc-tokens.yaml names
+// ---------------------------------------------------------------------------
+
+function enrichFromNames(terms, nameEntries) {
+  for (const termKey of Object.keys(terms)) {
+    const entry = nameEntries.find((n) => n.id === termKey);
+    if (entry) {
+      terms[termKey].canonical_name = entry.display || entry.canonical;
+    }
+  }
+}
+
+function enrichFeatureIds(terms, featureDomains) {
+  for (const termKey of Object.keys(terms)) {
+    const prefix = termKey.replace(/-\d{2}$/, "");
+    const domain = featureDomains.find((d) => d.id === prefix);
+    if (domain) {
+      terms[termKey].canonical_name = `${domain.canonical} (${domain.feature_range})`;
+    }
+  }
+}
+
+// ---------------------------------------------------------------------------
+// Query the index for a specific term
+// ---------------------------------------------------------------------------
+
+async function queryIndex(index, query, opts) {
+  const results = [];
+  const lowerQuery = query.toLowerCase();
+
+  for (const [catKey, cat] of Object.entries(index.categories || {})) {
+    if (opts.category && catKey !== opts.category) continue;
+
+    if (cat.metrics) {
+      // Counts category — search metric keys
+      for (const [metricKey, metricData] of Object.entries(cat.metrics)) {
+        if (metricKey.toLowerCase().includes(lowerQuery) || lowerQuery.includes(metricKey.toLowerCase())) {
+          results.push({ category: catKey, term: metricKey, metric: true, ...metricData });
+        }
+      }
+      continue;
+    }
+
+    for (const [termKey, termData] of Object.entries(cat.terms || {})) {
+      if (
+        termKey.toLowerCase().includes(lowerQuery) ||
+        lowerQuery.includes(termKey.toLowerCase())
+      ) {
+        results.push({ category: catKey, term: termKey, ...termData });
+      }
+    }
+  }
+
+  if (opts.json) {
+    console.log(JSON.stringify({ query, results }, null, 2));
+    return;
+  }
+
+  if (results.length === 0) {
+    console.log(`\n  No matches for "${query}" in the term index.`);
+    console.log("  Try `wwa update-docs --scan` to rebuild, or broaden your query.\n");
+    return;
+  }
+
+  console.log(`\n  Results for "${query}" (${results.length} match${results.length > 1 ? "es" : ""}):\n`);
+  for (const r of results) {
+    if (r.metric) {
+      // Counts metric display
+      console.log(`  [${r.category}] ${r.term} (expected: ${r.expected ?? "unknown"})`);
+      for (const v of r.values_found) {
+        const statusTag = v.status === "current" ? "current" : v.status === "stale" ? "STALE" : "unverified";
+        console.log(`    value ${v.value} [${statusTag}] — ${v.total_references} reference${v.total_references > 1 ? "s" : ""}`);
+        for (const f of v.referenced_in) {
+          console.log(`      ${f.file} (${f.occurrences}x)`);
+          printUsageSnippets(f.usages, "        ");
+        }
+      }
+    } else {
+      // Standard term display
+      const nameStr = r.canonical_name ? ` — ${r.canonical_name}` : "";
+      console.log(`  [${r.category}] ${r.term}${nameStr} (${r.total_references} reference${r.total_references > 1 ? "s" : ""})`);
+      for (const f of r.referenced_in) {
+        console.log(`    ${f.file} (${f.occurrences}x)`);
+        printUsageSnippets(f.usages, "      ");
+      }
+    }
+  }
+  console.log("");
+}
+
+function printUsageSnippets(usages, indent, max = 3) {
+  const show = usages.slice(0, max);
+  for (const u of show) {
+    console.log(`${indent}L${u.line}: ${u.context}`);
+  }
+  if (usages.length > max) {
+    console.log(`${indent}... (+${usages.length - max} more)`);
+  }
+}
+
+// ---------------------------------------------------------------------------
+// Show stale terms — where token value doesn't match docs content
+// ---------------------------------------------------------------------------
+
+async function showStale(dir, index, opts) {
+  const countsCategory = index.categories?.counts;
+  if (!countsCategory || !countsCategory.metrics) {
+    console.log("\n  No counts metrics in index. Run `wwa update-docs --scan` first.\n");
+    return;
+  }
+
+  const stale = [];
+  for (const [metricKey, metricData] of Object.entries(countsCategory.metrics)) {
+    for (const v of metricData.values_found) {
+      if (v.status === "stale") {
+        stale.push({
+          metric: metricKey,
+          expected: metricData.expected,
+          found: v.value,
+          total_references: v.total_references,
+          referenced_in: v.referenced_in,
+        });
+      }
+    }
+  }
+
+  if (opts.json) {
+    console.log(JSON.stringify({ stale }, null, 2));
+    return;
+  }
+
+  if (stale.length === 0) {
+    console.log("\n  No stale count values detected. All docs match doc-tokens.yaml.\n");
+    return;
+  }
+
+  console.log(`\n  Stale values (${stale.length}):\n`);
+  for (const s of stale) {
+    console.log(`  ${s.metric}: expected ${s.expected}, found ${s.found} (${s.total_references} reference${s.total_references > 1 ? "s" : ""})`);
+    for (const f of s.referenced_in) {
+      console.log(`    ${f.file} (${f.occurrences}x)`);
+      printUsageSnippets(f.usages, "      ");
+    }
+  }
+  console.log("");
+}
+
+// ---------------------------------------------------------------------------
+// Print index summary
+// ---------------------------------------------------------------------------
+
+function printIndexSummary(index, opts) {
+  if (opts.json) {
+    console.log(JSON.stringify(index, null, 2));
+    return;
+  }
+
+  console.log("\n  Documentation Term Index\n");
+  console.log(`  Generated: ${index.generated}`);
+  console.log(`  Source: ${index.source_directory}`);
+  console.log(`  Files scanned: ${index.files_scanned}`);
+  console.log(`  Total terms: ${index.total_terms}\n`);
+
+  for (const [key, cat] of Object.entries(index.categories || {})) {
+    if (cat.metrics) {
+      // Counts category
+      const topMetrics = Object.entries(cat.metrics)
+        .sort((a, b) => {
+          const aRefs = a[1].values_found.reduce((s, v) => s + v.total_references, 0);
+          const bRefs = b[1].values_found.reduce((s, v) => s + v.total_references, 0);
+          return bRefs - aRefs;
+        })
+        .slice(0, 3)
+        .map(([t]) => t);
+      console.log(`  [${key}] ${Object.keys(cat.metrics).length} metrics — ${cat.description}`);
+      console.log(`    Top: ${topMetrics.join(", ")}`);
+    } else {
+      const topTerms = Object.entries(cat.terms || {})
+        .sort((a, b) => (b[1].total_references || 0) - (a[1].total_references || 0))
+        .slice(0, 3)
+        .map(([t, d]) => {
+          const name = d.canonical_name ? ` (${d.canonical_name})` : "";
+          return `${t}${name}`;
+        });
+      console.log(`  [${key}] ${cat.term_count} terms — ${cat.description}`);
+      console.log(`    Top: ${topTerms.join(", ")}`);
+    }
+  }
+  console.log("");
+}
+
+// ---------------------------------------------------------------------------
+// Helpers
+// ---------------------------------------------------------------------------
+
+function collectMdFiles(absDir, rootDir, files) {
+  for (const entry of readdirSync(absDir, { withFileTypes: true })) {
+    const fullPath = join(absDir, entry.name);
+    if (entry.isDirectory()) {
+      if (!["node_modules", ".git", "dist", "build"].includes(entry.name)) {
+        collectMdFiles(fullPath, rootDir, files);
+      }
+    } else if (entry.name.endsWith(".md") || entry.name.endsWith(".yaml") || entry.name.endsWith(".yml")) {
+      files.push(relative(rootDir, fullPath).replace(/\\/g, "/"));
+    }
+  }
+}