@thanbir-tamim/enterprise-project-intelligence-agent 0.1.1 → 0.2.0

package/skills/presentation-compliance-validator/scripts/validate-deck.mjs

@@ -9,17 +9,46 @@ import { pathToFileURL } from "node:url";
  *                 citationsPresent, notes } ], sections: [string] }
  */
 
+export const RULE_IDS = ["required-sections", "require-title", "max-bullets", "require-footer", "citation-on-numeric-claim"];
+const KNOWN_KEYS = new Set(["name", "requiredSections", "maxBulletsPerSlide", "requireTitleOnEverySlide", "requireFooterOrPageNumber", "requireCitationOnNumericClaims", "palette", "typography", "blockingRuleIds", "humanReviewOnly", "notes", "source", "derivedRules", "optionalRecommendations"]);
+
+/** Validates a rule set against references/rule-schema.md. Returns a list of schema errors (empty = valid). */
+export function validateRules(rules) {
+  const errors = [];
+  if (!rules || typeof rules !== "object" || Array.isArray(rules)) return ["rule set must be a JSON object"];
+  for (const k of Object.keys(rules)) if (!KNOWN_KEYS.has(k)) errors.push(`unknown key "${k}"`);
+  const isStrArray = (v) => Array.isArray(v) && v.every((x) => typeof x === "string" && x.trim());
+  if ("requiredSections" in rules && !isStrArray(rules.requiredSections)) errors.push("requiredSections must be an array of non-empty strings");
+  if ("maxBulletsPerSlide" in rules && !(Number.isInteger(rules.maxBulletsPerSlide) && rules.maxBulletsPerSlide > 0)) errors.push("maxBulletsPerSlide must be a positive integer");
+  for (const k of ["requireTitleOnEverySlide", "requireFooterOrPageNumber", "requireCitationOnNumericClaims"]) if (k in rules && typeof rules[k] !== "boolean") errors.push(`${k} must be true or false`);
+  for (const k of ["palette", "typography"]) if (k in rules && (typeof rules[k] !== "object" || Array.isArray(rules[k]) || Object.values(rules[k]).some((v) => typeof v !== "string"))) errors.push(`${k} must be an object of string values`);
+  if (rules.palette) for (const [k, v] of Object.entries(rules.palette)) if (typeof v === "string" && !/^#[0-9A-Fa-f]{6}$/.test(v)) errors.push(`palette.${k} must be a #RRGGBB colour`);
+  if ("blockingRuleIds" in rules) {
+    if (!isStrArray(rules.blockingRuleIds)) errors.push("blockingRuleIds must be an array of strings");
+    else for (const id of rules.blockingRuleIds) if (!RULE_IDS.includes(id) && !/^(palette|typography)-[A-Za-z]+$/.test(id)) errors.push(`blockingRuleIds contains unknown rule id "${id}" (known: ${RULE_IDS.join(", ")}, palette-<key>, typography-<key>)`);
+  }
+  if ("humanReviewOnly" in rules && !isStrArray(rules.humanReviewOnly)) errors.push("humanReviewOnly must be an array of strings");
+  return errors;
+}
+
 export function validateDeck(deck, rules) {
+  const schemaErrors = validateRules(rules);
+  if (schemaErrors.length) {
+    // SKILL.md: a malformed rule set is reported and validation stops — never guess intent.
+    return { pass: false, schemaErrors, findings: [], humanReview: [], blockingFailureCount: 0, stopped: "invalid rule set" };
+  }
   const findings = [];
   const humanReview = [];
 
+  const deckObj = deck && typeof deck === "object" ? deck : {};
   const blocking = new Set(rules.blockingRuleIds || []);
 
-  // Required sections
+  // Required sections (matched case-insensitively against declared sections and slide titles)
   if (rules.requiredSections) {
-    const sections = new Set(deck.sections || []);
+    const norm = (s) => String(s || "").trim().toLowerCase();
+    const sections = new Set([...(deckObj.sections || []), ...(deckObj.slides || []).map((s) => s?.title)].map(norm));
     for (const s of rules.requiredSections) {
-      if (!sections.has(s)) {
+      if (![...sections].some((x) => x === norm(s) || x.startsWith(norm(s) + ":") || x.startsWith(norm(s) + " —") || x.startsWith(norm(s) + " -"))) {
         findings.push({
           ruleId: "required-sections",
           severity: blocking.has("required-sections") ? "blocking" : "recommendation",
@@ -31,7 +60,8 @@ export function validateDeck(deck, rules) {
     }
   }
 
-  (deck.slides || []).forEach((slide, idx) => {
+  (deckObj.slides || []).forEach((slide, idx) => {
+    slide = slide || {};
     const slideNum = idx + 1;
 
     if (rules.requireTitleOnEverySlide && !slide.title) {
@@ -71,10 +101,10 @@ export function validateDeck(deck, rules) {
   // Palette / typography: only checkable if the deck actually exposes these properties.
   for (const key of ["palette", "typography"]) {
     if (rules[key]) {
-      if (deck[key]) {
+      if (deckObj[key]) {
         for (const [prop, expected] of Object.entries(rules[key])) {
-          const actual = deck[key][prop];
-          if (actual && actual !== expected) {
+          const actual = deckObj[key][prop];
+          if (actual && String(actual).toLowerCase() !== String(expected).toLowerCase()) {
             findings.push({
               ruleId: `${key}-${prop}`, severity: blocking.has(`${key}-${prop}`) ? "blocking" : "recommendation",
               slide: null, finding: `${key}.${prop} is "${actual}", expected "${expected}".`,
@@ -111,10 +141,17 @@ async function main() {
     console.error("Usage: node validate-deck.mjs <deck.json> <rules.json>");
     process.exit(1);
   }
-  const fs = await import("node:fs");
-  const deck = JSON.parse(fs.readFileSync(deckPath, "utf8"));
-  const rules = JSON.parse(fs.readFileSync(rulesPath, "utf8"));
-  console.log(JSON.stringify(validateDeck(deck, rules), null, 2));
+  try {
+    const fs = await import("node:fs");
+    const deck = JSON.parse(fs.readFileSync(deckPath, "utf8"));
+    const rules = JSON.parse(fs.readFileSync(rulesPath, "utf8"));
+    const result = validateDeck(deck, rules);
+    console.log(JSON.stringify(result, null, 2));
+    process.exitCode = result.pass ? 0 : 1; // non-zero on blocking failures or an invalid rule set
+  } catch (err) {
+    console.error(`validate-deck: ${err.message}`);
+    process.exit(1);
+  }
 }
 
 if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) {

package/skills/project-documentation-discovery/SKILL.md

@@ -1,13 +1,6 @@
 ---
 name: project-documentation-discovery
-description: >-
-  Inventory and ingest an arbitrary project's documentation corpus (Markdown, HTML, plain text,
-  DOCX, PPTX, PDF, CSV, JSON, YAML, images/screenshots, source code and config files, plus
-  Keynote/Pages via their exported formats). Use this skill first, whenever a new project corpus is
-  supplied, whenever the corpus changes, or whenever another skill needs a source-evidence index. It
-  builds a document inventory and a source evidence index without loading the whole corpus into the
-  model's context, and it never claims a format is supported unless extraction was actually verified
-  for that file.
+description: "Inventory any project documentation/code corpus and build a citable source-evidence index. Extracts Markdown, plain text, HTML (headings/tables), CSV, JSON, YAML (syntax-checked), and source/config files; catalogues DOCX, PPTX, PDF, Keynote, Pages and images as unsupported until converted, instead of pretending to read them. Redacts secret-like values with file/line records. Use first whenever a new or changed corpus is supplied, or when another skill needs an evidence index; it processes files incrementally and never over-claims extraction."
 metadata:
   version: 1.0.0
   partOf: enterprise-project-intelligence-agent
@@ -16,6 +9,7 @@ metadata:
     - references/extraction-checklist.md
   scripts:
     - scripts/inventory.mjs
+    - scripts/secrets.mjs
 ---
 
 # project-documentation-discovery
@@ -40,6 +34,8 @@ extracted, what could not, and exactly where every extracted fact came from.
 
 - `corpusPath` — path to the folder containing the target project's documentation and/or source code.
   Treat it as read-only. Never modify, move, or delete files inside it.
+- Optional exclusions — directory/file names or globs (e.g. `drafts`, `**/*.min.js`) passed as
+  `--exclude a,b` (taken from the project profile's `excludePaths` when present).
 - Optional `project-profile.yaml` (see `templates/project-profile.example.yaml` in the repo root) that
   names the project, states confidentiality level, and lists any paths to exclude.
 - Optional previous inventory (`inventory.json`) for incremental re-scans.
@@ -76,6 +72,26 @@ extracted, what could not, and exactly where every extracted fact came from.
 8. **Report a summary**: file counts by status, formats seen, formats requested but unsupported,
    and the list of files that need visual/manual inspection.
 
+## Running the reference script
+
+```bash
+node scripts/inventory.mjs <corpusPath> --out ./.local-knowledge/<project-id> [--exclude drafts,**/*.min.js]
+# or, from the repository root: node scripts/cli.mjs inventory <corpusPath> --out <dir> [--exclude ...]
+```
+
+What it guarantees (each item covered by an automated test):
+- Markdown/text/source/JSON: `extracted-full` (JSON must parse, otherwise `extraction-failed`).
+- YAML: checked by a minimal built-in syntax checker — `extracted-partial` when it passes (values are
+  not parsed), `extraction-failed` with line-level reasons when it does not.
+- CSV: RFC-4180 quote-aware parsing; ragged rows downgrade to `extracted-partial`.
+- HTML: always `extracted-partial` (headings/tables only).
+- DOCX/PPTX/PDF/Keynote/Pages/images: `unsupported-format` with a conversion recommendation.
+- Secret-like values (keyed assignments such as `API_KEY=…`, private keys, credentialed URLs,
+  high-entropy tokens) are redacted before indexing in every text file, including config and source
+  files, and recorded as `evidence_type: "redacted-secret"` entries with `source_location: path#L<n>`.
+  Ordinary identifiers (SNAKE_CASE names, hashes, slash-separated words) are not redacted.
+- Outputs contain corpus-relative paths only.
+
 ## Expected outputs
 
 - `inventory.json` — one record per file: `{ path, type, title, status, hash, warnings[] }`.
@@ -128,6 +144,7 @@ extracted, what could not, and exactly where every extracted fact came from.
 
 - `references/format-matrix.md` — per-format support status and extraction method.
 - `references/extraction-checklist.md` — step-by-step checklist used during extraction.
+- `scripts/secrets.mjs` — shared secret-likeness detector (also used by the repository release audit).
 - `scripts/inventory.mjs` — reference implementation: walks a folder and emits `inventory.json`
   for the formats this toolkit can parse with built-in Node.js tooling (Markdown, text, JSON, YAML,
   CSV, HTML headings). Formats requiring external libraries (DOCX/PPTX/PDF binary parsing) are

package/skills/project-documentation-discovery/references/format-matrix.md

@@ -10,10 +10,10 @@ sync with `scripts/inventory.mjs` and the repo-level `docs/SUPPORTED_FORMATS.md`
 | Markdown (`.md`) | Native Node.js text read + heading regex | Verified | Full heading hierarchy. |
 | Plain text (`.txt`) | Native Node.js text read | Verified | No structure beyond paragraphs. |
 | JSON (`.json`) | `JSON.parse` | Verified | Treated as structured data, not prose. |
-| YAML (`.yml`, `.yaml`) | Minimal built-in YAML line scanner | Verified (subset) | Complex anchors/multi-doc YAML should be hand-checked. |
-| CSV (`.csv`) | Native split + quote-aware parser | Verified | First row treated as header. |
-| HTML (`.html`, `.htm`) | Regex-based heading/table extraction | Verified (subset) | Extracts `h1`-`h6`, `table`; does not render CSS/JS-driven content. |
-| Source code / config (`.ts`, `.js`, `.py`, `.java`, `.go`, `.rb`, `.sql`, `.env.example`, etc.) | Treated as text; language-aware parsing is out of scope | Verified (as text) | Used for reverse engineering, not prose extraction. |
+| YAML (`.yml`, `.yaml`) | Minimal built-in syntax checker (indentation, flow brackets, quotes, line shape); top-level keys indexed | Verified (syntax check only) | Passing files are `extracted-partial` (values not parsed); failing files are `extraction-failed` with line numbers. |
+| CSV (`.csv`) | Built-in RFC-4180 quote-aware parser | Verified | First row treated as header; ragged rows → `extracted-partial`. |
+| HTML (`.html`, `.htm`) | Regex-based heading/table detection | Verified (subset) | Always `extracted-partial`: `h1`-`h6` indexed, tables counted; paragraphs and CSS/JS-driven content not captured. |
+| Source code / config (`.ts`, `.js`, `.py`, `.java`, `.go`, `.rb`, `.sql`, `.env*`, `.ini`, `.toml`, etc.) | Treated as text; secret-scanned; one `code` evidence entry per file (`path#L1-Ln`) | Verified (as text) | No language-aware parsing; secrets recorded as `redacted-secret` entries. |
 | Images / screenshots (`.png`, `.jpg`, `.gif`) | Catalogued by filename/metadata only; no OCR in the reference implementation | Unsupported (cataloguing only) | Flag for manual/visual inspection; do not claim text extraction. |
 | DOCX | Not implemented in the bundled reference script | Unsupported (requires optional dependency) | Add a DOCX parser (e.g., an XML-zip reader) behind an explicit opt-in dependency before claiming support. |
 | PPTX | Not implemented in the bundled reference script | Unsupported (requires optional dependency) | Same as DOCX; slide numbers and notes require a PPTX-aware parser. |

package/skills/project-documentation-discovery/scripts/inventory.mjs

@@ -2,66 +2,68 @@
 /**
  * Reference documentation-discovery inventory tool.
  *
- * Walks a corpus directory and produces inventory.json + evidence-index.json using only
- * built-in Node.js capabilities. Formats that genuinely require an external parser (DOCX,
- * PPTX, PDF, Keynote, Pages, images) are honestly reported as "unsupported-format" rather
- * than silently skipped or falsely claimed as extracted.
+ * Walks a corpus directory and produces inventory.json + evidence-index.json + INVENTORY_SUMMARY.md
+ * using only built-in Node.js capabilities. Formats that genuinely require an external parser (DOCX,
+ * PPTX, PDF, Keynote, Pages, images) are honestly reported as "unsupported-format" rather than
+ * silently skipped or falsely claimed as extracted.
  *
  * Usage:
- *   node inventory.mjs <corpusPath> [--out <dir>] [--exclude <glob>,<glob>...]
+ *   node inventory.mjs <corpusPath> [--out <dir>] [--exclude <name-or-glob>[,<name-or-glob>...]]
+ *
+ * --exclude matches a file/directory name or a corpus-relative path; "*" and "**" globs supported.
  */
 import { pathToFileURL } from "node:url";
-import { readFileSync, statSync, readdirSync, writeFileSync, mkdirSync } from "node:fs";
-import { join, relative, extname, basename } from "node:path";
+import { readFileSync, statSync, readdirSync, writeFileSync, mkdirSync, existsSync } from "node:fs";
+import { join, relative, extname, basename, resolve, isAbsolute } from "node:path";
 import { createHash } from "node:crypto";
+import { redactText } from "./secrets.mjs";
 
 const DEFAULT_EXCLUDES = [".git", "node_modules", ".DS_Store", "dist", "dist-tests", ".idea"];
 
 const TEXT_LIKE_SOURCE_EXT = new Set([
   ".ts", ".tsx", ".js", ".jsx", ".mjs", ".cjs", ".py", ".java", ".go", ".rb", ".sql",
-  ".c", ".cpp", ".h", ".hpp", ".cs", ".php", ".sh", ".env", ".ini", ".toml", ".conf",
+  ".c", ".cpp", ".h", ".hpp", ".cs", ".php", ".sh", ".env", ".ini", ".toml", ".conf", ".properties",
 ]);
 
 const UNSUPPORTED_BINARY_EXT = new Set([
   ".docx", ".pptx", ".pdf", ".key", ".pages", ".png", ".jpg", ".jpeg", ".gif", ".bmp", ".webp",
 ]);
 
-function sha256(buf) {
-  return createHash("sha256").update(buf).digest("hex");
-}
+const sha256 = (buf) => createHash("sha256").update(buf).digest("hex");
+// GitHub-compatible heading slugs (each space -> "-", punctuation dropped) so anchors resolve on GitHub too.
+const slug = (text) => text.toLowerCase().trim().replace(/[`*_~]/g, (c) => (c === "_" ? "_" : "")).replace(/[^a-z0-9 _-]/g, "").replace(/ /g, "-").slice(0, 120);
 
-function slug(text) {
-  return text.toLowerCase().trim().replace(/[^a-z0-9\s-]/g, "").replace(/\s+/g, "-").slice(0, 80);
+/* ---------- exclusion ---------- */
+function globToRegExp(glob) {
+  const esc = glob.replace(/[.+^${}()|[\]\\]/g, "\\$&").replace(/\*\*/g, "\u0000").replace(/\*/g, "[^/]*").replace(/\u0000/g, ".*");
+  return new RegExp(`^${esc}$`);
 }
-
-function redactSecrets(text) {
-  return text
-    .replace(/-----BEGIN [A-Z ]*PRIVATE KEY-----[\s\S]*?-----END [A-Z ]*PRIVATE KEY-----/g, "[REDACTED-PRIVATE-KEY]")
-    .replace(/[A-Za-z0-9_\-]{32,}/g, (m) => (/^[A-F0-9]+$/i.test(m) && m.length === 32 ? m : "[REDACTED-TOKEN-LIKE-VALUE]"));
+export function makeExcluder(patterns) {
+  const res = patterns.map((p) => p.replace(/\/+$/, "")).filter(Boolean).map((p) => ({ raw: p, re: globToRegExp(p) }));
+  return (name, relPath) => res.some(({ raw, re }) => name === raw || relPath === raw || re.test(name) || re.test(relPath) || relPath.startsWith(raw + "/"));
 }
 
-function walk(dir, excludes, out = []) {
+function walk(root, dir, isExcluded, out = []) {
   for (const entry of readdirSync(dir)) {
-    if (excludes.includes(entry)) continue;
     const p = join(dir, entry);
-    const st = statSync(p);
-    if (st.isDirectory()) walk(p, excludes, out);
+    const rel = relative(root, p).split("\\").join("/");
+    if (isExcluded(entry, rel)) continue;
+    if (statSync(p).isDirectory()) walk(root, p, isExcluded, out);
     else out.push(p);
   }
   return out;
 }
 
-function extractMarkdownOrText(raw, isMarkdown) {
-  const headings = [];
-  if (isMarkdown) {
-    const lines = raw.split("\n");
-    for (const line of lines) {
-      const m = /^(#{1,6})\s+(.*)$/.exec(line);
-      if (m) headings.push({ level: m[1].length, text: m[2].trim(), slug: slug(m[2]) });
-    }
+/* ---------- extractors ---------- */
+function extractMarkdownHeadings(raw) {
+  const headings = []; let inFence = false;
+  for (const line of raw.split("\n")) {
+    if (/^\s*(```|~~~)/.test(line)) { inFence = !inFence; continue; }
+    if (inFence) continue;
+    const m = /^(#{1,6})\s+(.*?)\s*#*\s*$/.exec(line);
+    if (m) headings.push({ level: m[1].length, text: m[2].trim(), slug: slug(m[2]) });
   }
-  const title = headings[0]?.text ?? null;
-  return { title, headings };
+  return headings;
 }
 
 function extractHtml(raw) {
@@ -77,28 +79,84 @@ function extractHtml(raw) {
   return { title: titleMatch ? titleMatch[1].trim() : headings[0]?.text ?? null, headings, tableCount };
 }
 
-function extractCsv(raw) {
-  const lines = raw.split(/\r?\n/).filter(Boolean);
-  const header = lines[0]?.split(",") ?? [];
-  return { title: null, rowCount: Math.max(lines.length - 1, 0), columns: header };
+/** RFC-4180-style CSV parsing: quoted fields, escaped quotes (""), commas/newlines inside quotes. */
+export function parseCsv(raw) {
+  const rows = []; let row = []; let field = ""; let q = false;
+  for (let i = 0; i < raw.length; i++) {
+    const c = raw[i];
+    if (q) {
+      if (c === '"') { if (raw[i + 1] === '"') { field += '"'; i++; } else q = false; }
+      else field += c;
+    } else if (c === '"') q = true;
+    else if (c === ",") { row.push(field); field = ""; }
+    else if (c === "\n" || c === "\r") { if (c === "\r" && raw[i + 1] === "\n") i++; row.push(field); field = ""; if (row.some((f) => f !== "")) rows.push(row); row = []; }
+    else field += c;
+  }
+  if (field !== "" || row.length) { row.push(field); if (row.some((f) => f !== "")) rows.push(row); }
+  if (q) throw new Error("Unterminated quoted field in CSV");
+  return rows;
+}
+
+/**
+ * Minimal YAML syntax checker (no dependency). Detects common corruption: tab indentation,
+ * unbalanced flow collections ([ ] { }), unterminated quotes, and lines that are neither
+ * mappings, sequence items, comments, document markers, nor block-scalar continuation.
+ * It does NOT parse values, so a passing file is reported as extracted-partial, never full.
+ */
+export function checkYaml(raw) {
+  const errors = []; const topKeys = [];
+  let depth = 0; let blockIndent = -1; let flowStartLine = 0;
+  raw.split("\n").forEach((line, i) => {
+    const n = i + 1;
+    if (/^\t+/.test(line)) errors.push(`line ${n}: tab indentation`);
+    const indent = line.match(/^ */)[0].length;
+    if (blockIndent >= 0) { if (line.trim() === "" || indent > blockIndent) return; blockIndent = -1; }
+    const body = line.replace(/\s+#.*$/, "");
+    if (body.trim() === "" || /^\s*#/.test(line) || /^(---|\.\.\.)\s*$/.test(line)) return;
+    const depthAtStart = depth;
+    let sq = false, dq = false;
+    for (let j = 0; j < body.length; j++) {
+      const c = body[j];
+      if (c === "'" && !dq) sq = !sq; else if (c === '"' && !sq && body[j - 1] !== "\\") dq = !dq;
+      else if (!sq && !dq) { if (c === "[" || c === "{") { if (depth === 0) flowStartLine = n; depth++; } else if (c === "]" || c === "}") depth--; }
+    }
+    if (sq || dq) errors.push(`line ${n}: unterminated quoted string`);
+    if (depth < 0) { errors.push(`line ${n}: unmatched closing bracket`); depth = 0; }
+    if (depthAtStart === 0) { // only lines that start outside a flow collection must be mapping/sequence entries
+      const t = body.trim();
+      const ok = /^-(\s|$)/.test(t) || /^[^:\[{]+:(\s|$)/.test(t) || /^["'][^"']*["']\s*:(\s|$)/.test(t) || /^\? /.test(t) || /^[\[{]/.test(t);
+      if (!ok) errors.push(`line ${n}: not a mapping, sequence item, or comment`);
+    }
+    const km = /^([A-Za-z0-9_.-]+)\s*:/.exec(line); if (km && indent === 0) topKeys.push(km[1]);
+    if (/:\s*[|>][-+]?\s*$/.test(body)) blockIndent = indent;
+  });
+  if (depth > 0) errors.push(`line ${flowStartLine}: unclosed flow collection ([ or {) opened here`);
+  return { errors, topKeys };
 }
 
 function classify(filePath) {
   const ext = extname(filePath).toLowerCase();
+  const base = basename(filePath).toLowerCase();
   if (ext === ".md" || ext === ".markdown") return "markdown";
   if (ext === ".txt") return "text";
   if (ext === ".json") return "json";
   if (ext === ".yml" || ext === ".yaml") return "yaml";
   if (ext === ".csv") return "csv";
   if (ext === ".html" || ext === ".htm") return "html";
-  if (TEXT_LIKE_SOURCE_EXT.has(ext)) return "source-code";
+  if (TEXT_LIKE_SOURCE_EXT.has(ext) || base.startsWith(".env")) return "source-code";
   if (UNSUPPORTED_BINARY_EXT.has(ext)) return "unsupported-binary";
   return "unknown";
 }
 
+/** Records one evidence entry per redacted secret: file + line + rule, never the value. */
+function secretEvidence(relPath, findings) {
+  return findings.map((f) => ({ source_path: relPath, source_location: `${relPath}#L${f.line}`, heading_path: "[REDACTED-SECRET-LIKE-VALUE]", evidence_type: "redacted-secret", rule: f.rule }));
+}
+
 export function runInventory(corpusPath, { excludes = [] } = {}) {
-  const allExcludes = [...DEFAULT_EXCLUDES, ...excludes];
-  const files = walk(corpusPath, allExcludes);
+  if (!existsSync(corpusPath) || !statSync(corpusPath).isDirectory()) throw new Error(`Corpus path not found or not a directory: ${corpusPath}`);
+  const isExcluded = makeExcluder([...DEFAULT_EXCLUDES, ...excludes]);
+  const files = walk(corpusPath, corpusPath, isExcluded).sort();
   const inventory = [];
   const evidenceIndex = [];
 
@@ -106,24 +164,27 @@ export function runInventory(corpusPath, { excludes = [] } = {}) {
     const relPath = relative(corpusPath, absPath).split("\\").join("/");
     const kind = classify(absPath);
     let buf;
-    try {
-      buf = readFileSync(absPath);
-    } catch (err) {
+    try { buf = readFileSync(absPath); } catch (err) {
       inventory.push({ path: relPath, type: kind, title: null, status: "extraction-failed", hash: null, warnings: [String(err.message)] });
       continue;
     }
-    const hash = sha256(buf);
-    const record = { path: relPath, type: kind, title: null, status: "not-yet-processed", hash, warnings: [] };
-
+    const record = { path: relPath, type: kind, title: null, status: "not-yet-processed", hash: sha256(buf), warnings: [] };
+    const textKinds = new Set(["markdown", "text", "json", "yaml", "csv", "html", "source-code"]);
+    let raw = "";
+    if (textKinds.has(kind)) {
+      const r = redactText(buf.toString("utf8"));
+      raw = r.text;
+      if (r.findings.length) {
+        evidenceIndex.push(...secretEvidence(relPath, r.findings));
+        record.warnings.push(`${r.findings.length} secret-like value(s) redacted (lines ${[...new Set(r.findings.map((f) => f.line))].join(", ")}); review and rotate if real.`);
+      }
+    }
     try {
       if (kind === "markdown" || kind === "text") {
-        const raw = redactSecrets(buf.toString("utf8"));
-        const { title, headings } = extractMarkdownOrText(raw, kind === "markdown");
-        record.title = title ?? basename(relPath);
+        const headings = kind === "markdown" ? extractMarkdownHeadings(raw) : [];
+        record.title = headings[0]?.text ?? basename(relPath);
         record.status = "extracted-full";
-        for (const h of headings) {
-          evidenceIndex.push({ source_path: relPath, source_location: `${relPath}#${h.slug}`, heading_path: h.text, evidence_type: "text", level: h.level });
-        }
+        for (const h of headings) evidenceIndex.push({ source_path: relPath, source_location: `${relPath}#${h.slug}`, heading_path: h.text, evidence_type: "text", level: h.level });
       } else if (kind === "json") {
         JSON.parse(buf.toString("utf8"));
         record.title = basename(relPath);
@@ -131,34 +192,38 @@ export function runInventory(corpusPath, { excludes = [] } = {}) {
         evidenceIndex.push({ source_path: relPath, source_location: `${relPath}#root`, heading_path: "(structured data)", evidence_type: "structured-data" });
       } else if (kind === "yaml") {
         record.title = basename(relPath);
-        record.status = "extracted-full";
-        evidenceIndex.push({ source_path: relPath, source_location: `${relPath}#root`, heading_path: "(structured data)", evidence_type: "structured-data" });
+        const { errors, topKeys } = checkYaml(raw);
+        if (errors.length) {
+          record.status = "extraction-failed";
+          record.warnings.push(`YAML syntax problems: ${errors.slice(0, 5).join("; ")}`);
+        } else {
+          record.status = "extracted-partial";
+          record.warnings.push("YAML syntax checked by a minimal built-in checker; values are not parsed (top-level keys only).");
+          evidenceIndex.push({ source_path: relPath, source_location: `${relPath}#root`, heading_path: `(structured data; top-level keys: ${topKeys.join("|") || "none"})`, evidence_type: "structured-data" });
+        }
       } else if (kind === "csv") {
-        const raw = buf.toString("utf8");
-        const { rowCount, columns } = extractCsv(raw);
+        const rows = parseCsv(raw);
+        const columns = rows[0] ?? [];
+        const ragged = rows.slice(1).filter((r) => r.length !== columns.length).length;
         record.title = basename(relPath);
-        record.status = "extracted-full";
-        evidenceIndex.push({ source_path: relPath, source_location: `${relPath}#table`, heading_path: `table (${rowCount} rows, columns: ${columns.join("|")})`, evidence_type: "table" });
+        record.status = ragged ? "extracted-partial" : "extracted-full";
+        if (ragged) record.warnings.push(`${ragged} row(s) have a different column count than the header.`);
+        evidenceIndex.push({ source_path: relPath, source_location: `${relPath}#table`, heading_path: `table (${Math.max(rows.length - 1, 0)} rows, columns: ${columns.join("|")})`, evidence_type: "table", columns });
       } else if (kind === "html") {
-        const raw = redactSecrets(buf.toString("utf8"));
         const { title, headings, tableCount } = extractHtml(raw);
         record.title = title ?? basename(relPath);
-        record.status = headings.length || tableCount ? "extracted-partial" : "extracted-full";
-        if (headings.length === 0 && tableCount === 0) {
-          // no structure found at all; still a successful text pass
-        } else {
-          record.warnings.push("HTML extraction captures headings and tables only; layout, scripts, and embedded visuals are not evaluated.");
-        }
-        for (const h of headings) {
-          evidenceIndex.push({ source_path: relPath, source_location: `${relPath}#${h.slug}`, heading_path: h.text, evidence_type: "text", level: h.level });
-        }
+        record.status = "extracted-partial";
+        record.warnings.push("HTML extraction captures headings and tables only; paragraphs, layout, scripts, and embedded visuals are not evaluated.");
+        if (tableCount) record.warnings.push(`${tableCount} table(s) present; table contents not indexed.`);
+        for (const h of headings) evidenceIndex.push({ source_path: relPath, source_location: `${relPath}#${h.slug}`, heading_path: h.text, evidence_type: "text", level: h.level });
       } else if (kind === "source-code") {
         record.title = basename(relPath);
         record.status = "extracted-full";
         record.warnings.push("Treated as raw text for reverse-engineering purposes; no language-aware parsing performed.");
+        evidenceIndex.push({ source_path: relPath, source_location: `${relPath}#L1-L${raw.split("\n").length}`, heading_path: "(source file)", evidence_type: "code" });
       } else if (kind === "unsupported-binary") {
         record.status = "unsupported-format";
-        record.warnings.push(`Format ${extname(relPath)} requires an external parser or manual export (see references/format-matrix.md). Catalogued by filename only.`);
+        record.warnings.push(`Format ${extname(relPath)} requires an external parser or manual export (see references/format-matrix.md). Catalogued by filename only; flag for manual/visual review.`);
       } else {
         record.status = "unsupported-format";
         record.warnings.push("Unknown/unclassified file type; catalogued by filename only.");
@@ -167,47 +232,56 @@ export function runInventory(corpusPath, { excludes = [] } = {}) {
       record.status = "extraction-failed";
       record.warnings.push(String(err.message));
     }
-
     inventory.push(record);
   }
-
   return { inventory, evidenceIndex };
 }
 
-function main() {
-  const [corpusPath, ...rest] = process.argv.slice(2);
-  if (!corpusPath) {
-    console.error("Usage: node inventory.mjs <corpusPath> [--out <dir>]");
-    process.exit(1);
-  }
-  let outDir = "./.local-knowledge";
-  for (let i = 0; i < rest.length; i++) {
-    if (rest[i] === "--out") outDir = rest[++i];
-  }
-  const { inventory, evidenceIndex } = runInventory(corpusPath);
+/** Writes the three standard outputs. Shared by this script's CLI and scripts/cli.mjs. */
+export function writeInventoryOutputs(outDir, corpusPath, { inventory, evidenceIndex }) {
   mkdirSync(outDir, { recursive: true });
   writeFileSync(join(outDir, "inventory.json"), JSON.stringify(inventory, null, 2));
   writeFileSync(join(outDir, "evidence-index.json"), JSON.stringify(evidenceIndex, null, 2));
-
-  const byStatus = {};
-  for (const r of inventory) byStatus[r.status] = (byStatus[r.status] || 0) + 1;
-  const summaryLines = [
-    "# Inventory summary",
-    "",
-    `Corpus: \`${corpusPath}\``,
-    `Total files: ${inventory.length}`,
-    "",
-    "## By status",
-    ...Object.entries(byStatus).map(([k, v]) => `- ${k}: ${v}`),
-    "",
-    "## Files needing manual/visual review",
-    ...inventory.filter((r) => r.status === "unsupported-format" || r.status === "extraction-failed").map((r) => `- ${r.path} (${r.status})`),
+  const byStatus = {}; const byType = {};
+  for (const r of inventory) { byStatus[r.status] = (byStatus[r.status] || 0) + 1; byType[r.type] = (byType[r.type] || 0) + 1; }
+  const abs = resolve(corpusPath); const relToCwd = relative(process.cwd(), abs);
+  const corpusLabel = !relToCwd.startsWith("..") && !isAbsolute(relToCwd) ? relToCwd || "." : basename(abs); // never write absolute paths
+  const review = inventory.filter((r) => ["unsupported-format", "extraction-failed", "extracted-partial"].includes(r.status));
+  const secrets = evidenceIndex.filter((e) => e.evidence_type === "redacted-secret");
+  const lines = [
+    "# Inventory summary", "", `Corpus: \`${corpusLabel}\``, `Total files: ${inventory.length}`, "",
+    "## By status", ...Object.entries(byStatus).map(([k, v]) => `- ${k}: ${v}`), "",
+    "## By type", ...Object.entries(byType).map(([k, v]) => `- ${k}: ${v}`), "",
+    "## Files needing manual/visual review", ...(review.length ? review.map((r) => `- ${r.path} (${r.status})${r.warnings[0] ? ` — ${r.warnings[0]}` : ""}`) : ["- none"]), "",
+    "## Redacted secret-like values", ...(secrets.length ? secrets.map((s) => `- ${s.source_location} (${s.rule})`) : ["- none"]), "",
   ];
-  writeFileSync(join(outDir, "INVENTORY_SUMMARY.md"), summaryLines.join("\n") + "\n");
-  console.log(`Wrote inventory for ${inventory.length} files to ${outDir}`);
-  console.log(JSON.stringify(byStatus, null, 2));
+  writeFileSync(join(outDir, "INVENTORY_SUMMARY.md"), lines.join("\n"));
+  return { byStatus, files: ["inventory.json", "evidence-index.json", "INVENTORY_SUMMARY.md"] };
+}
+
+export function parseArgs(argv) {
+  const opts = { corpusPath: null, outDir: "./.local-knowledge", excludes: [] };
+  for (let i = 0; i < argv.length; i++) {
+    const a = argv[i];
+    if (a === "--out") opts.outDir = argv[++i];
+    else if (a === "--exclude") opts.excludes.push(...String(argv[++i] || "").split(",").map((s) => s.trim()).filter(Boolean));
+    else if (!opts.corpusPath) opts.corpusPath = a;
+  }
+  return opts;
 }
 
-if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) {
-  main();
+function main() {
+  const opts = parseArgs(process.argv.slice(2));
+  if (!opts.corpusPath) { console.error("Usage: node inventory.mjs <corpusPath> [--out <dir>] [--exclude <name-or-glob>,...]"); process.exit(1); }
+  try {
+    const result = runInventory(opts.corpusPath, { excludes: opts.excludes });
+    const { byStatus } = writeInventoryOutputs(opts.outDir, opts.corpusPath, result);
+    console.log(`Wrote inventory for ${result.inventory.length} files to ${opts.outDir}`);
+    console.log(JSON.stringify(byStatus, null, 2));
+  } catch (err) {
+    console.error(`inventory: ${err.message}`);
+    process.exit(1);
+  }
 }
+
+if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) main();