@descix/cli 1.0.1 → 1.0.3

package/lib/core/CorpusDenyLint.js

@@ -0,0 +1,326 @@
+/**
+ * CorpusDenyLint — the single canonical deny-class lint for Tier-P KB publishing.
+ *
+ * Ratified: CEO-D-2026-07-11-KB-CURATION-RATIFIED.
+ * Spec: docs/audit/kb-curation-2026-07-11/redteam-AUDIT.md §"PROPOSED DENY-LINT
+ *       PATTERN SET" (15 path patterns + 4 content patterns + lint_exempt) and
+ *       docs/design/kb-publishing-model-2026-07-11.md §4 (doc_class taxonomy).
+ *
+ * This module is the ONE definition of:
+ *   - the publishable/deny doc_class sets (imported by ManifestLoader — do NOT re-list),
+ *   - the deny path/content pattern set,
+ *   - the lint that a Tier-P (`publish_tier:"P"`) manifest MUST pass before any
+ *     walk/push (corpus.js runCorpusSync) and, via the git hook, before commit.
+ *
+ * The lint itself never throws — it returns a report. The CALLER decides to throw
+ * (assertNoViolations) so exemptions + warnings can be printed as the review
+ * artifact first (model §5.3). No hardcoded fallbacks; the manifest is the
+ * curation instrument.
+ */
+
+import * as fs from 'fs/promises';
+import * as path from 'path';
+import { createHash } from 'crypto';
+import { minimatch } from 'minimatch';
+
+/**
+ * Doc-class taxonomy (model §4). ONE canonical definition — ManifestLoader
+ * imports these; nothing re-enumerates the field list by hand.
+ */
+export const PUBLISHABLE_DOC_CLASSES = new Set([
+  'guide', 'primary-source', 'paper', 'proof', 'code', 'debate-record', 'patent'
+]);
+export const DENY_DOC_CLASSES = new Set([
+  'manuscript-draft', 'archive', 'internal-note'
+]);
+
+/**
+ * PATH patterns (hard deny — sync fails loud). redteam-AUDIT §"Path patterns".
+ * Each entry: { id, class, patterns:[glob...], caseFold?, excludeGlobs?, zeroByte?, reason }.
+ * `class` is the stable string a manifest source's `lint_exempt` names to exempt a match.
+ */
+export const DENY_PATH_PATTERNS = [
+  { id: 1,  class: 'archive',            patterns: ['**/archive/**', '**/Archive/**'], reason: 'archive/graveyard directory — never a public surface' },
+  { id: 2,  class: 'lean-dev',           patterns: ['**/Complexity/Dev/**'], reason: 'Lean non-load-bearing dev tree (explicitly non-load-bearing)' },
+  { id: 3,  class: 'draft-filename',     patterns: ['*DRAFT*', '*Draft*'], caseFold: true, reason: 'draft in path/filename' },
+  { id: 4,  class: 'nature-draft',       patterns: ['**/Nature_PeqNP/**'], reason: 'Nature_PeqNP draft lineage (v1/v2/v3/root) — deny the whole tree' },
+  { id: 5,  class: 'jsonl-working-log',  patterns: ['*.jsonl'], reason: 'working log format (.jsonl) — exempt with lint_exempt:["jsonl-working-log"] when public-record-by-intent' },
+  { id: 6,  class: 'dot-claude',         patterns: ['**/.claude/**'], reason: 'agent/internal .claude tree' },
+  { id: 7,  class: 'private',            patterns: ['**/PRIVATE/**'], reason: 'PRIVATE directory' },
+  { id: 8,  class: 'handoff',            patterns: ['**/HANDOFF*', '**/handoff*'], caseFold: true, reason: 'internal handoff document' },
+  { id: 9,  class: 'test-dump',          patterns: ['**/*_TEST_RESULTS*', '**/test/**', '**/tests/**', '**/TheoremTests/**', '**/benchmarks/**'], excludeGlobs: ['*_README.md'], reason: 'raw test/benchmark suite dump or bloat (named *_README.md summaries excluded)' },
+  { id: 10, class: 'internal-pm',        patterns: ['**/session_transcript/**', '**/debug-*/**', '**/tasks/**'], reason: 'internal PM/debug/session material' },
+  { id: 11, class: 'status-file',        patterns: ['STATUS.md'], reason: 'internal workstream-status file (STATUS.md) — never a public surface' },
+  { id: 12, class: 'investor-material',  patterns: ['*Business_Plan*', '*Executive_Summary*', '*Market_Research*', '**/Faster Abadir Transform BP/**'], reason: 'investor/fundraising material — not public KB content' },
+  { id: 13, class: 'patent-application', patterns: ['*Patent*Application*', '*Patent_Application*'], reason: 'patent application (unpublished filing)' },
+  { id: 14, class: 'wip',                patterns: ['**/PM/wip/**', '**/wip/**'], reason: 'work-in-progress notes carryover' },
+  { id: 15, class: 'zero-byte',          patterns: [], zeroByte: true, reason: 'zero-byte file (produces no content)' },
+];
+
+/**
+ * CONTENT patterns (soft-block / warn). redteam-AUDIT §"Content patterns".
+ * Each entry: { id, class, kind, ... , reason }.
+ *   kind 'header'  — first-10-lines regex; requires synced_from_edit (raw_path).
+ *   kind 'body'    — body regex(es); hard deny unless exempted.
+ *   kind 'warn'    — body regex; WARN only (collect, do not fail).
+ *   kind 'license' — primary-source doc_class with empty/missing license_basis; hard deny.
+ */
+export const DENY_CONTENT_PATTERNS = [
+  {
+    id: 16, class: 'debate-context-header', kind: 'header',
+    headerRegex: /@\[debate_context\]|@\[debate_system\]/,
+    reason: '@[debate_context]/@[debate_system] header requires a reviewer-edited synced copy (set raw_path / synced_from_edit)'
+  },
+  {
+    id: 17, class: 'active-draft-marker', kind: 'body',
+    bodyRegexes: [/status:\s*"?draft/i, /ACTIVE DRAFTING/, /Project Status:\s*ACTIVE/],
+    reason: 'active-draft marker in body (status: draft / ACTIVE DRAFTING / Project Status: ACTIVE)'
+  },
+  {
+    id: 18, class: 'mailto-in-doc', kind: 'warn',
+    bodyRegex: /mailto:[\w.+-]+@[\w.-]+/,
+    reason: 'mailto: link embedded in a non-PDF paper/guide doc — reviewer accepts or strips'
+  },
+  {
+    id: 19, class: 'missing-license-basis', kind: 'license',
+    reason: 'primary-source doc_class with empty/missing license_basis'
+  },
+];
+
+// ─────────────────────────────────────────────────────────────────────────────
+// Matching helpers
+// ─────────────────────────────────────────────────────────────────────────────
+
+/**
+ * True if `relPath` matches `glob` as a full path, as its basename, or as any
+ * single path segment (so segment-anywhere globs like `*DRAFT*` catch a middle
+ * directory such as `ReimannHypothesis Rota Draft/`).
+ */
+function pathMatchesGlob(relPath, glob, opts) {
+  if (minimatch(relPath, glob, opts)) return true;
+  const base = path.basename(relPath);
+  if (minimatch(base, glob, opts)) return true;
+  for (const seg of relPath.split('/')) {
+    if (seg && minimatch(seg, glob, opts)) return true;
+  }
+  return false;
+}
+
+/**
+ * Test a resolved source path against a single DENY_PATH_PATTERN.
+ * Returns the matched glob string, or `false` (no match), or `null` (explicitly
+ * excluded, e.g. a `*_README.md` summary inside a tests/ tree).
+ */
+export function matchPathPattern(relPath, pattern) {
+  if (pattern.zeroByte) return false; // stat-based, handled by the lint, not a glob
+  if (pattern.excludeGlobs) {
+    const base = path.basename(relPath);
+    for (const ex of pattern.excludeGlobs) {
+      if (minimatch(base, ex, { dot: true, nocase: true })) return null;
+    }
+  }
+  const opts = { dot: true, nocase: !!pattern.caseFold };
+  for (const g of pattern.patterns) {
+    if (pathMatchesGlob(relPath, g, opts)) return g;
+  }
+  return false;
+}
+
+// ─────────────────────────────────────────────────────────────────────────────
+// Report record builders (each carries a pre-rendered `line` for printing)
+// ─────────────────────────────────────────────────────────────────────────────
+
+function mkViolation(p, id, matched, reason) {
+  const pat = typeof id === 'number' ? `#${id}` : id;
+  const matchStr = matched ? ` [${matched}]` : '';
+  return {
+    path: p,
+    pattern: pat,
+    reason,
+    line: `VIOLATION: ${p} — pattern ${pat}: ${reason}${matchStr}`
+  };
+}
+
+function mkExemption(p, patternId, cls, matched, exemptReason) {
+  const pat = `#${patternId}`;
+  const why = exemptReason || '(no exempt_reason given)';
+  return {
+    path: p,
+    pattern: pat,
+    class: cls,
+    reason: why,
+    line: `EXEMPTED: ${p} matched pattern ${pat} ${matched} — exempted for "${cls}": ${why}`
+  };
+}
+
+function mkWarning(p, id, matched, reason) {
+  const pat = `#${id}`;
+  return {
+    path: p,
+    pattern: pat,
+    reason,
+    line: `WARNING: ${p} — pattern ${pat}: ${reason} ${matched}`
+  };
+}
+
+// ─────────────────────────────────────────────────────────────────────────────
+// The lint
+// ─────────────────────────────────────────────────────────────────────────────
+
+/**
+ * Lint a resolved manifest for deny-class violations. Never throws — returns a
+ * report. The caller prints exemptions/warnings then calls assertNoViolations.
+ *
+ * A source may point at a file or a directory. Tier-P manifests are file-level
+ * allowlists (the common case): for a file source every check runs. For a dir
+ * source, only the source-entry-level checks run (path pattern on `source.path`,
+ * doc_class deny re-check, license #19, and the raw_path byte-hash) — content
+ * reads are skipped since a directory cannot be read as a document.
+ *
+ * @param {Object} manifest - Validated manifest with `_resolvedSources`.
+ * @param {string} workspaceRoot - Absolute workspace root (resolves raw_path).
+ * @returns {Promise<{violations:Array, exemptions:Array, warnings:Array}>}
+ */
+export async function lintManifestForDenyClasses(manifest, workspaceRoot) {
+  const report = { violations: [], exemptions: [], warnings: [] };
+  const sources = manifest?._resolvedSources || [];
+
+  for (const source of sources) {
+    const relPath = source.path;
+    const absPath = source.absolutePath || path.resolve(workspaceRoot, relPath);
+    const exemptSet = new Set(source.lint_exempt || []);
+    const exemptReason = source.exempt_reason || null;
+
+    let stat = null;
+    try { stat = await fs.stat(absPath); } catch { stat = null; }
+    const isFileSource = !!(stat && stat.isFile());
+
+    // (a) PATH patterns — evaluated on source.path (works for file, dir, missing).
+    for (const pattern of DENY_PATH_PATTERNS) {
+      const matched = matchPathPattern(relPath, pattern);
+      if (matched) {
+        if (exemptSet.has(pattern.class)) {
+          report.exemptions.push(mkExemption(relPath, pattern.id, pattern.class, matched, exemptReason));
+        } else {
+          report.violations.push(mkViolation(relPath, pattern.id, matched, pattern.reason));
+        }
+      }
+    }
+
+    // (#15) zero-byte — stat-based, only meaningful for a real file.
+    if (isFileSource && stat.size === 0) {
+      const zb = DENY_PATH_PATTERNS.find(p => p.class === 'zero-byte');
+      if (exemptSet.has('zero-byte')) {
+        report.exemptions.push(mkExemption(relPath, zb.id, zb.class, '(zero-byte file)', exemptReason));
+      } else {
+        report.violations.push(mkViolation(relPath, zb.id, '(zero-byte file)', zb.reason));
+      }
+    }
+
+    // (f) doc_class deny-class re-check (defense-in-depth; ManifestLoader also blocks).
+    if (source.doc_class && DENY_DOC_CLASSES.has(source.doc_class)) {
+      report.violations.push(mkViolation(
+        relPath, 'doc_class', source.doc_class,
+        `doc_class "${source.doc_class}" is a deny-class — structurally invalid in a Tier-P manifest`
+      ));
+    }
+
+    // (g) content #19 — primary-source must carry a license_basis (source-level).
+    if (source.doc_class === 'primary-source' && !source.license_basis) {
+      report.violations.push(mkViolation(
+        relPath, 19, 'primary-source',
+        DENY_CONTENT_PATTERNS.find(p => p.id === 19).reason
+      ));
+    }
+
+    // (c) synced_from_edit byte-hash — whenever raw_path is set (file OR dir source).
+    if (source.raw_path) {
+      const rawAbs = path.resolve(workspaceRoot, source.raw_path);
+      let rawBytes = null;
+      try { rawBytes = await fs.readFile(rawAbs); } catch { rawBytes = null; }
+      if (rawBytes === null) {
+        report.violations.push(mkViolation(
+          relPath, 'synced_from_edit', source.raw_path,
+          `raw_path "${source.raw_path}" is missing on disk`
+        ));
+      } else if (isFileSource) {
+        let editBytes = null;
+        try { editBytes = await fs.readFile(absPath); } catch { editBytes = null; }
+        if (editBytes !== null) {
+          const rawHash = createHash('sha256').update(rawBytes).digest('hex');
+          const editHash = createHash('sha256').update(editBytes).digest('hex');
+          if (rawHash === editHash) {
+            report.violations.push(mkViolation(
+              relPath, 16, source.raw_path,
+              'edited copy byte-identical to raw source — the edit was not applied'
+            ));
+          }
+        }
+      }
+    }
+
+    // Content-body checks require reading the file (file sources only).
+    if (isFileSource) {
+      let content = null;
+      try { content = await fs.readFile(absPath, 'utf-8'); } catch { content = null; }
+      if (content !== null) {
+        // (b) content #16 — debate-context header in first 10 lines requires raw_path.
+        const p16 = DENY_CONTENT_PATTERNS.find(p => p.id === 16);
+        const first10 = content.split('\n').slice(0, 10).join('\n');
+        if (p16.headerRegex.test(first10) && !source.raw_path) {
+          report.violations.push(mkViolation(
+            relPath, 16, '@[debate_context]/@[debate_system]', p16.reason
+          ));
+        }
+
+        // (d) content #17 — active-draft marker in body → hard deny unless exempted.
+        const p17 = DENY_CONTENT_PATTERNS.find(p => p.id === 17);
+        if (p17.bodyRegexes.some(re => re.test(content))) {
+          if (exemptSet.has(p17.class)) {
+            report.exemptions.push(mkExemption(relPath, p17.id, p17.class, '(active-draft marker)', exemptReason));
+          } else {
+            report.violations.push(mkViolation(relPath, p17.id, '(active-draft marker)', p17.reason));
+          }
+        }
+
+        // (e) content #18 — mailto: in a non-PDF paper/guide doc → WARN only.
+        const p18 = DENY_CONTENT_PATTERNS.find(p => p.id === 18);
+        const dc = source.doc_class;
+        const isPaperOrGuide = dc === 'paper' || dc === 'guide';
+        const isPdf = relPath.toLowerCase().endsWith('.pdf');
+        if (isPaperOrGuide && !isPdf && p18.bodyRegex.test(content)) {
+          report.warnings.push(mkWarning(relPath, p18.id, '(mailto)', p18.reason));
+        }
+      }
+    }
+  }
+
+  return report;
+}
+
+/**
+ * Throw a single aggregated error listing every violation, or return quietly if
+ * the report is clean. The caller (corpus.js / git hook) uses this to fail loud
+ * AFTER printing exemptions + warnings.
+ *
+ * @param {{violations:Array}} report
+ */
+export function assertNoViolations(report) {
+  const violations = report?.violations || [];
+  if (violations.length === 0) return;
+  const lines = violations.map(v => `  - ${v.line || `${v.path}: ${v.reason}`}`);
+  throw new Error(
+    `Tier-P deny-class lint FAILED: ${violations.length} violation(s) — refusing to sync/commit.\n` +
+    lines.join('\n')
+  );
+}
+
+export default {
+  PUBLISHABLE_DOC_CLASSES,
+  DENY_DOC_CLASSES,
+  DENY_PATH_PATTERNS,
+  DENY_CONTENT_PATTERNS,
+  matchPathPattern,
+  lintManifestForDenyClasses,
+  assertNoViolations,
+};

package/lib/core/CorpusWalker.js

@@ -0,0 +1,357 @@
+/**
+ * CorpusWalker - Git-aware file walker for corpus sync
+ *
+ * Given a manifest, walks source directories, applies syncignore globs,
+ * and computes git blob SHAs for content-addressed chunk IDs.
+ *
+ * Architecture:
+ * - Files stay in place — no copying or staging
+ * - Git blob SHAs provide versioning (same content = same SHA)
+ * - Untracked/modified files use `git hash-object` for working tree version
+ * - Chunk IDs are content-addressed: {app_id}:{kb_name}:{blob_sha}:{chunk_idx}
+ */
+
+import * as fs from 'fs/promises';
+import * as path from 'path';
+import { execSync } from 'child_process';
+import { minimatch } from 'minimatch';
+// The source-ref default is owned by ManifestLoader — consume it, never re-default.
+import { DEFAULT_SOURCE_REF } from './ManifestLoader.js';
+
+/**
+ * File extensions that are processable for RAG chunking.
+ * Binary files, images, fonts, compiled assets, etc. are excluded.
+ */
+const PROCESSABLE_EXTENSIONS = new Set([
+  '.md', '.txt', '.js', '.mjs', '.cjs', '.ts', '.tsx', '.jsx',
+  '.py', '.rs', '.go', '.sol', '.lean', '.json', '.jsonl',
+  '.yaml', '.yml', '.toml', '.csv', '.tex', '.sh', '.bash'
+]);
+
+/**
+ * Check if a file has a processable text extension.
+ *
+ * @param {string} filePath - File path
+ * @returns {boolean} True if the file can be chunked for RAG
+ */
+function isProcessableFile(filePath) {
+  const ext = path.extname(filePath).toLowerCase();
+  return PROCESSABLE_EXTENSIONS.has(ext);
+}
+
+/**
+ * Check if a file path matches any of the syncignore patterns.
+ *
+ * @param {string} relativePath - Path relative to the source entry root
+ * @param {string[]} syncignore - Array of glob patterns to exclude
+ * @returns {boolean} True if the file should be ignored
+ */
+function isIgnored(relativePath, syncignore) {
+  for (const pattern of syncignore) {
+    if (minimatch(relativePath, pattern, { dot: true })) {
+      return true;
+    }
+    // Also check just the filename for patterns like "*.test.js"
+    const basename = path.basename(relativePath);
+    if (minimatch(basename, pattern, { dot: true })) {
+      return true;
+    }
+  }
+  return false;
+}
+
+/**
+ * Compute the git blob SHA for a file.
+ * First tries `git rev-parse {ref}:{workspace-relative-path}` for tracked files.
+ * Falls back to `git hash-object {file}` for untracked/modified files.
+ *
+ * @param {string} absolutePath - Absolute path to the file
+ * @param {string} workspaceRelativePath - Path relative to workspace root
+ * @param {string} ref - Git ref (branch/tag)
+ * @param {string} workspaceRoot - Workspace root directory
+ * @returns {string} Git blob SHA (40 hex chars)
+ */
+function computeBlobSha(absolutePath, workspaceRelativePath, ref, workspaceRoot) {
+  try {
+    // Try to get the blob SHA from the git tree at the specified ref
+    const sha = execSync(
+      `git rev-parse "${ref}:${workspaceRelativePath}"`,
+      { cwd: workspaceRoot, encoding: 'utf-8', stdio: ['pipe', 'pipe', 'pipe'] }
+    ).trim();
+    return sha;
+  } catch {
+    // File is untracked or path doesn't exist in git tree at that ref
+    // Fall back to hashing the working tree version
+    try {
+      const sha = execSync(
+        `git hash-object "${absolutePath}"`,
+        { cwd: workspaceRoot, encoding: 'utf-8', stdio: ['pipe', 'pipe', 'pipe'] }
+      ).trim();
+      return sha;
+    } catch {
+      // Last resort: return null — caller should skip this file
+      return null;
+    }
+  }
+}
+
+const SHA40 = /^[0-9a-f]{40}$/i;
+
+/**
+ * Resolve ONE source's provenance: which repository, which ref, and the exact commit synced.
+ *
+ * This is the single owner of "where did this content come from". `ref` may be a mutable branch,
+ * so the RESOLVED COMMIT SHA is what sync-state records — provenance stays exact after the branch
+ * moves. It FAILS LOUD rather than recording a placeholder: a source whose commit cannot be
+ * resolved has no honest provenance, and `"unknown"` in sync-state is a false record, not a gap.
+ *
+ * A CROSS-REPO source (`repo` set) is resolved against the REMOTE. It never falls back to a local
+ * read — that fallback is precisely the adjacency defect this contract exists to kill, and it would
+ * stamp a foreign repo's content with this repo's commit.
+ *
+ * @param {Object} source - a `_resolvedSources` entry from ManifestLoader
+ * @param {string} workspaceRoot - workspace root (the in-repo resolution context)
+ * @returns {Promise<{repo: string|null, ref: string, resolved_commit_sha: string, source_repo: string}>}
+ * @throws {Error} naming the repo/ref that could not be resolved
+ */
+export async function resolveSourceProvenance(source, workspaceRoot) {
+  const ref = source.ref || DEFAULT_SOURCE_REF;
+  const repo = source.repo ?? null;
+  // ONE derivation of "which repo this content belongs to": the explicit slug when the source is
+  // cross-repo, else the leading path segment (the historical in-repo value, preserved verbatim so
+  // existing chunk metadata does not churn).
+  const source_repo = repo ?? (String(source.path || '').split('/')[0] || 'unknown');
+
+  if (!repo) {
+    let sha;
+    try {
+      sha = execSync(`git rev-parse "${ref}^{commit}"`, {
+        cwd: workspaceRoot, encoding: 'utf-8', stdio: ['pipe', 'pipe', 'pipe'],
+      }).trim();
+    } catch {
+      throw new Error(
+        `Cannot resolve ref "${ref}" to a commit in ${workspaceRoot} for source "${source.path}". ` +
+        `Refusing to record provenance — a sync-state commit of "unknown" is a FALSE record, not a ` +
+        `missing one. Check that the ref exists locally.`
+      );
+    }
+    if (!SHA40.test(sha)) {
+      throw new Error(`Ref "${ref}" resolved to "${sha}", which is not a commit sha (source "${source.path}").`);
+    }
+    return { repo: null, ref, resolved_commit_sha: sha, source_repo };
+  }
+
+  // CROSS-REPO: resolve against the remote. Never a local read.
+  if (SHA40.test(ref)) {
+    return { repo, ref, resolved_commit_sha: ref.toLowerCase(), source_repo };
+  }
+  const remoteUrl = `https://github.com/${repo}.git`;
+  let out;
+  try {
+    out = execSync(`git ls-remote "${remoteUrl}" "${ref}" "refs/heads/${ref}" "refs/tags/${ref}"`, {
+      cwd: workspaceRoot, encoding: 'utf-8', stdio: ['pipe', 'pipe', 'pipe'],
+      env: { ...process.env, GIT_TERMINAL_PROMPT: '0' },
+    }).trim();
+  } catch (err) {
+    throw new Error(
+      `Cannot fetch cross-repo source "${source.path}" from repo "${repo}" at ref "${ref}": ` +
+      `${String(err.stderr || err.message).trim()}. REFUSING — a repo that cannot be fetched is ` +
+      `never silently skipped and never falls back to a local/adjacent working tree. Ensure git ` +
+      `credentials for ${remoteUrl} are available to this machine.`
+    );
+  }
+  const line = out.split('\n').map(l => l.trim()).filter(Boolean)[0];
+  const sha = line ? line.split(/\s+/)[0] : null;
+  if (!sha || !SHA40.test(sha)) {
+    throw new Error(
+      `Repo "${repo}" has no ref "${ref}" (source "${source.path}"). REFUSING rather than guessing.`
+    );
+  }
+  return { repo, ref, resolved_commit_sha: sha, source_repo };
+}
+
+/**
+ * Recursively walk a directory and collect file paths.
+ *
+ * @param {string} dir - Directory to walk
+ * @param {string} baseDir - Base directory for computing relative paths
+ * @param {string[]} syncignore - Glob patterns to exclude
+ * @returns {Promise<string[]>} Array of absolute file paths
+ */
+async function walkDir(dir, baseDir, syncignore) {
+  const results = [];
+
+  let entries;
+  try {
+    entries = await fs.readdir(dir, { withFileTypes: true });
+  } catch {
+    return results; // Directory doesn't exist or unreadable
+  }
+
+  for (const entry of entries) {
+    const fullPath = path.join(dir, entry.name);
+    const relativePath = path.relative(baseDir, fullPath);
+
+    if (isIgnored(relativePath, syncignore)) {
+      continue;
+    }
+
+    if (entry.isDirectory()) {
+      // Check directory-level ignore (patterns ending with /)
+      const dirRelative = relativePath + '/';
+      if (isIgnored(dirRelative, syncignore)) {
+        continue;
+      }
+      const subFiles = await walkDir(fullPath, baseDir, syncignore);
+      results.push(...subFiles);
+    } else if (entry.isFile()) {
+      // Only include text-processable files
+      if (isProcessableFile(fullPath)) {
+        results.push(fullPath);
+      }
+    }
+  }
+
+  return results;
+}
+
+/**
+ * Check if a path is a file (not a directory).
+ *
+ * @param {string} filePath - Path to check
+ * @returns {Promise<boolean>}
+ */
+async function isFile(filePath) {
+  try {
+    const stat = await fs.stat(filePath);
+    return stat.isFile();
+  } catch {
+    return false;
+  }
+}
+
+/**
+ * Walk a manifest and return file entries with git metadata.
+ *
+ * For each source entry in the manifest:
+ * 1. Resolve the local directory path (workspace-relative)
+ * 2. Walk the directory, applying syncignore globs
+ * 3. Compute git blob SHA for each file
+ * 4. Return file list with metadata
+ *
+ * @param {Object} manifest - Validated manifest from ManifestLoader (with _resolvedSources)
+ * @param {string} workspaceRoot - Absolute workspace root path
+ * @returns {Promise<Object>} { files: Array<FileEntry>, commitSha: string,
+ *                              provenance: Array<{path, repo, ref, resolved_commit_sha, source_repo}> }
+ */
+export async function walkCorpus(manifest, workspaceRoot) {
+  const files = [];
+  const provenance = [];
+  let commitSha = 'unknown';
+
+  // Nested-source dedup (WS-EVP-DESCIX-KB-CLEANUP item 4): a manifest may list both a directory
+  // source AND a more-specific source nested inside it (e.g. `.../services` at tier 3 AND
+  // `.../services/commandHandlers` at tier 2 to override the tier for that subtree). Without dedup
+  // the walker collects the nested files TWICE — once under each source — producing duplicate
+  // content-addressed chunk IDs and a deterministic upsert-vs-live dedup delta every full sync
+  // (the observed −175 on unk-beast/Corpus). Fix at the walker level so the class is killed for
+  // every manifest: each physical file belongs to EXACTLY ONE source — the DEEPEST (most-specific)
+  // source root that contains it. Walking an ancestor source skips a file owned by a descendant
+  // source, preserving the author's override intent (the nested source's tier/doc_type/syncignore
+  // win for its subtree). Order-independent and deterministic; a single-file source root === the
+  // file path, so an explicit file source always out-specifies any enclosing directory source.
+  const sourceRoots = manifest._resolvedSources.map(s => path.resolve(s.absolutePath));
+  const ownerSourceRoot = (fileAbs) => {
+    const fileResolved = path.resolve(fileAbs);
+    let deepest = null;
+    for (const root of sourceRoots) {
+      if (fileResolved === root || fileResolved.startsWith(root + path.sep)) {
+        if (deepest === null || root.length > deepest.length) deepest = root;
+      }
+    }
+    return deepest;
+  };
+
+  for (const source of manifest._resolvedSources) {
+    const { absolutePath, ref, tier, doc_type, syncignore } = source;
+    const thisSourceRoot = path.resolve(absolutePath);
+
+    // Resolve this source's exact provenance BEFORE reading anything. A cross-repo source that
+    // cannot be fetched REFUSES here, naming the repo — it is never silently skipped and never
+    // falls back to reading an adjacent working tree.
+    const prov = await resolveSourceProvenance(source, workspaceRoot);
+    provenance.push({ path: source.path, ...prov });
+
+    // A cross-repo source's CONTENT still lives in another repository. Reading `absolutePath`
+    // would resolve inside THIS workspace — the exact adjacency defect this contract kills — so
+    // refuse loudly rather than walk the wrong tree and stamp it with a foreign repo's sha.
+    if (prov.repo) {
+      throw new Error(
+        `Cross-repo source "${source.path}" (repo "${prov.repo}", ref "${ref}" @ ` +
+        `${prov.resolved_commit_sha.substring(0, 8)}) cannot be walked: cross-repo CONTENT FETCH is ` +
+        `not implemented in this CLI. Refusing rather than reading the local working tree at ` +
+        `${absolutePath}, which would attribute this workspace's files to "${prov.repo}".`
+      );
+    }
+
+    // The sync's headline commit is the FIRST source's resolved commit — read from the one owner
+    // (resolveSourceProvenance) rather than re-deriving it, so the headline and the per-source
+    // provenance records can never disagree.
+    if (commitSha === 'unknown') {
+      commitSha = prov.resolved_commit_sha;
+    }
+
+    // Check if source path is a file or directory
+    const sourceIsFile = await isFile(absolutePath);
+
+    let filePaths;
+    if (sourceIsFile) {
+      // Single file source entry — check if processable
+      filePaths = isProcessableFile(absolutePath) ? [absolutePath] : [];
+    } else {
+      // Directory: walk recursively with syncignore
+      filePaths = await walkDir(absolutePath, absolutePath, syncignore);
+    }
+
+    for (const filePath of filePaths) {
+      // Nested-source dedup: skip any file owned by a MORE-SPECIFIC (deeper) source root.
+      if (ownerSourceRoot(filePath) !== thisSourceRoot) {
+        continue;
+      }
+      // Compute workspace-relative path for git operations
+      const workspaceRelativePath = path.relative(workspaceRoot, filePath);
+
+      // Compute blob SHA
+      const blobSha = computeBlobSha(filePath, workspaceRelativePath, ref, workspaceRoot);
+      if (!blobSha) {
+        continue; // Skip files that can't be hashed
+      }
+
+      // "Which repo does this content belong to" has ONE owner: resolveSourceProvenance.
+      const sourceRepo = prov.source_repo;
+
+      files.push({
+        absolute_path: filePath,
+        relative_path: workspaceRelativePath,
+        blob_sha: blobSha,
+        source_entry: {
+          path: source.path,
+          ref,
+          tier,
+          doc_type,
+          syncignore,
+          // KB-curation metadata (rides into the chunk record → Pinecone).
+          doc_class: source.doc_class,
+          license_basis: source.license_basis ?? null,
+          raw_path: source.raw_path ?? null,
+          synced_from_edit: (source.raw_path ?? null) != null
+        },
+        source_repo: sourceRepo
+      });
+    }
+  }
+
+  return { files, commitSha, provenance };
+}
+
+export default { walkCorpus, resolveSourceProvenance };