@mmerterden/multi-agent-pipeline 20.8.3 → 20.9.0

package/pipeline/lib/claude-md-links.mjs

@@ -0,0 +1,328 @@
+#!/usr/bin/env node
+/**
+ * claude-md-links.mjs  -  the in-repo documents a repo's CLAUDE.md points at.
+ *
+ * A project CLAUDE.md is often an index: "naming rules are in docs/style.md".
+ * Phase 1 treats those linked files as project rules, so it needs them, and it
+ * needs them bounded. Three forms count as a reference: a markdown link, a
+ * document path written as inline code (`` See `.claude/RULES.md` ``), and a
+ * host import line (`@docs/rules.md`). A code path is resolved against the
+ * CLAUDE.md's own directory first, then the repo root; a link or import against
+ * the file's own directory; a leading `/` means the repo root when its first
+ * segment exists there, and is otherwise an absolute filesystem path, which is
+ * rejected like `~/`.
+ *
+ * One hop only (links inside a linked file are not followed), at most
+ * MAX_FILES files and MAX_BYTES bytes in total. References on a line that calls
+ * the target authoritative ("See X for ...", "X is authoritative", "source of
+ * truth") are read first, so the caps drop background reading before rules. The
+ * file that crosses the byte budget is truncated at a character boundary and
+ * reported in `truncated[]`, not dropped. SKILL.md files are skipped: skills are
+ * routed by scripts/skill-candidates.mjs. Nothing that resolves outside the repo
+ * root is read - checked on the written path and on the real path, for the
+ * CLAUDE.md files themselves too, because a symlink inside the repo can point
+ * anywhere.
+ *
+ * Usage:
+ *   node claude-md-links.mjs <repo-root> [--no-content]
+ *
+ * Prints JSON: {files:[{path, from, kind, priority, bytes, content, truncated?,
+ * fullBytes?}], rejected:[{link, from, kind, reason}], truncated:[path],
+ * totalBytes, sources}. `kind` is "link", "code-path", "import" or, for a
+ * rejected CLAUDE.md, "source". Exit 0 always when the root is readable; an
+ * absent CLAUDE.md is an empty result, not an error. Exit 2 on usage.
+ *
+ * @module pipeline/lib/claude-md-links
+ */
+
+import { existsSync, lstatSync, readFileSync, realpathSync, statSync } from "node:fs";
+import { basename, dirname, isAbsolute, join, relative, resolve, sep } from "node:path";
+
+import { invokedDirectly } from "./invoked-directly.mjs";
+
+export const SOURCES = ["CLAUDE.md", ".claude/CLAUDE.md"];
+export const MAX_FILES = 10;
+export const MAX_BYTES = 200 * 1024;
+
+const SCHEME = /^[a-z][a-z0-9+.-]*:/i;
+const AUTHORITATIVE =
+  /\bauthoritative\b|\bsource of truth\b|\bcanonical\b|\bmust (?:read|follow)\b|\bsee\b[^\n]*?\bfor\b|\brules? (?:are|live) in\b/i;
+
+function blank(m) {
+  return m.replace(/[^\n]/g, " ");
+}
+
+function stripFences(md) {
+  return md.replace(/^(```|~~~)[\s\S]*?^\1[^\n]*$/gm, blank);
+}
+
+function stripCode(md) {
+  return stripFences(md).replace(/`[^`\n]*`/g, blank);
+}
+
+function cleanTarget(raw) {
+  let t = raw.trim().replace(/^<|>$/g, "");
+  if (t.startsWith("#")) return null;
+  t = t.replace(/[#?].*$/, "");
+  if (!t) return null;
+  try {
+    t = decodeURI(t);
+  } catch {
+    /* keep the undecoded form */
+  }
+  return t;
+}
+
+function lineAt(text, index) {
+  const start = text.lastIndexOf("\n", index - 1) + 1;
+  const end = text.indexOf("\n", index);
+  return text.slice(start, end < 0 ? text.length : end);
+}
+
+function linkHits(md) {
+  const text = stripCode(String(md ?? ""));
+  const out = [];
+  const inline = /!?\[[^\]\n]*\]\(\s*(<[^>\n]*>|[^\s)]+)(?:\s+(?:"[^"\n]*"|'[^'\n]*'))?\s*\)/g;
+  const reference = /^[ \t]{0,3}\[[^\]\n]+\]:[ \t]*(<[^>\n]*>|\S+)/gm;
+  for (const re of [inline, reference]) {
+    for (const m of text.matchAll(re)) {
+      const t = cleanTarget(m[1]);
+      if (t) out.push({ at: m.index, t });
+    }
+  }
+  return out.sort((a, b) => a.at - b.at);
+}
+
+/**
+ * Link targets in a markdown document, in document order, fragments and titles
+ * removed. Anchor-only links and anything inside code are skipped.
+ * @param {string} md
+ * @returns {string[]}
+ */
+export function extractLinks(md) {
+  return linkHits(md).map((x) => x.t);
+}
+
+const DOC_PATH =
+  /^(?:\.{0,2}\/)?[A-Za-z0-9._-]+(?:\/[A-Za-z0-9._-]+)*\.(?:md|mdx|markdown|txt|rst)$/i;
+
+function codePathHits(md) {
+  const text = stripFences(String(md ?? ""));
+  const out = [];
+  for (const m of text.matchAll(/`([^`\n]+)`/g)) {
+    const t = m[1].trim();
+    if (DOC_PATH.test(t)) out.push({ at: m.index, t });
+  }
+  return out;
+}
+
+/**
+ * Document paths written as inline code, in document order. Only spans that are
+ * nothing but a relative-looking path to a text document count, so a command,
+ * a type name or a glob is never taken for a reference.
+ * @param {string} md
+ * @returns {string[]}
+ */
+export function extractCodePaths(md) {
+  return codePathHits(md).map((x) => x.t);
+}
+
+function importHits(md) {
+  const text = stripCode(String(md ?? ""));
+  const out = [];
+  const re = /(^|[\s(])@((?:~|\.{1,2})?\/?[A-Za-z0-9._-]+(?:\/[A-Za-z0-9._-]+)*)/g;
+  for (const m of text.matchAll(re)) {
+    const t = m[2].replace(/[.]+$/, "");
+    if (/\.[A-Za-z0-9]+$/.test(basename(t))) out.push({ at: m.index + m[1].length, t });
+  }
+  return out;
+}
+
+/**
+ * Host import lines (`@path/to/file.md`) outside code, in document order. An
+ * `@` inside a word (an e-mail address) is not an import, and a target needs a
+ * file extension, so a mention or a package scope is not taken for one.
+ * @param {string} md
+ * @returns {string[]}
+ */
+export function extractImports(md) {
+  return importHits(md).map((x) => x.t);
+}
+
+function inside(root, p) {
+  const rel = relative(root, p);
+  return rel === "" || (!rel.startsWith(`..${sep}`) && rel !== ".." && !isAbsolute(rel));
+}
+
+function toPosix(p) {
+  return p.split(sep).join("/");
+}
+
+function locate(root, ref) {
+  const { link } = ref;
+  if (SCHEME.test(link) || link.startsWith("//")) return { reason: "url" };
+  if (link.startsWith("~")) return { reason: "absolute path" };
+  let bases = ref.bases;
+  let rel = link;
+  if (isAbsolute(link)) {
+    const first = link.split("/").find(Boolean);
+    if (!first || !existsSync(join(root, first))) return { reason: "absolute path" };
+    bases = [root];
+    rel = link.replace(/^\/+/, "");
+  }
+  let sawInside = false;
+  for (const b of bases) {
+    const target = resolve(b, rel);
+    if (!inside(root, target)) continue;
+    sawInside = true;
+    try {
+      lstatSync(target);
+      return { target };
+    } catch {
+      /* try the next base */
+    }
+  }
+  return { reason: sawInside ? "missing" : "outside repo" };
+}
+
+/** Largest prefix of `buf` no longer than `limit` that ends on a UTF-8 character boundary. */
+function utf8Cut(buf, limit) {
+  let end = Math.min(limit, buf.length);
+  while (end > 0 && end < buf.length && (buf[end] & 0xc0) === 0x80) end--;
+  return end;
+}
+
+/**
+ * One reference: a file entry, a rejection reason, or null for a silent skip
+ * (a duplicate, or one of the CLAUDE.md files themselves).
+ */
+function admit(root, ref, seen, result, { maxFiles, maxBytes, withContent }) {
+  const found = locate(root, ref);
+  if (found.reason) return found.reason;
+  let real;
+  try {
+    real = realpathSync(found.target);
+  } catch {
+    return "missing";
+  }
+  if (!inside(root, real)) return "outside repo";
+  if (seen.has(real) || SOURCES.some((s) => join(root, s) === real)) return null;
+  if (basename(found.target) === "SKILL.md" || basename(real) === "SKILL.md") return "skill file";
+  let st;
+  try {
+    st = statSync(real);
+  } catch {
+    return "missing";
+  }
+  if (!st.isFile()) return "not a regular file";
+  if (result.files.length >= maxFiles) return "file cap";
+  const room = maxBytes - result.totalBytes;
+  if (room <= 0) return "byte budget";
+  let buf;
+  try {
+    buf = readFileSync(real);
+  } catch {
+    return "unreadable";
+  }
+  if (buf.includes(0)) return "binary";
+  const end = buf.length > room ? utf8Cut(buf, room) : buf.length;
+  if (end === 0) return "byte budget";
+  seen.add(real);
+  result.totalBytes += end;
+  const path = toPosix(relative(root, real));
+  const entry = {
+    path,
+    from: null,
+    kind: ref.kind,
+    priority: ref.priority,
+    bytes: end,
+  };
+  if (end < buf.length) {
+    entry.truncated = true;
+    entry.fullBytes = buf.length;
+    result.truncated.push(path);
+  }
+  if (withContent) entry.content = buf.subarray(0, end).toString("utf8");
+  return entry;
+}
+
+function refsOf(text, src, base, root) {
+  const tag = (kind, bases) => (h) => ({
+    link: h.t,
+    kind,
+    bases,
+    from: src,
+    at: h.at,
+    priority: AUTHORITATIVE.test(lineAt(text, h.at)) ? "authoritative" : "normal",
+  });
+  return [
+    ...linkHits(text).map(tag("link", [base])),
+    ...codePathHits(text).map(tag("code-path", base === root ? [base] : [base, root])),
+    ...importHits(text).map(tag("import", [base])),
+  ].sort((a, b) => a.at - b.at);
+}
+
+/**
+ * Follow relative links from the repo's CLAUDE.md files, one hop, bounded.
+ * @param {string} repoRoot
+ * @param {{maxFiles?: number, maxBytes?: number, content?: boolean}} [opts]
+ */
+export function followClaudeMdLinks(repoRoot, opts = {}) {
+  const maxFiles = opts.maxFiles ?? MAX_FILES;
+  const maxBytes = opts.maxBytes ?? MAX_BYTES;
+  const withContent = opts.content !== false;
+  const result = { files: [], rejected: [], truncated: [], totalBytes: 0, sources: [] };
+
+  let root;
+  try {
+    root = realpathSync(resolve(repoRoot));
+  } catch {
+    return result;
+  }
+  const seen = new Set();
+  const refs = [];
+
+  for (const src of SOURCES) {
+    const srcPath = join(root, src);
+    let text;
+    try {
+      if (!statSync(srcPath).isFile()) continue;
+      if (!inside(root, realpathSync(srcPath))) {
+        result.rejected.push({ link: src, from: src, kind: "source", reason: "outside repo" });
+        continue;
+      }
+      text = readFileSync(srcPath, "utf8");
+    } catch {
+      continue;
+    }
+    result.sources.push(src);
+    refs.push(...refsOf(text, src, dirname(srcPath), root));
+  }
+
+  const ordered = [
+    ...refs.filter((r) => r.priority === "authoritative"),
+    ...refs.filter((r) => r.priority !== "authoritative"),
+  ];
+  for (const ref of ordered) {
+    const verdict = admit(root, ref, seen, result, { maxFiles, maxBytes, withContent });
+    if (verdict === null) continue;
+    if (typeof verdict === "string") {
+      result.rejected.push({ link: ref.link, from: ref.from, kind: ref.kind, reason: verdict });
+    } else {
+      verdict.from = ref.from;
+      result.files.push(verdict);
+    }
+  }
+  return result;
+}
+
+if (invokedDirectly(import.meta.url)) {
+  const args = process.argv.slice(2);
+  const dir = args.find((a) => !a.startsWith("--"));
+  if (!dir) {
+    console.error("usage: claude-md-links.mjs <repo-root> [--no-content]");
+    process.exit(2);
+  }
+  const out = followClaudeMdLinks(dir, { content: !args.includes("--no-content") });
+  process.stdout.write(`${JSON.stringify(out, null, 2)}\n`);
+}