agent-doc-system 1.1.0 → 1.3.0

package/core/scaffold.mjs

@@ -13,8 +13,14 @@ import fs from "node:fs";
 import path from "node:path";
 import { fileURLToPath } from "node:url";
 import { AgentDocError, CODES } from "./codes.mjs";
-import { Repo } from "./fsx.mjs";
+import { Repo, assertRepoPath } from "./fsx.mjs";
+import { CONFIG_PATH } from "./descriptors.mjs";
+import { discoverUnits, isNotUnitRoot } from "./discover.mjs";
 import { classifyContractFile } from "./contracts.mjs";
+import { parse as parseToml } from "./toml.mjs";
+import { cargoUnitShape, isExcludedWorkspaceMember, isVirtualWorkspaceManifest } from "../adapters/rust.mjs";
+import { readPythonUnit } from "../adapters/python.mjs";
+import { loadAdapters } from "../adapters/index.mjs";
 
 const TEMPLATES_DIR = path.join(path.dirname(fileURLToPath(import.meta.url)), "..", "templates");
 
@@ -40,23 +46,96 @@ function sentence(s) {
   return /[.!?]$/.test(t) ? t : t + ".";
 }
 
-export function installTemplates(repo, cfg, { write, root, force = false, eligible = null } = {}) {
+// Where the documentation skeleton goes when no configuration exists yet. Once
+// a configuration exists, it is the only authority: `docs.*` is what audit and
+// compile read, so the scaffolder writes there and nowhere else (issue #49 —
+// `docs/` is often the product's own user documentation).
+//
+// The default lives under `agentdoc/` (issue #59): authored AgentDoc material
+// lives in `agentdoc/`, generated output in `.agentdoc/`. Not `docs/`, which is
+// the product's user documentation (and a MkDocs/Docusaurus/Pages source), and
+// not `.agentdoc/`, which the compiler skips and many repositories git-ignore.
+export const DEFAULT_DOCS = Object.freeze({
+  product: "agentdoc/docs/PRODUCT.md",
+  architecture: "agentdoc/docs/ARCHITECTURE.md",
+  constraints: "agentdoc/docs/CONSTRAINTS.md",
+  adrDir: "agentdoc/docs/adr",
+});
+
+// The skeleton roles that have a template. `runbookDir` and `normative` are
+// honoured by never writing anything for them: there is no runbook template,
+// and normative documents are the project's own.
+const SKELETON_ROLES = [
+  { key: "product", template: "PRODUCT.md", file: (p) => p },
+  { key: "architecture", template: "ARCHITECTURE.md", file: (p) => p },
+  { key: "constraints", template: "CONSTRAINTS.md", file: (p) => p },
+  { key: "adrDir", template: "ADR.md", file: (p) => p.replace(/\/+$/, "") + "/README.md" },
+];
+
+// Resolve the documentation skeleton for a loaded configuration (`null` when
+// there is none). A configured role is written to its configured path; a role
+// a configuration leaves out is not written anywhere and is reported, so an
+// omitted role can never fall back into `docs/`.
+export function docsSkeleton(cfg) {
+  const docs = cfg ? cfg.docs || {} : DEFAULT_DOCS;
+  const targets = [];
+  const notConfigured = [];
+  for (const role of SKELETON_ROLES) {
+    const configured = docs[role.key];
+    if (!configured) { notConfigured.push("docs." + role.key); continue; }
+    // loadConfig already validated these; re-asserting keeps this function
+    // safe for any caller that hands it an unvalidated object.
+    assertRepoPath(configured, CONFIG_PATH);
+    const rel = role.file(configured);
+    assertRepoPath(rel, CONFIG_PATH);
+    targets.push({ key: role.key, path: rel, template: role.template });
+  }
+  return { targets, notConfigured, docs: cfg ? docs : DEFAULT_DOCS };
+}
+
+// Refuse a write target that leaves the repository, including through a
+// symlinked directory or a (possibly dangling) symlink at the target itself.
+// Lexical checks alone (`isRepoPath`) cannot see a symlink.
+function assertWritableInside(root, rel) {
+  const abs = path.resolve(root, rel);
+  const inside = path.relative(path.resolve(root), abs);
+  if (!inside || inside === ".." || inside.startsWith(".." + path.sep) || path.isAbsolute(inside)) {
+    throw new AgentDocError(CODES.PATH_ESCAPE, "path escapes repository: " + rel, { path: rel });
+  }
+  const realRoot = fs.realpathSync(root);
+  let cur = path.resolve(root);
+  for (const seg of inside.split(path.sep)) {
+    cur = path.join(cur, seg);
+    let st;
+    try { st = fs.lstatSync(cur); } catch { return; } // nothing further exists yet
+    if (!st.isSymbolicLink()) continue;
+    let real = null;
+    try { real = fs.realpathSync(cur); } catch { /* dangling */ }
+    if (real === null || (real !== realRoot && !real.startsWith(realRoot + path.sep))) {
+      throw new AgentDocError(CODES.PATH_ESCAPE, "refusing to write through a symlink that leaves the repository: " + rel, { path: rel });
+    }
+  }
+}
+
+// `cfg` is the loaded configuration (scaffold) or null (init). `docsConfig`
+// overrides the configuration the documentation paths are read from: init
+// writes no component descriptors, but it must still honour an existing
+// configuration's `docs.*` paths.
+export function installTemplates(repo, cfg, { write, root, force = false, eligible = null, docsConfig } = {}) {
   const written = [];
   const skipped = [];
   const targetRoot = root || (repo ? repo.root : process.cwd());
   const r = repo || new Repo(targetRoot, { noGit: false });
+  const docsSource = docsConfig === undefined ? cfg : docsConfig;
+  const skeleton = docsSkeleton(docsSource);
 
-  const put = (rel, content) => {
-    const abs = path.join(targetRoot, rel);
-    if (fs.existsSync(abs) && !force) { skipped.push(rel); return; }
-    if (!write) { written.push(rel + " (dry run)"); return; }
-    fs.mkdirSync(path.dirname(abs), { recursive: true });
-    fs.writeFileSync(abs, content.endsWith("\n") ? content : content + "\n");
-    written.push(rel);
-  };
+  // Plan every file first and check every target before writing anything, so
+  // a refused path leaves the repository exactly as it was.
+  const plan = [];
+  const put = (rel, content) => { plan.push([rel, content]); };
 
   // ── configuration ───────────────────────────────────────────────────────
-  const config = buildConfig(scanLayout(r));
+  const config = buildConfig(scanLayout(r, cfg), skeleton.docs);
   put("agentdoc/agentdoc.config.yaml", config);
   put("agentdoc/journeys.yaml", "schemaVersion: agentdoc.dev/journeys/v1\njourneys: []\n");
   for (const f of ["domains.yaml", "systems.yaml", "apis.yaml", "resources.yaml"]) {
@@ -64,12 +143,10 @@ export function installTemplates(repo, cfg, { write, root, force = false, eligib
   }
   put("agentdoc/observations/README.md", OBSERVATIONS_README);
 
-  // ── documentation hierarchy ──────────────────────────────────────────────
-  for (const f of ["PRODUCT.md", "ARCHITECTURE.md", "CONSTRAINTS.md"]) {
-    const tpl = fs.readFileSync(path.join(TEMPLATES_DIR, f), "utf8");
-    put("docs/" + f, tpl);
+  // ── documentation hierarchy (configured docs.* paths) ───────────────────
+  for (const t of skeleton.targets) {
+    put(t.path, fs.readFileSync(path.join(TEMPLATES_DIR, t.template), "utf8"));
   }
-  put("docs/adr/README.md", fs.readFileSync(path.join(TEMPLATES_DIR, "ADR.md"), "utf8"));
   put(".github/workflows/agentdoc.yml", fs.readFileSync(path.join(TEMPLATES_DIR, "ci.yml"), "utf8"));
 
   // ── per-unit component descriptors ──────────────────────────────────────
@@ -83,26 +160,50 @@ export function installTemplates(repo, cfg, { write, root, force = false, eligib
       put(u.root + "/" + cfg.discovery.componentDescriptorName, componentDescriptor(u));
     }
   }
-  return { written, skipped, note: write ? "files written" : "dry run: pass --write to apply" };
+
+  const seen = new Set();
+  for (const [rel] of plan) assertWritableInside(targetRoot, rel);
+  for (const [rel, content] of plan) {
+    if (seen.has(rel)) continue; // two roles configured onto one file: first wins
+    seen.add(rel);
+    const abs = path.join(targetRoot, rel);
+    if (fs.existsSync(abs) && !force) { skipped.push(rel); continue; }
+    if (!write) { written.push(rel + " (dry run)"); continue; }
+    fs.mkdirSync(path.dirname(abs), { recursive: true });
+    fs.writeFileSync(abs, content.endsWith("\n") ? content : content + "\n");
+    written.push(rel);
+  }
+  return {
+    written,
+    skipped,
+    ...(docsSource && skeleton.notConfigured.length
+      ? {
+          notConfigured: skeleton.notConfigured,
+          notConfiguredNote: "the configuration declares no path for these roles, so no skeleton was written for them; add them under docs: to scaffold them",
+        }
+      : {}),
+    note: write ? "files written" : "dry run: pass --write to apply",
+  };
 }
 
 // Layout inference for CREATE mode: which directories look like unit roots.
-// Uses only file-presence heuristics that any repository can answer.
-function scanLayout(repo) {
+// Combines shallow marker heuristics with the compiler's adapter discovery.
+function scanLayout(repo, cfg) {
   const candidates = new Map();
   const note = (root, kind) => {
     if (!candidates.has(root)) candidates.set(root, { root, kinds: new Set(), evidence: [] });
     candidates.get(root).kinds.add(kind);
     candidates.get(root).evidence.push(kind);
   };
-  const skip = new Set(["node_modules", ".git", "dist", "build", "vendor", "target", "agentdoc", "docs", "scripts", ".github", "test", "tests", "examples", "example"]);
+  const skip = new Set(["node_modules", ".git", "dist", "build", "vendor", "target", "test", "tests", "examples", "example"]);
   const walk = (dir, depth) => {
-    if (depth > 3) return;
+    if (depth > 3 || isNotUnitRoot(dir, cfg)) return;
     let entries;
     try { entries = fs.readdirSync(path.join(repo.root, dir), { withFileTypes: true }); } catch { return; }
     for (const e of entries.sort((a, b) => (a.name < b.name ? -1 : 1))) {
       if (e.isSymbolicLink() || skip.has(e.name)) continue;
       const rel = dir ? dir + "/" + e.name : e.name;
+      if (repo.isIgnored && repo.isIgnored(rel)) continue;
       if (e.isDirectory()) {
         if (e.name.startsWith(".")) continue;
         walk(rel, depth + 1);
@@ -114,15 +215,62 @@ function scanLayout(repo) {
       const parent = () => rel.slice(0, -("/" + e.name).length);
       if (e.name === "package.json") note(parent(), "node");
       if (e.name === "go.mod") note(parent(), "go");
+      if (e.name === "Cargo.toml" && cargoUnitShape(repo, parent()) && !isExcludedWorkspaceMember(repo, parent())) note(parent(), "rust");
+      if (e.name === "pyproject.toml" || e.name === "setup.py" || e.name === "setup.cfg") note(parent(), "python");
+      if (/^requirements.*\.txt$/.test(e.name)) note(parent(), "python");
       if (e.name === "Dockerfile") note(parent(), "docker");
       if (/^wrangler\.(toml|json|jsonc)$/.test(e.name)) note(parent(), "platform");
       if (e.name === "go.mod") note(rel.slice(0, -"/go.mod".length), "go");
     }
   };
   walk("", 0);
+  // Directory-name/depth heuristics must not hide units declared by adapters
+  // (for example a uv or Cargo member under examples/). During init there is
+  // no config yet: probe supported marker adapters so those members can also
+  // select their adapter in the generated configuration.
+  const discoveryCfg = cfg || { discovery: { componentDescriptors: [] } };
+  const adapters = loadAdapters(cfg ? cfg.adapters : Object.values(ADAPTER_FOR_MARKER));
+  for (const unit of discoverUnits(repo, discoveryCfg, adapters).eligible) {
+    if (!unit.root || unit.root === "." || isNotUnitRoot(unit.root, cfg)) continue;
+    for (const [kind, adapter] of Object.entries(ADAPTER_FOR_MARKER)) {
+      if (unit.markers.some((marker) => marker.adapter === adapter)) note(unit.root, kind);
+    }
+  }
+  for (const u of [...candidates.values()]) {
+    // Layout-only candidates need the same artifact evidence as units already
+    // discovered by the compiler. A technology name alone proves no shape.
+    u.deployable = false;
+    u.importable = false;
+    for (const adapter of loadAdapters([...u.kinds].map((k) => ADAPTER_FOR_MARKER[k]))) {
+      const marker = adapter.detect({ repo, cfg }, u.root || ".");
+      if (!marker) continue;
+      const shape = marker.shape || "both";
+      u.deployable ||= shape === "deployable" || shape === "both";
+      u.importable ||= shape === "importable" || shape === "both";
+    }
+    if (!u.kinds.has("python")) continue;
+    const info = readPythonUnit(repo, u.root);
+    // A uv workspace root with no distribution is not a unit. Scaffolding it
+    // would invent a component for the virtual manifest.
+    if (!info.dist && !info.deployable && !info.importable && !info.error && virtualUvWorkspace(repo, u.root)) {
+      candidates.delete(u.root);
+    }
+  }
   return [...candidates.values()].sort((a, b) => (a.root < b.root ? -1 : 1));
 }
 
+function virtualUvWorkspace(repo, root) {
+  const file = !root || root === "." ? "pyproject.toml" : root + "/pyproject.toml";
+  if (!repo.exists(file)) return false;
+  try {
+    const doc = parseToml(repo.readText(file), file);
+    const ws = doc.tool && doc.tool.uv && doc.tool.uv.workspace;
+    return Boolean(ws && typeof ws === "object" && !Array.isArray(ws));
+  } catch {
+    return false;
+  }
+}
+
 function discoverUnitNames(repo, cfg, eligible = null) {
   const out = [];
   const nameFor = (root) => {
@@ -138,6 +286,18 @@ function discoverUnitNames(repo, cfg, eligible = null) {
       const m = /^module\s+(\S+)/m.exec(repo.readText(gomod));
       if (m) return kebab(m[1].split("/").pop());
     }
+    const cargo = root + "/Cargo.toml";
+    if (repo.exists(cargo)) {
+      try {
+        const doc = parseToml(repo.readText(cargo));
+        const n = doc && doc.package && typeof doc.package.name === "string" ? doc.package.name : "";
+        // `_` becomes `-u-` before kebab, so `foo_bar` does not collapse into
+        // the same component name as `foo-bar`.
+        if (n) return kebab(n.replace(/_/g, "-u-"));
+      } catch { /* a manifest that does not parse is not a name */ }
+    }
+    const python = readPythonUnit(repo, root);
+    if (python.dist) return kebab(python.dist);
     return kebab(path.basename(root));
   };
   const seen = new Set();
@@ -152,27 +312,32 @@ function discoverUnitNames(repo, cfg, eligible = null) {
       deployable: Boolean(u.deployable),
       importable: Boolean(u.importable),
     })),
-    ...scanLayout(repo).map((u) => ({ root: u.root, kinds: [...u.kinds].sort(), deployable: false, importable: false })),
+    ...scanLayout(repo, cfg).map((u) => ({
+      root: u.root,
+      kinds: [...u.kinds].sort(),
+      deployable: Boolean(u.deployable),
+      importable: Boolean(u.importable),
+    })),
   ];
-  for (const u of units) {
-    if (u.root === "." || !u.root || seenRoots.has(u.root)) continue;
+  for (const found of units) {
+    if (found.root === "." || !found.root || seenRoots.has(found.root)) continue;
+    if (isNotUnitRoot(found.root, cfg)) continue;
+    if (isVirtualWorkspaceManifest(repo, found.root)) continue;
+    const u = found;
     seenRoots.add(u.root);
     const name = nameFor(u.root);
     if (seen.has(name)) continue;
     seen.add(name);
     out.push({ ...u, name });
   }
-  void cfg;
   return out;
 }
 
 function componentDescriptor(u) {
-  const deployable = u.deployable || u.kinds.some((k) => k === "docker" || k === "platform");
-  const importable = u.importable || u.kinds.some((k) => k === "node" || k === "go");
   // "library" and "service" both have to be corroborated by artifact evidence
   // (AGENTDOC_ARTIFACT_TYPE); a unit the compiler only knows from configuration
   // shows neither, so claiming a concrete type would manufacture an error.
-  const type = deployable ? "service" : importable ? "library" : "other";
+  const type = u.deployable ? "service" : u.importable ? "library" : "other";
   return [
     "apiVersion: agentdoc.dev/v1",
     "kind: Component",
@@ -194,11 +359,13 @@ function componentDescriptor(u) {
 const ADAPTER_FOR_MARKER = {
   node: "node",
   go: "go",
+  rust: "rust",
+  python: "python",
   docker: "dockerfile",
   platform: "wrangler",
 };
 
-function buildConfig(layout) {
+function buildConfig(layout, docs = DEFAULT_DOCS) {
   const roots = layout.map((l) => l.root);
   const groups = new Map();
   for (const r of roots) {
@@ -206,13 +373,6 @@ function buildConfig(layout) {
     if (!groups.has(head)) groups.set(head, []);
     groups.get(head).push(r);
   }
-  // A manifest at the repository root yields an empty head; emitting
-  // "/agentdoc.yaml" would be an absolute glob, which the loader rejects.
-  const componentGlobs = [...groups.entries()]
-    .filter(([head]) => head.length > 0)
-    .sort((a, b) => (a[0] < b[0] ? -1 : 1))
-    .map(([head, list]) => (list.length === 1 && list[0] === head ? head + "/agentdoc.yaml" : head + "/**/agentdoc.yaml"));
-  if (!componentGlobs.length) componentGlobs.push("*/**/agentdoc.yaml");
 
   const adapters = new Set(["generic"]);
   for (const l of layout) {
@@ -239,9 +399,10 @@ function buildConfig(layout) {
     "discovery:",
     "  componentDescriptorName: agentdoc.yaml",
     "  componentDescriptors:",
-    // Globs are quoted: an unquoted leading "*" is an alias in strict YAML,
-    // and a configuration the compiler cannot parse is worse than none.
-    ...componentGlobs.map((g) => "    - " + yamlStr(g)),
+    // Discover every scaffolded descriptor, including adapter roots outside
+    // the shallow layout and units added after init. Keep the leading glob
+    // quoted because an unquoted "*" is an alias in strict YAML.
+    '    - "*/**/agentdoc.yaml"',
     "  centralDescriptors:",
     "    - agentdoc/domains.yaml",
     "    - agentdoc/systems.yaml",
@@ -281,15 +442,26 @@ function buildConfig(layout) {
     "  graph: .agentdoc/graph.json",
     "  observations: agentdoc/observations",
     "",
-    "docs:",
-    "  product: docs/PRODUCT.md",
-    "  architecture: docs/ARCHITECTURE.md",
-    "  constraints: docs/CONSTRAINTS.md",
-    "  adrDir: docs/adr",
+    ...docsSection(docs),
     "",
   ].join("\n");
 }
 
+// The `docs:` block of a generated configuration. It repeats the configured
+// paths (init --force regenerates the file and must not move documentation
+// elsewhere), or the agentdoc/docs/ defaults when there is no configuration yet.
+function docsSection(docs) {
+  const scalar = (v) => (/^[A-Za-z0-9._/-]+$/.test(v) ? v : yamlStr(v));
+  const lines = [];
+  for (const key of ["product", "architecture", "constraints", "adrDir", "runbookDir"]) {
+    if (docs[key]) lines.push("  " + key + ": " + scalar(docs[key]));
+  }
+  if (Array.isArray(docs.normative) && docs.normative.length) {
+    lines.push("  normative:", ...docs.normative.map((g) => "    - " + yamlStr(g)));
+  }
+  return lines.length ? ["docs:", ...lines] : ["# docs: {}  # no documentation roles configured"];
+}
+
 const OBSERVATIONS_README = `# Runtime observations
 
 An ObservationSet records what an authoritative external system actually said,

package/core/sourcescan.mjs

@@ -7,6 +7,7 @@
 // knows the namespace convention.
 import path from "node:path";
 import fs from "node:fs";
+import { joinPath } from "./paths.mjs";
 
 const SPECIFIER_RE = /["'`]([A-Za-z@][A-Za-z0-9@._/+-]*(?:\.[A-Za-z0-9@._/+-]+)*(?:\/[A-Za-z0-9@._+*-]+)*)["'`]/g;
 
@@ -27,7 +28,7 @@ export function sourceFiles(repo, root, { test = false, extensions } = {}) {
   const DEFAULT_EXT = /\.(ts|tsx|js|jsx|mts|cts|mjs|cjs|go|py|rb|java|kt|rs|ex|exs|cs|php|swift|scala|sh)$/;
   const re = extensions || DEFAULT_EXT;
   const out = [];
-  for (const f of repo.walk(root)) {
+  for (const f of repo.walkUnit(root)) {
     if (!re.test(f)) continue;
     const isTest = TEST_PATH.test(f) || TEST_FILE.test(f) || /_test\.[a-z]+$/.test(f);
     if (isTest !== test) continue;
@@ -64,7 +65,17 @@ export function findMigrationDirs(repo, root, maxDepth = 5) {
       if (!e.isDirectory() || e.isSymbolicLink()) continue;
       if (["node_modules", ".git", "vendor", "target", "dist", "build", ".next", "__pycache__"].includes(e.name)) continue;
       const abs = path.join(dirAbs, e.name);
+      const rel = path.relative(repo.root, abs).split(path.sep).join("/");
+      if (!repo.ownsUnitPath(root, rel)) continue;
+      if (repo.isIgnored && repo.isIgnored(rel)) continue;
       if (e.name === "migrations" || e.name === "migration") {
+        out.push(rel);
+        continue;
+      }
+      // Alembic keeps revisions in versions/ under a directory named alembic.
+      // The directory is the migration root; the adapter still reads alembic.ini
+      // (or [tool.alembic]) before treating it as ownership evidence.
+      if (e.name === "versions" && path.basename(dirAbs) === "alembic") {
         out.push(path.relative(repo.root, abs).split(path.sep).join("/"));
         continue;
       }
@@ -77,7 +88,8 @@ export function findMigrationDirs(repo, root, maxDepth = 5) {
 
 export function firstExistingFile(repo, root, names) {
   for (const n of names) {
-    if (repo.exists(root + "/" + n)) return root + "/" + n;
+    const p = joinPath(root, n);
+    if (repo.ownsUnitPath(root, p) && repo.exists(p)) return p;
   }
   return null;
 }