@norskvideo/ctl-dev-kit 0.1.48 → 0.1.50

package/conventions/checks.yml

@@ -7,9 +7,10 @@
 #     fenced CLAUDE.md core, flake.nix, the shared config files, the manifest
 #     seed. Copies exist so a human reads them on GitHub with no tooling; the
 #     gate is the only thing keeping them true.
-#   - quality: the repo's own lint + typecheck + unit tests, via the shared
-#     package.json script interface every product carries. The "whole project
-#     clean" convention in the CLAUDE core has no teeth without this.
+#   - quality: the repo's own lint + typecheck + unit tests + docs path lint,
+#     via the shared package.json script interface every product carries. The
+#     "whole project clean" convention in the CLAUDE core has no teeth without
+#     this; docs:check is what keeps README/CLAUDE/docs paths true.
 #
 # Both run the same proven way build-image does — inside `nix develop .#build`
 # (which provides bun) on the x64 self-hosted pool — but skip the Docker image
@@ -74,6 +75,7 @@ jobs:
             bun run lint
             bun run typecheck
             bun run test:unit
+            bun run docs:check
           '
 
   # Report this pipeline's result to the aggregated product CI dashboard

package/create-product/backend-turnkey.ts

@@ -44,7 +44,7 @@ container port 4321), then re-add it into the daemon —
    under \`tests/unit/__snapshots__/\` (the generator cannot compose without the
    Studio packages installed). Commit them; from then on any emission change is
    a reviewable fixture diff.
-3. \`bun run check:drift && bun run test:unit && bun run typecheck\`
+3. \`bun run check:drift && bun run docs:check && bun run test:unit && bun run typecheck\`
 
 ## Version pins
 
@@ -68,8 +68,9 @@ refreshed \`~/.norsk-ctl/product-templates/<name>/compose.yml\` carries the pins
 ## Build / test — the five layers
 
 - **Layer 1 — typed factories** (\`shared/src/workflow/components.ts\`): configs
-  \`import type\`d from each built-in's \`_gen/types\`; drift from Studio's
-  schema is a \`bun run typecheck\` failure.
+  \`import type\`d from each built-in's OpenAPI-generated types
+  (\`@norskvideo/norsk-studio-built-ins/lib/<component>/_gen/types\`); drift
+  from Studio's schema is a \`bun run typecheck\` failure.
 - **Layer 2 — \`builder.validate()\`**: the composer throws on any issue. Never
   demote it to a warning; declare out-of-library components in
   \`COMPONENT_STUBS\` (SDK \`stubbedLibrary\`) instead.
@@ -79,8 +80,8 @@ refreshed \`~/.norsk-ctl/product-templates/<name>/compose.yml\` carries the pins
 - **Layer 4 — studio-load round-trip** (\`tests/unit/loads-in-studio.test.ts\`):
   emitted YAML through Studio's real \`document.load()\`.
 - **Layer 5 — integration**: not scaffolded. Decide + record here whether this
-  product carries one (unattended nightly bumps need it; see the dev-kit's
-  \`conventions/integration-testing.md\`).
+  product carries one (unattended nightly bumps need it; see
+  \`node_modules/@norskvideo/ctl-dev-kit/conventions/integration-testing.md\`).
 
 Byte-snapshots (\`tests/unit/workflow-snapshot.test.ts\`) pin the emitted wire
 bytes; \`UPDATE_SNAPSHOTS=1\` refreshes deliberately.
@@ -90,6 +91,9 @@ bytes; \`UPDATE_SNAPSHOTS=1\` refreshes deliberately.
   runs the built container and asserts the surface \`product add\` probes.
 - \`bun run check:drift\` — the shared-conventions gate; CI runs it plus
   lint/typecheck/test:unit via the drift-gated \`checks.yml\`.
+- \`bun run docs:check\` — every backticked repo path in the README, this
+  file and the markdown under docs/ must exist (\`<!-- path-allow -->\` exempts
+  a line that names something outside the tree).
 `;
 }
 
@@ -111,6 +115,7 @@ function rootPackageJson(ctx: ShapeContext): string {
         "test:unit": "bun test shared/src/ backend/src/ tests/unit/",
         "test:image": "bun test tests/image/",
         "check:drift": "bun run node_modules/@norskvideo/ctl-dev-kit/conventions/check-drift.ts",
+        "docs:check": "bun run node_modules/@norskvideo/ctl-dev-kit/docs/path-existence.ts",
         "sync:drift": "bun run node_modules/@norskvideo/ctl-dev-kit/conventions/sync-drift.ts",
         typecheck: "bunx tsc --noEmit -p shared && bunx tsc --noEmit -p backend && bunx tsc --noEmit -p .",
         build: "bun run lint && bun run typecheck && bun run build:no-lint",

package/create-product/cli.ts

@@ -73,7 +73,7 @@ export function main(argv: string[]): void {
 Next steps (also in the generated CLAUDE.md):
   1. cd ${dir} && git init && bun install
   2. UPDATE_SNAPSHOTS=1 bun run test:unit   # bake the byte-snapshot fixtures, commit them
-  3. bun run check:drift && bun run test:unit && bun run typecheck
+  3. bun run check:drift && bun run docs:check && bun run test:unit && bun run typecheck
   4. Replace the starter graph in shared/src/workflow/ with the real one,
      keeping INVARIANTS.md and its rules tests in step.
 `);

package/docs/path-existence.ts

@@ -0,0 +1,286 @@
+// Path-existence lint over the hand-written docs (fleet review 02 s11.1). Every
+// backticked token in README.md, CLAUDE.md and docs/**/*.md that looks like a
+// repo path must resolve to something in the tree. Rot in these files is
+// invisible to every other gate: a CLAUDE.md that points an agent at a file
+// deleted two refactors ago still reads as authoritative.
+//
+// "The tree" is what git tracks or would track (cached + untracked, minus
+// ignored), so the answer is the same on a fresh CI checkout as on a dev box
+// with build output lying around. A path that git IGNORES is a declared output
+// (frontend/dist, docs/generated) rather than rot, so it counts as existing;
+// deciding how an index may link into generated output is the review's s13
+// follow-up, not this lint's call. Outside a git repo the tree is a filesystem
+// walk and nothing is ignored.
+import { readdirSync, readFileSync, realpathSync } from "node:fs";
+import { basename, dirname, join, posix } from "node:path";
+
+export interface PathFinding {
+  file: string;
+  line: number;
+  token: string;
+  message: string;
+}
+
+export interface PathExistenceOptions {
+  // Tracked paths relative to repoRoot; loaded via loadTree() when omitted.
+  tree?: string[];
+  // Extra bases a path may be relative to, on top of the repo root and the
+  // citing doc's own directory (ctl's CLAUDE.md writes paths relative to
+  // packages/norsk-ctl, and says so).
+  roots?: string[];
+  skipPrefixes?: string[];
+  // Which of the given candidate paths are gitignored; defaults to asking git.
+  ignored?: (candidates: string[]) => Set<string>;
+}
+
+export const PATH_ALLOW = "<!-- path-allow -->";
+// The fenced core a product's CLAUDE.md copies verbatim from the dev-kit is
+// fleet-generic prose under its own gate (check-drift); the paths it names are
+// conventions, not claims about this repo's tree.
+const CORE_BEGIN = "<!-- BEGIN ctl-shared-conventions v1 -->";
+const CORE_END = "<!-- END ctl-shared-conventions v1 -->";
+
+// The five prefixes the review names (its `@norskvideo/` widened to every
+// scoped npm name), plus the classes they imply: any absolute path (container
+// paths, URL paths) and the GitHub/Docker Hub org prefixes that read like a
+// path but name a repo or an image.
+export const DEFAULT_SKIP_PREFIXES = [
+  "~/",
+  "/api/",
+  "http",
+  "@",
+  "node_modules/",
+  "/",
+  "id3as/",
+  "norskvideo/",
+  "ghcr.io/",
+  "docker.io/",
+];
+
+const PATH_WITH_SLASH = /^[\w@./-]+\/[\w./*-]+$/;
+// A bare filename is only a repo-path claim when its extension says "source":
+// bare config names (config.yaml, default.yaml, package.json) far more often
+// name daemon state or a file inside the container.
+const SOURCE_EXTENSIONS = "md|mdx|ts|tsx|js|mjs|cjs|sh|nix";
+const BARE_FILENAME = new RegExp(`^[\\w.-]+\\.(${SOURCE_EXTENSIONS})$`);
+// `path:12`, `path:12-15`, `path#anchor` are citations of a path, not a path.
+const CITATION_SUFFIX = /(:\d+(-\d+)?|#[\w-]+)$/;
+const INLINE_CODE = /`([^`\n]+)`/g;
+
+export function extractPathTokens(line: string): string[] {
+  const tokens: string[] = [];
+  for (const match of line.matchAll(INLINE_CODE)) {
+    const raw = match[1]?.trim() ?? "";
+    const token = raw.replace(CITATION_SUFFIX, "");
+    if (PATH_WITH_SLASH.test(token) || BARE_FILENAME.test(token)) tokens.push(token);
+  }
+  return tokens;
+}
+
+export function docFilesFor(tree: string[], exclude: string[] = []): string[] {
+  const excluded = exclude.map((g) => new Bun.Glob(g));
+  return tree.filter((path) => {
+    const inScope =
+      path === "README.md" ||
+      path === "CLAUDE.md" ||
+      (path.startsWith("docs/") &&
+        path.endsWith(".md") &&
+        !path.startsWith("docs/internal/") &&
+        !path.startsWith("docs/generated/"));
+    return inScope && !excluded.some((g) => g.match(path));
+  });
+}
+
+const WALK_SKIP = new Set(["node_modules", ".git"]);
+
+function walk(root: string, rel = ""): string[] {
+  const out: string[] = [];
+  for (const entry of readdirSync(join(root, rel), { withFileTypes: true })) {
+    if (WALK_SKIP.has(entry.name)) continue;
+    const path = rel ? `${rel}/${entry.name}` : entry.name;
+    if (entry.isDirectory()) out.push(...walk(root, path));
+    else out.push(path);
+  }
+  return out;
+}
+
+function gitLines(repoRoot: string, args: string[], stdin?: string): string[] | undefined {
+  const git = Bun.spawnSync(["git", "-C", repoRoot, ...args], {
+    stdin: stdin === undefined ? undefined : Buffer.from(stdin),
+  });
+  // check-ignore exits 1 when nothing matched, which is an answer, not a failure.
+  if (git.exitCode !== 0 && git.exitCode !== 1) return undefined;
+  return git.stdout
+    .toString()
+    .split("\n")
+    .filter((l) => l.length > 0);
+}
+
+// Only a repo's own toplevel is git's to answer: a root nested somewhere inside
+// another repo (a generated skeleton under an ignored scratch dir) would come
+// back empty from ls-files, so it is walked instead.
+function isGitToplevel(repoRoot: string): boolean {
+  const top = gitLines(repoRoot, ["rev-parse", "--show-toplevel"])?.[0];
+  return top !== undefined && realpathSync(top) === realpathSync(repoRoot);
+}
+
+export function loadTree(repoRoot: string): string[] {
+  if (!isGitToplevel(repoRoot)) return walk(repoRoot);
+  return gitLines(repoRoot, ["ls-files", "--cached", "--others", "--exclude-standard"]) ?? walk(repoRoot);
+}
+
+// A directory pattern (`dist/`) only matches a path git knows is a directory,
+// and a path that is not on disk has no type, so ask about both spellings.
+export function gitIgnored(repoRoot: string): (candidates: string[]) => Set<string> {
+  return (candidates) => {
+    if (candidates.length === 0 || !isGitToplevel(repoRoot)) return new Set();
+    const asked = candidates.flatMap((c) => [c, `${c}/`]);
+    const hits = gitLines(repoRoot, ["check-ignore", "--stdin"], `${asked.join("\n")}\n`) ?? [];
+    return new Set(hits.map((h) => h.replace(/\/$/, "")));
+  };
+}
+
+class Tree {
+  private readonly files: Set<string>;
+  private readonly dirs: Set<string>;
+  private readonly basenames: Set<string>;
+
+  constructor(entries: string[]) {
+    this.files = new Set(entries);
+    this.dirs = new Set<string>();
+    this.basenames = new Set<string>();
+    for (const file of entries) {
+      this.basenames.add(basename(file));
+      let dir = dirname(file);
+      while (dir !== "." && dir !== "/" && !this.dirs.has(dir)) {
+        this.dirs.add(dir);
+        dir = dirname(dir);
+      }
+    }
+  }
+
+  has(path: string, directoryOnly: boolean): boolean {
+    if (path === "" || path === ".") return true;
+    if (path.includes("*")) {
+      const glob = new Bun.Glob(path);
+      for (const dir of this.dirs) if (glob.match(dir)) return true;
+      if (directoryOnly) return false;
+      for (const file of this.files) if (glob.match(file)) return true;
+      return false;
+    }
+    if (this.dirs.has(path)) return true;
+    return !directoryOnly && this.files.has(path);
+  }
+
+  hasBasename(name: string): boolean {
+    return this.basenames.has(name);
+  }
+}
+
+function isSkipped(token: string, skipPrefixes: string[]): boolean {
+  return skipPrefixes.some((p) => token.startsWith(p));
+}
+
+interface Miss {
+  file: string;
+  line: number;
+  token: string;
+  bases: string[];
+  candidates: string[];
+}
+
+export function checkPathExistence(
+  repoRoot: string,
+  files: string[],
+  {
+    tree: entries,
+    roots = [],
+    skipPrefixes = DEFAULT_SKIP_PREFIXES,
+    ignored = gitIgnored(repoRoot),
+  }: PathExistenceOptions = {},
+): PathFinding[] {
+  const tree = new Tree(entries ?? loadTree(repoRoot));
+  const misses: Miss[] = [];
+
+  for (const file of files) {
+    const bases = unique([dirname(file) === "." ? "" : dirname(file), "", ...roots]);
+    const lines = readFileSync(join(repoRoot, file), "utf8").split("\n");
+    let inFence = false;
+    let inCore = false;
+    lines.forEach((line, index) => {
+      if (/^\s*(```|~~~)/.test(line)) {
+        inFence = !inFence;
+        return;
+      }
+      if (line.includes(CORE_BEGIN)) inCore = true;
+      if (line.includes(CORE_END)) inCore = false;
+      if (inFence || inCore || line.includes(PATH_ALLOW)) return;
+      for (const token of extractPathTokens(line)) {
+        if (isSkipped(token, skipPrefixes)) continue;
+        if (!token.includes("/")) {
+          if (!tree.hasBasename(token)) misses.push({ file, line: index + 1, token, bases, candidates: [] });
+          continue;
+        }
+        const directoryOnly = token.endsWith("/");
+        const candidates = bases
+          .map((base) => posix.normalize(posix.join(base, token.replace(/\/+$/, ""))))
+          .filter((c) => !c.startsWith(".."));
+        // Every reading leaves the repo (`../norsk-ctl`): not this tree's claim.
+        if (candidates.length === 0) continue;
+        if (candidates.some((c) => tree.has(c, directoryOnly))) continue;
+        misses.push({ file, line: index + 1, token, bases, candidates });
+      }
+    });
+  }
+
+  // One git round-trip for every miss, not one per miss.
+  const declaredOutputs = ignored(unique(misses.flatMap((m) => m.candidates)));
+  return misses
+    .filter((m) => !m.candidates.some((c) => declaredOutputs.has(c)))
+    .map(({ file, line, token, bases }) => {
+      const looked = bases.map((b) => (b === "" ? "<repo root>" : `${b}/`)).join(", ");
+      return {
+        file,
+        line,
+        token,
+        message: `${file}:${line}: \`${token}\` does not exist (looked under ${looked}). Fix the path, or append ${PATH_ALLOW} to the line if it is not a repo path.`,
+      };
+    });
+}
+
+function unique<T>(xs: T[]): T[] {
+  return [...new Set(xs)];
+}
+
+function usage(): never {
+  console.error(
+    "usage: path-existence.ts [repoRoot] [--root <dir>]... [--exclude <glob>]...\n" +
+      "  --root     extra base a doc path may be relative to (repeatable)\n" +
+      "  --exclude  glob of docs to leave out, e.g. docs/review/** (repeatable)",
+  );
+  process.exit(2);
+}
+
+if (import.meta.main) {
+  const args = process.argv.slice(2);
+  let repoRoot = process.cwd();
+  const roots: string[] = [];
+  const exclude: string[] = [];
+  for (let i = 0; i < args.length; i++) {
+    const arg = args[i] as string;
+    if (arg === "--root") roots.push(args[++i] ?? usage());
+    else if (arg === "--exclude") exclude.push(args[++i] ?? usage());
+    else if (arg.startsWith("-")) usage();
+    else repoRoot = arg;
+  }
+  const tree = loadTree(repoRoot);
+  const files = docFilesFor(tree, exclude);
+  const findings = checkPathExistence(repoRoot, files, { tree, roots });
+  if (findings.length === 0) {
+    console.log(`docs path-check: ${files.length} file(s), every cited repo path exists.`);
+    process.exit(0);
+  }
+  console.error(`docs path-check FAILED (${findings.length} dead path(s) across ${files.length} file(s)):`);
+  for (const f of findings) console.error(`  - ${f.message}`);
+  process.exit(1);
+}

package/package.json

@@ -1,12 +1,13 @@
 {
   "name": "@norskvideo/ctl-dev-kit",
-  "version": "0.1.48",
+  "version": "0.1.50",
   "type": "module",
   "exports": {
     "./package.json": "./package.json",
     "./testing/invariants": "./testing/invariants.ts",
     "./testing/byte-snapshot": "./testing/byte-snapshot.ts",
     "./create-product": "./create-product/create-product.ts",
+    "./docs/path-existence": "./docs/path-existence.ts",
     "./doc-guide": "./doc-guide/doc-guide.js",
     "./doc-guide/build-manual": "./doc-guide/build-manual.js",
     "./doc-guide/guides-config": "./doc-guide/guides-config.js",