@avi2dg/checks 0.14.0 → 0.16.0

package/scripts/doc-snapshot.ts

@@ -0,0 +1,70 @@
+import { Effect, FileSystem, Path, Schema } from "effect";
+import { anchoredTargets, anchorsOf, PACKAGE_MANIFEST, packagesOf, snapshotOf, type Unresolved } from "./doc-references.ts";
+import { git, pathsAt } from "./git.ts";
+
+export class ManifestUnreadable extends Schema.TaggedError<ManifestUnreadable>()("ManifestUnreadable", {
+  message: Schema.String,
+}) {}
+
+const Scripts = Schema.fromJsonString(Schema.Struct({ scripts: Schema.optionalKey(Schema.Record(Schema.String, Schema.String)) }));
+const decodeScripts = Schema.decodeUnknownEffect(Scripts);
+
+export const readTexts = Effect.fn("readTexts")(function* (root: string, rev: string, paths: readonly string[]) {
+  const texts = yield* Effect.forEach(paths, (path) => git(["show", `${rev}:${path}`], root), { concurrency: 8 });
+  return new Map(paths.map((path, index) => [path, texts[index] ?? ""]));
+});
+
+function manifestIn(directory: string): string {
+  return directory === "" ? PACKAGE_MANIFEST : `${directory}/${PACKAGE_MANIFEST}`;
+}
+
+const scriptsAt = Effect.fn("scriptsAt")(function* (root: string, rev: string, directory: string) {
+  const manifest = manifestIn(directory);
+  const { scripts = {} } = yield* git(["show", `${rev}:${manifest}`], root).pipe(
+    Effect.flatMap(decodeScripts),
+    Effect.mapError((cause) => new ManifestUnreadable({ message: `${manifest} at ${rev}: ${cause.message}` })),
+  );
+  return new Set(Object.keys(scripts));
+});
+
+export const snapshotAt = Effect.fn("snapshotAt")(function* (
+  root: string,
+  rev: string,
+  docs: ReadonlyMap<string, string>,
+  alsoRoots: readonly string[],
+) {
+  const files = yield* pathsAt(rev, [], root);
+  const tracked = snapshotOf(files, new Map(), new Map());
+  const targets = [...new Set([...docs].flatMap(([doc, text]) => anchoredTargets(doc, text)))].filter((path) => tracked.files.has(path));
+  const read = yield* readTexts(root, rev, targets.filter((path) => !docs.has(path)));
+  const anchors = new Map(targets.map((path) => [path, anchorsOf(docs.get(path) ?? read.get(path) ?? "")]));
+  const packages = packagesOf([...docs.keys()], tracked);
+  const scripts = yield* Effect.forEach(packages, (directory) => scriptsAt(root, rev, directory), { concurrency: 8 });
+  return snapshotOf(files, anchors, new Map(packages.map((directory, index) => [directory, scripts[index] ?? new Set<string>()])), alsoRoots);
+});
+
+const ignored = (root: string, path: string) =>
+  git(["check-ignore", "-q", "--no-index", "--", path], root).pipe(
+    Effect.as(true),
+    Effect.catchTag("GitFailure", () => Effect.succeed(false)),
+  );
+
+// A script bun runs from an installed dependency's bin names no entry in package.json.
+const installedBin = Effect.fn("installedBin")(function* (root: string, directory: string, name: string) {
+  const fs = yield* FileSystem.FileSystem;
+  const path = yield* Path.Path;
+  const bins = [...new Set([directory, ""])].map((from) => path.join(root, from, "node_modules", ".bin", name));
+  const found = yield* Effect.forEach(bins, (bin) => fs.exists(bin).pipe(Effect.orElseSucceed(() => false)));
+  return found.some(Boolean);
+});
+
+// A path git ignores is generated or fetched, so a clean checkout lacks it by design.
+const stillBroken = Effect.fn("stillBroken")(function* (root: string, { missing }: Unresolved) {
+  if (missing.type === "file") return !(yield* ignored(root, missing.path));
+  if (missing.type === "script") return !(yield* installedBin(root, missing.packageDirectory, missing.name));
+  return true;
+});
+
+export const stillMissing = Effect.fn("stillMissing")(function* (root: string, unresolved: readonly Unresolved[]) {
+  return yield* Effect.forEach(unresolved, (one) => stillBroken(root, one), { concurrency: 8 });
+});

package/scripts/docs.ts

@@ -1,8 +1,11 @@
 #!/usr/bin/env bun
 import { Console, Effect } from "effect";
+import { rootsOf, unresolvedIn, type Judging, type Unresolved } from "./doc-references.ts";
 import { ADR_DIRECTORY, judge, placementOf, placementProblem, type Placement } from "./doc-rules.ts";
-import { changedPaths, git, pathsAt, rangeEnds } from "./git.ts";
+import { readTexts, snapshotAt, stillMissing } from "./doc-snapshot.ts";
+import { changedLines, changedPaths, git, pathsAt, rangeEnds } from "./git.ts";
 import { runMain, Usage } from "./main.ts";
+import { isLivingDoc, proseFindings, readerOf } from "./prose-matchers.ts";
 import { readQuality } from "./quality-file.ts";
 
 type Finding = {
@@ -13,46 +16,111 @@ type Finding = {
 
 type Judged = {
   readonly held: readonly string[];
+  readonly edited: { readonly docs: number; readonly lines: number };
+  readonly living: number;
   readonly findings: readonly Finding[];
   readonly advisory: ReadonlyMap<string, number>;
+  readonly brokenBefore: readonly Finding[];
 };
 
+type Range = {
+  readonly root: string;
+  readonly base: string;
+  readonly head: string;
+  readonly roots: readonly string[];
+  readonly changed: ReadonlyMap<string, ReadonlySet<number>>;
+  readonly renamedFrom: ReadonlyMap<string, string>;
+};
+
+type Located = Unresolved & { readonly path: string };
+
 const NAME = "docs";
 const USAGE = "usage: docs.ts <ref> | <base-ref> <head-ref>";
 const MARKDOWN = [":(glob)**/*.md"];
 
-const judgeFile = Effect.fn("judgeFile")(function* (
-  root: string,
-  head: string,
-  path: string,
-  placement: Placement,
-  records: readonly string[],
-) {
+function templateFindings(path: string, text: string, placement: Placement, records: readonly string[]): readonly Finding[] {
   const misplaced = placementProblem(placement);
   if (misplaced !== undefined) return [{ path, line: undefined, message: misplaced }];
   if (placement.type !== "judged") return [];
-  const text = yield* git(["show", `${head}:${path}`], root);
   return judge(placement.kind, { path, text }, records).map(({ line, message }) => ({ path, line, message }));
+}
+
+function inPathOrder(a: Finding, b: Finding): number {
+  return a.path === b.path ? (a.line ?? 0) - (b.line ?? 0) : a.path < b.path ? -1 : 1;
+}
+
+function locate(path: string, text: string, snapshot: Parameters<typeof unresolvedIn>[2], judging: Judging): readonly Located[] {
+  return unresolvedIn(path, text, snapshot, judging).map((unresolved) => ({ ...unresolved, path }));
+}
+
+const keyOf = ({ path, kind, named }: Located): string => `${path}\0${kind}\0${named}`;
+
+// A reference on a line the range leaves alone fails only when the range broke it, as by deleting the file it names.
+const brokenBeforeRange = Effect.fn("brokenBeforeRange")(function* (range: Range, found: readonly Located[], judging: (path: string) => Judging) {
+  const earlier = new Set(yield* pathsAt(range.base, MARKDOWN, range.root));
+  const docs = [...new Set(found.map(({ path }) => path))]
+    .map((path) => ({ path, from: range.renamedFrom.get(path) ?? path }))
+    .filter(({ from }) => earlier.has(from));
+  const texts = yield* readTexts(range.root, range.base, docs.map(({ from }) => from));
+  const snapshot = yield* snapshotAt(range.root, range.base, texts, range.roots);
+  return new Set(docs.flatMap(({ path, from }) => locate(from, texts.get(from) ?? "", snapshot, judging(path)).map((one) => keyOf({ ...one, path }))));
+});
+
+const referenceFindings = Effect.fn("referenceFindings")(function* (range: Range, texts: ReadonlyMap<string, string>, judging: (path: string) => Judging) {
+  const snapshot = yield* snapshotAt(range.root, range.head, texts, range.roots);
+  const unresolved = [...texts].flatMap(([path, text]) => locate(path, text, snapshot, judging(path)));
+  const missing = yield* stillMissing(range.root, unresolved);
+  const found = unresolved.filter((_, index) => missing[index] === true);
+  const onChangedLines = (one: Located): boolean => range.changed.get(one.path)?.has(one.line) === true;
+  const elsewhere = found.filter((one) => !onChangedLines(one));
+  const before = elsewhere.length === 0 ? new Set<string>() : yield* brokenBeforeRange(range, elsewhere, judging);
+  const finding = ({ path, line, message }: Located): Finding => ({ path, line, message });
+  return {
+    failing: found.filter((one) => onChangedLines(one) || !before.has(keyOf(one))).map(finding),
+    brokenBefore: elsewhere.filter((one) => before.has(keyOf(one))).map(finding),
+  };
 });
 
 const runDocs = Effect.fn("runDocs")(function* (root: string, base: string, head: string) {
   const { quality } = yield* readQuality(root);
-  const touched = new Set(
-    (yield* changedPaths(base, head, MARKDOWN, root)).flatMap((change) => (change.kind === "deleted" ? [] : [change.path])),
-  );
+  const changes = yield* changedPaths(base, head, MARKDOWN, root);
+  const touched = new Set(changes.flatMap((change) => (change.kind === "deleted" ? [] : [change.path])));
+  const renamedFrom = new Map(changes.flatMap((change) => (change.kind === "renamed" ? [[change.path, change.from] as const] : [])));
+  const changed = yield* changedLines(base, head, MARKDOWN, root);
   const present = yield* pathsAt(head, MARKDOWN, root);
   const records = present.filter((path) => path.startsWith(ADR_DIRECTORY));
-  const placed = present.map((path) => ({ path, placement: placementOf(path, quality.docs) }));
-  const judged = placed.filter(({ placement }) => placement.type !== "unjudged");
-  const findings = (yield* Effect.forEach(judged, ({ path, placement }) => judgeFile(root, head, path, placement, records), {
-    concurrency: 8,
-  })).flat();
+  const judged = present.map((path) => ({ path, placement: placementOf(path, quality.docs) })).filter(({ placement }) => placement.type !== "unjudged");
+  const living = present.filter(isLivingDoc);
+  const proseDocs = present.flatMap((path) => {
+    const reader = readerOf(path);
+    return reader === undefined ? [] : [{ path, reader }];
+  });
+  const texts = yield* readTexts(root, head, [...new Set([...judged.map(({ path }) => path), ...proseDocs.map(({ path }) => path)])]);
+  const text = (path: string): string => texts.get(path) ?? "";
+
+  const templated = judged.flatMap(({ path, placement }) => templateFindings(path, text(path), placement, records));
+  const edited = proseDocs.filter(({ path }) => changed.has(path));
+  const prose = edited.flatMap(({ path, reader }) =>
+    proseFindings(text(path), reader, changed.get(path)).map(({ line, message }) => ({ path, line, message })),
+  );
+  const forConsumers = (quality.docs?.forConsumers ?? []).map((glob) => new Bun.Glob(glob));
+  const judging = (path: string): Judging => ({ commands: !forConsumers.some((glob) => glob.match(path)) });
+  // A directory the range deletes still belongs to this repository, so a path under it is stale rather than another repository's.
+  const roots = rootsOf(yield* pathsAt(base, [], root));
+  const references = yield* referenceFindings(
+    { root, base, head, roots, changed, renamedFrom },
+    new Map(living.map((path) => [path, text(path)])),
+    judging,
+  );
   const advisory = new Map<string, number>();
-  for (const { path } of findings.filter((finding) => !touched.has(finding.path))) advisory.set(path, (advisory.get(path) ?? 0) + 1);
+  for (const { path } of templated.filter((finding) => !touched.has(finding.path))) advisory.set(path, (advisory.get(path) ?? 0) + 1);
   return {
     held: judged.map(({ path }) => path).filter((path) => touched.has(path)),
-    findings: findings.filter((finding) => touched.has(finding.path)),
+    edited: { docs: edited.length, lines: edited.reduce((sum, { path }) => sum + (changed.get(path)?.size ?? 0), 0) },
+    living: living.length,
+    findings: [...templated.filter((finding) => touched.has(finding.path)), ...prose, ...references.failing].toSorted(inPathOrder),
     advisory,
+    brokenBefore: references.brokenBefore.toSorted(inPathOrder),
   } satisfies Judged;
 });
 
@@ -60,19 +128,27 @@ function describe({ path, line, message }: Finding): string {
   return `  ${path}${line === undefined ? "" : `:${line}`}: ${message}`;
 }
 
-export function report({ held, findings, advisory }: Judged): string {
+export function report({ held, edited, living, findings, advisory, brokenBefore }: Judged): string {
   const verdict =
     findings.length === 0
-      ? [`${NAME}: ${held.length} doc file(s) the range touches hold to their templates`]
-      : [`${NAME}: ${findings.length} violation(s) in the doc files the range touches:`, ...findings.map(describe)];
-  const notice =
+      ? [
+          `${NAME}: ${held.length} doc file(s) the range touches hold to their templates`,
+          `${NAME}: ${edited.lines} line(s) the range adds or edits in ${edited.docs} living doc(s) or agent file(s) hold to the prose rules`,
+          `${NAME}: the range breaks no path, link or command the ${living} living doc(s) name`,
+        ]
+      : [`${NAME}: ${findings.length} violation(s):`, ...findings.map(describe)];
+  const unconformed =
     advisory.size === 0
       ? []
       : [
           `${NAME}: advisory, ${advisory.size} doc file(s) the range leaves alone do not hold to their templates yet:`,
           ...[...advisory].map(([path, count]) => `  ${path}: ${count} violation(s)`),
         ];
-  return [...verdict, ...notice].join("\n");
+  const broken =
+    brokenBefore.length === 0
+      ? []
+      : [`${NAME}: advisory, ${brokenBefore.length} path(s), link(s) or command(s) the living docs name were broken before the range:`, ...brokenBefore.map(describe)];
+  return [...verdict, ...unconformed, ...broken].join("\n");
 }
 
 const docs = Effect.gen(function* () {

package/scripts/gates.ts

@@ -40,6 +40,7 @@ export const KIT_GATES = [
   { bin: "checks-docs", script: "docs.ts", reads: "range", appliesTo: EVERY_REPOSITORY },
   { bin: "checks-quality", script: "quality.ts", reads: "tree", args: ["--check"], appliesTo: QUALITY_DECLARATION },
   { bin: "checks-size-budget", script: "size-budget.ts", reads: "range", appliesTo: TYPESCRIPT_SOURCE },
+  { bin: "checks-repetition", script: "repetition.ts", reads: "range", appliesTo: TYPESCRIPT_SOURCE },
   { bin: "checks-feature-owners", script: "feature-owners.ts", reads: "range", appliesTo: TYPESCRIPT_SOURCE },
 ] as const satisfies readonly KitGate[];
 

package/scripts/git.ts

@@ -1,5 +1,6 @@
-import { Effect, Schema, Stream } from "effect";
+import { Effect, Path, Schema, Stream } from "effect";
 import { ChildProcess, ChildProcessSpawner } from "effect/unstable/process";
+import { Usage } from "./main.ts";
 
 export class GitFailure extends Schema.TaggedError<GitFailure>()("GitFailure", {
   message: Schema.String,
@@ -71,6 +72,70 @@ export const changedPaths = Effect.fn("changedPaths")(function* (
   return parseNameStatus(yield* git(["diff", "--name-status", "-z", "-M", base, head, "--", ...pathspecs], cwd));
 });
 
+const ESCAPES: Readonly<Record<string, string>> = { a: "\x07", b: "\b", f: "\f", n: "\n", r: "\r", t: "\t", v: "\v" };
+
+// Git quotes a path holding a quote, a backslash or a control character, and ends one holding a space with a tab.
+function newPathOf(line: string): string | undefined {
+  const named = line.slice("+++ ".length).replace(/\t$/, "");
+  const path = named.startsWith('"')
+    ? named.slice(1, -1).replace(/\\(?:([0-7]{3})|(.))/g, (_, octal: string | undefined, char: string) =>
+        octal === undefined ? (ESCAPES[char] ?? char) : String.fromCharCode(Number.parseInt(octal, 8)),
+      )
+    : named;
+  return path === "/dev/null" ? undefined : path;
+}
+
+const HUNK = /^@@ -\d+(?:,(\d+))? \+(\d+)(?:,(\d+))? @@/;
+
+// A hunk's header counts its lines, so an added line reading `++ x` is not taken for the next file's `+++` header.
+export function parseAddedLines(diff: string): Map<string, Set<number>> {
+  const added = new Map<string, Set<number>>();
+  let path: string | undefined;
+  let line = 0;
+  let oldLeft = 0;
+  let newLeft = 0;
+  for (const row of diff.split("\n")) {
+    if (oldLeft > 0 || newLeft > 0) {
+      if (row.startsWith("+")) {
+        if (path !== undefined) added.set(path, (added.get(path) ?? new Set<number>()).add(line));
+        line += 1;
+        newLeft -= 1;
+      } else if (row.startsWith("-")) {
+        oldLeft -= 1;
+      } else if (row.startsWith(" ")) {
+        line += 1;
+        oldLeft -= 1;
+        newLeft -= 1;
+      }
+      continue;
+    }
+    if (row.startsWith("+++ ")) {
+      path = newPathOf(row);
+      continue;
+    }
+    const hunk = HUNK.exec(row);
+    if (hunk !== null) {
+      oldLeft = Number(hunk[1] ?? 1);
+      line = Number(hunk[2]);
+      newLeft = Number(hunk[3] ?? 1);
+    }
+  }
+  return added;
+}
+
+export const changedLines = Effect.fn("changedLines")(function* (
+  base: string,
+  head: string,
+  pathspecs: readonly string[],
+  cwd?: string,
+) {
+  const diff = yield* git(
+    ["-c", "core.quotePath=false", "diff", "-U0", "--no-color", "--no-prefix", "-M", base, head, "--", ...pathspecs],
+    cwd,
+  );
+  return parseAddedLines(diff);
+});
+
 const emptyTree = (cwd?: string) => git(["hash-object", "-t", "tree", "/dev/null"], cwd).pipe(Effect.map((sha) => sha.trim()));
 
 export const pathsAt = Effect.fn("pathsAt")(function* (rev: string, pathspecs: readonly string[], cwd?: string) {
@@ -95,3 +160,20 @@ export const rangeEnds = Effect.fn("rangeEnds")(function* (first: string, second
   if (second === undefined) return { base: yield* parentOrEmptyTree(first, cwd), head: first };
   return { base: (yield* git(["merge-base", first, second], cwd)).trim(), head: second };
 });
+
+export const rangeFromArgs = Effect.fn("rangeFromArgs")(function* (args: readonly string[], usage: string, cwd?: string) {
+  const [first, second, ...extra] = args;
+  if (first === undefined || extra.length > 0) return yield* new Usage({ message: usage });
+  return yield* rangeEnds(first, second, cwd);
+});
+
+// A scratch index leaves the repository's own index and working tree untouched.
+export const checkoutFiles = Effect.fn("checkoutFiles")(function* (rev: string, files: readonly string[], scratch: string, cwd?: string) {
+  const path = yield* Path.Path;
+  const env = { GIT_INDEX_FILE: path.join(scratch, "index") };
+  const tree = path.join(scratch, "tree");
+  yield* git(["read-tree", rev], cwd, { env });
+  const listed = files.map((file) => `${file}\0`).join("");
+  yield* git(["checkout-index", "-z", "--stdin", `--prefix=${tree}${path.sep}`], cwd, { env, input: listed });
+  return tree;
+});