@avi2dg/checks 0.31.0 → 0.33.0

package/src/docs/doc-agents.ts

@@ -1,11 +1,9 @@
+import { Parser, type NodeType } from "commonmark";
 import { commandNames, type Snapshot } from "./doc-references.ts";
 import { AGENT_NAMES, scanMarkdown, type MarkdownLine } from "./prose-matchers.ts";
 
 export const AGENT_CEILING = 3000;
 
-const LIST_ITEM = /^(?:\s*>)*\s*(?:[-*+]|\d{1,9}[.)])(?:\s|$)/;
-const INDENT = /^[ \t]*/;
-const CODE_INDENT = 4;
 const INLINE_LINK =
   /(?<![!\\])\[(?:[^[\]\\]|\\.|\[[^\]]*\])*\]\(\s*(?:<[^<>\n]+>|[^\s()<>]+(?:\([^\s()]*\)[^\s()<>]*)*)(?:\s+(?:"[^"]*"|'[^']*'))?\s*\)/g;
 const MAINTAINING = /^\s{0,3}#{1,6}\s+Maintaining this file(?:\s+#+)?\s*$/;
@@ -27,8 +25,24 @@ export function ceilingFinding(text: string): AgentFinding | undefined {
   };
 }
 
+function blockStarts(lines: readonly MarkdownLine[], type: NodeType): ReadonlySet<number> {
+  const read = lines.map(({ kind, raw }) => (kind === "front-matter" ? "" : raw)).join("\n");
+  const walker = new Parser().parse(read).walker();
+  const starts = new Set<number>();
+  for (let step = walker.next(); step !== null; step = walker.next()) {
+    if (step.entering && step.node.type === type) starts.add(step.node.sourcepos[0][0]);
+  }
+  return starts;
+}
+
+function linesOpening(text: string, type: NodeType): readonly MarkdownLine[] {
+  const lines = scanMarkdown(text);
+  const starts = blockStarts(lines, type);
+  return lines.filter(({ line }) => starts.has(line)).flatMap(({ line, raw }) => scanMarkdown(raw).map((alone) => ({ ...alone, line })));
+}
+
 export function maintainingFinding(text: string): AgentFinding | undefined {
-  const heading = scanMarkdown(text).find(({ kind, raw }) => kind === "heading" && MAINTAINING.test(raw));
+  const heading = linesOpening(text, "heading").find(({ raw }) => MAINTAINING.test(raw));
   if (heading === undefined) return undefined;
   return {
     line: heading.line,
@@ -36,27 +50,14 @@ export function maintainingFinding(text: string): AgentFinding | undefined {
   };
 }
 
-function indentOf(raw: string): number {
-  return (INDENT.exec(raw)?.[0] ?? "").replaceAll("\t", "    ").length;
-}
-
 export function entries(text: string): readonly MarkdownLine[] {
-  let inList = false;
-  let afterBlank = true;
-  return scanMarkdown(text).filter((line) => {
-    const blank = line.raw.trim() === "";
-    const indented = !blank && indentOf(line.raw) >= CODE_INDENT && !line.raw.trimStart().startsWith(">");
-    const listItem = line.kind === "prose" && LIST_ITEM.test(line.prose) && (inList || !indented);
-    if (listItem) inList = true;
-    else if (!blank && !indented && (afterBlank || line.kind !== "prose")) inList = false;
-    afterBlank = blank;
-    return listItem;
-  });
+  return linesOpening(text, "item");
 }
 
 type Tracked = Pick<Snapshot, "files" | "directories">;
 
 const STAYS = new Set(["", "."]);
+const NAMES_A_SEGMENT = /[^/]/;
 
 function joined(directory: string, segment: string): string {
   return directory === "" ? segment : `${directory}/${segment}`;
@@ -76,10 +77,11 @@ function resolvesFrom(directory: string, span: string, tracked: Tracked): boolea
   if (walked === undefined) return false;
   if (!STAYS.has(last) && last !== ".." && tracked.files.has(joined(walked, last))) return true;
   const end = enter(walked, last, tracked);
-  return end !== undefined && end !== "" && end !== directory;
+  return end !== undefined && end !== "";
 }
 
 function namesTrackedPath(agentFile: string, span: string, tracked: Tracked): boolean {
+  if (!NAMES_A_SEGMENT.test(span)) return false;
   const directory = agentFile.includes("/") ? agentFile.slice(0, agentFile.lastIndexOf("/")) : "";
   const fromFile = resolvesFrom(directory, span, tracked);
   return fromFile || (!span.startsWith("./") && !span.startsWith("../") && resolvesFrom("", span, tracked));

package/src/docs/doc-names.ts

@@ -1,5 +1,5 @@
 import { Effect, FileSystem, Option, Path, Schema } from "effect";
-import { ChildProcessSpawner } from "effect/unstable/process";
+import { ChildProcessSpawner } from "effect/process";
 import { collect, git, pathsAt } from "../core/git.ts";
 import type { Unresolved } from "./doc-references.ts";
 import { scanMarkdown } from "./prose-matchers.ts";

package/src/docs/doc-references.ts

@@ -78,20 +78,28 @@ function unresolvedPath(doc: string, line: number, span: string, snapshot: Snaps
 const SCHEME = /^(?:[a-z][a-z0-9+.-]*:|\/\/)/i;
 const ASCII_ESCAPE = /%([0-7][0-9a-f])/gi;
 
+function linkPath(doc: string, target: string, hash: number): string | undefined {
+  const written = (hash < 0 ? target : target.slice(0, hash)).split("?", 1)[0] ?? "";
+  const decoded = written.replace(ASCII_ESCAPE, (_, code: string) => String.fromCharCode(Number.parseInt(code, 16)));
+  return decoded === "" ? doc : decoded.startsWith("/") ? normalize(decoded) : within(directoryOf(doc), decoded);
+}
+
 function unresolvedLink(doc: string, line: number, target: string, snapshot: Snapshot): Unresolved | undefined {
   if (target === "" || SCHEME.test(target)) return undefined;
   const hash = target.indexOf("#");
-  const written = (hash < 0 ? target : target.slice(0, hash)).split("?", 1)[0] ?? "";
-  const decoded = written.replace(ASCII_ESCAPE, (_, code: string) => String.fromCharCode(Number.parseInt(code, 16)));
-  const path = decoded === "" ? doc : decoded.startsWith("/") ? normalize(decoded) : within(directoryOf(doc), decoded);
+  const path = linkPath(doc, target, hash);
   if (path === undefined) return undefined;
   const named = target;
   if (!exists(snapshot, path)) {
     return { kind: "link", line, named, message: `links to \`${named}\`, which is not in the repository`, missing: { type: "file", path } };
   }
   const anchor = hash < 0 ? "" : target.slice(hash + 1);
+  if (anchor === "") return undefined;
   const anchors = snapshot.anchors.get(path);
-  if (anchor === "" || anchors === undefined || anchors.has(anchor) || anchors.has(anchor.toLowerCase())) return undefined;
+  if (anchors === undefined) {
+    return { kind: "link", line, named, message: `links to \`${named}\`, and \`${path}\` has no headings to check`, missing: { type: "anchor" } };
+  }
+  if (anchors.has(anchor) || anchors.has(anchor.toLowerCase())) return undefined;
   return { kind: "link", line, named, message: `links to \`${named}\`, and \`${path}\` has no heading with that anchor`, missing: { type: "anchor" } };
 }
 
@@ -152,9 +160,8 @@ export function anchoredTargets(doc: string, text: string): readonly string[] {
     links.flatMap((target) => {
       const hash = target.indexOf("#");
       if (hash < 0 || SCHEME.test(target)) return [];
-      const written = target.slice(0, hash);
-      const path = written === "" ? doc : written.startsWith("/") ? normalize(written) : within(directoryOf(doc), written);
-      return path?.endsWith(".md") === true ? [path] : [];
+      const path = linkPath(doc, target, hash);
+      return path !== undefined && (path.endsWith(".md") || path.endsWith(".mdx")) ? [path] : [];
     }),
   );
 }

package/src/docs/docs.ts

@@ -8,6 +8,7 @@ import { readTexts, snapshotAt, stillMissing } from "./doc-snapshot.ts";
 import { changedLines, changedPaths, git, pathsAt, rangeEnds, refArgs } from "../core/git.ts";
 import { runMain } from "../core/main.ts";
 import { proseFindings, readerOf } from "./prose-matchers.ts";
+import { longSentences } from "./sentence-length.ts";
 
 type Finding = {
   readonly path: string;
@@ -22,6 +23,7 @@ type Judged = {
   readonly agents: number;
   readonly findings: readonly Finding[];
   readonly advisory: ReadonlyMap<string, number>;
+  readonly sentenceAdvisory: readonly Finding[];
   readonly brokenBefore: readonly Finding[];
 };
 
@@ -105,6 +107,9 @@ const runDocs = Effect.fn("runDocs")(function* (root: string, base: string, head
   const prose = edited.flatMap(({ path, reader }) =>
     proseFindings(text(path), reader, changed.get(path)).map(({ line, message }) => ({ path, line, message })),
   );
+  const sentenceAdvisory = edited
+    .flatMap(({ path }) => longSentences(text(path), changed.get(path)).map(({ line, message }) => ({ path, line, message })))
+    .toSorted(inPathOrder);
   const judging = (path: string): Judging => ({ commands: !speaksToConsumers(text(path)) });
   // A directory the range deletes still belongs to this repository, so a path under it is stale rather than another repository's.
   const roots = rootsOf(yield* pathsAt(base, [], root));
@@ -133,6 +138,7 @@ const runDocs = Effect.fn("runDocs")(function* (root: string, base: string, head
       ...entries,
     ].toSorted(inPathOrder),
     advisory,
+    sentenceAdvisory,
     brokenBefore: references.brokenBefore.toSorted(inPathOrder),
   } satisfies Judged;
 });
@@ -141,7 +147,7 @@ function describe({ path, line, message }: Finding): string {
   return `  ${path}${line === undefined ? "" : `:${line}`}: ${message}`;
 }
 
-export function report({ held, edited, named, agents, findings, advisory, brokenBefore }: Judged): string {
+export function report({ held, edited, named, agents, findings, advisory, sentenceAdvisory, brokenBefore }: Judged): string {
   const verdict =
     findings.length === 0
       ? [
@@ -158,6 +164,13 @@ export function report({ held, edited, named, agents, findings, advisory, broken
           `${NAME}: advisory, ${advisory.size} doc file(s) the range leaves alone do not hold to their templates yet:`,
           ...[...advisory].map(([path, count]) => `  ${path}: ${count} violation(s)`),
         ];
+  const sentences =
+    sentenceAdvisory.length === 0
+      ? []
+      : [
+          `${NAME}: advisory, ${sentenceAdvisory.length} sentence(s) over the trial length caps:`,
+          ...sentenceAdvisory.map(describe),
+        ];
   const broken =
     brokenBefore.length === 0
       ? []
@@ -165,7 +178,7 @@ export function report({ held, edited, named, agents, findings, advisory, broken
           `${NAME}: advisory, ${brokenBefore.length} path(s), link(s) or command(s) the living docs or agent files name were broken before the range:`,
           ...brokenBefore.map(describe),
         ];
-  return [...verdict, ...unconformed, ...broken].join("\n");
+  return [...verdict, ...unconformed, ...sentences, ...broken].join("\n");
 }
 
 const docs = Effect.gen(function* () {

package/src/docs/prose-matchers.ts

@@ -243,7 +243,7 @@ export function scanMarkdown(text: string): readonly MarkdownLine[] {
 
 const LEADING_MARKERS = /^(?:\s*>)*\s*(?:#{1,6}\s+|(?:[-*+]|\d{1,9}[.)])\s+(?:\[[ xX]\]\s+)?)?/;
 
-function bodyOf({ prose }: MarkdownLine): string {
+export function bodyOf({ prose }: MarkdownLine): string {
   const markers = LEADING_MARKERS.exec(prose)?.[0] ?? "";
   return BLANK.repeat(markers.length) + prose.slice(markers.length);
 }

package/src/docs/sentence-length.ts

@@ -0,0 +1,106 @@
+import { bodyOf, scanMarkdown } from "./prose-matchers.ts";
+
+export const PROCEDURAL_WORD_CAP = 20;
+export const DESCRIPTIVE_WORD_CAP = 25;
+
+export type SentenceKind = "procedural" | "descriptive";
+
+export type SentenceLength = {
+  readonly line: number;
+  readonly words: number;
+  readonly kind: SentenceKind;
+};
+
+export type LongSentence = SentenceLength & {
+  readonly message: string;
+};
+
+function graphemesOf(body: string): readonly string[] {
+  return [...new Intl.Segmenter().segment(body)].map(({ segment }) => segment);
+}
+
+function dropPrefix(raw: string): string {
+  const chars = graphemesOf(raw);
+  for (const [at, char] of chars.entries()) {
+    if (char !== " " && char !== "\t" && char !== ">") return chars.slice(at).join("");
+  }
+  return "";
+}
+
+function orderedAt(raw: string): boolean {
+  const rest = dropPrefix(raw);
+  const digits = /^\d+/.exec(rest)?.[0] ?? "";
+  if (digits === "") return false;
+  const marker = rest.charAt(digits.length);
+  if (marker !== "." && marker !== ")") return false;
+  const after = rest.charAt(digits.length + 1);
+  return after === " " || after === "\t" || after === "";
+}
+
+function isWordChar(char: string): boolean {
+  return (char >= "0" && char <= "9") || (char >= "A" && char <= "Z") || (char >= "a" && char <= "z");
+}
+
+function wordsIn(sentence: string): number {
+  let words = 0;
+  let inToken = false;
+  let holdsWordChar = false;
+  for (const char of sentence) {
+    if (char === " " || char === "\t") {
+      if (inToken && holdsWordChar) words += 1;
+      inToken = false;
+      holdsWordChar = false;
+    } else {
+      inToken = true;
+      if (isWordChar(char)) holdsWordChar = true;
+    }
+  }
+  return inToken && holdsWordChar ? words + 1 : words;
+}
+
+function isCloser(char: string): boolean {
+  return char === '"' || char === "'" || char === ")" || char === "]" || char === "*" || char === "_" || char === "”" || char === "’";
+}
+
+function sentencesIn(body: string): readonly string[] {
+  const chars = graphemesOf(body);
+  const sentences: string[] = [];
+  let start = 0;
+  let cut = -1;
+  for (const [at, char] of chars.entries()) {
+    if (char === "." || char === "!" || char === "?") cut = at + 1;
+    else if (cut === at && isCloser(char)) cut = at + 1;
+    else if (cut > start && (char === " " || char === "\t")) {
+      sentences.push(chars.slice(start, cut).join(""));
+      start = cut;
+      cut = -1;
+    } else if (cut > start) cut = -1;
+  }
+  if (cut === chars.length && cut > start) {
+    sentences.push(chars.slice(start, cut).join(""));
+    start = cut;
+  }
+  const tail = chars.slice(start).join("");
+  if (tail.trim() !== "") sentences.push(tail);
+  return sentences;
+}
+
+export function sentenceLengths(text: string, within?: ReadonlySet<number>): readonly SentenceLength[] {
+  return scanMarkdown(text).flatMap((line) => {
+    if (line.kind !== "prose" || (within !== undefined && !within.has(line.line))) return [];
+    const kind: SentenceKind = orderedAt(line.raw) ? "procedural" : "descriptive";
+    return sentencesIn(bodyOf(line)).map((sentence) => ({ line: line.line, words: wordsIn(sentence), kind }));
+  });
+}
+
+export function longSentences(text: string, within?: ReadonlySet<number>): readonly LongSentence[] {
+  return sentenceLengths(text, within).flatMap(({ line, words, kind }) => {
+    const cap = kind === "procedural" ? PROCEDURAL_WORD_CAP : DESCRIPTIVE_WORD_CAP;
+    if (words <= cap) return [];
+    const rule =
+      kind === "procedural"
+        ? `procedural means an ordered list item; descriptive caps at ${DESCRIPTIVE_WORD_CAP} words`
+        : `procedural means an ordered list item, capped at ${PROCEDURAL_WORD_CAP} words`;
+    return [{ line, words, kind, message: `carries a ${words}-word ${kind} sentence, over the ${cap}-word cap (${rule})` }];
+  });
+}

package/src/testing/flake.ts

@@ -1,6 +1,6 @@
 #!/usr/bin/env bun
 import { Config, Console, Effect, FileSystem, Option, Path, Random, Schema } from "effect";
-import { ChildProcess, ChildProcessSpawner } from "effect/unstable/process";
+import { ChildProcess, ChildProcessSpawner } from "effect/process";
 import { runMain, Usage } from "../core/main.ts";
 import { NAME_SEPARATOR, parseReport, ReportError, reporterArgs, type TestResult } from "./test-report.ts";
 

package/src/testing/mutation.ts

@@ -1,6 +1,6 @@
 #!/usr/bin/env bun
 import { Config, Effect, Schema } from "effect";
-import { ChildProcess, ChildProcessSpawner } from "effect/unstable/process";
+import { ChildProcess, ChildProcessSpawner } from "effect/process";
 import { runMain } from "../core/main.ts";
 import { fullRunRefusal } from "./mutation-scope.js";
 

package/src/testing/test.ts

@@ -1,6 +1,6 @@
 #!/usr/bin/env bun
 import { Config, Console, Effect, FileSystem, Path, Schema } from "effect";
-import { ChildProcess, ChildProcessSpawner } from "effect/unstable/process";
+import { ChildProcess, ChildProcessSpawner } from "effect/process";
 import { TEST_ENTRY_POINT } from "../core/gates.ts";
 import { runMain, Usage } from "../core/main.ts";
 import { readSkipDeclarations, type Environment, type SkipDeclaration, type TestTier } from "./test-skips.ts";