@jpeakai/prose-gates 0.2.0 → 0.3.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@jpeakai/prose-gates",
-  "version": "0.2.0",
+  "version": "0.3.0",
   "type": "module",
   "description": "Deterministic prose gates for markdown documentation",
   "license": "MIT",
@@ -48,6 +48,7 @@
     "examples"
   ],
   "dependencies": {
+    "compromise": "14.18.0",
     "mdast-util-from-markdown": "^2.0.3",
     "mdast-util-frontmatter": "^2.0.1",
     "mdast-util-gfm": "^3.1.0",
@@ -58,6 +59,7 @@
   },
   "devDependencies": {
     "@biomejs/biome": "^2",
+    "@jpeakai/mermaidjs-lint": "^0.7.2",
     "@types/bun": "latest",
     "@types/mdast": "^4",
     "fast-check": "^4.10.0",

package/src/check.ts

@@ -1,14 +1,14 @@
 // Checking: build the model once, run every active rule over it, and audit
 // embedded markdown fences as documents of their own.
 //
-// Nothing is modified in check mode and no rule reads another's result, so every
-// rule is started at once and the engine waits for all of them. A rule may be
+// Nothing is modified in check mode. Every rule is started at once, except a
+// rule that names categories in `after`, which waits for those findings. A rule may be
 // synchronous or return a promise, so concurrency is as good as parallelism
 // here: a rule that waits on a tool or a service overlaps with the others.
 
 import { buildDocModel, type DocModel } from "./model.ts";
 import { BUILTIN, type Registry, ruleContext, TEXT_HELPERS } from "./rules/registry.ts";
-import type { Finding } from "./rules/types.ts";
+import type { Finding, Rule } from "./rules/types.ts";
 
 export const DEFAULT_MAX_WORDS = 25;
 
@@ -24,19 +24,31 @@ export const checkModel = async (
   registry: Registry = BUILTIN,
 ): Promise<Finding[]> => {
   const shared = ruleContext(doc, registry);
-  const perRule = registry.rules.map(async (rule) => {
+  const run = async (rule: Rule, earlier: readonly Finding[]): Promise<Finding[]> => {
     const options = registry.options.get(rule.id) ?? {};
     return rule.check({
       ...shared,
       file,
       maxWords: budgetFor(maxWords, options.maxWords),
+      earlier,
       options,
       helpers: TEXT_HELPERS,
     });
-  });
+  };
+  // Rules that read no other rule's findings start at once. A rule with
+  // `after` waits for them and reads what its named categories reported.
+  const categoryOf = new Map(registry.rules.map((r) => [r.id, r.category]));
+  const first = Promise.all(registry.rules.filter((r) => !r.after).map((r) => run(r, [])));
+  const later = registry.rules
+    .filter((r) => r.after)
+    .map(async (rule) => {
+      const earlier = (await first).flat().filter((f) => rule.after?.includes(categoryOf.get(f.rule) ?? ""));
+      return run(rule, earlier);
+    });
+  const perRule = [first.then((found) => found.flat()), ...later];
   // The fence body starts one line below the opening fence.
   const perFence = doc.fences.map(async (fence) => {
-    const inner = await checkModel(buildDocModel(fence.value), file, maxWords, registry);
+    const inner = await checkModel(buildDocModel(fence.value, registry.concepts), file, maxWords, registry);
     return inner.map((f) => ({ ...f, line: fence.line + f.line }));
   });
   const findings = (await Promise.all([...perRule, ...perFence])).flat();
@@ -48,4 +60,4 @@ export const checkMarkdown = (
   file: string,
   maxWords?: number,
   registry: Registry = BUILTIN,
-): Promise<Finding[]> => checkModel(buildDocModel(src), file, maxWords, registry);
+): Promise<Finding[]> => checkModel(buildDocModel(src, registry.concepts), file, maxWords, registry);

package/src/cli.ts

@@ -97,6 +97,14 @@ export const skippedLines = (failures: PluginFailure[], ignored: string[]): stri
 // The reason without the "plugin <source>:" the error message begins with.
 const reasonOf = (f: PluginFailure): string => f.error.message.replace(`plugin ${f.source}: `, "");
 
+// A plugin failure as the JSON output and --validate-plugins report it.
+const failureRecord = (f: PluginFailure) => ({
+  source: f.source,
+  kind: f.kind,
+  code: f.error.code,
+  message: reasonOf(f),
+});
+
 // Every active rule under its category, then where each plugin came from.
 // Built-in categories come first, in their fixed order, then plugin ones.
 export const listRules = (registry: Registry): string => {
@@ -122,7 +130,7 @@ const validate = (
   json: boolean,
 ): { code: number; stdout: string | null; stderr: string[] } => {
   const sources = registry.sources.map((s) => ({ ...s }));
-  const failed = failures.map((f) => ({ source: f.source, kind: f.kind, code: f.error.code, message: reasonOf(f) }));
+  const failed = failures.map(failureRecord);
   const code = failures.length > 0 ? 1 : 0;
   if (json)
     return {
@@ -146,26 +154,29 @@ const validate = (
   return { code, stdout: line, stderr: [] };
 };
 
-const run = async (argv: string[], cwd: string): Promise<number> => {
-  let parsed: Parsed;
+// The flags, or a usage error naming what parseArgs rejected.
+const parseFlags = (argv: string[]): Parsed => {
   try {
-    parsed = parse(argv);
+    return parse(argv);
   } catch (err) {
     throw new FlagError("argv", err instanceof Error ? err.message : String(err));
   }
-  const { values, positionals } = parsed;
-  if (values.help) {
-    console.log(USAGE);
-    return 0;
-  }
-  // Absent means the rule's own option, then the default. Present is a flag
-  // that wins for this run.
+};
+
+// Absent means the rule's own option, then the default. Present is a flag
+// that wins for this run.
+const maxWordsOf = (values: Parsed["values"]): number | undefined => {
   const maxWords = values["max-words"] === undefined ? undefined : Number(values["max-words"]);
-  const listing = values["list-rules"];
-  const validating = values["validate-plugins"];
   if (maxWords !== undefined && (Number.isNaN(maxWords) || maxWords < 1)) {
     throw new FlagError("--max-words", `--max-words must be a number of at least 1, not "${values["max-words"]}"`);
   }
+  return maxWords;
+};
+
+// The flag combinations that contradict each other, or leave nothing to do.
+const checkFlags = ({ values, positionals }: Parsed): void => {
+  const listing = values["list-rules"];
+  const validating = values["validate-plugins"];
   if (values["no-plugins"] && values["lenient-plugins"]) {
     throw new FlagError("--lenient-plugins", "--no-plugins and --lenient-plugins contradict each other");
   }
@@ -176,59 +187,78 @@ const run = async (argv: string[], cwd: string): Promise<number> => {
     throw new FlagError("files", "--validate-plugins runs no check, so it takes no files");
   }
   if (positionals.length === 0 && !listing && !validating) throw new FlagError("files", "no files given");
+};
+
+// Prints the --validate-plugins result and returns its exit code.
+const printValidation = (registry: Registry, failures: PluginFailure[], json: boolean): number => {
+  const result = validate(registry, failures, json);
+  for (const line of result.stderr) console.error(line);
+  if (result.stdout) console.log(result.stdout);
+  return result.code;
+};
+
+// One file's findings, fixed first when asked and written back only if changed.
+const checkFile = async (file: string, fix: boolean, maxWords: number | undefined, registry: Registry) => {
+  let src = await readFile(file, "utf8");
+  if (fix) {
+    const fixed = (await fixMarkdownReport(src, registry)).output;
+    if (fixed !== src) {
+      await writeFile(file, fixed);
+      src = fixed;
+    }
+  }
+  return await checkModel(buildDocModel(src, registry.concepts), file, maxWords, registry);
+};
+
+type Finding = Awaited<ReturnType<typeof checkFile>>[number];
+type SetUp = Awaited<ReturnType<typeof setUp>>;
+
+// The findings as JSON or one line each, then the exit code.
+const report = (all: Finding[], files: string[], setup: SetUp, json: boolean): number => {
+  if (json) {
+    const skippedPlugins = setup.failures.map(failureRecord);
+    console.log(
+      JSON.stringify({ findings: all, files: files.length, plugins: setup.registry.sources, skippedPlugins }, null, 2),
+    );
+  } else {
+    for (const f of all) console.log(`${f.file}:${f.line} ${f.rule} ${f.message}`);
+    console.log(`${all.length} finding(s) in ${files.length} file(s)`);
+  }
+  return all.length > 0 ? 1 : 0;
+};
 
-  const { registry, skipped, failures } = await setUp({
+const run = async (argv: string[], cwd: string): Promise<number> => {
+  const parsed = parseFlags(argv);
+  const { values, positionals } = parsed;
+  if (values.help) {
+    console.log(USAGE);
+    return 0;
+  }
+  const maxWords = maxWordsOf(values);
+  checkFlags(parsed);
+  const validating = values["validate-plugins"];
+
+  const setup = await setUp({
     cwd,
     configPath: values.config,
     noPlugins: values["no-plugins"],
     lenient: values["lenient-plugins"],
     collect: validating,
   });
-
-  if (validating) {
-    const result = validate(registry, failures, Boolean(values.json));
-    for (const line of result.stderr) console.error(line);
-    if (result.stdout) console.log(result.stdout);
-    return result.code;
-  }
+  const { registry, skipped, failures } = setup;
+  if (validating) return printValidation(registry, failures, Boolean(values.json));
 
   const note = loadedLine(registry.sources, skipped);
   if (note) console.error(note);
   for (const line of skippedLines(failures, registry.ignoredSettings)) console.error(line);
-  if (listing) {
+  if (values["list-rules"]) {
     console.log(listRules(registry));
     return 0;
   }
   const perFile = await Promise.all(
-    positionals.map(async (file) => {
-      let src = await readFile(file, "utf8");
-      if (values.fix) {
-        const fixed = (await fixMarkdownReport(src, registry)).output;
-        if (fixed !== src) {
-          await writeFile(file, fixed);
-          src = fixed;
-        }
-      }
-      return await checkModel(buildDocModel(src), file, maxWords, registry);
-    }),
+    positionals.map((file) => checkFile(file, Boolean(values.fix), maxWords, registry)),
   );
-  const all = perFile.flat();
-
-  if (values.json) {
-    const skippedPlugins = failures.map((f) => ({
-      source: f.source,
-      kind: f.kind,
-      code: f.error.code,
-      message: reasonOf(f),
-    }));
-    console.log(
-      JSON.stringify({ findings: all, files: positionals.length, plugins: registry.sources, skippedPlugins }, null, 2),
-    );
-  } else {
-    for (const f of all) console.log(`${f.file}:${f.line} ${f.rule} ${f.message}`);
-    console.log(`${all.length} finding(s) in ${positionals.length} file(s)`);
-  }
-  return all.length > 0 ? 1 : 0;
+  return report(perFile.flat(), positionals, setup, Boolean(values.json));
 };
 
 // The line printed for a failure: its stable code, then the reason.

package/src/cohesion.ts

@@ -0,0 +1,70 @@
+// Lexical cohesion between the sentences of one paragraph: which sentences
+// talk about the same thing, and where the topic turns. It reads the language
+// model a paragraph exposes, so the words, tags and lemmas come from
+// src/language.ts. The scoring and the break plan are a guide, not a parse:
+// no JavaScript topic segmenter exists to use instead.
+
+import type { LanguageSentence } from "./language.ts";
+
+const shared = (a: ReadonlySet<string>, b: ReadonlySet<string>): number => [...a].filter((t) => b.has(t)).length;
+
+// How strongly the sentences either side of each gap hold together. Gap i
+// sits between sentence i and sentence i + 1, counted from zero here. A term
+// the two neighbours share counts double. A term shared by sentences two
+// apart across the gap counts once, so a sentence that strays for a moment
+// does not split a topic. A sentence that opens by referring back counts
+// double, and one that opens like its neighbour, in parallel, counts once.
+// Zero means nothing ties the two sides together: the topic turns.
+export const gapCohesion = (sentences: readonly LanguageSentence[]): number[] => {
+  const terms = sentences.map((s) => s.terms);
+  const openers = sentences.map((s) => s.opener);
+  const none: ReadonlySet<string> = new Set();
+  const at = (i: number): ReadonlySet<string> => terms[i] ?? none;
+  return sentences.slice(1).map((next, i) => {
+    const adjacent = shared(at(i), at(i + 1));
+    // A gap at either end has one pair two apart instead of two, so the pairs
+    // it has are scaled up. Otherwise an edge would always look weakest.
+    const pairs = [
+      i - 1 >= 0 ? shared(at(i - 1), at(i + 1)) : null,
+      i + 2 < terms.length ? shared(at(i), at(i + 2)) : null,
+    ];
+    const present = pairs.filter((p): p is number => p !== null);
+    const skips = present.length === 0 ? 0 : (2 * present.reduce((a, b) => a + b, 0)) / present.length;
+    const parallel = openers[i] !== null && openers[i] === openers[i + 1] ? 1 : 0;
+    return 2 * adjacent + skips + (next.refersBack ? 2 : 0) + parallel;
+  });
+};
+
+// Where to break a paragraph: after every sentence whose gap has no cohesion,
+// then inside any group still over the budget, at its weakest gap. A tie goes
+// to the gap nearest the middle of the group, so the halves come out even.
+// Breaks are sentence numbers counted from one: a break of 3 means a new
+// paragraph starts with sentence 4.
+export interface BreakPlan {
+  topics: number;
+  breaks: number[];
+}
+
+export const planBreaks = (cohesion: number[], maxSentences: number): BreakPlan => {
+  const count = cohesion.length + 1;
+  const topicBreaks = cohesion.flatMap((score, i) => (score === 0 ? [i + 1] : []));
+  const breaks = new Set(topicBreaks);
+  const split = (from: number, to: number): void => {
+    // The group holds sentences from + 1 through to.
+    if (to - from <= maxSentences) return;
+    let best = -1;
+    for (let at = from + 1; at < to; at++) {
+      const better =
+        best === -1 ||
+        (cohesion[at - 1] as number) < (cohesion[best - 1] as number) ||
+        (cohesion[at - 1] === cohesion[best - 1] && Math.abs(2 * at - from - to) < Math.abs(2 * best - from - to));
+      if (better) best = at;
+    }
+    breaks.add(best);
+    split(from, best);
+    split(best, to);
+  };
+  const edges = [0, ...topicBreaks, count];
+  for (let i = 0; i + 1 < edges.length; i++) split(edges[i] as number, edges[i + 1] as number);
+  return { topics: topicBreaks.length + 1, breaks: [...breaks].sort((a, b) => a - b) };
+};

package/src/concepts.ts

@@ -0,0 +1,209 @@
+// The concept model: a project's own vocabulary, read from the GLOSSARY.md or
+// CONCEPTS.md at its root. Each H2 heading is a concept, and each Mermaid
+// flowchart in the file relates them: a labelled edge is a relation, a dotted
+// one is a prohibition, and "is a kind of" makes one concept a kind of another.
+// The file is optional, but one that is there must parse, because a rule that
+// reads concepts would otherwise run against half a vocabulary.
+
+import { access, readFile } from "node:fs/promises";
+import { join } from "node:path";
+import nlp from "compromise";
+import type { Code, Heading, Node, Text } from "mdast";
+import { fromMarkdown } from "mdast-util-from-markdown";
+import { AmbiguousConceptsError, ConceptsFileError } from "./errors.ts";
+
+export const CONCEPT_FILES = ["GLOSSARY.md", "CONCEPTS.md"] as const;
+
+// The edge label that makes its source a kind of its target.
+export const KIND_OF = "is a kind of";
+
+// One concept. `kinds` is every concept it is a kind of, nearest first, so a
+// Local Rule that is a kind of Plugin has kinds ["Plugin"].
+export interface Concept {
+  name: string;
+  kinds: readonly string[];
+}
+
+// One edge of a diagram, by concept name. `verb` is the edge label, empty for
+// an unlabelled edge. `never` is true for a dotted edge: the source must never
+// do what the label says to the target.
+export interface ConceptRelation {
+  from: string;
+  verb: string;
+  to: string;
+  never: boolean;
+}
+
+export interface ConceptModel {
+  // The file it came from, relative to the project root. Null when there is none.
+  file: string | null;
+  concepts: readonly Concept[];
+  relations: readonly ConceptRelation[];
+}
+
+export const NO_CONCEPTS: ConceptModel = { file: null, concepts: [], relations: [] };
+
+// --- reading the file ------------------------------------------------------
+
+const textOf = (node: Node): string =>
+  node.type === "text" || node.type === "inlineCode"
+    ? (node as Text).value
+    : ((node as { children?: Node[] }).children ?? []).map(textOf).join("");
+
+const NODE = /^\s*([A-Za-z][\w-]*)\["([^"]+)"\]/;
+const EDGE = /^\s*([A-Za-z][\w-]*)\s+(?:(--|-\.)\s+(.+?)\s+(-->|\.->)|(-->|-\.->))\s+([A-Za-z][\w-]*)\s*$/;
+
+// One diagram's relations. Node ids are local to their diagram, so two
+// diagrams may reuse an id for different concepts.
+const relationsOf = (file: string, diagram: Code, names: ReadonlySet<string>): ConceptRelation[] => {
+  const label = new Map<string, string>();
+  const relations: ConceptRelation[] = [];
+  for (const line of diagram.value.split("\n")) {
+    const node = NODE.exec(line);
+    if (node) {
+      const [, id = "", name = ""] = node;
+      if (!names.has(name)) throw new ConceptsFileError(file, `diagram node "${name}" is not a term heading`);
+      label.set(id, name);
+      continue;
+    }
+    const edge = EDGE.exec(line);
+    if (!edge) continue;
+    const [, from = "", solidOrDotted, verb = "", , bare, to = ""] = edge;
+    const ends = [from, to].map((id) => {
+      const name = label.get(id);
+      if (!name) throw new ConceptsFileError(file, `edge "${line.trim()}" names "${id}", which no node declares`);
+      return name;
+    });
+    const never = solidOrDotted === "-." || bare === "-.->";
+    relations.push({ from: ends[0] as string, verb, to: ends[1] as string, never });
+  }
+  return relations;
+};
+
+// Every concept a concept is a kind of, nearest first. A cycle is an error,
+// because it leaves a concept a kind of itself.
+const kindsOf = (file: string, name: string, parents: ReadonlyMap<string, string[]>): string[] => {
+  const kinds: string[] = [];
+  const queue = [...(parents.get(name) ?? [])];
+  while (queue.length > 0) {
+    const next = queue.shift() as string;
+    if (next === name) throw new ConceptsFileError(file, `"${name}" is a kind of itself`);
+    if (kinds.includes(next)) continue;
+    kinds.push(next);
+    queue.push(...(parents.get(next) ?? []));
+  }
+  return kinds;
+};
+
+export const parseConcepts = (file: string, src: string): ConceptModel => {
+  const root = fromMarkdown(src);
+  const headings = root.children.filter((n): n is Heading => n.type === "heading" && n.depth === 2);
+  const names = headings.map((h) => textOf(h).trim());
+  if (names.length === 0) throw new ConceptsFileError(file, "it has no H2 headings, and each H2 is a term");
+  const repeated = names.find((n, i) => names.indexOf(n) !== i);
+  if (repeated) throw new ConceptsFileError(file, `the term "${repeated}" has two headings`);
+  const known = new Set(names);
+  const diagrams = root.children.filter((n): n is Code => n.type === "code" && n.lang === "mermaid");
+  const relations = diagrams.flatMap((d) => relationsOf(file, d, known));
+  const parents = new Map<string, string[]>();
+  for (const r of relations.filter((x) => x.verb === KIND_OF && !x.never)) {
+    parents.set(r.from, [...(parents.get(r.from) ?? []), r.to]);
+  }
+  return { file, concepts: names.map((name) => ({ name, kinds: kindsOf(file, name, parents) })), relations };
+};
+
+const exists = (path: string): Promise<boolean> =>
+  access(path).then(
+    () => true,
+    () => false,
+  );
+
+// The concept file at the root, or null. Two at once is ambiguous, so it is
+// an error rather than a pick.
+export const findConcepts = async (root: string): Promise<string | null> => {
+  const found: string[] = [];
+  for (const name of CONCEPT_FILES) if (await exists(join(root, name))) found.push(name);
+  if (found.length > 1) throw new AmbiguousConceptsError(root, found);
+  return found[0] ?? null;
+};
+
+export const loadConcepts = async (root: string): Promise<ConceptModel> => {
+  const name = await findConcepts(root);
+  return name ? parseConcepts(name, await readFile(join(root, name), "utf8")) : NO_CONCEPTS;
+};
+
+// --- finding concepts in a sentence ----------------------------------------
+
+// One place a sentence names a concept. `start` and `length` count words of
+// the sentence. `text` is the words as written, so "prose gates" names Prose Gate.
+export interface ConceptMention {
+  concept: string;
+  text: string;
+  start: number;
+  length: number;
+  kinds: readonly string[];
+}
+
+interface Form {
+  concept: Concept;
+  words: readonly string[];
+}
+
+// The word sequences that name each concept, its own and its plural, longest
+// first, keyed by their first word. compromise tokenises each so they line up
+// with the sentence's words, which it tokenised the same way.
+const formsCache = new WeakMap<ConceptModel, Map<string, Form[]>>();
+
+const tokens = (text: string): string[] =>
+  (nlp(text).json() as { terms: { text: string }[] }[]).flatMap((s) => s.terms.map((t) => t.text.toLowerCase()));
+
+const formsOf = (model: ConceptModel): Map<string, Form[]> => {
+  const cached = formsCache.get(model);
+  if (cached) return cached;
+  const byFirst = new Map<string, Form[]>();
+  for (const concept of model.concepts) {
+    const lower = concept.name.toLowerCase();
+    const plural = nlp(lower).nouns().toPlural().text();
+    for (const form of new Set([lower, plural].filter((f) => f.length > 0))) {
+      const words = tokens(form);
+      const first = words[0] as string;
+      byFirst.set(first, [...(byFirst.get(first) ?? []), { concept, words }]);
+    }
+  }
+  for (const forms of byFirst.values()) forms.sort((a, b) => b.words.length - a.words.length);
+  formsCache.set(model, byFirst);
+  return byFirst;
+};
+
+interface MatchWord {
+  text: string;
+  tags: readonly string[];
+}
+
+// A one-word concept must be read as a noun there, so "release the package"
+// names no Release. A longer one is specific enough as it stands.
+const matchAt = (words: readonly MatchWord[], at: number, forms: readonly Form[]): Form | undefined =>
+  forms.find(
+    (f) =>
+      f.words.every((w, i) => words[at + i]?.text.toLowerCase() === w) &&
+      (f.words.length > 1 || (words[at]?.tags.includes("Noun") ?? false)),
+  );
+
+// Every concept the words name, longest match first, never overlapping.
+export const mentionsIn = (words: readonly MatchWord[], model: ConceptModel): ConceptMention[] => {
+  if (model.concepts.length === 0) return [];
+  const byFirst = formsOf(model);
+  const mentions: ConceptMention[] = [];
+  for (let at = 0; at < words.length; at++) {
+    const form = matchAt(words, at, byFirst.get(words[at]?.text.toLowerCase() ?? "") ?? []);
+    if (!form) continue;
+    const length = form.words.length;
+    const text = words
+      .slice(at, at + length)
+      .map((w) => w.text)
+      .join(" ");
+    mentions.push({ concept: form.concept.name, text, start: at, length, kinds: form.concept.kinds });
+    at += length - 1;
+  }
+  return mentions;
+};

package/src/errors.ts

@@ -61,6 +61,27 @@ export class AmbiguousConfigError extends UsageError {
   }
 }
 
+// A concept file at the project root that does not parse as one: no term
+// headings, a repeated term, or a diagram that names something undefined.
+export class ConceptsFileError extends UsageError {
+  readonly file: string;
+
+  constructor(file: string, message: string) {
+    super(`${file}: not a concept file, because ${message}`, "concepts-file");
+    this.file = file;
+  }
+}
+
+// Both GLOSSARY.md and CONCEPTS.md at one root, so neither can be picked.
+export class AmbiguousConceptsError extends UsageError {
+  readonly files: string[];
+
+  constructor(root: string, files: string[]) {
+    super(`more than one concept file at ${root}: ${files.join(", ")}`, "concepts-ambiguous");
+    this.files = files;
+  }
+}
+
 // The config parsed, but its top-level shape is wrong.
 export class ConfigShapeError extends UsageError {
   readonly file: string;