@jpeakai/prose-gates 0.1.0

package/package.json

@@ -0,0 +1,59 @@
+{
+  "name": "@jpeakai/prose-gates",
+  "version": "0.1.0",
+  "type": "module",
+  "description": "Deterministic prose gates for markdown documentation",
+  "license": "MIT",
+  "keywords": [
+    "markdown",
+    "prose",
+    "lint",
+    "documentation"
+  ],
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/jpeakai/jpai-prose-gates.git"
+  },
+  "homepage": "https://github.com/jpeakai/jpai-prose-gates#readme",
+  "bugs": "https://github.com/jpeakai/jpai-prose-gates/issues",
+  "packageManager": "bun@1.3.12",
+  "publishConfig": {
+    "access": "public"
+  },
+  "scripts": {
+    "prepublishOnly": "make check",
+    "prepack": "make build"
+  },
+  "bin": {
+    "prose-gates": "dist/bin.js"
+  },
+  "exports": {
+    ".": {
+      "bun": "./src/index.ts",
+      "types": "./src/index.ts",
+      "default": "./dist/index.js"
+    }
+  },
+  "engines": {
+    "node": ">=20"
+  },
+  "files": [
+    "src",
+    "dist"
+  ],
+  "dependencies": {
+    "mdast-util-from-markdown": "^2.0.3",
+    "mdast-util-frontmatter": "^2.0.1",
+    "mdast-util-gfm": "^3.1.0",
+    "micromark-extension-frontmatter": "^2.0.0",
+    "micromark-extension-gfm": "^3.0.0",
+    "unist-util-visit-parents": "^6.0.2"
+  },
+  "devDependencies": {
+    "@biomejs/biome": "^2",
+    "@types/bun": "latest",
+    "@types/mdast": "^4",
+    "fast-check": "^4.10.0",
+    "typescript": "^5"
+  }
+}

package/src/bin.ts

@@ -0,0 +1,10 @@
+#!/usr/bin/env node
+import { main } from "./cli.ts";
+
+main().then(
+  (code) => process.exit(code),
+  (err: unknown) => {
+    console.error(`error: ${err instanceof Error ? err.message : String(err)}`);
+    process.exit(1);
+  },
+);

package/src/check.ts

@@ -0,0 +1,24 @@
+// Checking: build the model once, run every rule over it, and audit
+// embedded markdown fences as documents of their own.
+
+import { buildDocModel, type DocModel } from "./model.ts";
+import { RULES } from "./rules/index.ts";
+import { interpunctRuns } from "./rules/interpunct.ts";
+import type { Finding } from "./rules/types.ts";
+
+export const DEFAULT_MAX_WORDS = 25;
+
+export const checkModel = (doc: DocModel, file: string, maxWords: number): Finding[] => {
+  const ctx = { doc, file, maxWords, runs: interpunctRuns(doc) };
+  const findings = RULES.flatMap((rule) => rule.check(ctx));
+  // The fence body starts one line below the opening fence.
+  for (const fence of doc.fences) {
+    for (const f of checkModel(buildDocModel(fence.value), file, maxWords)) {
+      findings.push({ ...f, line: fence.line + f.line });
+    }
+  }
+  return findings.sort((a, b) => a.line - b.line || a.rule.localeCompare(b.rule));
+};
+
+export const checkMarkdown = (src: string, file: string, maxWords: number = DEFAULT_MAX_WORDS): Finding[] =>
+  checkModel(buildDocModel(src), file, maxWords);

package/src/cli.ts

@@ -0,0 +1,82 @@
+// The command line: check files, optionally fix them first. The process
+// glue lives in bin.ts so tests can call main() directly.
+
+import { readFile, writeFile } from "node:fs/promises";
+import { parseArgs } from "node:util";
+import { checkModel, DEFAULT_MAX_WORDS } from "./check.ts";
+import { fixMarkdown } from "./fix/engine.ts";
+import { buildDocModel } from "./model.ts";
+import { RULES } from "./rules/index.ts";
+import { CATEGORIES } from "./rules/types.ts";
+
+const USAGE = `usage: prose-gates <file.md> [...more] [--fix] [--json] [--max-words N]
+
+Deterministic prose gates (* = autofixable):
+${CATEGORIES.map((c) =>
+  [`  ${c}:`, ...RULES.filter((r) => r.category === c).map((r) => `    ${r.id}${r.fix ? "*" : " "} ${r.summary}`)].join(
+    "\n",
+  ),
+).join("\n")}
+
+Examples of every rule, with fixes: RULES.md
+
+Code blocks are exempt, except fences tagged markdown/md: those hold
+markdown templates and their body is audited recursively (check-only).
+
+--fix applies every fix it can prove safe, then reports what remains.
+A fixer that is unsure leaves the text alone and the finding stays.
+Exits 1 when findings remain, 2 on usage error.`;
+
+export const main = async (argv: string[] = process.argv.slice(2)): Promise<number> => {
+  let values: { fix?: boolean; json?: boolean; help?: boolean; "max-words"?: string };
+  let positionals: string[];
+  try {
+    ({ values, positionals } = parseArgs({
+      args: argv,
+      options: {
+        fix: { type: "boolean", default: false },
+        json: { type: "boolean", default: false },
+        help: { type: "boolean", short: "h", default: false },
+        "max-words": { type: "string" },
+      },
+      allowPositionals: true,
+      strict: true,
+    }));
+  } catch (err) {
+    console.error(`error: ${err instanceof Error ? err.message : String(err)}`);
+    console.error(USAGE);
+    return 2;
+  }
+  if (values.help) {
+    console.log(USAGE);
+    return 0;
+  }
+  const maxWords = values["max-words"] === undefined ? DEFAULT_MAX_WORDS : Number(values["max-words"]);
+  if (positionals.length === 0 || Number.isNaN(maxWords) || maxWords < 1) {
+    console.error(USAGE);
+    return 2;
+  }
+
+  const perFile = await Promise.all(
+    positionals.map(async (file) => {
+      let src = await readFile(file, "utf8");
+      if (values.fix) {
+        const fixed = fixMarkdown(src);
+        if (fixed !== src) {
+          await writeFile(file, fixed);
+          src = fixed;
+        }
+      }
+      return checkModel(buildDocModel(src), file, maxWords);
+    }),
+  );
+  const all = perFile.flat();
+
+  if (values.json) {
+    console.log(JSON.stringify({ findings: all, files: positionals.length }, null, 2));
+  } else {
+    for (const f of all) console.log(`${f.file}:${f.line} ${f.rule} ${f.message}`);
+    console.log(`${all.length} finding(s) in ${positionals.length} file(s)`);
+  }
+  return all.length > 0 ? 1 : 0;
+};

package/src/fix/engine.ts

@@ -0,0 +1,84 @@
+// The fix engine: a fixpoint loop over splice edits. Each pass rebuilds the
+// model, asks the fixers for edits in priority order, and keeps the first
+// edit that survives verification. Offsets are only ever valid for the
+// source they were computed from, so every accepted edit restarts the pass.
+
+import { buildDocModel, parse } from "../model.ts";
+import { FIX_ORDER } from "../rules/index.ts";
+import type { Edit } from "../rules/types.ts";
+import { verify } from "./verify.ts";
+
+const MAX_PASSES = 10_000;
+
+export interface FixReport {
+  output: string;
+  applied: Edit[];
+  refused: Edit[];
+}
+
+const splice = (src: string, edits: Edit[]): string => {
+  let out = src;
+  for (const e of [...edits].sort((a, b) => b.start - a.start)) out = out.slice(0, e.start) + e.text + out.slice(e.end);
+  return out;
+};
+
+const overlaps = (edits: Edit[]): boolean => {
+  const sorted = [...edits].sort((a, b) => a.start - b.start);
+  return sorted.some((e, i) => i > 0 && e.start < (sorted[i - 1] as Edit).end);
+};
+
+export const fixMarkdownReport = (src: string): FixReport => {
+  const applied: Edit[] = [];
+  const refused: Edit[] = [];
+  // An edit refused once stays refused while its source slice is unchanged.
+  const refusedKeys = new Set<string>();
+  let current = src;
+
+  for (let pass = 0; pass < MAX_PASSES; pass++) {
+    const doc = buildDocModel(current);
+    const before = { src: current, tree: doc.tree };
+    const keyOf = (e: Edit): string => `${e.rule}\u0000${current.slice(e.start, e.end)}\u0000${e.text}`;
+
+    const attempt = (edits: Edit[]): boolean => {
+      const next = splice(current, edits);
+      const afterTree = parse(next);
+      if (edits.every((e) => verify(e, before, afterTree) === "accept")) {
+        current = next;
+        applied.push(...edits);
+        return true;
+      }
+      return false;
+    };
+
+    let progressed = false;
+    for (const rule of FIX_ORDER) {
+      const edits = (rule.fix?.(doc) ?? []).filter(
+        (e) => current.slice(e.start, e.end) !== e.text && !refusedKeys.has(keyOf(e)),
+      );
+      if (edits.length === 0) continue;
+
+      // Whitespace and glyph edits leave the tree's shape alone, so a rule's
+      // whole batch can be proven in one re-parse. A batch that fails falls
+      // back to one edit at a time, so one bad edit never blocks the rest.
+      const batchable =
+        edits.length > 1 && !overlaps(edits) && edits.every((e) => e.expect.kind !== "replace-paragraph");
+      if (batchable && attempt(edits)) {
+        progressed = true;
+        break;
+      }
+      for (const edit of edits) {
+        if (attempt([edit])) {
+          progressed = true;
+          break;
+        }
+        refusedKeys.add(keyOf(edit));
+        refused.push(edit);
+      }
+      if (progressed) break;
+    }
+    if (!progressed) return { output: current, applied, refused };
+  }
+  throw new Error(`fix did not converge after ${MAX_PASSES} passes; a fixer keeps proposing edits`);
+};
+
+export const fixMarkdown = (src: string): string => fixMarkdownReport(src).output;

package/src/fix/promote.ts

@@ -0,0 +1,107 @@
+// List promotion: the shared fix for every "hidden list" rule. A sentence
+// that strings items together inline becomes a lead-in line and a real
+// markdown list. Item text is sliced from the source, never re-serialised,
+// so links, emphasis, escapes and code spans travel unchanged.
+
+import type { ParagraphView, Range } from "../model.ts";
+import type { Edit, RuleId } from "../rules/types.ts";
+import { collapse } from "./spans.ts";
+
+export interface PromotedItem {
+  text: string;
+  children?: string[];
+}
+
+export interface Promotion {
+  rule: RuleId;
+  src: string;
+  view: ParagraphView;
+  span: Range; // the sentences the list replaces
+  leadIn: string | null;
+  items: PromotedItem[];
+  ordered: boolean;
+}
+
+// A line that would parse as something other than paragraph text when it
+// starts a list item: a heading, quote, nested bullet, ordered marker,
+// fence, thematic break, table pipe or raw HTML.
+const UNSAFE_START =
+  /^(?:[-+*](?:\s|$)|#{1,6}(?:\s|$)|>|\d{1,9}[.)](?:\s|$)|`{3}|~{3}|<|\||[-_*=](?:\s*[-_*=]){2,}\s*$)/;
+
+const CONJUNCTION_HEAD = /^(?:and|or)\s+/i;
+const CONJUNCTION_TAIL = /\s+(?:and|or)$/i;
+
+export const cleanItem = (raw: string, { last }: { last: boolean }): string => {
+  let s = collapse(raw).trim();
+  // "b, and" needs the conjunction and the comma gone, in either order. Only
+  // the last item may open with the joining conjunction: "or" at the head of
+  // a middle item is content the list would otherwise swallow.
+  for (let prev = ""; prev !== s; ) {
+    prev = s;
+    s = s.replace(/[;,]+$/, "").trim();
+    s = s.replace(CONJUNCTION_TAIL, "").trim();
+    if (last) s = s.replace(CONJUNCTION_HEAD, "").trim();
+  }
+  if (last) s = s.replace(/(?<!\.)\.$/, "").trim();
+  return s;
+};
+
+export const cleanLeadIn = (raw: string): string | null => {
+  const s = collapse(raw)
+    .trim()
+    .replace(/[\s:;,—-]+$/, "")
+    .trim();
+  return s.length > 0 ? `${s}:` : null;
+};
+
+const dedent = (raw: string, indent: string): string =>
+  raw
+    .split("\n")
+    .map((line, i) => (i > 0 && line.startsWith(indent) ? line.slice(indent.length) : line.trimStart()))
+    .join("\n");
+
+export const promote = (p: Promotion): Edit | null => {
+  const { view, src, span } = p;
+  if (view.inTable || view.inBlockquote || view.hasBreak) return null;
+  if (p.items.length < 2) return null;
+
+  const texts = [...p.items.flatMap((i) => [i.text, ...(i.children ?? [])]), ...(p.leadIn ? [p.leadIn] : [])];
+  if (texts.some((t) => t.length === 0 || UNSAFE_START.test(t))) return null;
+
+  const indent = " ".repeat(view.startColumn - 1);
+  const before = dedent(src.slice(view.startOffset, span[0]), indent).trim();
+  const after = dedent(src.slice(span[1], view.endOffset), indent).trim();
+  const head = [before, p.leadIn].filter((s): s is string => s !== null && s.length > 0).join("\n");
+
+  const list = p.items
+    .map((item, i) => {
+      const marker = p.ordered ? `${i + 1}. ` : "- ";
+      const pad = " ".repeat(marker.length);
+      return [`${marker}${item.text}`, ...(item.children ?? []).map((c) => `${pad}- ${c}`)].join("\n");
+    })
+    .join("\n");
+
+  let fragment: string;
+  if (view.inListItem) {
+    // Inside a list item a blank line would loosen the parent list, and text
+    // after the nested list would become lazy continuation. Both change what
+    // renders, so only a lead-in followed by the list is promotable here.
+    if (head.length === 0 || after.length > 0) return null;
+    fragment = `${head}\n${list}`;
+  } else {
+    fragment = [head, list, after].filter((s) => s.length > 0).join("\n\n");
+  }
+
+  const text = fragment
+    .split("\n")
+    .map((line, i) => (i === 0 || line.length === 0 ? line : indent + line))
+    .join("\n");
+
+  return {
+    rule: p.rule,
+    start: view.startOffset,
+    end: view.endOffset,
+    text,
+    expect: { kind: "replace-paragraph", view, fragment, listItems: p.items.length },
+  };
+};

package/src/fix/spans.ts

@@ -0,0 +1,75 @@
+// Source-offset helpers shared by fixers. Everything here works on absolute
+// offsets into the document source, and only ever looks inside text the
+// parser proved is prose.
+
+import type { DirectText, ParagraphView, Range } from "../model.ts";
+
+// Absolute offsets of every match of `pattern` inside a text node, or null
+// when the source slice and the parsed value disagree (an entity like
+// `·` or an escape). A fixer that cannot map a glyph back to the
+// source refuses rather than guesses.
+export const matchesIn = (src: string, text: DirectText, pattern: RegExp): Range[] | null => {
+  const slice = src.slice(text.start, text.end);
+  const flags = pattern.flags.includes("g") ? pattern.flags : `${pattern.flags}g`;
+  const inSlice = [...slice.matchAll(new RegExp(pattern.source, flags))];
+  const inValue = [...text.value.matchAll(new RegExp(pattern.source, flags))];
+  if (inSlice.length !== inValue.length) return null;
+  return inSlice.map((m) => [text.start + (m.index ?? 0), text.start + (m.index ?? 0) + m[0].length]);
+};
+
+export const matchesInAll = (src: string, texts: DirectText[], pattern: RegExp): Range[] | null => {
+  const out: Range[] = [];
+  for (const t of texts) {
+    const found = matchesIn(src, t, pattern);
+    if (found === null) return null;
+    out.push(...found);
+  }
+  return out;
+};
+
+// Sentence boundaries a fixer trusts: a terminal followed by whitespace,
+// inside a direct text node. Terminals inside links, emphasis or code never
+// split a sentence. Each boundary is [end of terminal, start of next sentence].
+const boundaries = (src: string, view: ParagraphView): Range[] => {
+  const out: Range[] = [];
+  for (const t of view.directTexts) {
+    const slice = src.slice(t.start, t.end);
+    for (const m of slice.matchAll(/[.!?]+(\s+)/g)) {
+      const at = t.start + (m.index ?? 0);
+      out.push([at + m[0].length - (m[1]?.length ?? 0), at + m[0].length]);
+    }
+  }
+  return out.sort((a, b) => a[0] - b[0]);
+};
+
+// The sentence span that covers [from, to): from the start of the sentence
+// holding `from` to the end of the sentence holding `to`.
+export const sentenceSpan = (src: string, view: ParagraphView, from: number, to: number): Range => {
+  let start = view.startOffset;
+  let end = view.endOffset;
+  for (const [terminalEnd, nextStart] of boundaries(src, view)) {
+    if (nextStart <= from) start = nextStart;
+    if (terminalEnd >= to && end === view.endOffset) end = terminalEnd;
+  }
+  return [start, end];
+};
+
+// Every sentence span of a paragraph, in order.
+export const sentenceSpans = (src: string, view: ParagraphView): Range[] => {
+  const spans: Range[] = [];
+  let start = view.startOffset;
+  for (const [terminalEnd, nextStart] of boundaries(src, view)) {
+    spans.push([start, terminalEnd]);
+    start = nextStart;
+  }
+  spans.push([start, view.endOffset]);
+  return spans;
+};
+
+export const within = (r: Range, span: Range): boolean => r[0] >= span[0] && r[1] <= span[1];
+
+// Soft line breaks collapse to single spaces; nothing else in the slice moves.
+export const collapse = (s: string): string => s.replace(/[ \t]*\r?\n[ \t]*/g, " ");
+
+// A glyph the author escaped is literal text, never a separator to rewrite.
+export const escaped = (src: string, offset: number): boolean => src[offset - 1] === "\\";

package/src/fix/verify.ts

@@ -0,0 +1,134 @@
+// Proof obligations for a fix. The engine applies one edit, re-parses, and
+// checks two things: no word was lost (a fixer bug, so it throws), and the
+// document's structure changed exactly as the edit declared (a context the
+// fixer could not see, so the edit is refused and the finding stays).
+
+import type { Node, Parent, Root } from "mdast";
+import { visitParents as visit } from "unist-util-visit-parents";
+import { parse } from "../model.ts";
+import type { Edit } from "../rules/types.ts";
+
+export class FixInvariantError extends Error {
+  constructor(edit: Edit, detail: string) {
+    super(`${edit.rule} fixer lost content (${detail}); this is a bug in the fixer, not the document`);
+    this.name = "FixInvariantError";
+  }
+}
+
+// Words carried by the document: text, code, alt text and every URL. The
+// punctuation a fixer is allowed to rewrite (separators, enumeration markers,
+// joining conjunctions) is excluded so the comparison proves only that no
+// real content moved or vanished. A marker is recognised whatever follows
+// it: a separator swap that turns "(1) —" into "(1):" must not make the
+// marker count as a word it never was.
+const ENUM_MARKER = /(?<![\p{L}\p{N}_])\((?:[a-z]|[ivx]+|\d+)\)/giu;
+
+export const fingerprint = (tree: Root): string[] => {
+  const words: string[] = [];
+  const take = (s: string | null | undefined, stripMarkers = false): void => {
+    if (!s) return;
+    const text = stripMarkers ? s.replace(ENUM_MARKER, " ") : s;
+    for (const m of text.matchAll(/[\p{L}\p{N}_]+/gu)) {
+      const w = m[0];
+      if (!/^(?:and|or)$/i.test(w)) words.push(w);
+    }
+  };
+  visit(tree, (node: Node) => {
+    const n = node as Node & { value?: string; url?: string; alt?: string; title?: string };
+    if (node.type === "text") take(n.value, true);
+    else if (node.type === "inlineCode" || node.type === "code" || node.type === "html") take(n.value);
+    else if (node.type === "yaml" || node.type === "toml") take(n.value);
+    take(n.url);
+    take(n.alt);
+    take(n.title);
+  });
+  return words;
+};
+
+type Plain = { [k: string]: unknown };
+
+const OMIT = new Set(["position", "data"]);
+
+const normaliseValue = (node: Node, value: string, blankText: boolean): string => {
+  if (node.type !== "text") return value;
+  return blankText ? "" : value.replace(/\s+/g, " ").trim();
+};
+
+// A position-free copy of the tree for structural comparison. `swap`
+// replaces one node with a list of nodes, which is how the expected tree for
+// a paragraph replacement is built.
+export const normalise = (node: Node, opts: { blankText?: boolean; swap?: [Node, Node[]] } = {}): Plain => {
+  const out: Plain = {};
+  for (const [key, value] of Object.entries(node)) {
+    if (OMIT.has(key)) continue;
+    if (key === "children" && Array.isArray(value)) {
+      out.children = (value as Node[]).flatMap((child) =>
+        opts.swap && child === opts.swap[0]
+          ? opts.swap[1].map((c) => normalise(c, { blankText: opts.blankText }))
+          : [normalise(child, opts)],
+      );
+    } else if (key === "value" && typeof value === "string") {
+      out.value = normaliseValue(node, value, opts.blankText ?? false);
+    } else {
+      out[key] = value;
+    }
+  }
+  return out;
+};
+
+const sameJson = (a: unknown, b: unknown): boolean => JSON.stringify(a) === JSON.stringify(b);
+
+// Definitions resolve references document-wide, so a fragment parsed on its
+// own borrows every definition from the document it came from.
+const definitionsOf = (src: string, tree: Root): string[] => {
+  const defs: string[] = [];
+  visit(tree, (node: Node) => {
+    if ((node.type === "definition" || node.type === "footnoteDefinition") && node.position) {
+      const { start, end } = node.position;
+      if (start.offset != null && end.offset != null) defs.push(src.slice(start.offset, end.offset));
+    }
+  });
+  return defs;
+};
+
+const parseFragment = (fragment: string, defs: string[]): Node[] => {
+  const tree = parse([fragment, ...defs].join("\n\n"));
+  return tree.children.slice(0, tree.children.length - defs.length);
+};
+
+// The fragment must be the shape the fixer meant: optional paragraphs around
+// exactly one list whose items each open with paragraph text.
+const fragmentShapeOk = (nodes: Node[], listItems: number): boolean => {
+  const lists = nodes.filter((n) => n.type === "list") as Parent[];
+  if (lists.length !== 1 || nodes.some((n) => n.type !== "list" && n.type !== "paragraph")) return false;
+  const items = (lists[0] as Parent).children as Parent[];
+  return items.length === listItems && items.every((i) => i.children[0]?.type === "paragraph");
+};
+
+export type Verdict = "accept" | "refuse";
+
+export const verify = (edit: Edit, before: { src: string; tree: Root }, afterTree: Root): Verdict => {
+  const lost = fingerprint(before.tree).join(" ");
+  const kept = fingerprint(afterTree).join(" ");
+  if (lost !== kept) {
+    const a = lost.split(" ");
+    const b = kept.split(" ");
+    const diverge = a.findIndex((w, i) => w !== b[i]);
+    const at = diverge === -1 ? a.length : diverge;
+    throw new FixInvariantError(edit, `word ${at} was "${a[at] ?? ""}", now "${b[at] ?? ""}"`);
+  }
+
+  const expect = edit.expect;
+  if (expect.kind === "same-tree") {
+    return sameJson(normalise(before.tree), normalise(afterTree)) ? "accept" : "refuse";
+  }
+  if (expect.kind === "same-shape") {
+    return sameJson(normalise(before.tree, { blankText: true }), normalise(afterTree, { blankText: true }))
+      ? "accept"
+      : "refuse";
+  }
+  const nodes = parseFragment(expect.fragment, definitionsOf(before.src, before.tree));
+  if (!fragmentShapeOk(nodes, expect.listItems)) return "refuse";
+  const expected = normalise(before.tree, { swap: [expect.view.node, nodes] });
+  return sameJson(expected, normalise(afterTree)) ? "accept" : "refuse";
+};

package/src/index.ts

@@ -0,0 +1,10 @@
+// Public API.
+
+export { checkMarkdown, checkModel, DEFAULT_MAX_WORDS } from "./check.ts";
+export { main } from "./cli.ts";
+export { type FixReport, fixMarkdown, fixMarkdownReport } from "./fix/engine.ts";
+export { FixInvariantError } from "./fix/verify.ts";
+export { buildDocModel, type DocModel } from "./model.ts";
+export { FIX_ORDER, RULES } from "./rules/index.ts";
+export type { Category, Edit, Finding, Rule, RuleId } from "./rules/types.ts";
+export { CATEGORIES, RULE } from "./rules/types.ts";