@jpeakai/prose-gates 0.2.0 → 0.3.0

package/src/fix/engine.ts

@@ -3,9 +3,10 @@
 // edit that survives verification. Offsets are only ever valid for the
 // source they were computed from, so every accepted edit restarts the pass.
 
+import type { Root } from "mdast";
 import { buildDocModel, parse } from "../model.ts";
 import { BUILTIN, type Registry, ruleContext, TEXT_HELPERS } from "../rules/registry.ts";
-import type { Edit } from "../rules/types.ts";
+import type { Edit, Rule, RuleContext } from "../rules/types.ts";
 import { verify } from "./verify.ts";
 
 const MAX_PASSES = 10_000;
@@ -37,67 +38,84 @@ const wellFormed = (e: Edit, length: number): boolean =>
   e.end <= length &&
   typeof e.text === "string";
 
-export const fixMarkdownReport = async (src: string, registry: Registry = BUILTIN): Promise<FixReport> => {
-  const applied: Edit[] = [];
-  const refused: Edit[] = [];
-  // An edit refused once stays refused while its source slice is unchanged.
-  const refusedKeys = new Set<string>();
-  let current = src;
+// What the loop carries between passes. An edit refused once stays refused
+// while its source slice is unchanged, which `refusedKeys` remembers.
+interface FixState {
+  current: string;
+  applied: Edit[];
+  refused: Edit[];
+  refusedKeys: Set<string>;
+}
+
+type Before = { src: string; tree: Root };
+
+const keyOf = (state: FixState, e: Edit): string =>
+  `${e.rule}\u0000${state.current.slice(e.start, e.end)}\u0000${e.text}`;
+
+// Splice the edits, re-parse, and keep the result only if every edit is
+// proven against the tree the pass started from.
+const attempt = (state: FixState, edits: Edit[], before: Before): boolean => {
+  const next = splice(state.current, edits);
+  const afterTree = parse(next);
+  if (!edits.every((e) => verify(e, before, afterTree) === "accept")) return false;
+  state.current = next;
+  state.applied.push(...edits);
+  return true;
+};
+
+// An edit that points outside the source cannot be proven, so it is
+// refused before it is ever spliced. A built-in never proposes one.
+const keepWellFormed = (state: FixState, proposed: Edit[]): Edit[] =>
+  proposed.filter((e) => {
+    if (wellFormed(e, state.current.length)) return true;
+    const key = `${e.rule}\u0000malformed\u0000${e.start}:${e.end}`;
+    if (!state.refusedKeys.has(key)) {
+      state.refusedKeys.add(key);
+      state.refused.push(e);
+    }
+    return false;
+  });
 
+// The rule's edits that would change something and have not been refused.
+const proposeEdits = async (state: FixState, rule: Rule, ctx: RuleContext, registry: Registry): Promise<Edit[]> => {
+  const proposed =
+    (await rule.fix?.({ ...ctx, options: registry.options.get(rule.id) ?? {}, helpers: TEXT_HELPERS })) ?? [];
+  const sound = keepWellFormed(state, proposed);
+  return sound.filter((e) => state.current.slice(e.start, e.end) !== e.text && !state.refusedKeys.has(keyOf(state, e)));
+};
+
+// Whitespace and glyph edits leave the tree's shape alone, so a rule's
+// whole batch can be proven in one re-parse. A batch that fails falls
+// back to one edit at a time, so one bad edit never blocks the rest.
+const applyRuleEdits = (state: FixState, edits: Edit[], before: Before): boolean => {
+  const batchable = edits.length > 1 && !overlaps(edits) && edits.every((e) => e.expect.kind !== "replace-paragraph");
+  if (batchable && attempt(state, edits, before)) return true;
+  for (const edit of edits) {
+    if (attempt(state, [edit], before)) return true;
+    state.refusedKeys.add(keyOf(state, edit));
+    state.refused.push(edit);
+  }
+  return false;
+};
+
+// One pass: rules in fix order until one edit lands. True when it did.
+const runPass = async (state: FixState, registry: Registry): Promise<boolean> => {
+  const ctx = ruleContext(buildDocModel(state.current, registry.concepts), registry);
+  const before = { src: state.current, tree: ctx.doc.tree };
+  for (const rule of registry.fixOrder) {
+    const edits = await proposeEdits(state, rule, ctx, registry);
+    if (edits.length === 0) continue;
+    if (applyRuleEdits(state, edits, before)) return true;
+  }
+  return false;
+};
+
+export const fixMarkdownReport = async (src: string, registry: Registry = BUILTIN): Promise<FixReport> => {
+  const state: FixState = { current: src, applied: [], refused: [], refusedKeys: new Set<string>() };
   for (let pass = 0; pass < MAX_PASSES; pass++) {
-    const ctx = ruleContext(buildDocModel(current), registry);
-    const before = { src: current, tree: ctx.doc.tree };
-    const keyOf = (e: Edit): string => `${e.rule}\u0000${current.slice(e.start, e.end)}\u0000${e.text}`;
-
-    const attempt = (edits: Edit[]): boolean => {
-      const next = splice(current, edits);
-      const afterTree = parse(next);
-      if (edits.every((e) => verify(e, before, afterTree) === "accept")) {
-        current = next;
-        applied.push(...edits);
-        return true;
-      }
-      return false;
-    };
-
-    let progressed = false;
-    for (const rule of registry.fixOrder) {
-      const proposed =
-        (await rule.fix?.({ ...ctx, options: registry.options.get(rule.id) ?? {}, helpers: TEXT_HELPERS })) ?? [];
-      // An edit that points outside the source cannot be proven, so it is
-      // refused before it is ever spliced. A built-in never proposes one.
-      const sound = proposed.filter((e) => {
-        if (wellFormed(e, current.length)) return true;
-        const key = `${e.rule}\u0000malformed\u0000${e.start}:${e.end}`;
-        if (!refusedKeys.has(key)) {
-          refusedKeys.add(key);
-          refused.push(e);
-        }
-        return false;
-      });
-      const edits = sound.filter((e) => current.slice(e.start, e.end) !== e.text && !refusedKeys.has(keyOf(e)));
-      if (edits.length === 0) continue;
-
-      // Whitespace and glyph edits leave the tree's shape alone, so a rule's
-      // whole batch can be proven in one re-parse. A batch that fails falls
-      // back to one edit at a time, so one bad edit never blocks the rest.
-      const batchable =
-        edits.length > 1 && !overlaps(edits) && edits.every((e) => e.expect.kind !== "replace-paragraph");
-      if (batchable && attempt(edits)) {
-        progressed = true;
-        break;
-      }
-      for (const edit of edits) {
-        if (attempt([edit])) {
-          progressed = true;
-          break;
-        }
-        refusedKeys.add(keyOf(edit));
-        refused.push(edit);
-      }
-      if (progressed) break;
+    if (!(await runPass(state, registry))) {
+      return { output: state.current, applied: state.applied, refused: state.refused };
     }
-    if (!progressed) return { output: current, applied, refused };
   }
   throw new Error(`fix did not converge after ${MAX_PASSES} passes; a fixer keeps proposing edits`);
 };

package/src/fix/verify.ts

@@ -128,36 +128,48 @@ const withoutFrontmatter = (tree: Root): Root =>
 
 export type Verdict = "accept" | "refuse";
 
-export const verify = (edit: Edit, before: { src: string; tree: Root }, afterTree: Root): Verdict => {
-  const lost = fingerprint(before.tree).join(" ");
+// No word may be lost or moved. A mismatch is a fixer bug, so it throws and
+// names the first word where the two fingerprints part.
+const assertNoWordLost = (edit: Edit, beforeTree: Root, afterTree: Root): void => {
+  const lost = fingerprint(beforeTree).join(" ");
   const kept = fingerprint(afterTree).join(" ");
-  if (lost !== kept) {
-    const a = lost.split(" ");
-    const b = kept.split(" ");
-    const diverge = a.findIndex((w, i) => w !== b[i]);
-    const at = diverge === -1 ? a.length : diverge;
-    throw new FixInvariantError(edit, `word ${at} was "${a[at] ?? ""}", now "${b[at] ?? ""}"`);
-  }
+  if (lost === kept) return;
+  const a = lost.split(" ");
+  const b = kept.split(" ");
+  const diverge = a.findIndex((w, i) => w !== b[i]);
+  const at = diverge === -1 ? a.length : diverge;
+  throw new FixInvariantError(edit, `word ${at} was "${a[at] ?? ""}", now "${b[at] ?? ""}"`);
+};
 
-  const expect = edit.expect;
-  if (expect.kind === "same-tree") {
-    return sameJson(normalise(before.tree), normalise(afterTree)) ? "accept" : "refuse";
-  }
-  if (expect.kind === "same-frontmatter-data") {
-    const was = frontmatterData(before.tree);
-    const now = frontmatterData(afterTree);
-    if (was === INVALID || now === INVALID || !sameJson(was, now)) return "refuse";
-    return sameJson(normalise(withoutFrontmatter(before.tree)), normalise(withoutFrontmatter(afterTree)))
-      ? "accept"
-      : "refuse";
-  }
-  if (expect.kind === "same-shape") {
-    return sameJson(normalise(before.tree, { blankText: true }), normalise(afterTree, { blankText: true }))
-      ? "accept"
-      : "refuse";
-  }
+const verdictOf = (ok: boolean): Verdict => (ok ? "accept" : "refuse");
+
+// The frontmatter must mean the same data, and the rest of the tree must be
+// untouched.
+const sameFrontmatterData = (beforeTree: Root, afterTree: Root): Verdict => {
+  const was = frontmatterData(beforeTree);
+  const now = frontmatterData(afterTree);
+  if (was === INVALID || now === INVALID || !sameJson(was, now)) return "refuse";
+  return verdictOf(sameJson(normalise(withoutFrontmatter(beforeTree)), normalise(withoutFrontmatter(afterTree))));
+};
+
+type ReplaceParagraph = Extract<Edit["expect"], { kind: "replace-paragraph" }>;
+
+// The paragraph must become exactly the fragment the fixer declared, parsed
+// with the document's definitions, and nothing else may change.
+const replacedParagraph = (expect: ReplaceParagraph, before: { src: string; tree: Root }, afterTree: Root): Verdict => {
   const nodes = parseFragment(expect.fragment, definitionsOf(before.src, before.tree));
   if (!fragmentShapeOk(nodes, expect.listItems)) return "refuse";
   const expected = normalise(before.tree, { swap: [expect.view.node, nodes] });
-  return sameJson(expected, normalise(afterTree)) ? "accept" : "refuse";
+  return verdictOf(sameJson(expected, normalise(afterTree)));
+};
+
+export const verify = (edit: Edit, before: { src: string; tree: Root }, afterTree: Root): Verdict => {
+  assertNoWordLost(edit, before.tree, afterTree);
+  const expect = edit.expect;
+  if (expect.kind === "same-tree") return verdictOf(sameJson(normalise(before.tree), normalise(afterTree)));
+  if (expect.kind === "same-frontmatter-data") return sameFrontmatterData(before.tree, afterTree);
+  if (expect.kind === "same-shape") {
+    return verdictOf(sameJson(normalise(before.tree, { blankText: true }), normalise(afterTree, { blankText: true })));
+  }
+  return replacedParagraph(expect, before, afterTree);
 };

package/src/index.ts

@@ -2,16 +2,29 @@
 
 export { checkMarkdown, checkModel, DEFAULT_MAX_WORDS } from "./check.ts";
 export { main } from "./cli.ts";
+export {
+  CONCEPT_FILES,
+  type Concept,
+  type ConceptMention,
+  type ConceptModel,
+  type ConceptRelation,
+  KIND_OF,
+  loadConcepts,
+  NO_CONCEPTS,
+  parseConcepts,
+} from "./concepts.ts";
 export { type Config, loadConfig, type PluginLoading, parseConfig, type RuleSetting, type Severity } from "./config.ts";
 export * from "./errors.ts";
 export { type FixReport, fixMarkdown, fixMarkdownReport } from "./fix/engine.ts";
 export { FixInvariantError } from "./fix/verify.ts";
+export type { LanguageSentence, LanguageView, LanguageWord } from "./language.ts";
 export {
   buildDocModel,
   type DocModel,
   type FrontmatterEntry,
   type FrontmatterScalar,
   type FrontmatterView,
+  type ParagraphView,
   type ScalarStyle,
 } from "./model.ts";
 export type { Plugin, PluginEdit, PluginFinding, PluginMeta, PluginRule } from "./plugins/contract.ts";

package/src/language.ts

@@ -0,0 +1,107 @@
+// The language model of a paragraph: its sentences, and each sentence's words
+// with their part-of-speech tags and lemmas. Everything here takes a string and
+// returns plain data. compromise does the work, pinned exactly so a finding
+// never changes under an upgrade: it splits the sentences, tags each word and
+// gives its lemma. A paragraph exposes this as its lazy `language` view, so a
+// rule that never reads it pays nothing.
+
+import nlp from "compromise";
+import { type ConceptMention, type ConceptModel, mentionsIn, NO_CONCEPTS } from "./concepts.ts";
+
+// One word of a sentence. `lemma` is lower case, so "Parsing" reads as "parse".
+// `content` is true for a noun, verb or adjective that is not a pronoun or an
+// auxiliary, and never for the CODE token that stands for inline code.
+export interface LanguageWord {
+  text: string;
+  lemma: string;
+  tags: readonly string[];
+  content: boolean;
+}
+
+// One sentence, as the paragraph's prose holds it, with inline code as CODE.
+// `terms` is the lemmas of its content words. `opener` is CODE when the
+// sentence opens with inline code, else the lemma of a content word it opens
+// with, else null. `refersBack` is true when its first word is a pronoun, a
+// possessive, an adverb or a connective: "It", "Its", "Then", "However".
+// `verbPhrases` is how many verb phrases compromise finds, with an auxiliary
+// counted inside its verb, so "is shared" is one. `concepts` is each place it
+// names a concept of the project's concept file, empty when it has none.
+export interface LanguageSentence {
+  text: string;
+  words: readonly LanguageWord[];
+  terms: ReadonlySet<string>;
+  opener: string | null;
+  refersBack: boolean;
+  verbPhrases: number;
+  concepts: readonly ConceptMention[];
+}
+
+export interface LanguageView {
+  sentences: readonly LanguageSentence[];
+}
+
+const CONTENT = new Set(["Noun", "Verb", "Adjective"]);
+const FUNCTION = new Set(["Pronoun", "Possessive", "Auxiliary", "Copula", "Modal", "Determiner"]);
+const REFERS_BACK = new Set(["Pronoun", "Possessive", "Adverb", "Conjunction"]);
+
+// The shape compromise returns for a term once compute("root") has run.
+interface TaggedTerm {
+  text: string;
+  normal: string;
+  root?: string;
+  tags: string[];
+}
+
+const wordOf = (term: TaggedTerm): LanguageWord => ({
+  text: term.text,
+  lemma: (term.root ?? term.normal).toLowerCase(),
+  tags: term.tags,
+  content: term.text !== "CODE" && term.tags.some((t) => CONTENT.has(t)) && !term.tags.some((t) => FUNCTION.has(t)),
+});
+
+// One tagging pass gives the words and the verb phrases.
+const tag = (sentence: string): { words: LanguageWord[]; verbPhrases: number } => {
+  const doc = nlp(sentence);
+  // A participle between a determiner, adjective or number and a noun
+  // describes the noun, as in "the signed tag", so it is no verb phrase.
+  doc.match("(#Determiner|#Adjective|#Value) [(#PastTense|#Participle)] #Noun", 0).tag("Adjective");
+  doc.compute("root");
+  // compromise's option types omit root, which compute("root") adds.
+  const rows = doc.json({ terms: { normal: true, root: true, tags: true } } as object) as { terms: TaggedTerm[] }[];
+  return { words: rows.flatMap((row) => row.terms).map(wordOf), verbPhrases: doc.verbs().length };
+};
+
+const openerOf = (first: LanguageWord | undefined): string | null => {
+  if (!first) return null;
+  if (first.text === "CODE") return "CODE";
+  return first.content ? first.lemma : null;
+};
+
+// One sentence's words and what they say about cohesion, and the concepts it names.
+const sentenceWith = (text: string, concepts: ConceptModel): LanguageSentence => {
+  const { words, verbPhrases } = tag(text);
+  const first = words[0];
+  return {
+    text,
+    words,
+    terms: new Set(words.filter((w) => w.content).map((w) => w.lemma)),
+    opener: openerOf(first),
+    refersBack: first?.tags.some((t) => REFERS_BACK.has(t)) ?? false,
+    verbPhrases,
+    concepts: mentionsIn(words, concepts),
+  };
+};
+
+// One sentence on its own, with no concepts. It takes one argument, so it is
+// safe to pass to map.
+export const sentenceOf = (text: string): LanguageSentence => sentenceWith(text, NO_CONCEPTS);
+
+// The sentences of a paragraph's prose. compromise keeps abbreviations such
+// as e.g. and Dr., decimals and urls inside the sentence they belong to, and
+// still ends one after a number or before a lower-case word.
+export const sentenceTexts = (prose: string): string[] =>
+  (nlp(prose).sentences().out("array") as string[]).map((s) => s.trim()).filter((s) => s.length > 0);
+
+export const languageOf = (prose: string, concepts: ConceptModel = NO_CONCEPTS): LanguageView => ({
+  sentences: sentenceTexts(prose).map((s) => sentenceWith(s, concepts)),
+});