chaffjs 0.18.0 → 0.20.0

package/src/run.ts

@@ -8,14 +8,24 @@ import { missingList } from "./declared-lists.ts";
 import { unreadStructure, type Unread } from "./structure/unread.ts";
 import { isMarkdownPath } from "./structure/markdown-path.ts";
 import { maskSpans } from "./mask.ts";
+import { structureText } from "./document-reading.ts";
 import { textOutline } from "./page-furniture.ts";
 import { uiLanguageOf } from "./ui.ts";
 import { presetLevels } from "./genre-load.ts";
 import { bodySectionOf } from "./body-section.ts";
 import { optionValues, settleOptions, type OptionLayer } from "./rule-options.ts";
+import { byPosition } from "./finding-order.ts";
 import { PatternTimeout } from "./custom/bounded-match.ts";
-
-export type Skipped = { readonly rule: string; readonly why: string };
+import { PluginRuleFailure } from "./extension/module-detector.ts";
+import { failureReason } from "./extension/failure-text.ts";
+import { tagCoverage } from "./tag-coverage.ts";
+
+export type Skipped = {
+  readonly rule: string;
+  readonly why: string;
+  /** Off only because it is experimental: --experimental (or chaff.yaml's experimental) would run it. */
+  readonly offUntilExperimental?: true;
+};
 
 export type RunResult = {
   readonly findings: readonly Finding[];
@@ -28,7 +38,9 @@ export type RunResult = {
 
 export type Settings = Readonly<Record<string, Level>>;
 
-const levelFor = (rule: RuleDefinition, settings: Settings, experimental: boolean, preset: Settings): Level => {
+type Outcome = { readonly findings: Finding[]; readonly skipped: Skipped[] };
+
+export const levelFor = (rule: RuleDefinition, settings: Settings, experimental: boolean, preset: Settings): Level => {
   const explicit = settings[rule.id];
   // 明示設定は status の既定に勝つ。名指しで有効にしたものを黙って無効にしない。
   if (explicit !== undefined) return explicit;
@@ -40,9 +52,10 @@ const levelFor = (rule: RuleDefinition, settings: Settings, experimental: boolea
 };
 
 /** Why a rule at off did not run: chaff.yaml turned it off, the genre's preset did, or it is experimental. */
-const offReason = (rule: RuleDefinition, settings: Settings, preset: Settings, genre: string, reasons: Reasons): string => {
-  if (settings[rule.id] !== undefined) return reasons.turnedOff;
-  return preset[rule.id] === undefined ? reasons.experimental : reasons.presetOff(genre);
+const offSkip = (rule: RuleDefinition, settings: Settings, preset: Settings, genre: string, reasons: Reasons): Skipped => {
+  if (settings[rule.id] !== undefined) return { rule: rule.id, why: reasons.turnedOff };
+  if (preset[rule.id] !== undefined) return { rule: rule.id, why: reasons.presetOff(genre) };
+  return { rule: rule.id, why: reasons.experimental, offUntilExperimental: true };
 };
 
 const reasonsFor = (doc: ProseDocument): Reasons => REASONS[uiLanguageOf(doc.language)];
@@ -70,8 +83,11 @@ const treeProblem = (doc: ProseDocument): string | undefined => {
 };
 
 /** 木が読んだのと同じ本文。Markdown はコードを覆ったもの、.txt はページの飾りだけを覆ったもの（.txt の字下げはコードではない）。 */
-const textTheTreeRead = (doc: ProseDocument): string =>
-  isMarkdownPath(doc.path) ? (doc.prose ?? doc.source) : maskSpans(doc.source, textOutline(doc.source, doc.replyQuotes).opaque);
+const textTheTreeRead = (doc: ProseDocument): string => {
+  if (isMarkdownPath(doc.path)) return doc.prose ?? doc.source;
+  const text = structureText(doc.path, doc.source);
+  return maskSpans(text, textOutline(text, doc.replyQuotes).opaque);
+};
 
 const unreadByDocument = new WeakMap<ProseDocument, Unread | undefined>();
 
@@ -83,11 +99,16 @@ const unreadOf = (doc: ProseDocument): Unread | undefined => {
 
 /** 見出しを読む rule の要求。表題より下の見出しが無い文書では、本題の前を測れない。0 件を「前置きが短い」に見せない。 */
 /** 記法を読む rule の要求（markdown）。.txt には見出しの記法も画像もリンクの記法も無いので、0 件を「問題なし」に見せない。 */
-const DOCUMENT_NEEDS: ReadonlySet<string> = new Set(["headings", "markdown"]);
+/** documents: a rule that compares the documents of one run (cross-run.ts). Run on one document, it says so instead of finding nothing. */
+const DOCUMENT_NEEDS: ReadonlySet<string> = new Set(["headings", "markdown", "documents"]);
 
 const documentNeed = (rule: RuleDefinition, doc: ProseDocument): string | undefined => {
   if (rule.requires.includes("markdown") && !isMarkdownPath(doc.path)) return reasonsFor(doc).notMarkdown;
-  return rule.requires.includes("headings") && bodySectionOf(doc.sections) === undefined ? reasonsFor(doc).noHeadings : undefined;
+  if (rule.requires.includes("headings") && bodySectionOf(doc.sections) === undefined) return reasonsFor(doc).noHeadings;
+  if (!rule.requires.includes("documents")) return undefined;
+  // Set aside only what the cross pass can run: a missing word list is said here, as for any rule.
+  const absent = missingList(rule, doc.lexicons);
+  return absent === undefined ? reasonsFor(doc).oneDocument : reasonsFor(doc).noLexicon(doc.language, absent);
 };
 
 /** 知らない要求は満たされていないものとして扱う。黙って無視すると、要求なしで動いてしまう。 */
@@ -102,13 +123,21 @@ const has = (capabilities: ProseDocument["capabilities"], need: string): boolean
  *
  * `capabilities.pos: true` と言いながら token を返さないアダプタでも、rule は
  * `sentence.tokens ?? []` を見るので**例外にならず、指摘 0 件で終わる**。
- * 0 件は「問題なし」と見分けがつかない。
+ * 0 件は「問題なし」と見分けがつかない。一部の文にだけ token が無いとき（解析器が読めなかった段落）も、その段落の 0 件が同じく保証に見える。
  */
-const hasTokens = (doc: ProseDocument): boolean => doc.sentences.length === 0 || doc.sentences.some((sentence) => sentence.tokens !== undefined);
+const tagReason = (doc: ProseDocument): string | undefined => {
+  const coverage = tagCoverage(doc.sentences);
+  if (coverage === "all") return undefined;
+  return coverage === "none" ? reasonsFor(doc).noTags : reasonsFor(doc).unreadTags;
+};
 
 /** 要求を満たさない rule は動かせない。満たさないまま動かすと「指摘 0 件」が保証に見える。 */
+const otherLanguage = (rule: RuleDefinition, doc: ProseDocument): string | undefined =>
+  rule.languages !== undefined && !rule.languages.includes(doc.language) ? reasonsFor(doc).otherLanguage(doc.language) : undefined;
+
 const unmet = (rule: RuleDefinition, doc: ProseDocument): string | undefined => {
-  if (rule.languages !== undefined && !rule.languages.includes(doc.language)) return reasonsFor(doc).otherLanguage(doc.language);
+  const language = otherLanguage(rule, doc);
+  if (language !== undefined) return language;
   const missing = rule.requires.filter((need) => !TREE_NEEDS.has(need) && !DOCUMENT_NEEDS.has(need)).find((need) => !has(doc.capabilities, need));
   if (missing !== undefined) return reasonsFor(doc).noCapability(missing);
   return undefined;
@@ -119,10 +148,15 @@ const unmet = (rule: RuleDefinition, doc: ProseDocument): string | undefined =>
  * 先に聞くと、止めている rule まで「アダプタが品詞を返さなかった」と、違う理由で出る。
  */
 const untagged = (rule: RuleDefinition, doc: ProseDocument): string | undefined =>
-  rule.requires.some((need) => need === "pos" || need === "lemma") && !hasTokens(doc) ? reasonsFor(doc).noTags : undefined;
+  rule.requires.some((need) => need === "pos" || need === "lemma") ? tagReason(doc) : undefined;
+
+const suitsGenre = (rule: RuleDefinition, genre: string): boolean => rule.use_for.some((target) => genre.startsWith(target));
+
+const forGenre = (rules: readonly RuleDefinition[], genre: string): RuleDefinition[] => rules.filter((rule) => suitsGenre(rule, genre));
 
-const forGenre = (rules: readonly RuleDefinition[], genre: string): RuleDefinition[] =>
-  rules.filter((rule) => rule.use_for.some((target) => genre.startsWith(target)));
+/** A rule use_for keeps out of the genre. A rule for another language says that first: no genre would run it here. */
+const unsuited = (rule: RuleDefinition, doc: ProseDocument, genre: string): string | undefined =>
+  suitsGenre(rule, genre) ? undefined : (otherLanguage(rule, doc) ?? reasonsFor(doc).presetOff(genre));
 
 /** rule が品詞か見出し語を要求するか、使えるなら使うか。 */
 export const wantsTags = (rule: RuleDefinition): boolean => [...rule.requires, ...rule.uses].some((need) => need === "pos" || need === "lemma");
@@ -141,21 +175,26 @@ export const neededBy = (rules: readonly RuleDefinition[], settings: Settings, e
 /** The token features the rules read (RuleDefinition.token_features), each once. */
 export const tokenFeaturesOf = (rules: readonly RuleDefinition[]): string[] => [...new Set(rules.flatMap((rule) => rule.token_features ?? []))];
 
-/** A detector's findings, or how long it ran before a team's pattern was stopped (bounded-match.ts). Other errors are chaff's bugs and propagate. */
+/**
+ * A detector's findings, or why the rule did not run: a team's pattern stopped at its time limit (bounded-match.ts), or a
+ * plugin's detector that threw or returned something else than findings (module-detector.ts). Other errors are chaff's
+ * bugs and propagate.
+ */
 const runDetector = (
   detector: Detector,
   doc: ProseDocument,
   options: DetectorOptions,
-): { readonly findings: readonly Finding[] } | { readonly timedOut: number } => {
+): { readonly findings: readonly Finding[] } | { readonly notRun: string } => {
   try {
     return { findings: detector(doc, options) };
   } catch (error) {
-    if (error instanceof PatternTimeout) return { timedOut: error.budget_ms };
+    if (error instanceof PatternTimeout) return { notRun: reasonsFor(doc).patternTimeout(error.budget_ms) };
+    if (error instanceof PluginRuleFailure) return { notRun: failureReason(error.origin, error.failure, uiLanguageOf(doc.language)) };
     throw error;
   }
 };
 
-const place = (starts: readonly number[], finding: Finding): Finding => {
+export const place = (starts: readonly number[], finding: Finding): Finding => {
   const offset = finding.values["offset"];
   const at = placeOf(starts, typeof offset === "number" ? offset : 0);
   return { ...finding, line: at.line, column: at.column };
@@ -187,7 +226,7 @@ const compositeOf = (rule: RuleDefinition, found: readonly Finding[], limit: num
 /** chaff.yaml の rules に数値で書いた上限。段階の表より先に効く。 */
 export type Limits = Readonly<Record<string, number>>;
 
-const limitFor = (rule: RuleDefinition, level: Level, genre: string, limits: Limits): number => limits[rule.id] ?? resolve(rule, level, genre).limit;
+export const limitFor = (rule: RuleDefinition, level: Level, genre: string, limits: Limits): number => limits[rule.id] ?? resolve(rule, level, genre).limit;
 
 /** 文書と違う言語で書いた文の上限。chaff.yaml の数値は文書の言語の単位で書いたものなので、ここには効かせない。 */
 const embeddedLimitsFor = (rule: RuleDefinition, level: Level, genre: string, embedded: readonly string[]): Record<string, number> =>
@@ -210,6 +249,8 @@ export type RunContext = {
   readonly limits?: Limits;
   /** Where rule options come from, strongest first (chaff.yaml). An option no layer sets is at its default. */
   readonly optionLayers?: readonly OptionLayer[];
+  /** The detectors loaded from the code chaff.yaml names, by rule id. They come before chaff's own table. */
+  readonly detectors?: Readonly<Record<string, Detector>>;
 };
 
 export const runRules = (
@@ -222,7 +263,7 @@ export const runRules = (
 ): RunResult => runRulesWith(doc, rules, { settings, experimental, genre, limits });
 
 export const runRulesWith = (doc: ProseDocument, rules: readonly RuleDefinition[], context: RunContext): RunResult => {
-  const { settings, experimental, genre, limits = {}, optionLayers = [] } = context;
+  const { settings, experimental, genre, limits = {}, optionLayers = [], detectors = {} } = context;
   const preset = presetLevels(genre);
   const starts = lineStarts(doc.source);
   const applicable = forGenre(rules, genre);
@@ -230,47 +271,51 @@ export const runRulesWith = (doc: ProseDocument, rules: readonly RuleDefinition[
   const experimentalOn = applicable.filter((rule) => rule.status === "experimental" && levelFor(rule, settings, experimental, preset) !== "off");
   const forced = experimentalOn.filter((rule) => settings[rule.id] !== undefined).map((rule) => rule.id);
   const presetOn = experimentalOn.filter((rule) => settings[rule.id] === undefined && preset[rule.id] !== undefined).map((rule) => rule.id);
-  const outcome = applicable.reduce<{ findings: Finding[]; skipped: Skipped[] }>(
+  const runOne = (acc: Outcome, rule: RuleDefinition): Outcome => {
+    // L4 は意味を読む検査。chaff test が扱う。ここで「検出器が無い」と言わせない。
+    if (rule.layer === "L4") {
+      return { findings: acc.findings, skipped: [...acc.skipped, { rule: rule.id, why: reasonsFor(doc).semantic }] };
+    }
+    const blocked = unmet(rule, doc);
+    if (blocked !== undefined) return { findings: acc.findings, skipped: [...acc.skipped, { rule: rule.id, why: blocked }] };
+    const level = levelFor(rule, settings, experimental, preset);
+    if (level === "off") return { findings: acc.findings, skipped: [...acc.skipped, offSkip(rule, settings, preset, genre, reasonsFor(doc))] };
+    const noTags = untagged(rule, doc);
+    if (noTags !== undefined) return { findings: acc.findings, skipped: [...acc.skipped, { rule: rule.id, why: noTags }] };
+    // 木は capability ではなく、adapter が structure を持つかで決まる。持たない言語で動かすと「参照先が無い」が 0 件に見える。
+    // 段階を見た後で聞く。doc.structure は触れたときに木を作るので、止めている rule のために作らない。
+    const noTree = treeNeed(rule, doc);
+    if (noTree !== undefined) return { findings: acc.findings, skipped: [...acc.skipped, { rule: rule.id, why: noTree }] };
+    const noDocumentNeed = documentNeed(rule, doc);
+    if (noDocumentNeed !== undefined) return { findings: acc.findings, skipped: [...acc.skipped, { rule: rule.id, why: noDocumentNeed }] };
+    // 複合シグナルは二段目で扱う。一段目では「検出器が無い」と言わせない。
+    if (rule.from.length > 0) return acc;
+    const detector = detectors[rule.id] ?? DETECTORS[rule.how_to_find];
+    if (detector === undefined)
+      return { findings: acc.findings, skipped: [...acc.skipped, { rule: rule.id, why: reasonsFor(doc).noDetector(rule.how_to_find) }] };
+    // 語彙表を要求する rule で、その言語に語彙表が無ければ動かせない。黙って通さない。
+    const absent = missingList(rule, doc.lexicons);
+    if (absent !== undefined)
+      return { findings: acc.findings, skipped: [...acc.skipped, { rule: rule.id, why: reasonsFor(doc).noLexicon(doc.language, absent) }] };
+    const options = {
+      limit: limitFor(rule, level, genre, limits),
+      lexicon: rule.word_list === undefined ? undefined : doc.lexicons[rule.word_list],
+      where: rule.where,
+      fullSentence: rule.full_sentence,
+      embeddedLimits: embeddedLimitsFor(rule, level, genre, embedded),
+      ...(rule.options === undefined ? {} : { settings: optionValues(settleOptions(rule.id, rule.options, optionLayers)) }),
+      ...(rule.custom === undefined ? {} : { custom: rule.custom }),
+    };
+    const ran = runDetector(detector, doc, options);
+    if ("notRun" in ran) return { findings: acc.findings, skipped: [...acc.skipped, { rule: rule.id, why: ran.notRun }] };
+    const found = ran.findings.map((finding) => place(starts, { ...finding, rule: rule.id, severity: severityAt(rule, level, genre) }));
+    return { findings: [...acc.findings, ...found], skipped: acc.skipped };
+  };
+  const outcome = rules.reduce<Outcome>(
     (acc, rule) => {
-      // L4 は意味を読む検査。chaff test が扱う。ここで「検出器が無い」と言わせない。
-      if (rule.layer === "L4") {
-        return { findings: acc.findings, skipped: [...acc.skipped, { rule: rule.id, why: reasonsFor(doc).semantic }] };
-      }
-      const blocked = unmet(rule, doc);
-      if (blocked !== undefined) return { findings: acc.findings, skipped: [...acc.skipped, { rule: rule.id, why: blocked }] };
-      const level = levelFor(rule, settings, experimental, preset);
-      if (level === "off")
-        return { findings: acc.findings, skipped: [...acc.skipped, { rule: rule.id, why: offReason(rule, settings, preset, genre, reasonsFor(doc)) }] };
-      const noTags = untagged(rule, doc);
-      if (noTags !== undefined) return { findings: acc.findings, skipped: [...acc.skipped, { rule: rule.id, why: noTags }] };
-      // 木は capability ではなく、adapter が structure を持つかで決まる。持たない言語で動かすと「参照先が無い」が 0 件に見える。
-      // 段階を見た後で聞く。doc.structure は触れたときに木を作るので、止めている rule のために作らない。
-      const noTree = treeNeed(rule, doc);
-      if (noTree !== undefined) return { findings: acc.findings, skipped: [...acc.skipped, { rule: rule.id, why: noTree }] };
-      const noDocumentNeed = documentNeed(rule, doc);
-      if (noDocumentNeed !== undefined) return { findings: acc.findings, skipped: [...acc.skipped, { rule: rule.id, why: noDocumentNeed }] };
-      // 複合シグナルは二段目で扱う。一段目では「検出器が無い」と言わせない。
-      if (rule.from.length > 0) return acc;
-      const detector = DETECTORS[rule.how_to_find];
-      if (detector === undefined)
-        return { findings: acc.findings, skipped: [...acc.skipped, { rule: rule.id, why: reasonsFor(doc).noDetector(rule.how_to_find) }] };
-      // 語彙表を要求する rule で、その言語に語彙表が無ければ動かせない。黙って通さない。
-      const absent = missingList(rule, doc.lexicons);
-      if (absent !== undefined)
-        return { findings: acc.findings, skipped: [...acc.skipped, { rule: rule.id, why: reasonsFor(doc).noLexicon(doc.language, absent) }] };
-      const options = {
-        limit: limitFor(rule, level, genre, limits),
-        lexicon: rule.word_list === undefined ? undefined : doc.lexicons[rule.word_list],
-        where: rule.where,
-        fullSentence: rule.full_sentence,
-        embeddedLimits: embeddedLimitsFor(rule, level, genre, embedded),
-        ...(rule.options === undefined ? {} : { settings: optionValues(settleOptions(rule.id, rule.options, optionLayers)) }),
-        ...(rule.custom === undefined ? {} : { custom: rule.custom }),
-      };
-      const ran = runDetector(detector, doc, options);
-      if ("timedOut" in ran) return { findings: acc.findings, skipped: [...acc.skipped, { rule: rule.id, why: reasonsFor(doc).patternTimeout(ran.timedOut) }] };
-      const found = ran.findings.map((finding) => place(starts, { ...finding, rule: rule.id, severity: severityAt(rule, level, genre) }));
-      return { findings: [...acc.findings, ...found], skipped: acc.skipped };
+      // Listed, not dropped: a clean run in a genre that leaves rules out is not "checked and fine".
+      const notForGenre = unsuited(rule, doc, genre);
+      return notForGenre === undefined ? runOne(acc, rule) : { findings: acc.findings, skipped: [...acc.skipped, { rule: rule.id, why: notForGenre }] };
     },
     { findings: [], skipped: [] },
   );
@@ -280,5 +325,5 @@ export const runRulesWith = (doc: ProseDocument, rules: readonly RuleDefinition[
       compositeOf(rule, outcome.findings, limitFor(rule, levelFor(rule, settings, experimental, preset), genre, limits), starts, doc.language),
     );
   const all = [...outcome.findings, ...composites];
-  return { ...outcome, findings: all.toSorted((left, right) => left.line - right.line), forcedExperimental: forced, presetExperimental: presetOn };
+  return { ...outcome, findings: all.toSorted(byPosition), forcedExperimental: forced, presetExperimental: presetOn };
 };

package/src/sentence-load.ts

@@ -0,0 +1,77 @@
+// How much one sentence asks the reader to hold at once: brackets inside brackets, clauses chained one after another, and how
+// far the topic stands from the end where its predicate is. Pure: a sentence's text and tokens come in, numbers go out.
+import type { Token } from "./plugin.ts";
+
+/**
+ * Opening brackets and the mark that closes each. Quotation marks that open and close with the same character are left out, and
+ * so are curly braces, which prose rarely uses and code and formulas use everywhere.
+ */
+const BRACKETS: Readonly<Record<string, string>> = {
+  "(": ")",
+  "（": "）",
+  "[": "]",
+  "［": "］",
+  "「": "」",
+  "『": "』",
+  "【": "】",
+  "〔": "〕",
+  "〈": "〉",
+  "《": "》",
+  "“": "”",
+};
+
+/** A formula written in TeX ($d^k = -\nabla f(x^k)$): its brackets are the formula's, not the sentence's. */
+const FORMULA = /\$\$[^$]*\$\$|\$[^$\n]*\$/gu;
+
+/**
+ * The deepest the brackets of a text nest: 「計画（案）」 is one, 「（注（a）を参照）」 two, plain text none. A closer with no
+ * opener (the 1) of a list) is ignored, an opener never closed counts as open until the end, and formulas in $…$ are skipped.
+ */
+export const bracketDepth = (text: string): number => {
+  // The closers still awaited, innermost last. Pushed and popped in place: copying it per character is quadratic on deep nesting.
+  const awaited: string[] = [];
+  return [...text.replace(FORMULA, " ")].reduce((deepest, char) => {
+    if (awaited.at(-1) === char) awaited.pop();
+    else if (BRACKETS[char] !== undefined) awaited.push(BRACKETS[char]);
+    return Math.max(deepest, awaited.length);
+  }, 0);
+};
+
+const PREDICATE = new Set(["VERB", "ADJ", "AUX"]);
+const COMMA = new Set(["、", "，"]);
+
+/**
+ * Whether the token ends a clause that the sentence then goes on from: a conjunctive particle (調べて, 報告したが, ないので), or
+ * a verb or adjective stopped by a comma (内容をまとめ、). て or で before a helper verb (調べている, 読んでもらう) joins two verbs
+ * into one predicate and does not end a clause.
+ */
+const endsClause = (token: Token, next: Token | undefined): boolean => {
+  if (token.pos === "SCONJ") return next?.features?.["Bound"] !== "Yes";
+  return PREDICATE.has(token.pos) && next !== undefined && COMMA.has(next.surface);
+};
+
+const TRAILING = new Set(["PUNCT", "SYM"]);
+
+/**
+ * The number of clauses the tokens chain: one more than the clause ends before the last word. A sentence of one predicate is
+ * one, and a sentence that ends on a particle (返信して。) has not chained another clause by it.
+ */
+export const clauseCount = (tokens: readonly Token[]): number => {
+  const last = tokens.findLastIndex((token) => !TRAILING.has(token.pos));
+  return 1 + tokens.filter((token, at) => at < last && endsClause(token, tokens[at + 1])).length;
+};
+
+const TOPIC_BEFORE = new Set(["NOUN", "PRON", "PROPN"]);
+
+/**
+ * How far the sentence's first topic (会議は, これは) stands from the end of the sentence, where Japanese puts the predicate the
+ * topic belongs to, in characters of the words between them (spaces, and a link's address the document hides, are not read).
+ * undefined when the sentence has no topic. The distance is the reader's wait: everything between has to be held before the
+ * topic is said to do anything.
+ */
+export const topicDistance = (tokens: readonly Token[]): number | undefined => {
+  const topic = tokens.findIndex((token, at) => token.surface === "は" && token.pos === "ADP" && TOPIC_BEFORE.has(tokens[at - 1]?.pos ?? ""));
+  const last = tokens.findLastIndex((token) => !TRAILING.has(token.pos));
+  if (topic === -1 || last <= topic) return undefined;
+  return tokens.slice(topic + 1, last).reduce((sum, token) => sum + [...token.surface.replace(/\s/gu, "")].length, 0);
+};

package/src/setting-problems.ts

@@ -4,6 +4,7 @@ import type { UiLanguage } from "./ui.ts";
 import { styleProblems } from "./config/style.ts";
 import { loadStyles } from "./style-load.ts";
 import { customRuleProblems } from "./custom/problems.ts";
+import { extensionProblems } from "./extension/problem-text.ts";
 import { GENRES } from "./genre.ts";
 import { unknownGenres, writtenGenres } from "./genre-check.ts";
 import { loadProfiles } from "./profile/load.ts";
@@ -25,6 +26,7 @@ const profileProblems = (command: string, config: Config, text: CliText): string
 export const settingProblems = (command: string, genreFlag: string | undefined, config: Config, text: CliText, ui: UiLanguage = "en"): string[] => [
   ...genreProblems(command, genreFlag, config, text),
   ...profileProblems(command, config, text),
-  ...styleProblems(config, loadStyles(), ui),
+  ...styleProblems(config, [...loadStyles(), ...(config.extensions?.styles ?? [])], ui),
   ...customRuleProblems(config, ui),
+  ...extensionProblems(config, ui),
 ];

package/src/spacing-minority.ts

@@ -0,0 +1,27 @@
+import { minorityStyle } from "./orthography.ts";
+
+/**
+ * 少ないほうがこの割合を超えると、書き損じではなく、書き方が二通りある文書と読む。1 箇所ずつ「違う」と指すと、
+ * 半々に近い文書でどちらかの側に立つことになる。そのときは 1 件にまとめて、両方の数を言う。
+ */
+const MIXED_SHARE = 0.2;
+
+/** 割合が大きくても、これより少なければ 1 箇所ずつ言う。3 箇所のうち 1 箇所なら、その 1 箇所を見せるほうが早い。 */
+const MIN_MIXED = 5;
+
+/** each は少ないほうを 1 箇所ずつ、mixed は文書に書き方が二通りあることを 1 件で。first は少ないほうの最初の箇所。 */
+export type MinorityReport<T> =
+  | { readonly mode: "each"; readonly odd: readonly T[] }
+  | { readonly mode: "mixed"; readonly first: T; readonly odd: number; readonly spaced: number; readonly touching: number };
+
+/** 一つの種類の境目から、何をどう言うか。混ざっていない、または少ないほうが limit に届かなければ言わない。 */
+export const minorityReport = <T extends { readonly spaced: boolean }>(ofKind: readonly T[], limit: number): MinorityReport<T> | undefined => {
+  const minority = minorityStyle(ofKind);
+  if (minority === undefined) return undefined;
+  const odd = ofKind.filter((entry) => entry.spaced === minority);
+  const first = odd[0];
+  if (first === undefined || odd.length < limit) return undefined;
+  if (odd.length < MIN_MIXED || odd.length <= ofKind.length * MIXED_SHARE) return { mode: "each", odd };
+  const spaced = ofKind.filter((entry) => entry.spaced).length;
+  return { mode: "mixed", first, odd: odd.length, spaced, touching: ofKind.length - spaced };
+};

package/src/spelling-variants.ts

@@ -0,0 +1,42 @@
+// One word written two ways in one document, found without a hand list: the words come in already keyed (two spellings of one
+// word share a key), and this picks, per key, the spellings the document uses less. Pure: no document, no language here.
+
+/** A word as written: the key that says which word it is, and how this place spells it. */
+export type KeyedWord = { readonly key: string; readonly spelling: string };
+
+/** A word spelled the way the document uses less: the spelling it usually uses, and how many of the word's places differ. */
+export type OddSpelling<T extends KeyedWord> = { readonly word: T; readonly usual: string; readonly count: number; readonly of: number };
+
+const PERCENT = 100;
+
+/** Items by key, in the order each key first appears. */
+export const groupBy = <T>(items: readonly T[], keyOf: (item: T) => string): Map<string, T[]> =>
+  items.reduce((groups, item) => {
+    // Pushed in place: copying the group for each item is quadratic in a long document's commonest word.
+    const group = groups.get(keyOf(item));
+    if (group === undefined) groups.set(keyOf(item), [item]);
+    else group.push(item);
+    return groups;
+  }, new Map<string, T[]>());
+
+const groupsOf = <T extends KeyedWord>(words: readonly T[]): T[][] => [...groupBy(words, (word) => word.key).values()];
+
+/** Spellings by how often the group writes them, most first; ties keep the order they were first written in. */
+const spellingsByUse = (group: readonly KeyedWord[]): [string, number][] =>
+  [...groupBy(group, (word) => word.spelling).entries()].map(([spelling, uses]): [string, number] => [spelling, uses.length]).toSorted((a, b) => b[1] - a[1]);
+
+/**
+ * The places in one group that do not use its most common spelling. Nothing when the group has one spelling, when two spellings
+ * tie for the most (no side to take), or when the others are more than limitPercent of the group (the document uses both on purpose).
+ */
+const oddIn = <T extends KeyedWord>(group: readonly T[], limitPercent: number): OddSpelling<T>[] => {
+  const [first, second] = spellingsByUse(group);
+  if (first === undefined || second === undefined || first[1] === second[1]) return [];
+  const odd = group.filter((word) => word.spelling !== first[0]);
+  if (odd.length * PERCENT > limitPercent * group.length) return [];
+  return odd.map((word) => ({ word, usual: first[0], count: odd.length, of: group.length }));
+};
+
+/** Every place spelled the way its document uses less, key by key. */
+export const oddSpellings = <T extends KeyedWord>(words: readonly T[], limitPercent: number): OddSpelling<T>[] =>
+  groupsOf(words).flatMap((group) => oddIn(group, limitPercent));

package/src/spelling.ts

@@ -0,0 +1,81 @@
+// 辞書に無い語と、一字違いの言い直し。純粋な関数だけを置く。辞書と、語尾・接頭辞の知識は言語パッケージが渡す。
+
+/** 語尾の置き換え 1 つ。suffix を外して replacement を足すと辞書の語（carried → carry）。 */
+export type Suffix = { readonly suffix: string; readonly replacement: string };
+
+/**
+ * 語を辞書で引くのに要るもの。words は語の集まりの並び（辞書と、語彙表やチームの足した語）で、どれかにあれば載っている語。
+ * 大きい辞書を文書ごとに写さないよう、集まりは足し合わせずに並べる。suffixes は語尾、prefixes は接頭辞。
+ */
+export type Speller = { readonly words: ReadonlyArray<ReadonlySet<string>>; readonly suffixes: readonly Suffix[]; readonly prefixes: readonly string[] };
+
+const inLists = (word: string, speller: Speller): boolean => speller.words.some((list) => list.has(word));
+
+/** 語幹がこれより短くなる外し方はしない。"bed" を "b" + "ed" と読まない。 */
+const MIN_STEM = 2;
+
+/** 語尾を外した語幹。外した後ろに同じ子音が二つ並べば一つ減らした形も（stopped → stopp → stop）。 */
+const stemsOf = (word: string, suffixes: readonly Suffix[]): string[] =>
+  suffixes.flatMap(({ suffix, replacement }) => {
+    if (!word.endsWith(suffix) || word.length - suffix.length < MIN_STEM) return [];
+    const stem = word.slice(0, word.length - suffix.length);
+    const doubled = stem.length > MIN_STEM && stem.at(-1) === stem.at(-2) ? [stem.slice(0, -1)] : [];
+    return [`${stem}${replacement}`, ...doubled];
+  });
+
+const isListed = (word: string, speller: Speller): boolean => inLists(word, speller) || stemsOf(word, speller.suffixes).some((stem) => inLists(stem, speller));
+
+const isPrefixed = (word: string, speller: Speller): boolean =>
+  speller.prefixes.some((prefix) => word.startsWith(prefix) && word.length - prefix.length >= MIN_STEM && isListed(word.slice(prefix.length), speller));
+
+/** 辞書の語か。そのままか、語尾を外すか、接頭辞を一つ外して（語尾も外して）辞書にあれば知っている語。 */
+export const isKnownWord = (word: string, speller: Speller): boolean => isListed(word, speller) || isPrefixed(word, speller);
+
+const LETTERS = [..."abcdefghijklmnopqrstuvwxyz"];
+
+/** 一字違いの形。入れ替え（recieve → receive）、置き換え、抜け、余分の順。同じ形は一度だけ。 */
+export const editsOf = (word: string): string[] => {
+  const at = [...Array(word.length + 1).keys()];
+  const swaps = at.slice(0, -2).map((index) => `${word.slice(0, index)}${word.charAt(index + 1)}${word.charAt(index)}${word.slice(index + 2)}`);
+  const replaced = at.slice(0, -1).flatMap((index) => LETTERS.map((letter) => `${word.slice(0, index)}${letter}${word.slice(index + 1)}`));
+  const dropped = at.slice(0, -1).map((index) => `${word.slice(0, index)}${word.slice(index + 1)}`);
+  const inserted = at.flatMap((index) => LETTERS.map((letter) => `${word.slice(0, index)}${letter}${word.slice(index)}`));
+  return [...new Set([...swaps, ...replaced, ...dropped, ...inserted])].filter((edit) => edit !== word && edit !== "");
+};
+
+/**
+ * 辞書に無い語の、一字違いの語。言い直しに出すのは辞書にそのまま載る語だけ（語尾や接頭辞を足して作った語は、辞書が
+ * 知っていても言い直しとしては確かでない）。文書がほかで使っている語を先に、次に頭の字を変えない語を（打ち間違いは語の頭に少ない。
+ * pipline は hipline でなく pipeline）、その中で編集の順（入れ替えが先）。一字違いの語が無ければ undefined（言い直しを示せない語は、
+ * 誤りか新しい語か決められないので言わない）。
+ */
+export const suggestionFor = (word: string, speller: Speller, used: ReadonlySet<string>): string | undefined => {
+  const rankOf = (edit: string): number => (used.has(edit) ? 0 : 1) * 2 + (edit.charAt(0) === word.charAt(0) ? 0 : 1);
+  const listed = editsOf(word).filter((edit) => inLists(edit, speller) || used.has(edit));
+  return listed.toSorted((left, right) => rankOf(left) - rankOf(right))[0];
+};
+
+/** 本文の中の、確かめる英字の語 1 つ。offset は渡した文字列の中の位置。 */
+export type CheckedWord = { readonly offset: number; readonly word: string };
+
+/**
+ * 確かめる語。小文字だけで書いた英字の語で、ほかの語・番号・記号に続かないもの。大文字を含む語（固有名詞、略語、文頭の語）、
+ * 数字や下線を含む語、アポストロフィとハイフンの前後（don't、co-operate）、文字参照（&mdash;）、ファイル名やドメインの一部（index.html の index）、
+ * URL とメールの中、maxLength より長い字の並び（符号化したデータ）は見ない。
+ */
+const WORD = /(?<![\p{L}\p{N}_./@#&'’\\-])[a-z]+(?![\p{L}\p{N}_/@'’-]|\.[\p{L}\p{N}])/gu;
+/** 空白で区切った一続きの字。URL（://）、www. で始まるもの、メール（@）は丸ごと語として読まない。 */
+const CHUNK = /\S+/gu;
+const isAddress = (chunk: string): boolean => chunk.includes("://") || chunk.includes("@") || chunk.startsWith("www.");
+
+const addressSpans = (text: string): Array<readonly [number, number]> =>
+  [...text.matchAll(CHUNK)].filter((match) => isAddress(match[0])).map((match) => [match.index, match.index + match[0].length]);
+
+export const wordsToCheck = (text: string, minLength: number, maxLength: number): CheckedWord[] => {
+  const addresses = addressSpans(text);
+  return [...text.matchAll(WORD)]
+    .filter(
+      (match) => match[0].length >= minLength && match[0].length <= maxLength && !addresses.some(([start, end]) => match.index >= start && match.index < end),
+    )
+    .map((match) => ({ offset: match.index, word: match[0] }));
+};

package/src/stet-block.ts

@@ -0,0 +1,60 @@
+import { spanOf, type MarkdownNode } from "./markdown-node.ts";
+import type { Span } from "./plugin.ts";
+
+/**
+ * Where a `<!-- stet: … -->` (scope next) stops: the end of the block right after the comment, as the Markdown parser
+ * reads it — a paragraph, a heading, a whole list, a table, a code block, a quote. Inside a list item, the item's next
+ * block. Plain text reads the same way: its paragraphs are the runs between blank lines.
+ */
+
+/** Containers whose children are blocks. Anywhere else, an HTML comment sits inside a paragraph or a heading. */
+const BLOCK_CONTAINERS: ReadonlySet<string> = new Set(["root", "listItem", "blockquote"]);
+
+type Holder = { readonly parent: MarkdownNode; readonly index: number };
+
+/** The child holding `offset`, or -1. Binary search: siblings are in source order, and a document can hold thousands of stets. */
+const childHolding = (children: readonly MarkdownNode[], offset: number): number => {
+  let low = 0;
+  let high = children.length - 1;
+  while (low <= high) {
+    const middle = (low + high) >> 1;
+    const span = spanOf(children[middle] ?? { type: "" });
+    if (span === undefined) return -1;
+    if (offset < span.start) high = middle - 1;
+    else if (offset >= span.end) low = middle + 1;
+    else return middle;
+  }
+  return -1;
+};
+
+/** The deepest node holding `offset`, as its parent and its place among the parent's children. A loop: quotes nest without limit. */
+const holderOf = (root: MarkdownNode, offset: number): Holder | undefined => {
+  let found: Holder | undefined;
+  let parent: MarkdownNode | undefined = root;
+  while (parent !== undefined) {
+    const children: readonly MarkdownNode[] = parent.children ?? [];
+    const index = childHolding(children, offset);
+    if (index !== -1) found = { parent, index };
+    parent = children[index];
+  }
+  return found;
+};
+
+/** The node a stet covers: the next block when the comment stands alone, else the block the comment is part of. */
+const coveredNode = (source: string, holder: Holder, comment: Span): MarkdownNode | undefined => {
+  const siblings = holder.parent.children ?? [];
+  const node = siblings[holder.index];
+  if (node === undefined) return undefined;
+  if (!BLOCK_CONTAINERS.has(holder.parent.type)) return holder.parent;
+  const span = spanOf(node);
+  const alone = span !== undefined && source.slice(span.start, span.end).trim() === source.slice(comment.start, comment.end);
+  return alone ? siblings[holder.index + 1] : node;
+};
+
+/** The last line a `stet` (scope next) covers, for the comment at `comment` in a document parsed as `root`. */
+export const stetBlockEnd = (source: string, root: MarkdownNode, comment: Span, lineAt: (offset: number) => number): number => {
+  const holder = holderOf(root, comment.start);
+  const covered = holder === undefined ? undefined : coveredNode(source, holder, comment);
+  const end = covered === undefined ? undefined : spanOf(covered)?.end;
+  return lineAt(end ?? comment.end);
+};