chaffjs 0.17.0 → 0.18.0

package/src/custom/parse.ts

@@ -0,0 +1,274 @@
+import type { CustomSpec, LevelTable, Localized, RuleDefinition, Severity, TokenCondition } from "../plugin.ts";
+import { regexRefusal, type RegexRefusal } from "./regex-safety.ts";
+import { posTags } from "./token-pattern.ts";
+
+// custom_rules in chaff.yaml: a team's own deterministic rules, written without code. Each becomes a RuleDefinition like the
+// built-in ones, so findings, explain, rules --json, stet, the baseline and SARIF treat it the same. Pure: the YAML comes in parsed.
+
+export const CUSTOM_TYPES = ["words", "pattern", "tokens"] as const;
+
+/** Reserved for a small Node function a team writes. Refused with its own message until it is supported. */
+const RESERVED_TYPES: readonly string[] = ["module"];
+
+export type CustomProblem =
+  | { readonly kind: "not-a-list" }
+  | { readonly kind: "not-a-map"; readonly at: string }
+  | { readonly kind: "bad-id"; readonly at: string }
+  | { readonly kind: "duplicate-id"; readonly at: string }
+  | { readonly kind: "built-in-id"; readonly at: string }
+  | { readonly kind: "unknown-type"; readonly at: string; readonly written: string }
+  | { readonly kind: "not-yet"; readonly at: string; readonly written: string }
+  | { readonly kind: "missing"; readonly at: string; readonly field: string }
+  | { readonly kind: "unpaired-example"; readonly at: string }
+  | { readonly kind: "bad-level"; readonly at: string; readonly written: string }
+  | { readonly kind: "bad-languages"; readonly at: string }
+  | { readonly kind: "no-words"; readonly at: string }
+  | { readonly kind: "bad-pattern"; readonly at: string; readonly refusal: RegexRefusal }
+  | { readonly kind: "no-tokens"; readonly at: string }
+  | { readonly kind: "bad-token"; readonly at: string; readonly index: number }
+  | { readonly kind: "unknown-pos"; readonly at: string; readonly written: string };
+
+export type CustomRules = { readonly rules: readonly RuleDefinition[]; readonly problems: readonly CustomProblem[] };
+
+/** What parsing needs to know: the built-in rule ids (a team's rule may not take one) and the genres a rule applies to. */
+export type CustomContext = { readonly builtIn: ReadonlySet<string>; readonly useFor: readonly string[] };
+
+const RULE_ID = /^[a-z][a-z0-9-]*$/u;
+const SEVERITIES: readonly Severity[] = ["info", "warning", "error"];
+
+const isRecord = (value: unknown): value is Record<string, unknown> => typeof value === "object" && value !== null && !Array.isArray(value);
+
+const nonEmpty = (value: unknown): value is string => typeof value === "string" && value.trim() !== "";
+
+/** Texts by language ({ ja, en } or any language's id). undefined when there is none. */
+const byLanguageOf = (value: unknown): Localized | undefined => {
+  if (!isRecord(value)) return undefined;
+  const entries = Object.entries(value).filter((entry): entry is [string, string] => nonEmpty(entry[1]));
+  return entries.length === 0 ? undefined : Object.fromEntries(entries.map(([language, text]) => [language, text.trim()]));
+};
+
+/** A text by language, or one string for ja and en (and, through the "en" fallback, for any other language). */
+const localizedOf = (value: unknown): Localized | undefined => (nonEmpty(value) ? { ja: value.trim(), en: value.trim() } : byLanguageOf(value));
+
+/** The key an example side written as one string is kept under: it reads the same in every language. */
+const EVERY_LANGUAGE = "*";
+
+const exampleSideOf = (value: unknown): Localized | undefined => (nonEmpty(value) ? { [EVERY_LANGUAGE]: value.trim() } : byLanguageOf(value));
+
+/** One rule's own problems, with the rule's id (or its place in the list) as where. */
+type Checked<T> = { readonly value: T | undefined; readonly problems: readonly CustomProblem[] };
+
+const ok = <T>(value: T): Checked<T> => ({ value, problems: [] });
+const failed = <T>(...problems: CustomProblem[]): Checked<T> => ({ value: undefined, problems });
+
+type Pair = { readonly avoid: string; readonly use: string };
+
+/** words as a map (avoid: use) or a list (words to point at, with nothing to use instead). */
+const pairsOf = (raw: unknown): Pair[] => {
+  if (Array.isArray(raw)) return raw.filter(nonEmpty).map((avoid) => ({ avoid: avoid.trim(), use: "" }));
+  if (!isRecord(raw)) return [];
+  return Object.entries(raw).flatMap(([avoid, use]) => (avoid.trim() === "" ? [] : [{ avoid: avoid.trim(), use: nonEmpty(use) ? use.trim() : "" }]));
+};
+
+const printed = (value: unknown): string => JSON.stringify(value) ?? "undefined";
+
+const wordsOf = (raw: unknown, at: string): Checked<CustomSpec> => {
+  const usable = pairsOf(raw).filter((pair) => pair.avoid !== pair.use);
+  return usable.length === 0 ? failed({ kind: "no-words", at }) : ok({ type: "words", words: usable });
+};
+
+const patternOf = (raw: Record<string, unknown>, at: string): Checked<CustomSpec> => {
+  const pattern = raw["pattern"];
+  if (!nonEmpty(pattern)) return failed({ kind: "missing", at, field: "pattern" });
+  const flags = raw["ignore_case"] === true ? "iu" : "u";
+  const refusal = regexRefusal(pattern, flags);
+  return refusal === undefined ? ok({ type: "pattern", pattern, flags }) : failed({ kind: "bad-pattern", at, refusal });
+};
+
+const posOf = (raw: unknown, at: string): Checked<readonly string[]> => {
+  if (raw === undefined) return ok([]);
+  const written = (Array.isArray(raw) ? raw : [raw]).map((entry: unknown) => (typeof entry === "string" ? entry : printed(entry)));
+  const unknown = written.find((name) => posTags(name) === undefined);
+  if (unknown !== undefined) return failed({ kind: "unknown-pos", at, written: unknown });
+  return ok([...new Set(written.flatMap((name) => posTags(name) ?? []))]);
+};
+
+const conditionOf = (raw: unknown, index: number, at: string): Checked<TokenCondition> => {
+  if (!isRecord(raw)) return failed({ kind: "bad-token", at, index });
+  const pos = posOf(raw["pos"], at);
+  if (pos.value === undefined) return failed(...pos.problems);
+  // lemma is the linguists' word for the base form; either may be written.
+  const written = raw["base"] ?? raw["lemma"];
+  const base = nonEmpty(written) ? written.trim() : undefined;
+  const surface = nonEmpty(raw["surface"]) ? raw["surface"].trim() : undefined;
+  if (pos.value.length === 0 && base === undefined && surface === undefined) return failed({ kind: "bad-token", at, index });
+  return ok({ ...(pos.value.length === 0 ? {} : { pos: pos.value }), ...(base === undefined ? {} : { base }), ...(surface === undefined ? {} : { surface }) });
+};
+
+const tokensOf = (raw: unknown, at: string): Checked<CustomSpec> => {
+  if (!Array.isArray(raw) || raw.length === 0) return failed({ kind: "no-tokens", at });
+  const checked = raw.map((entry: unknown, index) => conditionOf(entry, index + 1, at));
+  const problems = checked.flatMap((entry) => entry.problems);
+  const tokens = checked.flatMap((entry) => (entry.value === undefined ? [] : [entry.value]));
+  return problems.length > 0 ? failed(...problems) : ok({ type: "tokens", tokens });
+};
+
+const specOf = (raw: Record<string, unknown>, type: string, at: string): Checked<CustomSpec> => {
+  if (type === "words") return wordsOf(raw["words"], at);
+  if (type === "pattern") return patternOf(raw, at);
+  return tokensOf(raw["tokens"], at);
+};
+
+const typeOf = (raw: Record<string, unknown>, at: string): Checked<string> => {
+  const type = raw["type"];
+  if (!nonEmpty(type)) return failed({ kind: "missing", at, field: "type" });
+  if (RESERVED_TYPES.includes(type)) return failed({ kind: "not-yet", at, written: type });
+  return CUSTOM_TYPES.some((known) => known === type) ? ok(type) : failed({ kind: "unknown-type", at, written: type });
+};
+
+const severityOf = (raw: unknown, at: string): Checked<Severity> => {
+  if (raw === undefined) return ok("warning");
+  const severity = SEVERITIES.find((entry) => entry === raw);
+  return severity === undefined ? failed({ kind: "bad-level", at, written: typeof raw === "string" ? raw : printed(raw) }) : ok(severity);
+};
+
+/** The four words as severities: normal is the rule's own level, relaxed one lower, strict one higher, where there is one. */
+const levelsFor = (severity: Severity): LevelTable => {
+  const rank = SEVERITIES.indexOf(severity) + 1;
+  return { ...(rank < SEVERITIES.length ? { strict: rank + 1 } : {}), normal: rank, ...(rank > 1 ? { relaxed: rank - 1 } : {}) };
+};
+
+type Texts = { readonly name: Localized; readonly why: Localized; readonly how_to_fix: Localized; readonly before: Localized; readonly after: Localized };
+
+const textsOf = (raw: Record<string, unknown>, at: string): Checked<Texts> => {
+  const example = isRecord(raw["example"]) ? raw["example"] : {};
+  const found = {
+    name: localizedOf(raw["name"]),
+    why: localizedOf(raw["why"]),
+    how_to_fix: localizedOf(raw["how_to_fix"]),
+    before: exampleSideOf(example["before"]),
+    after: exampleSideOf(example["after"]),
+  };
+  const missing = Object.entries(found).flatMap(([field, text]) =>
+    text === undefined ? [{ kind: "missing" as const, at, field: field === "before" || field === "after" ? `example.${field}` : field }] : [],
+  );
+  const { name, why, how_to_fix: howToFix, before, after } = found;
+  if (name === undefined || why === undefined || howToFix === undefined || before === undefined || after === undefined) return failed(...missing);
+  return ok({ name, why, how_to_fix: howToFix, before, after });
+};
+
+const languagesOf = (raw: unknown, at: string): Checked<readonly string[] | undefined> => {
+  if (raw === undefined) return ok(undefined);
+  const list = Array.isArray(raw) ? raw : [raw];
+  return list.length > 0 && list.every(nonEmpty) ? ok(list.map((entry: string) => entry.trim())) : failed({ kind: "bad-languages", at });
+};
+
+/** The message when a rule does not write one: the replacement for a word list, else the rule's name and what was found. */
+const defaultMessage = (spec: CustomSpec, name: Localized): Localized => {
+  if (spec.type === "words" && spec.words.every((pair) => pair.use !== ""))
+    return { ja: "「{matched}」は「{preferred}」と書きます", en: 'Write "{preferred}", not "{matched}"' };
+  return Object.fromEntries(Object.entries(name).map(([language, text]) => [language, language === "ja" ? `${text}:「{matched}」` : `${text}: "{matched}"`]));
+};
+
+const PLACEHOLDERS: Readonly<Record<string, Localized>> = {
+  matched: { ja: "見つけた語句", en: "the text found" },
+  preferred: { ja: "使う書き方", en: "the spelling to use" },
+};
+
+const HOW_TO_FIND: Readonly<Record<CustomSpec["type"], string>> = { words: "custom-words", pattern: "custom-pattern", tokens: "custom-tokens" };
+
+type Example = { readonly before: string; readonly after: string };
+
+type Parts = {
+  readonly spec: CustomSpec;
+  readonly severity: Severity;
+  readonly texts: Texts;
+  readonly languages: readonly string[] | undefined;
+  readonly examples: Readonly<Record<string, Example>>;
+};
+
+const READER_LANGUAGES: readonly string[] = ["ja", "en"];
+
+/** The example in each language the rule checks, where both before and after are written in it. A string counts for both. */
+const examplesOf = (texts: Texts, languages: readonly string[] | undefined): Record<string, Example> =>
+  Object.fromEntries(
+    (languages ?? READER_LANGUAGES).flatMap((language) => {
+      const [before, after] = [texts.before[language] ?? texts.before[EVERY_LANGUAGE], texts.after[language] ?? texts.after[EVERY_LANGUAGE]];
+      return before === undefined || after === undefined ? [] : [[language, { before, after }]];
+    }),
+  );
+
+const definitionOf = (id: string, raw: Record<string, unknown>, parts: Parts, useFor: readonly string[]): RuleDefinition => ({
+  id,
+  layer: parts.spec.type === "tokens" ? "L3" : "L2",
+  status: "stable",
+  name: parts.texts.name,
+  why: parts.texts.why,
+  how_to_fix: parts.texts.how_to_fix,
+  message: localizedOf(raw["message"]) ?? defaultMessage(parts.spec, parts.texts.name),
+  messages: {},
+  placeholders: PLACEHOLDERS,
+  levels: levelsFor(parts.severity),
+  level_sets: "severity",
+  by_genre: {},
+  how_to_find: HOW_TO_FIND[parts.spec.type],
+  word_list: undefined,
+  extra_word_lists: [],
+  what_to_check: undefined,
+  where: undefined,
+  full_sentence: undefined,
+  requires: parts.spec.type === "tokens" ? ["pos"] : [],
+  uses: [],
+  from: [],
+  languages: parts.languages,
+  use_for: useFor,
+  severity: parts.severity,
+  custom: parts.spec,
+  // The reference and explain read a rule's plain-language fields from guide; a team's rule is listed with the team's words.
+  guide: {
+    group: "team",
+    summary: parts.texts.name,
+    examples: parts.examples,
+    notFlagged: {},
+    levelMeaning: {},
+  },
+});
+
+const idOf = (raw: Record<string, unknown>, index: number, seen: ReadonlySet<string>, builtIn: ReadonlySet<string>): Checked<string> => {
+  const id = raw["id"];
+  const at = typeof id === "string" && id !== "" ? id : `#${String(index + 1)}`;
+  if (typeof id !== "string" || !RULE_ID.test(id)) return failed({ kind: "bad-id", at });
+  if (builtIn.has(id)) return failed({ kind: "built-in-id", at });
+  return seen.has(id) ? failed({ kind: "duplicate-id", at }) : ok(id);
+};
+
+const ruleOf = (raw: unknown, index: number, seen: ReadonlySet<string>, context: CustomContext): Checked<RuleDefinition> => {
+  if (!isRecord(raw)) return failed({ kind: "not-a-map", at: `#${String(index + 1)}` });
+  const id = idOf(raw, index, seen, context.builtIn);
+  const at = id.value ?? (nonEmpty(raw["id"]) ? raw["id"] : `#${String(index + 1)}`);
+  const type = typeOf(raw, at);
+  const spec = type.value === undefined ? failed<CustomSpec>() : specOf(raw, type.value, at);
+  const severity = severityOf(raw["level"], at);
+  const texts = textsOf(raw, at);
+  const languages = languagesOf(raw["languages"], at);
+  const examples = texts.value === undefined ? {} : examplesOf(texts.value, languages.value);
+  const unpaired: CustomProblem[] = texts.value !== undefined && Object.keys(examples).length === 0 ? [{ kind: "unpaired-example", at }] : [];
+  const problems = [...[id, type, spec, severity, texts, languages].flatMap((checked) => checked.problems), ...unpaired];
+  if (id.value === undefined || spec.value === undefined || severity.value === undefined || texts.value === undefined || problems.length > 0)
+    return failed(...problems);
+  const parts = { spec: spec.value, severity: severity.value, texts: texts.value, languages: languages.value, examples };
+  return ok(definitionOf(id.value, raw, parts, context.useFor));
+};
+
+/** custom_rules as written. Every problem is reported; a rule with a problem is left out, and the run is stopped by the caller. */
+export const parseCustomRules = (raw: unknown, context: CustomContext): CustomRules => {
+  if (raw === undefined || raw === null) return { rules: [], problems: [] };
+  if (!Array.isArray(raw)) return { rules: [], problems: [{ kind: "not-a-list" }] };
+  return raw.reduce<{ rules: RuleDefinition[]; problems: CustomProblem[] }>(
+    (acc, entry: unknown, index) => {
+      const checked = ruleOf(entry, index, new Set(acc.rules.map((rule) => rule.id)), context);
+      return { rules: checked.value === undefined ? acc.rules : [...acc.rules, checked.value], problems: [...acc.problems, ...checked.problems] };
+    },
+    { rules: [], problems: [] },
+  );
+};

package/src/custom/problems.ts

@@ -0,0 +1,89 @@
+import type { Config } from "../config/load.ts";
+import type { Texts, UiLanguage } from "../ui.ts";
+import type { CustomProblem } from "./parse.ts";
+import { customRulesOf } from "./load.ts";
+import { MAX_PATTERN_LENGTH, MAX_REPEATS, type RegexRefusal } from "./regex-safety.ts";
+import { POS_WRITTEN_NAMES } from "./token-pattern.ts";
+
+/** Each problem's sentence, with {at}, {written}, {field}, {index}, {refusal} and {names} filled in from the problem. */
+type Text = { readonly problems: Readonly<Record<CustomProblem["kind"], string>>; readonly refusal: Readonly<Record<RegexRefusal, string>> };
+
+const TEXT: Texts<Text> = {
+  ja: {
+    refusal: {
+      "too-long": `${String(MAX_PATTERN_LENGTH)} 字を超えています。いくつかのルールに分けてください`,
+      "nested-quantifier": "繰り返しの中に繰り返しがあります（(a+)+ のような形）。長い行で止まらなくなるので使えません",
+      "too-many-repeats": `回数の決まらない繰り返し（* や + や {1,9}）が ${String(MAX_REPEATS)} つを超えています。長い行で止まらなくなることがあります`,
+      backreference: "後方参照（\\1）は使えません。長い行で止まらなくなることがあるためです",
+      "empty-match": "何も無い所にも当たります。すべての文で指摘が出てしまいます",
+      invalid: "正規表現として読めません",
+    },
+    problems: {
+      "not-a-list": "custom_rules はルールの並び（- id: …）で書いてください",
+      "not-a-map": "custom_rules の {at}: id や type を持つ項目として書いてください",
+      "bad-id": "custom_rules の {at}: id は英小文字で始まり、英小文字・数字・ハイフンだけで書きます（例: team-no-tbd）",
+      "duplicate-id": "custom_rules の {at}: 同じ id のルールがもう一つあります",
+      "built-in-id": "custom_rules の {at}: chaff のルールと同じ id です。別の id にしてください",
+      "unknown-type": "custom_rules の {at}: type: {written} は使えません（words / pattern / tokens）",
+      "not-yet": "custom_rules の {at}: type: {written} はまだ使えません（words / pattern / tokens）",
+      missing: "custom_rules の {at}: {field} がありません",
+      "unpaired-example": "custom_rules の {at}: example の before と after を、ルールが見る言語の同じ言語で書いてください",
+      "bad-level": "custom_rules の {at}: level: {written} は読めません（error / warning / info）",
+      "bad-languages": "custom_rules の {at}: languages は言語の並びで書いてください（[ja] や [ja, en]）",
+      "no-words": "custom_rules の {at}: words に語がありません（「使わない書き方: 使う書き方」か語の並び）",
+      "no-tokens": "custom_rules の {at}: tokens に語の条件がありません（- { pos: 名詞 } のように並べます）",
+      "bad-token": "custom_rules の {at}: tokens の {index} 番目に pos・base・surface のどれもありません",
+      "unknown-pos": "custom_rules の {at}: 品詞 {written} は知りません（{names}）",
+      "bad-pattern": "custom_rules の {at}: pattern を使えません。{refusal}",
+    },
+  },
+  en: {
+    refusal: {
+      "too-long": `it is longer than ${String(MAX_PATTERN_LENGTH)} characters; split it into several rules`,
+      "nested-quantifier": "it repeats something that repeats (a shape like (a+)+), which can run for minutes on a long line",
+      "too-many-repeats": `it has more than ${String(MAX_REPEATS)} repeats of varying count (*, + or {1,9}), which can run for minutes on a long line`,
+      backreference: "backreferences (\\1) are not allowed, since they can run for minutes on a long line",
+      "empty-match": "it matches an empty string, so it would report every sentence",
+      invalid: "it is not a regular expression",
+    },
+    problems: {
+      "not-a-list": "write custom_rules as a list of rules (- id: …)",
+      "not-a-map": "custom_rules {at}: write each rule as an entry with id and type",
+      "bad-id": "custom_rules {at}: an id starts with a lowercase letter and has only lowercase letters, digits and hyphens (team-no-tbd)",
+      "duplicate-id": "custom_rules {at}: another rule has the same id",
+      "built-in-id": "custom_rules {at}: chaff has a rule with this id; choose another",
+      "unknown-type": "custom_rules {at}: type: {written} is not a type (words / pattern / tokens)",
+      "not-yet": "custom_rules {at}: type: {written} is not supported yet (words / pattern / tokens)",
+      missing: "custom_rules {at}: {field} is missing",
+      "unpaired-example": "custom_rules {at}: write example's before and after in the same language, one the rule checks",
+      "bad-level": "custom_rules {at}: cannot read level: {written} (error / warning / info)",
+      "bad-languages": "custom_rules {at}: write languages as a list ([en] or [ja, en])",
+      "no-words": "custom_rules {at}: words has no words (avoid: use pairs, or a list)",
+      "no-tokens": "custom_rules {at}: tokens has no conditions (- { pos: noun } and so on)",
+      "bad-token": "custom_rules {at}: token {index} has none of pos, base and surface",
+      "unknown-pos": "custom_rules {at}: unknown part of speech {written} ({names})",
+      "bad-pattern": "custom_rules {at}: the pattern cannot be used: {refusal}",
+    },
+  },
+};
+
+const PLACEHOLDER = /\{(at|written|field|index|refusal|names)\}/gu;
+
+/** The values a problem's sentence names. Each kind carries only some of them. */
+const valuesOf = (problem: CustomProblem, text: Text): Readonly<Record<string, string>> => ({
+  at: "at" in problem ? problem.at : "",
+  written: "written" in problem ? problem.written : "",
+  field: "field" in problem ? problem.field : "",
+  index: "index" in problem ? String(problem.index) : "",
+  refusal: "refusal" in problem ? text.refusal[problem.refusal] : "",
+  names: POS_WRITTEN_NAMES.join(" / "),
+});
+
+const sentenceOf = (problem: CustomProblem, text: Text): string => {
+  const values = valuesOf(problem, text);
+  return text.problems[problem.kind].replace(PLACEHOLDER, (_whole, key: string) => values[key] ?? "");
+};
+
+/** What in custom_rules cannot run. Each stops the run: a team's rule that silently does not run looks like a clean document. */
+export const customRuleProblems = (config: Pick<Config, "customRules" | "path">, ui: UiLanguage): string[] =>
+  customRulesOf(config).problems.map((problem) => `chaff: ${config.path ?? "chaff.yaml"}: ${sentenceOf(problem, TEXT[ui])}`);

package/src/custom/regex-safety.ts

@@ -0,0 +1,118 @@
+// Whether a team's regular expression can run away. JavaScript's engine backtracks, so a pattern such as (a+)+ takes time
+// exponential in the length of a line that almost matches. Pure: the pattern is read as text, never run on a document.
+
+/** Longer patterns are refused: a rule that needs one is better written as several. */
+export const MAX_PATTERN_LENGTH = 500;
+
+/**
+ * More repeats of varying count (*, +, {1,9}) than this are refused. Repeats side by side (a*a*a*b, a{0,99}a{0,99}…) do
+ * not nest, but their work multiplies: a line that almost matches costs its length to the power of their number.
+ */
+export const MAX_REPEATS = 3;
+
+export type RegexRefusal = "too-long" | "nested-quantifier" | "too-many-repeats" | "backreference" | "empty-match" | "invalid";
+
+/** An open group: where its body starts, and whether it holds a repeat of varying count (a+, a?, a{1,3}) or an alternation. */
+type Group = { readonly start: number; varies: boolean; alternation: boolean };
+
+/** A quantifier: its length, whether it has no upper bound, whether its count varies, and whether it can repeat more than once. */
+type Quantifier = { readonly length: number; readonly unbounded: boolean; readonly varies: boolean; readonly many: boolean };
+
+type Scan = { at: number; repeats: number; readonly stack: Group[] };
+
+const QUANTIFIER_START: ReadonlySet<string> = new Set(["*", "+", "?", "{"]);
+const BRACE = /^\{(\d+)(,(\d*))?\}\??/u;
+const BACKREFERENCE = /^\\(?:[1-9]|k<)/u;
+const CLASS_BODY = /^\^?\]?(?:\\.|[^\]\\])*\]/u;
+
+/** The quantifier at pattern[at], or undefined when none is there. */
+const quantifierAt = (pattern: string, at: number): Quantifier | undefined => {
+  const char = pattern[at];
+  if (char === undefined || !QUANTIFIER_START.has(char)) return undefined;
+  if (char !== "{") return { length: pattern[at + 1] === "?" ? 2 : 1, unbounded: char !== "?", varies: true, many: char !== "?" };
+  const brace = BRACE.exec(pattern.slice(at));
+  if (brace === null) return undefined;
+  const [least, most] = [Number(brace[1]), brace[2] === undefined ? Number(brace[1]) : Number(brace[3] || Infinity)];
+  return { length: brace[0].length, unbounded: most === Infinity, varies: most !== least, many: most > 1 };
+};
+
+/** Just past the "]" that closes the character class opened at pattern[at]. */
+const classEnd = (pattern: string, at: number): number => {
+  const close = CLASS_BODY.exec(pattern.slice(at + 1));
+  return close === null ? pattern.length : at + 1 + close[0].length;
+};
+
+const innermost = (scan: Scan): Group => scan.stack[scan.stack.length - 1] ?? { start: 0, varies: false, alternation: false };
+
+/** A fixed string: plain characters, and punctuation escaped (Mr\.). \d, \w and the like are classes, not literals. */
+const LITERAL = /^(?:[^\\[\](){}*+?.|^$]|\\[^A-Za-z0-9])+$/u;
+
+/**
+ * Alternatives that are plain words starting with different characters ((cat|dog)+) never compete for the same text, so
+ * repeating them is linear. Any other alternation under a repeat ((a|aa)*) may try every split of the line.
+ */
+const distinctWords = (body: string): boolean => {
+  const alternatives = body.replace(/^\?:/u, "").split("|");
+  const firsts = alternatives.map((alternative) => (alternative.startsWith("\\") ? alternative.charAt(1) : alternative.charAt(0)));
+  return alternatives.every((alternative) => LITERAL.test(alternative)) && new Set(firsts).size === firsts.length;
+};
+
+/** Whether the group's text can be split between repeats in more than one way: it varies in length, or its alternatives compete. */
+const competes = (pattern: string, closed: Group, end: number): boolean =>
+  closed.varies || (closed.alternation && !distinctWords(pattern.slice(closed.start, end - 1)));
+
+/**
+ * After an atom that ends at end (closed is the group it closes, if any). A group repeated a varying number of times, more than
+ * once ((…)+, (…)*, (…){1,10}), that can itself match in more than one way ((a+), (a{1,10}), (a|aa)) is refused.
+ */
+const afterAtom = (pattern: string, scan: Scan, end: number, closed: Group | undefined): RegexRefusal | undefined => {
+  const quantifier = quantifierAt(pattern, end);
+  scan.at = end + (quantifier?.length ?? 0);
+  const repeatsVariably = quantifier !== undefined && quantifier.varies && quantifier.many;
+  if (repeatsVariably) scan.repeats += 1;
+  if (scan.repeats > MAX_REPEATS) return "too-many-repeats";
+  // A group that can match in more than one way makes the group around it so too, quantified or not: ((a|aa))+ is (a|aa)+.
+  const closedCompetes = closed !== undefined && competes(pattern, closed, end);
+  if (closedCompetes && repeatsVariably) return "nested-quantifier";
+  if (quantifier?.varies === true || closedCompetes) innermost(scan).varies = true;
+  return undefined;
+};
+
+/** Reads one escape, class, bracket or character at scan.at. */
+const step = (pattern: string, scan: Scan): RegexRefusal | undefined => {
+  const char = pattern[scan.at];
+  if (char === "\\") return BACKREFERENCE.test(pattern.slice(scan.at)) ? "backreference" : afterAtom(pattern, scan, scan.at + 2, undefined);
+  if (char === "[") return afterAtom(pattern, scan, classEnd(pattern, scan.at), undefined);
+  if (char === "(") {
+    scan.stack.push({ start: scan.at + 1, varies: false, alternation: false });
+    scan.at += 1;
+    return undefined;
+  }
+  if (char === ")") return afterAtom(pattern, scan, scan.at + 1, scan.stack.length > 1 ? scan.stack.pop() : undefined);
+  if (char === "|") innermost(scan).alternation = true;
+  return afterAtom(pattern, scan, scan.at + 1, undefined);
+};
+
+/**
+ * A group repeated without bound that holds a repeat without bound or an alternation ((a+)+, (a|aa)*, (\w+\s?)*) is the shape
+ * that backtracks exponentially. The pattern is read once, left to right, with a stack of open groups.
+ */
+const nestedRepeat = (pattern: string): RegexRefusal | undefined => {
+  const scan: Scan = { at: 0, repeats: 0, stack: [{ start: 0, varies: false, alternation: false }] };
+  while (scan.at < pattern.length) {
+    const refusal = step(pattern, scan);
+    if (refusal !== undefined) return refusal;
+  }
+  return undefined;
+};
+
+/** Why the pattern is refused, or undefined when it may run. flags are the ones the rule runs with ("u", "iu"). */
+export const regexRefusal = (pattern: string, flags: string): RegexRefusal | undefined => {
+  if (pattern.length > MAX_PATTERN_LENGTH) return "too-long";
+  try {
+    if (new RegExp(pattern, flags).test("")) return "empty-match";
+  } catch {
+    return "invalid";
+  }
+  return nestedRepeat(pattern);
+};

package/src/custom/token-pattern.ts

@@ -0,0 +1,84 @@
+import type { Token, TokenCondition } from "../plugin.ts";
+
+// A team's morphology pattern: a run of tokens, each matching a condition on its part of speech, base form or surface.
+// 「〜を行う」 is [{ pos: 名詞 }, { surface: を }, { base: 行う }]. Pure: tokens come from the language adapter.
+
+/**
+ * Parts of speech are Universal Dependencies tags (NOUN, VERB, ADP), the same for every language. A team may also write the
+ * everyday name in Japanese or English; each maps to the tags the adapters give for it.
+ */
+const POS_NAMES: Readonly<Record<string, readonly string[]>> = {
+  名詞: ["NOUN", "PROPN", "PRON", "NUM"],
+  固有名詞: ["PROPN"],
+  代名詞: ["PRON"],
+  動詞: ["VERB"],
+  形容詞: ["ADJ"],
+  副詞: ["ADV"],
+  助詞: ["ADP", "PART", "SCONJ", "CCONJ"],
+  助動詞: ["AUX"],
+  接続詞: ["CCONJ", "SCONJ"],
+  連体詞: ["DET"],
+  記号: ["PUNCT", "SYM"],
+  noun: ["NOUN", "PROPN", "PRON"],
+  verb: ["VERB"],
+  adjective: ["ADJ"],
+  adverb: ["ADV"],
+  preposition: ["ADP"],
+  determiner: ["DET"],
+  pronoun: ["PRON"],
+  conjunction: ["CCONJ", "SCONJ"],
+  auxiliary: ["AUX"],
+};
+
+const UPOS: ReadonlySet<string> = new Set([
+  "ADJ",
+  "ADP",
+  "ADV",
+  "AUX",
+  "CCONJ",
+  "DET",
+  "INTJ",
+  "NOUN",
+  "NUM",
+  "PART",
+  "PRON",
+  "PROPN",
+  "PUNCT",
+  "SCONJ",
+  "SYM",
+  "VERB",
+  "X",
+]);
+
+/** The tags a written part of speech stands for, or undefined when chaff does not know the name. */
+export const posTags = (written: string): readonly string[] | undefined => {
+  const upper = written.toUpperCase();
+  if (UPOS.has(upper)) return [upper];
+  return POS_NAMES[written] ?? POS_NAMES[written.toLowerCase()];
+};
+
+/** The names a team may write, for the message about an unknown one. */
+export const POS_WRITTEN_NAMES: readonly string[] = [...UPOS, ...Object.keys(POS_NAMES)];
+
+const sameWord = (left: string, right: string): boolean => left.toLowerCase() === right.toLowerCase();
+
+const holds = (token: Token, condition: TokenCondition): boolean =>
+  (condition.pos === undefined || condition.pos.includes(token.pos)) &&
+  (condition.surface === undefined || sameWord(token.surface, condition.surface)) &&
+  (condition.base === undefined || sameWord(token.lemma ?? token.surface, condition.base));
+
+/** Every run of consecutive tokens where each holds its condition, as [first, last] token indexes. Runs do not overlap. */
+export const tokenRuns = (tokens: readonly Token[], conditions: readonly TokenCondition[]): (readonly [number, number])[] => {
+  if (conditions.length === 0) return [];
+  return tokens.reduce<{ runs: (readonly [number, number])[]; next: number }>(
+    (acc, _token, start) => {
+      if (start < acc.next || start + conditions.length > tokens.length) return acc;
+      const matches = conditions.every((condition, offset) => {
+        const token = tokens[start + offset];
+        return token !== undefined && holds(token, condition);
+      });
+      return matches ? { runs: [...acc.runs, [start, start + conditions.length - 1]], next: start + conditions.length } : acc;
+    },
+    { runs: [], next: 0 },
+  ).runs;
+};

package/src/detect.ts

@@ -1,4 +1,10 @@
+import { readFileSync } from "node:fs";
+import { dirname, join } from "node:path";
+import { fileURLToPath } from "node:url";
+import { parse } from "yaml";
 import { japaneseRatio, latinRatio } from "./detect-language.ts";
+import { referenceListSpans } from "./reference-lists.ts";
+import { withoutSpans } from "./soft-break.ts";
 
 export type LanguageGuess = { readonly language: string; readonly confidence: number; readonly from: string };
 
@@ -8,8 +14,28 @@ export type LanguageGuess = { readonly language: string; readonly confidence: nu
  */
 const JAPANESE_FLOOR = 0.15;
 
+const HEADINGS_FILE = join(dirname(fileURLToPath(import.meta.url)), "..", "reference-headings.yaml");
+
+const isStringList = (value: unknown): value is string[] => Array.isArray(value) && value.every((entry) => typeof entry === "string");
+
+const readHeadings = (): string[] => {
+  const raw: unknown = parse(readFileSync(HEADINGS_FILE, "utf8"));
+  const headings: unknown = typeof raw === "object" && raw !== null && "headings" in raw ? raw.headings : undefined;
+  if (!isStringList(headings)) throw new Error(`${HEADINGS_FILE}: headings は文字列の並びであること`);
+  return headings;
+};
+
+const REFERENCE_HEADINGS = readHeadings();
+
+/** 本文だけ。文献一覧は引いた文献の言語で書かれるので数えない。本文に字が無ければ（文献一覧だけの文書）全体。 */
+const bodyOf = (source: string): string => {
+  const body = withoutSpans(source, referenceListSpans(source, REFERENCE_HEADINGS));
+  return body.trim() === "" ? source : body;
+};
+
 export const guessLanguage = (source: string): LanguageGuess => {
-  const japanese = japaneseRatio(source);
+  const body = bodyOf(source);
+  const japanese = japaneseRatio(body);
   if (japanese >= JAPANESE_FLOOR) return { language: "ja", confidence: japanese, from: "content" };
-  return { language: "en", confidence: latinRatio(source), from: "content" };
+  return { language: "en", confidence: latinRatio(body), from: "content" };
 };