chaffjs 0.18.0 → 0.20.0

package/src/detectors/fullwidth-alnum.ts

@@ -0,0 +1,118 @@
+import type { Detector, Finding, Sentence, Span } from "../plugin.ts";
+import { minorityWithin } from "../orthography.ts";
+import { QUOTATION_MARKS, isWithinAny, quotedSpans } from "../quoted-span.ts";
+import { isCitation } from "./kutoten-consistency.ts";
+
+/** 英字と数字の並び。半角（ABC123）と全角（ＡＢＣ１２３）。 */
+const ALNUM_RUN = /[A-Za-z0-9Ａ-Ｚａ-ｚ０-９]+/gu;
+const FULLWIDTH = /^[Ａ-Ｚａ-ｚ０-９]+$/u;
+const HALFWIDTH = /^[A-Za-z0-9]+$/u;
+const LETTER = /[A-Za-zＡ-Ｚａ-ｚ]/u;
+
+/**
+ * 比べる組。英字と数字を分け、一字だけの並びと二字以上の並びも分ける。「１桁は全角、２桁以上は半角」「一字の英字（Ａ案）は全角」と
+ * 決めている文書は、組の中ではそろっている。
+ */
+export type AlnumKind = "letter" | "letters" | "digit" | "digits";
+
+const kindOf = (run: string): AlnumKind => {
+  const single = [...run].length === 1;
+  if (LETTER.test(run)) return single ? "letter" : "letters";
+  return single ? "digit" : "digits";
+};
+
+/** 項目の番号の並び（行の頭や括弧の後ろの「１．」「（１）」「1)」）。項目の印で、本文の数ではない。 */
+const LABEL_BEFORE = /(?:^|[\s、，,（(［[])$/u;
+const LABEL_AFTER = /^[．.）)］\]]/u;
+const LABEL_REACH = 1;
+
+/** 注の印（「※１」「（※３）」）。注どうしで決まった書き方をする。 */
+const NOTE_MARK = /[※＊*]$/u;
+
+/** 箇条書きの項目の頭（「- ４「勧告」…」）。目次の番号で、本文の数ではない。 */
+const LIST_ITEM_HEAD = /^[ \t]*(?:[-*+]|\d{1,9}[.)])[ \t]+$/u;
+
+const isLabel = (text: string, start: number, end: number): boolean =>
+  (LABEL_BEFORE.test(text.slice(Math.max(0, start - LABEL_REACH), start)) && LABEL_AFTER.test(text.slice(end, end + LABEL_REACH))) ||
+  NOTE_MARK.test(text.slice(0, start));
+
+/**
+ * メールアドレスやドメイン名の一部（support@example.com の support・example・com）。字の幅を変えると宛先が壊れる。
+ * 点の向こうが英字のときだけ名前の区切りと見る。数の小数点（39.4）は数として数える。
+ */
+const LETTER_BESIDE = /[A-Za-zＡ-Ｚａ-ｚ]/u;
+
+const isInAddress = (text: string, start: number, end: number): boolean =>
+  text.charAt(start - 1) === "@" ||
+  text.charAt(end) === "@" ||
+  (text.charAt(start - 1) === "." && LETTER_BESIDE.test(text.charAt(start - 2))) ||
+  (text.charAt(end) === "." && LETTER_BESIDE.test(text.charAt(end + 1)));
+
+/** 英字の後ろに数字を続けた名前（TLS1、Python3）と、その後ろの点でつないだ番号（Python3.12）。版の番号の頭になる。 */
+const VERSION_SO_FAR = /[A-Za-zＡ-Ｚａ-ｚ][0-9０-９]+(?:\.[0-9０-９]+)*$/u;
+
+/** 版の番号を後ろ向きに読む長さの上限。長い行で点ごとに行の頭から読み直さない。 */
+const VERSION_REACH = 32;
+
+/** 版の番号の続き（TLS1.2 の 2、Python3.12.4 の 12 と 4）。名前の数字と同じ幅なら、幅は名前に従い、文書の数の書き方の票にはしない（TLS1.２ の ２ は数える）。 */
+const continuesVersion = (text: string, start: number, run: string): boolean =>
+  text.charAt(start - 1) === "." &&
+  FULLWIDTH.test(run) === FULLWIDTH.test(text.charAt(start - 2)) &&
+  VERSION_SO_FAR.test(text.slice(Math.max(0, start - 1 - VERSION_REACH), start - 1));
+
+const isListItemNumber = (source: string, offset: number): boolean => LIST_ITEM_HEAD.test(source.slice(source.lastIndexOf("\n", offset - 1) + 1, offset));
+
+export type AlnumRun = { readonly kind: AlnumKind; readonly fullwidth: boolean; readonly written: string; readonly offset: number };
+
+/**
+ * 文の中の英字・数字の並び。全角と半角が混ざった並び、鉤括弧で引いたものの中、項目と注の番号、箇条書きの頭の番号、版の番号の続き（TLS1.2 の 2）は数えない。
+ * offset は文書の中の位置。
+ */
+export const alnumRunsIn = (sentence: Sentence, source: string): AlnumRun[] => {
+  const quoted: readonly Span[] = quotedSpans(sentence.text, QUOTATION_MARKS);
+  return [...sentence.text.matchAll(ALNUM_RUN)]
+    .filter((match) => FULLWIDTH.test(match[0]) || HALFWIDTH.test(match[0]))
+    .filter((match) => !isWithinAny(quoted, { start: match.index, end: match.index + match[0].length }))
+    .filter((match) => !isLabel(sentence.text, match.index, match.index + match[0].length))
+    .filter((match) => !isListItemNumber(source, sentence.span.start + match.index))
+    .filter((match) => !isInAddress(sentence.text, match.index, match.index + match[0].length))
+    .filter((match) => !continuesVersion(sentence.text, match.index, match[0]))
+    .map((match) => ({ kind: kindOf(match[0]), fullwidth: FULLWIDTH.test(match[0]), written: match[0], offset: sentence.span.start + match.index }));
+};
+
+const KINDS: readonly AlnumKind[] = ["letter", "letters", "digit", "digits"];
+
+/** 半角の英数字を全角に（NFKC の逆）。 */
+const FULLWIDTH_OFFSET = 0xfee0;
+const toFullwidth = (text: string): string => text.replace(/[A-Za-z0-9]/gu, (char) => String.fromCodePoint((char.codePointAt(0) ?? 0) + FULLWIDTH_OFFSET));
+
+/** 同じ並びの、もう一方の幅の書き方。 */
+const otherWidth = (run: AlnumRun): string => (run.fullwidth ? run.written.normalize("NFKC") : toFullwidth(run.written));
+
+type Located = { readonly sentence: Sentence; readonly run: AlnumRun };
+
+const findingOf = ({ sentence, run }: Located, count: number, of: number, limit: number): Finding => ({
+  rule: "",
+  severity: "warning",
+  line: 0,
+  column: 0,
+  quote: sentence.text.trim(),
+  values: { written: run.written, other: otherWidth(run), count, of, limit, offset: run.offset },
+  variant: run.fullwidth ? "fullwidth" : "halfwidth",
+});
+
+/**
+ * 英字・数字を全角で書くか半角で書くか。組（英字一字・英字の語・数字一字・数字の並び）ごとに、文書の中で少ないほうを指摘する。
+ * 少ないほうが組の limit パーセントを超えるほど多いなら、文書が使い分けている（数え方の語や見出しの書き方）と見て言わない。
+ * どちらが正しいかは決めない。
+ */
+export const fullwidthAlnum: Detector = (doc, options): Finding[] => {
+  const located: Located[] = doc.sentences
+    .filter((sentence) => sentence.embeddedLanguage === undefined && !isCitation(sentence, doc.source))
+    .flatMap((sentence) => alnumRunsIn(sentence, doc.source).map((run) => ({ sentence, run })));
+  return KINDS.flatMap((kind) => {
+    const ofKind = located.filter(({ run }) => run.kind === kind);
+    const odd = minorityWithin(ofKind, ({ run }) => run.fullwidth, options.limit);
+    return odd.map((entry) => findingOf(entry, odd.length, ofKind.length, options.limit));
+  });
+};

package/src/detectors/hankaku-kana.ts

@@ -0,0 +1,38 @@
+import type { Detector, Finding, Span } from "../plugin.ts";
+import { QUOTATION_MARKS, isWithinAny, quotedSpans } from "../quoted-span.ts";
+import { quoteAround } from "./quote-around.ts";
+
+/** 半角の片仮名と、半角の句読点・鉤括弧・中黒・長音・濁点（｡｢｣､･ｰﾞﾟ）。 */
+const HALFWIDTH_KANA_RUN = /[\uFF61-\uFF9F]+/gu;
+
+export type HalfwidthRun = { readonly span: Span; readonly written: string };
+
+/** 一行の中の、鉤括弧や引用符で引いたものの位置（行の頭を 0 とする）。引いた名前（「ｶﾅ表示」の画面名）は元の書き方のまま。 */
+const quotedOnLine = (source: string, at: number): { readonly lineStart: number; readonly quoted: readonly Span[] } => {
+  const lineStart = source.lastIndexOf("\n", at - 1) + 1;
+  const lineEnd = source.indexOf("\n", at);
+  return { lineStart, quoted: quotedSpans(source.slice(lineStart, lineEnd === -1 ? source.length : lineEnd), QUOTATION_MARKS) };
+};
+
+const isQuoted = (source: string, span: Span): boolean => {
+  const { lineStart, quoted } = quotedOnLine(source, span.start);
+  return isWithinAny(quoted, { start: span.start - lineStart, end: span.end - lineStart });
+};
+
+/** 字のまま見える範囲（texts）の中の、半角の片仮名の並び。引いたものの中は数えない。 */
+export const halfwidthKanaRuns = (source: string, texts: readonly Span[]): HalfwidthRun[] =>
+  texts.flatMap((text) =>
+    [...source.slice(text.start, text.end).matchAll(HALFWIDTH_KANA_RUN)]
+      .map((match) => ({ span: { start: text.start + match.index, end: text.start + match.index + match[0].length }, written: match[0] }))
+      .filter((run) => !isQuoted(source, run.span)),
+  );
+
+export const hankakuKana: Detector = (doc): Finding[] =>
+  halfwidthKanaRuns(doc.source, doc.markup?.texts ?? []).map(({ span, written }) => ({
+    rule: "",
+    severity: "warning",
+    line: 0,
+    column: 0,
+    quote: quoteAround(doc.source, span.start, span.end),
+    values: { kana: written, fullwidth: written.normalize("NFKC"), offset: span.start },
+  }));

package/src/detectors/heading-code-words.ts

@@ -1,8 +1,8 @@
 // A code span closes at the next run of exactly as many backticks as opened it: ``a ` b`` is one span.
 const CODE_SPAN = /(?<!`)(`+)(?!`)[\s\S]*?(?<!`)\1(?!`)/gu;
 
-const OPENING = new Set([..."(\"'“‘["]);
-const CLOSING = new Set([...")\"'”’].,:;!?"]);
+const OPENING = new Set("(\"'“‘[");
+const CLOSING = new Set(")\"'”’].,:;!?");
 
 /** The word without the punctuation around it: `(optional)` is `optional`, `Setup:` is `Setup`. */
 const bareWord = (token: string): string => {

package/src/detectors/heading-numbering.ts

@@ -0,0 +1,38 @@
+import type { Detector, Finding, LexiconEntry } from "../plugin.ts";
+import { findingAt, writtenHeadings } from "./markup-finding.ts";
+import { numberingMinorities, numberingStyleOf, UNNUMBERED, type NumberingMinority, type NumberLabel } from "../heading-numbering.ts";
+
+const labelsOf = (entries: readonly LexiconEntry[]): NumberLabel[] =>
+  entries.map((entry) => ({ word: entry.pattern, position: entry.position === "after" ? "after" : "before" }));
+
+/** 前後の見出しの語のあとに続いてよい区切り（「付録: 用語」「Appendix A」）。 */
+const AFTER_MATTER = /^[\s:：、]/u;
+
+/** 番号を付けない前後の見出し（目次・参考文献・付録 A）か。言葉全体が語彙表の語か、語彙表の語に区切りが続くとき。 */
+const isFrontOrBackMatter = (text: string, words: readonly string[]): boolean => {
+  const lowered = text.trim().toLowerCase();
+  return words.some((word) => lowered === word || (lowered.startsWith(word) && AFTER_MATTER.test(lowered.slice(word.length))));
+};
+
+const variantOf = (minority: NumberingMinority): string => {
+  if (minority.kind === "style") return "style";
+  return minority.majority === UNNUMBERED ? "numbered" : "unnumbered";
+};
+
+/**
+ * 兄弟の見出し（同じ親の下の同じ深さ）で、番号を付けたものと付けないもの、番号の書き方（1. と 1）と 第1章）が混ざっている所。
+ * 少ないほうを指す。どの語が番号を持つか（第1章、Chapter 1）は語彙表 heading-number-label、番号を付けない前後の見出しは unnumbered-heading が言う。
+ */
+export const headingNumbering: Detector = (doc): Finding[] => {
+  const headings = writtenHeadings(doc);
+  const labels = labelsOf(doc.lexicons["heading-number-label"] ?? []);
+  const matter = (doc.lexicons["unnumbered-heading"] ?? []).map((entry) => entry.pattern.toLowerCase());
+  const styles = headings.map((heading) => numberingStyleOf(heading.text, labels));
+  const read = headings.map((heading, at) => ({ depth: heading.depth, style: styles[at] ?? UNNUMBERED, skipped: isFrontOrBackMatter(heading.text, matter) }));
+  return numberingMinorities(read).flatMap((minority) => {
+    const heading = headings[minority.at];
+    if (heading === undefined) return [];
+    const values = { heading: heading.text.trim(), count: minority.count, example: headings[minority.example]?.text.trim() ?? "" };
+    return [findingAt(doc, heading, values, variantOf(minority))];
+  });
+};

package/src/detectors/index.ts

@@ -1,116 +1,20 @@
-import type { Detector } from "../plugin.ts";
-import { countPerSection } from "./count-per-section.ts";
-import { sentenceLength } from "./sentence-length.ts";
-import { headingEcho } from "./heading-echo.ts";
-import { repeatedHead } from "./repeated-head.ts";
-import { sentenceRhythm } from "./sentence-rhythm.ts";
-import { phraseCount, phraseMatch } from "./phrase-match.ts";
-import { agentlessPassive } from "./agentless-passive.ts";
-import { doubledWord } from "./doubled-word.ts";
-import { agreementSlip } from "./agreement-slip.ts";
-import { sentenceEnding } from "./sentence-ending.ts";
-import { doubledParticle, nounEnding } from "./token-shape.ts";
-import { kanjiRun, middleDot } from "./char-shape.ts";
-import { adverbDensity, conjunctionRun, expletive, titleCaseMix } from "./en-shape.ts";
-import { oxfordComma } from "./oxford-comma.ts";
-import { paragraphLength, paragraphVariance, preambleLength, ruleOfThree, sectionUniformity } from "./structure.ts";
-import { concreteEvidence, dashDensity, emojiDensity, ngramRepetition, undefinedAcronym } from "./signals.ts";
-import { aiTell, contractionMix, cushionDensity, hedging, repeatedConjunction, unqualifiedSuperlative } from "./lexicon.ts";
-import { internalJargon, properNounDensity, requiredSections } from "./team.ts";
-import { latinSpacing, preferredTerm } from "./orthography.ts";
-import { straySpace } from "./stray-space.ts";
-import { announcedCount } from "./announced-count.ts";
-import { danglingFigure } from "./dangling-figure.ts";
-import {
-  danglingReference,
-  dateOrder,
-  dateRangeReversed,
-  dateWeekdayMismatch,
-  duplicateDefinition,
-  numberingGap,
-  percentSumMismatch,
-  totalMismatch,
-} from "./structure-tree.ts";
-import { headingLevelSkip } from "./heading-level-skip.ts";
-import { imageAltText } from "./image-alt-text.ts";
-import { brokenLink } from "./broken-link.ts";
-import { urlRunOn } from "./url-run-on.ts";
-import { unbalancedBracket } from "./unbalanced-bracket.ts";
-import { doubledPunctuation } from "./doubled-punctuation.ts";
-import { kutotenConsistency } from "./kutoten-consistency.ts";
-import { katakanaLongVowel } from "./long-vowel.ts";
-import { customPattern, customTokens, customWords } from "./custom.ts";
-import { assistantResidue, colonLeadIn, contrastFraming, openerDensity, openerPile, unfilledPlaceholder } from "./ai-phrasing.ts";
+import { extname, join } from "node:path";
+import type { CrossDetector, Detector } from "../plugin.ts";
+import { loadRegistry } from "./registry-load.ts";
+
+// Each detector is registered by its own file, registry/<how_to_find>.ts, so a new rule adds a file and edits no shared
+// list. A rule that compares the documents of one run registers in cross-registry/<how_to_find>.ts instead.
+
+const isDetector = (value: unknown): value is Detector => typeof value === "function";
+
+const isCrossDetector = (value: unknown): value is CrossDetector => typeof value === "function";
 
 /** rule 定義の how_to_find がここを引く。rule 側は実装を知らない。 */
-export const DETECTORS: Readonly<Record<string, Detector>> = {
-  "count-per-section": countPerSection,
-  "sentence-length": sentenceLength,
-  "heading-echo": headingEcho,
-  "repeated-head": repeatedHead,
-  "sentence-rhythm": sentenceRhythm,
-  "phrase-match": phraseMatch,
-  "phrase-count": phraseCount,
-  "agentless-passive": agentlessPassive,
-  "sentence-ending": sentenceEnding,
-  "noun-ending": nounEnding,
-  "doubled-particle": doubledParticle,
-  "doubled-word": doubledWord,
-  "agreement-slip": agreementSlip,
-  "kanji-run": kanjiRun,
-  "middle-dot": middleDot,
-  "adverb-density": adverbDensity,
-  expletive: expletive,
-  "conjunction-run": conjunctionRun,
-  "title-case-mix": titleCaseMix,
-  "oxford-comma": oxfordComma,
-  "paragraph-length": paragraphLength,
-  "paragraph-variance": paragraphVariance,
-  "section-uniformity": sectionUniformity,
-  "rule-of-three": ruleOfThree,
-  "preamble-length": preambleLength,
-  "emoji-density": emojiDensity,
-  "dash-density": dashDensity,
-  "ngram-repetition": ngramRepetition,
-  "undefined-acronym": undefinedAcronym,
-  "concrete-evidence": concreteEvidence,
-  hedging: hedging,
-  "cushion-density": cushionDensity,
-  "unqualified-superlative": unqualifiedSuperlative,
-  "repeated-conjunction": repeatedConjunction,
-  "ai-tell": aiTell,
-  "contrast-framing": contrastFraming,
-  "stock-transition": openerDensity,
-  "opener-pile": openerPile,
-  "colon-lead-in": colonLeadIn,
-  "assistant-residue": assistantResidue,
-  "unfilled-placeholder": unfilledPlaceholder,
-  "contraction-mix": contractionMix,
-  "internal-jargon": internalJargon,
-  "required-sections": requiredSections,
-  "proper-noun-density": properNounDensity,
-  "preferred-term": preferredTerm,
-  "latin-spacing": latinSpacing,
-  "stray-space": straySpace,
-  "katakana-long-vowel": katakanaLongVowel,
-  "custom-words": customWords,
-  "custom-pattern": customPattern,
-  "custom-tokens": customTokens,
-  "dangling-reference": danglingReference,
-  "date-weekday-mismatch": dateWeekdayMismatch,
-  "date-order": dateOrder,
-  "total-mismatch": totalMismatch,
-  "numbering-gap": numberingGap,
-  "duplicate-definition": duplicateDefinition,
-  "announced-count": announcedCount,
-  "dangling-figure": danglingFigure,
-  "date-range-reversed": dateRangeReversed,
-  "percent-sum-mismatch": percentSumMismatch,
-  "heading-level-skip": headingLevelSkip,
-  "image-alt-text": imageAltText,
-  "broken-link": brokenLink,
-  "url-run-on": urlRunOn,
-  "unbalanced-bracket": unbalancedBracket,
-  "doubled-punctuation": doubledPunctuation,
-  "kutoten-consistency": kutotenConsistency,
-};
+export const DETECTORS: Readonly<Record<string, Detector>> = loadRegistry(join(import.meta.dirname, "registry"), extname(import.meta.filename), isDetector);
+
+/** The detectors of the rules that compare the documents of one run (requires: [documents]), by how_to_find. */
+export const CROSS_DETECTORS: Readonly<Record<string, CrossDetector>> = loadRegistry(
+  join(import.meta.dirname, "cross-registry"),
+  extname(import.meta.filename),
+  isCrossDetector,
+);

package/src/detectors/invisible-character.ts

@@ -0,0 +1,190 @@
+import type { Detector, Finding, Span } from "../plugin.ts";
+import { isJapanese } from "../orthography.ts";
+
+/**
+ * 見えない字の種類。zero-width はゼロ幅の空白・接合子と途中の BOM、soft-hyphen は任意の改行位置の印、direction は文字の向きの指定、
+ * control は改行・タブ・改ページ以外の制御文字、hidden は字の後ろに隠せる印（タグ文字、二つ以上続く異体字の指定）、
+ * nbsp は普通の空白が要る所のノーブレークスペース。どれも画面では見えず、検索・コピー・リンクを壊す。向きの指定と隠せる印は、
+ * 読み手に見えるものと機械が読むものを食い違わせるのにも使える。
+ */
+export type InvisibleKind = "zero-width" | "soft-hyphen" | "direction" | "control" | "hidden" | "nbsp";
+
+/**
+ * 種類ごとの字（正規表現の文字クラスの中身）。制御文字（Cc）には改行・タブ・改ページも入るので、それは isIntended が外す。
+ * hidden の異体字の指定は、字に一つ付いたものを isIntended が外す。
+ */
+const CLASSES: readonly (readonly [string, InvisibleKind])[] = [
+  ["\\u200B-\\u200D\\u2060-\\u2064\\uFEFF", "zero-width"],
+  ["\\u00AD", "soft-hyphen"],
+  ["\\u200E\\u200F\\u061C\\u202A-\\u202E\\u2066-\\u2069", "direction"],
+  ["\\p{Cc}", "control"],
+  ["\\u{E0001}\\u{E0020}-\\u{E007F}\\p{Variation_Selector}", "hidden"],
+  ["\\u00A0\\u2007\\u202F", "nbsp"],
+];
+
+const KIND_OF: readonly (readonly [RegExp, InvisibleKind])[] = CLASSES.map(([chars, kind]) => [new RegExp(`^[${chars}]$`, "u"), kind]);
+
+/** 一字ずつ当てる。タグ文字と異体字の指定は BMP の外にもあるので u で読む。 */
+const CANDIDATE = new RegExp(`[${CLASSES.map(([chars]) => chars).join("")}]`, "gu");
+
+const kindOf = (char: string): InvisibleKind | undefined => KIND_OF.find(([pattern]) => pattern.test(char))?.[1];
+
+/** 接合子（ZWJ・ZWNJ）で字の形を変える文字体系。アラビア文字やインドの文字では、見えない接合子が綴りの一部。 */
+const JOINING_SCRIPT = /[\p{L}\p{M}]/u;
+const NON_JOINING_SCRIPT = /[\p{Script=Latin}\p{Script=Greek}\p{Script=Cyrillic}\p{Script=Han}\p{Script=Hiragana}\p{Script=Katakana}\p{Script=Hangul}]/u;
+/** どの文字体系にも付くアクセント（U+0301 など）。ラテン文字の後ろの接合子を、接合子を使う文字体系のものと読まない。 */
+const INHERITED = /\p{Script=Inherited}/u;
+
+/** 右から左に書く文字。この字の隣の向きの印（LRM・RLM）は、英数字と混ぜて正しく並べるためのもの。 */
+const RIGHT_TO_LEFT = /[\p{Script=Hebrew}\p{Script=Arabic}\p{Script=Syriac}\p{Script=Thaana}\p{Script=Nko}\p{Script=Samaritan}]/u;
+
+/** 字の向きを強制する印（LRO・RLO）。右から左の文字の隣でも、見える順を入れ替えるのに使えるので、いつも指摘する。 */
+const OVERRIDE = /[\u202D\u202E]/u;
+
+/** 絵文字の部品。ZWJ でつないだ絵文字（家族・職業）と、肌の色や異体字の指定を挟んだもの。 */
+const PICTOGRAPH = /[\p{Extended_Pictographic}\p{Emoji_Modifier}\uFE0F]/u;
+
+/** 黒い旗の絵文字。後ろにタグ文字を並べて地域の旗（イングランドなど）を書く。地域の名は六字までで、旗は字の数で十四単位に収まる。 */
+const BLACK_FLAG = String.fromCodePoint(0x1f3f4);
+const FLAG_REACH = 14;
+const TAG = /[\u{E0020}-\u{E007F}]/u;
+
+const VARIATION_SELECTOR = /\p{Variation_Selector}/u;
+
+/** 文書の形を作る制御文字（タブ・改行・改ページ）。 */
+const LAYOUT_CONTROL = /[\t\n\r\f]/u;
+
+/** Markdown の行の頭の印（見出し・箇条・番号・引用）。後ろの空白がノーブレークスペースだと、印として読まれない。 */
+const BLOCK_MARKER = /^[ \t]*(?:#{1,6}|[-*+]|\d{1,9}[.)]|>)$/u;
+const JAPANESE_MARK = /[、。「」『』（）・]/u;
+
+/** at の前の一字（サロゲートの対は一字に）。文書の頭なら空。 */
+const charBefore = (source: string, at: number): string => {
+  if (at <= 0) return "";
+  const low = source.charCodeAt(at - 1);
+  const isLow = low >= 0xdc00 && low <= 0xdfff;
+  return isLow && at >= 2 ? source.slice(at - 2, at) : source.slice(at - 1, at);
+};
+
+/** at から始まる一字。文書の終わりなら空。 */
+const charAt = (source: string, at: number): string => {
+  const code = source.codePointAt(at);
+  return code === undefined ? "" : String.fromCodePoint(code);
+};
+
+const isJoiningLetter = (char: string): boolean => JOINING_SCRIPT.test(char) && !NON_JOINING_SCRIPT.test(char) && !INHERITED.test(char);
+
+const isJapaneseSide = (char: string): boolean => isJapanese(char) || JAPANESE_MARK.test(char);
+
+const CANCEL_TAG = String.fromCodePoint(0xe007f);
+
+/** タグ文字の並びをさかのぼった先が黒い旗で、並びが取り消しのタグ（U+E007F）で閉じていれば、地域の旗の中。 */
+const isInFlag = (source: string, at: number): boolean => {
+  const before = Array.from(source.slice(Math.max(0, at - FLAG_REACH), at));
+  const after = Array.from(source.slice(at, at + FLAG_REACH));
+  const closing = after.findIndex((char) => !TAG.test(char) || char === CANCEL_TAG);
+  return before[before.findLastIndex((char) => !TAG.test(char))] === BLACK_FLAG && after[closing] === CANCEL_TAG;
+};
+
+/** 異体字の指定が付く字。空白や文書の頭の後ろの指定は、付く字が無い。 */
+const SELECTOR_BASE = /\S/u;
+
+/** 字が普通に使う異体字の指定は、字の後ろに一つ。二つ以上続く指定と、付く字の無い指定は、何かを隠した並び。 */
+const isLoneSelector = (source: string, at: number, length: number): boolean => {
+  const before = charBefore(source, at);
+  return SELECTOR_BASE.test(before) && !VARIATION_SELECTOR.test(before) && !VARIATION_SELECTOR.test(charAt(source, at + length));
+};
+
+/** 向きの分離の開き（LRI・RLI・FSI）と閉じ（PDI）。囲んだ中に右から左の文字があれば、混ぜて並べるための印。 */
+const ISOLATE_OPENS: readonly string[] = ["\u2066", "\u2067", "\u2068"];
+const ISOLATE_CLOSE = "\u2069";
+const ISOLATE_REACH = 200;
+
+/** 分離の印 char（at の位置）が囲む中身。組になる印が近くに無ければ undefined。 */
+const isolatedText = (source: string, at: number, char: string): string | undefined => {
+  if (ISOLATE_OPENS.includes(char)) {
+    const inside = source.slice(at + 1, at + 1 + ISOLATE_REACH);
+    const end = inside.indexOf(ISOLATE_CLOSE);
+    return end === -1 ? undefined : inside.slice(0, end);
+  }
+  if (char !== ISOLATE_CLOSE) return undefined;
+  const inside = source.slice(Math.max(0, at - ISOLATE_REACH), at);
+  const start = Math.max(...ISOLATE_OPENS.map((open) => inside.lastIndexOf(open)));
+  return start === -1 ? undefined : inside.slice(start + 1);
+};
+
+const isolatesRightToLeft = (source: string, at: number, char: string): boolean => RIGHT_TO_LEFT.test(isolatedText(source, at, char) ?? "");
+
+/** 文書の頭の BOM はファイルの印で、本文の字ではない。 */
+const BYTE_ORDER_MARK = "\uFEFF";
+
+const lineBefore = (source: string, at: number): string => source.slice(source.lastIndexOf("\n", at - 1) + 1, at);
+
+/** 見えない字一つと、その前後の字。 */
+type Place = { readonly source: string; readonly at: number; readonly char: string; readonly before: string; readonly after: string };
+
+const ZERO_WIDTH_JOINER = "\u200D";
+const ZERO_WIDTH_NON_JOINER = "\u200C";
+
+const joinsLetters = ({ before, after }: Place): boolean => isJoiningLetter(before) || isJoiningLetter(after);
+
+/** 種類ごとの、書き手が意味を持たせて置いた見えない字。そうでないものだけを指摘する。 */
+const INTENDED: Readonly<Record<InvisibleKind, (place: Place) => boolean>> = {
+  "zero-width": (place) => {
+    if (place.char === BYTE_ORDER_MARK) return place.at === 0;
+    if (place.char === ZERO_WIDTH_JOINER) return (PICTOGRAPH.test(place.before) && PICTOGRAPH.test(place.after)) || joinsLetters(place);
+    return place.char === ZERO_WIDTH_NON_JOINER && joinsLetters(place);
+  },
+  "soft-hyphen": () => false,
+  direction: ({ source, at, char, before, after }) =>
+    !OVERRIDE.test(char) && (RIGHT_TO_LEFT.test(before) || RIGHT_TO_LEFT.test(after) || isolatesRightToLeft(source, at, char)),
+  control: ({ char }) => LAYOUT_CONTROL.test(char),
+  hidden: ({ source, at, char }) => (TAG.test(char) ? isInFlag(source, at) : isLoneSelector(source, at, char.length)),
+  nbsp: ({ source, at, before, after }) => !BLOCK_MARKER.test(lineBefore(source, at)) && !isJapaneseSide(before) && !isJapaneseSide(after),
+};
+
+const isIntended = (source: string, at: number, char: string, kind: InvisibleKind): boolean =>
+  INTENDED[kind]({ source, at, char, before: charBefore(source, at), after: charAt(source, at + char.length) });
+
+export type InvisibleRun = { readonly span: Span; readonly kind: InvisibleKind; readonly codes: readonly string[] };
+
+export const codePointName = (char: string): string => `U+${(char.codePointAt(0) ?? 0).toString(16).toUpperCase().padStart(4, "0")}`;
+
+const joinRuns = (runs: InvisibleRun[], next: InvisibleRun): InvisibleRun[] => {
+  const last = runs.at(-1);
+  if (last === undefined || last.span.end !== next.span.start || last.kind !== next.kind) return [...runs, next];
+  const codes = last.codes.includes(next.codes[0] ?? "") ? last.codes : [...last.codes, ...next.codes];
+  return [...runs.slice(0, -1), { span: { start: last.span.start, end: next.span.end }, kind: last.kind, codes }];
+};
+
+/** source の中の、見えない字の並び。隣り合う同じ種類の字は一つの並びにまとめる。 */
+export const invisibleRuns = (source: string): InvisibleRun[] =>
+  [...source.matchAll(CANDIDATE)]
+    .flatMap((match): InvisibleRun[] => {
+      const kind = kindOf(match[0]);
+      if (kind === undefined || isIntended(source, match.index, match[0], kind)) return [];
+      return [{ span: { start: match.index, end: match.index + match[0].length }, kind, codes: [codePointName(match[0])] }];
+    })
+    .reduce<InvisibleRun[]>(joinRuns, []);
+
+/** 見せる前後の字数。 */
+const QUOTE_REACH = 20;
+
+/** 見えない字を「⟨U+200B⟩」のように見える形にして、前後を引く。 */
+export const visibleQuote = (source: string, span: Span): string =>
+  source
+    .slice(Math.max(0, span.start - QUOTE_REACH), span.end + QUOTE_REACH)
+    .replace(CANDIDATE, (char) => (LAYOUT_CONTROL.test(char) ? char : `⟨${codePointName(char)}⟩`))
+    .replace(/\s+/gu, " ")
+    .trim();
+
+export const invisibleCharacter: Detector = (doc): Finding[] =>
+  invisibleRuns(doc.source).map((run) => ({
+    rule: "",
+    severity: "warning",
+    line: 0,
+    column: 0,
+    quote: visibleQuote(doc.source, run.span),
+    values: { code: run.codes.join(" "), count: Array.from(doc.source.slice(run.span.start, run.span.end)).length, offset: run.span.start },
+    variant: run.kind,
+  }));

package/src/detectors/katakana-density.ts

@@ -0,0 +1,62 @@
+import type { Detector, Finding, ProseDocument, Sentence } from "../plugin.ts";
+import { proseText } from "../measure.ts";
+import { rateOver } from "./lexicon.ts";
+
+/** A katakana word, full or half width: two or more katakana, not starting with the long-vowel mark. A middle dot (ジョン・スミス) splits it. */
+const KATAKANA_WORD = /[ァ-ヴ][ァ-ヴー]+|[ｦ-ﾝ][ｦ-ﾟ]+/gu;
+
+/** A line break inside a katakana word (アジェ↵ンダ): a wrapped line, not two words. */
+const WRAP_IN_WORD = /(?<=[ァ-ヴーｦ-ﾟ])[^\S\n]*\n[^\S\n]*(?=[ァ-ヴーｦ-ﾟ])/gu;
+
+/** How many of the most frequent words the message names. */
+const NAMED_WORDS = 3;
+
+type KatakanaWord = { readonly sentence: Sentence; readonly word: string };
+
+/** The text with each of the team's names (chaff.yaml names:) blanked out, longest first: a name is not a loanword to trim. */
+const withoutNames = (text: string, names: readonly string[]): string =>
+  names
+    .filter((name) => name !== "")
+    .toSorted((left, right) => right.length - left.length)
+    .reduce((rest, name) => rest.replaceAll(name, " "), text);
+
+/** The katakana words of one sentence's text, a line wrapped inside a word joined back, the team's names left out. */
+export const katakanaWordsIn = (text: string, names: readonly string[]): string[] =>
+  [...withoutNames(text.replace(WRAP_IN_WORD, ""), names).matchAll(KATAKANA_WORD)].map((match) => match[0]);
+
+/** The words used most, most frequent first; ties keep the order they first appear in. */
+export const mostFrequent = (words: readonly string[], count: number): string[] => {
+  const tally = words.reduce((counts, word) => counts.set(word, (counts.get(word) ?? 0) + 1), new Map<string, number>());
+  return [...tally.entries()]
+    .toSorted((left, right) => right[1] - left[1])
+    .slice(0, count)
+    .map(([word]) => word);
+};
+
+const wordsOfDocument = (doc: ProseDocument): KatakanaWord[] =>
+  doc.sentences.flatMap((sentence) => katakanaWordsIn(sentence.text, doc.names ?? []).map((word) => ({ sentence, word })));
+
+/**
+ * Katakana loanwords per 1000 characters, over the whole document. Reported once, at the first sentence holding one, with the
+ * words used most: those are the ones a Japanese word could replace first. Short documents are not measured.
+ */
+export const katakanaDensity: Detector = (doc, options): Finding[] => {
+  const found = wordsOfDocument(doc);
+  const density = rateOver(doc, found.length, options.limit);
+  const first = found[0];
+  if (density === undefined || first === undefined) return [];
+  const words = mostFrequent(
+    found.map((hit) => hit.word),
+    NAMED_WORDS,
+  );
+  return [
+    {
+      rule: "katakana-density",
+      severity: "info",
+      line: 0,
+      column: 0,
+      quote: proseText(first.sentence),
+      values: { count: found.length, density, limit: options.limit, words: words.join("、"), offset: first.sentence.span.start },
+    },
+  ];
+};