chaffjs 0.10.0 → 0.11.0

package/rules/double-keigo.yaml

@@ -3,6 +3,7 @@ layer: L3
 status: experimental
 how_to_find: phrase-match
 word_list: double-keigo
+uses: [pos]
 languages: [ja]
 severity: warning
 

package/rules/empty-intensifier.yaml

@@ -3,6 +3,7 @@ layer: L2
 status: stable
 how_to_find: phrase-match
 word_list: empty-intensifier
+uses: [pos]
 severity: warning
 
 name:

package/rules/excessive-hedging.yaml

@@ -3,6 +3,7 @@ layer: L2
 status: experimental
 how_to_find: hedging-density
 word_list: excessive-hedging
+uses: [pos]
 severity: warning
 
 name:

package/rules/hiragana-fukushi.yaml

@@ -3,6 +3,7 @@ layer: L3
 status: experimental
 how_to_find: phrase-match
 word_list: hiragana-fukushi
+uses: [pos]
 languages: [ja]
 severity: info
 

package/rules/max-kanji-continuous.yaml

@@ -2,6 +2,8 @@ id: max-kanji-continuous
 layer: L3
 status: experimental
 how_to_find: kanji-run
+# 品詞があれば、住所（地名と数だけでできた連なり）を数えない。無くても文字だけで動く。
+uses: [pos]
 languages: [ja]
 severity: warning
 

package/rules/ngram-repetition.yaml

@@ -2,6 +2,8 @@ id: ngram-repetition
 layer: L1
 status: stable
 how_to_find: ngram-repetition
+# 品詞があれば使う（数の語、言い回しの動詞）。無くても文字だけで動く。
+uses: [pos]
 severity: info
 
 name:

package/rules/padded-intro.yaml

@@ -3,6 +3,7 @@ layer: L2
 status: stable
 how_to_find: phrase-match
 word_list: padded-intro
+uses: [pos]
 where: opening
 severity: warning
 

package/rules/repeated-conjunction.yaml

@@ -3,6 +3,7 @@ layer: L2
 status: experimental
 how_to_find: repeated-conjunction
 word_list: paragraph-opener
+uses: [pos]
 severity: warning
 
 name:

package/rules/sasete-itadaku.yaml

@@ -3,6 +3,7 @@ layer: L3
 status: experimental
 how_to_find: phrase-match
 word_list: sasete-itadaku
+uses: [pos]
 languages: [ja]
 severity: info
 

package/rules/unqualified-superlative.yaml

@@ -3,6 +3,7 @@ layer: L2
 status: experimental
 how_to_find: unqualified-superlative
 word_list: superlative
+uses: [pos]
 severity: warning
 
 name:

package/src/cli-text.ts

@@ -12,6 +12,7 @@ const USAGE_JA = `chaff — 文章の読みにくいところを見つけます
   chaff test <file|dir>...       意味を読む検査も動かす（API key が要ります）
                                  判定役は chaff.yaml の ai_backend で選びます
   chaff init                     chaff.yaml を作る
+  chaff --version                chaffjs と言語パッケージの版
   chaff eval <dir>               手元の文書で閾値を測り直す
   chaff explain <rule>           そのルールの意図と根拠を読む
   chaff genres                   ジャンルの一覧
@@ -45,6 +46,7 @@ const USAGE_EN = `chaff — finds what makes writing hard to read. It never rewr
   chaff test <file|dir>...       also run the checks that read meaning (needs an API key)
                                  the judge is chosen by ai_backend in chaff.yaml
   chaff init                     create chaff.yaml
+  chaff --version                the version of chaffjs and its language packages
   chaff eval <dir>               re-measure the limits on your own documents
   chaff explain <rule>           read what a rule is for and why
   chaff genres                   list the genres

package/src/cli.ts

@@ -23,7 +23,7 @@ import { renderExplain } from "./render/explain.ts";
 import { renderFriendly } from "./render/friendly.ts";
 import { rulesJson } from "./render/rules-json.ts";
 import { renderSarif } from "./render/sarif.ts";
-import { VERSION } from "./version.ts";
+import { VERSION, VERSION_LINES } from "./version.ts";
 import { runTree, treeTargets, type TreeContext } from "./commands/tree.ts";
 import { citeTargets, runCite } from "./commands/cite.ts";
 import { runSkill } from "./commands/skill.ts";
@@ -328,6 +328,10 @@ export const main = async (argv: readonly string[]): Promise<number> => {
     console.log(hostText(readConfig()).usage);
     return first === undefined ? 1 : 0;
   }
+  if (first === "--version" || first === "-v") {
+    console.log(VERSION_LINES.join("\n"));
+    return 0;
+  }
   const cliGenre = flag(argv, "--genre");
   if (cliGenre !== undefined && !GENRES.includes(cliGenre)) {
     console.error(hostText(readConfig()).unknownGenre(cliGenre));

package/src/detectors/char-shape.ts

@@ -1,15 +1,50 @@
-import type { Detector, DocumentProfile, Finding, Sentence } from "../plugin.ts";
+import type { Detector, DocumentProfile, Finding, Sentence, Token } from "../plugin.ts";
+import { compacted, placeOf } from "./gram-place.ts";
 import { maskAddresses } from "../address-chain.ts";
 import { proseText } from "../measure.ts";
 
 /**
- * 文字の並びだけを見る検出。品詞も語彙表も要らない。
+ * 文字の並びを見る検出。漢字の連なりが住所かどうかだけは、形態素解析の地名と数で決める。
  * 覆った箇所は空白になっているので、proseText で潰してから数える。
  */
 const KANJI_RUN = /[一-鿿]+/gu;
 
+/** 地名（Geo）と、地名に付く単位（GeoUnit: 都・県・市・区・町）。 */
+const isGeoName = (token: Token | undefined): boolean => token?.features?.["NameType"] === "Geo";
+const isGeo = (token: Token | undefined): boolean => isGeoName(token) || token?.features?.["NameType"] === "GeoUnit";
+const isNumber = (token: Token | undefined): boolean => token?.features?.["NumType"] === "Card";
+
+/**
+ * 住所の語: 地名と地名の単位（NameType=Geo / GeoUnit）、数（NumType=Card）、数のすぐ後ろの助数詞（丁目、NounType=Class）。
+ * 辞書に無い町名（新橋）は、地名の後ろで数の前に来るときだけ住所の一部とする。
+ */
+const isPlacePart = (tokens: readonly Token[], index: number): boolean =>
+  isGeo(tokens[index]) ||
+  isNumber(tokens[index]) ||
+  (isNumber(tokens[index - 1]) && tokens[index]?.features?.["NounType"] === "Class") ||
+  (isGeo(tokens[index - 1]) && isNumber(tokens[index + 1]));
+
+/**
+ * 漢字の連なりが住所か（東京都港区新橋二丁目）。形態素解析が地名と読んだ語で始まり、住所の語だけでできている。
+ * 住所は決まった形で、ひらがなを挟んで書けない。組織名（日本経済団体連合会）は地名の後ろに普通の語が続くので住所ではない。品詞が無ければ判定しない。
+ */
+const isPlaceName = (sentence: Sentence, run: string): boolean => {
+  const tokens = sentence.tokens;
+  const place = tokens === undefined ? undefined : placeOf(compacted(sentence.text, "word"), run);
+  if (tokens === undefined || place === undefined) return false;
+  const [start, end] = [sentence.span.start + place.start, sentence.span.start + place.end];
+  const covering = tokens.flatMap((token, index) => (token.span.start < end && start < token.span.end ? [index] : []));
+  const first = covering[0];
+  // 地名が単位を挟まずに続けば（東京大阪名古屋福岡）、住所ではなく地名の並び。
+  const listed = covering.some((index) => isGeoName(tokens[index]) && isGeoName(tokens[index + 1]));
+  return first !== undefined && isGeo(tokens[first]) && !listed && covering.every((index) => isPlacePart(tokens, index));
+};
+
 const longestKanji = (sentence: Sentence, profile: DocumentProfile | undefined): string =>
-  [...maskAddresses(proseText(sentence), profile).matchAll(KANJI_RUN)].reduce((longest, match) => (match[0].length > longest.length ? match[0] : longest), "");
+  [...maskAddresses(proseText(sentence), profile).matchAll(KANJI_RUN)]
+    .map((match) => match[0])
+    .filter((run) => !isPlaceName(sentence, run))
+    .reduce((longest, run) => (run.length > longest.length ? run : longest), "");
 
 /**
  * 漢字が続くと、どこで語が切れるのか読み手が探すことになる。

package/src/detectors/en-shape.ts

@@ -150,6 +150,139 @@ export const titleCaseMix: Detector = (doc, options): Finding[] => {
 
 const LIST_CONJUNCTION = new Set(["and", "or"]);
 
+const isComma = (token: Token): boolean => token.surface === ",";
+
+const commaBefore = (tokens: readonly Token[], at: number): boolean => tokens[at - 1]?.surface === ",";
+
+/** 並列はこれをまたがない。セミコロンの前後は別の節。 */
+const CLAUSE_BREAK = new Set([";", ":", "—"]);
+
+/** 各 token の前で閉じていない括弧の数。括弧の中の読点（external users (e.g., guests), and ...）は外の並列を切らない。 */
+const PAREN_STEP: Readonly<Record<string, number>> = { "(": 1, ")": -1 };
+
+const depthsOf = (tokens: readonly Token[]): number[] =>
+  tokens.reduce<{ depths: number[]; open: number }>(
+    (acc, token) => ({ depths: [...acc.depths, acc.open], open: Math.max(0, acc.open + (PAREN_STEP[token.surface] ?? 0)) }),
+    { depths: [], open: 0 },
+  ).depths;
+
+type Clause = { readonly tokens: readonly Token[]; readonly depths: readonly number[] };
+
+/** 冠詞や引用符を飛ばした、項目の頭の品詞。the parser と an exporter と samples を同じ形と見る。 */
+const NOMINAL = new Set(["NOUN", "PROPN", "PRON", "NUM", "ADJ"]);
+
+const isContent = (token: Token): boolean => token.pos !== "DET" && token.pos !== "PUNCT" && token.pos !== "X";
+
+const shapeOf = (item: readonly Token[]): string | undefined => {
+  const head = item.find(isContent);
+  if (head === undefined) return undefined;
+  return NOMINAL.has(head.pos) ? "NOMINAL" : head.pos;
+};
+
+const VERBAL = new Set(["VERB", "AUX"]);
+
+/** 主語と述語のある項目。the team fixed the bug / these are crucial。gets us more は述語だけ。 */
+const isClause = (item: readonly Token[]): boolean => {
+  const first = item.find((token) => token.pos !== "PUNCT" && token.pos !== "X");
+  if (first === undefined || !(first.pos === "DET" || NOMINAL.has(first.pos))) return false;
+  return item.some((token) => token !== first && VERBAL.has(token.pos));
+};
+
+/** 節の頭から and / or の手前までを、同じ深さの読点で項目に切る。Oxford comma の読点のあとは空なので項目にならない。 */
+const itemsBefore = (clause: Clause, at: number): Token[][] => {
+  const level = clause.depths[at] ?? 0;
+  const start = clause.tokens.slice(0, at).findLastIndex((token) => CLAUSE_BREAK.has(token.surface)) + 1;
+  return clause.tokens
+    .slice(start, at)
+    .reduce<Token[][]>(
+      (items, token, offset) => {
+        if (isComma(token) && clause.depths[start + offset] === level) return [...items, []];
+        return [...items.slice(0, -1), [...(items.at(-1) ?? []), token]];
+      },
+      [[]],
+    )
+    .filter((item) => item.length > 0);
+};
+
+/** and / or の後ろの項目。次の読点か節の切れ目まで。 */
+const itemAfter = (tokens: readonly Token[], at: number): Token[] => {
+  const end = tokens.findIndex((token, index) => index > at && (isComma(token) || CLAUSE_BREAK.has(token.surface)));
+  return tokens.slice(at + 1, end === -1 ? undefined : end);
+};
+
+/**
+ * 導入の句（After the review, / If it fails, / Finally, / To test it, / Based on the review,）は並列の項目ではない。
+ * 節の最初の項目が前置詞・接続詞・副詞・to・過去分詞で始まり、次の項目と頭の形が違えば外す。
+ * Quickly, quietly and carefully は残る。For managers, engineers and designers, も外れるが、前置詞のあとの名詞を
+ * 項目と読むと、Over this period, the subcommittees and the full committee ... を並びと誤る。こちらのほうが多い。
+ */
+const LEAD_POS = new Set(["ADP", "SCONJ", "ADV", "PART"]);
+
+const isParticiple = (token: Token): boolean => token.features?.["VerbForm"] === "Part";
+
+const openingOf = (item: readonly Token[]): Token | undefined => item.find((token) => token.pos !== "PUNCT" && token.pos !== "X");
+
+const openingKind = (item: readonly Token[]): string | undefined => {
+  const opening = openingOf(item);
+  return opening !== undefined && isParticiple(opening) ? "PARTICIPLE" : shapeOf(item);
+};
+
+const withoutLead = (items: readonly Token[][]): readonly Token[][] => {
+  const [first, second] = items;
+  const opening = first === undefined ? undefined : openingOf(first);
+  if (first === undefined || second === undefined || opening === undefined) return items;
+  if (!LEAD_POS.has(opening.pos) && !isParticiple(opening)) return items;
+  return openingKind(first) === openingKind(second) ? items : items.slice(1);
+};
+
+/**
+ * 形容詞のあとの読点は、名詞の前で形容詞を重ねているだけのことが多い（the long, winding bridge）。
+ * 次の項目が名詞で終わるならつなげ直す。形容詞そのものの並び（quick, cheap and reliable）は切ったまま。
+ */
+const lastContent = (item: readonly Token[]): Token | undefined => item.findLast(isContent);
+
+const joinAdjectives = (items: readonly Token[][]): Token[][] =>
+  items.reduce<Token[][]>((joined, item) => {
+    const previous = joined.at(-1);
+    const stacked = previous !== undefined && lastContent(previous)?.pos === "ADJ" && lastContent(item)?.pos !== "ADJ";
+    return stacked ? [...joined.slice(0, -1), [...previous, ...item]] : [...joined, item];
+  }, []);
+
+/**
+ * 節を並べるなら、どの項目も節。Additionally, others can learn, and the mistake is rarer. の
+ * Additionally は項目ではなく、and の前の読点は節をつなぐ読点。名詞の並びは最初の項目に前置き
+ * （We shipped the parser）を抱えるので、この確かめは節の並びだけにする。
+ * and の後ろだけが節なのは主語の並び（The parser, the renderer and the exporter shipped.）なので外さない。
+ */
+const clausesAgree = (items: readonly Token[][], after: readonly Token[]): boolean => {
+  const last = items.at(-1);
+  if (last === undefined || !isClause(last)) return true;
+  return isClause(after) && items.every(isClause);
+};
+
+/**
+ * 動詞で始まる項目を並べるなら、どの項目にも動詞がある。The scope, in contrast, is larger, and covers ... の
+ * The scope は項目ではなく、後ろの述語の主語。
+ */
+const hasVerb = (item: readonly Token[]): boolean => item.some((token) => VERBAL.has(token.pos));
+
+const predicatesAgree = (items: readonly Token[][], shape: string): boolean => !VERBAL.has(shape) || items.every(hasVerb);
+
+/**
+ * 読点で区切った項目が and / or の前に 2 つ以上あり、最後の項目と and の後ろが同じ形のときだけ並列。
+ * 導入の読点（After the review, the team fixed the bug and shipped it.）や、節をつなぐ読点
+ * （We tested it, and the team shipped it.）は、Oxford comma を打つかどうかの選択を見せない。
+ */
+const listAt = (clause: Clause, at: number): boolean | undefined => {
+  const items = withoutLead(joinAdjectives(itemsBefore(clause, at)));
+  const after = itemAfter(clause.tokens, at);
+  const last = items.at(-1);
+  const shape = last === undefined ? undefined : shapeOf(last);
+  if (items.length < 2 || shape === undefined || shape !== shapeOf(after)) return undefined;
+  if (!clausesAgree(items, after) || !predicatesAgree(items, shape)) return undefined;
+  return commaBefore(clause.tokens, at);
+};
+
 /**
  * 3 つ以上の並列の最後の and / or の前に読点を打つか。Oxford comma。
  *
@@ -157,13 +290,11 @@ const LIST_CONJUNCTION = new Set(["and", "or"]);
  * 見るのは 1 つの文書で揃っているかだけ。spec §12.3。
  */
 const oxfordIn = (tokens: readonly Token[]): boolean | undefined => {
-  const at = tokens.findIndex((token) => LIST_CONJUNCTION.has(token.surface.toLowerCase()));
-  if (at < 1) return undefined;
-  const before = tokens.slice(0, at);
-  // 並列が 3 つ以上あるときだけ判定できる。読点が 1 つも無ければ 2 つの並列。
-  const commas = before.filter((token) => token.surface === ",").length;
-  if (commas === 0) return undefined;
-  return before.at(-1)?.surface === ",";
+  const clause = { tokens, depths: depthsOf(tokens) };
+  return tokens.reduce<boolean | undefined>(
+    (found, token, at) => found ?? (at > 0 && LIST_CONJUNCTION.has(token.surface.toLowerCase()) ? listAt(clause, at) : undefined),
+    undefined,
+  );
 };
 
 export const oxfordComma: Detector = (doc, options): Finding[] => {

package/src/detectors/gram-place.ts

@@ -0,0 +1,41 @@
+import type { LengthUnit, Span } from "../plugin.ts";
+
+/** 数えるために詰めた文と、詰めた文の各文字が元の文のどこにあったか。 */
+export type Compacted = { readonly text: string; readonly offsets: readonly number[] };
+
+/**
+ * n-gram を数えるときと同じ詰め方。word 単位は空白の並びを 1 つにして前後を削り、char 単位は空白を全部除く。
+ * 語句が元の文のどこにあるかを戻せるよう、残した文字ごとに元の位置を持つ。
+ */
+export const compacted = (text: string, unit: LengthUnit): Compacted => {
+  const chars: string[] = [];
+  const offsets: number[] = [];
+  const state = { pendingSpace: -1, at: 0 };
+  Array.from(text).forEach((char) => {
+    const at = state.at;
+    state.at += char.length;
+    if (/\s/u.test(char)) {
+      if (unit === "word" && chars.length > 0 && state.pendingSpace === -1) state.pendingSpace = at;
+      return;
+    }
+    if (state.pendingSpace !== -1) {
+      chars.push(" ");
+      offsets.push(state.pendingSpace);
+      state.pendingSpace = -1;
+    }
+    chars.push(char);
+    // 位置は UTF-16 の単位で持つ。placeOf は indexOf の位置で引くので、絵文字（2 単位）には 2 つ置く。
+    Array.from({ length: char.length }, (_, unit) => offsets.push(at + unit));
+  });
+  return { text: chars.join(""), offsets };
+};
+
+/** 詰めた文の中の gram の最初の出現を、元の文の範囲に戻す。無ければ undefined。 */
+export const placeOf = (source: Compacted, gram: string): Span | undefined => {
+  const at = source.text.indexOf(gram);
+  if (at === -1) return undefined;
+  const start = source.offsets[at];
+  const last = source.offsets[at + gram.length - 1];
+  if (start === undefined || last === undefined) return undefined;
+  return { start, end: last + 1 };
+};

package/src/detectors/lexicon-match.ts

@@ -0,0 +1,44 @@
+import type { LexiconEntry, Sentence, Token } from "../plugin.ts";
+import { proseText } from "../measure.ts";
+
+/** 活用する品詞。"best" を "good" と同じ語にはしない。 */
+const INFLECTING = new Set(["VERB", "AUX"]);
+
+/**
+ * 語彙表が原形で書いた語（いただく）だけが、活用した形（いただきます・いただいた）に当たる。
+ * 活用した形で書いた語（could・かもしれません）はその形だけ。"could" は "can" と言っていることが違う。
+ */
+const sameWord = (written: Token, entry: Token): boolean => {
+  const surface = entry.surface.toLowerCase();
+  if (written.surface.toLowerCase() === surface) return true;
+  const inflects = INFLECTING.has(written.pos) && INFLECTING.has(entry.pos);
+  return inflects && entry.lemma?.toLowerCase() === surface && written.lemma?.toLowerCase() === surface;
+};
+
+const runsAt = (tokens: readonly Token[], entry: readonly Token[], start: number): boolean =>
+  entry.every((word, offset) => {
+    const written = tokens[start + offset];
+    return written !== undefined && sameWord(written, word);
+  });
+
+/**
+ * 語彙の語が文の中にあるか。品詞が付いていれば語の並びで照らし、活用する語は原形で比べる（「させていただく」が「させていただきました」に当たる）。
+ * 語の途中には当たらない（「また」は「またいで」に当たらない）。品詞が無ければ、空白をまとめた文字列で照らす。
+ */
+export const entryIn = (sentence: Sentence, entry: LexiconEntry): boolean => {
+  const tokens = sentence.tokens;
+  const words = entry.tokens;
+  if (tokens === undefined || words === undefined || words.length === 0) return proseText(sentence).toLowerCase().includes(entry.pattern.toLowerCase());
+  return tokens.some((_token, start) => runsAt(tokens, words, start));
+};
+
+const LEADING_MARK = new Set(["PUNCT", "SYM"]);
+
+/** 語彙の語で文が始まるか。文頭の記号は飛ばす。品詞が無ければ文字列の前方一致。 */
+export const entryOpens = (sentence: Sentence, entry: LexiconEntry): boolean => {
+  const tokens = sentence.tokens;
+  const words = entry.tokens;
+  if (tokens === undefined || words === undefined || words.length === 0) return proseText(sentence).toLowerCase().startsWith(entry.pattern.toLowerCase());
+  const first = tokens.findIndex((token) => !LEADING_MARK.has(token.pos));
+  return runsAt(tokens, words, first);
+};

package/src/detectors/lexicon.ts

@@ -1,7 +1,7 @@
 import { joinWords } from "./word-list.ts";
-import { proseText } from "../measure.ts";
 import { wordsOf } from "./structure.ts";
-import type { Detector, Finding, ProseDocument, Sentence } from "../plugin.ts";
+import type { Detector, Finding, Lexicon, ProseDocument, Sentence } from "../plugin.ts";
+import { entryIn, entryOpens } from "./lexicon-match.ts";
 
 const PER = 1000;
 
@@ -10,11 +10,8 @@ const FLOOR = { word: 200, char: 500 };
 
 type Hit = { readonly sentence: Sentence; readonly matched: string };
 
-const hitsFor = (doc: ProseDocument, patterns: readonly string[]): Hit[] =>
-  doc.sentences.flatMap((sentence) => {
-    const text = sentence.text.toLowerCase();
-    return patterns.filter((pattern) => text.includes(pattern.toLowerCase())).map((matched) => ({ sentence, matched }));
-  });
+const hitsFor = (doc: ProseDocument, lexicon: Lexicon): Hit[] =>
+  doc.sentences.flatMap((sentence) => lexicon.filter((entry) => entryIn(sentence, entry)).map((entry) => ({ sentence, matched: entry.pattern })));
 
 /**
  * 単位長あたりの出現率。件数で数えると長い文書ほど当たる（bold-density と同じ）。
@@ -23,8 +20,7 @@ const hitsFor = (doc: ProseDocument, patterns: readonly string[]): Hit[] =>
 const densityRule =
   (rule: string): Detector =>
   (doc, options): Finding[] => {
-    const patterns = (options.lexicon ?? []).map((entry) => entry.pattern);
-    const hits = hitsFor(doc, patterns);
+    const hits = hitsFor(doc, options.lexicon ?? []);
     const length = wordsOf(doc);
     const rate = length === 0 ? 0 : Math.round((hits.length / length) * PER);
     const first = hits[0];
@@ -49,8 +45,7 @@ export const cushionDensity = densityRule("cushion-phrase-density");
 const QUALIFIER = /\d|より|に比べ|のうち|among|than|compared|based on|according/iu;
 
 export const unqualifiedSuperlative: Detector = (doc, options): Finding[] => {
-  const patterns = (options.lexicon ?? []).map((entry) => entry.pattern);
-  const bare = hitsFor(doc, patterns).filter((hit) => !QUALIFIER.test(hit.sentence.text));
+  const bare = hitsFor(doc, options.lexicon ?? []).filter((hit) => !QUALIFIER.test(hit.sentence.text));
   if (bare.length < options.limit) return [];
   return bare.map((hit) => ({
     rule: "unqualified-superlative",
@@ -62,10 +57,10 @@ export const unqualifiedSuperlative: Detector = (doc, options): Finding[] => {
   }));
 };
 
-const openerOf = (doc: ProseDocument, patterns: readonly string[]): (string | undefined)[] =>
+const openerOf = (doc: ProseDocument, lexicon: Lexicon): (string | undefined)[] =>
   doc.paragraphs.map((paragraph) => {
-    const head = paragraph.sentences[0]?.text.trim().toLowerCase() ?? "";
-    return patterns.find((pattern) => head.startsWith(pattern.toLowerCase()));
+    const head = paragraph.sentences[0];
+    return head === undefined ? undefined : lexicon.find((entry) => entryOpens(head, entry))?.pattern;
   });
 
 /**
@@ -73,8 +68,7 @@ const openerOf = (doc: ProseDocument, patterns: readonly string[]): (string | un
  * 前の段落の付け足しに見えて、話がどこへ向かっているのか分からなくなる。
  */
 export const repeatedConjunction: Detector = (doc, options): Finding[] => {
-  const patterns = (options.lexicon ?? []).map((entry) => entry.pattern);
-  const openers = openerOf(doc, patterns);
+  const openers = openerOf(doc, options.lexicon ?? []);
   const runs = openers.reduce<{ runs: number[][]; current: number[] }>(
     (acc, opener, index) => {
       if (opener === undefined) return { runs: [...acc.runs, acc.current], current: [] };
@@ -106,11 +100,10 @@ export const repeatedConjunction: Detector = (doc, options): Finding[] => {
  * 単独で断じない。§20.2 の複合シグナルの入口で、これだけで「AI が書いた」とは言わない。
  */
 export const aiTell: Detector = (doc, options): Finding[] => {
-  const weights = new Map((options.lexicon ?? []).map((entry) => [entry.pattern.toLowerCase(), entry.weight ?? 1]));
-  const body = doc.sentences.map(proseText).join(" ").toLowerCase();
-  const found = [...weights.entries()].filter(([pattern]) => body.includes(pattern));
-  const score = Math.round(found.reduce((sum, [, weight]) => sum + weight, 0) * 10);
-  const first = doc.sentences.find((sentence) => found.some(([pattern]) => sentence.text.toLowerCase().includes(pattern)));
+  const entries = [...new Map((options.lexicon ?? []).map((entry) => [entry.pattern.toLowerCase(), entry])).values()];
+  const found = entries.filter((entry) => doc.sentences.some((sentence) => entryIn(sentence, entry)));
+  const score = Math.round(found.reduce((sum, entry) => sum + (entry.weight ?? 1), 0) * 10);
+  const first = doc.sentences.find((sentence) => found.some((entry) => entryIn(sentence, entry)));
   if (first === undefined || score <= options.limit) return [];
   return [
     {
@@ -121,7 +114,7 @@ export const aiTell: Detector = (doc, options): Finding[] => {
       quote: first.text.trim(),
       values: {
         word: joinWords(
-          found.map(([pattern]) => pattern),
+          found.map((entry) => entry.pattern.toLowerCase()),
           doc.language,
         ),
         count: found.length,

package/src/detectors/phrase-match.ts

@@ -1,4 +1,5 @@
-import type { Detector, Finding, ProseDocument, Sentence } from "../plugin.ts";
+import type { Detector, Finding, Lexicon, ProseDocument, Sentence } from "../plugin.ts";
+import { entryIn } from "./lexicon-match.ts";
 
 /** 冒頭は先頭 2 段落ぶん。「どの記事にも当てはまる書き出し」は冒頭にあるときだけ問題。 */
 const OPENING_SENTENCES = 4;
@@ -15,10 +16,10 @@ const inScope = (doc: ProseDocument, where: string | undefined): readonly Senten
 
 type Hit = { readonly sentence: Sentence; readonly matched: string };
 
-const hitsIn = (sentences: readonly Sentence[], patterns: readonly string[]): Hit[] =>
+const hitsIn = (sentences: readonly Sentence[], lexicon: Lexicon): Hit[] =>
   sentences.flatMap((sentence) => {
-    const matched = patterns.find((pattern) => sentence.text.toLowerCase().includes(pattern.toLowerCase()));
-    return matched === undefined ? [] : [{ sentence, matched }];
+    const matched = lexicon.find((entry) => entryIn(sentence, entry));
+    return matched === undefined ? [] : [{ sentence, matched: matched.pattern }];
   });
 
 /**
@@ -26,9 +27,9 @@ const hitsIn = (sentences: readonly Sentence[], patterns: readonly string[]): Hi
  * これが L2 の全体像で、新しい言語は語彙表を書くだけで動く。
  */
 export const phraseMatch: Detector = (doc, options): Finding[] => {
-  const patterns = (options.lexicon ?? []).map((entry) => entry.pattern);
-  if (patterns.length === 0) return [];
-  const hits = hitsIn(inScope(doc, options.where), patterns);
+  const lexicon = options.lexicon ?? [];
+  if (lexicon.length === 0) return [];
+  const hits = hitsIn(inScope(doc, options.where), lexicon);
   if (hits.length < options.limit) return [];
   return hits.map((hit) => ({
     rule: "",

package/src/detectors/repeated-head.ts

@@ -1,4 +1,4 @@
-import type { Detector, Finding, LengthUnit, Sentence } from "../plugin.ts";
+import type { Detector, Finding, LengthUnit, Sentence, Span } from "../plugin.ts";
 
 const HEAD_CHARS = 6;
 const HEAD_WORDS = 3;
@@ -13,22 +13,36 @@ const headOf = (sentence: Sentence, unit: LengthUnit): string => {
   return text.split(/\s+/u).slice(0, HEAD_WORDS).join(" ");
 };
 
-type Run = { readonly head: string; readonly members: readonly Sentence[] };
+/** 文の入れ物。文の始まりを含む、いちばん内側の箇条書きの項目の始まり。地の文なら -1。 */
+const containerOf = (sentence: Sentence, lists: readonly Span[]): number =>
+  lists.filter((item) => item.start <= sentence.span.start && sentence.span.start < item.end).reduce((innermost, item) => Math.max(innermost, item.start), -1);
 
-const runsOf = (sentences: readonly Sentence[], unit: LengthUnit): Run[] =>
+type Run = { readonly head: string; readonly container: number; readonly members: readonly Sentence[] };
+
+/** 前の文と今の文のあいだで、箇条書きの項目が始まっているか。文を持たない項目（画像だけの - ![図](a.png)）も連なりを切る。 */
+const listBetween = (previous: Sentence | undefined, sentence: Sentence, lists: readonly Span[]): boolean =>
+  previous !== undefined && lists.some((item) => item.start >= previous.span.end && item.start < sentence.span.start);
+
+/**
+ * 連なりは同じ入れ物の中でだけ続く。箇条書きの項目どうしは同じ形で並べるのが書き方そのもの（「1. SRE に関する…」「2. SRE に関する…」）で、
+ * 項目をまたぐと切る。一つの項目の中の段落や、地の文の中では、これまでどおり数える。箇条書きの前後の地の文も、箇条書きで切れる。
+ */
+const runsOf = (sentences: readonly Sentence[], unit: LengthUnit, lists: readonly Span[]): Run[] =>
   sentences.reduce<Run[]>((acc, sentence) => {
     const head = headOf(sentence, unit);
+    const container = containerOf(sentence, lists);
     const last = acc.at(-1);
     // 短すぎる書き出しは「同じ」と言えない。単位ぶん揃って初めて連なりと見る。
     const full = unit === "char" ? head.length === HEAD_CHARS : head.split(" ").length === HEAD_WORDS;
-    if (last !== undefined && last.head === head && full) {
-      return [...acc.slice(0, -1), { head, members: [...last.members, sentence] }];
+    const joins = last !== undefined && last.head === head && last.container === container && !listBetween(last.members.at(-1), sentence, lists);
+    if (last !== undefined && joins && full) {
+      return [...acc.slice(0, -1), { head, container, members: [...last.members, sentence] }];
     }
-    return [...acc, { head, members: [sentence] }];
+    return [...acc, { head, container, members: [sentence] }];
   }, []);
 
 export const repeatedHead: Detector = (doc, options): Finding[] =>
-  runsOf(doc.sentences, doc.lengthUnit)
+  runsOf(doc.sentences, doc.lengthUnit, doc.listSpans)
     .filter((run) => run.members.length > options.limit)
     .map((run) => ({
       rule: "repeated-sentence-head",

package/src/detectors/sentence-ending.ts

@@ -1,4 +1,4 @@
-import { hasPredicate, isClosed } from "../sentence-shape.ts";
+import { endingTokens, hasPredicate, isClosed } from "../sentence-shape.ts";
 import type { Detector, Finding, Sentence } from "../plugin.ts";
 
 /**
@@ -11,10 +11,14 @@ import type { Detector, Finding, Sentence } from "../plugin.ts";
  * 述語を持たないものは文として数えない。見出しの下の `MaaSサービス` のような
  * 名前だけの行がそのまま「である調」の少数派になり、実文書の誤検知はすべてこれだった。
  */
-const isSentence = (sentence: Sentence): boolean => isClosed(sentence) && hasPredicate(sentence);
+const isSentence = (sentence: Sentence): boolean => isClosed(sentence) && hasPredicate(sentence) && endingTokens(sentence).length > 0;
 
+/**
+ * 文末の語だけを見る。文の途中の引用（「ご覧ください」という表現を使う。）は文末の調子ではない。
+ * 書いた形でも原形でも当てる。「ください」の原形は「くださる」で、原形だけを見ると丁寧な文末を見落とす。
+ */
 const isPolite = (sentence: Sentence, polite: readonly string[]): boolean =>
-  (sentence.tokens ?? []).some((token) => polite.includes(token.lemma ?? token.surface));
+  endingTokens(sentence).some((token) => polite.includes(token.surface) || (token.lemma !== undefined && polite.includes(token.lemma)));
 
 export const sentenceEnding: Detector = (doc, options): Finding[] => {
   const polite = (options.lexicon ?? []).map((entry) => entry.pattern);