chaffjs 0.16.0 → 0.17.0

package/src/detectors/agreement-slip.ts

@@ -0,0 +1,191 @@
+import type { Detector, Finding, Lexicon, Token } from "../plugin.ts";
+import { isPartOfLongerWord } from "./doubled-word.ts";
+
+/**
+ * 語の形が合っていない（a significant changes / one of the most important feature / Your can check）。
+ * 書き換えで片方の語だけ直した跡。解析器の読みが一通りに決まる形だけを数える。語はすべてアダプタの語彙表から来る。
+ */
+
+export type AgreementLists = {
+  /** 一つを数える限定詞（a / each）。 */
+  readonly singular: Lexicon;
+  /** 複数を数える限定詞と句（these / one of the）。 */
+  readonly plural: Lexicon;
+  /** 単数と複数が同じ綴りの名詞（series / data）。 */
+  readonly invariant: ReadonlySet<string>;
+  /** 単数の限定詞で複数を数える形容詞（a few days）。 */
+  readonly count: ReadonlySet<string>;
+  /** 名詞の前にしか立たない所有の語（your / their）。 */
+  readonly possessive: ReadonlySet<string>;
+  /** 名詞にならない be / have の形（is / has）。 */
+  readonly finite: ReadonlySet<string>;
+  /** 単数で書く所を複数で書いた句（ones of the most）。 */
+  readonly misnumbered: Lexicon;
+};
+
+export type Slip = { readonly variant: "number" | "possessive"; readonly first: Token; readonly last: Token; readonly word: string };
+
+const SPACE_ONLY = /^\s+$/u;
+const UPPER_START = /^\p{Lu}/u;
+const LETTER = /\p{L}/u;
+const LOWER = /\p{Ll}/u;
+/** 名詞句を閉じる記号。読点・括弧の始まり・ハイフンは、名詞を重ねた語（a sales, marketing and support team）の途中にも立つ。 */
+const CLOSING_MARK = new Set([".", ";", ":", "?", "!", ")"]);
+/**
+ * 名詞の後ろに来れば、名詞句はそこで終わっている品詞。名詞・形容詞・接続詞・所有の印は、名詞を重ねた語の途中。
+ * 動詞は、名詞にならない be / have の形（is / has）だけ。解析器は複数の名詞の後ろの名詞を動詞と読む（a sales team の team が VBP）。
+ */
+const PHRASE_END = new Set(["AUX", "ADP", "SCONJ", "PRON", "DET", "ADV"]);
+const PREDICATE_TAIL = new Set(["DET", "ADJ", "PRON", "NUM"]);
+
+const lower = (token: Token): string => token.surface.toLowerCase();
+
+const isParticiple = (token: Token): boolean => token.features?.["VerbForm"] === "Part" || token.features?.["VerbForm"] === "Ger";
+
+const isArticle = (token: Token): boolean => token.features?.["PronType"] === "Art";
+
+const isPlural = (token: Token): boolean => token.features?.["Number"] === "Plur";
+
+/** 名詞の後ろの語で、名詞句が終わるか。分詞（a data driven approach）も名詞を重ねた語の途中。 */
+export const endsNounPhrase = (next: Token | undefined, finite: ReadonlySet<string>): boolean => {
+  if (next === undefined || finite.has(lower(next))) return true;
+  return next.pos === "PUNCT" ? CLOSING_MARK.has(next.surface) : PHRASE_END.has(next.pos);
+};
+
+const wordsOf = (entry: Lexicon[number]): readonly string[] =>
+  entry.tokens === undefined ? entry.pattern.toLowerCase().split(/\s+/u) : entry.tokens.map((token) => token.surface.toLowerCase());
+
+/** tokens の at 語目から始まる語彙表の句の長さ。無ければ 0。 */
+const phraseAt = (tokens: readonly Token[], at: number, phrases: Lexicon): number => {
+  const matched = phrases.map(wordsOf).find((words) => words.every((word, k) => tokens[at + k]?.surface.toLowerCase() === word));
+  return matched?.length ?? 0;
+};
+
+/** first から last まで、語のあいだが空白だけ。コードの印やリンクを挟めば別々の語。 */
+const joinedBySpace = (source: string, tokens: readonly Token[]): boolean =>
+  tokens.slice(1).every((token, k) => SPACE_ONLY.test(source.slice(tokens[k]?.span.end ?? token.span.start, token.span.start)));
+
+const opensSentence = (tokens: readonly Token[], at: number): boolean => !tokens.slice(0, at).some((token) => LETTER.test(token.surface));
+
+/** 文の途中の大文字の限定詞は、名前の一部（Plan A users）。 */
+const isNamePart = (tokens: readonly Token[], at: number): boolean => UPPER_START.test(tokens[at]?.surface ?? "") && !opensSentence(tokens, at);
+
+const isModifier = (token: Token): boolean => token.pos === "ADJ" || token.pos === "ADV" || (token.pos === "VERB" && isParticiple(token));
+
+/** 限定詞の後ろの修飾語。数の形容詞（a few days）が挟まれば、単数の限定詞でも複数を数えるので undefined。 */
+const modifiersAfter = (tokens: readonly Token[], from: number, count: ReadonlySet<string>): readonly Token[] | undefined => {
+  const end = tokens.findIndex((token, index) => index >= from && !isModifier(token));
+  const modifiers = tokens.slice(from, end === -1 ? tokens.length : end);
+  return modifiers.some((token) => count.has(lower(token))) ? undefined : modifiers;
+};
+
+const isGuessed = (token: Token): boolean => token.features?.["Guess"] === "Yes";
+
+/** 単数と複数が同じ綴りの語、複数と付いても原形が同じ語（lens）、解析器が形から当てた語（stimuli）は、どちらの数とも決まらない。 */
+const hasOneNumber = (noun: Token, invariant: ReadonlySet<string>): boolean =>
+  !invariant.has(lower(noun)) && !isGuessed(noun) && !(isPlural(noun) && noun.lemma?.toLowerCase() === lower(noun));
+
+const isCommonNoun = (token: Token | undefined): token is Token => token?.pos === "NOUN" && !UPPER_START.test(token.surface);
+
+type Determiner = { readonly head: Token; readonly length: number; readonly plural: boolean };
+
+const determinerAt = (tokens: readonly Token[], at: number, lists: AgreementLists): Determiner | undefined => {
+  const head = tokens[at];
+  const plural = phraseAt(tokens, at, lists.plural);
+  const singular = phraseAt(tokens, at, lists.singular);
+  if (head === undefined || plural + singular === 0) return undefined;
+  return plural > 0 ? { head, length: plural, plural: true } : { head, length: singular, plural: false };
+};
+
+/**
+ * 冠詞と句でない限定詞（this / these）は代名詞にもなる。This results in a loss の results は動詞なので、
+ * 形容詞を挟んだ名詞（this new rules）だけを数える。
+ */
+const standsAlone = (determiner: Determiner): boolean => determiner.length === 1 && !isArticle(determiner.head);
+
+const readsAsDeterminer = (determiner: Determiner, modifiers: readonly Token[]): boolean =>
+  !standsAlone(determiner) || modifiers.some((token) => token.pos === "ADJ");
+
+const mayBeNoun = (token: Token): boolean => token.features?.["AlsoNoun"] === "Yes" || isGuessed(token);
+
+/**
+ * 名詞が動詞にも読め（works）、その前に名詞句の頭になれる語があれば、限定詞と名詞ではなく主語と動詞かもしれない。
+ * an individual works（individual は名詞にもなる）/ Those responsible report to the board（those は代名詞にもなる）。
+ * 後ろが is / has なら、名詞は動詞ではない（these key result is）。
+ */
+const mayBeVerb = (determiner: Determiner, modifiers: readonly Token[], noun: Token, next: Token | undefined, finite: ReadonlySet<string>): boolean =>
+  noun.features?.["AlsoVerb"] === "Yes" && !(next !== undefined && finite.has(lower(next))) && (standsAlone(determiner) || modifiers.some(mayBeNoun));
+
+const numberSlipAt = (source: string, tokens: readonly Token[], at: number, lists: AgreementLists): Slip | undefined => {
+  const determiner = determinerAt(tokens, at, lists);
+  if (determiner === undefined || isNamePart(tokens, at)) return undefined;
+  const modifiers = modifiersAfter(tokens, at + determiner.length, lists.count);
+  if (modifiers === undefined || !readsAsDeterminer(determiner, modifiers)) return undefined;
+  const nounAt = at + determiner.length + modifiers.length;
+  const noun = tokens[nounAt];
+  if (!isCommonNoun(noun) || !hasOneNumber(noun, lists.invariant) || isPlural(noun) === determiner.plural) return undefined;
+  if (mayBeVerb(determiner, modifiers, noun, tokens[nounAt + 1], lists.finite)) return undefined;
+  const span = tokens.slice(at, nounAt + 1);
+  if (!endsNounPhrase(tokens[nounAt + 1], lists.finite) || !joinedBySpace(source, span) || isPartOfLongerWord(source, determiner.head, noun)) return undefined;
+  return slipOf("number", source, determiner.head, noun);
+};
+
+const slipOf = (variant: Slip["variant"], source: string, first: Token, last: Token): Slip => ({
+  variant,
+  first,
+  last,
+  word: source.slice(first.span.start, last.span.end).replace(/\s+/gu, " "),
+});
+
+const isBaseVerb = (token: Token | undefined): boolean => token?.pos === "VERB" && token.features?.["VerbForm"] === undefined;
+
+/** 所有の語の直後の be（their is）か、助動詞 + 動詞の原形（Your can check）。名詞にもなる助動詞（your will）は動詞が続くときだけ。 */
+const possessiveSlipAt = (source: string, tokens: readonly Token[], at: number, lists: AgreementLists): Slip | undefined => {
+  const [possessive, verb] = [tokens[at], tokens[at + 1]];
+  if (possessive === undefined || verb === undefined || !lists.possessive.has(lower(possessive))) return undefined;
+  // 大文字だけの語は略語（Your AM = account manager）。
+  const verbal = LOWER.test(verb.surface) && (lists.finite.has(lower(verb)) || (verb.pos === "AUX" && isBaseVerb(tokens[at + 2])));
+  if (!verbal || !joinedBySpace(source, [possessive, verb]) || isPartOfLongerWord(source, possessive, verb)) return undefined;
+  return slipOf("possessive", source, possessive, verb);
+};
+
+/** ones of the most。前に限定詞・形容詞・代名詞があれば、ones は代名詞（the ones of the most use）。 */
+const misnumberedAt = (source: string, tokens: readonly Token[], at: number, lists: AgreementLists): Slip | undefined => {
+  const span = tokens.slice(at, at + phraseAt(tokens, at, lists.misnumbered));
+  const [first, last, previous] = [span[0], span.at(-1), tokens[at - 1]];
+  if (first === undefined || last === undefined || (previous !== undefined && PREDICATE_TAIL.has(previous.pos))) return undefined;
+  return joinedBySpace(source, span) ? slipOf("number", source, first, last) : undefined;
+};
+
+export const agreementIn = (source: string, tokens: readonly Token[], lists: AgreementLists): Slip[] =>
+  tokens.flatMap((_, at) => {
+    const slip = numberSlipAt(source, tokens, at, lists) ?? possessiveSlipAt(source, tokens, at, lists) ?? misnumberedAt(source, tokens, at, lists);
+    return slip === undefined ? [] : [slip];
+  });
+
+const setOf = (lexicon: Lexicon | undefined): ReadonlySet<string> => new Set((lexicon ?? []).map((entry) => entry.pattern.toLowerCase()));
+
+const listsOf = (lexicons: Readonly<Record<string, Lexicon>>): AgreementLists => ({
+  singular: lexicons["singular-determiner"] ?? [],
+  plural: lexicons["plural-determiner"] ?? [],
+  invariant: setOf(lexicons["invariant-noun"]),
+  count: setOf(lexicons["count-adjective"]),
+  possessive: setOf(lexicons["dependent-possessive"]),
+  finite: setOf(lexicons["finite-auxiliary"]),
+  misnumbered: lexicons["misnumbered-phrase"] ?? [],
+});
+
+export const agreementSlip: Detector = (doc): Finding[] => {
+  const lists = listsOf(doc.lexicons);
+  return doc.sentences.flatMap((sentence) =>
+    agreementIn(doc.source, sentence.tokens ?? [], lists).map((slip): Finding => ({
+      rule: "agreement-slip",
+      severity: "warning",
+      line: 0,
+      column: 0,
+      quote: sentence.text.trim(),
+      values: { word: slip.word, offset: slip.first.span.start },
+      variant: slip.variant,
+    })),
+  );
+};

package/src/detectors/content-morphemes.ts

@@ -0,0 +1,19 @@
+// What a sentence adds to a heading, counted in content morphemes: words that stand by themselves and name something.
+// Pure; heading-echo measures a language written without spaces with it, where a character count says nothing
+// (「会社に勤めています」 carries two new words, 「について説明します」 one, in the same nine characters).
+import type { Token } from "../plugin.ts";
+
+/** 代名詞・連体詞・接続詞は、すでに書いたものを指すかつなぐだけで、中身を足さない。 */
+const CONTENT_POS = new Set(["NOUN", "PROPN", "NUM", "VERB", "ADJ", "ADV"]);
+
+/** 接頭詞・接尾・非自立の語（Bound）と、動詞にするだけの「する」（VerbType=Light）は、付いた語が中身を持つ。 */
+const isContent = (token: Token): boolean => CONTENT_POS.has(token.pos) && token.features?.["Bound"] !== "Yes" && token.features?.["VerbType"] !== "Light";
+
+/** 原形で照らす（変えました → 変える）。原形の無い語は書いた形で。 */
+const baseForm = (token: Token): string => (token.lemma ?? token.surface).toLowerCase();
+
+/** The sentence's content morphemes whose base form is not one of the heading's content morphemes, every occurrence counted. */
+export const newContentMorphemes = (heading: readonly Token[], sentence: readonly Token[]): number => {
+  const said = new Set(heading.filter(isContent).map(baseForm));
+  return sentence.filter((token) => isContent(token) && !said.has(baseForm(token))).length;
+};

package/src/detectors/en-shape.ts

@@ -134,272 +134,3 @@ export const titleCaseMix: Detector = (doc, options): Finding[] => {
     values: { count: few.length, limit: options.limit, offset: section.span.start },
   }));
 };
-
-const LIST_CONJUNCTION = new Set(["and", "or"]);
-
-const isComma = (token: Token): boolean => token.surface === ",";
-
-const commaBefore = (tokens: readonly Token[], at: number): boolean => tokens[at - 1]?.surface === ",";
-
-/** 並列はこれをまたがない。セミコロンの前後は別の節。 */
-const CLAUSE_BREAK = new Set([";", ":", "—"]);
-
-/** 各 token の前で閉じていない括弧の数。括弧の中の読点（external users (e.g., guests), and ...）は外の並列を切らない。 */
-const PAREN_STEP: Readonly<Record<string, number>> = { "(": 1, ")": -1 };
-
-const depthsOf = (tokens: readonly Token[]): number[] =>
-  tokens.reduce<{ depths: number[]; open: number }>(
-    (acc, token) => ({ depths: [...acc.depths, acc.open], open: Math.max(0, acc.open + (PAREN_STEP[token.surface] ?? 0)) }),
-    { depths: [], open: 0 },
-  ).depths;
-
-/** participleWords は、解析器が分詞と読まないが読点のあとで分詞の句を始める語（meaning）。 */
-type Clause = { readonly tokens: readonly Token[]; readonly depths: readonly number[]; readonly participleWords: ReadonlySet<string> };
-
-/** 冠詞や引用符を飛ばした、項目の頭の品詞。the parser と an exporter と samples を同じ形と見る。 */
-const NOMINAL = new Set(["NOUN", "PROPN", "PRON", "NUM", "ADJ"]);
-
-const isContent = (token: Token): boolean => token.pos !== "DET" && token.pos !== "PUNCT" && token.pos !== "X";
-
-const shapeOf = (item: readonly Token[]): string | undefined => {
-  const head = item.find(isContent);
-  if (head === undefined) return undefined;
-  return NOMINAL.has(head.pos) ? "NOMINAL" : head.pos;
-};
-
-const VERBAL = new Set(["VERB", "AUX"]);
-
-/** 主語と述語のある項目。the team fixed the bug / these are crucial。gets us more は述語だけ。 */
-const isClause = (item: readonly Token[]): boolean => {
-  const first = item.find((token) => token.pos !== "PUNCT" && token.pos !== "X");
-  if (first === undefined || !(first.pos === "DET" || NOMINAL.has(first.pos))) return false;
-  return item.some((token) => token !== first && VERBAL.has(token.pos));
-};
-
-/** 節の頭から and / or の手前までを、同じ深さの読点で項目に切る。Oxford comma の読点のあとは空なので項目にならない。 */
-const itemsBefore = (clause: Clause, at: number): Token[][] => {
-  const level = clause.depths[at] ?? 0;
-  const start = clause.tokens.slice(0, at).findLastIndex((token) => CLAUSE_BREAK.has(token.surface)) + 1;
-  return clause.tokens
-    .slice(start, at)
-    .reduce<Token[][]>(
-      (items, token, offset) => {
-        if (isComma(token) && clause.depths[start + offset] === level) return [...items, []];
-        return [...items.slice(0, -1), [...(items.at(-1) ?? []), token]];
-      },
-      [[]],
-    )
-    .filter((item) => item.length > 0);
-};
-
-/**
- * and / or で始まる項目で、前の並びは閉じている（offering, giving, or receiving, directly or indirectly）。
- * 次の並びはその項目から始まる。項目が動詞で始まれば、その目的語から（Develop, maintain, and track courses, materials and events）。
- */
-const afterClosedList = (items: readonly Token[][]): Token[][] => {
-  const closed = items.findLastIndex((item) => LIST_CONJUNCTION.has(openingOf(item)?.surface.toLowerCase() ?? ""));
-  const closing = items[closed];
-  if (closing === undefined) return [...items];
-  const body = closing.slice(closing.findIndex((token) => LIST_CONJUNCTION.has(token.surface.toLowerCase())) + 1);
-  const head = body.findIndex(isContent);
-  const rest = VERBAL.has(body[head]?.pos ?? "") ? body.slice(head + 1) : body;
-  // 項目が残らなければ、後ろは新しい句の頭。導入の句（directly or indirectly,）を外す。
-  return rest.some(isContent) ? [rest, ...items.slice(closed + 1)] : [...withoutLead(items.slice(closed + 1))];
-};
-
-/** and / or の後ろの項目。次の読点か節の切れ目まで。 */
-const itemAfter = (tokens: readonly Token[], at: number): Token[] => {
-  const end = tokens.findIndex((token, index) => index > at && (isComma(token) || CLAUSE_BREAK.has(token.surface)));
-  return tokens.slice(at + 1, end === -1 ? undefined : end);
-};
-
-/**
- * 導入の句（After the review, / If it fails, / Finally, / To test it, / Based on the review,）は並列の項目ではない。
- * 節の最初の項目が前置詞・接続詞・副詞・to・過去分詞で始まり、次の項目と頭の形が違えば外す。
- * Quickly, quietly and carefully は残る。For managers, engineers and designers, も外れるが、前置詞のあとの名詞を
- * 項目と読むと、Over this period, the subcommittees and the full committee ... を並びと誤る。こちらのほうが多い。
- */
-const LEAD_POS = new Set(["ADP", "SCONJ", "ADV", "PART"]);
-
-const isParticiple = (token: Token): boolean => token.features?.["VerbForm"] === "Part";
-
-const openingOf = (item: readonly Token[]): Token | undefined => item.find((token) => token.pos !== "PUNCT" && token.pos !== "X");
-
-const openingKind = (item: readonly Token[]): string | undefined => {
-  const opening = openingOf(item);
-  return opening !== undefined && isParticiple(opening) ? "PARTICIPLE" : shapeOf(item);
-};
-
-const withoutLead = (items: readonly Token[][]): readonly Token[][] => {
-  const [first, second] = items;
-  const opening = first === undefined ? undefined : openingOf(first);
-  if (first === undefined || second === undefined || opening === undefined) return items;
-  if (!LEAD_POS.has(opening.pos) && !isParticiple(opening)) return items;
-  return openingKind(first) === openingKind(second) ? items : items.slice(1);
-};
-
-/**
- * 形容詞のあとの読点は、名詞の前で形容詞を重ねているだけのことが多い（the long, winding bridge）。
- * 次の項目が名詞で終わるならつなげ直す。形容詞そのものの並び（quick, cheap and reliable）は切ったまま。
- */
-const lastContent = (item: readonly Token[]): Token | undefined => item.findLast(isContent);
-
-const joinAdjectives = (items: readonly Token[][]): Token[][] =>
-  items.reduce<Token[][]>((joined, item) => {
-    const previous = joined.at(-1);
-    const stacked = previous !== undefined && lastContent(previous)?.pos === "ADJ" && lastContent(item)?.pos !== "ADJ";
-    if (stacked) joined[joined.length - 1] = [...previous, ...item];
-    else joined.push(item);
-    return joined;
-  }, []);
-
-/**
- * 節を並べるなら、どの項目も節。Additionally, others can learn, and the mistake is rarer. の
- * Additionally は項目ではなく、and の前の読点は節をつなぐ読点。名詞の並びは最初の項目に前置き
- * （We shipped the parser）を抱えるので、この確かめは節の並びだけにする。
- * and の後ろだけが節なのは主語の並び（The parser, the renderer and the exporter shipped.）なので外さない。
- */
-const clausesAgree = (items: readonly Token[][], after: readonly Token[]): boolean => {
-  const last = items.at(-1);
-  if (last === undefined || !isClause(last)) return true;
-  return isClause(after) && items.every(isClause);
-};
-
-/**
- * 動詞で始まる項目を並べるなら、どの項目にも動詞がある。The scope, in contrast, is larger, and covers ... の
- * The scope は項目ではなく、後ろの述語の主語。
- */
-const hasVerb = (item: readonly Token[]): boolean => item.some((token) => VERBAL.has(token.pos));
-
-const predicatesAgree = (items: readonly Token[][], shape: string): boolean => !VERBAL.has(shape) || items.every(hasVerb);
-
-/** -ing 形（VerbForm=Ger）と過去分詞（Part）。filling the gaps と sentenced to prison は並べても並列にならない。 */
-const VERB_FORMS = new Set(["Ger", "Part"]);
-
-const verbFormOf = (token: Token): string | undefined => {
-  const form = token.features?.["VerbForm"];
-  return form !== undefined && VERB_FORMS.has(form) ? form : undefined;
-};
-
-/** 項目の頭の分詞とその形。副詞は飛ばす（originally written）。引用符の中の語（‘modelling’）は語の例なので分詞と読まない。 */
-const participleOpening = (item: readonly Token[], words: ReadonlySet<string>): { readonly at: number; readonly form: string } | undefined => {
-  const at = item.findIndex((token) => token.pos !== "PUNCT" && token.pos !== "ADV");
-  const opening = item[at];
-  if (opening === undefined) return undefined;
-  const form = words.has(opening.surface.toLowerCase()) ? "Ger" : verbFormOf(opening);
-  return form === undefined ? undefined : { at, form };
-};
-
-const hasParticiple = (item: readonly Token[], form: string): boolean => item.some((token) => verbFormOf(token) === form);
-
-/**
- * 最初より後ろの項目が分詞で始まるとき。最初の項目にも同じ形の分詞があれば分詞の並び
- * （are hiring engineers, buying laptops and renting desks / Tested, reviewed and approved）。後ろの項目が動詞で始まれば述語の並びで、
- * 分詞は解析器の読み違い（the jury, cross-examined the witnesses, presented witnesses, and made）。そうでなければ分詞の句
- * （from Latin, meaning ship or boat / the judge, sitting without a jury, and sentenced）で、並びがあるならその句の中から始まる。
- * 分詞より前の項目と分詞そのものは項目ではない（programs, including the office, the desk and the team）。句の中の項目の形が揃わなければ
- * 句は挿入で、並びは句の後ろだけ（relief, including an injunction, in any court ... and without / including before the date, to a
- * Recipient ... “confidential”, “proprietary”, or the like）。
- */
-/** 解析器は過去形と過去分詞（fixed）を見分けられない。動詞のある項目のあとの「過去分詞」の並びは、述語の並びと読む。 */
-const pastPredicates = (first: readonly Token[], form: string, after: readonly Token[], words: ReadonlySet<string>): boolean =>
-  form === "Part" && participleOpening(after, words)?.form === "Part" && hasVerb(first);
-
-const fromParticiplePhrase = (items: readonly Token[][], after: readonly Token[], words: ReadonlySet<string>): readonly Token[][] => {
-  const at = items.findIndex((item, index) => index > 0 && participleOpening(item, words) !== undefined);
-  const phrase = items[at];
-  const first = items[0];
-  const opening = phrase === undefined ? undefined : participleOpening(phrase, words);
-  if (first === undefined || phrase === undefined || opening === undefined) return items;
-  const rest = items.slice(at + 1);
-  const predicates = rest.some((item) => VERBAL.has(shapeOf(item) ?? "")) || pastPredicates(first, opening.form, after, words);
-  if (hasParticiple(first, opening.form) || predicates) return items;
-  const inPhrase = withoutLead([phrase.slice(opening.at + 1), ...rest].filter((item) => item.some(isContent)));
-  return new Set(inPhrase.map(shapeOf)).size > 1 ? rest : inPhrase;
-};
-
-const APPOSITIVE_ANCHOR = new Set(["NOUN", "PROPN"]);
-
-/**
- * 名詞のすぐ後ろの、読点で挟まれた X and Y は同格（two HPV types, HPV16 and HPV18, that account / Governments, both state and
- * federal, spend）。項目が 2 つに見え、and の前に読点が無く、and の後ろの項目が読点で閉じ、後ろの 2 つに動詞が無いときに同格と読む。
- * 2 つを結ぶ同格に Oxford comma は打たないので、and の前に読点があれば 3 つの並び（the subcommittee, full committee, and
- * chamber levels, as well as）。読点の無い本当の並び（apples, pears and plums, then ...）はこの形と見分けられず、判定から外れる。
- */
-const isAppositive = (clause: Clause, at: number, items: readonly Token[][], after: readonly Token[]): boolean => {
-  const [anchor, head] = items;
-  if (items.length !== 2 || anchor === undefined || head === undefined || commaBefore(clause.tokens, at)) return false;
-  const end = at + 1 + after.length;
-  // 閉じた読点のあとに and / or が続くなら、並びのあとに節をつないでいる（apples, pears and plums, and went home）。
-  const joinsClause = LIST_CONJUNCTION.has(clause.tokens[end + 1]?.surface.toLowerCase() ?? "");
-  const closed = clause.tokens[end]?.surface === "," && clause.depths[end] === clause.depths[at] && !joinsClause;
-  return closed && APPOSITIVE_ANCHOR.has(lastContent(anchor)?.pos ?? "") && !hasVerb(head) && !hasVerb(after);
-};
-
-/** 副詞を飛ばした頭の形。explain and justify と ultimately ensure は同じ動詞の項目。 */
-const leadShape = (item: readonly Token[]): string | undefined => shapeOf(item.filter((token) => token.pos !== "ADV"));
-
-/**
- * and の後ろの項目に読点と、同じ形の項目を連れた and / or が続くなら、この and は項目の中にある（searches and seizures, and
- * the Eighth's ban）。頭の形か、節かどうかが違えば別の節をつなぐ and で、この and までが並び（apples, pears and plums, and went home / and figs fell）。
- */
-const listContinues = (clause: Clause, at: number, after: readonly Token[]): boolean => {
-  const next = at + 1 + after.length;
-  if (clause.tokens[next]?.surface !== "," || !LIST_CONJUNCTION.has(clause.tokens[next + 1]?.surface.toLowerCase() ?? "")) return false;
-  const following = itemAfter(clause.tokens, next + 1);
-  const shape = leadShape(following);
-  return shape !== undefined && shape === leadShape(after) && isClause(following) === isClause(after);
-};
-
-/**
- * 読点で区切った項目が and / or の前に 2 つ以上あり、最後の項目と and の後ろが同じ形のときだけ並列。
- * 導入の読点（After the review, the team fixed the bug and shipped it.）や、節をつなぐ読点
- * （We tested it, and the team shipped it.）は、Oxford comma を打つかどうかの選択を見せない。
- */
-const listAt = (clause: Clause, at: number): boolean | undefined => {
-  const after = itemAfter(clause.tokens, at);
-  const items = fromParticiplePhrase(afterClosedList(withoutLead(joinAdjectives(itemsBefore(clause, at)))), after, clause.participleWords);
-  if (listContinues(clause, at, after) || isAppositive(clause, at, items, after)) return undefined;
-  const last = items.at(-1);
-  const shape = last === undefined ? undefined : shapeOf(last);
-  if (items.length < 2 || shape === undefined || shape !== shapeOf(after)) return undefined;
-  if (!clausesAgree(items, after) || !predicatesAgree(items, shape)) return undefined;
-  return commaBefore(clause.tokens, at);
-};
-
-/**
- * 3 つ以上の並列の最後の and / or の前に読点を打つか。Oxford comma。
- *
- * どちらが正しいかは決めない。スタイルガイドで割れる論点に立場を取ると rule ごと無視される。
- * 見るのは 1 つの文書で揃っているかだけ。spec §12.3。
- */
-const oxfordIn = (tokens: readonly Token[], participleWords: ReadonlySet<string>): boolean | undefined => {
-  const clause = { tokens, depths: depthsOf(tokens), participleWords };
-  return tokens.reduce<boolean | undefined>(
-    (found, token, at) => found ?? (at > 0 && LIST_CONJUNCTION.has(token.surface.toLowerCase()) ? listAt(clause, at) : undefined),
-    undefined,
-  );
-};
-
-export const oxfordComma: Detector = (doc, options): Finding[] => {
-  const participleWords = new Set((doc.lexicons["participle-word"] ?? []).map((entry) => entry.pattern.toLowerCase()));
-  const judged = doc.sentences.flatMap((sentence) => {
-    const oxford = oxfordIn(sentence.tokens ?? [], participleWords);
-    return oxford === undefined ? [] : [{ sentence, oxford }];
-  });
-  const withComma = judged.filter((entry) => entry.oxford).length;
-  const minorityUsesComma = withComma <= judged.length - withComma;
-  const few = minorityUsesComma ? withComma : judged.length - withComma;
-  // 同数なら少数派は無い。
-  if (few === 0 || few * 2 === judged.length || few > options.limit) return [];
-  return judged
-    .filter((entry) => entry.oxford === minorityUsesComma)
-    .map(({ sentence }) => ({
-      rule: "oxford-comma-consistency",
-      severity: "info",
-      line: 0,
-      column: 0,
-      quote: sentence.text.trim(),
-      values: { count: few, limit: options.limit, offset: sentence.span.start },
-    }));
-};

package/src/detectors/heading-echo.ts

@@ -1,4 +1,6 @@
-import { lengthOf } from "../measure.ts";
+import { lengthOf, proseText } from "../measure.ts";
+import { newContentMorphemes } from "./content-morphemes.ts";
+import { echoedHeadingUnits, trigrams } from "./heading-overlap.ts";
 import { handsOver } from "./lead-in.ts";
 import { withoutQuotedVariants } from "./quoted-variant.ts";
 import type { Detector, Finding, LengthUnit, ProseDocument, Section } from "../plugin.ts";
@@ -6,17 +8,6 @@ import type { Detector, Finding, LengthUnit, ProseDocument, Section } from "../p
 /** 見出しが短すぎると、偶然の一致で 100% になる。これ未満の見出しは見ない。 */
 const MIN_GRAMS = 4;
 
-/**
- * 文字 3-gram で測る。
- * 語 n-gram にすると wordSplit capability が要り、L1（言語を問わず動く）から外れる。
- */
-const trigrams = (text: string): Set<string> => {
-  // 大文字小文字を畳む。英語では見出しが Title Case、本文が小文字になり、
-  // 同じ語でも一致しなくなる（Generating Output → generated output）。
-  const clean = [...text.toLowerCase().replace(/\s+/gu, "")];
-  return new Set(clean.slice(0, Math.max(0, clean.length - 2)).map((__char, index) => clean.slice(index, index + 3).join("")));
-};
-
 /**
  * Jaccard ではなく包含率を使う。
  *
@@ -39,18 +30,19 @@ const containment = (heading: Set<string>, sentence: Set<string>): number => {
  */
 const NEW_MATERIAL = { word: 6, char: 20 };
 
-const headingUnits = (heading: string, unit: LengthUnit): number =>
-  unit === "word"
-    ? heading
-        .trim()
-        .split(/\s+/u)
-        .filter((word) => word.length > 0).length
-    : heading.replace(/\s+/gu, "").length;
+/**
+ * 空白で語を分けない言語は、品詞が読めれば文字ではなく見出しに無い内容語で測る。文字数は中身の量を言わない:
+ * 「会社に勤めています」は 9 文字で 会社・勤める を足し、「について説明します」は同じ 9 文字で 説明 しか足さない。
+ * 内容語 2 つで足りる（こころさんのお父さんは、会社に勤めています。）。日本語の corpus で測った（yarn corpus）。
+ */
+const NEW_CONTENT_MORPHEMES = 1;
 
 const addsLittle = (section: Section, unit: LengthUnit): boolean => {
   const first = section.firstSentence;
   if (first === undefined) return false;
-  return lengthOf(first, unit) - headingUnits(section.heading, unit) <= NEW_MATERIAL[unit];
+  if (unit === "char" && first.tokens !== undefined && section.headingTokens !== undefined)
+    return newContentMorphemes(section.headingTokens, first.tokens) <= NEW_CONTENT_MORPHEMES;
+  return lengthOf(first, unit) - echoedHeadingUnits(section.heading, proseText(first), unit) <= NEW_MATERIAL[unit];
 };
 
 /** 最初の文が、同じ節の後ろ（箇条書き・表・コード）へ読者を渡している。 */

package/src/detectors/heading-overlap.ts

@@ -0,0 +1,65 @@
+// How much of a section's heading its first sentence takes up: the heading's character trigrams, and the part of the
+// heading the sentence repeats, in the document's length unit. Pure; heading-echo decides with these.
+import type { LengthUnit } from "../plugin.ts";
+
+const foldedChars = (text: string): string[] => [...text.toLowerCase().replace(/\s+/gu, "")];
+
+const trigramAt = (chars: readonly string[], index: number): string => chars.slice(index, index + 3).join("");
+
+/**
+ * 文字 3-gram で測る。
+ * 語 n-gram にすると wordSplit capability が要り、L1（言語を問わず動く）から外れる。
+ */
+export const trigrams = (text: string): Set<string> => {
+  // 大文字小文字を畳む。英語では見出しが Title Case、本文が小文字になり、
+  // 同じ語でも一致しなくなる（Generating Output → generated output）。
+  const clean = foldedChars(text);
+  return new Set(clean.slice(0, Math.max(0, clean.length - 2)).map((__char, index) => trigramAt(clean, index)));
+};
+
+/** A word as compared across heading and sentence: case folded, the punctuation around it dropped ("Spacewalks?" is spacewalks). */
+const foldedWord = (word: string): string => {
+  const chars = [...word.toLowerCase()];
+  const isWordChar = (char: string): boolean => /[\p{L}\p{N}]/u.test(char);
+  return chars.slice(chars.findIndex(isWordChar), chars.findLastIndex(isWordChar) + 1).join("");
+};
+
+const headingWords = (heading: string): string[] =>
+  heading
+    .trim()
+    .split(/\s+/u)
+    .filter((word) => word.length > 0);
+
+/** Below this length a word repeats another only as itself: "on" is not repeated by "one", nor "go" by "going". */
+const MIN_STEM = 4;
+
+/** The same word, or one of them the other with an ending added (level → levels, reference → references). */
+const isSameWord = (headingWord: string, sentenceWord: string): boolean => {
+  if (headingWord === sentenceWord) return true;
+  const [shorter, longer] = headingWord.length < sentenceWord.length ? [headingWord, sentenceWord] : [sentenceWord, headingWord];
+  return shorter.length >= MIN_STEM && longer.startsWith(shorter);
+};
+
+/** The heading's words that the sentence repeats; a word of punctuation alone is never repeated. */
+const echoedWords = (heading: string, sentence: string): number => {
+  const said = sentence.split(/\s+/u).map(foldedWord);
+  return headingWords(heading)
+    .map(foldedWord)
+    .filter((word) => word !== "" && said.some((other) => isSameWord(word, other))).length;
+};
+
+/** The heading's characters (spaces aside) that a trigram the sentence also has covers. */
+const echoedChars = (heading: string, sentence: string): number => {
+  const said = trigrams(sentence);
+  const clean = foldedChars(heading);
+  const covered = new Set(clean.flatMap((__char, index) => (said.has(trigramAt(clean, index)) ? [index, index + 1, index + 2] : [])));
+  return covered.size;
+};
+
+/**
+ * How much of the heading the sentence repeats, in the document's length unit. A sentence that repeats only part of it
+ * ("How Do Astronauts Go on Spacewalks?" → "When astronauts go on spacewalks, they wear spacesuits…") keeps the rest of
+ * its length as its own material.
+ */
+export const echoedHeadingUnits = (heading: string, sentence: string, unit: LengthUnit): number =>
+  unit === "word" ? echoedWords(heading, sentence) : echoedChars(heading, sentence);

package/src/detectors/index.ts

@@ -7,15 +7,18 @@ import { sentenceRhythm } from "./sentence-rhythm.ts";
 import { phraseCount, phraseMatch } from "./phrase-match.ts";
 import { agentlessPassive } from "./agentless-passive.ts";
 import { doubledWord } from "./doubled-word.ts";
+import { agreementSlip } from "./agreement-slip.ts";
 import { sentenceEnding } from "./sentence-ending.ts";
 import { doubledParticle, nounEnding } from "./token-shape.ts";
 import { kanjiRun, middleDot } from "./char-shape.ts";
-import { adverbDensity, conjunctionRun, expletive, oxfordComma, titleCaseMix } from "./en-shape.ts";
+import { adverbDensity, conjunctionRun, expletive, titleCaseMix } from "./en-shape.ts";
+import { oxfordComma } from "./oxford-comma.ts";
 import { paragraphLength, paragraphVariance, preambleLength, ruleOfThree, sectionUniformity } from "./structure.ts";
 import { concreteEvidence, dashDensity, emojiDensity, ngramRepetition, undefinedAcronym } from "./signals.ts";
 import { aiTell, contractionMix, cushionDensity, hedging, repeatedConjunction, unqualifiedSuperlative } from "./lexicon.ts";
 import { internalJargon, properNounDensity, requiredSections } from "./team.ts";
 import { latinSpacing, preferredTerm } from "./orthography.ts";
+import { straySpace } from "./stray-space.ts";
 import { danglingReference, dateOrder, dateWeekdayMismatch, duplicateDefinition, numberingGap, totalMismatch } from "./structure-tree.ts";
 
 /** rule 定義の how_to_find がここを引く。rule 側は実装を知らない。 */
@@ -32,6 +35,7 @@ export const DETECTORS: Readonly<Record<string, Detector>> = {
   "noun-ending": nounEnding,
   "doubled-particle": doubledParticle,
   "doubled-word": doubledWord,
+  "agreement-slip": agreementSlip,
   "kanji-run": kanjiRun,
   "middle-dot": middleDot,
   "adverb-density": adverbDensity,
@@ -60,6 +64,7 @@ export const DETECTORS: Readonly<Record<string, Detector>> = {
   "proper-noun-density": properNounDensity,
   "preferred-term": preferredTerm,
   "latin-spacing": latinSpacing,
+  "stray-space": straySpace,
   "dangling-reference": danglingReference,
   "date-weekday-mismatch": dateWeekdayMismatch,
   "date-order": dateOrder,