chaffjs 0.1.0 → 0.3.0

package/src/detectors/en-shape.ts

@@ -0,0 +1,187 @@
+import { lengthOf } from "../measure.ts";
+import { isClosed } from "../sentence-shape.ts";
+import type { Detector, Finding, ProseDocument, Section, Sentence, Token } from "../plugin.ts";
+
+const PER = 1000;
+
+/** これより短い文書では密度が暴れる。1 語で「1000 語あたり 100」になる。 */
+const MIN_WORDS = 200;
+
+const wordsIn = (doc: ProseDocument): number => doc.sentences.reduce((sum, sentence) => sum + lengthOf(sentence, "word"), 0);
+
+const isLyAdverb = (token: Token): boolean => token.pos === "ADV" && token.surface.toLowerCase().endsWith("ly");
+
+/**
+ * -ly 副詞は、動詞が弱いことの目印になる。「walked quickly」より「hurried」。
+ * 1 つずつは正しいので、件数ではなく密度で見る（bold-density と同じ）。
+ */
+export const adverbDensity: Detector = (doc, options): Finding[] => {
+  const words = wordsIn(doc);
+  const hits = doc.sentences.flatMap((sentence) => (sentence.tokens ?? []).filter(isLyAdverb).map((token) => ({ sentence, token })));
+  const density = words === 0 ? 0 : Math.round((hits.length / words) * PER);
+  const first = hits[0];
+  if (words < MIN_WORDS || first === undefined || density <= options.limit) return [];
+  return [
+    {
+      rule: "adverb-overuse",
+      severity: "info",
+      line: 0,
+      column: 0,
+      quote: first.sentence.text.trim(),
+      values: { count: hits.length, density, words, limit: options.limit, offset: first.token.span.start },
+    },
+  ];
+};
+
+const BE = new Set(["be", "is", "are", "was", "were"]);
+
+/**
+ * There is / It is ... that は、主語を後ろへ押しやって誰が何をするのかを消す。
+ * 「There is a need to review」は、誰が必要としているのかを言っていない。
+ */
+const expletiveAt = (tokens: readonly Token[]): Token | undefined => {
+  const head = tokens.find((token) => token.pos !== "PUNCT");
+  if (head === undefined) return undefined;
+  const lead = head.surface.toLowerCase();
+  if (lead !== "there" && lead !== "it") return undefined;
+  const next = tokens.find((token) => token.span.start >= head.span.end && token.pos !== "PUNCT");
+  if (next === undefined || !BE.has(next.lemma ?? next.surface.toLowerCase())) return undefined;
+  // "it is" は "it is raining" のような正当な用法があるので、that 節を伴うときだけ数える。
+  if (lead === "it" && !tokens.some((token) => token.surface.toLowerCase() === "that")) return undefined;
+  return head;
+};
+
+export const expletive: Detector = (doc, options): Finding[] => {
+  const hits = doc.sentences.flatMap((sentence) => {
+    const at = expletiveAt(sentence.tokens ?? []);
+    return at === undefined ? [] : [{ sentence, at }];
+  });
+  if (hits.length <= options.limit) return [];
+  return hits.map(({ sentence, at }) => ({
+    rule: "expletive-construction",
+    severity: "info",
+    line: 0,
+    column: 0,
+    quote: sentence.text.trim(),
+    values: { word: at.surface, count: hits.length, limit: options.limit, offset: at.span.start },
+  }));
+};
+
+const firstWord = (sentence: Sentence): string =>
+  sentence.text
+    .trim()
+    .split(/\s+/u)[0]
+    ?.replace(/[^A-Za-z]/gu, "") ?? "";
+
+/**
+ * 接続詞で始まる文が続く。1 つなら効くが、続くと文が前の文の付け足しに見えて、
+ * 何が主張なのかが分からなくなる。
+ */
+export const conjunctionRun: Detector = (doc, options): Finding[] => {
+  const heads = (options.lexicon ?? []).map((entry) => entry.pattern.toLowerCase());
+  const runs = doc.sentences.filter(isClosed).reduce<Sentence[][]>(
+    (acc, sentence) => {
+      const last = acc.at(-1) ?? [];
+      if (!heads.includes(firstWord(sentence).toLowerCase())) return [...acc.slice(0, -1), last, []];
+      return [...acc.slice(0, -1), [...last, sentence]];
+    },
+    [[]],
+  );
+  return runs
+    .filter((run) => run.length > options.limit)
+    .flatMap((run) => {
+      const first = run[0];
+      return first === undefined
+        ? []
+        : [
+            {
+              rule: "sentence-initial-conjunction-run",
+              severity: "info" as const,
+              line: 0,
+              column: 0,
+              quote: run.map((sentence) => sentence.text.trim()).join(" "),
+              values: { count: run.length, limit: options.limit, offset: first.span.start },
+            },
+          ];
+    });
+};
+
+const WORD = /[A-Za-z][A-Za-z'-]*/gu;
+
+/** 小さい語は Title Case でも小文字のままなので、大文字化の判定から外す。 */
+const MINOR = new Set(["a", "an", "the", "and", "or", "but", "of", "in", "on", "at", "to", "for", "with", "as", "by", "from", "is"]);
+
+const isTitleCase = (heading: string): boolean | undefined => {
+  const words = [...heading.matchAll(WORD)].map((match) => match[0]).filter((word) => !MINOR.has(word.toLowerCase()));
+  // 1 語の見出しは、どちらの流儀でも先頭が大文字になる。判定できない。
+  if (words.length < 2) return undefined;
+  const capitalized = words.filter((word) => word[0] === word[0]?.toUpperCase()).length;
+  return capitalized === words.length;
+};
+
+const headingsOf = (sections: readonly Section[]): { readonly section: Section; readonly title: boolean }[] =>
+  sections.flatMap((section) => {
+    const title = isTitleCase(section.heading);
+    return title === undefined ? [] : [{ section, title }];
+  });
+
+/**
+ * 見出しの大文字化が混ざっている。どちらの流儀が正しいかは決めない。spec §12.3。
+ * 見るのは文書の中で揃っているかだけで、少数派のほうを指摘する。
+ */
+export const titleCaseMix: Detector = (doc, options): Finding[] => {
+  const judged = headingsOf(doc.sections);
+  const title = judged.filter((entry) => entry.title).length;
+  const minorityIsTitle = title <= judged.length - title;
+  const few = minorityIsTitle ? title : judged.length - title;
+  if (few === 0 || few > options.limit) return [];
+  return judged
+    .filter((entry) => entry.title === minorityIsTitle)
+    .map(({ section }) => ({
+      rule: "title-case-consistency",
+      severity: "info",
+      line: 0,
+      column: 0,
+      quote: section.heading,
+      values: { count: few, limit: options.limit, offset: section.span.start },
+    }));
+};
+
+const LIST_CONJUNCTION = new Set(["and", "or"]);
+
+/**
+ * 3 つ以上の並列の最後の and / or の前に読点を打つか。Oxford comma。
+ *
+ * どちらが正しいかは決めない。スタイルガイドで割れる論点に立場を取ると rule ごと無視される。
+ * 見るのは 1 つの文書で揃っているかだけ。spec §12.3。
+ */
+const oxfordIn = (tokens: readonly Token[]): boolean | undefined => {
+  const at = tokens.findIndex((token) => LIST_CONJUNCTION.has(token.surface.toLowerCase()));
+  if (at < 1) return undefined;
+  const before = tokens.slice(0, at);
+  // 並列が 3 つ以上あるときだけ判定できる。読点が 1 つも無ければ 2 つの並列。
+  const commas = before.filter((token) => token.surface === ",").length;
+  if (commas === 0) return undefined;
+  return before.at(-1)?.surface === ",";
+};
+
+export const oxfordComma: Detector = (doc, options): Finding[] => {
+  const judged = doc.sentences.flatMap((sentence) => {
+    const oxford = oxfordIn(sentence.tokens ?? []);
+    return oxford === undefined ? [] : [{ sentence, oxford }];
+  });
+  const withComma = judged.filter((entry) => entry.oxford).length;
+  const minorityUsesComma = withComma <= judged.length - withComma;
+  const few = minorityUsesComma ? withComma : judged.length - withComma;
+  if (few === 0 || few > options.limit) return [];
+  return judged
+    .filter((entry) => entry.oxford === minorityUsesComma)
+    .map(({ sentence }) => ({
+      rule: "oxford-comma-consistency",
+      severity: "info",
+      line: 0,
+      column: 0,
+      quote: sentence.text.trim(),
+      values: { count: few, limit: options.limit, offset: sentence.span.start },
+    }));
+};

package/src/detectors/index.ts

@@ -5,6 +5,14 @@ import { headingEcho } from "./heading-echo.ts";
 import { repeatedHead } from "./repeated-head.ts";
 import { sentenceRhythm } from "./sentence-rhythm.ts";
 import { phraseMatch } from "./phrase-match.ts";
+import { agentlessPassive } from "./agentless-passive.ts";
+import { sentenceEnding } from "./sentence-ending.ts";
+import { doubledParticle, nounEnding } from "./token-shape.ts";
+import { kanjiRun, middleDot } from "./char-shape.ts";
+import { adverbDensity, conjunctionRun, expletive, oxfordComma, titleCaseMix } from "./en-shape.ts";
+import { paragraphLength, paragraphVariance, preambleLength, ruleOfThree, sectionUniformity } from "./structure.ts";
+import { concreteEvidence, emojiDensity, ngramRepetition, undefinedAcronym } from "./signals.ts";
+import { aiTell, cushionDensity, hedgingDensity, repeatedConjunction, unqualifiedSuperlative } from "./lexicon.ts";
 
 /** rule 定義の how_to_find がここを引く。rule 側は実装を知らない。 */
 export const DETECTORS: Readonly<Record<string, Detector>> = {
@@ -14,4 +22,29 @@ export const DETECTORS: Readonly<Record<string, Detector>> = {
   "repeated-head": repeatedHead,
   "sentence-rhythm": sentenceRhythm,
   "phrase-match": phraseMatch,
+  "agentless-passive": agentlessPassive,
+  "sentence-ending": sentenceEnding,
+  "noun-ending": nounEnding,
+  "doubled-particle": doubledParticle,
+  "kanji-run": kanjiRun,
+  "middle-dot": middleDot,
+  "adverb-density": adverbDensity,
+  expletive: expletive,
+  "conjunction-run": conjunctionRun,
+  "title-case-mix": titleCaseMix,
+  "oxford-comma": oxfordComma,
+  "paragraph-length": paragraphLength,
+  "paragraph-variance": paragraphVariance,
+  "section-uniformity": sectionUniformity,
+  "rule-of-three": ruleOfThree,
+  "preamble-length": preambleLength,
+  "emoji-density": emojiDensity,
+  "ngram-repetition": ngramRepetition,
+  "undefined-acronym": undefinedAcronym,
+  "concrete-evidence": concreteEvidence,
+  "hedging-density": hedgingDensity,
+  "cushion-density": cushionDensity,
+  "unqualified-superlative": unqualifiedSuperlative,
+  "repeated-conjunction": repeatedConjunction,
+  "ai-tell": aiTell,
 };

package/src/detectors/lexicon.ts

@@ -0,0 +1,124 @@
+import { proseText } from "../measure.ts";
+import { wordsOf } from "./structure.ts";
+import type { Detector, Finding, ProseDocument, Sentence } from "../plugin.ts";
+
+const PER = 1000;
+
+/** 密度を見る rule は短い文書を測らない。単位は言語で違うので床も分ける。 */
+const FLOOR = { word: 200, char: 500 };
+
+type Hit = { readonly sentence: Sentence; readonly matched: string };
+
+const hitsFor = (doc: ProseDocument, patterns: readonly string[]): Hit[] =>
+  doc.sentences.flatMap((sentence) => {
+    const text = sentence.text.toLowerCase();
+    return patterns.filter((pattern) => text.includes(pattern.toLowerCase())).map((matched) => ({ sentence, matched }));
+  });
+
+/**
+ * 単位長あたりの出現率。件数で数えると長い文書ほど当たる（bold-density と同じ）。
+ * rule の id は呼び出し側が持つ。detector は「密度が閾値を超えたか」しか知らない。
+ */
+const densityRule =
+  (rule: string): Detector =>
+  (doc, options): Finding[] => {
+    const patterns = (options.lexicon ?? []).map((entry) => entry.pattern);
+    const hits = hitsFor(doc, patterns);
+    const length = wordsOf(doc);
+    const rate = length === 0 ? 0 : Math.round((hits.length / length) * PER);
+    const first = hits[0];
+    if (length < FLOOR[doc.lengthUnit] || first === undefined || rate <= options.limit) return [];
+    return hits.map((hit) => ({
+      rule,
+      severity: "warning",
+      line: 0,
+      column: 0,
+      quote: hit.sentence.text.trim(),
+      values: { matched: hit.matched, count: hits.length, density: rate, limit: options.limit, offset: hit.sentence.span.start },
+    }));
+  };
+
+export const hedgingDensity = densityRule("excessive-hedging");
+export const cushionDensity = densityRule("cushion-phrase-density");
+
+/**
+ * 限定のない最上級。「最も速い」だけでは、何と比べて最もなのかが無い。
+ * 同じ文に比較対象や条件があれば、それは主張であって誇張ではない。
+ */
+const QUALIFIER = /\d|より|に比べ|のうち|among|than|compared|based on|according/iu;
+
+export const unqualifiedSuperlative: Detector = (doc, options): Finding[] => {
+  const patterns = (options.lexicon ?? []).map((entry) => entry.pattern);
+  const bare = hitsFor(doc, patterns).filter((hit) => !QUALIFIER.test(hit.sentence.text));
+  if (bare.length < options.limit) return [];
+  return bare.map((hit) => ({
+    rule: "unqualified-superlative",
+    severity: "warning",
+    line: 0,
+    column: 0,
+    quote: hit.sentence.text.trim(),
+    values: { matched: hit.matched, count: bare.length, limit: options.limit, offset: hit.sentence.span.start },
+  }));
+};
+
+const openerOf = (doc: ProseDocument, patterns: readonly string[]): (string | undefined)[] =>
+  doc.paragraphs.map((paragraph) => {
+    const head = paragraph.sentences[0]?.text.trim().toLowerCase() ?? "";
+    return patterns.find((pattern) => head.startsWith(pattern.toLowerCase()));
+  });
+
+/**
+ * 段落が同じ接続詞で始まり続ける。1 つなら流れを作るが、続くとどの段落も
+ * 前の段落の付け足しに見えて、話がどこへ向かっているのか分からなくなる。
+ */
+export const repeatedConjunction: Detector = (doc, options): Finding[] => {
+  const patterns = (options.lexicon ?? []).map((entry) => entry.pattern);
+  const openers = openerOf(doc, patterns);
+  const runs = openers.reduce<{ runs: number[][]; current: number[] }>(
+    (acc, opener, index) => {
+      if (opener === undefined) return { runs: [...acc.runs, acc.current], current: [] };
+      return { runs: acc.runs, current: [...acc.current, index] };
+    },
+    { runs: [], current: [] },
+  );
+  return [...runs.runs, runs.current]
+    .filter((run) => run.length > options.limit)
+    .flatMap((run) => {
+      const at = doc.paragraphs[run[0] ?? 0];
+      return at === undefined
+        ? []
+        : [
+            {
+              rule: "repeated-conjunction",
+              severity: "warning" as const,
+              line: 0,
+              column: 0,
+              quote: at.sentences[0]?.text.trim() ?? "",
+              values: { count: run.length, limit: options.limit, offset: at.span.start },
+            },
+          ];
+    });
+};
+
+/**
+ * AI 生成の signal。1 つでは何も言えないので、重みを足し合わせて文書の点にする。
+ * 単独で断じない。§20.2 の複合シグナルの入口で、これだけで「AI が書いた」とは言わない。
+ */
+export const aiTell: Detector = (doc, options): Finding[] => {
+  const weights = new Map((options.lexicon ?? []).map((entry) => [entry.pattern.toLowerCase(), entry.weight ?? 1]));
+  const body = doc.sentences.map(proseText).join(" ").toLowerCase();
+  const found = [...weights.entries()].filter(([pattern]) => body.includes(pattern));
+  const score = Math.round(found.reduce((sum, [, weight]) => sum + weight, 0) * 10);
+  const first = doc.sentences.find((sentence) => found.some(([pattern]) => sentence.text.toLowerCase().includes(pattern)));
+  if (first === undefined || score <= options.limit) return [];
+  return [
+    {
+      rule: "ai-tell",
+      severity: "info",
+      line: 0,
+      column: 0,
+      quote: first.text.trim(),
+      values: { word: found.map(([pattern]) => pattern).join("、"), count: found.length, density: score, limit: options.limit, offset: first.span.start },
+    },
+  ];
+};

package/src/detectors/repeated-head.ts

@@ -1,23 +1,34 @@
-import type { Detector, Finding, Sentence } from "../plugin.ts";
+import type { Detector, Finding, LengthUnit, Sentence } from "../plugin.ts";
 
 const HEAD_CHARS = 6;
+const HEAD_WORDS = 3;
 
-const headOf = (sentence: Sentence): string => sentence.text.trim().replace(/\s+/gu, "").slice(0, HEAD_CHARS);
+/**
+ * 書き出しの「同じさ」を測る単位は言語で違う。日本語は文字、英語は語。
+ * 文字で切ると "We continued" が "Wecont" になり、引用がそのまま読み手に出る。
+ */
+const headOf = (sentence: Sentence, unit: LengthUnit): string => {
+  const text = sentence.text.trim();
+  if (unit === "char") return text.replace(/\s+/gu, "").slice(0, HEAD_CHARS);
+  return text.split(/\s+/u).slice(0, HEAD_WORDS).join(" ");
+};
 
 type Run = { readonly head: string; readonly members: readonly Sentence[] };
 
-const runsOf = (sentences: readonly Sentence[]): Run[] =>
+const runsOf = (sentences: readonly Sentence[], unit: LengthUnit): Run[] =>
   sentences.reduce<Run[]>((acc, sentence) => {
-    const head = headOf(sentence);
+    const head = headOf(sentence, unit);
     const last = acc.at(-1);
-    if (last !== undefined && last.head === head && head.length === HEAD_CHARS) {
+    // 短すぎる書き出しは「同じ」と言えない。単位ぶん揃って初めて連なりと見る。
+    const full = unit === "char" ? head.length === HEAD_CHARS : head.split(" ").length === HEAD_WORDS;
+    if (last !== undefined && last.head === head && full) {
       return [...acc.slice(0, -1), { head, members: [...last.members, sentence] }];
     }
     return [...acc, { head, members: [sentence] }];
   }, []);
 
 export const repeatedHead: Detector = (doc, options): Finding[] =>
-  runsOf(doc.sentences)
+  runsOf(doc.sentences, doc.lengthUnit)
     .filter((run) => run.members.length > options.limit)
     .map((run) => ({
       rule: "repeated-sentence-head",

package/src/detectors/sentence-ending.ts

@@ -0,0 +1,37 @@
+import { hasPredicate, isClosed } from "../sentence-shape.ts";
+import type { Detector, Finding, Sentence } from "../plugin.ts";
+
+/**
+ * 文末の調子が混ざっているかを見る。日本語の「ですます / である」がこれ。
+ *
+ * どちらが正しいかは決めない。決めると、方針の違う書き手にそのまま無視される。
+ * 見るのは文書の中での一貫性だけで、少数派のほうを指摘する。spec §12.3 の方針を日本語にも。
+ */
+/**
+ * 述語を持たないものは文として数えない。見出しの下の `MaaSサービス` のような
+ * 名前だけの行がそのまま「である調」の少数派になり、実文書の誤検知はすべてこれだった。
+ */
+const isSentence = (sentence: Sentence): boolean => isClosed(sentence) && hasPredicate(sentence);
+
+const isPolite = (sentence: Sentence, polite: readonly string[]): boolean =>
+  (sentence.tokens ?? []).some((token) => polite.includes(token.lemma ?? token.surface));
+
+export const sentenceEnding: Detector = (doc, options): Finding[] => {
+  const polite = (options.lexicon ?? []).map((entry) => entry.pattern);
+  const judged = doc.sentences.filter(isSentence).map((sentence) => ({ sentence, polite: isPolite(sentence, polite) }));
+  const counts = { polite: judged.filter((entry) => entry.polite).length, plain: judged.length - judged.filter((entry) => entry.polite).length };
+  const minorityIsPolite = counts.polite <= counts.plain;
+  const few = minorityIsPolite ? counts.polite : counts.plain;
+  // 片方しか無ければ一貫している。少数派が閾値を超えて多ければ、混在ではなく別の文体。
+  if (few === 0 || few > options.limit) return [];
+  return judged
+    .filter((entry) => entry.polite === minorityIsPolite)
+    .map(({ sentence }) => ({
+      rule: "no-mixed-desumasu",
+      severity: "warning",
+      line: 0,
+      column: 0,
+      quote: sentence.text.trim(),
+      values: { count: few, limit: options.limit, offset: sentence.span.start },
+    }));
+};

package/src/detectors/sentence-length.ts

@@ -1,21 +1,15 @@
-import type { Detector, Finding, ProseDocument, Sentence } from "../plugin.ts";
-
-/** 日本語は文字、英語は語。単位は adapter が宣言する。spec §7。 */
-const measure = (doc: ProseDocument, sentence: Sentence): number => {
-  const text = sentence.text.trim();
-  if (doc.lengthUnit === "char") return text.length;
-  return text.split(/\s+/u).filter((word) => word.length > 0).length;
-};
+import { lengthOf, proseText } from "../measure.ts";
+import type { Detector, Finding } from "../plugin.ts";
 
 export const sentenceLength: Detector = (doc, options): Finding[] =>
   doc.sentences
-    .map((sentence) => ({ sentence, size: measure(doc, sentence) }))
+    .map((sentence) => ({ sentence, size: lengthOf(sentence, doc.lengthUnit) }))
     .filter(({ size }) => size > options.limit)
     .map(({ sentence, size }) => ({
       rule: "max-sentence-length",
       severity: "warning",
       line: 0,
       column: 0,
-      quote: sentence.text.trim(),
+      quote: proseText(sentence),
       values: { count: size, limit: options.limit, offset: sentence.span.start },
     }));

package/src/detectors/sentence-rhythm.ts

@@ -1,3 +1,4 @@
+import { charLength, proseText } from "../measure.ts";
 import type { Detector, Finding } from "../plugin.ts";
 
 /** 文が少ないと変動係数は当てにならない。ここを下回る文書は見ない。 */
@@ -11,7 +12,7 @@ const coefficientOfVariation = (sizes: readonly number[]): number => {
 };
 
 export const sentenceRhythm: Detector = (doc, options): Finding[] => {
-  const sizes = doc.sentences.map((sentence) => sentence.text.trim().length);
+  const sizes = doc.sentences.map((sentence) => charLength(sentence));
   if (sizes.length < MIN_SENTENCES) return [];
   const spread = Math.round(coefficientOfVariation(sizes) * 100);
   if (spread >= options.limit) return [];
@@ -21,7 +22,7 @@ export const sentenceRhythm: Detector = (doc, options): Finding[] => {
       severity: "warning",
       line: 0,
       column: 0,
-      quote: doc.sentences[0]?.text.trim() ?? "",
+      quote: doc.sentences[0] === undefined ? "" : proseText(doc.sentences[0]),
       values: { count: spread, limit: options.limit, offset: doc.sentences[0]?.span.start ?? 0 },
     },
   ];