chaffjs 0.0.1 → 0.2.0

package/src/checks.ts

@@ -0,0 +1,74 @@
+import { existsSync, readFileSync } from "node:fs";
+import { parse } from "yaml";
+import type { Level, Severity } from "./plugin.ts";
+
+export const CHECKS_FILE = "checks.yaml";
+
+/**
+ * 利用者が自然文で足す検査。キーは英語、中身は書き手の言葉。
+ * プログラムは書かせない。spec §18.3。
+ */
+export type UserCheck = {
+  readonly id: string;
+  readonly name: string;
+  readonly use_for: readonly string[];
+  readonly check: string;
+  readonly look_at: string | undefined;
+  readonly level: Level;
+  readonly how_to_fix: string;
+  readonly source: string;
+};
+
+const isRecord = (value: unknown): value is Record<string, unknown> => typeof value === "object" && value !== null && !Array.isArray(value);
+
+const str = (value: unknown): string | undefined => (typeof value === "string" ? value.trim() : undefined);
+
+const LEVELS: readonly Level[] = ["strict", "normal", "relaxed", "off"];
+const asLevel = (value: unknown): Level => LEVELS.find((level) => level === value) ?? "normal";
+
+/** id は名前から作る。利用者に id を書かせない。 */
+const idOf = (name: string, index: number): string => {
+  const slug = name
+    .toLowerCase()
+    .replace(/[^a-z0-9]+/gu, "-")
+    .replace(/^-|-$/gu, "");
+  return slug.length > 0 ? `check:${slug}` : `check:${index + 1}`;
+};
+
+/** use_for は 1 つでも配列でも書ける。書かなければ全ジャンル。 */
+const useForOf = (value: unknown): string[] => {
+  if (typeof value === "string") return [value];
+  if (Array.isArray(value)) return value.map((entry) => String(entry));
+  return ["blog", "business"];
+};
+
+const toCheck = (raw: unknown, index: number, source: string): UserCheck | undefined => {
+  if (!isRecord(raw)) return undefined;
+  const name = str(raw["name"]);
+  const check = str(raw["check"]);
+  if (name === undefined || check === undefined) return undefined;
+  return {
+    id: idOf(name, index),
+    name,
+    use_for: useForOf(raw["use_for"]),
+    check,
+    look_at: str(raw["look_at"]),
+    level: asLevel(raw["level"]),
+    how_to_fix: str(raw["how_to_fix"]) ?? "",
+    source,
+  };
+};
+
+export const loadChecks = (path: string): UserCheck[] => {
+  if (!existsSync(path)) return [];
+  const raw: unknown = parse(readFileSync(path, "utf8"));
+  if (!isRecord(raw) || !Array.isArray(raw["checks"])) return [];
+  return raw["checks"].map((entry, index) => toCheck(entry, index, path)).filter((entry) => entry !== undefined);
+};
+
+/** 4 語を severity にする。L4 は件数ではなく深刻度を段で持つ。 */
+export const severityOf = (level: Level): Severity => {
+  if (level === "strict") return "error";
+  if (level === "relaxed") return "info";
+  return "warning";
+};

package/src/cli.ts

@@ -3,6 +3,7 @@ import { existsSync } from "node:fs";
 import { join } from "node:path";
 import { loadAdapter, packageFor } from "./adapter-load.ts";
 import { CONFIG_FILE, EMPTY, loadConfig, type Config } from "./config/load.ts";
+import { applyByPath } from "./config/by-path.ts";
 import { applyLevel } from "./config/write.ts";
 import { buildDocument } from "./document.ts";
 import { guessLanguage } from "./detect.ts";
@@ -11,6 +12,8 @@ import { BASELINE_FILE, fingerprint, readBaseline, splitByBaseline, writeBaselin
 import { applySuppressions } from "./stet.ts";
 import { renderSuppressions, type PerFile } from "./render/suppressions.ts";
 import { clock, describeChange, snapshotOf, watchPaths, type Snapshot } from "./watch.ts";
+import { runEval } from "./commands/eval.ts";
+import { runTest } from "./commands/test.ts";
 import { frontMatterGenre, guessGenre } from "./genre.ts";
 import { GENRES, runInit } from "./init.ts";
 import { loadRules } from "./rule-load.ts";
@@ -26,7 +29,9 @@ const USAGE = `chaff — 文章の読みにくいところを見つけます。
 
   chaff <file|dir|glob>...       検査する（設定も API key も要りません）
   chaff .                        この場所の Markdown を全部
+  chaff test <file|dir>...       意味を読む検査も動かす（API key が要ります）
   chaff init                     chaff.yaml を作る
+  chaff eval <dir>               手元の文書で閾値を測り直す
   chaff explain <rule>           そのルールの意図と根拠を読む
   chaff genres                   ジャンルの一覧
   chaff rules --json             いまの設定を JSON で出す（AI に渡す用）
@@ -47,6 +52,9 @@ const USAGE = `chaff — 文章の読みにくいところを見つけます。
 const readConfig = (): Config => (existsSync(join(process.cwd(), CONFIG_FILE)) ? loadConfig(join(process.cwd(), CONFIG_FILE)) : EMPTY);
 
 const resolveGenre = (path: string, source: string, config: Config): { genre: string; from: string } => {
+  // パスごとの上書きが最優先。「全体はこう、ここだけは違う」を書けるようにする。
+  const override = applyByPath(config.byPath, config.baseDir, path);
+  if (override.genre !== undefined) return { genre: override.genre, from: `${CONFIG_FILE} の by_path` };
   if (config.genre !== undefined) return { genre: config.genre, from: CONFIG_FILE };
   const guess = guessGenre(path, source, frontMatterGenre(source));
   return guess === undefined ? { genre: "blog/tech", from: "既定" } : { genre: guess.genre, from: guess.from };
@@ -81,7 +89,7 @@ const headerFor = (path: string, genre: string, from: string, language: string,
 
 const inspect = async (path: string, config: Config, argv: readonly string[]): Promise<Inspected> => {
   const source = await readFile(path, "utf8");
-  const language = config.language ?? guessLanguage(source).language;
+  const language = applyByPath(config.byPath, config.baseDir, path).language ?? config.language ?? guessLanguage(source).language;
   const adapter = await loadAdapter(language);
   const { genre, from } = resolveGenre(path, source, config);
   const doc = buildDocument(path, source, adapter);
@@ -175,6 +183,7 @@ const explain = (ruleId: string | undefined): number => {
 
 /** 閾値の単位。explain で「4 とは何の 4 か」を示すため。 */
 const unitOf = (ruleId: string, language: string): string => {
+  if (ruleId === "bold-density") return "1000 字あたりの箇所数";
   if (ruleId !== "max-sentence-length") return "回";
   return language === "ja" ? "文字" : "語";
 };
@@ -239,6 +248,8 @@ const HANDLERS: Readonly<Record<string, Handler>> = {
   genres: showGenres,
   rules: showRules,
   explain: (argv) => explain(argv[1]),
+  eval: (argv) => runEval(positional(argv), argv, { config: readConfig(), resolveGenre, flag }),
+  test: (argv) => runTest(positional(argv), argv, { config: readConfig(), resolveGenre, inspect }),
   baseline: (argv) => runBaseline(positional(argv), argv),
   suppressions: (argv) => runSuppressions(positional(argv), argv),
   relax: (argv) => changeSetting("relaxed", argv[1], flag(argv, "--why")),

package/src/commands/eval.ts

@@ -0,0 +1,64 @@
+import { readFile } from "node:fs/promises";
+import { loadAdapter } from "../adapter-load.ts";
+import { applyByPath } from "../config/by-path.ts";
+import { buildDocument } from "../document.ts";
+import { guessLanguage } from "../detect.ts";
+import { collectTargets } from "../files.ts";
+import { loadRules } from "../rule-load.ts";
+import { evaluate } from "../eval.ts";
+import { renderEval } from "../render/eval.ts";
+import type { Config } from "../config/load.ts";
+
+export type Context = {
+  readonly config: Config;
+  readonly resolveGenre: (path: string, source: string, config: Config) => { genre: string; from: string };
+  readonly flag: (argv: readonly string[], name: string) => string | undefined;
+};
+
+/**
+ * 閾値の較正。手元の文書を「人間の良い文書」として扱い、どの閾値なら
+ * 誤検知が目標を下回るかを測る。書き換えはしない。
+ */
+export const runEval = async (targets: readonly string[], argv: readonly string[], context: Context): Promise<number> => {
+  const paths = collectTargets(targets.length > 0 ? targets : ["."]);
+  if (paths.length === 0) {
+    console.error(`Markdown が 1 つも見つかりませんでした: ${targets.join(", ")}`);
+    return 1;
+  }
+  const { config, resolveGenre, flag } = context;
+  const only = flag(argv, "--rule");
+  const docs = await Promise.all(
+    paths.map(async (path) => {
+      const source = await readFile(path, "utf8");
+      const language = applyByPath(config.byPath, config.baseDir, path).language ?? config.language ?? guessLanguage(source).language;
+      const adapter = await loadAdapter(language);
+      return { doc: buildDocument(path, source, adapter), language, genre: resolveGenre(path, source, config).genre };
+    }),
+  );
+  const language = docs[0]?.language ?? "ja";
+  const genre = docs[0]?.genre ?? "blog/tech";
+  // 言語とジャンルが混ざった corpus は測れない。閾値の単位が違うため。
+  const mixed = new Set(docs.map((entry) => `${entry.language}/${entry.genre}`));
+  if (mixed.size > 1) {
+    console.error(`言語かジャンルが混ざっています: ${[...mixed].join(", ")}\n1 つに絞って測ってください（例: npx chaff eval examples/blog-ja/）。`);
+    return 1;
+  }
+  const rules = loadRules(language).filter((rule) => only === undefined || rule.id === only);
+  if (rules.length === 0) {
+    console.error(`${only ?? "(名前なし)"} という rule はありません。`);
+    return 1;
+  }
+  console.log(
+    renderEval(
+      evaluate(
+        docs.map((entry) => entry.doc),
+        rules,
+        genre,
+        language,
+      ),
+      docs.length,
+      paths.length,
+    ),
+  );
+  return 0;
+};

package/src/commands/test.ts

@@ -0,0 +1,89 @@
+import { readFile } from "node:fs/promises";
+import { join } from "node:path";
+import { loadAdapter } from "../adapter-load.ts";
+import { applyByPath } from "../config/by-path.ts";
+import { buildDocument } from "../document.ts";
+import { guessLanguage } from "../detect.ts";
+import { collectTargets } from "../files.ts";
+import { loadRules } from "../rule-load.ts";
+import { CHECKS_FILE, loadChecks, type UserCheck } from "../checks.ts";
+import { CACHE_DIR, hasCredentials, isAuthFailure } from "../judge.ts";
+import { runSemantic } from "../run-semantic.ts";
+import { MACHINE_BANNER, renderSemantic } from "../render/semantic.ts";
+import type { Config } from "../config/load.ts";
+import type { Finding, RuleDefinition } from "../plugin.ts";
+
+export type TestContext = {
+  readonly config: Config;
+  readonly resolveGenre: (path: string, source: string, config: Config) => { genre: string; from: string };
+  readonly inspect: (path: string, config: Config, argv: readonly string[]) => Promise<{ text: string; outcome: { findings: readonly Finding[] } }>;
+};
+
+type Judged = { path: string; outcome: Awaited<ReturnType<typeof runSemantic>>; rules: RuleDefinition[]; language: string };
+
+const judgeAll = async (
+  paths: readonly string[],
+  config: Config,
+  checks: readonly UserCheck[],
+  options: Parameters<typeof runSemantic>[5],
+  resolveGenre: TestContext["resolveGenre"],
+): Promise<Judged[]> =>
+  Promise.all(
+    paths.map(async (path) => {
+      const source = await readFile(path, "utf8");
+      const language = applyByPath(config.byPath, config.baseDir, path).language ?? config.language ?? guessLanguage(source).language;
+      const adapter = await loadAdapter(language);
+      const doc = buildDocument(path, source, adapter);
+      const { genre } = resolveGenre(path, source, config);
+      const rules = loadRules(language);
+      return { path, outcome: await runSemantic(doc, rules, checks, config.rules, genre, options), rules, language };
+    }),
+  );
+
+/**
+ * 機械と AI を見出しで分けて出す。読む人が「これは揺れる判定か」を知らないと、
+ * AI の誤検知に振り回される。workflow spec §7、samples/README の軸 2。
+ */
+export const runTest = async (targets: readonly string[], argv: readonly string[], context: TestContext): Promise<number> => {
+  const paths = collectTargets(targets.length > 0 ? targets : ["."]);
+  if (paths.length === 0) {
+    console.error(`Markdown が 1 つも見つかりませんでした: ${targets.join(", ")}`);
+    return 1;
+  }
+  const { config, resolveGenre, inspect } = context;
+  const checks = loadChecks(join(process.cwd(), CHECKS_FILE));
+  const results = await Promise.all(paths.map((path) => inspect(path, config, argv)));
+  results.forEach((result) => {
+    console.log(result.text.replace(/\n/u, `\n${MACHINE_BANNER.join("\n")}`));
+  });
+  const machineOnly = results.some((result) => result.outcome.findings.some((finding) => finding.severity === "error")) ? 1 : 0;
+  const noCredentials = [
+    "",
+    "  意味を読む検査は動かしていません。Anthropic の認証情報がありません。",
+    "  ANTHROPIC_API_KEY を設定するか、ant auth login を実行してください。",
+    "  機械による判定はすべて動いています。",
+    "",
+  ].join("\n");
+  // 呼んでから落ちるのを待たない。認証が無いときの SDK の例外は型で判別できない。
+  if (!hasCredentials()) {
+    console.log(noCredentials);
+    return machineOnly;
+  }
+  const options = { model: config.aiModel, cacheDir: join(process.cwd(), CACHE_DIR), confidenceThreshold: config.confidenceThreshold };
+  const semantic = await judgeAll(paths, config, checks, options, resolveGenre).catch((error: unknown) => {
+    if (!isAuthFailure(error)) throw error;
+    console.log(noCredentials);
+    return undefined;
+  });
+  if (semantic === undefined) return machineOnly;
+  semantic.forEach((entry) => {
+    if (entry.outcome.findings.length === 0 && entry.outcome.asked === 0) return;
+    console.log([`${entry.path}`, ...renderSemantic(entry.outcome, entry.rules, checks, entry.language)].join("\n"));
+  });
+  const machine = results.flatMap((result) => result.outcome.findings);
+  const ai = semantic.flatMap((entry) => entry.outcome.findings);
+  console.log(
+    ["", "─".repeat(60), "", `  機械 ${machine.length} 件 / AI ${ai.length} 件`, "  文章は書き換えていません。直すのは書いた人です。", ""].join("\n"),
+  );
+  return [...machine, ...ai].some((finding) => finding.severity === "error") ? 1 : 0;
+};

package/src/config/by-path.ts

@@ -0,0 +1,47 @@
+import { relative, sep } from "node:path";
+
+export type PathRule = { readonly files: readonly string[]; readonly genre: string | undefined; readonly language: string | undefined };
+
+/**
+ * glob を正規表現にする。使うのは 3 つだけ。
+ *   **\/  区切りをまたいでディレクトリ 0 個以上（docs/**\/*.md は docs/a.md にも当たる）
+ *   *    区切りをまたがずに何文字でも
+ *   ?    区切り以外の 1 文字
+ * これ以上を足すと、設定を書く人が覚えることが増える。
+ *
+ * 「ディレクトリ 0 個」を落とすと docs/**\/*.md が docs/a.md に当たらない。
+ * glob の典型的な落とし穴で、設定を書いた人は当たらない理由に気づけない。
+ */
+const DIRS = "\u0000";
+const ANY = "\u0001";
+
+const toPattern = (glob: string): RegExp => {
+  const body = glob
+    .replace(/[.+^${}()|[\]\\]/gu, "\\$&")
+    .replace(/\*\*\//gu, DIRS)
+    .replace(/\*\*/gu, ANY)
+    .replace(/\*/gu, "[^/]*")
+    .replace(/\?/gu, "[^/]")
+    .replaceAll(DIRS, "(?:[^/]*/)*")
+    .replaceAll(ANY, ".*");
+  return new RegExp(`^${body}$`, "u");
+};
+
+/** 設定ファイルからの相対で照合する。実行した場所に左右されないため。 */
+const normalize = (base: string, path: string): string => relative(base, path).split(sep).join("/");
+
+export const matches = (glob: string, base: string, path: string): boolean => toPattern(glob).test(normalize(base, path));
+
+/**
+ * 後に書いたものが勝つ。上から順に当て、最後に当たったものを使う。
+ * 「全体はこう、ここだけは違う」と書けるようにするため。
+ */
+export const applyByPath = (rules: readonly PathRule[], base: string, path: string): { genre?: string; language?: string } =>
+  rules.reduce<{ genre?: string; language?: string }>((acc, rule) => {
+    if (!rule.files.some((glob) => matches(glob, base, path))) return acc;
+    return {
+      ...acc,
+      ...(rule.genre === undefined ? {} : { genre: rule.genre }),
+      ...(rule.language === undefined ? {} : { language: rule.language }),
+    };
+  }, {});

package/src/config/load.ts

@@ -1,7 +1,9 @@
 import { readFileSync } from "node:fs";
+import { dirname } from "node:path";
 import { parse } from "yaml";
 import { isLevel } from "../levels.ts";
 import type { Level } from "../plugin.ts";
+import type { PathRule } from "./by-path.ts";
 
 export const CONFIG_FILE = "chaff.yaml";
 
@@ -11,9 +13,27 @@ export type Config = {
   readonly rules: Readonly<Record<string, Level>>;
   readonly experimental: boolean;
   readonly path: string | undefined;
+  readonly aiModel: string;
+  readonly confidenceThreshold: number;
+  /** パスごとの上書き。設定ファイルのある場所からの相対で照合する。 */
+  readonly byPath: readonly PathRule[];
+  readonly baseDir: string;
 };
 
-export const EMPTY: Config = { genre: undefined, language: undefined, rules: {}, experimental: false, path: undefined };
+/** 判定の質が誤検知に直結するので、既定は最上位のモデル。cost は絞り込みで削る。spec §14。 */
+export const DEFAULT_MODEL = "claude-opus-5";
+
+export const EMPTY: Config = {
+  genre: undefined,
+  language: undefined,
+  rules: {},
+  experimental: false,
+  path: undefined,
+  aiModel: DEFAULT_MODEL,
+  confidenceThreshold: 0.7,
+  byPath: [],
+  baseDir: process.cwd(),
+};
 
 const isRecord = (value: unknown): value is Record<string, unknown> => typeof value === "object" && value !== null && !Array.isArray(value);
 
@@ -24,6 +44,22 @@ const rulesOf = (raw: unknown): Record<string, Level> => {
 
 const str = (value: unknown): string | undefined => (typeof value === "string" ? value : undefined);
 
+/** files は 1 つの文字列でも配列でも書ける。 */
+const globsOf = (value: unknown): string[] => {
+  if (typeof value === "string") return [value];
+  return Array.isArray(value) ? value.map((entry) => String(entry)) : [];
+};
+
+const toPathRule = (raw: unknown): PathRule | undefined => {
+  if (!isRecord(raw)) return undefined;
+  const files = raw["files"];
+  const globs = globsOf(files);
+  if (globs.length === 0) return undefined;
+  return { files: globs, genre: str(raw["genre"]), language: str(raw["language"]) };
+};
+
+const byPathOf = (raw: unknown): PathRule[] => (Array.isArray(raw) ? raw.map(toPathRule).filter((rule) => rule !== undefined) : []);
+
 /** 設定ファイルが無くても動く。あっても、既定から変えたものだけが書かれている。spec §18。 */
 export const loadConfig = (path: string): Config => {
   const raw: unknown = parse(readFileSync(path, "utf8"));
@@ -34,5 +70,9 @@ export const loadConfig = (path: string): Config => {
     rules: rulesOf(raw["rules"]),
     experimental: raw["experimental"] === true,
     path,
+    aiModel: str(raw["ai_model"]) ?? DEFAULT_MODEL,
+    confidenceThreshold: typeof raw["confidence_threshold"] === "number" ? raw["confidence_threshold"] : 0.7,
+    byPath: byPathOf(raw["by_path"]),
+    baseDir: dirname(path),
   };
 };

package/src/detectors/count-per-section.ts

@@ -1,15 +1,39 @@
-import type { Detector, Finding } from "../plugin.ts";
+import { charLength, proseText } from "../measure.ts";
+import type { Detector, Finding, Section } from "../plugin.ts";
 
-/** 節あたりの強調の数。bold-density が使う。 */
+/** これより短い節では密度が暴れる。43 字に 1 箇所で「1000 字あたり 23」になる。 */
+const MIN_CHARS = 200;
+
+const PER = 1000;
+
+const charsOf = (section: Section): number => section.sentences.reduce((sum, sentence) => sum + charLength(sentence), 0);
+
+/**
+ * 節あたりの「密度」を見る。件数ではない。
+ *
+ * 件数で数えると、節が長いほど当たる。実文書で測ると、1095 字に 2 箇所の節と
+ * 557 字に 9 箇所の節が同じ「多すぎる」になっていた。読者が受け取る印象は逆なのに。
+ *
+ * rule の名前（bold-density）が最初から密度と言っていた。件数で実装していたのが誤り。
+ */
 export const countPerSection: Detector = (doc, options): Finding[] =>
   doc.sections
-    .filter((section) => section.strongCount > options.limit)
-    .map((section) => ({
+    .map((section) => ({ section, chars: charsOf(section) }))
+    .filter(({ chars }) => chars >= MIN_CHARS)
+    .map(({ section, chars }) => ({ section, chars, density: Math.round((section.strongCount / chars) * PER) }))
+    .filter(({ density }) => density > options.limit)
+    .map(({ section, chars, density }) => ({
       rule: "bold-density",
       severity: "warning",
       line: 0,
       column: 0,
       // 節全体を引くと段落をまたいで読めなくなる。最初の文だけ見せて場所を示す。
-      quote: section.firstSentence?.text.trim() ?? "",
-      values: { count: section.strongCount, limit: options.limit, offset: section.firstSentence?.span.start ?? section.span.start },
+      quote: section.firstSentence === undefined ? "" : proseText(section.firstSentence),
+      values: {
+        count: density,
+        limit: options.limit,
+        bolds: section.strongCount,
+        chars,
+        offset: section.firstSentence?.span.start ?? section.span.start,
+      },
     }));

package/src/detectors/sentence-length.ts

@@ -1,21 +1,15 @@
-import type { Detector, Finding, ProseDocument, Sentence } from "../plugin.ts";
-
-/** 日本語は文字、英語は語。単位は adapter が宣言する。spec §7。 */
-const measure = (doc: ProseDocument, sentence: Sentence): number => {
-  const text = sentence.text.trim();
-  if (doc.lengthUnit === "char") return text.length;
-  return text.split(/\s+/u).filter((word) => word.length > 0).length;
-};
+import { lengthOf, proseText } from "../measure.ts";
+import type { Detector, Finding } from "../plugin.ts";
 
 export const sentenceLength: Detector = (doc, options): Finding[] =>
   doc.sentences
-    .map((sentence) => ({ sentence, size: measure(doc, sentence) }))
+    .map((sentence) => ({ sentence, size: lengthOf(sentence, doc.lengthUnit) }))
     .filter(({ size }) => size > options.limit)
     .map(({ sentence, size }) => ({
       rule: "max-sentence-length",
       severity: "warning",
       line: 0,
       column: 0,
-      quote: sentence.text.trim(),
+      quote: proseText(sentence),
       values: { count: size, limit: options.limit, offset: sentence.span.start },
     }));

package/src/detectors/sentence-rhythm.ts

@@ -1,3 +1,4 @@
+import { charLength, proseText } from "../measure.ts";
 import type { Detector, Finding } from "../plugin.ts";
 
 /** 文が少ないと変動係数は当てにならない。ここを下回る文書は見ない。 */
@@ -11,7 +12,7 @@ const coefficientOfVariation = (sizes: readonly number[]): number => {
 };
 
 export const sentenceRhythm: Detector = (doc, options): Finding[] => {
-  const sizes = doc.sentences.map((sentence) => sentence.text.trim().length);
+  const sizes = doc.sentences.map((sentence) => charLength(sentence));
   if (sizes.length < MIN_SENTENCES) return [];
   const spread = Math.round(coefficientOfVariation(sizes) * 100);
   if (spread >= options.limit) return [];
@@ -21,7 +22,7 @@ export const sentenceRhythm: Detector = (doc, options): Finding[] => {
       severity: "warning",
       line: 0,
       column: 0,
-      quote: doc.sentences[0]?.text.trim() ?? "",
+      quote: doc.sentences[0] === undefined ? "" : proseText(doc.sentences[0]),
       values: { count: spread, limit: options.limit, offset: doc.sentences[0]?.span.start ?? 0 },
     },
   ];

package/src/document.ts

@@ -13,8 +13,14 @@ type Node = {
   readonly children?: readonly Node[] | undefined;
 };
 
-/** 本文として数えないもの。コードは文章ではなく、表は文章の形をしていない。 */
-const NOT_PROSE = new Set(["code", "inlineCode", "html", "yaml", "toml", "table", "thematicBreak", "definition", "image", "imageReference"]);
+/**
+ * 本文として数えないもの。
+ *
+ * コードは文章ではなく、表は文章の形をしていない。
+ * **引用は自分の文章ではない。** chaff が言えるのは「2 文に割ってください」までで、
+ * 引用文にそれはできない。直せないものを指摘しても、書いた人は動けない。
+ */
+const NOT_PROSE = new Set(["code", "inlineCode", "html", "yaml", "toml", "table", "blockquote", "thematicBreak", "definition", "image", "imageReference"]);
 
 const spanOf = (node: Node): Span | undefined => {
   const start = node.position?.start.offset;
@@ -80,12 +86,21 @@ const headingsOf = (root: Node, source: string): Heading[] => {
   return found;
 };
 
-const strongSpans = (root: Node): Span[] => {
+const startsInside = (span: Span, regions: readonly Span[]): boolean => regions.some((region) => span.start >= region.start && span.start < region.end);
+
+/**
+ * 本文の強調だけを数える。
+ *
+ * 表のセルや見出しの中の太字はラベルであって強調ではない。
+ * 「太字は読者の目を止める道具」という bold-density の理屈が当てはまらない。
+ * 「本文でないもの」の定義は覆う範囲（collectMasks）に 1 つだけ置き、ここはそれを使う。
+ */
+const strongSpans = (root: Node, masked: readonly Span[]): Span[] => {
   const found: Span[] = [];
   walk(root, (node) => {
     if (node.type !== "strong") return;
     const span = spanOf(node);
-    if (span !== undefined) found.push(span);
+    if (span !== undefined && !startsInside(span, masked)) found.push(span);
   });
   return found;
 };
@@ -140,14 +155,15 @@ const sectionsOf = (headings: readonly Heading[], sentences: readonly Sentence[]
 
 export const buildDocument = (path: string, source: string, adapter: LanguageAdapter): ProseDocument => {
   const root = parse(source);
-  const prose = maskSpans(source, collectMasks(root));
+  const masked = collectMasks(root);
+  const prose = maskSpans(source, masked);
   const sentences = sentencesOf(prose, paragraphSpans(root), adapter);
   return {
     path,
     source,
     language: adapter.id,
     lengthUnit: adapter.capabilities.lengthUnit,
-    sections: sectionsOf(headingsOf(root, source), sentences, strongSpans(root), source.length),
+    sections: sectionsOf(headingsOf(root, source), sentences, strongSpans(root, masked), source.length),
     sentences,
     lexicons: adapter.lexicons,
   };

package/src/eval.ts

@@ -0,0 +1,80 @@
+import { DETECTORS } from "./detectors/index.ts";
+import { charLength } from "./measure.ts";
+import type { ProseDocument, RuleDefinition } from "./plugin.ts";
+
+/**
+ * 閾値の較正。
+ *
+ * examples/ に置いた文書は、実際に公開した「人間の良い文書」。そこで多く発火する
+ * rule は、閾値が現実に合っていない疑いがある。spec §21 の目標（人間の良い文書での
+ * 誤検知率 5% 未満）を、そのまま推奨の判断に使う。
+ *
+ * これは較正であって正解ではない。corpus が変われば答えも変わる。だから自動では
+ * 適用せず、提示して選ばせる。
+ */
+export const TARGET_HIT_RATE = 0.05;
+
+/** 文書単位の割合は corpus が小さいと粗い。文字数あたりの密度を併記する。 */
+export type Point = { readonly limit: number; readonly findings: number; readonly documents: number; readonly per10k: number };
+
+/** 5% を表せるだけの文書数。これを下回ると割合が「0 か全部」になる。 */
+export const MIN_CORPUS = 20;
+
+export type RuleReport = {
+  readonly rule: string;
+  readonly name: string;
+  readonly current: number;
+  readonly sweep: readonly Point[];
+  readonly recommended: number | undefined;
+  readonly total: number;
+};
+
+/** strict が relaxed より大きい rule は、値が大きいほど厳しい（sentence-rhythm）。 */
+const stricterIsHigher = (rule: RuleDefinition): boolean => (rule.levels.strict ?? 0) > (rule.levels.relaxed ?? 0);
+
+const MULTIPLIERS = [0.5, 0.7, 0.85, 1, 1.25, 1.5, 2, 3];
+
+/** rule 自身が持つ段を基点に掃引する。rule ごとに単位が違うので、倍率で散らす。 */
+const candidates = (rule: RuleDefinition): number[] => {
+  const base = rule.levels.normal ?? 1;
+  const fromLevels = [rule.levels.strict, rule.levels.normal, rule.levels.relaxed].filter((value) => value !== undefined);
+  const scaled = MULTIPLIERS.map((factor) => Math.max(1, Math.round(base * factor)));
+  return [...new Set([...fromLevels, ...scaled])].sort((left, right) => left - right);
+};
+
+const countAt = (docs: readonly ProseDocument[], rule: RuleDefinition, limit: number): Point => {
+  const detector = DETECTORS[rule.how_to_find];
+  if (detector === undefined) return { limit, findings: 0, documents: 0, per10k: 0 };
+  const perDoc = docs.map((doc) => {
+    const lexicon = rule.word_list === undefined ? undefined : doc.lexicons[rule.word_list];
+    if (rule.word_list !== undefined && lexicon === undefined) return 0;
+    return detector(doc, { limit, lexicon, where: rule.where }).length;
+  });
+  const findings = perDoc.reduce((sum, count) => sum + count, 0);
+  const chars = docs.reduce((sum, doc) => sum + doc.sentences.reduce((inner, sentence) => inner + charLength(sentence), 0), 0);
+  return { limit, findings, documents: perDoc.filter((count) => count > 0).length, per10k: chars === 0 ? 0 : (findings / chars) * 10000 };
+};
+
+/** 目標の誤検知率を満たす中で、いちばん厳しい閾値を選ぶ。 */
+const recommend = (rule: RuleDefinition, sweep: readonly Point[], total: number): number | undefined => {
+  const ok = sweep.filter((point) => point.documents / Math.max(1, total) <= TARGET_HIT_RATE);
+  if (ok.length === 0) return undefined;
+  const strictest = stricterIsHigher(rule) ? Math.max(...ok.map((p) => p.limit)) : Math.min(...ok.map((p) => p.limit));
+  return strictest;
+};
+
+export const evaluate = (docs: readonly ProseDocument[], rules: readonly RuleDefinition[], genre: string, language: string): RuleReport[] =>
+  rules
+    .filter((rule) => rule.layer !== "L4" && rule.use_for.some((target) => genre.startsWith(target)))
+    .map((rule) => {
+      const sweep = candidates(rule).map((limit) => countAt(docs, rule, limit));
+      const current = rule.levels.normal ?? 1;
+      return {
+        rule: rule.id,
+        name: rule.name[language] ?? rule.id,
+        current,
+        sweep,
+        recommended: recommend(rule, sweep, docs.length),
+        total: docs.length,
+      };
+    });