chaffjs 0.0.1 → 0.2.0

package/src/genre.ts

@@ -1,12 +1,19 @@
 export type GenreGuess = { readonly genre: string; readonly from: string };
 
+/**
+ * 仕様書・README は「読ませる文章」ではなく「間違えさせない文章」。
+ * 書き出しのつかみもリズムも要らない。技術文書として別に見る。
+ */
 const BY_PATH: readonly (readonly [RegExp, string])[] = [
+  [/(^|\/)readme\.md$/iu, "technical/readme"],
+  [/spec[^/]*\.md$/iu, "technical/spec"],
+  [/(^|\/)(spec|specs)\//iu, "technical/spec"],
   [/(^|\/)(proposals?|teian)\//iu, "business/proposal"],
   [/(^|\/)(minutes|gijiroku)\//iu, "business/meeting-notes"],
   [/(^|\/)(press|pr)\//iu, "business/press-release"],
   [/(^|\/)(reports?)\//iu, "business/report"],
   [/(^|\/)(blog|posts?|articles?)\//iu, "blog/tech"],
-  [/(^|\/)docs?\//iu, "blog/tech"],
+  [/(^|\/)docs?\//iu, "technical/readme"],
 ];
 
 /**

package/src/init.ts

@@ -65,6 +65,8 @@ export const runInit = (dir: string, genre: string): string[] => {
 };
 
 export const GENRES: readonly string[] = [
+  "technical/spec",
+  "technical/readme",
   "blog/tech",
   "blog/essay",
   "blog/owned-media",

package/src/measure.ts

@@ -0,0 +1,21 @@
+import type { Sentence } from "./plugin.ts";
+
+/**
+ * 文の長さ。**覆った部分の空白を数えない。**
+ *
+ * 非 prose は同じ長さの空白で覆っている（オフセットを保つため）。そのまま数えると、
+ * URL を 1 つ含む文が「245 文字」になる。実文書で max-sentence-length が指摘の
+ * 8 割を占めていた原因はこれで、読み手が長いと感じる長さではなかった。
+ *
+ * 日本語は文字数、英語は語数。単位は adapter が宣言する。
+ */
+export const proseText = (sentence: Sentence): string => sentence.text.replace(/\s+/gu, " ").trim();
+
+export const charLength = (sentence: Sentence): number => sentence.text.replace(/\s+/gu, "").length;
+
+export const wordLength = (sentence: Sentence): number =>
+  proseText(sentence)
+    .split(" ")
+    .filter((word) => word.length > 0).length;
+
+export const lengthOf = (sentence: Sentence, unit: "char" | "word"): number => (unit === "char" ? charLength(sentence) : wordLength(sentence));

package/src/plugin.ts

@@ -118,6 +118,8 @@ export type RuleDefinition = {
   readonly levels: LevelTable;
   readonly how_to_find: string;
   readonly word_list: string | undefined;
+  /** L4 のみ。LLM に渡す決まり。言語別。 */
+  readonly what_to_check: Localized | undefined;
   readonly where: string | undefined;
   readonly use_for: readonly string[];
   readonly severity: Severity;

package/src/render/eval.ts

@@ -0,0 +1,62 @@
+import { MIN_CORPUS, TARGET_HIT_RATE, type RuleReport } from "../eval.ts";
+
+const pct = (part: number, whole: number): string => `${((part / Math.max(1, whole)) * 100).toFixed(1)}%`;
+
+const mark = (report: RuleReport, limit: number): string => {
+  if (limit === report.current && limit === report.recommended) return "  ← 現在 / 推奨";
+  if (limit === report.current) return "  ← 現在";
+  if (limit === report.recommended) return "  ← 推奨";
+  return "";
+};
+
+const sweepLines = (report: RuleReport): string[] =>
+  report.sweep.map((point) => {
+    const limit = String(point.limit).padStart(6);
+    const docs = `${String(point.documents).padStart(3)} 文書`;
+    const density = `1万字あたり ${point.per10k.toFixed(1)}`;
+    return `    ${limit}   ${docs} (${pct(point.documents, report.total).padStart(6)})   指摘 ${String(point.findings).padStart(3)} 件   ${density}${mark(report, point.limit)}`;
+  });
+
+const verdict = (report: RuleReport): string[] => {
+  const now = report.sweep.find((point) => point.limit === report.current);
+  const rate = now === undefined ? 0 : now.documents / Math.max(1, report.total);
+  if (rate <= TARGET_HIT_RATE) return [`    いまの閾値で目標（${TARGET_HIT_RATE * 100}% 未満）を満たしています。`];
+  if (report.recommended === undefined) {
+    return [`    どの閾値でも目標を満たしません。この corpus に対して rule 自体が合っていない可能性があります。`];
+  }
+  return [
+    `    いまの閾値では ${pct(now?.documents ?? 0, report.total)} の文書が該当し、目標（${TARGET_HIT_RATE * 100}% 未満）を超えています。`,
+    `    推奨: ${report.recommended}`,
+    "",
+    `    chaff.yaml に書くなら:  rules:\n                              ${report.rule}: ${report.recommended}`,
+  ];
+};
+
+const block = (report: RuleReport): string[] => ["", `  ${report.name}   (${report.rule})`, "", ...sweepLines(report), "", ...verdict(report), ""];
+
+/**
+ * 閾値は自動で書き換えない。提示して選ばせる。
+ * 較正は corpus に対する答えであって、正解ではないため。
+ */
+export const renderEval = (reports: readonly RuleReport[], total: number, paths: number): string =>
+  [
+    "",
+    `  ${paths} ファイルを corpus として ${reports.length} 本の rule を測りました。`,
+    "",
+    "  ここにある文書は「人間が書いて公開した良い文書」として扱っています。",
+    `  そこで多く発火する rule は、閾値が現実に合っていない疑いがあります（目標: ${TARGET_HIT_RATE * 100}% 未満）。`,
+    ...(total < MIN_CORPUS
+      ? [
+          "",
+          `  ※ corpus が ${total} 文書しかありません。${TARGET_HIT_RATE * 100}% を表すには ${MIN_CORPUS} 文書以上が要ります。`,
+          "     いまは割合が「0 か全部」に振れるので、右端の密度（1万字あたりの指摘数）のほうを見てください。",
+        ]
+      : []),
+    ...reports.flatMap(block),
+    "  " + "─".repeat(58),
+    "",
+    `  閾値は自動で書き換えていません。corpus が変われば答えも変わるためです。`,
+    "",
+  ]
+    .join("\n")
+    .replace(/\{total\}/gu, String(total));

package/src/render/semantic.ts

@@ -0,0 +1,60 @@
+import type { Finding, RuleDefinition } from "../plugin.ts";
+import type { UserCheck } from "../checks.ts";
+import type { SemanticResult } from "../run-semantic.ts";
+import { MARK, localized } from "./text.ts";
+
+const RULE = 60;
+
+const indent = (text: string, pad: string): string[] => text.split("\n").map((line) => `${pad}${line}`);
+
+export const MACHINE_BANNER = ["", `═══ 機械による判定 ${"═".repeat(RULE - 18)}`, "    同じ文章なら何度実行しても同じ結果になります", ""];
+
+export const aiBanner = (result: SemanticResult): string[] => [
+  "",
+  `═══ AI による判定 ${"═".repeat(RULE - 17)}`,
+  "    文章の意味を読んでいます。実行するたび結果が変わることが",
+  "    あります。おかしいと思ったら、そのまま無視して構いません。",
+  "",
+  `    ${result.sentencesSeen} 文のうち ${result.asked} 箇所を読みました（残りは機械が対象外と判断）`,
+  "",
+];
+
+type Named = { readonly name: string; readonly howToFix: string };
+
+const nameOf = (rule: string, rules: readonly RuleDefinition[], checks: readonly UserCheck[], language: string): Named => {
+  const built = rules.find((entry) => entry.id === rule);
+  if (built !== undefined) return { name: localized(built.name, language), howToFix: localized(built.how_to_fix, language) };
+  const user = checks.find((entry) => entry.id === rule);
+  return { name: user?.name ?? rule, howToFix: user?.how_to_fix ?? "" };
+};
+
+/**
+ * AI の指摘にだけ確からしさを出し、納得できないときの逃げ道を 2 つ添える。
+ * 揺れる判定だと読む人が知らないと、誤検知に振り回される。
+ */
+const block = (finding: Finding, named: Named): string[] => {
+  const confidence = finding.values["confidence"];
+  const reason = String(finding.values["reason"] ?? "");
+  const sure = confidence === undefined ? "" : `            確からしさ ${String(confidence)}`;
+  const fix = named.howToFix.length > 0 ? ["", ...indent(`→ ${named.howToFix}`, "     ")] : [];
+  return [
+    "",
+    `─── ${finding.line} 行目 ${"─".repeat(Math.max(0, RULE - String(finding.line).length - 8))}`,
+    "",
+    `  ${MARK[finding.severity] ?? "·"}  ${named.name}${sure}`,
+    "",
+    ...indent(reason, "     "),
+    ...fix,
+    "",
+    "     この指摘が違うと思ったら:",
+    `       この箇所だけ黙らせる    <!-- stet: ${finding.rule} — 理由 -->`,
+    `       ルールごとゆるめる      npx chaff relax ${finding.rule}`,
+    "",
+  ];
+};
+
+export const renderSemantic = (result: SemanticResult, rules: readonly RuleDefinition[], checks: readonly UserCheck[], language: string): string[] => [
+  ...aiBanner(result),
+  ...result.findings.flatMap((finding) => block(finding, nameOf(finding.rule, rules, checks, language))),
+  ...(result.skipped.length > 0 ? [`  ${result.skipped.length} 件の検査は、見るところが無いため動いていません。`, ""] : []),
+];

package/src/rule-load.ts

@@ -10,15 +10,25 @@ const isRecord = (value: unknown): value is Record<string, unknown> => typeof va
 
 const isLocalized = (value: unknown): value is Record<string, string> => isRecord(value) && Object.values(value).every((entry) => typeof entry === "string");
 
+const SEVERITY_BY_NAME: Readonly<Record<string, number>> = { info: 1, warning: 2, error: 3 };
+
+/** L4 は件数ではなく深刻度を段で持つ。error/warning/info を数に写して同じ器に入れる。 */
+const asNumber = (entry: unknown): number | undefined => {
+  if (typeof entry === "number") return entry;
+  return typeof entry === "string" ? SEVERITY_BY_NAME[entry] : undefined;
+};
+
 const isLevelTable = (value: unknown): value is LevelTable =>
-  isRecord(value) && Object.entries(value).every(([key, entry]) => ["strict", "normal", "relaxed"].includes(key) && typeof entry === "number");
+  isRecord(value) && Object.entries(value).every(([key, entry]) => ["strict", "normal", "relaxed"].includes(key) && asNumber(entry) !== undefined);
 
 /** 言語別の levels を、その言語のぶんだけに平坦化する。単位が言語で違う rule のため。 */
+const normalize = (table: LevelTable): LevelTable => Object.fromEntries(Object.entries(table).map(([key, entry]) => [key, asNumber(entry) ?? 0]));
+
 const flattenLevels = (raw: unknown, language: string): LevelTable | undefined => {
-  if (isLevelTable(raw)) return raw;
+  if (isLevelTable(raw)) return normalize(raw);
   if (!isRecord(raw)) return undefined;
   const forLanguage = raw[language] ?? raw["default"];
-  return isLevelTable(forLanguage) ? forLanguage : undefined;
+  return isLevelTable(forLanguage) ? normalize(forLanguage) : undefined;
 };
 
 const SEVERITIES: readonly Severity[] = ["error", "warning", "info"];
@@ -65,6 +75,7 @@ const toRule = (raw: unknown, language: string, file: string): RuleDefinition =>
     levels,
     how_to_find: String(raw["how_to_find"]),
     word_list: typeof raw["word_list"] === "string" ? raw["word_list"] : undefined,
+    what_to_check: isLocalized(raw["what_to_check"]) ? raw["what_to_check"] : undefined,
     where: typeof raw["where"] === "string" ? raw["where"] : undefined,
     use_for: Array.isArray(raw["use_for"]) ? raw["use_for"].map((entry) => String(entry)) : [],
     severity: isSeverity(raw["severity"]) ? raw["severity"] : "warning",

package/src/run-semantic.ts

@@ -0,0 +1,107 @@
+import { severityOf, type UserCheck } from "./checks.ts";
+import { ask, type JudgeOptions } from "./judge.ts";
+import { FILTERS, type Candidate } from "./semantic.ts";
+import { localized } from "./render/text.ts";
+import type { Finding, Level, ProseDocument, RuleDefinition, Severity } from "./plugin.ts";
+import { lineStarts, placeOf } from "./position.ts";
+
+export type SemanticResult = {
+  readonly findings: readonly Finding[];
+  readonly skipped: readonly { rule: string; why: string }[];
+  readonly asked: number;
+  readonly sentencesSeen: number;
+};
+
+type Job = {
+  readonly rule: string;
+  readonly name: string;
+  readonly rubric: string;
+  readonly howToFix: string;
+  readonly severity: Severity;
+  readonly candidates: readonly Candidate[];
+};
+
+const filterFor = (name: string | undefined): (typeof FILTERS)[string] => FILTERS[name ?? "whole-document"] ?? FILTERS["whole-document"] ?? (() => []);
+
+const SEVERITY_BY_VALUE: readonly Severity[] = ["info", "info", "warning", "error"];
+
+const severityFromLevel = (rule: RuleDefinition, level: Level): Severity => {
+  if (level === "off") return "info";
+  const value = rule.levels[level] ?? rule.levels.normal ?? 2;
+  return SEVERITY_BY_VALUE[Math.min(3, Math.max(1, Math.round(value)))] ?? "warning";
+};
+
+const builtInJobs = (doc: ProseDocument, rules: readonly RuleDefinition[], settings: Readonly<Record<string, Level>>, genre: string): Job[] =>
+  rules
+    .filter((rule) => rule.layer === "L4" && rule.use_for.some((target) => genre.startsWith(target)) && settings[rule.id] !== "off")
+    .flatMap((rule) => {
+      const rubric = rule.what_to_check === undefined ? undefined : localized(rule.what_to_check, doc.language);
+      if (rubric === undefined || rubric.length === 0) return [];
+      return [
+        {
+          rule: rule.id,
+          name: localized(rule.name, doc.language),
+          rubric,
+          howToFix: localized(rule.how_to_fix, doc.language),
+          severity: severityFromLevel(rule, settings[rule.id] ?? "normal"),
+          candidates: filterFor(rule.how_to_find)(doc),
+        },
+      ];
+    });
+
+const userJobs = (doc: ProseDocument, checks: readonly UserCheck[], genre: string): Job[] =>
+  checks
+    .filter((check) => check.level !== "off" && check.use_for.some((target) => genre.startsWith(target)))
+    .map((check) => ({
+      rule: check.id,
+      name: check.name,
+      // look_at は自然文なので、まだ絞り込みに変換できない。文書全体を渡す。spec §26-7。
+      rubric: check.look_at === undefined ? check.check : `${check.check}\n\n見るところ: ${check.look_at}`,
+      howToFix: check.how_to_fix,
+      severity: severityOf(check.level),
+      candidates: filterFor("whole-document")(doc),
+    }));
+
+const findingOf = (job: Job, candidate: Candidate, reason: string, confidence: number, threshold: number, starts: readonly number[]): Finding => {
+  const at = placeOf(starts, candidate.offset);
+  // 確からしさが低いものは指摘として出すが、重さを下げる。spec §14。
+  const severity = confidence < threshold ? "info" : job.severity;
+  return {
+    rule: job.rule,
+    severity,
+    line: at.line,
+    column: at.column,
+    quote: candidate.text.slice(0, 200),
+    values: { reason, confidence: confidence.toFixed(2), offset: candidate.offset },
+  };
+};
+
+export type SemanticOptions = JudgeOptions & { readonly confidenceThreshold: number };
+
+export const runSemantic = async (
+  doc: ProseDocument,
+  rules: readonly RuleDefinition[],
+  checks: readonly UserCheck[],
+  settings: Readonly<Record<string, Level>>,
+  genre: string,
+  options: SemanticOptions,
+): Promise<SemanticResult> => {
+  const jobs = [...builtInJobs(doc, rules, settings, genre), ...userJobs(doc, checks, genre)];
+  const starts = lineStarts(doc.source);
+  const skipped = jobs.filter((job) => job.candidates.length === 0).map((job) => ({ rule: job.rule, why: "見るところが無かったため" }));
+  const live = jobs.filter((job) => job.candidates.length > 0);
+  const answers = await Promise.all(
+    live.flatMap((job) =>
+      job.candidates.map(async (candidate) => {
+        const verdict = await ask({ rule: job.rule, rubric: job.rubric, candidate: candidate.text, language: doc.language }, options);
+        return verdict.violated ? findingOf(job, candidate, verdict.reason, verdict.confidence, options.confidenceThreshold, starts) : undefined;
+      }),
+    ),
+  );
+  return {
+    findings: answers.filter((finding) => finding !== undefined).sort((left, right) => left.line - right.line),
+    skipped,
+    asked: live.reduce((sum, job) => sum + job.candidates.length, 0),
+    sentencesSeen: doc.sentences.length,
+  };
+};

package/src/run.ts

@@ -36,6 +36,10 @@ export const runRules = (doc: ProseDocument, rules: readonly RuleDefinition[], s
     .map((rule) => rule.id);
   const outcome = applicable.reduce<{ findings: Finding[]; skipped: Skipped[] }>(
     (acc, rule) => {
+      // L4 は意味を読む検査。chaff test が扱う。ここで「検出器が無い」と言わせない。
+      if (rule.layer === "L4") {
+        return { findings: acc.findings, skipped: [...acc.skipped, { rule: rule.id, why: "意味を読む検査のため（npx chaff test で動きます）" }] };
+      }
       const level = levelFor(rule, settings, experimental);
       if (level === "off") {
         const why = rule.status === "experimental" && settings[rule.id] === undefined ? "まだ試験中のため" : "設定で止めているため";

package/src/semantic.ts

@@ -0,0 +1,74 @@
+import type { ProseDocument, Section } from "./plugin.ts";
+
+/**
+ * 二段構えの 1 段目。文書全体を LLM に投げない。spec §14。
+ *
+ * 絞り込みを持たない L4 rule は登録できない。持たせないと、1 記事ごとに全文を
+ * 読ませることになり、速度もコストも成り立たない。
+ */
+export type Candidate = { readonly text: string; readonly offset: number };
+
+export type Filter = (doc: ProseDocument) => Candidate[];
+
+const heads = (doc: ProseDocument): string => doc.sections.map((section) => section.heading).join(" ");
+
+const sectionText = (doc: ProseDocument, section: Section): string => doc.source.slice(section.span.start, section.span.end).trim();
+
+/** 見出しに答えが書いてあるなら、本文を読ませる必要はない。 */
+const skipIfHeadingMatches = (doc: ProseDocument, pattern: RegExp): boolean => pattern.test(heads(doc));
+
+const RISK_HEADING = /リスク|懸念|デメリット|risk|tradeoff|downside/iu;
+
+/** 提案文書にリスクが書かれているか。見出しにあれば見ない。 */
+export const riskDisclosure: Filter = (doc) => {
+  if (skipIfHeadingMatches(doc, RISK_HEADING)) return [];
+  const body = doc.sentences.map((sentence) => sentence.text.trim()).join("");
+  if (body.length === 0) return [];
+  return [{ text: body.slice(0, 4000), offset: doc.sentences[0]?.span.start ?? 0 }];
+};
+
+const EVIDENCE = /\d|`|https?:\/\//u;
+
+/**
+ * 結びが本文の要約でしかないか。
+ * 最後の節だけを見て、そこに具体物が無いときにだけ問い合わせる。
+ */
+export const emptyConclusion: Filter = (doc) => {
+  const last = doc.sections.at(-1);
+  if (last === undefined || last.sentences.length === 0) return [];
+  const text = sectionText(doc, last);
+  if (EVIDENCE.test(text)) return [];
+  return [{ text: text.slice(0, 2000), offset: last.sentences[0]?.span.start ?? last.span.start }];
+};
+
+// \s* を挟むとバックトラックする。単位の前の空白は 1 つまでで足りる。
+const NUMBER = /\d ?(?:%|％|倍|割|件|人|円|ドル|時間|分|秒)/u;
+const EFFECT = /向上|改善|削減|短縮|増加|減少|成長|伸び/u;
+/**
+ * 根拠と認めるもの。ここを狭く取ると、根拠つきの文まで LLM に送ってしまう。
+ * 二段構えの 1 段目が防ぐべきなのは、まさにその無駄。
+ */
+const EVIDENCE_WORDS = ["出典", "調査", "自社調べ", "参考", "期間", "比較", "対比", "前年", "昨年", "同期", "前月", "前四半期", "vs"];
+const EVIDENCE_SHAPE = /n ?=|\[\d+\]|\d{4} ?年|Q[1-4]/iu;
+
+/** 語の照合は正規表現にしない。数が増えるほど式が読めなくなり、複雑度も上がる。 */
+const hasEvidence = (text: string): boolean => EVIDENCE_WORDS.some((word) => text.includes(word)) || EVIDENCE_SHAPE.test(text);
+
+/** 効果を主張する数値に根拠が無い文だけを渡す。 */
+export const unsourcedNumber: Filter = (doc) =>
+  doc.sentences
+    .filter((sentence) => NUMBER.test(sentence.text) && EFFECT.test(sentence.text) && !hasEvidence(sentence.text))
+    .map((sentence) => ({ text: sentence.text.trim(), offset: sentence.span.start }));
+
+/** 利用者が checks.yaml に書いた検査の既定。絞り込みを書かなければ文書全体。 */
+export const wholeDocument: Filter = (doc) => {
+  const body = doc.sentences.map((sentence) => sentence.text.trim()).join("");
+  return body.length === 0 ? [] : [{ text: body.slice(0, 4000), offset: doc.sentences[0]?.span.start ?? 0 }];
+};
+
+export const FILTERS: Readonly<Record<string, Filter>> = {
+  "risk-disclosure": riskDisclosure,
+  "empty-conclusion": emptyConclusion,
+  "unsourced-number": unsourcedNumber,
+  "whole-document": wholeDocument,
+};