chaffjs 0.4.0 → 0.5.1

package/src/files.ts

@@ -26,7 +26,7 @@ const expand = (target: string): string[] => {
  * ロケールを明示する。既定のロケールに任せると、同じ入力でも機械によって順序が変わる。
  * 指摘の並びは CI のログにも baseline にも入るので、順序が揺れてはいけない。
  */
-export const byPath = (left: string, right: string): number => left.localeCompare(right, "en");
+const byPath = (left: string, right: string): number => left.localeCompare(right, "en");
 
 export const collectTargets = (targets: readonly string[]): string[] => {
   const found = targets.flatMap(expand).filter((path) => !isSkipped(path));

package/src/levels.ts

@@ -1,6 +1,6 @@
 import type { Level, LevelTable, RuleDefinition } from "./plugin.ts";
 
-export const LEVELS: readonly Level[] = ["strict", "normal", "relaxed", "off"];
+const LEVELS: readonly Level[] = ["strict", "normal", "relaxed", "off"];
 
 export const isLevel = (value: unknown): value is Level => typeof value === "string" && LEVELS.some((level) => level === value);
 

package/src/mask.ts

@@ -7,7 +7,14 @@ import type { Span } from "./plugin.ts";
  * 切り出して繋ぐのではなく覆うのは、オフセットを元文字列と一致させ続けるため。
  * 行・列の計算も、引用して見せる範囲も、これで元の位置のまま使える。
  */
-const blankOut = (text: string): string => text.replace(/[^\n]/gu, " ");
+/**
+ * 空白は**元の文字と同じ UTF-16 長**にする。
+ *
+ * `u` 付きの正規表現は 1 文字（コードポイント）ずつ当たるので、絵文字のような
+ * サロゲートペアを空白 1 つに置き換えると文字列が 1 だけ縮む。mdast のオフセットは
+ * UTF-16 単位なので、そこから先の指摘がすべて 1 ずれる。
+ */
+const blankOut = (text: string): string => text.replace(/[^\n]/gu, (char) => " ".repeat(char.length));
 
 const merge = (spans: readonly Span[]): Span[] =>
   [...spans]

package/src/measure.ts

@@ -13,7 +13,7 @@ export const proseText = (sentence: Sentence): string => sentence.text.replace(/
 
 export const charLength = (sentence: Sentence): number => sentence.text.replace(/\s+/gu, "").length;
 
-export const wordLength = (sentence: Sentence): number =>
+const wordLength = (sentence: Sentence): number =>
   proseText(sentence)
     .split(" ")
     .filter((word) => word.length > 0).length;

package/src/plugin.ts

@@ -32,7 +32,11 @@ export type Segmentation = {
 export type LengthUnit = "char" | "word";
 
 /** L2 の語彙表。detector は共通で、これだけが言語別。spec §11。 */
-export type LexiconEntry = { readonly pattern: string; readonly weight?: number | undefined };
+/**
+ * `instead_of` は「同じことを言う別の書きかた」。文体の一貫性を見る rule が使う。
+ * 2 つの書きかたのどちらが正しいかは決めず、**1 つの文書で混ざっていないか**だけを見る。
+ */
+export type LexiconEntry = { readonly pattern: string; readonly weight?: number | undefined; readonly instead_of?: string | undefined };
 
 export type Lexicon = readonly LexiconEntry[];
 
@@ -163,6 +167,8 @@ export type RuleDefinition = {
   readonly where: string | undefined;
   /** adapter に要る capability。"pos" / "lemma"。満たさなければ動かさない。spec §16。 */
   readonly requires: readonly string[];
+  /** 複合シグナル。ここに並べた rule のうち何本が出たかを見る。spec §20.2。 */
+  readonly from: readonly string[];
   /** 動かす言語。未指定は全言語。「ですます調」のように言語に固有の rule が使う。 */
   readonly languages: readonly string[] | undefined;
   readonly use_for: readonly string[];

package/src/position.ts

@@ -1,7 +1,16 @@
 export type Place = { readonly line: number; readonly column: number };
 
-/** 行頭のオフセット表。1 文書につき 1 度だけ作る。 */
-export const lineStarts = (source: string): number[] => [...source].reduce<number[]>((acc, char, index) => (char === "\n" ? [...acc, index + 1] : acc), [0]);
+/**
+ * 行頭のオフセット表。1 文書につき 1 度だけ作る。
+ *
+ * 数えるのは **UTF-16 単位**。`[...source]` はコードポイントで数えるので、
+ * 絵文字より後ろの行がすべて 1 ずれる。指摘が持つオフセットは UTF-16 なので、そちらに合わせる。
+ */
+export const lineStarts = (source: string): number[] => {
+  const starts = [0];
+  for (let at = source.indexOf("\n"); at !== -1; at = source.indexOf("\n", at + 1)) starts.push(at + 1);
+  return starts;
+};
 
 export const placeOf = (starts: readonly number[], offset: number): Place => {
   const index = starts.reduce((best, start, at) => (start <= offset ? at : best), 0);

package/src/render/rules-json.ts

@@ -1,6 +1,6 @@
 import { definedLevels, resolve } from "../levels.ts";
 import type { Config } from "../config/load.ts";
-import type { Level, RuleDefinition } from "../plugin.ts";
+import type { RuleDefinition } from "../plugin.ts";
 
 const now = (rule: RuleDefinition, config: Config, genre: string): Record<string, unknown> => {
   const explicit = config.rules[rule.id];
@@ -53,5 +53,3 @@ export const rulesJson = (rules: readonly RuleDefinition[], config: Config, lang
     null,
     2,
   );
-
-export const levelsForCli = (): readonly Level[] => ["strict", "normal", "relaxed", "off"];

package/src/render/sarif.ts

@@ -0,0 +1,103 @@
+import { localized, messageOf } from "./text.ts";
+import type { Finding, RuleDefinition, Severity } from "../plugin.ts";
+
+/**
+ * 指摘を SARIF 2.1.0 で出す。GitHub の code scanning に上げるため。
+ *
+ * 上げると Security タブに並ぶだけでなく、**PR の変更行に直接出る**。
+ * ログの中の指摘は読みに行かないと見えないが、行の上の指摘は書いた本人の目の前にある。
+ * chaff は文章を直す人のための道具なので、ここが効く。
+ */
+const SCHEMA = "https://json.schemastore.org/sarif-2.1.0.json";
+const INFORMATION_URI = "https://www.npmjs.com/package/chaffjs";
+
+/** SARIF に info は無い。上げるときに弾かれるので、note に写す。 */
+type Level = "error" | "warning" | "note";
+
+const levelOf = (severity: Severity): Level => (severity === "info" ? "note" : severity);
+
+type SarifRule = {
+  readonly id: string;
+  readonly name: string;
+  readonly shortDescription: { readonly text: string };
+  readonly fullDescription: { readonly text: string };
+  readonly help: { readonly text: string };
+  readonly properties: { readonly tags: readonly string[] };
+};
+
+type SarifResult = {
+  readonly ruleId: string;
+  readonly level: Level;
+  readonly message: { readonly text: string };
+  readonly locations: readonly {
+    readonly physicalLocation: {
+      readonly artifactLocation: { readonly uri: string };
+      readonly region: { readonly startLine: number; readonly startColumn: number };
+    };
+  }[];
+};
+
+/** `chaff/<rule>` と名前空間を切る。同じ PR に上がる他のツールの rule id と衝突させない。 */
+const ruleIdOf = (rule: string): string => `chaff/${rule}`;
+
+/** 指摘 1 件と、それを描くのに要るもの。言語はファイルごとに違う。 */
+export type Located = { readonly path: string; readonly finding: Finding; readonly language: string; readonly rules: readonly RuleDefinition[] };
+
+const ruleFor = (located: Located): RuleDefinition | undefined => located.rules.find((rule) => rule.id === located.finding.rule);
+
+const sarifRule = (rule: RuleDefinition, language: string): SarifRule => ({
+  id: ruleIdOf(rule.id),
+  name: rule.id,
+  shortDescription: { text: localized(rule.name, language) },
+  // なぜ直すのかと、どう直すのかを一緒に上げる。指摘だけでは書いた人が動けない。
+  fullDescription: { text: localized(rule.why, language) },
+  help: { text: localized(rule.how_to_fix, language) },
+  properties: { tags: [rule.layer, ...rule.use_for] },
+});
+
+const resultOf = (located: Located): SarifResult => {
+  const rule = ruleFor(located);
+  return {
+    ruleId: ruleIdOf(located.finding.rule),
+    level: levelOf(located.finding.severity),
+    message: { text: rule === undefined ? located.finding.rule : messageOf(rule, located.finding, located.language) },
+    locations: [
+      {
+        physicalLocation: {
+          artifactLocation: { uri: located.path },
+          // SARIF の行と桁は 1 始まり。0 を上げると弾かれる。
+          region: { startLine: Math.max(1, located.finding.line), startColumn: Math.max(1, located.finding.column) },
+        },
+      },
+    ],
+  };
+};
+
+/** 同じ rule が 2 言語で出たら、先に出たほうの言語で 1 度だけ書く。SARIF は rule を 1 つしか持てない。 */
+const usedRules = (located: readonly Located[]): SarifRule[] => {
+  const seen = new Map<string, Located>();
+  located.forEach((entry) => {
+    if (!seen.has(entry.finding.rule)) seen.set(entry.finding.rule, entry);
+  });
+  return [...seen.values()].flatMap((entry) => {
+    const rule = ruleFor(entry);
+    return rule === undefined ? [] : [sarifRule(rule, entry.language)];
+  });
+};
+
+export const renderSarif = (located: readonly Located[], version: string): string => {
+  return `${JSON.stringify(
+    {
+      $schema: SCHEMA,
+      version: "2.1.0",
+      runs: [
+        {
+          tool: { driver: { name: "chaff", version, informationUri: INFORMATION_URI, rules: usedRules(located) } },
+          results: located.map(resultOf),
+        },
+      ],
+    },
+    null,
+    2,
+  )}\n`;
+};

package/src/render/semantic.ts

@@ -22,7 +22,7 @@ const narrowingLines = (result: SemanticResult): string[] =>
     return [`    ${name}: ${what} を含む ${narrowing.kept} 文だけを読みました（全 ${narrowing.total} 文）`];
   });
 
-export const aiBanner = (result: SemanticResult): string[] => [
+const aiBanner = (result: SemanticResult): string[] => [
   "",
   `═══ AI による判定 ${"═".repeat(RULE - 17)}`,
   "    文章の意味を読んでいます。実行するたび結果が変わることが",

package/src/render/text.ts

@@ -2,7 +2,7 @@ import type { Finding, Localized, RuleDefinition } from "../plugin.ts";
 
 export const localized = (field: Localized, language: string): string => field[language] ?? field["en"] ?? Object.values(field)[0] ?? "";
 
-export const fill = (template: string, values: Readonly<Record<string, string | number>>): string =>
+const fill = (template: string, values: Readonly<Record<string, string | number>>): string =>
   template.replace(/\{(\w+)\}/gu, (whole, key: string) => {
     const value = values[key];
     return value === undefined ? whole : String(value);

package/src/rule-load.ts

@@ -63,11 +63,22 @@ const missingFields = (raw: Record<string, unknown>, levels: LevelTable | undefi
   ...(levels?.normal === undefined ? ["levels.normal"] : []),
   ...(typeof raw["how_to_find"] === "string" ? [] : ["how_to_find"]),
   ...(Array.isArray(raw["use_for"]) ? [] : ["use_for"]),
-  ...(isSeverity(raw["severity"]) ? [] : ["severity"]),
+  ...(isSeverity(raw["severity"]) || isRecord(raw["severity"]) ? [] : ["severity"]),
 ];
 
 const localizedOf = (value: unknown): Record<string, string> => (isLocalized(value) ? value : {});
 
+/**
+ * severity も言語別に書ける。levels と同じ畳みかた。
+ * 記号の許容度は言語で大きく違う（ダッシュは英語では普通、日本語の組版では扱いが難しい）。
+ */
+const severityOf = (raw: unknown, language: string): Severity => {
+  if (isSeverity(raw)) return raw;
+  if (!isRecord(raw)) return "warning";
+  const picked: unknown = raw[language] ?? raw["default"];
+  return isSeverity(picked) ? picked : "warning";
+};
+
 const stringList = (value: unknown): string[] | undefined => (Array.isArray(value) ? value.map((entry) => String(entry)) : undefined);
 
 const toRule = (raw: unknown, language: string, file: string): RuleDefinition => {
@@ -91,9 +102,10 @@ const toRule = (raw: unknown, language: string, file: string): RuleDefinition =>
     what_to_check: isLocalized(raw["what_to_check"]) ? raw["what_to_check"] : undefined,
     where: typeof raw["where"] === "string" ? raw["where"] : undefined,
     requires: stringList(raw["requires"]) ?? [],
+    from: stringList(raw["from"]) ?? [],
     languages: stringList(raw["languages"]),
     use_for: Array.isArray(raw["use_for"]) ? raw["use_for"].map((entry) => String(entry)) : [],
-    severity: isSeverity(raw["severity"]) ? raw["severity"] : "warning",
+    severity: severityOf(raw["severity"], language),
   };
 };
 

package/src/run-semantic.ts

@@ -25,7 +25,11 @@ export type Job = {
   readonly narrowing?: Narrowing;
 };
 
-const filterFor = (name: string | undefined): (typeof FILTERS)[string] => FILTERS[name ?? "whole-document"] ?? FILTERS["whole-document"] ?? (() => []);
+/**
+ * 知らない絞り込みを文書全体に落とさない。spec §14 は「絞り込みを持たない L4 rule は
+ * 登録できない」と決めている。落とすと、**書いていない範囲が黙って LLM に送られる**。
+ */
+const filterFor = (name: string | undefined): (typeof FILTERS)[string] | undefined => FILTERS[name ?? "whole-document"];
 
 const SEVERITY_BY_VALUE: readonly Severity[] = ["info", "info", "warning", "error"];
 
@@ -41,6 +45,9 @@ const builtInJobs = (doc: ProseDocument, rules: readonly RuleDefinition[], setti
     .flatMap((rule) => {
       const rubric = rule.what_to_check === undefined ? undefined : localized(rule.what_to_check, doc.language);
       if (rubric === undefined || rubric.length === 0) return [];
+      // 絞り込みが無い rule は動かさない。黙って落とさず、下の skipped に出す。
+      const filter = filterFor(rule.how_to_find);
+      if (filter === undefined) return [];
       return [
         {
           rule: rule.id,
@@ -48,7 +55,7 @@ const builtInJobs = (doc: ProseDocument, rules: readonly RuleDefinition[], setti
           rubric,
           howToFix: localized(rule.how_to_fix, doc.language),
           severity: severityFromLevel(rule, settings[rule.id] ?? "normal"),
-          candidates: filterFor(rule.how_to_find)(doc),
+          candidates: filter(doc),
         },
       ];
     });
@@ -58,7 +65,7 @@ const userJobs = (doc: ProseDocument, checks: readonly UserCheck[], genre: strin
     .filter((check) => check.level !== "off" && check.use_for.some((target) => genre.startsWith(target)))
     .map((check) => {
       // look_at の括弧の中を取り出して絞り込む。取り出せなければ文書全体。spec §14。
-      const { candidates, narrowing } = narrow(doc, check.look_at, filterFor("whole-document"));
+      const { candidates, narrowing } = narrow(doc, check.look_at, FILTERS["whole-document"] ?? (() => []));
       return {
         rule: check.id,
         name: check.name,
@@ -108,7 +115,10 @@ export const runSemantic = async (
 ): Promise<SemanticResult> => {
   const jobs = planSemantic(doc, rules, checks, settings, genre);
   const starts = lineStarts(doc.source);
-  const skipped = jobs.filter((job) => job.candidates.length === 0).map((job) => ({ rule: job.rule, why: "見るところが無かったため" }));
+  const unfiltered = rules
+    .filter((rule) => rule.layer === "L4" && rule.use_for.some((target) => genre.startsWith(target)) && filterFor(rule.how_to_find) === undefined)
+    .map((rule) => ({ rule: rule.id, why: `絞り込み ${rule.how_to_find} が無いため（文書全体は送りません）` }));
+  const skipped = [...jobs.filter((job) => job.candidates.length === 0).map((job) => ({ rule: job.rule, why: "見るところが無かったため" })), ...unfiltered];
   const live = jobs.filter((job) => job.candidates.length > 0);
   const answers = await Promise.all(
     live.flatMap((job) =>

package/src/run.ts

@@ -31,12 +31,22 @@ const has = (capabilities: ProseDocument["capabilities"], need: string): boolean
   return false;
 };
 
+/**
+ * 宣言だけでなく、実際に token が来ているかも見る。
+ *
+ * `capabilities.pos: true` と言いながら token を返さないアダプタでも、rule は
+ * `sentence.tokens ?? []` を見るので**例外にならず、指摘 0 件で終わる**。
+ * 0 件は「問題なし」と見分けがつかない。
+ */
+const hasTokens = (doc: ProseDocument): boolean => doc.sentences.length === 0 || doc.sentences.some((sentence) => sentence.tokens !== undefined);
+
 /** 要求を満たさない rule は動かせない。満たさないまま動かすと「指摘 0 件」が保証に見える。 */
 const unmet = (rule: RuleDefinition, doc: ProseDocument): string | undefined => {
   if (rule.languages !== undefined && !rule.languages.includes(doc.language)) return `${doc.language} 向けの rule ではないため`;
   const missing = rule.requires.find((need) => !has(doc.capabilities, need));
-  if (missing === undefined) return undefined;
-  return `この言語では${CAPABILITY_NAME[missing] ?? missing}が使えないため`;
+  if (missing !== undefined) return `この言語では${CAPABILITY_NAME[missing] ?? missing}が使えないため`;
+  if (rule.requires.length > 0 && !hasTokens(doc)) return "アダプタが品詞を返さなかったため";
+  return undefined;
 };
 
 const forGenre = (rules: readonly RuleDefinition[], genre: string): RuleDefinition[] =>
@@ -59,6 +69,29 @@ const place = (starts: readonly number[], finding: Finding): Finding => {
   return { ...finding, line: at.line, column: at.column };
 };
 
+/**
+ * 複合シグナル。他の rule の結果を読むので、detector の形には収まらない。run の二段目。spec §20.2。
+ *
+ * 1 本ずつでは何も言えないものが揃ったときだけ、1 件にまとめて出す。
+ * 元の指摘は消さない。`padded-intro` のように単独でも正しい指摘が混ざっており、
+ * まとめるために消すと本物の指摘が見えなくなる。
+ */
+const compositeOf = (rule: RuleDefinition, found: readonly Finding[], limit: number, starts: readonly number[]): Finding[] => {
+  const fired = rule.from.filter((id) => found.some((finding) => finding.rule === id));
+  if (fired.length < limit) return [];
+  const first = found.find((finding) => fired.includes(finding.rule));
+  return [
+    place(starts, {
+      rule: rule.id,
+      severity: rule.severity,
+      line: 0,
+      column: 0,
+      quote: first?.quote ?? "",
+      values: { word: fired.join("、"), count: fired.length, limit, offset: first === undefined ? 0 : Number(first.values["offset"] ?? 0) },
+    }),
+  ];
+};
+
 export const runRules = (doc: ProseDocument, rules: readonly RuleDefinition[], settings: Settings, experimental: boolean, genre: string): RunResult => {
   const starts = lineStarts(doc.source);
   const applicable = forGenre(rules, genre);
@@ -78,6 +111,8 @@ export const runRules = (doc: ProseDocument, rules: readonly RuleDefinition[], s
         const why = rule.status === "experimental" && settings[rule.id] === undefined ? "まだ試験中のため" : "設定で止めているため";
         return { findings: acc.findings, skipped: [...acc.skipped, { rule: rule.id, why }] };
       }
+      // 複合シグナルは二段目で扱う。一段目では「検出器が無い」と言わせない。
+      if (rule.from.length > 0) return acc;
       const detector = DETECTORS[rule.how_to_find];
       if (detector === undefined) return { findings: acc.findings, skipped: [...acc.skipped, { rule: rule.id, why: `検出器 ${rule.how_to_find} がないため` }] };
       const options = {
@@ -94,5 +129,9 @@ export const runRules = (doc: ProseDocument, rules: readonly RuleDefinition[], s
     },
     { findings: [], skipped: [] },
   );
-  return { ...outcome, findings: [...outcome.findings].sort((left, right) => left.line - right.line), forcedExperimental: forced };
+  const composites = applicable
+    .filter((rule) => rule.from.length > 0 && levelFor(rule, settings, experimental) !== "off")
+    .flatMap((rule) => compositeOf(rule, outcome.findings, resolve(rule, levelFor(rule, settings, experimental), genre).limit, starts));
+  const all = [...outcome.findings, ...composites];
+  return { ...outcome, findings: [...all].sort((left, right) => left.line - right.line), forcedExperimental: forced };
 };

package/src/sentence-shape.ts

@@ -19,4 +19,22 @@ export const hasParticle = (sentence: Sentence): boolean => (sentence.tokens ??
 
 const SKIP = new Set(["PUNCT", "PART", "SYM"]);
 
-export const lastContent = (sentence: Sentence): Token | undefined => [...(sentence.tokens ?? [])].reverse().find((token) => !SKIP.has(token.pos));
+/**
+ * 文末の括弧は、文の終わりではなく添え物。
+ *
+ * 「これを最優先制約とする（§17）。」の述語は「とする」であって「17」ではない。
+ * 仕様書は相互参照を括弧で添えるので、これを数えると文末が全部そこになる。
+ */
+const TRAILING = /[(（][^(（]*[)）][\s。．！？!?]*$/u;
+
+const beforeTrailing = (sentence: Sentence): readonly Token[] => {
+  const tokens = sentence.tokens ?? [];
+  const match = TRAILING.exec(sentence.text);
+  if (match?.index === undefined) return tokens;
+  const cut = sentence.span.start + match.index;
+  const kept = tokens.filter((token) => token.span.end <= cut);
+  // 括弧を外したら何も残らない文は、括弧そのものが中身。そのまま見る。
+  return kept.length === 0 ? tokens : kept;
+};
+
+export const lastContent = (sentence: Sentence): Token | undefined => [...beforeTrailing(sentence)].reverse().find((token) => !SKIP.has(token.pos));

package/src/version.ts

@@ -0,0 +1,14 @@
+import { readFileSync } from "node:fs";
+import { fileURLToPath } from "node:url";
+import { dirname, join } from "node:path";
+
+/**
+ * 版は manifest から読む。定数に書くと、上げ忘れたまま古い版を名乗るものが出る。
+ * SARIF の tool.version がそれで、指摘を出していない版に指摘が紐づく。
+ */
+const MANIFEST = join(dirname(fileURLToPath(import.meta.url)), "..", "package.json");
+
+const versionOf = (raw: unknown): string =>
+  typeof raw === "object" && raw !== null && "version" in raw && typeof raw.version === "string" ? raw.version : "0.0.0";
+
+export const VERSION = versionOf(JSON.parse(readFileSync(MANIFEST, "utf8")));