chaffjs 0.11.0 → 0.12.0

package/src/detectors/structure.ts

@@ -1,5 +1,7 @@
 import { charLength, lengthOf } from "../measure.ts";
-import type { BulletList, Detector, Finding, Paragraph, ProseDocument, Section } from "../plugin.ts";
+import type { BulletList, Detector, Finding, Paragraph, ProseDocument, Section, Span } from "../plugin.ts";
+import { dateStampIndexes, stampCandidates } from "../date-stamp.ts";
+import { inDocumentOrder } from "../structure/issues.ts";
 
 /**
  * ばらつきは変動係数（標準偏差 ÷ 平均）で測る。
@@ -90,6 +92,21 @@ export const ruleOfThree: Detector = (doc, options): Finding[] => {
   ];
 };
 
+const dateSpans = (doc: ProseDocument): Span[] =>
+  doc.structure === undefined ? [] : inDocumentOrder(doc.structure).flatMap((node) => (node.kind === "date" ? [node.span] : []));
+
+/** 更新日の刻印（「最終更新日:」「2025年6月20日」）は前置きに数えない。木を作るのは、数えて上限を超えたときだけ。 */
+const withoutDateStamps = (doc: ProseDocument, paragraphs: readonly Paragraph[]): Paragraph[] => {
+  const labels = (doc.lexicons["date-stamp-label"] ?? []).map((entry) => entry.pattern);
+  const candidates = stampCandidates(
+    doc.source,
+    paragraphs.map((paragraph) => paragraph.span),
+    dateSpans(doc),
+  );
+  const stamps = dateStampIndexes(candidates, labels);
+  return paragraphs.filter((_, index) => !stamps.has(index));
+};
+
 /**
  * 本題に入るまでが長い。業務文書では、読み手は結論を探しに来ている。
  *
@@ -99,7 +116,9 @@ export const ruleOfThree: Detector = (doc, options): Finding[] => {
 export const preambleLength: Detector = (doc, options): Finding[] => {
   const body = doc.sections.find((section) => section.depth >= 2);
   if (body === undefined) return [];
-  const before = doc.paragraphs.filter((paragraph) => paragraph.span.start < body.span.start);
+  const all = doc.paragraphs.filter((paragraph) => paragraph.span.start < body.span.start);
+  if (all.length <= options.limit) return [];
+  const before = withoutDateStamps(doc, all);
   if (before.length <= options.limit) return [];
   const first = before[0];
   return first === undefined

package/src/detectors/superlative-scope.ts

@@ -0,0 +1,66 @@
+import type { Lexicon, Token } from "../plugin.ts";
+import type { TokenRange } from "./lexicon-match.ts";
+
+/**
+ * 最上級が範囲を持っているか。「日本で最も」「国内最大」"the best in the world" は、何の中で最もなのかを言っている。
+ * 範囲を示す語は言語で違い、置く側も違う（日本語は最上級の前、英語は後ろ）。どちらの語も言語パッケージの語彙表が持つ。
+ */
+export type ScopeMarkers = {
+  /** 最上級の直前に置く語（で）。その前に固有名詞か地名が要る。 */
+  readonly before: Lexicon;
+  /** 最上級の名詞句の直後に置く語（in・of）。その後ろに名詞句が要る。 */
+  readonly after: Lexicon;
+};
+
+/** 語彙表の position で分ける。position の無い語は、どちら側の語か分からないので使わない。 */
+export const scopeMarkersOf = (lexicon: Lexicon): ScopeMarkers => ({
+  before: lexicon.filter((entry) => entry.position === "before"),
+  after: lexicon.filter((entry) => entry.position === "after"),
+});
+
+const NOMINAL = new Set(["NOUN", "PROPN"]);
+
+/** 名詞句の始まり。the world・Chicago・our products・the three・all。 */
+const PHRASE_START = new Set(["DET", "NOUN", "PROPN", "PRON", "NUM", "ADJ"]);
+
+/** 名詞の前に来る修飾（most skilled・widely used・stuffed）。名詞が出た後は名詞の連なりだけが続く。 */
+const PREMODIFIER = new Set(["ADJ", "ADV", "VERB", "NOUN", "PROPN", "NUM"]);
+const NOUN_RUN = new Set(["NOUN", "PROPN", "NUM"]);
+
+/** 範囲の前に置ける語。修飾だけで終わった句（the best is in …）は範囲ではなく述語。 */
+const PHRASE_END = new Set(["NOUN", "PROPN", "NUM", "ADJ"]);
+
+const isMarker = (token: Token | undefined, markers: Lexicon): boolean =>
+  token !== undefined && markers.some((entry) => entry.pattern.toLowerCase() === token.surface.toLowerCase());
+
+const isNamed = (token: Token | undefined): boolean => token !== undefined && (token.pos === "PROPN" || token.features?.["NameType"] !== undefined);
+
+/** 空白も助詞も挟まずに名詞が付いた最上級（国内最大・業界最速・世界唯一）。前の名詞が範囲。 */
+const compound = (tokens: readonly Token[], range: TokenRange): boolean => {
+  const before = tokens[range.start - 1];
+  const first = tokens[range.start];
+  return before !== undefined && first !== undefined && NOMINAL.has(before.pos) && before.span.end === first.span.start;
+};
+
+/** 名前 + 範囲の語 + 最上級（日本で最も・トヨタで最も・東京都で最大）。「費用で最大の効果」の「費用」は名前ではないので範囲にしない。 */
+const namedBefore = (tokens: readonly Token[], range: TokenRange, markers: Lexicon): boolean =>
+  isMarker(tokens[range.start - 1], markers) && isNamed(tokens[range.start - 2]);
+
+/** 最上級に続く名詞句の終わり。名詞が出るまでは修飾を、出た後は名詞の連なりを読む。 */
+const phraseEnd = (tokens: readonly Token[], at: number, nounSeen = false): number => {
+  const token = tokens[at];
+  if (token === undefined || !(nounSeen ? NOUN_RUN : PREMODIFIER).has(token.pos)) return at;
+  return phraseEnd(tokens, at + 1, nounSeen || NOMINAL.has(token.pos));
+};
+
+/** 最上級 + 名詞句 + 範囲の語 + 名詞句（the best pizza in Chicago・the most famous of the sculptures）。 */
+const phraseAfter = (tokens: readonly Token[], range: TokenRange, markers: Lexicon): boolean => {
+  const at = phraseEnd(tokens, range.end);
+  const last = tokens[at - 1];
+  const endsPhrase = at === range.end || (last !== undefined && PHRASE_END.has(last.pos));
+  const opens = tokens[at + 1];
+  return endsPhrase && isMarker(tokens[at], markers) && opens !== undefined && PHRASE_START.has(opens.pos);
+};
+
+export const scoped = (tokens: readonly Token[], range: TokenRange, markers: ScopeMarkers): boolean =>
+  compound(tokens, range) || namedBefore(tokens, range, markers.before) || phraseAfter(tokens, range, markers.after);

package/src/document.ts

@@ -4,16 +4,21 @@ import { gfmTableFromMarkdown } from "mdast-util-gfm-table";
 import { frontmatter } from "micromark-extension-frontmatter";
 import { frontmatterFromMarkdown } from "mdast-util-frontmatter";
 import { atxHeadingText, headingText } from "./heading-text.ts";
+import { hasTitle } from "./heading-title.ts";
 import { maskSpans } from "./mask.ts";
 import { buildTree, type Outline } from "./structure/build.ts";
 import { isMarkdownPath } from "./structure/markdown-path.ts";
 import { pageFurniture, textOutline } from "./page-furniture.ts";
 import { tokenizedLexicons } from "./lexicon-tokens.ts";
+import { inPageAnchors, isInPageNavigation, isNavigationList, type InPageAnchors } from "./in-page-nav.ts";
 import type { BulletList, LanguageAdapter, Paragraph, ProseDocument, Section, Sentence, Span, StructureNode, DocumentProfile } from "./plugin.ts";
 
 type Place = { readonly offset?: number | undefined };
 type Node = {
   readonly type: string;
+  readonly url?: string | undefined;
+  readonly value?: string | undefined;
+  readonly identifier?: string | undefined;
   readonly position?: { readonly start: Place; readonly end: Place } | undefined;
   readonly children?: readonly Node[] | undefined;
 };
@@ -93,14 +98,14 @@ const matchSpans = (source: string, pattern: RegExp): Span[] =>
 
 const directiveSpans = (source: string): Span[] => [...matchSpans(source, DIRECTIVE), ...matchSpans(source, BARE_URL)];
 
-const collectMasks = (root: Node, source: string): Span[] => {
+const collectMasks = (root: Node, source: string, anchors: InPageAnchors): Span[] => {
   const spans: Span[] = [...directiveSpans(source)];
   walk(root, (node) => {
     if (node.type === "link" || node.type === "linkReference") {
       spans.push(...linkChrome(node));
       return;
     }
-    if (!NOT_PROSE.has(node.type) && node.type !== "heading") return;
+    if (!NOT_PROSE.has(node.type) && node.type !== "heading" && !isInPageNavigation(node, anchors)) return;
     const span = spanOf(node);
     if (span !== undefined) spans.push(span);
   });
@@ -136,7 +141,8 @@ const headingsOf = (root: Node, source: string): Heading[] => {
     if (span === undefined) return;
     // ATX（行頭が #）なら閉じの # も外す。setext の見出しの末尾の # は言葉なので残す。
     const read = source.startsWith("#", span.start) ? atxHeadingText : headingText;
-    found.push({ depth, text: read(textOf(node, source)), start: span.start, end: span.end });
+    const text = read(textOf(node, source));
+    if (hasTitle(text)) found.push({ depth, text, start: span.start, end: span.end });
   });
   return found;
 };
@@ -162,10 +168,10 @@ const strongSpans = (root: Node, masked: readonly Span[]): Span[] => {
 
 const within = (span: Span, from: number, to: number): boolean => span.start >= from && span.start < to;
 
-const spansOfType = (root: Node, type: string): Span[] => {
+const spansOfType = (root: Node, type: string, keep: (node: Node) => boolean = () => true): Span[] => {
   const found: Span[] = [];
   walk(root, (node) => {
-    if (node.type !== type) return;
+    if (node.type !== type || !keep(node)) return;
     const span = spanOf(node);
     if (span !== undefined) found.push(span);
   });
@@ -218,11 +224,11 @@ const paragraphsOf = (spans: readonly Span[], sentences: readonly Sentence[], li
     .filter((span) => !listSpans.some((list) => span.start >= list.start && span.start < list.end))
     .map((span) => ({ span, sentences: sentences.filter((sentence) => within(sentence.span, span.start, span.end)) }));
 
-/** 箇条書きは list ノードの直下の項目を数える。入れ子の項目は内側の list のものとして数える。 */
-const listsOf = (root: Node, source: string): BulletList[] => {
+/** 箇条書きは list ノードの直下の項目を数える。入れ子の項目は内側の list のものとして数える。目次のような案内だけの箇条書きは数えない。 */
+const listsOf = (root: Node, source: string, anchors: InPageAnchors): BulletList[] => {
   const found: BulletList[] = [];
   walk(root, (node) => {
-    if (node.type !== "list") return;
+    if (node.type !== "list" || isNavigationList(node, anchors)) return;
     const span = spanOf(node);
     if (span === undefined) return;
     const items = (node.children ?? []).flatMap((child) => {
@@ -263,13 +269,15 @@ export const buildDocument = (
   profile: DocumentProfile | undefined = undefined,
 ): ProseDocument => {
   const root = parse(source);
+  const anchors = inPageAnchors(root);
   // 強調の記号は「本文でないもの」だが、太字の数を数えるときの「覆われた場所」ではない。
   // 同じ集合にすると、太字が自分の記号のせいで覆われた場所にあることになり、1 つも数えられなくなる。
   // テキストの文書は、ページのヘッダーとフッターも本文ではない（Markdown には改ページが無い）。
-  const blocks = [...collectMasks(root, source), ...(isMarkdownPath(path) ? [] : pageFurniture(source))];
+  const blocks = [...collectMasks(root, source, anchors), ...(isMarkdownPath(path) ? [] : pageFurniture(source))];
   const masked = [...blocks, ...emphasisSpans(root, source)];
   const prose = maskSpans(source, masked);
-  const paragraphSpans = spansOfType(root, "paragraph");
+  // ページの案内は段落としても数えない。数えると、目次の行が「本題までの段落」に入る。
+  const paragraphSpans = spansOfType(root, "paragraph", (node) => !isInPageNavigation(node, anchors));
   const listItems = spansOfType(root, "listItem");
   const sentences = sentencesOf(prose, paragraphSpans, adapter);
   const lexicons = {
@@ -292,7 +300,7 @@ export const buildDocument = (
     sentences,
     listSpans: listItems,
     paragraphs: paragraphsOf(paragraphSpans, sentences, listItems),
-    lists: listsOf(root, source),
+    lists: listsOf(root, source, anchors),
     lexicons: tagged ? tokenizedLexicons(lexicons, adapter) : lexicons,
     requiredSections: team.requiredSections,
     // 構造の rule（参照先が無い・番号の抜け）が読む木。どの rule も読まなければ作らない。何万行の契約書で、他の rule の lint に代金を払わせない。

package/src/eval.ts

@@ -1,6 +1,7 @@
 import { DETECTORS } from "./detectors/index.ts";
 import { charLength } from "./measure.ts";
 import type { ProseDocument, RuleDefinition } from "./plugin.ts";
+import { missingList } from "./declared-lists.ts";
 
 /**
  * 閾値の較正。
@@ -48,7 +49,6 @@ const countAt = (docs: readonly ProseDocument[], rule: RuleDefinition, limit: nu
   if (detector === undefined) return { limit, findings: 0, documents: 0, per10k: 0 };
   const perDoc = docs.map((doc) => {
     const lexicon = rule.word_list === undefined ? undefined : doc.lexicons[rule.word_list];
-    if (rule.word_list !== undefined && lexicon === undefined) return 0;
     return detector(doc, { limit, lexicon, where: rule.where }).length;
   });
   const findings = perDoc.reduce((sum, count) => sum + count, 0);
@@ -82,6 +82,7 @@ const measurable = (rule: RuleDefinition, docs: readonly ProseDocument[], genre:
   if (rule.layer === "L4" || !rule.use_for.some((target) => genre.startsWith(target))) return false;
   if (rule.languages !== undefined && !rule.languages.includes(language)) return false;
   if (rule.from.length > 0) return false;
+  if (docs.some((doc) => missingList(rule, doc.lexicons) !== undefined)) return false;
   return rule.requires.every((need) => meets(need, docs));
 };
 

package/src/heading-title.ts

@@ -0,0 +1,8 @@
+/** 文字か数字。記号・空白・絵文字だけの並びには、読者が指せる題が無い。 */
+const TITLE_CHARACTER = /[\p{L}\p{N}]/u;
+
+/**
+ * 見出しに題があるか。`###` だけの行や `## ---` は、変換で残った空の `<h3>` のような区切りで、
+ * 本文の始まりでも読者が辿れる節でもない。題の無い見出しは見出しとして数えず、前後の本文を 1 つの節として読む。
+ */
+export const hasTitle = (headingWords: string): boolean => TITLE_CHARACTER.test(headingWords);

package/src/in-page-nav.ts

@@ -0,0 +1,67 @@
+/** mdast の節のうち、ページ内の案内かどうかの判定に要る部分だけ。 */
+export type NavNode = {
+  readonly type: string;
+  readonly url?: string | undefined;
+  readonly value?: string | undefined;
+  readonly identifier?: string | undefined;
+  readonly children?: readonly NavNode[] | undefined;
+};
+
+/** 参照形式のリンク（`[目次][toc]`）の名前のうち、定義（`[toc]: #目次`）が同じページの中を指すもの。 */
+export type InPageAnchors = ReadonlySet<string>;
+
+/** 文字か数字。これが 1 つでもあれば、書いた人の言葉がある。記号（▲ ↑ | ・）だけなら飾り。 */
+const WORD = /[\p{L}\p{N}]/u;
+
+const WRAPPERS = new Set(["strong", "emphasis", "delete"]);
+
+const pointsInPage = (url: string | undefined): boolean => (url ?? "").startsWith("#");
+
+const collectAnchors = (node: NavNode, found: Set<string>): void => {
+  if (node.type === "definition" && node.identifier !== undefined && pointsInPage(node.url)) found.add(node.identifier);
+  (node.children ?? []).forEach((child) => collectAnchors(child, found));
+};
+
+export const inPageAnchors = (root: NavNode): InPageAnchors => {
+  const found = new Set<string>();
+  collectAnchors(root, found);
+  return found;
+};
+
+const isInPageLink = (node: NavNode, anchors: InPageAnchors): boolean =>
+  (node.type === "link" && pointsInPage(node.url)) || (node.type === "linkReference" && anchors.has(node.identifier ?? ""));
+
+const isNavigationPart = (node: NavNode, anchors: InPageAnchors): boolean => {
+  if (isInPageLink(node, anchors) || node.type === "break") return true;
+  if (node.type === "text") return !WORD.test(node.value ?? "");
+  return WRAPPERS.has(node.type) && (node.children ?? []).every((child) => isNavigationPart(child, anchors));
+};
+
+const containsInPageLink = (node: NavNode, anchors: InPageAnchors): boolean =>
+  isInPageLink(node, anchors) || (node.children ?? []).some((child) => containsInPageLink(child, anchors));
+
+/**
+ * 同じページの中へのリンク（`](#…)`）と記号だけでできた段落。「▲ 目次に戻る」や目次の項目で、本文ではなくページの案内。
+ * 言葉が 1 つでも混ざれば本文（「詳しくは[第3章](#第3章)を参照」）。他のページへのリンクは書いた人が選んだ一覧なので残す。
+ */
+export const isInPageNavigation = (paragraph: NavNode, anchors: InPageAnchors = new Set()): boolean => {
+  const children = paragraph.children ?? [];
+  return (
+    paragraph.type === "paragraph" &&
+    children.every((child) => isNavigationPart(child, anchors)) &&
+    children.some((child) => containsInPageLink(child, anchors))
+  );
+};
+
+const isNavigationBlock = (node: NavNode, anchors: InPageAnchors): boolean => isInPageNavigation(node, anchors) || isNavigationList(node, anchors);
+
+const isNavigationItem = (item: NavNode, anchors: InPageAnchors): boolean => {
+  const children = item.children ?? [];
+  return item.type === "listItem" && children.length > 0 && children.every((child) => isNavigationBlock(child, anchors));
+};
+
+/** 項目がどれもページの案内（入れ子も含む）の箇条書き。目次の項目の数は節の数で決まり、書いた人が選んだ並べ方ではない。 */
+export const isNavigationList = (list: NavNode, anchors: InPageAnchors = new Set()): boolean => {
+  const items = list.children ?? [];
+  return list.type === "list" && items.length > 0 && items.every((item) => isNavigationItem(item, anchors));
+};

package/src/orthography.ts

@@ -1,6 +1,6 @@
 // 表記のそろい。書き方の決まりのうち、文字だけで決まるもの。
 
-const escapeRegExp = (text: string): string => text.replace(/[.*+?^${}()|[\]\\]/gu, "\\$&");
+export const escapeRegExp = (text: string): string => text.replace(/[.*+?^${}()|[\]\\]/gu, "\\$&");
 
 /** text の中で word が現れる [始まり, 終わり) を、左から順に。大文字小文字を区別しないときは i で探す（位置は元の文字列のまま）。 */
 const spansOf = (text: string, word: string, ignoreCase: boolean): [number, number][] => {
@@ -67,6 +67,7 @@ const ALPHANUMERIC = /[A-Za-z0-9.]/u;
 /**
  * 日本語の左にある英数字の並びの、先頭の字。「3GBの」の「GB」は数量の単位で、並びは数字で始まる。
  * そうした並びと日本語の境目は、英字の空け方ではなく数字の後ろの空け方として数える（「3回」と同じ）。
+ * 逆に「H30 等」「EC2 で」は英字で始まる名前で、書き手は英単語と同じに空ける。数字の後ろとしては数えない。
  */
 const runStart = (chars: readonly string[], last: number): string | undefined => {
   let first = last;
@@ -74,10 +75,10 @@ const runStart = (chars: readonly string[], last: number): string | undefined =>
   return chars[first];
 };
 
-/** 左の並びの種類。数字で始まる並び（3GB、10ms）は数字として扱う。 */
+/** 左の並びの種類は先頭の字で決める。数字で始まる並び（3GB、10ms）は数字、英字で始まる並び（H30、v1.2）は英字。 */
 const leftRunBeside = (chars: readonly string[], last: number, japanese: string | undefined): SpacingKind | undefined => {
   const kind = latinBeside(japanese, chars[last], "after-digit");
-  return kind === "letter" && DIGIT.test(runStart(chars, last) ?? "") ? "after-digit" : kind;
+  return kind === undefined ? undefined : (kindOf(runStart(chars, last), "after-digit") ?? kind);
 };
 
 /**

package/src/plugin.ts

@@ -40,6 +40,8 @@ export type LexiconEntry = {
   readonly pattern: string;
   readonly weight?: number | undefined;
   readonly instead_of?: string | undefined;
+  /** 語が、かかる語のどちら側に立つか。語順が言語で違うものを語彙表が言う（範囲の「で」は前、"in" は後ろ）。 */
+  readonly position?: "before" | "after" | undefined;
   /** pattern を adapter が語に分けたもの。品詞が読めるときだけ core が入れる。語彙表を書く側は書かない。 */
   readonly tokens?: readonly Token[] | undefined;
 };
@@ -344,6 +346,8 @@ export type RuleDefinition = {
   readonly by_genre: Readonly<Record<string, LevelTable>>;
   readonly how_to_find: string;
   readonly word_list: string | undefined;
+  /** word_list のほかに detector が名前で引く語彙表。どれかが無い言語では rule を動かさない。 */
+  readonly extra_word_lists: readonly string[];
   /** L4 のみ。LLM に渡す決まり。言語別。 */
   readonly what_to_check: Localized | undefined;
   readonly where: string | undefined;

package/src/rule-load.ts

@@ -99,6 +99,7 @@ const toRule = (raw: unknown, language: string, file: string): RuleDefinition =>
     by_genre: genreTables(raw["by_genre"], language),
     how_to_find: String(raw["how_to_find"]),
     word_list: typeof raw["word_list"] === "string" ? raw["word_list"] : undefined,
+    extra_word_lists: stringList(raw["extra_word_lists"]) ?? [],
     what_to_check: isLocalized(raw["what_to_check"]) ? raw["what_to_check"] : undefined,
     where: typeof raw["where"] === "string" ? raw["where"] : undefined,
     requires: stringList(raw["requires"]) ?? [],

package/src/run.ts

@@ -4,6 +4,7 @@ import type { AdapterNeeds, Finding, Level, ProseDocument, RuleDefinition } from
 import { lineStarts, placeOf } from "./position.ts";
 import { REASONS, type Reasons } from "./reasons.ts";
 import { joinWords } from "./detectors/word-list.ts";
+import { missingList } from "./declared-lists.ts";
 import { unreadStructure, type Unread } from "./structure/unread.ts";
 import { isMarkdownPath } from "./structure/markdown-path.ts";
 import { maskSpans } from "./mask.ts";
@@ -181,15 +182,15 @@ export const runRules = (
       const detector = DETECTORS[rule.how_to_find];
       if (detector === undefined)
         return { findings: acc.findings, skipped: [...acc.skipped, { rule: rule.id, why: reasonsFor(doc).noDetector(rule.how_to_find) }] };
+      // 語彙表を要求する rule で、その言語に語彙表が無ければ動かせない。黙って通さない。
+      const absent = missingList(rule, doc.lexicons);
+      if (absent !== undefined)
+        return { findings: acc.findings, skipped: [...acc.skipped, { rule: rule.id, why: reasonsFor(doc).noLexicon(doc.language, absent) }] };
       const options = {
         limit: limitFor(rule, level, genre, limits),
         lexicon: rule.word_list === undefined ? undefined : doc.lexicons[rule.word_list],
         where: rule.where,
       };
-      // 語彙表を要求する rule で、その言語に語彙表が無ければ動かせない。黙って通さない。
-      if (rule.word_list !== undefined && options.lexicon === undefined) {
-        return { findings: acc.findings, skipped: [...acc.skipped, { rule: rule.id, why: reasonsFor(doc).noLexicon(doc.language, rule.word_list) }] };
-      }
       const found = detector(doc, options).map((finding) => place(starts, { ...finding, rule: rule.id, severity: rule.severity }));
       return { findings: [...acc.findings, ...found], skipped: acc.skipped };
     },