chaffjs 0.17.0 → 0.19.0

package/src/detectors/token-column.ts

@@ -0,0 +1,149 @@
+/** A stretch of a sentence's tokens, [start, end). */
+export type TokenRange = { readonly start: number; readonly end: number };
+
+/**
+ * Tokens set aside (a list's commas, which an item holds or not by their depth) with a key each. A question leaves out
+ * the ones with one key and counts the rest.
+ */
+type Aside = {
+  /** Indices of the passing tokens set aside, in order. */
+  readonly at: readonly number[];
+  readonly keys: readonly number[];
+  /** Per key, the positions in `at` that have it. */
+  readonly byKey: ReadonlyMap<number, readonly number[]>;
+  /** For each position in `at`, the next and previous position whose key differs, or at.length / -1. */
+  readonly nextOther: readonly number[];
+  readonly previousOther: readonly number[];
+};
+
+/**
+ * One test over a sentence, read once: how many tokens before each index pass it, and the nearest passing index at or
+ * after, and before, each index. A range then answers "how many", "the first" and "the last" without reading it.
+ * Tokens set aside are not in these counts; the ones that pass are in `aside`.
+ */
+export type Column = {
+  readonly before: readonly number[];
+  readonly next: readonly number[];
+  readonly previous: readonly number[];
+  readonly aside: Aside;
+};
+
+/** The key of a token set aside, or undefined for an ordinary token. */
+export type AsideKey<T> = (item: T, index: number) => number | undefined;
+
+/** The first index in a sorted list at which the value is at least `value`. */
+export const lowerBound = (sorted: readonly number[], value: number): number => {
+  const search = (low: number, high: number): number => {
+    if (low >= high) return low;
+    const middle = Math.floor((low + high) / 2);
+    return (sorted[middle] ?? value) < value ? search(middle + 1, high) : search(low, middle);
+  };
+  return search(0, sorted.length);
+};
+
+const prefixCounts = (passes: readonly boolean[]): number[] =>
+  passes.reduce<number[]>(
+    (counts, passed, index) => {
+      counts.push((counts[index] ?? 0) + (passed ? 1 : 0));
+      return counts;
+    },
+    [0],
+  );
+
+const nextPassing = (passes: readonly boolean[]): number[] =>
+  passes.reduceRight<number[]>(
+    (found, passed, index) => {
+      found[index] = passed ? index : (found[index + 1] ?? passes.length);
+      return found;
+    },
+    Array.from({ length: passes.length + 1 }, () => passes.length),
+  );
+
+const previousPassing = (passes: readonly boolean[]): number[] =>
+  passes.reduce<number[]>(
+    (found, passed, index) => {
+      found.push(passed ? index : (found[index] ?? -1));
+      return found;
+    },
+    [-1],
+  );
+
+const asideOf = (at: readonly number[], keys: readonly number[]): Aside => {
+  const byKey = keys.reduce((found, key, position) => {
+    const list = found.get(key);
+    if (list === undefined) found.set(key, [position]);
+    else list.push(position);
+    return found;
+  }, new Map<number, number[]>());
+  const nextOther = keys.reduceRight<number[]>((found, key, position) => {
+    found[position] = keys[position + 1] !== key ? position + 1 : (found[position + 1] ?? keys.length);
+    return found;
+  }, []);
+  const previousOther = keys.reduce<number[]>((found, key, position) => {
+    found.push(position === 0 || keys[position - 1] !== key ? position - 1 : (found[position - 1] ?? -1));
+    return found;
+  }, []);
+  return { at, keys, byKey, nextOther, previousOther };
+};
+
+export const columnOf = <T>(items: readonly T[], test: (item: T, index: number) => boolean, asideKey: AsideKey<T> = () => undefined): Column => {
+  const keys = items.map(asideKey);
+  const tested = items.map(test);
+  const passes = tested.map((passed, index) => passed && keys[index] === undefined);
+  const asideAt = tested.flatMap((passed, index) => (passed && keys[index] !== undefined ? [index] : []));
+  return {
+    before: prefixCounts(passes),
+    next: nextPassing(passes),
+    previous: previousPassing(passes),
+    aside: asideOf(
+      asideAt,
+      asideAt.map((index) => keys[index] ?? 0),
+    ),
+  };
+};
+
+/** Positions in `aside.at` of the tokens set aside within the range, [low, high). */
+const asideSpan = (aside: Aside, range: TokenRange): { readonly low: number; readonly high: number } => ({
+  low: lowerBound(aside.at, range.start),
+  high: lowerBound(aside.at, range.end),
+});
+
+const asideCount = (aside: Aside, range: TokenRange, excluded: number | undefined): number => {
+  if (excluded === undefined || aside.at.length === 0) return 0;
+  const { low, high } = asideSpan(aside, range);
+  const same = aside.byKey.get(excluded) ?? [];
+  return high - low - (lowerBound(same, high) - lowerBound(same, low));
+};
+
+const firstAside = (aside: Aside, range: TokenRange, excluded: number | undefined): number => {
+  if (excluded === undefined || aside.at.length === 0) return range.end;
+  const { low, high } = asideSpan(aside, range);
+  const position = aside.keys[low] === excluded ? (aside.nextOther[low] ?? high) : low;
+  return position < high ? (aside.at[position] ?? range.end) : range.end;
+};
+
+const lastAside = (aside: Aside, range: TokenRange, excluded: number | undefined): number => {
+  if (excluded === undefined || aside.at.length === 0) return -1;
+  const { low, high } = asideSpan(aside, range);
+  const position = aside.keys[high - 1] === excluded ? (aside.previousOther[high - 1] ?? -1) : high - 1;
+  return position >= low ? (aside.at[position] ?? -1) : -1;
+};
+
+/**
+ * How many tokens in the range pass. `excluded` is the key of the tokens set aside that do not count; without it, none of
+ * them count.
+ */
+export const countIn = (column: Column, range: TokenRange, excluded?: number): number =>
+  (column.before[range.end] ?? 0) - (column.before[range.start] ?? 0) + asideCount(column.aside, range, excluded);
+
+/** The first passing index in the range, or -1. */
+export const firstIn = (column: Column, range: TokenRange, excluded?: number): number => {
+  const first = Math.min(column.next[range.start] ?? range.end, firstAside(column.aside, range, excluded));
+  return first < range.end ? first : -1;
+};
+
+/** The last passing index in the range, or -1. */
+export const lastIn = (column: Column, range: TokenRange, excluded?: number): number => {
+  const last = Math.max(column.previous[range.end] ?? -1, lastAside(column.aside, range, excluded));
+  return last >= range.start ? last : -1;
+};

package/src/detectors/unbalanced-bracket.ts

@@ -0,0 +1,153 @@
+import type { Detector, Finding, Span } from "../plugin.ts";
+import { BARE_URL } from "../bare-url.ts";
+import { escapeRegExp } from "../orthography.ts";
+import { quoteAround } from "./quote-around.ts";
+import { mergeSpans } from "../span-merge.ts";
+import { firstEndingAfter } from "../soft-break.ts";
+
+/**
+ * 括弧の組。どの言語でも同じ字の決まりなので、語彙表ではなくここに置く。family は形（丸・角）で、全角と半角は同じ形の別の幅。
+ * 英語の "…" と '…' は開きと閉じが同じ字で、' は省略の印（don't）と見分けられないので入れない。
+ */
+type Bracket = { readonly open: string; readonly close: string; readonly family: string };
+
+const BRACKETS: readonly Bracket[] = [
+  { open: "「", close: "」", family: "kagi" },
+  { open: "『", close: "』", family: "double-kagi" },
+  { open: "（", close: "）", family: "round" },
+  { open: "(", close: ")", family: "round" },
+  { open: "［", close: "］", family: "square" },
+  { open: "[", close: "]", family: "square" },
+  { open: "【", close: "】", family: "lenticular" },
+  { open: "〔", close: "〕", family: "tortoise" },
+  { open: "〈", close: "〉", family: "angle" },
+  { open: "《", close: "》", family: "double-angle" },
+  { open: "“", close: "”", family: "quote" },
+];
+
+const BY_OPEN = new Map(BRACKETS.map((bracket) => [bracket.open, bracket]));
+const BY_CLOSE = new Map(BRACKETS.map((bracket) => [bracket.close, bracket]));
+
+/** 括弧の字のどれか。どれも BMP の字なので、見つけた位置がそのまま UTF-16 の位置になる。 */
+const MARKS = new RegExp(
+  BRACKETS.flatMap((bracket) => [bracket.open, bracket.close])
+    .map(escapeRegExp)
+    .join("|"),
+  "gu",
+);
+
+export type BracketProblem = {
+  /** unclosed は節の中で閉じない開き、unopened は開きの無い閉じ、mismatch は全角と半角で組んだもの（（…)）。 */
+  readonly kind: "unclosed" | "unopened" | "mismatch";
+  readonly offset: number;
+  readonly mark: string;
+  /** mismatch のときの、組んだ相手の字。 */
+  readonly partner?: string;
+};
+
+type Open = { readonly bracket: Bracket; readonly offset: number };
+type Scan = { readonly open: Open[]; readonly problems: BracketProblem[] };
+
+/**
+ * 丸括弧の閉じの前の、行の頭か空白・句読点の後ろに書いた短い印（「1)」「a)」「事例）」「※）」）と、数の後ろの閉じ（「事例2）及び事例3）」）。
+ * 箇条の番号や見出しの印で、開きを持たない書き方。顔文字の「:)」も同じ形。印に使うのは丸括弧だけ。
+ */
+const LABEL_BEFORE = /(?:^|[\s、。,;:：])[^\s()（）[\]［］「」『』]{1,3}[ \t\u3000]?$|[\d０-９]$/u;
+
+const isLabel = (text: string, bracket: Bracket, offset: number): boolean => {
+  if (bracket.family !== "round") return false;
+  const lineStart = text.lastIndexOf("\n", offset - 1) + 1;
+  return LABEL_BEFORE.test(text.slice(lineStart, offset));
+};
+
+const unclosedOf = (entry: Open): BracketProblem => ({ kind: "unclosed", offset: entry.offset, mark: entry.bracket.open });
+
+const closeWith = (scan: Scan, text: string, bracket: Bracket, offset: number): void => {
+  const at = scan.open.findLastIndex((entry) => entry.bracket.family === bracket.family);
+  if (at === -1) {
+    if (!isLabel(text, bracket, offset)) scan.problems.push({ kind: "unopened", offset, mark: bracket.close });
+    return;
+  }
+  const opened = scan.open[at];
+  // 間に残った開きは、この閉じより先に閉じるはずだったもの。
+  scan.open
+    .splice(at)
+    .slice(1)
+    .forEach((inner) => scan.problems.push(unclosedOf(inner)));
+  if (opened !== undefined && opened.bracket !== bracket) scan.problems.push({ kind: "mismatch", offset, mark: bracket.close, partner: opened.bracket.open });
+};
+
+/** 段落（空行で区切った塊）の最初の字。 */
+const PARAGRAPH_HEAD = /(?:^|\n[ \t\u3000]*\n)[ \t\u3000]*(?=\S)/gu;
+
+const paragraphHeads = (text: string): ReadonlySet<number> => new Set([...text.matchAll(PARAGRAPH_HEAD)].map((match) => match.index + match[0].length));
+
+/** 人の言葉を引く括弧。続く段落を同じ開きで始めて、引用が続くことを示す書き方がある。 */
+const QUOTATIONS: ReadonlySet<string> = new Set(["quote", "kagi", "double-kagi"]);
+
+const openWith = (scan: Scan, bracket: Bracket, offset: number, opensParagraph: boolean): void => {
+  // 段落をまたぐ台詞・引用（英語の小説）は、閉じずに次の段落を同じ開きで始める。前の開きはここで続く。
+  // 行の頭では続きと読まない。折り返した台詞の行の頭の開きは、中に引いた別の言葉のこともある。
+  if (opensParagraph && QUOTATIONS.has(bracket.family) && scan.open.at(-1)?.bracket === bracket) scan.open.pop();
+  scan.open.push({ bracket, offset });
+};
+
+/**
+ * 一つの節の中の括弧の組を読む。text は節の文字列で、offset は text の中の位置。
+ * 詩の連や、段落をまたぐ引用があるので、括弧は段落を越えて組にする。節の終わりまで閉じない開きを指摘する。
+ */
+export const bracketProblems = (text: string): BracketProblem[] => {
+  const scan: Scan = { open: [], problems: [] };
+  const heads = paragraphHeads(text);
+  [...text.matchAll(MARKS)].forEach((match) => {
+    const opening = BY_OPEN.get(match[0]);
+    const closing = BY_CLOSE.get(match[0]);
+    if (opening !== undefined) openWith(scan, opening, match.index, heads.has(match.index));
+    else if (closing !== undefined) closeWith(scan, text, closing, match.index);
+  });
+  return [...scan.problems, ...scan.open.map(unclosedOf)].toSorted((left, right) => left.offset - right.offset);
+};
+
+/** sorted（昇順・重ならない）のどれかが offset を含むか。何万もの字の節があっても、URL ごとに全部をなめない。 */
+const coversOffset = (sorted: readonly Span[], offset: number): boolean => {
+  const span = sorted[firstEndingAfter(sorted, offset)];
+  return span !== undefined && span.start <= offset;
+};
+
+/**
+ * 本文（prose）では URL を覆うが、覆いは空白まで取るので、URL の直後に続けて書いた字（「（https://example.jp/）」の「）」）まで消える。
+ * 読み手には見えている字なので、ASCII でない字から先を元に戻す。コードの中の URL（texts の外）は戻さない。
+ */
+export const withRunOnRestored = (prose: string, source: string, texts: readonly Span[]): string => {
+  const readable = mergeSpans(texts, false);
+  const parts: string[] = [];
+  const cursor = { at: 0 };
+  [...source.matchAll(BARE_URL)].forEach((match) => {
+    const tail = match[0].search(/[^\p{ASCII}]/u);
+    if (tail === -1 || !coversOffset(readable, match.index)) return;
+    parts.push(prose.slice(cursor.at, match.index + tail), source.slice(match.index + tail, match.index + match[0].length));
+    cursor.at = match.index + match[0].length;
+  });
+  parts.push(prose.slice(cursor.at));
+  return parts.join("");
+};
+
+const findingOf = (text: string, problem: BracketProblem): Finding => ({
+  rule: "",
+  severity: "warning",
+  line: 0,
+  column: 0,
+  quote: quoteAround(text, problem.offset),
+  values: { mark: problem.mark, partner: problem.partner ?? "", offset: problem.offset },
+  variant: problem.kind,
+});
+
+/** 見出しで区切った節ごとに読む。見出しの行は本文で覆ってある。 */
+export const unbalancedBracket: Detector = (doc): Finding[] => {
+  const text = withRunOnRestored(doc.prose ?? doc.source, doc.source, doc.markup?.texts ?? []);
+  return doc.sections.flatMap((section) =>
+    bracketProblems(text.slice(section.span.start, section.span.end)).map((problem) =>
+      findingOf(text, { ...problem, offset: section.span.start + problem.offset }),
+    ),
+  );
+};

package/src/detectors/url-run-on.ts

@@ -0,0 +1,33 @@
+import type { Detector, Finding, Span } from "../plugin.ts";
+import { findingAt, quoteOf } from "./markup-finding.ts";
+
+/** そのまま書いた URL の、ASCII の字だけの並び。 */
+const BARE_URL = /https?:\/\/[!-~]+/gu;
+
+/**
+ * URL は ASCII の字を空白まで取るので、後ろに来るのは空白か ASCII でない字。GFM の自動リンクや多くのメールソフトは空白が来るまでを
+ * リンクにするので、空白でない字はリンクに入る。見えない字（ゼロ幅の空白など、Cf）は折り返しのために入れたもので、読み手には続いて見えない。
+ */
+const RUNS_ON = /[^\s\p{Cf}]/u;
+
+/** index から始まる一字（サロゲートの対は一字）。末尾より後ろは空。 */
+const characterAt = (text: string, index: number): string => {
+  const point = text.codePointAt(index);
+  return point === undefined ? "" : String.fromCodePoint(point);
+};
+
+export type RunOnUrl = { readonly url: Span; readonly next: string };
+
+/** 字のまま見える範囲（texts）の中の、すぐ後ろに空白を置かずに日本語や全角の記号が続く URL。 */
+export const runOnUrls = (source: string, texts: readonly Span[]): RunOnUrl[] =>
+  texts.flatMap((text) => {
+    const written = source.slice(text.start, text.end);
+    return [...written.matchAll(BARE_URL)].flatMap((match) => {
+      const end = match.index + match[0].length;
+      const next = characterAt(written, end);
+      return RUNS_ON.test(next) ? [{ url: { start: text.start + match.index, end: text.start + end }, next }] : [];
+    });
+  });
+
+export const urlRunOn: Detector = (doc): Finding[] =>
+  runOnUrls(doc.source, doc.markup?.texts ?? []).map(({ url, next }) => findingAt(doc, url, { url: quoteOf(doc.source, url), next }));

package/src/detectors/vague-figure-pointer.ts

@@ -0,0 +1,58 @@
+import type { Detector, Finding, Lexicon, ProseDocument } from "../plugin.ts";
+import { FIGURE_NUMBER, labelledKindsIn } from "../figure-references.ts";
+import { escapeRegExp } from "../orthography.ts";
+import { labelWordsOf } from "./dangling-figure.ts";
+import { quoteAt } from "./structure-tree.ts";
+import { numbersClauses } from "../structure/numbered-clauses.ts";
+
+// A figure or a table pointed at by where it is (上記の図, the table below) in a document that numbers that kind and
+// could name it (図3, Table 2), and a clause (以下の箇条, the clause below) in a document that numbers its clauses.
+// JIS Z 8301:2019 10.6. The phrases and the kind each points at come from the lexicons.
+
+export type VaguePointer = { readonly offset: number; readonly written: string; readonly kind: string };
+
+/** A phrase inside a longer word (向上の図, 上の図表) is not one. */
+const EDGE = "[\\p{Script=Han}\\p{Script=Katakana}A-Za-z0-9０-９]";
+
+/** Matched in any case (The figure below). */
+const pointerPattern = (pattern: string): RegExp => new RegExp(`(?<!${EDGE})${escapeRegExp(pattern)}(?!${EDGE})`, "giu");
+
+/** A pointer followed by a number names the figure (上の図 １, the figure below 2, the table above IV): the numbers figure labels take. */
+const NUMBERED = new RegExp(`^${FIGURE_NUMBER}`, "u");
+/** Enough characters after a pointer to hold a figure's number (第十二, 3.2.1). */
+const NUMBER_LOOKAHEAD = 16;
+
+/** Pure: every pointer in the prose that points at a kind the document labels. */
+export const vaguePointers = (prose: string, pointers: Lexicon, labelledKinds: ReadonlySet<string>): VaguePointer[] =>
+  pointers
+    .flatMap((entry) => {
+      const kind = entry.instead_of;
+      if (kind === undefined || !labelledKinds.has(kind)) return [];
+      return [...prose.matchAll(pointerPattern(entry.pattern))]
+        .filter((match) => !NUMBERED.test(prose.slice(match.index + match[0].length, match.index + match[0].length + NUMBER_LOOKAHEAD)))
+        .map((match) => ({ offset: match.index, written: match[0], kind }));
+    })
+    .toSorted((left, right) => left.offset - right.offset);
+
+const kindsOf = (pointers: Lexicon): Set<string> => new Set(pointers.flatMap((entry) => entry.instead_of ?? []));
+
+/** The clause kinds to check. The tree is built only when a clause pointer is written, so other documents do not pay for it. */
+const clauseKindsIn = (doc: ProseDocument, prose: string, pointers: Lexicon): string[] => {
+  const kinds = kindsOf(pointers);
+  if (vaguePointers(prose, pointers, kinds).length === 0) return [];
+  return doc.structure !== undefined && numbersClauses(doc.structure) ? [...kinds] : [];
+};
+
+export const vagueFigurePointer: Detector = (doc): Finding[] => {
+  const prose = doc.prose ?? doc.source;
+  const clausePointers = doc.lexicons["vague-clause-pointer"] ?? [];
+  const kinds = new Set([...labelledKindsIn(doc.source, labelWordsOf(doc)), ...clauseKindsIn(doc, prose, clausePointers)]);
+  return vaguePointers(prose, [...(doc.lexicons["vague-figure-pointer"] ?? []), ...clausePointers], kinds).map((pointer) => ({
+    rule: "",
+    severity: "warning",
+    line: 0,
+    column: 0,
+    quote: quoteAt(doc.source, pointer.offset),
+    values: { matched: pointer.written, kind: pointer.kind, offset: pointer.offset },
+  }));
+};

package/src/detectors/word-list.ts

@@ -1,2 +1,7 @@
+const separatorOf = (language: string): string => (language === "ja" ? "、" : ", ");
+
 /** 語の並べ方は文書の言語に合わせる。日本語は「、」、ほかは ", "。英語の画面に「Risks、Costs」と出さない。 */
-export const joinWords = (words: readonly string[], language: string): string => words.join(language === "ja" ? "、" : ", ");
+export const joinWords = (words: readonly string[], language: string): string => words.join(separatorOf(language));
+
+/** joinWords で並べた語を、元の語に戻す。 */
+export const splitWords = (joined: string, language: string): string[] => (joined === "" ? [] : joined.split(separatorOf(language)));

package/src/document-sections.ts

@@ -0,0 +1,46 @@
+import type { Heading } from "./document.ts";
+import { unlabeledReader } from "./heading-label.ts";
+import type { LanguageAdapter, Section, Sentence, Span, Token } from "./plugin.ts";
+
+const within = (span: Span, from: number, to: number): boolean => span.start >= from && span.start < to;
+
+/** 見出しの語。品詞を読んでいない文書（文が tokens を持たない）では分けない。 */
+const headingTokensOf = (adapter: LanguageAdapter, tagged: boolean, heading: string): { headingTokens?: readonly Token[] } =>
+  tagged && heading !== "" ? { headingTokens: adapter.segment(heading).sentences.flatMap((sentence) => sentence.tokens ?? []) } : {};
+
+type HeadingReaders = { readonly tokensOf: (heading: string) => { headingTokens?: readonly Token[] }; readonly unlabeledOf: (heading: string) => () => string };
+
+/** 見出しから読むもの: 語（品詞を読んだ文書だけ）と、頭の番号の札を除いた題（読まれたときに）。 */
+export const headingReadersOf = (adapter: LanguageAdapter, tagged: boolean, lexicons: LanguageAdapter["lexicons"]): HeadingReaders => ({
+  tokensOf: (heading) => headingTokensOf(adapter, tagged, heading),
+  unlabeledOf: unlabeledReader(adapter.structure, lexicons),
+});
+
+export const sectionsOf = (
+  headings: readonly Heading[],
+  sentences: readonly Sentence[],
+  strongs: readonly Span[],
+  length: number,
+  readers: HeadingReaders,
+): Section[] => {
+  const bounds = headings.map((heading, index) => ({ heading, from: heading.end, to: headings[index + 1]?.start ?? length }));
+  const lead = { heading: { depth: 0, text: "", start: 0, end: 0 }, from: 0, to: headings[0]?.start ?? length };
+  return [lead, ...bounds]
+    .filter((bound) => bound.to > bound.from)
+    .map(({ heading, from, to }) => {
+      const inside = sentences.filter((sentence) => within(sentence.span, from, to));
+      const unlabeled = readers.unlabeledOf(heading.text);
+      return {
+        depth: heading.depth,
+        heading: heading.text,
+        ...readers.tokensOf(heading.text),
+        get unlabeledHeading(): string {
+          return unlabeled();
+        },
+        span: { start: from, end: to },
+        sentences: inside,
+        strongCount: strongs.filter((span) => within(span, from, to)).length,
+        firstSentence: inside[0],
+      };
+    });
+};