chaffjs 0.28.0 → 0.30.0

package/src/unnamed-speaker.ts

@@ -0,0 +1,224 @@
+// A quotation whose speaker, in the words right beside it, is a role and not a name (業界のアナリストは「…」と述べた,
+// "…," said an analyst). The words come from the language's word lists: unnamed-speaker (roles, what may follow a role,
+// titles, words pointing back, the quoting particle) and quote-attribution (said, according to, と述べ). Pure.
+import { QUOTATION_MARKS, quotedSpans } from "./quoted-span.ts";
+import type { LexiconEntry, Span } from "./plugin.ts";
+
+export type SpeakerSentence = { readonly start: number; readonly text: string };
+
+export type UnnamedSpeakerQuote = { readonly offset: number; readonly quote: string; readonly speaker: string };
+
+/** The word list unnamed-speaker, by group, and quote-attribution with the list's own cues (group cue), longest first. */
+export type SpeakerWords = {
+  readonly roles: readonly string[];
+  readonly afterRole: readonly string[];
+  readonly honorifics: readonly string[];
+  readonly anaphors: readonly string[];
+  readonly afterQuote: readonly string[];
+  readonly cues: readonly LexiconEntry[];
+};
+
+export const speakerWords = (speaker: readonly LexiconEntry[], cues: readonly LexiconEntry[]): SpeakerWords => {
+  const group = (name: string): string[] => speaker.filter((entry) => entry.group === name && entry.pattern !== "").map((entry) => entry.pattern);
+  return {
+    roles: group("role"),
+    afterRole: group("after-role"),
+    honorifics: group("honorific"),
+    anaphors: group("anaphor"),
+    afterQuote: group("after-quote"),
+    cues: [...cues, ...speaker.filter((entry) => entry.group === "cue")]
+      .filter((cue) => cue.pattern !== "")
+      .toSorted((left, right) => right.pattern.length - left.pattern.length),
+  };
+};
+
+/** Shorter than this, a quotation is a term or a label (「保存」, "Save"), not someone's words. */
+const MIN_QUOTE_CHARS = 10;
+
+const LATIN = /\p{Script=Latin}/u;
+const LETTER = /\p{L}/u;
+const UPPER_START = /^\p{Lu}/u;
+const LOWER_START = /^\p{Ll}/u;
+const WORD = /[\p{L}\p{N}'’-]+/gu;
+/** Words, and the commas between them: a comma ends a run of capitalised words (an analyst at Acme, Hanako Mori). */
+const WORD_OR_COMMA = /[\p{L}\p{N}'’-]+|[,，]/gu;
+/** A Japanese title is not one when a kanji follows it (氏名, 様々). */
+const JOINED_KANJI = /^[\p{Script=Han}々]/u;
+/** How far from the quotation the speaker is looked for, on each side: keeps a long sentence of many quotations linear. */
+const SPEAKER_REACH = 200;
+/** Where the clause holding the speaker before a quotation starts: after these marks. */
+const CLAUSE_MARKS_BEFORE = /[。．！？!?;；「」『』“”"、，]/u;
+/** Where the speaker's phrase after a quotation ends. A comma does not end it in English (said Hanako Mori, an analyst). */
+const CLAUSE_MARKS_AFTER = /[。．！？.!?;；「」『』“”"]/u;
+/** Where a phrase after the quoting particle (「…」と業界関係者は話す) ends. */
+const CLAUSE_MARKS_PARTICLE = /[。．！？.!?;；「」『』“”"、，]/u;
+/** Where a speaker standing before its cue ("…," an analyst said) must not reach past. */
+const CLAUSE_MARKS_INNER = /[,，、。．！？.!?;；:：]/u;
+/** Spaces and marks that join a speaker to a quotation or a cue, and are not part of either. */
+const GLUE = /[\s,，、:：—–-]/u;
+const PHRASE_END = /^[,，、。．.;；:：)）]/u;
+
+/** A Latin word is matched whole and without case; a Japanese one as written, inside any text. */
+const matchesAt = (text: string, at: number, word: string): boolean => {
+  if (!LATIN.test(word)) return text.startsWith(word, at);
+  return (
+    text.slice(at, at + word.length).toLowerCase() === word.toLowerCase() && !LETTER.test(text.charAt(at - 1)) && !LETTER.test(text.charAt(at + word.length))
+  );
+};
+
+const positionsOf = (text: string, word: string): number[] =>
+  word === "" ? [] : [...Array(Math.max(0, text.length - word.length + 1)).keys()].filter((at) => matchesAt(text, at, word));
+
+const startsWithWord = (text: string, word: string): boolean => matchesAt(text, 0, word);
+
+const endsWithWord = (text: string, word: string): boolean => text.length >= word.length && matchesAt(text, text.length - word.length, word);
+
+const afterLast = (text: string, marks: RegExp): string => {
+  const cut = [...text].reduce((last, char, index) => (marks.test(char) ? index : last), -1);
+  return [...text].slice(cut + 1).join("");
+};
+
+const beforeFirst = (text: string, marks: RegExp): string => {
+  const at = [...text].findIndex((char) => marks.test(char));
+  return at < 0 ? text : [...text].slice(0, at).join("");
+};
+
+/** The text without glue at either end. */
+const clean = (text: string): string => {
+  const chars = [...text];
+  const first = chars.findIndex((char) => !GLUE.test(char));
+  return first < 0 ? "" : chars.slice(first, chars.findLastIndex((char) => !GLUE.test(char)) + 1).join("");
+};
+
+/** leavesSpeakerBefore: after the quotation, whether the speaker may still be the one before it (「…」と述べた), not one after it (said X). */
+type Side = { readonly speaker: string; readonly attributed: boolean; readonly leavesSpeakerBefore?: boolean };
+
+/** The clause before the quotation: the speaker, with a cue at its end (An analyst said, 関係者によれば) or before it (according to). */
+const speakerBefore = (before: string, words: SpeakerWords): Side => {
+  const clause = clean(afterLast(clean(before), CLAUSE_MARKS_BEFORE));
+  const cues = words.cues.filter((cue) => cue.position !== "after");
+  const ending = cues.find((cue) => endsWithWord(clause, cue.pattern));
+  if (ending !== undefined) return { speaker: clean(clause.slice(0, clause.length - ending.pattern.length)), attributed: true };
+  const leads = cues
+    .filter((cue) => cue.position === "before")
+    .flatMap((cue) => positionsOf(clause, cue.pattern).map((at) => at + cue.pattern.length))
+    .toSorted((left, right) => right - left);
+  const lead = leads[0];
+  if (lead !== undefined) return { speaker: clean(clause.slice(lead)), attributed: true };
+  return { speaker: clause, attributed: false };
+};
+
+/** The words after the quotation: a quoting particle (「…」と業界関係者は), a cue then the speaker (said an analyst), or the speaker then a cue. */
+const speakerAfter = (after: string, words: SpeakerWords): Side => {
+  const rest = clean(after);
+  const particle = words.afterQuote.find((word) => startsWithWord(rest, word));
+  if (particle !== undefined)
+    return { speaker: clean(beforeFirst(clean(rest.slice(particle.length)), CLAUSE_MARKS_PARTICLE)), attributed: true, leavesSpeakerBefore: true };
+  const cues = words.cues.filter((cue) => cue.position !== "before");
+  const leading = cues.find((cue) => startsWithWord(rest, cue.pattern));
+  if (leading !== undefined) {
+    const speaker = clean(beforeFirst(rest.slice(leading.pattern.length), CLAUSE_MARKS_AFTER));
+    return { speaker, attributed: true, leavesSpeakerBefore: speaker === "" };
+  }
+  const clause = beforeFirst(rest, CLAUSE_MARKS_INNER);
+  const cueAt = cues.flatMap((cue) => positionsOf(clause, cue.pattern)).toSorted((left, right) => left - right)[0];
+  if (cueAt !== undefined) return { speaker: clean(clause.slice(0, cueAt)), attributed: true, leavesSpeakerBefore: cueAt === 0 };
+  return { speaker: "", attributed: false };
+};
+
+const isCapitalised = (word: string | undefined): boolean => word !== undefined && UPPER_START.test(word);
+
+const isRoleWord = (word: string | undefined, words: SpeakerWords): boolean =>
+  word !== undefined && words.roles.some((role) => LATIN.test(role) && role.toLowerCase() === word.toLowerCase());
+
+const isAfterRoleWord = (word: string | undefined, words: SpeakerWords): boolean =>
+  word !== undefined && words.afterRole.some((entry) => entry.toLowerCase() === word.toLowerCase());
+
+/**
+ * A run of capitalised words that is a person's name (Hanako Mori, analyst Hanako Mori). Not one: a run after a word like
+ * "at" (an analyst at Minato Research), a run right before a role (an Acme spokesperson), or, when the phrase opens the
+ * clause, its first word alone ("An analyst", "Experts"). After a cue it is a name: said Mori, an analyst.
+ */
+const holdsCapitalisedName = (phrase: string, words: SpeakerWords, opensSentence: boolean): boolean => {
+  const tokens = [...phrase.matchAll(WORD_OR_COMMA)].map((match) => match[0]);
+  return tokens.some((token, index) => {
+    if (!isCapitalised(token) || isCapitalised(tokens[index - 1])) return false;
+    const runEnd = tokens.findIndex((later, at) => at > index && !isCapitalised(later));
+    const end = runEnd < 0 ? tokens.length : runEnd;
+    if (isAfterRoleWord(tokens[index - 1], words) || isRoleWord(tokens[end], words)) return false;
+    const next = tokens[1];
+    return !(opensSentence && index === 0 && end === 1 && (next === undefined || LOWER_START.test(next)));
+  });
+};
+
+const holdsTitle = (phrase: string, title: string): boolean =>
+  positionsOf(phrase, title).some((at) => LATIN.test(title) || !JOINED_KANJI.test(phrase.slice(at + title.length)));
+
+const isNamed = (phrase: string, words: SpeakerWords, opensSentence: boolean): boolean =>
+  words.honorifics.some((title) => holdsTitle(phrase, title)) || holdsCapitalisedName(phrase, words, opensSentence);
+
+const lastWord = (text: string): string => [...text.matchAll(WORD)].map((match) => match[0]).at(-1) ?? "";
+
+/** Whether the role at this place is the head of the phrase: nothing after it, or only a word like は or "at". */
+const headsPhrase = (phrase: string, at: number, role: string, words: SpeakerWords): boolean => {
+  const after = phrase.slice(at + role.length).trimStart();
+  return after === "" || PHRASE_END.test(after) || words.afterRole.some((word) => startsWithWord(after, word));
+};
+
+const pointsBack = (phrase: string, at: number, words: SpeakerWords): boolean => {
+  const before = phrase.slice(0, at).trimEnd();
+  return words.anaphors.some((word) => (LATIN.test(word) ? lastWord(before).toLowerCase() === word.toLowerCase() : before.endsWith(word)));
+};
+
+/**
+ * The phrase as the message shows it, when a role heads it and does not point back at someone named before (同アナリスト,
+ * the analyst). A Japanese phrase is shown up to its role (業界関係者は話す as 業界関係者); an English one whole.
+ */
+const unnamedRoleIn = (phrase: string, words: SpeakerWords): string | undefined => {
+  const heads = words.roles.flatMap((role) =>
+    positionsOf(phrase, role)
+      .filter((at) => headsPhrase(phrase, at, role, words) && !pointsBack(phrase, at, words))
+      .map((at) => (LATIN.test(role) ? phrase : phrase.slice(0, at + role.length))),
+  );
+  return heads[0];
+};
+
+type Phrase = { readonly text: string; readonly opensSentence: boolean };
+
+/** The phrases that may be the speaker: the attributed side's, and the one before when a cue after leaves the speaker there. */
+const speakerPhrases = (before: Side, after: Side, beforeOpens: boolean): Phrase[] => {
+  const useBefore = before.attributed || (after.attributed && after.leavesSpeakerBefore === true);
+  return [
+    ...(useBefore ? [{ text: before.speaker, opensSentence: beforeOpens }] : []),
+    ...(after.attributed ? [{ text: after.speaker, opensSentence: false }] : []),
+  ].filter((phrase) => phrase.text !== "");
+};
+
+/** The phrase that gives this quotation to a role and to no name, if any. */
+const unnamedSpeakerOf = (text: string, span: Span, words: SpeakerWords): string | undefined => {
+  const reach = Math.max(0, span.start - 1 - SPEAKER_REACH);
+  const before = speakerBefore(text.slice(reach, span.start - 1), words);
+  const after = speakerAfter(text.slice(span.end + 1, span.end + 1 + SPEAKER_REACH), words);
+  const phrases = speakerPhrases(
+    before,
+    after,
+    reach === 0 &&
+      text
+        .slice(0, span.start - 1)
+        .trim()
+        .startsWith(before.speaker),
+  );
+  if (phrases.some((phrase) => isNamed(phrase.text, words, phrase.opensSentence))) return undefined;
+  return phrases.map((phrase) => unnamedRoleIn(phrase.text, words)).find((speaker) => speaker !== undefined);
+};
+
+/** Each quotation in the sentence given to a role with no name. */
+export const unnamedSpeakerQuotes = (sentence: SpeakerSentence, words: SpeakerWords): UnnamedSpeakerQuote[] => {
+  if (words.roles.length === 0) return [];
+  return quotedSpans(sentence.text, QUOTATION_MARKS).flatMap((span) => {
+    const quote = clean(sentence.text.slice(span.start, span.end));
+    if ([...quote].length < MIN_QUOTE_CHARS) return [];
+    const speaker = unnamedSpeakerOf(sentence.text, span, words);
+    return speaker === undefined ? [] : [{ offset: sentence.start + span.start, quote, speaker }];
+  });
+};

package/src/warning-action.ts

@@ -0,0 +1,73 @@
+// A warning (Warning:, 注意：, > [!WARNING]) that names a risk and gives the reader nothing to do. Pure: reads a paragraph's
+// text and its sentences; the labels and the words of an action come from the language's warning-label and warning-action
+// lexicons.
+import { escapeRegExp } from "./orthography.ts";
+
+export type WarningWords = {
+  /** What a warning opens with: Warning, Caution, 注意, 警告. */
+  readonly labels: readonly string[];
+  /**
+   * What makes a sentence an action. A Latin entry is matched as whole words anywhere (do not, make sure, must); another is
+   * matched at the end of the sentence (ください, ましょう), where Japanese puts what the reader is to do.
+   */
+  readonly actions: readonly string[];
+};
+
+/** What may come before the label: a quote mark, bold, a GitHub alert's [! (> [!WARNING], **Warning:**). */
+const BEFORE_LABEL = /^(?:>[ \t]*)*(?:\*\*|__)?(?:\[!)?/u;
+/** What closes the label: the end of bold or of an alert, then a colon or an exclamation mark, or the end of the line. */
+const LABEL_CLOSE = /^(?:\*\*|__)?\]?[ \t]*/u;
+const LABEL_MARK = /^[:：!！](?:\*\*|__)?[ \t]*/u;
+const SENTENCE_END_MARKS = new Set(" \t\r\n。．.!！?？」』)）");
+const LATIN_START = /^[A-Za-z]/u;
+const LETTER = /\p{L}/u;
+
+/**
+ * Where the text after a paragraph's warning label starts, or undefined when the paragraph opens with no label. The label
+ * must stand alone: "Warning:", "注意：", "**Caution**", "> [!WARNING]", not "Warnings are logged" or 「注意深く」.
+ */
+export const warningTextStart = (paragraph: string, labels: readonly string[]): number | undefined => {
+  const lead = BEFORE_LABEL.exec(paragraph)?.[0].length ?? 0;
+  const rest = paragraph.slice(lead);
+  const label = labels.find((word) => rest.toLowerCase().startsWith(word.toLowerCase()));
+  if (label === undefined) return undefined;
+  const after = rest.slice(label.length);
+  const close = LABEL_CLOSE.exec(after)?.[0].length ?? 0;
+  const mark = LABEL_MARK.exec(after.slice(close))?.[0].length;
+  if (mark !== undefined) return lead + label.length + close + mark;
+  const next = after.slice(close);
+  return next === "" || next.startsWith("\n") || next.startsWith("\r\n") ? lead + label.length + close : undefined;
+};
+
+const SPACES = /\s+/gu;
+
+/** A word or phrase as whole words, with any run of spaces or a line break between its words ("Do\nnot"). */
+const hasWholeWord = (text: string, word: string): boolean => {
+  const lowered = text.replaceAll(SPACES, " ").toLowerCase();
+  const target = word.toLowerCase();
+  return [...lowered.matchAll(new RegExp(escapeRegExp(target), "gu"))].some(
+    (match) => !LETTER.test(lowered[match.index - 1] ?? "") && !LETTER.test(lowered[match.index + target.length] ?? ""),
+  );
+};
+
+/** Whether a sentence tells the reader what to do, by the words of the warning-action lexicon. */
+export const namesAction = (sentence: string, actions: readonly string[]): boolean => {
+  const chars = [...sentence];
+  const ending = chars.slice(0, chars.findLastIndex((char) => !SENTENCE_END_MARKS.has(char)) + 1).join("");
+  return actions.some((word) => (LATIN_START.test(word) ? hasWholeWord(sentence, word) : ending.endsWith(word)));
+};
+
+const QUOTED = /^[ \t]*>/u;
+
+/**
+ * Where the block quote holding an offset ends, or undefined when its line is not quoted. A GitHub alert (> [!WARNING])
+ * runs over every quoted line after it, blank quoted lines (>) included, so its action may be in a later paragraph.
+ */
+export const quoteBlockEnd = (source: string, offset: number): number | undefined => {
+  const lineStart = source.lastIndexOf("\n", offset - 1) + 1;
+  const lines = source.slice(lineStart).split("\n");
+  const quoted = lines.findIndex((line) => !QUOTED.test(line));
+  const count = quoted < 0 ? lines.length : quoted;
+  if (count === 0) return undefined;
+  return lineStart + lines.slice(0, count).join("\n").length;
+};

package/styles/koyobun.yaml

@@ -5,6 +5,8 @@
 #   意識するとよい」。chaff の上限は「ここを超えたら見直す」所なので、その上の端の 60 字を和文の上限にする。英文には言っていないので、英文の上限は変えない。
 # Ⅲ－1 イ「一つの文書内では、常体と敬体のどちらかで統一する」: no-mixed-desumasu。どちらにするかは文書の相手で決まる（Ⅲ－1 ア）ので、決めない。
 # Ⅰ－3 エ「長音は、原則として長音符号を使って書く」: 英語の語末の -er、-or、-ar などは長音符号を付ける（コンピューター）。katakana-long-vowel の keep。
+# Ⅰ－1（3）ア「常用漢字表に使える漢字があっても仮名で書く場合」: 補助動詞（〜ていただく、〜てくださる）と形式名詞（こと、とき、ため、よう）。
+#   「公用文における漢字使用等について」（平成 22 年内閣訓令第 1 号）1 (2) キ の〜ておく、〜てくださいも同じ。kana-function-word。
 id: koyobun
 
 name:
@@ -12,19 +14,20 @@ name:
   en: 公用文作成の考え方 (Council for Cultural Affairs, 2022)
 
 summary:
-  ja: 一文が 60 字を超えたら見直します。一つの文書の中で「です・ます」と「である」を混ぜません。カタカナ語の語末の「ー」は付けます（コンピューター）。
-  en: Flags a Japanese sentence over 60 characters, a document that mixes です・ます with である, and a katakana word that drops its final ー (コンピューター).
+  ja: 一文が 60 字を超えたら見直します。一つの文書の中で「です・ます」と「である」を混ぜません。カタカナ語の語末の「ー」は付けます（コンピューター）。補助動詞と形式名詞はかなで書きます（〜てください、〜すること）。
+  en: Flags a Japanese sentence over 60 characters, a document that mixes です・ます with である, a katakana word that drops its final ー (コンピューター), and a helper verb or formal noun written in kanji where official style writes kana (〜てください, 〜すること).
 
 source:
   title:
-    ja: 文化審議会「公用文作成の考え方」（令和 4 年 1 月 7 日 建議）Ⅲ－3 ア、Ⅲ－1 イ、Ⅰ－3 エ
-    en: 公用文作成の考え方 (recommendation of the Council for Cultural Affairs, 7 January 2022), III-3 a, III-1 b and I-3 d
+    ja: 文化審議会「公用文作成の考え方」（令和 4 年 1 月 7 日 建議）Ⅲ－3 ア、Ⅲ－1 イ、Ⅰ－3 エ、Ⅰ－1（3）ア
+    en: 公用文作成の考え方 (recommendation of the Council for Cultural Affairs, 7 January 2022), III-3 a, III-1 b, I-3 d and I-1 (3) a
   url: https://www.bunka.go.jp/seisaku/bunkashingikai/kokugo/hokoku/pdf/93651301_01.pdf
 
 rules:
   max-sentence-length: normal
   no-mixed-desumasu: normal
   katakana-long-vowel: normal
+  kana-function-word: normal
 
 limits:
   max-sentence-length: