chaffjs 0.29.0 → 0.30.0

package/src/structure/requirement-lists.ts

@@ -0,0 +1,145 @@
+// required-listed-as-preferred: one skill or qualification listed under a required heading or label and again under a
+// preferred one. The headings and labels, the units and words of a length of experience, and the words that only end an
+// item come from the lexicons. Pure.
+import { escapeRegExp } from "../orthography.ts";
+import type { StructureIssue } from "./issues.ts";
+
+/** A heading, a line standing alone (a label if it is one of the words), or one list item, in document order. */
+export type RequirementBlock =
+  | { readonly kind: "heading"; readonly depth: number; readonly text: string; readonly offset: number }
+  | { readonly kind: "line"; readonly text: string; readonly offset: number }
+  | { readonly kind: "item"; readonly text: string; readonly offset: number };
+
+export type ValuedWord = { readonly word: string; readonly value: number };
+
+export type RequirementWords = {
+  /** Headings and labels of what an applicant must have (必須, Required) and of what is welcome (歓迎, Preferred). */
+  readonly required: readonly string[];
+  readonly preferred: readonly string[];
+  /** Units of a length of experience, valued in months (年 12, month 1). */
+  readonly units: readonly ValuedWord[];
+  /** Words that stand before or after a length and belong to it (at least, 以上, of). */
+  readonly before: readonly string[];
+  readonly after: readonly string[];
+  /** Numbers written as words (three, 三). */
+  readonly numbers: readonly ValuedWord[];
+  /** Words that only end an item and are not the skill (の方, こと). */
+  readonly endings: readonly string[];
+};
+
+type Kind = "required" | "preferred";
+
+/** container is the offset of the heading holding the lists (a posting, 応募の条件), -1 for none; only one container's items are compared. */
+type Requirement = { readonly kind: Kind; readonly text: string; readonly offset: number; readonly container: number };
+
+/** The skill an item names with its length of experience taken out, and that length in months (0 when none). */
+type Reading = { readonly core: string; readonly months: number };
+
+const MARKS = /[*_`~]/gu;
+
+const LATIN = /^[a-z]/u;
+
+const plain = (text: string): string => text.normalize("NFKC").toLowerCase().replace(MARKS, "").replace(/\s+/gu, " ").trim();
+
+/** A Latin word is matched at word edges; other scripts as written. */
+const wordPattern = (word: string): string => {
+  const escaped = escapeRegExp(plain(word));
+  return LATIN.test(plain(word)) ? `(?<![a-z])${escaped}(?![a-z])` : escaped;
+};
+
+const alternation = (words: readonly string[]): string =>
+  words
+    .toSorted((left, right) => right.length - left.length)
+    .map(wordPattern)
+    .join("|");
+
+/** The text without the characters at either end that the test accepts (no backtracking regex on long lines). */
+const trimWhere = (text: string, isEdge: (char: string) => boolean): string => {
+  const chars = [...text];
+  const start = chars.findIndex((char) => !isEdge(char));
+  return start === -1 ? "" : chars.slice(start, chars.findLastIndex((char) => !isEdge(char)) + 1).join("");
+};
+
+const LABEL_EDGE = /[\s【［[「(（】］\]」)）:：]/u;
+
+const asLabel = (text: string): string => trimWhere(plain(text).replace(/[-‐–]/gu, " "), (char) => LABEL_EDGE.test(char)).replace(/\s+/gu, " ");
+
+const kindOfLabel = (label: string, words: RequirementWords): Kind | undefined => {
+  if (label === "") return undefined;
+  if (words.required.some((word) => asLabel(word) === label)) return "required";
+  return words.preferred.some((word) => asLabel(word) === label) ? "preferred" : undefined;
+};
+
+const HEADING_PARTS = /[・/／、,，(（:：]/u;
+
+/** A heading names its kind by its first part (必須スキル・経験, 歓迎（あれば尚可）); a line standing alone, by all of it. */
+const kindOfHeading = (text: string, words: RequirementWords): Kind | undefined => kindOfLabel(asLabel(text.split(HEADING_PARTS)[0] ?? ""), words);
+
+type Scope = { readonly depth: number; readonly kind: Kind | undefined; readonly offset: number };
+
+type Walk = { readonly scopes: readonly Scope[]; readonly label: Kind | undefined; readonly items: readonly Requirement[] };
+
+/** A heading opens a scope that its subheadings inherit unless they name their own kind; a label holds until the next heading or label. */
+const step = (walk: Walk, block: RequirementBlock, words: RequirementWords): Walk => {
+  if (block.kind === "heading") {
+    const outer = walk.scopes.filter((scope) => scope.depth < block.depth);
+    const kind = kindOfHeading(block.text, words) ?? outer.at(-1)?.kind;
+    return { ...walk, scopes: [...outer, { depth: block.depth, kind, offset: block.offset }], label: undefined };
+  }
+  if (block.kind === "line") {
+    const kind = kindOfLabel(asLabel(block.text), words);
+    return kind === undefined ? walk : { ...walk, label: kind };
+  }
+  const kind = walk.label ?? walk.scopes.at(-1)?.kind;
+  const container = walk.scopes.findLast((scope) => scope.kind === undefined)?.offset ?? -1;
+  return kind === undefined ? walk : { ...walk, items: [...walk.items, { kind, text: block.text.trim(), offset: block.offset, container }] };
+};
+
+const numberPattern = (words: RequirementWords): string =>
+  [String.raw`\d+(?:\.\d+)?`, alternation(words.numbers.map((entry) => entry.word))].filter((part) => part !== "").join("|");
+
+const lengthPattern = (words: RequirementWords): RegExp | undefined => {
+  if (words.units.length === 0) return undefined;
+  const before = words.before.length === 0 ? "" : `(?:(?:${alternation(words.before)})\\s*)?`;
+  const after = words.after.length === 0 ? "" : `(?:\\s*(?:${alternation(words.after)}))*`;
+  return new RegExp(`${before}(${numberPattern(words)})\\s*\\+?\\s*(${alternation(words.units.map((unit) => unit.word))})['’]?${after}`, "gu");
+};
+
+const valueOf = (written: string, words: readonly ValuedWord[]): number => words.find((entry) => plain(entry.word) === written)?.value ?? Number(written);
+
+/** Punctuation, symbols and spaces, but not the + and # that end a name (C++, C#). */
+const PUNCTUATION = /(?<![\p{L}\p{N}+#])[+#]|[^\p{L}\p{N}+#]/gu;
+
+const EDGE_PUNCTUATION = /[\p{P}\s]/u;
+
+const withoutEnding = (written: string, endings: readonly string[]): string => {
+  const text = trimWhere(written, (char) => char !== "#" && EDGE_PUNCTUATION.test(char));
+  const ending = endings.map(plain).find((word) => word !== "" && text.endsWith(word) && text.length > word.length);
+  return ending === undefined ? text : withoutEnding(text.slice(0, -ending.length), endings);
+};
+
+const readingOf = (text: string, words: RequirementWords): Reading => {
+  const written = plain(text);
+  const pattern = lengthPattern(words);
+  const lengths = pattern === undefined ? [] : [...written.matchAll(pattern)];
+  const months = lengths.map((match) => valueOf(match[1] ?? "", words.numbers) * (words.units.find((unit) => plain(unit.word) === match[2])?.value ?? 0));
+  const rest = pattern === undefined ? written : written.replace(pattern, " ");
+  return { core: withoutEnding(rest.replace(/\s+/gu, " ").trim(), words.endings).replace(PUNCTUATION, ""), months: Math.max(0, ...months) };
+};
+
+/** A preferred item repeats a required one when it names the same skill and asks for no longer experience. */
+const repeats = (preferred: Reading, required: Reading): boolean =>
+  preferred.core !== "" && preferred.core === required.core && preferred.months <= required.months;
+
+/** The preferred items that repeat a required item of the same document, once each, in document order. */
+export const requiredListedAsPreferred = (blocks: readonly RequirementBlock[], words: RequirementWords): StructureIssue[] => {
+  const { items } = blocks.reduce<Walk>((walk, block) => step(walk, block, words), { scopes: [], label: undefined, items: [] });
+  const required = items.filter((item) => item.kind === "required").map((item) => ({ item, reading: readingOf(item.text, words) }));
+  return items
+    .filter((item) => item.kind === "preferred")
+    .flatMap((item) => {
+      const reading = readingOf(item.text, words);
+      const match = required.find((candidate) => candidate.item.container === item.container && repeats(reading, candidate.reading));
+      return match === undefined ? [] : [{ offset: item.offset, values: { item: item.text, required: match.item.text } }];
+    });
+};

package/src/structure/time-order.ts

@@ -13,7 +13,7 @@ import { crossesZones, dayShiftOf, withoutDayShiftBefore, zonesIn, SECONDS_PER_D
  */
 
 /** 頭に時刻を書いた項目。seconds は 0 時からの秒、shift は翌日の印（1）か無印（0）。 */
-type Stop = {
+export type Stop = {
   readonly offset: number;
   readonly written: string;
   readonly seconds: number;
@@ -130,11 +130,12 @@ const breaksIn = (day: readonly Stop[]): StructureIssue[] => {
   return values.slice(1).flatMap((value, index) => (value < (values[index] ?? "") ? issueAt(day, outOfPlace(values, index + 1, 1)) : []));
 };
 
-/** 一日の予定の時刻のうち、早い順から外れたもの。 */
-export const timeOrderBreaks = (source: string, times: readonly TimeSpan[], dates: readonly DateSpan[], words: TimeWords): StructureIssue[] => {
+/** 一日の予定として並べた時刻を、日ごとに。並びの中で日付か時間帯が変われば別の日。 */
+export const scheduleDays = (source: string, times: readonly TimeSpan[], dates: readonly DateSpan[], words: TimeWords): Stop[][] => {
   const inputs = { source, times, dates, words };
-  return runsOf(source).flatMap((run) => {
-    const stops = run.flatMap((line) => stopOf(line, inputs) ?? []);
-    return daysOf(stops).flatMap(breaksIn);
-  });
+  return runsOf(source).flatMap((run) => daysOf(run.flatMap((line) => stopOf(line, inputs) ?? [])));
 };
+
+/** 一日の予定の時刻のうち、早い順から外れたもの。 */
+export const timeOrderBreaks = (source: string, times: readonly TimeSpan[], dates: readonly DateSpan[], words: TimeWords): StructureIssue[] =>
+  scheduleDays(source, times, dates, words).flatMap(breaksIn);

package/src/structure/version-order.ts

@@ -32,6 +32,16 @@ const compareVersions = (left: Version, right: Version): number => {
   return differing === -1 ? comparePrerelease(left.prerelease, right.prerelease) : Math.sign((left.parts[differing] ?? 0) - (right.parts[differing] ?? 0));
 };
 
+/** Whether every label is a version and no step goes against direction (equal versions do not break it). */
+export const versionsAlong = (labels: readonly (string | undefined)[], direction: number): boolean => {
+  const versions = labels.flatMap((label) => {
+    const version = label === undefined ? undefined : parse(label);
+    return version === undefined ? [] : [version];
+  });
+  if (versions.length !== labels.length) return false;
+  return versions.slice(1).every((version, index) => compareVersions(version, versions[index] ?? version) !== -direction);
+};
+
 /** The longest run along direction ending at each version (equal versions do not break it). */
 const longestEndingAt = (versions: readonly Version[], direction: number): number[] =>
   versions.reduce<number[]>((lengths, version, index) => {

package/src/surname-near.ts

@@ -0,0 +1,31 @@
+// 名が同じ人の名前の、姓だけの書き損じ（Whitford と Whitfield）。一字違い（isNearWord）より遠い違いを、英字の長い姓に限って見る。
+
+const LATIN_WORD = /^[a-z]+$/u;
+
+/** 二字違いまで見る姓の長さ。短い姓（Martin と Marten、Hansen と Hanson）は別の姓のことが多い。 */
+const MIN_SURNAME_LENGTH = 7;
+/** 頭からそろう字の数。頭が違えば別の姓（Robertson と Robinson）。 */
+const MIN_SURNAME_PREFIX = 4;
+/** 頭と終わりのそろった字のあいだに残る字の数。短いほうで二字まで、長いほうで三字まで（Whitford の or と Whitfield の iel）。 */
+const MAX_SURNAME_GAP = 2;
+
+const sharedPrefix = (left: string, right: string): number => {
+  const differ = [...left].findIndex((char, index) => char !== right.charAt(index));
+  return differ < 0 ? left.length : differ;
+};
+
+/**
+ * 二つの英字の姓が、頭が四字以上そろい、終わりの字もそろい、あいだの二字（長いほうは三字）までだけ違うか。どちらも七字以上で。
+ * 終わりだけの違い（Whitford と Whitfords）は語の形の違い。
+ */
+export const isNearSurname = (left: string, right: string): boolean => {
+  if (left === right || !LATIN_WORD.test(left) || !LATIN_WORD.test(right) || Math.min(left.length, right.length) < MIN_SURNAME_LENGTH) return false;
+  const prefix = sharedPrefix(left, right);
+  if (prefix < MIN_SURNAME_PREFIX) return false;
+  const [leftRest, rightRest] = [left.slice(prefix), right.slice(prefix)];
+  const suffix = sharedPrefix([...leftRest].toReversed().join(""), [...rightRest].toReversed().join(""));
+  if (suffix === 0) return false;
+  const gaps = [leftRest.length - suffix, rightRest.length - suffix].toSorted((first, second) => first - second);
+  const [shorter = 0, longer = 0] = gaps;
+  return shorter <= MAX_SURNAME_GAP && longer <= MAX_SURNAME_GAP + 1;
+};

package/src/table-cells.ts

@@ -0,0 +1,41 @@
+import type { ProseDocument, Segmentation, Span, TableCell, Token } from "./plugin.ts";
+import { proseAndTablesOf } from "./table-text.ts";
+import { tableBodyCells, type Cell } from "./facts/table-facts.ts";
+
+// 表の升の語。本文は表を覆うので、升の字は文の分割も品詞解析も通らない。升ごとに adapter に渡し、語の位置を source の上に戻す。
+
+const LETTER = /\p{L}/u;
+
+const shiftToken = (token: Token, by: number): Token => ({ ...token, span: { start: token.span.start + by, end: token.span.end + by } });
+
+/** 升一つの語。字の無い升（数だけ、空）は解析しない。adapter が語を返さなければ tokens は無い。 */
+export const cellWithTokens = (cell: Cell, segment: (text: string) => Segmentation): TableCell => {
+  if (!LETTER.test(cell.text)) return cell;
+  const sentences = segment(cell.text).sentences;
+  if (!sentences.some((sentence) => sentence.tokens !== undefined)) return cell;
+  return { ...cell, tokens: sentences.flatMap((sentence) => (sentence.tokens ?? []).map((token) => shiftToken(token, cell.start))) };
+};
+
+/** 文書の表（引用とコードの外）の本体の升と、その語。 */
+export const tableCellsOf = (doc: ProseDocument, segment: (text: string) => Segmentation): TableCell[] =>
+  tableBodyCells(proseAndTablesOf(doc)).map((cell) => cellWithTokens(cell, segment));
+
+/** prose に、升の字だけを書き戻したもの（表の見出しの行と区切りの | は覆ったまま）。位置は変えない。 */
+export const proseWithCells = (prose: string, cells: readonly (Span & { readonly text: string })[]): string => {
+  const pieces: string[] = [];
+  const end = cells
+    .toSorted((left, right) => left.start - right.start)
+    .reduce((from, cell) => {
+      if (cell.start < from || cell.end - cell.start !== cell.text.length) return from;
+      pieces.push(prose.slice(from, cell.start), cell.text);
+      return cell.end;
+    }, 0);
+  pieces.push(prose.slice(end));
+  return pieces.join("");
+};
+
+/** 表の升の語は、読む rule があるときに一度だけ作る。升ごとの解析を、表を読まない rule に払わせない。 */
+export const lazyCells = (make: () => readonly TableCell[]): (() => readonly TableCell[]) => {
+  const cells: { value?: readonly TableCell[] } = {};
+  return () => (cells.value ??= make());
+};

package/src/table-names.ts

@@ -0,0 +1,113 @@
+import { isNearWord, nameKey, type NameMention } from "./name-variants.ts";
+import type { TableCell, Token } from "./plugin.ts";
+import { withoutEdgeMarks } from "./facts/trim-marks.ts";
+
+// 表の升に書いた名前と、本文の名前の書き分け（本文は Sofia Mendes、担当者の表は Sofia Mendez）。表の升は品詞解析を通らないので、
+// 名前の形をした升だけを読み、本文で読めた名前と比べる。比べるのは姓と名のように二語以上の名前で、一語だけが違うときに限る。
+// 同じ語がほかにあることが、同じ人を指す手がかりになる。漢字・かなの名前は、本文と同じく読みが同じときに限る（佐々木 美穂 と
+// 佐々木 美保）。読みの違う一字違い（佐藤 太郎 と 佐藤 次郎）は別の人のことが多い。
+
+/** 表の升の名前。words は空白で分けた語。reading は升の語の読み（読めなければ無い）。 */
+export type CellName = { readonly surface: string; readonly offset: number; readonly words: readonly string[]; readonly reading?: string | undefined };
+
+/** 本文の名前の書き方と、その数。reading は最初の現れの読み。 */
+export type ProseName = { readonly surface: string; readonly words: readonly string[]; readonly count: number; readonly reading?: string | undefined };
+
+export type CellNameVariant = { readonly name: CellName; readonly usual: string; readonly kind: "spelling" | "reading" | "near" };
+
+/** 英字の名前の升: 大文字で始まる語が二つから四つ。小文字の語（Write the procedure）を含む升は名前でない。 */
+const LATIN_NAME = /^\p{Lu}[\p{L}'’.-]*(?: \p{Lu}[\p{L}'’.-]*){1,3}$/u;
+/** 漢字・カタカナの名前の升: 名前の字の語が、空白一つを挟んで二つか三つ（佐々木 美穂）。 */
+const CJK_NAME = /^[\p{Script=Han}\p{Script=Katakana}ー々]+(?:[ \u3000][\p{Script=Han}\p{Script=Katakana}ー々]+){1,2}$/u;
+const SPACE = /[ \u3000]/u;
+const LOWER = /\p{Ll}/u;
+
+/** 範囲の中の語の読みをつないだもの。記号と空白の語は読まない。読めない語があれば無い。 */
+export const readingWithin = (tokens: readonly Token[], start: number, end: number): string | undefined => {
+  const words = tokens.filter((token) => token.span.start >= start && token.span.end <= end && token.pos !== "PUNCT" && token.surface.trim() !== "");
+  const readings = words.map((token) => token.reading);
+  return words.length > 0 && readings.every((reading) => reading !== undefined && reading !== "") ? readings.join("") : undefined;
+};
+
+/** 名前の形をした升。升の字の前後の空白と強調の印は名前の外。 */
+export const cellNamesIn = (cells: readonly TableCell[]): CellName[] =>
+  cells.flatMap((cell) => {
+    const surface = withoutEdgeMarks(cell.text.trim());
+    if (!(LATIN_NAME.test(surface) && LOWER.test(surface)) && !CJK_NAME.test(surface)) return [];
+    const offset = cell.start + cell.text.indexOf(surface);
+    return [{ surface, offset, words: surface.split(SPACE), reading: readingWithin(cell.tokens ?? [], offset, offset + surface.length) }];
+  });
+
+/**
+ * 本文の名前の書き方ごとの数。空白一つで並んだ名前の現れ（佐々木 と 美穂）は、一つの名前（佐々木 美穂）としても数える。
+ * 日本語の解析器は姓と名のあいだの空白で名前を切る。
+ */
+export const proseNamesOf = (mentions: readonly NameMention[], source: string): ProseName[] => {
+  const sorted = mentions.toSorted((left, right) => left.offset - right.offset);
+  const joined = sorted.flatMap((mention, index): Written[] => {
+    const next = sorted[index + 1];
+    const end = mention.offset + mention.surface.length;
+    if (next === undefined || next.offset !== end + 1 || !SPACE.test(source.charAt(end))) return [];
+    const reading = mention.reading === undefined || next.reading === undefined ? undefined : `${mention.reading}${next.reading}`;
+    return [{ surface: `${mention.surface}${source.charAt(end)}${next.surface}`, reading }];
+  });
+  const forms = new Map<string, ProseName>();
+  [...sorted, ...joined].forEach(({ surface, reading }) => {
+    const form = forms.get(surface);
+    forms.set(surface, form === undefined ? { surface, words: surface.split(SPACE), count: 1, reading } : { ...form, count: form.count + 1 });
+  });
+  return [...forms.values()];
+};
+
+type Written = { readonly surface: string; readonly reading: string | undefined };
+
+const LATIN = /\p{Script=Latin}/u;
+/** 英字の語の中の記号（O'Connor、Anne-Marie）。一字違いは字だけで見る。 */
+const WORD_MARKS = /['’.-]/gu;
+
+/**
+ * 二つの英字の語が一字違いか（名前の見方と同じ isNearWord）。漢字・かなの語は見ない。升は読みが分からず、一字違いの名前
+ * （佐藤 太郎 と 佐藤 次郎）は別の人のことが多い。
+ */
+const isNearNameWord = (left: string, right: string): boolean =>
+  LATIN.test(left) && LATIN.test(right) && isNearWord(left.toLowerCase().replaceAll(WORD_MARKS, ""), right.toLowerCase().replaceAll(WORD_MARKS, ""));
+
+/**
+ * 升の名前が、本文の名前の書き分けか。記号・幅・大小だけの違いか、語の数が同じで一語だけが違い、残りの語が同じとき。違う一語は、
+ * 英字なら一字違い、漢字・かなら名前全体の読みが同じ（本文の名前の見方と同じ）。この二つは、本文の形が二度以上、升の形が文書の
+ * 中で一度だけのとき（別の人のこともある）。
+ */
+export const cellNameRelation = (name: CellName, uses: number, prose: ProseName): CellNameVariant["kind"] | undefined => {
+  if (name.surface === prose.surface || name.words.length !== prose.words.length) return undefined;
+  if (nameKey(name.surface) === nameKey(prose.surface)) return "spelling";
+  const differ = name.words.flatMap((word, index) => (word === prose.words[index] ? [] : [index]));
+  const [at] = differ;
+  if (differ.length !== 1 || at === undefined || uses !== 1 || prose.count < 2) return undefined;
+  if (isNearNameWord(name.words[at] ?? "", prose.words[at] ?? "")) return "near";
+  return name.reading !== undefined && name.reading === prose.reading ? "reading" : undefined;
+};
+
+/** 記号・幅・大小だけの違いを、読みの同じものと一字違いより先に取る。 */
+const KIND_ORDER: readonly CellNameVariant["kind"][] = ["spelling", "reading", "near"];
+
+/**
+ * 表の升の名前のうち、本文の名前の書き分けのもの。記号だけの違いを先に、その中で本文の形のうち一番多いものと比べる。
+ * 書き方ごとに最初の升を一つ。
+ */
+export const cellNameVariants = (names: readonly CellName[], prose: readonly ProseName[]): CellNameVariant[] => {
+  const uses = new Map(prose.map((form) => [form.surface, form.count]));
+  const firsts = new Map<string, CellName>();
+  names.forEach((name) => {
+    uses.set(name.surface, (uses.get(name.surface) ?? 0) + 1);
+    if (!firsts.has(name.surface)) firsts.set(name.surface, name);
+  });
+  return [...firsts.values()].flatMap((name) => {
+    const [usual] = prose
+      .flatMap((form) => {
+        const kind = cellNameRelation(name, uses.get(name.surface) ?? 0, form);
+        return kind === undefined ? [] : [{ form, kind }];
+      })
+      .toSorted((left, right) => KIND_ORDER.indexOf(left.kind) - KIND_ORDER.indexOf(right.kind) || right.form.count - left.form.count);
+    return usual === undefined ? [] : [{ name, usual: usual.form.surface, kind: usual.kind }];
+  });
+};