chaffjs 0.31.0 → 0.33.0

package/src/structure/gpa.ts

@@ -0,0 +1,218 @@
+// gpa-mismatch: a written grade-point average ("GPA: 3.00") that the course rows of the same document do not give, as
+// Σ(grade points × credits) / Σ credits. Columns are named by heading words, grade points come from the document's own
+// grading key ("S (4), A (3)") or from one unambiguous scale of the grade-point lexicon. Pure.
+import type { StructureIssue } from "./issues.ts";
+import { holds, plain, type ChangeTable, type TableCell } from "./change-rate-table.ts";
+import { escapeRegExp } from "../orthography.ts";
+
+export type TranscriptRole = "points" | "grade" | "credits" | "course" | "total";
+export type TranscriptWord = { readonly pattern: string; readonly role: TranscriptRole };
+/** A grade word on one scale, with its points; undefined points is a grade that carries none (Pass, 認定). */
+export type GradeWord = { readonly pattern: string; readonly scale: string; readonly points?: number };
+export type GpaLabel = { readonly pattern: string; readonly part: boolean };
+export type GpaWords = {
+  readonly columns: readonly TranscriptWord[];
+  readonly grades: readonly GradeWord[];
+  readonly labels: readonly GpaLabel[];
+};
+export type TextLine = { readonly text: string; readonly start: number };
+
+/** Grade-points columns first, so "Grade points" is not the grade column. */
+const ROLE_ORDER: readonly TranscriptRole[] = ["points", "grade", "credits", "course"];
+
+const roleOf = (heading: string, words: readonly TranscriptWord[]): TranscriptRole | undefined => {
+  const key = plain(heading).toLowerCase();
+  return ROLE_ORDER.find((role) => words.some((word) => word.role === role && holds(key, word.pattern)));
+};
+
+type CourseColumns = { readonly grade: number; readonly credits: number };
+
+/** A course table: at least one course column, and exactly one grade and one credits column. */
+export const courseColumnsOf = (headings: readonly string[], words: readonly TranscriptWord[]): CourseColumns | undefined => {
+  const roles = headings.map((heading) => roleOf(heading, words));
+  const all = (role: TranscriptRole): number[] => roles.flatMap((found, index) => (found === role ? [index] : []));
+  const [grades, credits] = [all("grade"), all("credits")];
+  const [grade, credit] = [grades[0], credits[0]];
+  if (all("course").length === 0 || grades.length !== 1 || credits.length !== 1 || grade === undefined || credit === undefined) return undefined;
+  return { grade, credits: credit };
+};
+
+/** One course row as written: its grade word and its credits cell. */
+export type CourseRow = { readonly grade: string; readonly credits: string };
+
+const holdsAny = (text: string, patterns: readonly string[]): boolean => patterns.some((pattern) => holds(plain(text).toLowerCase(), pattern));
+
+/** The course rows of every course table; a total row and a row that writes the GPA itself are not courses. */
+export const courseRowsOf = (tables: readonly ChangeTable[], words: GpaWords): CourseRow[] => {
+  const skip = [...words.columns.filter((word) => word.role === "total"), ...words.labels.filter((label) => !label.part)].map((word) => word.pattern);
+  return tables.flatMap((table) => {
+    const columns = courseColumnsOf(
+      table.header.map((cell) => cell.text),
+      words.columns,
+    );
+    if (columns === undefined) return [];
+    const rows = table.rows.filter((row) => row.length === table.header.length && !holdsAny(row[0]?.text ?? "", skip));
+    const textOf = (row: readonly TableCell[], index: number): string => plain(row[index]?.text ?? "");
+    return rows
+      .filter((row) => row.some((cell) => plain(cell.text) !== ""))
+      .map((row) => ({ grade: textOf(row, columns.grade), credits: textOf(row, columns.credits) }));
+  });
+};
+
+const CREDITS = /^\d+(?:\.\d+)?$/u;
+
+type GradedRow = { readonly grade: string; readonly credits: number };
+
+/** The rows that carry points; undefined when a row's grade is on no scale or its credits are not a number. */
+export const gradedRowsOf = (rows: readonly CourseRow[], grades: readonly GradeWord[]): GradedRow[] | undefined => {
+  const noPoints = new Set(grades.filter((word) => word.points === undefined).map((word) => word.pattern));
+  const onScale = new Set(grades.filter((word) => word.points !== undefined).map((word) => word.pattern));
+  const counted = rows.filter((row) => !noPoints.has(row.grade));
+  if (counted.some((row) => !onScale.has(row.grade) || !CREDITS.test(row.credits))) return undefined;
+  return counted.map((row) => ({ grade: row.grade, credits: Number(row.credits) }));
+};
+
+/** Above this, a number beside a grade is a range of marks (S (90)), not grade points. */
+const GRADE_POINT_CEILING = 5;
+
+const keyPattern = (grades: readonly GradeWord[]): RegExp => {
+  const words = [...new Set(grades.filter((word) => word.points !== undefined).map((word) => word.pattern))].toSorted(
+    (left, right) => right.length - left.length,
+  );
+  const number = String.raw`(\d+(?:\.\d+)?)`;
+  return new RegExp(
+    String.raw`(?<![A-Za-z0-9])(${words.map(escapeRegExp).join("|")})\s*(?:\(\s*${number}\s*(?:点|pts?\.?|points?)?\s*\)|=\s*${number}\s*点?)`,
+    "gu",
+  );
+};
+
+/**
+ * The grade points a key line writes: a line naming grades (評価, Grades) with "S（4）" or "A = 4" pairs, so "Room A (4)"
+ * is not read. "conflict" when the document gives one grade two different points, or numbers too large to be points.
+ */
+export const gradingKeyOf = (lines: readonly TextLine[], grades: readonly GradeWord[], cues: readonly string[]): ReadonlyMap<string, number> | "conflict" => {
+  const pattern = keyPattern(grades);
+  const keyLines = lines.filter((line) => holdsAny(line.text, cues));
+  const pairs = keyLines.flatMap((line) =>
+    [...plain(line.text).matchAll(pattern)].map((match) => ({ grade: match[1] ?? "", points: Number(match[2] ?? match[3]) })),
+  );
+  const key = new Map<string, number>();
+  const clash = pairs.some(({ grade, points }) => {
+    const known = key.get(grade);
+    key.set(grade, points);
+    return points > GRADE_POINT_CEILING || (known !== undefined && known !== points);
+  });
+  return clash ? "conflict" : key;
+};
+
+/** The points every lexicon scale holding all the grades agrees on; undefined when no scale holds them or two disagree. */
+export const scalePointsOf = (used: readonly string[], grades: readonly GradeWord[]): ReadonlyMap<string, number> | undefined => {
+  const scales = [...new Set(grades.flatMap((word) => (word.points === undefined ? [] : [word.scale])))];
+  const pointsIn = (scale: string, grade: string): number | undefined => grades.find((word) => word.scale === scale && word.pattern === grade)?.points;
+  const fitting = scales.filter((scale) => used.every((grade) => pointsIn(scale, grade) !== undefined));
+  const points = new Map(used.map((grade) => [grade, new Set(fitting.map((scale) => pointsIn(scale, grade)))]));
+  if (fitting.length === 0 || [...points.values()].some((values) => values.size !== 1)) return undefined;
+  return new Map(used.map((grade) => [grade, [...(points.get(grade) ?? [])][0] ?? 0]));
+};
+
+/**
+ * The document's key when it gives every used grade its points; otherwise the lexicon's scale, when the key gives none
+ * of the used grades points other than the scale's.
+ */
+export const pointsFor = (
+  used: readonly string[],
+  key: ReadonlyMap<string, number> | "conflict",
+  grades: readonly GradeWord[],
+): ReadonlyMap<string, number> | undefined => {
+  if (key === "conflict") return undefined;
+  if (used.every((grade) => key.has(grade))) return key;
+  const scale = scalePointsOf(used, grades);
+  return scale !== undefined && used.every((grade) => !key.has(grade) || key.get(grade) === scale.get(grade)) ? scale : undefined;
+};
+
+/** A written GPA: where its number starts, the number as written, and its decimals. */
+export type GpaStatement = { readonly offset: number; readonly text: string; readonly value: number; readonly decimals: number };
+
+const LINE_MARK = /^\s*(?:>\s*)*(?:(?:[-*+]|\d+[.)])\s+)?/u;
+const COLON = /[:：]/u;
+const BRACKET = /[()]/u;
+const WRITTEN_GPA = /^(\s*)(\d+(?:\.(\d+))?)/u;
+
+/** A label without what it writes in brackets: 「評価平均(gpa)」 is 「評価平均」. */
+const outsideBrackets = (label: string): string =>
+  label
+    .split(BRACKET)
+    .filter((_, index) => index % 2 === 0)
+    .join(" ")
+    .replace(/\s+/gu, " ")
+    .trim();
+
+/** What one "label: value" line says of a GPA: a statement, one of a part of the record, another figure, or nothing. */
+type GpaLine = { readonly kind: "statement"; readonly statement: GpaStatement } | { readonly kind: "part" | "other" } | undefined;
+
+const gpaLineOf = (line: TextLine, labels: readonly GpaLabel[]): GpaLine => {
+  const mark = LINE_MARK.exec(line.text)?.[0] ?? "";
+  const colon = line.text.slice(mark.length).search(COLON);
+  if (colon <= 0) return undefined;
+  const label = plain(line.text.slice(mark.length, mark.length + colon)).toLowerCase();
+  const named = (part: boolean): boolean => labels.some((word) => word.part === part && holds(label, word.pattern));
+  if (!named(false)) return undefined;
+  if (named(true)) return { kind: "part" };
+  const valueFrom = mark.length + colon + 1;
+  const number = WRITTEN_GPA.exec(line.text.slice(valueFrom));
+  if (number === null) return undefined;
+  const outside = outsideBrackets(label);
+  if (!labels.some((word) => !word.part && plain(word.pattern).toLowerCase() === outside)) return { kind: "other" };
+  const [, space = "", text = "", fraction = ""] = number;
+  return { kind: "statement", statement: { offset: line.start + valueFrom + space.length, text, value: Number(text), decimals: fraction.length } };
+};
+
+/** The written GPAs; none when a line gives one part of the record's, or a GPA figure under another label, or two values. */
+export const gpaStatementsOf = (lines: readonly TextLine[], labels: readonly GpaLabel[]): GpaStatement[] => {
+  const read = lines.flatMap((line) => gpaLineOf(line, labels) ?? []);
+  const statements = read.flatMap((found) => (found.kind === "statement" ? [found.statement] : []));
+  if (read.some((found) => found.kind !== "statement") || new Set(statements.map((statement) => statement.value)).size > 1) return [];
+  return statements;
+};
+
+const DECIMAL_BASE = 10;
+const SLACK = 1e-9;
+/** Decimals of the computed GPA shown beside a GPA written as a whole number. */
+const WHOLE_SHOWN_DECIMALS = 2;
+
+/** The written GPA is the computed one rounded, or cut off, to the written decimals. */
+export const withinRounding = (written: GpaStatement, computed: number): boolean => {
+  const step = DECIMAL_BASE ** -written.decimals;
+  const rounded = Math.abs(computed - written.value) <= step / 2 + SLACK;
+  const cutOff = written.value <= computed + SLACK && computed - written.value < step - SLACK;
+  return rounded || cutOff;
+};
+
+/** Σ(points × credits) / Σ credits, undefined with no credits. */
+export const averageOf = (rows: readonly GradedRow[], points: ReadonlyMap<string, number>): number | undefined => {
+  const credits = rows.reduce((sum, row) => sum + row.credits, 0);
+  if (credits === 0) return undefined;
+  return rows.reduce((sum, row) => sum + (points.get(row.grade) ?? 0) * row.credits, 0) / credits;
+};
+
+const computedGpa = (tables: readonly ChangeTable[], lines: readonly TextLine[], words: GpaWords): number | undefined => {
+  const rows = gradedRowsOf(courseRowsOf(tables, words), words.grades);
+  if (rows === undefined || rows.length === 0) return undefined;
+  const cues = words.columns.filter((word) => word.role === "grade" || word.role === "points").map((word) => word.pattern);
+  const points = pointsFor([...new Set(rows.map((row) => row.grade))], gradingKeyOf(lines, words.grades, cues), words.grades);
+  return points === undefined ? undefined : averageOf(rows, points);
+};
+
+/** Each written GPA the document's course rows do not give, with the GPA they do give. */
+export const gpaMismatches = (tables: readonly ChangeTable[], lines: readonly TextLine[], words: GpaWords): StructureIssue[] => {
+  const statements = gpaStatementsOf(lines, words.labels);
+  if (statements.length === 0) return [];
+  const computed = computedGpa(tables, lines, words);
+  if (computed === undefined) return [];
+  return statements
+    .filter((statement) => !withinRounding(statement, computed))
+    .map((statement) => ({
+      offset: statement.offset,
+      values: { gpa: statement.text, computed: computed.toFixed(statement.decimals === 0 ? WHOLE_SHOWN_DECIMALS : statement.decimals) },
+    }));
+};

package/src/structure/installment-fee.ts

@@ -0,0 +1,201 @@
+import { escapeRegExp } from "../orthography.ts";
+import type { StructureIssue } from "./issues.ts";
+import {
+  alternation,
+  describe,
+  hasWord,
+  intervalOf,
+  isPlainGap,
+  isRough,
+  isTotalOf,
+  lineBounds,
+  occurrences,
+  sentenceAround,
+  sumOf,
+  writtenLike,
+  type InstallmentAmount,
+  type InstallmentWords,
+  type Part,
+} from "./installment-total.ts";
+
+/**
+ * installment-total-mismatch, the fee form: numbered instalments listed one per row or item (第1回：32,000円, | 1st
+ * payment | £150 |, Final payment: $60) that do not add up to the fee stated in the same section (旅行代金は、1人あたり
+ * 62,000円; "The trip costs £300 per student"). Compared only when the section states one fee, every instalment of the
+ * series is read (numbered 1, 2, … with no gap or repeat, each with one plain amount), the section names a payment
+ * (積立金, instalments), and every amount is in the fee's currency. A deposit counts only as an item of the series; one
+ * named anywhere else stops the comparison. An instalment as large as the fee says the fee is per payment (a monthly
+ * fee), not split, and is not compared. Words come from the lexicons. Pure.
+ */
+
+type Span = { readonly start: number; readonly end: number };
+
+export type InstallmentSeriesWords = {
+  /** Item labels with {n} for the instalment's number (第{n}回, {n} payment, Instalment {n}). */
+  readonly numbered: readonly string[];
+  /** Item labels of the last instalment, numbered after the others (最終回, Final payment). */
+  readonly finals: readonly string[];
+  /** Numbers written as words that {n} may be (first, 一) and their values. */
+  readonly ordinals: readonly { readonly word: string; readonly value: number }[];
+  /** Letters after a number that make it an ordinal (1st). */
+  readonly ordinalSuffixes: readonly string[];
+  /** Words that say the section is about paying (積立金, instalment); one must stand in the section. */
+  readonly payments: readonly string[];
+  /** A deposit (申込金, deposit): an item of the series, or a reason not to compare. */
+  readonly deposits: readonly string[];
+  /** Words that state the fee (旅行代金, costs). */
+  readonly fees: readonly string[];
+  /** Words that may stand between a fee word and its amount (1人あたり, per student). */
+  readonly feeLinks: readonly string[];
+};
+
+/** One read item of a series: its number (undefined for a deposit, Infinity for a final one) and its amount. */
+type Item = { readonly line: Span; readonly number: number | undefined; readonly amount: InstallmentAmount | undefined };
+
+/** What may stand before an item label on its line: a table bar, a list mark, spaces. */
+const ITEM_LEAD = /^[\s|｜\-*+・•]*$/u;
+const DIGITS = "[0-9０-９]+";
+const FINAL = Number.POSITIVE_INFINITY;
+
+const numberPattern = (series: InstallmentSeriesWords): string => {
+  const suffix = alternation(series.ordinalSuffixes);
+  const words = alternation(series.ordinals.map((entry) => entry.word));
+  const digits = suffix === undefined ? DIGITS : `${DIGITS}(?:${suffix})?`;
+  return words === undefined ? digits : `${digits}|${words}`;
+};
+
+const numberOf = (written: string, series: InstallmentSeriesWords): number | undefined => {
+  const digits = /^[0-9０-９]+/u.exec(written)?.[0];
+  if (digits !== undefined) return Number(digits.normalize("NFKC"));
+  return series.ordinals.find((entry) => entry.word.toLowerCase() === written.toLowerCase())?.value;
+};
+
+const labelRegExp = (label: string, number: string): RegExp => {
+  const [before = "", after = ""] = label.split("{n}").map(escapeRegExp);
+  const edgeStart = /^[A-Za-z]/u.test(label) ? "(?<![A-Za-z])" : "";
+  const edgeEnd = /[A-Za-z]$/u.test(label) ? "(?![A-Za-z])" : "";
+  return new RegExp(`${edgeStart}${before}(${number})${after}${edgeEnd}`, "giu");
+};
+
+type Label = Span & { readonly number: number | undefined };
+
+const numberedLabels = (text: string, series: InstallmentSeriesWords): Label[] =>
+  series.numbered.flatMap((label) =>
+    [...text.matchAll(labelRegExp(label, numberPattern(series)))].map((match) => ({
+      start: match.index,
+      end: match.index + match[0].length,
+      number: numberOf(match[1] ?? "", series),
+    })),
+  );
+
+const wordLabels = (text: string, words: readonly string[], number: number | undefined): Label[] =>
+  occurrences(text, words).map((span) => ({ ...span, number }));
+
+const startsItem = (text: string, label: Span): boolean => ITEM_LEAD.test(text.slice(lineBounds(text, label.start).start, label.start));
+
+/** The longest label at each place wins (Final payment over a bare payment), and only labels that open a row or an item. */
+const itemLabels = (text: string, series: InstallmentSeriesWords): Label[] => {
+  const all = [...numberedLabels(text, series), ...wordLabels(text, series.finals, FINAL), ...wordLabels(text, series.deposits, undefined)];
+  const longest = all.filter(
+    (label) =>
+      !all.some((other) => other !== label && other.start <= label.start && other.end >= label.end && other.end - other.start > label.end - label.start),
+  );
+  return longest.filter((label) => startsItem(text, label)).toSorted((left, right) => left.start - right.start);
+};
+
+/** A label's amount: the one amount of its line, after a gap of separators, links and payment words; else unread. */
+const itemOf = (text: string, label: Label, amounts: readonly InstallmentAmount[], words: InstallmentWords, series: InstallmentSeriesWords): Item => {
+  const line = lineBounds(text, label.start);
+  const onLine = amounts.filter((amount) => amount.offset >= label.end && amount.end <= line.end);
+  const [amount] = onLine;
+  const read =
+    onLine.length === 1 &&
+    amount !== undefined &&
+    isPlainGap(text.slice(label.end, amount.offset), [...words.links, ...series.payments]) &&
+    !isRough(text, { start: amount.offset, end: amount.end }, words);
+  return { line, number: label.number, amount: read ? amount : undefined };
+};
+
+/** Whether the numbered items run 1, 2, …, k (a final one only last), each read; deposits may stand anywhere. */
+export const isWholeSeries = (items: readonly Item[]): boolean => {
+  const numbers = items.flatMap((item) => (item.number === undefined ? [] : [item.number]));
+  const finalAt = numbers.indexOf(FINAL);
+  const counted = numbers.filter((number) => number !== FINAL);
+  const inOrder = counted.every((number, index) => number === index + 1);
+  const finalLast = finalAt === -1 || finalAt === numbers.length - 1;
+  return items.length >= 2 && counted.length >= 1 && inOrder && finalLast && items.every((item) => item.amount !== undefined);
+};
+
+/** The sections of the text: from one heading to the next; the whole text when there are none. */
+export const sectionsOf = (text: string, headingStarts: readonly number[]): Span[] => {
+  const starts = [0, ...headingStarts.filter((start) => start > 0 && start < text.length).toSorted((left, right) => left - right)];
+  return starts.map((start, index) => ({ start, end: starts[index + 1] ?? text.length }));
+};
+
+/** Every fee the section states: a fee word, then only separators, links and per-person words before an amount. */
+export const feesIn = (
+  text: string,
+  section: Span,
+  amounts: readonly InstallmentAmount[],
+  words: InstallmentWords,
+  series: InstallmentSeriesWords,
+): InstallmentAmount[] => {
+  const body = text.slice(section.start, section.end);
+  return occurrences(body, series.fees).flatMap((span) => {
+    const at = { start: section.start + span.start, end: section.start + span.end };
+    const sentence = sentenceAround(text, at);
+    const amount = amounts.find((candidate) => candidate.offset >= at.end && candidate.end <= sentence.end);
+    if (amount === undefined || !isPlainGap(text.slice(at.end, amount.offset), [...words.links, ...series.feeLinks])) return [];
+    return [amount];
+  });
+};
+
+const within = (span: Span, section: Span): boolean => span.start >= section.start && span.end <= section.end;
+
+const sameAmount = (left: InstallmentAmount, right: InstallmentAmount): boolean => left.value === right.value && left.currency === right.currency;
+
+const partOf = (text: string, amount: InstallmentAmount): Part => ({ count: 1, amount, written: text.slice(amount.offset, amount.end) });
+
+/** The comparison of one section: its series of instalments against its one fee; undefined when anything is not read. */
+const sectionIssue = (
+  text: string,
+  section: Span,
+  items: readonly Item[],
+  amounts: readonly InstallmentAmount[],
+  words: InstallmentWords,
+  series: InstallmentSeriesWords,
+): StructureIssue | undefined => {
+  const own = items.filter((item) => within(item.line, section));
+  if (!isWholeSeries(own) || !hasWord(text.slice(section.start, section.end), series.payments)) return undefined;
+  const fees = feesIn(text, section, amounts, words, series).filter((fee) => !own.some((item) => within({ start: fee.offset, end: fee.end }, item.line)));
+  const [fee] = fees;
+  if (fee === undefined || !fees.every((other) => sameAmount(other, fee)) || isRough(text, { start: fee.offset, end: fee.end }, words)) return undefined;
+  const parts = own.flatMap((item) => (item.amount === undefined ? [] : [partOf(text, item.amount)]));
+  const [first] = parts;
+  if (first === undefined || parts.some((part) => part.amount.currency !== fee.currency || part.amount.value >= fee.value)) return undefined;
+  const written = text.slice(fee.offset, fee.end);
+  const sum = sumOf(parts);
+  const stated = fee.scale === undefined ? { low: fee.value, high: fee.value } : intervalOf(fee, written);
+  if (isTotalOf(stated, sum)) return undefined;
+  return { offset: first.amount.offset, values: { payments: describe(parts), sum: writtenLike((sum.low + sum.high) / 2, first), total: written } };
+};
+
+/** Whether a deposit is named outside the items of a series: then whether it is part of the fee cannot be read. */
+const depositOutside = (text: string, items: readonly Item[], series: InstallmentSeriesWords): boolean =>
+  occurrences(text, series.deposits).some((span) => !items.some((item) => item.number === undefined && within(span, item.line)));
+
+/** Every section whose numbered instalments do not add up to its stated fee; reported at the first instalment. */
+export const installmentFeeMismatches = (
+  text: string,
+  amounts: readonly InstallmentAmount[],
+  headingStarts: readonly number[],
+  words: InstallmentWords,
+  series: InstallmentSeriesWords,
+): StructureIssue[] => {
+  const items = itemLabels(text, series).map((label) => itemOf(text, label, amounts, words, series));
+  if (items.length < 2 || depositOutside(text, items, series)) return [];
+  return sectionsOf(text, headingStarts).flatMap((section) => {
+    const issue = sectionIssue(text, section, items, amounts, words, series);
+    return issue === undefined ? [] : [issue];
+  });
+};