chaffjs 0.30.0 → 0.31.0

package/src/structure/layout-codes.ts

@@ -0,0 +1,210 @@
+// layout-code-mismatch: a layout code (2LDK, "2 bedrooms") against the rooms a listing names. Pure: the words come in as
+// LayoutWords (the layout-code and layout-room lexicons), the text as strings.
+
+export type LayoutPart = "living" | "dining" | "kitchen";
+export type LetterGroup = LayoutPart | "storage" | "one-room";
+export type RoomGroup = LayoutPart | "room" | "storage" | "unclear";
+
+export type LayoutWords = {
+  /** One-letter marks written right after the number (L, D, K, S, R). */
+  readonly letters: ReadonlyMap<string, LetterGroup>;
+  /** Marks that add a room not counted (2LDK+S). */
+  readonly joiners: readonly string[];
+  /** Words after the number that say what it counts ("bedrooms", "BR"). */
+  readonly counted: readonly string[];
+  readonly rooms: readonly { readonly pattern: string; readonly group: RoomGroup }[];
+  /** Words after a number that make it a count of rooms (洋室2室): an entry with one is not read as one room. */
+  readonly counters: readonly string[];
+};
+
+export type LayoutCode = {
+  readonly text: string;
+  /** Where the code starts and ends in the text it was read from. */
+  readonly start: number;
+  readonly end: number;
+  readonly rooms: number;
+  /** The L, D and K a lettered code has. Undefined for a code that says nothing about them ("2 bedrooms", 1R). */
+  readonly parts: ReadonlySet<LayoutPart> | undefined;
+};
+
+export type Breakdown = {
+  readonly rooms: number;
+  /** Each L, D or K part the breakdown names, with the entry that names it. */
+  readonly parts: ReadonlyMap<LayoutPart, string>;
+  /** An entry that is a room in some listings and not in others, or names two kinds at once. */
+  readonly unclear: boolean;
+};
+
+export type LayoutSlip =
+  { readonly kind: "count"; readonly code: LayoutCode; readonly listed: number } | { readonly kind: "part"; readonly code: LayoutCode; readonly entry: string };
+
+const FULLWIDTH_OFFSET = 0xfee0;
+const FULLWIDTH_FIRST = 0xff01;
+const FULLWIDTH_LAST = 0xff5e;
+
+/** Full-width ASCII (２ＬＤＫ) as ASCII, one character for one, so offsets stay those of the source. */
+export const foldWidth = (text: string): string =>
+  text.replace(/[！-～]/gu, (char) => {
+    const code = char.charCodeAt(0);
+    return code >= FULLWIDTH_FIRST && code <= FULLWIDTH_LAST ? String.fromCharCode(code - FULLWIDTH_OFFSET) : char;
+  });
+
+const DIGITS = /\d+/uy;
+const LATIN = /[a-z]/iu;
+const WORD_CHAR = /[\p{L}\p{N}]/u;
+
+/** Whether text has pattern at index, and does not run on into a longer word when the pattern is Latin (2BR is a word after a number). */
+const wordAt = (text: string, index: number, pattern: string): boolean => {
+  if (text.slice(index, index + pattern.length).toLowerCase() !== pattern.toLowerCase()) return false;
+  if (!LATIN.test(pattern)) return true;
+  return !WORD_CHAR.test(text.charAt(index + pattern.length)) && !/\p{L}/u.test(text.charAt(index - 1));
+};
+
+/** The longest code letters run (3SLDK). A longer run is a word, not a code. */
+const MAX_CODE_LETTERS = 5;
+/** The most "+S" a code adds (2LDK+S+納戸). */
+const MAX_JOINS = 3;
+
+/** The run of code letters at index (LDK, SLDK), with what they say; empty when the run is longer than a code. */
+const lettersAt = (text: string, index: number, words: LayoutWords): LetterGroup[] => {
+  const head = Array.from(text.slice(index, index + MAX_CODE_LETTERS + 1));
+  const stop = head.findIndex((char) => !words.letters.has(char));
+  const length = stop === -1 ? head.length : stop;
+  if (length > MAX_CODE_LETTERS) return [];
+  return head.slice(0, length).flatMap((char) => words.letters.get(char) ?? []);
+};
+
+/** After a lettered code, each "+S" or "+納戸" that adds a room not counted: the index where they end. */
+const joinedEnd = (text: string, index: number, words: LayoutWords, joins = 0): number => {
+  const joiner = words.joiners.find((mark) => text.startsWith(mark, index));
+  if (joiner === undefined || joins === MAX_JOINS) return index;
+  const after = index + joiner.length;
+  const letter = words.letters.get(text.charAt(after)) === "storage" ? 1 : 0;
+  const word = words.rooms.find((room) => room.group === "storage" && wordAt(text, after, room.pattern))?.pattern.length ?? letter;
+  return word === 0 ? index : joinedEnd(text, after + word, words, joins + 1);
+};
+
+const letteredCode = (text: string, start: number, digitsEnd: number, words: LayoutWords): LayoutCode | undefined => {
+  const groups = lettersAt(text, digitsEnd, words);
+  const lettersEnd = digitsEnd + groups.length;
+  if (!groups.includes("kitchen") && !groups.includes("one-room")) return undefined;
+  if (LATIN.test(text.charAt(lettersEnd))) return undefined;
+  const end = joinedEnd(text, lettersEnd, words);
+  const parts = groups.includes("one-room") ? undefined : new Set(groups.filter((group): group is LayoutPart => group !== "storage"));
+  return { text: text.slice(start, end), start, end, rooms: Number(text.slice(start, digitsEnd)), parts };
+};
+
+const SPACER = /[ \-‐–]?/uy;
+
+const countedCode = (text: string, start: number, digitsEnd: number, words: LayoutWords): LayoutCode | undefined => {
+  SPACER.lastIndex = digitsEnd;
+  const after = digitsEnd + (SPACER.exec(text)?.[0].length ?? 0);
+  const word = words.counted.find((pattern) => wordAt(text, after, pattern));
+  if (word === undefined) return undefined;
+  const end = after + word.length;
+  return { text: text.slice(start, end), start, end, rooms: Number(text.slice(start, digitsEnd)), parts: undefined };
+};
+
+const codeAt = (text: string, start: number, words: LayoutWords): LayoutCode | undefined => {
+  if (/[\p{L}\p{N}.,]/u.test(text.charAt(start - 1))) return undefined;
+  DIGITS.lastIndex = start;
+  const digits = DIGITS.exec(text)?.[0];
+  if (digits === undefined) return undefined;
+  const digitsEnd = start + digits.length;
+  return letteredCode(text, start, digitsEnd, words) ?? countedCode(text, start, digitsEnd, words);
+};
+
+/**
+ * The one layout code a field value gives (2LDK, 3SLDK, 2LDK+S, "2 bedrooms", 2-bed, 2BR). A value with two codes of
+ * different counts ("2LDK or 3DK") gives none: which one the breakdown is for cannot be told.
+ */
+export const parseLayoutCode = (value: string, words: LayoutWords): LayoutCode | undefined => {
+  const text = foldWidth(value);
+  const codes = [...text.matchAll(/\d+/gu)].flatMap((match) => codeAt(text, match.index, words) ?? []);
+  const first = codes[0];
+  if (first === undefined || codes.some((code) => code.rooms !== first.rooms)) return undefined;
+  return { ...first, text: value.slice(first.start, first.end) };
+};
+
+const ENTRY_SEPARATOR = /[、，,・/／+＋;；\n]/u;
+const MULTIPLIER = /(?:[×✕]|(?<=[\s\d])[xX])\s*(\d+)\s*$/u;
+
+/** The kinds of room an entry names, by the room words in it; a lone run of code letters (LDK 12帖, S 4帖) by its letters. */
+const groupsOfEntry = (entry: string, words: LayoutWords): RoomGroup[] => {
+  const named = words.rooms.filter((room) => positions(entry).some((index) => wordAt(entry, index, room.pattern))).map((room) => room.group);
+  if (named.length > 0) return [...new Set(named)];
+  const letters = /^[A-Z]+(?![A-Za-z])/u.exec(entry)?.[0] ?? "";
+  const groups = [...letters].map((letter) => words.letters.get(letter));
+  if (letters === "" || groups.some((group) => group === undefined || group === "one-room")) return [];
+  return [...new Set(groups.flatMap((group): RoomGroup[] => (group === undefined || group === "one-room" ? [] : [group])))];
+};
+
+const positions = (entry: string): number[] => Array.from({ length: entry.length }, (_, index) => index);
+
+/** How many rooms an entry names in words (Bedroom 1 and Bedroom 2 names two). */
+const roomMentions = (entry: string, words: LayoutWords): number => {
+  const rooms = words.rooms.filter((room) => room.group === "room");
+  return positions(entry).filter((index) => rooms.some((room) => wordAt(entry, index, room.pattern))).length;
+};
+
+/** Whether the entry counts rooms with a number (洋室2室, 和室 2間). */
+const countsRooms = (entry: string, words: LayoutWords): boolean =>
+  [...entry.matchAll(/\d+\s*/gu)].some((match) => words.counters.some((counter) => entry.startsWith(counter, match.index + match[0].length)));
+
+const isPart = (group: RoomGroup): group is LayoutPart => group === "living" || group === "dining" || group === "kitchen";
+
+type Tally = { rooms: number; parts: Map<LayoutPart, string>; unclear: boolean };
+
+const tallyEntry = (tally: Tally, entry: string, words: LayoutWords): Tally => {
+  const groups = groupsOfEntry(entry, words);
+  const rooms = groups.includes("room");
+  const several = rooms && (groups.length > 1 || roomMentions(entry, words) > 1 || countsRooms(entry, words));
+  if (groups.includes("unclear") || several) return { ...tally, unclear: true };
+  const times = Number(MULTIPLIER.exec(entry)?.[1] ?? 1);
+  const parts = new Map(tally.parts);
+  groups.filter(isPart).forEach((part) => parts.set(part, parts.get(part) ?? entry));
+  return { rooms: tally.rooms + (rooms ? times : 0), parts, unclear: tally.unclear };
+};
+
+/** The rooms a breakdown names ("LDK 11.5帖、洋室 6帖、洋室 5帖", "Bedroom 1, Bedroom 2, Bathroom"). Entries it cannot name are left out. */
+export const parseBreakdown = (entries: readonly string[], words: LayoutWords): Breakdown =>
+  entries
+    .flatMap((entry) => foldWidth(entry).split(ENTRY_SEPARATOR))
+    .map((entry) => entry.trim())
+    .filter((entry) => entry !== "")
+    .reduce<Tally>((tally, entry) => tallyEntry(tally, entry, words), { rooms: 0, parts: new Map(), unclear: false });
+
+/**
+ * Where a code and the breakdown of the same listing disagree. Silent when the breakdown names no room (it may list only
+ * the LDK), or names one that is counted in some listings and not in others. A lettered code is also checked for an L, D or
+ * K part the breakdown names and the code lacks; a one-room code (1R) and a counted code ("2 bedrooms") are not.
+ */
+export const layoutSlip = (code: LayoutCode, breakdown: Breakdown): LayoutSlip | undefined => {
+  if (breakdown.unclear || breakdown.rooms === 0) return undefined;
+  if (breakdown.rooms !== code.rooms) return { kind: "count", code, listed: breakdown.rooms };
+  const parts = code.parts;
+  if (parts === undefined) return undefined;
+  const extra = [...breakdown.parts].find(([part]) => !parts.has(part));
+  return extra === undefined ? undefined : { kind: "part", code, entry: extra[1] };
+};
+
+/**
+ * The codes and breakdowns of one page, paired listing by listing: within one scope (a section), the first code with the
+ * first breakdown and so on, but only when the scope has as many of each. Any other count is several listings that cannot
+ * be told apart, and none is paired.
+ */
+export const pairByScope = <C extends { readonly scope: number }, B extends { readonly scope: number }>(
+  codes: readonly C[],
+  breakdowns: readonly B[],
+): [C, B][] => {
+  const scopes = [...new Set(codes.map((code) => code.scope))];
+  return scopes.flatMap((scope) => {
+    const inCodes = codes.filter((code) => code.scope === scope);
+    const inBreakdowns = breakdowns.filter((breakdown) => breakdown.scope === scope);
+    if (inCodes.length !== inBreakdowns.length) return [];
+    return inCodes.flatMap((code, index): [C, B][] => {
+      const breakdown = inBreakdowns[index];
+      return breakdown === undefined ? [] : [[code, breakdown]];
+    });
+  });
+};

package/src/structure/measured-amounts.ts

@@ -0,0 +1,138 @@
+import type { Amount } from "./total.ts";
+import type { MeasureUnit } from "../facts/measures.ts";
+import { CONTINUES_WORD, NUMBER, overlapsAny, valueOf } from "./counted-amounts.ts";
+
+/**
+ * 足してよい、単位の付いた量（300 mg、0.5 g、2 L、3 cm）。単位は量の種類（dimension）と、基準の単位への倍率（factors）を持つ。
+ * 同じ種類の量は、その種類で一番小さい単位に直して足す（500 mg と 0.5 g は 1,000 mg）。倍率が二つある単位（cup）は換算が決まらない。
+ */
+export type SummedMeasure = Pick<MeasureUnit, "pattern" | "dimension" | "factors" | "before" | "context">;
+
+/** 足せない量の印。目安の印（約、about、程度）、範囲の印（〜）、何あたりかの印（数の前の $3 / 200 g、単位の後ろの 300 mg/錠、per tablet）。 */
+export type MeasureMarks = {
+  readonly roughBefore: readonly string[];
+  readonly roughAfter: readonly string[];
+  readonly connectors: readonly string[];
+  readonly perMarks: readonly string[];
+};
+
+/**
+ * 足せない量の単位に付ける印。量は読むが、単位に位置も入れて、ほかのどの量とも同じ単位にならないようにする。その量がある列の和は取らない。
+ * 読まずに捨てると、その行が空の升に見えて、残りの行の和を合計と比べてしまう。
+ */
+const UNSUMMABLE = "?";
+
+const LOOKAHEAD = 32;
+const LATIN = /\p{Script=Latin}/u;
+const DIGIT_AT_END = /\p{N}$/u;
+
+const lineStartOf = (source: string, offset: number): number => source.lastIndexOf("\n", offset - 1) + 1;
+
+/** text が word で終わり、英字の語なら語の途中（awesome の some）でない。 */
+const endsWithWord = (text: string, word: string): boolean => {
+  if (!text.toLowerCase().endsWith(word.toLowerCase())) return false;
+  return !LATIN.test(word.charAt(0)) || !LATIN.test(text.charAt(text.length - word.length - 1));
+};
+
+/** text が word で始まり、英字の語なら語の途中（personal の per）でない。 */
+const startsWithWord = (text: string, word: string): boolean => {
+  if (!text.toLowerCase().startsWith(word.toLowerCase())) return false;
+  return !LATIN.test(word.at(-1) ?? "") || !LATIN.test(text.charAt(word.length));
+};
+
+/**
+ * 数の前の目安の印（約300mg、about 300 mg）、何あたりかの印（$3 / 200 g、$3 per 200 g）、
+ * 範囲の後ろの端（300〜400mg、300 - 400 mg）。
+ */
+const markedBefore = (source: string, start: number, marks: MeasureMarks): boolean => {
+  const before = source.slice(lineStartOf(source, start), start);
+  const trimmed = before.trimEnd();
+  if ([...marks.roughBefore, ...marks.perMarks].some((word) => endsWithWord(trimmed, word))) return true;
+  return marks.connectors.some(
+    (connector) => before.endsWith(connector) || (trimmed.endsWith(connector) && DIGIT_AT_END.test(trimmed.slice(0, -connector.length).trimEnd())),
+  );
+};
+
+/** 印の前に置かれうる、開き括弧と空白（5 mg (per tablet)）。 */
+const OPENING = /^[\s(（[［]+/u;
+
+/** 単位の後ろの、目安の印（300mg程度）か何あたりかの印（300 mg/tablet、300 mg per tablet、5 mg (per tablet)）。 */
+const markedAfter = (source: string, end: number, marks: MeasureMarks): boolean => {
+  const after = source.slice(end, end + LOOKAHEAD).replace(OPENING, "");
+  return [...marks.roughAfter, ...marks.perMarks].some((word) => startsWithWord(after, word));
+};
+
+type Found = { readonly measure: SummedMeasure; readonly end: number };
+
+const lineAt = (source: string, offset: number): string => {
+  const end = source.indexOf("\n", offset);
+  return source.slice(lineStartOf(source, offset), end === -1 ? source.length : end);
+};
+
+/** 文脈の語が要る単位（unit-context）は、同じ行にその語があるときだけ単位。 */
+const inContext = (measure: SummedMeasure, source: string, offset: number): boolean => {
+  if (measure.context.length === 0) return true;
+  const line = lineAt(source, offset).toLowerCase();
+  return measure.context.some((word) => line.includes(word.toLowerCase()));
+};
+
+/** 数のすぐ後ろ（空白一つまで）の単位。長い単位から試す（mg を m で切らない）。大文字と小文字は区別する（mL と ML は別）。 */
+const measureAfter = (source: string, end: number, measures: readonly SummedMeasure[], marks: MeasureMarks): Found | undefined => {
+  const gap = source.charAt(end) === " " ? 1 : 0;
+  const rest = source.slice(end + gap, end + gap + LOOKAHEAD);
+  const measure = measures.find((candidate) => {
+    if (candidate.before || !rest.startsWith(candidate.pattern) || !inContext(candidate, source, end)) return false;
+    const unitEnd = end + gap + candidate.pattern.length;
+    return markedAfter(source, unitEnd, marks) || !CONTINUES_WORD.test(rest.slice(candidate.pattern.length));
+  });
+  return measure === undefined ? undefined : { measure, end: end + gap + measure.pattern.length };
+};
+
+/** 量の種類ごとの、一番小さい倍率。倍率が一つの単位だけから取る。 */
+const baseFactors = (measures: readonly SummedMeasure[]): Map<string, number> =>
+  measures.reduce((bases, measure) => {
+    const [factor] = measure.factors;
+    if (measure.factors.length !== 1 || factor === undefined || factor <= 0) return bases;
+    return bases.set(measure.dimension, Math.min(bases.get(measure.dimension) ?? factor, factor));
+  }, new Map<string, number>());
+
+const unsummable = (offset: number, end: number, value: number, measure: SummedMeasure): Amount => ({
+  offset,
+  end,
+  value,
+  unit: `${measure.dimension}${UNSUMMABLE}${String(offset)}`,
+});
+
+const amountOf = (source: string, start: number, written: string, found: Found, bases: Map<string, number>, marks: MeasureMarks): Amount => {
+  const value = valueOf(written);
+  const { measure } = found;
+  const [factor] = measure.factors;
+  const base = bases.get(measure.dimension);
+  const sure = measure.factors.length === 1 && factor !== undefined && base !== undefined;
+  if (!sure || markedBefore(source, start, marks) || markedAfter(source, found.end, marks)) return unsummable(start, found.end, value, measure);
+  const scale = factor / base;
+  return { offset: start, end: found.end, value: value * scale, unit: measure.dimension, scale };
+};
+
+/**
+ * 数のすぐ前（空白一つまで）の単位（大さじ1）は、足さない量として読む。読まずに捨てると、その行が空の升に見えてしまう。
+ */
+const measureBefore = (source: string, start: number, end: number, measures: readonly SummedMeasure[]): Amount | undefined => {
+  const gap = source.charAt(start - 1) === " " ? 1 : 0;
+  const head = source.slice(Math.max(0, start - gap - LOOKAHEAD), start - gap);
+  const measure = measures.find((candidate) => candidate.before && head.endsWith(candidate.pattern) && inContext(candidate, source, start));
+  return measure === undefined ? undefined : unsummable(start - gap - measure.pattern.length, end, valueOf(source.slice(start, end)), measure);
+};
+
+/** 文書の中の、単位の付いた量。値は量の種類で一番小さい単位に直す。木がすでに数量として読んだもの（known）とは重ねない。 */
+export const measuredAmounts = (source: string, measures: readonly SummedMeasure[], marks: MeasureMarks, known: readonly Amount[]): Amount[] => {
+  const longestFirst = measures.toSorted((left, right) => right.pattern.length - left.pattern.length);
+  const bases = baseFactors(measures);
+  return [...source.matchAll(NUMBER)].flatMap((match): Amount[] => {
+    const end = match.index + match[0].length;
+    const found = measureAfter(source, end, longestFirst, marks);
+    if (found !== undefined) return overlapsAny(match.index, found.end, known) ? [] : [amountOf(source, match.index, match[0], found, bases, marks)];
+    const before = measureBefore(source, match.index, end, longestFirst);
+    return before === undefined || overlapsAny(before.offset, before.end, known) ? [] : [before];
+  });
+};

package/src/structure/quantity-range.ts

@@ -0,0 +1,54 @@
+// quantity-range-reversed: which measured quantities may be one end of a range, read as the ends structure/amount-range.ts
+// compares. Pure.
+import type { Measured } from "../facts/measures.ts";
+import { escapeRegExp } from "../orthography.ts";
+import type { StructureIssue } from "./issues.ts";
+import type { RangeAmount } from "./amount-range.ts";
+
+const LATIN_EDGE = /[A-Za-z]$/u;
+const LATIN_START = /^[A-Za-z]/u;
+
+/** A Latin unit running into a word is part of that word: the t of "40 to" is not a tonne. */
+const standsAlone = (text: string, measured: Measured): boolean =>
+  !(LATIN_EDGE.test(measured.unit) && LATIN_START.test(text.slice(measured.end, measured.end + 1)));
+
+/** 10 and a minus sign right before the number is a power of ten written flat ("10-8 cm" for 10⁻⁸ cm), not a range. */
+const POWER_OF_TEN = /(?<![0-9０-９.,，．])[1１][0０][-−－]$/u;
+
+/** "10-" and the character before it, which must not be a digit (110-8 cm is a range). */
+const POWER_OF_TEN_LOOKBACK = 4;
+
+const isExponent = (text: string, measured: Measured): boolean =>
+  POWER_OF_TEN.test(text.slice(Math.max(0, measured.start - POWER_OF_TEN_LOOKBACK), measured.start));
+
+/** A quantity as one end of a range: two ends compare only in one unit, however it is spelled (℃ with °C, m with meters). */
+const rangeEndOf = (text: string, measured: Measured): RangeAmount => ({
+  offset: measured.start,
+  end: measured.end,
+  currency: `${measured.dimension}:${measured.factors.join(",")}`,
+  value: measured.amount,
+  scale: undefined,
+  position: text.startsWith(measured.unit, measured.start) ? "before" : "after",
+});
+
+/** The measured quantities that can end a range, as range ends. */
+export const quantityRangeEnds = (text: string, measured: readonly Measured[]): RangeAmount[] =>
+  measured.filter((value) => standsAlone(text, value) && !isExponent(text, value)).map((value) => rangeEndOf(text, value));
+
+const LATIN_WORD = /^[A-Za-z]/u;
+
+/** Whether a lead word stands right before the range on its line ("went from 9 minutes to 2 minutes" tells of a change). */
+const ledBy = (text: string, offset: number, leads: readonly string[]): boolean => {
+  const before = text
+    .slice(text.lastIndexOf("\n", offset - 1) + 1, offset)
+    .trimEnd()
+    .toLowerCase();
+  return leads.some((lead) => {
+    const edge = LATIN_WORD.test(lead) ? "(?<![a-z])" : "";
+    return new RegExp(`${edge}${escapeRegExp(lead.toLowerCase())}$`, "u").test(before);
+  });
+};
+
+/** The reversed ranges that no lead word frames as a change. */
+export const unledRanges = (text: string, issues: readonly StructureIssue[], leads: readonly string[]): StructureIssue[] =>
+  issues.filter((issue) => !ledBy(text, issue.offset, leads));