chaffjs 0.30.0 → 0.32.0

package/src/facts/labelled-facts.ts

@@ -1,5 +1,7 @@
 import type { FactValue } from "./fact-values.ts";
 import { EDGE_MARKS, trimEndOf, withoutEdgeMarks } from "./trim-marks.ts";
+import { qualifiedLabelOf, type QualifierWords } from "./qualified-labels.ts";
+import { isRate, joinsNextClause, rateHead, rateTail, type RateBasis, type RateWords } from "./rate-values.ts";
 
 /**
  * 名前の付いた値（「締切：10月5日」「参加費は3,000円です」"Fee: $300" "The deadline is May 3."）と、
@@ -22,9 +24,20 @@ export type FactWords = {
   /** 名前の前に置かれる決定の前置き（協議の結果、, it was decided that）。読点を含んでも条件ではないので、外してから名前を読む。 */
   readonly leads: readonly string[];
   readonly attributes: readonly AttributePhrase[];
+  /** 率のまわりの、値を変えない語（年、per year、（固定金利））。 */
+  readonly rates?: RateWords;
+  /** 区切りと値のあいだに置ける目安の語（約、about）。あれば値を目安として読む。無ければ、区切りのすぐ後ろの値だけ。 */
+  readonly valueLeads?: readonly string[];
+  /** 名前に条件を付ける型（（*）、*の、for *）。条件の付いた名前は、名前と条件をそれぞれ一つの名前の長さで測る。 */
+  readonly qualifiers?: QualifierWords;
+  /** 計器の読みを言う動詞（read, shows）、計器の名前（odometer, meter）、読みのあとに置ける時の節の頭（when）。 */
+  readonly readings?: ReadingWords;
 };
 
-export type Fact = { readonly label: string; readonly key: string; readonly value: FactValue };
+export type ReadingWords = { readonly verbs: readonly string[]; readonly gauges: readonly string[]; readonly clauses: readonly string[] };
+
+/** approximate: 値の前に目安の語（約、about）があった。table: 表の升の値。 */
+export type Fact = { readonly label: string; readonly key: string; readonly value: FactValue; readonly approximate?: boolean; readonly table?: boolean };
 
 const MAX_LABEL_LENGTH = 24;
 const MAX_LABEL_WORDS = 5;
@@ -57,6 +70,15 @@ const beforeSeparator = (head: string, separator: string): string | undefined =>
   return rest;
 };
 
+/** 頭の後ろの目安の語（「重さは約」の約、"is about" の about）を外したもの。英字の語は、前が語の切れ目のときだけ。 */
+const withoutValueLead = (head: string, leads: readonly string[]): string => {
+  const lowered = head.toLowerCase();
+  const lead = leads.find(
+    (word) => lowered.endsWith(word.toLowerCase()) && !(LATIN_LETTER.test(word.charAt(0)) && LATIN_LETTER.test(head.charAt(head.length - word.length - 1))),
+  );
+  return lead === undefined ? head : trimEndOf(head.slice(0, head.length - lead.length), EDGE_MARKS);
+};
+
 /** 英字の前置きが語の途中で切れているか（「we agreed that」は「we agreed thatching」の頭ではない）。 */
 const endsInsideWord = (text: string, length: number): boolean => LATIN_LETTER.test(text.charAt(length - 1)) && LATIN_LETTER.test(text.charAt(length));
 
@@ -75,11 +97,18 @@ const afterLastBreak = (text: string): string => {
 
 const wordsOf = (text: string): string[] => text.split(/\s+/u).filter((word) => word !== "");
 
+/** 語を空白で分けない言語の、名前の頭の前置き（「本体の幅」の 本体の）。前置きだけの名前は落とさない。 */
+const withoutPrefix = (key: string, determiners: readonly string[]): string => {
+  const prefix = determiners.find((word) => !LATIN_WORD.test(word) && key.length > word.length && key.startsWith(word));
+  return prefix === undefined ? key : key.slice(prefix.length);
+};
+
 /** 名前を一つの書き方に。全角と半角、大文字と小文字、空白の数を揃え、頭の冠詞を落とす。 */
 const keyOf = (label: string, words: FactWords): string => {
   const [first, ...rest] = wordsOf(label.normalize("NFKC").toLowerCase());
-  const determiners = new Set(words.determiners.map((word) => word.toLowerCase()));
-  return (first !== undefined && determiners.has(first) ? rest : [first ?? "", ...rest]).join(" ");
+  const determiners = words.determiners.map((word) => word.normalize("NFKC").toLowerCase());
+  const key = (first !== undefined && determiners.includes(first) ? rest : [first ?? "", ...rest]).join(" ");
+  return withoutPrefix(key, determiners);
 };
 
 const containsSeparator = (label: string, words: FactWords): boolean =>
@@ -87,25 +116,51 @@ const containsSeparator = (label: string, words: FactWords): boolean =>
     LATIN_WORD.test(separator) ? wordsOf(label.toLowerCase()).includes(separator.toLowerCase()) : label.includes(separator),
   );
 
+const isShort = (text: string): boolean => text.length <= MAX_LABEL_LENGTH && wordsOf(text).length <= MAX_LABEL_WORDS;
+
+/** 名前の長さ。条件の付いた名前（waiting period for cancellation cover）は、名前と条件がそれぞれ短いこと。 */
+const isShortLabel = (label: string, key: string, words: FactWords): boolean => {
+  const qualified = words.qualifiers === undefined ? undefined : qualifiedLabelOf(key, words.qualifiers);
+  return qualified === undefined ? isShort(label) : isShort(qualified.head) && isShort(qualified.qualifier);
+};
+
 /** 名前として使える書き方か。短く、字を含み、区切りを含まず、指すだけの語（それ、it）でない。 */
 const isLabel = (label: string, key: string, words: FactWords): boolean =>
   label.length > 0 &&
-  label.length <= MAX_LABEL_LENGTH &&
-  wordsOf(label).length <= MAX_LABEL_WORDS &&
+  isShortLabel(label, key, words) &&
   LETTER.test(key) &&
   !CODE_MARK.test(label) &&
   !containsSeparator(label, words) &&
   !words.vague.some((word) => word.toLowerCase() === key);
 
-/** 値の後ろが文の切れ目か。行の終わり、表の升の終わり、語彙表の終わりの語。 */
+/** 値の後ろが文の切れ目か。行の終わり、表の升の終わり、語彙表の終わりの語。半角の数のまわりに空白を置く書き方（「14 cm です」）もある。 */
 const endsAfter = (source: string, value: FactValue, words: FactWords): boolean => {
-  const after = source.slice(value.end, lineEndOf(source, value.end)).replace(MARKS_AFTER_VALUE, "");
-  if (after.trim() === "" || after.trimStart().startsWith("|")) return true;
+  const after = source.slice(value.end, lineEndOf(source, value.end)).replace(MARKS_AFTER_VALUE, "").trimStart();
+  if (after === "" || after.startsWith("|")) return true;
   return words.valueEnds.some((end) => after.startsWith(end));
 };
 
-const labelledFact = (source: string, value: FactValue, words: FactWords): Fact | undefined => {
-  const head = trimEndOf(source.slice(lineStartOf(source, value.start), value.start), EDGE_MARKS);
+/** 値の終わりまで読めた値。率は、後ろの値を変えない語を越えた所で終わってもよく、そこに書いた基準を持つ。 */
+const endedValue = (source: string, value: FactValue, words: FactWords): FactValue | undefined => {
+  if (words.rates === undefined || !isRate(value, words.rates)) return endsAfter(source, value, words) ? value : undefined;
+  const tail = rateTail(source, value, words.rates);
+  const ended = endsAfter(source, { ...value, end: tail.end }, words) || joinsNextClause(source, tail.end, words.rates);
+  return ended ? withBasis(value, tail.basis) : undefined;
+};
+
+const withBasis = (value: FactValue, basis: RateBasis): FactValue => (basis.length === 0 ? value : { ...value, basis: [...(value.basis ?? []), ...basis] });
+
+/** 率の前の期間の語（年3.6% の 年）は名前の側に入れない。値の基準になる。 */
+const headOfRate = (written: string, value: FactValue, words: FactWords): { readonly head: string; readonly value: FactValue } => {
+  if (words.rates === undefined || !isRate(value, words.rates)) return { head: written, value };
+  const { head, basis } = rateHead(written, words.rates);
+  return { head, value: withBasis(value, basis) };
+};
+
+const labelledFact = (source: string, read: FactValue, words: FactWords): Fact | undefined => {
+  const { head: written, value } = headOfRate(trimEndOf(source.slice(lineStartOf(source, read.start), read.start), EDGE_MARKS), read, words);
+  const head = withoutValueLead(written, words.valueLeads ?? []);
+  const approximate = head.length < written.length ? { approximate: true } : {};
   for (const separator of words.separators) {
     const before = beforeSeparator(head, separator);
     if (before === undefined) continue;
@@ -113,7 +168,7 @@ const labelledFact = (source: string, value: FactValue, words: FactWords): Fact
     if (CLAUSE_COMMA.test(clause)) return undefined;
     const label = withoutEdgeMarks(clause);
     const key = keyOf(label, words);
-    return isLabel(label, key, words) ? { label, key, value } : undefined;
+    return isLabel(label, key, words) ? { label, key, value, ...approximate } : undefined;
   }
   return undefined;
 };
@@ -132,12 +187,36 @@ const attributeFact = (source: string, value: FactValue, words: FactWords): Fact
   return { label: found.pattern, key: keyOf(found.pattern, words), value };
 };
 
+/** 単位の語彙表の量か（"45,210 km"）。金額、率、ただの数は、読む動詞のあとでも読まない（"He read 3 chapters"）。 */
+const isMeasured = (value: FactValue): boolean => "dimension" in value;
+
+/** 値の後ろが、空白と時の節の頭の語と空白か（"45,210 km when the car came in"）。 */
+const startsClause = (source: string, end: number, clauses: readonly string[]): boolean => {
+  const after = source.slice(end, lineEndOf(source, end)).toLowerCase();
+  return /^\s/u.test(after) && clauses.some((clause) => after.trimStart().startsWith(`${clause.toLowerCase()} `));
+};
+
+/** 名前の最後の語が計器か（odometer、fuel gauge）。"The chart showed 5 kg" の chart は計器ではない。 */
+const namesGauge = (key: string, gauges: readonly string[]): boolean => {
+  const last = wordsOf(key).at(-1);
+  return last !== undefined && gauges.some((gauge) => gauge.toLowerCase() === last);
+};
+
+/** 計器が示した量（"The odometer read 45,210 km."）。読む動詞を区切りにし、値の後ろの時の節も値の終わりにする。 */
+const readingFact = (source: string, value: FactValue, words: FactWords): Fact | undefined => {
+  const readings = words.readings;
+  if (readings === undefined || !isMeasured(value)) return undefined;
+  if (!endsAfter(source, value, words) && !startsClause(source, value.end, readings.clauses)) return undefined;
+  const fact = labelledFact(source, value, { ...words, separators: readings.verbs });
+  return fact !== undefined && namesGauge(fact.key, readings.gauges) ? fact : undefined;
+};
+
 /** 文に書いた名前付きの値。値ごとに、属性の言い回しか、区切りの前の名前を探す。 */
 export const labelledFacts = (source: string, values: readonly FactValue[], words: FactWords): Fact[] =>
   values.flatMap((value) => {
     const attribute = attributeFact(source, value, words);
     if (attribute !== undefined) return [attribute];
-    if (!endsAfter(source, value, words)) return [];
-    const fact = labelledFact(source, value, words);
+    const ended = endedValue(source, value, words);
+    const fact = (ended === undefined ? undefined : labelledFact(source, ended, words)) ?? readingFact(source, value, words);
     return fact === undefined ? [] : [fact];
   });

package/src/facts/measure-units.ts

@@ -0,0 +1,49 @@
+import type { LexiconEntry, ProseDocument } from "../plugin.ts";
+import type { MeasureUnit } from "./measures.ts";
+
+/**
+ * 量の種類ごとの語彙表。weight が基準の単位への倍率で、同じ単位を二度書けば倍率が二つ（GB の 10^9 と 2^30）。
+ * group のある語は、語彙表の中の別の種類（mg/dL と mmol/L は換算が物質で違うので、組をまたいで比べない）。
+ */
+const MEASURE_DIMENSIONS = [
+  "unit-length",
+  "unit-mass",
+  "unit-time",
+  "unit-volume",
+  "unit-data",
+  "unit-temperature",
+  "unit-pressure",
+  "unit-area",
+  "unit-concentration",
+] as const;
+
+const dimensionOf = (lexicon: string, entry: LexiconEntry): string => (entry.group === undefined ? lexicon : `${lexicon}:${entry.group}`);
+
+const unitKey = (dimension: string, pattern: string, before: boolean): string => `${dimension}\u0000${before ? "<" : ">"}${pattern}`;
+
+type Factors = { readonly pattern: string; readonly dimension: string; readonly before: boolean; readonly weights: readonly number[] };
+
+/** 単位ごとの、零点（unit-zero）と、要る文脈の語（unit-context の group がその単位の語）。 */
+export const measureUnitsOf = (doc: ProseDocument): MeasureUnit[] => {
+  const zeros = new Map((doc.lexicons["unit-zero"] ?? []).map((entry) => [entry.pattern, entry.weight ?? 0]));
+  const contexts = doc.lexicons["unit-context"] ?? [];
+  return MEASURE_DIMENSIONS.flatMap((lexicon) => {
+    const factors = new Map<string, Factors>();
+    (doc.lexicons[lexicon] ?? []).forEach((entry) => {
+      if (entry.weight === undefined) return;
+      const before = entry.position === "before";
+      const dimension = dimensionOf(lexicon, entry);
+      const key = unitKey(dimension, entry.pattern, before);
+      const found = factors.get(key) ?? { pattern: entry.pattern, dimension, before, weights: [] };
+      factors.set(key, { ...found, weights: [...found.weights, entry.weight] });
+    });
+    return [...factors.values()].map(({ pattern, dimension, before, weights }) => ({
+      pattern,
+      dimension,
+      factors: weights,
+      zero: zeros.get(pattern) ?? 0,
+      before,
+      context: contexts.filter((entry) => entry.group === pattern).map((entry) => entry.pattern),
+    }));
+  });
+};

package/src/facts/measures.ts

@@ -1,54 +1,142 @@
 import type { FactValue } from "./fact-values.ts";
+import { lineHasContext, runsIntoWord, standsAsUnitWord } from "./unit-word.ts";
 
 /**
- * 単位の付いた量（5 km、3000 m、1.5時間、2 GB）。単位は言語パッケージの語彙表から取り、量の種類（長さ、重さ…）と、基準の単位への
- * 倍率を持つ。倍率が二つある単位（1 GB は 10^9 か 2^30 バイト）は、どちらで数えても合えば合う。
+ * 単位の付いた量（5 km、3000 m、1.5時間、2 GB、180℃、大さじ1）。単位は言語パッケージの語彙表から取り、量の種類（長さ、重さ…）と、基準の単位への
+ * 倍率を持つ。倍率が二つある単位（1 GB は 10^9 か 2^30 バイト）は、どちらで数えても合えば合う。零点がずれた単位（°F）は zero を引いてから掛ける。
  */
-export type MeasureUnit = { readonly pattern: string; readonly dimension: string; readonly factors: readonly number[] };
+export type MeasureUnit = {
+  readonly pattern: string;
+  readonly dimension: string;
+  readonly factors: readonly number[];
+  /** 基準の単位で 0 のときに、この単位で読む値（°F なら 32）。 */
+  readonly zero: number;
+  /** 数の前に書く単位（大さじ1）。 */
+  readonly before: boolean;
+  /** 同じ行にどれかが無ければ単位として読まない語（「度」はオーブンの行だけ温度）。空なら要らない。 */
+  readonly context: readonly string[];
+};
 
-export type Measured = FactValue & { readonly dimension: string; readonly factors: readonly number[]; readonly amount: number };
+export type Measured = FactValue & {
+  readonly dimension: string;
+  readonly factors: readonly number[];
+  readonly zero: number;
+  readonly amount: number;
+  /** 書いた小数の桁数（1.20 なら 2）。amount は数なので末尾の 0 を持たない。 */
+  readonly decimals: number;
+};
 
-/** 前に英字や数が続く数（v2、A4、1.2.3 の 2）は量ではない。日本語は数の前に空白を置かないので、仮名や漢字の後ろはよい。 */
-const NUMBER = /(?<![\p{Script=Latin}\p{N}.,])[0-9０-９]+(?:[,，][0-9０-９]{3})*(?:[.．][0-9０-９]+)?/gu;
+/** 換算して合うとみなす差。relative は大きいほうの値に対する割合、absolute は基準の単位での差。どちらかに収まれば合う。 */
+export type Tolerance = { readonly relative: number; readonly absolute: number };
 
 /**
- * 数のすぐ後ろ（空白一つまで）の単位。長い単位から試す（「5分間」を 分 で切らない、min を m にしない）。
- * 単位の後ろに語が続く（5 miners）ものは、名前付きの値の終わり（fact-value-end）で落ちるので、ここでは見ない。
+ * 前に英字や数が続く数（v2、A4、1.2.3 の 2）と、分数の分母（1/2 の 2）と、冪の指数（×10^4/µL の 4）は量ではない。日本語は数の前に空白を置かないので、仮名や漢字の後ろはよい。
+ * 字や数のすぐ後ろでない負の符号は数に含める（-20 °C）。20-25°C の - は範囲の印。
  */
-const unitAfter = (source: string, end: number, units: readonly MeasureUnit[]): { unit: MeasureUnit; end: number } | undefined => {
-  const gap = source.charAt(end) === " " ? 1 : 0;
-  const rest = source.slice(end + gap, end + gap + 16);
-  const unit = units.find((candidate) => rest.startsWith(candidate.pattern));
-  return unit === undefined ? undefined : { unit, end: end + gap + unit.pattern.length };
+const NUMBER = /(?<![\p{Script=Latin}\p{N}.,^])(?<![0-9０-９][/／])[0-9０-９]+(?:[,，][0-9０-９]{3})*(?:[.．][0-9０-９]+)?/gu;
+const MINUS_SIGNS = new Set(["-", "−", "－"]);
+const BEFORE_SIGN = /[\p{L}\p{N}.,]/u;
+
+/** 数の頭の負の符号の位置。符号が無ければ数の頭。 */
+const signedStart = (source: string, start: number): number =>
+  MINUS_SIGNS.has(source.charAt(start - 1)) && !BEFORE_SIGN.test(source.charAt(start - 2)) ? start - 1 : start;
+
+/** 数のすぐ後ろに分数が続く（大さじ1と1/2、1 1/2 cups、1½）。数だけを読むと量を取り違える。 */
+const FRACTION_AFTER = /^.?[0-9０-９]*[/／][0-9０-９]|^.?[½⅓⅔¼¾⅛]/u;
+
+const MAX_UNIT_LENGTH = 20;
+
+const lineAt = (source: string, offset: number): string => {
+  const end = source.indexOf("\n", offset);
+  return source.slice(source.lastIndexOf("\n", offset - 1) + 1, end === -1 ? source.length : end);
+};
+
+const inContext = (unit: MeasureUnit, source: string, offset: number): boolean =>
+  unit.context.length === 0 || lineHasContext(lineAt(source, offset), unit.context);
+
+/** 数と単位のあいだに置ける字。空白一つか、英語の複合語のハイフン（13.3-inch display）。 */
+const GAPS = new Set([" ", "-"]);
+
+type Found = { readonly unit: MeasureUnit; readonly start: number; readonly end: number };
+
+/**
+ * 数のすぐ後ろ（空白かハイフン一つまで）の単位。長い単位から試す（「5分間」を 分 で切らない、min を m にしない）。一番長く書かれた単位が
+ * 語に続いていれば、短い単位にも読み替えない（"5 g/dLs" を 5 g にしない）。
+ */
+const unitAfter = (source: string, start: number, end: number, units: readonly MeasureUnit[]): Found | undefined => {
+  const gap = GAPS.has(source.charAt(end)) ? 1 : 0;
+  const rest = source.slice(end + gap, end + gap + MAX_UNIT_LENGTH);
+  const afterUnit = (candidate: MeasureUnit): string => source.slice(end + gap + candidate.pattern.length);
+  const written = units.filter((candidate) => !candidate.before && rest.startsWith(candidate.pattern));
+  const [longest] = written;
+  if (longest === undefined || runsIntoWord(longest.pattern, afterUnit(longest))) return undefined;
+  const unit = written.find(
+    (candidate) => standsAsUnitWord(candidate.pattern, candidate.context.length > 0, afterUnit(candidate)) && inContext(candidate, source, end),
+  );
+  return unit === undefined ? undefined : { unit, start, end: end + gap + unit.pattern.length };
+};
+
+/** 数のすぐ前（空白一つまで）の単位（大さじ1）。 */
+const unitBefore = (source: string, start: number, end: number, units: readonly MeasureUnit[]): Found | undefined => {
+  const gap = source.charAt(start - 1) === " " ? 1 : 0;
+  const head = source.slice(Math.max(0, start - gap - MAX_UNIT_LENGTH), start - gap);
+  const unit = units.find((candidate) => candidate.before && head.endsWith(candidate.pattern) && inContext(candidate, source, start));
+  return unit === undefined ? undefined : { unit, start: start - gap - unit.pattern.length, end };
 };
 
 /** 文書の中の、単位の付いた量。 */
 export const measuredValues = (source: string, units: readonly MeasureUnit[]): Measured[] => {
   const longestFirst = units.toSorted((left, right) => right.pattern.length - left.pattern.length);
   return [...source.matchAll(NUMBER)].flatMap((match): Measured[] => {
-    const found = unitAfter(source, match.index + match[0].length, longestFirst);
+    const end = match.index + match[0].length;
+    if (FRACTION_AFTER.test(source.slice(end, end + MAX_UNIT_LENGTH))) return [];
+    const start = signedStart(source, match.index);
+    const found = unitAfter(source, start, end, longestFirst) ?? unitBefore(source, start, end, longestFirst);
     if (found === undefined) return [];
-    const amount = Number(match[0].normalize("NFKC").replace(/,/gu, ""));
+    const written = match[0].normalize("NFKC").replace(/,/gu, "");
+    const magnitude = Number(written);
+    const amount = start < match.index ? -magnitude : magnitude;
+    const { unit } = found;
     return [
       {
-        start: match.index,
+        start: found.start,
         end: found.end,
         kind: "quantity",
         key: String(amount),
-        unit: found.unit.pattern,
-        dimension: found.unit.dimension,
-        factors: found.unit.factors,
+        unit: unit.pattern,
+        dimension: unit.dimension,
+        factors: unit.factors,
+        zero: unit.zero,
         amount,
+        decimals: written.split(".")[1]?.length ?? 0,
       },
     ];
   });
 };
 
-/** 二つの量が同じ量か。倍率のどの組み合わせかで、基準の単位に直した値の差が大きいほうの tolerance 以内なら同じ。 */
-export const sameMeasure = (left: Measured, right: Measured, tolerance: number): boolean =>
-  left.factors.some((a) =>
-    right.factors.some((b) => {
-      const [x, y] = [left.amount * a, right.amount * b];
-      return Math.abs(x - y) <= Math.max(Math.abs(x), Math.abs(y)) * tolerance;
-    }),
+/** 二つの量の単位が、書き方だけ違う同じ単位か（mL と ml、tablespoon と tablespoons）。 */
+export const sameUnit = (left: Measured, right: Measured): boolean =>
+  left.zero === right.zero && left.factors.length === right.factors.length && left.factors.every((factor, index) => factor === right.factors[index]);
+
+const agrees = (x: number, y: number, tolerance: Tolerance): boolean => {
+  const difference = Math.abs(x - y);
+  return difference <= tolerance.absolute || difference <= Math.max(Math.abs(x), Math.abs(y)) * tolerance.relative;
+};
+
+/** 零点のずれた単位で、零点より小さい量（1 °F）。温度の読みより、差（0.5 ℃ の上昇は 0.9 °F）のことが多い。 */
+const couldBeDifference = (value: Measured): boolean => value.zero !== 0 && Math.abs(value.amount) < Math.abs(value.zero);
+
+/**
+ * 二つの量が同じ量か。倍率のどの組み合わせかで、基準の単位に直した値の差が tolerance に収まれば同じ。零点より小さい °F の量は、差として
+ * 零点を引かずに直しても合えば合う（0.5°C (1°F)）。180 °F は差としては読まないので、100 °C (180 °F) は合わない。
+ */
+export const sameMeasure = (left: Measured, right: Measured, tolerance: Tolerance): boolean => {
+  const difference = couldBeDifference(left) || couldBeDifference(right);
+  return left.factors.some((a) =>
+    right.factors.some(
+      (b) =>
+        agrees((left.amount - left.zero) * a, (right.amount - right.zero) * b, tolerance) ||
+        (difference && agrees(left.amount * a, right.amount * b, tolerance)),
+    ),
   );
+};

package/src/facts/qualified-labels.ts

@@ -0,0 +1,126 @@
+/**
+ * 条件の付いた名前（「待機期間（旅行キャンセル費用）」「旅行キャンセル費用の待機期間」"waiting period for cancellation cover"）を
+ * 一つの書き方（名前と条件）にする。語彙表の型は * が条件の位置: 「（*）」「*の」"(*)" "for *"。
+ * 条件の違う名前（待機期間（疾病）と待機期間（けが））と、条件の無い名前（待機期間）は別の名前のまま。
+ * 英語の "cancellation cover waiting period" は、同じ文書が "waiting period (cancellation cover)" とも書いたときだけ同じ名前。
+ */
+export type QualifierWords = {
+  /** 条件を書く型。* が条件。* が頭なら条件は名前の前（*の）、そうでなければ後ろ（（*）、for *）。 */
+  readonly templates: readonly string[];
+  /** 名前と条件の頭から落とす語（the, our）。 */
+  readonly determiners: readonly string[];
+};
+
+export type QualifiedLabel = { readonly head: string; readonly qualifier: string };
+
+const PLACEHOLDER = "*";
+const LETTER = /\p{L}/u;
+const LATIN_WORD = /^[a-z]+(?: [a-z]+)*$/u;
+
+const folded = (text: string): string => text.normalize("NFKC").toLowerCase();
+
+const wordsOf = (text: string): string[] => text.split(/\s+/u).filter((word) => word !== "");
+
+const withoutDeterminers = (text: string, determiners: readonly string[]): string => {
+  const words = wordsOf(text);
+  const start = words.findIndex((word) => !determiners.includes(word));
+  return start === -1 ? "" : words.slice(start).join(" ");
+};
+
+const labelOf = (head: string, qualifier: string, words: QualifierWords): QualifiedLabel | undefined => {
+  const determiners = words.determiners.map(folded);
+  const label = { head: withoutDeterminers(head, determiners), qualifier: withoutDeterminers(qualifier, determiners) };
+  return LETTER.test(label.head) && LETTER.test(label.qualifier) ? label : undefined;
+};
+
+/** 英字の語の区切り（for、of）は、前後が空白のときだけ。最初のものから後ろが条件。 */
+const splitAtWord = (key: string, joiner: string): [string, string] | undefined => {
+  const at = key.indexOf(` ${joiner} `);
+  return at === -1 ? undefined : [key.slice(0, at), key.slice(at + joiner.length + 2)];
+};
+
+/** 記号の区切り（（）は、最後のものから後ろが条件。 */
+const splitAtMark = (key: string, open: string): [string, string] | undefined => {
+  const at = key.lastIndexOf(open);
+  return at <= 0 ? undefined : [key.slice(0, at), key.slice(at + open.length)];
+};
+
+/** 条件が名前の後ろの型（（*）、for *）。 */
+const qualifierAfter = (key: string, open: string, close: string): [string, string] | undefined => {
+  if (!key.endsWith(close)) return undefined;
+  const body = key.slice(0, key.length - close.length);
+  const trimmed = open.trim();
+  return LATIN_WORD.test(trimmed) ? splitAtWord(body, trimmed) : splitAtMark(body, open);
+};
+
+/** 条件が名前の前の型（*の）。最後のつなぎの語から後ろが名前。 */
+const qualifierBefore = (key: string, joiner: string): [string, string] | undefined => {
+  const at = key.lastIndexOf(joiner);
+  return at <= 0 ? undefined : [key.slice(at + joiner.length), key.slice(0, at)];
+};
+
+const splitBy = (key: string, template: string): [string, string] | undefined => {
+  const at = template.indexOf(PLACEHOLDER);
+  if (at === -1) return undefined;
+  const open = template.slice(0, at);
+  const close = template.slice(at + PLACEHOLDER.length);
+  if (open === "") return close === "" ? undefined : qualifierBefore(key, close);
+  return qualifierAfter(key, open, close);
+};
+
+/** 名前の key を、名前と条件に分ける。型に合わなければ undefined。語彙表の順に試す。 */
+export const qualifiedLabelOf = (key: string, words: QualifierWords): QualifiedLabel | undefined => {
+  const trimmed = folded(key).trim();
+  for (const template of words.templates.map(folded)) {
+    const parts = splitBy(trimmed, template);
+    const label = parts === undefined ? undefined : labelOf(parts[0], parts[1], words);
+    if (label !== undefined) return label;
+  }
+  return undefined;
+};
+
+const keyOfLabel = ({ head, qualifier }: QualifiedLabel): string => `${head} (${qualifier})`;
+
+/** 名前と条件を別々に持つ値（表の列の見出しと行の見出し）の key。どちらかに字が無ければ undefined。 */
+export const keyOfParts = (head: string, qualifier: string, words: QualifierWords): string | undefined => {
+  const spaced = (text: string): string => folded(text).replace(/\s+/gu, " ").trim();
+  const label = labelOf(spaced(head), spaced(qualifier), words);
+  return label === undefined ? undefined : keyOfLabel(label);
+};
+
+/** 条件の付いた名前は「名前 (条件)」に。条件の無い名前は書いたまま。 */
+export const qualifiedKeyOf = (key: string, words: QualifierWords): string => {
+  const label = qualifiedLabelOf(key, words);
+  return label === undefined ? key : keyOfLabel(label);
+};
+
+/** 前に置いた条件（cancellation cover waiting period）として読める条件か。語だけで、冠詞もつなぎの語も含まない。 */
+const isPremodifier = (qualifier: string, words: QualifierWords): boolean => {
+  const joiners = words.templates.map((template) => folded(template).replace(PLACEHOLDER, "").trim());
+  const determiners = words.determiners.map(folded);
+  return LATIN_WORD.test(qualifier) && wordsOf(qualifier).every((word) => !determiners.includes(word) && !joiners.includes(word));
+};
+
+/** 前に置いた書き方が、二つの名前と条件（claim limit (baggage) と limit (baggage claim)）に読めるものは、どちらにもしない。 */
+const unambiguous = (pairs: readonly [string, string][]): Map<string, string> => {
+  const targets = new Map<string, Set<string>>();
+  pairs.forEach(([surface, key]) => targets.set(surface, (targets.get(surface) ?? new Set<string>()).add(key)));
+  return new Map(pairs.filter(([surface]) => targets.get(surface)?.size === 1));
+};
+
+/**
+ * 文書の名前の key を一つの書き方に。条件を前に置いた名前（"cancellation cover waiting period"）は、同じ文書に同じ名前と条件の
+ * 型の書き方があるときだけ、それと同じ key にする。
+ */
+export const qualifiedKeys = (keys: readonly string[], words: QualifierWords): string[] => {
+  const labels = keys.map((key) => qualifiedLabelOf(key, words));
+  const premodified = unambiguous(
+    labels.flatMap((label): [string, string][] =>
+      label === undefined || !isPremodifier(label.qualifier, words) ? [] : [[`${label.qualifier} ${label.head}`, keyOfLabel(label)]],
+    ),
+  );
+  return keys.map((key, index) => {
+    const label = labels[index];
+    return label === undefined ? (premodified.get(key.trim()) ?? key) : keyOfLabel(label);
+  });
+};

package/src/facts/rate-values.ts

@@ -0,0 +1,117 @@
+import type { FactValue } from "./fact-values.ts";
+import { EDGE_MARKS, trimEndOf } from "./trim-marks.ts";
+
+/**
+ * 率（百分率）の値のまわりに書く、値を変えない語。期間（年3.6%、4.8% per year）、種類（固定、variable）、何に対する率
+ * （of the purchase price）、括弧の注記（3.6%（固定金利））。これらを越えた所で値が終わっていれば、率は名前の値として読む。
+ * 語彙表に無い語が続く率（3.6% of applicants、3.6%増）は、別の量の一部なので読まない。
+ * 期間・種類・対象は値の基準（basis）になり、両方に書いてあって違えば別の量として比べない（月0.5% と 年6%）。
+ * 語彙表に無い括弧の注記（(of applicants)、(men)）は、同じ注記の率とだけ比べる。何の率かを言っているかもしれない。
+ */
+export type RateWord = { readonly pattern: string; readonly group?: string | undefined; readonly position: "before" | "after" };
+
+export type RateWords = {
+  /** 百分率の単位（%、パーセント）。 */
+  readonly units: readonly string[];
+  /** 基準の語彙表ごとの語。鍵は基準の種類（period、kind、base）。 */
+  readonly notes: Readonly<Record<string, readonly RateWord[]>>;
+  /** 率のすぐ後ろで、値を言い終えて次の節を始める語句（and stays、and does not）。 */
+  readonly joiners: readonly string[];
+};
+
+/** 基準の種類と、その値（period:year）。 */
+export type RateBasis = readonly string[];
+
+const OPEN_BRACKETS: ReadonlySet<string> = new Set(["(", "（"]);
+const CLOSE_BRACKETS = /[)）]/u;
+const NESTED_OR_LINE = /[(（\n]/u;
+const SPACES = /^[ \t]*/u;
+const LATIN_LETTER = /[a-z]/iu;
+/** 語彙表に無い括弧の注記の基準。この基準だけは、片方に無くても食い違いとする。 */
+const NOTE = "note:";
+
+const normalized = (text: string): string => text.normalize("NFKC").toLowerCase();
+
+export const isRate = (value: FactValue, words: RateWords): boolean =>
+  value.kind === "quantity" && words.units.some((unit) => normalized(unit) === normalized(value.unit));
+
+const basisOf = (dimension: string, word: RateWord): RateBasis => (word.group === undefined ? [] : [`${dimension}:${word.group}`]);
+
+type NoteMatch = { readonly length: number; readonly basis: RateBasis };
+
+/** 英字で終わる語は、語の途中で切れていないこと（a year を a yearly の頭として読まない）。 */
+const endsWord = (text: string, length: number): boolean => !(LATIN_LETTER.test(text.charAt(length - 1)) && LATIN_LETTER.test(text.charAt(length)));
+
+const longestOf = (matches: readonly NoteMatch[]): NoteMatch | undefined =>
+  matches.reduce<NoteMatch | undefined>((longest, match) => (longest === undefined || match.length > longest.length ? match : longest), undefined);
+
+/** 基準の語のうち、置く側が position のもの（undefined ならどちらでも）を、語彙表の種類と組にして。 */
+const notesAt = (words: RateWords, position: RateWord["position"] | undefined): { readonly dimension: string; readonly word: RateWord }[] =>
+  Object.entries(words.notes).flatMap(([dimension, entries]) =>
+    entries.filter((word) => position === undefined || word.position === position).map((word) => ({ dimension, word })),
+  );
+
+/** text の頭に置いた基準の語のうち一番長いもの。 */
+const noteAtStart = (text: string, words: RateWords, position: RateWord["position"] | undefined): NoteMatch | undefined =>
+  longestOf(
+    notesAt(words, position)
+      .filter(({ word }) => normalized(text.slice(0, word.pattern.length)) === normalized(word.pattern) && endsWord(text, word.pattern.length))
+      .map(({ dimension, word }): NoteMatch => ({ length: word.pattern.length, basis: basisOf(dimension, word) })),
+  );
+
+/** 頭の括弧の注記。中身が基準の語そのもの（（固定金利）、(fixed)）なら、その基準も。行をまたぐ括弧や入れ子は注記にしない。 */
+const bracketAtStart = (text: string, words: RateWords): NoteMatch | undefined => {
+  if (!OPEN_BRACKETS.has(text.charAt(0))) return undefined;
+  const close = text.search(CLOSE_BRACKETS);
+  const inside = close === -1 ? "" : text.slice(1, close);
+  if (close === -1 || NESTED_OR_LINE.test(inside)) return undefined;
+  const whole = noteAtStart(inside, words, undefined);
+  return { length: close + 1, basis: whole?.length === inside.length ? whole.basis : [`${NOTE}${normalized(inside).trim()}`] };
+};
+
+export type RateTail = { readonly end: number; readonly basis: RateBasis };
+
+/** 率の後ろの、値を変えない語と括弧の注記を越えた所と、そこまでに書いた基準。 */
+const tailFrom = (source: string, tail: RateTail, words: RateWords): RateTail => {
+  const text = source.slice(tail.end).replace(SPACES, "");
+  const note = bracketAtStart(text, words) ?? noteAtStart(text, words, "after");
+  if (note === undefined) return tail;
+  return tailFrom(source, { end: source.length - text.length + note.length, basis: [...tail.basis, ...note.basis] }, words);
+};
+
+export const rateTail = (source: string, value: FactValue, words: RateWords): RateTail => tailFrom(source, { end: value.end, basis: [] }, words);
+
+export type RateHead = { readonly head: string; readonly basis: RateBasis };
+
+/** 率の前の期間の語（年3.6% の 年）を、名前の側から外したものと、その基準。 */
+export const rateHead = (head: string, words: RateWords): RateHead => {
+  const trimmed = trimEndOf(head, EDGE_MARKS);
+  const found = longestOf(
+    notesAt(words, "before")
+      .filter(({ word }) => normalized(trimmed.slice(-word.pattern.length)) === normalized(word.pattern))
+      .map(({ dimension, word }): NoteMatch => ({ length: word.pattern.length, basis: basisOf(dimension, word) })),
+  );
+  if (found === undefined) return { head, basis: [] };
+  return { head: trimEndOf(trimmed.slice(0, trimmed.length - found.length), EDGE_MARKS), basis: found.basis };
+};
+
+/** 率の後ろが、値を言い終えて次の節を始める語句（4.5% and does not change）。語句の後ろは語の切れ目。 */
+export const joinsNextClause = (source: string, end: number, words: RateWords): boolean => {
+  const text = source.slice(end).replace(SPACES, "");
+  return words.joiners.some((phrase) => normalized(text.slice(0, phrase.length)) === normalized(phrase) && endsWord(text, phrase.length));
+};
+
+const dimensionOf = (entry: string): string => entry.slice(0, entry.indexOf(":") + 1);
+
+const notesOf = (basis: RateBasis): string =>
+  basis
+    .filter((entry) => entry.startsWith(NOTE))
+    .toSorted((left, right) => left.localeCompare(right, "en"))
+    .join("\u0000");
+
+/** 二つの値の基準が食い違わないか。同じ種類の基準を両方に書いて違うか、語彙表に無い注記が揃わないときに食い違う。 */
+export const basesAgree = (left: RateBasis | undefined, right: RateBasis | undefined): boolean => {
+  const [ours, theirs] = [left ?? [], right ?? []];
+  if (notesOf(ours) !== notesOf(theirs)) return false;
+  return ours.every((entry) => !theirs.some((other) => dimensionOf(other) === dimensionOf(entry) && other !== entry));
+};