chaffjs 0.27.0 → 0.28.0

package/src/structure/change-rate.ts

@@ -12,6 +12,10 @@ export type Rate = Span & { readonly value: number; readonly decimals: number };
 export type Direction = Span & { readonly sign: 1 | -1 };
 /** A word that marks a value: the earlier one after it (「1,000社から」) or before it ("from 1,000"), the later one (「1,200社に」, "to 1,200"). */
 export type BaseMark = Span & { readonly position: "before" | "after" };
+/** A calendar year written in the sentence (「2025年」, "in 2025"). */
+export type Period = Span & { readonly year: number };
+/** A word that starts another subject or clause; one that joins two values ("… in 2025 and …") is read only before the rate. */
+export type Break = Span & { readonly beforeRateOnly: boolean };
 
 export type ChangeText = {
   readonly sentences: readonly Span[];
@@ -21,6 +25,11 @@ export type ChangeText = {
   readonly marks: readonly BaseMark[];
   /** Words that mark the later value ("to", 「に」). */
   readonly targets: readonly BaseMark[];
+  readonly periods: readonly Period[];
+  /** Words that start another subject or clause (「利益は」, "and", ";"): a rate after one is not about the values before it. */
+  readonly breaks: readonly Break[];
+  /** The document's text, which the gaps of a pair are measured in. */
+  readonly source: string;
 };
 
 /** How far a direction word may stand from the rate (「20%の増加」「20%増えました」, "an increase of 20%", "up 20%"). */
@@ -32,6 +41,16 @@ const MARK_GAP = 3;
  * away, the rate is of something else in the sentence (「100店から120店に増え、売上は15%増えた」).
  */
 const BASE_REACH = 12;
+/**
+ * The two values written together, the earlier then the later or the other way round (「昨年の100万円から今年は150万円」
+ * "from $1.0 million last year to $1.5 million"): how far apart they may stand, and how far after them the rate may
+ * (「150万円になり、前年比20%」", an increase of 20%"). Measured with a Latin word counting as one character.
+ */
+const PAIR_GAP = 16;
+const PAIR_REACH = 16;
+/** How far a year may stand from the value it dates (「2025年の100万円」"$1.0 million in 2025"), a word counting as one. */
+const PERIOD_GAP = 8;
+const LATIN_WORD = /[A-Za-z]+/gu;
 const PERCENT = 100;
 const HALF = 0.5;
 const DECIMAL_BASE = 10;
@@ -40,6 +59,11 @@ const within = (outer: Span, inner: Span): boolean => inner.start >= outer.start
 
 const distance = (left: Span, right: Span): number => Math.max(left.start - right.end, right.start - left.end, 0);
 
+const gapBetween = (left: Span, right: Span): Span => ({ start: left.end, end: right.start });
+
+/** The length of a gap, a Latin word counting as one character: "an increase of" is as short as 「前年比」. */
+export const gapLength = (source: string, gap: Span): number => source.slice(gap.start, gap.end).replaceAll(LATIN_WORD, "w").length;
+
 const markOf = (figure: Figure, marks: readonly BaseMark[]): BaseMark | undefined =>
   marks.find((mark) => {
     const gap = mark.position === "after" ? mark.start - figure.end : figure.start - mark.end;
@@ -76,28 +100,124 @@ const rateRange = (base: Figure, current: Figure): readonly [number, number] | u
   return [(currentLow / baseHigh - 1) * PERCENT, (currentHigh / baseLow - 1) * PERCENT];
 };
 
-const issueIn = (text: ChangeText, sentence: Span): StructureIssue[] => {
-  const rates = text.rates.filter((rate) => within(sentence, rate));
-  const figures = text.figures.filter((figure) => within(sentence, figure));
-  const [rate] = rates;
-  if (rates.length !== 1 || rate === undefined) return [];
-  const sign = signNear(
-    rate,
-    text.directions.filter((direction) => within(sentence, direction)),
-  );
-  const bases = figures.filter((figure) => isBaseFor(figure, rate, text.marks));
+type Pair = { readonly base: Figure; readonly current: Figure };
+/** One sentence's parts, with the rate: the one percentage that has a word of direction beside it. */
+type Reading = ChangeText & { readonly rate: Rate; readonly sign: 1 | -1 };
+
+/** A word between the two that starts another subject or clause (「100万円から150万円に増え、利益は20%」). */
+const brokenBetween = (text: Reading, one: Span, other: Span): boolean => {
+  const gap = one.end <= other.start ? gapBetween(one, other) : gapBetween(other, one);
+  return text.breaks.some((word) => within(gap, word));
+};
+
+/** The earlier value marked beside the rate, and the one value in its unit before the rate or marked as reached. */
+const besideRate = (text: Reading): Pair | undefined => {
+  const bases = text.figures.filter((figure) => isBaseFor(figure, text.rate, text.marks) && !brokenBetween(text, figure, text.rate));
   const [base] = bases;
-  if (sign === undefined || bases.length !== 1 || base === undefined) return [];
-  const currents = figures.filter((figure) => figure !== base && figure.unit === base.unit && isCurrentFor(figure, rate, text.targets));
+  if (bases.length !== 1 || base === undefined) return undefined;
+  const currents = text.figures.filter((figure) => figure !== base && figure.unit === base.unit && isCurrentFor(figure, text.rate, text.targets));
   const [current] = currents;
-  const range = currents.length === 1 && current !== undefined ? rateRange(base, current) : undefined;
-  if (range === undefined || current === undefined) return [];
+  return currents.length === 1 && current !== undefined ? { base, current } : undefined;
+};
+
+const yearsBeside = (text: Reading, figure: Figure, side: "before" | "after", bound: Span): Period[] =>
+  text.periods.filter((period) => {
+    const [left, right] = side === "before" ? [period, figure] : [figure, period];
+    return left.end <= right.start && within(bound, period) && gapLength(text.source, gapBetween(left, right)) <= PERIOD_GAP;
+  });
+
+/**
+ * Of two values each dated by a year, the earlier year's. The years stand on the same side of both values: before them
+ * (「2025年は100万円、2026年は150万円」) or after them ("$1.0 million in 2025 and $1.5 million in 2026").
+ */
+const datedEarlier = (text: Reading, first: Figure, second: Figure): Figure | undefined => {
+  const middle = gapBetween(first, second);
+  const sides = [
+    [yearsBeside(text, first, "before", { start: 0, end: first.start }), yearsBeside(text, second, "before", middle)],
+    [yearsBeside(text, first, "after", middle), yearsBeside(text, second, "after", { start: second.end, end: text.source.length })],
+  ].filter(([early, late]) => early?.length === 1 && late?.length === 1);
+  const [[early, late] = []] = sides;
+  const [firstYear, secondYear] = [early?.[0]?.year, late?.[0]?.year];
+  if (sides.length !== 1 || firstYear === undefined || secondYear === undefined || firstYear === secondYear) return undefined;
+  return firstYear < secondYear ? first : second;
+};
+
+/** Which of two values written together is the earlier: the one marked (「から」, "from", 「前年の」), else the one dated earlier. */
+const pairOf = (text: Reading, first: Figure, second: Figure): Pair | undefined => {
+  const marked = [first, second].filter((figure) => markOf(figure, text.marks) !== undefined);
+  if (marked.length > 1) return undefined;
+  const base = marked[0] ?? datedEarlier(text, first, second);
+  if (base === undefined) return undefined;
+  return { base, current: base === first ? second : first };
+};
+
+/** Nothing between the later value and the rate that makes the rate another one's: a value, another subject, another base. */
+const leadsTo = (text: Reading, second: Figure): boolean => {
+  const gap = gapBetween(second, text.rate);
+  if (gap.start > gap.end || gapLength(text.source, gap) > PAIR_REACH) return false;
+  return ![...text.figures, ...text.breaks, ...text.marks].some((span) => within(gap, span));
+};
+
+/** Two values of one subject written close, with no rate between them (「売上は100万円、費用は150万円」 is two subjects). */
+const standTogether = (text: Reading, first: Figure, second: Figure): boolean => {
+  const gap = gapBetween(first, second);
+  const inGap = (span: Span): boolean => within(gap, span);
+  if (gapLength(text.source, gap) > PAIR_GAP || text.rates.some(inGap)) return false;
+  return !text.breaks.some((word) => !word.beforeRateOnly && inGap(word));
+};
+
+/** The two values written together right before the rate (「昨年の100万円から今年は150万円になり、前年比20%の増加」). */
+const beforeRate = (text: Reading): Pair | undefined => {
+  const pairs = text.figures.flatMap((first, index) => {
+    const second = text.figures.slice(index + 1).find((figure) => figure.unit === first.unit);
+    const pair = second !== undefined && standTogether(text, first, second) && leadsTo(text, second) ? pairOf(text, first, second) : undefined;
+    return pair === undefined ? [] : [pair];
+  });
+  const [pair] = pairs;
+  return pairs.length === 1 ? pair : undefined;
+};
+
+/**
+ * The sentence's one rate with a word of direction beside it, a word counting for the rate it stands nearest. A percentage
+ * without one (「20%の増加、利益率は10%」) is about something else.
+ */
+const readingIn = (text: ChangeText, sentence: Span): Reading | undefined => {
+  const inSentence = (span: Span): boolean => within(sentence, span);
+  const directions = text.directions.filter(inSentence);
+  const rates = text.rates.filter(inSentence);
+  const nearestTo = (rate: Rate, direction: Direction): boolean => rates.every((other) => distance(direction, rate) <= distance(direction, other));
+  const signed = rates.flatMap((rate) => {
+    const sign = signNear(
+      rate,
+      directions.filter((direction) => nearestTo(rate, direction)),
+    );
+    return sign === undefined ? [] : [{ rate, sign }];
+  });
+  const [only] = signed;
+  if (signed.length !== 1 || only === undefined) return undefined;
+  const parts = {
+    figures: text.figures.filter(inSentence).toSorted((left, right) => left.start - right.start),
+    periods: text.periods.filter(inSentence),
+    breaks: text.breaks.filter(inSentence),
+  };
+  return { ...text, ...parts, ...only, rates };
+};
+
+const issueIn = (text: ChangeText, sentence: Span): StructureIssue[] => {
+  const reading = readingIn(text, sentence);
+  const pair = reading === undefined ? undefined : (besideRate(reading) ?? beforeRate(reading));
+  const range = pair === undefined ? undefined : rateRange(pair.base, pair.current);
+  if (reading === undefined || pair === undefined || range === undefined) return [];
+  const { rate, sign } = reading;
   const written = sign * rate.value;
   const slack = HALF * DECIMAL_BASE ** -rate.decimals;
   if (written >= range[0] - slack && written <= range[1] + slack) return [];
-  const computed = Math.abs((current.value / base.value - 1) * PERCENT).toFixed(rate.decimals);
+  const computed = Math.abs((pair.current.value / pair.base.value - 1) * PERCENT).toFixed(rate.decimals);
   return [{ offset: rate.start, values: { rate: String(rate.value), computed } }];
 };
 
-/** Each sentence with one rate of change, one direction, one marked earlier value and one other value in its unit, that disagree. */
+/**
+ * Each sentence with one rate of change and its direction, and the two values it is computed from, that disagree: the
+ * earlier value marked beside the rate, or the two values written together right before it.
+ */
 export const changeRateMismatches = (text: ChangeText): StructureIssue[] => text.sentences.flatMap((sentence) => issueIn(text, sentence));

package/src/structure/subject-change.ts

@@ -0,0 +1,35 @@
+import type { Token } from "../plugin.ts";
+
+// Whether a joining word ("and") starts a clause about another subject: "Revenue was $1,000 in 2025 and costs were $1,500
+// in 2026" joins two subjects, "… in 2025 and $1,500 in 2026" or "… and revenue was $1,500" one. Two words for one thing
+// ("sales" and "revenue") read as two subjects, so the pair is left alone rather than read wrongly. Pure: reads the tagger's
+// parts of speech and lemmas.
+
+const NOUN: ReadonlySet<string> = new Set(["NOUN", "PROPN"]);
+/** Words that may stand before the noun that starts the clause ("and the costs", "and our operating costs"). */
+const BEFORE_NOUN: ReadonlySet<string> = new Set(["DET", "ADJ", "PRON"]);
+
+const wordOf = (token: Token): string => (token.lemma ?? token.surface).toLowerCase();
+
+/** The last noun of the run of nouns at the start of tokens, past determiners and adjectives ("the operating costs" → cost). */
+const leadingNoun = (tokens: readonly Token[]): Token | undefined => {
+  const start = tokens.findIndex((token) => !BEFORE_NOUN.has(token.pos));
+  if (start < 0) return undefined;
+  const end = tokens.slice(start).findIndex((token) => !NOUN.has(token.pos));
+  const nouns = tokens.slice(start, end < 0 ? tokens.length : start + end);
+  return nouns.at(-1);
+};
+
+/**
+ * The joining word at index is followed by a noun the sentence has not used before it: "and costs were", "and costs $1,500"
+ * after "Revenue was $1,000", where "and revenue was" after "ACME's revenue was $1,000" names the same thing again. A
+ * sentence with no noun before the joining word gives nothing to compare with.
+ */
+export const startsOtherSubject = (tokens: readonly Token[], index: number): boolean => {
+  const next = leadingNoun(tokens.slice(index + 1));
+  const before = tokens
+    .slice(0, index)
+    .filter((token) => NOUN.has(token.pos))
+    .map(wordOf);
+  return next !== undefined && before.length > 0 && !before.includes(wordOf(next));
+};