@chaffjs/lang-en 0.14.0 → 0.16.0

package/lexicons/finite-auxiliary.yaml

@@ -0,0 +1,9 @@
+# Forms of "be" and "have" that are never a noun, so a possessive directly before one ("their is") is a slip.
+id: finite-auxiliary
+language: en
+entries:
+  - pattern: is
+  - pattern: are
+  - pattern: was
+  - pattern: were
+  - pattern: has

package/lexicons/invariant-noun.yaml

@@ -0,0 +1,26 @@
+# Nouns whose singular and plural are spelled alike, or that name one thing while ending in -s. The tagger may read
+# either number into them, so agreement-slip does not count them: "a series", "a means", "these staff", "this data".
+id: invariant-noun
+language: en
+entries:
+  - pattern: news
+  - pattern: series
+  - pattern: species
+  - pattern: means
+  - pattern: data
+  - pattern: media
+  - pattern: criteria
+  - pattern: phenomena
+  - pattern: staff
+  - pattern: personnel
+  - pattern: police
+  - pattern: headquarters
+  - pattern: crossroads
+  - pattern: whereabouts
+  - pattern: savings
+  - pattern: sheep
+  - pattern: fish
+  - pattern: deer
+  - pattern: aircraft
+  - pattern: spacecraft
+  - pattern: offspring

package/lexicons/measure-unit.yaml

@@ -0,0 +1,78 @@
+# Unit symbols written after a measured number (1.5 mM, 37 °C, 2.5 mg/kg). A decimal at the start of a line followed
+# by one of these is an amount, not a dotted section number. Matched as written, and only when no letter, digit or
+# hyphen follows ("2.1 mmap" and "5.2.2 min-fresh" stay section titles). Symbols that begin with a capital letter
+# (Da, Pa, GB, W, V) are left out: a section title begins with a capital too, and "3.1 Da Vinci" is a title.
+id: measure-unit
+language: en
+entries:
+  - pattern: "mM"
+  - pattern: "µM"
+  - pattern: "μM"
+  - pattern: "nM"
+  - pattern: "pM"
+  - pattern: "mol"
+  - pattern: "mmol"
+  - pattern: "µmol"
+  - pattern: "μmol"
+  - pattern: "nmol"
+  - pattern: "pmol"
+  - pattern: "kg"
+  - pattern: "mg"
+  - pattern: "µg"
+  - pattern: "μg"
+  - pattern: "ng"
+  - pattern: "pg"
+  - pattern: "kDa"
+  - pattern: "mL"
+  - pattern: "ml"
+  - pattern: "µL"
+  - pattern: "μL"
+  - pattern: "µl"
+  - pattern: "μl"
+  - pattern: "dL"
+  - pattern: "nL"
+  - pattern: "km"
+  - pattern: "cm"
+  - pattern: "mm"
+  - pattern: "µm"
+  - pattern: "μm"
+  - pattern: "nm"
+  - pattern: "ms"
+  - pattern: "µs"
+  - pattern: "μs"
+  - pattern: "ns"
+  - pattern: "min"
+  - pattern: "sec"
+  - pattern: "hr"
+  - pattern: "hrs"
+  - pattern: "kHz"
+  - pattern: "mA"
+  - pattern: "mV"
+  - pattern: "kV"
+  - pattern: "kW"
+  - pattern: "kWh"
+  - pattern: "mAh"
+  - pattern: "kPa"
+  - pattern: "mmHg"
+  - pattern: "atm"
+  - pattern: "psi"
+  - pattern: "kB"
+  - pattern: "bp"
+  - pattern: "kb"
+  - pattern: "kbp"
+  - pattern: "ppm"
+  - pattern: "ppb"
+  - pattern: "rpm"
+  - pattern: "dB"
+  - pattern: "mCi"
+  - pattern: "µCi"
+  - pattern: "μCi"
+  - pattern: "mGy"
+  - pattern: "mSv"
+  - pattern: "°C"
+  - pattern: "°F"
+  - pattern: "℃"
+  - pattern: "℉"
+  - pattern: "°"
+  - pattern: "%"
+  - pattern: "‰"

package/lexicons/misnumbered-phrase.yaml

@@ -0,0 +1,7 @@
+# Phrases with a plural where English takes the singular: "remains ones of the most intriguing" is "one of the most".
+# Not counted after a determiner, an adjective or a pronoun, where "ones" is the pronoun ("the ones of the most use").
+id: misnumbered-phrase
+language: en
+entries:
+  - pattern: ones of the most
+  - pattern: ones of the least

package/lexicons/name-title.yaml

@@ -0,0 +1,20 @@
+# Official titles set before a surname printed in capitals: a hearing transcript's speakers (Senator HAWLEY., Secretary
+# BLINKEN.) and the Japanese government's romanised names, surname first (Prime Minister ABE Shinzo). A word in capitals
+# right after one of these is a name, not an acronym. "Minister" also covers Prime Minister and Foreign Minister, "President"
+# Vice President. Titles often followed by an acronym (Chair, General, Justice) are left out. Matched as written.
+id: name-title
+language: en
+entries:
+  - pattern: "President"
+  - pattern: "Minister"
+  - pattern: "Secretary"
+  - pattern: "Secretary-General"
+  - pattern: "Chief Justice"
+  - pattern: "Governor"
+  - pattern: "Senator"
+  - pattern: "Ambassador"
+  - pattern: "Mayor"
+  - pattern: "Chancellor"
+  - pattern: "Commissioner"
+  - pattern: "Chairman"
+  - pattern: "Chairwoman"

package/lexicons/numbered-label.yaml

@@ -0,0 +1,26 @@
+# Words written before a number to name an item of a series (Step 3, Example 4). At the head of a heading they are a
+# label, which the text below never repeats, so heading-echo does not count them in the overlap. position is the side
+# of the number the word is written on. Matched as written.
+id: numbered-label
+language: en
+entries:
+  - pattern: Step
+    position: before
+  - pattern: Example
+    position: before
+  - pattern: Case
+    position: before
+  - pattern: Exercise
+    position: before
+  - pattern: Question
+    position: before
+  - pattern: Problem
+    position: before
+  - pattern: Lesson
+    position: before
+  - pattern: Stage
+    position: before
+  - pattern: Figure
+    position: before
+  - pattern: Table
+    position: before

package/lexicons/pair-opener.yaml

@@ -0,0 +1,13 @@
+# Words that take exactly two things, each with the conjunction that joins its pair: "between Provider and Customer",
+# "both state and federal", "either email or post", "whether written or oral". When one stands in the item right
+# before its own conjunction, oxford-comma-consistency reads that conjunction as part of the item, not as the list's
+# last joint: "the Key Terms between Provider and Customer, and any policies". "between Acme, Beta and Gamma" is still
+# a list: "between" is not in the last item. "the impact of either option and implementation" is still a list: "either"
+# pairs with "or", not "and".
+id: pair-opener
+language: en
+entries:
+  - pattern: between and
+  - pattern: both and
+  - pattern: either or
+  - pattern: whether or

package/lexicons/percent-unit.yaml

@@ -0,0 +1,7 @@
+# The marks and words written as the unit of a percentage.
+id: percent-unit
+language: en
+entries:
+  - pattern: "%"
+  - pattern: percent
+  - pattern: per cent

package/lexicons/placeholder-word.yaml

@@ -0,0 +1,17 @@
+# What a template writes inside the brackets of a blank: [Your Name], [Insert Date], [Company Name].
+# A bracket counts when it holds one of these, alone or followed by more words.
+id: placeholder-word
+language: en
+entries:
+  - pattern: your
+  - pattern: insert
+  - pattern: recipient
+  - pattern: name
+  - pattern: company name
+  - pattern: client name
+  - pattern: project name
+  - pattern: product name
+  - pattern: date
+  - pattern: describe
+  - pattern: specific
+  - pattern: add

package/lexicons/plural-determiner.yaml

@@ -0,0 +1,10 @@
+# Determiners that count several things. A singular noun after one of these ("these new version") is a slip. They also
+# stand alone ("These help us"), so a noun is counted only when an adjective comes between.
+#
+# "one of the" is left out. It takes a plural ("one of the most important features"), but in the corpus it only ever
+# stood before a singular in "some one of the name of Cecily", which is right: "some one" is "someone".
+id: plural-determiner
+language: en
+entries:
+  - pattern: these
+  - pattern: those

package/lexicons/range-connector.yaml

@@ -0,0 +1,12 @@
+# Marks and words that, alone between two dates, make a period ("5 April – 9 April", "May 1 through May 5").
+# "to" is not one: "moved from March 10 to March 3" changes a date rather than spanning a period.
+id: range-connector
+language: en
+entries:
+  - pattern: "-"
+  - pattern: –
+  - pattern: —
+  - pattern: through
+  - pattern: thru
+  - pattern: until
+  - pattern: till

package/lexicons/share-exception.yaml

@@ -0,0 +1,10 @@
+# Words that mark a count whose percentages need not add up to 100%. When the header or the text above has one, the
+# percentages are not added.
+id: share-exception
+language: en
+entries:
+  - pattern: multiple answers
+  - pattern: multiple responses
+  - pattern: more than one
+  - pattern: select all
+  - pattern: all that apply

package/lexicons/share-label.yaml

@@ -0,0 +1,12 @@
+# Words that name the parts of one whole (they add up to 100%). When a table's header, or the sentence or heading right
+# above a list or a table, has one of these, its percentages are added and compared with 100%.
+# "rate" and "percentage" are left out: they also name a rate per row. So is a bare "share" ("share price return").
+id: share-label
+language: en
+entries:
+  - pattern: breakdown
+  - pattern: market share
+  - pattern: share of
+  - pattern: composition
+  - pattern: split
+  - pattern: allocation

package/lexicons/singular-determiner.yaml

@@ -0,0 +1,13 @@
+# Determiners that count one thing. A plural noun after one of these ("a significant changes", "each new users") is a
+# slip. "that" is left out: the tagger cannot tell the demonstrative from the conjunction ("so that users can").
+# Only the articles never stand alone; after the others a noun is counted only when an adjective comes between, because
+# "This results in a loss" is a pronoun and a verb, not a determiner and a noun.
+id: singular-determiner
+language: en
+entries:
+  - pattern: a
+  - pattern: an
+  - pattern: another
+  - pattern: each
+  - pattern: every
+  - pattern: this

package/lexicons/stock-transition.yaml

@@ -0,0 +1,17 @@
+# Sentence openers that glue without saying how. Words that carry the argument (However, Therefore, Thus) are not here.
+# "In addition," keeps its comma: "In addition to X" opens a phrase, not a transition.
+id: stock-transition
+language: en
+entries:
+  - pattern: Additionally
+  - pattern: Moreover
+  - pattern: Furthermore
+  - pattern: In addition,
+  - pattern: Notably
+  - pattern: Importantly
+  - pattern: Crucially
+  - pattern: Ultimately
+  - pattern: Overall,
+  - pattern: In conclusion
+  - pattern: In summary
+  - pattern: That said

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@chaffjs/lang-en",
-  "version": "0.14.0",
+  "version": "0.16.0",
   "description": "English language adapter for chaff",
   "license": "MIT",
   "author": "isamu",

package/src/citation.ts

@@ -52,17 +52,18 @@ export const listMembers = (rest: string, plural: boolean): ListMember[] =>
  */
 const TAG = "(?<tag>[A-Z][A-Z0-9]{1,30})";
 /**
- * "[HTTP-CACHING]" is written like a contract's placeholder "[BUYER-1]". Only the document tells them apart, by listing
- * the tag or not, so this tag is a candidate that the core checks against the document (attrs.citedTag).
+ * "[HTTP-CACHING]" is written like a contract's placeholder "[BUYER-1]", and a numbered citation "[19]" like a blank to
+ * fill in. Only the document tells them apart, by listing the tag or not, so this tag is a candidate that the core checks
+ * against the document (attrs.citedTag).
  */
-const HYPHENATED_TAG = "(?<tag>[A-Z][A-Z0-9]{0,30}(?:-[A-Z0-9]{1,30}){1,4})";
+const LISTED_TAG = "(?<tag>\\d{1,3}|[A-Z][A-Z0-9]{0,30}(?:-[A-Z0-9]{1,30}){1,4})";
 const tagAfter = (tag: string): RegExp => new RegExp(`^\\[${tag}\\]`, "u");
 /** "[HTTP], Section 12.1": the tag written just before the reference. */
 const tagBefore = (tag: string): RegExp => new RegExp(`\\[${tag}\\],?\\s?$`, "u");
 const TAG_AFTER = tagAfter(TAG);
 const TAG_BEFORE = tagBefore(TAG);
-const HYPHENATED_AFTER = tagAfter(HYPHENATED_TAG);
-const HYPHENATED_BEFORE = tagBefore(HYPHENATED_TAG);
+const LISTED_AFTER = tagAfter(LISTED_TAG);
+const LISTED_BEFORE = tagBefore(LISTED_TAG);
 const TAG_REACH = 40;
 
 const tagEndingAt = (pattern: RegExp, text: string, start: number): string | undefined =>
@@ -120,9 +121,12 @@ export const citedDocumentAfter = (text: string, end: number): string | undefine
   return words.length === 1 && SELF.has(name) ? undefined : name;
 };
 
-/** A hyphenated tag right after a reference ("Section 4.2.3 of [HTTP-CACHING]") or just before it ("[HTTP-CACHING], Section 4"). */
-export const hyphenatedTagAround = (text: string, start: number, end: number): string | undefined => {
+/**
+ * A tag that names another document only if this document lists it, right after a reference ("Section 4.2.3 of
+ * [HTTP-CACHING]", "Section 4.2.2.17 of [19]") or just before it ("[HTTP-CACHING], Section 4", "[23], Section 2.17").
+ */
+export const listedTagAround = (text: string, start: number, end: number): string | undefined => {
   const named = afterOf(text, end);
-  const following = named === undefined ? undefined : HYPHENATED_AFTER.exec(named)?.groups?.["tag"];
-  return following ?? tagEndingAt(HYPHENATED_BEFORE, text, start);
+  const following = named === undefined ? undefined : LISTED_AFTER.exec(named)?.groups?.["tag"];
+  return following ?? tagEndingAt(LISTED_BEFORE, text, start);
 };

package/src/code-citation.ts

@@ -0,0 +1,52 @@
+import type { Lexicon, LexiconEntry } from "chaffjs/plugin";
+import { escapeRegExp } from "./regexp.ts";
+
+// "35 CFR §122", "42 U.S.C. § 1983", "RFC 1122, Section 3.3.4.2": a section of a code or a numbered document named
+// right before the reference, not of this document. Which names are codes is the lexicon's (document-kind).
+
+/** Built once from the lexicon; each pattern is undefined when the lexicon names no code of its shape. */
+export type CodeVocabulary = {
+  readonly before: RegExp | undefined;
+  readonly numberedBefore: RegExp | undefined;
+  /** The name of a code that takes a title number, at the start of the text and ending at a word boundary. */
+  readonly titled: RegExp | undefined;
+};
+
+/** The code's title number ("35 CFR"), then the name, then at most a second "§" ("§§ 1981 and 1983") up to the reference. */
+const TITLE_NUMBER = String.raw`(?:\d{1,3}\s+)?`;
+const SECOND_SIGN = String.raw`\s*(?:§\s*)?$`;
+/** The document's own number after its name, then a comma, an opening parenthesis or a space up to the reference. */
+const OWN_NUMBER = String.raw`\s+\d{1,5}`;
+const NUMBER_TO_REFERENCE = String.raw`(?:,\s*|\s*\(\s*|\s+)(?:§\s*)?$`;
+const NOT_INSIDE_A_WORD = String.raw`(?<![\p{L}\p{N}_.])`;
+const WORD_ENDS = String.raw`(?![\p{L}\p{N}_])`;
+
+/** "35 CFR " is a few characters. Only this much before a reference is read, however long the line. */
+const REACH = 40;
+
+const namesOf = (entries: readonly LexiconEntry[]): string | undefined => {
+  const names = entries.map((entry) => entry.pattern).toSorted((left, right) => right.length - left.length);
+  return names.length === 0 ? undefined : `(?:${names.map(escapeRegExp).join("|")})`;
+};
+
+/** An entry with position "before" is a name written before its own number ("RFC 1122"); the others follow a title number. */
+export const codeVocabulary = (lexicons: Readonly<Record<string, Lexicon>>): CodeVocabulary => {
+  const entries = lexicons["document-kind"] ?? [];
+  const titled = namesOf(entries.filter((entry) => entry.position !== "before"));
+  const numbered = namesOf(entries.filter((entry) => entry.position === "before"));
+  return {
+    before: titled === undefined ? undefined : new RegExp(String.raw`${NOT_INSIDE_A_WORD}(?<code>${TITLE_NUMBER}${titled})${SECOND_SIGN}`, "u"),
+    numberedBefore:
+      numbered === undefined ? undefined : new RegExp(String.raw`${NOT_INSIDE_A_WORD}(?<code>${numbered}${OWN_NUMBER})${NUMBER_TO_REFERENCE}`, "u"),
+    titled: titled === undefined ? undefined : new RegExp(String.raw`^${titled}${WORD_ENDS}`, "u"),
+  };
+};
+
+/** The code named right before the reference at `start`, or undefined when none is. */
+export const citedCodeBefore = (text: string, start: number, vocabulary: CodeVocabulary): string | undefined => {
+  const before = text.slice(Math.max(0, start - REACH), start);
+  return vocabulary.before?.exec(before)?.groups?.["code"] ?? vocabulary.numberedBefore?.exec(before)?.groups?.["code"];
+};
+
+/** "40 CFR § 163.25" at the start of a heading: the text after the number starts with a code named after a title number. */
+export const titledCodeAt = (rest: string, vocabulary: CodeVocabulary): boolean => vocabulary.titled?.test(rest) === true;

package/src/dates.ts

@@ -3,7 +3,12 @@ import type { Mention } from "chaffjs/plugin";
 // Dates in English text, and the weekday written beside one.
 
 const MONTHS = ["january", "february", "march", "april", "may", "june", "july", "august", "september", "october", "november", "december"];
-const MONTH_WORD = /\b(?<month>[A-Z][a-z]{2,8})\b/gu;
+/** "Sep", "Sept": a month cut short, which may carry a period ("Sept. 12, 2025"). */
+const ABBREVIATED: ReadonlyMap<string, number> = new Map([...MONTHS.map((name, index) => [name.slice(0, 3), index + 1] as const), ["sept", 9]]);
+/** Title case or capitals ("SEP 01, 2022" on a web page): a lower-case "mar" or "may" is never a month. */
+const MONTH_WORD = /\b(?<month>[A-Z][a-z]{2,8}|[A-Z]{3,9})\b/gu;
+/** "12-SEP-2025", the day-month-year form of labels and logs. Only a three-letter month is written this way. */
+const HYPHENATED_DATE = /(?<![\w-])(?<d>\d{1,2})-(?<month>[A-Z][a-z]{2}|[A-Z]{3})-(?<y>\d{4})(?![\w-])/gu;
 const ISO_DATE = /\b(?<y>\d{4})-(?<m>\d{2})-(?<d>\d{2})\b/gu;
 const DAY_BEFORE = /(?<d>\d{1,2})(?:st|nd|rd|th)? $/u;
 const DAY_YEAR_AFTER = /^ (?<d>\d{1,2})(?:st|nd|rd|th)?,? (?<y>\d{4})\b/u;
@@ -41,12 +46,21 @@ const monthYear = (text: string, at: number, end: number, month: number): Mentio
  * The month is found first and its neighbours read with anchored patterns, never one long alternation.
  */
 const namedDate = (text: string, match: RegExpExecArray): Mention | undefined => {
-  const month = MONTHS.indexOf((match.groups?.["month"] ?? "").toLowerCase()) + 1;
+  const word = (match.groups?.["month"] ?? "").toLowerCase();
+  const full = MONTHS.indexOf(word) + 1;
+  const month = full === 0 ? (ABBREVIATED.get(word) ?? 0) : full;
   if (month === 0) return undefined;
-  const end = match.index + match[0].length;
+  const end = match.index + match[0].length + (full === 0 && text.charAt(match.index + match[0].length) === "." ? 1 : 0);
   return monthDayYear(text, match.index, end, month) ?? monthYear(text, match.index, end, month);
 };
 
+const hyphenatedDate = (match: RegExpExecArray): Mention[] => {
+  const month = ABBREVIATED.get((match.groups?.["month"] ?? "").toLowerCase());
+  if (month === undefined) return [];
+  const value = `${match.groups?.["y"] ?? ""}-${pad(String(month))}-${pad(match.groups?.["d"] ?? "")}`;
+  return [{ start: match.index, end: match.index + match[0].length, attrs: { value } }];
+};
+
 const isoDate = (match: RegExpExecArray): Mention => ({
   start: match.index,
   end: match.index + match[0].length,
@@ -88,6 +102,10 @@ const withWeekday = (text: string, date: Mention): Mention => {
 };
 
 export const dates = (text: string): Mention[] =>
-  [...[...text.matchAll(MONTH_WORD)].flatMap((match) => namedDate(text, match) ?? []), ...[...text.matchAll(ISO_DATE)].map(isoDate)]
+  [
+    ...[...text.matchAll(MONTH_WORD)].flatMap((match) => namedDate(text, match) ?? []),
+    ...[...text.matchAll(HYPHENATED_DATE)].flatMap(hyphenatedDate),
+    ...[...text.matchAll(ISO_DATE)].map(isoDate),
+  ]
     .toSorted((left, right) => left.start - right.start)
     .map((date) => withWeekday(text, date));

package/src/emphasis.ts

@@ -0,0 +1,16 @@
+// A word written in capitals for emphasis (the office will NEVER call you), told from an acronym by the dictionary.
+
+const CAPITALS = /^\p{Lu}{2,}$/u;
+
+const ADVERB_TAGS: ReadonlySet<string> = new Set(["RB", "RBR", "RBS"]);
+
+/**
+ * The dictionary knows the lower-case word, and only as an adverb (never, always, also). An acronym names a thing, so a word
+ * with no noun or adjective reading cannot be one. A word that is also a noun or an adjective (fast, eagle, cheese) is kept:
+ * FAST and EAGLE are a telescope and a simulation, and the context tagger cannot tell them from emphasis.
+ */
+export const isEmphasisedAdverb = (surface: string, tagsOf: (word: string) => readonly string[] | undefined): boolean => {
+  if (!CAPITALS.test(surface)) return false;
+  const tags = tagsOf(surface.toLowerCase()) ?? [];
+  return tags.length > 0 && tags.every((tag) => ADVERB_TAGS.has(tag));
+};

package/src/index.ts

@@ -1,14 +1,19 @@
 import { loadLexicons } from "./lexicons.ts";
 import { unmarkNumberStops } from "./number-stop.ts";
+import { labelStops, unmarkLabelStops } from "./label-stop.ts";
 import { sentenceSpans } from "./sentence-split.ts";
 import { splitAtQuotedStops } from "./quoted-stop.ts";
 import { reattachClosingQuotes } from "./closing-quote.ts";
 import { structure } from "./structure.ts";
 import { isReady, prepare, tokenize } from "./pos.ts";
-import type { AdapterNeeds, LanguageAdapter, Segmentation, Sentence } from "chaffjs/plugin";
+import { isJapaneseRun } from "./japanese-run.ts";
+import type { AdapterNeeds, EmbeddedLanguage, LanguageAdapter, Segmentation, Sentence, Span } from "chaffjs/plugin";
 
 // chaff からは型だけを取る。実行時の値依存を作らない。アダプタは単体で動く。
 
+const LEXICONS = loadLexicons();
+const LABEL_STOPS = labelStops((LEXICONS["abbreviated-label"] ?? []).map((entry) => entry.pattern));
+
 const LATIN_LETTER = /[a-z]/giu;
 const COUNTABLE = /\S/gu;
 
@@ -25,9 +30,13 @@ const withTokens = (sentence: Sentence): Sentence => {
   };
 };
 
+const JAPANESE: EmbeddedLanguage = { id: "ja", lengthUnit: "char" };
+
+const withLanguage = (text: string, span: Span): Sentence => (isJapaneseRun(text) ? { span, text, embeddedLanguage: JAPANESE } : { span, text });
+
 /**
  * 英語は sentence-splitter の既定にほぼ任せる。"Dr." "e.g." "U.S." "$3.50" を
- * いずれも文末と誤認しない。前処理は行の途中の番号を箇条書きと読ませること、後処理は閉じ引用符の内側で閉じた文を切ることと、文頭に取り残された閉じ引用符を前の文へ戻すこと。spec §7.2。
+ * いずれも文末と誤認しない。前処理は行の途中の番号を箇条書きと読ませることと、番号の前の略した名前（FIG. 1、Vol. XLIII）の点で切らないこと、後処理は閉じ引用符の内側で閉じた文を切ることと、文頭に取り残された閉じ引用符を前の文へ戻すこと。spec §7.2。
  */
 export const adapter: LanguageAdapter = {
   kind: "language",
@@ -51,11 +60,11 @@ export const adapter: LanguageAdapter = {
     if (total === 0) return 0;
     return [...source.matchAll(LATIN_LETTER)].length / total;
   },
-  lexicons: loadLexicons(),
+  lexicons: LEXICONS,
   structure,
   segment: (text: string): Segmentation => {
-    const quotedStops = sentenceSpans(unmarkNumberStops(text)).flatMap((span) => splitAtQuotedStops(text, span));
-    const sentences: Sentence[] = reattachClosingQuotes(text, quotedStops).map((span) => ({ span, text: text.slice(span.start, span.end) }));
+    const quotedStops = sentenceSpans(unmarkNumberStops(unmarkLabelStops(text, LABEL_STOPS))).flatMap((span) => splitAtQuotedStops(text, span));
+    const sentences: Sentence[] = reattachClosingQuotes(text, quotedStops).map((span) => withLanguage(text.slice(span.start, span.end), span));
     return { sentences: isReady() ? sentences.map(withTokens) : sentences };
   },
 };

package/src/japanese-run.ts

@@ -0,0 +1,9 @@
+/**
+ * A Japanese sentence in an English document (a quoted notice, a bilingual title): it has kana, and kana or kanji make
+ * up at least half of its letters (digits and symbols are not counted). Kanji alone do not count, as a Chinese name in English text is not Japanese.
+ */
+const KANA = /[\p{Script=Hiragana}\p{Script=Katakana}]/u;
+const JAPANESE = /[\p{Script=Hiragana}\p{Script=Katakana}\p{Script=Han}]/gu;
+const LETTER = /\p{L}/gu;
+
+export const isJapaneseRun = (text: string): boolean => KANA.test(text) && [...text.matchAll(JAPANESE)].length * 2 >= [...text.matchAll(LETTER)].length;

package/src/label-stop.ts

@@ -0,0 +1,25 @@
+import { escapeRegExp } from "./regexp.ts";
+
+/**
+ * sentence-splitter ends a sentence at the full stop of a label before its number: "FIG." and "1 illustrates …" were two
+ * sentences, and so were "Vol." and "XLIII (1979)". Where a number follows, that full stop is replaced with a letter
+ * before splitting. The length does not change, so the spans fit the original text.
+ * A lone "I" after a label is the pronoun ("He said No. I left."), and a word ("the last Fig. The tree") is not a number.
+ */
+
+/** Built once from the lexicon; undefined when no label ends in a full stop. */
+export type LabelStops = RegExp | undefined;
+
+const NUMBER_AFTER = String.raw`(?=\s+(?:\d|(?:[IVXLCDM]{2,}|[VXLCDM])(?![\p{L}\p{N}_])))`;
+const PLAIN_LETTER = "n";
+
+/** The labels as written and in capitals, without their full stop ("Fig." → Fig, FIG). */
+export const labelStops = (labels: readonly string[]): LabelStops => {
+  const stems = labels.filter((label) => label.endsWith(".")).flatMap((label) => [label.slice(0, -1), label.slice(0, -1).toUpperCase()]);
+  if (stems.length === 0) return undefined;
+  return new RegExp(String.raw`(?<![\p{L}\p{N}_.])(?:${[...new Set(stems)].map(escapeRegExp).join("|")})\.${NUMBER_AFTER}`, "gu");
+};
+
+/** The text with the full stop of every label before a number replaced by a letter. */
+export const unmarkLabelStops = (text: string, stops: LabelStops): string =>
+  stops === undefined ? text : text.replace(stops, (label: string) => `${label.slice(0, -1)}${PLAIN_LETTER}`);