@chaffjs/lang-en 0.12.0 → 0.14.0

package/lexicons/fixed-phrase.yaml

@@ -0,0 +1,14 @@
+# Fixed phrases, mostly borrowed, that open with a word the tagger reads as an article and act as one adjective or
+# adverb: "the a priori approach", "an a fortiori case", "the a la carte menu". The "a" belongs to the phrase, so an
+# article before it is not a doubled article. The tagger gives no sign of a foreign word here ("priori" is a noun to
+# it), so the phrases are listed.
+id: fixed-phrase
+language: en
+entries:
+  - pattern: a priori
+  - pattern: a posteriori
+  - pattern: a fortiori
+  - pattern: a la carte
+  - pattern: a la mode
+  - pattern: a cappella
+  - pattern: a capella

package/lexicons/hedge-frame.yaml

@@ -0,0 +1,9 @@
+# Words that soften a claim without hedging it on their own: "may" is often permission or plain possibility.
+# Next to a hedge in the same sentence they stack ("may possibly", "it could perhaps be argued").
+# Used only by excessive-hedging's count of hedges in one sentence, never by its density.
+id: hedge-frame
+language: en
+entries:
+  - pattern: may
+  - pattern: might
+  - pattern: could

package/lexicons/hedge-scope.yaml

@@ -0,0 +1,7 @@
+# Entries of excessive-hedging that say where a claim holds, not how sure the writer is.
+# "In some cases, the API may accept the key as a parameter" states a fact about some cases.
+# They count towards the density, but never towards hedges stacked in one sentence.
+id: hedge-scope
+language: en
+entries:
+  - pattern: in some cases

package/lexicons/honorific.yaml

@@ -0,0 +1,15 @@
+# Honorifics a transcript sets before a speaker's surname, which it prints in capitals (Mr. HAWLEY., Mrs. CAPITO.,
+# Mr BLAKE in British style). A word in capitals right after one of these is a name, not an acronym. Matched as written.
+id: honorific
+language: en
+entries:
+  - pattern: "Mr."
+  - pattern: "Mrs."
+  - pattern: "Ms."
+  - pattern: "Dr."
+  - pattern: "Mr"
+  - pattern: "Mrs"
+  - pattern: "Ms"
+  - pattern: "Dr"
+  - pattern: "Miss"
+  - pattern: "Madam"

package/lexicons/http-method.yaml

@@ -0,0 +1,14 @@
+# Names of HTTP request methods (RFC 9110, and PATCH from RFC 5789). They are written in capitals but are words, not
+# abbreviations, so there is nothing to expand. Matched as written.
+id: http-method
+language: en
+entries:
+  - pattern: GET
+  - pattern: HEAD
+  - pattern: POST
+  - pattern: PUT
+  - pattern: DELETE
+  - pattern: CONNECT
+  - pattern: OPTIONS
+  - pattern: TRACE
+  - pattern: PATCH

package/lexicons/participle-word.yaml

@@ -0,0 +1,8 @@
+# Words that open a participle phrase after a comma but that the tagger reads as another part of speech:
+# "It comes from Latin, meaning ship or boat." oxford-comma-consistency reads them as participles, so the phrase
+# they open is not an item of the list before it. Words the tagger already reads as -ing verbs (including,
+# regarding) need no entry.
+id: participle-word
+language: en
+entries:
+  - pattern: meaning

package/lexicons/quantity-noun.yaml

@@ -0,0 +1,6 @@
+# Nouns that name a measured quantity. A superlative joined to such a noun with no space names the quantity (最大風速
+# in Japanese) rather than claiming anything is the greatest. English writes a space between a superlative and its noun,
+# so there are none; the list exists so both languages name the same lists.
+id: quantity-noun
+language: en
+entries: []

package/lexicons/relative-word.yaml

@@ -0,0 +1,10 @@
+# Words that open a relative clause after a noun ("the best option which we chose", "the threats that we have seen").
+# A superlative whose noun phrase such a clause restricts says what it is the most of.
+# A language without this list puts no clause after a noun, and unqualified-superlative reads no clause or participle there.
+id: relative-word
+language: en
+entries:
+  - pattern: that
+  - pattern: which
+  - pattern: who
+  - pattern: whom

package/lexicons/subject-pronoun.yaml

@@ -0,0 +1,13 @@
+# Pronouns that can be the subject of a clause ("the best we have measured", "the best option they could find").
+# A reflexive or emphatic pronoun right after a noun ("The best players themselves get a bonus") is not a subject.
+# A language without this list reads no clause that starts with a pronoun after a superlative.
+id: subject-pronoun
+language: en
+entries:
+  - pattern: i
+  - pattern: you
+  - pattern: he
+  - pattern: she
+  - pattern: it
+  - pattern: we
+  - pattern: they

package/lexicons/superlative-amount.yaml

@@ -0,0 +1,7 @@
+# Superlatives that also name an amount. With a noun right after them and nothing else joined to that noun ("the most
+# work", "the most students", "for the most part"), they say which has the largest amount, not that anything is best.
+# A language without this list reads no superlative as an amount.
+id: superlative-amount
+language: en
+entries:
+  - pattern: the most

package/lexicons/superlative-bound.yaml

@@ -0,0 +1,8 @@
+# Adjectives that bound a superlative to what can be had: right after it ("the best possible outcome") or after its noun
+# ("the best education possible", "the best evidence available"). The superlative is the most of what is possible.
+# A language without this list reads no superlative as bounded.
+id: superlative-bound
+language: en
+entries:
+  - pattern: possible
+  - pattern: available

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@chaffjs/lang-en",
-  "version": "0.12.0",
+  "version": "0.14.0",
   "description": "English language adapter for chaff",
   "license": "MIT",
   "author": "isamu",

package/src/apostrophe.ts

@@ -0,0 +1,8 @@
+/**
+ * 解析器（wink）は ' しかアポストロフィと読まず、that’s を that / ’ / s に割る。’ を ' に置き換えてから渡す。
+ * ’ は閉じの一重引用符でもあるので、字と字に挟まれたとき（don’t, team’s, 1990’s）だけ置き換える。閉じの引用符の後ろに字は続かない。
+ * ʼ（U+02BC）は引用符には使わないので、いつも置き換える。一字を一字に置き換えるので、位置は本文のまま。
+ */
+const APOSTROPHE = /(?<=[\p{L}\p{N}])’(?=\p{L})|ʼ/gu;
+
+export const straightApostrophes = (text: string): string => text.replace(APOSTROPHE, "'");

package/src/citation.ts

@@ -4,8 +4,11 @@ const OF = /^,? of (?:the |that |those )?/u;
 /** "section 4(2)(a) (exception to liability …) of the Damages (Scotland) Act 2011": the gloss sits between the number and the name. */
 const GLOSS = /^ \([^()]{1,100}\)/u;
 const CONNECTOR = /^(?:,? (?:to|and|or)|,) /u;
-/** "9", "29(2)", "(g)", and a roman "V" for a list of Articles. The roman numeral must end the word: "VIII", not "Vendor". */
-const LISTED_NUMBERS = [/^\d{1,3}[A-Z]{0,2}(?:\([a-z0-9]{1,4}\))*/u, /^(?:\([a-z0-9]{1,4}\))+/u, /^[IVXLC]{1,7}\b/u];
+/**
+ * "9", "29(2)", "(g)", and a roman "V" for a list of Articles. The roman numeral must end the word: "VIII", not "Vendor".
+ * A dotted "310.5" is not read: its "310" alone is not the section the list names.
+ */
+const LISTED_NUMBERS = [/^\d{1,3}(?!\.?\d)[A-Z]{0,2}(?:\([a-z0-9]{1,4}\))*/u, /^(?:\([a-z0-9]{1,4}\))+/u, /^[IVXLC]{1,7}\b/u];
 
 const listedNumber = (text: string): string | undefined => LISTED_NUMBERS.map((pattern) => pattern.exec(text)?.[0]).find((found) => found !== undefined);
 
@@ -48,13 +51,25 @@ export const listMembers = (rest: string, plural: boolean): ListMember[] =>
  * word of a contract ("[Company]", "[Reserved]") is not taken for another document and still has to exist here.
  */
 const TAG = "(?<tag>[A-Z][A-Z0-9]{1,30})";
-const TAG_AFTER = new RegExp(`^\\[${TAG}\\]`, "u");
+/**
+ * "[HTTP-CACHING]" is written like a contract's placeholder "[BUYER-1]". Only the document tells them apart, by listing
+ * the tag or not, so this tag is a candidate that the core checks against the document (attrs.citedTag).
+ */
+const HYPHENATED_TAG = "(?<tag>[A-Z][A-Z0-9]{0,30}(?:-[A-Z0-9]{1,30}){1,4})";
+const tagAfter = (tag: string): RegExp => new RegExp(`^\\[${tag}\\]`, "u");
 /** "[HTTP], Section 12.1": the tag written just before the reference. */
-const TAG_BEFORE = new RegExp(`\\[${TAG}\\],?\\s?$`, "u");
+const tagBefore = (tag: string): RegExp => new RegExp(`\\[${tag}\\],?\\s?$`, "u");
+const TAG_AFTER = tagAfter(TAG);
+const TAG_BEFORE = tagBefore(TAG);
+const HYPHENATED_AFTER = tagAfter(HYPHENATED_TAG);
+const HYPHENATED_BEFORE = tagBefore(HYPHENATED_TAG);
+const TAG_REACH = 40;
+
+const tagEndingAt = (pattern: RegExp, text: string, start: number): string | undefined =>
+  pattern.exec(text.slice(Math.max(0, start - TAG_REACH), start))?.groups?.["tag"];
 
 /** The document cited by a tag just before a reference, as in "see [HTTP], Section 12.1". */
-export const citedDocumentBefore = (text: string, start: number): string | undefined =>
-  TAG_BEFORE.exec(text.slice(Math.max(0, start - 40), start))?.groups?.["tag"];
+export const citedDocumentBefore = (text: string, start: number): string | undefined => tagEndingAt(TAG_BEFORE, text, start);
 
 const CAPITALISED = /^[A-Z][\w'’-]*/u;
 /**
@@ -81,20 +96,33 @@ const titleWords = (rest: string): string[] => {
   return words;
 };
 
+/** What follows the "of" after a reference and its list, or undefined when no "of" follows. */
+const afterOf = (text: string, end: number): string | undefined => {
+  const rest = text.slice(end, end + 200);
+  const afterList = listEnd(rest);
+  const afterGloss = afterList + (GLOSS.exec(rest.slice(afterList))?.[0].length ?? 0);
+  const of = OF.exec(rest.slice(afterGloss));
+  return of === null ? undefined : rest.slice(afterGloss + of[0].length);
+};
+
 /**
  * The document named right after a reference, or undefined when the reference is into this document.
  * "of this Agreement" and "of the Agreement" are this document; "of the Master Agreement" is another.
  */
 export const citedDocumentAfter = (text: string, end: number): string | undefined => {
-  const rest = text.slice(end, end + 200);
-  const afterList = listEnd(rest);
-  const listed = afterList + (GLOSS.exec(rest.slice(afterList))?.[0].length ?? 0);
-  const of = OF.exec(rest.slice(listed));
-  if (of === null) return undefined;
-  const tag = TAG_AFTER.exec(rest.slice(listed + of[0].length))?.groups?.["tag"];
+  const named = afterOf(text, end);
+  if (named === undefined) return undefined;
+  const tag = TAG_AFTER.exec(named)?.groups?.["tag"];
   if (tag !== undefined) return tag;
-  const words = titleWords(rest.slice(listed + of[0].length));
+  const words = titleWords(named);
   if (words.length === 0) return undefined;
   const name = words.join(" ");
   return words.length === 1 && SELF.has(name) ? undefined : name;
 };
+
+/** A hyphenated tag right after a reference ("Section 4.2.3 of [HTTP-CACHING]") or just before it ("[HTTP-CACHING], Section 4"). */
+export const hyphenatedTagAround = (text: string, start: number, end: number): string | undefined => {
+  const named = afterOf(text, end);
+  const following = named === undefined ? undefined : HYPHENATED_AFTER.exec(named)?.groups?.["tag"];
+  return following ?? tagEndingAt(HYPHENATED_BEFORE, text, start);
+};

package/src/closing-quote.ts

@@ -0,0 +1,52 @@
+import type { Span } from "chaffjs/plugin";
+import { NEXT_SENTENCE_HEAD } from "./quoted-stop.ts";
+
+/**
+ * sentence-splitter は疑問符・感嘆符の後の曲がった閉じ引用符（“Is it done?” Nobody …）と直線の一重引用符の手前で文を切る。
+ * 次の文が「”」で始まり、前の文は引用符が開いたまま終わる。
+ *
+ * 前の文が句点で終わり、次の文がそこから間を置かずに閉じ引用符（閉じ括弧も）で始まるとき、それを前の文の末尾へ戻す。
+ * 戻した後に空白と大文字が続けば文は二つのまま、何も続かなければ前の文だけ、
+ * 小文字・数字・句読点が続けば引用は文の途中にあるので一つの文につなぐ（直線の二重引用符で分割器がそうするのと同じ）。
+ * 開き引用符（“ ‘）は閉じの分類に入らない。語頭のアポストロフィ（’Tis ’90s）は後ろに字が続くので閉じと読まない。
+ */
+const CLOSING_RUN = /^[\p{Pf}\p{Pe}"']+/u;
+const WORD_CHARACTER = /^[\p{L}\p{N}]/u;
+const STOP_AT_END = /[.?!]$/u;
+// 語頭のアポストロフィの後の大文字（’Tis）も次の文の頭。
+const NEXT_SENTENCE = new RegExp(String.raw`^(?:${NEXT_SENTENCE_HEAD}|\s+’\p{Lu})`, "u");
+const LEADING_SPACE = /^\s*/u;
+
+/** 文頭の閉じ引用符の並びの長さ。後ろに字が続けば開き引用符かアポストロフィなので 0。 */
+export const closingRunLength = (sentence: string): number => {
+  const run = CLOSING_RUN.exec(sentence)?.[0].length ?? 0;
+  return run > 0 && !WORD_CHARACTER.test(sentence.slice(run)) ? run : 0;
+};
+
+const follows = (text: string, previous: Span, span: Span): boolean =>
+  previous.end === span.start && STOP_AT_END.test(text.slice(previous.start, previous.end));
+
+/** previous と span を置き換える文。閉じ引用符が戻らなければ undefined。 */
+const reattached = (text: string, previous: Span, span: Span): Span[] | undefined => {
+  const run = follows(text, previous, span) ? closingRunLength(text.slice(span.start, span.end)) : 0;
+  if (run === 0) return undefined;
+  const quoteEnd = span.start + run;
+  const rest = text.slice(quoteEnd, span.end);
+  if (rest.trim() === "") return [{ start: previous.start, end: quoteEnd }];
+  if (!NEXT_SENTENCE.test(rest)) return [{ start: previous.start, end: span.end }];
+  const next = quoteEnd + (LEADING_SPACE.exec(rest)?.[0].length ?? 0);
+  return [
+    { start: previous.start, end: quoteEnd },
+    { start: next, end: span.end },
+  ];
+};
+
+/** 文の span の並び（text 上の位置）で、文頭に取り残された閉じ引用符を前の文へ戻したもの。 */
+export const reattachClosingQuotes = (text: string, spans: readonly Span[]): Span[] =>
+  spans.reduce<Span[]>((sentences, span) => {
+    const previous = sentences.at(-1);
+    const replaced = previous === undefined ? undefined : reattached(text, previous, span);
+    if (replaced === undefined) sentences.push(span);
+    else sentences.splice(-1, 1, ...replaced);
+    return sentences;
+  }, []);

package/src/dates.ts

@@ -89,5 +89,5 @@ const withWeekday = (text: string, date: Mention): Mention => {
 
 export const dates = (text: string): Mention[] =>
   [...[...text.matchAll(MONTH_WORD)].flatMap((match) => namedDate(text, match) ?? []), ...[...text.matchAll(ISO_DATE)].map(isoDate)]
-    .sort((left, right) => left.start - right.start)
+    .toSorted((left, right) => left.start - right.start)
     .map((date) => withWeekday(text, date));

package/src/index.ts

@@ -1,6 +1,8 @@
 import { loadLexicons } from "./lexicons.ts";
 import { unmarkNumberStops } from "./number-stop.ts";
 import { sentenceSpans } from "./sentence-split.ts";
+import { splitAtQuotedStops } from "./quoted-stop.ts";
+import { reattachClosingQuotes } from "./closing-quote.ts";
 import { structure } from "./structure.ts";
 import { isReady, prepare, tokenize } from "./pos.ts";
 import type { AdapterNeeds, LanguageAdapter, Segmentation, Sentence } from "chaffjs/plugin";
@@ -25,7 +27,7 @@ const withTokens = (sentence: Sentence): Sentence => {
 
 /**
  * 英語は sentence-splitter の既定にほぼ任せる。"Dr." "e.g." "U.S." "$3.50" を
- * いずれも文末と誤認しない。前処理は、行の途中の番号を箇条書きと読ませないことだけ。spec §7.2。
+ * いずれも文末と誤認しない。前処理は行の途中の番号を箇条書きと読ませること、後処理は閉じ引用符の内側で閉じた文を切ることと、文頭に取り残された閉じ引用符を前の文へ戻すこと。spec §7.2。
  */
 export const adapter: LanguageAdapter = {
   kind: "language",
@@ -52,7 +54,8 @@ export const adapter: LanguageAdapter = {
   lexicons: loadLexicons(),
   structure,
   segment: (text: string): Segmentation => {
-    const sentences: Sentence[] = sentenceSpans(unmarkNumberStops(text)).map((span) => ({ span, text: text.slice(span.start, span.end) }));
+    const quotedStops = sentenceSpans(unmarkNumberStops(text)).flatMap((span) => splitAtQuotedStops(text, span));
+    const sentences: Sentence[] = reattachClosingQuotes(text, quotedStops).map((span) => ({ span, text: text.slice(span.start, span.end) }));
     return { sentences: isReady() ? sentences.map(withTokens) : sentences };
   },
 };

package/src/item-style.ts

@@ -0,0 +1,130 @@
+import type { NumberedLine, NumberingContext } from "chaffjs/plugin";
+import { parseRoman } from "./roman.ts";
+
+/**
+ * An amendment inserts a subsection between two others and numbers it "(A1)" or "(2A)". It is a subsection, written
+ * outside the sequence: it has no ordinal, so "(1)" after "(A1)" is still the first.
+ */
+export const INSERTED = "\\d{1,3}[A-Z]{1,2}|[A-Z]{1,2}\\d{1,3}";
+const IS_INSERTED = new RegExp(`^(?:${INSERTED})$`, "u");
+const MULTI_ROMAN = /^(?:ii|iii|iv|vi|vii|viii|ix)$/u;
+const AMBIGUOUS = /^[ivx]$/u;
+const DIGITS = /^\d+$/u;
+const CAPITALS = /^[A-Z]+$/u;
+const ONE_CAPITAL = /^[A-Z]$/u;
+
+export type Style = "letter" | "roman" | "digit" | "capital" | "capital-roman";
+
+/** 大文字で書いた同じ並び。"(B)" は英字の、"(II)" はローマ数字の大文字。 */
+const CAPITAL_OF: Readonly<Record<"letter" | "roman", Style>> = { letter: "capital", roman: "capital-roman" };
+
+/** 小文字にした番号を形で読む。一文字の "i" は、開いたときに付けた並びの位置で見分ける。ローマ数字なら 1、英字なら 9。 */
+const shapeOf = (lower: string, ordinal: number | undefined): "letter" | "roman" => {
+  if (MULTI_ROMAN.test(lower)) return "roman";
+  return AMBIGUOUS.test(lower) && ordinal === parseRoman(lower) ? "roman" : "letter";
+};
+
+/** 開いている項目の書き方。"(ii)" はローマ数字、"(b)" は英字、"(B)" は大文字の英字。 */
+export const styleOfOpen = (open: NumberedLine): Style | undefined => {
+  const inner = /^\((?<n>[A-Za-z0-9]{1,5})\)$/u.exec(open.label)?.groups?.["n"];
+  if (inner === undefined) return undefined;
+  if (DIGITS.test(inner) || IS_INSERTED.test(inner)) return "digit";
+  const shape = shapeOf(inner.toLowerCase(), open.ordinal);
+  return CAPITALS.test(inner) ? CAPITAL_OF[shape] : shape;
+};
+
+const LETTER_BEFORE_A = "a".charCodeAt(0) - 1;
+
+/** "(b)" と "(B)" は 2 番目、"(ii)" と "(II)" も 2 番目。二文字以上の英字（"(aa)"）は並びが決まらないので付けない。 */
+export const ordinalOf = (raw: string, style: Style): number | undefined => {
+  if (style === "digit") return IS_INSERTED.test(raw) ? undefined : Number(raw);
+  if (style === "roman" || style === "capital-roman") return parseRoman(raw);
+  return raw.length === 1 ? raw.toLowerCase().charCodeAt(0) - LETTER_BEFORE_A : undefined;
+};
+
+const styles = (context: NumberingContext): (Style | undefined)[] => context.open.map(styleOfOpen);
+
+/** "(h)" の次の "(i)"、"(H)" の次の "(I)" は英字。同じ書き方で開いている英字の次の文字なら、ローマ数字とは読まない。 */
+const follows = (raw: string, style: "letter" | "capital", context: NumberingContext): boolean =>
+  context.open.some((open, index) => styles(context)[index] === style && open.number.charCodeAt(0) + 1 === raw.charCodeAt(0));
+
+/**
+ * 英字の並びは (a) から始まるので、"(a)" や "(1)" のすぐ下に来た "(i)" は一段深いローマ数字。
+ * 米国の規則は (a)(1)(i) の順に下る。見出しのすぐ下の "(i)" は、どちらとも決まらないので英字。
+ */
+const OPENS_ROMAN: ReadonlySet<Style | undefined> = new Set(["letter", "digit"]);
+
+/** "(I)" は米国の法典で (i) の一段下の大文字ローマ数字。"(H)" の次なら英字。 */
+const capitalRoman = (raw: string, open: readonly (Style | undefined)[], context: NumberingContext): boolean => {
+  const lower = raw.toLowerCase();
+  if (MULTI_ROMAN.test(lower)) return open.includes("capital-roman");
+  if (!AMBIGUOUS.test(lower) || follows(raw, "capital", context)) return false;
+  return open.includes("capital-roman") || (raw === "I" && open.at(-1) === "roman");
+};
+
+/**
+ * 米国の規則は (a)(1)(i)(A) と下る。大文字の "(A)" は、開いているローマ数字のすぐ下で始まるか、開いている大文字の続きのときだけ項目。
+ * 英国の法令や契約書が一段目に使う "(A)" は、何の下とも決まらないので本文のまま読む。
+ */
+const capitalStyleOf = (raw: string, context: NumberingContext): Style | undefined => {
+  const open = styles(context);
+  if (capitalRoman(raw, open, context)) return "capital-roman";
+  if (!ONE_CAPITAL.test(raw)) return undefined;
+  return open.includes("capital") || (raw === "A" && open.at(-1) === "roman") ? "capital" : undefined;
+};
+
+/**
+ * "(i)" is a roman numeral right under "(a)" or "(1)", or when a roman list is already open; the letter i otherwise.
+ * A capital "(A)" is an item only where a US regulation puts it, under a roman item; undefined elsewhere.
+ */
+export const styleOf = (raw: string, context: NumberingContext): Style | undefined => {
+  if (DIGITS.test(raw) || IS_INSERTED.test(raw)) return "digit";
+  if (CAPITALS.test(raw)) return capitalStyleOf(raw, context);
+  if (MULTI_ROMAN.test(raw)) return "roman";
+  if (!AMBIGUOUS.test(raw) || follows(raw, "letter", context)) return "letter";
+  const open = styles(context);
+  return open.includes("roman") || OPENS_ROMAN.has(open.at(-1)) ? "roman" : "letter";
+};
+
+/** The nearest open item written the same way: "(b)" closes "(i)" and sits beside "(a)". */
+const siblingOf = (style: Style, context: NumberingContext): NumberedLine | undefined => {
+  const open = styles(context);
+  return context.open.findLast((_item, index) => open[index] === style);
+};
+
+const IS_CAPITAL: ReadonlySet<Style | undefined> = new Set(["capital", "capital-roman"]);
+
+/**
+ * "(A)" の下の "(1)" や "(i)"、"(i)" の下の "(A)" は、上に開いている同じ書き方の項目の続きではなく、一段深い並びの始まり。
+ * 米国の規則は (a)(1)(i)(A) の下を、もう一度 (1) や (i) で数える。
+ */
+const startsBelow = (style: Style, ordinal: number | undefined, innermost: Style | undefined): boolean =>
+  ordinal === 1 && style !== innermost && (IS_CAPITAL.has(innermost) || (IS_CAPITAL.has(style) && innermost === "roman"));
+
+/**
+ * A sibling has the depth of the open item written the same way. A new way of numbering goes one deeper than whatever
+ * is open, and so does a first item right under a capital, or a first capital right under a roman item.
+ */
+export const depthFor = (style: Style, context: NumberingContext, ordinal?: number): number => {
+  const innermost = context.open.at(-1);
+  const deeper = (innermost?.depth ?? 0) + 1;
+  if (innermost !== undefined && startsBelow(style, ordinal, styleOfOpen(innermost))) return deeper;
+  return siblingOf(style, context)?.depth ?? deeper;
+};
+
+/**
+ * 参照の "(A)" や "(I)" は、ローマ数字の "(iii)" の次に書かれたときだけ番地の続き。木も、大文字はローマ数字の下でしか読まない。
+ * "section 5(A)" の "(A)" は番地に入れず、これまでどおり "section 5" を引く。"(h)(i)" や番号のすぐ後の "(i)" は、木と同じく英字と読む。
+ */
+export const continuesAddress = (part: string, before: readonly string[]): boolean => {
+  if (!CAPITALS.test(part)) return true;
+  const [previous, earlier] = [before.at(-1) ?? "", before.at(-2) ?? ""];
+  if (MULTI_ROMAN.test(previous)) return true;
+  return AMBIGUOUS.test(previous) && earlier !== "" && earlier.charCodeAt(0) + 1 !== previous.charCodeAt(0);
+};
+
+/** 番号だけの行 "(5)" は、開いている "(4)" の次のときだけ項目。本文は次の行から始まる。 */
+export const continuesOpen = (style: Style, ordinal: number | undefined, context: NumberingContext): boolean => {
+  const previous = siblingOf(style, context)?.ordinal;
+  return ordinal !== undefined && previous !== undefined && ordinal === previous + 1;
+};

package/src/lexicons.ts

@@ -27,11 +27,12 @@ const toEntry = (raw: unknown): LexiconEntry | undefined => {
  * 新しい言語のサポートは、ここを書くところから始まる。
  */
 export const loadLexicons = (dir: string = DIR): Record<string, Lexicon> =>
-  readdirSync(dir)
-    .filter((file) => file.endsWith(".yaml"))
-    .reduce<Record<string, Lexicon>>((acc, file) => {
-      const raw: unknown = parse(readFileSync(join(dir, file), "utf8"));
-      if (!isRecord(raw) || typeof raw["id"] !== "string" || !Array.isArray(raw["entries"])) return acc;
-      const entries = raw["entries"].map(toEntry).filter((entry) => entry !== undefined);
-      return { ...acc, [raw["id"]]: entries };
-    }, {});
+  Object.fromEntries(
+    readdirSync(dir)
+      .filter((file) => file.endsWith(".yaml"))
+      .flatMap((file): [string, Lexicon][] => {
+        const raw: unknown = parse(readFileSync(join(dir, file), "utf8"));
+        if (!isRecord(raw) || typeof raw["id"] !== "string" || !Array.isArray(raw["entries"])) return [];
+        return [[raw["id"], raw["entries"].map(toEntry).filter((entry) => entry !== undefined)]];
+      }),
+  );

package/src/long-runs.ts

@@ -0,0 +1,6 @@
+/**
+ * 空白の無いまま limit 文字を超える並びを、同じ長さの空白で覆う。語ではなく（base64、ハッシュ、区切りの無い記号の列）、
+ * 解析器（wink）はその長さの二乗で遅くなる。長さを変えないので、残りの語の位置はそのまま。
+ */
+export const blankLongRuns = (text: string, limit: number): string =>
+  text.replace(new RegExp(`\\S{${String(limit + 1)},}`, "gu"), (run) => " ".repeat(run.length));

package/src/pos.ts

@@ -1,7 +1,9 @@
 import { createRequire } from "node:module";
 import type { Token } from "chaffjs/plugin";
+import { straightApostrophes } from "./apostrophe.ts";
 import { loadLexicons } from "./lexicons.ts";
-import { properNounChecked } from "./proper-noun.ts";
+import { blankLongRuns } from "./long-runs.ts";
+import { lowercasedAt, properNounChecked, rereadAt, sentenceInitialCommonWord } from "./proper-noun.ts";
 import { isStativeParticiple, stativeVocabulary } from "./stative-participle.ts";
 
 const require = createRequire(import.meta.url);
@@ -157,18 +159,43 @@ const determinerFeatures = (entry: Tagged): Features => {
   return entry.pos === "DT" && ARTICLES.has(entry.value.toLowerCase()) ? { features: { PronType: "Art" } } : {};
 };
 
-/** 過去分詞は VerbForm=Part。Based on the review, のような分詞の導入句を、命令形の並び（fix the parser, ship it）と見分ける。 */
+/** 複数形の名詞は UD の Number=Plur。数の語がそれを数えていれば（five minutes）、one of のような言い回しではなく量。 */
+const PLURAL_TAG = new Set(["NNS", "NNPS"]);
+
+const nounOrDeterminerFeatures = (entry: Tagged): Features => (PLURAL_TAG.has(entry.pos) ? { features: { Number: "Plur" } } : determinerFeatures(entry));
+
+/**
+ * 過去分詞は VerbForm=Part。Based on the review, のような分詞の導入句を、命令形の並び（fix the parser, ship it）と見分ける。
+ * -ing 形は VerbForm=Ger。解析器は動名詞と現在分詞を分けないので、過去分詞を見る判断（Part）には混ぜない。
+ */
 const featuresOf = (tagged: readonly Tagged[], at: number): Features => {
   const entry = tagged[at];
   if (entry === undefined) return {};
-  if (entry.pos !== "VBN") return determinerFeatures(entry);
+  if (entry.pos === "VBG") return { features: { VerbForm: "Ger" } };
+  if (entry.pos !== "VBN") return nounOrDeterminerFeatures(entry);
   return { features: isPassive(tagged, at) ? { VerbForm: "Part", Voice: "Pass" } : { VerbForm: "Part" } };
 };
 
-const state: { ready: Tagger | undefined } = { ready: undefined };
+/** 解析器が引く語彙（語 → Penn Treebank の品詞の並び）。wink-pos-tagger が自分の依存から読むものと同じ一つを、同じ場所から読む。 */
+type Vocabulary = (word: string) => readonly string[] | undefined;
+
+const isTags = (value: unknown): value is readonly string[] => Array.isArray(value) && value.every((tag) => typeof tag === "string");
+
+const buildVocabulary = (): Vocabulary => {
+  const words: unknown = createRequire(require.resolve("wink-pos-tagger"))("wink-lexicon/src/lexicon.js");
+  if (!isRecord(words)) throw new Error("wink-lexicon の語彙が object ではありません");
+  return (word) => {
+    const tags = Object.hasOwn(words, word) ? words[word] : undefined;
+    return isTags(tags) ? tags : undefined;
+  };
+};
+
+const state: { ready: Tagger | undefined; vocabulary: Vocabulary } = { ready: undefined, vocabulary: () => undefined };
 
 export const prepare = (): void => {
-  state.ready ??= build();
+  if (state.ready !== undefined) return;
+  state.ready = build();
+  state.vocabulary = buildVocabulary();
 };
 
 export const isReady = (): boolean => state.ready !== undefined;
@@ -195,8 +222,22 @@ const locate = (text: string, tagged: readonly Tagged[]): Token[] =>
     { tokens: [], cursor: 0 },
   ).tokens;
 
+/** 英語の語はこれより長くならない。超える並びは語として読まない。 */
+const RUN_LIMIT = 1000;
+
+const tagged = (tagger: Tagger, text: string): readonly Tagged[] => toArray(callMethod(tagger, "tagSentence", [text])).filter(isTagged);
+
+/** 文頭で大文字になっただけの普通の語を、小文字で書いたときの品詞に戻す。前後の語による判断も効くよう、文ごと解析し直す。 */
+const withSentenceInitialCase = (tagger: Tagger, text: string, entries: readonly Tagged[]): readonly Tagged[] => {
+  const at = sentenceInitialCommonWord(entries, state.vocabulary);
+  const lowered = lowercasedAt(text, entries[at]?.value);
+  return lowered === undefined ? entries : rereadAt(entries, at, tagged(tagger, lowered));
+};
+
 export const tokenize = (text: string): Token[] | undefined => {
   const tagger = state.ready;
   if (tagger === undefined) return undefined;
-  return locate(text, toArray(callMethod(tagger, "tagSentence", [text])).filter(isTagged));
+  // 語は解析させた字（don't）で返す。語彙表の語と同じ字で比べられる。span は本文を指したまま。
+  const words = straightApostrophes(blankLongRuns(text, RUN_LIMIT));
+  return locate(words, withSentenceInitialCase(tagger, words, tagged(tagger, words)));
 };

package/src/proper-noun.ts

@@ -19,3 +19,46 @@ export const properNounChecked = (surface: string, pos: string): string => {
   if (!LETTER.test(surface)) return nonWord(surface);
   return SMALL.test(surface) && !CAPITAL.test(surface) ? "NOUN" : "PROPN";
 };
+
+/** 解析器が返す 1 語。pos は Penn Treebank。 */
+type TaggedWord = { readonly value: string; readonly pos: string; readonly lemma?: string };
+
+const PROPER_TAG = new Set(["NNP", "NNPS"]);
+
+/** 頭の一字だけが大文字の語（Containers、Such）。文頭の大文字はこの形になる。API や iPhone は名前の書き方なので含めない。 */
+const CAPITALISED = /^\p{Lu}\p{Ll}+$/u;
+
+/** 解析器の語彙に、その語が普通の語として載っているか。一度でも固有名詞として載っていれば（may の May）、名前かもしれない。 */
+const isCommonWord = (tags: readonly string[] | undefined): boolean => tags !== undefined && tags.length > 0 && !tags.some((tag) => PROPER_TAG.has(tag));
+
+/**
+ * 文頭で大文字になっただけの普通の語の位置。無ければ -1。
+ * wink は大文字で始まる名詞と形容詞をすべて固有名詞にするので、Containers are … の Containers も Traditional servers … の Traditional も固有名詞になる。
+ * 文頭の大文字は英語の書き方で、名前の印ではない。解析器の語彙が小文字の形を普通の語として知っていれば、名前ではなくその語として読む。
+ * 語彙に無い語（Kubernetes、Congress）と、文の途中の大文字の語は名前のまま。
+ */
+export const sentenceInitialCommonWord = (tagged: readonly TaggedWord[], tagsOf: (word: string) => readonly string[] | undefined): number => {
+  const at = tagged.findIndex((entry) => LETTER.test(entry.value));
+  const first = tagged[at];
+  if (first === undefined || !PROPER_TAG.has(first.pos) || !CAPITALISED.test(first.value)) return -1;
+  return isCommonWord(tagsOf(first.value.toLowerCase())) ? at : -1;
+};
+
+/** 文の中の最初の word を小文字にした文。word が文に無ければ undefined。 */
+export const lowercasedAt = (text: string, word: string | undefined): string | undefined => {
+  const start = word === undefined ? -1 : text.indexOf(word);
+  if (word === undefined || start === -1) return undefined;
+  return `${text.slice(0, start)}${word.toLowerCase()}${text.slice(start + word.length)}`;
+};
+
+/**
+ * at の語の品詞と原形を、小文字にして解析し直した結果（again）から取る。表層は元のまま。
+ * 解析し直した文の同じ位置に、同じ語の小文字が無ければ、語の切り方が変わったので元のまま。
+ */
+export const rereadAt = (entries: readonly TaggedWord[], at: number, again: readonly TaggedWord[]): readonly TaggedWord[] => {
+  const entry = entries[at];
+  const reread = again[at];
+  if (entry === undefined || reread === undefined || reread.value !== entry.value.toLowerCase()) return entries;
+  const word: TaggedWord = { value: entry.value, pos: reread.pos, ...(reread.lemma === undefined ? {} : { lemma: reread.lemma }) };
+  return entries.map((original, index) => (index === at ? word : original));
+};