@chaffjs/lang-en 0.32.0 → 0.34.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (57) hide show
  1. package/dist/dates.d.ts.map +1 -1
  2. package/dist/dates.js +35 -3
  3. package/dist/dates.js.map +1 -1
  4. package/dist/index.d.ts +1 -1
  5. package/dist/index.d.ts.map +1 -1
  6. package/dist/index.js +14 -3
  7. package/dist/index.js.map +1 -1
  8. package/dist/list-line.d.ts +12 -0
  9. package/dist/list-line.d.ts.map +1 -0
  10. package/dist/list-line.js +118 -0
  11. package/dist/list-line.js.map +1 -0
  12. package/lexicons/abbreviation-mention.yaml +14 -0
  13. package/lexicons/any-year-marker.yaml +6 -1
  14. package/lexicons/article-sound.yaml +3 -0
  15. package/lexicons/attached-document-cue.yaml +33 -0
  16. package/lexicons/attached-instrument-label.yaml +12 -0
  17. package/lexicons/breakdown-partial-word.yaml +8 -0
  18. package/lexicons/caps-convention.yaml +8 -2
  19. package/lexicons/citation-signal.yaml +8 -0
  20. package/lexicons/common-acronym.yaml +9 -0
  21. package/lexicons/comparative-baseline.yaml +5 -0
  22. package/lexicons/concrete-deadline.yaml +7 -0
  23. package/lexicons/count-title.yaml +10 -0
  24. package/lexicons/date-stamp-label.yaml +1 -0
  25. package/lexicons/defined-term-pointer.yaml +26 -0
  26. package/lexicons/definition-alternative.yaml +6 -0
  27. package/lexicons/demonstrative-not-opener.yaml +3 -0
  28. package/lexicons/duration-per-period.yaml +18 -0
  29. package/lexicons/email-greeting.yaml +1 -0
  30. package/lexicons/extra-word.yaml +54 -1
  31. package/lexicons/figure-scale-joiner.yaml +6 -0
  32. package/lexicons/figure-scale-word.yaml +9 -0
  33. package/lexicons/instruction-verb.yaml +1 -0
  34. package/lexicons/known-misspelling.yaml +8 -0
  35. package/lexicons/list-note-lead.yaml +9 -0
  36. package/lexicons/literal-quote-cue.yaml +16 -0
  37. package/lexicons/memo-header-label.yaml +16 -0
  38. package/lexicons/mention-cue.yaml +15 -0
  39. package/lexicons/named-deadline.yaml +12 -0
  40. package/lexicons/negation-word.yaml +13 -0
  41. package/lexicons/placeholder-word.yaml +5 -0
  42. package/lexicons/requirement-open-condition.yaml +9 -0
  43. package/lexicons/requirement-open-end.yaml +3 -0
  44. package/lexicons/requirement-open-question.yaml +6 -0
  45. package/lexicons/rounded-amount-unit.yaml +9 -0
  46. package/lexicons/self-prefix.yaml +6 -0
  47. package/lexicons/sequence-opener.yaml +17 -0
  48. package/lexicons/side-step-label.yaml +7 -0
  49. package/lexicons/term-mention-cue.yaml +15 -0
  50. package/lexicons/title-cue.yaml +9 -0
  51. package/lexicons/verb-or-adjective.yaml +14 -0
  52. package/lexicons/wordy-phrase.yaml +3 -3
  53. package/lexicons/year-span-joiner.yaml +5 -1
  54. package/package.json +1 -1
  55. package/src/dates.ts +39 -3
  56. package/src/index.ts +16 -3
  57. package/src/list-line.ts +136 -0
package/src/index.ts CHANGED
@@ -8,6 +8,7 @@ import { structure } from "./structure.ts";
8
8
  import { dictionary } from "./dictionary.ts";
9
9
  import { isReady, prepare, tokenize } from "./pos.ts";
10
10
  import { isJapaneseRun } from "./japanese-run.ts";
11
+ import { cutAtBreaks, listItemHeads, listLineBreaks, startsListItem } from "./list-line.ts";
11
12
  import type { AdapterNeeds, EmbeddedLanguage, LanguageAdapter, Segmentation, Sentence, Span } from "chaffjs/plugin";
12
13
 
13
14
  // chaff からは型だけを取る。実行時の値依存を作らない。アダプタは単体で動く。
@@ -35,9 +36,22 @@ const JAPANESE: EmbeddedLanguage = { id: "ja", lengthUnit: "char" };
35
36
 
36
37
  const withLanguage = (text: string, span: Span): Sentence => (isJapaneseRun(text) ? { span, text, embeddedLanguage: JAPANESE } : { span, text });
37
38
 
39
+ const asListItem = (sentence: Sentence): Sentence => ({ ...sentence, listItem: true });
40
+
41
+ /** Spans of the splitter, cut again where a line break before a list item ends a sentence. */
42
+ const sentencesOf = (text: string): Sentence[] => {
43
+ const quotedStops = sentenceSpans(unmarkNumberStops(unmarkLabelStops(text, LABEL_STOPS))).flatMap((span) => splitAtQuotedStops(text, span));
44
+ const listBreaks = listLineBreaks(text);
45
+ const itemHeads = listItemHeads(text);
46
+ return reattachClosingQuotes(text, quotedStops)
47
+ .flatMap((span) => cutAtBreaks(span, listBreaks))
48
+ .map((span) => withLanguage(text.slice(span.start, span.end), span))
49
+ .map((sentence) => (startsListItem(itemHeads, sentence.span.start) ? asListItem(sentence) : sentence));
50
+ };
51
+
38
52
  /**
39
53
  * 英語は sentence-splitter の既定にほぼ任せる。"Dr." "e.g." "U.S." "$3.50" を
40
- * いずれも文末と誤認しない。前処理は行の途中の番号を箇条書きと読ませることと、番号の前の略した名前(FIG. 1、Vol. XLIII)の点で切らないこと、後処理は閉じ引用符の内側で閉じた文を切ることと、文頭に取り残された閉じ引用符を前の文へ戻すこと。spec §7.2。
54
+ * いずれも文末と誤認しない。前処理は行の途中の番号を箇条書きと読ませることと、番号の前の略した名前(FIG. 1、Vol. XLIII)の点で切らないこと、後処理は閉じ引用符の内側で閉じた文を切ることと、文頭に取り残された閉じ引用符を前の文へ戻すこと、1 行 1 項目の箇条書きを項目ごとの文に切ること。spec §7.2。
41
55
  */
42
56
  export const adapter: LanguageAdapter = {
43
57
  kind: "language",
@@ -65,8 +79,7 @@ export const adapter: LanguageAdapter = {
65
79
  structure,
66
80
  dictionary,
67
81
  segment: (text: string): Segmentation => {
68
- const quotedStops = sentenceSpans(unmarkNumberStops(unmarkLabelStops(text, LABEL_STOPS))).flatMap((span) => splitAtQuotedStops(text, span));
69
- const sentences: Sentence[] = reattachClosingQuotes(text, quotedStops).map((span) => withLanguage(text.slice(span.start, span.end), span));
82
+ const sentences = sentencesOf(text);
70
83
  return { sentences: isReady() ? sentences.map(withTokens) : sentences };
71
84
  },
72
85
  };
@@ -0,0 +1,136 @@
1
+ import type { Span } from "chaffjs/plugin";
2
+
3
+ /**
4
+ * Items written one per line with no blank line between them ("(a) …;\n(b) …", plain-text "1." or "-" lines, lines that
5
+ * were `<br>`). An item seldom ends in a full stop, so the splitter reads the whole run as one sentence.
6
+ *
7
+ * English is hard-wrapped, and a wrapped line can start like an item ("in Section 2\n(a) of this Agreement", "in
8
+ * RFC\n793. It"). So a break before an item is cut only on a cue: the line before ends a lead-in or an item, the next
9
+ * marker continues the list ("-" after "-", "(b)" after "(a)"), both lines are items and the next starts with a capital,
10
+ * or a bracketed marker is followed by a capital. A break after
11
+ * an item into a line with no marker is never cut: that line continues the item as often as it starts new text.
12
+ */
13
+
14
+ const PAREN = String.raw`\((?:\d{1,3}[A-Z]?|[a-z]{1,2}|[A-Z]|[ivxlc]{2,6}|[IVXLC]{2,6})\)`;
15
+ const NUMBER = String.raw`\d{1,3}(?:\.\d{1,3})*[.)]|\d{1,3}(?:\.\d{1,3})+`;
16
+ const LETTER = String.raw`[a-z][.)]`;
17
+ /** An appendix's numbered part ("A.1.", "B.2.3"), as an RFC writes it in the body and in its contents. */
18
+ const APPENDIX = String.raw`[A-Z](?:\.\d{1,3})+\.?`;
19
+ const BULLET = "[-–•●▪◦‣*·]";
20
+
21
+ /** The marker, then the first character of the item's text. A marker other than a bracketed one needs a space after it. */
22
+ const MARKER = new RegExp(String.raw`^[ \t]*(?:(${PAREN})[ \t]*|(${NUMBER}|${APPENDIX}|${LETTER}|${BULLET})[ \t]+)(\S)?`, "u");
23
+
24
+ /** marker: as written ("(a)", "2.", "-"). capital: the item's text starts with a capital or a digit. */
25
+ type Item = { readonly marker: string; readonly bracketed: boolean; readonly capital: boolean };
26
+
27
+ const itemOf = (line: string): Item | undefined => {
28
+ const match = MARKER.exec(line);
29
+ if (match === null) return undefined;
30
+ return { marker: match[1] ?? match[2] ?? "", bracketed: match[1] !== undefined, capital: /[\p{Lu}\d]/u.test(match[3] ?? "") };
31
+ };
32
+
33
+ const BULLET_ONLY = new RegExp(`^${BULLET}$`, "u");
34
+
35
+ /** A number or a single letter between the same brackets or before the same stop ("(a)", "2.", "b)"). */
36
+ const SEQUENCED = /^(\(?)(\d{1,3}|[a-zA-Z])([.)]?)$/u;
37
+
38
+ const nextLabel = (label: string): string => (/\d/u.test(label) ? String(Number(label) + 1) : String.fromCharCode(label.charCodeAt(0) + 1));
39
+
40
+ /** The second marker continues the first: the same bullet ("-", "-"), or the next number or letter written alike ("(a)", "(b)"). */
41
+ const continuesList = (first: string, second: string): boolean => {
42
+ if (BULLET_ONLY.test(first)) return first === second;
43
+ const [, open, label, close] = SEQUENCED.exec(first) ?? [];
44
+ if (label === undefined) return false;
45
+ return second === `${open ?? ""}${nextLabel(label)}${close ?? ""}`;
46
+ };
47
+
48
+ /** Whether the line starts with a list marker. */
49
+ export const startsWithListMarker = (line: string): boolean => itemOf(line) !== undefined;
50
+
51
+ /** The end of a line, without a Markdown hard break ("\" or trailing spaces). */
52
+ const endOf = (line: string): string => {
53
+ const trimmed = line.trimEnd();
54
+ return trimmed.endsWith("\\") ? trimmed.slice(0, -1).trimEnd() : trimmed;
55
+ };
56
+
57
+ /** A lead-in or an item that ends here: the next item is a new sentence whatever it starts with. */
58
+ const LEAD_IN_END = /(?:[:;—–]|[;,:][ \t]*(?:and|or|and\/or|but|then))$/u;
59
+ /** Ends the line's own item: a comma or a conjunction after it, when the line is an item too. */
60
+ const ITEM_END = /(?:,|\b(?:and|or))$/u;
61
+ /** A full stop the splitter may have read as an abbreviation ("Inc.", "U.S."). */
62
+ const STOP_END = /[.?!]["”’)]?$/u;
63
+
64
+ const isCut = (previous: string, next: string): boolean => {
65
+ const item = itemOf(next);
66
+ if (item === undefined) return false;
67
+ const end = endOf(previous);
68
+ const before = itemOf(previous);
69
+ const previousIsItem = before !== undefined;
70
+ if (LEAD_IN_END.test(end)) return true;
71
+ if (before !== undefined && continuesList(before.marker, item.marker)) return true;
72
+ if (previousIsItem && ITEM_END.test(end)) return true;
73
+ if (!item.capital) return false;
74
+ return previousIsItem || item.bracketed || STOP_END.test(end);
75
+ };
76
+
77
+ type Line = { readonly start: number; readonly end: number; readonly text: string };
78
+
79
+ const NEWLINE = /\r?\n/gu;
80
+
81
+ const linesOf = (text: string): Line[] => {
82
+ const breaks = [...text.matchAll(NEWLINE)];
83
+ const starts = [0, ...breaks.map((match) => match.index + match[0].length)];
84
+ return starts.map((start, index) => {
85
+ const end = breaks[index]?.index ?? text.length;
86
+ return { start, end, text: text.slice(start, end) };
87
+ });
88
+ };
89
+
90
+ const indentOf = (line: Line): number => line.text.length - line.text.trimStart().length;
91
+
92
+ /** The line breaks that end a sentence: from the end of the line before to the first character of the item. */
93
+ export const listLineBreaks = (text: string): Span[] => {
94
+ const lines = linesOf(text);
95
+ return lines.slice(1).flatMap((next, index) => {
96
+ const previous = lines[index];
97
+ if (previous === undefined || previous.text.trim() === "" || !isCut(previous.text, next.text)) return [];
98
+ return [{ start: previous.end, end: next.start + indentOf(next) }];
99
+ });
100
+ };
101
+
102
+ /** Each item's head: from the start of its line to the marker's first character. A sentence starting here starts an item. */
103
+ export const listItemHeads = (text: string): Span[] =>
104
+ linesOf(text).flatMap((line) => (startsWithListMarker(line.text) ? [{ start: line.start, end: line.start + indentOf(line) + 1 }] : []));
105
+
106
+ /** Whether offset is inside one of the heads (in order, not overlapping). */
107
+ export const startsListItem = (heads: readonly Span[], offset: number): boolean => {
108
+ const search = (low: number, high: number): Span | undefined => {
109
+ if (low >= high) return heads[low - 1];
110
+ const middle = (low + high) >> 1;
111
+ return (heads[middle]?.start ?? Number.POSITIVE_INFINITY) > offset ? search(low, middle) : search(middle + 1, high);
112
+ };
113
+ const head = search(0, heads.length);
114
+ return head !== undefined && offset < head.end;
115
+ };
116
+
117
+ /** In breaks (in order, not overlapping), the index of the first one whose start passes the test. */
118
+ const firstWhere = (breaks: readonly Span[], test: (part: Span) => boolean): number => {
119
+ const search = (low: number, high: number): number => {
120
+ if (low >= high) return low;
121
+ const middle = (low + high) >> 1;
122
+ const part = breaks[middle];
123
+ return part !== undefined && test(part) ? search(low, middle) : search(middle + 1, high);
124
+ };
125
+ return search(0, breaks.length);
126
+ };
127
+
128
+ /** The span cut at the breaks inside it. The piece before a break ends where the line ends; the next starts at the item. */
129
+ export const cutAtBreaks = (span: Span, breaks: readonly Span[]): Span[] => {
130
+ const inside = breaks.slice(
131
+ firstWhere(breaks, (part) => part.start > span.start),
132
+ firstWhere(breaks, (part) => part.end >= span.end),
133
+ );
134
+ const starts = [span.start, ...inside.map((part) => part.end)];
135
+ return starts.map((start, index) => ({ start, end: inside[index]?.start ?? span.end }));
136
+ };