@chaffjs/lang-en 0.32.0 → 0.33.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (50) hide show
  1. package/dist/dates.d.ts.map +1 -1
  2. package/dist/dates.js +35 -3
  3. package/dist/dates.js.map +1 -1
  4. package/dist/index.d.ts +1 -1
  5. package/dist/index.d.ts.map +1 -1
  6. package/dist/index.js +14 -3
  7. package/dist/index.js.map +1 -1
  8. package/dist/list-line.d.ts +12 -0
  9. package/dist/list-line.d.ts.map +1 -0
  10. package/dist/list-line.js +118 -0
  11. package/dist/list-line.js.map +1 -0
  12. package/lexicons/any-year-marker.yaml +6 -1
  13. package/lexicons/article-sound.yaml +3 -0
  14. package/lexicons/attached-instrument-label.yaml +12 -0
  15. package/lexicons/breakdown-partial-word.yaml +8 -0
  16. package/lexicons/caps-convention.yaml +8 -2
  17. package/lexicons/common-acronym.yaml +7 -0
  18. package/lexicons/comparative-baseline.yaml +5 -0
  19. package/lexicons/count-title.yaml +10 -0
  20. package/lexicons/date-stamp-label.yaml +1 -0
  21. package/lexicons/defined-term-pointer.yaml +26 -0
  22. package/lexicons/definition-alternative.yaml +6 -0
  23. package/lexicons/demonstrative-not-opener.yaml +3 -0
  24. package/lexicons/duration-per-period.yaml +18 -0
  25. package/lexicons/extra-word.yaml +37 -1
  26. package/lexicons/figure-scale-joiner.yaml +6 -0
  27. package/lexicons/figure-scale-word.yaml +9 -0
  28. package/lexicons/instruction-verb.yaml +1 -0
  29. package/lexicons/known-misspelling.yaml +8 -0
  30. package/lexicons/list-note-lead.yaml +9 -0
  31. package/lexicons/literal-quote-cue.yaml +16 -0
  32. package/lexicons/memo-header-label.yaml +16 -0
  33. package/lexicons/mention-cue.yaml +15 -0
  34. package/lexicons/named-deadline.yaml +12 -0
  35. package/lexicons/negation-word.yaml +13 -0
  36. package/lexicons/placeholder-word.yaml +5 -0
  37. package/lexicons/requirement-open-condition.yaml +9 -0
  38. package/lexicons/requirement-open-question.yaml +6 -0
  39. package/lexicons/rounded-amount-unit.yaml +9 -0
  40. package/lexicons/self-prefix.yaml +6 -0
  41. package/lexicons/sequence-opener.yaml +17 -0
  42. package/lexicons/side-step-label.yaml +7 -0
  43. package/lexicons/term-mention-cue.yaml +15 -0
  44. package/lexicons/title-cue.yaml +9 -0
  45. package/lexicons/verb-or-adjective.yaml +14 -0
  46. package/lexicons/wordy-phrase.yaml +3 -3
  47. package/package.json +1 -1
  48. package/src/dates.ts +39 -3
  49. package/src/index.ts +16 -3
  50. package/src/list-line.ts +136 -0
@@ -0,0 +1,136 @@
1
+ import type { Span } from "chaffjs/plugin";
2
+
3
+ /**
4
+ * Items written one per line with no blank line between them ("(a) …;\n(b) …", plain-text "1." or "-" lines, lines that
5
+ * were `<br>`). An item seldom ends in a full stop, so the splitter reads the whole run as one sentence.
6
+ *
7
+ * English is hard-wrapped, and a wrapped line can start like an item ("in Section 2\n(a) of this Agreement", "in
8
+ * RFC\n793. It"). So a break before an item is cut only on a cue: the line before ends a lead-in or an item, the next
9
+ * marker continues the list ("-" after "-", "(b)" after "(a)"), both lines are items and the next starts with a capital,
10
+ * or a bracketed marker is followed by a capital. A break after
11
+ * an item into a line with no marker is never cut: that line continues the item as often as it starts new text.
12
+ */
13
+
14
+ const PAREN = String.raw`\((?:\d{1,3}[A-Z]?|[a-z]{1,2}|[A-Z]|[ivxlc]{2,6}|[IVXLC]{2,6})\)`;
15
+ const NUMBER = String.raw`\d{1,3}(?:\.\d{1,3})*[.)]|\d{1,3}(?:\.\d{1,3})+`;
16
+ const LETTER = String.raw`[a-z][.)]`;
17
+ /** An appendix's numbered part ("A.1.", "B.2.3"), as an RFC writes it in the body and in its contents. */
18
+ const APPENDIX = String.raw`[A-Z](?:\.\d{1,3})+\.?`;
19
+ const BULLET = "[-–•●▪◦‣*·]";
20
+
21
+ /** The marker, then the first character of the item's text. A marker other than a bracketed one needs a space after it. */
22
+ const MARKER = new RegExp(String.raw`^[ \t]*(?:(${PAREN})[ \t]*|(${NUMBER}|${APPENDIX}|${LETTER}|${BULLET})[ \t]+)(\S)?`, "u");
23
+
24
+ /** marker: as written ("(a)", "2.", "-"). capital: the item's text starts with a capital or a digit. */
25
+ type Item = { readonly marker: string; readonly bracketed: boolean; readonly capital: boolean };
26
+
27
+ const itemOf = (line: string): Item | undefined => {
28
+ const match = MARKER.exec(line);
29
+ if (match === null) return undefined;
30
+ return { marker: match[1] ?? match[2] ?? "", bracketed: match[1] !== undefined, capital: /[\p{Lu}\d]/u.test(match[3] ?? "") };
31
+ };
32
+
33
+ const BULLET_ONLY = new RegExp(`^${BULLET}$`, "u");
34
+
35
+ /** A number or a single letter between the same brackets or before the same stop ("(a)", "2.", "b)"). */
36
+ const SEQUENCED = /^(\(?)(\d{1,3}|[a-zA-Z])([.)]?)$/u;
37
+
38
+ const nextLabel = (label: string): string => (/\d/u.test(label) ? String(Number(label) + 1) : String.fromCharCode(label.charCodeAt(0) + 1));
39
+
40
+ /** The second marker continues the first: the same bullet ("-", "-"), or the next number or letter written alike ("(a)", "(b)"). */
41
+ const continuesList = (first: string, second: string): boolean => {
42
+ if (BULLET_ONLY.test(first)) return first === second;
43
+ const [, open, label, close] = SEQUENCED.exec(first) ?? [];
44
+ if (label === undefined) return false;
45
+ return second === `${open ?? ""}${nextLabel(label)}${close ?? ""}`;
46
+ };
47
+
48
+ /** Whether the line starts with a list marker. */
49
+ export const startsWithListMarker = (line: string): boolean => itemOf(line) !== undefined;
50
+
51
+ /** The end of a line, without a Markdown hard break ("\" or trailing spaces). */
52
+ const endOf = (line: string): string => {
53
+ const trimmed = line.trimEnd();
54
+ return trimmed.endsWith("\\") ? trimmed.slice(0, -1).trimEnd() : trimmed;
55
+ };
56
+
57
+ /** A lead-in or an item that ends here: the next item is a new sentence whatever it starts with. */
58
+ const LEAD_IN_END = /(?:[:;—–]|[;,:][ \t]*(?:and|or|and\/or|but|then))$/u;
59
+ /** Ends the line's own item: a comma or a conjunction after it, when the line is an item too. */
60
+ const ITEM_END = /(?:,|\b(?:and|or))$/u;
61
+ /** A full stop the splitter may have read as an abbreviation ("Inc.", "U.S."). */
62
+ const STOP_END = /[.?!]["”’)]?$/u;
63
+
64
+ const isCut = (previous: string, next: string): boolean => {
65
+ const item = itemOf(next);
66
+ if (item === undefined) return false;
67
+ const end = endOf(previous);
68
+ const before = itemOf(previous);
69
+ const previousIsItem = before !== undefined;
70
+ if (LEAD_IN_END.test(end)) return true;
71
+ if (before !== undefined && continuesList(before.marker, item.marker)) return true;
72
+ if (previousIsItem && ITEM_END.test(end)) return true;
73
+ if (!item.capital) return false;
74
+ return previousIsItem || item.bracketed || STOP_END.test(end);
75
+ };
76
+
77
+ type Line = { readonly start: number; readonly end: number; readonly text: string };
78
+
79
+ const NEWLINE = /\r?\n/gu;
80
+
81
+ const linesOf = (text: string): Line[] => {
82
+ const breaks = [...text.matchAll(NEWLINE)];
83
+ const starts = [0, ...breaks.map((match) => match.index + match[0].length)];
84
+ return starts.map((start, index) => {
85
+ const end = breaks[index]?.index ?? text.length;
86
+ return { start, end, text: text.slice(start, end) };
87
+ });
88
+ };
89
+
90
+ const indentOf = (line: Line): number => line.text.length - line.text.trimStart().length;
91
+
92
+ /** The line breaks that end a sentence: from the end of the line before to the first character of the item. */
93
+ export const listLineBreaks = (text: string): Span[] => {
94
+ const lines = linesOf(text);
95
+ return lines.slice(1).flatMap((next, index) => {
96
+ const previous = lines[index];
97
+ if (previous === undefined || previous.text.trim() === "" || !isCut(previous.text, next.text)) return [];
98
+ return [{ start: previous.end, end: next.start + indentOf(next) }];
99
+ });
100
+ };
101
+
102
+ /** Each item's head: from the start of its line to the marker's first character. A sentence starting here starts an item. */
103
+ export const listItemHeads = (text: string): Span[] =>
104
+ linesOf(text).flatMap((line) => (startsWithListMarker(line.text) ? [{ start: line.start, end: line.start + indentOf(line) + 1 }] : []));
105
+
106
+ /** Whether offset is inside one of the heads (in order, not overlapping). */
107
+ export const startsListItem = (heads: readonly Span[], offset: number): boolean => {
108
+ const search = (low: number, high: number): Span | undefined => {
109
+ if (low >= high) return heads[low - 1];
110
+ const middle = (low + high) >> 1;
111
+ return (heads[middle]?.start ?? Number.POSITIVE_INFINITY) > offset ? search(low, middle) : search(middle + 1, high);
112
+ };
113
+ const head = search(0, heads.length);
114
+ return head !== undefined && offset < head.end;
115
+ };
116
+
117
+ /** In breaks (in order, not overlapping), the index of the first one whose start passes the test. */
118
+ const firstWhere = (breaks: readonly Span[], test: (part: Span) => boolean): number => {
119
+ const search = (low: number, high: number): number => {
120
+ if (low >= high) return low;
121
+ const middle = (low + high) >> 1;
122
+ const part = breaks[middle];
123
+ return part !== undefined && test(part) ? search(low, middle) : search(middle + 1, high);
124
+ };
125
+ return search(0, breaks.length);
126
+ };
127
+
128
+ /** The span cut at the breaks inside it. The piece before a break ends where the line ends; the next starts at the item. */
129
+ export const cutAtBreaks = (span: Span, breaks: readonly Span[]): Span[] => {
130
+ const inside = breaks.slice(
131
+ firstWhere(breaks, (part) => part.start > span.start),
132
+ firstWhere(breaks, (part) => part.end >= span.end),
133
+ );
134
+ const starts = [span.start, ...inside.map((part) => part.end)];
135
+ return starts.map((start, index) => ({ start, end: inside[index]?.start ?? span.end }));
136
+ };