@chaffjs/lang-en 0.34.0 → 0.36.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (121) hide show
  1. package/dist/citation.d.ts +5 -0
  2. package/dist/citation.d.ts.map +1 -1
  3. package/dist/citation.js +29 -3
  4. package/dist/citation.js.map +1 -1
  5. package/dist/index.d.ts.map +1 -1
  6. package/dist/index.js +6 -1
  7. package/dist/index.js.map +1 -1
  8. package/dist/initial-stop.d.ts +27 -0
  9. package/dist/initial-stop.d.ts.map +1 -0
  10. package/dist/initial-stop.js +47 -0
  11. package/dist/initial-stop.js.map +1 -0
  12. package/dist/item-style.d.ts +5 -2
  13. package/dist/item-style.d.ts.map +1 -1
  14. package/dist/item-style.js +31 -6
  15. package/dist/item-style.js.map +1 -1
  16. package/dist/law-citation.d.ts +14 -0
  17. package/dist/law-citation.d.ts.map +1 -0
  18. package/dist/law-citation.js +58 -0
  19. package/dist/law-citation.js.map +1 -0
  20. package/dist/lead-initial.d.ts +5 -0
  21. package/dist/lead-initial.d.ts.map +1 -0
  22. package/dist/lead-initial.js +24 -0
  23. package/dist/lead-initial.js.map +1 -0
  24. package/dist/names-read-as-verbs.d.ts +21 -0
  25. package/dist/names-read-as-verbs.d.ts.map +1 -0
  26. package/dist/names-read-as-verbs.js +40 -0
  27. package/dist/names-read-as-verbs.js.map +1 -0
  28. package/dist/pos.d.ts.map +1 -1
  29. package/dist/pos.js +30 -1
  30. package/dist/pos.js.map +1 -1
  31. package/dist/structure.d.ts.map +1 -1
  32. package/dist/structure.js +67 -9
  33. package/dist/structure.js.map +1 -1
  34. package/lexicons/abbreviated-label.yaml +2 -0
  35. package/lexicons/ai-tell.yaml +61 -27
  36. package/lexicons/alternative-clause-label.yaml +6 -0
  37. package/lexicons/announcing-opener.yaml +7 -0
  38. package/lexicons/article-sound.yaml +1 -0
  39. package/lexicons/capital-phrase.yaml +3 -1
  40. package/lexicons/case-party-joiner.yaml +7 -0
  41. package/lexicons/citation-name-connector.yaml +28 -0
  42. package/lexicons/citation-series.yaml +5 -1
  43. package/lexicons/clause-conjunction.yaml +7 -0
  44. package/lexicons/closing-cliche.yaml +9 -0
  45. package/lexicons/closing-exhortation.yaml +11 -0
  46. package/lexicons/cold-open.yaml +30 -0
  47. package/lexicons/common-acronym.yaml +21 -1
  48. package/lexicons/comparative-baseline.yaml +27 -0
  49. package/lexicons/condition-heading.yaml +21 -0
  50. package/lexicons/conjunction-lookalike.yaml +13 -0
  51. package/lexicons/connective-opener.yaml +23 -0
  52. package/lexicons/contraction-head.yaml +10 -0
  53. package/lexicons/contrast-amount.yaml +13 -0
  54. package/lexicons/contrast-lead.yaml +23 -0
  55. package/lexicons/contrast-turn.yaml +5 -0
  56. package/lexicons/definition-scope-lead.yaml +11 -0
  57. package/lexicons/determiner-after-article.yaml +14 -0
  58. package/lexicons/drug-stem.yaml +48 -0
  59. package/lexicons/elision-mark.yaml +9 -0
  60. package/lexicons/email-closing.yaml +7 -1
  61. package/lexicons/email-salutation-end.yaml +7 -0
  62. package/lexicons/email-stock-courtesy.yaml +29 -0
  63. package/lexicons/enclosing-unit.yaml +14 -0
  64. package/lexicons/everyday-name.yaml +24 -0
  65. package/lexicons/extra-word.yaml +28 -1
  66. package/lexicons/figure-label-plural.yaml +19 -0
  67. package/lexicons/figure-number-joiner.yaml +20 -0
  68. package/lexicons/headnote-heading.yaml +8 -0
  69. package/lexicons/honorific.yaml +3 -0
  70. package/lexicons/indefinite-article.yaml +8 -0
  71. package/lexicons/initial-lead-word.yaml +7 -0
  72. package/lexicons/intent-phrase.yaml +21 -0
  73. package/lexicons/interviewer-mark.yaml +10 -0
  74. package/lexicons/link-label-ending.yaml +6 -0
  75. package/lexicons/list-conjunction.yaml +7 -0
  76. package/lexicons/list-continues.yaml +9 -0
  77. package/lexicons/name-abbreviation.yaml +21 -0
  78. package/lexicons/name-capitals-prefix.yaml +16 -0
  79. package/lexicons/name-direction-word.yaml +10 -0
  80. package/lexicons/name-title-head.yaml +23 -0
  81. package/lexicons/name-word-ending.yaml +10 -0
  82. package/lexicons/navigation-line.yaml +11 -0
  83. package/lexicons/newsletter-issue.yaml +9 -0
  84. package/lexicons/numbered-division.yaml +4 -1
  85. package/lexicons/open-list-end.yaml +10 -0
  86. package/lexicons/over-general-claim.yaml +62 -0
  87. package/lexicons/padded-intro.yaml +14 -0
  88. package/lexicons/placeholder-division.yaml +7 -0
  89. package/lexicons/prepositive-abbreviation.yaml +10 -0
  90. package/lexicons/product-name.yaml +9 -0
  91. package/lexicons/prohibition-opener.yaml +6 -0
  92. package/lexicons/question-wording-lead.yaml +7 -0
  93. package/lexicons/rejected-wording.yaml +8 -0
  94. package/lexicons/request-deadline-section.yaml +11 -0
  95. package/lexicons/request-prohibition.yaml +8 -0
  96. package/lexicons/sentence-start-word.yaml +64 -0
  97. package/lexicons/session-law.yaml +19 -0
  98. package/lexicons/superlative-goal.yaml +40 -0
  99. package/lexicons/superlative-part.yaml +22 -0
  100. package/lexicons/superlative-phrase.yaml +21 -0
  101. package/lexicons/superlative-purpose.yaml +7 -0
  102. package/lexicons/superlative-referent.yaml +15 -0
  103. package/lexicons/superlative-scope.yaml +5 -1
  104. package/lexicons/telling-cue.yaml +6 -0
  105. package/lexicons/template-heading.yaml +35 -0
  106. package/lexicons/text-message-verb.yaml +11 -0
  107. package/lexicons/uninflected-word.yaml +22 -0
  108. package/lexicons/unprompted-denial.yaml +113 -0
  109. package/lexicons/word-suffix.yaml +34 -1
  110. package/lexicons/wordy-phrase.yaml +2 -2
  111. package/lexicons/year-reference.yaml +12 -0
  112. package/package.json +1 -1
  113. package/src/citation.ts +33 -3
  114. package/src/index.ts +13 -1
  115. package/src/initial-stop.ts +80 -0
  116. package/src/item-style.ts +33 -7
  117. package/src/law-citation.ts +68 -0
  118. package/src/lead-initial.ts +28 -0
  119. package/src/names-read-as-verbs.ts +65 -0
  120. package/src/pos.ts +33 -1
  121. package/src/structure.ts +73 -9
@@ -0,0 +1,11 @@
1
+ # Verbs that send a keyword by text message (text GNPCGS to 333111). The capitals between the verb and the short code are
2
+ # a keyword to type as it stands, not an acronym to spell out. Matched as written, in any case.
3
+ id: text-message-verb
4
+ language: en
5
+ entries:
6
+ - pattern: text
7
+ - pattern: texts
8
+ - pattern: texting
9
+ - pattern: txt
10
+ - pattern: reply
11
+ - pattern: replying
@@ -0,0 +1,22 @@
1
+ # Function words that take no ending (word-suffix): a determiner, a pronoun or a conjunction never has a plural or a
2
+ # past. unknown-word does not read "thoses" as "those" + s, so the slip is reported. Only closed-class words are listed;
3
+ # a word that does take an ending in some use (a noun, a verb) never belongs here.
4
+ id: uninflected-word
5
+ language: en
6
+ entries:
7
+ - pattern: although
8
+ - pattern: because
9
+ - pattern: these
10
+ - pattern: those
11
+ - pattern: their
12
+ - pattern: there
13
+ - pattern: thus
14
+ - pattern: unless
15
+ - pattern: where
16
+ - pattern: whether
17
+ - pattern: which
18
+ - pattern: whom
19
+ - pattern: whose
20
+ - pattern: would
21
+ - pattern: could
22
+ - pattern: should
@@ -0,0 +1,113 @@
1
+ # A sentence that denies a premise nobody stated ("This isn't about tooling.", "That's not the point."). Groups as
2
+ # unprompted-denial.ts reads them:
3
+ # open the sentence opens with it, and the next word is a follow word: a denial of what something is
4
+ # follow the word after open ("about", "a", "the", "what"); "It's not ready." negates a fact and is not counted
5
+ # basis a reason in the same sentence: the denial states a fact
6
+ # premise in the previous sentence: the denial answers something the text said ("You might think ...")
7
+ id: unprompted-denial
8
+ language: en
9
+ entries:
10
+ - pattern: this isn't
11
+ group: open
12
+ - pattern: this is not
13
+ group: open
14
+ - pattern: that's not
15
+ group: open
16
+ - pattern: that is not
17
+ group: open
18
+ - pattern: that isn't
19
+ group: open
20
+ - pattern: it's not
21
+ group: open
22
+ - pattern: it is not
23
+ group: open
24
+ - pattern: it isn't
25
+ group: open
26
+ - pattern: these aren't
27
+ group: open
28
+ - pattern: these are not
29
+ group: open
30
+ - pattern: those aren't
31
+ group: open
32
+ - pattern: those are not
33
+ group: open
34
+ - pattern: the point isn't
35
+ group: open
36
+ - pattern: the point is not
37
+ group: open
38
+ - pattern: the goal isn't
39
+ group: open
40
+ - pattern: the goal is not
41
+ group: open
42
+ - pattern: the problem isn't
43
+ group: open
44
+ - pattern: the problem is not
45
+ group: open
46
+ - pattern: the question isn't
47
+ group: open
48
+ - pattern: the question is not
49
+ group: open
50
+ - pattern: the answer isn't
51
+ group: open
52
+ - pattern: the answer is not
53
+ group: open
54
+ - pattern: about
55
+ group: follow
56
+ - pattern: a
57
+ group: follow
58
+ - pattern: an
59
+ group: follow
60
+ - pattern: the
61
+ group: follow
62
+ - pattern: just
63
+ group: follow
64
+ - pattern: only
65
+ group: follow
66
+ - pattern: what
67
+ group: follow
68
+ - pattern: why
69
+ group: follow
70
+ - pattern: how
71
+ group: follow
72
+ - pattern: where
73
+ group: follow
74
+ - pattern: who
75
+ group: follow
76
+ - pattern: my
77
+ group: follow
78
+ - pattern: your
79
+ group: follow
80
+ - pattern: our
81
+ group: follow
82
+ - pattern: their
83
+ group: follow
84
+ - pattern: something
85
+ group: follow
86
+ - pattern: really
87
+ group: follow
88
+ - pattern: simply
89
+ group: follow
90
+ - pattern: merely
91
+ group: follow
92
+ - pattern: because
93
+ group: basis
94
+ - pattern: since
95
+ group: basis
96
+ - pattern: due to
97
+ group: basis
98
+ - pattern: you might think
99
+ group: premise
100
+ - pattern: you may think
101
+ group: premise
102
+ - pattern: people think
103
+ group: premise
104
+ - pattern: some say
105
+ group: premise
106
+ - pattern: it's tempting
107
+ group: premise
108
+ - pattern: assume
109
+ group: premise
110
+ - pattern: myth
111
+ group: premise
112
+ - pattern: misconception
113
+ group: premise
@@ -1,45 +1,69 @@
1
1
  # Endings that make an inflected or derived form of a dictionary word: unknown-word strips the ending (pattern) and
2
2
  # puts the replacement (instead_of, empty when nothing goes back) before looking the stem up. "carried" is "carry",
3
3
  # "stopped" is "stop" (a doubled last consonant is undone too), "baking" is "bake". The dictionary itself is
4
- # dictionary.txt; this list only saves listing every inflected form there.
4
+ # dictionary.txt; this list only saves listing every inflected form there. An ending of group inflection (a plural, a
5
+ # past, a comparative) closes a word: unknown-word takes it on top of a derived word ("operationalises" is
6
+ # "operationalise" + s), never a derived ending on top of it, and never two of them ("oversers" is not "overs" + ers).
5
7
  id: word-suffix
6
8
  language: en
7
9
  entries:
8
10
  - pattern: s
9
11
  instead_of: ""
12
+ group: inflection
10
13
  - pattern: es
11
14
  instead_of: ""
15
+ group: inflection
12
16
  - pattern: ies
13
17
  instead_of: "y"
18
+ group: inflection
14
19
  # The Greek plural: "pneumoconioses" is "pneumoconiosis".
15
20
  - pattern: ses
16
21
  instead_of: "sis"
22
+ group: inflection
17
23
  - pattern: ied
18
24
  instead_of: "y"
25
+ group: inflection
19
26
  - pattern: ed
20
27
  instead_of: ""
28
+ group: inflection
21
29
  - pattern: ed
22
30
  instead_of: "e"
31
+ group: inflection
23
32
  - pattern: ing
24
33
  instead_of: ""
34
+ group: inflection
25
35
  - pattern: ing
26
36
  instead_of: "e"
37
+ group: inflection
27
38
  - pattern: er
28
39
  instead_of: ""
40
+ group: inflection
29
41
  - pattern: er
30
42
  instead_of: "e"
43
+ group: inflection
31
44
  - pattern: ers
32
45
  instead_of: ""
46
+ group: inflection
33
47
  - pattern: ers
34
48
  instead_of: "e"
49
+ group: inflection
35
50
  - pattern: ier
36
51
  instead_of: "y"
52
+ group: inflection
37
53
  - pattern: iest
38
54
  instead_of: "y"
55
+ group: inflection
39
56
  - pattern: est
40
57
  instead_of: ""
58
+ group: inflection
41
59
  - pattern: est
42
60
  instead_of: "e"
61
+ group: inflection
62
+ # The adjective of a noun: "crunchy" is "crunch", "possessory" is "possessor", "stony" is "stone".
63
+ - pattern: y
64
+ instead_of: ""
65
+ - pattern: y
66
+ instead_of: "e"
43
67
  - pattern: ly
44
68
  instead_of: ""
45
69
  - pattern: ily
@@ -82,3 +106,12 @@ entries:
82
106
  instead_of: "ible"
83
107
  - pattern: men
84
108
  instead_of: "man"
109
+ group: inflection
110
+ # The follower of a doctrine and the doctrine: "relativist" is "relativism", "generalism" is "generalist".
111
+ - pattern: ist
112
+ instead_of: "ism"
113
+ - pattern: ism
114
+ instead_of: "ist"
115
+ # The adjective of an -ion noun: "propositional" is "proposition".
116
+ - pattern: ional
117
+ instead_of: "ion"
@@ -2,6 +2,8 @@
2
2
  # Only phrases whose short form says the same in every context. "with regard to", "prior to" and "in accordance with"
3
3
  # are left out: contracts and statutes use them on purpose. "a small number of" is left out too: "a few" names a
4
4
  # different count, and minutes use both as separate points of one scale (a few, a small number, some, several, many).
5
+ # "at all times" is left out: a rule states its scope with it (pets must be on a leash at all times), and "always" has to
6
+ # move before the verb and reads as a habit rather than a duty.
5
7
  # Matched as whole words, case aside.
6
8
  id: wordy-phrase
7
9
  language: en
@@ -72,8 +74,6 @@ entries:
72
74
  rewrite: near
73
75
  - pattern: in excess of
74
76
  rewrite: more than
75
- - pattern: at all times
76
- rewrite: always
77
77
  - pattern: has the capacity to
78
78
  rewrite: can
79
79
  - pattern: have the capacity to
@@ -34,6 +34,18 @@ entries:
34
34
  position: after
35
35
  - pattern: of the fiscal year
36
36
  position: after
37
+ - pattern: of each calendar year
38
+ position: after
39
+ - pattern: of every calendar year
40
+ position: after
41
+ - pattern: of each fiscal year
42
+ position: after
43
+ - pattern: of the calendar year
44
+ position: after
45
+ - pattern: of the year in which
46
+ position: after
47
+ - pattern: of the year following
48
+ position: after
37
49
  - pattern: annually
38
50
  position: before
39
51
  - pattern: each year
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@chaffjs/lang-en",
3
- "version": "0.34.0",
3
+ "version": "0.36.0",
4
4
  "description": "English language adapter for chaff",
5
5
  "license": "MIT",
6
6
  "author": "isamu",
package/src/citation.ts CHANGED
@@ -97,15 +97,32 @@ const titleWords = (rest: string): string[] => {
97
97
  return words;
98
98
  };
99
99
 
100
+ /** Labels the reference did not take into its address: the "(E)" of "section 552(a)(6)(E) of title 5". */
101
+ const UNREAD_LABELS = /^(?:\([A-Za-z0-9]{1,4}\))+/u;
102
+
103
+ /**
104
+ * "chapter 15 (§401 et seq.) of Title 50": the sections from this one on, glossing the unit before the bracket, which the
105
+ * name after the bracket governs.
106
+ */
107
+ const RUN_ON = /^ et seq\.\)?/u;
108
+
100
109
  /** What follows the "of" after a reference and its list, or undefined when no "of" follows. */
101
110
  const afterOf = (text: string, end: number): string | undefined => {
102
111
  const rest = text.slice(end, end + 200);
103
- const afterList = listEnd(rest);
112
+ const unread = UNREAD_LABELS.exec(rest)?.[0].length ?? 0;
113
+ const afterList = unread + listEnd(rest.slice(unread));
104
114
  const afterGloss = afterList + (GLOSS.exec(rest.slice(afterList))?.[0].length ?? 0);
105
- const of = OF.exec(rest.slice(afterGloss));
106
- return of === null ? undefined : rest.slice(afterGloss + of[0].length);
115
+ const afterRun = afterGloss + (RUN_ON.exec(rest.slice(afterGloss))?.[0].length ?? 0);
116
+ const of = OF.exec(rest.slice(afterRun));
117
+ return of === null ? undefined : rest.slice(afterRun + of[0].length);
107
118
  };
108
119
 
120
+ /**
121
+ * A code's title cited by its number: "section 552(b)(1) of title 5, United States Code", "section 1029 of title 18".
122
+ * An Act numbers its own titles in Roman ("title IV"), and "title 2 of this Act" says it is this document's.
123
+ */
124
+ const CODE_TITLE = /^[Tt]itle (?<n>\d{1,3})(?![\p{L}\p{N}]|\.\d)(?! of this\b)/u;
125
+
109
126
  /**
110
127
  * The document named right after a reference, or undefined when the reference is into this document.
111
128
  * "of this Agreement" and "of the Agreement" are this document; "of the Master Agreement" is another.
@@ -115,12 +132,25 @@ export const citedDocumentAfter = (text: string, end: number): string | undefine
115
132
  if (named === undefined) return undefined;
116
133
  const tag = TAG_AFTER.exec(named)?.groups?.["tag"];
117
134
  if (tag !== undefined) return tag;
135
+ const title = CODE_TITLE.exec(named)?.groups?.["n"];
136
+ if (title !== undefined) return `title ${title}`;
118
137
  const words = titleWords(named);
119
138
  if (words.length === 0) return undefined;
120
139
  const name = words.join(" ");
121
140
  return words.length === 1 && SELF.has(name) ? undefined : name;
122
141
  };
123
142
 
143
+ const THIS_UNIT = /^this (?<unit>\p{L}+)(?![\p{L}\p{N}])/u;
144
+
145
+ /**
146
+ * The unit the reference says it is in, "title" in "section 552 of this title", when that word is one of `units`
147
+ * (lower case); undefined otherwise.
148
+ */
149
+ export const enclosingUnitAfter = (text: string, end: number, units: ReadonlySet<string>): string | undefined => {
150
+ const unit = THIS_UNIT.exec(afterOf(text, end) ?? "")?.groups?.["unit"]?.toLowerCase();
151
+ return unit !== undefined && units.has(unit) ? unit : undefined;
152
+ };
153
+
124
154
  /**
125
155
  * A tag that names another document only if this document lists it, right after a reference ("Section 4.2.3 of
126
156
  * [HTTP-CACHING]", "Section 4.2.2.17 of [19]") or just before it ("[HTTP-CACHING], Section 4", "[23], Section 2.17").
package/src/index.ts CHANGED
@@ -1,6 +1,8 @@
1
1
  import { loadLexicons } from "./lexicons.ts";
2
2
  import { unmarkNumberStops } from "./number-stop.ts";
3
3
  import { labelStops, unmarkLabelStops } from "./label-stop.ts";
4
+ import { initialStops, unmarkInitialStops } from "./initial-stop.ts";
5
+ import { leadInitials, unmarkLeadInitials } from "./lead-initial.ts";
4
6
  import { sentenceSpans } from "./sentence-split.ts";
5
7
  import { splitAtQuotedStops } from "./quoted-stop.ts";
6
8
  import { reattachClosingQuotes } from "./closing-quote.ts";
@@ -15,6 +17,15 @@ import type { AdapterNeeds, EmbeddedLanguage, LanguageAdapter, Segmentation, Sen
15
17
 
16
18
  const LEXICONS = loadLexicons();
17
19
  const LABEL_STOPS = labelStops((LEXICONS["abbreviated-label"] ?? []).map((entry) => entry.pattern));
20
+ const INITIAL_STOPS = initialStops(
21
+ (LEXICONS["sentence-start-word"] ?? []).map((entry) => entry.pattern),
22
+ (LEXICONS["prepositive-abbreviation"] ?? []).map((entry) => entry.pattern),
23
+ (LEXICONS["name-abbreviation"] ?? []).map((entry) => entry.pattern),
24
+ );
25
+ const LEAD_INITIALS = leadInitials(
26
+ (LEXICONS["initial-lead-word"] ?? []).map((entry) => entry.pattern),
27
+ (LEXICONS["sentence-start-word"] ?? []).map((entry) => entry.pattern),
28
+ );
18
29
 
19
30
  const LATIN_LETTER = /[a-z]/giu;
20
31
  const COUNTABLE = /\S/gu;
@@ -40,7 +51,8 @@ const asListItem = (sentence: Sentence): Sentence => ({ ...sentence, listItem: t
40
51
 
41
52
  /** Spans of the splitter, cut again where a line break before a list item ends a sentence. */
42
53
  const sentencesOf = (text: string): Sentence[] => {
43
- const quotedStops = sentenceSpans(unmarkNumberStops(unmarkLabelStops(text, LABEL_STOPS))).flatMap((span) => splitAtQuotedStops(text, span));
54
+ const unmarked = unmarkLeadInitials(unmarkInitialStops(unmarkNumberStops(unmarkLabelStops(text, LABEL_STOPS)), INITIAL_STOPS), LEAD_INITIALS);
55
+ const quotedStops = sentenceSpans(unmarked).flatMap((span) => splitAtQuotedStops(text, span));
44
56
  const listBreaks = listLineBreaks(text);
45
57
  const itemHeads = listItemHeads(text);
46
58
  return reattachClosingQuotes(text, quotedStops)
@@ -0,0 +1,80 @@
1
+ import { escapeRegExp } from "./regexp.ts";
2
+
3
+ /**
4
+ * sentence-splitter reads a capital and a full stop as an initial and never ends a sentence there: "Regulation C. The
5
+ * Guide applies" and "Appendix B. In 2020 …" were one sentence. It also ends a sentence after an abbreviation such as
6
+ * "U.S." before any capital, so "the U.S. Census Bureau" was two. Both are decided by the word after the full stop:
7
+ * - capitals with full stops (C., D.C.) before a sentence-start-word (The, In) end the sentence. Their capitals are
8
+ * replaced with a plain letter, which the splitter does not read as an initial;
9
+ * - a prepositive abbreviation (U.S.) before any other capitalised word does not end it, unless a comma follows that
10
+ * word ("in the U.S. Yesterday, …"). Its last full stop is replaced with a letter;
11
+ * - a name abbreviation (Jas., Wm.) stands before a name, so it does not end it before any word but a
12
+ * sentence-start-word, even before a name and a comma ("Jas. Whittaker, J. Hocknell"). Its full stop is replaced
13
+ * with a letter;
14
+ * - capitals with full stops (J., U.K.) before a word in lower case ("J. meacham", "Hart, K. and J. Reed") do not end
15
+ * it, since no sentence opens in lower case. A word that is lower case only at its head (iPhone) may be a name that
16
+ * opens one, so it is left out. Their last full stop is replaced with a letter.
17
+ * The length does not change, so the spans fit the original text. Initials first on their line are a list's labels
18
+ * (A. The first item) and are left as they are.
19
+ */
20
+
21
+ export type InitialStops = { readonly ends: RegExp | undefined; readonly joins: RegExp | undefined; readonly names: RegExp | undefined };
22
+
23
+ const EDGE_BEFORE = String.raw`(?<![\p{L}\p{N}_.'’])`;
24
+ const WORD_END = String.raw`(?![\p{L}\p{N}_.'’-])`;
25
+ const PLAIN_LETTER = "n";
26
+ const CAPITAL = /\p{Lu}/gu;
27
+ const BEFORE_LOWER_CASE = new RegExp(String.raw`${EDGE_BEFORE}(?:\p{Lu}\.){1,3}(?=[ \t\r\n]+\p{Ll}(?!\p{Lu}))`, "gu");
28
+
29
+ const oneOf = (words: readonly string[]): string => `(?:${words.map(escapeRegExp).join("|")})`;
30
+
31
+ const startOf = (startWords: readonly string[]): string | undefined => (startWords.length === 0 ? undefined : String.raw`\s+${oneOf(startWords)}${WORD_END}`);
32
+
33
+ const sentenceStops = (start: string | undefined, prepositive: readonly string[]): Pick<InitialStops, "ends" | "joins"> => {
34
+ if (start === undefined) return { ends: undefined, joins: undefined };
35
+ const ends = new RegExp(String.raw`${EDGE_BEFORE}(?:\p{Lu}\.){1,3}(?=${start})`, "gu");
36
+ if (prepositive.length === 0) return { ends, joins: undefined };
37
+ return { ends, joins: new RegExp(String.raw`${EDGE_BEFORE}${oneOf(prepositive)}(?=\s+\p{Lu})(?!${start})(?!\s+\p{Lu}\p{L}*,)`, "gu") };
38
+ };
39
+
40
+ // "I" is not a sentence-start word because it is also an initial (John I. Smith); after a name abbreviation the pronoun
41
+ // is told by having no full stop of its own.
42
+ const PRONOUN_I = String.raw`(?!\s+I${WORD_END})`;
43
+
44
+ const notBefore = (start: string | undefined): string => (start === undefined ? "" : `(?!${start})`);
45
+
46
+ /** A name abbreviation before any word but a sentence-start word or the pronoun I. */
47
+ const nameJoins = (start: string | undefined, names: readonly string[]): RegExp | undefined => {
48
+ if (names.length === 0) return undefined;
49
+ return new RegExp(String.raw`${EDGE_BEFORE}${oneOf(names)}(?=\s+\p{L})${PRONOUN_I}${notBefore(start)}`, "gu");
50
+ };
51
+
52
+ /** Built once from the lexicons. */
53
+ export const initialStops = (startWords: readonly string[], prepositive: readonly string[], names: readonly string[] = []): InitialStops => {
54
+ const start = startOf(startWords);
55
+ return { ...sentenceStops(start, prepositive), names: nameJoins(start, names) };
56
+ };
57
+
58
+ /** Only a list's marker, or nothing, before the initials on their line (A. The first item; - B. The second item). */
59
+ const LABEL_HEAD = /^[ \t]*(?:[-*+•][ \t]+)?$/u;
60
+
61
+ const lineStartOf = (text: string, index: number): number => Math.max(text.lastIndexOf("\n", index - 1), text.lastIndexOf("\r", index - 1)) + 1;
62
+
63
+ const isFirstOnLine = (text: string, index: number): boolean => LABEL_HEAD.test(text.slice(lineStartOf(text, index), index));
64
+
65
+ const withoutLastStop = (abbreviation: string): string => `${abbreviation.slice(0, -1)}${PLAIN_LETTER}`;
66
+
67
+ const joined = (text: string, pattern: RegExp | undefined): string => (pattern === undefined ? text : text.replace(pattern, withoutLastStop));
68
+
69
+ /** Initials before a word in lower case, not first on their line (a list's label stays as it is), rewritten. */
70
+ const joinedBeforeLowerCase = (text: string): string =>
71
+ text.replace(BEFORE_LOWER_CASE, (initials: string, index: number) => (isFirstOnLine(text, index) ? initials : withoutLastStop(initials)));
72
+
73
+ /** The text with every initial that ends a sentence, and every prepositive or name abbreviation or initial that does not, rewritten. */
74
+ export const unmarkInitialStops = (text: string, stops: InitialStops): string => {
75
+ const ended =
76
+ stops.ends === undefined
77
+ ? text
78
+ : text.replace(stops.ends, (initials: string, index: number) => (isFirstOnLine(text, index) ? initials : initials.replace(CAPITAL, PLAIN_LETTER)));
79
+ return joinedBeforeLowerCase(joined(joined(ended, stops.joins), stops.names));
80
+ };
package/src/item-style.ts CHANGED
@@ -12,12 +12,19 @@ const AMBIGUOUS = /^[ivx]$/u;
12
12
  const DIGITS = /^\d+$/u;
13
13
  const CAPITALS = /^[A-Z]+$/u;
14
14
  const ONE_CAPITAL = /^[A-Z]$/u;
15
+ /**
16
+ * "(aa)", "(bb)": the US Code numbers the items under a subclause "(I)" with a doubled letter. Elsewhere a doubled letter
17
+ * is a letter: a UK Act inserts "(aa)" after "(a)", and a long list goes on from "(z)" to "(aa)".
18
+ */
19
+ const DOUBLED = /^(?<letter>[a-z])\k<letter>$/u;
15
20
 
16
- export type Style = "letter" | "roman" | "digit" | "capital" | "capital-roman";
21
+ export type Style = "letter" | "roman" | "digit" | "capital" | "capital-roman" | "doubled";
17
22
 
18
23
  /** 大文字で書いた同じ並び。"(B)" は英字の、"(II)" はローマ数字の大文字。 */
19
24
  const CAPITAL_OF: Readonly<Record<"letter" | "roman", Style>> = { letter: "capital", roman: "capital-roman" };
20
25
 
26
+ const isDoubled = (raw: string): boolean => DOUBLED.test(raw) && !MULTI_ROMAN.test(raw);
27
+
21
28
  /** 小文字にした番号を形で読む。一文字の "i" は、開いたときに付けた並びの位置で見分ける。ローマ数字なら 1、英字なら 9。 */
22
29
  const shapeOf = (lower: string, ordinal: number | undefined): "letter" | "roman" => {
23
30
  if (MULTI_ROMAN.test(lower)) return "roman";
@@ -29,15 +36,20 @@ export const styleOfOpen = (open: NumberedLine): Style | undefined => {
29
36
  const inner = /^\((?<n>[A-Za-z0-9]{1,5})\)$/u.exec(open.label)?.groups?.["n"];
30
37
  if (inner === undefined) return undefined;
31
38
  if (DIGITS.test(inner) || IS_INSERTED.test(inner)) return "digit";
39
+ if (isDoubled(inner) && open.ordinal !== undefined) return "doubled";
32
40
  const shape = shapeOf(inner.toLowerCase(), open.ordinal);
33
41
  return CAPITALS.test(inner) ? CAPITAL_OF[shape] : shape;
34
42
  };
35
43
 
36
44
  const LETTER_BEFORE_A = "a".charCodeAt(0) - 1;
37
45
 
38
- /** "(b)" と "(B)" は 2 番目、"(ii)" と "(II)" も 2 番目。二文字以上の英字("(aa)")は並びが決まらないので付けない。 */
46
+ /**
47
+ * "(b)" と "(B)" は 2 番目、"(ii)" と "(II)" も 2 番目、細目の下の重ねた "(bb)" も 2 番目。英字の並びの中の "(aa)" は、
48
+ * 何番目かは決めない。
49
+ */
39
50
  export const ordinalOf = (raw: string, style: Style): number | undefined => {
40
51
  if (style === "digit") return IS_INSERTED.test(raw) ? undefined : Number(raw);
52
+ if (style === "doubled") return raw.charCodeAt(0) - LETTER_BEFORE_A;
41
53
  if (style === "roman" || style === "capital-roman") return parseRoman(raw);
42
54
  return raw.length === 1 ? raw.toLowerCase().charCodeAt(0) - LETTER_BEFORE_A : undefined;
43
55
  };
@@ -49,10 +61,10 @@ const follows = (raw: string, style: "letter" | "capital", context: NumberingCon
49
61
  context.open.some((open, index) => styles(context)[index] === style && open.number.charCodeAt(0) + 1 === raw.charCodeAt(0));
50
62
 
51
63
  /**
52
- * 英字の並びは (a) から始まるので、"(a)" や "(1)" のすぐ下に来た "(i)" は一段深いローマ数字。
53
- * 米国の規則は (a)(1)(i) の順に下る。見出しのすぐ下の "(i)" は、どちらとも決まらないので英字。
64
+ * 英字の並びは (a) から始まるので、"(a)" や "(1)" や "(A)" のすぐ下に来た "(i)" は一段深いローマ数字。
65
+ * 米国の規則は (a)(1)(i)、法典は (a)(1)(A)(i) の順に下る。見出しのすぐ下の "(i)" は、どちらとも決まらないので英字。
54
66
  */
55
- const OPENS_ROMAN: ReadonlySet<Style | undefined> = new Set(["letter", "digit"]);
67
+ const OPENS_ROMAN: ReadonlySet<Style | undefined> = new Set(["letter", "digit", "capital"]);
56
68
 
57
69
  /** "(I)" は米国の法典で (i) の一段下の大文字ローマ数字。"(H)" の次なら英字。 */
58
70
  const capitalRoman = (raw: string, open: readonly (Style | undefined)[], context: NumberingContext): boolean => {
@@ -63,14 +75,27 @@ const capitalRoman = (raw: string, open: readonly (Style | undefined)[], context
63
75
  };
64
76
 
65
77
  /**
66
- * 米国の規則は (a)(1)(i)(A) と下る。大文字の "(A)" は、開いているローマ数字のすぐ下で始まるか、開いている大文字の続きのときだけ項目。
78
+ * 米国の規則(CFR)は (a)(1)(i)(A)、米国の法典(U.S.C.)は (a)(1)(A)(i) と下る。"(A)" はローマ数字のすぐ下か、英字の下の数字の
79
+ * すぐ下で始まる。見出しのすぐ下の "(1)" の下の "(A)" は、どちらの並びとも決まらない。
80
+ */
81
+ const opensCapital = (open: readonly (Style | undefined)[]): boolean =>
82
+ open.at(-1) === "roman" || (open.at(-1) === "digit" && open.slice(0, -1).includes("letter"));
83
+
84
+ /**
85
+ * 大文字の "(A)" は、上の並びのすぐ下で始まるか(opensCapital)、開いている大文字の続きのときだけ項目。
67
86
  * 英国の法令や契約書が一段目に使う "(A)" は、何の下とも決まらないので本文のまま読む。
68
87
  */
69
88
  const capitalStyleOf = (raw: string, context: NumberingContext): Style | undefined => {
70
89
  const open = styles(context);
71
90
  if (capitalRoman(raw, open, context)) return "capital-roman";
72
91
  if (!ONE_CAPITAL.test(raw)) return undefined;
73
- return open.includes("capital") || (raw === "A" && open.at(-1) === "roman") ? "capital" : undefined;
92
+ return open.includes("capital") || (raw === "A" && opensCapital(open)) ? "capital" : undefined;
93
+ };
94
+
95
+ /** A doubled letter right under a subclause "(I)", or beside another doubled letter. */
96
+ const underSubclause = (context: NumberingContext): boolean => {
97
+ const innermost = styles(context).at(-1);
98
+ return innermost === "capital-roman" || innermost === "doubled";
74
99
  };
75
100
 
76
101
  /**
@@ -80,6 +105,7 @@ const capitalStyleOf = (raw: string, context: NumberingContext): Style | undefin
80
105
  export const styleOf = (raw: string, context: NumberingContext): Style | undefined => {
81
106
  if (DIGITS.test(raw) || IS_INSERTED.test(raw)) return "digit";
82
107
  if (CAPITALS.test(raw)) return capitalStyleOf(raw, context);
108
+ if (isDoubled(raw)) return underSubclause(context) ? "doubled" : "letter";
83
109
  if (MULTI_ROMAN.test(raw)) return "roman";
84
110
  if (!AMBIGUOUS.test(raw) || follows(raw, "letter", context)) return "letter";
85
111
  const open = styles(context);
@@ -0,0 +1,68 @@
1
+ import type { Lexicon, LexiconEntry } from "chaffjs/plugin";
2
+ import { escapeRegExp } from "./regexp.ts";
3
+
4
+ // "Pub. L. 98–620, title IV, §402(2)" and "ch. 324, §2(a), 60 Stat. 237": a statute's notes cite a law by its number or by
5
+ // the volume that prints it. The section is that law's, not this document's. Which names are laws is the lexicon's
6
+ // (session-law). Pure.
7
+
8
+ /** Built once from the lexicon; each pattern is undefined when the lexicon names no law of its kind. */
9
+ export type LawVocabulary = { readonly before: RegExp | undefined; readonly after: RegExp | undefined };
10
+
11
+ /** A law's number is its Congress and its number in that Congress: "98–620", "89-487". */
12
+ const LAW_NUMBER = String.raw`\s+\d{1,3}[–-]\d{1,3}(?!\d)`;
13
+ /** "Pub. L. 98–620, " is far from "§402(2)" once "div. A, title IV, §201, title VI, " stands between them. */
14
+ const REACH = 160;
15
+ /**
16
+ * What may stand between a law's number and its section: its divisions, titles, subtitles and parts, other sections of
17
+ * it, and "formerly".
18
+ * The gap is these and nothing else; a word of a sentence ("see", "and the Act") ends the citation.
19
+ */
20
+ const CITATION_PARTS: readonly RegExp[] = [
21
+ /(?:div\.|subtitle|part|subchapter|ch\.)\s+[A-Z0-9]{1,4}/u,
22
+ /title\s+[IVXLC]{1,7}/u,
23
+ /formerly|§/u,
24
+ /\d{1,5}[A-Za-z]{0,2}/u,
25
+ /\([A-Za-z0-9]{1,4}\)/u,
26
+ /[\s,–-]/u,
27
+ ];
28
+ const CITATION_PART = new RegExp(CITATION_PARTS.map((part) => part.source).join("|"), "gu");
29
+ /** "§3.6, Dec. 29, 2009, 75 F.R. 718": a date may stand between a section and the volume's page. */
30
+ const DATE = String.raw`(?:,\s*[A-Z][a-z]{2,8}\.?\s+\d{1,2},\s+\d{4})?`;
31
+ /** "§2 (less (a)), 60 Stat. 237": a derivation table says in brackets which part of the section it takes. */
32
+ const PART_TAKEN = String.raw`(?:\s\((?:[^()]|\([^()]{1,10}\)){1,40}\))?`;
33
+
34
+ const namesOf = (entries: readonly LexiconEntry[]): string | undefined => {
35
+ const names = entries.map((entry) => entry.pattern).toSorted((left, right) => right.length - left.length);
36
+ return names.length === 0 ? undefined : `(?:${names.map(escapeRegExp).join("|")})`;
37
+ };
38
+
39
+ export const lawVocabulary = (lexicons: Readonly<Record<string, Lexicon>>): LawVocabulary => {
40
+ const entries = lexicons["session-law"] ?? [];
41
+ const named = namesOf(entries.filter((entry) => entry.position === "before"));
42
+ const volumes = namesOf(entries.filter((entry) => entry.position === "after"));
43
+ return {
44
+ before: named === undefined ? undefined : new RegExp(String.raw`(?<![\p{L}\p{N}])${named}${LAW_NUMBER}`, "gu"),
45
+ after: volumes === undefined ? undefined : new RegExp(String.raw`^${PART_TAKEN}${DATE},\s*\d{1,3}\s+${volumes}\s+\d`, "u"),
46
+ };
47
+ };
48
+
49
+ /** The last law named before `start` whose citation runs on, part by part, up to the reference. */
50
+ const lawBefore = (text: string, start: number, pattern: RegExp): string | undefined => {
51
+ const window = text.slice(Math.max(0, start - REACH), start);
52
+ const last = [...window.matchAll(pattern)].at(-1);
53
+ if (last === undefined) return undefined;
54
+ const gap = window.slice(last.index + last[0].length);
55
+ return gap.replaceAll(CITATION_PART, "") === "" ? last[0] : undefined;
56
+ };
57
+
58
+ /**
59
+ * The law a reference from `start` to `end` is a section of, or undefined when it is not cited as a law's section.
60
+ * Named before it ("Pub. L. 98–620, §402(2)"), or by the volume after it ("§2(a), 60 Stat. 237"), which names the law
61
+ * only by its page, so the volume stands for it.
62
+ */
63
+ export const citedLawAround = (text: string, start: number, end: number, vocabulary: LawVocabulary): string | undefined => {
64
+ const named = vocabulary.before === undefined ? undefined : lawBefore(text, start, vocabulary.before);
65
+ if (named !== undefined) return named;
66
+ const volume = vocabulary.after?.exec(text.slice(end, end + REACH))?.[0];
67
+ return volume === undefined ? undefined : volume.replace(/^[\s,]+/u, "");
68
+ };
@@ -0,0 +1,28 @@
1
+ import { escapeRegExp } from "./regexp.ts";
2
+
3
+ /**
4
+ * sentence-splitter joins an initial to the next word only when a capitalised word stands before it, so "written by
5
+ * J. Smith", "Smith and C.W. Landsea" and "by F. W. Evans" were each cut after the initials. After a word that leads a
6
+ * name (by, and), initials before a capitalised word are a name's, unless that word is a sentence-start-word
7
+ * ("items B and D. The rest") or the initials are the pronoun I ("you and I. Then"). Their full stops are replaced with
8
+ * a plain letter; the length does not change, so the spans fit the original text.
9
+ */
10
+
11
+ const EDGE_BEFORE = String.raw`(?<![\p{L}\p{N}_.'’])`;
12
+ const WORD_END = String.raw`(?![\p{L}\p{N}_.'’-])`;
13
+ const PLAIN_LETTER = "n";
14
+ const FULL_STOP = /\./gu;
15
+ const PRONOUN_I = /^I\.$/u;
16
+
17
+ const oneOf = (words: readonly string[]): string => `(?:${words.map(escapeRegExp).join("|")})`;
18
+
19
+ /** Built once from the lexicons: initials after a lead word, before a capitalised word that is not a sentence-start-word. */
20
+ export const leadInitials = (leadWords: readonly string[], startWords: readonly string[]): RegExp | undefined => {
21
+ if (leadWords.length === 0) return undefined;
22
+ const notStart = startWords.length === 0 ? "" : String.raw`(?!\s+${oneOf(startWords)}${WORD_END})`;
23
+ return new RegExp(String.raw`(?<=${EDGE_BEFORE}${oneOf(leadWords)}[ \t]+)(?:\p{Lu}\.[ \t]?){0,2}\p{Lu}\.(?=\s+\p{Lu})${notStart}`, "gu");
24
+ };
25
+
26
+ /** The text with the full stops of every name's initials after a lead word rewritten. */
27
+ export const unmarkLeadInitials = (text: string, pattern: RegExp | undefined): string =>
28
+ pattern === undefined ? text : text.replace(pattern, (initials: string) => (PRONOUN_I.test(initials) ? initials : initials.replace(FULL_STOP, PLAIN_LETTER)));