@chaffjs/lang-en 0.44.0 → 0.45.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +1 -1
- package/dist/index.js.map +1 -1
- package/dist/initial-stop.d.ts +7 -4
- package/dist/initial-stop.d.ts.map +1 -1
- package/dist/initial-stop.js +9 -5
- package/dist/initial-stop.js.map +1 -1
- package/lexicons/closing-abbreviation.yaml +7 -0
- package/lexicons/common-acronym.yaml +2 -0
- package/lexicons/product-name.yaml +10 -2
- package/lexicons/vague-word-law-reference.yaml +61 -0
- package/package.json +1 -1
- package/src/index.ts +1 -0
- package/src/initial-stop.ts +28 -8
package/dist/index.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAaA,OAAO,KAAK,EAAkC,eAAe,EAAgC,MAAM,gBAAgB,CAAC;
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAaA,OAAO,KAAK,EAAkC,eAAe,EAAgC,MAAM,gBAAgB,CAAC;AAmDpH;;;GAGG;AACH,eAAO,MAAM,OAAO,EAAE,eA6BrB,CAAC"}
|
package/dist/index.js
CHANGED
|
@@ -14,7 +14,7 @@ import { cutAtBreaks, listItemHeads, listLineBreaks, startsListItem } from "./li
|
|
|
14
14
|
// chaff からは型だけを取る。実行時の値依存を作らない。アダプタは単体で動く。
|
|
15
15
|
const LEXICONS = loadLexicons();
|
|
16
16
|
const LABEL_STOPS = labelStops((LEXICONS["abbreviated-label"] ?? []).map((entry) => entry.pattern));
|
|
17
|
-
const INITIAL_STOPS = initialStops((LEXICONS["sentence-start-word"] ?? []).map((entry) => entry.pattern), (LEXICONS["prepositive-abbreviation"] ?? []).map((entry) => entry.pattern), (LEXICONS["name-abbreviation"] ?? []).map((entry) => entry.pattern));
|
|
17
|
+
const INITIAL_STOPS = initialStops((LEXICONS["sentence-start-word"] ?? []).map((entry) => entry.pattern), (LEXICONS["prepositive-abbreviation"] ?? []).map((entry) => entry.pattern), (LEXICONS["name-abbreviation"] ?? []).map((entry) => entry.pattern), (LEXICONS["closing-abbreviation"] ?? []).map((entry) => entry.pattern));
|
|
18
18
|
const LEAD_INITIALS = leadInitials((LEXICONS["initial-lead-word"] ?? []).map((entry) => entry.pattern), (LEXICONS["sentence-start-word"] ?? []).map((entry) => entry.pattern));
|
|
19
19
|
const LATIN_LETTER = /[a-z]/giu;
|
|
20
20
|
const COUNTABLE = /\S/gu;
|
package/dist/index.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAE,MAAM,eAAe,CAAC;AAC7C,OAAO,EAAE,iBAAiB,EAAE,MAAM,kBAAkB,CAAC;AACrD,OAAO,EAAE,UAAU,EAAE,gBAAgB,EAAE,MAAM,iBAAiB,CAAC;AAC/D,OAAO,EAAE,YAAY,EAAE,kBAAkB,EAAE,MAAM,mBAAmB,CAAC;AACrE,OAAO,EAAE,YAAY,EAAE,kBAAkB,EAAE,MAAM,mBAAmB,CAAC;AACrE,OAAO,EAAE,aAAa,EAAE,MAAM,qBAAqB,CAAC;AACpD,OAAO,EAAE,kBAAkB,EAAE,MAAM,kBAAkB,CAAC;AACtD,OAAO,EAAE,qBAAqB,EAAE,MAAM,oBAAoB,CAAC;AAC3D,OAAO,EAAE,SAAS,EAAE,MAAM,gBAAgB,CAAC;AAC3C,OAAO,EAAE,UAAU,EAAE,MAAM,iBAAiB,CAAC;AAC7C,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,QAAQ,EAAE,MAAM,UAAU,CAAC;AACtD,OAAO,EAAE,aAAa,EAAE,MAAM,mBAAmB,CAAC;AAClD,OAAO,EAAE,WAAW,EAAE,aAAa,EAAE,cAAc,EAAE,cAAc,EAAE,MAAM,gBAAgB,CAAC;AAG5F,2CAA2C;AAE3C,MAAM,QAAQ,GAAG,YAAY,EAAE,CAAC;AAChC,MAAM,WAAW,GAAG,UAAU,CAAC,CAAC,QAAQ,CAAC,mBAAmB,CAAC,IAAI,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC;AACpG,MAAM,aAAa,GAAG,YAAY,CAChC,CAAC,QAAQ,CAAC,qBAAqB,CAAC,IAAI,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,OAAO,CAAC,EACrE,CAAC,QAAQ,CAAC,0BAA0B,CAAC,IAAI,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,OAAO,CAAC,EAC1E,CAAC,QAAQ,CAAC,mBAAmB,CAAC,IAAI,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,OAAO,CAAC,
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAE,MAAM,eAAe,CAAC;AAC7C,OAAO,EAAE,iBAAiB,EAAE,MAAM,kBAAkB,CAAC;AACrD,OAAO,EAAE,UAAU,EAAE,gBAAgB,EAAE,MAAM,iBAAiB,CAAC;AAC/D,OAAO,EAAE,YAAY,EAAE,kBAAkB,EAAE,MAAM,mBAAmB,CAAC;AACrE,OAAO,EAAE,YAAY,EAAE,kBAAkB,EAAE,MAAM,mBAAmB,CAAC;AACrE,OAAO,EAAE,aAAa,EAAE,MAAM,qBAAqB,CAAC;AACpD,OAAO,EAAE,kBAAkB,EAAE,MAAM,kBAAkB,CAAC;AACtD,OAAO,EAAE,qBAAqB,EAAE,MAAM,oBAAoB,CAAC;AAC3D,OAAO,EAAE,SAAS,EAAE,MAAM,gBAAgB,CAAC;AAC3C,OAAO,EAAE,UAAU,EAAE,MAAM,iBAAiB,CAAC;AAC7C,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,QAAQ,EAAE,MAAM,UAAU,CAAC;AACtD,OAAO,EAAE,aAAa,EAAE,MAAM,mBAAmB,CAAC;AAClD,OAAO,EAAE,WAAW,EAAE,aAAa,EAAE,cAAc,EAAE,cAAc,EAAE,MAAM,gBAAgB,CAAC;AAG5F,2CAA2C;AAE3C,MAAM,QAAQ,GAAG,YAAY,EAAE,CAAC;AAChC,MAAM,WAAW,GAAG,UAAU,CAAC,CAAC,QAAQ,CAAC,mBAAmB,CAAC,IAAI,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC;AACpG,MAAM,aAAa,GAAG,YAAY,CAChC,CAAC,QAAQ,CAAC,qBAAqB,CAAC,IAAI,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,OAAO,CAAC,EACrE,CAAC,QAAQ,CAAC,0BAA0B,CAAC,IAAI,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,OAAO,CAAC,EAC1E,CAAC,QAAQ,CAAC,mBAAmB,CAAC,IAAI,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,OAAO,CAAC,EACnE,CAAC,QAAQ,CAAC,sBAAsB,CAAC,IAAI,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,OAAO,CAAC,CACvE,CAAC;AACF,MAAM,aAAa,GAAG,YAAY,CAChC,CAAC,QAAQ,CAAC,mBAAmB,CAAC,IAAI,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,OAAO,CAAC,EACnE,CAAC,QAAQ,CAAC,qBAAqB,CAAC,IAAI,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,OAAO,CAAC,CACtE,CAAC;AAEF,MAAM,YAAY,GAAG,UAAU,CAAC;AAChC,MAAM,SAAS,GAAG,MAAM,CAAC;AAEzB;;;GAGG;AACH,MAAM,UAAU,GAAG,CAAC,QAAkB,EAAY,EAAE;IAClD,MAAM,MAAM,GAAG,QAAQ,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC;IACvC,IAAI,MAAM,KAAK,SAAS;QAAE,OAAO,QAAQ,CAAC;IAC1C,OAAO;QACL,GAAG,QAAQ;QACX,MAAM,EAAE,MAAM,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC,EAAE,GAAG,KAAK,EAAE,IAAI,EAAE,EAAE,KAAK,EAAE,QAAQ,CAAC,IAAI,CAAC,KAAK,GAAG,KAAK,CAAC,IAAI,CAAC,KAAK,EAAE,GAAG,EAAE,QAAQ,CAAC,IAAI,CAAC,KAAK,GAAG,KAAK,CAAC,IAAI,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC;KAClJ,CAAC;AACJ,CAAC,CAAC;AAEF,MAAM,QAAQ,GAAqB,EAAE,EAAE,EAAE,IAAI,EAAE,UAAU,EAAE,MAAM,EAAE,CAAC;AAEpE,MAAM,YAAY,GAAG,CAAC,IAAY,EAAE,IAAU,EAAY,EAAE,CAAC,CAAC,aAAa,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,IAAI,EAAE,gBAAgB,EAAE,QAAQ,EAAE,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC,CAAC;AAEjJ,MAAM,UAAU,GAAG,CAAC,QAAkB,EAAY,EAAE,CAAC,CAAC,EAAE,GAAG,QAAQ,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC,CAAC;AAEvF,8FAA8F;AAC9F,MAAM,WAAW,GAAG,CAAC,IAAY,EAAc,EAAE;IAC/C,MAAM,QAAQ,GAAG,kBAAkB,CAAC,kBAAkB,CAAC,iBAAiB,CAAC,gBAAgB,CAAC,IAAI,EAAE,WAAW,CAAC,CAAC,EAAE,aAAa,CAAC,EAAE,aAAa,CAAC,CAAC;IAC9I,MAAM,WAAW,GAAG,aAAa,CAAC,QAAQ,CAAC,CAAC,OAAO,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,kBAAkB,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC,CAAC;IAC9F,MAAM,UAAU,GAAG,cAAc,CAAC,IAAI,CAAC,CAAC;IACxC,MAAM,SAAS,GAAG,aAAa,CAAC,IAAI,CAAC,CAAC;IACtC,OAAO,qBAAqB,CAAC,IAAI,EAAE,WAAW,CAAC;SAC5C,OAAO,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,WAAW,CAAC,IAAI,EAAE,UAAU,CAAC,CAAC;SAChD,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,YAAY,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,KAAK,EAAE,IAAI,CAAC,GAAG,CAAC,EAAE,IAAI,CAAC,CAAC;SACnE,GAAG,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC,cAAc,CAAC,SAAS,EAAE,QAAQ,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,UAAU,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC;AAC3G,CAAC,CAAC;AAEF;;;GAGG;AACH,MAAM,CAAC,MAAM,OAAO,GAAoB;IACtC,IAAI,EAAE,UAAU;IAChB,EAAE,EAAE,IAAI;IACR,UAAU,EAAE,CAAC;IACb,YAAY,EAAE;QACZ,aAAa,EAAE,IAAI;QACnB,SAAS,EAAE,IAAI;QACf,6CAA6C;QAC7C,GAAG,EAAE,IAAI;QACT,KAAK,EAAE,IAAI;QACX,UAAU,EAAE,MAAM;KACnB;IACD,OAAO,EAAE,CAAC,IAAkB,EAAiB,EAAE;QAC7C,0CAA0C;QAC1C,IAAI,IAAI,CAAC,GAAG;YAAE,OAAO,EAAE,CAAC;QACxB,OAAO,OAAO,CAAC,OAAO,EAAE,CAAC;IAC3B,CAAC;IACD,MAAM,EAAE,CAAC,MAAc,EAAU,EAAE;QACjC,MAAM,KAAK,GAAG,CAAC,GAAG,MAAM,CAAC,QAAQ,CAAC,SAAS,CAAC,CAAC,CAAC,MAAM,CAAC;QACrD,IAAI,KAAK,KAAK,CAAC;YAAE,OAAO,CAAC,CAAC;QAC1B,OAAO,CAAC,GAAG,MAAM,CAAC,QAAQ,CAAC,YAAY,CAAC,CAAC,CAAC,MAAM,GAAG,KAAK,CAAC;IAC3D,CAAC;IACD,QAAQ,EAAE,QAAQ;IAClB,SAAS;IACT,UAAU;IACV,OAAO,EAAE,CAAC,IAAY,EAAgB,EAAE;QACtC,MAAM,SAAS,GAAG,WAAW,CAAC,IAAI,CAAC,CAAC;QACpC,OAAO,EAAE,SAAS,EAAE,OAAO,EAAE,CAAC,CAAC,CAAC,SAAS,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,SAAS,EAAE,CAAC;IAC1E,CAAC;CACF,CAAC"}
|
package/dist/initial-stop.d.ts
CHANGED
|
@@ -9,8 +9,10 @@
|
|
|
9
9
|
* - a name abbreviation (Jas., Wm.) stands before a name, so it does not end it before any word but a
|
|
10
10
|
* sentence-start-word, even before a name and a comma ("Jas. Whittaker, J. Hocknell"). Its full stop is replaced
|
|
11
11
|
* with a letter;
|
|
12
|
-
* -
|
|
13
|
-
*
|
|
12
|
+
* - an abbreviation that closes a list (etc.) before a sentence-start-word ends the sentence. The splitter joins it to
|
|
13
|
+
* the next sentence when more than one space or a line break comes between. Its letters are replaced with a plain letter;
|
|
14
|
+
* - capitals with full stops (J., U.K.), or two or three small letters with them (a.m., p.m.), before a word in lower case
|
|
15
|
+
* ("J. meacham", "Hart, K. and J. Reed", "9 a.m. and 4 p.m.") do not end it, since no sentence opens in lower case. A word that is lower case only at its head (iPhone) may be a name that
|
|
14
16
|
* opens one, so it is left out. Their last full stop is replaced with a letter.
|
|
15
17
|
* The length does not change, so the spans fit the original text. Initials first on their line are a list's labels
|
|
16
18
|
* (A. The first item) and are left as they are.
|
|
@@ -19,9 +21,10 @@ export type InitialStops = {
|
|
|
19
21
|
readonly ends: RegExp | undefined;
|
|
20
22
|
readonly joins: RegExp | undefined;
|
|
21
23
|
readonly names: RegExp | undefined;
|
|
24
|
+
readonly closings: RegExp | undefined;
|
|
22
25
|
};
|
|
23
26
|
/** Built once from the lexicons. */
|
|
24
|
-
export declare const initialStops: (startWords: readonly string[], prepositive: readonly string[], names?: readonly string[]) => InitialStops;
|
|
25
|
-
/** The text with every initial that ends a sentence, and every prepositive or name abbreviation or initial that does not, rewritten. */
|
|
27
|
+
export declare const initialStops: (startWords: readonly string[], prepositive: readonly string[], names?: readonly string[], closings?: readonly string[]) => InitialStops;
|
|
28
|
+
/** The text with every initial or closing abbreviation that ends a sentence, and every prepositive or name abbreviation or initial that does not, rewritten. */
|
|
26
29
|
export declare const unmarkInitialStops: (text: string, stops: InitialStops) => string;
|
|
27
30
|
//# sourceMappingURL=initial-stop.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"initial-stop.d.ts","sourceRoot":"","sources":["../src/initial-stop.ts"],"names":[],"mappings":"AAEA
|
|
1
|
+
{"version":3,"file":"initial-stop.d.ts","sourceRoot":"","sources":["../src/initial-stop.ts"],"names":[],"mappings":"AAEA;;;;;;;;;;;;;;;;;;GAkBG;AAEH,MAAM,MAAM,YAAY,GAAG;IACzB,QAAQ,CAAC,IAAI,EAAE,MAAM,GAAG,SAAS,CAAC;IAClC,QAAQ,CAAC,KAAK,EAAE,MAAM,GAAG,SAAS,CAAC;IACnC,QAAQ,CAAC,KAAK,EAAE,MAAM,GAAG,SAAS,CAAC;IACnC,QAAQ,CAAC,QAAQ,EAAE,MAAM,GAAG,SAAS,CAAC;CACvC,CAAC;AAoCF,oCAAoC;AACpC,eAAO,MAAM,YAAY,GACvB,YAAY,SAAS,MAAM,EAAE,EAC7B,aAAa,SAAS,MAAM,EAAE,EAC9B,QAAO,SAAS,MAAM,EAAO,EAC7B,WAAU,SAAS,MAAM,EAAO,KAC/B,YAGF,CAAC;AAoBF,gKAAgK;AAChK,eAAO,MAAM,kBAAkB,GAAI,MAAM,MAAM,EAAE,OAAO,YAAY,KAAG,MAMtE,CAAC"}
|
package/dist/initial-stop.js
CHANGED
|
@@ -3,7 +3,8 @@ const EDGE_BEFORE = String.raw `(?<![\p{L}\p{N}_.'’])`;
|
|
|
3
3
|
const WORD_END = String.raw `(?![\p{L}\p{N}_.'’-])`;
|
|
4
4
|
const PLAIN_LETTER = "n";
|
|
5
5
|
const CAPITAL = /\p{Lu}/gu;
|
|
6
|
-
const
|
|
6
|
+
const LETTER = /\p{L}/gu;
|
|
7
|
+
const BEFORE_LOWER_CASE = new RegExp(String.raw `${EDGE_BEFORE}(?:(?:\p{Lu}\.){1,3}|(?:\p{Ll}\.){2,3})(?=[ \t\r\n]+\p{Ll}(?!\p{Lu}))`, "gu");
|
|
7
8
|
const oneOf = (words) => `(?:${words.map(escapeRegExp).join("|")})`;
|
|
8
9
|
const startOf = (startWords) => (startWords.length === 0 ? undefined : String.raw `\s+${oneOf(startWords)}${WORD_END}`);
|
|
9
10
|
const sentenceStops = (start, prepositive) => {
|
|
@@ -24,10 +25,12 @@ const nameJoins = (start, names) => {
|
|
|
24
25
|
return undefined;
|
|
25
26
|
return new RegExp(String.raw `${EDGE_BEFORE}${oneOf(names)}(?=\s+\p{L})${PRONOUN_I}${notBefore(start)}`, "gu");
|
|
26
27
|
};
|
|
28
|
+
/** A closing abbreviation (etc.) before a sentence-start word. */
|
|
29
|
+
const closingEnds = (start, closings) => start === undefined || closings.length === 0 ? undefined : new RegExp(String.raw `${EDGE_BEFORE}${oneOf(closings)}(?=${start})`, "gu");
|
|
27
30
|
/** Built once from the lexicons. */
|
|
28
|
-
export const initialStops = (startWords, prepositive, names = []) => {
|
|
31
|
+
export const initialStops = (startWords, prepositive, names = [], closings = []) => {
|
|
29
32
|
const start = startOf(startWords);
|
|
30
|
-
return { ...sentenceStops(start, prepositive), names: nameJoins(start, names) };
|
|
33
|
+
return { ...sentenceStops(start, prepositive), names: nameJoins(start, names), closings: closingEnds(start, closings) };
|
|
31
34
|
};
|
|
32
35
|
/** Only a list's marker, or nothing, before the initials on their line (A. The first item; - B. The second item). */
|
|
33
36
|
const LABEL_HEAD = /^[ \t]*(?:[-*+•][ \t]+)?$/u;
|
|
@@ -35,13 +38,14 @@ const lineStartOf = (text, index) => Math.max(text.lastIndexOf("\n", index - 1),
|
|
|
35
38
|
const isFirstOnLine = (text, index) => LABEL_HEAD.test(text.slice(lineStartOf(text, index), index));
|
|
36
39
|
const withoutLastStop = (abbreviation) => `${abbreviation.slice(0, -1)}${PLAIN_LETTER}`;
|
|
37
40
|
const joined = (text, pattern) => (pattern === undefined ? text : text.replace(pattern, withoutLastStop));
|
|
41
|
+
const closed = (text, pattern) => pattern === undefined ? text : text.replace(pattern, (abbreviation) => abbreviation.replace(LETTER, PLAIN_LETTER));
|
|
38
42
|
/** Initials before a word in lower case, not first on their line (a list's label stays as it is), rewritten. */
|
|
39
43
|
const joinedBeforeLowerCase = (text) => text.replace(BEFORE_LOWER_CASE, (initials, index) => (isFirstOnLine(text, index) ? initials : withoutLastStop(initials)));
|
|
40
|
-
/** The text with every initial that ends a sentence, and every prepositive or name abbreviation or initial that does not, rewritten. */
|
|
44
|
+
/** The text with every initial or closing abbreviation that ends a sentence, and every prepositive or name abbreviation or initial that does not, rewritten. */
|
|
41
45
|
export const unmarkInitialStops = (text, stops) => {
|
|
42
46
|
const ended = stops.ends === undefined
|
|
43
47
|
? text
|
|
44
48
|
: text.replace(stops.ends, (initials, index) => (isFirstOnLine(text, index) ? initials : initials.replace(CAPITAL, PLAIN_LETTER)));
|
|
45
|
-
return joinedBeforeLowerCase(joined(joined(ended, stops.joins), stops.names));
|
|
49
|
+
return joinedBeforeLowerCase(joined(joined(closed(ended, stops.closings), stops.joins), stops.names));
|
|
46
50
|
};
|
|
47
51
|
//# sourceMappingURL=initial-stop.js.map
|
package/dist/initial-stop.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"initial-stop.js","sourceRoot":"","sources":["../src/initial-stop.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;
|
|
1
|
+
{"version":3,"file":"initial-stop.js","sourceRoot":"","sources":["../src/initial-stop.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AA6B3C,MAAM,WAAW,GAAG,MAAM,CAAC,GAAG,CAAA,uBAAuB,CAAC;AACtD,MAAM,QAAQ,GAAG,MAAM,CAAC,GAAG,CAAA,uBAAuB,CAAC;AACnD,MAAM,YAAY,GAAG,GAAG,CAAC;AACzB,MAAM,OAAO,GAAG,UAAU,CAAC;AAC3B,MAAM,MAAM,GAAG,SAAS,CAAC;AACzB,MAAM,iBAAiB,GAAG,IAAI,MAAM,CAAC,MAAM,CAAC,GAAG,CAAA,GAAG,WAAW,uEAAuE,EAAE,IAAI,CAAC,CAAC;AAE5I,MAAM,KAAK,GAAG,CAAC,KAAwB,EAAU,EAAE,CAAC,MAAM,KAAK,CAAC,GAAG,CAAC,YAAY,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC;AAE/F,MAAM,OAAO,GAAG,CAAC,UAA6B,EAAsB,EAAE,CAAC,CAAC,UAAU,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAA,MAAM,KAAK,CAAC,UAAU,CAAC,GAAG,QAAQ,EAAE,CAAC,CAAC;AAE9J,MAAM,aAAa,GAAG,CAAC,KAAyB,EAAE,WAA8B,EAAwC,EAAE;IACxH,IAAI,KAAK,KAAK,SAAS;QAAE,OAAO,EAAE,IAAI,EAAE,SAAS,EAAE,KAAK,EAAE,SAAS,EAAE,CAAC;IACtE,MAAM,IAAI,GAAG,IAAI,MAAM,CAAC,MAAM,CAAC,GAAG,CAAA,GAAG,WAAW,uBAAuB,KAAK,GAAG,EAAE,IAAI,CAAC,CAAC;IACvF,IAAI,WAAW,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,EAAE,IAAI,EAAE,KAAK,EAAE,SAAS,EAAE,CAAC;IAChE,OAAO,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,MAAM,CAAC,MAAM,CAAC,GAAG,CAAA,GAAG,WAAW,GAAG,KAAK,CAAC,WAAW,CAAC,mBAAmB,KAAK,uBAAuB,EAAE,IAAI,CAAC,EAAE,CAAC;AACzI,CAAC,CAAC;AAEF,wHAAwH;AACxH,6CAA6C;AAC7C,MAAM,SAAS,GAAG,MAAM,CAAC,GAAG,CAAA,UAAU,QAAQ,GAAG,CAAC;AAElD,MAAM,SAAS,GAAG,CAAC,KAAyB,EAAU,EAAE,CAAC,CAAC,KAAK,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,MAAM,KAAK,GAAG,CAAC,CAAC;AAErG,sFAAsF;AACtF,MAAM,SAAS,GAAG,CAAC,KAAyB,EAAE,KAAwB,EAAsB,EAAE;IAC5F,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,SAAS,CAAC;IACzC,OAAO,IAAI,MAAM,CAAC,MAAM,CAAC,GAAG,CAAA,GAAG,WAAW,GAAG,KAAK,CAAC,KAAK,CAAC,eAAe,SAAS,GAAG,SAAS,CAAC,KAAK,CAAC,EAAE,EAAE,IAAI,CAAC,CAAC;AAChH,CAAC,CAAC;AAEF,kEAAkE;AAClE,MAAM,WAAW,GAAG,CAAC,KAAyB,EAAE,QAA2B,EAAsB,EAAE,CACjG,KAAK,KAAK,SAAS,IAAI,QAAQ,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI,MAAM,CAAC,MAAM,CAAC,GAAG,CAAA,GAAG,WAAW,GAAG,KAAK,CAAC,QAAQ,CAAC,MAAM,KAAK,GAAG,EAAE,IAAI,CAAC,CAAC;AAExI,oCAAoC;AACpC,MAAM,CAAC,MAAM,YAAY,GAAG,CAC1B,UAA6B,EAC7B,WAA8B,EAC9B,QAA2B,EAAE,EAC7B,WAA8B,EAAE,EAClB,EAAE;IAChB,MAAM,KAAK,GAAG,OAAO,CAAC,UAAU,CAAC,CAAC;IAClC,OAAO,EAAE,GAAG,aAAa,CAAC,KAAK,EAAE,WAAW,CAAC,EAAE,KAAK,EAAE,SAAS,CAAC,KAAK,EAAE,KAAK,CAAC,EAAE,QAAQ,EAAE,WAAW,CAAC,KAAK,EAAE,QAAQ,CAAC,EAAE,CAAC;AAC1H,CAAC,CAAC;AAEF,qHAAqH;AACrH,MAAM,UAAU,GAAG,4BAA4B,CAAC;AAEhD,MAAM,WAAW,GAAG,CAAC,IAAY,EAAE,KAAa,EAAU,EAAE,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,WAAW,CAAC,IAAI,EAAE,KAAK,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,WAAW,CAAC,IAAI,EAAE,KAAK,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC;AAEhJ,MAAM,aAAa,GAAG,CAAC,IAAY,EAAE,KAAa,EAAW,EAAE,CAAC,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,WAAW,CAAC,IAAI,EAAE,KAAK,CAAC,EAAE,KAAK,CAAC,CAAC,CAAC;AAE7H,MAAM,eAAe,GAAG,CAAC,YAAoB,EAAU,EAAE,CAAC,GAAG,YAAY,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,GAAG,YAAY,EAAE,CAAC;AAExG,MAAM,MAAM,GAAG,CAAC,IAAY,EAAE,OAA2B,EAAU,EAAE,CAAC,CAAC,OAAO,KAAK,SAAS,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,OAAO,EAAE,eAAe,CAAC,CAAC,CAAC;AAE9I,MAAM,MAAM,GAAG,CAAC,IAAY,EAAE,OAA2B,EAAU,EAAE,CACnE,OAAO,KAAK,SAAS,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,OAAO,EAAE,CAAC,YAAoB,EAAE,EAAE,CAAC,YAAY,CAAC,OAAO,CAAC,MAAM,EAAE,YAAY,CAAC,CAAC,CAAC;AAE7H,gHAAgH;AAChH,MAAM,qBAAqB,GAAG,CAAC,IAAY,EAAU,EAAE,CACrD,IAAI,CAAC,OAAO,CAAC,iBAAiB,EAAE,CAAC,QAAgB,EAAE,KAAa,EAAE,EAAE,CAAC,CAAC,aAAa,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,eAAe,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC;AAE5I,gKAAgK;AAChK,MAAM,CAAC,MAAM,kBAAkB,GAAG,CAAC,IAAY,EAAE,KAAmB,EAAU,EAAE;IAC9E,MAAM,KAAK,GACT,KAAK,CAAC,IAAI,KAAK,SAAS;QACtB,CAAC,CAAC,IAAI;QACN,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC,QAAgB,EAAE,KAAa,EAAE,EAAE,CAAC,CAAC,aAAa,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,QAAQ,CAAC,OAAO,CAAC,OAAO,EAAE,YAAY,CAAC,CAAC,CAAC,CAAC;IACvJ,OAAO,qBAAqB,CAAC,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,KAAK,EAAE,KAAK,CAAC,QAAQ,CAAC,EAAE,KAAK,CAAC,KAAK,CAAC,EAAE,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC;AACxG,CAAC,CAAC"}
|
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
# Abbreviations that close a list or a phrase ("phone, mail, etc."), so their full stop can end a sentence. Before a
|
|
2
|
+
# sentence-start-word the full stop ends it, even across two spaces or a line break, where the splitter joined the two
|
|
3
|
+
# sentences. Matched as written.
|
|
4
|
+
id: closing-abbreviation
|
|
5
|
+
language: en
|
|
6
|
+
entries:
|
|
7
|
+
- pattern: etc.
|
|
@@ -76,6 +76,7 @@ entries:
|
|
|
76
76
|
- pattern: CEO
|
|
77
77
|
- pattern: CTO
|
|
78
78
|
- pattern: CFO
|
|
79
|
+
- pattern: HR # the office every workplace has (call HR), an everyday word
|
|
79
80
|
- pattern: US
|
|
80
81
|
- pattern: USA
|
|
81
82
|
- pattern: UK
|
|
@@ -105,6 +106,7 @@ entries:
|
|
|
105
106
|
- pattern: HIV
|
|
106
107
|
- pattern: UV
|
|
107
108
|
- pattern: FYI
|
|
109
|
+
- pattern: NB # nota bene, a Latin abbreviation read like i.e. and e.g.
|
|
108
110
|
- pattern: LGBTQ
|
|
109
111
|
- pattern: CD
|
|
110
112
|
- pattern: ROM
|
|
@@ -1,5 +1,6 @@
|
|
|
1
|
-
# Product names that end in capitals (Adobe Acrobat Reader DC)
|
|
2
|
-
# not an acronym the writer spells out. Only the whole name is
|
|
1
|
+
# Product names that end in capitals (Adobe Acrobat Reader DC), and names led by the maker's short name in capitals
|
|
2
|
+
# (MS Excel). The capitals are part of the product's name, not an acronym the writer spells out. Only the whole name is
|
|
3
|
+
# matched, as written: DC or MS alone stays an acronym.
|
|
3
4
|
id: product-name
|
|
4
5
|
language: en
|
|
5
6
|
entries:
|
|
@@ -7,3 +8,10 @@ entries:
|
|
|
7
8
|
- pattern: Acrobat Pro DC
|
|
8
9
|
- pattern: Acrobat Standard DC
|
|
9
10
|
- pattern: Acrobat DC
|
|
11
|
+
- pattern: MS Excel
|
|
12
|
+
- pattern: MS Word
|
|
13
|
+
- pattern: MS Office
|
|
14
|
+
- pattern: MS PowerPoint
|
|
15
|
+
- pattern: MS Outlook
|
|
16
|
+
- pattern: MS Access
|
|
17
|
+
- pattern: MS Teams
|
|
@@ -0,0 +1,61 @@
|
|
|
1
|
+
# A vague word that a law or a section governs: "as required by applicable law", "as required under 45 CFR §164.528",
|
|
2
|
+
# "as required by §164.410 of HIPAA". The law it names fixes what is required, so vague-word-density does not count the
|
|
3
|
+
# word there. group: governor is a word written right after the vague word that hands it to a law (by, under); group:
|
|
4
|
+
# reference is a word that names a law or a section, found among the next few words. A name a party gives ("as required
|
|
5
|
+
# by Licensor") names no law, and the word is still counted.
|
|
6
|
+
id: vague-word-law-reference
|
|
7
|
+
language: en
|
|
8
|
+
entries:
|
|
9
|
+
- pattern: by
|
|
10
|
+
group: governor
|
|
11
|
+
- pattern: under
|
|
12
|
+
group: governor
|
|
13
|
+
- pattern: pursuant to
|
|
14
|
+
group: governor
|
|
15
|
+
- pattern: in accordance with
|
|
16
|
+
group: governor
|
|
17
|
+
- pattern: law
|
|
18
|
+
group: reference
|
|
19
|
+
- pattern: laws
|
|
20
|
+
group: reference
|
|
21
|
+
- pattern: statute
|
|
22
|
+
group: reference
|
|
23
|
+
- pattern: statutes
|
|
24
|
+
group: reference
|
|
25
|
+
- pattern: regulation
|
|
26
|
+
group: reference
|
|
27
|
+
- pattern: regulations
|
|
28
|
+
group: reference
|
|
29
|
+
- pattern: rule
|
|
30
|
+
group: reference
|
|
31
|
+
- pattern: rules
|
|
32
|
+
group: reference
|
|
33
|
+
- pattern: act
|
|
34
|
+
group: reference
|
|
35
|
+
- pattern: code
|
|
36
|
+
group: reference
|
|
37
|
+
- pattern: section
|
|
38
|
+
group: reference
|
|
39
|
+
- pattern: subsection
|
|
40
|
+
group: reference
|
|
41
|
+
- pattern: article
|
|
42
|
+
group: reference
|
|
43
|
+
- pattern: §
|
|
44
|
+
group: reference
|
|
45
|
+
- pattern: cfr
|
|
46
|
+
group: reference
|
|
47
|
+
- pattern: c.f.r.
|
|
48
|
+
group: reference
|
|
49
|
+
- pattern: usc
|
|
50
|
+
group: reference
|
|
51
|
+
- pattern: u.s.c.
|
|
52
|
+
group: reference
|
|
53
|
+
# Laws and standards known by their short names.
|
|
54
|
+
- pattern: hipaa
|
|
55
|
+
group: reference
|
|
56
|
+
- pattern: gdpr
|
|
57
|
+
group: reference
|
|
58
|
+
- pattern: ccpa
|
|
59
|
+
group: reference
|
|
60
|
+
- pattern: gaap
|
|
61
|
+
group: reference
|
package/package.json
CHANGED
package/src/index.ts
CHANGED
|
@@ -21,6 +21,7 @@ const INITIAL_STOPS = initialStops(
|
|
|
21
21
|
(LEXICONS["sentence-start-word"] ?? []).map((entry) => entry.pattern),
|
|
22
22
|
(LEXICONS["prepositive-abbreviation"] ?? []).map((entry) => entry.pattern),
|
|
23
23
|
(LEXICONS["name-abbreviation"] ?? []).map((entry) => entry.pattern),
|
|
24
|
+
(LEXICONS["closing-abbreviation"] ?? []).map((entry) => entry.pattern),
|
|
24
25
|
);
|
|
25
26
|
const LEAD_INITIALS = leadInitials(
|
|
26
27
|
(LEXICONS["initial-lead-word"] ?? []).map((entry) => entry.pattern),
|
package/src/initial-stop.ts
CHANGED
|
@@ -11,20 +11,28 @@ import { escapeRegExp } from "./regexp.ts";
|
|
|
11
11
|
* - a name abbreviation (Jas., Wm.) stands before a name, so it does not end it before any word but a
|
|
12
12
|
* sentence-start-word, even before a name and a comma ("Jas. Whittaker, J. Hocknell"). Its full stop is replaced
|
|
13
13
|
* with a letter;
|
|
14
|
-
* -
|
|
15
|
-
*
|
|
14
|
+
* - an abbreviation that closes a list (etc.) before a sentence-start-word ends the sentence. The splitter joins it to
|
|
15
|
+
* the next sentence when more than one space or a line break comes between. Its letters are replaced with a plain letter;
|
|
16
|
+
* - capitals with full stops (J., U.K.), or two or three small letters with them (a.m., p.m.), before a word in lower case
|
|
17
|
+
* ("J. meacham", "Hart, K. and J. Reed", "9 a.m. and 4 p.m.") do not end it, since no sentence opens in lower case. A word that is lower case only at its head (iPhone) may be a name that
|
|
16
18
|
* opens one, so it is left out. Their last full stop is replaced with a letter.
|
|
17
19
|
* The length does not change, so the spans fit the original text. Initials first on their line are a list's labels
|
|
18
20
|
* (A. The first item) and are left as they are.
|
|
19
21
|
*/
|
|
20
22
|
|
|
21
|
-
export type InitialStops = {
|
|
23
|
+
export type InitialStops = {
|
|
24
|
+
readonly ends: RegExp | undefined;
|
|
25
|
+
readonly joins: RegExp | undefined;
|
|
26
|
+
readonly names: RegExp | undefined;
|
|
27
|
+
readonly closings: RegExp | undefined;
|
|
28
|
+
};
|
|
22
29
|
|
|
23
30
|
const EDGE_BEFORE = String.raw`(?<![\p{L}\p{N}_.'’])`;
|
|
24
31
|
const WORD_END = String.raw`(?![\p{L}\p{N}_.'’-])`;
|
|
25
32
|
const PLAIN_LETTER = "n";
|
|
26
33
|
const CAPITAL = /\p{Lu}/gu;
|
|
27
|
-
const
|
|
34
|
+
const LETTER = /\p{L}/gu;
|
|
35
|
+
const BEFORE_LOWER_CASE = new RegExp(String.raw`${EDGE_BEFORE}(?:(?:\p{Lu}\.){1,3}|(?:\p{Ll}\.){2,3})(?=[ \t\r\n]+\p{Ll}(?!\p{Lu}))`, "gu");
|
|
28
36
|
|
|
29
37
|
const oneOf = (words: readonly string[]): string => `(?:${words.map(escapeRegExp).join("|")})`;
|
|
30
38
|
|
|
@@ -49,10 +57,19 @@ const nameJoins = (start: string | undefined, names: readonly string[]): RegExp
|
|
|
49
57
|
return new RegExp(String.raw`${EDGE_BEFORE}${oneOf(names)}(?=\s+\p{L})${PRONOUN_I}${notBefore(start)}`, "gu");
|
|
50
58
|
};
|
|
51
59
|
|
|
60
|
+
/** A closing abbreviation (etc.) before a sentence-start word. */
|
|
61
|
+
const closingEnds = (start: string | undefined, closings: readonly string[]): RegExp | undefined =>
|
|
62
|
+
start === undefined || closings.length === 0 ? undefined : new RegExp(String.raw`${EDGE_BEFORE}${oneOf(closings)}(?=${start})`, "gu");
|
|
63
|
+
|
|
52
64
|
/** Built once from the lexicons. */
|
|
53
|
-
export const initialStops = (
|
|
65
|
+
export const initialStops = (
|
|
66
|
+
startWords: readonly string[],
|
|
67
|
+
prepositive: readonly string[],
|
|
68
|
+
names: readonly string[] = [],
|
|
69
|
+
closings: readonly string[] = [],
|
|
70
|
+
): InitialStops => {
|
|
54
71
|
const start = startOf(startWords);
|
|
55
|
-
return { ...sentenceStops(start, prepositive), names: nameJoins(start, names) };
|
|
72
|
+
return { ...sentenceStops(start, prepositive), names: nameJoins(start, names), closings: closingEnds(start, closings) };
|
|
56
73
|
};
|
|
57
74
|
|
|
58
75
|
/** Only a list's marker, or nothing, before the initials on their line (A. The first item; - B. The second item). */
|
|
@@ -66,15 +83,18 @@ const withoutLastStop = (abbreviation: string): string => `${abbreviation.slice(
|
|
|
66
83
|
|
|
67
84
|
const joined = (text: string, pattern: RegExp | undefined): string => (pattern === undefined ? text : text.replace(pattern, withoutLastStop));
|
|
68
85
|
|
|
86
|
+
const closed = (text: string, pattern: RegExp | undefined): string =>
|
|
87
|
+
pattern === undefined ? text : text.replace(pattern, (abbreviation: string) => abbreviation.replace(LETTER, PLAIN_LETTER));
|
|
88
|
+
|
|
69
89
|
/** Initials before a word in lower case, not first on their line (a list's label stays as it is), rewritten. */
|
|
70
90
|
const joinedBeforeLowerCase = (text: string): string =>
|
|
71
91
|
text.replace(BEFORE_LOWER_CASE, (initials: string, index: number) => (isFirstOnLine(text, index) ? initials : withoutLastStop(initials)));
|
|
72
92
|
|
|
73
|
-
/** The text with every initial that ends a sentence, and every prepositive or name abbreviation or initial that does not, rewritten. */
|
|
93
|
+
/** The text with every initial or closing abbreviation that ends a sentence, and every prepositive or name abbreviation or initial that does not, rewritten. */
|
|
74
94
|
export const unmarkInitialStops = (text: string, stops: InitialStops): string => {
|
|
75
95
|
const ended =
|
|
76
96
|
stops.ends === undefined
|
|
77
97
|
? text
|
|
78
98
|
: text.replace(stops.ends, (initials: string, index: number) => (isFirstOnLine(text, index) ? initials : initials.replace(CAPITAL, PLAIN_LETTER)));
|
|
79
|
-
return joinedBeforeLowerCase(joined(joined(ended, stops.joins), stops.names));
|
|
99
|
+
return joinedBeforeLowerCase(joined(joined(closed(ended, stops.closings), stops.joins), stops.names));
|
|
80
100
|
};
|