@echogarden/text-segmentation 0.3.2 → 0.3.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/Patterns.d.ts +7 -2
- package/dist/Patterns.js +29 -23
- package/dist/Patterns.js.map +1 -1
- package/dist/Suppressions.js +2 -3
- package/dist/Suppressions.js.map +1 -1
- package/dist/Test.js +20 -22
- package/dist/Test.js.map +1 -1
- package/dist/TextSegmentation.js +50 -14
- package/dist/TextSegmentation.js.map +1 -1
- package/package.json +2 -2
- package/src/Patterns.ts +47 -30
- package/src/Suppressions.ts +2 -3
- package/src/Test.ts +22 -23
- package/src/TextSegmentation.ts +62 -15
package/dist/Patterns.d.ts
CHANGED
|
@@ -1,9 +1,14 @@
|
|
|
1
1
|
export declare function buildWordOrNumberPattern(suppressions: string[]): import("regexp-composer").AnyOf;
|
|
2
2
|
export declare function buildSuppressionPattern(suppressions: string[]): import("regexp-composer").Pattern;
|
|
3
|
+
export declare const whitespacePattern: (import("regexp-composer").SpecialToken | import("regexp-composer").OneOrMore)[];
|
|
4
|
+
export declare const startsWithWhitespacePattern: import("regexp-composer").SpecialToken[];
|
|
5
|
+
export declare const endsWithWhitespacePattern: import("regexp-composer").SpecialToken[];
|
|
3
6
|
export declare const wordCharacterRegExp: RegExp;
|
|
4
7
|
export declare const whitespacePatternRegExp: RegExp;
|
|
5
8
|
export declare const letterPatternGlobalRegExp: RegExp;
|
|
6
9
|
export declare const phraseSeparatorRegExp: RegExp;
|
|
7
10
|
export declare const phraseSeparatorTrailingPunctuationRegExp: RegExp;
|
|
8
|
-
export declare const
|
|
9
|
-
export declare const
|
|
11
|
+
export declare const sentenceSeparatorCharacterRegExp: RegExp;
|
|
12
|
+
export declare const sentenceSeparatorTrailingPunctuationCharacterRegExp: RegExp;
|
|
13
|
+
export declare const startsWithWhitespaceRegExp: RegExp;
|
|
14
|
+
export declare const endsWithWhitespaceRegExp: RegExp;
|
package/dist/Patterns.js
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { anyOf, buildRegExp, charRange, inputEnd, inputStart, matches, oneOrMore, possibly, repeated, tab, unicodeProperty, whitespace, zeroOrMore } from 'regexp-composer';
|
|
1
|
+
import { anyOf, buildRegExp, charRange, codepoint, inputEnd, inputStart, matches, oneOrMore, possibly, repeated, tab, unicodeProperty, whitespace, zeroOrMore } from 'regexp-composer';
|
|
2
2
|
////////////////////////////////////////////////////////////////////////////////////////////////
|
|
3
3
|
// Pattern builder methods
|
|
4
4
|
////////////////////////////////////////////////////////////////////////////////////////////////
|
|
@@ -14,12 +14,14 @@ export function buildSuppressionPattern(suppressions) {
|
|
|
14
14
|
////////////////////////////////////////////////////////////////////////////////////////////////
|
|
15
15
|
const letterPattern = unicodeProperty('Letter');
|
|
16
16
|
const markPattern = unicodeProperty('Mark');
|
|
17
|
-
const letterOrMarkPattern = anyOf(letterPattern, markPattern);
|
|
18
|
-
const apostrophPattern = anyOf(`'`, `’`, `‘`);
|
|
19
|
-
const punctuationPattern = unicodeProperty('Punctuation');
|
|
20
17
|
const digitPattern = unicodeProperty('Decimal_Number');
|
|
18
|
+
const punctuationPattern = unicodeProperty('Punctuation');
|
|
19
|
+
const apostrophPattern = anyOf(`'`, `’`, `‘`);
|
|
21
20
|
const arabicNumeralPattern = charRange('0', '9');
|
|
22
|
-
const
|
|
21
|
+
const letterOrMarkPattern = anyOf(letterPattern, markPattern);
|
|
22
|
+
const letterOrMarkOrDigitPattern = anyOf(letterPattern, markPattern, digitPattern);
|
|
23
|
+
// See: https://mathiasbynens.be/notes/es-unicode-property-escapes
|
|
24
|
+
const emojiPattern = anyOf([unicodeProperty('Emoji_Modifier_Base'), unicodeProperty('Emoji_Modifier')], unicodeProperty('Emoji_Presentation'), [unicodeProperty('Emoji'), codepoint('FE0F')]);
|
|
23
25
|
const percentageCharacters = ['%'];
|
|
24
26
|
const currencyCharacters = ['$', '¥', '€', '£', '₩', '₭', '₽', '₫', '฿', '¢', '₮', '؋', '₦', '₱', '₴', '₪'];
|
|
25
27
|
const percentageOrCurrencyCharacterPattern = anyOf(...percentageCharacters, ...currencyCharacters);
|
|
@@ -61,12 +63,15 @@ const exponentPattern = [
|
|
|
61
63
|
possibly(anyOf('+', '-')),
|
|
62
64
|
oneOrMore(arabicNumeralPattern),
|
|
63
65
|
];
|
|
64
|
-
const numberPossiblyFollowedByExponentOrLettersPattern = [
|
|
66
|
+
const numberPossiblyFollowedByExponentOrLettersPattern = matches([
|
|
65
67
|
numberPattern,
|
|
66
|
-
possibly(anyOf(exponentPattern, matches(zeroOrMore(unicodeProperty('Letter')), {
|
|
68
|
+
possibly(anyOf(exponentPattern, matches(zeroOrMore(anyOf(unicodeProperty('Letter'), '_')), {
|
|
67
69
|
ifNotFollowedBy: digitPattern,
|
|
68
70
|
})))
|
|
69
|
-
]
|
|
71
|
+
], {
|
|
72
|
+
ifPrecededBy: anyOf(whitespace, punctuationPattern, inputStart),
|
|
73
|
+
ifFollowedBy: anyOf(whitespace, punctuationPattern, inputEnd),
|
|
74
|
+
});
|
|
70
75
|
const precedingPercentageOrCurrencyPattern = matches([
|
|
71
76
|
percentageOrCurrencyCharacterPattern,
|
|
72
77
|
numberPattern,
|
|
@@ -94,7 +99,7 @@ const timePattern = matches([
|
|
|
94
99
|
], {
|
|
95
100
|
ifPrecededBy: anyOf(whitespace, punctuationPattern, inputStart),
|
|
96
101
|
ifNotPrecededBy: ':',
|
|
97
|
-
ifFollowedBy: anyOf(whitespace, punctuationPattern, inputEnd),
|
|
102
|
+
ifFollowedBy: anyOf(whitespace, punctuationPattern, inputEnd, 'am', 'AM', 'pm', 'PM'),
|
|
98
103
|
ifNotFollowedBy: ':',
|
|
99
104
|
});
|
|
100
105
|
////////////////////////////////////////////////////////////////////////////////////////////////
|
|
@@ -121,14 +126,14 @@ const dottedAbbreviationSequencePattern = matches(anyOf([
|
|
|
121
126
|
});
|
|
122
127
|
const wordCharacterPattern = anyOf(letterPattern, markPattern, digitPattern);
|
|
123
128
|
const wordInnerApostrophPattern = matches(apostrophPattern, {
|
|
124
|
-
ifPrecededBy:
|
|
129
|
+
ifPrecededBy: letterOrMarkOrDigitPattern,
|
|
125
130
|
ifFollowedBy: letterOrMarkPattern,
|
|
126
131
|
});
|
|
127
132
|
const wordStartApostrophPattern = matches(apostrophPattern, {
|
|
128
133
|
ifPrecededBy: whitespace,
|
|
129
134
|
ifFollowedBy: [letterOrMarkPattern, letterOrMarkPattern, anyOf(whitespace, inputEnd)]
|
|
130
135
|
});
|
|
131
|
-
const basicWordPattern = oneOrMore(anyOf(wordCharacterPattern, wordInnerApostrophPattern));
|
|
136
|
+
const basicWordPattern = oneOrMore(anyOf(wordCharacterPattern, wordInnerApostrophPattern, '_'));
|
|
132
137
|
const hyphenatedWordPattern = [
|
|
133
138
|
basicWordPattern,
|
|
134
139
|
oneOrMore([
|
|
@@ -153,14 +158,7 @@ const interpunctSeparatedWordPattern = [
|
|
|
153
158
|
basicWordPattern,
|
|
154
159
|
]),
|
|
155
160
|
];
|
|
156
|
-
const
|
|
157
|
-
basicWordPattern,
|
|
158
|
-
oneOrMore([
|
|
159
|
-
'_',
|
|
160
|
-
basicWordPattern,
|
|
161
|
-
]),
|
|
162
|
-
];
|
|
163
|
-
const wordSegmentPattern = anyOf(timePattern, dimensionsPattern, percentageOrCurrencyPattern, numberPossiblyFollowedByExponentOrLettersPattern, emojiPattern, interpunctSeparatedWordPattern, dotSeparatedWordPattern, dottedAbbreviationSequencePattern, underscoreSeparatedWordPattern, basicWordPattern);
|
|
161
|
+
const wordSegmentPattern = anyOf(emojiPattern, timePattern, percentageOrCurrencyPattern, dotSeparatedWordPattern, numberPossiblyFollowedByExponentOrLettersPattern, interpunctSeparatedWordPattern, dottedAbbreviationSequencePattern, basicWordPattern);
|
|
164
162
|
////////////////////////////////////////////////////////////////////////////////////////////////
|
|
165
163
|
// Phrase separation patterns
|
|
166
164
|
////////////////////////////////////////////////////////////////////////////////////////////////
|
|
@@ -179,7 +177,7 @@ const phraseSeparatorTrailingPunctuationPattern = [
|
|
|
179
177
|
// Sentence separation patterns
|
|
180
178
|
////////////////////////////////////////////////////////////////////////////////////////////////
|
|
181
179
|
const sentenceSeparatorCharacters = ['.', '。', '?', '？', '!', '！', '\n'];
|
|
182
|
-
const
|
|
180
|
+
const sentenceSeparatorCharacterPattern = [
|
|
183
181
|
inputStart,
|
|
184
182
|
anyOf(...sentenceSeparatorCharacters),
|
|
185
183
|
inputEnd
|
|
@@ -190,13 +188,21 @@ const sentenceSeparatorTrailingPunctuationPattern = [
|
|
|
190
188
|
inputEnd
|
|
191
189
|
];
|
|
192
190
|
////////////////////////////////////////////////////////////////////////////////////////////////
|
|
191
|
+
// Other patterns
|
|
192
|
+
////////////////////////////////////////////////////////////////////////////////////////////////
|
|
193
|
+
export const whitespacePattern = [inputStart, oneOrMore(whitespace), inputEnd];
|
|
194
|
+
export const startsWithWhitespacePattern = [inputStart, whitespace];
|
|
195
|
+
export const endsWithWhitespacePattern = [whitespace, inputEnd];
|
|
196
|
+
////////////////////////////////////////////////////////////////////////////////////////////////
|
|
193
197
|
// Prebuilt regular expressions
|
|
194
198
|
////////////////////////////////////////////////////////////////////////////////////////////////
|
|
195
199
|
export const wordCharacterRegExp = buildRegExp(wordCharacterPattern);
|
|
196
|
-
export const whitespacePatternRegExp = buildRegExp(
|
|
200
|
+
export const whitespacePatternRegExp = buildRegExp(whitespacePattern);
|
|
197
201
|
export const letterPatternGlobalRegExp = buildRegExp(letterPattern, { global: true });
|
|
198
202
|
export const phraseSeparatorRegExp = buildRegExp(phraseSeparatorPattern);
|
|
199
203
|
export const phraseSeparatorTrailingPunctuationRegExp = buildRegExp(phraseSeparatorTrailingPunctuationPattern);
|
|
200
|
-
export const
|
|
201
|
-
export const
|
|
204
|
+
export const sentenceSeparatorCharacterRegExp = buildRegExp(sentenceSeparatorCharacterPattern);
|
|
205
|
+
export const sentenceSeparatorTrailingPunctuationCharacterRegExp = buildRegExp(sentenceSeparatorTrailingPunctuationPattern);
|
|
206
|
+
export const startsWithWhitespaceRegExp = buildRegExp(startsWithWhitespacePattern);
|
|
207
|
+
export const endsWithWhitespaceRegExp = buildRegExp(endsWithWhitespacePattern);
|
|
202
208
|
//# sourceMappingURL=Patterns.js.map
|
package/dist/Patterns.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"Patterns.js","sourceRoot":"","sources":["../src/Patterns.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,EAAE,WAAW,EAAE,SAAS,EAAE,QAAQ,EAAE,UAAU,EAAE,OAAO,EAAE,SAAS,EAAE,QAAQ,EAAE,QAAQ,EAAE,GAAG,EAAE,eAAe,EAAE,UAAU,EAAE,UAAU,EAAE,MAAM,iBAAiB,CAAA;
|
|
1
|
+
{"version":3,"file":"Patterns.js","sourceRoot":"","sources":["../src/Patterns.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,EAAE,WAAW,EAAE,SAAS,EAAE,SAAS,EAAE,QAAQ,EAAE,UAAU,EAAE,OAAO,EAAE,SAAS,EAAE,QAAQ,EAAE,QAAQ,EAAE,GAAG,EAAE,eAAe,EAAE,UAAU,EAAE,UAAU,EAAE,MAAM,iBAAiB,CAAA;AAEtL,gGAAgG;AAChG,0BAA0B;AAC1B,gGAAgG;AAChG,MAAM,UAAU,wBAAwB,CAAC,YAAsB;IAC9D,OAAO,KAAK,CACX,uBAAuB,CAAC,YAAY,CAAC,EACrC,kBAAkB,CAClB,CAAA;AACF,CAAC;AAED,MAAM,UAAU,uBAAuB,CAAC,YAAsB;IAC7D,MAAM,mBAAmB,GACxB,OAAO,CACN,KAAK,CAAC,GAAG,YAAY,CAAC,EACtB,EAAE,eAAe,EAAE,oBAAoB,EAAE,CACzC,CAAA;IAEF,OAAO,mBAAmB,CAAA;AAC3B,CAAC;AAED,gGAAgG;AAChG,4BAA4B;AAC5B,gGAAgG;AAChG,MAAM,aAAa,GAAG,eAAe,CAAC,QAAQ,CAAC,CAAA;AAC/C,MAAM,WAAW,GAAG,eAAe,CAAC,MAAM,CAAC,CAAA;AAC3C,MAAM,YAAY,GAAG,eAAe,CAAC,gBAAgB,CAAC,CAAA;AACtD,MAAM,kBAAkB,GAAG,eAAe,CAAC,aAAa,CAAC,CAAA;AAEzD,MAAM,gBAAgB,GAAG,KAAK,CAAC,GAAG,EAAE,GAAG,EAAE,GAAG,CAAC,CAAA;AAC7C,MAAM,oBAAoB,GAAG,SAAS,CAAC,GAAG,EAAE,GAAG,CAAC,CAAA;AAEhD,MAAM,mBAAmB,GAAG,KAAK,CAChC,aAAa,EACb,WAAW,CACX,CAAA;AAED,MAAM,0BAA0B,GAAG,KAAK,CACvC,aAAa,EACb,WAAW,EACX,YAAY,CACZ,CAAA;AAED,kEAAkE;AAClE,MAAM,YAAY,GAAG,KAAK,CACzB,CAAC,eAAe,CAAC,qBAAqB,CAAC,EAAE,eAAe,CAAC,gBAAgB,CAAC,CAAC,EAC3E,eAAe,CAAC,oBAAoB,CAAC,EACrC,CAAC,eAAe,CAAC,OAAO,CAAC,EAAE,SAAS,CAAC,MAAM,CAAC,CAAC,CAC7C,CAAA;AAED,MAAM,oBAAoB,GAAG,CAAC,GAAG,CAAC,CAAA;AAClC,MAAM,kBAAkB,GAAG,CAAC,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,CAAC,CAAA;AAE3G,MAAM,oCAAoC,GAAG,KAAK,CAAC,GAAG,oBAAoB,EAAE,GAAG,kBAAkB,CAAC,CAAA;AAElG,gGAAgG;AAChG,mBAAmB;AACnB,gGAAgG;AAChG,MAAM,uBAAuB,GAC5B,OAAO,CACN,KAAK,CAAC,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,CAAC,EAAE;IAC3B,YAAY,EAAE,oBAAoB;IAClC,YAAY,EAAE,oBAAoB;CAClC,CAAC,CAAA;AAEH,MAAM,iBAAiB,GAAG,OAAO,CAAC;IACjC,SAAS,CAAC,oBAAoB,CAAC;IAE/B,SAAS,CAAC;QACT,GAAG;QACH,SAAS,CAAC,oBAAoB,CAAC;KAC/B,CAAC;CACF,EAAE;IACF,YAAY,EAAE,KAAK,CAAC,UAAU,EAAE,kBAAkB,EAAE,UAAU,CAAC;IAC/D,YAAY,EAAE,KAAK,CAAC,UAAU,EAAE,kBAAkB,EAAE,QAAQ,CAAC;CAC7D,CAAC,CAAA;AAEF,MAAM,+BAA+B,GACpC,OAAO,CACN,GAAG,EAAE;IACL,YAAY,EAAE,oBAAoB;IAClC,YAAY,EAAE;QACb,QAAQ,CAAC,CAAC,EAAE,oBAAoB,CAAC;QACjC,KAAK,CAAC,UAAU,EAAE,kBAAkB,EAAE,QAAQ,CAAC;KAC/C;CACD,CAAC,CAAA;AAEH,MAAM,kBAAkB,GACvB,OAAO,CACN,KAAK,CAAC,GAAG,EAAE,GAAG,CAAC,EAAE;IACjB,YAAY,EAAE,KAAK,CAAC,UAAU,EAAE,kBAAkB,EAAE,UAAU,CAAC;IAC/D,YAAY,EAAE,oBAAoB;CAClC,CAAC,CAAA;AAEH,MAAM,aAAa,GAAG;IACrB,QAAQ,CAAC,kBAAkB,CAAC;IAC5B,YAAY;IAEZ,UAAU,CAAC,KAAK,CACf,YAAY,EACZ,uBAAuB,EACvB,+BAA+B,CAC/B,CAAC;CACF,CAAA;AAED,MAAM,eAAe,GAAG;IACvB,KAAK,CAAC,GAAG,EAAE,GAAG,CAAC;IACf,QAAQ,CAAC,KAAK,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC;IACzB,SAAS,CAAC,oBAAoB,CAAC;CAC/B,CAAA;AAED,MAAM,gDAAgD,GAAG,OAAO,CAAC;IAChE,aAAa;IAEb,QAAQ,CAAC,KAAK,CACb,eAAe,EAEf,OAAO,CACN,UAAU,CAAC,KAAK,CAAC,eAAe,CAAC,QAAQ,CAAC,EAAE,GAAG,CAAC,CAAC,EAAE;QACnD,eAAe,EAAE,YAAY;KAC7B,CAAC,CACF,CAAC;CAAC,EAAE;IAEL,YAAY,EAAE,KAAK,CAAC,UAAU,EAAE,kBAAkB,EAAE,UAAU,CAAC;IAC/D,YAAY,EAAE,KAAK,CAAC,UAAU,EAAE,kBAAkB,EAAE,QAAQ,CAAC;CAC7D,CAAC,CAAA;AAGF,MAAM,oCAAoC,GACzC,OAAO,CAAC;IACP,oCAAoC;IACpC,aAAa;CACb,EAAE;IACF,eAAe,EAAE,YAAY;IAC7B,YAAY,EAAE,KAAK,CAAC,UAAU,EAAE,kBAAkB,EAAE,QAAQ,CAAC;CAC7D,CAAC,CAAA;AAEH,MAAM,oCAAoC,GACzC,OAAO,CAAC;IACP,aAAa;IACb,oCAAoC;CACpC,EAAE;IACF,YAAY,EAAE,KAAK,CAAC,UAAU,EAAE,kBAAkB,EAAE,UAAU,CAAC;IAC/D,eAAe,EAAE,KAAK,CAAC,YAAY,CAAC;CACpC,CAAC,CAAA;AAEH,MAAM,2BAA2B,GAAG,KAAK,CACxC,oCAAoC,EACpC,oCAAoC,CACpC,CAAA;AAED,gGAAgG;AAChG,gBAAgB;AAChB,gGAAgG;AAChG,MAAM,WAAW,GAAG,OAAO,CAAC;IAC3B,KAAK,CAAC,oBAAoB,EAAE,CAAC,SAAS,CAAC,GAAG,EAAE,GAAG,CAAC,EAAE,oBAAoB,CAAC,EAAE,CAAC,GAAG,EAAE,SAAS,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC,CAAC;IAEpG,QAAQ,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE;QAChB,GAAG;QACH,KAAK,CAAC,oBAAoB,EAAE,CAAC,SAAS,CAAC,GAAG,EAAE,GAAG,CAAC,EAAE,oBAAoB,CAAC,CAAC;KACxE,CAAC;CACF,EAAE;IACF,YAAY,EAAE,KAAK,CAAC,UAAU,EAAE,kBAAkB,EAAE,UAAU,CAAC;IAC/D,eAAe,EAAE,GAAG;IACpB,YAAY,EAAE,KAAK,CAAC,UAAU,EAAE,kBAAkB,EAAE,QAAQ,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,CAAC;IACrF,eAAe,EAAE,GAAG;CACpB,CAAC,CAAA;AAEF,gGAAgG;AAChG,kBAAkB;AAClB,gGAAgG;AAChG,MAAM,iCAAiC,GACtC,OAAO,CACN,KAAK,CACJ;IACC,aAAa;IAEb,SAAS,CAAC;QACT,IAAI;QACJ,aAAa;KACb,CAAC;IAEF,QAAQ,CAAC,GAAG,CAAC;CACb,EACD;IACC,aAAa;IAEb,SAAS,CAAC;QACT,GAAG;QACH,QAAQ,CAAC,GAAG,CAAC;QACb,aAAa;KACb,CAAC;IAEF,QAAQ,CAAC,GAAG,CAAC;CACb,CACD,EAAE;IACH,eAAe,EAAE,KAAK,CAAC,mBAAmB,EAAE,YAAY,CAAC;IACzD,eAAe,EAAE,KAAK,CAAC,mBAAmB,EAAE,YAAY,CAAC;CACzD,CAAC,CAAA;AAEH,MAAM,oBAAoB,GACzB,KAAK,CACJ,aAAa,EACb,WAAW,EACX,YAAY,CACZ,CAAA;AAEF,MAAM,yBAAyB,GAC9B,OAAO,CACN,gBAAgB,EAAE;IAElB,YAAY,EAAE,0BAA0B;IACxC,YAAY,EAAE,mBAAmB;CACjC,CAAC,CAAA;AAEH,MAAM,yBAAyB,GAC9B,OAAO,CACN,gBAAgB,EAAE;IAElB,YAAY,EAAE,UAAU;IACxB,YAAY,EAAE,CAAC,mBAAmB,EAAE,mBAAmB,EAAE,KAAK,CAAC,UAAU,EAAE,QAAQ,CAAC,CAAC;CACrF,CAAC,CAAA;AAEH,MAAM,gBAAgB,GACrB,SAAS,CACR,KAAK,CACJ,oBAAoB,EACpB,yBAAyB,EACzB,GAAG,CACH,CACD,CAAA;AAEF,MAAM,qBAAqB,GAAG;IAC7B,gBAAgB;IAEhB,SAAS,CAAC;QACT,GAAG;QACH,gBAAgB;KAChB,CAAC;CACF,CAAA;AAED,MAAM,uBAAuB,GAAG,OAAO,CAAC;IACvC,gBAAgB;IAEhB,SAAS,CAAC;QACT,GAAG;QACH,gBAAgB;KAChB,CAAC;CACF,EAAE;AACF,8CAA8C;AAC9C,2CAA2C;CAC3C,CAAC,CAAA;AAEF,MAAM,8BAA8B,GAAG;IACtC,gBAAgB;IAEhB,SAAS,CAAC;QACT,KAAK,CAAC,GAAG,EAAE,GAAG,CAAC;QACf,gBAAgB;KAChB,CAAC;CACF,CAAA;AAED,MAAM,kBAAkB,GAAG,KAAK,CAC/B,YAAY,EAEZ,WAAW,EACX,2BAA2B,EAE3B,uBAAuB,EACvB,gDAAgD,EAEhD,8BAA8B,EAC9B,iCAAiC,EACjC,gBAAgB,CAChB,CAAA;AAED,gGAAgG;AAChG,6BAA6B;AAC7B,gGAAgG;AAChG,MAAM,yBAAyB,GAAG,CAAC,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,CAAC,CAAA;AAE/E,MAAM,sBAAsB,GAAG;IAC9B,UAAU;IACV,KAAK,CAAC,GAAG,yBAAyB,CAAC;IACnC,QAAQ;CACR,CAAA;AAED,MAAM,yCAAyC,GAAG;IACjD,UAAU;IACV,KAAK,CAAC,GAAG,yBAAyB,EAAE,GAAG,EAAE,GAAG,CAAC;IAC7C,QAAQ;CACR,CAAA;AAED,gGAAgG;AAChG,+BAA+B;AAC/B,gGAAgG;AAChG,MAAM,2BAA2B,GAAG,CAAC,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,IAAI,CAAC,CAAA;AAExE,MAAM,iCAAiC,GAAG;IACzC,UAAU;IACV,KAAK,CAAC,GAAG,2BAA2B,CAAC;IACrC,QAAQ;CACR,CAAA;AAED,MAAM,2CAA2C,GAAG;IACnD,UAAU;IACV,KAAK,CAAC,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,2BAA2B,EAAE,GAAG,yBAAyB,EAAE,SAAS,CAAC,UAAU,CAAC,CAAC;IAC7H,QAAQ;CACR,CAAA;AAED,gGAAgG;AAChG,iBAAiB;AACjB,gGAAgG;AAEhG,MAAM,CAAC,MAAM,iBAAiB,GAAG,CAAC,UAAU,EAAE,SAAS,CAAC,UAAU,CAAC,EAAE,QAAQ,CAAC,CAAA;AAC9E,MAAM,CAAC,MAAM,2BAA2B,GAAG,CAAC,UAAU,EAAE,UAAU,CAAC,CAAA;AACnE,MAAM,CAAC,MAAM,yBAAyB,GAAG,CAAC,UAAU,EAAE,QAAQ,CAAC,CAAA;AAE/D,gGAAgG;AAChG,+BAA+B;AAC/B,gGAAgG;AAChG,MAAM,CAAC,MAAM,mBAAmB,GAAG,WAAW,CAAC,oBAAoB,CAAC,CAAA;AACpE,MAAM,CAAC,MAAM,uBAAuB,GAAG,WAAW,CAAC,iBAAiB,CAAC,CAAA;AAErE,MAAM,CAAC,MAAM,yBAAyB,GAAG,WAAW,CAAC,aAAa,EAAE,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC,CAAA;AAErF,MAAM,CAAC,MAAM,qBAAqB,GAAG,WAAW,CAAC,sBAAsB,CAAC,CAAA;AACxE,MAAM,CAAC,MAAM,wCAAwC,GAAG,WAAW,CAAC,yCAAyC,CAAC,CAAA;AAE9G,MAAM,CAAC,MAAM,gCAAgC,GAAG,WAAW,CAAC,iCAAiC,CAAC,CAAA;AAC9F,MAAM,CAAC,MAAM,mDAAmD,GAAG,WAAW,CAAC,2CAA2C,CAAC,CAAA;AAE3H,MAAM,CAAC,MAAM,0BAA0B,GAAG,WAAW,CAAC,2BAA2B,CAAC,CAAA;AAClF,MAAM,CAAC,MAAM,wBAAwB,GAAG,WAAW,CAAC,yBAAyB,CAAC,CAAA"}
|
package/dist/Suppressions.js
CHANGED
|
@@ -9,9 +9,8 @@ export const cldrSuppressions = {
|
|
|
9
9
|
ru: ['руб.', 'янв.', 'до н. э.', 'сент.', 'тел.', 'дек.', 'февр.', 'нояб.', 'апр.', 'н. э.', 'окт.', 'тыс.', 'авг.', 'проф.', 'н.э.', 'кв.', 'ул.', 'отд.'],
|
|
10
10
|
};
|
|
11
11
|
export const additionalSuppressions = {
|
|
12
|
-
en: [
|
|
13
|
-
|
|
14
|
-
],
|
|
12
|
+
en: ['TL;DR'],
|
|
13
|
+
th: ['ค.ศ.'],
|
|
15
14
|
};
|
|
16
15
|
export const leadingApostropheContractionSuppressions = {
|
|
17
16
|
'en': [
|
package/dist/Suppressions.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"Suppressions.js","sourceRoot":"","sources":["../src/Suppressions.ts"],"names":[],"mappings":"AAAA,MAAM,CAAC,MAAM,gBAAgB,GAA6B;IACzD,6pCAA6pC;IAC7pC,EAAE,EAAE,CAAC,MAAM,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,EAAE,KAAK,EAAE,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,KAAK,EAAE,KAAK,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,QAAQ,EAAE,KAAK,EAAE,MAAM,EAAE,QAAQ,EAAE,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,IAAI,EAAE,OAAO,EAAE,KAAK,EAAE,MAAM,EAAE,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,EAAE,KAAK,EAAE,OAAO,EAAE,MAAM,EAAE,OAAO,EAAE,UAAU,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,KAAK,EAAE,KAAK,EAAE,QAAQ,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,KAAK,CAAC;IAC/1B,EAAE,EAAE,CAAC,OAAO,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,SAAS,EAAE,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,QAAQ,EAAE,KAAK,EAAE,MAAM,EAAE,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,KAAK,EAAE,KAAK,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,EAAE,MAAM,EAAE,IAAI,EAAE,OAAO,EAAE,UAAU,EAAE,KAAK,EAAE,SAAS,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,IAAI,EAAE,OAAO,EAAE,SAAS,EAAE,MAAM,EAAE,MAAM,EAAE,YAAY,EAAE,IAAI,EAAE,IAAI,EAAE,OAAO,EAAE,IAAI,EAAE,KAAK,EAAE,KAAK,EAAE,QAAQ,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,UAAU,EAAE,OAAO,EAAE,MAAM,EAAE,IAAI,EAAE,IAAI,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,EAAE,KAAK,EAAE,KAAK,EAAE,KAAK,EAAE,MAAM,EAAE,KAAK,EAAE,OAAO,EAAE,QAAQ,EAAE,IAAI,EAAE,MAAM,EAAE,SAAS,EAAE,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,QAAQ,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,IAAI,EAAE,OAAO,EAAE,MAAM,EAAE,KAAK,EAAE,QAAQ,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,QAAQ,EAAE,QAAQ,EAAE,IAAI,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,UAAU,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,IAAI,EAAE,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,QAAQ,EAAE,UAAU,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,UAAU,EAAE,MAAM,EAAE,IAAI,EAAE,YAAY,EAAE,MAAM,EAAE,OAAO,EAAE,KAAK,EAAE,MAAM,EAAE,IAAI,EAAE,IAAI,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,QAAQ,EAAE,OAAO,EAAE,KAAK,EAAE,MAAM,EAAE,WAAW,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,KAAK,EAAE,OAAO,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,IAAI,EAAE,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,MAAM,EAAE,QAAQ,EAAE,QAAQ,EAAE,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,OAAO,EAAE,WAAW,EAAE,KAAK,EAAE,QAAQ,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,IAAI,EAAE,UAAU,EAAE,QAAQ,EAAE,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,IAAI,EAAE,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,KAAK,EAAE,IAAI,EAAE,OAAO,EAAE,IAAI,EAAE,YAAY,EAAE,YAAY,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,IAAI,EAAE,QAAQ,EAAE,OAAO,EAAE,WAAW,EAAE,MAAM,EAAE,QAAQ,EAAE,SAAS,CAAC;IAC9+D,EAAE,EAAE,CAAC,OAAO,EAAE,QAAQ,EAAE,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,QAAQ,EAAE,OAAO,EAAE,SAAS,EAAE,MAAM,EAAE,MAAM,EAAE,QAAQ,EAAE,KAAK,EAAE,MAAM,EAAE,OAAO,EAAE,KAAK,EAAE,QAAQ,EAAE,KAAK,EAAE,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,EAAE,MAAM,EAAE,SAAS,EAAE,OAAO,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,OAAO,EAAE,KAAK,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,KAAK,EAAE,QAAQ,EAAE,OAAO,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,KAAK,EAAE,OAAO,EAAE,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,QAAQ,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,QAAQ,EAAE,OAAO,EAAE,QAAQ,EAAE,OAAO,EAAE,QAAQ,EAAE,KAAK,EAAE,IAAI,EAAE,OAAO,EAAE,KAAK,EAAE,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,EAAE,OAAO,EAAE,KAAK,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,EAAE,MAAM,EAAE,IAAI,EAAE,IAAI,EAAE,MAAM,EAAE,MAAM,EAAE,SAAS,EAAE,KAAK,EAAE,OAAO,EAAE,QAAQ,EAAE,MAAM,EAAE,QAAQ,EAAE,OAAO,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,EAAE,KAAK,EAAE,OAAO,EAAE,OAAO,EAAE,QAAQ,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,SAAS,EAAE,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,KAAK,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,SAAS,EAAE,KAAK,EAAE,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,MAAM,EAAE,OAAO,EAAE,OAAO,EAAE,QAAQ,EAAE,UAAU,EAAE,OAAO,EAAE,QAAQ,EAAE,KAAK,EAAE,MAAM,EAAE,KAAK,EAAE,OAAO,EAAE,MAAM,EAAE,MAAM,CAAC;IAC92C,EAAE,EAAE,CAAC,MAAM,EAAE,SAAS,EAAE,SAAS,EAAE,IAAI,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,EAAE,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,OAAO,EAAE,QAAQ,EAAE,MAAM,EAAE,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,KAAK,EAAE,QAAQ,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,EAAE,OAAO,EAAE,OAAO,EAAE,MAAM,EAAE,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,QAAQ,EAAE,KAAK,EAAE,QAAQ,EAAE,UAAU,EAAE,SAAS,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,EAAE,MAAM,EAAE,YAAY,EAAE,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,UAAU,EAAE,MAAM,EAAE,OAAO,EAAE,SAAS,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,IAAI,EAAE,SAAS,EAAE,MAAM,EAAE,QAAQ,EAAE,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,EAAE,MAAM,EAAE,IAAI,EAAE,QAAQ,EAAE,OAAO,EAAE,WAAW,EAAE,IAAI,EAAE,KAAK,EAAE,WAAW,EAAE,OAAO,EAAE,aAAa,CAAC;IACxtB,EAAE,EAAE,CAAC,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,QAAQ,EAAE,QAAQ,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,QAAQ,EAAE,QAAQ,EAAE,MAAM,EAAE,QAAQ,EAAE,OAAO,EAAE,OAAO,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,QAAQ,EAAE,KAAK,EAAE,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,KAAK,EAAE,IAAI,EAAE,MAAM,EAAE,YAAY,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,IAAI,EAAE,QAAQ,EAAE,MAAM,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,EAAE,IAAI,CAAC;IAC9X,EAAE,EAAE,CAAC,SAAS,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,SAAS,EAAE,OAAO,EAAE,SAAS,EAAE,OAAO,EAAE,UAAU,EAAE,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,SAAS,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,KAAK,EAAE,OAAO,EAAE,MAAM,EAAE,IAAI,EAAE,SAAS,EAAE,KAAK,EAAE,IAAI,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,QAAQ,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,OAAO,EAAE,MAAM,EAAE,OAAO,EAAE,SAAS,EAAE,MAAM,EAAE,OAAO,EAAE,KAAK,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,EAAE,OAAO,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,QAAQ,EAAE,QAAQ,EAAE,UAAU,EAAE,MAAM,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,KAAK,EAAE,QAAQ,EAAE,OAAO,EAAE,QAAQ,EAAE,UAAU,EAAE,OAAO,EAAE,KAAK,EAAE,OAAO,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,KAAK,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,KAAK,EAAE,OAAO,EAAE,KAAK,EAAE,UAAU,EAAE,OAAO,EAAE,QAAQ,EAAE,IAAI,EAAE,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,QAAQ,EAAE,KAAK,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,SAAS,EAAE,OAAO,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,QAAQ,EAAE,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,YAAY,EAAE,OAAO,EAAE,OAAO,CAAC;IAC17C,EAAE,EAAE,CAAC,MAAM,EAAE,MAAM,EAAE,UAAU,EAAE,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,OAAO,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,KAAK,EAAE,KAAK,EAAE,MAAM,CAAC;CAC3J,CAAA;AAED,MAAM,CAAC,MAAM,sBAAsB,GAA8B;IAChE,EAAE,EAAE
|
|
1
|
+
{"version":3,"file":"Suppressions.js","sourceRoot":"","sources":["../src/Suppressions.ts"],"names":[],"mappings":"AAAA,MAAM,CAAC,MAAM,gBAAgB,GAA6B;IACzD,6pCAA6pC;IAC7pC,EAAE,EAAE,CAAC,MAAM,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,EAAE,KAAK,EAAE,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,KAAK,EAAE,KAAK,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,QAAQ,EAAE,KAAK,EAAE,MAAM,EAAE,QAAQ,EAAE,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,IAAI,EAAE,OAAO,EAAE,KAAK,EAAE,MAAM,EAAE,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,EAAE,KAAK,EAAE,OAAO,EAAE,MAAM,EAAE,OAAO,EAAE,UAAU,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,KAAK,EAAE,KAAK,EAAE,QAAQ,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,KAAK,CAAC;IAC/1B,EAAE,EAAE,CAAC,OAAO,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,SAAS,EAAE,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,QAAQ,EAAE,KAAK,EAAE,MAAM,EAAE,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,KAAK,EAAE,KAAK,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,EAAE,MAAM,EAAE,IAAI,EAAE,OAAO,EAAE,UAAU,EAAE,KAAK,EAAE,SAAS,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,IAAI,EAAE,OAAO,EAAE,SAAS,EAAE,MAAM,EAAE,MAAM,EAAE,YAAY,EAAE,IAAI,EAAE,IAAI,EAAE,OAAO,EAAE,IAAI,EAAE,KAAK,EAAE,KAAK,EAAE,QAAQ,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,UAAU,EAAE,OAAO,EAAE,MAAM,EAAE,IAAI,EAAE,IAAI,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,EAAE,KAAK,EAAE,KAAK,EAAE,KAAK,EAAE,MAAM,EAAE,KAAK,EAAE,OAAO,EAAE,QAAQ,EAAE,IAAI,EAAE,MAAM,EAAE,SAAS,EAAE,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,QAAQ,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,IAAI,EAAE,OAAO,EAAE,MAAM,EAAE,KAAK,EAAE,QAAQ,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,QAAQ,EAAE,QAAQ,EAAE,IAAI,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,UAAU,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,IAAI,EAAE,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,QAAQ,EAAE,UAAU,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,UAAU,EAAE,MAAM,EAAE,IAAI,EAAE,YAAY,EAAE,MAAM,EAAE,OAAO,EAAE,KAAK,EAAE,MAAM,EAAE,IAAI,EAAE,IAAI,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,QAAQ,EAAE,OAAO,EAAE,KAAK,EAAE,MAAM,EAAE,WAAW,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,KAAK,EAAE,OAAO,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,IAAI,EAAE,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,MAAM,EAAE,QAAQ,EAAE,QAAQ,EAAE,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,OAAO,EAAE,WAAW,EAAE,KAAK,EAAE,QAAQ,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,IAAI,EAAE,UAAU,EAAE,QAAQ,EAAE,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,IAAI,EAAE,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,KAAK,EAAE,IAAI,EAAE,OAAO,EAAE,IAAI,EAAE,YAAY,EAAE,YAAY,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,IAAI,EAAE,QAAQ,EAAE,OAAO,EAAE,WAAW,EAAE,MAAM,EAAE,QAAQ,EAAE,SAAS,CAAC;IAC9+D,EAAE,EAAE,CAAC,OAAO,EAAE,QAAQ,EAAE,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,QAAQ,EAAE,OAAO,EAAE,SAAS,EAAE,MAAM,EAAE,MAAM,EAAE,QAAQ,EAAE,KAAK,EAAE,MAAM,EAAE,OAAO,EAAE,KAAK,EAAE,QAAQ,EAAE,KAAK,EAAE,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,EAAE,MAAM,EAAE,SAAS,EAAE,OAAO,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,OAAO,EAAE,KAAK,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,KAAK,EAAE,QAAQ,EAAE,OAAO,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,KAAK,EAAE,OAAO,EAAE,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,QAAQ,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,QAAQ,EAAE,OAAO,EAAE,QAAQ,EAAE,OAAO,EAAE,QAAQ,EAAE,KAAK,EAAE,IAAI,EAAE,OAAO,EAAE,KAAK,EAAE,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,EAAE,OAAO,EAAE,KAAK,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,EAAE,MAAM,EAAE,IAAI,EAAE,IAAI,EAAE,MAAM,EAAE,MAAM,EAAE,SAAS,EAAE,KAAK,EAAE,OAAO,EAAE,QAAQ,EAAE,MAAM,EAAE,QAAQ,EAAE,OAAO,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,EAAE,KAAK,EAAE,OAAO,EAAE,OAAO,EAAE,QAAQ,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,SAAS,EAAE,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,KAAK,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,SAAS,EAAE,KAAK,EAAE,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,MAAM,EAAE,OAAO,EAAE,OAAO,EAAE,QAAQ,EAAE,UAAU,EAAE,OAAO,EAAE,QAAQ,EAAE,KAAK,EAAE,MAAM,EAAE,KAAK,EAAE,OAAO,EAAE,MAAM,EAAE,MAAM,CAAC;IAC92C,EAAE,EAAE,CAAC,MAAM,EAAE,SAAS,EAAE,SAAS,EAAE,IAAI,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,EAAE,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,OAAO,EAAE,QAAQ,EAAE,MAAM,EAAE,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,KAAK,EAAE,QAAQ,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,EAAE,OAAO,EAAE,OAAO,EAAE,MAAM,EAAE,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,QAAQ,EAAE,KAAK,EAAE,QAAQ,EAAE,UAAU,EAAE,SAAS,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,EAAE,MAAM,EAAE,YAAY,EAAE,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,UAAU,EAAE,MAAM,EAAE,OAAO,EAAE,SAAS,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,IAAI,EAAE,SAAS,EAAE,MAAM,EAAE,QAAQ,EAAE,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,EAAE,MAAM,EAAE,IAAI,EAAE,QAAQ,EAAE,OAAO,EAAE,WAAW,EAAE,IAAI,EAAE,KAAK,EAAE,WAAW,EAAE,OAAO,EAAE,aAAa,CAAC;IACxtB,EAAE,EAAE,CAAC,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,QAAQ,EAAE,QAAQ,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,QAAQ,EAAE,QAAQ,EAAE,MAAM,EAAE,QAAQ,EAAE,OAAO,EAAE,OAAO,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,QAAQ,EAAE,KAAK,EAAE,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,KAAK,EAAE,IAAI,EAAE,MAAM,EAAE,YAAY,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,IAAI,EAAE,QAAQ,EAAE,MAAM,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,EAAE,IAAI,CAAC;IAC9X,EAAE,EAAE,CAAC,SAAS,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,SAAS,EAAE,OAAO,EAAE,SAAS,EAAE,OAAO,EAAE,UAAU,EAAE,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,SAAS,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,KAAK,EAAE,OAAO,EAAE,MAAM,EAAE,IAAI,EAAE,SAAS,EAAE,KAAK,EAAE,IAAI,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,QAAQ,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,OAAO,EAAE,MAAM,EAAE,OAAO,EAAE,SAAS,EAAE,MAAM,EAAE,OAAO,EAAE,KAAK,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,EAAE,OAAO,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,QAAQ,EAAE,QAAQ,EAAE,UAAU,EAAE,MAAM,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,KAAK,EAAE,QAAQ,EAAE,OAAO,EAAE,QAAQ,EAAE,UAAU,EAAE,OAAO,EAAE,KAAK,EAAE,OAAO,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,KAAK,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,KAAK,EAAE,OAAO,EAAE,KAAK,EAAE,UAAU,EAAE,OAAO,EAAE,QAAQ,EAAE,IAAI,EAAE,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,QAAQ,EAAE,KAAK,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,SAAS,EAAE,OAAO,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,QAAQ,EAAE,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,YAAY,EAAE,OAAO,EAAE,OAAO,CAAC;IAC17C,EAAE,EAAE,CAAC,MAAM,EAAE,MAAM,EAAE,UAAU,EAAE,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,OAAO,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,KAAK,EAAE,KAAK,EAAE,MAAM,CAAC;CAC3J,CAAA;AAED,MAAM,CAAC,MAAM,sBAAsB,GAA8B;IAChE,EAAE,EAAE,CAAC,OAAO,CAAC;IACb,EAAE,EAAE,CAAC,MAAM,CAAC;CACZ,CAAA;AAED,MAAM,CAAC,MAAM,wCAAwC,GAA6B;IACjF,IAAI,EAAE;QACL,QAAQ,EAAE,MAAM,EAAE,OAAO,EAAE,OAAO,EAAE,MAAM;KAC1C;IACD,IAAI,EAAE;QACL,IAAI;KACJ;CACD,CAAA;AAED,MAAM,CAAC,MAAM,gBAAgB,GAAG;IAC/B,IAAI,EAAE,IAAI,EAAE,KAAK,EAAE,QAAQ,EAAE,WAAW,EAAE,SAAS,EAAE,OAAO,EAAE,KAAK,EAAE,MAAM,EAAE,KAAK,EAAE,KAAK;CACzF,CAAA;AAED,MAAM,CAAC,MAAM,eAAe,GAAG;IAC9B,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,KAAK,EAAE,KAAK;CACpC,CAAA;AAED,kDAAkD;AAClD,wBAAwB;AACxB,yBAAyB;AACzB,gBAAgB;AAChB,wBAAwB;AACxB,EAAE;AACF,0CAA0C;AAC1C,qEAAqE;AACrE,gDAAgD;AAChD,iFAAiF;AACjF,0FAA0F"}
|
package/dist/Test.js
CHANGED
|
@@ -2,20 +2,20 @@ import { addMissingPunctuationWordsToWordSequence, segmentText, segmentWordSeque
|
|
|
2
2
|
import { Timer } from "./utilities/Timer.js";
|
|
3
3
|
const log = console.log;
|
|
4
4
|
async function test1() {
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
5
|
+
//const text = 'Hello! 1. Say who? 2. How are you? This is v2.0 that good. I have 2 344 234ms C# is good games'
|
|
6
|
+
const text = 'the 6v34abc6g6 😊😊 -43.45 x -2445.65rty the';
|
|
7
|
+
//const text = 'Hello world! 23rd? 123% 3/4/7 Привет мир! 你好世界！'
|
|
8
|
+
const options = {
|
|
9
|
+
language: 'en'
|
|
10
|
+
};
|
|
11
|
+
const wordSequence = await splitToWords(text, options);
|
|
12
|
+
console.log(JSON.stringify(wordSequence.wordArray));
|
|
13
|
+
const segmentedText = await segmentWordSequence(wordSequence);
|
|
14
|
+
const x = 0;
|
|
15
|
+
}
|
|
16
|
+
async function test2() {
|
|
17
17
|
const { readFileSync, writeFileSync } = await import('fs');
|
|
18
|
-
const text = readFileSync('test-data/
|
|
18
|
+
const text = readFileSync('test-data/Test2.txt', 'utf-8');
|
|
19
19
|
const timer = new Timer();
|
|
20
20
|
let result;
|
|
21
21
|
for (let i = 0; i < 5; i++) {
|
|
@@ -45,7 +45,7 @@ async function test1() {
|
|
|
45
45
|
}
|
|
46
46
|
writeFileSync('out/segmented.txt', segmentedText);
|
|
47
47
|
}
|
|
48
|
-
async function
|
|
48
|
+
async function test3() {
|
|
49
49
|
const text = ` Hello, how are you today ?? `;
|
|
50
50
|
const words = await splitToWords(text);
|
|
51
51
|
const nonPunctuationWordEntries = words.nonPunctuationEntries;
|
|
@@ -54,9 +54,8 @@ async function test2() {
|
|
|
54
54
|
const extendedWords = addMissingPunctuationWordsToWordSequence(nonPunctuationWordSequence, text);
|
|
55
55
|
const x = 0;
|
|
56
56
|
}
|
|
57
|
-
async function
|
|
58
|
-
|
|
59
|
-
const text = `
|
|
57
|
+
async function test4() {
|
|
58
|
+
const text = `
|
|
60
59
|
Hello 12/43 yo good-go bobo_baba man!
|
|
61
60
|
2.4a, 5.6 x&y x'v·y v‧z x·y·5
|
|
62
61
|
abc123 23+42.534 645
|
|
@@ -85,10 +84,9 @@ async function test3() {
|
|
|
85
84
|
Price is $60 or 60$
|
|
86
85
|
|
|
87
86
|
`;
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
}
|
|
87
|
+
const result = (await splitToWords(text)).wordArray;
|
|
88
|
+
log(result.join(' | '));
|
|
91
89
|
}
|
|
92
|
-
test1()
|
|
93
|
-
|
|
90
|
+
//test1()
|
|
91
|
+
test2();
|
|
94
92
|
//# sourceMappingURL=Test.js.map
|
package/dist/Test.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"Test.js","sourceRoot":"","sources":["../src/Test.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,wCAAwC,EAA2C,WAAW,EAAE,mBAAmB,EAAE,YAAY,EAAE,YAAY,EAAE,MAAM,uBAAuB,CAAA;AACvL,OAAO,EAAE,KAAK,EAAE,MAAM,sBAAsB,CAAA;AAE5C,MAAM,GAAG,GAAG,OAAO,CAAC,GAAG,CAAA;AAEvB,KAAK,UAAU,KAAK;IACnB
|
|
1
|
+
{"version":3,"file":"Test.js","sourceRoot":"","sources":["../src/Test.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,wCAAwC,EAA2C,WAAW,EAAE,mBAAmB,EAAE,YAAY,EAAE,YAAY,EAAE,MAAM,uBAAuB,CAAA;AACvL,OAAO,EAAE,KAAK,EAAE,MAAM,sBAAsB,CAAA;AAE5C,MAAM,GAAG,GAAG,OAAO,CAAC,GAAG,CAAA;AAEvB,KAAK,UAAU,KAAK;IACnB,+GAA+G;IAC/G,MAAM,IAAI,GAAG,8CAA8C,CAAA;IAC3D,iEAAiE;IAEjE,MAAM,OAAO,GAAwB;QACpC,QAAQ,EAAE,IAAI;KACd,CAAA;IAED,MAAM,YAAY,GAAG,MAAM,YAAY,CAAC,IAAI,EAAE,OAAO,CAAC,CAAA;IAEtD,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,SAAS,CAAC,YAAY,CAAC,SAAS,CAAC,CAAC,CAAA;IAEnD,MAAM,aAAa,GAAG,MAAM,mBAAmB,CAAC,YAAY,CAAC,CAAA;IAE7D,MAAM,CAAC,GAAG,CAAC,CAAA;AACZ,CAAC;AAED,KAAK,UAAU,KAAK;IACnB,MAAM,EAAE,YAAY,EAAE,aAAa,EAAE,GAAG,MAAM,MAAM,CAAC,IAAI,CAAC,CAAA;IAC1D,MAAM,IAAI,GAAG,YAAY,CAAC,qBAAqB,EAAE,OAAO,CAAC,CAAA;IAEzD,MAAM,KAAK,GAAG,IAAI,KAAK,EAAE,CAAA;IAEzB,IAAI,MAA0B,CAAA;IAE9B,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;QAC5B,MAAM,GAAG,MAAM,WAAW,CAAC,IAAI,EAAE;YAChC,QAAQ,EAAE,IAAI;YACd,kBAAkB,EAAE,EAAE;YACtB,6BAA6B,EAAE,IAAI;SACnC,CAAC,CAAA;QAEF,KAAK,CAAC,aAAa,CAAC,sBAAsB,CAAC,CAAA;IAC5C,CAAC;IAED,GAAG,CAAC,EAAE,CAAC,CAAA;IAEP,EAAE;IAEF,IAAI,aAAa,GAAG,EAAE,CAAA;IAEtB,KAAK,IAAI,aAAa,GAAG,CAAC,EAAE,aAAa,GAAG,MAAO,CAAC,SAAS,CAAC,MAAM,EAAE,aAAa,EAAE,EAAE,CAAC;QACvF,MAAM,QAAQ,GAAG,MAAO,CAAC,SAAS,CAAC,aAAa,CAAC,CAAA;QACjD,MAAM,OAAO,GAAG,QAAQ,CAAC,OAAO,CAAA;QAEhC,KAAK,IAAI,WAAW,GAAG,CAAC,EAAE,WAAW,GAAG,OAAO,CAAC,MAAM,EAAE,WAAW,EAAE,EAAE,CAAC;YACvE,MAAM,MAAM,GAAG,OAAO,CAAC,WAAW,CAAC,CAAA;YAEnC,aAAa,IAAI,MAAM,CAAC,KAAK,CAAC,SAAS,CAAC,IAAI,CAAC,KAAK,CAAC,CAAA;YAEnD,IAAI,WAAW,GAAG,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBACtC,aAAa,IAAI,KAAK,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,IAAI,CAAA;YAC1C,CAAC;QACF,CAAC;QAED,IAAI,aAAa,GAAG,MAAO,CAAC,SAAS,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAClD,aAAa,IAAI,KAAK,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,KAAK,CAAA;QAC3C,CAAC;IACF,CAAC;IAED,aAAa,CAAC,mBAAmB,EAAE,aAAa,CAAC,CAAA;AAClD,CAAC;AAED,KAAK,UAAU,KAAK;IACnB,MAAM,IAAI,GAAG,mDAAmD,CAAA;IAEhE,MAAM,KAAK,GAAG,MAAM,YAAY,CAAC,IAAI,CAAC,CAAA;IAEtC,MAAM,yBAAyB,GAAG,KAAK,CAAC,qBAAqB,CAAA;IAC7D,MAAM,0BAA0B,GAAG,IAAI,YAAY,EAAE,CAAA;IACrD,0BAA0B,CAAC,OAAO,GAAG,yBAAyB,CAAA;IAE9D,MAAM,aAAa,GAAG,wCAAwC,CAAC,0BAA0B,EAAE,IAAI,CAAC,CAAA;IAEhG,MAAM,CAAC,GAAG,CAAC,CAAA;AACZ,CAAC;AAED,KAAK,UAAU,KAAK;IACnB,MAAM,IAAI,GAAG;;;;;;;;;;;;;;;;;;;;;;;;;;;;CA4Bb,CAAA;IAEA,MAAM,MAAM,GAAG,CAAC,MAAM,YAAY,CAAC,IAAI,CAAC,CAAC,CAAC,SAAS,CAAA;IACnD,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAA;AAExB,CAAC;AAED,SAAS;AACT,KAAK,EAAE,CAAA"}
|
package/dist/TextSegmentation.js
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { buildWordOrNumberPattern as buildWordSplitterPattern, phraseSeparatorRegExp,
|
|
1
|
+
import { buildWordOrNumberPattern as buildWordSplitterPattern, phraseSeparatorRegExp, sentenceSeparatorTrailingPunctuationCharacterRegExp, sentenceSeparatorCharacterRegExp, whitespacePatternRegExp, letterPatternGlobalRegExp } from './Patterns.js';
|
|
2
2
|
import { cldrSuppressions, additionalSuppressions, leadingApostropheContractionSuppressions, nounSuppressions, tldSuppressions } from './Suppressions.js';
|
|
3
3
|
import { eastAsianCharRangesRegExp } from './EastAsianCharacterPatterns.js';
|
|
4
4
|
import { WordSequence } from './WordSequence.js';
|
|
@@ -21,27 +21,39 @@ export async function segmentWordSequence(wordSequence) {
|
|
|
21
21
|
const word = wordSequence.getWordAt(wordIndex);
|
|
22
22
|
if (currentSentenceLetterCount < minimumSentenceLetterCount) {
|
|
23
23
|
const matches = word.matchAll(letterPatternGlobalRegExp);
|
|
24
|
-
for (const
|
|
24
|
+
for (const _ of matches) {
|
|
25
25
|
currentSentenceLetterCount += 1;
|
|
26
26
|
if (currentSentenceLetterCount >= minimumSentenceLetterCount) {
|
|
27
27
|
break;
|
|
28
28
|
}
|
|
29
29
|
}
|
|
30
30
|
}
|
|
31
|
-
if (currentSentenceLetterCount >= minimumSentenceLetterCount &&
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
31
|
+
if (currentSentenceLetterCount >= minimumSentenceLetterCount && sentenceSeparatorCharacterRegExp.test(word)) {
|
|
32
|
+
let trailingSequenceEndIndex = wordIndex;
|
|
33
|
+
let trailingSequenceContainedWhitespace = false;
|
|
34
|
+
while (trailingSequenceEndIndex < wordSequence.length) {
|
|
35
|
+
const trailingWord = wordSequence.getWordAt(trailingSequenceEndIndex);
|
|
36
|
+
if (sentenceSeparatorTrailingPunctuationCharacterRegExp.test(trailingWord)) {
|
|
37
|
+
if (!trailingSequenceContainedWhitespace && whitespacePatternRegExp.test(trailingWord)) {
|
|
38
|
+
trailingSequenceContainedWhitespace = true;
|
|
39
|
+
}
|
|
40
|
+
trailingSequenceEndIndex++;
|
|
41
|
+
}
|
|
42
|
+
else {
|
|
35
43
|
break;
|
|
36
44
|
}
|
|
37
|
-
wordIndex += 1;
|
|
38
45
|
}
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
46
|
+
if (trailingSequenceEndIndex === wordSequence.length ||
|
|
47
|
+
trailingSequenceContainedWhitespace ||
|
|
48
|
+
['。', '？', '！'].includes(word)) {
|
|
49
|
+
sentenceWordRanges.push({
|
|
50
|
+
start: sentenceStartWordOffset,
|
|
51
|
+
end: trailingSequenceEndIndex
|
|
52
|
+
});
|
|
53
|
+
sentenceStartWordOffset = trailingSequenceEndIndex;
|
|
54
|
+
currentSentenceLetterCount = 0;
|
|
55
|
+
wordIndex = trailingSequenceEndIndex - 1;
|
|
56
|
+
}
|
|
45
57
|
}
|
|
46
58
|
}
|
|
47
59
|
if (sentenceStartWordOffset < wordSequence.length) {
|
|
@@ -62,7 +74,7 @@ export async function segmentWordSequence(wordSequence) {
|
|
|
62
74
|
let whitespaceSeenOnce = false;
|
|
63
75
|
while (wordIndex < sentenceEndWordOffset - 1) {
|
|
64
76
|
const nextWord = wordSequence.getWordAt(wordIndex + 1);
|
|
65
|
-
if (!
|
|
77
|
+
if (!sentenceSeparatorTrailingPunctuationCharacterRegExp.test(nextWord)) {
|
|
66
78
|
break;
|
|
67
79
|
}
|
|
68
80
|
if (whitespacePatternRegExp.test(nextWord)) {
|
|
@@ -226,6 +238,30 @@ export function addMissingPunctuationWordsToWordSequence(wordSequence, sourceTex
|
|
|
226
238
|
}
|
|
227
239
|
return { wordSequenceWithPunctuation, originalWordsReverseMapping };
|
|
228
240
|
}
|
|
241
|
+
function getPunctuationRanges(wordSequence, text) {
|
|
242
|
+
const punctuationRanges = [];
|
|
243
|
+
const wordEntries = wordSequence.entries;
|
|
244
|
+
if (wordEntries[0].startOffset > 0) {
|
|
245
|
+
punctuationRanges.push({ start: 0, end: wordEntries[0].startOffset });
|
|
246
|
+
}
|
|
247
|
+
for (let i = 0; i < wordEntries.length; i++) {
|
|
248
|
+
const entry = wordEntries[i];
|
|
249
|
+
const previousEndOffset = wordEntries[i - 1]?.endOffset ?? 0;
|
|
250
|
+
if (entry.startOffset > previousEndOffset) {
|
|
251
|
+
punctuationRanges.push({ start: previousEndOffset, end: entry.startOffset });
|
|
252
|
+
}
|
|
253
|
+
if (entry.isPunctuation) {
|
|
254
|
+
punctuationRanges.push({ start: entry.startOffset, end: entry.endOffset });
|
|
255
|
+
}
|
|
256
|
+
}
|
|
257
|
+
{
|
|
258
|
+
const lastEndOffset = wordEntries[wordEntries.length - 1]?.endOffset;
|
|
259
|
+
if (lastEndOffset && lastEndOffset < text.length) {
|
|
260
|
+
punctuationRanges.push({ start: lastEndOffset, end: text.length });
|
|
261
|
+
}
|
|
262
|
+
}
|
|
263
|
+
return punctuationRanges;
|
|
264
|
+
}
|
|
229
265
|
////////////////////////////////////////////////////////////////////////////////////////////////
|
|
230
266
|
// Helper methods
|
|
231
267
|
////////////////////////////////////////////////////////////////////////////////////////////////
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"TextSegmentation.js","sourceRoot":"","sources":["../src/TextSegmentation.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,wBAAwB,IAAI,wBAAwB,EAAE,qBAAqB,EAAE,0CAA0C,EAAE,uBAAuB,EAAE,uBAAuB,EAAE,yBAAyB,EAAE,MAAM,eAAe,CAAA;AACpO,OAAO,EAAE,gBAAgB,EAAE,sBAAsB,EAAE,wCAAwC,EAAE,gBAAgB,EAAE,eAAe,EAAE,MAAM,mBAAmB,CAAA;AACzJ,OAAO,EAAE,yBAAyB,EAAE,MAAM,iCAAiC,CAAA;AAC3E,OAAO,EAAE,YAAY,EAAE,MAAM,mBAAmB,CAAA;AAChD,OAAO,EAAE,oBAAoB,EAAE,MAAM,0BAA0B,CAAA;AAC/D,OAAO,EAAE,YAAY,EAAkB,MAAM,mBAAmB,CAAA;AAEhE,OAAO,EAAE,WAAW,EAAE,MAAM,iBAAiB,CAAA;AAE7C,gGAAgG;AAChG,mBAAmB;AACnB,gGAAgG;AAChG,MAAM,CAAC,KAAK,UAAU,WAAW,CAAC,IAAY,EAAE,OAA6B;IAC5E,MAAM,YAAY,GAAG,MAAM,YAAY,CAAC,IAAI,EAAE,OAAO,CAAC,CAAA;IAEtD,OAAO,mBAAmB,CAAC,YAAY,CAAC,CAAA;AACzC,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,mBAAmB,CAAC,YAA0B;IACnE,MAAM,kBAAkB,GAAY,EAAE,CAAA;IAEtC,MAAM,0BAA0B,GAAG,CAAC,CAAA;IAEpC,IAAI,uBAAuB,GAAG,CAAC,CAAA;IAC/B,IAAI,0BAA0B,GAAG,CAAC,CAAA;IAElC,KAAK,IAAI,SAAS,GAAG,CAAC,EAAE,SAAS,GAAG,YAAY,CAAC,MAAM,EAAE,SAAS,EAAE,EAAE,CAAC;QACtE,MAAM,IAAI,GAAG,YAAY,CAAC,SAAS,CAAC,SAAS,CAAC,CAAA;QAE9C,IAAI,0BAA0B,GAAG,0BAA0B,EAAE,CAAC;YAC7D,MAAM,OAAO,GAAG,IAAI,CAAC,QAAQ,CAAC,yBAAyB,CAAC,CAAA;YAExD,KAAK,MAAM,KAAK,IAAI,OAAO,EAAE,CAAC;gBAC7B,0BAA0B,IAAI,CAAC,CAAA;gBAE/B,IAAI,0BAA0B,IAAI,0BAA0B,EAAE,CAAC;oBAC9D,MAAK;gBACN,CAAC;YACF,CAAC;QACF,CAAC;QAED,IAAI,0BAA0B,IAAI,0BAA0B,IAAI,uBAAuB,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;YACpG,OAAO,SAAS,GAAG,YAAY,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBAC5C,MAAM,QAAQ,GAAG,YAAY,CAAC,SAAS,CAAC,SAAS,GAAG,CAAC,CAAC,CAAA;gBAEtD,IAAI,CAAC,0CAA0C,CAAC,IAAI,CAAC,QAAQ,CAAC,EAAE,CAAC;oBAChE,MAAK;gBACN,CAAC;gBAED,SAAS,IAAI,CAAC,CAAA;YACf,CAAC;YAED,kBAAkB,CAAC,IAAI,CAAC;gBACvB,KAAK,EAAE,uBAAuB;gBAC9B,GAAG,EAAE,SAAS,GAAG,CAAC;aAClB,CAAC,CAAA;YAEF,uBAAuB,GAAG,SAAS,GAAG,CAAC,CAAA;YACvC,0BAA0B,GAAG,CAAC,CAAA;QAC/B,CAAC;IACF,CAAC;IAED,IAAI,uBAAuB,GAAG,YAAY,CAAC,MAAM,EAAE,CAAC;QACnD,kBAAkB,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,uBAAuB,EAAE,GAAG,EAAE,YAAY,CAAC,MAAM,EAAE,CAAC,CAAA;IACtF,CAAC;IAED,MAAM,SAAS,GAAe,EAAE,CAAA;IAEhC,KAAK,MAAM,SAAS,IAAI,kBAAkB,EAAE,CAAC;QAC5C,MAAM,oBAAoB,GAAG,YAAY,CAAC,KAAK,CAAC,SAAS,CAAC,KAAK,EAAE,SAAS,CAAC,GAAG,CAAC,CAAA;QAE/E,SAAS,CAAC,IAAI,CAAC,IAAI,QAAQ,CAAC,SAAS,EAAE,oBAAoB,CAAC,CAAC,CAAA;IAC9D,CAAC;IAED,KAAK,MAAM,QAAQ,IAAI,SAAS,EAAE,CAAC;QAClC,MAAM,gBAAgB,GAAY,EAAE,CAAA;QAEpC,IAAI,qBAAqB,GAAG,QAAQ,CAAC,SAAS,CAAC,GAAG,CAAA;QAClD,IAAI,qBAAqB,GAAG,QAAQ,CAAC,SAAS,CAAC,KAAK,CAAA;QAEpD,KAAK,IAAI,SAAS,GAAG,qBAAqB,EAAE,SAAS,GAAG,qBAAqB,EAAE,SAAS,EAAE,EAAE,CAAC;YAC5F,MAAM,WAAW,GAAG,YAAY,CAAC,SAAS,CAAC,SAAS,CAAC,CAAA;YAErD,IAAI,qBAAqB,CAAC,IAAI,CAAC,WAAW,CAAC,EAAE,CAAC;gBAC7C,IAAI,kBAAkB,GAAG,KAAK,CAAA;gBAE9B,OAAO,SAAS,GAAG,qBAAqB,GAAG,CAAC,EAAE,CAAC;oBAC9C,MAAM,QAAQ,GAAG,YAAY,CAAC,SAAS,CAAC,SAAS,GAAG,CAAC,CAAC,CAAA;oBAEtD,IAAI,CAAC,0CAA0C,CAAC,IAAI,CAAC,QAAQ,CAAC,EAAE,CAAC;wBAChE,MAAK;oBACN,CAAC;oBAED,IAAI,uBAAuB,CAAC,IAAI,CAAC,QAAQ,CAAC,EAAE,CAAC;wBAC5C,kBAAkB,GAAG,IAAI,CAAA;oBAC1B,CAAC;oBAED,IAAI,QAAQ,KAAK,GAAG,IAAI,kBAAkB,EAAE,CAAC;wBAC5C,MAAK;oBACN,CAAC;oBAED,SAAS,IAAI,CAAC,CAAA;gBACf,CAAC;gBAED,gBAAgB,CAAC,IAAI,CAAC;oBACrB,KAAK,EAAE,qBAAqB;oBAC5B,GAAG,EAAE,SAAS,GAAG,CAAC;iBAClB,CAAC,CAAA;gBAEF,qBAAqB,GAAG,SAAS,GAAG,CAAC,CAAA;YACtC,CAAC;QACF,CAAC;QAED,IAAI,qBAAqB,GAAG,qBAAqB,EAAE,CAAC;YACnD,gBAAgB,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,qBAAqB,EAAE,GAAG,EAAE,qBAAqB,EAAE,CAAC,CAAA;QACpF,CAAC;QAED,KAAK,MAAM,SAAS,IAAI,gBAAgB,EAAE,CAAC;YAC1C,MAAM,kBAAkB,GAAG,YAAY,CAAC,KAAK,CAAC,SAAS,CAAC,KAAK,EAAE,SAAS,CAAC,GAAG,CAAC,CAAA;YAE7E,QAAQ,CAAC,OAAO,CAAC,IAAI,CAAC,IAAI,MAAM,CAAC,SAAS,EAAE,kBAAkB,CAAC,CAAC,CAAA;QACjE,CAAC;IACF,CAAC;IAED,MAAM,MAAM,GAAuB;QAClC,KAAK,EAAE,YAAY;QACnB,SAAS;KACT,CAAA;IAED,OAAO,MAAM,CAAA;AACd,CAAC;AAED,MAAM,yBAAyB,GAAG,IAAI,GAAG,EAAkB,CAAA;AAE3D,MAAM,CAAC,KAAK,UAAU,YAAY,CAAC,IAAY,EAAE,OAA6B;IAC7E,IAAI,CAAC,OAAO,EAAE,CAAC;QACd,OAAO,GAAG,EAAE,CAAA;IACb,CAAC;IAED,OAAO,GAAG,EAAE,GAAG,0BAA0B,EAAE,GAAG,OAAO,EAAE,CAAA;IAEvD,IAAI,OAAO,CAAC,QAAQ,EAAE,CAAC;QACtB,OAAO,CAAC,QAAQ,GAAG,oBAAoB,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAA;IAC1D,CAAC;IAED,MAAM,aAAa,GAAG,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,CAAA;IAE7C,IAAI,kBAAkB,GAAG,yBAAyB,CAAC,GAAG,CAAC,aAAa,CAAC,CAAA;IAErE,IAAI,CAAC,kBAAkB,EAAE,CAAC;QACzB,kBAAkB,GAAG,iCAAiC,CAAC,OAAO,CAAC,CAAA;QAE/D,yBAAyB,CAAC,GAAG,CAAC,aAAa,EAAE,kBAAkB,CAAC,CAAA;IACjE,CAAC;IAED,IAAI,YAAY,GAAG,IAAI,YAAY,EAAE,CAAA;IAErC,SAAS,0BAA0B,CAAC,WAAmB,EAAE,SAAiB;QACzE,MAAM,wBAAwB,GAAG,IAAI,CAAC,SAAS,CAAC,WAAW,EAAE,SAAS,CAAC,CAAA;QAEvE,IAAI,UAAU,GAAG,WAAW,CAAA;QAC5B,IAAI,0BAA0B,GAAG,WAAW,CAAA;QAE5C,SAAS,0BAA0B;YAClC,IAAI,UAAU,GAAG,0BAA0B,EAAE,CAAC;gBAC7C,MAAM,QAAQ,GAAG,IAAI,CAAC,SAAS,CAAC,0BAA0B,EAAE,UAAU,CAAC,CAAA;gBACvE,YAAY,CAAC,OAAO,CAAC,QAAQ,EAAE,0BAA0B,EAAE,IAAI,CAAC,CAAA;gBAEhE,0BAA0B,GAAG,UAAU,CAAA;YACxC,CAAC;QACF,CAAC;QAED,KAAK,MAAM,IAAI,IAAI,wBAAwB,EAAE,CAAC;YAC7C,IAAI,IAAI,KAAK,GAAG,EAAE,CAAC;gBAClB,UAAU,IAAI,CAAC,CAAA;gBAEf,SAAQ;YACT,CAAC;YAED,0BAA0B,EAAE,CAAA;YAE5B,UAAU,IAAI,IAAI,CAAC,MAAM,CAAA;YAEzB,0BAA0B,EAAE,CAAA;QAC7B,CAAC;QAED,0BAA0B,EAAE,CAAA;IAC7B,CAAC;IAED,MAAM,WAAW,GAAG,IAAI,CAAC,QAAQ,CAAC,kBAAkB,CAAC,CAAA;IAErD,IAAI,kBAAkB,GAAG,CAAC,CAAA;IAE1B,IAAI,WAAW,EAAE,CAAC;QACjB,KAAK,MAAM,KAAK,IAAI,WAAW,EAAE,CAAC;YACjC,MAAM,OAAO,GAAG,KAAK,CAAC,OAAQ,CAAC,CAAC,CAAC,CAAA;YACjC,MAAM,gBAAgB,GAAG,OAAO,CAAC,CAAC,CAAC,CAAA;YACnC,MAAM,cAAc,GAAG,OAAO,CAAC,CAAC,CAAC,CAAA;YAEjC,IAAI,gBAAgB,GAAG,kBAAkB,EAAE,CAAC;gBAC3C,0BAA0B,CAAC,kBAAkB,EAAE,gBAAgB,CAAC,CAAA;YACjE,CAAC;YAED,MAAM,QAAQ,GAAG,IAAI,CAAC,SAAS,CAAC,gBAAgB,EAAE,cAAc,CAAC,CAAA;YACjE,YAAY,CAAC,OAAO,CAAC,QAAQ,EAAE,gBAAgB,EAAE,KAAK,CAAC,CAAA;YAEvD,kBAAkB,GAAG,cAAc,CAAA;QACpC,CAAC;QAED,0BAA0B,CAAC,kBAAkB,EAAE,IAAI,CAAC,MAAM,CAAC,CAAA;IAC5D,CAAC;IAED,IAAI,OAAO,CAAC,6BAA6B,EAAE,CAAC;QAC3C,YAAY,GAAG,MAAM,yBAAyB,CAAC,IAAI,EAAE,YAAY,CAAC,CAAA;IACnE,CAAC;IAED,OAAO,YAAY,CAAA;AACpB,CAAC;AAED,KAAK,UAAU,yBAAyB,CAAC,cAAsB,EAAE,YAA0B;IAC1F,MAAM,eAAe,GAAG,MAAM,kBAAkB,EAAE,CAAA;IAElD,IAAI,eAAe,KAAK,SAAS,EAAE,CAAC;QACnC,OAAO,YAAY,CAAA;IACpB,CAAC;IAED,IAAI,cAAc,GAAG,KAAK,CAAA;IAE1B,MAAM,eAAe,GAAG,IAAI,YAAY,EAAE,CAAA;IAE1C,KAAK,IAAI,SAAS,GAAG,CAAC,EAAE,SAAS,GAAG,YAAY,CAAC,MAAM,EAAE,SAAS,EAAE,EAAE,CAAC;QACtE,MAAM,SAAS,GAAG,YAAY,CAAC,OAAO,CAAC,SAAS,CAAC,CAAA;QACjD,MAAM,eAAe,GAAG,SAAS,CAAC,WAAW,CAAA;QAE7C,MAAM,IAAI,GAAG,YAAY,CAAC,SAAS,CAAC,SAAS,CAAC,CAAA;QAE9C,IAAI,yBAAyB,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;YAC1C,IAAI,CAAC,cAAc,EAAE,CAAC;gBACrB,MAAM,eAAe,CAAC,UAAU,EAAE,CAAA;gBAElC,cAAc,GAAG,IAAI,CAAA;YACtB,CAAC;YAED,MAAM,UAAU,GAAG,CAAC,GAAG,eAAe,CAAC,uBAAuB,CAAC,IAAI,CAAC,CAAC,CAAA;YAErE,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,UAAU,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;gBAChD,MAAM,kBAAkB,GAAG,eAAe,GAAG,UAAU,CAAC,CAAC,CAAC,CAAA;gBAC1D,MAAM,gBAAgB,GAAG,eAAe,GAAG,UAAU,CAAC,CAAC,GAAG,CAAC,CAAC,CAAA;gBAE5D,MAAM,WAAW,GAAG,cAAc,CAAC,SAAS,CAAC,kBAAkB,EAAE,gBAAgB,CAAC,CAAA;gBAElF,eAAe,CAAC,OAAO,CACtB,WAAW,EACX,kBAAkB,EAClB,KAAK,CACL,CAAA;YACF,CAAC;QACF,CAAC;aAAM,CAAC;YACP,MAAM,QAAQ,GAAG,cAAc,CAAC,SAAS,CAAC,SAAS,CAAC,WAAW,EAAE,SAAS,CAAC,SAAS,CAAC,CAAA;YAErF,eAAe,CAAC,OAAO,CAAC,QAAQ,EAAE,SAAS,CAAC,WAAW,EAAE,SAAS,CAAC,aAAa,CAAC,CAAA;QAClF,CAAC;IACF,CAAC;IAED,OAAO,eAAe,CAAA;AACvB,CAAC;AAED,uDAAuD;AACvD,MAAM,UAAU,wCAAwC,CAAC,YAA0B,EAAE,UAAkB;IACtG,MAAM,2BAA2B,GAAG,IAAI,GAAG,EAAkB,CAAA;IAE7D,MAAM,2BAA2B,GAAG,IAAI,YAAY,EAAE,CAAA;IAEtD,SAAS,0BAA0B,CAAC,SAAiB,EAAE,iBAAyB;QAC/E,IAAI,UAAU,GAAG,iBAAiB,CAAA;QAElC,kFAAkF;QAClF,KAAK,MAAM,IAAI,IAAI,SAAS,EAAE,CAAC;YAC9B,MAAM,aAAa,GAAG,UAAU,GAAG,IAAI,CAAC,MAAM,CAAA;YAE9C,MAAM,SAAS,GAAG,2BAA2B,CAAC,SAAS,CAAA;YAEvD,IAAI,IAAI,KAAK,GAAG,IAAI,SAAS,IAAI,SAAS,CAAC,aAAa,IAAI,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC,KAAK,GAAG,EAAE,CAAC;gBACvF,2BAA2B,CAAC,SAAS,CAAC,IAAI,IAAI,GAAG,CAAA;gBACjD,2BAA2B,CAAC,SAAS,CAAC,SAAS,GAAG,aAAa,CAAA;YAChE,CAAC;iBAAM,CAAC;gBACP,MAAM,QAAQ,GAAG,UAAU,CAAC,SAAS,CAAC,UAAU,EAAE,aAAa,CAAC,CAAA;gBAEhE,2BAA2B,CAAC,OAAO,CAAC,QAAQ,EAAE,UAAU,EAAE,IAAI,CAAC,CAAA;YAChE,CAAC;YAED,UAAU,GAAG,aAAa,CAAA;QAC3B,CAAC;IACF,CAAC;IAED,KAAK,IAAI,SAAS,GAAG,CAAC,EAAE,SAAS,GAAG,YAAY,CAAC,MAAM,EAAE,SAAS,EAAE,EAAE,CAAC;QACtE,MAAM,SAAS,GAAG,YAAY,CAAC,UAAU,CAAC,SAAS,CAAC,CAAA;QACpD,MAAM,eAAe,GAAG,SAAS,CAAC,WAAW,CAAA;QAE7C,MAAM,qBAAqB,GAAG,SAAS,GAAG,CAAC,CAAC,CAAC,CAAC,YAAY,CAAC,OAAO,CAAC,SAAS,GAAG,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,CAAA;QAE/F,8FAA8F;QAC9F,IAAI,qBAAqB,KAAK,eAAe,EAAE,CAAC;YAC/C,MAAM,SAAS,GAAG,UAAU,CAAC,SAAS,CAAC,qBAAqB,EAAE,eAAe,CAAC,CAAA;YAE9E,0BAA0B,CAAC,SAAS,EAAE,qBAAqB,CAAC,CAAA;QAC7D,CAAC;QAED,2BAA2B,CAAC,OAAO,CAAC,IAAI,CAAC,SAAS,CAAC,CAAA;QACnD,2BAA2B,CAAC,GAAG,CAAC,2BAA2B,CAAC,MAAM,GAAG,CAAC,EAAE,SAAS,CAAC,CAAA;QAElF,oEAAoE;QACpE,IAAI,SAAS,KAAK,YAAY,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAC3C,IAAI,UAAU,CAAC,MAAM,KAAK,SAAS,CAAC,SAAS,EAAE,CAAC;gBAC/C,MAAM,SAAS,GAAG,UAAU,CAAC,SAAS,CAAC,SAAS,CAAC,SAAS,EAAE,UAAU,CAAC,MAAM,CAAC,CAAA;gBAE9E,0BAA0B,CAAC,SAAS,EAAE,SAAS,CAAC,SAAS,CAAC,CAAA;YAC3D,CAAC;QACF,CAAC;IACF,CAAC;IAED,OAAO,EAAE,2BAA2B,EAAE,2BAA2B,EAAE,CAAA;AACpE,CAAC;AAED,gGAAgG;AAChG,iBAAiB;AACjB,gGAAgG;AAChG,SAAS,iCAAiC,CAAC,OAA4B;IACtE,MAAM,uBAAuB,GAAG,gBAAgB,CAAC,OAAO,CAAC,QAAQ,IAAI,EAAE,CAAC,IAAI,EAAE,CAAA;IAC9E,MAAM,2BAA2B,GAAG,sBAAsB,CAAC,OAAO,CAAC,QAAQ,IAAI,EAAE,CAAC,IAAI,EAAE,CAAA;IACxF,MAAM,8BAA8B,GAAG,wCAAwC,CAAC,OAAO,CAAC,QAAQ,IAAI,EAAE,CAAC,IAAI,EAAE,CAAA;IAC7G,MAAM,6CAA6C,GAAG,8BAA8B,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,UAAU,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC,CAAA;IACzH,MAAM,kBAAkB,GAAG,OAAO,CAAC,kBAAkB,IAAI,EAAE,CAAA;IAE3D,IAAI,YAAY,GAAG;QAClB,GAAG,kBAAkB;QACrB,GAAG,uBAAuB;QAC1B,GAAG,2BAA2B;QAC9B,GAAG,8BAA8B;QACjC,GAAG,6CAA6C;QAChD,GAAG,gBAAgB;QACnB,GAAG,eAAe;KAClB,CAAA;IAED,MAAM,WAAW,GAAG,wBAAwB,CAAC;QAC5C,GAAG,YAAY;QACf,GAAG,YAAY,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,iBAAiB,EAAE,CAAC;QACrD,GAAG,YAAY,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,iBAAiB,EAAE,CAAC;KACrD,CAAC,CAAA;IAEF,MAAM,kBAAkB,GAAG,WAAW,CAAC,WAAW,EAAE,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC,CAAA;IAErE,OAAO,kBAAkB,CAAA;AAC1B,CAAC;AAED,KAAK,UAAU,kBAAkB;IAChC,IAAI,CAAC;QACJ,MAAM,eAAe,GAAG,MAAM,MAAM,CAAC,mCAAmC,CAAC,CAAA;QAEzE,OAAO,eAAe,CAAA;IACvB,CAAC;IAAC,MAAM,CAAC;QACR,OAAO,SAAS,CAAA;IACjB,CAAC;AACF,CAAC;AAUD,MAAM,OAAO,YAAY;IACxB,SAAS,CAAO;IAChB,KAAK,CAAc;IAEnB,YAAY,SAAgB,EAAE,KAAmB;QAChD,IAAI,CAAC,SAAS,GAAG,SAAS,CAAA;QAC1B,IAAI,CAAC,KAAK,GAAG,KAAK,CAAA;IACnB,CAAC;IAED,IAAI,IAAI;QACP,OAAO,IAAI,CAAC,KAAK,CAAC,IAAI,CAAA;IACvB,CAAC;IAED,IAAI,SAAS;QACZ,OAAO;YACN,KAAK,EAAE,IAAI,CAAC,KAAK,CAAC,UAAU,CAAC,WAAW;YACxC,GAAG,EAAE,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC,SAAS;SACnC,CAAA;IACF,CAAC;CACD;AAED,MAAM,OAAO,QAAS,SAAQ,YAAY;IACzC,OAAO,GAAa,EAAE,CAAA;CACtB;AAED,MAAM,OAAO,MAAO,SAAQ,YAAY;CACvC;AAaD,MAAM,CAAC,MAAM,0BAA0B,GAAwB;IAC9D,QAAQ,EAAE,EAAE;IACZ,kBAAkB,EAAE,EAAE;IACtB,6BAA6B,EAAE,IAAI;CACnC,CAAA"}
|
|
1
|
+
{"version":3,"file":"TextSegmentation.js","sourceRoot":"","sources":["../src/TextSegmentation.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,wBAAwB,IAAI,wBAAwB,EAAE,qBAAqB,EAAE,mDAAmD,EAAE,gCAAgC,EAAE,uBAAuB,EAAE,yBAAyB,EAA0D,MAAM,eAAe,CAAA;AAC9S,OAAO,EAAE,gBAAgB,EAAE,sBAAsB,EAAE,wCAAwC,EAAE,gBAAgB,EAAE,eAAe,EAAE,MAAM,mBAAmB,CAAA;AACzJ,OAAO,EAAE,yBAAyB,EAAE,MAAM,iCAAiC,CAAA;AAC3E,OAAO,EAAE,YAAY,EAAE,MAAM,mBAAmB,CAAA;AAChD,OAAO,EAAE,oBAAoB,EAAE,MAAM,0BAA0B,CAAA;AAC/D,OAAO,EAAE,YAAY,EAAkB,MAAM,mBAAmB,CAAA;AAEhE,OAAO,EAAE,WAAW,EAAE,MAAM,iBAAiB,CAAA;AAE7C,gGAAgG;AAChG,mBAAmB;AACnB,gGAAgG;AAChG,MAAM,CAAC,KAAK,UAAU,WAAW,CAAC,IAAY,EAAE,OAA6B;IAC5E,MAAM,YAAY,GAAG,MAAM,YAAY,CAAC,IAAI,EAAE,OAAO,CAAC,CAAA;IAEtD,OAAO,mBAAmB,CAAC,YAAY,CAAC,CAAA;AACzC,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,mBAAmB,CAAC,YAA0B;IACnE,MAAM,kBAAkB,GAAY,EAAE,CAAA;IAEtC,MAAM,0BAA0B,GAAG,CAAC,CAAA;IAEpC,IAAI,uBAAuB,GAAG,CAAC,CAAA;IAC/B,IAAI,0BAA0B,GAAG,CAAC,CAAA;IAElC,KAAK,IAAI,SAAS,GAAG,CAAC,EAAE,SAAS,GAAG,YAAY,CAAC,MAAM,EAAE,SAAS,EAAE,EAAE,CAAC;QACtE,MAAM,IAAI,GAAG,YAAY,CAAC,SAAS,CAAC,SAAS,CAAC,CAAA;QAE9C,IAAI,0BAA0B,GAAG,0BAA0B,EAAE,CAAC;YAC7D,MAAM,OAAO,GAAG,IAAI,CAAC,QAAQ,CAAC,yBAAyB,CAAC,CAAA;YAExD,KAAK,MAAM,CAAC,IAAI,OAAO,EAAE,CAAC;gBACzB,0BAA0B,IAAI,CAAC,CAAA;gBAE/B,IAAI,0BAA0B,IAAI,0BAA0B,EAAE,CAAC;oBAC9D,MAAK;gBACN,CAAC;YACF,CAAC;QACF,CAAC;QAED,IAAI,0BAA0B,IAAI,0BAA0B,IAAI,gCAAgC,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;YAC7G,IAAI,wBAAwB,GAAG,SAAS,CAAA;YACxC,IAAI,mCAAmC,GAAG,KAAK,CAAA;YAE/C,OAAO,wBAAwB,GAAG,YAAY,CAAC,MAAM,EAAE,CAAC;gBACvD,MAAM,YAAY,GAAG,YAAY,CAAC,SAAS,CAAC,wBAAwB,CAAC,CAAA;gBAErE,IAAI,mDAAmD,CAAC,IAAI,CAAC,YAAY,CAAC,EAAE,CAAC;oBAC5E,IAAI,CAAC,mCAAmC,IAAI,uBAAuB,CAAC,IAAI,CAAC,YAAY,CAAC,EAAE,CAAC;wBACxF,mCAAmC,GAAG,IAAI,CAAA;oBAC3C,CAAC;oBAED,wBAAwB,EAAE,CAAA;gBAC3B,CAAC;qBAAM,CAAC;oBACP,MAAK;gBACN,CAAC;YACF,CAAC;YAED,IAAI,wBAAwB,KAAK,YAAY,CAAC,MAAM;gBACnD,mCAAmC;gBACnC,CAAC,GAAG,EAAE,GAAG,EAAE,GAAG,CAAC,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,CAAC;gBAEjC,kBAAkB,CAAC,IAAI,CAAC;oBACvB,KAAK,EAAE,uBAAuB;oBAC9B,GAAG,EAAE,wBAAwB;iBAC7B,CAAC,CAAA;gBAEF,uBAAuB,GAAG,wBAAwB,CAAA;gBAClD,0BAA0B,GAAG,CAAC,CAAA;gBAE9B,SAAS,GAAG,wBAAwB,GAAG,CAAC,CAAA;YACzC,CAAC;QACF,CAAC;IACF,CAAC;IAED,IAAI,uBAAuB,GAAG,YAAY,CAAC,MAAM,EAAE,CAAC;QACnD,kBAAkB,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,uBAAuB,EAAE,GAAG,EAAE,YAAY,CAAC,MAAM,EAAE,CAAC,CAAA;IACtF,CAAC;IAED,MAAM,SAAS,GAAe,EAAE,CAAA;IAEhC,KAAK,MAAM,SAAS,IAAI,kBAAkB,EAAE,CAAC;QAC5C,MAAM,oBAAoB,GAAG,YAAY,CAAC,KAAK,CAAC,SAAS,CAAC,KAAK,EAAE,SAAS,CAAC,GAAG,CAAC,CAAA;QAE/E,SAAS,CAAC,IAAI,CAAC,IAAI,QAAQ,CAAC,SAAS,EAAE,oBAAoB,CAAC,CAAC,CAAA;IAC9D,CAAC;IAED,KAAK,MAAM,QAAQ,IAAI,SAAS,EAAE,CAAC;QAClC,MAAM,gBAAgB,GAAY,EAAE,CAAA;QAEpC,IAAI,qBAAqB,GAAG,QAAQ,CAAC,SAAS,CAAC,GAAG,CAAA;QAClD,IAAI,qBAAqB,GAAG,QAAQ,CAAC,SAAS,CAAC,KAAK,CAAA;QAEpD,KAAK,IAAI,SAAS,GAAG,qBAAqB,EAAE,SAAS,GAAG,qBAAqB,EAAE,SAAS,EAAE,EAAE,CAAC;YAC5F,MAAM,WAAW,GAAG,YAAY,CAAC,SAAS,CAAC,SAAS,CAAC,CAAA;YAErD,IAAI,qBAAqB,CAAC,IAAI,CAAC,WAAW,CAAC,EAAE,CAAC;gBAC7C,IAAI,kBAAkB,GAAG,KAAK,CAAA;gBAE9B,OAAO,SAAS,GAAG,qBAAqB,GAAG,CAAC,EAAE,CAAC;oBAC9C,MAAM,QAAQ,GAAG,YAAY,CAAC,SAAS,CAAC,SAAS,GAAG,CAAC,CAAC,CAAA;oBAEtD,IAAI,CAAC,mDAAmD,CAAC,IAAI,CAAC,QAAQ,CAAC,EAAE,CAAC;wBACzE,MAAK;oBACN,CAAC;oBAED,IAAI,uBAAuB,CAAC,IAAI,CAAC,QAAQ,CAAC,EAAE,CAAC;wBAC5C,kBAAkB,GAAG,IAAI,CAAA;oBAC1B,CAAC;oBAED,IAAI,QAAQ,KAAK,GAAG,IAAI,kBAAkB,EAAE,CAAC;wBAC5C,MAAK;oBACN,CAAC;oBAED,SAAS,IAAI,CAAC,CAAA;gBACf,CAAC;gBAED,gBAAgB,CAAC,IAAI,CAAC;oBACrB,KAAK,EAAE,qBAAqB;oBAC5B,GAAG,EAAE,SAAS,GAAG,CAAC;iBAClB,CAAC,CAAA;gBAEF,qBAAqB,GAAG,SAAS,GAAG,CAAC,CAAA;YACtC,CAAC;QACF,CAAC;QAED,IAAI,qBAAqB,GAAG,qBAAqB,EAAE,CAAC;YACnD,gBAAgB,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,qBAAqB,EAAE,GAAG,EAAE,qBAAqB,EAAE,CAAC,CAAA;QACpF,CAAC;QAED,KAAK,MAAM,SAAS,IAAI,gBAAgB,EAAE,CAAC;YAC1C,MAAM,kBAAkB,GAAG,YAAY,CAAC,KAAK,CAAC,SAAS,CAAC,KAAK,EAAE,SAAS,CAAC,GAAG,CAAC,CAAA;YAE7E,QAAQ,CAAC,OAAO,CAAC,IAAI,CAAC,IAAI,MAAM,CAAC,SAAS,EAAE,kBAAkB,CAAC,CAAC,CAAA;QACjE,CAAC;IACF,CAAC;IAED,MAAM,MAAM,GAAuB;QAClC,KAAK,EAAE,YAAY;QACnB,SAAS;KACT,CAAA;IAED,OAAO,MAAM,CAAA;AACd,CAAC;AAED,MAAM,yBAAyB,GAAG,IAAI,GAAG,EAAkB,CAAA;AAE3D,MAAM,CAAC,KAAK,UAAU,YAAY,CAAC,IAAY,EAAE,OAA6B;IAC7E,IAAI,CAAC,OAAO,EAAE,CAAC;QACd,OAAO,GAAG,EAAE,CAAA;IACb,CAAC;IAED,OAAO,GAAG,EAAE,GAAG,0BAA0B,EAAE,GAAG,OAAO,EAAE,CAAA;IAEvD,IAAI,OAAO,CAAC,QAAQ,EAAE,CAAC;QACtB,OAAO,CAAC,QAAQ,GAAG,oBAAoB,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAA;IAC1D,CAAC;IAED,MAAM,aAAa,GAAG,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,CAAA;IAE7C,IAAI,kBAAkB,GAAG,yBAAyB,CAAC,GAAG,CAAC,aAAa,CAAC,CAAA;IAErE,IAAI,CAAC,kBAAkB,EAAE,CAAC;QACzB,kBAAkB,GAAG,iCAAiC,CAAC,OAAO,CAAC,CAAA;QAE/D,yBAAyB,CAAC,GAAG,CAAC,aAAa,EAAE,kBAAkB,CAAC,CAAA;IACjE,CAAC;IAED,IAAI,YAAY,GAAG,IAAI,YAAY,EAAE,CAAA;IAErC,SAAS,0BAA0B,CAAC,WAAmB,EAAE,SAAiB;QACzE,MAAM,wBAAwB,GAAG,IAAI,CAAC,SAAS,CAAC,WAAW,EAAE,SAAS,CAAC,CAAA;QAEvE,IAAI,UAAU,GAAG,WAAW,CAAA;QAC5B,IAAI,0BAA0B,GAAG,WAAW,CAAA;QAE5C,SAAS,0BAA0B;YAClC,IAAI,UAAU,GAAG,0BAA0B,EAAE,CAAC;gBAC7C,MAAM,QAAQ,GAAG,IAAI,CAAC,SAAS,CAAC,0BAA0B,EAAE,UAAU,CAAC,CAAA;gBACvE,YAAY,CAAC,OAAO,CAAC,QAAQ,EAAE,0BAA0B,EAAE,IAAI,CAAC,CAAA;gBAEhE,0BAA0B,GAAG,UAAU,CAAA;YACxC,CAAC;QACF,CAAC;QAED,KAAK,MAAM,IAAI,IAAI,wBAAwB,EAAE,CAAC;YAC7C,IAAI,IAAI,KAAK,GAAG,EAAE,CAAC;gBAClB,UAAU,IAAI,CAAC,CAAA;gBAEf,SAAQ;YACT,CAAC;YAED,0BAA0B,EAAE,CAAA;YAE5B,UAAU,IAAI,IAAI,CAAC,MAAM,CAAA;YAEzB,0BAA0B,EAAE,CAAA;QAC7B,CAAC;QAED,0BAA0B,EAAE,CAAA;IAC7B,CAAC;IAED,MAAM,WAAW,GAAG,IAAI,CAAC,QAAQ,CAAC,kBAAkB,CAAC,CAAA;IAErD,IAAI,kBAAkB,GAAG,CAAC,CAAA;IAE1B,IAAI,WAAW,EAAE,CAAC;QACjB,KAAK,MAAM,KAAK,IAAI,WAAW,EAAE,CAAC;YACjC,MAAM,OAAO,GAAG,KAAK,CAAC,OAAQ,CAAC,CAAC,CAAC,CAAA;YACjC,MAAM,gBAAgB,GAAG,OAAO,CAAC,CAAC,CAAC,CAAA;YACnC,MAAM,cAAc,GAAG,OAAO,CAAC,CAAC,CAAC,CAAA;YAEjC,IAAI,gBAAgB,GAAG,kBAAkB,EAAE,CAAC;gBAC3C,0BAA0B,CAAC,kBAAkB,EAAE,gBAAgB,CAAC,CAAA;YACjE,CAAC;YAED,MAAM,QAAQ,GAAG,IAAI,CAAC,SAAS,CAAC,gBAAgB,EAAE,cAAc,CAAC,CAAA;YACjE,YAAY,CAAC,OAAO,CAAC,QAAQ,EAAE,gBAAgB,EAAE,KAAK,CAAC,CAAA;YAEvD,kBAAkB,GAAG,cAAc,CAAA;QACpC,CAAC;QAED,0BAA0B,CAAC,kBAAkB,EAAE,IAAI,CAAC,MAAM,CAAC,CAAA;IAC5D,CAAC;IAED,IAAI,OAAO,CAAC,6BAA6B,EAAE,CAAC;QAC3C,YAAY,GAAG,MAAM,yBAAyB,CAAC,IAAI,EAAE,YAAY,CAAC,CAAA;IACnE,CAAC;IAED,OAAO,YAAY,CAAA;AACpB,CAAC;AAED,KAAK,UAAU,yBAAyB,CAAC,cAAsB,EAAE,YAA0B;IAC1F,MAAM,eAAe,GAAG,MAAM,kBAAkB,EAAE,CAAA;IAElD,IAAI,eAAe,KAAK,SAAS,EAAE,CAAC;QACnC,OAAO,YAAY,CAAA;IACpB,CAAC;IAED,IAAI,cAAc,GAAG,KAAK,CAAA;IAE1B,MAAM,eAAe,GAAG,IAAI,YAAY,EAAE,CAAA;IAE1C,KAAK,IAAI,SAAS,GAAG,CAAC,EAAE,SAAS,GAAG,YAAY,CAAC,MAAM,EAAE,SAAS,EAAE,EAAE,CAAC;QACtE,MAAM,SAAS,GAAG,YAAY,CAAC,OAAO,CAAC,SAAS,CAAC,CAAA;QACjD,MAAM,eAAe,GAAG,SAAS,CAAC,WAAW,CAAA;QAE7C,MAAM,IAAI,GAAG,YAAY,CAAC,SAAS,CAAC,SAAS,CAAC,CAAA;QAE9C,IAAI,yBAAyB,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;YAC1C,IAAI,CAAC,cAAc,EAAE,CAAC;gBACrB,MAAM,eAAe,CAAC,UAAU,EAAE,CAAA;gBAElC,cAAc,GAAG,IAAI,CAAA;YACtB,CAAC;YAED,MAAM,UAAU,GAAG,CAAC,GAAG,eAAe,CAAC,uBAAuB,CAAC,IAAI,CAAC,CAAC,CAAA;YAErE,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,UAAU,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;gBAChD,MAAM,kBAAkB,GAAG,eAAe,GAAG,UAAU,CAAC,CAAC,CAAC,CAAA;gBAC1D,MAAM,gBAAgB,GAAG,eAAe,GAAG,UAAU,CAAC,CAAC,GAAG,CAAC,CAAC,CAAA;gBAE5D,MAAM,WAAW,GAAG,cAAc,CAAC,SAAS,CAAC,kBAAkB,EAAE,gBAAgB,CAAC,CAAA;gBAElF,eAAe,CAAC,OAAO,CACtB,WAAW,EACX,kBAAkB,EAClB,KAAK,CACL,CAAA;YACF,CAAC;QACF,CAAC;aAAM,CAAC;YACP,MAAM,QAAQ,GAAG,cAAc,CAAC,SAAS,CAAC,SAAS,CAAC,WAAW,EAAE,SAAS,CAAC,SAAS,CAAC,CAAA;YAErF,eAAe,CAAC,OAAO,CAAC,QAAQ,EAAE,SAAS,CAAC,WAAW,EAAE,SAAS,CAAC,aAAa,CAAC,CAAA;QAClF,CAAC;IACF,CAAC;IAED,OAAO,eAAe,CAAA;AACvB,CAAC;AAED,uDAAuD;AACvD,MAAM,UAAU,wCAAwC,CAAC,YAA0B,EAAE,UAAkB;IACtG,MAAM,2BAA2B,GAAG,IAAI,GAAG,EAAkB,CAAA;IAE7D,MAAM,2BAA2B,GAAG,IAAI,YAAY,EAAE,CAAA;IAEtD,SAAS,0BAA0B,CAAC,SAAiB,EAAE,iBAAyB;QAC/E,IAAI,UAAU,GAAG,iBAAiB,CAAA;QAElC,kFAAkF;QAClF,KAAK,MAAM,IAAI,IAAI,SAAS,EAAE,CAAC;YAC9B,MAAM,aAAa,GAAG,UAAU,GAAG,IAAI,CAAC,MAAM,CAAA;YAE9C,MAAM,SAAS,GAAG,2BAA2B,CAAC,SAAS,CAAA;YAEvD,IAAI,IAAI,KAAK,GAAG,IAAI,SAAS,IAAI,SAAS,CAAC,aAAa,IAAI,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC,KAAK,GAAG,EAAE,CAAC;gBACvF,2BAA2B,CAAC,SAAS,CAAC,IAAI,IAAI,GAAG,CAAA;gBACjD,2BAA2B,CAAC,SAAS,CAAC,SAAS,GAAG,aAAa,CAAA;YAChE,CAAC;iBAAM,CAAC;gBACP,MAAM,QAAQ,GAAG,UAAU,CAAC,SAAS,CAAC,UAAU,EAAE,aAAa,CAAC,CAAA;gBAEhE,2BAA2B,CAAC,OAAO,CAAC,QAAQ,EAAE,UAAU,EAAE,IAAI,CAAC,CAAA;YAChE,CAAC;YAED,UAAU,GAAG,aAAa,CAAA;QAC3B,CAAC;IACF,CAAC;IAED,KAAK,IAAI,SAAS,GAAG,CAAC,EAAE,SAAS,GAAG,YAAY,CAAC,MAAM,EAAE,SAAS,EAAE,EAAE,CAAC;QACtE,MAAM,SAAS,GAAG,YAAY,CAAC,UAAU,CAAC,SAAS,CAAC,CAAA;QACpD,MAAM,eAAe,GAAG,SAAS,CAAC,WAAW,CAAA;QAE7C,MAAM,qBAAqB,GAAG,SAAS,GAAG,CAAC,CAAC,CAAC,CAAC,YAAY,CAAC,OAAO,CAAC,SAAS,GAAG,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,CAAA;QAE/F,8FAA8F;QAC9F,IAAI,qBAAqB,KAAK,eAAe,EAAE,CAAC;YAC/C,MAAM,SAAS,GAAG,UAAU,CAAC,SAAS,CAAC,qBAAqB,EAAE,eAAe,CAAC,CAAA;YAE9E,0BAA0B,CAAC,SAAS,EAAE,qBAAqB,CAAC,CAAA;QAC7D,CAAC;QAED,2BAA2B,CAAC,OAAO,CAAC,IAAI,CAAC,SAAS,CAAC,CAAA;QACnD,2BAA2B,CAAC,GAAG,CAAC,2BAA2B,CAAC,MAAM,GAAG,CAAC,EAAE,SAAS,CAAC,CAAA;QAElF,oEAAoE;QACpE,IAAI,SAAS,KAAK,YAAY,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAC3C,IAAI,UAAU,CAAC,MAAM,KAAK,SAAS,CAAC,SAAS,EAAE,CAAC;gBAC/C,MAAM,SAAS,GAAG,UAAU,CAAC,SAAS,CAAC,SAAS,CAAC,SAAS,EAAE,UAAU,CAAC,MAAM,CAAC,CAAA;gBAE9E,0BAA0B,CAAC,SAAS,EAAE,SAAS,CAAC,SAAS,CAAC,CAAA;YAC3D,CAAC;QACF,CAAC;IACF,CAAC;IAED,OAAO,EAAE,2BAA2B,EAAE,2BAA2B,EAAE,CAAA;AACpE,CAAC;AAED,SAAS,oBAAoB,CAAC,YAA0B,EAAE,IAAY;IACrE,MAAM,iBAAiB,GAAY,EAAE,CAAA;IACrC,MAAM,WAAW,GAAG,YAAY,CAAC,OAAO,CAAA;IAExC,IAAI,WAAW,CAAC,CAAC,CAAC,CAAC,WAAW,GAAG,CAAC,EAAE,CAAC;QACpC,iBAAiB,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,CAAC,EAAE,GAAG,EAAE,WAAW,CAAC,CAAC,CAAC,CAAC,WAAW,EAAE,CAAC,CAAA;IACtE,CAAC;IAED,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,WAAW,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QAC7C,MAAM,KAAK,GAAG,WAAW,CAAC,CAAC,CAAC,CAAA;QAE5B,MAAM,iBAAiB,GAAG,WAAW,CAAC,CAAC,GAAG,CAAC,CAAC,EAAE,SAAS,IAAI,CAAC,CAAA;QAE5D,IAAI,KAAK,CAAC,WAAW,GAAG,iBAAiB,EAAE,CAAC;YAC3C,iBAAiB,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,iBAAiB,EAAE,GAAG,EAAE,KAAK,CAAC,WAAW,EAAE,CAAC,CAAA;QAC7E,CAAC;QAED,IAAI,KAAK,CAAC,aAAa,EAAE,CAAC;YACzB,iBAAiB,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,KAAK,CAAC,WAAW,EAAE,GAAG,EAAE,KAAK,CAAC,SAAS,EAAE,CAAC,CAAA;QAC3E,CAAC;IACF,CAAC;IAED,CAAC;QACA,MAAM,aAAa,GAAG,WAAW,CAAC,WAAW,CAAC,MAAM,GAAG,CAAC,CAAC,EAAE,SAAS,CAAA;QAEpE,IAAI,aAAa,IAAI,aAAa,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC;YAClD,iBAAiB,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,aAAa,EAAE,GAAG,EAAE,IAAI,CAAC,MAAM,EAAE,CAAC,CAAA;QACnE,CAAC;IACF,CAAC;IAED,OAAO,iBAAiB,CAAA;AACzB,CAAC;AAED,gGAAgG;AAChG,iBAAiB;AACjB,gGAAgG;AAChG,SAAS,iCAAiC,CAAC,OAA4B;IACtE,MAAM,uBAAuB,GAAG,gBAAgB,CAAC,OAAO,CAAC,QAAQ,IAAI,EAAE,CAAC,IAAI,EAAE,CAAA;IAC9E,MAAM,2BAA2B,GAAG,sBAAsB,CAAC,OAAO,CAAC,QAAQ,IAAI,EAAE,CAAC,IAAI,EAAE,CAAA;IACxF,MAAM,8BAA8B,GAAG,wCAAwC,CAAC,OAAO,CAAC,QAAQ,IAAI,EAAE,CAAC,IAAI,EAAE,CAAA;IAC7G,MAAM,6CAA6C,GAAG,8BAA8B,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,UAAU,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC,CAAA;IACzH,MAAM,kBAAkB,GAAG,OAAO,CAAC,kBAAkB,IAAI,EAAE,CAAA;IAE3D,IAAI,YAAY,GAAG;QAClB,GAAG,kBAAkB;QACrB,GAAG,uBAAuB;QAC1B,GAAG,2BAA2B;QAC9B,GAAG,8BAA8B;QACjC,GAAG,6CAA6C;QAChD,GAAG,gBAAgB;QACnB,GAAG,eAAe;KAClB,CAAA;IAED,MAAM,WAAW,GAAG,wBAAwB,CAAC;QAC5C,GAAG,YAAY;QACf,GAAG,YAAY,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,iBAAiB,EAAE,CAAC;QACrD,GAAG,YAAY,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,iBAAiB,EAAE,CAAC;KACrD,CAAC,CAAA;IAEF,MAAM,kBAAkB,GAAG,WAAW,CAAC,WAAW,EAAE,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC,CAAA;IAErE,OAAO,kBAAkB,CAAA;AAC1B,CAAC;AAED,KAAK,UAAU,kBAAkB;IAChC,IAAI,CAAC;QACJ,MAAM,eAAe,GAAG,MAAM,MAAM,CAAC,mCAAmC,CAAC,CAAA;QAEzE,OAAO,eAAe,CAAA;IACvB,CAAC;IAAC,MAAM,CAAC;QACR,OAAO,SAAS,CAAA;IACjB,CAAC;AACF,CAAC;AAUD,MAAM,OAAO,YAAY;IACxB,SAAS,CAAO;IAChB,KAAK,CAAc;IAEnB,YAAY,SAAgB,EAAE,KAAmB;QAChD,IAAI,CAAC,SAAS,GAAG,SAAS,CAAA;QAC1B,IAAI,CAAC,KAAK,GAAG,KAAK,CAAA;IACnB,CAAC;IAED,IAAI,IAAI;QACP,OAAO,IAAI,CAAC,KAAK,CAAC,IAAI,CAAA;IACvB,CAAC;IAED,IAAI,SAAS;QACZ,OAAO;YACN,KAAK,EAAE,IAAI,CAAC,KAAK,CAAC,UAAU,CAAC,WAAW;YACxC,GAAG,EAAE,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC,SAAS;SACnC,CAAA;IACF,CAAC;CACD;AAED,MAAM,OAAO,QAAS,SAAQ,YAAY;IACzC,OAAO,GAAa,EAAE,CAAA;CACtB;AAED,MAAM,OAAO,MAAO,SAAQ,YAAY;CACvC;AAaD,MAAM,CAAC,MAAM,0BAA0B,GAAwB;IAC9D,QAAQ,EAAE,EAAE;IACZ,kBAAkB,EAAE,EAAE;IACtB,6BAA6B,EAAE,IAAI;CACnC,CAAA"}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@echogarden/text-segmentation",
|
|
3
|
-
"version": "0.3.
|
|
3
|
+
"version": "0.3.3",
|
|
4
4
|
"description": "A library for multilingual word, phrase and sentence segmentation.",
|
|
5
5
|
"author": "Rotem Dan",
|
|
6
6
|
"license": "MIT",
|
|
@@ -50,6 +50,6 @@
|
|
|
50
50
|
}
|
|
51
51
|
},
|
|
52
52
|
"devDependencies": {
|
|
53
|
-
"@types/node": "~22.15.
|
|
53
|
+
"@types/node": "~22.15.19"
|
|
54
54
|
}
|
|
55
55
|
}
|
package/src/Patterns.ts
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { anyOf, buildRegExp, charRange, inputEnd, inputStart, matches, oneOrMore, possibly, repeated, tab, unicodeProperty, whitespace, zeroOrMore } from 'regexp-composer'
|
|
1
|
+
import { anyOf, buildRegExp, charRange, codepoint, inputEnd, inputStart, matches, oneOrMore, possibly, repeated, tab, unicodeProperty, whitespace, zeroOrMore } from 'regexp-composer'
|
|
2
2
|
|
|
3
3
|
////////////////////////////////////////////////////////////////////////////////////////////////
|
|
4
4
|
// Pattern builder methods
|
|
@@ -25,19 +25,29 @@ export function buildSuppressionPattern(suppressions: string[]) {
|
|
|
25
25
|
////////////////////////////////////////////////////////////////////////////////////////////////
|
|
26
26
|
const letterPattern = unicodeProperty('Letter')
|
|
27
27
|
const markPattern = unicodeProperty('Mark')
|
|
28
|
+
const digitPattern = unicodeProperty('Decimal_Number')
|
|
29
|
+
const punctuationPattern = unicodeProperty('Punctuation')
|
|
30
|
+
|
|
31
|
+
const apostrophPattern = anyOf(`'`, `’`, `‘`)
|
|
32
|
+
const arabicNumeralPattern = charRange('0', '9')
|
|
28
33
|
|
|
29
34
|
const letterOrMarkPattern = anyOf(
|
|
30
35
|
letterPattern,
|
|
31
36
|
markPattern,
|
|
32
37
|
)
|
|
33
38
|
|
|
34
|
-
const
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
+
const letterOrMarkOrDigitPattern = anyOf(
|
|
40
|
+
letterPattern,
|
|
41
|
+
markPattern,
|
|
42
|
+
digitPattern,
|
|
43
|
+
)
|
|
39
44
|
|
|
40
|
-
|
|
45
|
+
// See: https://mathiasbynens.be/notes/es-unicode-property-escapes
|
|
46
|
+
const emojiPattern = anyOf(
|
|
47
|
+
[unicodeProperty('Emoji_Modifier_Base'), unicodeProperty('Emoji_Modifier')],
|
|
48
|
+
unicodeProperty('Emoji_Presentation'),
|
|
49
|
+
[unicodeProperty('Emoji'), codepoint('FE0F')]
|
|
50
|
+
)
|
|
41
51
|
|
|
42
52
|
const percentageCharacters = ['%']
|
|
43
53
|
const currencyCharacters = ['$', '¥', '€', '£', '₩', '₭', '₽', '₫', '฿', '¢', '₮', '؋', '₦', '₱', '₴', '₪']
|
|
@@ -100,18 +110,22 @@ const exponentPattern = [
|
|
|
100
110
|
oneOrMore(arabicNumeralPattern),
|
|
101
111
|
]
|
|
102
112
|
|
|
103
|
-
const numberPossiblyFollowedByExponentOrLettersPattern = [
|
|
113
|
+
const numberPossiblyFollowedByExponentOrLettersPattern = matches([
|
|
104
114
|
numberPattern,
|
|
105
115
|
|
|
106
116
|
possibly(anyOf(
|
|
107
117
|
exponentPattern,
|
|
108
118
|
|
|
109
119
|
matches(
|
|
110
|
-
zeroOrMore(unicodeProperty('Letter')), {
|
|
120
|
+
zeroOrMore(anyOf(unicodeProperty('Letter'), '_')), {
|
|
111
121
|
ifNotFollowedBy: digitPattern,
|
|
112
122
|
}),
|
|
113
|
-
))
|
|
114
|
-
|
|
123
|
+
))], {
|
|
124
|
+
|
|
125
|
+
ifPrecededBy: anyOf(whitespace, punctuationPattern, inputStart),
|
|
126
|
+
ifFollowedBy: anyOf(whitespace, punctuationPattern, inputEnd),
|
|
127
|
+
})
|
|
128
|
+
|
|
115
129
|
|
|
116
130
|
const precedingPercentageOrCurrencyPattern =
|
|
117
131
|
matches([
|
|
@@ -149,7 +163,7 @@ const timePattern = matches([
|
|
|
149
163
|
], {
|
|
150
164
|
ifPrecededBy: anyOf(whitespace, punctuationPattern, inputStart),
|
|
151
165
|
ifNotPrecededBy: ':',
|
|
152
|
-
ifFollowedBy: anyOf(whitespace, punctuationPattern, inputEnd),
|
|
166
|
+
ifFollowedBy: anyOf(whitespace, punctuationPattern, inputEnd, 'am', 'AM', 'pm', 'PM'),
|
|
153
167
|
ifNotFollowedBy: ':',
|
|
154
168
|
})
|
|
155
169
|
|
|
@@ -196,7 +210,7 @@ const wordInnerApostrophPattern =
|
|
|
196
210
|
matches(
|
|
197
211
|
apostrophPattern, {
|
|
198
212
|
|
|
199
|
-
ifPrecededBy:
|
|
213
|
+
ifPrecededBy: letterOrMarkOrDigitPattern,
|
|
200
214
|
ifFollowedBy: letterOrMarkPattern,
|
|
201
215
|
})
|
|
202
216
|
|
|
@@ -213,6 +227,7 @@ const basicWordPattern =
|
|
|
213
227
|
anyOf(
|
|
214
228
|
wordCharacterPattern,
|
|
215
229
|
wordInnerApostrophPattern,
|
|
230
|
+
'_'
|
|
216
231
|
),
|
|
217
232
|
)
|
|
218
233
|
|
|
@@ -246,26 +261,17 @@ const interpunctSeparatedWordPattern = [
|
|
|
246
261
|
]),
|
|
247
262
|
]
|
|
248
263
|
|
|
249
|
-
const underscoreSeparatedWordPattern = [
|
|
250
|
-
basicWordPattern,
|
|
251
|
-
|
|
252
|
-
oneOrMore([
|
|
253
|
-
'_',
|
|
254
|
-
basicWordPattern,
|
|
255
|
-
]),
|
|
256
|
-
]
|
|
257
|
-
|
|
258
264
|
const wordSegmentPattern = anyOf(
|
|
265
|
+
emojiPattern,
|
|
266
|
+
|
|
259
267
|
timePattern,
|
|
260
|
-
dimensionsPattern,
|
|
261
268
|
percentageOrCurrencyPattern,
|
|
269
|
+
|
|
270
|
+
dotSeparatedWordPattern,
|
|
262
271
|
numberPossiblyFollowedByExponentOrLettersPattern,
|
|
263
272
|
|
|
264
|
-
emojiPattern,
|
|
265
273
|
interpunctSeparatedWordPattern,
|
|
266
|
-
dotSeparatedWordPattern,
|
|
267
274
|
dottedAbbreviationSequencePattern,
|
|
268
|
-
underscoreSeparatedWordPattern,
|
|
269
275
|
basicWordPattern,
|
|
270
276
|
)
|
|
271
277
|
|
|
@@ -291,7 +297,7 @@ const phraseSeparatorTrailingPunctuationPattern = [
|
|
|
291
297
|
////////////////////////////////////////////////////////////////////////////////////////////////
|
|
292
298
|
const sentenceSeparatorCharacters = ['.', '。', '?', '？', '!', '！', '\n']
|
|
293
299
|
|
|
294
|
-
const
|
|
300
|
+
const sentenceSeparatorCharacterPattern = [
|
|
295
301
|
inputStart,
|
|
296
302
|
anyOf(...sentenceSeparatorCharacters),
|
|
297
303
|
inputEnd
|
|
@@ -303,16 +309,27 @@ const sentenceSeparatorTrailingPunctuationPattern = [
|
|
|
303
309
|
inputEnd
|
|
304
310
|
]
|
|
305
311
|
|
|
312
|
+
////////////////////////////////////////////////////////////////////////////////////////////////
|
|
313
|
+
// Other patterns
|
|
314
|
+
////////////////////////////////////////////////////////////////////////////////////////////////
|
|
315
|
+
|
|
316
|
+
export const whitespacePattern = [inputStart, oneOrMore(whitespace), inputEnd]
|
|
317
|
+
export const startsWithWhitespacePattern = [inputStart, whitespace]
|
|
318
|
+
export const endsWithWhitespacePattern = [whitespace, inputEnd]
|
|
319
|
+
|
|
306
320
|
////////////////////////////////////////////////////////////////////////////////////////////////
|
|
307
321
|
// Prebuilt regular expressions
|
|
308
322
|
////////////////////////////////////////////////////////////////////////////////////////////////
|
|
309
323
|
export const wordCharacterRegExp = buildRegExp(wordCharacterPattern)
|
|
310
|
-
export const whitespacePatternRegExp = buildRegExp(
|
|
324
|
+
export const whitespacePatternRegExp = buildRegExp(whitespacePattern)
|
|
311
325
|
|
|
312
326
|
export const letterPatternGlobalRegExp = buildRegExp(letterPattern, { global: true })
|
|
313
327
|
|
|
314
328
|
export const phraseSeparatorRegExp = buildRegExp(phraseSeparatorPattern)
|
|
315
329
|
export const phraseSeparatorTrailingPunctuationRegExp = buildRegExp(phraseSeparatorTrailingPunctuationPattern)
|
|
316
330
|
|
|
317
|
-
export const
|
|
318
|
-
export const
|
|
331
|
+
export const sentenceSeparatorCharacterRegExp = buildRegExp(sentenceSeparatorCharacterPattern)
|
|
332
|
+
export const sentenceSeparatorTrailingPunctuationCharacterRegExp = buildRegExp(sentenceSeparatorTrailingPunctuationPattern)
|
|
333
|
+
|
|
334
|
+
export const startsWithWhitespaceRegExp = buildRegExp(startsWithWhitespacePattern)
|
|
335
|
+
export const endsWithWhitespaceRegExp = buildRegExp(endsWithWhitespacePattern)
|
package/src/Suppressions.ts
CHANGED
|
@@ -10,9 +10,8 @@ export const cldrSuppressions: Record<string, string[]> = {
|
|
|
10
10
|
}
|
|
11
11
|
|
|
12
12
|
export const additionalSuppressions: Record<string, string[] > = {
|
|
13
|
-
en: [
|
|
14
|
-
|
|
15
|
-
],
|
|
13
|
+
en: ['TL;DR'],
|
|
14
|
+
th: ['ค.ศ.'],
|
|
16
15
|
}
|
|
17
16
|
|
|
18
17
|
export const leadingApostropheContractionSuppressions: Record<string, string[]> = {
|
package/src/Test.ts
CHANGED
|
@@ -4,26 +4,26 @@ import { Timer } from "./utilities/Timer.js"
|
|
|
4
4
|
const log = console.log
|
|
5
5
|
|
|
6
6
|
async function test1() {
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
//const text = 'Hello world! 23rd? 123% 3/4/7 Привет мир! 你好世界！'
|
|
7
|
+
//const text = 'Hello! 1. Say who? 2. How are you? This is v2.0 that good. I have 2 344 234ms C# is good games'
|
|
8
|
+
const text = 'the 6v34abc6g6 😊😊 -43.45 x -2445.65rty the'
|
|
9
|
+
//const text = 'Hello world! 23rd? 123% 3/4/7 Привет мир! 你好世界！'
|
|
11
10
|
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
11
|
+
const options: SegmentationOptions = {
|
|
12
|
+
language: 'en'
|
|
13
|
+
}
|
|
15
14
|
|
|
16
|
-
|
|
15
|
+
const wordSequence = await splitToWords(text, options)
|
|
17
16
|
|
|
18
|
-
|
|
17
|
+
console.log(JSON.stringify(wordSequence.wordArray))
|
|
19
18
|
|
|
20
|
-
|
|
19
|
+
const segmentedText = await segmentWordSequence(wordSequence)
|
|
21
20
|
|
|
22
|
-
|
|
23
|
-
|
|
21
|
+
const x = 0
|
|
22
|
+
}
|
|
24
23
|
|
|
24
|
+
async function test2() {
|
|
25
25
|
const { readFileSync, writeFileSync } = await import('fs')
|
|
26
|
-
const text = readFileSync('test-data/
|
|
26
|
+
const text = readFileSync('test-data/Test2.txt', 'utf-8')
|
|
27
27
|
|
|
28
28
|
const timer = new Timer()
|
|
29
29
|
|
|
@@ -60,14 +60,14 @@ async function test1() {
|
|
|
60
60
|
}
|
|
61
61
|
|
|
62
62
|
if (sentenceIndex < result!.sentences.length - 1) {
|
|
63
|
-
segmentedText += `\n${
|
|
63
|
+
segmentedText += `\n${'='.repeat(100)} \n`
|
|
64
64
|
}
|
|
65
65
|
}
|
|
66
66
|
|
|
67
67
|
writeFileSync('out/segmented.txt', segmentedText)
|
|
68
68
|
}
|
|
69
69
|
|
|
70
|
-
async function
|
|
70
|
+
async function test3() {
|
|
71
71
|
const text = ` Hello, how are you today ?? `
|
|
72
72
|
|
|
73
73
|
const words = await splitToWords(text)
|
|
@@ -81,9 +81,8 @@ async function test2() {
|
|
|
81
81
|
const x = 0
|
|
82
82
|
}
|
|
83
83
|
|
|
84
|
-
async function
|
|
85
|
-
|
|
86
|
-
const text = `
|
|
84
|
+
async function test4() {
|
|
85
|
+
const text = `
|
|
87
86
|
Hello 12/43 yo good-go bobo_baba man!
|
|
88
87
|
2.4a, 5.6 x&y x'v·y v‧z x·y·5
|
|
89
88
|
abc123 23+42.534 645
|
|
@@ -113,10 +112,10 @@ async function test3() {
|
|
|
113
112
|
|
|
114
113
|
`
|
|
115
114
|
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
115
|
+
const result = (await splitToWords(text)).wordArray
|
|
116
|
+
log(result.join(' | '))
|
|
117
|
+
|
|
119
118
|
}
|
|
120
119
|
|
|
121
|
-
test1()
|
|
122
|
-
|
|
120
|
+
//test1()
|
|
121
|
+
test2()
|
package/src/TextSegmentation.ts
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { buildWordOrNumberPattern as buildWordSplitterPattern, phraseSeparatorRegExp,
|
|
1
|
+
import { buildWordOrNumberPattern as buildWordSplitterPattern, phraseSeparatorRegExp, sentenceSeparatorTrailingPunctuationCharacterRegExp, sentenceSeparatorCharacterRegExp, whitespacePatternRegExp, letterPatternGlobalRegExp, startsWithWhitespacePattern, endsWithWhitespacePattern } from './Patterns.js'
|
|
2
2
|
import { cldrSuppressions, additionalSuppressions, leadingApostropheContractionSuppressions, nounSuppressions, tldSuppressions } from './Suppressions.js'
|
|
3
3
|
import { eastAsianCharRangesRegExp } from './EastAsianCharacterPatterns.js'
|
|
4
4
|
import { WordSequence } from './WordSequence.js'
|
|
@@ -30,7 +30,7 @@ export async function segmentWordSequence(wordSequence: WordSequence) {
|
|
|
30
30
|
if (currentSentenceLetterCount < minimumSentenceLetterCount) {
|
|
31
31
|
const matches = word.matchAll(letterPatternGlobalRegExp)
|
|
32
32
|
|
|
33
|
-
for (const
|
|
33
|
+
for (const _ of matches) {
|
|
34
34
|
currentSentenceLetterCount += 1
|
|
35
35
|
|
|
36
36
|
if (currentSentenceLetterCount >= minimumSentenceLetterCount) {
|
|
@@ -39,24 +39,38 @@ export async function segmentWordSequence(wordSequence: WordSequence) {
|
|
|
39
39
|
}
|
|
40
40
|
}
|
|
41
41
|
|
|
42
|
-
if (currentSentenceLetterCount >= minimumSentenceLetterCount &&
|
|
43
|
-
|
|
44
|
-
|
|
42
|
+
if (currentSentenceLetterCount >= minimumSentenceLetterCount && sentenceSeparatorCharacterRegExp.test(word)) {
|
|
43
|
+
let trailingSequenceEndIndex = wordIndex
|
|
44
|
+
let trailingSequenceContainedWhitespace = false
|
|
45
45
|
|
|
46
|
-
|
|
46
|
+
while (trailingSequenceEndIndex < wordSequence.length) {
|
|
47
|
+
const trailingWord = wordSequence.getWordAt(trailingSequenceEndIndex)
|
|
48
|
+
|
|
49
|
+
if (sentenceSeparatorTrailingPunctuationCharacterRegExp.test(trailingWord)) {
|
|
50
|
+
if (!trailingSequenceContainedWhitespace && whitespacePatternRegExp.test(trailingWord)) {
|
|
51
|
+
trailingSequenceContainedWhitespace = true
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
trailingSequenceEndIndex++
|
|
55
|
+
} else {
|
|
47
56
|
break
|
|
48
57
|
}
|
|
49
|
-
|
|
50
|
-
wordIndex += 1
|
|
51
58
|
}
|
|
52
59
|
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
60
|
+
if (trailingSequenceEndIndex === wordSequence.length ||
|
|
61
|
+
trailingSequenceContainedWhitespace ||
|
|
62
|
+
['。', '？', '！'].includes(word)) {
|
|
63
|
+
|
|
64
|
+
sentenceWordRanges.push({
|
|
65
|
+
start: sentenceStartWordOffset,
|
|
66
|
+
end: trailingSequenceEndIndex
|
|
67
|
+
})
|
|
68
|
+
|
|
69
|
+
sentenceStartWordOffset = trailingSequenceEndIndex
|
|
70
|
+
currentSentenceLetterCount = 0
|
|
57
71
|
|
|
58
|
-
|
|
59
|
-
|
|
72
|
+
wordIndex = trailingSequenceEndIndex - 1
|
|
73
|
+
}
|
|
60
74
|
}
|
|
61
75
|
}
|
|
62
76
|
|
|
@@ -87,7 +101,7 @@ export async function segmentWordSequence(wordSequence: WordSequence) {
|
|
|
87
101
|
while (wordIndex < sentenceEndWordOffset - 1) {
|
|
88
102
|
const nextWord = wordSequence.getWordAt(wordIndex + 1)
|
|
89
103
|
|
|
90
|
-
if (!
|
|
104
|
+
if (!sentenceSeparatorTrailingPunctuationCharacterRegExp.test(nextWord)) {
|
|
91
105
|
break
|
|
92
106
|
}
|
|
93
107
|
|
|
@@ -322,6 +336,39 @@ export function addMissingPunctuationWordsToWordSequence(wordSequence: WordSeque
|
|
|
322
336
|
return { wordSequenceWithPunctuation, originalWordsReverseMapping }
|
|
323
337
|
}
|
|
324
338
|
|
|
339
|
+
function getPunctuationRanges(wordSequence: WordSequence, text: string) {
|
|
340
|
+
const punctuationRanges: Range[] = []
|
|
341
|
+
const wordEntries = wordSequence.entries
|
|
342
|
+
|
|
343
|
+
if (wordEntries[0].startOffset > 0) {
|
|
344
|
+
punctuationRanges.push({ start: 0, end: wordEntries[0].startOffset })
|
|
345
|
+
}
|
|
346
|
+
|
|
347
|
+
for (let i = 0; i < wordEntries.length; i++) {
|
|
348
|
+
const entry = wordEntries[i]
|
|
349
|
+
|
|
350
|
+
const previousEndOffset = wordEntries[i - 1]?.endOffset ?? 0
|
|
351
|
+
|
|
352
|
+
if (entry.startOffset > previousEndOffset) {
|
|
353
|
+
punctuationRanges.push({ start: previousEndOffset, end: entry.startOffset })
|
|
354
|
+
}
|
|
355
|
+
|
|
356
|
+
if (entry.isPunctuation) {
|
|
357
|
+
punctuationRanges.push({ start: entry.startOffset, end: entry.endOffset })
|
|
358
|
+
}
|
|
359
|
+
}
|
|
360
|
+
|
|
361
|
+
{
|
|
362
|
+
const lastEndOffset = wordEntries[wordEntries.length - 1]?.endOffset
|
|
363
|
+
|
|
364
|
+
if (lastEndOffset && lastEndOffset < text.length) {
|
|
365
|
+
punctuationRanges.push({ start: lastEndOffset, end: text.length })
|
|
366
|
+
}
|
|
367
|
+
}
|
|
368
|
+
|
|
369
|
+
return punctuationRanges
|
|
370
|
+
}
|
|
371
|
+
|
|
325
372
|
////////////////////////////////////////////////////////////////////////////////////////////////
|
|
326
373
|
// Helper methods
|
|
327
374
|
////////////////////////////////////////////////////////////////////////////////////////////////
|