@echogarden/text-segmentation 0.1.0 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/Test.js +20 -8
- package/dist/Test.js.map +1 -1
- package/dist/TextSegmentation.d.ts +5 -0
- package/dist/TextSegmentation.js +87 -33
- package/dist/TextSegmentation.js.map +1 -1
- package/dist/WordSequence.d.ts +1 -0
- package/dist/WordSequence.js +3 -0
- package/dist/WordSequence.js.map +1 -1
- package/package.json +3 -3
- package/src/Test.ts +28 -11
- package/src/TextSegmentation.ts +107 -33
- package/src/WordSequence.ts +4 -0
package/dist/Test.js
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { segmentText, splitToWords } from "./TextSegmentation.js";
|
|
1
|
+
import { addMissingPunctuationWordsToWordSequence, segmentText, splitToWords, WordSequence } from "./TextSegmentation.js";
|
|
2
2
|
import { Timer } from "./utilities/Timer.js";
|
|
3
3
|
const log = console.log;
|
|
4
4
|
async function test1() {
|
|
@@ -9,13 +9,15 @@ async function test1() {
|
|
|
9
9
|
const { readFileSync, writeFileSync } = await import('fs');
|
|
10
10
|
const text = readFileSync('test-data/Test.txt', 'utf-8');
|
|
11
11
|
const timer = new Timer();
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
12
|
+
let result;
|
|
13
|
+
for (let i = 0; i < 3; i++) {
|
|
14
|
+
result = await segmentText(text, {
|
|
15
|
+
language: 'en',
|
|
16
|
+
customSuppressions: [],
|
|
17
|
+
enableEastAsianPostprocessing: true
|
|
18
|
+
});
|
|
19
|
+
timer.logAndRestart(`Total execution time`);
|
|
20
|
+
}
|
|
19
21
|
log('');
|
|
20
22
|
//
|
|
21
23
|
let segmentedText = '';
|
|
@@ -35,5 +37,15 @@ async function test1() {
|
|
|
35
37
|
}
|
|
36
38
|
writeFileSync('out/segmented.txt', segmentedText);
|
|
37
39
|
}
|
|
40
|
+
async function test2() {
|
|
41
|
+
const text = ` Hello, how are you today ?? `;
|
|
42
|
+
const words = await splitToWords(text);
|
|
43
|
+
const nonPunctuationWordEntries = words.nonPunctuationEntries;
|
|
44
|
+
const nonPunctuationWordSequence = new WordSequence();
|
|
45
|
+
nonPunctuationWordSequence.entries = nonPunctuationWordEntries;
|
|
46
|
+
const extendedWords = addMissingPunctuationWordsToWordSequence(nonPunctuationWordSequence, text);
|
|
47
|
+
const x = 0;
|
|
48
|
+
}
|
|
38
49
|
test1();
|
|
50
|
+
//test2()
|
|
39
51
|
//# sourceMappingURL=Test.js.map
|
package/dist/Test.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"Test.js","sourceRoot":"","sources":["../src/Test.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,WAAW,EAAE,YAAY,EAAE,MAAM,uBAAuB,CAAA;
|
|
1
|
+
{"version":3,"file":"Test.js","sourceRoot":"","sources":["../src/Test.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,wCAAwC,EAAsB,WAAW,EAAE,YAAY,EAAE,YAAY,EAAE,MAAM,uBAAuB,CAAA;AAC7I,OAAO,EAAE,KAAK,EAAE,MAAM,sBAAsB,CAAA;AAE5C,MAAM,GAAG,GAAG,OAAO,CAAC,GAAG,CAAA;AAEvB,KAAK,UAAU,KAAK;IACnB,CAAC;QACA,MAAM,YAAY,GAAG,MAAM,YAAY,CAAC,gCAAgC,CAAC,CAAA;QACzE,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,SAAS,CAAC,YAAY,CAAC,KAAK,CAAC,CAAC,CAAA;IAChD,CAAC;IAED,MAAM,EAAE,YAAY,EAAE,aAAa,EAAE,GAAG,MAAM,MAAM,CAAC,IAAI,CAAC,CAAA;IAC1D,MAAM,IAAI,GAAG,YAAY,CAAC,oBAAoB,EAAE,OAAO,CAAC,CAAA;IAExD,MAAM,KAAK,GAAG,IAAI,KAAK,EAAE,CAAA;IAEzB,IAAI,MAA0B,CAAA;IAE9B,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;QAC5B,MAAM,GAAG,MAAM,WAAW,CAAC,IAAI,EAAE;YAChC,QAAQ,EAAE,IAAI;YACd,kBAAkB,EAAE,EAAE;YACtB,6BAA6B,EAAE,IAAI;SACnC,CAAC,CAAA;QAEF,KAAK,CAAC,aAAa,CAAC,sBAAsB,CAAC,CAAA;IAC5C,CAAC;IAED,GAAG,CAAC,EAAE,CAAC,CAAA;IAEP,EAAE;IAEF,IAAI,aAAa,GAAG,EAAE,CAAA;IAEtB,KAAK,IAAI,aAAa,GAAG,CAAC,EAAE,aAAa,GAAG,MAAO,CAAC,SAAS,CAAC,MAAM,EAAE,aAAa,EAAE,EAAE,CAAC;QACvF,MAAM,QAAQ,GAAG,MAAO,CAAC,SAAS,CAAC,aAAa,CAAC,CAAA;QACjD,MAAM,OAAO,GAAG,QAAQ,CAAC,OAAO,CAAA;QAEhC,KAAK,IAAI,WAAW,GAAG,CAAC,EAAE,WAAW,GAAG,OAAO,CAAC,MAAM,EAAE,WAAW,EAAE,EAAE,CAAC;YACvE,MAAM,MAAM,GAAG,OAAO,CAAC,WAAW,CAAC,CAAA;YAEnC,aAAa,IAAI,MAAM,CAAC,YAAY,CAAC,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,CAAA;YAEtD,IAAI,WAAW,GAAG,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBACtC,aAAa,IAAI,KAAK,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,IAAI,CAAA;YAC1C,CAAC;QACF,CAAC;QAED,IAAI,aAAa,GAAG,MAAO,CAAC,SAAS,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAClD,aAAa,IAAI,KAAM,GAAG,CAAC,MAAM,CAAC,GAAG,CAAE,KAAK,CAAA;QAC7C,CAAC;IACF,CAAC;IAED,aAAa,CAAC,mBAAmB,EAAE,aAAa,CAAC,CAAA;AAClD,CAAC;AAED,KAAK,UAAU,KAAK;IACnB,MAAM,IAAI,GAAG,mDAAmD,CAAA;IAEhE,MAAM,KAAK,GAAG,MAAM,YAAY,CAAC,IAAI,CAAC,CAAA;IAEtC,MAAM,yBAAyB,GAAG,KAAK,CAAC,qBAAqB,CAAA;IAC7D,MAAM,0BAA0B,GAAG,IAAI,YAAY,EAAE,CAAA;IACrD,0BAA0B,CAAC,OAAO,GAAG,yBAAyB,CAAA;IAE9D,MAAM,aAAa,GAAG,wCAAwC,CAAC,0BAA0B,EAAE,IAAI,CAAC,CAAA;IAEhG,MAAM,CAAC,GAAG,CAAC,CAAA;AACZ,CAAC;AAED,KAAK,EAAE,CAAA;AACP,SAAS"}
|
|
@@ -1,8 +1,13 @@
|
|
|
1
1
|
import { WordSequence } from './WordSequence.js';
|
|
2
2
|
export { cldrSuppressions } from './Suppressions.js';
|
|
3
|
+
export { WordSequence, type WordEntry } from './WordSequence.js';
|
|
3
4
|
export declare function segmentText(text: string, options?: SegmentationOptions): Promise<SegmentationResult>;
|
|
4
5
|
export declare function segmentWordSequence(wordSequence: WordSequence): Promise<SegmentationResult>;
|
|
5
6
|
export declare function splitToWords(text: string, options?: SegmentationOptions): Promise<WordSequence>;
|
|
7
|
+
export declare function addMissingPunctuationWordsToWordSequence(wordSequence: WordSequence, sourceText: string): {
|
|
8
|
+
wordSequnceWithPunctuation: WordSequence;
|
|
9
|
+
originalWordsReverseMapping: Map<number, number>;
|
|
10
|
+
};
|
|
6
11
|
export interface SegmentationResult {
|
|
7
12
|
wordSequence: WordSequence;
|
|
8
13
|
sentences: Sentence[];
|
package/dist/TextSegmentation.js
CHANGED
|
@@ -4,6 +4,7 @@ import { cldrSuppressions, leadingApostropheContractionSuppressions, nounSuppres
|
|
|
4
4
|
import { eastAsianCharRangesRegExp } from './EastAsianCharacterPatterns.js';
|
|
5
5
|
import { WordSequence } from './WordSequence.js';
|
|
6
6
|
export { cldrSuppressions } from './Suppressions.js';
|
|
7
|
+
export { WordSequence } from './WordSequence.js';
|
|
7
8
|
export async function segmentText(text, options) {
|
|
8
9
|
options = { ...defaultSegmentationOptions, ...(options ?? {}) };
|
|
9
10
|
const wordSequence = await splitToWords(text, options);
|
|
@@ -79,48 +80,39 @@ export async function segmentWordSequence(wordSequence) {
|
|
|
79
80
|
};
|
|
80
81
|
return result;
|
|
81
82
|
}
|
|
83
|
+
const cachedWordSplitterRegExps = new Map();
|
|
82
84
|
export async function splitToWords(text, options) {
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
const
|
|
88
|
-
let
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
...nounSuppressions,
|
|
94
|
-
...tldSuppressions,
|
|
95
|
-
];
|
|
96
|
-
const wordPattern = buildWordSplitterPattern([
|
|
97
|
-
...suppressions,
|
|
98
|
-
...suppressions.map(word => word.toLocaleLowerCase()),
|
|
99
|
-
...suppressions.map(word => word.toLocaleUpperCase()),
|
|
100
|
-
]);
|
|
101
|
-
const wordSplitterRegExp = buildRegExp(wordPattern, { global: true });
|
|
102
|
-
//console.log(`Encoded pattern: ${wordSplitterRegExp.source}\n`)
|
|
85
|
+
if (!options) {
|
|
86
|
+
options = {};
|
|
87
|
+
}
|
|
88
|
+
options = { ...defaultSegmentationOptions, ...options };
|
|
89
|
+
const optionsAsJson = JSON.stringify(options);
|
|
90
|
+
let wordSplitterRegExp = cachedWordSplitterRegExps.get(optionsAsJson);
|
|
91
|
+
if (!wordSplitterRegExp) {
|
|
92
|
+
wordSplitterRegExp = buildWordSplitterRegExpForOptions(options);
|
|
93
|
+
cachedWordSplitterRegExps.set(optionsAsJson, wordSplitterRegExp);
|
|
94
|
+
}
|
|
103
95
|
let wordSequence = new WordSequence();
|
|
104
|
-
function
|
|
105
|
-
const
|
|
96
|
+
function addPunctuationWordsBetween(startOffset, endOffset) {
|
|
97
|
+
const punctuationWordSubstring = text.substring(startOffset, endOffset);
|
|
106
98
|
let charOffset = startOffset;
|
|
107
|
-
let
|
|
108
|
-
function
|
|
109
|
-
if (charOffset >
|
|
110
|
-
wordSequence.addWordEntry(text,
|
|
111
|
-
|
|
99
|
+
let punctuationWordStartOffset = startOffset;
|
|
100
|
+
function addPunctuationWordIfNeeded() {
|
|
101
|
+
if (charOffset > punctuationWordStartOffset) {
|
|
102
|
+
wordSequence.addWordEntry(text, punctuationWordStartOffset, charOffset, true);
|
|
103
|
+
punctuationWordStartOffset = charOffset;
|
|
112
104
|
}
|
|
113
105
|
}
|
|
114
|
-
for (const char of
|
|
106
|
+
for (const char of punctuationWordSubstring) {
|
|
115
107
|
if (char === ' ') {
|
|
116
108
|
charOffset += 1;
|
|
117
109
|
continue;
|
|
118
110
|
}
|
|
119
|
-
|
|
111
|
+
addPunctuationWordIfNeeded();
|
|
120
112
|
charOffset += char.length;
|
|
121
|
-
|
|
113
|
+
addPunctuationWordIfNeeded();
|
|
122
114
|
}
|
|
123
|
-
|
|
115
|
+
addPunctuationWordIfNeeded();
|
|
124
116
|
}
|
|
125
117
|
const wordMatches = text.matchAll(wordSplitterRegExp);
|
|
126
118
|
let lastMatchEndOffset = 0;
|
|
@@ -130,12 +122,12 @@ export async function splitToWords(text, options) {
|
|
|
130
122
|
const matchStartOffset = offsets[0];
|
|
131
123
|
const matchEndOffset = offsets[1];
|
|
132
124
|
if (matchStartOffset > lastMatchEndOffset) {
|
|
133
|
-
|
|
125
|
+
addPunctuationWordsBetween(lastMatchEndOffset, matchStartOffset);
|
|
134
126
|
}
|
|
135
127
|
wordSequence.addWordEntry(text, matchStartOffset, matchEndOffset, false);
|
|
136
128
|
lastMatchEndOffset = matchEndOffset;
|
|
137
129
|
}
|
|
138
|
-
|
|
130
|
+
addPunctuationWordsBetween(lastMatchEndOffset, text.length);
|
|
139
131
|
}
|
|
140
132
|
if (options.enableEastAsianPostprocessing) {
|
|
141
133
|
wordSequence = await postprocessEastAsianWords(text, wordSequence);
|
|
@@ -169,6 +161,68 @@ async function postprocessEastAsianWords(containingText, wordSequence) {
|
|
|
169
161
|
}
|
|
170
162
|
return newWordSequence;
|
|
171
163
|
}
|
|
164
|
+
function buildWordSplitterRegExpForOptions(options) {
|
|
165
|
+
const cldrSuppressionsForLang = cldrSuppressions[options.language ?? ''] ?? [];
|
|
166
|
+
const contractionSuppressionsForLang = leadingApostropheContractionSuppressions[options.language ?? ''] ?? [];
|
|
167
|
+
const contractionSuppressionsForLangWithSingleQuote = contractionSuppressionsForLang.map(str => str.replaceAll(`'`, `’`));
|
|
168
|
+
const customSuppressions = options.customSuppressions ?? [];
|
|
169
|
+
let suppressions = [
|
|
170
|
+
...customSuppressions,
|
|
171
|
+
...cldrSuppressionsForLang,
|
|
172
|
+
...contractionSuppressionsForLang,
|
|
173
|
+
...contractionSuppressionsForLangWithSingleQuote,
|
|
174
|
+
...nounSuppressions,
|
|
175
|
+
...tldSuppressions,
|
|
176
|
+
];
|
|
177
|
+
const wordPattern = buildWordSplitterPattern([
|
|
178
|
+
...suppressions,
|
|
179
|
+
...suppressions.map(word => word.toLocaleLowerCase()),
|
|
180
|
+
...suppressions.map(word => word.toLocaleUpperCase()),
|
|
181
|
+
]);
|
|
182
|
+
const wordSplitterRegExp = buildRegExp(wordPattern, { global: true });
|
|
183
|
+
return wordSplitterRegExp;
|
|
184
|
+
}
|
|
185
|
+
// Add any missing punctuation words to a word sequence
|
|
186
|
+
export function addMissingPunctuationWordsToWordSequence(wordSequence, sourceText) {
|
|
187
|
+
const originalWordsReverseMapping = new Map();
|
|
188
|
+
const wordSequnceWithPunctuation = new WordSequence();
|
|
189
|
+
function addWordEntriesForTextSlice(textSlice, initialCharOffset) {
|
|
190
|
+
let charOffset = initialCharOffset;
|
|
191
|
+
// Add entry for every codepoint (this will correctly treat characters beyond BMP)
|
|
192
|
+
for (const char of textSlice) {
|
|
193
|
+
const charEndOffset = charOffset + char.length;
|
|
194
|
+
const lastEntry = wordSequnceWithPunctuation.lastEntry;
|
|
195
|
+
if (char === ' ' && lastEntry && lastEntry.isPunctuation && lastEntry.text[0] === ' ') {
|
|
196
|
+
wordSequnceWithPunctuation.lastEntry.text += ' ';
|
|
197
|
+
wordSequnceWithPunctuation.lastEntry.endOffset = charEndOffset;
|
|
198
|
+
}
|
|
199
|
+
else {
|
|
200
|
+
wordSequnceWithPunctuation.addWordEntry(sourceText, charOffset, charEndOffset, true);
|
|
201
|
+
}
|
|
202
|
+
charOffset = charEndOffset;
|
|
203
|
+
}
|
|
204
|
+
}
|
|
205
|
+
for (let wordIndex = 0; wordIndex < wordSequence.length; wordIndex++) {
|
|
206
|
+
const wordEntry = wordSequence.getEntryAt(wordIndex);
|
|
207
|
+
const wordStartOffset = wordEntry.startOffset;
|
|
208
|
+
const previousWordEndOffset = wordIndex > 0 ? wordSequence.entries[wordIndex - 1].endOffset : 0;
|
|
209
|
+
// Add entries for any punctuation characters between the current and previous word (or start)
|
|
210
|
+
if (previousWordEndOffset !== wordStartOffset) {
|
|
211
|
+
const textSlice = sourceText.substring(previousWordEndOffset, wordStartOffset);
|
|
212
|
+
addWordEntriesForTextSlice(textSlice, previousWordEndOffset);
|
|
213
|
+
}
|
|
214
|
+
wordSequnceWithPunctuation.entries.push(wordEntry);
|
|
215
|
+
originalWordsReverseMapping.set(wordSequnceWithPunctuation.length - 1, wordIndex);
|
|
216
|
+
// If last word, add entries for any trailing punctuation characters
|
|
217
|
+
if (wordIndex === wordSequence.length - 1) {
|
|
218
|
+
if (sourceText.length !== wordEntry.endOffset) {
|
|
219
|
+
const textSlice = sourceText.substring(wordEntry.endOffset, sourceText.length);
|
|
220
|
+
addWordEntriesForTextSlice(textSlice, wordEntry.endOffset);
|
|
221
|
+
}
|
|
222
|
+
}
|
|
223
|
+
}
|
|
224
|
+
return { wordSequnceWithPunctuation, originalWordsReverseMapping };
|
|
225
|
+
}
|
|
172
226
|
async function getIcuSegmentation() {
|
|
173
227
|
try {
|
|
174
228
|
const icuSegmentation = await import('@echogarden/icu-segmentation-wasm');
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"TextSegmentation.js","sourceRoot":"","sources":["../src/TextSegmentation.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,WAAW,EAAE,MAAM,iBAAiB,CAAA;AAC7C,OAAO,EAAE,wBAAwB,IAAI,wBAAwB,EAAE,qBAAqB,EAAE,0CAA0C,EAAE,uBAAuB,EAAE,uBAAuB,EAAE,MAAM,eAAe,CAAA;AACzM,OAAO,EAAE,gBAAgB,EAAE,wCAAwC,EAAE,gBAAgB,EAAE,eAAe,EAAE,MAAM,mBAAmB,CAAA;AACjI,OAAO,EAAE,yBAAyB,EAAE,MAAM,iCAAiC,CAAA;AAC3E,OAAO,EAAE,YAAY,EAAE,MAAM,mBAAmB,CAAA;AAEhD,OAAO,EAAE,gBAAgB,EAAE,MAAM,mBAAmB,CAAA;
|
|
1
|
+
{"version":3,"file":"TextSegmentation.js","sourceRoot":"","sources":["../src/TextSegmentation.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,WAAW,EAAE,MAAM,iBAAiB,CAAA;AAC7C,OAAO,EAAE,wBAAwB,IAAI,wBAAwB,EAAE,qBAAqB,EAAE,0CAA0C,EAAE,uBAAuB,EAAE,uBAAuB,EAAE,MAAM,eAAe,CAAA;AACzM,OAAO,EAAE,gBAAgB,EAAE,wCAAwC,EAAE,gBAAgB,EAAE,eAAe,EAAE,MAAM,mBAAmB,CAAA;AACjI,OAAO,EAAE,yBAAyB,EAAE,MAAM,iCAAiC,CAAA;AAC3E,OAAO,EAAE,YAAY,EAAE,MAAM,mBAAmB,CAAA;AAEhD,OAAO,EAAE,gBAAgB,EAAE,MAAM,mBAAmB,CAAA;AACpD,OAAO,EAAE,YAAY,EAAkB,MAAM,mBAAmB,CAAA;AAEhE,MAAM,CAAC,KAAK,UAAU,WAAW,CAAC,IAAY,EAAE,OAA6B;IAC5E,OAAO,GAAG,EAAE,GAAG,0BAA0B,EAAE,GAAG,CAAC,OAAO,IAAI,EAAE,CAAC,EAAE,CAAA;IAE/D,MAAM,YAAY,GAAG,MAAM,YAAY,CAAC,IAAI,EAAE,OAAO,CAAC,CAAA;IAEtD,OAAO,mBAAmB,CAAC,YAAY,CAAC,CAAA;AACzC,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,mBAAmB,CAAC,YAA0B;IACnE,MAAM,kBAAkB,GAAY,EAAE,CAAA;IAEtC,IAAI,uBAAuB,GAAG,CAAC,CAAA;IAE/B,KAAK,IAAI,SAAS,GAAG,CAAC,EAAE,SAAS,GAAG,YAAY,CAAC,MAAM,EAAE,SAAS,EAAE,EAAE,CAAC;QACtE,MAAM,IAAI,GAAG,YAAY,CAAC,SAAS,CAAC,SAAS,CAAC,CAAA;QAE9C,IAAI,uBAAuB,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;YACxC,OAAO,SAAS,GAAG,YAAY,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBAC5C,MAAM,QAAQ,GAAG,YAAY,CAAC,SAAS,CAAC,SAAS,GAAG,CAAC,CAAC,CAAA;gBAEtD,IAAI,CAAC,0CAA0C,CAAC,IAAI,CAAC,QAAQ,CAAC,EAAE,CAAC;oBAChE,MAAK;gBACN,CAAC;gBAED,SAAS,IAAI,CAAC,CAAA;YACf,CAAC;YAED,kBAAkB,CAAC,IAAI,CAAC;gBACvB,KAAK,EAAE,uBAAuB;gBAC9B,GAAG,EAAE,SAAS,GAAG,CAAC;aAClB,CAAC,CAAA;YAEF,uBAAuB,GAAG,SAAS,GAAG,CAAC,CAAA;QACxC,CAAC;IACF,CAAC;IAED,IAAI,uBAAuB,GAAG,YAAY,CAAC,MAAM,EAAE,CAAC;QACnD,kBAAkB,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,uBAAuB,EAAE,GAAG,EAAE,YAAY,CAAC,MAAM,EAAE,CAAC,CAAA;IACtF,CAAC;IAED,MAAM,SAAS,GAAe,EAAE,CAAA;IAEhC,KAAK,MAAM,SAAS,IAAI,kBAAkB,EAAE,CAAC;QAC5C,MAAM,oBAAoB,GAAG,YAAY,CAAC,KAAK,CAAC,SAAS,CAAC,KAAK,EAAE,SAAS,CAAC,GAAG,CAAC,CAAA;QAE/E,SAAS,CAAC,IAAI,CAAC,IAAI,QAAQ,CAAC,SAAS,EAAE,oBAAoB,CAAC,CAAC,CAAA;IAC9D,CAAC;IAED,KAAK,MAAM,QAAQ,IAAI,SAAS,EAAE,CAAC;QAClC,MAAM,gBAAgB,GAAY,EAAE,CAAA;QAEpC,IAAI,qBAAqB,GAAG,QAAQ,CAAC,SAAS,CAAC,GAAG,CAAA;QAClD,IAAI,qBAAqB,GAAG,QAAQ,CAAC,SAAS,CAAC,KAAK,CAAA;QAEpD,KAAK,IAAI,SAAS,GAAG,qBAAqB,EAAE,SAAS,GAAG,qBAAqB,EAAE,SAAS,EAAE,EAAE,CAAC;YAC5F,MAAM,WAAW,GAAG,YAAY,CAAC,SAAS,CAAC,SAAS,CAAC,CAAA;YAErD,IAAI,qBAAqB,CAAC,IAAI,CAAC,WAAW,CAAC,EAAE,CAAC;gBAC7C,IAAI,kBAAkB,GAAG,KAAK,CAAA;gBAE9B,OAAO,SAAS,GAAG,qBAAqB,GAAG,CAAC,EAAE,CAAC;oBAC9C,MAAM,QAAQ,GAAG,YAAY,CAAC,SAAS,CAAC,SAAS,GAAG,CAAC,CAAC,CAAA;oBAEtD,IAAI,CAAC,0CAA0C,CAAC,IAAI,CAAC,QAAQ,CAAC,EAAE,CAAC;wBAChE,MAAK;oBACN,CAAC;oBAED,IAAI,uBAAuB,CAAC,IAAI,CAAC,QAAQ,CAAC,EAAE,CAAC;wBAC5C,kBAAkB,GAAG,IAAI,CAAA;oBAC1B,CAAC;oBAED,IAAI,QAAQ,KAAK,GAAG,IAAI,kBAAkB,EAAE,CAAC;wBAC5C,MAAK;oBACN,CAAC;oBAED,SAAS,IAAI,CAAC,CAAA;gBACf,CAAC;gBAED,gBAAgB,CAAC,IAAI,CAAC;oBACrB,KAAK,EAAE,qBAAqB;oBAC5B,GAAG,EAAE,SAAS,GAAG,CAAC;iBAClB,CAAC,CAAA;gBAEF,qBAAqB,GAAG,SAAS,GAAG,CAAC,CAAA;YACtC,CAAC;QACF,CAAC;QAED,IAAI,qBAAqB,GAAG,qBAAqB,EAAE,CAAC;YACnD,gBAAgB,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,qBAAqB,EAAE,GAAG,EAAE,qBAAqB,EAAE,CAAC,CAAA;QACpF,CAAC;QAED,KAAK,MAAM,SAAS,IAAI,gBAAgB,EAAE,CAAC;YAC1C,MAAM,kBAAkB,GAAG,YAAY,CAAC,KAAK,CAAC,SAAS,CAAC,KAAK,EAAE,SAAS,CAAC,GAAG,CAAC,CAAA;YAE7E,QAAQ,CAAC,OAAO,CAAC,IAAI,CAAC,IAAI,MAAM,CAAC,SAAS,EAAE,kBAAkB,CAAC,CAAC,CAAA;QACjE,CAAC;IACF,CAAC;IAED,MAAM,MAAM,GAAuB;QAClC,YAAY;QACZ,SAAS;KACT,CAAA;IAED,OAAO,MAAM,CAAA;AACd,CAAC;AAED,MAAM,yBAAyB,GAAG,IAAI,GAAG,EAAkB,CAAA;AAE3D,MAAM,CAAC,KAAK,UAAU,YAAY,CAAC,IAAY,EAAE,OAA6B;IAC7E,IAAI,CAAC,OAAO,EAAE,CAAC;QACd,OAAO,GAAG,EAAE,CAAA;IACb,CAAC;IAED,OAAO,GAAG,EAAE,GAAG,0BAA0B,EAAE,GAAG,OAAO,EAAE,CAAA;IAEvD,MAAM,aAAa,GAAG,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,CAAA;IAE7C,IAAI,kBAAkB,GAAG,yBAAyB,CAAC,GAAG,CAAC,aAAa,CAAC,CAAA;IAErE,IAAI,CAAC,kBAAkB,EAAE,CAAC;QACzB,kBAAkB,GAAG,iCAAiC,CAAC,OAAO,CAAC,CAAA;QAE/D,yBAAyB,CAAC,GAAG,CAAC,aAAa,EAAE,kBAAkB,CAAC,CAAA;IACjE,CAAC;IAED,IAAI,YAAY,GAAG,IAAI,YAAY,EAAE,CAAA;IAErC,SAAS,0BAA0B,CAAC,WAAmB,EAAE,SAAiB;QACzE,MAAM,wBAAwB,GAAG,IAAI,CAAC,SAAS,CAAC,WAAW,EAAE,SAAS,CAAC,CAAA;QAEvE,IAAI,UAAU,GAAG,WAAW,CAAA;QAC5B,IAAI,0BAA0B,GAAG,WAAW,CAAA;QAE5C,SAAS,0BAA0B;YAClC,IAAI,UAAU,GAAG,0BAA0B,EAAE,CAAC;gBAC7C,YAAY,CAAC,YAAY,CAAC,IAAI,EAAE,0BAA0B,EAAE,UAAU,EAAE,IAAI,CAAC,CAAA;gBAC7E,0BAA0B,GAAG,UAAU,CAAA;YACxC,CAAC;QACF,CAAC;QAED,KAAK,MAAM,IAAI,IAAI,wBAAwB,EAAE,CAAC;YAC7C,IAAI,IAAI,KAAK,GAAG,EAAE,CAAC;gBAClB,UAAU,IAAI,CAAC,CAAA;gBAEf,SAAQ;YACT,CAAC;YAED,0BAA0B,EAAE,CAAA;YAE5B,UAAU,IAAI,IAAI,CAAC,MAAM,CAAA;YAEzB,0BAA0B,EAAE,CAAA;QAC7B,CAAC;QAED,0BAA0B,EAAE,CAAA;IAC7B,CAAC;IAED,MAAM,WAAW,GAAG,IAAI,CAAC,QAAQ,CAAC,kBAAkB,CAAC,CAAA;IAErD,IAAI,kBAAkB,GAAG,CAAC,CAAA;IAE1B,IAAI,WAAW,EAAE,CAAC;QACjB,KAAK,MAAM,KAAK,IAAI,WAAW,EAAE,CAAC;YACjC,MAAM,OAAO,GAAG,KAAK,CAAC,OAAQ,CAAC,CAAC,CAAC,CAAA;YACjC,MAAM,gBAAgB,GAAG,OAAO,CAAC,CAAC,CAAC,CAAA;YACnC,MAAM,cAAc,GAAG,OAAO,CAAC,CAAC,CAAC,CAAA;YAEjC,IAAI,gBAAgB,GAAG,kBAAkB,EAAE,CAAC;gBAC3C,0BAA0B,CAAC,kBAAkB,EAAE,gBAAgB,CAAC,CAAA;YACjE,CAAC;YAED,YAAY,CAAC,YAAY,CAAC,IAAI,EAAE,gBAAgB,EAAE,cAAc,EAAE,KAAK,CAAC,CAAA;YAExE,kBAAkB,GAAG,cAAc,CAAA;QACpC,CAAC;QAED,0BAA0B,CAAC,kBAAkB,EAAE,IAAI,CAAC,MAAM,CAAC,CAAA;IAC5D,CAAC;IAED,IAAI,OAAO,CAAC,6BAA6B,EAAE,CAAC;QAC3C,YAAY,GAAG,MAAM,yBAAyB,CAAC,IAAI,EAAE,YAAY,CAAC,CAAA;IACnE,CAAC;IAED,OAAO,YAAY,CAAA;AACpB,CAAC;AAED,KAAK,UAAU,yBAAyB,CAAC,cAAsB,EAAE,YAA0B;IAC1F,MAAM,eAAe,GAAG,MAAM,kBAAkB,EAAE,CAAA;IAElD,IAAI,eAAe,KAAK,SAAS,EAAE,CAAC;QACnC,OAAO,YAAY,CAAA;IACpB,CAAC;IAED,IAAI,cAAc,GAAG,KAAK,CAAA;IAE1B,MAAM,eAAe,GAAG,IAAI,YAAY,EAAE,CAAA;IAE1C,KAAK,IAAI,SAAS,GAAG,CAAC,EAAE,SAAS,GAAG,YAAY,CAAC,MAAM,EAAE,SAAS,EAAE,EAAE,CAAC;QACtE,MAAM,SAAS,GAAG,YAAY,CAAC,OAAO,CAAC,SAAS,CAAC,CAAA;QACjD,MAAM,eAAe,GAAG,SAAS,CAAC,WAAW,CAAA;QAE7C,MAAM,IAAI,GAAG,YAAY,CAAC,SAAS,CAAC,SAAS,CAAC,CAAA;QAE9C,IAAI,yBAAyB,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;YAC1C,IAAI,CAAC,cAAc,EAAE,CAAC;gBACrB,MAAM,eAAe,CAAC,UAAU,EAAE,CAAA;gBAElC,cAAc,GAAG,IAAI,CAAA;YACtB,CAAC;YAED,MAAM,UAAU,GAAG,CAAC,GAAG,eAAe,CAAC,uBAAuB,CAAC,IAAI,CAAC,CAAC,CAAA;YAErE,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,UAAU,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;gBAChD,eAAe,CAAC,YAAY,CAC3B,cAAc,EACd,eAAe,GAAG,UAAU,CAAC,CAAC,CAAC,EAC/B,eAAe,GAAG,UAAU,CAAC,CAAC,GAAG,CAAC,CAAC,EACnC,KAAK,CACL,CAAA;YACF,CAAC;QACF,CAAC;aAAM,CAAC;YACP,eAAe,CAAC,YAAY,CAAC,cAAc,EAAE,SAAS,CAAC,WAAW,EAAE,SAAS,CAAC,SAAS,EAAE,SAAS,CAAC,aAAa,CAAC,CAAA;QAClH,CAAC;IACF,CAAC;IAED,OAAO,eAAe,CAAA;AACvB,CAAC;AAED,SAAS,iCAAiC,CAAC,OAA4B;IACtE,MAAM,uBAAuB,GAAG,gBAAgB,CAAC,OAAO,CAAC,QAAQ,IAAI,EAAE,CAAC,IAAI,EAAE,CAAA;IAC9E,MAAM,8BAA8B,GAAG,wCAAwC,CAAC,OAAO,CAAC,QAAQ,IAAI,EAAE,CAAC,IAAI,EAAE,CAAA;IAC7G,MAAM,6CAA6C,GAAG,8BAA8B,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,UAAU,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC,CAAA;IACzH,MAAM,kBAAkB,GAAG,OAAO,CAAC,kBAAkB,IAAI,EAAE,CAAA;IAE3D,IAAI,YAAY,GAAG;QAClB,GAAG,kBAAkB;QACrB,GAAG,uBAAuB;QAC1B,GAAG,8BAA8B;QACjC,GAAG,6CAA6C;QAChD,GAAG,gBAAgB;QACnB,GAAG,eAAe;KAClB,CAAA;IAED,MAAM,WAAW,GAAG,wBAAwB,CAAC;QAC5C,GAAG,YAAY;QACf,GAAG,YAAY,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,iBAAiB,EAAE,CAAC;QACrD,GAAG,YAAY,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,iBAAiB,EAAE,CAAC;KACrD,CAAC,CAAA;IAEF,MAAM,kBAAkB,GAAG,WAAW,CAAC,WAAW,EAAE,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC,CAAA;IAErE,OAAO,kBAAkB,CAAA;AAC1B,CAAC;AAED,uDAAuD;AACvD,MAAM,UAAU,wCAAwC,CAAC,YAA0B,EAAE,UAAkB;IACtG,MAAM,2BAA2B,GAAG,IAAI,GAAG,EAAkB,CAAA;IAE7D,MAAM,0BAA0B,GAAG,IAAI,YAAY,EAAE,CAAA;IAErD,SAAS,0BAA0B,CAAC,SAAiB,EAAE,iBAAyB;QAC/E,IAAI,UAAU,GAAG,iBAAiB,CAAA;QAElC,kFAAkF;QAClF,KAAK,MAAM,IAAI,IAAI,SAAS,EAAE,CAAC;YAC9B,MAAM,aAAa,GAAG,UAAU,GAAG,IAAI,CAAC,MAAM,CAAA;YAE9C,MAAM,SAAS,GAAG,0BAA0B,CAAC,SAAS,CAAA;YAEtD,IAAI,IAAI,KAAK,GAAG,IAAI,SAAS,IAAI,SAAS,CAAC,aAAa,IAAI,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC,KAAK,GAAG,EAAE,CAAC;gBACvF,0BAA0B,CAAC,SAAS,CAAC,IAAI,IAAI,GAAG,CAAA;gBAChD,0BAA0B,CAAC,SAAS,CAAC,SAAS,GAAG,aAAa,CAAA;YAC/D,CAAC;iBAAM,CAAC;gBACP,0BAA0B,CAAC,YAAY,CAAC,UAAU,EAAE,UAAU,EAAE,aAAa,EAAE,IAAI,CAAC,CAAA;YACrF,CAAC;YAED,UAAU,GAAG,aAAa,CAAA;QAC3B,CAAC;IACF,CAAC;IAED,KAAK,IAAI,SAAS,GAAG,CAAC,EAAE,SAAS,GAAG,YAAY,CAAC,MAAM,EAAE,SAAS,EAAE,EAAE,CAAC;QACtE,MAAM,SAAS,GAAG,YAAY,CAAC,UAAU,CAAC,SAAS,CAAC,CAAA;QACpD,MAAM,eAAe,GAAG,SAAS,CAAC,WAAW,CAAA;QAE7C,MAAM,qBAAqB,GAAG,SAAS,GAAG,CAAC,CAAC,CAAC,CAAC,YAAY,CAAC,OAAO,CAAC,SAAS,GAAG,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,CAAA;QAE/F,8FAA8F;QAC9F,IAAI,qBAAqB,KAAK,eAAe,EAAE,CAAC;YAC/C,MAAM,SAAS,GAAG,UAAU,CAAC,SAAS,CAAC,qBAAqB,EAAE,eAAe,CAAC,CAAA;YAE9E,0BAA0B,CAAC,SAAS,EAAE,qBAAqB,CAAC,CAAA;QAC7D,CAAC;QAED,0BAA0B,CAAC,OAAO,CAAC,IAAI,CAAC,SAAS,CAAC,CAAA;QAClD,2BAA2B,CAAC,GAAG,CAAC,0BAA0B,CAAC,MAAM,GAAG,CAAC,EAAE,SAAS,CAAC,CAAA;QAEjF,oEAAoE;QACpE,IAAI,SAAS,KAAK,YAAY,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAC3C,IAAI,UAAU,CAAC,MAAM,KAAK,SAAS,CAAC,SAAS,EAAE,CAAC;gBAC/C,MAAM,SAAS,GAAG,UAAU,CAAC,SAAS,CAAC,SAAS,CAAC,SAAS,EAAE,UAAU,CAAC,MAAM,CAAC,CAAA;gBAE9E,0BAA0B,CAAC,SAAS,EAAE,SAAS,CAAC,SAAS,CAAC,CAAA;YAC3D,CAAC;QACF,CAAC;IACF,CAAC;IAED,OAAO,EAAE,0BAA0B,EAAE,2BAA2B,EAAE,CAAA;AACnE,CAAC;AAED,KAAK,UAAU,kBAAkB;IAChC,IAAI,CAAC;QACJ,MAAM,eAAe,GAAG,MAAM,MAAM,CAAC,mCAAmC,CAAC,CAAA;QAEzE,OAAO,eAAe,CAAA;IACvB,CAAC;IAAC,MAAM,CAAC;QACR,OAAO,SAAS,CAAA;IACjB,CAAC;AACF,CAAC;AAOD,MAAM,OAAO,YAAY;IACxB,SAAS,CAAO;IAChB,YAAY,CAAc;IAE1B,YAAY,SAAgB,EAAE,YAA0B;QACvD,IAAI,CAAC,SAAS,GAAG,SAAS,CAAA;QAC1B,IAAI,CAAC,YAAY,GAAG,YAAY,CAAA;IACjC,CAAC;IAED,IAAI,IAAI;QACP,OAAO,IAAI,CAAC,YAAY,CAAC,IAAI,CAAA;IAC9B,CAAC;IAED,IAAI,SAAS;QACZ,OAAO;YACN,KAAK,EAAE,IAAI,CAAC,YAAY,CAAC,UAAU,CAAC,WAAW;YAC/C,GAAG,EAAE,IAAI,CAAC,YAAY,CAAC,SAAS,CAAC,SAAS;SAC1C,CAAA;IACF,CAAC;CACD;AAED,MAAM,OAAO,QAAS,SAAQ,YAAY;IACzC,OAAO,GAAa,EAAE,CAAA;CACtB;AAED,MAAM,OAAO,MAAO,SAAQ,YAAY;CACvC;AAaD,MAAM,CAAC,MAAM,0BAA0B,GAAwB;IAC9D,QAAQ,EAAE,EAAE;IACZ,kBAAkB,EAAE,EAAE;IACtB,6BAA6B,EAAE,IAAI;CACnC,CAAA"}
|
package/dist/WordSequence.d.ts
CHANGED
|
@@ -7,6 +7,7 @@ export declare class WordSequence {
|
|
|
7
7
|
getEntryRange(startIndex: number, endIndex: number): WordEntry[];
|
|
8
8
|
iterateEntryRange(startIndex: number, endIndex: number): Generator<WordEntry, void, unknown>;
|
|
9
9
|
getEntryAt(index: number): WordEntry;
|
|
10
|
+
clone(): WordSequence;
|
|
10
11
|
slice(startIndex: number, endIndex: number): WordSequence;
|
|
11
12
|
get words(): string[];
|
|
12
13
|
get firstWord(): string;
|
package/dist/WordSequence.js
CHANGED
|
@@ -31,6 +31,9 @@ export class WordSequence {
|
|
|
31
31
|
getEntryAt(index) {
|
|
32
32
|
return this.entries[index];
|
|
33
33
|
}
|
|
34
|
+
clone() {
|
|
35
|
+
return this.slice(0, this.length);
|
|
36
|
+
}
|
|
34
37
|
slice(startIndex, endIndex) {
|
|
35
38
|
const slicedSequence = new WordSequence();
|
|
36
39
|
slicedSequence.entries = this.entries.slice(startIndex, endIndex);
|
package/dist/WordSequence.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"WordSequence.js","sourceRoot":"","sources":["../src/WordSequence.ts"],"names":[],"mappings":"AAAA,MAAM,OAAO,YAAY;IACxB,OAAO,GAAgB,EAAE,CAAA;IAEzB,YAAY,CAAC,cAAsB,EAAE,WAAmB,EAAE,SAAiB,EAAE,aAAsB;QAClG,MAAM,QAAQ,GAAG,cAAc,CAAC,SAAS,CAAC,WAAW,EAAE,SAAS,CAAC,CAAA;QAEjE,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC;YACjB,IAAI,EAAE,QAAQ;YACd,WAAW;YACX,SAAS;YACT,aAAa;SACb,CAAC,CAAA;IACH,CAAC;IAED,YAAY,CAAC,UAAkB,EAAE,QAAgB;QAChD,OAAO,CAAC,GAAG,IAAI,CAAC,gBAAgB,CAAC,UAAU,EAAE,QAAQ,CAAC,CAAC,CAAA;IACxD,CAAC;IAED,CAAC,gBAAgB,CAAC,UAAkB,EAAE,QAAgB;QACrD,KAAK,IAAI,CAAC,GAAG,UAAU,EAAE,CAAC,GAAG,QAAQ,EAAE,CAAC,EAAE,EAAE,CAAC;YAC5C,MAAM,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,CAAA;QACxB,CAAC;IACF,CAAC;IAED,SAAS,CAAC,KAAa;QACtB,OAAO,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,IAAI,CAAA;IAChC,CAAC;IAED,aAAa,CAAC,UAAkB,EAAE,QAAgB;QACjD,OAAO,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,UAAU,EAAE,QAAQ,CAAC,CAAA;IAChD,CAAC;IAED,CAAC,iBAAiB,CAAC,UAAkB,EAAE,QAAgB;QACtD,KAAK,IAAI,CAAC,GAAG,UAAU,EAAE,CAAC,GAAG,QAAQ,EAAE,CAAC,EAAE,EAAE,CAAC;YAC5C,MAAM,IAAI,CAAC,UAAU,CAAC,CAAC,CAAC,CAAA;QACzB,CAAC;IACF,CAAC;IAED,UAAU,CAAC,KAAa;QACvB,OAAO,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,CAAA;IAC3B,CAAC;IAED,KAAK,CAAC,UAAkB,EAAE,QAAgB;QACzC,MAAM,cAAc,GAAG,IAAI,YAAY,EAAE,CAAA;QAEzC,cAAc,CAAC,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,UAAU,EAAE,QAAQ,CAAC,CAAA;QAEjE,OAAO,cAAc,CAAA;IACtB,CAAC;IAED,IAAI,KAAK;QACR,OAAO,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC,KAAK,CAAC,IAAI,CAAC,CAAA;IAC7C,CAAC;IAED,IAAI,SAAS;QACZ,OAAO,IAAI,CAAC,UAAU,EAAE,IAAI,CAAA;IAC7B,CAAC;IAED,IAAI,QAAQ;QACX,OAAO,IAAI,CAAC,SAAS,EAAE,IAAI,CAAA;IAC5B,CAAC;IAED,IAAI,UAAU;QACb,OAAO,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,CAAA;IACvB,CAAC;IAED,IAAI,SAAS;QACZ,OAAO,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC,CAAA;IACrC,CAAC;IAED,IAAI,qBAAqB;QACxB,OAAO,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,KAAK,CAAC,aAAa,KAAK,KAAK,CAAC,CAAA;IACnE,CAAC;IAED,IAAI,mBAAmB;QACtB,OAAO,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,KAAK,CAAC,aAAa,KAAK,KAAK,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC,KAAK,CAAC,IAAI,CAAC,CAAA;IAC5F,CAAC;IAED,IAAI,IAAI;QACP,OAAO,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAA;IAC3B,CAAC;IAED,IAAI,MAAM;QACT,OAAO,IAAI,CAAC,OAAO,CAAC,MAAM,CAAA;IAC3B,CAAC;CACD"}
|
|
1
|
+
{"version":3,"file":"WordSequence.js","sourceRoot":"","sources":["../src/WordSequence.ts"],"names":[],"mappings":"AAAA,MAAM,OAAO,YAAY;IACxB,OAAO,GAAgB,EAAE,CAAA;IAEzB,YAAY,CAAC,cAAsB,EAAE,WAAmB,EAAE,SAAiB,EAAE,aAAsB;QAClG,MAAM,QAAQ,GAAG,cAAc,CAAC,SAAS,CAAC,WAAW,EAAE,SAAS,CAAC,CAAA;QAEjE,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC;YACjB,IAAI,EAAE,QAAQ;YACd,WAAW;YACX,SAAS;YACT,aAAa;SACb,CAAC,CAAA;IACH,CAAC;IAED,YAAY,CAAC,UAAkB,EAAE,QAAgB;QAChD,OAAO,CAAC,GAAG,IAAI,CAAC,gBAAgB,CAAC,UAAU,EAAE,QAAQ,CAAC,CAAC,CAAA;IACxD,CAAC;IAED,CAAC,gBAAgB,CAAC,UAAkB,EAAE,QAAgB;QACrD,KAAK,IAAI,CAAC,GAAG,UAAU,EAAE,CAAC,GAAG,QAAQ,EAAE,CAAC,EAAE,EAAE,CAAC;YAC5C,MAAM,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,CAAA;QACxB,CAAC;IACF,CAAC;IAED,SAAS,CAAC,KAAa;QACtB,OAAO,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,IAAI,CAAA;IAChC,CAAC;IAED,aAAa,CAAC,UAAkB,EAAE,QAAgB;QACjD,OAAO,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,UAAU,EAAE,QAAQ,CAAC,CAAA;IAChD,CAAC;IAED,CAAC,iBAAiB,CAAC,UAAkB,EAAE,QAAgB;QACtD,KAAK,IAAI,CAAC,GAAG,UAAU,EAAE,CAAC,GAAG,QAAQ,EAAE,CAAC,EAAE,EAAE,CAAC;YAC5C,MAAM,IAAI,CAAC,UAAU,CAAC,CAAC,CAAC,CAAA;QACzB,CAAC;IACF,CAAC;IAED,UAAU,CAAC,KAAa;QACvB,OAAO,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,CAAA;IAC3B,CAAC;IAED,KAAK;QACJ,OAAO,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,IAAI,CAAC,MAAM,CAAC,CAAA;IAClC,CAAC;IAED,KAAK,CAAC,UAAkB,EAAE,QAAgB;QACzC,MAAM,cAAc,GAAG,IAAI,YAAY,EAAE,CAAA;QAEzC,cAAc,CAAC,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,UAAU,EAAE,QAAQ,CAAC,CAAA;QAEjE,OAAO,cAAc,CAAA;IACtB,CAAC;IAED,IAAI,KAAK;QACR,OAAO,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC,KAAK,CAAC,IAAI,CAAC,CAAA;IAC7C,CAAC;IAED,IAAI,SAAS;QACZ,OAAO,IAAI,CAAC,UAAU,EAAE,IAAI,CAAA;IAC7B,CAAC;IAED,IAAI,QAAQ;QACX,OAAO,IAAI,CAAC,SAAS,EAAE,IAAI,CAAA;IAC5B,CAAC;IAED,IAAI,UAAU;QACb,OAAO,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,CAAA;IACvB,CAAC;IAED,IAAI,SAAS;QACZ,OAAO,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC,CAAA;IACrC,CAAC;IAED,IAAI,qBAAqB;QACxB,OAAO,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,KAAK,CAAC,aAAa,KAAK,KAAK,CAAC,CAAA;IACnE,CAAC;IAED,IAAI,mBAAmB;QACtB,OAAO,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,KAAK,CAAC,aAAa,KAAK,KAAK,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC,KAAK,CAAC,IAAI,CAAC,CAAA;IAC5F,CAAC;IAED,IAAI,IAAI;QACP,OAAO,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAA;IAC3B,CAAC;IAED,IAAI,MAAM;QACT,OAAO,IAAI,CAAC,OAAO,CAAC,MAAM,CAAA;IAC3B,CAAC;CACD"}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@echogarden/text-segmentation",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.2.0",
|
|
4
4
|
"description": "A library for multilingual word, phrase and sentence segmentation.",
|
|
5
5
|
"author": "Rotem Dan",
|
|
6
6
|
"license": "MIT",
|
|
@@ -39,7 +39,7 @@
|
|
|
39
39
|
"regexp-composer": "../../regexp-composer"
|
|
40
40
|
},
|
|
41
41
|
"dependencies": {
|
|
42
|
-
"regexp-composer": "^0.
|
|
42
|
+
"regexp-composer": "^0.2.2"
|
|
43
43
|
},
|
|
44
44
|
"peerDependencies": {
|
|
45
45
|
"@echogarden/icu-segmentation-wasm": "^0.1.0"
|
|
@@ -50,6 +50,6 @@
|
|
|
50
50
|
}
|
|
51
51
|
},
|
|
52
52
|
"devDependencies": {
|
|
53
|
-
"@types/node": "^22.
|
|
53
|
+
"@types/node": "^22.15.16"
|
|
54
54
|
}
|
|
55
55
|
}
|
package/src/Test.ts
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { segmentText, splitToWords } from "./TextSegmentation.js"
|
|
1
|
+
import { addMissingPunctuationWordsToWordSequence, SegmentationResult, segmentText, splitToWords, WordSequence } from "./TextSegmentation.js"
|
|
2
2
|
import { Timer } from "./utilities/Timer.js"
|
|
3
3
|
|
|
4
4
|
const log = console.log
|
|
@@ -14,15 +14,17 @@ async function test1() {
|
|
|
14
14
|
|
|
15
15
|
const timer = new Timer()
|
|
16
16
|
|
|
17
|
-
|
|
18
|
-
language: 'en',
|
|
19
|
-
customSuppressions: [],
|
|
20
|
-
enableEastAsianPostprocessing: true
|
|
21
|
-
})
|
|
17
|
+
let result: SegmentationResult
|
|
22
18
|
|
|
23
|
-
|
|
19
|
+
for (let i = 0; i < 3; i++) {
|
|
20
|
+
result = await segmentText(text, {
|
|
21
|
+
language: 'en',
|
|
22
|
+
customSuppressions: [],
|
|
23
|
+
enableEastAsianPostprocessing: true
|
|
24
|
+
})
|
|
24
25
|
|
|
25
|
-
|
|
26
|
+
timer.logAndRestart(`Total execution time`)
|
|
27
|
+
}
|
|
26
28
|
|
|
27
29
|
log('')
|
|
28
30
|
|
|
@@ -30,8 +32,8 @@ async function test1() {
|
|
|
30
32
|
|
|
31
33
|
let segmentedText = ''
|
|
32
34
|
|
|
33
|
-
for (let sentenceIndex = 0; sentenceIndex < result
|
|
34
|
-
const sentence = result
|
|
35
|
+
for (let sentenceIndex = 0; sentenceIndex < result!.sentences.length; sentenceIndex++) {
|
|
36
|
+
const sentence = result!.sentences[sentenceIndex]
|
|
35
37
|
const phrases = sentence.phrases
|
|
36
38
|
|
|
37
39
|
for (let phraseIndex = 0; phraseIndex < phrases.length; phraseIndex++) {
|
|
@@ -44,7 +46,7 @@ async function test1() {
|
|
|
44
46
|
}
|
|
45
47
|
}
|
|
46
48
|
|
|
47
|
-
if (sentenceIndex < result
|
|
49
|
+
if (sentenceIndex < result!.sentences.length - 1) {
|
|
48
50
|
segmentedText += `\n${ '='.repeat(100) } \n`
|
|
49
51
|
}
|
|
50
52
|
}
|
|
@@ -52,4 +54,19 @@ async function test1() {
|
|
|
52
54
|
writeFileSync('out/segmented.txt', segmentedText)
|
|
53
55
|
}
|
|
54
56
|
|
|
57
|
+
async function test2() {
|
|
58
|
+
const text = ` Hello, how are you today ?? `
|
|
59
|
+
|
|
60
|
+
const words = await splitToWords(text)
|
|
61
|
+
|
|
62
|
+
const nonPunctuationWordEntries = words.nonPunctuationEntries
|
|
63
|
+
const nonPunctuationWordSequence = new WordSequence()
|
|
64
|
+
nonPunctuationWordSequence.entries = nonPunctuationWordEntries
|
|
65
|
+
|
|
66
|
+
const extendedWords = addMissingPunctuationWordsToWordSequence(nonPunctuationWordSequence, text)
|
|
67
|
+
|
|
68
|
+
const x = 0
|
|
69
|
+
}
|
|
70
|
+
|
|
55
71
|
test1()
|
|
72
|
+
//test2()
|
package/src/TextSegmentation.ts
CHANGED
|
@@ -5,6 +5,7 @@ import { eastAsianCharRangesRegExp } from './EastAsianCharacterPatterns.js'
|
|
|
5
5
|
import { WordSequence } from './WordSequence.js'
|
|
6
6
|
|
|
7
7
|
export { cldrSuppressions } from './Suppressions.js'
|
|
8
|
+
export { WordSequence, type WordEntry } from './WordSequence.js'
|
|
8
9
|
|
|
9
10
|
export async function segmentText(text: string, options?: SegmentationOptions) {
|
|
10
11
|
options = { ...defaultSegmentationOptions, ...(options ?? {}) }
|
|
@@ -112,63 +113,55 @@ export async function segmentWordSequence(wordSequence: WordSequence) {
|
|
|
112
113
|
return result
|
|
113
114
|
}
|
|
114
115
|
|
|
116
|
+
const cachedWordSplitterRegExps = new Map<string, RegExp>()
|
|
117
|
+
|
|
115
118
|
export async function splitToWords(text: string, options?: SegmentationOptions) {
|
|
116
|
-
|
|
119
|
+
if (!options) {
|
|
120
|
+
options = {}
|
|
121
|
+
}
|
|
117
122
|
|
|
118
|
-
|
|
119
|
-
const contractionSuppressionsForLang = leadingApostropheContractionSuppressions[options.language ?? ''] ?? []
|
|
120
|
-
const contractionSuppressionsForLangWithSingleQuote = contractionSuppressionsForLang.map(str => str.replaceAll(`'`, `’`))
|
|
121
|
-
const customSuppressions = options.customSuppressions ?? []
|
|
123
|
+
options = { ...defaultSegmentationOptions, ...options }
|
|
122
124
|
|
|
123
|
-
|
|
124
|
-
...customSuppressions,
|
|
125
|
-
...cldrSuppressionsForLang,
|
|
126
|
-
...contractionSuppressionsForLang,
|
|
127
|
-
...contractionSuppressionsForLangWithSingleQuote,
|
|
128
|
-
...nounSuppressions,
|
|
129
|
-
...tldSuppressions,
|
|
130
|
-
]
|
|
125
|
+
const optionsAsJson = JSON.stringify(options)
|
|
131
126
|
|
|
132
|
-
|
|
133
|
-
...suppressions,
|
|
134
|
-
...suppressions.map(word => word.toLocaleLowerCase()),
|
|
135
|
-
...suppressions.map(word => word.toLocaleUpperCase()),
|
|
136
|
-
])
|
|
127
|
+
let wordSplitterRegExp = cachedWordSplitterRegExps.get(optionsAsJson)
|
|
137
128
|
|
|
138
|
-
|
|
129
|
+
if (!wordSplitterRegExp) {
|
|
130
|
+
wordSplitterRegExp = buildWordSplitterRegExpForOptions(options)
|
|
139
131
|
|
|
140
|
-
|
|
132
|
+
cachedWordSplitterRegExps.set(optionsAsJson, wordSplitterRegExp)
|
|
133
|
+
}
|
|
141
134
|
|
|
142
135
|
let wordSequence = new WordSequence()
|
|
143
136
|
|
|
144
|
-
function
|
|
145
|
-
const
|
|
137
|
+
function addPunctuationWordsBetween(startOffset: number, endOffset: number) {
|
|
138
|
+
const punctuationWordSubstring = text.substring(startOffset, endOffset)
|
|
146
139
|
|
|
147
140
|
let charOffset = startOffset
|
|
148
|
-
let
|
|
141
|
+
let punctuationWordStartOffset = startOffset
|
|
149
142
|
|
|
150
|
-
function
|
|
151
|
-
if (charOffset >
|
|
152
|
-
wordSequence.addWordEntry(text,
|
|
153
|
-
|
|
143
|
+
function addPunctuationWordIfNeeded() {
|
|
144
|
+
if (charOffset > punctuationWordStartOffset) {
|
|
145
|
+
wordSequence.addWordEntry(text, punctuationWordStartOffset, charOffset, true)
|
|
146
|
+
punctuationWordStartOffset = charOffset
|
|
154
147
|
}
|
|
155
148
|
}
|
|
156
149
|
|
|
157
|
-
for (const char of
|
|
150
|
+
for (const char of punctuationWordSubstring) {
|
|
158
151
|
if (char === ' ') {
|
|
159
152
|
charOffset += 1
|
|
160
153
|
|
|
161
154
|
continue
|
|
162
155
|
}
|
|
163
156
|
|
|
164
|
-
|
|
157
|
+
addPunctuationWordIfNeeded()
|
|
165
158
|
|
|
166
159
|
charOffset += char.length
|
|
167
160
|
|
|
168
|
-
|
|
161
|
+
addPunctuationWordIfNeeded()
|
|
169
162
|
}
|
|
170
163
|
|
|
171
|
-
|
|
164
|
+
addPunctuationWordIfNeeded()
|
|
172
165
|
}
|
|
173
166
|
|
|
174
167
|
const wordMatches = text.matchAll(wordSplitterRegExp)
|
|
@@ -182,7 +175,7 @@ export async function splitToWords(text: string, options?: SegmentationOptions)
|
|
|
182
175
|
const matchEndOffset = offsets[1]
|
|
183
176
|
|
|
184
177
|
if (matchStartOffset > lastMatchEndOffset) {
|
|
185
|
-
|
|
178
|
+
addPunctuationWordsBetween(lastMatchEndOffset, matchStartOffset)
|
|
186
179
|
}
|
|
187
180
|
|
|
188
181
|
wordSequence.addWordEntry(text, matchStartOffset, matchEndOffset, false)
|
|
@@ -190,7 +183,7 @@ export async function splitToWords(text: string, options?: SegmentationOptions)
|
|
|
190
183
|
lastMatchEndOffset = matchEndOffset
|
|
191
184
|
}
|
|
192
185
|
|
|
193
|
-
|
|
186
|
+
addPunctuationWordsBetween(lastMatchEndOffset, text.length)
|
|
194
187
|
}
|
|
195
188
|
|
|
196
189
|
if (options.enableEastAsianPostprocessing) {
|
|
@@ -242,6 +235,87 @@ async function postprocessEastAsianWords(containingText: string, wordSequence: W
|
|
|
242
235
|
return newWordSequence
|
|
243
236
|
}
|
|
244
237
|
|
|
238
|
+
function buildWordSplitterRegExpForOptions(options: SegmentationOptions) {
|
|
239
|
+
const cldrSuppressionsForLang = cldrSuppressions[options.language ?? ''] ?? []
|
|
240
|
+
const contractionSuppressionsForLang = leadingApostropheContractionSuppressions[options.language ?? ''] ?? []
|
|
241
|
+
const contractionSuppressionsForLangWithSingleQuote = contractionSuppressionsForLang.map(str => str.replaceAll(`'`, `’`))
|
|
242
|
+
const customSuppressions = options.customSuppressions ?? []
|
|
243
|
+
|
|
244
|
+
let suppressions = [
|
|
245
|
+
...customSuppressions,
|
|
246
|
+
...cldrSuppressionsForLang,
|
|
247
|
+
...contractionSuppressionsForLang,
|
|
248
|
+
...contractionSuppressionsForLangWithSingleQuote,
|
|
249
|
+
...nounSuppressions,
|
|
250
|
+
...tldSuppressions,
|
|
251
|
+
]
|
|
252
|
+
|
|
253
|
+
const wordPattern = buildWordSplitterPattern([
|
|
254
|
+
...suppressions,
|
|
255
|
+
...suppressions.map(word => word.toLocaleLowerCase()),
|
|
256
|
+
...suppressions.map(word => word.toLocaleUpperCase()),
|
|
257
|
+
])
|
|
258
|
+
|
|
259
|
+
const wordSplitterRegExp = buildRegExp(wordPattern, { global: true })
|
|
260
|
+
|
|
261
|
+
return wordSplitterRegExp
|
|
262
|
+
}
|
|
263
|
+
|
|
264
|
+
// Add any missing punctuation words to a word sequence
|
|
265
|
+
export function addMissingPunctuationWordsToWordSequence(wordSequence: WordSequence, sourceText: string) {
|
|
266
|
+
const originalWordsReverseMapping = new Map<number, number>()
|
|
267
|
+
|
|
268
|
+
const wordSequnceWithPunctuation = new WordSequence()
|
|
269
|
+
|
|
270
|
+
function addWordEntriesForTextSlice(textSlice: string, initialCharOffset: number) {
|
|
271
|
+
let charOffset = initialCharOffset
|
|
272
|
+
|
|
273
|
+
// Add entry for every codepoint (this will correctly treat characters beyond BMP)
|
|
274
|
+
for (const char of textSlice) {
|
|
275
|
+
const charEndOffset = charOffset + char.length
|
|
276
|
+
|
|
277
|
+
const lastEntry = wordSequnceWithPunctuation.lastEntry
|
|
278
|
+
|
|
279
|
+
if (char === ' ' && lastEntry && lastEntry.isPunctuation && lastEntry.text[0] === ' ') {
|
|
280
|
+
wordSequnceWithPunctuation.lastEntry.text += ' '
|
|
281
|
+
wordSequnceWithPunctuation.lastEntry.endOffset = charEndOffset
|
|
282
|
+
} else {
|
|
283
|
+
wordSequnceWithPunctuation.addWordEntry(sourceText, charOffset, charEndOffset, true)
|
|
284
|
+
}
|
|
285
|
+
|
|
286
|
+
charOffset = charEndOffset
|
|
287
|
+
}
|
|
288
|
+
}
|
|
289
|
+
|
|
290
|
+
for (let wordIndex = 0; wordIndex < wordSequence.length; wordIndex++) {
|
|
291
|
+
const wordEntry = wordSequence.getEntryAt(wordIndex)
|
|
292
|
+
const wordStartOffset = wordEntry.startOffset
|
|
293
|
+
|
|
294
|
+
const previousWordEndOffset = wordIndex > 0 ? wordSequence.entries[wordIndex - 1].endOffset : 0
|
|
295
|
+
|
|
296
|
+
// Add entries for any punctuation characters between the current and previous word (or start)
|
|
297
|
+
if (previousWordEndOffset !== wordStartOffset) {
|
|
298
|
+
const textSlice = sourceText.substring(previousWordEndOffset, wordStartOffset)
|
|
299
|
+
|
|
300
|
+
addWordEntriesForTextSlice(textSlice, previousWordEndOffset)
|
|
301
|
+
}
|
|
302
|
+
|
|
303
|
+
wordSequnceWithPunctuation.entries.push(wordEntry)
|
|
304
|
+
originalWordsReverseMapping.set(wordSequnceWithPunctuation.length - 1, wordIndex)
|
|
305
|
+
|
|
306
|
+
// If last word, add entries for any trailing punctuation characters
|
|
307
|
+
if (wordIndex === wordSequence.length - 1) {
|
|
308
|
+
if (sourceText.length !== wordEntry.endOffset) {
|
|
309
|
+
const textSlice = sourceText.substring(wordEntry.endOffset, sourceText.length)
|
|
310
|
+
|
|
311
|
+
addWordEntriesForTextSlice(textSlice, wordEntry.endOffset)
|
|
312
|
+
}
|
|
313
|
+
}
|
|
314
|
+
}
|
|
315
|
+
|
|
316
|
+
return { wordSequnceWithPunctuation, originalWordsReverseMapping }
|
|
317
|
+
}
|
|
318
|
+
|
|
245
319
|
async function getIcuSegmentation() {
|
|
246
320
|
try {
|
|
247
321
|
const icuSegmentation = await import('@echogarden/icu-segmentation-wasm')
|
package/src/WordSequence.ts
CHANGED