@echogarden/text-segmentation 0.1.1 → 0.2.1

package/dist/Test.js

@@ -1,4 +1,4 @@
-import { segmentText, splitToWords } from "./TextSegmentation.js";
+import { addMissingPunctuationWordsToWordSequence, segmentText, splitToWords, WordSequence } from "./TextSegmentation.js";
 import { Timer } from "./utilities/Timer.js";
 const log = console.log;
 async function test1() {
@@ -9,13 +9,15 @@ async function test1() {
     const { readFileSync, writeFileSync } = await import('fs');
     const text = readFileSync('test-data/Test.txt', 'utf-8');
     const timer = new Timer();
-    const result = await segmentText(text, {
-        language: 'en',
-        customSuppressions: [],
-        enableEastAsianPostprocessing: true
-    });
-    //const json = JSON.stringify(result.sentences)
-    timer.logAndRestart(`Total execution time`);
+    let result;
+    for (let i = 0; i < 3; i++) {
+        result = await segmentText(text, {
+            language: 'en',
+            customSuppressions: [],
+            enableEastAsianPostprocessing: true
+        });
+        timer.logAndRestart(`Total execution time`);
+    }
     log('');
     //
     let segmentedText = '';
@@ -35,5 +37,15 @@ async function test1() {
     }
     writeFileSync('out/segmented.txt', segmentedText);
 }
+async function test2() {
+    const text = `  Hello,  how are you today                ??    `;
+    const words = await splitToWords(text);
+    const nonPunctuationWordEntries = words.nonPunctuationEntries;
+    const nonPunctuationWordSequence = new WordSequence();
+    nonPunctuationWordSequence.entries = nonPunctuationWordEntries;
+    const extendedWords = addMissingPunctuationWordsToWordSequence(nonPunctuationWordSequence, text);
+    const x = 0;
+}
 test1();
+//test2()
 //# sourceMappingURL=Test.js.map

package/dist/Test.js.map

@@ -1 +1 @@
-{"version":3,"file":"Test.js","sourceRoot":"","sources":["../src/Test.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,WAAW,EAAE,YAAY,EAAE,MAAM,uBAAuB,CAAA;AACjE,OAAO,EAAE,KAAK,EAAE,MAAM,sBAAsB,CAAA;AAE5C,MAAM,GAAG,GAAG,OAAO,CAAC,GAAG,CAAA;AAEvB,KAAK,UAAU,KAAK;IACnB,CAAC;QACA,MAAM,YAAY,GAAG,MAAM,YAAY,CAAC,gCAAgC,CAAC,CAAA;QACzE,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,SAAS,CAAC,YAAY,CAAC,KAAK,CAAC,CAAC,CAAA;IAChD,CAAC;IAED,MAAM,EAAE,YAAY,EAAE,aAAa,EAAE,GAAG,MAAM,MAAM,CAAC,IAAI,CAAC,CAAA;IAC1D,MAAM,IAAI,GAAG,YAAY,CAAC,oBAAoB,EAAE,OAAO,CAAC,CAAA;IAExD,MAAM,KAAK,GAAG,IAAI,KAAK,EAAE,CAAA;IAEzB,MAAM,MAAM,GAAG,MAAM,WAAW,CAAC,IAAI,EAAE;QACtC,QAAQ,EAAE,IAAI;QACd,kBAAkB,EAAE,EAAE;QACtB,6BAA6B,EAAE,IAAI;KACnC,CAAC,CAAA;IAEF,+CAA+C;IAE/C,KAAK,CAAC,aAAa,CAAC,sBAAsB,CAAC,CAAA;IAE3C,GAAG,CAAC,EAAE,CAAC,CAAA;IAEP,EAAE;IAEF,IAAI,aAAa,GAAG,EAAE,CAAA;IAEtB,KAAK,IAAI,aAAa,GAAG,CAAC,EAAE,aAAa,GAAG,MAAM,CAAC,SAAS,CAAC,MAAM,EAAE,aAAa,EAAE,EAAE,CAAC;QACtF,MAAM,QAAQ,GAAG,MAAM,CAAC,SAAS,CAAC,aAAa,CAAC,CAAA;QAChD,MAAM,OAAO,GAAG,QAAQ,CAAC,OAAO,CAAA;QAEhC,KAAK,IAAI,WAAW,GAAG,CAAC,EAAE,WAAW,GAAG,OAAO,CAAC,MAAM,EAAE,WAAW,EAAE,EAAE,CAAC;YACvE,MAAM,MAAM,GAAG,OAAO,CAAC,WAAW,CAAC,CAAA;YAEnC,aAAa,IAAI,MAAM,CAAC,YAAY,CAAC,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,CAAA;YAEtD,IAAI,WAAW,GAAG,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBACtC,aAAa,IAAI,KAAK,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,IAAI,CAAA;YAC1C,CAAC;QACF,CAAC;QAED,IAAI,aAAa,GAAG,MAAM,CAAC,SAAS,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACjD,aAAa,IAAI,KAAM,GAAG,CAAC,MAAM,CAAC,GAAG,CAAE,KAAK,CAAA;QAC7C,CAAC;IACF,CAAC;IAED,aAAa,CAAC,mBAAmB,EAAE,aAAa,CAAC,CAAA;AAClD,CAAC;AAED,KAAK,EAAE,CAAA"}
+{"version":3,"file":"Test.js","sourceRoot":"","sources":["../src/Test.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,wCAAwC,EAAsB,WAAW,EAAE,YAAY,EAAE,YAAY,EAAE,MAAM,uBAAuB,CAAA;AAC7I,OAAO,EAAE,KAAK,EAAE,MAAM,sBAAsB,CAAA;AAE5C,MAAM,GAAG,GAAG,OAAO,CAAC,GAAG,CAAA;AAEvB,KAAK,UAAU,KAAK;IACnB,CAAC;QACA,MAAM,YAAY,GAAG,MAAM,YAAY,CAAC,gCAAgC,CAAC,CAAA;QACzE,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,SAAS,CAAC,YAAY,CAAC,KAAK,CAAC,CAAC,CAAA;IAChD,CAAC;IAED,MAAM,EAAE,YAAY,EAAE,aAAa,EAAE,GAAG,MAAM,MAAM,CAAC,IAAI,CAAC,CAAA;IAC1D,MAAM,IAAI,GAAG,YAAY,CAAC,oBAAoB,EAAE,OAAO,CAAC,CAAA;IAExD,MAAM,KAAK,GAAG,IAAI,KAAK,EAAE,CAAA;IAEzB,IAAI,MAA0B,CAAA;IAE9B,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;QAC5B,MAAM,GAAG,MAAM,WAAW,CAAC,IAAI,EAAE;YAChC,QAAQ,EAAE,IAAI;YACd,kBAAkB,EAAE,EAAE;YACtB,6BAA6B,EAAE,IAAI;SACnC,CAAC,CAAA;QAEF,KAAK,CAAC,aAAa,CAAC,sBAAsB,CAAC,CAAA;IAC5C,CAAC;IAED,GAAG,CAAC,EAAE,CAAC,CAAA;IAEP,EAAE;IAEF,IAAI,aAAa,GAAG,EAAE,CAAA;IAEtB,KAAK,IAAI,aAAa,GAAG,CAAC,EAAE,aAAa,GAAG,MAAO,CAAC,SAAS,CAAC,MAAM,EAAE,aAAa,EAAE,EAAE,CAAC;QACvF,MAAM,QAAQ,GAAG,MAAO,CAAC,SAAS,CAAC,aAAa,CAAC,CAAA;QACjD,MAAM,OAAO,GAAG,QAAQ,CAAC,OAAO,CAAA;QAEhC,KAAK,IAAI,WAAW,GAAG,CAAC,EAAE,WAAW,GAAG,OAAO,CAAC,MAAM,EAAE,WAAW,EAAE,EAAE,CAAC;YACvE,MAAM,MAAM,GAAG,OAAO,CAAC,WAAW,CAAC,CAAA;YAEnC,aAAa,IAAI,MAAM,CAAC,YAAY,CAAC,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,CAAA;YAEtD,IAAI,WAAW,GAAG,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBACtC,aAAa,IAAI,KAAK,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,IAAI,CAAA;YAC1C,CAAC;QACF,CAAC;QAED,IAAI,aAAa,GAAG,MAAO,CAAC,SAAS,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAClD,aAAa,IAAI,KAAM,GAAG,CAAC,MAAM,CAAC,GAAG,CAAE,KAAK,CAAA;QAC7C,CAAC;IACF,CAAC;IAED,aAAa,CAAC,mBAAmB,EAAE,aAAa,CAAC,CAAA;AAClD,CAAC;AAED,KAAK,UAAU,KAAK;IACnB,MAAM,IAAI,GAAG,mDAAmD,CAAA;IAEhE,MAAM,KAAK,GAAG,MAAM,YAAY,CAAC,IAAI,CAAC,CAAA;IAEtC,MAAM,yBAAyB,GAAG,KAAK,CAAC,qBAAqB,CAAA;IAC7D,MAAM,0BAA0B,GAAG,IAAI,YAAY,EAAE,CAAA;IACrD,0BAA0B,CAAC,OAAO,GAAG,yBAAyB,CAAA;IAE9D,MAAM,aAAa,GAAG,wCAAwC,CAAC,0BAA0B,EAAE,IAAI,CAAC,CAAA;IAEhG,MAAM,CAAC,GAAG,CAAC,CAAA;AACZ,CAAC;AAED,KAAK,EAAE,CAAA;AACP,SAAS"}

package/dist/TextSegmentation.d.ts

@@ -4,6 +4,10 @@ export { WordSequence, type WordEntry } from './WordSequence.js';
 export declare function segmentText(text: string, options?: SegmentationOptions): Promise<SegmentationResult>;
 export declare function segmentWordSequence(wordSequence: WordSequence): Promise<SegmentationResult>;
 export declare function splitToWords(text: string, options?: SegmentationOptions): Promise<WordSequence>;
+export declare function addMissingPunctuationWordsToWordSequence(wordSequence: WordSequence, sourceText: string): {
+    wordSequenceWithPunctuation: WordSequence;
+    originalWordsReverseMapping: Map<number, number>;
+};
 export interface SegmentationResult {
     wordSequence: WordSequence;
     sentences: Sentence[];

package/dist/TextSegmentation.js

@@ -80,48 +80,39 @@ export async function segmentWordSequence(wordSequence) {
     };
     return result;
 }
+const cachedWordSplitterRegExps = new Map();
 export async function splitToWords(text, options) {
-    options = { ...defaultSegmentationOptions, ...(options ?? {}) };
-    const cldrSuppressionsForLang = cldrSuppressions[options.language ?? ''] ?? [];
-    const contractionSuppressionsForLang = leadingApostropheContractionSuppressions[options.language ?? ''] ?? [];
-    const contractionSuppressionsForLangWithSingleQuote = contractionSuppressionsForLang.map(str => str.replaceAll(`'`, `’`));
-    const customSuppressions = options.customSuppressions ?? [];
-    let suppressions = [
-        ...customSuppressions,
-        ...cldrSuppressionsForLang,
-        ...contractionSuppressionsForLang,
-        ...contractionSuppressionsForLangWithSingleQuote,
-        ...nounSuppressions,
-        ...tldSuppressions,
-    ];
-    const wordPattern = buildWordSplitterPattern([
-        ...suppressions,
-        ...suppressions.map(word => word.toLocaleLowerCase()),
-        ...suppressions.map(word => word.toLocaleUpperCase()),
-    ]);
-    const wordSplitterRegExp = buildRegExp(wordPattern, { global: true });
-    //console.log(`Encoded pattern: ${wordSplitterRegExp.source}\n`)
+    if (!options) {
+        options = {};
+    }
+    options = { ...defaultSegmentationOptions, ...options };
+    const optionsAsJson = JSON.stringify(options);
+    let wordSplitterRegExp = cachedWordSplitterRegExps.get(optionsAsJson);
+    if (!wordSplitterRegExp) {
+        wordSplitterRegExp = buildWordSplitterRegExpForOptions(options);
+        cachedWordSplitterRegExps.set(optionsAsJson, wordSplitterRegExp);
+    }
     let wordSequence = new WordSequence();
-    function addNonWordsBetween(startOffset, endOffset) {
-        const nonWordSubstring = text.substring(startOffset, endOffset);
+    function addPunctuationWordsBetween(startOffset, endOffset) {
+        const punctuationWordSubstring = text.substring(startOffset, endOffset);
         let charOffset = startOffset;
-        let nonwordStartOffset = startOffset;
-        function addNonWordIfNeeded() {
-            if (charOffset > nonwordStartOffset) {
-                wordSequence.addWordEntry(text, nonwordStartOffset, charOffset, true);
-                nonwordStartOffset = charOffset;
+        let punctuationWordStartOffset = startOffset;
+        function addPunctuationWordIfNeeded() {
+            if (charOffset > punctuationWordStartOffset) {
+                wordSequence.addWordEntry(text, punctuationWordStartOffset, charOffset, true);
+                punctuationWordStartOffset = charOffset;
             }
         }
-        for (const char of nonWordSubstring) {
+        for (const char of punctuationWordSubstring) {
             if (char === ' ') {
                 charOffset += 1;
                 continue;
             }
-            addNonWordIfNeeded();
+            addPunctuationWordIfNeeded();
             charOffset += char.length;
-            addNonWordIfNeeded();
+            addPunctuationWordIfNeeded();
         }
-        addNonWordIfNeeded();
+        addPunctuationWordIfNeeded();
     }
     const wordMatches = text.matchAll(wordSplitterRegExp);
     let lastMatchEndOffset = 0;
@@ -131,12 +122,12 @@ export async function splitToWords(text, options) {
             const matchStartOffset = offsets[0];
             const matchEndOffset = offsets[1];
             if (matchStartOffset > lastMatchEndOffset) {
-                addNonWordsBetween(lastMatchEndOffset, matchStartOffset);
+                addPunctuationWordsBetween(lastMatchEndOffset, matchStartOffset);
             }
             wordSequence.addWordEntry(text, matchStartOffset, matchEndOffset, false);
             lastMatchEndOffset = matchEndOffset;
         }
-        addNonWordsBetween(lastMatchEndOffset, text.length);
+        addPunctuationWordsBetween(lastMatchEndOffset, text.length);
     }
     if (options.enableEastAsianPostprocessing) {
         wordSequence = await postprocessEastAsianWords(text, wordSequence);
@@ -170,6 +161,68 @@ async function postprocessEastAsianWords(containingText, wordSequence) {
     }
     return newWordSequence;
 }
+function buildWordSplitterRegExpForOptions(options) {
+    const cldrSuppressionsForLang = cldrSuppressions[options.language ?? ''] ?? [];
+    const contractionSuppressionsForLang = leadingApostropheContractionSuppressions[options.language ?? ''] ?? [];
+    const contractionSuppressionsForLangWithSingleQuote = contractionSuppressionsForLang.map(str => str.replaceAll(`'`, `’`));
+    const customSuppressions = options.customSuppressions ?? [];
+    let suppressions = [
+        ...customSuppressions,
+        ...cldrSuppressionsForLang,
+        ...contractionSuppressionsForLang,
+        ...contractionSuppressionsForLangWithSingleQuote,
+        ...nounSuppressions,
+        ...tldSuppressions,
+    ];
+    const wordPattern = buildWordSplitterPattern([
+        ...suppressions,
+        ...suppressions.map(word => word.toLocaleLowerCase()),
+        ...suppressions.map(word => word.toLocaleUpperCase()),
+    ]);
+    const wordSplitterRegExp = buildRegExp(wordPattern, { global: true });
+    return wordSplitterRegExp;
+}
+// Add any missing punctuation words to a word sequence
+export function addMissingPunctuationWordsToWordSequence(wordSequence, sourceText) {
+    const originalWordsReverseMapping = new Map();
+    const wordSequenceWithPunctuation = new WordSequence();
+    function addWordEntriesForTextSlice(textSlice, initialCharOffset) {
+        let charOffset = initialCharOffset;
+        // Add entry for every codepoint (this will correctly treat characters beyond BMP)
+        for (const char of textSlice) {
+            const charEndOffset = charOffset + char.length;
+            const lastEntry = wordSequenceWithPunctuation.lastEntry;
+            if (char === ' ' && lastEntry && lastEntry.isPunctuation && lastEntry.text[0] === ' ') {
+                wordSequenceWithPunctuation.lastEntry.text += ' ';
+                wordSequenceWithPunctuation.lastEntry.endOffset = charEndOffset;
+            }
+            else {
+                wordSequenceWithPunctuation.addWordEntry(sourceText, charOffset, charEndOffset, true);
+            }
+            charOffset = charEndOffset;
+        }
+    }
+    for (let wordIndex = 0; wordIndex < wordSequence.length; wordIndex++) {
+        const wordEntry = wordSequence.getEntryAt(wordIndex);
+        const wordStartOffset = wordEntry.startOffset;
+        const previousWordEndOffset = wordIndex > 0 ? wordSequence.entries[wordIndex - 1].endOffset : 0;
+        // Add entries for any punctuation characters between the current and previous word (or start)
+        if (previousWordEndOffset !== wordStartOffset) {
+            const textSlice = sourceText.substring(previousWordEndOffset, wordStartOffset);
+            addWordEntriesForTextSlice(textSlice, previousWordEndOffset);
+        }
+        wordSequenceWithPunctuation.entries.push(wordEntry);
+        originalWordsReverseMapping.set(wordSequenceWithPunctuation.length - 1, wordIndex);
+        // If last word, add entries for any trailing punctuation characters
+        if (wordIndex === wordSequence.length - 1) {
+            if (sourceText.length !== wordEntry.endOffset) {
+                const textSlice = sourceText.substring(wordEntry.endOffset, sourceText.length);
+                addWordEntriesForTextSlice(textSlice, wordEntry.endOffset);
+            }
+        }
+    }
+    return { wordSequenceWithPunctuation, originalWordsReverseMapping };
+}
 async function getIcuSegmentation() {
     try {
         const icuSegmentation = await import('@echogarden/icu-segmentation-wasm');

package/dist/TextSegmentation.js.map

@@ -1 +1 @@
-{"version":3,"file":"TextSegmentation.js","sourceRoot":"","sources":["../src/TextSegmentation.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,WAAW,EAAE,MAAM,iBAAiB,CAAA;AAC7C,OAAO,EAAE,wBAAwB,IAAI,wBAAwB,EAAE,qBAAqB,EAAE,0CAA0C,EAAE,uBAAuB,EAAE,uBAAuB,EAAE,MAAM,eAAe,CAAA;AACzM,OAAO,EAAE,gBAAgB,EAAE,wCAAwC,EAAE,gBAAgB,EAAE,eAAe,EAAE,MAAM,mBAAmB,CAAA;AACjI,OAAO,EAAE,yBAAyB,EAAE,MAAM,iCAAiC,CAAA;AAC3E,OAAO,EAAE,YAAY,EAAE,MAAM,mBAAmB,CAAA;AAEhD,OAAO,EAAE,gBAAgB,EAAE,MAAM,mBAAmB,CAAA;AACpD,OAAO,EAAE,YAAY,EAAkB,MAAM,mBAAmB,CAAA;AAEhE,MAAM,CAAC,KAAK,UAAU,WAAW,CAAC,IAAY,EAAE,OAA6B;IAC5E,OAAO,GAAG,EAAE,GAAG,0BAA0B,EAAE,GAAG,CAAC,OAAO,IAAI,EAAE,CAAC,EAAE,CAAA;IAE/D,MAAM,YAAY,GAAG,MAAM,YAAY,CAAC,IAAI,EAAE,OAAO,CAAC,CAAA;IAEtD,OAAO,mBAAmB,CAAC,YAAY,CAAC,CAAA;AACzC,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,mBAAmB,CAAC,YAA0B;IACnE,MAAM,kBAAkB,GAAY,EAAE,CAAA;IAEtC,IAAI,uBAAuB,GAAG,CAAC,CAAA;IAE/B,KAAK,IAAI,SAAS,GAAG,CAAC,EAAE,SAAS,GAAG,YAAY,CAAC,MAAM,EAAE,SAAS,EAAE,EAAE,CAAC;QACtE,MAAM,IAAI,GAAG,YAAY,CAAC,SAAS,CAAC,SAAS,CAAC,CAAA;QAE9C,IAAI,uBAAuB,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;YACxC,OAAO,SAAS,GAAG,YAAY,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBAC5C,MAAM,QAAQ,GAAG,YAAY,CAAC,SAAS,CAAC,SAAS,GAAG,CAAC,CAAC,CAAA;gBAEtD,IAAI,CAAC,0CAA0C,CAAC,IAAI,CAAC,QAAQ,CAAC,EAAE,CAAC;oBAChE,MAAK;gBACN,CAAC;gBAED,SAAS,IAAI,CAAC,CAAA;YACf,CAAC;YAED,kBAAkB,CAAC,IAAI,CAAC;gBACvB,KAAK,EAAE,uBAAuB;gBAC9B,GAAG,EAAE,SAAS,GAAG,CAAC;aAClB,CAAC,CAAA;YAEF,uBAAuB,GAAG,SAAS,GAAG,CAAC,CAAA;QACxC,CAAC;IACF,CAAC;IAED,IAAI,uBAAuB,GAAG,YAAY,CAAC,MAAM,EAAE,CAAC;QACnD,kBAAkB,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,uBAAuB,EAAE,GAAG,EAAE,YAAY,CAAC,MAAM,EAAE,CAAC,CAAA;IACtF,CAAC;IAED,MAAM,SAAS,GAAe,EAAE,CAAA;IAEhC,KAAK,MAAM,SAAS,IAAI,kBAAkB,EAAE,CAAC;QAC5C,MAAM,oBAAoB,GAAG,YAAY,CAAC,KAAK,CAAC,SAAS,CAAC,KAAK,EAAE,SAAS,CAAC,GAAG,CAAC,CAAA;QAE/E,SAAS,CAAC,IAAI,CAAC,IAAI,QAAQ,CAAC,SAAS,EAAE,oBAAoB,CAAC,CAAC,CAAA;IAC9D,CAAC;IAED,KAAK,MAAM,QAAQ,IAAI,SAAS,EAAE,CAAC;QAClC,MAAM,gBAAgB,GAAY,EAAE,CAAA;QAEpC,IAAI,qBAAqB,GAAG,QAAQ,CAAC,SAAS,CAAC,GAAG,CAAA;QAClD,IAAI,qBAAqB,GAAG,QAAQ,CAAC,SAAS,CAAC,KAAK,CAAA;QAEpD,KAAK,IAAI,SAAS,GAAG,qBAAqB,EAAE,SAAS,GAAG,qBAAqB,EAAE,SAAS,EAAE,EAAE,CAAC;YAC5F,MAAM,WAAW,GAAG,YAAY,CAAC,SAAS,CAAC,SAAS,CAAC,CAAA;YAErD,IAAI,qBAAqB,CAAC,IAAI,CAAC,WAAW,CAAC,EAAE,CAAC;gBAC7C,IAAI,kBAAkB,GAAG,KAAK,CAAA;gBAE9B,OAAO,SAAS,GAAG,qBAAqB,GAAG,CAAC,EAAE,CAAC;oBAC9C,MAAM,QAAQ,GAAG,YAAY,CAAC,SAAS,CAAC,SAAS,GAAG,CAAC,CAAC,CAAA;oBAEtD,IAAI,CAAC,0CAA0C,CAAC,IAAI,CAAC,QAAQ,CAAC,EAAE,CAAC;wBAChE,MAAK;oBACN,CAAC;oBAED,IAAI,uBAAuB,CAAC,IAAI,CAAC,QAAQ,CAAC,EAAE,CAAC;wBAC5C,kBAAkB,GAAG,IAAI,CAAA;oBAC1B,CAAC;oBAED,IAAI,QAAQ,KAAK,GAAG,IAAI,kBAAkB,EAAE,CAAC;wBAC5C,MAAK;oBACN,CAAC;oBAED,SAAS,IAAI,CAAC,CAAA;gBACf,CAAC;gBAED,gBAAgB,CAAC,IAAI,CAAC;oBACrB,KAAK,EAAE,qBAAqB;oBAC5B,GAAG,EAAE,SAAS,GAAG,CAAC;iBAClB,CAAC,CAAA;gBAEF,qBAAqB,GAAG,SAAS,GAAG,CAAC,CAAA;YACtC,CAAC;QACF,CAAC;QAED,IAAI,qBAAqB,GAAG,qBAAqB,EAAE,CAAC;YACnD,gBAAgB,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,qBAAqB,EAAE,GAAG,EAAE,qBAAqB,EAAE,CAAC,CAAA;QACpF,CAAC;QAED,KAAK,MAAM,SAAS,IAAI,gBAAgB,EAAE,CAAC;YAC1C,MAAM,kBAAkB,GAAG,YAAY,CAAC,KAAK,CAAC,SAAS,CAAC,KAAK,EAAE,SAAS,CAAC,GAAG,CAAC,CAAA;YAE7E,QAAQ,CAAC,OAAO,CAAC,IAAI,CAAC,IAAI,MAAM,CAAC,SAAS,EAAE,kBAAkB,CAAC,CAAC,CAAA;QACjE,CAAC;IACF,CAAC;IAED,MAAM,MAAM,GAAuB;QAClC,YAAY;QACZ,SAAS;KACT,CAAA;IAED,OAAO,MAAM,CAAA;AACd,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,YAAY,CAAC,IAAY,EAAE,OAA6B;IAC7E,OAAO,GAAG,EAAE,GAAG,0BAA0B,EAAE,GAAG,CAAC,OAAO,IAAI,EAAE,CAAC,EAAE,CAAA;IAE/D,MAAM,uBAAuB,GAAG,gBAAgB,CAAC,OAAO,CAAC,QAAQ,IAAI,EAAE,CAAC,IAAI,EAAE,CAAA;IAC9E,MAAM,8BAA8B,GAAG,wCAAwC,CAAC,OAAO,CAAC,QAAQ,IAAI,EAAE,CAAC,IAAI,EAAE,CAAA;IAC7G,MAAM,6CAA6C,GAAG,8BAA8B,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,UAAU,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC,CAAA;IACzH,MAAM,kBAAkB,GAAG,OAAO,CAAC,kBAAkB,IAAI,EAAE,CAAA;IAE3D,IAAI,YAAY,GAAG;QAClB,GAAG,kBAAkB;QACrB,GAAG,uBAAuB;QAC1B,GAAG,8BAA8B;QACjC,GAAG,6CAA6C;QAChD,GAAG,gBAAgB;QACnB,GAAG,eAAe;KAClB,CAAA;IAED,MAAM,WAAW,GAAG,wBAAwB,CAAC;QAC5C,GAAG,YAAY;QACf,GAAG,YAAY,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,iBAAiB,EAAE,CAAC;QACrD,GAAG,YAAY,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,iBAAiB,EAAE,CAAC;KACrD,CAAC,CAAA;IAEF,MAAM,kBAAkB,GAAG,WAAW,CAAC,WAAW,EAAE,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC,CAAA;IAErE,gEAAgE;IAEhE,IAAI,YAAY,GAAG,IAAI,YAAY,EAAE,CAAA;IAErC,SAAS,kBAAkB,CAAC,WAAmB,EAAE,SAAiB;QACjE,MAAM,gBAAgB,GAAG,IAAI,CAAC,SAAS,CAAC,WAAW,EAAE,SAAS,CAAC,CAAA;QAE/D,IAAI,UAAU,GAAG,WAAW,CAAA;QAC5B,IAAI,kBAAkB,GAAG,WAAW,CAAA;QAEpC,SAAS,kBAAkB;YAC1B,IAAI,UAAU,GAAG,kBAAkB,EAAE,CAAC;gBACrC,YAAY,CAAC,YAAY,CAAC,IAAI,EAAE,kBAAkB,EAAE,UAAU,EAAE,IAAI,CAAC,CAAA;gBACrE,kBAAkB,GAAG,UAAU,CAAA;YAChC,CAAC;QACF,CAAC;QAED,KAAK,MAAM,IAAI,IAAI,gBAAgB,EAAE,CAAC;YACrC,IAAI,IAAI,KAAK,GAAG,EAAE,CAAC;gBAClB,UAAU,IAAI,CAAC,CAAA;gBAEf,SAAQ;YACT,CAAC;YAED,kBAAkB,EAAE,CAAA;YAEpB,UAAU,IAAI,IAAI,CAAC,MAAM,CAAA;YAEzB,kBAAkB,EAAE,CAAA;QACrB,CAAC;QAED,kBAAkB,EAAE,CAAA;IACrB,CAAC;IAED,MAAM,WAAW,GAAG,IAAI,CAAC,QAAQ,CAAC,kBAAkB,CAAC,CAAA;IAErD,IAAI,kBAAkB,GAAG,CAAC,CAAA;IAE1B,IAAI,WAAW,EAAE,CAAC;QACjB,KAAK,MAAM,KAAK,IAAI,WAAW,EAAE,CAAC;YACjC,MAAM,OAAO,GAAG,KAAK,CAAC,OAAQ,CAAC,CAAC,CAAC,CAAA;YACjC,MAAM,gBAAgB,GAAG,OAAO,CAAC,CAAC,CAAC,CAAA;YACnC,MAAM,cAAc,GAAG,OAAO,CAAC,CAAC,CAAC,CAAA;YAEjC,IAAI,gBAAgB,GAAG,kBAAkB,EAAE,CAAC;gBAC3C,kBAAkB,CAAC,kBAAkB,EAAE,gBAAgB,CAAC,CAAA;YACzD,CAAC;YAED,YAAY,CAAC,YAAY,CAAC,IAAI,EAAE,gBAAgB,EAAE,cAAc,EAAE,KAAK,CAAC,CAAA;YAExE,kBAAkB,GAAG,cAAc,CAAA;QACpC,CAAC;QAED,kBAAkB,CAAC,kBAAkB,EAAE,IAAI,CAAC,MAAM,CAAC,CAAA;IACpD,CAAC;IAED,IAAI,OAAO,CAAC,6BAA6B,EAAE,CAAC;QAC3C,YAAY,GAAG,MAAM,yBAAyB,CAAC,IAAI,EAAE,YAAY,CAAC,CAAA;IACnE,CAAC;IAED,OAAO,YAAY,CAAA;AACpB,CAAC;AAED,KAAK,UAAU,yBAAyB,CAAC,cAAsB,EAAE,YAA0B;IAC1F,MAAM,eAAe,GAAG,MAAM,kBAAkB,EAAE,CAAA;IAElD,IAAI,eAAe,KAAK,SAAS,EAAE,CAAC;QACnC,OAAO,YAAY,CAAA;IACpB,CAAC;IAED,IAAI,cAAc,GAAG,KAAK,CAAA;IAE1B,MAAM,eAAe,GAAG,IAAI,YAAY,EAAE,CAAA;IAE1C,KAAK,IAAI,SAAS,GAAG,CAAC,EAAE,SAAS,GAAG,YAAY,CAAC,MAAM,EAAE,SAAS,EAAE,EAAE,CAAC;QACtE,MAAM,SAAS,GAAG,YAAY,CAAC,OAAO,CAAC,SAAS,CAAC,CAAA;QACjD,MAAM,eAAe,GAAG,SAAS,CAAC,WAAW,CAAA;QAE7C,MAAM,IAAI,GAAG,YAAY,CAAC,SAAS,CAAC,SAAS,CAAC,CAAA;QAE9C,IAAI,yBAAyB,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;YAC1C,IAAI,CAAC,cAAc,EAAE,CAAC;gBACrB,MAAM,eAAe,CAAC,UAAU,EAAE,CAAA;gBAElC,cAAc,GAAG,IAAI,CAAA;YACtB,CAAC;YAED,MAAM,UAAU,GAAG,CAAC,GAAG,eAAe,CAAC,uBAAuB,CAAC,IAAI,CAAC,CAAC,CAAA;YAErE,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,UAAU,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;gBAChD,eAAe,CAAC,YAAY,CAC3B,cAAc,EACd,eAAe,GAAG,UAAU,CAAC,CAAC,CAAC,EAC/B,eAAe,GAAG,UAAU,CAAC,CAAC,GAAG,CAAC,CAAC,EACnC,KAAK,CACL,CAAA;YACF,CAAC;QACF,CAAC;aAAM,CAAC;YACP,eAAe,CAAC,YAAY,CAAC,cAAc,EAAE,SAAS,CAAC,WAAW,EAAE,SAAS,CAAC,SAAS,EAAE,SAAS,CAAC,aAAa,CAAC,CAAA;QAClH,CAAC;IACF,CAAC;IAED,OAAO,eAAe,CAAA;AACvB,CAAC;AAED,KAAK,UAAU,kBAAkB;IAChC,IAAI,CAAC;QACJ,MAAM,eAAe,GAAG,MAAM,MAAM,CAAC,mCAAmC,CAAC,CAAA;QAEzE,OAAO,eAAe,CAAA;IACvB,CAAC;IAAC,MAAM,CAAC;QACR,OAAO,SAAS,CAAA;IACjB,CAAC;AACF,CAAC;AAOD,MAAM,OAAO,YAAY;IACxB,SAAS,CAAO;IAChB,YAAY,CAAc;IAE1B,YAAY,SAAgB,EAAE,YAA0B;QACvD,IAAI,CAAC,SAAS,GAAG,SAAS,CAAA;QAC1B,IAAI,CAAC,YAAY,GAAG,YAAY,CAAA;IACjC,CAAC;IAED,IAAI,IAAI;QACP,OAAO,IAAI,CAAC,YAAY,CAAC,IAAI,CAAA;IAC9B,CAAC;IAED,IAAI,SAAS;QACZ,OAAO;YACN,KAAK,EAAE,IAAI,CAAC,YAAY,CAAC,UAAU,CAAC,WAAW;YAC/C,GAAG,EAAE,IAAI,CAAC,YAAY,CAAC,SAAS,CAAC,SAAS;SAC1C,CAAA;IACF,CAAC;CACD;AAED,MAAM,OAAO,QAAS,SAAQ,YAAY;IACzC,OAAO,GAAa,EAAE,CAAA;CACtB;AAED,MAAM,OAAO,MAAO,SAAQ,YAAY;CACvC;AAaD,MAAM,CAAC,MAAM,0BAA0B,GAAwB;IAC9D,QAAQ,EAAE,EAAE;IACZ,kBAAkB,EAAE,EAAE;IACtB,6BAA6B,EAAE,IAAI;CACnC,CAAA"}
+{"version":3,"file":"TextSegmentation.js","sourceRoot":"","sources":["../src/TextSegmentation.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,WAAW,EAAE,MAAM,iBAAiB,CAAA;AAC7C,OAAO,EAAE,wBAAwB,IAAI,wBAAwB,EAAE,qBAAqB,EAAE,0CAA0C,EAAE,uBAAuB,EAAE,uBAAuB,EAAE,MAAM,eAAe,CAAA;AACzM,OAAO,EAAE,gBAAgB,EAAE,wCAAwC,EAAE,gBAAgB,EAAE,eAAe,EAAE,MAAM,mBAAmB,CAAA;AACjI,OAAO,EAAE,yBAAyB,EAAE,MAAM,iCAAiC,CAAA;AAC3E,OAAO,EAAE,YAAY,EAAE,MAAM,mBAAmB,CAAA;AAEhD,OAAO,EAAE,gBAAgB,EAAE,MAAM,mBAAmB,CAAA;AACpD,OAAO,EAAE,YAAY,EAAkB,MAAM,mBAAmB,CAAA;AAEhE,MAAM,CAAC,KAAK,UAAU,WAAW,CAAC,IAAY,EAAE,OAA6B;IAC5E,OAAO,GAAG,EAAE,GAAG,0BAA0B,EAAE,GAAG,CAAC,OAAO,IAAI,EAAE,CAAC,EAAE,CAAA;IAE/D,MAAM,YAAY,GAAG,MAAM,YAAY,CAAC,IAAI,EAAE,OAAO,CAAC,CAAA;IAEtD,OAAO,mBAAmB,CAAC,YAAY,CAAC,CAAA;AACzC,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,mBAAmB,CAAC,YAA0B;IACnE,MAAM,kBAAkB,GAAY,EAAE,CAAA;IAEtC,IAAI,uBAAuB,GAAG,CAAC,CAAA;IAE/B,KAAK,IAAI,SAAS,GAAG,CAAC,EAAE,SAAS,GAAG,YAAY,CAAC,MAAM,EAAE,SAAS,EAAE,EAAE,CAAC;QACtE,MAAM,IAAI,GAAG,YAAY,CAAC,SAAS,CAAC,SAAS,CAAC,CAAA;QAE9C,IAAI,uBAAuB,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;YACxC,OAAO,SAAS,GAAG,YAAY,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBAC5C,MAAM,QAAQ,GAAG,YAAY,CAAC,SAAS,CAAC,SAAS,GAAG,CAAC,CAAC,CAAA;gBAEtD,IAAI,CAAC,0CAA0C,CAAC,IAAI,CAAC,QAAQ,CAAC,EAAE,CAAC;oBAChE,MAAK;gBACN,CAAC;gBAED,SAAS,IAAI,CAAC,CAAA;YACf,CAAC;YAED,kBAAkB,CAAC,IAAI,CAAC;gBACvB,KAAK,EAAE,uBAAuB;gBAC9B,GAAG,EAAE,SAAS,GAAG,CAAC;aAClB,CAAC,CAAA;YAEF,uBAAuB,GAAG,SAAS,GAAG,CAAC,CAAA;QACxC,CAAC;IACF,CAAC;IAED,IAAI,uBAAuB,GAAG,YAAY,CAAC,MAAM,EAAE,CAAC;QACnD,kBAAkB,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,uBAAuB,EAAE,GAAG,EAAE,YAAY,CAAC,MAAM,EAAE,CAAC,CAAA;IACtF,CAAC;IAED,MAAM,SAAS,GAAe,EAAE,CAAA;IAEhC,KAAK,MAAM,SAAS,IAAI,kBAAkB,EAAE,CAAC;QAC5C,MAAM,oBAAoB,GAAG,YAAY,CAAC,KAAK,CAAC,SAAS,CAAC,KAAK,EAAE,SAAS,CAAC,GAAG,CAAC,CAAA;QAE/E,SAAS,CAAC,IAAI,CAAC,IAAI,QAAQ,CAAC,SAAS,EAAE,oBAAoB,CAAC,CAAC,CAAA;IAC9D,CAAC;IAED,KAAK,MAAM,QAAQ,IAAI,SAAS,EAAE,CAAC;QAClC,MAAM,gBAAgB,GAAY,EAAE,CAAA;QAEpC,IAAI,qBAAqB,GAAG,QAAQ,CAAC,SAAS,CAAC,GAAG,CAAA;QAClD,IAAI,qBAAqB,GAAG,QAAQ,CAAC,SAAS,CAAC,KAAK,CAAA;QAEpD,KAAK,IAAI,SAAS,GAAG,qBAAqB,EAAE,SAAS,GAAG,qBAAqB,EAAE,SAAS,EAAE,EAAE,CAAC;YAC5F,MAAM,WAAW,GAAG,YAAY,CAAC,SAAS,CAAC,SAAS,CAAC,CAAA;YAErD,IAAI,qBAAqB,CAAC,IAAI,CAAC,WAAW,CAAC,EAAE,CAAC;gBAC7C,IAAI,kBAAkB,GAAG,KAAK,CAAA;gBAE9B,OAAO,SAAS,GAAG,qBAAqB,GAAG,CAAC,EAAE,CAAC;oBAC9C,MAAM,QAAQ,GAAG,YAAY,CAAC,SAAS,CAAC,SAAS,GAAG,CAAC,CAAC,CAAA;oBAEtD,IAAI,CAAC,0CAA0C,CAAC,IAAI,CAAC,QAAQ,CAAC,EAAE,CAAC;wBAChE,MAAK;oBACN,CAAC;oBAED,IAAI,uBAAuB,CAAC,IAAI,CAAC,QAAQ,CAAC,EAAE,CAAC;wBAC5C,kBAAkB,GAAG,IAAI,CAAA;oBAC1B,CAAC;oBAED,IAAI,QAAQ,KAAK,GAAG,IAAI,kBAAkB,EAAE,CAAC;wBAC5C,MAAK;oBACN,CAAC;oBAED,SAAS,IAAI,CAAC,CAAA;gBACf,CAAC;gBAED,gBAAgB,CAAC,IAAI,CAAC;oBACrB,KAAK,EAAE,qBAAqB;oBAC5B,GAAG,EAAE,SAAS,GAAG,CAAC;iBAClB,CAAC,CAAA;gBAEF,qBAAqB,GAAG,SAAS,GAAG,CAAC,CAAA;YACtC,CAAC;QACF,CAAC;QAED,IAAI,qBAAqB,GAAG,qBAAqB,EAAE,CAAC;YACnD,gBAAgB,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,qBAAqB,EAAE,GAAG,EAAE,qBAAqB,EAAE,CAAC,CAAA;QACpF,CAAC;QAED,KAAK,MAAM,SAAS,IAAI,gBAAgB,EAAE,CAAC;YAC1C,MAAM,kBAAkB,GAAG,YAAY,CAAC,KAAK,CAAC,SAAS,CAAC,KAAK,EAAE,SAAS,CAAC,GAAG,CAAC,CAAA;YAE7E,QAAQ,CAAC,OAAO,CAAC,IAAI,CAAC,IAAI,MAAM,CAAC,SAAS,EAAE,kBAAkB,CAAC,CAAC,CAAA;QACjE,CAAC;IACF,CAAC;IAED,MAAM,MAAM,GAAuB;QAClC,YAAY;QACZ,SAAS;KACT,CAAA;IAED,OAAO,MAAM,CAAA;AACd,CAAC;AAED,MAAM,yBAAyB,GAAG,IAAI,GAAG,EAAkB,CAAA;AAE3D,MAAM,CAAC,KAAK,UAAU,YAAY,CAAC,IAAY,EAAE,OAA6B;IAC7E,IAAI,CAAC,OAAO,EAAE,CAAC;QACd,OAAO,GAAG,EAAE,CAAA;IACb,CAAC;IAED,OAAO,GAAG,EAAE,GAAG,0BAA0B,EAAE,GAAG,OAAO,EAAE,CAAA;IAEvD,MAAM,aAAa,GAAG,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,CAAA;IAE7C,IAAI,kBAAkB,GAAG,yBAAyB,CAAC,GAAG,CAAC,aAAa,CAAC,CAAA;IAErE,IAAI,CAAC,kBAAkB,EAAE,CAAC;QACzB,kBAAkB,GAAG,iCAAiC,CAAC,OAAO,CAAC,CAAA;QAE/D,yBAAyB,CAAC,GAAG,CAAC,aAAa,EAAE,kBAAkB,CAAC,CAAA;IACjE,CAAC;IAED,IAAI,YAAY,GAAG,IAAI,YAAY,EAAE,CAAA;IAErC,SAAS,0BAA0B,CAAC,WAAmB,EAAE,SAAiB;QACzE,MAAM,wBAAwB,GAAG,IAAI,CAAC,SAAS,CAAC,WAAW,EAAE,SAAS,CAAC,CAAA;QAEvE,IAAI,UAAU,GAAG,WAAW,CAAA;QAC5B,IAAI,0BAA0B,GAAG,WAAW,CAAA;QAE5C,SAAS,0BAA0B;YAClC,IAAI,UAAU,GAAG,0BAA0B,EAAE,CAAC;gBAC7C,YAAY,CAAC,YAAY,CAAC,IAAI,EAAE,0BAA0B,EAAE,UAAU,EAAE,IAAI,CAAC,CAAA;gBAC7E,0BAA0B,GAAG,UAAU,CAAA;YACxC,CAAC;QACF,CAAC;QAED,KAAK,MAAM,IAAI,IAAI,wBAAwB,EAAE,CAAC;YAC7C,IAAI,IAAI,KAAK,GAAG,EAAE,CAAC;gBAClB,UAAU,IAAI,CAAC,CAAA;gBAEf,SAAQ;YACT,CAAC;YAED,0BAA0B,EAAE,CAAA;YAE5B,UAAU,IAAI,IAAI,CAAC,MAAM,CAAA;YAEzB,0BAA0B,EAAE,CAAA;QAC7B,CAAC;QAED,0BAA0B,EAAE,CAAA;IAC7B,CAAC;IAED,MAAM,WAAW,GAAG,IAAI,CAAC,QAAQ,CAAC,kBAAkB,CAAC,CAAA;IAErD,IAAI,kBAAkB,GAAG,CAAC,CAAA;IAE1B,IAAI,WAAW,EAAE,CAAC;QACjB,KAAK,MAAM,KAAK,IAAI,WAAW,EAAE,CAAC;YACjC,MAAM,OAAO,GAAG,KAAK,CAAC,OAAQ,CAAC,CAAC,CAAC,CAAA;YACjC,MAAM,gBAAgB,GAAG,OAAO,CAAC,CAAC,CAAC,CAAA;YACnC,MAAM,cAAc,GAAG,OAAO,CAAC,CAAC,CAAC,CAAA;YAEjC,IAAI,gBAAgB,GAAG,kBAAkB,EAAE,CAAC;gBAC3C,0BAA0B,CAAC,kBAAkB,EAAE,gBAAgB,CAAC,CAAA;YACjE,CAAC;YAED,YAAY,CAAC,YAAY,CAAC,IAAI,EAAE,gBAAgB,EAAE,cAAc,EAAE,KAAK,CAAC,CAAA;YAExE,kBAAkB,GAAG,cAAc,CAAA;QACpC,CAAC;QAED,0BAA0B,CAAC,kBAAkB,EAAE,IAAI,CAAC,MAAM,CAAC,CAAA;IAC5D,CAAC;IAED,IAAI,OAAO,CAAC,6BAA6B,EAAE,CAAC;QAC3C,YAAY,GAAG,MAAM,yBAAyB,CAAC,IAAI,EAAE,YAAY,CAAC,CAAA;IACnE,CAAC;IAED,OAAO,YAAY,CAAA;AACpB,CAAC;AAED,KAAK,UAAU,yBAAyB,CAAC,cAAsB,EAAE,YAA0B;IAC1F,MAAM,eAAe,GAAG,MAAM,kBAAkB,EAAE,CAAA;IAElD,IAAI,eAAe,KAAK,SAAS,EAAE,CAAC;QACnC,OAAO,YAAY,CAAA;IACpB,CAAC;IAED,IAAI,cAAc,GAAG,KAAK,CAAA;IAE1B,MAAM,eAAe,GAAG,IAAI,YAAY,EAAE,CAAA;IAE1C,KAAK,IAAI,SAAS,GAAG,CAAC,EAAE,SAAS,GAAG,YAAY,CAAC,MAAM,EAAE,SAAS,EAAE,EAAE,CAAC;QACtE,MAAM,SAAS,GAAG,YAAY,CAAC,OAAO,CAAC,SAAS,CAAC,CAAA;QACjD,MAAM,eAAe,GAAG,SAAS,CAAC,WAAW,CAAA;QAE7C,MAAM,IAAI,GAAG,YAAY,CAAC,SAAS,CAAC,SAAS,CAAC,CAAA;QAE9C,IAAI,yBAAyB,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;YAC1C,IAAI,CAAC,cAAc,EAAE,CAAC;gBACrB,MAAM,eAAe,CAAC,UAAU,EAAE,CAAA;gBAElC,cAAc,GAAG,IAAI,CAAA;YACtB,CAAC;YAED,MAAM,UAAU,GAAG,CAAC,GAAG,eAAe,CAAC,uBAAuB,CAAC,IAAI,CAAC,CAAC,CAAA;YAErE,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,UAAU,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;gBAChD,eAAe,CAAC,YAAY,CAC3B,cAAc,EACd,eAAe,GAAG,UAAU,CAAC,CAAC,CAAC,EAC/B,eAAe,GAAG,UAAU,CAAC,CAAC,GAAG,CAAC,CAAC,EACnC,KAAK,CACL,CAAA;YACF,CAAC;QACF,CAAC;aAAM,CAAC;YACP,eAAe,CAAC,YAAY,CAAC,cAAc,EAAE,SAAS,CAAC,WAAW,EAAE,SAAS,CAAC,SAAS,EAAE,SAAS,CAAC,aAAa,CAAC,CAAA;QAClH,CAAC;IACF,CAAC;IAED,OAAO,eAAe,CAAA;AACvB,CAAC;AAED,SAAS,iCAAiC,CAAC,OAA4B;IACtE,MAAM,uBAAuB,GAAG,gBAAgB,CAAC,OAAO,CAAC,QAAQ,IAAI,EAAE,CAAC,IAAI,EAAE,CAAA;IAC9E,MAAM,8BAA8B,GAAG,wCAAwC,CAAC,OAAO,CAAC,QAAQ,IAAI,EAAE,CAAC,IAAI,EAAE,CAAA;IAC7G,MAAM,6CAA6C,GAAG,8BAA8B,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,UAAU,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC,CAAA;IACzH,MAAM,kBAAkB,GAAG,OAAO,CAAC,kBAAkB,IAAI,EAAE,CAAA;IAE3D,IAAI,YAAY,GAAG;QAClB,GAAG,kBAAkB;QACrB,GAAG,uBAAuB;QAC1B,GAAG,8BAA8B;QACjC,GAAG,6CAA6C;QAChD,GAAG,gBAAgB;QACnB,GAAG,eAAe;KAClB,CAAA;IAED,MAAM,WAAW,GAAG,wBAAwB,CAAC;QAC5C,GAAG,YAAY;QACf,GAAG,YAAY,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,iBAAiB,EAAE,CAAC;QACrD,GAAG,YAAY,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,iBAAiB,EAAE,CAAC;KACrD,CAAC,CAAA;IAEF,MAAM,kBAAkB,GAAG,WAAW,CAAC,WAAW,EAAE,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC,CAAA;IAErE,OAAO,kBAAkB,CAAA;AAC1B,CAAC;AAED,uDAAuD;AACvD,MAAM,UAAU,wCAAwC,CAAC,YAA0B,EAAE,UAAkB;IACtG,MAAM,2BAA2B,GAAG,IAAI,GAAG,EAAkB,CAAA;IAE7D,MAAM,2BAA2B,GAAG,IAAI,YAAY,EAAE,CAAA;IAEtD,SAAS,0BAA0B,CAAC,SAAiB,EAAE,iBAAyB;QAC/E,IAAI,UAAU,GAAG,iBAAiB,CAAA;QAElC,kFAAkF;QAClF,KAAK,MAAM,IAAI,IAAI,SAAS,EAAE,CAAC;YAC9B,MAAM,aAAa,GAAG,UAAU,GAAG,IAAI,CAAC,MAAM,CAAA;YAE9C,MAAM,SAAS,GAAG,2BAA2B,CAAC,SAAS,CAAA;YAEvD,IAAI,IAAI,KAAK,GAAG,IAAI,SAAS,IAAI,SAAS,CAAC,aAAa,IAAI,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC,KAAK,GAAG,EAAE,CAAC;gBACvF,2BAA2B,CAAC,SAAS,CAAC,IAAI,IAAI,GAAG,CAAA;gBACjD,2BAA2B,CAAC,SAAS,CAAC,SAAS,GAAG,aAAa,CAAA;YAChE,CAAC;iBAAM,CAAC;gBACP,2BAA2B,CAAC,YAAY,CAAC,UAAU,EAAE,UAAU,EAAE,aAAa,EAAE,IAAI,CAAC,CAAA;YACtF,CAAC;YAED,UAAU,GAAG,aAAa,CAAA;QAC3B,CAAC;IACF,CAAC;IAED,KAAK,IAAI,SAAS,GAAG,CAAC,EAAE,SAAS,GAAG,YAAY,CAAC,MAAM,EAAE,SAAS,EAAE,EAAE,CAAC;QACtE,MAAM,SAAS,GAAG,YAAY,CAAC,UAAU,CAAC,SAAS,CAAC,CAAA;QACpD,MAAM,eAAe,GAAG,SAAS,CAAC,WAAW,CAAA;QAE7C,MAAM,qBAAqB,GAAG,SAAS,GAAG,CAAC,CAAC,CAAC,CAAC,YAAY,CAAC,OAAO,CAAC,SAAS,GAAG,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,CAAA;QAE/F,8FAA8F;QAC9F,IAAI,qBAAqB,KAAK,eAAe,EAAE,CAAC;YAC/C,MAAM,SAAS,GAAG,UAAU,CAAC,SAAS,CAAC,qBAAqB,EAAE,eAAe,CAAC,CAAA;YAE9E,0BAA0B,CAAC,SAAS,EAAE,qBAAqB,CAAC,CAAA;QAC7D,CAAC;QAED,2BAA2B,CAAC,OAAO,CAAC,IAAI,CAAC,SAAS,CAAC,CAAA;QACnD,2BAA2B,CAAC,GAAG,CAAC,2BAA2B,CAAC,MAAM,GAAG,CAAC,EAAE,SAAS,CAAC,CAAA;QAElF,oEAAoE;QACpE,IAAI,SAAS,KAAK,YAAY,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAC3C,IAAI,UAAU,CAAC,MAAM,KAAK,SAAS,CAAC,SAAS,EAAE,CAAC;gBAC/C,MAAM,SAAS,GAAG,UAAU,CAAC,SAAS,CAAC,SAAS,CAAC,SAAS,EAAE,UAAU,CAAC,MAAM,CAAC,CAAA;gBAE9E,0BAA0B,CAAC,SAAS,EAAE,SAAS,CAAC,SAAS,CAAC,CAAA;YAC3D,CAAC;QACF,CAAC;IACF,CAAC;IAED,OAAO,EAAE,2BAA2B,EAAE,2BAA2B,EAAE,CAAA;AACpE,CAAC;AAED,KAAK,UAAU,kBAAkB;IAChC,IAAI,CAAC;QACJ,MAAM,eAAe,GAAG,MAAM,MAAM,CAAC,mCAAmC,CAAC,CAAA;QAEzE,OAAO,eAAe,CAAA;IACvB,CAAC;IAAC,MAAM,CAAC;QACR,OAAO,SAAS,CAAA;IACjB,CAAC;AACF,CAAC;AAOD,MAAM,OAAO,YAAY;IACxB,SAAS,CAAO;IAChB,YAAY,CAAc;IAE1B,YAAY,SAAgB,EAAE,YAA0B;QACvD,IAAI,CAAC,SAAS,GAAG,SAAS,CAAA;QAC1B,IAAI,CAAC,YAAY,GAAG,YAAY,CAAA;IACjC,CAAC;IAED,IAAI,IAAI;QACP,OAAO,IAAI,CAAC,YAAY,CAAC,IAAI,CAAA;IAC9B,CAAC;IAED,IAAI,SAAS;QACZ,OAAO;YACN,KAAK,EAAE,IAAI,CAAC,YAAY,CAAC,UAAU,CAAC,WAAW;YAC/C,GAAG,EAAE,IAAI,CAAC,YAAY,CAAC,SAAS,CAAC,SAAS;SAC1C,CAAA;IACF,CAAC;CACD;AAED,MAAM,OAAO,QAAS,SAAQ,YAAY;IACzC,OAAO,GAAa,EAAE,CAAA;CACtB;AAED,MAAM,OAAO,MAAO,SAAQ,YAAY;CACvC;AAaD,MAAM,CAAC,MAAM,0BAA0B,GAAwB;IAC9D,QAAQ,EAAE,EAAE;IACZ,kBAAkB,EAAE,EAAE;IACtB,6BAA6B,EAAE,IAAI;CACnC,CAAA"}

package/dist/WordSequence.d.ts

@@ -7,6 +7,7 @@ export declare class WordSequence {
     getEntryRange(startIndex: number, endIndex: number): WordEntry[];
     iterateEntryRange(startIndex: number, endIndex: number): Generator<WordEntry, void, unknown>;
     getEntryAt(index: number): WordEntry;
+    clone(): WordSequence;
     slice(startIndex: number, endIndex: number): WordSequence;
     get words(): string[];
     get firstWord(): string;

package/dist/WordSequence.js

@@ -31,6 +31,9 @@ export class WordSequence {
     getEntryAt(index) {
         return this.entries[index];
     }
+    clone() {
+        return this.slice(0, this.length);
+    }
     slice(startIndex, endIndex) {
         const slicedSequence = new WordSequence();
         slicedSequence.entries = this.entries.slice(startIndex, endIndex);

package/dist/WordSequence.js.map

@@ -1 +1 @@
-{"version":3,"file":"WordSequence.js","sourceRoot":"","sources":["../src/WordSequence.ts"],"names":[],"mappings":"AAAA,MAAM,OAAO,YAAY;IACxB,OAAO,GAAgB,EAAE,CAAA;IAEzB,YAAY,CAAC,cAAsB,EAAE,WAAmB,EAAE,SAAiB,EAAE,aAAsB;QAClG,MAAM,QAAQ,GAAG,cAAc,CAAC,SAAS,CAAC,WAAW,EAAE,SAAS,CAAC,CAAA;QAEjE,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC;YACjB,IAAI,EAAE,QAAQ;YACd,WAAW;YACX,SAAS;YACT,aAAa;SACb,CAAC,CAAA;IACH,CAAC;IAED,YAAY,CAAC,UAAkB,EAAE,QAAgB;QAChD,OAAO,CAAC,GAAG,IAAI,CAAC,gBAAgB,CAAC,UAAU,EAAE,QAAQ,CAAC,CAAC,CAAA;IACxD,CAAC;IAED,CAAC,gBAAgB,CAAC,UAAkB,EAAE,QAAgB;QACrD,KAAK,IAAI,CAAC,GAAG,UAAU,EAAE,CAAC,GAAG,QAAQ,EAAE,CAAC,EAAE,EAAE,CAAC;YAC5C,MAAM,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,CAAA;QACxB,CAAC;IACF,CAAC;IAED,SAAS,CAAC,KAAa;QACtB,OAAO,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,IAAI,CAAA;IAChC,CAAC;IAED,aAAa,CAAC,UAAkB,EAAE,QAAgB;QACjD,OAAO,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,UAAU,EAAE,QAAQ,CAAC,CAAA;IAChD,CAAC;IAED,CAAC,iBAAiB,CAAC,UAAkB,EAAE,QAAgB;QACtD,KAAK,IAAI,CAAC,GAAG,UAAU,EAAE,CAAC,GAAG,QAAQ,EAAE,CAAC,EAAE,EAAE,CAAC;YAC5C,MAAM,IAAI,CAAC,UAAU,CAAC,CAAC,CAAC,CAAA;QACzB,CAAC;IACF,CAAC;IAED,UAAU,CAAC,KAAa;QACvB,OAAO,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,CAAA;IAC3B,CAAC;IAED,KAAK,CAAC,UAAkB,EAAE,QAAgB;QACzC,MAAM,cAAc,GAAG,IAAI,YAAY,EAAE,CAAA;QAEzC,cAAc,CAAC,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,UAAU,EAAE,QAAQ,CAAC,CAAA;QAEjE,OAAO,cAAc,CAAA;IACtB,CAAC;IAED,IAAI,KAAK;QACR,OAAO,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC,KAAK,CAAC,IAAI,CAAC,CAAA;IAC7C,CAAC;IAED,IAAI,SAAS;QACZ,OAAO,IAAI,CAAC,UAAU,EAAE,IAAI,CAAA;IAC7B,CAAC;IAED,IAAI,QAAQ;QACX,OAAO,IAAI,CAAC,SAAS,EAAE,IAAI,CAAA;IAC5B,CAAC;IAED,IAAI,UAAU;QACb,OAAO,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,CAAA;IACvB,CAAC;IAED,IAAI,SAAS;QACZ,OAAO,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC,CAAA;IACrC,CAAC;IAED,IAAI,qBAAqB;QACxB,OAAO,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,KAAK,CAAC,aAAa,KAAK,KAAK,CAAC,CAAA;IACnE,CAAC;IAED,IAAI,mBAAmB;QACtB,OAAO,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,KAAK,CAAC,aAAa,KAAK,KAAK,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC,KAAK,CAAC,IAAI,CAAC,CAAA;IAC5F,CAAC;IAED,IAAI,IAAI;QACP,OAAO,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAA;IAC3B,CAAC;IAED,IAAI,MAAM;QACT,OAAO,IAAI,CAAC,OAAO,CAAC,MAAM,CAAA;IAC3B,CAAC;CACD"}
+{"version":3,"file":"WordSequence.js","sourceRoot":"","sources":["../src/WordSequence.ts"],"names":[],"mappings":"AAAA,MAAM,OAAO,YAAY;IACxB,OAAO,GAAgB,EAAE,CAAA;IAEzB,YAAY,CAAC,cAAsB,EAAE,WAAmB,EAAE,SAAiB,EAAE,aAAsB;QAClG,MAAM,QAAQ,GAAG,cAAc,CAAC,SAAS,CAAC,WAAW,EAAE,SAAS,CAAC,CAAA;QAEjE,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC;YACjB,IAAI,EAAE,QAAQ;YACd,WAAW;YACX,SAAS;YACT,aAAa;SACb,CAAC,CAAA;IACH,CAAC;IAED,YAAY,CAAC,UAAkB,EAAE,QAAgB;QAChD,OAAO,CAAC,GAAG,IAAI,CAAC,gBAAgB,CAAC,UAAU,EAAE,QAAQ,CAAC,CAAC,CAAA;IACxD,CAAC;IAED,CAAC,gBAAgB,CAAC,UAAkB,EAAE,QAAgB;QACrD,KAAK,IAAI,CAAC,GAAG,UAAU,EAAE,CAAC,GAAG,QAAQ,EAAE,CAAC,EAAE,EAAE,CAAC;YAC5C,MAAM,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,CAAA;QACxB,CAAC;IACF,CAAC;IAED,SAAS,CAAC,KAAa;QACtB,OAAO,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,IAAI,CAAA;IAChC,CAAC;IAED,aAAa,CAAC,UAAkB,EAAE,QAAgB;QACjD,OAAO,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,UAAU,EAAE,QAAQ,CAAC,CAAA;IAChD,CAAC;IAED,CAAC,iBAAiB,CAAC,UAAkB,EAAE,QAAgB;QACtD,KAAK,IAAI,CAAC,GAAG,UAAU,EAAE,CAAC,GAAG,QAAQ,EAAE,CAAC,EAAE,EAAE,CAAC;YAC5C,MAAM,IAAI,CAAC,UAAU,CAAC,CAAC,CAAC,CAAA;QACzB,CAAC;IACF,CAAC;IAED,UAAU,CAAC,KAAa;QACvB,OAAO,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,CAAA;IAC3B,CAAC;IAED,KAAK;QACJ,OAAO,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,IAAI,CAAC,MAAM,CAAC,CAAA;IAClC,CAAC;IAED,KAAK,CAAC,UAAkB,EAAE,QAAgB;QACzC,MAAM,cAAc,GAAG,IAAI,YAAY,EAAE,CAAA;QAEzC,cAAc,CAAC,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,UAAU,EAAE,QAAQ,CAAC,CAAA;QAEjE,OAAO,cAAc,CAAA;IACtB,CAAC;IAED,IAAI,KAAK;QACR,OAAO,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC,KAAK,CAAC,IAAI,CAAC,CAAA;IAC7C,CAAC;IAED,IAAI,SAAS;QACZ,OAAO,IAAI,CAAC,UAAU,EAAE,IAAI,CAAA;IAC7B,CAAC;IAED,IAAI,QAAQ;QACX,OAAO,IAAI,CAAC,SAAS,EAAE,IAAI,CAAA;IAC5B,CAAC;IAED,IAAI,UAAU;QACb,OAAO,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,CAAA;IACvB,CAAC;IAED,IAAI,SAAS;QACZ,OAAO,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC,CAAA;IACrC,CAAC;IAED,IAAI,qBAAqB;QACxB,OAAO,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,KAAK,CAAC,aAAa,KAAK,KAAK,CAAC,CAAA;IACnE,CAAC;IAED,IAAI,mBAAmB;QACtB,OAAO,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,KAAK,CAAC,aAAa,KAAK,KAAK,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC,KAAK,CAAC,IAAI,CAAC,CAAA;IAC5F,CAAC;IAED,IAAI,IAAI;QACP,OAAO,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAA;IAC3B,CAAC;IAED,IAAI,MAAM;QACT,OAAO,IAAI,CAAC,OAAO,CAAC,MAAM,CAAA;IAC3B,CAAC;CACD"}

package/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "@echogarden/text-segmentation",
-	"version": "0.1.1",
+	"version": "0.2.1",
 	"description": "A library for multilingual word, phrase and sentence segmentation.",
 	"author": "Rotem Dan",
 	"license": "MIT",
@@ -50,6 +50,6 @@
 		}
 	},
 	"devDependencies": {
-		"@types/node": "^22.15.12"
+		"@types/node": "^22.15.16"
 	}
 }

package/src/Test.ts

@@ -1,4 +1,4 @@
-import { segmentText, splitToWords } from "./TextSegmentation.js"
+import { addMissingPunctuationWordsToWordSequence, SegmentationResult, segmentText, splitToWords, WordSequence } from "./TextSegmentation.js"
 import { Timer } from "./utilities/Timer.js"
 
 const log = console.log
@@ -14,15 +14,17 @@ async function test1() {
 
 	const timer = new Timer()
 
-	const result = await segmentText(text, {
-		language: 'en',
-		customSuppressions: [],
-		enableEastAsianPostprocessing: true
-	})
+	let result: SegmentationResult
 
-	//const json = JSON.stringify(result.sentences)
+	for (let i = 0; i < 3; i++) {
+		result = await segmentText(text, {
+			language: 'en',
+			customSuppressions: [],
+			enableEastAsianPostprocessing: true
+		})
 
-	timer.logAndRestart(`Total execution time`)
+		timer.logAndRestart(`Total execution time`)
+	}
 
 	log('')
 
@@ -30,8 +32,8 @@ async function test1() {
 
 	let segmentedText = ''
 
-	for (let sentenceIndex = 0; sentenceIndex < result.sentences.length; sentenceIndex++) {
-		const sentence = result.sentences[sentenceIndex]
+	for (let sentenceIndex = 0; sentenceIndex < result!.sentences.length; sentenceIndex++) {
+		const sentence = result!.sentences[sentenceIndex]
 		const phrases = sentence.phrases
 
 		for (let phraseIndex = 0; phraseIndex < phrases.length; phraseIndex++) {
@@ -44,7 +46,7 @@ async function test1() {
 			}
 		}
 
-		if (sentenceIndex < result.sentences.length - 1) {
+		if (sentenceIndex < result!.sentences.length - 1) {
 			segmentedText += `\n${ '='.repeat(100) } \n`
 		}
 	}
@@ -52,4 +54,19 @@ async function test1() {
 	writeFileSync('out/segmented.txt', segmentedText)
 }
 
+async function test2() {
+	const text = `  Hello,  how are you today                ??    `
+
+	const words = await splitToWords(text)
+
+	const nonPunctuationWordEntries = words.nonPunctuationEntries
+	const nonPunctuationWordSequence = new WordSequence()
+	nonPunctuationWordSequence.entries = nonPunctuationWordEntries
+
+	const extendedWords = addMissingPunctuationWordsToWordSequence(nonPunctuationWordSequence, text)
+
+	const x = 0
+}
+
 test1()
+//test2()

package/src/TextSegmentation.ts

@@ -113,63 +113,55 @@ export async function segmentWordSequence(wordSequence: WordSequence) {
 	return result
 }
 
+const cachedWordSplitterRegExps = new Map<string, RegExp>()
+
 export async function splitToWords(text: string, options?: SegmentationOptions) {
-	options = { ...defaultSegmentationOptions, ...(options ?? {}) }
+	if (!options) {
+		options = {}
+	}
 
-	const cldrSuppressionsForLang = cldrSuppressions[options.language ?? ''] ?? []
-	const contractionSuppressionsForLang = leadingApostropheContractionSuppressions[options.language ?? ''] ?? []
-	const contractionSuppressionsForLangWithSingleQuote = contractionSuppressionsForLang.map(str => str.replaceAll(`'`, `’`))
-	const customSuppressions = options.customSuppressions ?? []
+	options = { ...defaultSegmentationOptions, ...options }
 
-	let suppressions = [
-		...customSuppressions,
-		...cldrSuppressionsForLang,
-		...contractionSuppressionsForLang,
-		...contractionSuppressionsForLangWithSingleQuote,
-		...nounSuppressions,
-		...tldSuppressions,
-	]
+	const optionsAsJson = JSON.stringify(options)
 
-	const wordPattern = buildWordSplitterPattern([
-		...suppressions,
-		...suppressions.map(word => word.toLocaleLowerCase()),
-		...suppressions.map(word => word.toLocaleUpperCase()),
-	])
+	let wordSplitterRegExp = cachedWordSplitterRegExps.get(optionsAsJson)
 
-	const wordSplitterRegExp = buildRegExp(wordPattern, { global: true })
+	if (!wordSplitterRegExp) {
+		wordSplitterRegExp = buildWordSplitterRegExpForOptions(options)
 
-	//console.log(`Encoded pattern: ${wordSplitterRegExp.source}\n`)
+		cachedWordSplitterRegExps.set(optionsAsJson, wordSplitterRegExp)
+	}
 
 	let wordSequence = new WordSequence()
 
-	function addNonWordsBetween(startOffset: number, endOffset: number) {
-		const nonWordSubstring = text.substring(startOffset, endOffset)
+	function addPunctuationWordsBetween(startOffset: number, endOffset: number) {
+		const punctuationWordSubstring = text.substring(startOffset, endOffset)
 
 		let charOffset = startOffset
-		let nonwordStartOffset = startOffset
+		let punctuationWordStartOffset = startOffset
 
-		function addNonWordIfNeeded() {
-			if (charOffset > nonwordStartOffset) {
-				wordSequence.addWordEntry(text, nonwordStartOffset, charOffset, true)
-				nonwordStartOffset = charOffset
+		function addPunctuationWordIfNeeded() {
+			if (charOffset > punctuationWordStartOffset) {
+				wordSequence.addWordEntry(text, punctuationWordStartOffset, charOffset, true)
+				punctuationWordStartOffset = charOffset
 			}
 		}
 
-		for (const char of nonWordSubstring) {
+		for (const char of punctuationWordSubstring) {
 			if (char === ' ') {
 				charOffset += 1
 
 				continue
 			}
 
-			addNonWordIfNeeded()
+			addPunctuationWordIfNeeded()
 
 			charOffset += char.length
 
-			addNonWordIfNeeded()
+			addPunctuationWordIfNeeded()
 		}
 
-		addNonWordIfNeeded()
+		addPunctuationWordIfNeeded()
 	}
 
 	const wordMatches = text.matchAll(wordSplitterRegExp)
@@ -183,7 +175,7 @@ export async function splitToWords(text: string, options?: SegmentationOptions)
 			const matchEndOffset = offsets[1]
 
 			if (matchStartOffset > lastMatchEndOffset) {
-				addNonWordsBetween(lastMatchEndOffset, matchStartOffset)
+				addPunctuationWordsBetween(lastMatchEndOffset, matchStartOffset)
 			}
 
 			wordSequence.addWordEntry(text, matchStartOffset, matchEndOffset, false)
@@ -191,7 +183,7 @@ export async function splitToWords(text: string, options?: SegmentationOptions)
 			lastMatchEndOffset = matchEndOffset
 		}
 
-		addNonWordsBetween(lastMatchEndOffset, text.length)
+		addPunctuationWordsBetween(lastMatchEndOffset, text.length)
 	}
 
 	if (options.enableEastAsianPostprocessing) {
@@ -243,6 +235,87 @@ async function postprocessEastAsianWords(containingText: string, wordSequence: W
 	return newWordSequence
 }
 
+function buildWordSplitterRegExpForOptions(options: SegmentationOptions) {
+	const cldrSuppressionsForLang = cldrSuppressions[options.language ?? ''] ?? []
+	const contractionSuppressionsForLang = leadingApostropheContractionSuppressions[options.language ?? ''] ?? []
+	const contractionSuppressionsForLangWithSingleQuote = contractionSuppressionsForLang.map(str => str.replaceAll(`'`, `’`))
+	const customSuppressions = options.customSuppressions ?? []
+
+	let suppressions = [
+		...customSuppressions,
+		...cldrSuppressionsForLang,
+		...contractionSuppressionsForLang,
+		...contractionSuppressionsForLangWithSingleQuote,
+		...nounSuppressions,
+		...tldSuppressions,
+	]
+
+	const wordPattern = buildWordSplitterPattern([
+		...suppressions,
+		...suppressions.map(word => word.toLocaleLowerCase()),
+		...suppressions.map(word => word.toLocaleUpperCase()),
+	])
+
+	const wordSplitterRegExp = buildRegExp(wordPattern, { global: true })
+
+	return wordSplitterRegExp
+}
+
+// Add any missing punctuation words to a word sequence
+export function addMissingPunctuationWordsToWordSequence(wordSequence: WordSequence, sourceText: string) {
+	const originalWordsReverseMapping = new Map<number, number>()
+
+	const wordSequenceWithPunctuation = new WordSequence()
+
+	function addWordEntriesForTextSlice(textSlice: string, initialCharOffset: number) {
+		let charOffset = initialCharOffset
+
+		// Add entry for every codepoint (this will correctly treat characters beyond BMP)
+		for (const char of textSlice) {
+			const charEndOffset = charOffset + char.length
+
+			const lastEntry = wordSequenceWithPunctuation.lastEntry
+
+			if (char === ' ' && lastEntry && lastEntry.isPunctuation && lastEntry.text[0] === ' ') {
+				wordSequenceWithPunctuation.lastEntry.text += ' '
+				wordSequenceWithPunctuation.lastEntry.endOffset = charEndOffset
+			} else {
+				wordSequenceWithPunctuation.addWordEntry(sourceText, charOffset, charEndOffset, true)
+			}
+
+			charOffset = charEndOffset
+		}
+	}
+
+	for (let wordIndex = 0; wordIndex < wordSequence.length; wordIndex++) {
+		const wordEntry = wordSequence.getEntryAt(wordIndex)
+		const wordStartOffset = wordEntry.startOffset
+
+		const previousWordEndOffset = wordIndex > 0 ? wordSequence.entries[wordIndex - 1].endOffset : 0
+
+		// Add entries for any punctuation characters between the current and previous word (or start)
+		if (previousWordEndOffset !== wordStartOffset) {
+			const textSlice = sourceText.substring(previousWordEndOffset, wordStartOffset)
+
+			addWordEntriesForTextSlice(textSlice, previousWordEndOffset)
+		}
+
+		wordSequenceWithPunctuation.entries.push(wordEntry)
+		originalWordsReverseMapping.set(wordSequenceWithPunctuation.length - 1, wordIndex)
+
+		// If last word, add entries for any trailing punctuation characters
+		if (wordIndex === wordSequence.length - 1) {
+			if (sourceText.length !== wordEntry.endOffset) {
+				const textSlice = sourceText.substring(wordEntry.endOffset, sourceText.length)
+
+				addWordEntriesForTextSlice(textSlice, wordEntry.endOffset)
+			}
+		}
+	}
+
+	return { wordSequenceWithPunctuation, originalWordsReverseMapping }
+}
+
 async function getIcuSegmentation() {
 	try {
 		const icuSegmentation = await import('@echogarden/icu-segmentation-wasm')

package/src/WordSequence.ts

@@ -40,6 +40,10 @@ export class WordSequence {
 		return this.entries[index]
 	}
 
+	clone() {
+		return this.slice(0, this.length)
+	}
+
 	slice(startIndex: number, endIndex: number) {
 		const slicedSequence = new WordSequence()