@echogarden/text-segmentation 0.1.0

package/dist/TextSegmentation.js

@@ -0,0 +1,208 @@
+import { buildRegExp } from 'regexp-composer';
+import { buildWordOrNumberPattern as buildWordSplitterPattern, phraseSeparatorRegExp, sentenceSeparatorTrailingPunctuationRegExp, sentenceSeparatorRegExp, whitespacePatternRegExp } from './Patterns.js';
+import { cldrSuppressions, leadingApostropheContractionSuppressions, nounSuppressions, tldSuppressions } from './Suppressions.js';
+import { eastAsianCharRangesRegExp } from './EastAsianCharacterPatterns.js';
+import { WordSequence } from './WordSequence.js';
+export { cldrSuppressions } from './Suppressions.js';
+export async function segmentText(text, options) {
+    options = { ...defaultSegmentationOptions, ...(options ?? {}) };
+    const wordSequence = await splitToWords(text, options);
+    return segmentWordSequence(wordSequence);
+}
+export async function segmentWordSequence(wordSequence) {
+    const sentenceWordRanges = [];
+    let sentenceStartWordOffset = 0;
+    for (let wordIndex = 0; wordIndex < wordSequence.length; wordIndex++) {
+        const word = wordSequence.getWordAt(wordIndex);
+        if (sentenceSeparatorRegExp.test(word)) {
+            while (wordIndex < wordSequence.length - 1) {
+                const nextWord = wordSequence.getWordAt(wordIndex + 1);
+                if (!sentenceSeparatorTrailingPunctuationRegExp.test(nextWord)) {
+                    break;
+                }
+                wordIndex += 1;
+            }
+            sentenceWordRanges.push({
+                start: sentenceStartWordOffset,
+                end: wordIndex + 1
+            });
+            sentenceStartWordOffset = wordIndex + 1;
+        }
+    }
+    if (sentenceStartWordOffset < wordSequence.length) {
+        sentenceWordRanges.push({ start: sentenceStartWordOffset, end: wordSequence.length });
+    }
+    const sentences = [];
+    for (const wordRange of sentenceWordRanges) {
+        const sentenceWordSequence = wordSequence.slice(wordRange.start, wordRange.end);
+        sentences.push(new Sentence(wordRange, sentenceWordSequence));
+    }
+    for (const sentence of sentences) {
+        const phraseWordRanges = [];
+        let sentenceEndWordOffset = sentence.wordRange.end;
+        let phraseStartWordOffset = sentence.wordRange.start;
+        for (let wordIndex = phraseStartWordOffset; wordIndex < sentenceEndWordOffset; wordIndex++) {
+            const currentWord = wordSequence.getWordAt(wordIndex);
+            if (phraseSeparatorRegExp.test(currentWord)) {
+                let whitespaceSeenOnce = false;
+                while (wordIndex < sentenceEndWordOffset - 1) {
+                    const nextWord = wordSequence.getWordAt(wordIndex + 1);
+                    if (!sentenceSeparatorTrailingPunctuationRegExp.test(nextWord)) {
+                        break;
+                    }
+                    if (whitespacePatternRegExp.test(nextWord)) {
+                        whitespaceSeenOnce = true;
+                    }
+                    if (nextWord === '"' && whitespaceSeenOnce) {
+                        break;
+                    }
+                    wordIndex += 1;
+                }
+                phraseWordRanges.push({
+                    start: phraseStartWordOffset,
+                    end: wordIndex + 1
+                });
+                phraseStartWordOffset = wordIndex + 1;
+            }
+        }
+        if (phraseStartWordOffset < sentenceEndWordOffset) {
+            phraseWordRanges.push({ start: phraseStartWordOffset, end: sentenceEndWordOffset });
+        }
+        for (const wordRange of phraseWordRanges) {
+            const phraseWordSequence = wordSequence.slice(wordRange.start, wordRange.end);
+            sentence.phrases.push(new Phrase(wordRange, phraseWordSequence));
+        }
+    }
+    const result = {
+        wordSequence,
+        sentences,
+    };
+    return result;
+}
+export async function splitToWords(text, options) {
+    options = { ...defaultSegmentationOptions, ...(options ?? {}) };
+    const cldrSuppressionsForLang = cldrSuppressions[options.language ?? ''] ?? [];
+    const contractionSuppressionsForLang = leadingApostropheContractionSuppressions[options.language ?? ''] ?? [];
+    const contractionSuppressionsForLangWithSingleQuote = contractionSuppressionsForLang.map(str => str.replaceAll(`'`, `’`));
+    const customSuppressions = options.customSuppressions ?? [];
+    let suppressions = [
+        ...customSuppressions,
+        ...cldrSuppressionsForLang,
+        ...contractionSuppressionsForLang,
+        ...contractionSuppressionsForLangWithSingleQuote,
+        ...nounSuppressions,
+        ...tldSuppressions,
+    ];
+    const wordPattern = buildWordSplitterPattern([
+        ...suppressions,
+        ...suppressions.map(word => word.toLocaleLowerCase()),
+        ...suppressions.map(word => word.toLocaleUpperCase()),
+    ]);
+    const wordSplitterRegExp = buildRegExp(wordPattern, { global: true });
+    //console.log(`Encoded pattern: ${wordSplitterRegExp.source}\n`)
+    let wordSequence = new WordSequence();
+    function addNonWordsBetween(startOffset, endOffset) {
+        const nonWordSubstring = text.substring(startOffset, endOffset);
+        let charOffset = startOffset;
+        let nonwordStartOffset = startOffset;
+        function addNonWordIfNeeded() {
+            if (charOffset > nonwordStartOffset) {
+                wordSequence.addWordEntry(text, nonwordStartOffset, charOffset, true);
+                nonwordStartOffset = charOffset;
+            }
+        }
+        for (const char of nonWordSubstring) {
+            if (char === ' ') {
+                charOffset += 1;
+                continue;
+            }
+            addNonWordIfNeeded();
+            charOffset += char.length;
+            addNonWordIfNeeded();
+        }
+        addNonWordIfNeeded();
+    }
+    const wordMatches = text.matchAll(wordSplitterRegExp);
+    let lastMatchEndOffset = 0;
+    if (wordMatches) {
+        for (const match of wordMatches) {
+            const offsets = match.indices[0];
+            const matchStartOffset = offsets[0];
+            const matchEndOffset = offsets[1];
+            if (matchStartOffset > lastMatchEndOffset) {
+                addNonWordsBetween(lastMatchEndOffset, matchStartOffset);
+            }
+            wordSequence.addWordEntry(text, matchStartOffset, matchEndOffset, false);
+            lastMatchEndOffset = matchEndOffset;
+        }
+        addNonWordsBetween(lastMatchEndOffset, text.length);
+    }
+    if (options.enableEastAsianPostprocessing) {
+        wordSequence = await postprocessEastAsianWords(text, wordSequence);
+    }
+    return wordSequence;
+}
+async function postprocessEastAsianWords(containingText, wordSequence) {
+    const icuSegmentation = await getIcuSegmentation();
+    if (icuSegmentation === undefined) {
+        return wordSequence;
+    }
+    let icuInitialized = false;
+    const newWordSequence = new WordSequence();
+    for (let wordIndex = 0; wordIndex < wordSequence.length; wordIndex++) {
+        const wordEntry = wordSequence.entries[wordIndex];
+        const wordStartOffset = wordEntry.startOffset;
+        const word = wordSequence.getWordAt(wordIndex);
+        if (eastAsianCharRangesRegExp.test(word)) {
+            if (!icuInitialized) {
+                await icuSegmentation.initialize();
+                icuInitialized = true;
+            }
+            const wordBreaks = [...icuSegmentation.createWordBreakIterator(word)];
+            for (let i = 0; i < wordBreaks.length - 1; i++) {
+                newWordSequence.addWordEntry(containingText, wordStartOffset + wordBreaks[i], wordStartOffset + wordBreaks[i + 1], false);
+            }
+        }
+        else {
+            newWordSequence.addWordEntry(containingText, wordEntry.startOffset, wordEntry.endOffset, wordEntry.isPunctuation);
+        }
+    }
+    return newWordSequence;
+}
+async function getIcuSegmentation() {
+    try {
+        const icuSegmentation = await import('@echogarden/icu-segmentation-wasm');
+        return icuSegmentation;
+    }
+    catch {
+        return undefined;
+    }
+}
+export class TextFragment {
+    wordRange;
+    wordSequence;
+    constructor(wordRange, wordSequence) {
+        this.wordRange = wordRange;
+        this.wordSequence = wordSequence;
+    }
+    get text() {
+        return this.wordSequence.text;
+    }
+    get charRange() {
+        return {
+            start: this.wordSequence.firstEntry.startOffset,
+            end: this.wordSequence.lastEntry.endOffset
+        };
+    }
+}
+export class Sentence extends TextFragment {
+    phrases = [];
+}
+export class Phrase extends TextFragment {
+}
+export const defaultSegmentationOptions = {
+    language: '',
+    customSuppressions: [],
+    enableEastAsianPostprocessing: true,
+};
+//# sourceMappingURL=TextSegmentation.js.map

package/dist/TextSegmentation.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"TextSegmentation.js","sourceRoot":"","sources":["../src/TextSegmentation.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,WAAW,EAAE,MAAM,iBAAiB,CAAA;AAC7C,OAAO,EAAE,wBAAwB,IAAI,wBAAwB,EAAE,qBAAqB,EAAE,0CAA0C,EAAE,uBAAuB,EAAE,uBAAuB,EAAE,MAAM,eAAe,CAAA;AACzM,OAAO,EAAE,gBAAgB,EAAE,wCAAwC,EAAE,gBAAgB,EAAE,eAAe,EAAE,MAAM,mBAAmB,CAAA;AACjI,OAAO,EAAE,yBAAyB,EAAE,MAAM,iCAAiC,CAAA;AAC3E,OAAO,EAAE,YAAY,EAAE,MAAM,mBAAmB,CAAA;AAEhD,OAAO,EAAE,gBAAgB,EAAE,MAAM,mBAAmB,CAAA;AAEpD,MAAM,CAAC,KAAK,UAAU,WAAW,CAAC,IAAY,EAAE,OAA6B;IAC5E,OAAO,GAAG,EAAE,GAAG,0BAA0B,EAAE,GAAG,CAAC,OAAO,IAAI,EAAE,CAAC,EAAE,CAAA;IAE/D,MAAM,YAAY,GAAG,MAAM,YAAY,CAAC,IAAI,EAAE,OAAO,CAAC,CAAA;IAEtD,OAAO,mBAAmB,CAAC,YAAY,CAAC,CAAA;AACzC,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,mBAAmB,CAAC,YAA0B;IACnE,MAAM,kBAAkB,GAAY,EAAE,CAAA;IAEtC,IAAI,uBAAuB,GAAG,CAAC,CAAA;IAE/B,KAAK,IAAI,SAAS,GAAG,CAAC,EAAE,SAAS,GAAG,YAAY,CAAC,MAAM,EAAE,SAAS,EAAE,EAAE,CAAC;QACtE,MAAM,IAAI,GAAG,YAAY,CAAC,SAAS,CAAC,SAAS,CAAC,CAAA;QAE9C,IAAI,uBAAuB,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;YACxC,OAAO,SAAS,GAAG,YAAY,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBAC5C,MAAM,QAAQ,GAAG,YAAY,CAAC,SAAS,CAAC,SAAS,GAAG,CAAC,CAAC,CAAA;gBAEtD,IAAI,CAAC,0CAA0C,CAAC,IAAI,CAAC,QAAQ,CAAC,EAAE,CAAC;oBAChE,MAAK;gBACN,CAAC;gBAED,SAAS,IAAI,CAAC,CAAA;YACf,CAAC;YAED,kBAAkB,CAAC,IAAI,CAAC;gBACvB,KAAK,EAAE,uBAAuB;gBAC9B,GAAG,EAAE,SAAS,GAAG,CAAC;aAClB,CAAC,CAAA;YAEF,uBAAuB,GAAG,SAAS,GAAG,CAAC,CAAA;QACxC,CAAC;IACF,CAAC;IAED,IAAI,uBAAuB,GAAG,YAAY,CAAC,MAAM,EAAE,CAAC;QACnD,kBAAkB,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,uBAAuB,EAAE,GAAG,EAAE,YAAY,CAAC,MAAM,EAAE,CAAC,CAAA;IACtF,CAAC;IAED,MAAM,SAAS,GAAe,EAAE,CAAA;IAEhC,KAAK,MAAM,SAAS,IAAI,kBAAkB,EAAE,CAAC;QAC5C,MAAM,oBAAoB,GAAG,YAAY,CAAC,KAAK,CAAC,SAAS,CAAC,KAAK,EAAE,SAAS,CAAC,GAAG,CAAC,CAAA;QAE/E,SAAS,CAAC,IAAI,CAAC,IAAI,QAAQ,CAAC,SAAS,EAAE,oBAAoB,CAAC,CAAC,CAAA;IAC9D,CAAC;IAED,KAAK,MAAM,QAAQ,IAAI,SAAS,EAAE,CAAC;QAClC,MAAM,gBAAgB,GAAY,EAAE,CAAA;QAEpC,IAAI,qBAAqB,GAAG,QAAQ,CAAC,SAAS,CAAC,GAAG,CAAA;QAClD,IAAI,qBAAqB,GAAG,QAAQ,CAAC,SAAS,CAAC,KAAK,CAAA;QAEpD,KAAK,IAAI,SAAS,GAAG,qBAAqB,EAAE,SAAS,GAAG,qBAAqB,EAAE,SAAS,EAAE,EAAE,CAAC;YAC5F,MAAM,WAAW,GAAG,YAAY,CAAC,SAAS,CAAC,SAAS,CAAC,CAAA;YAErD,IAAI,qBAAqB,CAAC,IAAI,CAAC,WAAW,CAAC,EAAE,CAAC;gBAC7C,IAAI,kBAAkB,GAAG,KAAK,CAAA;gBAE9B,OAAO,SAAS,GAAG,qBAAqB,GAAG,CAAC,EAAE,CAAC;oBAC9C,MAAM,QAAQ,GAAG,YAAY,CAAC,SAAS,CAAC,SAAS,GAAG,CAAC,CAAC,CAAA;oBAEtD,IAAI,CAAC,0CAA0C,CAAC,IAAI,CAAC,QAAQ,CAAC,EAAE,CAAC;wBAChE,MAAK;oBACN,CAAC;oBAED,IAAI,uBAAuB,CAAC,IAAI,CAAC,QAAQ,CAAC,EAAE,CAAC;wBAC5C,kBAAkB,GAAG,IAAI,CAAA;oBAC1B,CAAC;oBAED,IAAI,QAAQ,KAAK,GAAG,IAAI,kBAAkB,EAAE,CAAC;wBAC5C,MAAK;oBACN,CAAC;oBAED,SAAS,IAAI,CAAC,CAAA;gBACf,CAAC;gBAED,gBAAgB,CAAC,IAAI,CAAC;oBACrB,KAAK,EAAE,qBAAqB;oBAC5B,GAAG,EAAE,SAAS,GAAG,CAAC;iBAClB,CAAC,CAAA;gBAEF,qBAAqB,GAAG,SAAS,GAAG,CAAC,CAAA;YACtC,CAAC;QACF,CAAC;QAED,IAAI,qBAAqB,GAAG,qBAAqB,EAAE,CAAC;YACnD,gBAAgB,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,qBAAqB,EAAE,GAAG,EAAE,qBAAqB,EAAE,CAAC,CAAA;QACpF,CAAC;QAED,KAAK,MAAM,SAAS,IAAI,gBAAgB,EAAE,CAAC;YAC1C,MAAM,kBAAkB,GAAG,YAAY,CAAC,KAAK,CAAC,SAAS,CAAC,KAAK,EAAE,SAAS,CAAC,GAAG,CAAC,CAAA;YAE7E,QAAQ,CAAC,OAAO,CAAC,IAAI,CAAC,IAAI,MAAM,CAAC,SAAS,EAAE,kBAAkB,CAAC,CAAC,CAAA;QACjE,CAAC;IACF,CAAC;IAED,MAAM,MAAM,GAAuB;QAClC,YAAY;QACZ,SAAS;KACT,CAAA;IAED,OAAO,MAAM,CAAA;AACd,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,YAAY,CAAC,IAAY,EAAE,OAA6B;IAC7E,OAAO,GAAG,EAAE,GAAG,0BAA0B,EAAE,GAAG,CAAC,OAAO,IAAI,EAAE,CAAC,EAAE,CAAA;IAE/D,MAAM,uBAAuB,GAAG,gBAAgB,CAAC,OAAO,CAAC,QAAQ,IAAI,EAAE,CAAC,IAAI,EAAE,CAAA;IAC9E,MAAM,8BAA8B,GAAG,wCAAwC,CAAC,OAAO,CAAC,QAAQ,IAAI,EAAE,CAAC,IAAI,EAAE,CAAA;IAC7G,MAAM,6CAA6C,GAAG,8BAA8B,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,UAAU,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC,CAAA;IACzH,MAAM,kBAAkB,GAAG,OAAO,CAAC,kBAAkB,IAAI,EAAE,CAAA;IAE3D,IAAI,YAAY,GAAG;QAClB,GAAG,kBAAkB;QACrB,GAAG,uBAAuB;QAC1B,GAAG,8BAA8B;QACjC,GAAG,6CAA6C;QAChD,GAAG,gBAAgB;QACnB,GAAG,eAAe;KAClB,CAAA;IAED,MAAM,WAAW,GAAG,wBAAwB,CAAC;QAC5C,GAAG,YAAY;QACf,GAAG,YAAY,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,iBAAiB,EAAE,CAAC;QACrD,GAAG,YAAY,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,iBAAiB,EAAE,CAAC;KACrD,CAAC,CAAA;IAEF,MAAM,kBAAkB,GAAG,WAAW,CAAC,WAAW,EAAE,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC,CAAA;IAErE,gEAAgE;IAEhE,IAAI,YAAY,GAAG,IAAI,YAAY,EAAE,CAAA;IAErC,SAAS,kBAAkB,CAAC,WAAmB,EAAE,SAAiB;QACjE,MAAM,gBAAgB,GAAG,IAAI,CAAC,SAAS,CAAC,WAAW,EAAE,SAAS,CAAC,CAAA;QAE/D,IAAI,UAAU,GAAG,WAAW,CAAA;QAC5B,IAAI,kBAAkB,GAAG,WAAW,CAAA;QAEpC,SAAS,kBAAkB;YAC1B,IAAI,UAAU,GAAG,kBAAkB,EAAE,CAAC;gBACrC,YAAY,CAAC,YAAY,CAAC,IAAI,EAAE,kBAAkB,EAAE,UAAU,EAAE,IAAI,CAAC,CAAA;gBACrE,kBAAkB,GAAG,UAAU,CAAA;YAChC,CAAC;QACF,CAAC;QAED,KAAK,MAAM,IAAI,IAAI,gBAAgB,EAAE,CAAC;YACrC,IAAI,IAAI,KAAK,GAAG,EAAE,CAAC;gBAClB,UAAU,IAAI,CAAC,CAAA;gBAEf,SAAQ;YACT,CAAC;YAED,kBAAkB,EAAE,CAAA;YAEpB,UAAU,IAAI,IAAI,CAAC,MAAM,CAAA;YAEzB,kBAAkB,EAAE,CAAA;QACrB,CAAC;QAED,kBAAkB,EAAE,CAAA;IACrB,CAAC;IAED,MAAM,WAAW,GAAG,IAAI,CAAC,QAAQ,CAAC,kBAAkB,CAAC,CAAA;IAErD,IAAI,kBAAkB,GAAG,CAAC,CAAA;IAE1B,IAAI,WAAW,EAAE,CAAC;QACjB,KAAK,MAAM,KAAK,IAAI,WAAW,EAAE,CAAC;YACjC,MAAM,OAAO,GAAG,KAAK,CAAC,OAAQ,CAAC,CAAC,CAAC,CAAA;YACjC,MAAM,gBAAgB,GAAG,OAAO,CAAC,CAAC,CAAC,CAAA;YACnC,MAAM,cAAc,GAAG,OAAO,CAAC,CAAC,CAAC,CAAA;YAEjC,IAAI,gBAAgB,GAAG,kBAAkB,EAAE,CAAC;gBAC3C,kBAAkB,CAAC,kBAAkB,EAAE,gBAAgB,CAAC,CAAA;YACzD,CAAC;YAED,YAAY,CAAC,YAAY,CAAC,IAAI,EAAE,gBAAgB,EAAE,cAAc,EAAE,KAAK,CAAC,CAAA;YAExE,kBAAkB,GAAG,cAAc,CAAA;QACpC,CAAC;QAED,kBAAkB,CAAC,kBAAkB,EAAE,IAAI,CAAC,MAAM,CAAC,CAAA;IACpD,CAAC;IAED,IAAI,OAAO,CAAC,6BAA6B,EAAE,CAAC;QAC3C,YAAY,GAAG,MAAM,yBAAyB,CAAC,IAAI,EAAE,YAAY,CAAC,CAAA;IACnE,CAAC;IAED,OAAO,YAAY,CAAA;AACpB,CAAC;AAED,KAAK,UAAU,yBAAyB,CAAC,cAAsB,EAAE,YAA0B;IAC1F,MAAM,eAAe,GAAG,MAAM,kBAAkB,EAAE,CAAA;IAElD,IAAI,eAAe,KAAK,SAAS,EAAE,CAAC;QACnC,OAAO,YAAY,CAAA;IACpB,CAAC;IAED,IAAI,cAAc,GAAG,KAAK,CAAA;IAE1B,MAAM,eAAe,GAAG,IAAI,YAAY,EAAE,CAAA;IAE1C,KAAK,IAAI,SAAS,GAAG,CAAC,EAAE,SAAS,GAAG,YAAY,CAAC,MAAM,EAAE,SAAS,EAAE,EAAE,CAAC;QACtE,MAAM,SAAS,GAAG,YAAY,CAAC,OAAO,CAAC,SAAS,CAAC,CAAA;QACjD,MAAM,eAAe,GAAG,SAAS,CAAC,WAAW,CAAA;QAE7C,MAAM,IAAI,GAAG,YAAY,CAAC,SAAS,CAAC,SAAS,CAAC,CAAA;QAE9C,IAAI,yBAAyB,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;YAC1C,IAAI,CAAC,cAAc,EAAE,CAAC;gBACrB,MAAM,eAAe,CAAC,UAAU,EAAE,CAAA;gBAElC,cAAc,GAAG,IAAI,CAAA;YACtB,CAAC;YAED,MAAM,UAAU,GAAG,CAAC,GAAG,eAAe,CAAC,uBAAuB,CAAC,IAAI,CAAC,CAAC,CAAA;YAErE,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,UAAU,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;gBAChD,eAAe,CAAC,YAAY,CAC3B,cAAc,EACd,eAAe,GAAG,UAAU,CAAC,CAAC,CAAC,EAC/B,eAAe,GAAG,UAAU,CAAC,CAAC,GAAG,CAAC,CAAC,EACnC,KAAK,CACL,CAAA;YACF,CAAC;QACF,CAAC;aAAM,CAAC;YACP,eAAe,CAAC,YAAY,CAAC,cAAc,EAAE,SAAS,CAAC,WAAW,EAAE,SAAS,CAAC,SAAS,EAAE,SAAS,CAAC,aAAa,CAAC,CAAA;QAClH,CAAC;IACF,CAAC;IAED,OAAO,eAAe,CAAA;AACvB,CAAC;AAED,KAAK,UAAU,kBAAkB;IAChC,IAAI,CAAC;QACJ,MAAM,eAAe,GAAG,MAAM,MAAM,CAAC,mCAAmC,CAAC,CAAA;QAEzE,OAAO,eAAe,CAAA;IACvB,CAAC;IAAC,MAAM,CAAC;QACR,OAAO,SAAS,CAAA;IACjB,CAAC;AACF,CAAC;AAOD,MAAM,OAAO,YAAY;IACxB,SAAS,CAAO;IAChB,YAAY,CAAc;IAE1B,YAAY,SAAgB,EAAE,YAA0B;QACvD,IAAI,CAAC,SAAS,GAAG,SAAS,CAAA;QAC1B,IAAI,CAAC,YAAY,GAAG,YAAY,CAAA;IACjC,CAAC;IAED,IAAI,IAAI;QACP,OAAO,IAAI,CAAC,YAAY,CAAC,IAAI,CAAA;IAC9B,CAAC;IAED,IAAI,SAAS;QACZ,OAAO;YACN,KAAK,EAAE,IAAI,CAAC,YAAY,CAAC,UAAU,CAAC,WAAW;YAC/C,GAAG,EAAE,IAAI,CAAC,YAAY,CAAC,SAAS,CAAC,SAAS;SAC1C,CAAA;IACF,CAAC;CACD;AAED,MAAM,OAAO,QAAS,SAAQ,YAAY;IACzC,OAAO,GAAa,EAAE,CAAA;CACtB;AAED,MAAM,OAAO,MAAO,SAAQ,YAAY;CACvC;AAaD,MAAM,CAAC,MAAM,0BAA0B,GAAwB;IAC9D,QAAQ,EAAE,EAAE;IACZ,kBAAkB,EAAE,EAAE;IACtB,6BAA6B,EAAE,IAAI;CACnC,CAAA"}

package/dist/WordSequence.d.ts

@@ -0,0 +1,26 @@
+export declare class WordSequence {
+    entries: WordEntry[];
+    addWordEntry(containingText: string, startOffset: number, endOffset: number, isPunctuation: boolean): void;
+    getWordRange(startIndex: number, endIndex: number): string[];
+    iterateWordRange(startIndex: number, endIndex: number): Generator<string, void, unknown>;
+    getWordAt(index: number): string;
+    getEntryRange(startIndex: number, endIndex: number): WordEntry[];
+    iterateEntryRange(startIndex: number, endIndex: number): Generator<WordEntry, void, unknown>;
+    getEntryAt(index: number): WordEntry;
+    slice(startIndex: number, endIndex: number): WordSequence;
+    get words(): string[];
+    get firstWord(): string;
+    get lastWord(): string;
+    get firstEntry(): WordEntry;
+    get lastEntry(): WordEntry;
+    get nonPunctuationEntries(): WordEntry[];
+    get nonPunctuationWords(): string[];
+    get text(): string;
+    get length(): number;
+}
+export interface WordEntry {
+    text: string;
+    isPunctuation: boolean;
+    startOffset: number;
+    endOffset: number;
+}

package/dist/WordSequence.js

@@ -0,0 +1,67 @@
+export class WordSequence {
+    entries = [];
+    addWordEntry(containingText, startOffset, endOffset, isPunctuation) {
+        const wordText = containingText.substring(startOffset, endOffset);
+        this.entries.push({
+            text: wordText,
+            startOffset,
+            endOffset,
+            isPunctuation
+        });
+    }
+    getWordRange(startIndex, endIndex) {
+        return [...this.iterateWordRange(startIndex, endIndex)];
+    }
+    *iterateWordRange(startIndex, endIndex) {
+        for (let i = startIndex; i < endIndex; i++) {
+            yield this.getWordAt(i);
+        }
+    }
+    getWordAt(index) {
+        return this.entries[index].text;
+    }
+    getEntryRange(startIndex, endIndex) {
+        return this.entries.slice(startIndex, endIndex);
+    }
+    *iterateEntryRange(startIndex, endIndex) {
+        for (let i = startIndex; i < endIndex; i++) {
+            yield this.getEntryAt(i);
+        }
+    }
+    getEntryAt(index) {
+        return this.entries[index];
+    }
+    slice(startIndex, endIndex) {
+        const slicedSequence = new WordSequence();
+        slicedSequence.entries = this.entries.slice(startIndex, endIndex);
+        return slicedSequence;
+    }
+    get words() {
+        return this.entries.map(entry => entry.text);
+    }
+    get firstWord() {
+        return this.firstEntry?.text;
+    }
+    get lastWord() {
+        return this.lastEntry?.text;
+    }
+    get firstEntry() {
+        return this.entries[0];
+    }
+    get lastEntry() {
+        return this.entries[this.length - 1];
+    }
+    get nonPunctuationEntries() {
+        return this.entries.filter(entry => entry.isPunctuation === false);
+    }
+    get nonPunctuationWords() {
+        return this.entries.filter(entry => entry.isPunctuation === false).map(entry => entry.text);
+    }
+    get text() {
+        return this.words.join('');
+    }
+    get length() {
+        return this.entries.length;
+    }
+}
+//# sourceMappingURL=WordSequence.js.map

package/dist/WordSequence.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"WordSequence.js","sourceRoot":"","sources":["../src/WordSequence.ts"],"names":[],"mappings":"AAAA,MAAM,OAAO,YAAY;IACxB,OAAO,GAAgB,EAAE,CAAA;IAEzB,YAAY,CAAC,cAAsB,EAAE,WAAmB,EAAE,SAAiB,EAAE,aAAsB;QAClG,MAAM,QAAQ,GAAG,cAAc,CAAC,SAAS,CAAC,WAAW,EAAE,SAAS,CAAC,CAAA;QAEjE,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC;YACjB,IAAI,EAAE,QAAQ;YACd,WAAW;YACX,SAAS;YACT,aAAa;SACb,CAAC,CAAA;IACH,CAAC;IAED,YAAY,CAAC,UAAkB,EAAE,QAAgB;QAChD,OAAO,CAAC,GAAG,IAAI,CAAC,gBAAgB,CAAC,UAAU,EAAE,QAAQ,CAAC,CAAC,CAAA;IACxD,CAAC;IAED,CAAC,gBAAgB,CAAC,UAAkB,EAAE,QAAgB;QACrD,KAAK,IAAI,CAAC,GAAG,UAAU,EAAE,CAAC,GAAG,QAAQ,EAAE,CAAC,EAAE,EAAE,CAAC;YAC5C,MAAM,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,CAAA;QACxB,CAAC;IACF,CAAC;IAED,SAAS,CAAC,KAAa;QACtB,OAAO,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,IAAI,CAAA;IAChC,CAAC;IAED,aAAa,CAAC,UAAkB,EAAE,QAAgB;QACjD,OAAO,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,UAAU,EAAE,QAAQ,CAAC,CAAA;IAChD,CAAC;IAED,CAAC,iBAAiB,CAAC,UAAkB,EAAE,QAAgB;QACtD,KAAK,IAAI,CAAC,GAAG,UAAU,EAAE,CAAC,GAAG,QAAQ,EAAE,CAAC,EAAE,EAAE,CAAC;YAC5C,MAAM,IAAI,CAAC,UAAU,CAAC,CAAC,CAAC,CAAA;QACzB,CAAC;IACF,CAAC;IAED,UAAU,CAAC,KAAa;QACvB,OAAO,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,CAAA;IAC3B,CAAC;IAED,KAAK,CAAC,UAAkB,EAAE,QAAgB;QACzC,MAAM,cAAc,GAAG,IAAI,YAAY,EAAE,CAAA;QAEzC,cAAc,CAAC,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,UAAU,EAAE,QAAQ,CAAC,CAAA;QAEjE,OAAO,cAAc,CAAA;IACtB,CAAC;IAED,IAAI,KAAK;QACR,OAAO,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC,KAAK,CAAC,IAAI,CAAC,CAAA;IAC7C,CAAC;IAED,IAAI,SAAS;QACZ,OAAO,IAAI,CAAC,UAAU,EAAE,IAAI,CAAA;IAC7B,CAAC;IAED,IAAI,QAAQ;QACX,OAAO,IAAI,CAAC,SAAS,EAAE,IAAI,CAAA;IAC5B,CAAC;IAED,IAAI,UAAU;QACb,OAAO,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,CAAA;IACvB,CAAC;IAED,IAAI,SAAS;QACZ,OAAO,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC,CAAA;IACrC,CAAC;IAED,IAAI,qBAAqB;QACxB,OAAO,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,KAAK,CAAC,aAAa,KAAK,KAAK,CAAC,CAAA;IACnE,CAAC;IAED,IAAI,mBAAmB;QACtB,OAAO,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,KAAK,CAAC,aAAa,KAAK,KAAK,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC,KAAK,CAAC,IAAI,CAAC,CAAA;IAC5F,CAAC;IAED,IAAI,IAAI;QACP,OAAO,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAA;IAC3B,CAAC;IAED,IAAI,MAAM;QACT,OAAO,IAAI,CAAC,OAAO,CAAC,MAAM,CAAA;IAC3B,CAAC;CACD"}

package/dist/utilities/Timer.d.ts

@@ -0,0 +1,13 @@
+export declare class Timer {
+    startTime: number;
+    constructor();
+    restart(): void;
+    get elapsedTime(): number;
+    get elapsedTimeSeconds(): number;
+    getElapsedTimeAndRestart(): number;
+    logAndRestart(title: string, timePrecision?: number): number;
+    static get currentTime(): number;
+    static get microsecondTimestamp(): number;
+    private static createGlobalTimestampFunction;
+    private static timestampFunc;
+}

package/dist/utilities/Timer.js

@@ -0,0 +1,70 @@
+import { roundToDigits } from "./Utilities.js";
+export class Timer {
+    startTime = 0;
+    constructor() {
+        this.restart();
+    }
+    restart() {
+        this.startTime = Timer.currentTime;
+    }
+    get elapsedTime() {
+        // Elapsed time (milliseconds)
+        return Timer.currentTime - this.startTime;
+    }
+    get elapsedTimeSeconds() {
+        // Elapsed time (seconds)
+        return this.elapsedTime / 1000;
+    }
+    getElapsedTimeAndRestart() {
+        const elapsedTime = this.elapsedTime;
+        this.restart();
+        return elapsedTime;
+    }
+    logAndRestart(title, timePrecision = 3) {
+        const elapsedTime = this.elapsedTime;
+        //
+        const message = `${title}: ${roundToDigits(elapsedTime, timePrecision)}ms`;
+        console.log(message);
+        //
+        this.restart();
+        return elapsedTime;
+    }
+    static get currentTime() {
+        if (!this.timestampFunc) {
+            this.createGlobalTimestampFunction();
+        }
+        return this.timestampFunc();
+    }
+    static get microsecondTimestamp() {
+        return Math.floor(Timer.currentTime * 1000);
+    }
+    static createGlobalTimestampFunction() {
+        if (typeof process === 'object' && typeof process.hrtime === 'function') {
+            let baseTimestamp = 0;
+            this.timestampFunc = () => {
+                const nodeTimeStamp = process.hrtime();
+                const millisecondTime = (nodeTimeStamp[0] * 1000) + (nodeTimeStamp[1] / 1000000);
+                return baseTimestamp + millisecondTime;
+            };
+            baseTimestamp = Date.now() - this.timestampFunc();
+        }
+        else if (typeof chrome === 'object' && chrome.Interval) {
+            const baseTimestamp = Date.now();
+            const chromeIntervalObject = new chrome.Interval();
+            chromeIntervalObject.start();
+            this.timestampFunc = () => baseTimestamp + chromeIntervalObject.microseconds() / 1000;
+        }
+        else if (typeof performance === 'object' && performance.now) {
+            const baseTimestamp = Date.now() - performance.now();
+            this.timestampFunc = () => baseTimestamp + performance.now();
+        }
+        else if (Date.now) {
+            this.timestampFunc = () => Date.now();
+        }
+        else {
+            this.timestampFunc = () => (new Date()).getTime();
+        }
+    }
+    static timestampFunc;
+}
+//# sourceMappingURL=Timer.js.map

package/dist/utilities/Timer.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"Timer.js","sourceRoot":"","sources":["../../src/utilities/Timer.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,aAAa,EAAE,MAAM,gBAAgB,CAAA;AAK9C,MAAM,OAAO,KAAK;IACjB,SAAS,GAAG,CAAC,CAAA;IAEb;QACC,IAAI,CAAC,OAAO,EAAE,CAAA;IACf,CAAC;IAED,OAAO;QACN,IAAI,CAAC,SAAS,GAAG,KAAK,CAAC,WAAW,CAAA;IACnC,CAAC;IAED,IAAI,WAAW;QACd,8BAA8B;QAC9B,OAAO,KAAK,CAAC,WAAW,GAAG,IAAI,CAAC,SAAS,CAAA;IAC1C,CAAC;IAED,IAAI,kBAAkB;QACrB,yBAAyB;QACzB,OAAO,IAAI,CAAC,WAAW,GAAG,IAAI,CAAA;IAC/B,CAAC;IAED,wBAAwB;QACvB,MAAM,WAAW,GAAG,IAAI,CAAC,WAAW,CAAA;QACpC,IAAI,CAAC,OAAO,EAAE,CAAA;QAEd,OAAO,WAAW,CAAA;IACnB,CAAC;IAED,aAAa,CAAC,KAAa,EAAE,aAAa,GAAG,CAAC;QAC7C,MAAM,WAAW,GAAG,IAAI,CAAC,WAAW,CAAA;QAEpC,EAAE;QACF,MAAM,OAAO,GAAG,GAAG,KAAK,KAAK,aAAa,CAAC,WAAW,EAAE,aAAa,CAAC,IAAI,CAAA;QAE1E,OAAO,CAAC,GAAG,CAAC,OAAO,CAAC,CAAA;QACpB,EAAE;QAEF,IAAI,CAAC,OAAO,EAAE,CAAA;QAEd,OAAO,WAAW,CAAA;IACnB,CAAC;IAED,MAAM,KAAK,WAAW;QACrB,IAAI,CAAC,IAAI,CAAC,aAAa,EAAE,CAAC;YACzB,IAAI,CAAC,6BAA6B,EAAE,CAAA;QACrC,CAAC;QAED,OAAO,IAAI,CAAC,aAAa,EAAE,CAAA;IAC5B,CAAC;IAED,MAAM,KAAK,oBAAoB;QAC9B,OAAO,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,WAAW,GAAG,IAAI,CAAC,CAAA;IAC5C,CAAC;IAEO,MAAM,CAAC,6BAA6B;QAC3C,IAAI,OAAO,OAAO,KAAK,QAAQ,IAAI,OAAO,OAAO,CAAC,MAAM,KAAK,UAAU,EAAE,CAAC;YACzE,IAAI,aAAa,GAAG,CAAC,CAAA;YAErB,IAAI,CAAC,aAAa,GAAG,GAAG,EAAE;gBACzB,MAAM,aAAa,GAAG,OAAO,CAAC,MAAM,EAAE,CAAA;gBACtC,MAAM,eAAe,GAAG,CAAC,aAAa,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,aAAa,CAAC,CAAC,CAAC,GAAG,OAAO,CAAC,CAAA;gBAEhF,OAAO,aAAa,GAAG,eAAe,CAAA;YACvC,CAAC,CAAA;YAED,aAAa,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,IAAI,CAAC,aAAa,EAAE,CAAA;QAClD,CAAC;aACI,IAAI,OAAO,MAAM,KAAK,QAAQ,IAAI,MAAM,CAAC,QAAQ,EAAE,CAAC;YACxD,MAAM,aAAa,GAAG,IAAI,CAAC,GAAG,EAAE,CAAA;YAEhC,MAAM,oBAAoB,GAAG,IAAI,MAAM,CAAC,QAAQ,EAAE,CAAA;YAClD,oBAAoB,CAAC,KAAK,EAAE,CAAA;YAE5B,IAAI,CAAC,aAAa,GAAG,GAAG,EAAE,CAAC,aAAa,GAAG,oBAAoB,CAAC,YAAY,EAAE,GAAG,IAAI,CAAA;QACtF,CAAC;aACI,IAAI,OAAO,WAAW,KAAK,QAAQ,IAAI,WAAW,CAAC,GAAG,EAAE,CAAC;YAC7D,MAAM,aAAa,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,WAAW,CAAC,GAAG,EAAE,CAAA;YAEpD,IAAI,CAAC,aAAa,GAAG,GAAG,EAAE,CAAC,aAAa,GAAG,WAAW,CAAC,GAAG,EAAE,CAAA;QAC7D,CAAC;aACI,IAAI,IAAI,CAAC,GAAG,EAAE,CAAC;YACnB,IAAI,CAAC,aAAa,GAAG,GAAG,EAAE,CAAC,IAAI,CAAC,GAAG,EAAE,CAAA;QACtC,CAAC;aACI,CAAC;YACL,IAAI,CAAC,aAAa,GAAG,GAAG,EAAE,CAAC,CAAC,IAAI,IAAI,EAAE,CAAC,CAAC,OAAO,EAAE,CAAA;QAClD,CAAC;IACF,CAAC;IAEO,MAAM,CAAC,aAAa,CAAc;CAC1C"}

package/dist/utilities/Utilities.d.ts

@@ -0,0 +1,6 @@
+import { Pattern } from 'regexp-composer';
+export declare function roundToDigits(val: number, digits?: number): number;
+export declare function listAllCharsMatching(charPattern: Pattern): string[];
+export declare function extractSuppressions(entries: {
+    suppression: string;
+}[]): string[];

package/dist/utilities/Utilities.js

@@ -0,0 +1,24 @@
+import { buildRegExp } from 'regexp-composer';
+export function roundToDigits(val, digits = 3) {
+    const multiplier = 10 ** digits;
+    return Math.round(val * multiplier) / multiplier;
+}
+export function listAllCharsMatching(charPattern) {
+    const regExp = buildRegExp(charPattern);
+    const matchingChars = [];
+    for (let codepoint = 0; codepoint < 1114111; codepoint++) {
+        const char = String.fromCodePoint(codepoint);
+        if (regExp.test(char)) {
+            matchingChars.push(char);
+        }
+    }
+    return matchingChars;
+}
+export function extractSuppressions(entries) {
+    const suppressions = [];
+    for (const entry of entries) {
+        suppressions.push(entry.suppression);
+    }
+    return suppressions;
+}
+//# sourceMappingURL=Utilities.js.map

package/dist/utilities/Utilities.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"Utilities.js","sourceRoot":"","sources":["../../src/utilities/Utilities.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,WAAW,EAAW,MAAM,iBAAiB,CAAA;AAEtD,MAAM,UAAU,aAAa,CAAC,GAAW,EAAE,MAAM,GAAG,CAAC;IACpD,MAAM,UAAU,GAAG,EAAE,IAAI,MAAM,CAAA;IAE/B,OAAO,IAAI,CAAC,KAAK,CAAC,GAAG,GAAG,UAAU,CAAC,GAAG,UAAU,CAAA;AACjD,CAAC;AAED,MAAM,UAAU,oBAAoB,CAAC,WAAoB;IACxD,MAAM,MAAM,GAAG,WAAW,CAAC,WAAW,CAAC,CAAA;IAEvC,MAAM,aAAa,GAAa,EAAE,CAAA;IAElC,KAAK,IAAI,SAAS,GAAG,CAAC,EAAE,SAAS,GAAG,OAAO,EAAE,SAAS,EAAE,EAAE,CAAC;QAC1D,MAAM,IAAI,GAAG,MAAM,CAAC,aAAa,CAAC,SAAS,CAAC,CAAA;QAE5C,IAAI,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;YACvB,aAAa,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;QACzB,CAAC;IACF,CAAC;IAED,OAAO,aAAa,CAAA;AACrB,CAAC;AAED,MAAM,UAAU,mBAAmB,CAAC,OAAiC;IACpE,MAAM,YAAY,GAAa,EAAE,CAAA;IAEjC,KAAK,MAAM,KAAK,IAAI,OAAO,EAAE,CAAC;QAC7B,YAAY,CAAC,IAAI,CAAC,KAAK,CAAC,WAAW,CAAC,CAAA;IACrC,CAAC;IAED,OAAO,YAAY,CAAA;AACpB,CAAC"}

package/package.json

@@ -0,0 +1,55 @@
+{
+	"name": "@echogarden/text-segmentation",
+	"version": "0.1.0",
+	"description": "A library for multilingual word, phrase and sentence segmentation.",
+	"author": "Rotem Dan",
+	"license": "MIT",
+	"keywords": [
+		"text segmentation",
+		"word segmentation",
+		"word boundary detection",
+		"phrase segmentation",
+		"phrase boundary detection",
+		"sentence segmentation",
+		"sentence boundary detection"
+	],
+	"publishConfig": {
+		"access": "public"
+	},
+	"repository": {
+		"type": "git",
+		"url": "https://github.com/echogarden-project/text-segmentation"
+	},
+	"bugs": {
+		"url": "https://github.com/echogarden-project/text-segmentation/issues"
+	},
+	"main": "./dist/TextSegmentation.js",
+	"type": "module",
+	"files": [
+		"dist",
+		"src",
+		"tsconfig.json",
+		"LICENSE.md",
+		"README.md"
+	],
+	"scripts": {
+		"test": "node dist/Test.js"
+	},
+	"//dependencies": {
+		"regexp-composer": "../../regexp-composer"
+	},
+	"dependencies": {
+		"regexp-composer": "^0.1.2"
+	},
+	"peerDependencies": {
+		"@echogarden/icu-segmentation-wasm": "^0.1.0"
+	},
+	"peerDependenciesMeta": {
+		"@echogarden/icu-segmentation-wasm": {
+			"optional": true
+		}
+	},
+	"devDependencies": {
+		"@types/node": "^22.10.1"
+	}
+}

package/src/EastAsianCharacterPatterns.ts

@@ -0,0 +1,45 @@
+import { anyOf, buildRegExp, codepointRange } from 'regexp-composer'
+
+export const chineseCharacterRanges = anyOf(
+	codepointRange('4E00', '9FFF'), // Main
+	codepointRange('3400', '4DBF'), // CJK Unified Ideographs Extension A
+	codepointRange('20000', '2A6DF') // CJK Unified Ideographs Extension B
+)
+
+export const japaneseHiraganaCharacterRanges = anyOf(
+	codepointRange('3040', '309F'), // Hiragana
+	codepointRange('1AFF0', '1AFFF'), // Kana Extended-B
+	codepointRange('1B000', '1B0FF'), // Kana Supplement
+	codepointRange('1B100', '1B12F'), // Kana Extended-A
+	codepointRange('1B130', '1B16F'), // Small Kana Extension
+)
+
+export const japaneseKatakanaCharacterRanges = anyOf(
+	codepointRange('30A0', '30FF'), // Katakana
+	codepointRange('31F0', '31FF'), // Katakana Phonetic Extensions
+	codepointRange('3200', '32FF'), // Enclosed CJK Letters and Months
+	codepointRange('FF00', 'FFEF'), // Halfwidth and Fullwidth Forms
+	codepointRange('1AFF0', '1AFFF'), // Kana Extended-B
+	codepointRange('1B000', '1B0FF'), // Kana Supplement
+	codepointRange('1B100', '1B12F'), // Kana Extended-A
+	codepointRange('1B130', '1B16F'), // Small Kana Extension
+)
+
+export const thaiLetterRanges = anyOf(
+	codepointRange('0E00', '0E7F')
+)
+
+export const khmerLetterRanges = anyOf(
+	codepointRange('1780', '17FF'), // Letters
+	codepointRange('19E0', '19FF'), // Symbols
+)
+
+export const eastAsianCharRanges = anyOf(
+	chineseCharacterRanges,
+	japaneseHiraganaCharacterRanges,
+	japaneseKatakanaCharacterRanges,
+	thaiLetterRanges,
+	khmerLetterRanges,
+)
+
+export const eastAsianCharRangesRegExp = buildRegExp(eastAsianCharRanges)