@echogarden/text-segmentation 0.7.0 → 0.8.0

package/dist/exports/Exports.d.ts

@@ -0,0 +1,3 @@
+export * from '../segmentation/TextSegmentation.js';
+export { WordSequence, type WordEntry } from '../segmentation/WordSequence.js';
+//# sourceMappingURL=Exports.d.ts.map

package/dist/exports/Exports.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"Exports.d.ts","sourceRoot":"","sources":["../../src/exports/Exports.ts"],"names":[],"mappings":"AAAA,cAAc,qCAAqC,CAAA;AACnD,OAAO,EAAE,YAAY,EAAE,KAAK,SAAS,EAAE,MAAM,iCAAiC,CAAA"}

package/dist/exports/Exports.js

@@ -0,0 +1,3 @@
+export * from '../segmentation/TextSegmentation.js';
+export { WordSequence } from '../segmentation/WordSequence.js';
+//# sourceMappingURL=Exports.js.map

package/dist/exports/Exports.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"Exports.js","sourceRoot":"","sources":["../../src/exports/Exports.ts"],"names":[],"mappings":"AAAA,cAAc,qCAAqC,CAAA;AACnD,OAAO,EAAE,YAAY,EAAkB,MAAM,iCAAiC,CAAA"}

package/dist/{exports → segmentation}/TextSegmentation.d.ts

@@ -1,5 +1,4 @@
 import { WordSequence } from './WordSequence.js';
-export { WordSequence, type WordEntry } from './WordSequence.js';
 export declare function segmentText(text: string, options?: SegmentationOptions): Promise<SegmentationResult>;
 export declare function segmentWordSequence(wordSequence: WordSequence, options?: WordSequenceSegmentationOptions): Promise<SegmentationResult>;
 export declare function splitToWords(text: string, options?: SegmentationOptions): Promise<WordSequence>;

package/dist/segmentation/TextSegmentation.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"TextSegmentation.d.ts","sourceRoot":"","sources":["../../src/segmentation/TextSegmentation.ts"],"names":[],"mappings":"AAGA,OAAO,EAAE,YAAY,EAAE,MAAM,mBAAmB,CAAA;AAWhD,wBAAsB,WAAW,CAAC,IAAI,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,mBAAmB,+BAM5E;AAKD,wBAAsB,mBAAmB,CAAC,YAAY,EAAE,YAAY,EAAE,OAAO,CAAC,EAAE,+BAA+B,+BA8P9G;AAQD,wBAAsB,YAAY,CAAC,IAAI,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,mBAAmB,yBAuH7E;AAOD,wBAAgB,wCAAwC,CAAC,YAAY,EAAE,YAAY,EAAE,UAAU,EAAE,MAAM;;;EA2EtG;AAkLD,MAAM,WAAW,kBAAkB;IAClC,KAAK,EAAE,YAAY,CAAA;IACnB,SAAS,EAAE,QAAQ,EAAE,CAAA;IACrB,qBAAqB,EAAE,KAAK,EAAE,CAAA;CAC9B;AAID,qBAAa,YAAY;IAExB,SAAS,EAAE,KAAK,CAAA;IAEhB,KAAK,EAAE,YAAY,CAAA;IAGnB,YAAY,SAAS,EAAE,KAAK,EAAE,KAAK,EAAE,YAAY,EAGhD;IAGD,IAAI,IAAI,WAEP;IAID,IAAI,SAAS,IAAI,KAAK,CAKrB;CACD;AAID,qBAAa,QAAS,SAAQ,YAAY;IAEzC,OAAO,EAAE,MAAM,EAAE,CAAK;CACtB;AAID,qBAAa,MAAO,SAAQ,YAAY;CACvC;AAID,MAAM,WAAW,KAAK;IACrB,KAAK,EAAE,MAAM,CAAA;IACb,GAAG,EAAE,MAAM,CAAA;CACX;AAGD,MAAM,WAAW,mBAAmB;IACnC,QAAQ,CAAC,EAAE,MAAM,CAAA;IACjB,kBAAkB,CAAC,EAAE,MAAM,EAAE,CAAA;IAC7B,6BAA6B,CAAC,EAAE,OAAO,CAAA;CACvC;AAGD,eAAO,MAAM,0BAA0B,EAAE,mBAIxC,CAAA;AAGD,MAAM,WAAW,+BAA+B;IAC/C,mCAAmC,EAAE,OAAO,CAAA;CAC5C;AAGD,eAAO,MAAM,sCAAsC,EAAE,+BAEpD,CAAA"}

package/dist/segmentation/TextSegmentation.js

@@ -0,0 +1,575 @@
+import { buildWordOrNumberPattern as buildWordSplitterPattern, phraseSeparatorRegExp, sentenceSeparatorTrailingPunctuationCharacterRegExp, sentenceSeparatorCharacterRegExp, whitespacePatternRegExp, letterPatternGlobalRegExp } from '../patterns/Patterns.js';
+import { cldrSuppressions, additionalSuppressions, leadingApostropheContractionSuppressions, nounSuppressions, tldSuppressions } from '../patterns/Suppressions.js';
+import { eastAsianCharRangesRegExp } from '../patterns/EastAsianCharacterPatterns.js';
+import { WordSequence } from './WordSequence.js';
+import { getShortLanguageCode } from '../utilities/Utilities.js';
+import { buildRegExp } from 'regexp-composer';
+////////////////////////////////////////////////////////////////////////////////////////////////
+// Exported methods
+////////////////////////////////////////////////////////////////////////////////////////////////
+// Splits a text string into words and then segments the resulting word sequence into sentences and phrases
+// (this is the main high-level entry point; it is async because splitting may involve ICU postprocessing)
+export async function segmentText(text, options) {
+    // Split the text into a sequence of individual words (this may await East Asian postprocessing)
+    const wordSequence = await splitToWords(text, options);
+    // Segment the word sequence into segments (paragraphs), sentences, and phrases, and return the result
+    return segmentWordSequence(wordSequence);
+}
+// Segments an already-tokenized word sequence into segments (paragraphs), sentences, and phrases
+// This is the core segmentation algorithm, working purely on the word level
+// (options only control how colons/semicolons are treated when splitting phrases)
+export async function segmentWordSequence(wordSequence, options) {
+    // Fill in any option values the caller didn't provide with their defaults
+    options = { ...defaultWordSequenceSegmentationOptions, ...options };
+    // This will hold the start and end word offsets of each paragraph (called a "segment" here)
+    const segmentWordRanges = [];
+    // Step 1: find segment (paragraph) boundaries.
+    // A new segment begins right after a newline character that follows at least one non-whitespace word.
+    {
+        // The word offset at which the current segment starts
+        let segmentStartWordOffset = 0;
+        // Whether at least one non-whitespace word has been seen in the current segment
+        let nonWhitespaceWordSeen = false;
+        // Whether a newline has been seen in the current segment (i.e. we may be at a boundary)
+        let newlineSeenInCurrentSegment = false;
+        // Walk through every word of the sequence
+        for (let wordIndex = 0; wordIndex < wordSequence.length; wordIndex++) {
+            // Get the text of the word at the current index
+            const word = wordSequence.getWordAt(wordIndex);
+            // If we haven't seen a newline yet, the segment already has content, and this word is a newline, mark the newline as seen
+            if (!newlineSeenInCurrentSegment && nonWhitespaceWordSeen && word.endsWith('\n')) {
+                newlineSeenInCurrentSegment = true;
+            }
+            // If a newline was seen, decide whether the segment actually ends at this word
+            if (newlineSeenInCurrentSegment) {
+                // Look at the word after the current one (or an empty string if the current word is last)
+                const nextWord = wordIndex < wordSequence.length - 1 ? wordSequence.getWordAt(wordIndex + 1) : '';
+                // End the segment here if the newline is followed by real content (or by nothing at all)
+                if (nextWord === '' || !whitespacePatternRegExp.test(nextWord)) {
+                    // Record the current segment, ending after the current (newline) word
+                    segmentWordRanges.push({ start: segmentStartWordOffset, end: wordIndex + 1 });
+                    // The next segment starts at the word after the newline
+                    segmentStartWordOffset = wordIndex + 1;
+                    // Reset the newline flag for the new segment
+                    newlineSeenInCurrentSegment = false;
+                }
+            }
+            // If no non-whitespace word has been seen yet, check whether the current word is non-whitespace
+            if (nonWhitespaceWordSeen === false) {
+                nonWhitespaceWordSeen = !whitespacePatternRegExp.test(word);
+            }
+        }
+        // If the last segment didn't reach the end of the sequence, close it with the remaining words
+        if (segmentStartWordOffset < wordSequence.length) {
+            segmentWordRanges.push({ start: segmentStartWordOffset, end: wordSequence.length });
+        }
+    }
+    // These will hold the word offsets of each sentence, and (per segment) the range of sentence indices in it
+    const sentenceWordRanges = [];
+    const segmentSentenceRanges = [];
+    // Step 2: find sentence boundaries inside each segment.
+    // A sentence ends at a sentence-separator character (like '.', '!', '?') once it contains a minimum number of letters.
+    {
+        // A sentence must contain at least this many letters before a separator may end it
+        const minimumSentenceLetterCount = 2;
+        // Process each segment independently
+        for (let segmentIndex = 0; segmentIndex < segmentWordRanges.length; segmentIndex++) {
+            // Get the word range of the current segment
+            const segmentWordRange = segmentWordRanges[segmentIndex];
+            // Record the segment's first and last word index
+            const segmentStartWordIndex = segmentWordRange.start;
+            const segmentEndWordIndex = segmentWordRange.end;
+            // Add a placeholder entry for this segment's sentence range, starting (and initially ending) at the current sentence count
+            segmentSentenceRanges.push({ start: sentenceWordRanges.length, end: sentenceWordRanges.length });
+            // Whether a non-whitespace word has been seen in the current sentence
+            let nonWhitespaceWordSeen = false;
+            // The word offset at which the current sentence starts
+            let sentenceStartWordOffset = segmentStartWordIndex;
+            // The number of letters accumulated in the current sentence
+            let currentSentenceLetterCount = 0;
+            // Walk through the words of this segment
+            for (let wordIndex = segmentStartWordIndex; wordIndex < segmentEndWordIndex; wordIndex++) {
+                // Get the text of the current word
+                const word = wordSequence.getWordAt(wordIndex);
+                // If no non-whitespace word has been seen yet, check whether the current word is non-whitespace
+                if (nonWhitespaceWordSeen === false) {
+                    nonWhitespaceWordSeen = !whitespacePatternRegExp.test(word);
+                }
+                // Count letters, but only while the sentence hasn't yet reached the minimum (to save work)
+                if (currentSentenceLetterCount < minimumSentenceLetterCount) {
+                    // Find every letter character in the current word
+                    const matches = word.matchAll(letterPatternGlobalRegExp);
+                    // Count the letters, stopping early once the minimum is reached
+                    for (const _ of matches) {
+                        currentSentenceLetterCount += 1;
+                        if (currentSentenceLetterCount >= minimumSentenceLetterCount) {
+                            break;
+                        }
+                    }
+                }
+                // End the sentence here if it has content, enough letters, and the current word is a sentence-separator character
+                if (nonWhitespaceWordSeen && currentSentenceLetterCount >= minimumSentenceLetterCount && sentenceSeparatorCharacterRegExp.test(word)) {
+                    // Start scanning at the separator; this index marks where the sentence's trailing punctuation ends
+                    let trailingSequenceEndIndex = wordIndex;
+                    // Consume any trailing punctuation words (closing quotes, brackets, etc.) that belong to this sentence
+                    while (trailingSequenceEndIndex < segmentEndWordIndex) {
+                        // Get the text of the candidate trailing-punctuation word
+                        const trailingWord = wordSequence.getWordAt(trailingSequenceEndIndex);
+                        // Keep consuming it if it is trailing punctuation; otherwise stop
+                        if (sentenceSeparatorTrailingPunctuationCharacterRegExp.test(trailingWord)) {
+                            trailingSequenceEndIndex++;
+                        }
+                        else {
+                            break;
+                        }
+                    }
+                    // Record the completed sentence, from its start offset up to (but not including) the trailing punctuation
+                    sentenceWordRanges.push({
+                        start: sentenceStartWordOffset,
+                        end: trailingSequenceEndIndex
+                    });
+                    // Extend the current segment's sentence range to include the sentence we just recorded
+                    segmentSentenceRanges[segmentSentenceRanges.length - 1].end += 1;
+                    // The next sentence starts right after the trailing punctuation
+                    sentenceStartWordOffset = trailingSequenceEndIndex;
+                    // Reset the letter count for the next sentence
+                    currentSentenceLetterCount = 0;
+                    // Skip the words already consumed by the trailing-punctuation scan (the for loop will increment again)
+                    wordIndex = trailingSequenceEndIndex - 1;
+                }
+            }
+            // If the segment ends with words that don't complete a new sentence, close the final sentence at the segment end
+            if (sentenceStartWordOffset < segmentEndWordIndex) {
+                sentenceWordRanges.push({ start: sentenceStartWordOffset, end: segmentEndWordIndex });
+                segmentSentenceRanges[segmentSentenceRanges.length - 1].end += 1;
+            }
+        }
+    }
+    // This will hold the Sentence objects built from the sentence word ranges
+    const sentences = [];
+    // Build a Sentence object for every sentence range found above
+    for (const wordRange of sentenceWordRanges) {
+        // Extract the slice of the word sequence that belongs to this sentence
+        const sentenceWordSequence = wordSequence.slice(wordRange.start, wordRange.end);
+        // Create the Sentence for this range and append it to the list
+        sentences.push(new Sentence(wordRange, sentenceWordSequence));
+    }
+    // Step 3: split each sentence into phrases at phrase-separator characters (commas, colons, semicolons, etc.)
+    for (const sentence of sentences) {
+        // This will hold the start and end word offsets of each phrase in the current sentence
+        const phraseWordRanges = [];
+        // The word offset just past the end of this sentence
+        let sentenceEndWordOffset = sentence.wordRange.end;
+        // The word offset at which the current phrase starts
+        let phraseStartWordOffset = sentence.wordRange.start;
+        // Walk through the words of this sentence
+        for (let wordIndex = phraseStartWordOffset; wordIndex < sentenceEndWordOffset; wordIndex++) {
+            // Get the text of the current word
+            const currentWord = wordSequence.getWordAt(wordIndex);
+            // Determine whether this word is a phrase separator:
+            // it must match the phrase-separator pattern, and if the option is set, ':' or ';' only count when followed by whitespace
+            const isCurrentWordPhraseSeparator = phraseSeparatorRegExp.test(currentWord) &&
+                (!options.requireSpaceAfterColonsOrSemicolons ||
+                    (currentWord !== ':' && currentWord !== ';') ||
+                    whitespacePatternRegExp.test(wordSequence.getWordAt(wordIndex + 1)));
+            // If the current word is a phrase separator, close the current phrase at this word
+            if (isCurrentWordPhraseSeparator) {
+                // Tracks whether at least one whitespace word was seen while scanning trailing characters
+                let whitespaceSeenOnce = false;
+                // Include any trailing punctuation (closing quotes, brackets, etc.) in the current phrase
+                while (wordIndex < sentenceEndWordOffset - 1) {
+                    // Get the text of the next word
+                    const nextWord = wordSequence.getWordAt(wordIndex + 1);
+                    // Stop consuming if the next word is not trailing punctuation
+                    if (!sentenceSeparatorTrailingPunctuationCharacterRegExp.test(nextWord)) {
+                        break;
+                    }
+                    // Remember if this trailing word is whitespace
+                    if (whitespacePatternRegExp.test(nextWord)) {
+                        whitespaceSeenOnce = true;
+                    }
+                    // A closing double quote that follows whitespace starts the next phrase, so stop before it
+                    if (nextWord === '"' && whitespaceSeenOnce) {
+                        break;
+                    }
+                    // Otherwise consume this trailing-punctuation word
+                    wordIndex += 1;
+                }
+                // Record the phrase, from its start offset through the separator (and any trailing punctuation)
+                phraseWordRanges.push({
+                    start: phraseStartWordOffset,
+                    end: wordIndex + 1
+                });
+                // The next phrase starts right after the current separator
+                phraseStartWordOffset = wordIndex + 1;
+            }
+        }
+        // If the sentence ends with words that weren't part of any phrase, close the final phrase at the sentence end
+        if (phraseStartWordOffset < sentenceEndWordOffset) {
+            phraseWordRanges.push({ start: phraseStartWordOffset, end: sentenceEndWordOffset });
+        }
+        // Build a Phrase object for every phrase range and attach it to the sentence
+        for (const wordRange of phraseWordRanges) {
+            // Extract the slice of the word sequence that belongs to this phrase
+            const phraseWordSequence = wordSequence.slice(wordRange.start, wordRange.end);
+            // Create the Phrase for this range and add it to the sentence's phrase list
+            sentence.phrases.push(new Phrase(wordRange, phraseWordSequence));
+        }
+    }
+    // Assemble the final result: the full word sequence, the list of sentences, and the segment-to-sentence mapping
+    const result = {
+        words: wordSequence,
+        sentences,
+        segmentSentenceRanges,
+    };
+    // Return the assembled segmentation result
+    return result;
+}
+// Cache of compiled word-splitter regular expressions, keyed by the JSON-serialized options
+// (compiling a regex is expensive, so the same options reuse the same compiled expression)
+const cachedWordSplitterRegExps = new Map();
+// Splits a string of text into a WordSequence of individual words and punctuation marks
+// The returned sequence contains both the matched words and the punctuation found between/beside them
+export async function splitToWords(text, options) {
+    // If no options were given, start from an empty options object
+    if (!options) {
+        options = {};
+    }
+    // Fill in any option values the caller didn't provide with their defaults
+    options = { ...defaultSegmentationOptions, ...options };
+    // If a language was specified, normalize it to its short base code (e.g. "en-US" becomes "en")
+    // so that language-specific data is looked up consistently
+    if (options.language) {
+        options.language = getShortLanguageCode(options.language);
+    }
+    // Serialize the (now normalized) options to JSON so they can be used as a cache key
+    const optionsAsJson = JSON.stringify(options);
+    // Look up a previously compiled splitter expression for these exact options
+    let wordSplitterRegExp = cachedWordSplitterRegExps.get(optionsAsJson);
+    // If no cached expression exists, compile a new one and store it in the cache
+    if (!wordSplitterRegExp) {
+        wordSplitterRegExp = buildWordSplitterRegExpForOptions(options);
+        cachedWordSplitterRegExps.set(optionsAsJson, wordSplitterRegExp);
+    }
+    // Start with an empty word sequence that will accumulate the results
+    let wordSequence = new WordSequence();
+    // Adds the words found in a run of text between two matched words (typically punctuation and whitespace)
+    // to the word sequence. Spaces are skipped; every other character becomes its own punctuation word,
+    // except that consecutive punctuation characters are grouped into a single word.
+    function addPunctuationWordsBetween(startOffset, endOffset) {
+        // Extract the substring that lies between the two given character offsets
+        const punctuationWordSubstring = text.substring(startOffset, endOffset);
+        // The current character offset within the source text
+        let charOffset = startOffset;
+        // The character offset at which the current punctuation word starts
+        let punctuationWordStartOffset = startOffset;
+        // If any characters have been accumulated since the last flush, add them as a single punctuation word
+        function addPunctuationWordIfNeeded() {
+            // Check whether there are pending characters to add
+            if (charOffset > punctuationWordStartOffset) {
+                // Extract the pending word text from the source
+                const wordText = text.substring(punctuationWordStartOffset, charOffset);
+                // Add it to the sequence, marking it as punctuation
+                wordSequence.addWord(wordText, punctuationWordStartOffset, true);
+                // The next pending word (if any) starts at the current character offset
+                punctuationWordStartOffset = charOffset;
+            }
+        }
+        // Iterate over every character (codepoint) in the gap between words
+        for (const char of punctuationWordSubstring) {
+            // Spaces are skipped: they just advance the character offset and are not added as words
+            if (char === ' ') {
+                charOffset += 1;
+                continue;
+            }
+            // Flush any pending punctuation word before the current character
+            addPunctuationWordIfNeeded();
+            // Advance the character offset past the current character (char.length handles astral codepoints)
+            charOffset += char.length;
+            // Flush again so this single character becomes its own punctuation word
+            addPunctuationWordIfNeeded();
+        }
+        // Flush any remaining pending punctuation word at the end of the gap
+        addPunctuationWordIfNeeded();
+    }
+    // Find all word matches (as defined by the splitter expression) throughout the text
+    const wordMatches = text.matchAll(wordSplitterRegExp);
+    // Track the character offset at which the last matched word ended
+    let lastMatchEndOffset = 0;
+    // Process each matched word in order of appearance
+    if (wordMatches) {
+        for (const match of wordMatches) {
+            // Read the match's start and end character offsets (available because the regex has the 'd' flag)
+            const offsets = match.indices[0];
+            const matchStartOffset = offsets[0];
+            const matchEndOffset = offsets[1];
+            // If there is a gap between the previous match and this one, add the gap's punctuation as words
+            if (matchStartOffset > lastMatchEndOffset) {
+                addPunctuationWordsBetween(lastMatchEndOffset, matchStartOffset);
+            }
+            // Extract the matched word's text from the source
+            const wordText = text.substring(matchStartOffset, matchEndOffset);
+            // Add the matched word to the sequence (it is a real word, not punctuation)
+            wordSequence.addWord(wordText, matchStartOffset, false);
+            // Remember where this match ended so gaps after it can be found
+            lastMatchEndOffset = matchEndOffset;
+        }
+        // If the text extends beyond the last match, add the trailing punctuation as words
+        addPunctuationWordsBetween(lastMatchEndOffset, text.length);
+    }
+    // If East Asian postprocessing is enabled, further split runs of East Asian characters into subwords using ICU word breaking
+    if (options.enableEastAsianPostprocessing) {
+        wordSequence = await postprocessEastAsianWords(text, wordSequence);
+    }
+    // Return the resulting word sequence
+    return wordSequence;
+}
+// Add any missing punctuation words to a word sequence
+// Given a word sequence that was built from the source text (with its character offsets) and the source text itself,
+// produce a new word sequence that also includes every punctuation character found between, before, and after the words.
+// Also returns a mapping from each new entry's index back to the original word index it came from
+// (punctuation-only entries are not present as keys in this mapping).
+export function addMissingPunctuationWordsToWordSequence(wordSequence, sourceText) {
+    // Maps each index in the new (punctuation-included) sequence to the index of the original word it was copied from
+    const originalWordsReverseMapping = new Map();
+    // The new word sequence that will contain both the original words and the missing punctuation words
+    const wordSequenceWithPunctuation = new WordSequence();
+    // Adds entries for every character of a text slice, grouping runs of non-space characters into single punctuation words
+    // (spaces are merged into the preceding punctuation word, preserving them for offset correctness)
+    function addWordEntriesForTextSlice(textSlice, initialCharOffset) {
+        // The current character offset within the source text
+        let charOffset = initialCharOffset;
+        // Add entry for every codepoint (this will correctly treat characters beyond BMP)
+        for (const char of textSlice) {
+            // Compute the source-text offset just past this character
+            const charEndOffset = charOffset + char.length;
+            // Get the last entry added so far (if any)
+            const lastEntry = wordSequenceWithPunctuation.lastEntry;
+            // If this is a space and the last entry is a punctuation word that already starts with a space, extend that entry
+            if (char === ' ' && lastEntry && lastEntry.isPunctuation && lastEntry.text[0] === ' ') {
+                wordSequenceWithPunctuation.lastEntry.text += ' ';
+                wordSequenceWithPunctuation.lastEntry.endOffset = charEndOffset;
+            }
+            else {
+                // Otherwise, extract this single character from the source and add it as a new punctuation word
+                const wordText = sourceText.substring(charOffset, charEndOffset);
+                wordSequenceWithPunctuation.addWord(wordText, charOffset, true);
+            }
+            // Advance the character offset past the current character
+            charOffset = charEndOffset;
+        }
+    }
+    // Process each original word in order
+    for (let wordIndex = 0; wordIndex < wordSequence.length; wordIndex++) {
+        // Get the current word entry and the offset where it starts in the source text
+        const wordEntry = wordSequence.getEntryAt(wordIndex);
+        const wordStartOffset = wordEntry.startOffset;
+        // Get the end offset of the previous word (or 0 if this is the first word)
+        const previousWordEndOffset = wordIndex > 0 ? wordSequence.entries[wordIndex - 1].endOffset : 0;
+        // Add entries for any punctuation characters between the current and previous word (or the start of the text)
+        if (previousWordEndOffset !== wordStartOffset) {
+            // Extract the slice of source text lying between the two words
+            const textSlice = sourceText.substring(previousWordEndOffset, wordStartOffset);
+            // Add entries for the punctuation characters in that slice
+            addWordEntriesForTextSlice(textSlice, previousWordEndOffset);
+        }
+        // Copy the original word entry into the new sequence
+        wordSequenceWithPunctuation.entries.push(wordEntry);
+        // Record that this new entry corresponds to the current original word index
+        originalWordsReverseMapping.set(wordSequenceWithPunctuation.length - 1, wordIndex);
+        // If last word, add entries for any trailing punctuation characters
+        if (wordIndex === wordSequence.length - 1) {
+            // If the source text extends beyond the last word, the remainder is trailing punctuation
+            if (sourceText.length !== wordEntry.endOffset) {
+                // Extract the slice of source text after the last word
+                const textSlice = sourceText.substring(wordEntry.endOffset, sourceText.length);
+                // Add entries for the trailing punctuation characters
+                addWordEntriesForTextSlice(textSlice, wordEntry.endOffset);
+            }
+        }
+    }
+    // Return the enriched word sequence along with the reverse mapping to the original words
+    return { wordSequenceWithPunctuation, originalWordsReverseMapping };
+}
+////////////////////////////////////////////////////////////////////////////////////////////////
+// Helper methods
+////////////////////////////////////////////////////////////////////////////////////////////////
+// Builds a global regular expression that matches words in the given language, using the configured options
+function buildWordSplitterRegExpForOptions(options) {
+    // Get the CLDR suppression list (standard abbreviations like "Dr.") for this language, or an empty list if none exists
+    const cldrSuppressionsForLang = cldrSuppressions[options.language ?? ''] ?? [];
+    // Get the extended, project-specific suppression list for this language, or an empty list
+    const extendedSuppressionsForLang = additionalSuppressions[options.language ?? ''] ?? [];
+    // Get the leading-apostrophe contraction suppressions for this language, or an empty list
+    const contractionSuppressionsForLang = leadingApostropheContractionSuppressions[options.language ?? ''] ?? [];
+    // Build a variant of those contractions that uses the typographic apostrophe (') instead of the straight one (')
+    const contractionSuppressionsForLangWithSingleQuote = contractionSuppressionsForLang.map(str => str.replaceAll(`'`, `’`));
+    // Get any custom suppressions the caller provided, or an empty list
+    const customSuppressions = options.customSuppressions ?? [];
+    // Combine all suppression sources into one flat list
+    let suppressions = [
+        ...customSuppressions,
+        ...cldrSuppressionsForLang,
+        ...extendedSuppressionsForLang,
+        ...contractionSuppressionsForLang,
+        ...contractionSuppressionsForLangWithSingleQuote,
+        ...nounSuppressions,
+        ...tldSuppressions,
+    ];
+    // Build the word pattern: match any suppression (in original, lowercase, and uppercase forms) or a regular word
+    const wordPattern = buildWordSplitterPattern([
+        ...suppressions,
+        ...suppressions.map(word => word.toLocaleLowerCase()),
+        ...suppressions.map(word => word.toLocaleUpperCase()),
+    ]);
+    // Compile the pattern into a global regular expression (the 'd' flag, added by regexp-composer, gives match indices)
+    const wordSplitterRegExp = buildRegExp(wordPattern, { global: true });
+    // Return the compiled regular expression
+    return wordSplitterRegExp;
+}
+// Tries to load the ICU segmentation WebAssembly module; returns undefined if it is not installed
+async function getIcuSegmentation() {
+    // Attempt to dynamically import the ICU segmentation module
+    try {
+        const icuSegmentation = await import('@echogarden/icu-segmentation-wasm');
+        // If the import succeeded, return the module
+        return icuSegmentation;
+    }
+    catch {
+        // If the import failed (module not available), signal that ICU segmentation is unavailable
+        return undefined;
+    }
+}
+// Further splits words that contain East Asian characters (Chinese, Japanese, Thai, Khmer) into smaller subwords
+// using ICU's word-break rules, which are much better suited to these scripts than the generic splitter pattern
+async function postprocessEastAsianWords(containingText, wordSequence) {
+    // Try to load the ICU segmentation module
+    const icuSegmentation = await getIcuSegmentation();
+    // If ICU segmentation is unavailable, return the original word sequence unchanged
+    if (icuSegmentation === undefined) {
+        return wordSequence;
+    }
+    // Tracks whether the ICU module has been initialized (initialization only needs to happen once)
+    let icuInitialized = false;
+    // A new word sequence that will hold the postprocessed words
+    const newWordSequence = new WordSequence();
+    // Process each word in the original sequence
+    for (let wordIndex = 0; wordIndex < wordSequence.length; wordIndex++) {
+        // Get the current word entry and the offset where it starts in the source text
+        const wordEntry = wordSequence.entries[wordIndex];
+        const wordStartOffset = wordEntry.startOffset;
+        // Get the text of the current word
+        const word = wordSequence.getWordAt(wordIndex);
+        // If the word contains East Asian characters, split it into subwords
+        if (eastAsianCharRangesRegExp.test(word)) {
+            // Initialize the ICU module on first use
+            if (!icuInitialized) {
+                await icuSegmentation.initialize();
+                icuInitialized = true;
+            }
+            // Get the word-break positions within the word (offsets into the word string)
+            const wordBreaks = [...icuSegmentation.createWordBreakIterator(word)];
+            // Create a subword for every pair of consecutive break positions
+            for (let i = 0; i < wordBreaks.length - 1; i++) {
+                // Compute the subword's start and end offsets within the source text
+                const subwordStartOffset = wordStartOffset + wordBreaks[i];
+                const subwordEndOffset = wordStartOffset + wordBreaks[i + 1];
+                // Extract the subword text from the containing text
+                const subwordText = containingText.substring(subwordStartOffset, subwordEndOffset);
+                // Add the subword to the new sequence (it is a real word, not punctuation)
+                newWordSequence.addWord(subwordText, subwordStartOffset, false);
+            }
+        }
+        else {
+            // Otherwise, extract the word text from the source and copy it to the new sequence unchanged
+            const wordText = containingText.substring(wordEntry.startOffset, wordEntry.endOffset);
+            newWordSequence.addWord(wordText, wordEntry.startOffset, wordEntry.isPunctuation);
+        }
+    }
+    // Return the postprocessed word sequence
+    return newWordSequence;
+}
+// Computes the character ranges of all punctuation in the text.
+// This includes gaps between words, entries that are themselves punctuation, and any leading/trailing text
+// that is not covered by a word entry.
+function getPunctuationRanges(wordSequence, text) {
+    // This will hold the character ranges of the punctuation found
+    const punctuationRanges = [];
+    // Get the list of word entries
+    const wordEntries = wordSequence.entries;
+    // If the text starts before the first word, the leading part is punctuation
+    if (wordEntries[0].startOffset > 0) {
+        punctuationRanges.push({ start: 0, end: wordEntries[0].startOffset });
+    }
+    // Examine every word entry
+    for (let i = 0; i < wordEntries.length; i++) {
+        // Get the current entry
+        const entry = wordEntries[i];
+        // Get the end offset of the previous entry (or 0 for the first entry)
+        const previousEndOffset = wordEntries[i - 1]?.endOffset ?? 0;
+        // If there is a gap between the previous entry and this one, that gap is punctuation
+        if (entry.startOffset > previousEndOffset) {
+            punctuationRanges.push({ start: previousEndOffset, end: entry.startOffset });
+        }
+        // If the current entry is itself a punctuation word, its whole range is punctuation
+        if (entry.isPunctuation) {
+            punctuationRanges.push({ start: entry.startOffset, end: entry.endOffset });
+        }
+    }
+    {
+        // Get the end offset of the last entry (if any)
+        const lastEndOffset = wordEntries[wordEntries.length - 1]?.endOffset;
+        // If the text extends beyond the last entry, the trailing part is punctuation
+        if (lastEndOffset && lastEndOffset < text.length) {
+            punctuationRanges.push({ start: lastEndOffset, end: text.length });
+        }
+    }
+    // Return the collected punctuation ranges
+    return punctuationRanges;
+}
+// A fragment of text (either a sentence or a phrase): it holds the word range it occupies and its words
+// (this is the shared base class of Sentence and Phrase)
+export class TextFragment {
+    // The range of word indices this fragment covers (in the overall word sequence)
+    wordRange;
+    // The words that make up this fragment
+    words;
+    // Creates a fragment from a word range and the corresponding slice of words
+    constructor(wordRange, words) {
+        this.wordRange = wordRange;
+        this.words = words;
+    }
+    // Returns the fragment's text by concatenating all of its words
+    get text() {
+        return this.words.text;
+    }
+    // Returns the character range of this fragment within the source text,
+    // from the start of its first word to the end of its last word
+    get charRange() {
+        return {
+            start: this.words.firstEntry.startOffset,
+            end: this.words.lastEntry.endOffset
+        };
+    }
+}
+// A sentence: a text fragment that additionally contains the phrases it was split into
+// (phrases are filled in during segmentation)
+export class Sentence extends TextFragment {
+    // The phrases that make up this sentence
+    phrases = [];
+}
+// A phrase: a text fragment representing one part of a sentence
+// (it has no additional members; it exists to give phrases their own type)
+export class Phrase extends TextFragment {
+}
+// Default options for word splitting: no language, no custom suppressions, and East Asian postprocessing enabled
+export const defaultSegmentationOptions = {
+    language: '',
+    customSuppressions: [],
+    enableEastAsianPostprocessing: true,
+};
+// Default options for word-sequence segmentation: ':' and ';' only count as phrase separators when followed by a space
+export const defaultWordSequenceSegmentationOptions = {
+    requireSpaceAfterColonsOrSemicolons: true
+};
+//# sourceMappingURL=TextSegmentation.js.map

package/dist/segmentation/TextSegmentation.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"TextSegmentation.js","sourceRoot":"","sources":["../../src/segmentation/TextSegmentation.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,wBAAwB,IAAI,wBAAwB,EAAE,qBAAqB,EAAE,mDAAmD,EAAE,gCAAgC,EAAE,uBAAuB,EAAE,yBAAyB,EAA0D,MAAM,yBAAyB,CAAA;AACxT,OAAO,EAAE,gBAAgB,EAAE,sBAAsB,EAAE,wCAAwC,EAAE,gBAAgB,EAAE,eAAe,EAAE,MAAM,6BAA6B,CAAA;AACnK,OAAO,EAAE,yBAAyB,EAAE,MAAM,2CAA2C,CAAA;AACrF,OAAO,EAAE,YAAY,EAAE,MAAM,mBAAmB,CAAA;AAChD,OAAO,EAAE,oBAAoB,EAAE,MAAM,2BAA2B,CAAA;AAEhE,OAAO,EAAE,WAAW,EAAE,MAAM,iBAAiB,CAAA;AAE7C,gGAAgG;AAChG,mBAAmB;AACnB,gGAAgG;AAEhG,2GAA2G;AAC3G,0GAA0G;AAC1G,MAAM,CAAC,KAAK,UAAU,WAAW,CAAC,IAAY,EAAE,OAA6B;IAC5E,gGAAgG;IAChG,MAAM,YAAY,GAAG,MAAM,YAAY,CAAC,IAAI,EAAE,OAAO,CAAC,CAAA;IAEtD,sGAAsG;IACtG,OAAO,mBAAmB,CAAC,YAAY,CAAC,CAAA;AACzC,CAAC;AAED,iGAAiG;AACjG,4EAA4E;AAC5E,kFAAkF;AAClF,MAAM,CAAC,KAAK,UAAU,mBAAmB,CAAC,YAA0B,EAAE,OAAyC;IAC9G,0EAA0E;IAC1E,OAAO,GAAG,EAAE,GAAG,sCAAsC,EAAE,GAAG,OAAO,EAAE,CAAA;IAEnE,4FAA4F;IAC5F,MAAM,iBAAiB,GAAY,EAAE,CAAA;IAErC,+CAA+C;IAC/C,sGAAsG;IACtG,CAAC;QACA,sDAAsD;QACtD,IAAI,sBAAsB,GAAG,CAAC,CAAA;QAC9B,gFAAgF;QAChF,IAAI,qBAAqB,GAAG,KAAK,CAAA;QACjC,wFAAwF;QACxF,IAAI,2BAA2B,GAAG,KAAK,CAAA;QAEvC,0CAA0C;QAC1C,KAAK,IAAI,SAAS,GAAG,CAAC,EAAE,SAAS,GAAG,YAAY,CAAC,MAAM,EAAE,SAAS,EAAE,EAAE,CAAC;YACtE,gDAAgD;YAChD,MAAM,IAAI,GAAG,YAAY,CAAC,SAAS,CAAC,SAAS,CAAC,CAAA;YAE9C,0HAA0H;YAC1H,IAAI,CAAC,2BAA2B,IAAI,qBAAqB,IAAI,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,CAAC;gBAClF,2BAA2B,GAAG,IAAI,CAAA;YACnC,CAAC;YAED,+EAA+E;YAC/E,IAAI,2BAA2B,EAAE,CAAC;gBACjC,0FAA0F;gBAC1F,MAAM,QAAQ,GAAG,SAAS,GAAG,YAAY,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,YAAY,CAAC,SAAS,CAAC,SAAS,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,CAAA;gBAEjG,yFAAyF;gBACzF,IAAI,QAAQ,KAAK,EAAE,IAAI,CAAC,uBAAuB,CAAC,IAAI,CAAC,QAAQ,CAAC,EAAE,CAAC;oBAChE,sEAAsE;oBACtE,iBAAiB,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,sBAAsB,EAAE,GAAG,EAAE,SAAS,GAAG,CAAC,EAAE,CAAC,CAAA;oBAC7E,wDAAwD;oBACxD,sBAAsB,GAAG,SAAS,GAAG,CAAC,CAAA;oBAEtC,6CAA6C;oBAC7C,2BAA2B,GAAG,KAAK,CAAA;gBACpC,CAAC;YACF,CAAC;YAED,gGAAgG;YAChG,IAAI,qBAAqB,KAAK,KAAK,EAAE,CAAC;gBACrC,qBAAqB,GAAG,CAAC,uBAAuB,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;YAC5D,CAAC;QACF,CAAC;QAED,8FAA8F;QAC9F,IAAI,sBAAsB,GAAG,YAAY,CAAC,MAAM,EAAE,CAAC;YAClD,iBAAiB,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,sBAAsB,EAAE,GAAG,EAAE,YAAY,CAAC,MAAM,EAAE,CAAC,CAAA;QACpF,CAAC;IACF,CAAC;IAED,2GAA2G;IAC3G,MAAM,kBAAkB,GAAY,EAAE,CAAA;IACtC,MAAM,qBAAqB,GAAY,EAAE,CAAA;IAEzC,wDAAwD;IACxD,uHAAuH;IACvH,CAAC;QACA,mFAAmF;QACnF,MAAM,0BAA0B,GAAG,CAAC,CAAA;QAEpC,qCAAqC;QACrC,KAAK,IAAI,YAAY,GAAG,CAAC,EAAE,YAAY,GAAG,iBAAiB,CAAC,MAAM,EAAE,YAAY,EAAE,EAAE,CAAC;YACpF,4CAA4C;YAC5C,MAAM,gBAAgB,GAAG,iBAAiB,CAAC,YAAY,CAAC,CAAA;YAExD,iDAAiD;YACjD,MAAM,qBAAqB,GAAG,gBAAgB,CAAC,KAAK,CAAA;YACpD,MAAM,mBAAmB,GAAG,gBAAgB,CAAC,GAAG,CAAA;YAEhD,2HAA2H;YAC3H,qBAAqB,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,kBAAkB,CAAC,MAAM,EAAE,GAAG,EAAE,kBAAkB,CAAC,MAAM,EAAE,CAAC,CAAA;YAEhG,sEAAsE;YACtE,IAAI,qBAAqB,GAAG,KAAK,CAAA;YACjC,uDAAuD;YACvD,IAAI,uBAAuB,GAAG,qBAAqB,CAAA;YACnD,4DAA4D;YAC5D,IAAI,0BAA0B,GAAG,CAAC,CAAA;YAElC,yCAAyC;YACzC,KAAK,IAAI,SAAS,GAAG,qBAAqB,EAAE,SAAS,GAAG,mBAAmB,EAAE,SAAS,EAAE,EAAE,CAAC;gBAC1F,mCAAmC;gBACnC,MAAM,IAAI,GAAG,YAAY,CAAC,SAAS,CAAC,SAAS,CAAC,CAAA;gBAE9C,gGAAgG;gBAChG,IAAI,qBAAqB,KAAK,KAAK,EAAE,CAAC;oBACrC,qBAAqB,GAAG,CAAC,uBAAuB,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;gBAC5D,CAAC;gBAED,2FAA2F;gBAC3F,IAAI,0BAA0B,GAAG,0BAA0B,EAAE,CAAC;oBAC7D,kDAAkD;oBAClD,MAAM,OAAO,GAAG,IAAI,CAAC,QAAQ,CAAC,yBAAyB,CAAC,CAAA;oBAExD,gEAAgE;oBAChE,KAAK,MAAM,CAAC,IAAI,OAAO,EAAE,CAAC;wBACzB,0BAA0B,IAAI,CAAC,CAAA;wBAE/B,IAAI,0BAA0B,IAAI,0BAA0B,EAAE,CAAC;4BAC9D,MAAK;wBACN,CAAC;oBACF,CAAC;gBACF,CAAC;gBAED,kHAAkH;gBAClH,IAAI,qBAAqB,IAAI,0BAA0B,IAAI,0BAA0B,IAAI,gCAAgC,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;oBACtI,mGAAmG;oBACnG,IAAI,wBAAwB,GAAG,SAAS,CAAA;oBAExC,uGAAuG;oBACvG,OAAO,wBAAwB,GAAG,mBAAmB,EAAE,CAAC;wBACvD,0DAA0D;wBAC1D,MAAM,YAAY,GAAG,YAAY,CAAC,SAAS,CAAC,wBAAwB,CAAC,CAAA;wBAErE,kEAAkE;wBAClE,IAAI,mDAAmD,CAAC,IAAI,CAAC,YAAY,CAAC,EAAE,CAAC;4BAC5E,wBAAwB,EAAE,CAAA;wBAC3B,CAAC;6BAAM,CAAC;4BACP,MAAK;wBACN,CAAC;oBACF,CAAC;oBAED,0GAA0G;oBAC1G,kBAAkB,CAAC,IAAI,CAAC;wBACvB,KAAK,EAAE,uBAAuB;wBAC9B,GAAG,EAAE,wBAAwB;qBAC7B,CAAC,CAAA;oBAEF,uFAAuF;oBACvF,qBAAqB,CAAC,qBAAqB,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC,CAAA;oBAEhE,gEAAgE;oBAChE,uBAAuB,GAAG,wBAAwB,CAAA;oBAClD,+CAA+C;oBAC/C,0BAA0B,GAAG,CAAC,CAAA;oBAE9B,uGAAuG;oBACvG,SAAS,GAAG,wBAAwB,GAAG,CAAC,CAAA;gBACzC,CAAC;YACF,CAAC;YAED,iHAAiH;YACjH,IAAI,uBAAuB,GAAG,mBAAmB,EAAE,CAAC;gBACnD,kBAAkB,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,uBAAuB,EAAE,GAAG,EAAE,mBAAmB,EAAE,CAAC,CAAA;gBACrF,qBAAqB,CAAC,qBAAqB,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC,CAAA;YACjE,CAAC;QACF,CAAC;IACF,CAAC;IAED,0EAA0E;IAC1E,MAAM,SAAS,GAAe,EAAE,CAAA;IAEhC,+DAA+D;IAC/D,KAAK,MAAM,SAAS,IAAI,kBAAkB,EAAE,CAAC;QAC5C,uEAAuE;QACvE,MAAM,oBAAoB,GAAG,YAAY,CAAC,KAAK,CAAC,SAAS,CAAC,KAAK,EAAE,SAAS,CAAC,GAAG,CAAC,CAAA;QAE/E,+DAA+D;QAC/D,SAAS,CAAC,IAAI,CAAC,IAAI,QAAQ,CAAC,SAAS,EAAE,oBAAoB,CAAC,CAAC,CAAA;IAC9D,CAAC;IAED,6GAA6G;IAC7G,KAAK,MAAM,QAAQ,IAAI,SAAS,EAAE,CAAC;QAClC,uFAAuF;QACvF,MAAM,gBAAgB,GAAY,EAAE,CAAA;QAEpC,qDAAqD;QACrD,IAAI,qBAAqB,GAAG,QAAQ,CAAC,SAAS,CAAC,GAAG,CAAA;QAClD,qDAAqD;QACrD,IAAI,qBAAqB,GAAG,QAAQ,CAAC,SAAS,CAAC,KAAK,CAAA;QAEpD,0CAA0C;QAC1C,KAAK,IAAI,SAAS,GAAG,qBAAqB,EAAE,SAAS,GAAG,qBAAqB,EAAE,SAAS,EAAE,EAAE,CAAC;YAC5F,mCAAmC;YACnC,MAAM,WAAW,GAAG,YAAY,CAAC,SAAS,CAAC,SAAS,CAAC,CAAA;YAErD,qDAAqD;YACrD,0HAA0H;YAC1H,MAAM,4BAA4B,GACjC,qBAAqB,CAAC,IAAI,CAAC,WAAW,CAAC;gBACvC,CAAC,CAAC,OAAO,CAAC,mCAAmC;oBAC5C,CAAC,WAAW,KAAK,GAAG,IAAI,WAAW,KAAK,GAAG,CAAC;oBAC5C,uBAAuB,CAAC,IAAI,CAAC,YAAY,CAAC,SAAS,CAAC,SAAS,GAAG,CAAC,CAAC,CAAC,CAAC,CAAA;YAEtE,mFAAmF;YACnF,IAAI,4BAA4B,EAAE,CAAC;gBAClC,0FAA0F;gBAC1F,IAAI,kBAAkB,GAAG,KAAK,CAAA;gBAE9B,0FAA0F;gBAC1F,OAAO,SAAS,GAAG,qBAAqB,GAAG,CAAC,EAAE,CAAC;oBAC9C,gCAAgC;oBAChC,MAAM,QAAQ,GAAG,YAAY,CAAC,SAAS,CAAC,SAAS,GAAG,CAAC,CAAC,CAAA;oBAEtD,8DAA8D;oBAC9D,IAAI,CAAC,mDAAmD,CAAC,IAAI,CAAC,QAAQ,CAAC,EAAE,CAAC;wBACzE,MAAK;oBACN,CAAC;oBAED,+CAA+C;oBAC/C,IAAI,uBAAuB,CAAC,IAAI,CAAC,QAAQ,CAAC,EAAE,CAAC;wBAC5C,kBAAkB,GAAG,IAAI,CAAA;oBAC1B,CAAC;oBAED,2FAA2F;oBAC3F,IAAI,QAAQ,KAAK,GAAG,IAAI,kBAAkB,EAAE,CAAC;wBAC5C,MAAK;oBACN,CAAC;oBAED,mDAAmD;oBACnD,SAAS,IAAI,CAAC,CAAA;gBACf,CAAC;gBAED,gGAAgG;gBAChG,gBAAgB,CAAC,IAAI,CAAC;oBACrB,KAAK,EAAE,qBAAqB;oBAC5B,GAAG,EAAE,SAAS,GAAG,CAAC;iBAClB,CAAC,CAAA;gBAEF,2DAA2D;gBAC3D,qBAAqB,GAAG,SAAS,GAAG,CAAC,CAAA;YACtC,CAAC;QACF,CAAC;QAED,8GAA8G;QAC9G,IAAI,qBAAqB,GAAG,qBAAqB,EAAE,CAAC;YACnD,gBAAgB,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,qBAAqB,EAAE,GAAG,EAAE,qBAAqB,EAAE,CAAC,CAAA;QACpF,CAAC;QAED,6EAA6E;QAC7E,KAAK,MAAM,SAAS,IAAI,gBAAgB,EAAE,CAAC;YAC1C,qEAAqE;YACrE,MAAM,kBAAkB,GAAG,YAAY,CAAC,KAAK,CAAC,SAAS,CAAC,KAAK,EAAE,SAAS,CAAC,GAAG,CAAC,CAAA;YAE7E,4EAA4E;YAC5E,QAAQ,CAAC,OAAO,CAAC,IAAI,CAAC,IAAI,MAAM,CAAC,SAAS,EAAE,kBAAkB,CAAC,CAAC,CAAA;QACjE,CAAC;IACF,CAAC;IAED,gHAAgH;IAChH,MAAM,MAAM,GAAuB;QAClC,KAAK,EAAE,YAAY;QACnB,SAAS;QACT,qBAAqB;KACrB,CAAA;IAED,2CAA2C;IAC3C,OAAO,MAAM,CAAA;AACd,CAAC;AAED,4FAA4F;AAC5F,2FAA2F;AAC3F,MAAM,yBAAyB,GAAG,IAAI,GAAG,EAAkB,CAAA;AAE3D,wFAAwF;AACxF,sGAAsG;AACtG,MAAM,CAAC,KAAK,UAAU,YAAY,CAAC,IAAY,EAAE,OAA6B;IAC7E,+DAA+D;IAC/D,IAAI,CAAC,OAAO,EAAE,CAAC;QACd,OAAO,GAAG,EAAE,CAAA;IACb,CAAC;IAED,0EAA0E;IAC1E,OAAO,GAAG,EAAE,GAAG,0BAA0B,EAAE,GAAG,OAAO,EAAE,CAAA;IAEvD,+FAA+F;IAC/F,2DAA2D;IAC3D,IAAI,OAAO,CAAC,QAAQ,EAAE,CAAC;QACtB,OAAO,CAAC,QAAQ,GAAG,oBAAoB,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAA;IAC1D,CAAC;IAED,oFAAoF;IACpF,MAAM,aAAa,GAAG,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,CAAA;IAE7C,4EAA4E;IAC5E,IAAI,kBAAkB,GAAG,yBAAyB,CAAC,GAAG,CAAC,aAAa,CAAC,CAAA;IAErE,8EAA8E;IAC9E,IAAI,CAAC,kBAAkB,EAAE,CAAC;QACzB,kBAAkB,GAAG,iCAAiC,CAAC,OAAO,CAAC,CAAA;QAE/D,yBAAyB,CAAC,GAAG,CAAC,aAAa,EAAE,kBAAkB,CAAC,CAAA;IACjE,CAAC;IAED,qEAAqE;IACrE,IAAI,YAAY,GAAG,IAAI,YAAY,EAAE,CAAA;IAErC,yGAAyG;IACzG,oGAAoG;IACpG,iFAAiF;IACjF,SAAS,0BAA0B,CAAC,WAAmB,EAAE,SAAiB;QACzE,0EAA0E;QAC1E,MAAM,wBAAwB,GAAG,IAAI,CAAC,SAAS,CAAC,WAAW,EAAE,SAAS,CAAC,CAAA;QAEvE,sDAAsD;QACtD,IAAI,UAAU,GAAG,WAAW,CAAA;QAC5B,oEAAoE;QACpE,IAAI,0BAA0B,GAAG,WAAW,CAAA;QAE5C,sGAAsG;QACtG,SAAS,0BAA0B;YAClC,oDAAoD;YACpD,IAAI,UAAU,GAAG,0BAA0B,EAAE,CAAC;gBAC7C,gDAAgD;gBAChD,MAAM,QAAQ,GAAG,IAAI,CAAC,SAAS,CAAC,0BAA0B,EAAE,UAAU,CAAC,CAAA;gBACvE,oDAAoD;gBACpD,YAAY,CAAC,OAAO,CAAC,QAAQ,EAAE,0BAA0B,EAAE,IAAI,CAAC,CAAA;gBAEhE,wEAAwE;gBACxE,0BAA0B,GAAG,UAAU,CAAA;YACxC,CAAC;QACF,CAAC;QAED,oEAAoE;QACpE,KAAK,MAAM,IAAI,IAAI,wBAAwB,EAAE,CAAC;YAC7C,wFAAwF;YACxF,IAAI,IAAI,KAAK,GAAG,EAAE,CAAC;gBAClB,UAAU,IAAI,CAAC,CAAA;gBAEf,SAAQ;YACT,CAAC;YAED,kEAAkE;YAClE,0BAA0B,EAAE,CAAA;YAE5B,kGAAkG;YAClG,UAAU,IAAI,IAAI,CAAC,MAAM,CAAA;YAEzB,wEAAwE;YACxE,0BAA0B,EAAE,CAAA;QAC7B,CAAC;QAED,qEAAqE;QACrE,0BAA0B,EAAE,CAAA;IAC7B,CAAC;IAED,oFAAoF;IACpF,MAAM,WAAW,GAAG,IAAI,CAAC,QAAQ,CAAC,kBAAkB,CAAC,CAAA;IAErD,kEAAkE;IAClE,IAAI,kBAAkB,GAAG,CAAC,CAAA;IAE1B,mDAAmD;IACnD,IAAI,WAAW,EAAE,CAAC;QACjB,KAAK,MAAM,KAAK,IAAI,WAAW,EAAE,CAAC;YACjC,kGAAkG;YAClG,MAAM,OAAO,GAAG,KAAK,CAAC,OAAQ,CAAC,CAAC,CAAE,CAAA;YAClC,MAAM,gBAAgB,GAAG,OAAO,CAAC,CAAC,CAAC,CAAA;YACnC,MAAM,cAAc,GAAG,OAAO,CAAC,CAAC,CAAC,CAAA;YAEjC,gGAAgG;YAChG,IAAI,gBAAgB,GAAG,kBAAkB,EAAE,CAAC;gBAC3C,0BAA0B,CAAC,kBAAkB,EAAE,gBAAgB,CAAC,CAAA;YACjE,CAAC;YAED,kDAAkD;YAClD,MAAM,QAAQ,GAAG,IAAI,CAAC,SAAS,CAAC,gBAAgB,EAAE,cAAc,CAAC,CAAA;YACjE,4EAA4E;YAC5E,YAAY,CAAC,OAAO,CAAC,QAAQ,EAAE,gBAAgB,EAAE,KAAK,CAAC,CAAA;YAEvD,gEAAgE;YAChE,kBAAkB,GAAG,cAAc,CAAA;QACpC,CAAC;QAED,mFAAmF;QACnF,0BAA0B,CAAC,kBAAkB,EAAE,IAAI,CAAC,MAAM,CAAC,CAAA;IAC5D,CAAC;IAED,6HAA6H;IAC7H,IAAI,OAAO,CAAC,6BAA6B,EAAE,CAAC;QAC3C,YAAY,GAAG,MAAM,yBAAyB,CAAC,IAAI,EAAE,YAAY,CAAC,CAAA;IACnE,CAAC;IAED,qCAAqC;IACrC,OAAO,YAAY,CAAA;AACpB,CAAC;AAED,uDAAuD;AACvD,qHAAqH;AACrH,yHAAyH;AACzH,kGAAkG;AAClG,sEAAsE;AACtE,MAAM,UAAU,wCAAwC,CAAC,YAA0B,EAAE,UAAkB;IACtG,kHAAkH;IAClH,MAAM,2BAA2B,GAAG,IAAI,GAAG,EAAkB,CAAA;IAE7D,oGAAoG;IACpG,MAAM,2BAA2B,GAAG,IAAI,YAAY,EAAE,CAAA;IAEtD,wHAAwH;IACxH,kGAAkG;IAClG,SAAS,0BAA0B,CAAC,SAAiB,EAAE,iBAAyB;QAC/E,sDAAsD;QACtD,IAAI,UAAU,GAAG,iBAAiB,CAAA;QAElC,kFAAkF;QAClF,KAAK,MAAM,IAAI,IAAI,SAAS,EAAE,CAAC;YAC9B,0DAA0D;YAC1D,MAAM,aAAa,GAAG,UAAU,GAAG,IAAI,CAAC,MAAM,CAAA;YAE9C,2CAA2C;YAC3C,MAAM,SAAS,GAAG,2BAA2B,CAAC,SAAS,CAAA;YAEvD,kHAAkH;YAClH,IAAI,IAAI,KAAK,GAAG,IAAI,SAAS,IAAI,SAAS,CAAC,aAAa,IAAI,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC,KAAK,GAAG,EAAE,CAAC;gBACvF,2BAA2B,CAAC,SAAS,CAAC,IAAI,IAAI,GAAG,CAAA;gBACjD,2BAA2B,CAAC,SAAS,CAAC,SAAS,GAAG,aAAa,CAAA;YAChE,CAAC;iBAAM,CAAC;gBACP,gGAAgG;gBAChG,MAAM,QAAQ,GAAG,UAAU,CAAC,SAAS,CAAC,UAAU,EAAE,aAAa,CAAC,CAAA;gBAEhE,2BAA2B,CAAC,OAAO,CAAC,QAAQ,EAAE,UAAU,EAAE,IAAI,CAAC,CAAA;YAChE,CAAC;YAED,0DAA0D;YAC1D,UAAU,GAAG,aAAa,CAAA;QAC3B,CAAC;IACF,CAAC;IAED,sCAAsC;IACtC,KAAK,IAAI,SAAS,GAAG,CAAC,EAAE,SAAS,GAAG,YAAY,CAAC,MAAM,EAAE,SAAS,EAAE,EAAE,CAAC;QACtE,+EAA+E;QAC/E,MAAM,SAAS,GAAG,YAAY,CAAC,UAAU,CAAC,SAAS,CAAC,CAAA;QACpD,MAAM,eAAe,GAAG,SAAS,CAAC,WAAW,CAAA;QAE7C,2EAA2E;QAC3E,MAAM,qBAAqB,GAAG,SAAS,GAAG,CAAC,CAAC,CAAC,CAAC,YAAY,CAAC,OAAO,CAAC,SAAS,GAAG,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,CAAA;QAE/F,8GAA8G;QAC9G,IAAI,qBAAqB,KAAK,eAAe,EAAE,CAAC;YAC/C,+DAA+D;YAC/D,MAAM,SAAS,GAAG,UAAU,CAAC,SAAS,CAAC,qBAAqB,EAAE,eAAe,CAAC,CAAA;YAE9E,2DAA2D;YAC3D,0BAA0B,CAAC,SAAS,EAAE,qBAAqB,CAAC,CAAA;QAC7D,CAAC;QAED,qDAAqD;QACrD,2BAA2B,CAAC,OAAO,CAAC,IAAI,CAAC,SAAS,CAAC,CAAA;QACnD,4EAA4E;QAC5E,2BAA2B,CAAC,GAAG,CAAC,2BAA2B,CAAC,MAAM,GAAG,CAAC,EAAE,SAAS,CAAC,CAAA;QAElF,oEAAoE;QACpE,IAAI,SAAS,KAAK,YAAY,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAC3C,yFAAyF;YACzF,IAAI,UAAU,CAAC,MAAM,KAAK,SAAS,CAAC,SAAS,EAAE,CAAC;gBAC/C,uDAAuD;gBACvD,MAAM,SAAS,GAAG,UAAU,CAAC,SAAS,CAAC,SAAS,CAAC,SAAS,EAAE,UAAU,CAAC,MAAM,CAAC,CAAA;gBAE9E,sDAAsD;gBACtD,0BAA0B,CAAC,SAAS,EAAE,SAAS,CAAC,SAAS,CAAC,CAAA;YAC3D,CAAC;QACF,CAAC;IACF,CAAC;IAED,yFAAyF;IACzF,OAAO,EAAE,2BAA2B,EAAE,2BAA2B,EAAE,CAAA;AACpE,CAAC;AAED,gGAAgG;AAChG,iBAAiB;AACjB,gGAAgG;AAChG,4GAA4G;AAC5G,SAAS,iCAAiC,CAAC,OAA4B;IACtE,uHAAuH;IACvH,MAAM,uBAAuB,GAAG,gBAAgB,CAAC,OAAO,CAAC,QAAQ,IAAI,EAAE,CAAC,IAAI,EAAE,CAAA;IAC9E,0FAA0F;IAC1F,MAAM,2BAA2B,GAAG,sBAAsB,CAAC,OAAO,CAAC,QAAQ,IAAI,EAAE,CAAC,IAAI,EAAE,CAAA;IACxF,0FAA0F;IAC1F,MAAM,8BAA8B,GAAG,wCAAwC,CAAC,OAAO,CAAC,QAAQ,IAAI,EAAE,CAAC,IAAI,EAAE,CAAA;IAC7G,iHAAiH;IACjH,MAAM,6CAA6C,GAAG,8BAA8B,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,UAAU,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC,CAAA;IACzH,oEAAoE;IACpE,MAAM,kBAAkB,GAAG,OAAO,CAAC,kBAAkB,IAAI,EAAE,CAAA;IAE3D,qDAAqD;IACrD,IAAI,YAAY,GAAG;QAClB,GAAG,kBAAkB;QACrB,GAAG,uBAAuB;QAC1B,GAAG,2BAA2B;QAC9B,GAAG,8BAA8B;QACjC,GAAG,6CAA6C;QAChD,GAAG,gBAAgB;QACnB,GAAG,eAAe;KAClB,CAAA;IAED,gHAAgH;IAChH,MAAM,WAAW,GAAG,wBAAwB,CAAC;QAC5C,GAAG,YAAY;QACf,GAAG,YAAY,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,iBAAiB,EAAE,CAAC;QACrD,GAAG,YAAY,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,iBAAiB,EAAE,CAAC;KACrD,CAAC,CAAA;IAEF,qHAAqH;IACrH,MAAM,kBAAkB,GAAG,WAAW,CAAC,WAAW,EAAE,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC,CAAA;IAErE,yCAAyC;IACzC,OAAO,kBAAkB,CAAA;AAC1B,CAAC;AAED,kGAAkG;AAClG,KAAK,UAAU,kBAAkB;IAChC,4DAA4D;IAC5D,IAAI,CAAC;QACJ,MAAM,eAAe,GAAG,MAAM,MAAM,CAAC,mCAAmC,CAAC,CAAA;QAEzE,6CAA6C;QAC7C,OAAO,eAAe,CAAA;IACvB,CAAC;IAAC,MAAM,CAAC;QACR,2FAA2F;QAC3F,OAAO,SAAS,CAAA;IACjB,CAAC;AACF,CAAC;AAED,iHAAiH;AACjH,gHAAgH;AAChH,KAAK,UAAU,yBAAyB,CAAC,cAAsB,EAAE,YAA0B;IAC1F,0CAA0C;IAC1C,MAAM,eAAe,GAAG,MAAM,kBAAkB,EAAE,CAAA;IAElD,kFAAkF;IAClF,IAAI,eAAe,KAAK,SAAS,EAAE,CAAC;QACnC,OAAO,YAAY,CAAA;IACpB,CAAC;IAED,gGAAgG;IAChG,IAAI,cAAc,GAAG,KAAK,CAAA;IAE1B,6DAA6D;IAC7D,MAAM,eAAe,GAAG,IAAI,YAAY,EAAE,CAAA;IAE1C,6CAA6C;IAC7C,KAAK,IAAI,SAAS,GAAG,CAAC,EAAE,SAAS,GAAG,YAAY,CAAC,MAAM,EAAE,SAAS,EAAE,EAAE,CAAC;QACtE,+EAA+E;QAC/E,MAAM,SAAS,GAAG,YAAY,CAAC,OAAO,CAAC,SAAS,CAAC,CAAA;QACjD,MAAM,eAAe,GAAG,SAAS,CAAC,WAAW,CAAA;QAE7C,mCAAmC;QACnC,MAAM,IAAI,GAAG,YAAY,CAAC,SAAS,CAAC,SAAS,CAAC,CAAA;QAE9C,qEAAqE;QACrE,IAAI,yBAAyB,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;YAC1C,yCAAyC;YACzC,IAAI,CAAC,cAAc,EAAE,CAAC;gBACrB,MAAM,eAAe,CAAC,UAAU,EAAE,CAAA;gBAElC,cAAc,GAAG,IAAI,CAAA;YACtB,CAAC;YAED,8EAA8E;YAC9E,MAAM,UAAU,GAAG,CAAC,GAAG,eAAe,CAAC,uBAAuB,CAAC,IAAI,CAAC,CAAC,CAAA;YAErE,iEAAiE;YACjE,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,UAAU,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;gBAChD,qEAAqE;gBACrE,MAAM,kBAAkB,GAAG,eAAe,GAAG,UAAU,CAAC,CAAC,CAAC,CAAA;gBAC1D,MAAM,gBAAgB,GAAG,eAAe,GAAG,UAAU,CAAC,CAAC,GAAG,CAAC,CAAC,CAAA;gBAE5D,oDAAoD;gBACpD,MAAM,WAAW,GAAG,cAAc,CAAC,SAAS,CAAC,kBAAkB,EAAE,gBAAgB,CAAC,CAAA;gBAElF,2EAA2E;gBAC3E,eAAe,CAAC,OAAO,CACtB,WAAW,EACX,kBAAkB,EAClB,KAAK,CACL,CAAA;YACF,CAAC;QACF,CAAC;aAAM,CAAC;YACP,6FAA6F;YAC7F,MAAM,QAAQ,GAAG,cAAc,CAAC,SAAS,CAAC,SAAS,CAAC,WAAW,EAAE,SAAS,CAAC,SAAS,CAAC,CAAA;YAErF,eAAe,CAAC,OAAO,CAAC,QAAQ,EAAE,SAAS,CAAC,WAAW,EAAE,SAAS,CAAC,aAAa,CAAC,CAAA;QAClF,CAAC;IACF,CAAC;IAED,yCAAyC;IACzC,OAAO,eAAe,CAAA;AACvB,CAAC;AAED,gEAAgE;AAChE,2GAA2G;AAC3G,uCAAuC;AACvC,SAAS,oBAAoB,CAAC,YAA0B,EAAE,IAAY;IACrE,+DAA+D;IAC/D,MAAM,iBAAiB,GAAY,EAAE,CAAA;IACrC,+BAA+B;IAC/B,MAAM,WAAW,GAAG,YAAY,CAAC,OAAO,CAAA;IAExC,4EAA4E;IAC5E,IAAI,WAAW,CAAC,CAAC,CAAC,CAAC,WAAW,GAAG,CAAC,EAAE,CAAC;QACpC,iBAAiB,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,CAAC,EAAE,GAAG,EAAE,WAAW,CAAC,CAAC,CAAC,CAAC,WAAW,EAAE,CAAC,CAAA;IACtE,CAAC;IAED,2BAA2B;IAC3B,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,WAAW,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QAC7C,wBAAwB;QACxB,MAAM,KAAK,GAAG,WAAW,CAAC,CAAC,CAAC,CAAA;QAE5B,sEAAsE;QACtE,MAAM,iBAAiB,GAAG,WAAW,CAAC,CAAC,GAAG,CAAC,CAAC,EAAE,SAAS,IAAI,CAAC,CAAA;QAE5D,qFAAqF;QACrF,IAAI,KAAK,CAAC,WAAW,GAAG,iBAAiB,EAAE,CAAC;YAC3C,iBAAiB,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,iBAAiB,EAAE,GAAG,EAAE,KAAK,CAAC,WAAW,EAAE,CAAC,CAAA;QAC7E,CAAC;QAED,oFAAoF;QACpF,IAAI,KAAK,CAAC,aAAa,EAAE,CAAC;YACzB,iBAAiB,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,KAAK,CAAC,WAAW,EAAE,GAAG,EAAE,KAAK,CAAC,SAAS,EAAE,CAAC,CAAA;QAC3E,CAAC;IACF,CAAC;IAED,CAAC;QACA,gDAAgD;QAChD,MAAM,aAAa,GAAG,WAAW,CAAC,WAAW,CAAC,MAAM,GAAG,CAAC,CAAC,EAAE,SAAS,CAAA;QAEpE,8EAA8E;QAC9E,IAAI,aAAa,IAAI,aAAa,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC;YAClD,iBAAiB,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,aAAa,EAAE,GAAG,EAAE,IAAI,CAAC,MAAM,EAAE,CAAC,CAAA;QACnE,CAAC;IACF,CAAC;IAED,0CAA0C;IAC1C,OAAO,iBAAiB,CAAA;AACzB,CAAC;AAgBD,wGAAwG;AACxG,yDAAyD;AACzD,MAAM,OAAO,YAAY;IACxB,gFAAgF;IAChF,SAAS,CAAO;IAChB,uCAAuC;IACvC,KAAK,CAAc;IAEnB,4EAA4E;IAC5E,YAAY,SAAgB,EAAE,KAAmB;QAChD,IAAI,CAAC,SAAS,GAAG,SAAS,CAAA;QAC1B,IAAI,CAAC,KAAK,GAAG,KAAK,CAAA;IACnB,CAAC;IAED,gEAAgE;IAChE,IAAI,IAAI;QACP,OAAO,IAAI,CAAC,KAAK,CAAC,IAAI,CAAA;IACvB,CAAC;IAED,uEAAuE;IACvE,+DAA+D;IAC/D,IAAI,SAAS;QACZ,OAAO;YACN,KAAK,EAAE,IAAI,CAAC,KAAK,CAAC,UAAU,CAAC,WAAW;YACxC,GAAG,EAAE,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC,SAAS;SACnC,CAAA;IACF,CAAC;CACD;AAED,uFAAuF;AACvF,8CAA8C;AAC9C,MAAM,OAAO,QAAS,SAAQ,YAAY;IACzC,yCAAyC;IACzC,OAAO,GAAa,EAAE,CAAA;CACtB;AAED,gEAAgE;AAChE,2EAA2E;AAC3E,MAAM,OAAO,MAAO,SAAQ,YAAY;CACvC;AAgBD,iHAAiH;AACjH,MAAM,CAAC,MAAM,0BAA0B,GAAwB;IAC9D,QAAQ,EAAE,EAAE;IACZ,kBAAkB,EAAE,EAAE;IACtB,6BAA6B,EAAE,IAAI;CACnC,CAAA;AAOD,uHAAuH;AACvH,MAAM,CAAC,MAAM,sCAAsC,GAAoC;IACtF,mCAAmC,EAAE,IAAI;CACzC,CAAA"}