@echogarden/text-segmentation 0.7.0 → 0.8.0

package/src/segmentation/TextSegmentation.ts

@@ -0,0 +1,743 @@
+import { buildWordOrNumberPattern as buildWordSplitterPattern, phraseSeparatorRegExp, sentenceSeparatorTrailingPunctuationCharacterRegExp, sentenceSeparatorCharacterRegExp, whitespacePatternRegExp, letterPatternGlobalRegExp, startsWithWhitespacePattern, endsWithWhitespacePattern } from '../patterns/Patterns.js'
+import { cldrSuppressions, additionalSuppressions, leadingApostropheContractionSuppressions, nounSuppressions, tldSuppressions } from '../patterns/Suppressions.js'
+import { eastAsianCharRangesRegExp } from '../patterns/EastAsianCharacterPatterns.js'
+import { WordSequence } from './WordSequence.js'
+import { getShortLanguageCode } from '../utilities/Utilities.js'
+
+import { buildRegExp } from 'regexp-composer'
+
+////////////////////////////////////////////////////////////////////////////////////////////////
+// Exported methods
+////////////////////////////////////////////////////////////////////////////////////////////////
+
+// Splits a text string into words and then segments the resulting word sequence into sentences and phrases
+// (this is the main high-level entry point; it is async because splitting may involve ICU postprocessing)
+export async function segmentText(text: string, options?: SegmentationOptions) {
+	// Split the text into a sequence of individual words (this may await East Asian postprocessing)
+	const wordSequence = await splitToWords(text, options)
+
+	// Segment the word sequence into segments (paragraphs), sentences, and phrases, and return the result
+	return segmentWordSequence(wordSequence)
+}
+
+// Segments an already-tokenized word sequence into segments (paragraphs), sentences, and phrases
+// This is the core segmentation algorithm, working purely on the word level
+// (options only control how colons/semicolons are treated when splitting phrases)
+export async function segmentWordSequence(wordSequence: WordSequence, options?: WordSequenceSegmentationOptions) {
+	// Fill in any option values the caller didn't provide with their defaults
+	options = { ...defaultWordSequenceSegmentationOptions, ...options }
+
+	// This will hold the start and end word offsets of each paragraph (called a "segment" here)
+	const segmentWordRanges: Range[] = []
+
+	// Step 1: find segment (paragraph) boundaries.
+	// A new segment begins right after a newline character that follows at least one non-whitespace word.
+	{
+		// The word offset at which the current segment starts
+		let segmentStartWordOffset = 0
+		// Whether at least one non-whitespace word has been seen in the current segment
+		let nonWhitespaceWordSeen = false
+		// Whether a newline has been seen in the current segment (i.e. we may be at a boundary)
+		let newlineSeenInCurrentSegment = false
+
+		// Walk through every word of the sequence
+		for (let wordIndex = 0; wordIndex < wordSequence.length; wordIndex++) {
+			// Get the text of the word at the current index
+			const word = wordSequence.getWordAt(wordIndex)
+
+			// If we haven't seen a newline yet, the segment already has content, and this word is a newline, mark the newline as seen
+			if (!newlineSeenInCurrentSegment && nonWhitespaceWordSeen && word.endsWith('\n')) {
+				newlineSeenInCurrentSegment = true
+			}
+
+			// If a newline was seen, decide whether the segment actually ends at this word
+			if (newlineSeenInCurrentSegment) {
+				// Look at the word after the current one (or an empty string if the current word is last)
+				const nextWord = wordIndex < wordSequence.length - 1 ? wordSequence.getWordAt(wordIndex + 1) : ''
+
+				// End the segment here if the newline is followed by real content (or by nothing at all)
+				if (nextWord === '' || !whitespacePatternRegExp.test(nextWord)) {
+					// Record the current segment, ending after the current (newline) word
+					segmentWordRanges.push({ start: segmentStartWordOffset, end: wordIndex + 1 })
+					// The next segment starts at the word after the newline
+					segmentStartWordOffset = wordIndex + 1
+
+					// Reset the newline flag for the new segment
+					newlineSeenInCurrentSegment = false
+				}
+			}
+
+			// If no non-whitespace word has been seen yet, check whether the current word is non-whitespace
+			if (nonWhitespaceWordSeen === false) {
+				nonWhitespaceWordSeen = !whitespacePatternRegExp.test(word)
+			}
+		}
+
+		// If the last segment didn't reach the end of the sequence, close it with the remaining words
+		if (segmentStartWordOffset < wordSequence.length) {
+			segmentWordRanges.push({ start: segmentStartWordOffset, end: wordSequence.length })
+		}
+	}
+
+	// These will hold the word offsets of each sentence, and (per segment) the range of sentence indices in it
+	const sentenceWordRanges: Range[] = []
+	const segmentSentenceRanges: Range[] = []
+
+	// Step 2: find sentence boundaries inside each segment.
+	// A sentence ends at a sentence-separator character (like '.', '!', '?') once it contains a minimum number of letters.
+	{
+		// A sentence must contain at least this many letters before a separator may end it
+		const minimumSentenceLetterCount = 2
+
+		// Process each segment independently
+		for (let segmentIndex = 0; segmentIndex < segmentWordRanges.length; segmentIndex++) {
+			// Get the word range of the current segment
+			const segmentWordRange = segmentWordRanges[segmentIndex]
+
+			// Record the segment's first and last word index
+			const segmentStartWordIndex = segmentWordRange.start
+			const segmentEndWordIndex = segmentWordRange.end
+
+			// Add a placeholder entry for this segment's sentence range, starting (and initially ending) at the current sentence count
+			segmentSentenceRanges.push({ start: sentenceWordRanges.length, end: sentenceWordRanges.length })
+
+			// Whether a non-whitespace word has been seen in the current sentence
+			let nonWhitespaceWordSeen = false
+			// The word offset at which the current sentence starts
+			let sentenceStartWordOffset = segmentStartWordIndex
+			// The number of letters accumulated in the current sentence
+			let currentSentenceLetterCount = 0
+
+			// Walk through the words of this segment
+			for (let wordIndex = segmentStartWordIndex; wordIndex < segmentEndWordIndex; wordIndex++) {
+				// Get the text of the current word
+				const word = wordSequence.getWordAt(wordIndex)
+
+				// If no non-whitespace word has been seen yet, check whether the current word is non-whitespace
+				if (nonWhitespaceWordSeen === false) {
+					nonWhitespaceWordSeen = !whitespacePatternRegExp.test(word)
+				}
+
+				// Count letters, but only while the sentence hasn't yet reached the minimum (to save work)
+				if (currentSentenceLetterCount < minimumSentenceLetterCount) {
+					// Find every letter character in the current word
+					const matches = word.matchAll(letterPatternGlobalRegExp)
+
+					// Count the letters, stopping early once the minimum is reached
+					for (const _ of matches) {
+						currentSentenceLetterCount += 1
+
+						if (currentSentenceLetterCount >= minimumSentenceLetterCount) {
+							break
+						}
+					}
+				}
+
+				// End the sentence here if it has content, enough letters, and the current word is a sentence-separator character
+				if (nonWhitespaceWordSeen && currentSentenceLetterCount >= minimumSentenceLetterCount && sentenceSeparatorCharacterRegExp.test(word)) {
+					// Start scanning at the separator; this index marks where the sentence's trailing punctuation ends
+					let trailingSequenceEndIndex = wordIndex
+
+					// Consume any trailing punctuation words (closing quotes, brackets, etc.) that belong to this sentence
+					while (trailingSequenceEndIndex < segmentEndWordIndex) {
+						// Get the text of the candidate trailing-punctuation word
+						const trailingWord = wordSequence.getWordAt(trailingSequenceEndIndex)
+
+						// Keep consuming it if it is trailing punctuation; otherwise stop
+						if (sentenceSeparatorTrailingPunctuationCharacterRegExp.test(trailingWord)) {
+							trailingSequenceEndIndex++
+						} else {
+							break
+						}
+					}
+
+					// Record the completed sentence, from its start offset up to (but not including) the trailing punctuation
+					sentenceWordRanges.push({
+						start: sentenceStartWordOffset,
+						end: trailingSequenceEndIndex
+					})
+
+					// Extend the current segment's sentence range to include the sentence we just recorded
+					segmentSentenceRanges[segmentSentenceRanges.length - 1].end += 1
+
+					// The next sentence starts right after the trailing punctuation
+					sentenceStartWordOffset = trailingSequenceEndIndex
+					// Reset the letter count for the next sentence
+					currentSentenceLetterCount = 0
+
+					// Skip the words already consumed by the trailing-punctuation scan (the for loop will increment again)
+					wordIndex = trailingSequenceEndIndex - 1
+				}
+			}
+
+			// If the segment ends with words that don't complete a new sentence, close the final sentence at the segment end
+			if (sentenceStartWordOffset < segmentEndWordIndex) {
+				sentenceWordRanges.push({ start: sentenceStartWordOffset, end: segmentEndWordIndex })
+				segmentSentenceRanges[segmentSentenceRanges.length - 1].end += 1
+			}
+		}
+	}
+
+	// This will hold the Sentence objects built from the sentence word ranges
+	const sentences: Sentence[] = []
+
+	// Build a Sentence object for every sentence range found above
+	for (const wordRange of sentenceWordRanges) {
+		// Extract the slice of the word sequence that belongs to this sentence
+		const sentenceWordSequence = wordSequence.slice(wordRange.start, wordRange.end)
+
+		// Create the Sentence for this range and append it to the list
+		sentences.push(new Sentence(wordRange, sentenceWordSequence))
+	}
+
+	// Step 3: split each sentence into phrases at phrase-separator characters (commas, colons, semicolons, etc.)
+	for (const sentence of sentences) {
+		// This will hold the start and end word offsets of each phrase in the current sentence
+		const phraseWordRanges: Range[] = []
+
+		// The word offset just past the end of this sentence
+		let sentenceEndWordOffset = sentence.wordRange.end
+		// The word offset at which the current phrase starts
+		let phraseStartWordOffset = sentence.wordRange.start
+
+		// Walk through the words of this sentence
+		for (let wordIndex = phraseStartWordOffset; wordIndex < sentenceEndWordOffset; wordIndex++) {
+			// Get the text of the current word
+			const currentWord = wordSequence.getWordAt(wordIndex)
+
+			// Determine whether this word is a phrase separator:
+			// it must match the phrase-separator pattern, and if the option is set, ':' or ';' only count when followed by whitespace
+			const isCurrentWordPhraseSeparator =
+				phraseSeparatorRegExp.test(currentWord) &&
+				(!options.requireSpaceAfterColonsOrSemicolons ||
+					(currentWord !== ':' && currentWord !== ';') ||
+					whitespacePatternRegExp.test(wordSequence.getWordAt(wordIndex + 1)))
+
+			// If the current word is a phrase separator, close the current phrase at this word
+			if (isCurrentWordPhraseSeparator) {
+				// Tracks whether at least one whitespace word was seen while scanning trailing characters
+				let whitespaceSeenOnce = false
+
+				// Include any trailing punctuation (closing quotes, brackets, etc.) in the current phrase
+				while (wordIndex < sentenceEndWordOffset - 1) {
+					// Get the text of the next word
+					const nextWord = wordSequence.getWordAt(wordIndex + 1)
+
+					// Stop consuming if the next word is not trailing punctuation
+					if (!sentenceSeparatorTrailingPunctuationCharacterRegExp.test(nextWord)) {
+						break
+					}
+
+					// Remember if this trailing word is whitespace
+					if (whitespacePatternRegExp.test(nextWord)) {
+						whitespaceSeenOnce = true
+					}
+
+					// A closing double quote that follows whitespace starts the next phrase, so stop before it
+					if (nextWord === '"' && whitespaceSeenOnce) {
+						break
+					}
+
+					// Otherwise consume this trailing-punctuation word
+					wordIndex += 1
+				}
+
+				// Record the phrase, from its start offset through the separator (and any trailing punctuation)
+				phraseWordRanges.push({
+					start: phraseStartWordOffset,
+					end: wordIndex + 1
+				})
+
+				// The next phrase starts right after the current separator
+				phraseStartWordOffset = wordIndex + 1
+			}
+		}
+
+		// If the sentence ends with words that weren't part of any phrase, close the final phrase at the sentence end
+		if (phraseStartWordOffset < sentenceEndWordOffset) {
+			phraseWordRanges.push({ start: phraseStartWordOffset, end: sentenceEndWordOffset })
+		}
+
+		// Build a Phrase object for every phrase range and attach it to the sentence
+		for (const wordRange of phraseWordRanges) {
+			// Extract the slice of the word sequence that belongs to this phrase
+			const phraseWordSequence = wordSequence.slice(wordRange.start, wordRange.end)
+
+			// Create the Phrase for this range and add it to the sentence's phrase list
+			sentence.phrases.push(new Phrase(wordRange, phraseWordSequence))
+		}
+	}
+
+	// Assemble the final result: the full word sequence, the list of sentences, and the segment-to-sentence mapping
+	const result: SegmentationResult = {
+		words: wordSequence,
+		sentences,
+		segmentSentenceRanges,
+	}
+
+	// Return the assembled segmentation result
+	return result
+}
+
+// Cache of compiled word-splitter regular expressions, keyed by the JSON-serialized options
+// (compiling a regex is expensive, so the same options reuse the same compiled expression)
+const cachedWordSplitterRegExps = new Map<string, RegExp>()
+
+// Splits a string of text into a WordSequence of individual words and punctuation marks
+// The returned sequence contains both the matched words and the punctuation found between/beside them
+export async function splitToWords(text: string, options?: SegmentationOptions) {
+	// If no options were given, start from an empty options object
+	if (!options) {
+		options = {}
+	}
+
+	// Fill in any option values the caller didn't provide with their defaults
+	options = { ...defaultSegmentationOptions, ...options }
+
+	// If a language was specified, normalize it to its short base code (e.g. "en-US" becomes "en")
+	// so that language-specific data is looked up consistently
+	if (options.language) {
+		options.language = getShortLanguageCode(options.language)
+	}
+
+	// Serialize the (now normalized) options to JSON so they can be used as a cache key
+	const optionsAsJson = JSON.stringify(options)
+
+	// Look up a previously compiled splitter expression for these exact options
+	let wordSplitterRegExp = cachedWordSplitterRegExps.get(optionsAsJson)
+
+	// If no cached expression exists, compile a new one and store it in the cache
+	if (!wordSplitterRegExp) {
+		wordSplitterRegExp = buildWordSplitterRegExpForOptions(options)
+
+		cachedWordSplitterRegExps.set(optionsAsJson, wordSplitterRegExp)
+	}
+
+	// Start with an empty word sequence that will accumulate the results
+	let wordSequence = new WordSequence()
+
+	// Adds the words found in a run of text between two matched words (typically punctuation and whitespace)
+	// to the word sequence. Spaces are skipped; every other character becomes its own punctuation word,
+	// except that consecutive punctuation characters are grouped into a single word.
+	function addPunctuationWordsBetween(startOffset: number, endOffset: number) {
+		// Extract the substring that lies between the two given character offsets
+		const punctuationWordSubstring = text.substring(startOffset, endOffset)
+
+		// The current character offset within the source text
+		let charOffset = startOffset
+		// The character offset at which the current punctuation word starts
+		let punctuationWordStartOffset = startOffset
+
+		// If any characters have been accumulated since the last flush, add them as a single punctuation word
+		function addPunctuationWordIfNeeded() {
+			// Check whether there are pending characters to add
+			if (charOffset > punctuationWordStartOffset) {
+				// Extract the pending word text from the source
+				const wordText = text.substring(punctuationWordStartOffset, charOffset)
+				// Add it to the sequence, marking it as punctuation
+				wordSequence.addWord(wordText, punctuationWordStartOffset, true)
+
+				// The next pending word (if any) starts at the current character offset
+				punctuationWordStartOffset = charOffset
+			}
+		}
+
+		// Iterate over every character (codepoint) in the gap between words
+		for (const char of punctuationWordSubstring) {
+			// Spaces are skipped: they just advance the character offset and are not added as words
+			if (char === ' ') {
+				charOffset += 1
+
+				continue
+			}
+
+			// Flush any pending punctuation word before the current character
+			addPunctuationWordIfNeeded()
+
+			// Advance the character offset past the current character (char.length handles astral codepoints)
+			charOffset += char.length
+
+			// Flush again so this single character becomes its own punctuation word
+			addPunctuationWordIfNeeded()
+		}
+
+		// Flush any remaining pending punctuation word at the end of the gap
+		addPunctuationWordIfNeeded()
+	}
+
+	// Find all word matches (as defined by the splitter expression) throughout the text
+	const wordMatches = text.matchAll(wordSplitterRegExp)
+
+	// Track the character offset at which the last matched word ended
+	let lastMatchEndOffset = 0
+
+	// Process each matched word in order of appearance
+	if (wordMatches) {
+		for (const match of wordMatches) {
+			// Read the match's start and end character offsets (available because the regex has the 'd' flag)
+			const offsets = match.indices![0]!
+			const matchStartOffset = offsets[0]
+			const matchEndOffset = offsets[1]
+
+			// If there is a gap between the previous match and this one, add the gap's punctuation as words
+			if (matchStartOffset > lastMatchEndOffset) {
+				addPunctuationWordsBetween(lastMatchEndOffset, matchStartOffset)
+			}
+
+			// Extract the matched word's text from the source
+			const wordText = text.substring(matchStartOffset, matchEndOffset)
+			// Add the matched word to the sequence (it is a real word, not punctuation)
+			wordSequence.addWord(wordText, matchStartOffset, false)
+
+			// Remember where this match ended so gaps after it can be found
+			lastMatchEndOffset = matchEndOffset
+		}
+
+		// If the text extends beyond the last match, add the trailing punctuation as words
+		addPunctuationWordsBetween(lastMatchEndOffset, text.length)
+	}
+
+	// If East Asian postprocessing is enabled, further split runs of East Asian characters into subwords using ICU word breaking
+	if (options.enableEastAsianPostprocessing) {
+		wordSequence = await postprocessEastAsianWords(text, wordSequence)
+	}
+
+	// Return the resulting word sequence
+	return wordSequence
+}
+
+// Add any missing punctuation words to a word sequence
+// Given a word sequence that was built from the source text (with its character offsets) and the source text itself,
+// produce a new word sequence that also includes every punctuation character found between, before, and after the words.
+// Also returns a mapping from each new entry's index back to the original word index it came from
+// (punctuation-only entries are not present as keys in this mapping).
+export function addMissingPunctuationWordsToWordSequence(wordSequence: WordSequence, sourceText: string) {
+	// Maps each index in the new (punctuation-included) sequence to the index of the original word it was copied from
+	const originalWordsReverseMapping = new Map<number, number>()
+
+	// The new word sequence that will contain both the original words and the missing punctuation words
+	const wordSequenceWithPunctuation = new WordSequence()
+
+	// Adds entries for every character of a text slice, grouping runs of non-space characters into single punctuation words
+	// (spaces are merged into the preceding punctuation word, preserving them for offset correctness)
+	function addWordEntriesForTextSlice(textSlice: string, initialCharOffset: number) {
+		// The current character offset within the source text
+		let charOffset = initialCharOffset
+
+		// Add entry for every codepoint (this will correctly treat characters beyond BMP)
+		for (const char of textSlice) {
+			// Compute the source-text offset just past this character
+			const charEndOffset = charOffset + char.length
+
+			// Get the last entry added so far (if any)
+			const lastEntry = wordSequenceWithPunctuation.lastEntry
+
+			// If this is a space and the last entry is a punctuation word that already starts with a space, extend that entry
+			if (char === ' ' && lastEntry && lastEntry.isPunctuation && lastEntry.text[0] === ' ') {
+				wordSequenceWithPunctuation.lastEntry.text += ' '
+				wordSequenceWithPunctuation.lastEntry.endOffset = charEndOffset
+			} else {
+				// Otherwise, extract this single character from the source and add it as a new punctuation word
+				const wordText = sourceText.substring(charOffset, charEndOffset)
+
+				wordSequenceWithPunctuation.addWord(wordText, charOffset, true)
+			}
+
+			// Advance the character offset past the current character
+			charOffset = charEndOffset
+		}
+	}
+
+	// Process each original word in order
+	for (let wordIndex = 0; wordIndex < wordSequence.length; wordIndex++) {
+		// Get the current word entry and the offset where it starts in the source text
+		const wordEntry = wordSequence.getEntryAt(wordIndex)
+		const wordStartOffset = wordEntry.startOffset
+
+		// Get the end offset of the previous word (or 0 if this is the first word)
+		const previousWordEndOffset = wordIndex > 0 ? wordSequence.entries[wordIndex - 1].endOffset : 0
+
+		// Add entries for any punctuation characters between the current and previous word (or the start of the text)
+		if (previousWordEndOffset !== wordStartOffset) {
+			// Extract the slice of source text lying between the two words
+			const textSlice = sourceText.substring(previousWordEndOffset, wordStartOffset)
+
+			// Add entries for the punctuation characters in that slice
+			addWordEntriesForTextSlice(textSlice, previousWordEndOffset)
+		}
+
+		// Copy the original word entry into the new sequence
+		wordSequenceWithPunctuation.entries.push(wordEntry)
+		// Record that this new entry corresponds to the current original word index
+		originalWordsReverseMapping.set(wordSequenceWithPunctuation.length - 1, wordIndex)
+
+		// If last word, add entries for any trailing punctuation characters
+		if (wordIndex === wordSequence.length - 1) {
+			// If the source text extends beyond the last word, the remainder is trailing punctuation
+			if (sourceText.length !== wordEntry.endOffset) {
+				// Extract the slice of source text after the last word
+				const textSlice = sourceText.substring(wordEntry.endOffset, sourceText.length)
+
+				// Add entries for the trailing punctuation characters
+				addWordEntriesForTextSlice(textSlice, wordEntry.endOffset)
+			}
+		}
+	}
+
+	// Return the enriched word sequence along with the reverse mapping to the original words
+	return { wordSequenceWithPunctuation, originalWordsReverseMapping }
+}
+
+////////////////////////////////////////////////////////////////////////////////////////////////
+// Helper methods
+////////////////////////////////////////////////////////////////////////////////////////////////
+// Builds a global regular expression that matches words in the given language, using the configured options
+function buildWordSplitterRegExpForOptions(options: SegmentationOptions) {
+	// Get the CLDR suppression list (standard abbreviations like "Dr.") for this language, or an empty list if none exists
+	const cldrSuppressionsForLang = cldrSuppressions[options.language ?? ''] ?? []
+	// Get the extended, project-specific suppression list for this language, or an empty list
+	const extendedSuppressionsForLang = additionalSuppressions[options.language ?? ''] ?? []
+	// Get the leading-apostrophe contraction suppressions for this language, or an empty list
+	const contractionSuppressionsForLang = leadingApostropheContractionSuppressions[options.language ?? ''] ?? []
+	// Build a variant of those contractions that uses the typographic apostrophe (') instead of the straight one (')
+	const contractionSuppressionsForLangWithSingleQuote = contractionSuppressionsForLang.map(str => str.replaceAll(`'`, `’`))
+	// Get any custom suppressions the caller provided, or an empty list
+	const customSuppressions = options.customSuppressions ?? []
+
+	// Combine all suppression sources into one flat list
+	let suppressions = [
+		...customSuppressions,
+		...cldrSuppressionsForLang,
+		...extendedSuppressionsForLang,
+		...contractionSuppressionsForLang,
+		...contractionSuppressionsForLangWithSingleQuote,
+		...nounSuppressions,
+		...tldSuppressions,
+	]
+
+	// Build the word pattern: match any suppression (in original, lowercase, and uppercase forms) or a regular word
+	const wordPattern = buildWordSplitterPattern([
+		...suppressions,
+		...suppressions.map(word => word.toLocaleLowerCase()),
+		...suppressions.map(word => word.toLocaleUpperCase()),
+	])
+
+	// Compile the pattern into a global regular expression (the 'd' flag, added by regexp-composer, gives match indices)
+	const wordSplitterRegExp = buildRegExp(wordPattern, { global: true })
+
+	// Return the compiled regular expression
+	return wordSplitterRegExp
+}
+
+// Tries to load the ICU segmentation WebAssembly module; returns undefined if it is not installed
+async function getIcuSegmentation() {
+	// Attempt to dynamically import the ICU segmentation module
+	try {
+		const icuSegmentation = await import('@echogarden/icu-segmentation-wasm')
+
+		// If the import succeeded, return the module
+		return icuSegmentation
+	} catch {
+		// If the import failed (module not available), signal that ICU segmentation is unavailable
+		return undefined
+	}
+}
+
+// Further splits words that contain East Asian characters (Chinese, Japanese, Thai, Khmer) into smaller subwords
+// using ICU's word-break rules, which are much better suited to these scripts than the generic splitter pattern
+async function postprocessEastAsianWords(containingText: string, wordSequence: WordSequence) {
+	// Try to load the ICU segmentation module
+	const icuSegmentation = await getIcuSegmentation()
+
+	// If ICU segmentation is unavailable, return the original word sequence unchanged
+	if (icuSegmentation === undefined) {
+		return wordSequence
+	}
+
+	// Tracks whether the ICU module has been initialized (initialization only needs to happen once)
+	let icuInitialized = false
+
+	// A new word sequence that will hold the postprocessed words
+	const newWordSequence = new WordSequence()
+
+	// Process each word in the original sequence
+	for (let wordIndex = 0; wordIndex < wordSequence.length; wordIndex++) {
+		// Get the current word entry and the offset where it starts in the source text
+		const wordEntry = wordSequence.entries[wordIndex]
+		const wordStartOffset = wordEntry.startOffset
+
+		// Get the text of the current word
+		const word = wordSequence.getWordAt(wordIndex)
+
+		// If the word contains East Asian characters, split it into subwords
+		if (eastAsianCharRangesRegExp.test(word)) {
+			// Initialize the ICU module on first use
+			if (!icuInitialized) {
+				await icuSegmentation.initialize()
+
+				icuInitialized = true
+			}
+
+			// Get the word-break positions within the word (offsets into the word string)
+			const wordBreaks = [...icuSegmentation.createWordBreakIterator(word)]
+
+			// Create a subword for every pair of consecutive break positions
+			for (let i = 0; i < wordBreaks.length - 1; i++) {
+				// Compute the subword's start and end offsets within the source text
+				const subwordStartOffset = wordStartOffset + wordBreaks[i]
+				const subwordEndOffset = wordStartOffset + wordBreaks[i + 1]
+
+				// Extract the subword text from the containing text
+				const subwordText = containingText.substring(subwordStartOffset, subwordEndOffset)
+
+				// Add the subword to the new sequence (it is a real word, not punctuation)
+				newWordSequence.addWord(
+					subwordText,
+					subwordStartOffset,
+					false,
+				)
+			}
+		} else {
+			// Otherwise, extract the word text from the source and copy it to the new sequence unchanged
+			const wordText = containingText.substring(wordEntry.startOffset, wordEntry.endOffset)
+
+			newWordSequence.addWord(wordText, wordEntry.startOffset, wordEntry.isPunctuation)
+		}
+	}
+
+	// Return the postprocessed word sequence
+	return newWordSequence
+}
+
+// Computes the character ranges of all punctuation in the text.
+// This includes gaps between words, entries that are themselves punctuation, and any leading/trailing text
+// that is not covered by a word entry.
+function getPunctuationRanges(wordSequence: WordSequence, text: string) {
+	// This will hold the character ranges of the punctuation found
+	const punctuationRanges: Range[] = []
+	// Get the list of word entries
+	const wordEntries = wordSequence.entries
+
+	// If the text starts before the first word, the leading part is punctuation
+	if (wordEntries[0].startOffset > 0) {
+		punctuationRanges.push({ start: 0, end: wordEntries[0].startOffset })
+	}
+
+	// Examine every word entry
+	for (let i = 0; i < wordEntries.length; i++) {
+		// Get the current entry
+		const entry = wordEntries[i]
+
+		// Get the end offset of the previous entry (or 0 for the first entry)
+		const previousEndOffset = wordEntries[i - 1]?.endOffset ?? 0
+
+		// If there is a gap between the previous entry and this one, that gap is punctuation
+		if (entry.startOffset > previousEndOffset) {
+			punctuationRanges.push({ start: previousEndOffset, end: entry.startOffset })
+		}
+
+		// If the current entry is itself a punctuation word, its whole range is punctuation
+		if (entry.isPunctuation) {
+			punctuationRanges.push({ start: entry.startOffset, end: entry.endOffset })
+		}
+	}
+
+	{
+		// Get the end offset of the last entry (if any)
+		const lastEndOffset = wordEntries[wordEntries.length - 1]?.endOffset
+
+		// If the text extends beyond the last entry, the trailing part is punctuation
+		if (lastEndOffset && lastEndOffset < text.length) {
+			punctuationRanges.push({ start: lastEndOffset, end: text.length })
+		}
+	}
+
+	// Return the collected punctuation ranges
+	return punctuationRanges
+}
+
+
+////////////////////////////////////////////////////////////////////////////////////////////////
+// Types
+////////////////////////////////////////////////////////////////////////////////////////////////
+
+// Describes the result of segmenting a text: the full word sequence, the detected sentences,
+// and how the sentences map back to the segments (paragraphs) they belong to
+// (segmentSentenceRanges[i] is the range of sentence indices within segment i)
+export interface SegmentationResult {
+	words: WordSequence
+	sentences: Sentence[]
+	segmentSentenceRanges: Range[]
+}
+
+// A fragment of text (either a sentence or a phrase): it holds the word range it occupies and its words
+// (this is the shared base class of Sentence and Phrase)
+export class TextFragment {
+	// The range of word indices this fragment covers (in the overall word sequence)
+	wordRange: Range
+	// The words that make up this fragment
+	words: WordSequence
+
+	// Creates a fragment from a word range and the corresponding slice of words
+	constructor(wordRange: Range, words: WordSequence) {
+		this.wordRange = wordRange
+		this.words = words
+	}
+
+	// Returns the fragment's text by concatenating all of its words
+	get text() {
+		return this.words.text
+	}
+
+	// Returns the character range of this fragment within the source text,
+	// from the start of its first word to the end of its last word
+	get charRange(): Range {
+		return {
+			start: this.words.firstEntry.startOffset,
+			end: this.words.lastEntry.endOffset
+		}
+	}
+}
+
+// A sentence: a text fragment that additionally contains the phrases it was split into
+// (phrases are filled in during segmentation)
+export class Sentence extends TextFragment {
+	// The phrases that make up this sentence
+	phrases: Phrase[] = []
+}
+
+// A phrase: a text fragment representing one part of a sentence
+// (it has no additional members; it exists to give phrases their own type)
+export class Phrase extends TextFragment {
+}
+
+// Represents a range of indices (either word indices or character offsets)
+// with an inclusive start and an exclusive end
+export interface Range {
+	start: number
+	end: number
+}
+
+// Options that control how text is split into words
+export interface SegmentationOptions {
+	language?: string
+	customSuppressions?: string[]
+	enableEastAsianPostprocessing?: boolean
+}
+
+// Default options for word splitting: no language, no custom suppressions, and East Asian postprocessing enabled
+export const defaultSegmentationOptions: SegmentationOptions = {
+	language: '',
+	customSuppressions: [],
+	enableEastAsianPostprocessing: true,
+}
+
+// Options that control how a word sequence is segmented into sentences and phrases
+export interface WordSequenceSegmentationOptions {
+	requireSpaceAfterColonsOrSemicolons: boolean
+}
+
+// Default options for word-sequence segmentation: ':' and ';' only count as phrase separators when followed by a space
+export const defaultWordSequenceSegmentationOptions: WordSequenceSegmentationOptions = {
+	requireSpaceAfterColonsOrSemicolons: true
+}