@echogarden/text-segmentation 0.7.0 → 0.8.0

package/src/tests/Test.ts

@@ -1,5 +1,5 @@
-import { addMissingPunctuationWordsToWordSequence, SegmentationOptions, SegmentationResult, segmentText, segmentWordSequence, splitToWords, WordSequence } from "../exports/TextSegmentation.js"
-import { Timer } from "../utilities/Timer.js"
+import { addMissingPunctuationWordsToWordSequence, SegmentationOptions, SegmentationResult, segmentText, segmentWordSequence, splitToWords, WordSequence } from '../exports/Exports.js'
+import { Timer } from '../utilities/Timer.js'
 
 const log = console.log
 

package/src/utilities/Timer.ts

@@ -1,95 +1,97 @@
-import { roundToDigits } from "./Utilities.js"
+export class Timer {
+	private readonly logger: TimerLogger
 
-declare const chrome: any
-declare const process: any
+	private startTime = 0
 
-export class Timer {
-	startTime = 0
+	constructor(logger?: TimerLogger) {
+		if (logger) {
+			this.logger = logger
+		} else {
+			this.logger = console.log
+		}
 
-	constructor() {
 		this.restart()
 	}
 
-	restart() {
+	// Resets the timer to the current time.
+	restart(): void {
 		this.startTime = Timer.currentTime
 	}
 
+	// Elapsed time in milliseconds (monotonic where supported).
 	get elapsedTime(): number {
-		// Elapsed time (milliseconds)
 		return Timer.currentTime - this.startTime
 	}
 
+	// Elapsed time in seconds.
 	get elapsedTimeSeconds(): number {
-		// Elapsed time (seconds)
 		return this.elapsedTime / 1000
 	}
 
+	// Returns elapsed ms and restarts the timer.
 	getElapsedTimeAndRestart(): number {
-		const elapsedTime = this.elapsedTime
+		const elapsed = this.elapsedTime
 		this.restart()
 
-		return elapsedTime
+		return elapsed
 	}
 
+	// Logs elapsed time (in ms) and restarts the timer.
 	logAndRestart(title: string, timePrecision = 3): number {
-		const elapsedTime = this.elapsedTime
-
-		//
-		const message = `${title}: ${roundToDigits(elapsedTime, timePrecision)}ms`
-
-		console.log(message)
-		//
-
+		const elapsedMs = this.elapsedTime
+		this.logger(`${title}: ${roundToDigits(elapsedMs, timePrecision)}ms`)
 		this.restart()
 
-		return elapsedTime
+		return elapsedMs
 	}
 
+	// Current high-resolution timestamp in milliseconds since Unix epoch.
 	static get currentTime(): number {
-		if (!this.timestampFunc) {
-			this.createGlobalTimestampFunction()
-		}
-
 		return this.timestampFunc()
 	}
 
+	// Current timestamp in microseconds (integer).
 	static get microsecondTimestamp(): number {
 		return Math.floor(Timer.currentTime * 1000)
 	}
 
-	private static createGlobalTimestampFunction() {
-		if (typeof process === 'object' && typeof process.hrtime === 'function') {
-			let baseTimestamp = 0
+	// Clock setup
+	private static timestampFunc: () => number = Timer.createTimestampFunction()
 
-			this.timestampFunc = () => {
-				const nodeTimeStamp = process.hrtime()
-				const millisecondTime = (nodeTimeStamp[0] * 1000) + (nodeTimeStamp[1] / 1000000)
+	private static createTimestampFunction(): () => number {
+		const g = globalThis as any
 
-				return baseTimestamp + millisecondTime
-			}
+		// 1. Modern standard: performance.now() (Browsers & Node 16+)
+		if (typeof g.performance === 'object' && typeof g.performance.now === 'function') {
+			const timeOrigin =
+				g.performance.timeOrigin ?? (Date.now() - g.performance.now())
 
-			baseTimestamp = Date.now() - this.timestampFunc()
+			return () => timeOrigin + g.performance.now()
 		}
-		else if (typeof chrome === 'object' && chrome.Interval) {
-			const baseTimestamp = Date.now()
 
-			const chromeIntervalObject = new chrome.Interval()
-			chromeIntervalObject.start()
+		// 2. Node.js high resolution timer (BigInt variant, Node 10.4+)
+		if (typeof g.process === 'object' && typeof g.process.hrtime === 'function') {
+			const startNs = g.process.hrtime.bigint()
 
-			this.timestampFunc = () => baseTimestamp + chromeIntervalObject.microseconds() / 1000
-		}
-		else if (typeof performance === 'object' && performance.now) {
-			const baseTimestamp = Date.now() - performance.now()
+			const epochBaseMs = Date.now() - (Number(startNs) / 1e6)
 
-			this.timestampFunc = () => baseTimestamp + performance.now()
-		}
-		else if (Date.now) {
-			this.timestampFunc = () => Date.now()
+			return () =>
+				epochBaseMs + Number(g.process.hrtime.bigint()) / 1e6
 		}
-		else {
-			this.timestampFunc = () => (new Date()).getTime()
+
+		// 3. Last-resort fallback (non-monotonic)
+		if (typeof Date.now === 'function') {
+			return () => Date.now()
 		}
+
+		return () => new Date().getTime()
 	}
+}
+
+export function roundToDigits(value: number, digits: number): number {
+	const factor = 10 ** digits
 
-	private static timestampFunc: () => number
+	return Math.round(value * factor) / factor
 }
+
+type TimerLogger = (msg: string) => void

package/dist/Test.d.ts

@@ -1,2 +0,0 @@
-export {};
-//# sourceMappingURL=Test.d.ts.map

package/dist/Test.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"Test.d.ts","sourceRoot":"","sources":["../src/Test.ts"],"names":[],"mappings":""}

package/dist/Test.js

@@ -1,120 +0,0 @@
-import { addMissingPunctuationWordsToWordSequence, segmentText, segmentWordSequence, splitToWords, WordSequence } from "./exports/TextSegmentation.js";
-import { Timer } from "./utilities/Timer.js";
-const log = console.log;
-async function test1() {
-    //const text = 'Hello! 1. Say who? 2. How are you? This is v2.0 that good. I have 2 344 234ms C# is good games'
-    const text = 'the 6v34abc6g6 😊😊 -43.45 x -2445.65rty the';
-    //const text = 'Hello world! 23rd? 123% 3/4/7  Привет мир! 你好世界！'
-    const options = {
-        language: 'en'
-    };
-    const wordSequence = await splitToWords(text, options);
-    console.log(JSON.stringify(wordSequence.wordArray));
-    const segmentedText = await segmentWordSequence(wordSequence);
-    const x = 0;
-}
-async function test2() {
-    const { readFileSync, writeFileSync } = await import('fs');
-    const text = readFileSync('test-data/Japanese3.txt', 'utf-8');
-    const timer = new Timer();
-    let result;
-    for (let i = 0; i < 5; i++) {
-        result = await segmentText(text, {
-            language: 'en',
-            customSuppressions: [],
-            enableEastAsianPostprocessing: true
-        });
-        timer.logAndRestart(`Total execution time`);
-    }
-    log('');
-    //
-    let segmentedText = '';
-    for (let sentenceIndex = 0; sentenceIndex < result.sentences.length; sentenceIndex++) {
-        const sentence = result.sentences[sentenceIndex];
-        const phrases = sentence.phrases;
-        for (let phraseIndex = 0; phraseIndex < phrases.length; phraseIndex++) {
-            const phrase = phrases[phraseIndex];
-            segmentedText += phrase.words.wordArray.join(' | ');
-            if (phraseIndex < phrases.length - 1) {
-                segmentedText += `\n${'-'.repeat(100)}\n`;
-            }
-        }
-        if (sentenceIndex < result.sentences.length - 1) {
-            segmentedText += `\n${'='.repeat(100)} \n`;
-        }
-    }
-    writeFileSync('out/segmented.txt', segmentedText);
-}
-async function test3() {
-    const text = `  Hello,  how are you today                ??    `;
-    const words = await splitToWords(text);
-    const nonPunctuationWordEntries = words.nonPunctuationEntries;
-    const nonPunctuationWordSequence = new WordSequence();
-    nonPunctuationWordSequence.entries = nonPunctuationWordEntries;
-    const extendedWords = addMissingPunctuationWordsToWordSequence(nonPunctuationWordSequence, text);
-    const x = 0;
-}
-async function test4() {
-    const text = `
-	Hello 12/43 yo good-go bobo_baba man!
-	2.4a, 5.6 x&y x'v·y v‧z x·y·5
-	abc123 23+42.534 645
-	年代主演兩123部電影系列後
-	2004年-12月，公园被国家旅游局评定为国家4A级旅游景区.
-	ah'f.bf5.c.d.
-	-345.45%
-
-	Hello 1/ how are you?
-	76.54af's567
-	in an 8.2x8x3 grid
-
-	5343.234$
-
-	Hello World. How are you?
-
-	x·y·z
-	756.534-54
-
-	This is 23 GB.
-
-	That’s great if you want to cram as many of your friends’ genomes 'cause that's not.
-
-	I like C# and C++ languages!
-
-	Price is $60 or 60$
-
-`;
-    const result = (await splitToWords(text)).wordArray;
-    log(result.join(' | '));
-}
-async function test5() {
-    const text1 = `
-A.B.C.D.E.
-
-This A.I. techonology, tells the Dr. goodbye and nothing else. Yes.
-
-ただ
-◆
-
-
-街の
-`;
-    const text2 = `
-
-
-Hello World. Yo!
-\t
-
-gooooogoo
-
-, Hi. BOBO.
-`;
-    const wordSequence = await splitToWords(text2, { language: 'en' });
-    const result = await segmentWordSequence(wordSequence);
-    const sentencesText = result.sentences.map(x => x.text);
-    log(sentencesText.join(' \n '));
-}
-//test1()
-//test2()
-test5();
-//# sourceMappingURL=Test.js.map

package/dist/Test.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"Test.js","sourceRoot":"","sources":["../src/Test.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,wCAAwC,EAA2C,WAAW,EAAE,mBAAmB,EAAE,YAAY,EAAE,YAAY,EAAE,MAAM,+BAA+B,CAAA;AAC/L,OAAO,EAAE,KAAK,EAAE,MAAM,sBAAsB,CAAA;AAE5C,MAAM,GAAG,GAAG,OAAO,CAAC,GAAG,CAAA;AAEvB,KAAK,UAAU,KAAK;IACnB,+GAA+G;IAC/G,MAAM,IAAI,GAAG,8CAA8C,CAAA;IAC3D,iEAAiE;IAEjE,MAAM,OAAO,GAAwB;QACpC,QAAQ,EAAE,IAAI;KACd,CAAA;IAED,MAAM,YAAY,GAAG,MAAM,YAAY,CAAC,IAAI,EAAE,OAAO,CAAC,CAAA;IAEtD,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,SAAS,CAAC,YAAY,CAAC,SAAS,CAAC,CAAC,CAAA;IAEnD,MAAM,aAAa,GAAG,MAAM,mBAAmB,CAAC,YAAY,CAAC,CAAA;IAE7D,MAAM,CAAC,GAAG,CAAC,CAAA;AACZ,CAAC;AAED,KAAK,UAAU,KAAK;IACnB,MAAM,EAAE,YAAY,EAAE,aAAa,EAAE,GAAG,MAAM,MAAM,CAAC,IAAI,CAAC,CAAA;IAC1D,MAAM,IAAI,GAAG,YAAY,CAAC,yBAAyB,EAAE,OAAO,CAAC,CAAA;IAE7D,MAAM,KAAK,GAAG,IAAI,KAAK,EAAE,CAAA;IAEzB,IAAI,MAA0B,CAAA;IAE9B,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;QAC5B,MAAM,GAAG,MAAM,WAAW,CAAC,IAAI,EAAE;YAChC,QAAQ,EAAE,IAAI;YACd,kBAAkB,EAAE,EAAE;YACtB,6BAA6B,EAAE,IAAI;SACnC,CAAC,CAAA;QAEF,KAAK,CAAC,aAAa,CAAC,sBAAsB,CAAC,CAAA;IAC5C,CAAC;IAED,GAAG,CAAC,EAAE,CAAC,CAAA;IAEP,EAAE;IAEF,IAAI,aAAa,GAAG,EAAE,CAAA;IAEtB,KAAK,IAAI,aAAa,GAAG,CAAC,EAAE,aAAa,GAAG,MAAO,CAAC,SAAS,CAAC,MAAM,EAAE,aAAa,EAAE,EAAE,CAAC;QACvF,MAAM,QAAQ,GAAG,MAAO,CAAC,SAAS,CAAC,aAAa,CAAC,CAAA;QACjD,MAAM,OAAO,GAAG,QAAQ,CAAC,OAAO,CAAA;QAEhC,KAAK,IAAI,WAAW,GAAG,CAAC,EAAE,WAAW,GAAG,OAAO,CAAC,MAAM,EAAE,WAAW,EAAE,EAAE,CAAC;YACvE,MAAM,MAAM,GAAG,OAAO,CAAC,WAAW,CAAC,CAAA;YAEnC,aAAa,IAAI,MAAM,CAAC,KAAK,CAAC,SAAS,CAAC,IAAI,CAAC,KAAK,CAAC,CAAA;YAEnD,IAAI,WAAW,GAAG,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBACtC,aAAa,IAAI,KAAK,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,IAAI,CAAA;YAC1C,CAAC;QACF,CAAC;QAED,IAAI,aAAa,GAAG,MAAO,CAAC,SAAS,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAClD,aAAa,IAAI,KAAK,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,KAAK,CAAA;QAC3C,CAAC;IACF,CAAC;IAED,aAAa,CAAC,mBAAmB,EAAE,aAAa,CAAC,CAAA;AAClD,CAAC;AAED,KAAK,UAAU,KAAK;IACnB,MAAM,IAAI,GAAG,mDAAmD,CAAA;IAEhE,MAAM,KAAK,GAAG,MAAM,YAAY,CAAC,IAAI,CAAC,CAAA;IAEtC,MAAM,yBAAyB,GAAG,KAAK,CAAC,qBAAqB,CAAA;IAC7D,MAAM,0BAA0B,GAAG,IAAI,YAAY,EAAE,CAAA;IACrD,0BAA0B,CAAC,OAAO,GAAG,yBAAyB,CAAA;IAE9D,MAAM,aAAa,GAAG,wCAAwC,CAAC,0BAA0B,EAAE,IAAI,CAAC,CAAA;IAEhG,MAAM,CAAC,GAAG,CAAC,CAAA;AACZ,CAAC;AAED,KAAK,UAAU,KAAK;IACnB,MAAM,IAAI,GAAG;;;;;;;;;;;;;;;;;;;;;;;;;;;;CA4Bb,CAAA;IAEA,MAAM,MAAM,GAAG,CAAC,MAAM,YAAY,CAAC,IAAI,CAAC,CAAC,CAAC,SAAS,CAAA;IACnD,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAA;AACxB,CAAC;AAGD,KAAK,UAAU,KAAK;IACnB,MAAM,KAAK,GAAG;;;;;;;;;;CAUd,CAAA;IAEA,MAAM,KAAK,GAAG;;;;;;;;;CASd,CAAA;IAEA,MAAM,YAAY,GAAG,MAAM,YAAY,CAAC,KAAK,EAAE,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC,CAAA;IAElE,MAAM,MAAM,GAAG,MAAM,mBAAmB,CAAC,YAAY,CAAC,CAAA;IAEtD,MAAM,aAAa,GAAG,MAAM,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAA;IAEvD,GAAG,CAAC,aAAa,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,CAAA;AAChC,CAAC;AAED,SAAS;AACT,SAAS;AACT,KAAK,EAAE,CAAA"}

package/dist/exports/TextSegmentation.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"TextSegmentation.d.ts","sourceRoot":"","sources":["../../src/exports/TextSegmentation.ts"],"names":[],"mappings":"AAGA,OAAO,EAAE,YAAY,EAAE,MAAM,mBAAmB,CAAA;AAEhD,OAAO,EAAE,YAAY,EAAE,KAAK,SAAS,EAAE,MAAM,mBAAmB,CAAA;AAOhE,wBAAsB,WAAW,CAAC,IAAI,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,mBAAmB,+BAI5E;AAED,wBAAsB,mBAAmB,CAAC,YAAY,EAAE,YAAY,EAAE,OAAO,CAAC,EAAE,+BAA+B,+BAoL9G;AAID,wBAAsB,YAAY,CAAC,IAAI,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,mBAAmB,yBAmF7E;AAmDD,wBAAgB,wCAAwC,CAAC,YAAY,EAAE,YAAY,EAAE,UAAU,EAAE,MAAM;;;EAsDtG;AA+ED,MAAM,WAAW,kBAAkB;IAClC,KAAK,EAAE,YAAY,CAAA;IACnB,SAAS,EAAE,QAAQ,EAAE,CAAA;IACrB,qBAAqB,EAAE,KAAK,EAAE,CAAA;CAC9B;AAED,qBAAa,YAAY;IACxB,SAAS,EAAE,KAAK,CAAA;IAChB,KAAK,EAAE,YAAY,CAAA;IAEnB,YAAY,SAAS,EAAE,KAAK,EAAE,KAAK,EAAE,YAAY,EAGhD;IAED,IAAI,IAAI,WAEP;IAED,IAAI,SAAS,IAAI,KAAK,CAKrB;CACD;AAED,qBAAa,QAAS,SAAQ,YAAY;IACzC,OAAO,EAAE,MAAM,EAAE,CAAK;CACtB;AAED,qBAAa,MAAO,SAAQ,YAAY;CACvC;AAED,MAAM,WAAW,KAAK;IACrB,KAAK,EAAE,MAAM,CAAA;IACb,GAAG,EAAE,MAAM,CAAA;CACX;AAED,MAAM,WAAW,mBAAmB;IACnC,QAAQ,CAAC,EAAE,MAAM,CAAA;IACjB,kBAAkB,CAAC,EAAE,MAAM,EAAE,CAAA;IAC7B,6BAA6B,CAAC,EAAE,OAAO,CAAA;CACvC;AAED,eAAO,MAAM,0BAA0B,EAAE,mBAIxC,CAAA;AAED,MAAM,WAAW,+BAA+B;IAC/C,mCAAmC,EAAE,OAAO,CAAA;CAC5C;AAED,eAAO,MAAM,sCAAsC,EAAE,+BAEpD,CAAA"}

package/dist/exports/TextSegmentation.js

@@ -1,369 +0,0 @@
-import { buildWordOrNumberPattern as buildWordSplitterPattern, phraseSeparatorRegExp, sentenceSeparatorTrailingPunctuationCharacterRegExp, sentenceSeparatorCharacterRegExp, whitespacePatternRegExp, letterPatternGlobalRegExp } from '../patterns/Patterns.js';
-import { cldrSuppressions, additionalSuppressions, leadingApostropheContractionSuppressions, nounSuppressions, tldSuppressions } from '../patterns/Suppressions.js';
-import { eastAsianCharRangesRegExp } from '../patterns/EastAsianCharacterPatterns.js';
-import { WordSequence } from './WordSequence.js';
-import { getShortLanguageCode } from '../utilities/Utilities.js';
-export { WordSequence } from './WordSequence.js';
-import { buildRegExp } from 'regexp-composer';
-////////////////////////////////////////////////////////////////////////////////////////////////
-// Exported methods
-////////////////////////////////////////////////////////////////////////////////////////////////
-export async function segmentText(text, options) {
-    const wordSequence = await splitToWords(text, options);
-    return segmentWordSequence(wordSequence);
-}
-export async function segmentWordSequence(wordSequence, options) {
-    options = { ...defaultWordSequenceSegmentationOptions, ...options };
-    const segmentWordRanges = [];
-    {
-        let segmentStartWordOffset = 0;
-        let nonWhitespaceWordSeen = false;
-        let newlineSeenInCurrentSegment = false;
-        for (let wordIndex = 0; wordIndex < wordSequence.length; wordIndex++) {
-            const word = wordSequence.getWordAt(wordIndex);
-            if (!newlineSeenInCurrentSegment && nonWhitespaceWordSeen && word.endsWith('\n')) {
-                newlineSeenInCurrentSegment = true;
-            }
-            if (newlineSeenInCurrentSegment) {
-                const nextWord = wordIndex < wordSequence.length - 1 ? wordSequence.getWordAt(wordIndex + 1) : '';
-                if (nextWord === '' || !whitespacePatternRegExp.test(nextWord)) {
-                    segmentWordRanges.push({ start: segmentStartWordOffset, end: wordIndex + 1 });
-                    segmentStartWordOffset = wordIndex + 1;
-                    newlineSeenInCurrentSegment = false;
-                }
-            }
-            if (nonWhitespaceWordSeen === false) {
-                nonWhitespaceWordSeen = !whitespacePatternRegExp.test(word);
-            }
-        }
-        if (segmentStartWordOffset < wordSequence.length) {
-            segmentWordRanges.push({ start: segmentStartWordOffset, end: wordSequence.length });
-        }
-    }
-    const sentenceWordRanges = [];
-    const segmentSentenceRanges = [];
-    {
-        const minimumSentenceLetterCount = 2;
-        for (let segmentIndex = 0; segmentIndex < segmentWordRanges.length; segmentIndex++) {
-            const segmentWordRange = segmentWordRanges[segmentIndex];
-            const segmentStartWordIndex = segmentWordRange.start;
-            const segmentEndWordIndex = segmentWordRange.end;
-            segmentSentenceRanges.push({ start: sentenceWordRanges.length, end: sentenceWordRanges.length });
-            let nonWhitespaceWordSeen = false;
-            let sentenceStartWordOffset = segmentStartWordIndex;
-            let currentSentenceLetterCount = 0;
-            for (let wordIndex = segmentStartWordIndex; wordIndex < segmentEndWordIndex; wordIndex++) {
-                const word = wordSequence.getWordAt(wordIndex);
-                if (nonWhitespaceWordSeen === false) {
-                    nonWhitespaceWordSeen = !whitespacePatternRegExp.test(word);
-                }
-                if (currentSentenceLetterCount < minimumSentenceLetterCount) {
-                    const matches = word.matchAll(letterPatternGlobalRegExp);
-                    for (const _ of matches) {
-                        currentSentenceLetterCount += 1;
-                        if (currentSentenceLetterCount >= minimumSentenceLetterCount) {
-                            break;
-                        }
-                    }
-                }
-                if (nonWhitespaceWordSeen && currentSentenceLetterCount >= minimumSentenceLetterCount && sentenceSeparatorCharacterRegExp.test(word)) {
-                    let trailingSequenceEndIndex = wordIndex;
-                    while (trailingSequenceEndIndex < segmentEndWordIndex) {
-                        const trailingWord = wordSequence.getWordAt(trailingSequenceEndIndex);
-                        if (sentenceSeparatorTrailingPunctuationCharacterRegExp.test(trailingWord)) {
-                            trailingSequenceEndIndex++;
-                        }
-                        else {
-                            break;
-                        }
-                    }
-                    sentenceWordRanges.push({
-                        start: sentenceStartWordOffset,
-                        end: trailingSequenceEndIndex
-                    });
-                    segmentSentenceRanges[segmentSentenceRanges.length - 1].end += 1;
-                    sentenceStartWordOffset = trailingSequenceEndIndex;
-                    currentSentenceLetterCount = 0;
-                    wordIndex = trailingSequenceEndIndex - 1;
-                }
-            }
-            if (sentenceStartWordOffset < segmentEndWordIndex) {
-                sentenceWordRanges.push({ start: sentenceStartWordOffset, end: segmentEndWordIndex });
-                segmentSentenceRanges[segmentSentenceRanges.length - 1].end += 1;
-            }
-        }
-    }
-    const sentences = [];
-    for (const wordRange of sentenceWordRanges) {
-        const sentenceWordSequence = wordSequence.slice(wordRange.start, wordRange.end);
-        sentences.push(new Sentence(wordRange, sentenceWordSequence));
-    }
-    for (const sentence of sentences) {
-        const phraseWordRanges = [];
-        let sentenceEndWordOffset = sentence.wordRange.end;
-        let phraseStartWordOffset = sentence.wordRange.start;
-        for (let wordIndex = phraseStartWordOffset; wordIndex < sentenceEndWordOffset; wordIndex++) {
-            const currentWord = wordSequence.getWordAt(wordIndex);
-            const isCurrentWordPhraseSeparator = phraseSeparatorRegExp.test(currentWord) &&
-                (!options.requireSpaceAfterColonsOrSemicolons ||
-                    (currentWord !== ':' && currentWord !== ';') ||
-                    whitespacePatternRegExp.test(wordSequence.getWordAt(wordIndex + 1)));
-            if (isCurrentWordPhraseSeparator) {
-                let whitespaceSeenOnce = false;
-                while (wordIndex < sentenceEndWordOffset - 1) {
-                    const nextWord = wordSequence.getWordAt(wordIndex + 1);
-                    if (!sentenceSeparatorTrailingPunctuationCharacterRegExp.test(nextWord)) {
-                        break;
-                    }
-                    if (whitespacePatternRegExp.test(nextWord)) {
-                        whitespaceSeenOnce = true;
-                    }
-                    if (nextWord === '"' && whitespaceSeenOnce) {
-                        break;
-                    }
-                    wordIndex += 1;
-                }
-                phraseWordRanges.push({
-                    start: phraseStartWordOffset,
-                    end: wordIndex + 1
-                });
-                phraseStartWordOffset = wordIndex + 1;
-            }
-        }
-        if (phraseStartWordOffset < sentenceEndWordOffset) {
-            phraseWordRanges.push({ start: phraseStartWordOffset, end: sentenceEndWordOffset });
-        }
-        for (const wordRange of phraseWordRanges) {
-            const phraseWordSequence = wordSequence.slice(wordRange.start, wordRange.end);
-            sentence.phrases.push(new Phrase(wordRange, phraseWordSequence));
-        }
-    }
-    const result = {
-        words: wordSequence,
-        sentences,
-        segmentSentenceRanges,
-    };
-    return result;
-}
-const cachedWordSplitterRegExps = new Map();
-export async function splitToWords(text, options) {
-    if (!options) {
-        options = {};
-    }
-    options = { ...defaultSegmentationOptions, ...options };
-    if (options.language) {
-        options.language = getShortLanguageCode(options.language);
-    }
-    const optionsAsJson = JSON.stringify(options);
-    let wordSplitterRegExp = cachedWordSplitterRegExps.get(optionsAsJson);
-    if (!wordSplitterRegExp) {
-        wordSplitterRegExp = buildWordSplitterRegExpForOptions(options);
-        cachedWordSplitterRegExps.set(optionsAsJson, wordSplitterRegExp);
-    }
-    let wordSequence = new WordSequence();
-    function addPunctuationWordsBetween(startOffset, endOffset) {
-        const punctuationWordSubstring = text.substring(startOffset, endOffset);
-        let charOffset = startOffset;
-        let punctuationWordStartOffset = startOffset;
-        function addPunctuationWordIfNeeded() {
-            if (charOffset > punctuationWordStartOffset) {
-                const wordText = text.substring(punctuationWordStartOffset, charOffset);
-                wordSequence.addWord(wordText, punctuationWordStartOffset, true);
-                punctuationWordStartOffset = charOffset;
-            }
-        }
-        for (const char of punctuationWordSubstring) {
-            if (char === ' ') {
-                charOffset += 1;
-                continue;
-            }
-            addPunctuationWordIfNeeded();
-            charOffset += char.length;
-            addPunctuationWordIfNeeded();
-        }
-        addPunctuationWordIfNeeded();
-    }
-    const wordMatches = text.matchAll(wordSplitterRegExp);
-    let lastMatchEndOffset = 0;
-    if (wordMatches) {
-        for (const match of wordMatches) {
-            const offsets = match.indices[0];
-            const matchStartOffset = offsets[0];
-            const matchEndOffset = offsets[1];
-            if (matchStartOffset > lastMatchEndOffset) {
-                addPunctuationWordsBetween(lastMatchEndOffset, matchStartOffset);
-            }
-            const wordText = text.substring(matchStartOffset, matchEndOffset);
-            wordSequence.addWord(wordText, matchStartOffset, false);
-            lastMatchEndOffset = matchEndOffset;
-        }
-        addPunctuationWordsBetween(lastMatchEndOffset, text.length);
-    }
-    if (options.enableEastAsianPostprocessing) {
-        wordSequence = await postprocessEastAsianWords(text, wordSequence);
-    }
-    return wordSequence;
-}
-async function postprocessEastAsianWords(containingText, wordSequence) {
-    const icuSegmentation = await getIcuSegmentation();
-    if (icuSegmentation === undefined) {
-        return wordSequence;
-    }
-    let icuInitialized = false;
-    const newWordSequence = new WordSequence();
-    for (let wordIndex = 0; wordIndex < wordSequence.length; wordIndex++) {
-        const wordEntry = wordSequence.entries[wordIndex];
-        const wordStartOffset = wordEntry.startOffset;
-        const word = wordSequence.getWordAt(wordIndex);
-        if (eastAsianCharRangesRegExp.test(word)) {
-            if (!icuInitialized) {
-                await icuSegmentation.initialize();
-                icuInitialized = true;
-            }
-            const wordBreaks = [...icuSegmentation.createWordBreakIterator(word)];
-            for (let i = 0; i < wordBreaks.length - 1; i++) {
-                const subwordStartOffset = wordStartOffset + wordBreaks[i];
-                const subwordEndOffset = wordStartOffset + wordBreaks[i + 1];
-                const subwordText = containingText.substring(subwordStartOffset, subwordEndOffset);
-                newWordSequence.addWord(subwordText, subwordStartOffset, false);
-            }
-        }
-        else {
-            const wordText = containingText.substring(wordEntry.startOffset, wordEntry.endOffset);
-            newWordSequence.addWord(wordText, wordEntry.startOffset, wordEntry.isPunctuation);
-        }
-    }
-    return newWordSequence;
-}
-// Add any missing punctuation words to a word sequence
-export function addMissingPunctuationWordsToWordSequence(wordSequence, sourceText) {
-    const originalWordsReverseMapping = new Map();
-    const wordSequenceWithPunctuation = new WordSequence();
-    function addWordEntriesForTextSlice(textSlice, initialCharOffset) {
-        let charOffset = initialCharOffset;
-        // Add entry for every codepoint (this will correctly treat characters beyond BMP)
-        for (const char of textSlice) {
-            const charEndOffset = charOffset + char.length;
-            const lastEntry = wordSequenceWithPunctuation.lastEntry;
-            if (char === ' ' && lastEntry && lastEntry.isPunctuation && lastEntry.text[0] === ' ') {
-                wordSequenceWithPunctuation.lastEntry.text += ' ';
-                wordSequenceWithPunctuation.lastEntry.endOffset = charEndOffset;
-            }
-            else {
-                const wordText = sourceText.substring(charOffset, charEndOffset);
-                wordSequenceWithPunctuation.addWord(wordText, charOffset, true);
-            }
-            charOffset = charEndOffset;
-        }
-    }
-    for (let wordIndex = 0; wordIndex < wordSequence.length; wordIndex++) {
-        const wordEntry = wordSequence.getEntryAt(wordIndex);
-        const wordStartOffset = wordEntry.startOffset;
-        const previousWordEndOffset = wordIndex > 0 ? wordSequence.entries[wordIndex - 1].endOffset : 0;
-        // Add entries for any punctuation characters between the current and previous word (or start)
-        if (previousWordEndOffset !== wordStartOffset) {
-            const textSlice = sourceText.substring(previousWordEndOffset, wordStartOffset);
-            addWordEntriesForTextSlice(textSlice, previousWordEndOffset);
-        }
-        wordSequenceWithPunctuation.entries.push(wordEntry);
-        originalWordsReverseMapping.set(wordSequenceWithPunctuation.length - 1, wordIndex);
-        // If last word, add entries for any trailing punctuation characters
-        if (wordIndex === wordSequence.length - 1) {
-            if (sourceText.length !== wordEntry.endOffset) {
-                const textSlice = sourceText.substring(wordEntry.endOffset, sourceText.length);
-                addWordEntriesForTextSlice(textSlice, wordEntry.endOffset);
-            }
-        }
-    }
-    return { wordSequenceWithPunctuation, originalWordsReverseMapping };
-}
-function getPunctuationRanges(wordSequence, text) {
-    const punctuationRanges = [];
-    const wordEntries = wordSequence.entries;
-    if (wordEntries[0].startOffset > 0) {
-        punctuationRanges.push({ start: 0, end: wordEntries[0].startOffset });
-    }
-    for (let i = 0; i < wordEntries.length; i++) {
-        const entry = wordEntries[i];
-        const previousEndOffset = wordEntries[i - 1]?.endOffset ?? 0;
-        if (entry.startOffset > previousEndOffset) {
-            punctuationRanges.push({ start: previousEndOffset, end: entry.startOffset });
-        }
-        if (entry.isPunctuation) {
-            punctuationRanges.push({ start: entry.startOffset, end: entry.endOffset });
-        }
-    }
-    {
-        const lastEndOffset = wordEntries[wordEntries.length - 1]?.endOffset;
-        if (lastEndOffset && lastEndOffset < text.length) {
-            punctuationRanges.push({ start: lastEndOffset, end: text.length });
-        }
-    }
-    return punctuationRanges;
-}
-////////////////////////////////////////////////////////////////////////////////////////////////
-// Helper methods
-////////////////////////////////////////////////////////////////////////////////////////////////
-function buildWordSplitterRegExpForOptions(options) {
-    const cldrSuppressionsForLang = cldrSuppressions[options.language ?? ''] ?? [];
-    const extendedSuppressionsForLang = additionalSuppressions[options.language ?? ''] ?? [];
-    const contractionSuppressionsForLang = leadingApostropheContractionSuppressions[options.language ?? ''] ?? [];
-    const contractionSuppressionsForLangWithSingleQuote = contractionSuppressionsForLang.map(str => str.replaceAll(`'`, `’`));
-    const customSuppressions = options.customSuppressions ?? [];
-    let suppressions = [
-        ...customSuppressions,
-        ...cldrSuppressionsForLang,
-        ...extendedSuppressionsForLang,
-        ...contractionSuppressionsForLang,
-        ...contractionSuppressionsForLangWithSingleQuote,
-        ...nounSuppressions,
-        ...tldSuppressions,
-    ];
-    const wordPattern = buildWordSplitterPattern([
-        ...suppressions,
-        ...suppressions.map(word => word.toLocaleLowerCase()),
-        ...suppressions.map(word => word.toLocaleUpperCase()),
-    ]);
-    const wordSplitterRegExp = buildRegExp(wordPattern, { global: true });
-    return wordSplitterRegExp;
-}
-async function getIcuSegmentation() {
-    try {
-        const icuSegmentation = await import('@echogarden/icu-segmentation-wasm');
-        return icuSegmentation;
-    }
-    catch {
-        return undefined;
-    }
-}
-export class TextFragment {
-    wordRange;
-    words;
-    constructor(wordRange, words) {
-        this.wordRange = wordRange;
-        this.words = words;
-    }
-    get text() {
-        return this.words.text;
-    }
-    get charRange() {
-        return {
-            start: this.words.firstEntry.startOffset,
-            end: this.words.lastEntry.endOffset
-        };
-    }
-}
-export class Sentence extends TextFragment {
-    phrases = [];
-}
-export class Phrase extends TextFragment {
-}
-export const defaultSegmentationOptions = {
-    language: '',
-    customSuppressions: [],
-    enableEastAsianPostprocessing: true,
-};
-export const defaultWordSequenceSegmentationOptions = {
-    requireSpaceAfterColonsOrSemicolons: true
-};
-//# sourceMappingURL=TextSegmentation.js.map