@echogarden/text-segmentation 0.1.0

package/LICENSE.md

@@ -0,0 +1,7 @@
+Copyright (c) 2024 Rotem Dan
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the “Software”), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

package/README.md

@@ -0,0 +1,178 @@
+# Echogarden text segmentation library
+
+A library providing word, phrase and sentence segmentation for natural language text.
+
+* Fully **multilingual**. Covers all languages and writing systems representable as Unicode characters. When needed, applies second stage processing using a [WebAssembly port](https://github.com/echogarden-project/icu-segmentation-wasm) of the [ICU C++ Library](https://icu.unicode.org/) (if available) for handling difficult to segment east-Asian languages, like Chinese, Japanese, Thai and Khmer
+* Segments **mixtures of different languages**, including mixtures of different scripts, like Chinese, Cyrillic and Latin, all within a single sentence or phrase, without needing to explicitly specify a particular language
+* Includes **built-in suppression lists** for language-specific abbreviations and special word patterns. Currently included languages are English, German, Spanish, French, Italian, Portuguese and Russian, mostly extracted from the [CLDR JSON datasets](https://github.com/unicode-org/cldr-json)
+* Accepts **user-provided suppression lists**
+* **Very fast**. For most languages, word segmentation is done mostly via a single regular expression, dynamically built for the given options, and suppression set. Majority of processing is done within the JavaScript regular expression engine, or within optimized WebAssembly binaries
+* Written in TypeScript
+
+## Installation
+
+```
+npm install @echogarden/text-segmentation
+```
+
+To enable fine-grained segmentation of Chinese, Japanese, Thai and Khmer words, install this additional package:
+
+```
+npm install @echogarden/icu-segmentation-wasm
+```
+
+(`@echogarden/icu-segmentation-wasm` is a 27 MB package, and is set as a [peer dependency](https://docs.npmjs.com/cli/v10/configuring-npm/package-json#peerdependencies). For size reasons, it's not installed by default, to ease on deployments that may not require it - especially browsers)
+
+## Usage
+
+### Splitting to words
+```ts
+import { splitToWords } from '@echogarden/text-segmentation'
+
+const wordSequence: WordSequence =
+  await splitToWords('Hello, world! How are you doing today?', { language: 'en' })
+
+console.log(wordSequence.words)
+```
+Prints a list of words (including spaces and punctuation):
+
+```ts
+['Hello', ',', ' ', 'world', '!', ' ', 'How',' ', 'are', ' ', 'you', ' ', 'doing', ' ', 'today', '?']
+```
+
+### Language mixtures
+
+A language option provided like `language: 'en'` is only used for loading suppression dictionaries. You can still include mixtures of different languages and scripts within the same input text:
+
+```ts
+await splitToWords('Hello world! Привет мир! 你好世界！')
+```
+
+Producing:
+```ts
+['Hello', ' ', 'world', '!', ' ', 'Привет', ' ', 'мир', '!', ' ', '你好', '世界', '！']
+```
+
+### Getting detailed metadata
+
+To get more detailed information,
+```ts
+console.log(wordSequence.entries)
+```
+
+prints a list of objects, including metadata on each word:
+
+```ts
+[
+  { text: 'Hello', startOffset: 0, endOffset: 5, isPunctuation: false },
+  { text: ',', startOffset: 5, endOffset: 6, isPunctuation: true },
+  { text: ' ', startOffset: 6, endOffset: 7, isPunctuation: true },
+  { text: 'world', startOffset: 7, endOffset: 12, isPunctuation: false },
+  { text: '!', startOffset: 12, endOffset: 13, isPunctuation: true },
+  { text: ' ', startOffset: 13, endOffset: 14, isPunctuation: true },
+  { text: 'How', startOffset: 14, endOffset: 17, isPunctuation: false },
+  { text: ' ', startOffset: 17, endOffset: 18, isPunctuation: true },
+  { text: 'are', startOffset: 18, endOffset: 21, isPunctuation: false },
+  { text: ' ', startOffset: 21, endOffset: 22, isPunctuation: true },
+  { text: 'you', startOffset: 22, endOffset: 25, isPunctuation: false },
+  { text: ' ', startOffset: 25, endOffset: 26, isPunctuation: true },
+  { text: 'doing', startOffset: 26, endOffset: 31, isPunctuation: false },
+  { text: ' ', startOffset: 31, endOffset: 32, isPunctuation: true },
+  { text: 'today', startOffset: 32, endOffset: 37, isPunctuation: false },
+  { text: '?', startOffset: 37, endOffset: 38, isPunctuation: true }
+]
+```
+
+### Segmenting to words, phrases and sentences
+
+```ts
+const result: SegmentationResult =
+  await segmentText(`Hello, world! How are you doing today?`)
+```
+
+`result` is a nested object containing a breakdown of sentences, phrases and words in the given text. It is described by these TypeScript types:
+```ts
+interface SegmentationResult {
+	wordSequence: WordSequence
+	sentences: Sentence[]
+}
+
+interface Sentence {
+	text: string
+	charRange: Range
+	wordRange: Range
+	wordSequence: WordSequence
+
+	phrases: Phrase[]
+}
+
+interface Phrase {
+	text: string
+	charRange: Range
+	wordRange: Range
+	wordSequence: WordSequence
+}
+
+interface Range {
+	start: number
+	end: number
+}
+```
+
+### Segmenting a predefined word sequence
+
+If you have a pre-existing `WordSequence` object (or possibly a modified form of an existing one), you can segment it to sentences and phrases without needing to recompute the word boundaries:
+
+```ts
+const result: SegmentationResult = await segmentWordSequence(wordSequence: WordSequence)
+```
+
+In this way, you can also specify custom word boundaries, and the phrase and sentence segmentation operations would ensure that break characters are never identified within the spans of non-punctuation words.
+
+## Algorithm outline
+
+### Word segmentation
+
+Looks for one out of several accepted character sequence patterns, and identifies that pattern as an individual word.
+
+There are several types of accepted patterns, evaluated in this order:
+* User-provided suppressions
+* Language-specific suppressions. For example, in English it would include abbreviations like `Mr.`, `Mrs.`, `e.g.`, `i.e`, `etc.`, or contractions like and `'cause`, `'bout` in English, `'n` in Afrikaans
+* Noun suppressions, shared in all languages, like names of brands, misc. abbreviations and programming languages. Examples: `C#`, `F#`, `C++`, `Yahoo!`, `Toys"R"Us`, `Dunkin'`, `Ke$ha`, `Sky+`, `I/O`, `A/C`, `A/V`
+* Top-level domains, like `.com`, `.org`, `.net` (also doubling as the noun `.NET` as in the ".NET framework")
+* Number patterns, consisting of a sequences of digits separated by various separator characters, like decimal separators `3.14`, `3,14`, and thousands separators like `233,421` (`,`), `233.421` (`.`) or `233 421` (` `). optional `-` or `+` signs like `-34,534.123`, `+43 345,344`
+* Date, time, and phone number patterns, consisting of digits separated by characters like `/`, `:`, `-`, like `15:23:23`, `1953/11/06`, `64-534-756`
+* Percentage patterns, which are a subset of number patterns preceded or followed by `%`, like `53.243%` or `%34.12`
+* Currency patterns like `$101.25`, `€50`, `20£`, `-53.23¥`, which, like percentage patterns, are number patterns preceded or followed by a currency symbol
+* Abbreviation patterns like `Y.M.C.A`: these patterns will be automatically matched (no special suppressions needed) if there is a sequence of single `.` alternating between single letters (like `x.y.z`), optionally, there may be a space between the characters, like `x. y. z.`
+* **And finally** (but most importantly): word character sequences consisting of letter characters (Unicode category `Letter`), mark characters (Unicode category `Mark`) or digit characters (Unicode category `Decimal_Number`), which may include inner apostrophes like `'` and `’` and inner separators like `-`, `_` `·`
+
+**Current limitations**:
+* No current general identification of preceding or trailing apostrophes, like in English possessive plurals "The brothers' friend" or "The diplomats' contracts". **Reason**: this requires a large lexicon or more sophisticated language understanding, since the apostrophe is generally ambiguous with a single quote.
+
+#### Further segmentation for words containing Chinese, Japanese, Thai, or Khmer characters
+
+This requires the the `@echogarden/icu-segmentation-wasm` package to be **manually** installed.
+
+* Iterate each previously identified word span
+* Check if the word span contains at at least one codepoint belonging to the supported east Asian languages character ranges
+* Load the ICU module on the first match that is found (due to a startup delay of about `40ms` - `80ms`, the module is only loaded if absolutely needed)
+* Split the word span further using the ICU library
+
+
+### Sentence and phrase segmentation
+
+**Sentence segmentation**:
+* Split to words, or use a pre-supplied word sequence
+* Identify sentence enders, like `.`, `。`, `?`, `？`, `!`, `！` or a line break, outside word spans, and identify sentence boundaries based on them
+* Extend sentence spans with any number of possible trailing punctuation characters, after the ender character, like `”`, `’`, `)`, `]`, `}`, `»`, line breaks or other whitespace
+
+**Phrase segmentation**:
+* Iterate over the sentences detected
+* Within each sentence, identify phrase separators like `,`, `、`, `，`, `،`, `；`, `;`, `:`, `：`, `—` outside word spans, and split phrases based on them
+* Extend phrase spans with any number of trailing punctuation characters, up to the end of the containing sentence
+
+
+## License
+
+MIT

package/dist/EastAsianCharacterPatterns.d.ts

@@ -0,0 +1,7 @@
+export declare const chineseCharacterRanges: import("regexp-composer").AnyOf;
+export declare const japaneseHiraganaCharacterRanges: import("regexp-composer").AnyOf;
+export declare const japaneseKatakanaCharacterRanges: import("regexp-composer").AnyOf;
+export declare const thaiLetterRanges: import("regexp-composer").AnyOf;
+export declare const khmerLetterRanges: import("regexp-composer").AnyOf;
+export declare const eastAsianCharRanges: import("regexp-composer").AnyOf;
+export declare const eastAsianCharRangesRegExp: RegExp;

package/dist/EastAsianCharacterPatterns.js

@@ -0,0 +1,24 @@
+import { anyOf, buildRegExp, codepointRange } from 'regexp-composer';
+export const chineseCharacterRanges = anyOf(codepointRange('4E00', '9FFF'), // Main
+codepointRange('3400', '4DBF'), // CJK Unified Ideographs Extension A
+codepointRange('20000', '2A6DF') // CJK Unified Ideographs Extension B
+);
+export const japaneseHiraganaCharacterRanges = anyOf(codepointRange('3040', '309F'), // Hiragana
+codepointRange('1AFF0', '1AFFF'), // Kana Extended-B
+codepointRange('1B000', '1B0FF'), // Kana Supplement
+codepointRange('1B100', '1B12F'), // Kana Extended-A
+codepointRange('1B130', '1B16F'));
+export const japaneseKatakanaCharacterRanges = anyOf(codepointRange('30A0', '30FF'), // Katakana
+codepointRange('31F0', '31FF'), // Katakana Phonetic Extensions
+codepointRange('3200', '32FF'), // Enclosed CJK Letters and Months
+codepointRange('FF00', 'FFEF'), // Halfwidth and Fullwidth Forms
+codepointRange('1AFF0', '1AFFF'), // Kana Extended-B
+codepointRange('1B000', '1B0FF'), // Kana Supplement
+codepointRange('1B100', '1B12F'), // Kana Extended-A
+codepointRange('1B130', '1B16F'));
+export const thaiLetterRanges = anyOf(codepointRange('0E00', '0E7F'));
+export const khmerLetterRanges = anyOf(codepointRange('1780', '17FF'), // Letters
+codepointRange('19E0', '19FF'));
+export const eastAsianCharRanges = anyOf(chineseCharacterRanges, japaneseHiraganaCharacterRanges, japaneseKatakanaCharacterRanges, thaiLetterRanges, khmerLetterRanges);
+export const eastAsianCharRangesRegExp = buildRegExp(eastAsianCharRanges);
+//# sourceMappingURL=EastAsianCharacterPatterns.js.map

package/dist/EastAsianCharacterPatterns.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"EastAsianCharacterPatterns.js","sourceRoot":"","sources":["../src/EastAsianCharacterPatterns.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,EAAE,WAAW,EAAE,cAAc,EAAE,MAAM,iBAAiB,CAAA;AAEpE,MAAM,CAAC,MAAM,sBAAsB,GAAG,KAAK,CAC1C,cAAc,CAAC,MAAM,EAAE,MAAM,CAAC,EAAE,OAAO;AACvC,cAAc,CAAC,MAAM,EAAE,MAAM,CAAC,EAAE,qCAAqC;AACrE,cAAc,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC,qCAAqC;CACtE,CAAA;AAED,MAAM,CAAC,MAAM,+BAA+B,GAAG,KAAK,CACnD,cAAc,CAAC,MAAM,EAAE,MAAM,CAAC,EAAE,WAAW;AAC3C,cAAc,CAAC,OAAO,EAAE,OAAO,CAAC,EAAE,kBAAkB;AACpD,cAAc,CAAC,OAAO,EAAE,OAAO,CAAC,EAAE,kBAAkB;AACpD,cAAc,CAAC,OAAO,EAAE,OAAO,CAAC,EAAE,kBAAkB;AACpD,cAAc,CAAC,OAAO,EAAE,OAAO,CAAC,CAChC,CAAA;AAED,MAAM,CAAC,MAAM,+BAA+B,GAAG,KAAK,CACnD,cAAc,CAAC,MAAM,EAAE,MAAM,CAAC,EAAE,WAAW;AAC3C,cAAc,CAAC,MAAM,EAAE,MAAM,CAAC,EAAE,+BAA+B;AAC/D,cAAc,CAAC,MAAM,EAAE,MAAM,CAAC,EAAE,kCAAkC;AAClE,cAAc,CAAC,MAAM,EAAE,MAAM,CAAC,EAAE,gCAAgC;AAChE,cAAc,CAAC,OAAO,EAAE,OAAO,CAAC,EAAE,kBAAkB;AACpD,cAAc,CAAC,OAAO,EAAE,OAAO,CAAC,EAAE,kBAAkB;AACpD,cAAc,CAAC,OAAO,EAAE,OAAO,CAAC,EAAE,kBAAkB;AACpD,cAAc,CAAC,OAAO,EAAE,OAAO,CAAC,CAChC,CAAA;AAED,MAAM,CAAC,MAAM,gBAAgB,GAAG,KAAK,CACpC,cAAc,CAAC,MAAM,EAAE,MAAM,CAAC,CAC9B,CAAA;AAED,MAAM,CAAC,MAAM,iBAAiB,GAAG,KAAK,CACrC,cAAc,CAAC,MAAM,EAAE,MAAM,CAAC,EAAE,UAAU;AAC1C,cAAc,CAAC,MAAM,EAAE,MAAM,CAAC,CAC9B,CAAA;AAED,MAAM,CAAC,MAAM,mBAAmB,GAAG,KAAK,CACvC,sBAAsB,EACtB,+BAA+B,EAC/B,+BAA+B,EAC/B,gBAAgB,EAChB,iBAAiB,CACjB,CAAA;AAED,MAAM,CAAC,MAAM,yBAAyB,GAAG,WAAW,CAAC,mBAAmB,CAAC,CAAA"}

package/dist/Patterns.d.ts

@@ -0,0 +1,34 @@
+export declare function buildWordOrNumberPattern(suppressions: string[]): import("regexp-composer").AnyOf;
+export declare function buildSuppressionPattern(suppressions: string[]): import("regexp-composer").Pattern;
+export declare const punctuationPattern: import("regexp-composer").SpecialToken;
+export declare const digitPattern: import("regexp-composer").SpecialToken;
+export declare const arabicNumeralPattern: import("regexp-composer").SpecialToken;
+export declare const numericSeparatorPattern: import("regexp-composer").Pattern;
+export declare const dateTimeSeparatorPattern: import("regexp-composer").Pattern;
+export declare const dateTimePattern: import("regexp-composer").OneOrMore;
+export declare const spacedThousandsSeparatorPattern: import("regexp-composer").Pattern;
+export declare const numericSignPattern: import("regexp-composer").Pattern;
+export declare const numberPattern: import("regexp-composer").OneOrMore[];
+export declare const prefixPercentageOrCurrencyPattern: import("regexp-composer").Pattern;
+export declare const suffixPercentagePattern: import("regexp-composer").Pattern;
+export declare const percentagePattern: import("regexp-composer").AnyOf;
+export declare const letterPattern: import("regexp-composer").SpecialToken;
+export declare const markPattern: import("regexp-composer").SpecialToken;
+export declare const apostrophPattern: import("regexp-composer").AnyOf;
+export declare const letterOrMarkPattern: import("regexp-composer").AnyOf;
+export declare const dottedAbbreviationSequencePattern: import("regexp-composer").Pattern;
+export declare const wordCharacterPattern: import("regexp-composer").AnyOf;
+export declare const wordSeparatorPattern: import("regexp-composer").Pattern;
+export declare const wordInnerApostrophPattern: import("regexp-composer").Pattern;
+export declare const wordStartApostrophPattern: import("regexp-composer").Pattern;
+export declare const basicWordPattern: import("regexp-composer").OneOrMore;
+export declare const wordSegmentPattern: import("regexp-composer").AnyOf;
+export declare const phraseSeparators: string[];
+export declare const phraseSeparatorRegExp: RegExp;
+export declare const sentenceSeparators: string[];
+export declare const sentenceSeparatorRegExp: RegExp;
+export declare const sentenceSeparatorTrailingPunctuationRegExp: RegExp;
+export declare const phraseSeparatorTrailingPunctuationRegExp: RegExp;
+export declare const oneOrMoreSpacesRegExp: RegExp;
+export declare const wordCharacterRegExp: RegExp;
+export declare const whitespacePatternRegExp: RegExp;

package/dist/Patterns.js

@@ -0,0 +1,126 @@
+import { anyOf, buildRegExp, charRange, inputEnd, inputStart, matches, oneOrMore, possibly, repeated, tab, unicodeProperty, whitespace } from 'regexp-composer';
+export function buildWordOrNumberPattern(suppressions) {
+    return anyOf(buildSuppressionPattern(suppressions), wordSegmentPattern);
+}
+export function buildSuppressionPattern(suppressions) {
+    const suppressionsPattern = matches(anyOf(...suppressions), { ifNotFollowedBy: wordCharacterPattern });
+    return suppressionsPattern;
+}
+////////////////////////////////////////////////////////////////////////////////////////////////
+// Numeric patterns
+////////////////////////////////////////////////////////////////////////////////////////////////
+export const punctuationPattern = unicodeProperty('Punctuation');
+export const digitPattern = unicodeProperty('Decimal_Number');
+export const arabicNumeralPattern = charRange('0', '9');
+export const numericSeparatorPattern = matches(anyOf('.', ',', '٬', '_'), {
+    ifPrecededBy: arabicNumeralPattern,
+    ifFollowedBy: arabicNumeralPattern
+});
+export const dateTimeSeparatorPattern = matches(anyOf('/', '-', ':'), {
+    ifPrecededBy: arabicNumeralPattern,
+    ifFollowedBy: arabicNumeralPattern
+});
+export const dateTimePattern = oneOrMore(anyOf(arabicNumeralPattern, dateTimeSeparatorPattern));
+export const spacedThousandsSeparatorPattern = matches(' ', {
+    ifPrecededBy: arabicNumeralPattern,
+    ifFollowedBy: repeated(2, arabicNumeralPattern)
+});
+export const numericSignPattern = matches(anyOf('-', '+'), {
+    ifPrecededBy: anyOf(whitespace, punctuationPattern),
+    ifFollowedBy: arabicNumeralPattern,
+});
+export const numberPattern = [
+    oneOrMore(anyOf(digitPattern, numericSeparatorPattern, spacedThousandsSeparatorPattern, numericSignPattern)),
+];
+const percentageChars = ['%'];
+const currencySpecialChars = ['$', '¥', '€', '£', '¥', '₩', '₭', '₽', '₫', '฿', '¢', '₮', '؋', '₦', '₱', '₴', '₪'];
+const percentageOrCurrencyPattern = anyOf(...percentageChars, ...currencySpecialChars);
+export const prefixPercentageOrCurrencyPattern = matches([
+    percentageOrCurrencyPattern,
+    numberPattern,
+], {
+    ifNotPrecededBy: digitPattern,
+    ifFollowedBy: anyOf(whitespace, punctuationPattern),
+});
+export const suffixPercentagePattern = matches([
+    numberPattern,
+    percentageOrCurrencyPattern,
+], {
+    ifPrecededBy: anyOf(whitespace, punctuationPattern),
+    ifNotFollowedBy: digitPattern,
+});
+export const percentagePattern = anyOf(prefixPercentageOrCurrencyPattern, suffixPercentagePattern);
+////////////////////////////////////////////////////////////////////////////////////////////////
+// Letter patterns
+////////////////////////////////////////////////////////////////////////////////////////////////
+export const letterPattern = unicodeProperty('Letter');
+export const markPattern = unicodeProperty('Mark');
+export const apostrophPattern = anyOf(`'`, `’`, `‘`);
+export const letterOrMarkPattern = anyOf(letterPattern, markPattern);
+export const dottedAbbreviationSequencePattern = matches(anyOf([
+    letterPattern,
+    oneOrMore([
+        '. ',
+        letterPattern,
+    ]),
+    possibly('.'),
+], [
+    letterPattern,
+    oneOrMore([
+        '.',
+        possibly(' '),
+        letterPattern,
+    ]),
+    possibly('.'),
+]), {
+    ifNotPrecededBy: anyOf(letterOrMarkPattern, digitPattern),
+    ifNotFollowedBy: anyOf(letterOrMarkPattern, digitPattern)
+});
+export const wordCharacterPattern = anyOf(letterPattern, markPattern, digitPattern);
+export const wordSeparatorPattern = matches(anyOf('-', '_', '·', '‧', '&'), {
+    ifPrecededBy: letterOrMarkPattern,
+    ifFollowedBy: letterOrMarkPattern
+});
+export const wordInnerApostrophPattern = matches(apostrophPattern, {
+    ifPrecededBy: letterOrMarkPattern,
+    ifFollowedBy: letterOrMarkPattern,
+});
+export const wordStartApostrophPattern = matches(apostrophPattern, {
+    ifPrecededBy: whitespace,
+    ifFollowedBy: [letterOrMarkPattern, letterOrMarkPattern, whitespace]
+});
+export const basicWordPattern = oneOrMore(anyOf(wordCharacterPattern, wordSeparatorPattern, wordInnerApostrophPattern));
+export const wordSegmentPattern = anyOf(dottedAbbreviationSequencePattern, dateTimeSeparatorPattern, percentagePattern, numberPattern, basicWordPattern);
+////////////////////////////////////////////////////////////////////////////////////////////////
+// Prebuilt regular expressions
+////////////////////////////////////////////////////////////////////////////////////////////////
+export const phraseSeparators = [',', '、', '，', '،', ';', '；', ':', '：', '—'];
+export const phraseSeparatorRegExp = buildRegExp([
+    inputStart,
+    anyOf(...phraseSeparators),
+    inputEnd
+]);
+export const sentenceSeparators = ['.', '。', '?', '？', '!', '！', '\n'];
+export const sentenceSeparatorRegExp = buildRegExp([
+    inputStart,
+    anyOf(...sentenceSeparators),
+    inputEnd
+]);
+export const sentenceSeparatorTrailingPunctuationRegExp = buildRegExp([
+    inputStart,
+    anyOf('"', '”', '’', ')', ']', '}', '»', ...sentenceSeparators, ...phraseSeparators, oneOrMore(whitespace)),
+    inputEnd
+]);
+export const phraseSeparatorTrailingPunctuationRegExp = buildRegExp([
+    inputStart,
+    anyOf(...phraseSeparators, ' ', tab),
+    inputEnd
+]);
+export const oneOrMoreSpacesRegExp = buildRegExp([
+    inputStart,
+    oneOrMore(' '),
+    inputEnd
+]);
+export const wordCharacterRegExp = buildRegExp(wordCharacterPattern);
+export const whitespacePatternRegExp = buildRegExp(whitespace);
+//# sourceMappingURL=Patterns.js.map

package/dist/Patterns.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"Patterns.js","sourceRoot":"","sources":["../src/Patterns.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,EAAE,WAAW,EAAE,SAAS,EAAE,QAAQ,EAAE,UAAU,EAAE,OAAO,EAAE,SAAS,EAAE,QAAQ,EAAE,QAAQ,EAAE,GAAG,EAAE,eAAe,EAAE,UAAU,EAAE,MAAM,iBAAiB,CAAA;AAE/J,MAAM,UAAU,wBAAwB,CAAC,YAAsB;IAC9D,OAAO,KAAK,CACX,uBAAuB,CAAC,YAAY,CAAC,EACrC,kBAAkB,CAClB,CAAA;AACF,CAAC;AAED,MAAM,UAAU,uBAAuB,CAAC,YAAsB;IAC7D,MAAM,mBAAmB,GACxB,OAAO,CACN,KAAK,CAAC,GAAG,YAAY,CAAC,EACtB,EAAE,eAAe,EAAE,oBAAoB,EAAE,CACzC,CAAA;IAEF,OAAO,mBAAmB,CAAA;AAC3B,CAAC;AAED,gGAAgG;AAChG,mBAAmB;AACnB,gGAAgG;AAChG,MAAM,CAAC,MAAM,kBAAkB,GAAG,eAAe,CAAC,aAAa,CAAC,CAAA;AAChE,MAAM,CAAC,MAAM,YAAY,GAAG,eAAe,CAAC,gBAAgB,CAAC,CAAA;AAC7D,MAAM,CAAC,MAAM,oBAAoB,GAAG,SAAS,CAAC,GAAG,EAAE,GAAG,CAAC,CAAA;AAEvD,MAAM,CAAC,MAAM,uBAAuB,GACnC,OAAO,CACN,KAAK,CAAC,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,CAAC,EAAE;IAC3B,YAAY,EAAE,oBAAoB;IAClC,YAAY,EAAE,oBAAoB;CAClC,CAAC,CAAA;AAEH,MAAM,CAAC,MAAM,wBAAwB,GAAG,OAAO,CAC9C,KAAK,CAAC,GAAG,EAAE,GAAG,EAAE,GAAG,CAAC,EAAE;IACtB,YAAY,EAAE,oBAAoB;IAClC,YAAY,EAAE,oBAAoB;CAClC,CAAC,CAAA;AAEF,MAAM,CAAC,MAAM,eAAe,GAC3B,SAAS,CAAC,KAAK,CACd,oBAAoB,EACpB,wBAAwB,CACxB,CAAC,CAAA;AAEH,MAAM,CAAC,MAAM,+BAA+B,GAC3C,OAAO,CACN,GAAG,EAAE;IACL,YAAY,EAAE,oBAAoB;IAClC,YAAY,EAAE,QAAQ,CAAC,CAAC,EAAE,oBAAoB,CAAC;CAC/C,CAAC,CAAA;AAEH,MAAM,CAAC,MAAM,kBAAkB,GAC9B,OAAO,CACN,KAAK,CAAC,GAAG,EAAE,GAAG,CAAC,EAAE;IACjB,YAAY,EAAE,KAAK,CAAC,UAAU,EAAE,kBAAkB,CAAC;IACnD,YAAY,EAAE,oBAAoB;CAClC,CAAC,CAAA;AAEH,MAAM,CAAC,MAAM,aAAa,GAAG;IAC5B,SAAS,CAAC,KAAK,CACd,YAAY,EACZ,uBAAuB,EACvB,+BAA+B,EAC/B,kBAAkB,CAClB,CAAC;CACF,CAAA;AAED,MAAM,eAAe,GAAG,CAAC,GAAG,CAAC,CAAA;AAC7B,MAAM,oBAAoB,GAAG,CAAC,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,CAAC,CAAA;AAElH,MAAM,2BAA2B,GAAG,KAAK,CAAC,GAAG,eAAe,EAAE,GAAG,oBAAoB,CAAC,CAAA;AAEtF,MAAM,CAAC,MAAM,iCAAiC,GAC7C,OAAO,CAAC;IACP,2BAA2B;IAC3B,aAAa;CACb,EAAE;IACF,eAAe,EAAE,YAAY;IAC7B,YAAY,EAAE,KAAK,CAAC,UAAU,EAAE,kBAAkB,CAAC;CACnD,CAAC,CAAA;AAEH,MAAM,CAAC,MAAM,uBAAuB,GACnC,OAAO,CAAC;IACP,aAAa;IACb,2BAA2B;CAC3B,EAAE;IACF,YAAY,EAAE,KAAK,CAAC,UAAU,EAAE,kBAAkB,CAAC;IACnD,eAAe,EAAE,YAAY;CAC7B,CAAC,CAAA;AAEH,MAAM,CAAC,MAAM,iBAAiB,GAAG,KAAK,CACrC,iCAAiC,EACjC,uBAAuB,CACvB,CAAA;AAED,gGAAgG;AAChG,kBAAkB;AAClB,gGAAgG;AAChG,MAAM,CAAC,MAAM,aAAa,GAAG,eAAe,CAAC,QAAQ,CAAC,CAAA;AACtD,MAAM,CAAC,MAAM,WAAW,GAAG,eAAe,CAAC,MAAM,CAAC,CAAA;AAClD,MAAM,CAAC,MAAM,gBAAgB,GAAG,KAAK,CAAC,GAAG,EAAE,GAAG,EAAE,GAAG,CAAC,CAAA;AAEpD,MAAM,CAAC,MAAM,mBAAmB,GAAG,KAAK,CACvC,aAAa,EACb,WAAW,CACX,CAAA;AAED,MAAM,CAAC,MAAM,iCAAiC,GAC7C,OAAO,CACN,KAAK,CACJ;IACC,aAAa;IAEb,SAAS,CAAC;QACT,IAAI;QACJ,aAAa;KACb,CAAC;IAEF,QAAQ,CAAC,GAAG,CAAC;CACb,EACD;IACC,aAAa;IAEb,SAAS,CAAC;QACT,GAAG;QACH,QAAQ,CAAC,GAAG,CAAC;QACb,aAAa;KACb,CAAC;IAEF,QAAQ,CAAC,GAAG,CAAC;CACb,CACD,EAAE;IACH,eAAe,EAAE,KAAK,CAAC,mBAAmB,EAAE,YAAY,CAAC;IACzD,eAAe,EAAE,KAAK,CAAC,mBAAmB,EAAE,YAAY,CAAC;CACzD,CAAC,CAAA;AAGH,MAAM,CAAC,MAAM,oBAAoB,GAChC,KAAK,CACJ,aAAa,EACb,WAAW,EACX,YAAY,CACZ,CAAA;AAEF,MAAM,CAAC,MAAM,oBAAoB,GAChC,OAAO,CACN,KAAK,CAAC,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,CAAC,EAAE;IAEhC,YAAY,EAAE,mBAAmB;IACjC,YAAY,EAAE,mBAAmB;CACjC,CAAC,CAAA;AAEH,MAAM,CAAC,MAAM,yBAAyB,GACrC,OAAO,CACN,gBAAgB,EAAE;IAElB,YAAY,EAAE,mBAAmB;IACjC,YAAY,EAAE,mBAAmB;CACjC,CAAC,CAAA;AAEH,MAAM,CAAC,MAAM,yBAAyB,GACrC,OAAO,CACN,gBAAgB,EAAE;IAElB,YAAY,EAAE,UAAU;IACxB,YAAY,EAAE,CAAC,mBAAmB,EAAE,mBAAmB,EAAE,UAAU,CAAC;CACpE,CAAC,CAAA;AAEH,MAAM,CAAC,MAAM,gBAAgB,GAC5B,SAAS,CACR,KAAK,CACJ,oBAAoB,EACpB,oBAAoB,EACpB,yBAAyB,CACzB,CACD,CAAA;AAEF,MAAM,CAAC,MAAM,kBAAkB,GAAG,KAAK,CACtC,iCAAiC,EACjC,wBAAwB,EACxB,iBAAiB,EACjB,aAAa,EACb,gBAAgB,CAChB,CAAA;AAED,gGAAgG;AAChG,+BAA+B;AAC/B,gGAAgG;AAChG,MAAM,CAAC,MAAM,gBAAgB,GAAG,CAAC,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,CAAC,CAAA;AAE7E,MAAM,CAAC,MAAM,qBAAqB,GAAG,WAAW,CAAC;IAChD,UAAU;IACV,KAAK,CAAC,GAAG,gBAAgB,CAAC;IAC1B,QAAQ;CACR,CAAC,CAAA;AAEF,MAAM,CAAC,MAAM,kBAAkB,GAAG,CAAC,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,IAAI,CAAC,CAAA;AAEtE,MAAM,CAAC,MAAM,uBAAuB,GAAG,WAAW,CAAC;IAClD,UAAU;IACV,KAAK,CAAC,GAAG,kBAAkB,CAAC;IAC5B,QAAQ;CACR,CAAC,CAAA;AAEF,MAAM,CAAC,MAAM,0CAA0C,GAAG,WAAW,CAAC;IACrE,UAAU;IACV,KAAK,CAAC,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,kBAAkB,EAAE,GAAG,gBAAgB,EAAE,SAAS,CAAC,UAAU,CAAC,CAAC;IAC3G,QAAQ;CACR,CAAC,CAAA;AAEF,MAAM,CAAC,MAAM,wCAAwC,GAAG,WAAW,CAAC;IACnE,UAAU;IACV,KAAK,CAAC,GAAG,gBAAgB,EAAE,GAAG,EAAE,GAAG,CAAC;IACpC,QAAQ;CACR,CAAC,CAAA;AAEF,MAAM,CAAC,MAAM,qBAAqB,GAAG,WAAW,CAAC;IAChD,UAAU;IACV,SAAS,CAAC,GAAG,CAAC;IACd,QAAQ;CACR,CAAC,CAAA;AAEF,MAAM,CAAC,MAAM,mBAAmB,GAAG,WAAW,CAAC,oBAAoB,CAAC,CAAA;AACpE,MAAM,CAAC,MAAM,uBAAuB,GAAG,WAAW,CAAC,UAAU,CAAC,CAAA"}

package/dist/Suppressions.d.ts

@@ -0,0 +1,4 @@
+export declare const cldrSuppressions: Record<string, string[]>;
+export declare const leadingApostropheContractionSuppressions: Record<string, string[]>;
+export declare const nounSuppressions: string[];
+export declare const tldSuppressions: string[];

package/dist/Suppressions.js

@@ -0,0 +1,31 @@
+export const cldrSuppressions = {
+    'en': ['L.P.', 'Alt.', 'Approx.', 'E.G.', 'O.', 'Maj.', 'Misc.', 'P.O.', 'J.D.', 'Jam.', 'Card.', 'Dec.', 'Sept.', 'MR.', 'Long.', 'Hat.', 'G.', 'Link.', 'DC.', 'D.C.', 'M.T.', 'Hz.', 'Mrs.', 'By.', 'Act.', 'Var.', 'N.V.', 'Aug.', 'B.', 'S.A.', 'Up.', 'Job.', 'Num.', 'M.I.T.', 'Ok.', 'Org.', 'Ex.', 'Cont.', 'U.', 'Mart.', 'Fn.', 'Abs.', 'Lt.', 'OK.', 'Z.', 'E.', 'Kb.', 'Est.', 'A.M.', 'L.A.', 'Prof.', 'U.S.', 'Nov.', 'Ph.D.', 'Mar.', 'I.T.', 'exec.', 'Jan.', 'N.Y.', 'X.', 'Md.', 'Op.', 'vs.', 'D.A.', 'A.D.', 'R.L.', 'P.M.', 'Or.', 'M.R.', 'Cap.', 'PC.', 'Feb.', 'Exec.', 'I.e.', 'Sep.', 'Gb.', 'K.', 'U.S.C.', 'Mt.', 'S.', 'A.S.', 'C.O.D.', 'Capt.', 'Col.', 'In.', 'C.F.', 'Adj.', 'AD.', 'I.D.', 'Mgr.', 'R.T.', 'B.V.', 'M.', 'Conn.', 'Yr.', 'Rev.', 'Phys.', 'pp.', 'Ms.', 'To.', 'Sgt.', 'J.K.', 'Nr.', 'Jun.', 'Fri.', 'S.A.R.', 'Lev.', 'Lt.Cdr.', 'Def.', 'F.', 'Do.', 'Joe.', 'Id.', 'Mr.', 'Dept.', 'Is.', 'Pvt.', 'Diff.', 'Hon.B.A.', 'Q.', 'Mb.', 'On.', 'Min.', 'J.B.', 'Ed.', 'AB.', 'A.', 'S.p.A.', 'I.', 'a.m.', 'Comm.', 'Go.', 'VS.', 'L.', 'All.', 'PP.', 'P.V.', 'T.', 'K.R.', 'Etc.', 'D.', 'Adv.', 'Lib.', 'E.g.', 'Pro.', 'U.S.A.', 'S.E.', 'AA.', 'Rep.', 'Sq.', 'As.'],
+    'de': ['Port.', 'Alt.', 'Di.', 'Ges.', 'frz.', 'entspr.', 'Gebr.', 'erw.', 'Frl.', 'Inh.', 'k.u.k.', 'Ca.', 'J.D.', 'Ausg.', 'evtl.', 'So.', 'i.B.', 's.a.', 'kgl.', 'Sept.', 'o.B.', 'Sa.', 'ev.', 'Dez.', 'am.', 'i.R.', 'eigtl.', 'i.J.', 'u.U.', 'G.', 'z.Hd.', 'u.A.w.g.', 'Kl.', 'Spezif.', 'Obj.', 'Ing.', 'D. h.', 'Folg.', 'Akt.', 'i.A.', 'Msp.', 'U.U.', 'Chr.', 'R.', 'Einh.', 'schwäb.', 'Vgl.', 'Aug.', 'Dipl.-Ing.', 'W.', 'B.', 'U. U.', 'J.', 'Fa.', 'Mo.', 'n.u.Z.', 'Op.', 'Mrd.', 'e.h.', 'Hr.', 'Hrn.', 'Ztr.', 'k. u. k.', 'Bibl.', 'd.Ä.', 'b.', 'M.', 'i.H.', 'v.R.w.', 'o.A.', 'St.', 'Dr.', 'Fn.', 'Abs.', 'Rd.', 'Dtzd.', 'Jahrh.', 'Z.', 'Std.', 'n. Chr.', 'möbl.', 'tägl.', 'gest.', 'gesch.', 'z.B.', 'Hbf.', 'Abt.', 'A.M.', 'e.Wz.', 'v.T.', 'Nov.', 'z.', 'Prot.', 'U.S.', 'Wg.', 'u.v.a.', 'Adr.', 'App.', 'ggf.', 'ggfs.', 'Jan.', 'O.', 'Rel.', 'od.', 'Pfd.', 'a.a.O.', 'p.Adr.', 'P.', 'Gem.', 'v. Chr.', 'Art.', 'z.Z.', 'S.A.', 'i.V.', 'verh.', 'Ausschl.', 'm.W.', 'Dir.', 'Verf.', 'Sek.', 'r.', 'Chin.', 'Feb.', 'Int.', 'Sep.', 'Gesch.', 'schweiz.', 'Bed.', 'a.Rh.', 'jew.', 'vgl.', 'a.M.', 'Str.', 'exkl.', 'gek.', 'Erf.', 'u.Ä.', 'ehem.', 'näml.', 'u. Z.', 'v. u. Z.', 'sog.', 'C.', 'Dipl.-Kfm.', 'mtl.', 'Hrsg.', 'Qu.', 'röm.', 'u.', 'U.', 'Adj.', 'Kap.', 'hpts.', 'a.D.', 'gedr.', 'Best.', 'N.', 'v.u.Z.', 'Phys.', 'Fr.', 'd.J.', 'Reg.-Bez.', 'm.E.', 'schles.', 'Max.', 'Ltd.', 'südd.', 'inkl.', 'geb.', 'Ggf.', 'Inc.', 'kath.', 'kfm.', 'Nr.', 'Proz.', 'Dim.', 'verw.', 'Reg.', 'Dat.', 'Evtl.', 'led.', 'F.', 'Test.', 'Schr.', 'Do.', 'PIN.', 'Z. Zt.', 'v.Chr.', 'Tägl.', 's.', 'amtl.', 'Temp.', 'Mind.', 'e.V.', 'Abw.', 'P.M.', 'F.f.', 'a.a.S.', 'Mod.', 'Co.', 'Min.', 'Allg.', 'Geograph.', 'Jr.', 'Urspr.', 'Apr.', 'Z. B.', 'v.H.', 'A.', 'einschl.', 'Trans.', 'zzgl.', 'StR.', 'Fam.', 'I.', 'jhrl.', 'u.a.', 'Ben.', 'o.g.', 'Kfm.', 'Konv.', 'Mi.', 'L.', 'beil.', 'T.', 'Ursprüngl.', 'röm.-kath.', 'Okt.', 'u.ä.', 'Tel.', 'D.', 'Ber.', 'Kop.', 'Mio.', 'Y.', 'U.S.A.', 'v. H.', 'Forts. f.', 'Rep.', 'Hptst.', 'österr.'],
+    'es': ['Rdos.', 'JJ.OO.', 'Sres.', 'fig.', 'may.', 'RR.HH.', 'oct.', 'cap.', 'mié.', 'doc.', 'Excmo.', 'Trab.', 'Excmos.', 'Kit.', 'Inc.', 'FF.CC.', 'DC.', 'ago.', 'trad.', 'SA.', 'Rvdos.', 'ed.', 'Exmo.', 'jul.', 'col.', 'RAM.', 'Srtas.', 'ene.', 'Rol.', 'Fabric.', 'Comm.', 'vid.', 'Da.', 'dic.', 'ss.', 'abr.', 'ntra.', 'Sra.', 'dtor.', 'cf.', 'dom.', 'prov.', 'Emm.', 'Sr.', 'licdo.', 'p.ej.', 'bol.', 'figs.', 'Vda.', 'Dr.', 'ntro.', 'Desv.', 'O.M.', 'Ldo.', 'Drs.', 'sáb.', 'feb.', 'Ltda.', 'Lcda.', 'Exma.', 'C.V.', 'SS.MM.', 'Lda.', 'U.S.', 'hnos.', 'R.D.', 'Korn.', 'v.gr.', 'vs.', 'Ilmas.', 'Rdo.', 'ej.', 'vie.', 'jue.', 'a. C.', 'Ilmos.', 'e. c.', 'Excma.', 'afma.', 'licda.', 'Em.', 'K.', 'sras.', 'MM.', 'fund.', 'Mons.', 'Lcdo.', 'afmo.', 'C.', 'A.C.', 'dptos.', 'Col.', 'Srta.', 'Av.', 'Ant.', 'depto.', 'Var.', 'H.P.', 'D.', 'M.', 'C.P.', 'Rev.', 'Rvdmos.', 'Fr.', 'Ilmo.', 'afmos.', 'Ltd.', 'afmas.', 'prof.', 'lun.', 'SS.AA.', 'Sol.', 'nov.', 'mss.', 'Dña.', 'Seg.', 'mar.', 'Rvdmo.', 'Reg.', 'ms.', 'Sras.', 'sres.', 'U.S.A.', 'Sta.', 'Sdad.', 'Dra.', 'srs.', 'R.U.', 'deptos.', 'dpto.', 'jun.', 'bco.', 'Cía.', 'Id.', 'Mr.', 'e.g.', 'C.S.', 'Excmas.', 'Dª.', 'Rvdo.', 'Lic.', 'cfr.', 'Corp.', 'Dto.', 'Ilma.', 'L.', 'All.', 'PP.', 'd. C.', 'Ltdo.', 'mtro.', 'Mrs.', 'Desc.', 'Avda.', 'Exmas.', 'a. e. c.', 'Bien.', 'Exmos.', 'AA.', 'Sto.', 'CA.', 'sept.', 'Exc.', 'c/c.'],
+    'fr': ['aux.', 'config.', 'collab.', 'M.', 'dim.', 'imprim.', 'oct.', 'syst.', 'bull.', 'MM.', 'doc.', 'P.O.', 'hôp.', 'Mart.', 'juil.', 'broch.', 'adr.', 'symb.', 'C.', 'anc.', 'voit.', 'Jr.', 'graph.', 'dir.', 'éd.', 'fig.', 'édit.', 'niv.', 'quart.', 'cam.', 'éval.', 'anon.', 'réf.', 'Comm.', 'Prof.', 'févr.', 'indus.', 'DC.', 'équiv.', 'illustr.', 'acoust.', 'nov.', 'L.', 'All.', 'U.S.', 'S.M.A.R.T.', 'sept.', 'avr.', 'jeu.', 'dest.', 'P.-D. G.', 'ill.', 'coll.', 'encycl.', 'mer.', 'Desc.', 'ven.', 'P.', 'lun.', 'Inc.', 'sam.', 'D.', 'append.', 'Var.', 'categ.', 'janv.', 'S.A.', 'imm.', 'U.S.A.', 'mar.', 'exempl.', 'déc.', 'ann.', 'U.', 'synth.', 'dict.', 'av. J.-C.', 'W.', 'Op.', 'ap. J.-C.', 'gouv.', 'trav. publ.'],
+    'it': ['N.B.', 'div.', 'a.C.', 'fig.', 'd.p.R.', 'c.c.p.', 'Cfr.', 'vol.', 'Geom.', 'O.d.G.', 'S.p.A.', 'ver.', 'N.d.A.', 'dott.', 'arch.', 'd.C.', 'N.d.T.', 'rag.', 'Sig.', 'Mod.', 'pag.', 'dr.', 'tav.', 'N.d.E.', 'DC.', 'mitt.', 'Ing.', 'int.', 'on.', 'C.P.', 'ag.', 'L.', 'U.S.', 'S.M.A.R.T.', 'p.i.', 'tab.', 'Ltd.', 'Liv.', 'D.', 'U.S.A.', 'sez.', 'avv.', 'S.A.R.', 'all.', 'p.'],
+    'pt': ['psicol.', 'fig.', 'compl.', 'rep.', 'cap.', 'doc.', 'fisiol.', 'dipl.', 'astron.', 'port.', 'eletrôn.', 'geom.', 'mov.', 'ago.', 'trad.', 'arquit.', 'dez.', 'ed.', 'apt.', 'Exmo.', 'col.', 'ff.', 'univ.', 'res.', 'R.', 'transp.', 'D.C', 'l.', 'des.', 'fev.', 'abr.', 'liter.', 'lat.', 'Dir.', 'cf.', 'adm.', 'fot.', 'p.m.', 'P.M.', 'créd.', 'jur.', 'com.', 'anat.', 'dir.', 'end.', 'fís.', 'E.', 'Est.', 'cont.', 'matem.', 'Drs.', 'gên.', 'neol.', 'pág.', 'índ.', 'Ltda.', 'Exma.', 'esp.', 'ingl.', 'tecnol.', 'Mar.', 'símb.', 'Pe.', 'pal.', 'filos.', 'V.T.', 'fasc.', 'vs.', 'mai.', 'S.A.', 'profa.', 'N.Sra.', 'r.s.v.p.', 'cel.', 'mat.', 'abrev.', 'out.', 'long.', 'aux.', 'arit.', 'aer.', 'jul.', 'lin.', 'S.', 'méd.', 'odontol.', 'org.', 'A.C.', 'jun.', 'déb.', 'Av.', 'álg.', 'sup.', 'fl.', 'odont.', 'caps.', 'relat.', 'organiz.', 'hist.', 'Fr.', 'Ilmo.', 'fem.', 'ap.', 'Ltd.', 'pol.', 'séc.', 'prof.', 'cx.', 'nov.', 'quím.', 'mús.', 'agric.', 'mar.', 'W.C.', 'fr.', 'cat.', 'jan.', 'pron.', 'rel.', 'autom.', 'Sta.', 'Dra.', 'p.', 'tel.', 'div.', 'p. ex.', 'a.C.', 'bras.', 'Alm.', 'Dr.', 'comp.', 'pq.', 'arqueol.', 'náut.', 'biogr.', 'f.', 'círc.', 'fac.', 'd.C.', 'apart.', 'ex.', 'Jr.', 'set.', 'tec.', 'sociol.', 'gram.', 'ind.', 'Ilma.', 'vol.', 'eng.', 'rod.', 'Ph.D.', 'Dras.', 'pp.', 'elem.', 'máq.', 'cód.', 'eletr.', 'prod.', 'ref.', 'fil.', 'a.m.', 'A.M', 'obs.', 'N.T.', 'contab.', 'Sto.', 'lit.', 'educ.', 'rementente', 'desc.', 'próx.'],
+    'ru': ['руб.', 'янв.', 'до н. э.', 'сент.', 'тел.', 'дек.', 'февр.', 'нояб.', 'апр.', 'н. э.', 'окт.', 'тыс.', 'авг.', 'проф.', 'н.э.', 'кв.', 'ул.', 'отд.'],
+};
+export const leadingApostropheContractionSuppressions = {
+    'en': [`'cause`, `'til`, `'bout`, `'twas`, `'tis`],
+    'af': [`'n`]
+};
+export const nounSuppressions = [
+    'C#', 'F#', 'C++', 'Yahoo!', 'Toys"R"Us', `Dunkin'`, 'Ke$ha', 'I/O', 'Sky+', 'A/C', 'A/V'
+];
+export const tldSuppressions = [
+    '.com', '.org', '.net', '.co', '.us'
+];
+// What about plural possesive English nouns like:
+// The brothers' friend.
+// The professors' books.
+// Trees' roots.
+// Diplomats' contracts.
+//
+// Not easy to confidently identify these.
+// The apostrophe can also represent an ending single quotation mark,
+// or a misplaced opening single quotation mark.
+// Maybe if a word ends with `s'` and there is no opening `'` seen anywhere near,
+// but that is still not 100% certain. The opening `'` may have appeared long before that.
+//# sourceMappingURL=Suppressions.js.map

package/dist/Suppressions.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"Suppressions.js","sourceRoot":"","sources":["../src/Suppressions.ts"],"names":[],"mappings":"AAAA,MAAM,CAAC,MAAM,gBAAgB,GAA6B;IACzD,IAAI,EAAE,CAAC,MAAM,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,OAAO,EAAE,KAAK,EAAE,OAAO,EAAE,MAAM,EAAE,IAAI,EAAE,OAAO,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,QAAQ,EAAE,KAAK,EAAE,MAAM,EAAE,KAAK,EAAE,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,KAAK,EAAE,MAAM,EAAE,KAAK,EAAE,KAAK,EAAE,IAAI,EAAE,IAAI,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,IAAI,EAAE,KAAK,EAAE,KAAK,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,KAAK,EAAE,IAAI,EAAE,QAAQ,EAAE,KAAK,EAAE,IAAI,EAAE,MAAM,EAAE,QAAQ,EAAE,OAAO,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,IAAI,EAAE,OAAO,EAAE,KAAK,EAAE,MAAM,EAAE,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,EAAE,IAAI,EAAE,KAAK,EAAE,MAAM,EAAE,KAAK,EAAE,KAAK,EAAE,OAAO,EAAE,KAAK,EAAE,MAAM,EAAE,OAAO,EAAE,UAAU,EAAE,IAAI,EAAE,KAAK,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,KAAK,EAAE,KAAK,EAAE,IAAI,EAAE,QAAQ,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,KAAK,EAAE,KAAK,CAAC;IAC5pC,IAAI,EAAE,CAAC,OAAO,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,SAAS,EAAE,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,QAAQ,EAAE,KAAK,EAAE,MAAM,EAAE,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,KAAK,EAAE,KAAK,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,EAAE,MAAM,EAAE,IAAI,EAAE,OAAO,EAAE,UAAU,EAAE,KAAK,EAAE,SAAS,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,IAAI,EAAE,OAAO,EAAE,SAAS,EAAE,MAAM,EAAE,MAAM,EAAE,YAAY,EAAE,IAAI,EAAE,IAAI,EAAE,OAAO,EAAE,IAAI,EAAE,KAAK,EAAE,KAAK,EAAE,QAAQ,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,UAAU,EAAE,OAAO,EAAE,MAAM,EAAE,IAAI,EAAE,IAAI,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,EAAE,KAAK,EAAE,KAAK,EAAE,KAAK,EAAE,MAAM,EAAE,KAAK,EAAE,OAAO,EAAE,QAAQ,EAAE,IAAI,EAAE,MAAM,EAAE,SAAS,EAAE,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,QAAQ,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,IAAI,EAAE,OAAO,EAAE,MAAM,EAAE,KAAK,EAAE,QAAQ,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,QAAQ,EAAE,QAAQ,EAAE,IAAI,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,UAAU,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,IAAI,EAAE,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,QAAQ,EAAE,UAAU,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,UAAU,EAAE,MAAM,EAAE,IAAI,EAAE,YAAY,EAAE,MAAM,EAAE,OAAO,EAAE,KAAK,EAAE,MAAM,EAAE,IAAI,EAAE,IAAI,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,QAAQ,EAAE,OAAO,EAAE,KAAK,EAAE,MAAM,EAAE,WAAW,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,KAAK,EAAE,OAAO,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,IAAI,EAAE,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,MAAM,EAAE,QAAQ,EAAE,QAAQ,EAAE,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,OAAO,EAAE,WAAW,EAAE,KAAK,EAAE,QAAQ,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,IAAI,EAAE,UAAU,EAAE,QAAQ,EAAE,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,IAAI,EAAE,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,KAAK,EAAE,IAAI,EAAE,OAAO,EAAE,IAAI,EAAE,YAAY,EAAE,YAAY,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,IAAI,EAAE,QAAQ,EAAE,OAAO,EAAE,WAAW,EAAE,MAAM,EAAE,QAAQ,EAAE,SAAS,CAAC;IACh/D,IAAI,EAAE,CAAC,OAAO,EAAE,QAAQ,EAAE,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,QAAQ,EAAE,OAAO,EAAE,SAAS,EAAE,MAAM,EAAE,MAAM,EAAE,QAAQ,EAAE,KAAK,EAAE,MAAM,EAAE,OAAO,EAAE,KAAK,EAAE,QAAQ,EAAE,KAAK,EAAE,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,EAAE,MAAM,EAAE,SAAS,EAAE,OAAO,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,OAAO,EAAE,KAAK,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,KAAK,EAAE,QAAQ,EAAE,OAAO,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,KAAK,EAAE,OAAO,EAAE,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,QAAQ,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,QAAQ,EAAE,OAAO,EAAE,QAAQ,EAAE,OAAO,EAAE,QAAQ,EAAE,KAAK,EAAE,IAAI,EAAE,OAAO,EAAE,KAAK,EAAE,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,EAAE,OAAO,EAAE,KAAK,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,EAAE,MAAM,EAAE,IAAI,EAAE,IAAI,EAAE,MAAM,EAAE,MAAM,EAAE,SAAS,EAAE,KAAK,EAAE,OAAO,EAAE,QAAQ,EAAE,MAAM,EAAE,QAAQ,EAAE,OAAO,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,EAAE,KAAK,EAAE,OAAO,EAAE,OAAO,EAAE,QAAQ,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,SAAS,EAAE,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,KAAK,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,SAAS,EAAE,KAAK,EAAE,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,MAAM,EAAE,OAAO,EAAE,OAAO,EAAE,QAAQ,EAAE,UAAU,EAAE,OAAO,EAAE,QAAQ,EAAE,KAAK,EAAE,MAAM,EAAE,KAAK,EAAE,OAAO,EAAE,MAAM,EAAE,MAAM,CAAC;IACh3C,IAAI,EAAE,CAAC,MAAM,EAAE,SAAS,EAAE,SAAS,EAAE,IAAI,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,EAAE,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,OAAO,EAAE,QAAQ,EAAE,MAAM,EAAE,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,KAAK,EAAE,QAAQ,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,EAAE,OAAO,EAAE,OAAO,EAAE,MAAM,EAAE,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,QAAQ,EAAE,KAAK,EAAE,QAAQ,EAAE,UAAU,EAAE,SAAS,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,EAAE,MAAM,EAAE,YAAY,EAAE,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,UAAU,EAAE,MAAM,EAAE,OAAO,EAAE,SAAS,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,IAAI,EAAE,SAAS,EAAE,MAAM,EAAE,QAAQ,EAAE,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,EAAE,MAAM,EAAE,IAAI,EAAE,QAAQ,EAAE,OAAO,EAAE,WAAW,EAAE,IAAI,EAAE,KAAK,EAAE,WAAW,EAAE,OAAO,EAAE,aAAa,CAAC;IAC1tB,IAAI,EAAE,CAAC,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,QAAQ,EAAE,QAAQ,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,QAAQ,EAAE,QAAQ,EAAE,MAAM,EAAE,QAAQ,EAAE,OAAO,EAAE,OAAO,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,QAAQ,EAAE,KAAK,EAAE,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,KAAK,EAAE,IAAI,EAAE,MAAM,EAAE,YAAY,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,IAAI,EAAE,QAAQ,EAAE,MAAM,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,EAAE,IAAI,CAAC;IAChY,IAAI,EAAE,CAAC,SAAS,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,SAAS,EAAE,OAAO,EAAE,SAAS,EAAE,OAAO,EAAE,UAAU,EAAE,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,SAAS,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,KAAK,EAAE,OAAO,EAAE,MAAM,EAAE,IAAI,EAAE,SAAS,EAAE,KAAK,EAAE,IAAI,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,QAAQ,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,OAAO,EAAE,MAAM,EAAE,OAAO,EAAE,SAAS,EAAE,MAAM,EAAE,OAAO,EAAE,KAAK,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,EAAE,OAAO,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,QAAQ,EAAE,QAAQ,EAAE,UAAU,EAAE,MAAM,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,KAAK,EAAE,QAAQ,EAAE,OAAO,EAAE,QAAQ,EAAE,UAAU,EAAE,OAAO,EAAE,KAAK,EAAE,OAAO,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,KAAK,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,KAAK,EAAE,OAAO,EAAE,KAAK,EAAE,UAAU,EAAE,OAAO,EAAE,QAAQ,EAAE,IAAI,EAAE,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,QAAQ,EAAE,KAAK,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,SAAS,EAAE,OAAO,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,QAAQ,EAAE,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,YAAY,EAAE,OAAO,EAAE,OAAO,CAAC;IAC57C,IAAI,EAAE,CAAC,MAAM,EAAE,MAAM,EAAE,UAAU,EAAE,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,OAAO,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,KAAK,EAAE,KAAK,EAAE,MAAM,CAAC;CAC7J,CAAA;AAED,MAAM,CAAC,MAAM,wCAAwC,GAA6B;IACjF,IAAI,EAAE,CAAC,QAAQ,EAAE,MAAM,EAAE,OAAO,EAAE,OAAO,EAAE,MAAM,CAAC;IAClD,IAAI,EAAE,CAAC,IAAI,CAAC;CACZ,CAAA;AAED,MAAM,CAAC,MAAM,gBAAgB,GAAG;IAC/B,IAAI,EAAE,IAAI,EAAE,KAAK,EAAE,QAAQ,EAAE,WAAW,EAAE,SAAS,EAAE,OAAO,EAAE,KAAK,EAAE,MAAM,EAAE,KAAK,EAAE,KAAK;CACzF,CAAA;AAED,MAAM,CAAC,MAAM,eAAe,GAAG;IAC9B,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,KAAK,EAAE,KAAK;CACpC,CAAA;AAED,kDAAkD;AAClD,wBAAwB;AACxB,yBAAyB;AACzB,gBAAgB;AAChB,wBAAwB;AACxB,EAAE;AACF,0CAA0C;AAC1C,qEAAqE;AACrE,gDAAgD;AAChD,iFAAiF;AACjF,0FAA0F"}

package/dist/Test.d.ts

@@ -0,0 +1 @@
+export {};

package/dist/Test.js

@@ -0,0 +1,39 @@
+import { segmentText, splitToWords } from "./TextSegmentation.js";
+import { Timer } from "./utilities/Timer.js";
+const log = console.log;
+async function test1() {
+    {
+        const wordSequence = await splitToWords('Hello world! Привет мир! 你好世界！');
+        console.log(JSON.stringify(wordSequence.words));
+    }
+    const { readFileSync, writeFileSync } = await import('fs');
+    const text = readFileSync('test-data/Test.txt', 'utf-8');
+    const timer = new Timer();
+    const result = await segmentText(text, {
+        language: 'en',
+        customSuppressions: [],
+        enableEastAsianPostprocessing: true
+    });
+    //const json = JSON.stringify(result.sentences)
+    timer.logAndRestart(`Total execution time`);
+    log('');
+    //
+    let segmentedText = '';
+    for (let sentenceIndex = 0; sentenceIndex < result.sentences.length; sentenceIndex++) {
+        const sentence = result.sentences[sentenceIndex];
+        const phrases = sentence.phrases;
+        for (let phraseIndex = 0; phraseIndex < phrases.length; phraseIndex++) {
+            const phrase = phrases[phraseIndex];
+            segmentedText += phrase.wordSequence.words.join(' | ');
+            if (phraseIndex < phrases.length - 1) {
+                segmentedText += `\n${'-'.repeat(100)}\n`;
+            }
+        }
+        if (sentenceIndex < result.sentences.length - 1) {
+            segmentedText += `\n${'='.repeat(100)} \n`;
+        }
+    }
+    writeFileSync('out/segmented.txt', segmentedText);
+}
+test1();
+//# sourceMappingURL=Test.js.map

package/dist/Test.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"Test.js","sourceRoot":"","sources":["../src/Test.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,WAAW,EAAE,YAAY,EAAE,MAAM,uBAAuB,CAAA;AACjE,OAAO,EAAE,KAAK,EAAE,MAAM,sBAAsB,CAAA;AAE5C,MAAM,GAAG,GAAG,OAAO,CAAC,GAAG,CAAA;AAEvB,KAAK,UAAU,KAAK;IACnB,CAAC;QACA,MAAM,YAAY,GAAG,MAAM,YAAY,CAAC,gCAAgC,CAAC,CAAA;QACzE,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,SAAS,CAAC,YAAY,CAAC,KAAK,CAAC,CAAC,CAAA;IAChD,CAAC;IAED,MAAM,EAAE,YAAY,EAAE,aAAa,EAAE,GAAG,MAAM,MAAM,CAAC,IAAI,CAAC,CAAA;IAC1D,MAAM,IAAI,GAAG,YAAY,CAAC,oBAAoB,EAAE,OAAO,CAAC,CAAA;IAExD,MAAM,KAAK,GAAG,IAAI,KAAK,EAAE,CAAA;IAEzB,MAAM,MAAM,GAAG,MAAM,WAAW,CAAC,IAAI,EAAE;QACtC,QAAQ,EAAE,IAAI;QACd,kBAAkB,EAAE,EAAE;QACtB,6BAA6B,EAAE,IAAI;KACnC,CAAC,CAAA;IAEF,+CAA+C;IAE/C,KAAK,CAAC,aAAa,CAAC,sBAAsB,CAAC,CAAA;IAE3C,GAAG,CAAC,EAAE,CAAC,CAAA;IAEP,EAAE;IAEF,IAAI,aAAa,GAAG,EAAE,CAAA;IAEtB,KAAK,IAAI,aAAa,GAAG,CAAC,EAAE,aAAa,GAAG,MAAM,CAAC,SAAS,CAAC,MAAM,EAAE,aAAa,EAAE,EAAE,CAAC;QACtF,MAAM,QAAQ,GAAG,MAAM,CAAC,SAAS,CAAC,aAAa,CAAC,CAAA;QAChD,MAAM,OAAO,GAAG,QAAQ,CAAC,OAAO,CAAA;QAEhC,KAAK,IAAI,WAAW,GAAG,CAAC,EAAE,WAAW,GAAG,OAAO,CAAC,MAAM,EAAE,WAAW,EAAE,EAAE,CAAC;YACvE,MAAM,MAAM,GAAG,OAAO,CAAC,WAAW,CAAC,CAAA;YAEnC,aAAa,IAAI,MAAM,CAAC,YAAY,CAAC,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,CAAA;YAEtD,IAAI,WAAW,GAAG,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBACtC,aAAa,IAAI,KAAK,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,IAAI,CAAA;YAC1C,CAAC;QACF,CAAC;QAED,IAAI,aAAa,GAAG,MAAM,CAAC,SAAS,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACjD,aAAa,IAAI,KAAM,GAAG,CAAC,MAAM,CAAC,GAAG,CAAE,KAAK,CAAA;QAC7C,CAAC;IACF,CAAC;IAED,aAAa,CAAC,mBAAmB,EAAE,aAAa,CAAC,CAAA;AAClD,CAAC;AAED,KAAK,EAAE,CAAA"}

package/dist/TextSegmentation.d.ts

@@ -0,0 +1,31 @@
+import { WordSequence } from './WordSequence.js';
+export { cldrSuppressions } from './Suppressions.js';
+export declare function segmentText(text: string, options?: SegmentationOptions): Promise<SegmentationResult>;
+export declare function segmentWordSequence(wordSequence: WordSequence): Promise<SegmentationResult>;
+export declare function splitToWords(text: string, options?: SegmentationOptions): Promise<WordSequence>;
+export interface SegmentationResult {
+    wordSequence: WordSequence;
+    sentences: Sentence[];
+}
+export declare class TextFragment {
+    wordRange: Range;
+    wordSequence: WordSequence;
+    constructor(wordRange: Range, wordSequence: WordSequence);
+    get text(): string;
+    get charRange(): Range;
+}
+export declare class Sentence extends TextFragment {
+    phrases: Phrase[];
+}
+export declare class Phrase extends TextFragment {
+}
+export interface Range {
+    start: number;
+    end: number;
+}
+export interface SegmentationOptions {
+    language?: string;
+    customSuppressions?: string[];
+    enableEastAsianPostprocessing?: boolean;
+}
+export declare const defaultSegmentationOptions: SegmentationOptions;