echogarden 0.0.1 → 0.1.0

package/dist/nlp/PhoneConversion.js

@@ -0,0 +1,467 @@
+export function ipaPhoneToKirshenbaum(ipaPhone) {
+    let result = '';
+    for (const char of ipaPhone) {
+        const convertedChar = ipaToKirshenbaum[char];
+        if (convertedChar == undefined) {
+            throw new Error(`Could not convert phone character '${char}' to Kirshenbaum encoding`);
+        }
+        result += convertedChar || '_';
+    }
+    return result;
+}
+export function ipaWordToTimitTokens(ipaWord, subphoneCount = 0) {
+    let result = [];
+    for (const ipaPhone of ipaWord) {
+        const convertedPhoneSequence = ipaPhoneToTimit(ipaPhone, subphoneCount);
+        if (convertedPhoneSequence == undefined) {
+            throw new Error(`Could not find a TIMIT equivalent for ipa phone '${ipaPhone}'`);
+        }
+        result = [...result, ...convertedPhoneSequence];
+    }
+    return result;
+}
+export function ipaPhoneToTimit(ipaPhone, subphoneCount = 0) {
+    let tokens = ipaToTimit[ipaPhone];
+    if (!tokens) {
+        return undefined;
+    }
+    if (subphoneCount > 0) {
+        tokens = splitTokensToSubphones(tokens, subphoneCount);
+    }
+    return tokens;
+}
+export function arpabetPhoneToIpa(arpabetPhone) {
+    return arpabetToIPA[arpabetPhone.toUpperCase()];
+}
+export function timitPhoneToIpa(arpabetPhone) {
+    return timitToIPA[arpabetPhone.toUpperCase()];
+}
+export function splitTokensToSubphones(tokens, subphoneCount = 3) {
+    const result = [];
+    for (const token of tokens) {
+        if (token == 'pause') {
+            result.push(token);
+            continue;
+        }
+        for (let i = 1; i <= subphoneCount; i++) {
+            result.push(`${token}/${i}`);
+        }
+    }
+    return result;
+}
+// ARPABET was invented for English.
+// The standard dictionary written in ARPABET is the CMU dictionary.
+// TIMIT is written in a variant of ARPABET that includes a couple
+// of non-standard allophones, and most significantly, includes
+// separate symbols for the closure and release portions of each stop.
+const arpabetToIPA = {
+    'AA': 'ɑ',
+    'AE': 'æ',
+    'AH': 'ʌ',
+    'AH0': 'ə',
+    'AO': 'ɔ',
+    'AW': 'aʊ',
+    'AY': 'aɪ',
+    'EH': 'ɛ',
+    'ER': 'ɝ',
+    'ER0': 'ɚ',
+    'EY': 'eɪ',
+    'IH': 'ɪ',
+    'IH0': 'ɨ',
+    'IY': 'i',
+    'OW': 'oʊ',
+    'OY': 'ɔɪ',
+    'UH': 'ʊ',
+    'UW': 'u',
+    'B': 'b',
+    'CH': 'tʃ',
+    'D': 'd',
+    'DH': 'ð',
+    'EL': 'l̩ ',
+    'EM': 'm̩',
+    'EN': 'n̩',
+    'F': 'f',
+    'G': 'ɡ',
+    'HH': 'h',
+    'JH': 'dʒ',
+    'K': 'k',
+    'L': 'l',
+    'M': 'm',
+    'N': 'n',
+    'NG': 'ŋ',
+    'P': 'p',
+    'Q': 'ʔ',
+    'R': 'ɹ',
+    'S': 's',
+    'SH': 'ʃ',
+    'T': 't',
+    'TH': 'θ',
+    'V': 'v',
+    'W': 'w',
+    'WH': 'ʍ',
+    'Y': 'j',
+    'Z': 'z',
+    'ZH': 'ʒ'
+};
+const timitToIPA = {
+    ...arpabetToIPA,
+    'AX': 'ə',
+    'AX-H': 'ə̥',
+    'AXR': 'ɚ',
+    'B': '',
+    'BCL': 'b',
+    'D': '',
+    'DCL': 'd',
+    'DX': 'ɾ',
+    'ENG': 'ŋ̍',
+    'EPI': '',
+    'G': '',
+    'GCL': 'g',
+    'HV': 'ɦ',
+    'H//': '',
+    'IX': 'ɨ',
+    'KCL': 'k',
+    'K': '',
+    'NX': 'ɾ̃',
+    'P': '',
+    'PAU': '',
+    'PCL': 'p',
+    'T': '',
+    'TCL': 't',
+    'UX': 'ʉ',
+};
+const ipaToTimit = {
+    // Vowels
+    'ɑ': ['aa'],
+    'ɑː': ['aa'],
+    'ɑːɹ': ['aa', 'r'],
+    'a': ['aa'],
+    'aʊ': ['aw'],
+    'aɪ': ['ay'],
+    'aɪɚ': ['ay', 'r'],
+    'aɪə': ['ay', 'ax'],
+    'æ': ['ae'],
+    'æʊ': ['ae', 'uh'],
+    'ʌ': ['ah'],
+    'ɐ': ['ah'],
+    'ə': ['ax'],
+    'ə̥': ['ax-h'],
+    'əʊ': ['ax', 'uh'],
+    'ɚ': ['axr'],
+    'ᵻ': ['ax'],
+    'ɔ': ['ao'],
+    'ɔː': ['ao'],
+    'ɔːɹ': ['ao', 'r'],
+    'ɔɪ': ['oy'],
+    'ɒ': ['ao'],
+    'ɛ': ['eh'],
+    'ɜː': ['eh'],
+    'ɝ': ['er'],
+    'ɛɹ': ['er'],
+    'eɪ': ['ey'],
+    'eə': ['ey', 'ax'],
+    'eː': ['ey'],
+    'ɪ': ['ih'],
+    'ɪɹ': ['ih', 'r'],
+    'ɨ': ['ix'],
+    'i': ['iy'],
+    'iː': ['iy'],
+    'iə': ['iy', 'ax'],
+    'oʊ': ['ow'],
+    'oː': ['ow'],
+    'oːɹ': ['ow', 'r'],
+    'ʊ': ['uh'],
+    'ʊɹ': ['uh', 'r'],
+    'ʊə': ['uh', 'ax'],
+    'u': ['uw'],
+    'uː': ['uw'],
+    // Consonants
+    'b': ['bcl', 'b'],
+    'tʃ': ['tcl', 'ch'],
+    'd': ['dcl', 'd'],
+    'ð': ['dh'],
+    'l̩': ['el'],
+    'əl': ['el'],
+    'm̩': ['em'],
+    'n̩': ['en'],
+    'f': ['f'],
+    'ɡ': ['gcl', 'g'],
+    'h': ['hh'],
+    'dʒ': ['dcl', 'jh'],
+    'k': ['kcl', 'k'],
+    'l': ['l'],
+    'm': ['m'],
+    'n': ['n'],
+    'ŋ': ['ng'],
+    'p': ['pcl', 'p'],
+    'ʔ': ['q'],
+    'ɹ': ['r'],
+    's': ['s'],
+    'ʃ': ['sh'],
+    't': ['tcl', 't'],
+    'θ': ['th'],
+    'v': ['v'],
+    'w': ['w'],
+    'ʍ': ['wh'],
+    'j': ['y'],
+    'z': ['z'],
+    'ʒ': ['zh'],
+    'ɾ̃': ['nx'],
+    'ʉ': ['ux'],
+    'ɦ': ['hv'],
+    'ŋ̍': ['eng'],
+    'ɾ': ['dx'],
+};
+// This is adapted from a lookup table on the eSpeak-ng source code
+const ipaToKirshenbaum = {
+    '1': '1',
+    '2': '2',
+    '4': '4',
+    '5': '5',
+    '6': '6',
+    '7': '7',
+    '9': '9',
+    ' ': ' ',
+    '!': '!',
+    '\'': '\'',
+    'ʰ': '#',
+    '$': '$',
+    '%': '%',
+    //'æ': '&',
+    'æ': 'a',
+    'ˈ': '\'',
+    '(': '(',
+    ')': ')',
+    'ɾ': '*',
+    '+': '+',
+    'ˌ': ',',
+    '-': '-',
+    '.': '.',
+    '/': '/',
+    'ɒ': '0',
+    'ɜ': '3',
+    'ɵ': '8',
+    'ː': ':',
+    'ʲ': ';',
+    '<': '<',
+    '=': '=',
+    '>': '>',
+    'ʔ': '?',
+    'ə': '@',
+    'ɑ': 'A',
+    'β': 'B',
+    'ç': 'C',
+    'ð': 'D',
+    'ɛ': 'E',
+    'F': 'F',
+    'ɢ': 'G',
+    'ħ': 'H',
+    'ɪ': 'I',
+    'ɟ': 'J',
+    'K': 'K',
+    'ɫ': 'L',
+    'ɱ': 'M',
+    'ŋ': 'N',
+    'ɔ': 'O',
+    'Φ': 'P',
+    'ɣ': 'Q',
+    'ʀ': 'R',
+    'ʃ': 'S',
+    'θ': 'T',
+    'ʊ': 'U',
+    'ʌ': 'V',
+    'œ': 'W',
+    'χ': 'X',
+    'ø': 'Y',
+    'ʒ': 'Z',
+    '̪': '[',
+    '\\': '\\',
+    ']': ']',
+    '^': '^',
+    '_': '_',
+    '`': '`',
+    'a': 'a',
+    'b': 'b',
+    'c': 'c',
+    'd': 'd',
+    'e': 'e',
+    'f': 'f',
+    'ɡ': 'g',
+    'h': 'h',
+    'i': 'i',
+    'j': 'j',
+    'k': 'k',
+    'l': 'l',
+    'm': 'm',
+    'n': 'n',
+    'o': 'o',
+    'p': 'p',
+    'q': 'q',
+    'r': 'r',
+    's': 's',
+    't': 't',
+    'u': 'u',
+    'v': 'v',
+    'w': 'w',
+    'x': 'x',
+    'y': 'y',
+    'z': 'z',
+    '{': '{',
+    '|': '|',
+    '}': '}',
+    '̃': '~',
+    '': '',
+    // Extensions
+    'ɚ': '3',
+    'ɹ': 'r',
+    'ɐ': 'a#',
+    'ᵻ': 'i',
+    '̩': ','
+};
+/*
+// Source: https://github.com/coruus/ascii-ipa/blob/master/kirshenbaum.py
+// Conversion regex: \('.*?': '.*?'\)
+// Replace: $2: $1
+const ipaToKirshenbaum2: { [p: string]: string | undefined } = {
+    'm': 'm',
+    'p': 'p',
+    'b': 'b',
+    'Φ': 'P',
+    'β': 'B',
+    'ʙ': 'b<trl>',
+    'pʼ': 'p`',
+    'ɓ': 'b`',
+    'ʘ': 'p!',
+    'ɱ': 'M',
+    'f': 'f',
+    'v': 'v',
+    'ʋ': 'r<lbd>',
+    'n\u032a': 'n[',
+    't\u032a': 't[',
+    'θ': 'T',
+    'ð': 'D',
+    'r\u032a': 'r[',
+    'l\u032a': 'l[',
+    't\u032a\u02bc': 't[`',
+    'ɗ': 'd`',
+    'ʇ': 't!',
+    'n': 'n',
+    't': 't',
+    'd': 'd',
+    's': 's',
+    'z': 'z',
+    'ɬ': 's<lat>',
+    'ɮ': 'z<lat>',
+    'ɹ': 'r',
+    'l': 'l',
+    'ʀ': 'r<trl>',
+    'ɾ': '*',
+    'ɺ': '*<lat>',
+    't\u02bc': 't`',
+    'ʗ': 'c!',
+    'ʖ': 'l!',
+    'ɳ': 'n.',
+    'ʈ': 't.',
+    'ɖ': 'd.',
+    'ʂ': 's.',
+    'ʐ': 'z.',
+    //'ɖ': 'r.',
+    'ɭ': 'l.',
+    'ɽ': '*.',
+    'ʃ': 'S',
+    'ʒ': 'Z',
+    'n^': 'n^',
+    'c': 'c',
+    'ɟ': 'J',
+    'ç': 'C',
+    'ʝ': 'C<vcd>',
+    'j': 'j',
+    'ɥ': 'j<rnd>',
+    'ʎ': 'l^',
+    'ʄ': 'J`',
+    'ŋ': 'N',
+    'k': 'k',
+    'g': 'g',
+    'x': 'x',
+    'ɣ': 'Q',
+    'ɰ': 'j<vel>',
+    'ɫ': 'L',
+    //'ɬ': '{vls,alv,lat,frc}',
+    'k\u02bc': 'k`',
+    'g\u02bc': 'g`',
+    'ʞ': 'k!',
+    'n\u2030g': 'n<lbv>',
+    'k\u2030p': 't<lbv>',
+    'g\u2030b': 'n<lbv>',
+    'ʍ': 'w<vls>',
+    'w': 'w',
+    'ɴ': 'n'',
+    'q': 'q',
+    'ɢ': 'G',
+    'χ': 'X',
+    'ʁ': 'g'',
+    //'ʀ': 'r'',
+    'ʠ': 'q`',
+    'ʛ': 'G`',
+    'ħ': 'H',
+    'ʕ': 'H<vcd>',
+    'ʔ': '?',
+    'h': 'h',
+    'ɦ': 'h<?>',
+    'i': 'i',
+    'y': 'y',
+    'ɪ': 'I',
+    'ʏ': 'I.',
+    'e': 'e',
+    'ø': 'Y',
+    'ɛ': 'E',
+    'œ': 'W',
+    'æ': '&',
+    'ɶ': '&.',
+    'ɨ': 'i'',
+    'ʉ': 'u'',
+    'ɘ': '@<umd>',
+    'ɝ': 'R<umd>',
+    'ə': '@',
+    'ɚ': 'R',
+    'ɵ': '@.',
+    //'ɜ': 'V'',
+    'ɜ': '3',
+    'ɞ': 'O'',
+    'a': 'a',
+    'ɯ': 'u-',
+    'u': 'u',
+    'ʊ': 'U',
+    'ɤ': 'o-',
+    'o': 'o',
+    'ʌ': 'V',
+    'ɔ': 'O',
+    'ɑ': 'A',
+    'ɒ': 'A.',
+
+    'ː': ':',
+    '\u0322': '.',
+    '\u02bc': '`',
+    '\u032a': '[',
+    '\u02b2': ';',
+    ''': ''',
+    '^': '^',
+    '\u0334': '<H>',
+    '\u02b0': '<h>',
+    '\u02da': '<unx>',
+    '\u0325': '<vls>',
+    //'\u02da': '<o>',
+    '\u02b3': '<r>',
+    '\u02b7': '<w>',
+    '\u02b1': '<?>',
+
+    '\u0303': '~',
+    //'\u0334': '~'
+
+    'ˈ': ''',
+    'ˌ': ',',
+    ' ': ' ',
+    '\n': '\n'
+}
+*/
+//# sourceMappingURL=PhoneConversion.js.map

package/dist/nlp/PhoneConversion.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"PhoneConversion.js","sourceRoot":"","sources":["../../src/nlp/PhoneConversion.ts"],"names":[],"mappings":"AAAA,MAAM,UAAU,qBAAqB,CAAC,QAAgB;IACrD,IAAI,MAAM,GAAG,EAAE,CAAA;IAEf,KAAK,MAAM,IAAI,IAAI,QAAQ,EAAE;QAC5B,MAAM,aAAa,GAAG,gBAAgB,CAAC,IAAI,CAAC,CAAA;QAE5C,IAAI,aAAa,IAAI,SAAS,EAAE;YAC/B,MAAM,IAAI,KAAK,CAAC,sCAAsC,IAAI,2BAA2B,CAAC,CAAA;SACtF;QAED,MAAM,IAAI,aAAa,IAAI,GAAG,CAAA;KAC9B;IAED,OAAO,MAAM,CAAA;AACd,CAAC;AAED,MAAM,UAAU,oBAAoB,CAAC,OAAe,EAAE,aAAa,GAAG,CAAC;IACtE,IAAI,MAAM,GAAa,EAAE,CAAA;IAEzB,KAAK,MAAM,QAAQ,IAAI,OAAO,EAAE;QAC/B,MAAM,sBAAsB,GAAG,eAAe,CAAC,QAAQ,EAAE,aAAa,CAAC,CAAA;QAEvE,IAAI,sBAAsB,IAAI,SAAS,EAAE;YACxC,MAAM,IAAI,KAAK,CAAC,oDAAoD,QAAQ,GAAG,CAAC,CAAA;SAChF;QAED,MAAM,GAAG,CAAC,GAAG,MAAM,EAAE,GAAG,sBAAsB,CAAC,CAAA;KAC/C;IAED,OAAO,MAAM,CAAA;AACd,CAAC;AAED,MAAM,UAAU,eAAe,CAAC,QAAgB,EAAE,aAAa,GAAG,CAAC;IAClE,IAAI,MAAM,GAAG,UAAU,CAAC,QAAQ,CAAC,CAAA;IAEjC,IAAI,CAAC,MAAM,EAAE;QACZ,OAAO,SAAS,CAAA;KAChB;IAED,IAAI,aAAa,GAAG,CAAC,EAAE;QACtB,MAAM,GAAG,sBAAsB,CAAC,MAAM,EAAE,aAAa,CAAC,CAAA;KACtD;IAED,OAAO,MAAM,CAAA;AACd,CAAC;AAED,MAAM,UAAU,iBAAiB,CAAC,YAAoB;IACrD,OAAO,YAAY,CAAC,YAAY,CAAC,WAAW,EAAE,CAAC,CAAA;AAChD,CAAC;AAED,MAAM,UAAU,eAAe,CAAC,YAAoB;IACnD,OAAO,UAAU,CAAC,YAAY,CAAC,WAAW,EAAE,CAAC,CAAA;AAC9C,CAAC;AAED,MAAM,UAAU,sBAAsB,CAAC,MAAgB,EAAE,aAAa,GAAG,CAAC;IACzE,MAAM,MAAM,GAAa,EAAE,CAAA;IAE3B,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE;QAC3B,IAAI,KAAK,IAAI,OAAO,EAAE;YACrB,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAA;YAElB,SAAQ;SACR;QAED,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,IAAI,aAAa,EAAE,CAAC,EAAE,EAAE;YACxC,MAAM,CAAC,IAAI,CAAC,GAAG,KAAK,IAAI,CAAC,EAAE,CAAC,CAAA;SAC5B;KACD;IAED,OAAO,MAAM,CAAA;AACd,CAAC;AAED,oCAAoC;AACpC,oEAAoE;AACpE,kEAAkE;AAClE,+DAA+D;AAC/D,sEAAsE;AACtE,MAAM,YAAY,GAAwC;IACzD,IAAI,EAAE,GAAG;IACT,IAAI,EAAE,GAAG;IACT,IAAI,EAAE,GAAG;IACT,KAAK,EAAE,GAAG;IACV,IAAI,EAAE,GAAG;IACT,IAAI,EAAE,IAAI;IACV,IAAI,EAAE,IAAI;IACV,IAAI,EAAE,GAAG;IACT,IAAI,EAAE,GAAG;IACT,KAAK,EAAE,GAAG;IACV,IAAI,EAAE,IAAI;IACV,IAAI,EAAE,GAAG;IACT,KAAK,EAAE,GAAG;IACV,IAAI,EAAE,GAAG;IACT,IAAI,EAAE,IAAI;IACV,IAAI,EAAE,IAAI;IACV,IAAI,EAAE,GAAG;IACT,IAAI,EAAE,GAAG;IACT,GAAG,EAAE,GAAG;IACR,IAAI,EAAE,IAAI;IACV,GAAG,EAAE,GAAG;IACR,IAAI,EAAE,GAAG;IACT,IAAI,EAAE,KAAK;IACX,IAAI,EAAE,IAAI;IACV,IAAI,EAAE,IAAI;IACV,GAAG,EAAE,GAAG;IACR,GAAG,EAAE,GAAG;IACR,IAAI,EAAE,GAAG;IACT,IAAI,EAAE,IAAI;IACV,GAAG,EAAE,GAAG;IACR,GAAG,EAAE,GAAG;IACR,GAAG,EAAE,GAAG;IACR,GAAG,EAAE,GAAG;IACR,IAAI,EAAE,GAAG;IACT,GAAG,EAAE,GAAG;IACR,GAAG,EAAE,GAAG;IACR,GAAG,EAAE,GAAG;IACR,GAAG,EAAE,GAAG;IACR,IAAI,EAAE,GAAG;IACT,GAAG,EAAE,GAAG;IACR,IAAI,EAAE,GAAG;IACT,GAAG,EAAE,GAAG;IACR,GAAG,EAAE,GAAG;IACR,IAAI,EAAE,GAAG;IACT,GAAG,EAAE,GAAG;IACR,GAAG,EAAE,GAAG;IACR,IAAI,EAAE,GAAG;CACT,CAAA;AAED,MAAM,UAAU,GAAwC;IACvD,GAAG,YAAY;IAEf,IAAI,EAAE,GAAG;IACT,MAAM,EAAE,IAAI;IACZ,KAAK,EAAE,GAAG;IACV,GAAG,EAAE,EAAE;IACP,KAAK,EAAE,GAAG;IACV,GAAG,EAAE,EAAE;IACP,KAAK,EAAE,GAAG;IACV,IAAI,EAAE,GAAG;IACT,KAAK,EAAE,IAAI;IACX,KAAK,EAAE,EAAE;IACT,GAAG,EAAE,EAAE;IACP,KAAK,EAAE,GAAG;IACV,IAAI,EAAE,GAAG;IACT,KAAK,EAAE,EAAE;IACT,IAAI,EAAE,GAAG;IACT,KAAK,EAAE,GAAG;IACV,GAAG,EAAE,EAAE;IACP,IAAI,EAAE,IAAI;IACV,GAAG,EAAE,EAAE;IACP,KAAK,EAAE,EAAE;IACT,KAAK,EAAE,GAAG;IACV,GAAG,EAAE,EAAE;IACP,KAAK,EAAE,GAAG;IACV,IAAI,EAAE,GAAG;CACT,CAAA;AAED,MAAM,UAAU,GAA0C;IACzD,SAAS;IACT,GAAG,EAAE,CAAC,IAAI,CAAC;IACX,IAAI,EAAE,CAAC,IAAI,CAAC;IACZ,KAAK,EAAE,CAAC,IAAI,EAAE,GAAG,CAAC;IAClB,GAAG,EAAE,CAAC,IAAI,CAAC;IAEX,IAAI,EAAE,CAAC,IAAI,CAAC;IACZ,IAAI,EAAE,CAAC,IAAI,CAAC;IACZ,KAAK,EAAE,CAAC,IAAI,EAAE,GAAG,CAAC;IAClB,KAAK,EAAE,CAAC,IAAI,EAAE,IAAI,CAAC;IAEnB,GAAG,EAAE,CAAC,IAAI,CAAC;IACX,IAAI,EAAE,CAAC,IAAI,EAAE,IAAI,CAAC;IAElB,GAAG,EAAE,CAAC,IAAI,CAAC;IACX,GAAG,EAAE,CAAC,IAAI,CAAC;IAEX,GAAG,EAAE,CAAC,IAAI,CAAC;IACX,IAAI,EAAE,CAAC,MAAM,CAAC;IACd,IAAI,EAAE,CAAC,IAAI,EAAE,IAAI,CAAC;IAClB,GAAG,EAAE,CAAC,KAAK,CAAC;IACZ,GAAG,EAAE,CAAC,IAAI,CAAC;IAEX,GAAG,EAAE,CAAC,IAAI,CAAC;IACX,IAAI,EAAE,CAAC,IAAI,CAAC;IACZ,KAAK,EAAE,CAAC,IAAI,EAAE,GAAG,CAAC;IAClB,IAAI,EAAE,CAAC,IAAI,CAAC;IACZ,GAAG,EAAE,CAAC,IAAI,CAAC;IAEX,GAAG,EAAE,CAAC,IAAI,CAAC;IACX,IAAI,EAAE,CAAC,IAAI,CAAC;IACZ,GAAG,EAAE,CAAC,IAAI,CAAC;IACX,IAAI,EAAE,CAAC,IAAI,CAAC;IAEZ,IAAI,EAAE,CAAC,IAAI,CAAC;IACZ,IAAI,EAAE,CAAC,IAAI,EAAE,IAAI,CAAC;IAClB,IAAI,EAAE,CAAC,IAAI,CAAC;IAEZ,GAAG,EAAE,CAAC,IAAI,CAAC;IACX,IAAI,EAAE,CAAC,IAAI,EAAE,GAAG,CAAC;IAEjB,GAAG,EAAE,CAAC,IAAI,CAAC;IAEX,GAAG,EAAE,CAAC,IAAI,CAAC;IACX,IAAI,EAAE,CAAC,IAAI,CAAC;IACZ,IAAI,EAAE,CAAC,IAAI,EAAE,IAAI,CAAC;IAElB,IAAI,EAAE,CAAC,IAAI,CAAC;IACZ,IAAI,EAAE,CAAC,IAAI,CAAC;IACZ,KAAK,EAAE,CAAC,IAAI,EAAE,GAAG,CAAC;IAElB,GAAG,EAAE,CAAC,IAAI,CAAC;IACX,IAAI,EAAE,CAAC,IAAI,EAAE,GAAG,CAAC;IACjB,IAAI,EAAE,CAAC,IAAI,EAAE,IAAI,CAAC;IAElB,GAAG,EAAE,CAAC,IAAI,CAAC;IACX,IAAI,EAAE,CAAC,IAAI,CAAC;IAEZ,aAAa;IACb,GAAG,EAAE,CAAC,KAAK,EAAE,GAAG,CAAC;IACjB,IAAI,EAAE,CAAC,KAAK,EAAE,IAAI,CAAC;IACnB,GAAG,EAAE,CAAC,KAAK,EAAE,GAAG,CAAC;IACjB,GAAG,EAAE,CAAC,IAAI,CAAC;IACX,IAAI,EAAE,CAAC,IAAI,CAAC;IACZ,IAAI,EAAE,CAAC,IAAI,CAAC;IACZ,IAAI,EAAE,CAAC,IAAI,CAAC;IACZ,IAAI,EAAE,CAAC,IAAI,CAAC;IACZ,GAAG,EAAE,CAAC,GAAG,CAAC;IACV,GAAG,EAAE,CAAC,KAAK,EAAE,GAAG,CAAC;IACjB,GAAG,EAAE,CAAC,IAAI,CAAC;IACX,IAAI,EAAE,CAAC,KAAK,EAAE,IAAI,CAAC;IACnB,GAAG,EAAE,CAAC,KAAK,EAAE,GAAG,CAAC;IACjB,GAAG,EAAE,CAAC,GAAG,CAAC;IACV,GAAG,EAAE,CAAC,GAAG,CAAC;IACV,GAAG,EAAE,CAAC,GAAG,CAAC;IACV,GAAG,EAAE,CAAC,IAAI,CAAC;IACX,GAAG,EAAE,CAAC,KAAK,EAAE,GAAG,CAAC;IACjB,GAAG,EAAE,CAAC,GAAG,CAAC;IACV,GAAG,EAAE,CAAC,GAAG,CAAC;IACV,GAAG,EAAE,CAAC,GAAG,CAAC;IACV,GAAG,EAAE,CAAC,IAAI,CAAC;IACX,GAAG,EAAE,CAAC,KAAK,EAAE,GAAG,CAAC;IACjB,GAAG,EAAE,CAAC,IAAI,CAAC;IACX,GAAG,EAAE,CAAC,GAAG,CAAC;IACV,GAAG,EAAE,CAAC,GAAG,CAAC;IACV,GAAG,EAAE,CAAC,IAAI,CAAC;IACX,GAAG,EAAE,CAAC,GAAG,CAAC;IACV,GAAG,EAAE,CAAC,GAAG,CAAC;IACV,GAAG,EAAE,CAAC,IAAI,CAAC;IAEX,IAAI,EAAE,CAAC,IAAI,CAAC;IACZ,GAAG,EAAE,CAAC,IAAI,CAAC;IACX,GAAG,EAAE,CAAC,IAAI,CAAC;IACX,IAAI,EAAE,CAAC,KAAK,CAAC;IACb,GAAG,EAAE,CAAC,IAAI,CAAC;CACX,CAAA;AAED,mEAAmE;AACnE,MAAM,gBAAgB,GAAwC;IAC1D,GAAG,EAAE,GAAG;IACR,GAAG,EAAE,GAAG;IACR,GAAG,EAAE,GAAG;IACR,GAAG,EAAE,GAAG;IACR,GAAG,EAAE,GAAG;IACR,GAAG,EAAE,GAAG;IACR,GAAG,EAAE,GAAG;IACR,GAAG,EAAE,GAAG;IACR,GAAG,EAAE,GAAG;IACR,IAAI,EAAE,IAAI;IACV,GAAG,EAAE,GAAG;IACR,GAAG,EAAE,GAAG;IACR,GAAG,EAAE,GAAG;IACR,WAAW;IACd,GAAG,EAAE,GAAG;IACL,GAAG,EAAE,IAAI;IACT,GAAG,EAAE,GAAG;IACR,GAAG,EAAE,GAAG;IACR,GAAG,EAAE,GAAG;IACR,GAAG,EAAE,GAAG;IACR,GAAG,EAAE,GAAG;IACR,GAAG,EAAE,GAAG;IACR,GAAG,EAAE,GAAG;IACR,GAAG,EAAE,GAAG;IACR,GAAG,EAAE,GAAG;IACR,GAAG,EAAE,GAAG;IACR,GAAG,EAAE,GAAG;IACR,GAAG,EAAE,GAAG;IACR,GAAG,EAAE,GAAG;IACR,GAAG,EAAE,GAAG;IACR,GAAG,EAAE,GAAG;IACR,GAAG,EAAE,GAAG;IACR,GAAG,EAAE,GAAG;IACR,GAAG,EAAE,GAAG;IACR,GAAG,EAAE,GAAG;IACR,GAAG,EAAE,GAAG;IACR,GAAG,EAAE,GAAG;IACR,GAAG,EAAE,GAAG;IACR,GAAG,EAAE,GAAG;IACR,GAAG,EAAE,GAAG;IACR,GAAG,EAAE,GAAG;IACR,GAAG,EAAE,GAAG;IACR,GAAG,EAAE,GAAG;IACR,GAAG,EAAE,GAAG;IACR,GAAG,EAAE,GAAG;IACR,GAAG,EAAE,GAAG;IACR,GAAG,EAAE,GAAG;IACR,GAAG,EAAE,GAAG;IACR,GAAG,EAAE,GAAG;IACR,GAAG,EAAE,GAAG;IACR,GAAG,EAAE,GAAG;IACR,GAAG,EAAE,GAAG;IACR,GAAG,EAAE,GAAG;IACR,GAAG,EAAE,GAAG;IACR,GAAG,EAAE,GAAG;IACR,GAAG,EAAE,GAAG;IACR,GAAG,EAAE,GAAG;IACR,GAAG,EAAE,GAAG;IACR,GAAG,EAAE,GAAG;IACR,GAAG,EAAE,GAAG;IACR,GAAG,EAAE,GAAG;IACR,IAAI,EAAE,IAAI;IACV,GAAG,EAAE,GAAG;IACR,GAAG,EAAE,GAAG;IACR,GAAG,EAAE,GAAG;IACR,GAAG,EAAE,GAAG;IACR,GAAG,EAAE,GAAG;IACR,GAAG,EAAE,GAAG;IACR,GAAG,EAAE,GAAG;IACR,GAAG,EAAE,GAAG;IACR,GAAG,EAAE,GAAG;IACR,GAAG,EAAE,GAAG;IACR,GAAG,EAAE,GAAG;IACR,GAAG,EAAE,GAAG;IACR,GAAG,EAAE,GAAG;IACR,GAAG,EAAE,GAAG;IACR,GAAG,EAAE,GAAG;IACR,GAAG,EAAE,GAAG;IACR,GAAG,EAAE,GAAG;IACR,GAAG,EAAE,GAAG;IACR,GAAG,EAAE,GAAG;IACR,GAAG,EAAE,GAAG;IACR,GAAG,EAAE,GAAG;IACR,GAAG,EAAE,GAAG;IACR,GAAG,EAAE,GAAG;IACR,GAAG,EAAE,GAAG;IACR,GAAG,EAAE,GAAG;IACR,GAAG,EAAE,GAAG;IACX,GAAG,EAAE,GAAG;IACL,GAAG,EAAE,GAAG;IACR,GAAG,EAAE,GAAG;IACR,GAAG,EAAE,GAAG;IACR,GAAG,EAAE,GAAG;IACR,GAAG,EAAE,GAAG;IACR,GAAG,EAAE,GAAG;IACR,GAAG,EAAE,GAAG;IACR,GAAG,EAAE,GAAG;IAEX,aAAa;IACb,GAAG,EAAE,GAAG;IACR,GAAG,EAAE,GAAG;IACR,GAAG,EAAE,IAAI;IACT,GAAG,EAAE,GAAG;IACR,GAAG,EAAE,GAAG;CACR,CAAA;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;EAiJE"}

package/dist/nlp/Segmentation.d.ts

@@ -0,0 +1,41 @@
+export declare const wordCharacterPattern: RegExp;
+export declare const phraseSeparators: string[];
+export declare const sentenceSeparators: string[];
+export declare const symbolWords: string[];
+export declare function isWord(str: string): boolean;
+export declare class Sentence {
+    phrases: Phrase[];
+    readonly isSentenceFinalizer = true;
+    get length(): number;
+    get text(): string;
+}
+export declare class Phrase {
+    words: Word[];
+    get length(): number;
+    get text(): string;
+    get lastWord(): Word | undefined;
+    get isSentenceFinalizer(): boolean;
+}
+export declare class Word {
+    readonly text: string;
+    isSentenceFinalizer: boolean;
+    constructor(text: string, isSentenceFinalizer: boolean);
+    get containsOnlyPunctuation(): boolean;
+    get isSymbolWord(): boolean;
+    get isPhraseSeperator(): boolean;
+    get length(): number;
+}
+export type Segment = Sentence | Phrase | Word;
+export declare class Fragment {
+    segments: Segment[];
+    get length(): number;
+    get text(): string;
+    get isEmpty(): boolean;
+    get isNonempty(): boolean;
+    get lastSegment(): Segment | undefined;
+}
+export declare function splitToFragments(text: string, maxFragmentLength: number, langCode: string, preserveSentences?: boolean, preservePhrases?: boolean): Promise<Fragment[]>;
+export declare function parse(text: string, langCode: string): Promise<Sentence[]>;
+export declare function splitToSentences(text: string, langCode: string): string[];
+export declare function splitToWords(text: string, langCode: string): Promise<string[]>;
+export declare function splitToParagraphs(text: string): string[];

package/dist/nlp/Segmentation.js

@@ -0,0 +1,158 @@
+import XRegExp from 'xregexp';
+import * as CldrSegmentation from 'cldr-segmentation';
+import { splitChineseTextToWords_Jieba } from './ChineseSegmentation.js';
+import { sumArray, includesAnyOf, indexOfAnyOf, logToStderr } from '../utilities/Utilities.js';
+import { getShortLanguageCode } from '../utilities/Locale.js';
+import { splitJapaneseTextToWords_Kuromoji } from './JapaneseSegmentation.js';
+const log = logToStderr;
+export const wordCharacterPattern = XRegExp('[\\p{Letter}\\p{Number}]');
+export const phraseSeparators = [",", ";", ":"];
+export const sentenceSeparators = [".", "?", "!"];
+export const symbolWords = ["$", "%", "&", "@", "+", "/", "*", "="];
+export function isWord(str) {
+    str = str.trim();
+    return wordCharacterPattern.test(str) || symbolWords.includes(str);
+}
+export class Sentence {
+    phrases = [];
+    isSentenceFinalizer = true;
+    get length() { return sumArray(this.phrases, (phrase) => phrase.length); }
+    get text() { return this.phrases.reduce((result, phrase) => result + phrase.text, ""); }
+}
+export class Phrase {
+    words = [];
+    get length() { return sumArray(this.words, (word) => word.length); }
+    get text() { return this.words.reduce((result, word) => result + word.text, ""); }
+    get lastWord() {
+        if (this.words.length == 0) {
+            return undefined;
+        }
+        return this.words[this.words.length - 1];
+    }
+    get isSentenceFinalizer() { return this.lastWord != null ? this.lastWord.isSentenceFinalizer : false; }
+}
+export class Word {
+    text;
+    isSentenceFinalizer;
+    constructor(text, isSentenceFinalizer) {
+        this.text = text;
+        this.isSentenceFinalizer = isSentenceFinalizer;
+    }
+    get containsOnlyPunctuation() { return !wordCharacterPattern.test(this.text) && !this.isSymbolWord; }
+    get isSymbolWord() { return symbolWords.includes(this.text); }
+    get isPhraseSeperator() { return this.containsOnlyPunctuation && includesAnyOf(this.text, phraseSeparators); }
+    get length() { return this.text.length; }
+}
+export class Fragment {
+    segments = [];
+    get length() { return sumArray(this.segments, (phrase) => phrase.length); }
+    get text() { return this.segments.reduce((result, segment) => result + segment.text, ""); }
+    get isEmpty() { return this.length == 0; }
+    get isNonempty() { return !this.isEmpty; }
+    get lastSegment() {
+        if (this.isEmpty) {
+            return undefined;
+        }
+        return this.segments[this.segments.length - 1];
+    }
+}
+export async function splitToFragments(text, maxFragmentLength, langCode, preserveSentences = true, preservePhrases = true) {
+    const parsedText = await parse(text, langCode);
+    const fragments = [];
+    let currentFragment = new Fragment();
+    const remainingCharactersInCurrentFragment = () => maxFragmentLength - currentFragment.length;
+    const createNewFragmentIfNeeded = () => {
+        if (currentFragment.isNonempty) {
+            fragments.push(currentFragment);
+            currentFragment = new Fragment();
+        }
+    };
+    const fitsCurrentFragment = (segment) => segment.length <= remainingCharactersInCurrentFragment();
+    for (const sentence of parsedText) {
+        if (fitsCurrentFragment(sentence)) {
+            currentFragment.segments.push(sentence);
+            continue;
+        }
+        if (preserveSentences) {
+            createNewFragmentIfNeeded();
+            if (fitsCurrentFragment(sentence)) {
+                currentFragment.segments.push(sentence);
+                continue;
+            }
+        }
+        for (const phrase of sentence.phrases) {
+            if (fitsCurrentFragment(phrase)) {
+                currentFragment.segments.push(phrase);
+                continue;
+            }
+            if (preservePhrases) {
+                createNewFragmentIfNeeded();
+                if (fitsCurrentFragment(phrase)) {
+                    currentFragment.segments.push(phrase);
+                    continue;
+                }
+            }
+            for (const word of phrase.words) {
+                if (fitsCurrentFragment(word)) {
+                    currentFragment.segments.push(word);
+                    continue;
+                }
+                createNewFragmentIfNeeded();
+                if (fitsCurrentFragment(word)) {
+                    currentFragment.segments.push(word);
+                    continue;
+                }
+                throw new Error(`Encountered a word of length ${word.length}, which excceeds the maximum fragment length of ${maxFragmentLength}`);
+            }
+        }
+    }
+    createNewFragmentIfNeeded();
+    return fragments;
+}
+export async function parse(text, langCode) {
+    const sentencesText = splitToSentences(text, langCode);
+    const sentences = [];
+    for (const sentenceText of sentencesText) {
+        const sentence = new Sentence();
+        let currentPhrase = new Phrase();
+        const wordTexts = await splitToWords(sentenceText, langCode);
+        for (let wordIndex = 0; wordIndex < wordTexts.length; wordIndex++) {
+            const word = new Word(wordTexts[wordIndex], wordIndex == wordTexts.length - 1);
+            if (word.isPhraseSeperator) {
+                const separatorIndex = indexOfAnyOf(word.text, phraseSeparators);
+                currentPhrase.words.push(new Word(word.text.substring(0, separatorIndex + 1), word.isSentenceFinalizer));
+                sentence.phrases.push(currentPhrase);
+                currentPhrase = new Phrase();
+                currentPhrase.words.push(new Word(word.text.substring(separatorIndex + 1), false));
+            }
+            else {
+                currentPhrase.words.push(word);
+            }
+        }
+        if (currentPhrase.words.length > 0) {
+            sentence.phrases.push(currentPhrase);
+        }
+        sentences.push(sentence);
+    }
+    return sentences;
+}
+export function splitToSentences(text, langCode) {
+    const shortLangCode = getShortLanguageCode(langCode || "");
+    return CldrSegmentation.sentenceSplit(text, CldrSegmentation.suppressions[shortLangCode]);
+}
+export async function splitToWords(text, langCode) {
+    const shortLangCode = getShortLanguageCode(langCode || "");
+    if (shortLangCode == "zh" || shortLangCode == "cmn") {
+        return splitChineseTextToWords_Jieba(text, true, true);
+    }
+    else if (shortLangCode == "ja") {
+        return splitJapaneseTextToWords_Kuromoji(text);
+    }
+    else {
+        return CldrSegmentation.wordSplit(text, CldrSegmentation.suppressions[shortLangCode]);
+    }
+}
+export function splitToParagraphs(text) {
+    return text.split(/(\r?\n)+/g).map(p => p.trim()).filter(p => p.length > 0);
+}
+//# sourceMappingURL=Segmentation.js.map