@livekit/agents 0.5.0 → 0.5.2

package/dist/tokenize/basic/sentence.cjs.map

@@ -1 +1 @@
-{"version":3,"sources":["../../../src/tokenize/basic/sentence.ts"],"sourcesContent":["// SPDX-FileCopyrightText: 2024 LiveKit, Inc.\n//\n// SPDX-License-Identifier: Apache-2.0\n\n/**\n * Split the text into sentences.\n */\nexport const splitSentences = (text: string, minLength = 20): [string, number, number][] => {\n  const alphabets = /([A-Za-z])/g;\n  const prefixes = /(Mr|St|Mrs|Ms|Dr)[.]/g;\n  const suffixes = /(Inc|Ltd|Jr|Sr|Co)/g;\n  const starters =\n    /(Mr|Mrs|Ms|Dr|Prof|Capt|Cpt|Lt|He\\s|She\\s|It\\s|They\\s|Their\\s|Our\\s|We\\s|But\\s|However\\s|That\\s|This\\s|Wherever)/g;\n  const acronyms = /([A-Z][.][A-Z][.](?:[A-Z][.])?)/g;\n  const websites = /[.](com|net|org|io|gov|edu|me)/g;\n  const digits = /([0-9])/g;\n  const dots = /\\.{2,}/g;\n\n  text = text.replaceAll('\\n', ' ');\n  text = text.replaceAll(prefixes, '$1<prd>');\n  text = text.replaceAll(websites, '<prd>$2');\n  text = text.replaceAll(new RegExp(`${digits}[.]${digits}`, 'g'), '$1<prd>$2');\n  text = text.replaceAll(dots, (match) => '<prd>'.repeat(match.length));\n  text = text.replaceAll('Ph.D.', 'Ph<prd>D<prd>');\n  text = text.replaceAll(new RegExp(`\\s${alphabets}[.]`, 'g'), '$1<prd>');\n  text = text.replaceAll(new RegExp(`${acronyms} ${starters}`, 'g'), '$1<stop> $2');\n  text = text.replaceAll(\n    new RegExp(`${alphabets}[.]${alphabets}[.]${alphabets}[.]`, 'g'),\n    '$1<prd>$2<prd>$3<prd>',\n  );\n  text = text.replaceAll(new RegExp(`${alphabets}[.]${alphabets}[.]`, 'g'), '$1<prd>$2<prd>');\n  text = text.replaceAll(new RegExp(` ${suffixes}[.] ${starters}`, 'g'), '$1<stop> $2');\n  text = text.replaceAll(new RegExp(` ${suffixes}[.]`, 'g'), '$1<prd>');\n  text = text.replaceAll(new RegExp(` ${alphabets}[.]`, 'g'), '$1<prd>');\n  text = text.replaceAll('.”', '”.');\n  text = text.replaceAll('.\"', '\".');\n  text = text.replaceAll('!\"', '\"!');\n  text = text.replaceAll('?\"', '\"?');\n  text = text.replaceAll('.', '.<stop>');\n  text = text.replaceAll('?', '?<stop>');\n  text = text.replaceAll('!', '!<stop>');\n  text = text.replaceAll('<prd>', '.');\n\n  const split = text.split('<stop>');\n  text = text.replaceAll('<stop>', '');\n\n  const sentences: [string, number, number][] = [];\n  let buf = '';\n  let start = 0;\n  let end = 0;\n  for (const match of split) {\n    const sentence = match.trim();\n    if (!sentence) continue;\n\n    buf += ' ' + sentence;\n    end += match.length;\n    if (buf.length > minLength) {\n      sentences.push([buf.slice(1), start, end]);\n      start = end;\n      buf = '';\n    }\n  }\n\n  if (buf) {\n    sentences.push([buf.slice(1), start, text.length - 1]);\n  }\n\n  return sentences;\n};\n"],"mappings":";;;;;;;;;;;;;;;;;;AAAA;AAAA;AAAA;AAAA;AAAA;AAOO,MAAM,iBAAiB,CAAC,MAAc,YAAY,OAAmC;AAC1F,QAAM,YAAY;AAClB,QAAM,WAAW;AACjB,QAAM,WAAW;AACjB,QAAM,WACJ;AACF,QAAM,WAAW;AACjB,QAAM,WAAW;AACjB,QAAM,SAAS;AACf,QAAM,OAAO;AAEb,SAAO,KAAK,WAAW,MAAM,GAAG;AAChC,SAAO,KAAK,WAAW,UAAU,SAAS;AAC1C,SAAO,KAAK,WAAW,UAAU,SAAS;AAC1C,SAAO,KAAK,WAAW,IAAI,OAAO,GAAG,MAAM,MAAM,MAAM,IAAI,GAAG,GAAG,WAAW;AAC5E,SAAO,KAAK,WAAW,MAAM,CAAC,UAAU,QAAQ,OAAO,MAAM,MAAM,CAAC;AACpE,SAAO,KAAK,WAAW,SAAS,eAAe;AAC/C,SAAO,KAAK,WAAW,IAAI,OAAO,IAAK,SAAS,OAAO,GAAG,GAAG,SAAS;AACtE,SAAO,KAAK,WAAW,IAAI,OAAO,GAAG,QAAQ,IAAI,QAAQ,IAAI,GAAG,GAAG,aAAa;AAChF,SAAO,KAAK;AAAA,IACV,IAAI,OAAO,GAAG,SAAS,MAAM,SAAS,MAAM,SAAS,OAAO,GAAG;AAAA,IAC/D;AAAA,EACF;AACA,SAAO,KAAK,WAAW,IAAI,OAAO,GAAG,SAAS,MAAM,SAAS,OAAO,GAAG,GAAG,gBAAgB;AAC1F,SAAO,KAAK,WAAW,IAAI,OAAO,IAAI,QAAQ,OAAO,QAAQ,IAAI,GAAG,GAAG,aAAa;AACpF,SAAO,KAAK,WAAW,IAAI,OAAO,IAAI,QAAQ,OAAO,GAAG,GAAG,SAAS;AACpE,SAAO,KAAK,WAAW,IAAI,OAAO,IAAI,SAAS,OAAO,GAAG,GAAG,SAAS;AACrE,SAAO,KAAK,WAAW,WAAM,SAAI;AACjC,SAAO,KAAK,WAAW,MAAM,IAAI;AACjC,SAAO,KAAK,WAAW,MAAM,IAAI;AACjC,SAAO,KAAK,WAAW,MAAM,IAAI;AACjC,SAAO,KAAK,WAAW,KAAK,SAAS;AACrC,SAAO,KAAK,WAAW,KAAK,SAAS;AACrC,SAAO,KAAK,WAAW,KAAK,SAAS;AACrC,SAAO,KAAK,WAAW,SAAS,GAAG;AAEnC,QAAM,QAAQ,KAAK,MAAM,QAAQ;AACjC,SAAO,KAAK,WAAW,UAAU,EAAE;AAEnC,QAAM,YAAwC,CAAC;AAC/C,MAAI,MAAM;AACV,MAAI,QAAQ;AACZ,MAAI,MAAM;AACV,aAAW,SAAS,OAAO;AACzB,UAAM,WAAW,MAAM,KAAK;AAC5B,QAAI,CAAC,SAAU;AAEf,WAAO,MAAM;AACb,WAAO,MAAM;AACb,QAAI,IAAI,SAAS,WAAW;AAC1B,gBAAU,KAAK,CAAC,IAAI,MAAM,CAAC,GAAG,OAAO,GAAG,CAAC;AACzC,cAAQ;AACR,YAAM;AAAA,IACR;AAAA,EACF;AAEA,MAAI,KAAK;AACP,cAAU,KAAK,CAAC,IAAI,MAAM,CAAC,GAAG,OAAO,KAAK,SAAS,CAAC,CAAC;AAAA,EACvD;AAEA,SAAO;AACT;","names":[]}
+{"version":3,"sources":["../../../src/tokenize/basic/sentence.ts"],"sourcesContent":["// SPDX-FileCopyrightText: 2024 LiveKit, Inc.\n//\n// SPDX-License-Identifier: Apache-2.0\n\n/**\n * Split the text into sentences.\n */\nexport const splitSentences = (text: string, minLength = 20): [string, number, number][] => {\n  const alphabets = /([A-Za-z])/g;\n  const prefixes = /(Mr|St|Mrs|Ms|Dr)[.]/g;\n  const suffixes = /(Inc|Ltd|Jr|Sr|Co)/g;\n  const starters =\n    /(Mr|Mrs|Ms|Dr|Prof|Capt|Cpt|Lt|He\\s|She\\s|It\\s|They\\s|Their\\s|Our\\s|We\\s|But\\s|However\\s|That\\s|This\\s|Wherever)/g;\n  const acronyms = /([A-Z][.][A-Z][.](?:[A-Z][.])?)/g;\n  const websites = /[.](com|net|org|io|gov|edu|me)/g;\n  const digits = /([0-9])/g;\n  const dots = /\\.{2,}/g;\n\n  text = text.replaceAll('\\n', ' ');\n  text = text.replaceAll(prefixes, '$1<prd>');\n  text = text.replaceAll(websites, '<prd>$2');\n  text = text.replaceAll(new RegExp(`${digits.source}[.]${digits.source}`, 'g'), '$1<prd>$2');\n  text = text.replaceAll(dots, (match) => '<prd>'.repeat(match.length));\n  text = text.replaceAll('Ph.D.', 'Ph<prd>D<prd>');\n  text = text.replaceAll(new RegExp(`\\\\s${alphabets.source}[.] `, 'g'), ' $1<prd> ');\n  text = text.replaceAll(new RegExp(`${acronyms.source} ${starters.source}`, 'g'), '$1<stop> $2');\n  text = text.replaceAll(\n    new RegExp(`${alphabets.source}[.]${alphabets.source}[.]${alphabets.source}[.]`, 'g'),\n    '$1<prd>$2<prd>$3<prd>',\n  );\n  text = text.replaceAll(\n    new RegExp(`${alphabets.source}[.]${alphabets.source}[.]`, 'g'),\n    '$1<prd>$2<prd>',\n  );\n  text = text.replaceAll(\n    new RegExp(` ${suffixes.source}[.] ${starters.source}`, 'g'),\n    '$1<stop> $2',\n  );\n  text = text.replaceAll(new RegExp(` ${suffixes.source}[.]`, 'g'), '$1<prd>');\n  text = text.replaceAll(new RegExp(` ${alphabets.source}[.]`, 'g'), '$1<prd>');\n  text = text.replaceAll('.”', '”.');\n  text = text.replaceAll('.\"', '\".');\n  text = text.replaceAll('!\"', '\"!');\n  text = text.replaceAll('?\"', '\"?');\n  text = text.replaceAll('.', '.<stop>');\n  text = text.replaceAll('?', '?<stop>');\n  text = text.replaceAll('!', '!<stop>');\n  text = text.replaceAll('<prd>', '.');\n\n  const split = text.split('<stop>');\n  text = text.replaceAll('<stop>', '');\n\n  const sentences: [string, number, number][] = [];\n  let buf = '';\n  let start = 0;\n  let end = 0;\n  for (const match of split) {\n    const sentence = match.trim();\n    if (!sentence) continue;\n\n    buf += ' ' + sentence;\n    end += match.length;\n    if (buf.length > minLength) {\n      sentences.push([buf.slice(1), start, end]);\n      start = end;\n      buf = '';\n    }\n  }\n\n  if (buf) {\n    sentences.push([buf.slice(1), start, text.length - 1]);\n  }\n\n  return sentences;\n};\n"],"mappings":";;;;;;;;;;;;;;;;;;AAAA;AAAA;AAAA;AAAA;AAAA;AAOO,MAAM,iBAAiB,CAAC,MAAc,YAAY,OAAmC;AAC1F,QAAM,YAAY;AAClB,QAAM,WAAW;AACjB,QAAM,WAAW;AACjB,QAAM,WACJ;AACF,QAAM,WAAW;AACjB,QAAM,WAAW;AACjB,QAAM,SAAS;AACf,QAAM,OAAO;AAEb,SAAO,KAAK,WAAW,MAAM,GAAG;AAChC,SAAO,KAAK,WAAW,UAAU,SAAS;AAC1C,SAAO,KAAK,WAAW,UAAU,SAAS;AAC1C,SAAO,KAAK,WAAW,IAAI,OAAO,GAAG,OAAO,MAAM,MAAM,OAAO,MAAM,IAAI,GAAG,GAAG,WAAW;AAC1F,SAAO,KAAK,WAAW,MAAM,CAAC,UAAU,QAAQ,OAAO,MAAM,MAAM,CAAC;AACpE,SAAO,KAAK,WAAW,SAAS,eAAe;AAC/C,SAAO,KAAK,WAAW,IAAI,OAAO,MAAM,UAAU,MAAM,QAAQ,GAAG,GAAG,WAAW;AACjF,SAAO,KAAK,WAAW,IAAI,OAAO,GAAG,SAAS,MAAM,IAAI,SAAS,MAAM,IAAI,GAAG,GAAG,aAAa;AAC9F,SAAO,KAAK;AAAA,IACV,IAAI,OAAO,GAAG,UAAU,MAAM,MAAM,UAAU,MAAM,MAAM,UAAU,MAAM,OAAO,GAAG;AAAA,IACpF;AAAA,EACF;AACA,SAAO,KAAK;AAAA,IACV,IAAI,OAAO,GAAG,UAAU,MAAM,MAAM,UAAU,MAAM,OAAO,GAAG;AAAA,IAC9D;AAAA,EACF;AACA,SAAO,KAAK;AAAA,IACV,IAAI,OAAO,IAAI,SAAS,MAAM,OAAO,SAAS,MAAM,IAAI,GAAG;AAAA,IAC3D;AAAA,EACF;AACA,SAAO,KAAK,WAAW,IAAI,OAAO,IAAI,SAAS,MAAM,OAAO,GAAG,GAAG,SAAS;AAC3E,SAAO,KAAK,WAAW,IAAI,OAAO,IAAI,UAAU,MAAM,OAAO,GAAG,GAAG,SAAS;AAC5E,SAAO,KAAK,WAAW,WAAM,SAAI;AACjC,SAAO,KAAK,WAAW,MAAM,IAAI;AACjC,SAAO,KAAK,WAAW,MAAM,IAAI;AACjC,SAAO,KAAK,WAAW,MAAM,IAAI;AACjC,SAAO,KAAK,WAAW,KAAK,SAAS;AACrC,SAAO,KAAK,WAAW,KAAK,SAAS;AACrC,SAAO,KAAK,WAAW,KAAK,SAAS;AACrC,SAAO,KAAK,WAAW,SAAS,GAAG;AAEnC,QAAM,QAAQ,KAAK,MAAM,QAAQ;AACjC,SAAO,KAAK,WAAW,UAAU,EAAE;AAEnC,QAAM,YAAwC,CAAC;AAC/C,MAAI,MAAM;AACV,MAAI,QAAQ;AACZ,MAAI,MAAM;AACV,aAAW,SAAS,OAAO;AACzB,UAAM,WAAW,MAAM,KAAK;AAC5B,QAAI,CAAC,SAAU;AAEf,WAAO,MAAM;AACb,WAAO,MAAM;AACb,QAAI,IAAI,SAAS,WAAW;AAC1B,gBAAU,KAAK,CAAC,IAAI,MAAM,CAAC,GAAG,OAAO,GAAG,CAAC;AACzC,cAAQ;AACR,YAAM;AAAA,IACR;AAAA,EACF;AAEA,MAAI,KAAK;AACP,cAAU,KAAK,CAAC,IAAI,MAAM,CAAC,GAAG,OAAO,KAAK,SAAS,CAAC,CAAC;AAAA,EACvD;AAEA,SAAO;AACT;","names":[]}

package/dist/tokenize/basic/sentence.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"sentence.d.ts","sourceRoot":"","sources":["../../../src/tokenize/basic/sentence.ts"],"names":[],"mappings":"AAIA;;GAEG;AACH,eAAO,MAAM,cAAc,SAAU,MAAM,yBAAmB,CAAC,MAAM,EAAE,MAAM,EAAE,MAAM,CAAC,EA6DrF,CAAC"}
+{"version":3,"file":"sentence.d.ts","sourceRoot":"","sources":["../../../src/tokenize/basic/sentence.ts"],"names":[],"mappings":"AAIA;;GAEG;AACH,eAAO,MAAM,cAAc,SAAU,MAAM,yBAAmB,CAAC,MAAM,EAAE,MAAM,EAAE,MAAM,CAAC,EAmErF,CAAC"}

package/dist/tokenize/basic/sentence.js

@@ -10,19 +10,25 @@ const splitSentences = (text, minLength = 20) => {
   text = text.replaceAll("\n", " ");
   text = text.replaceAll(prefixes, "$1<prd>");
   text = text.replaceAll(websites, "<prd>$2");
-  text = text.replaceAll(new RegExp(`${digits}[.]${digits}`, "g"), "$1<prd>$2");
+  text = text.replaceAll(new RegExp(`${digits.source}[.]${digits.source}`, "g"), "$1<prd>$2");
   text = text.replaceAll(dots, (match) => "<prd>".repeat(match.length));
   text = text.replaceAll("Ph.D.", "Ph<prd>D<prd>");
-  text = text.replaceAll(new RegExp(`s${alphabets}[.]`, "g"), "$1<prd>");
-  text = text.replaceAll(new RegExp(`${acronyms} ${starters}`, "g"), "$1<stop> $2");
+  text = text.replaceAll(new RegExp(`\\s${alphabets.source}[.] `, "g"), " $1<prd> ");
+  text = text.replaceAll(new RegExp(`${acronyms.source} ${starters.source}`, "g"), "$1<stop> $2");
   text = text.replaceAll(
-    new RegExp(`${alphabets}[.]${alphabets}[.]${alphabets}[.]`, "g"),
+    new RegExp(`${alphabets.source}[.]${alphabets.source}[.]${alphabets.source}[.]`, "g"),
     "$1<prd>$2<prd>$3<prd>"
   );
-  text = text.replaceAll(new RegExp(`${alphabets}[.]${alphabets}[.]`, "g"), "$1<prd>$2<prd>");
-  text = text.replaceAll(new RegExp(` ${suffixes}[.] ${starters}`, "g"), "$1<stop> $2");
-  text = text.replaceAll(new RegExp(` ${suffixes}[.]`, "g"), "$1<prd>");
-  text = text.replaceAll(new RegExp(` ${alphabets}[.]`, "g"), "$1<prd>");
+  text = text.replaceAll(
+    new RegExp(`${alphabets.source}[.]${alphabets.source}[.]`, "g"),
+    "$1<prd>$2<prd>"
+  );
+  text = text.replaceAll(
+    new RegExp(` ${suffixes.source}[.] ${starters.source}`, "g"),
+    "$1<stop> $2"
+  );
+  text = text.replaceAll(new RegExp(` ${suffixes.source}[.]`, "g"), "$1<prd>");
+  text = text.replaceAll(new RegExp(` ${alphabets.source}[.]`, "g"), "$1<prd>");
   text = text.replaceAll(".\u201D", "\u201D.");
   text = text.replaceAll('."', '".');
   text = text.replaceAll('!"', '"!');

package/dist/tokenize/basic/sentence.js.map

@@ -1 +1 @@
-{"version":3,"sources":["../../../src/tokenize/basic/sentence.ts"],"sourcesContent":["// SPDX-FileCopyrightText: 2024 LiveKit, Inc.\n//\n// SPDX-License-Identifier: Apache-2.0\n\n/**\n * Split the text into sentences.\n */\nexport const splitSentences = (text: string, minLength = 20): [string, number, number][] => {\n  const alphabets = /([A-Za-z])/g;\n  const prefixes = /(Mr|St|Mrs|Ms|Dr)[.]/g;\n  const suffixes = /(Inc|Ltd|Jr|Sr|Co)/g;\n  const starters =\n    /(Mr|Mrs|Ms|Dr|Prof|Capt|Cpt|Lt|He\\s|She\\s|It\\s|They\\s|Their\\s|Our\\s|We\\s|But\\s|However\\s|That\\s|This\\s|Wherever)/g;\n  const acronyms = /([A-Z][.][A-Z][.](?:[A-Z][.])?)/g;\n  const websites = /[.](com|net|org|io|gov|edu|me)/g;\n  const digits = /([0-9])/g;\n  const dots = /\\.{2,}/g;\n\n  text = text.replaceAll('\\n', ' ');\n  text = text.replaceAll(prefixes, '$1<prd>');\n  text = text.replaceAll(websites, '<prd>$2');\n  text = text.replaceAll(new RegExp(`${digits}[.]${digits}`, 'g'), '$1<prd>$2');\n  text = text.replaceAll(dots, (match) => '<prd>'.repeat(match.length));\n  text = text.replaceAll('Ph.D.', 'Ph<prd>D<prd>');\n  text = text.replaceAll(new RegExp(`\\s${alphabets}[.]`, 'g'), '$1<prd>');\n  text = text.replaceAll(new RegExp(`${acronyms} ${starters}`, 'g'), '$1<stop> $2');\n  text = text.replaceAll(\n    new RegExp(`${alphabets}[.]${alphabets}[.]${alphabets}[.]`, 'g'),\n    '$1<prd>$2<prd>$3<prd>',\n  );\n  text = text.replaceAll(new RegExp(`${alphabets}[.]${alphabets}[.]`, 'g'), '$1<prd>$2<prd>');\n  text = text.replaceAll(new RegExp(` ${suffixes}[.] ${starters}`, 'g'), '$1<stop> $2');\n  text = text.replaceAll(new RegExp(` ${suffixes}[.]`, 'g'), '$1<prd>');\n  text = text.replaceAll(new RegExp(` ${alphabets}[.]`, 'g'), '$1<prd>');\n  text = text.replaceAll('.”', '”.');\n  text = text.replaceAll('.\"', '\".');\n  text = text.replaceAll('!\"', '\"!');\n  text = text.replaceAll('?\"', '\"?');\n  text = text.replaceAll('.', '.<stop>');\n  text = text.replaceAll('?', '?<stop>');\n  text = text.replaceAll('!', '!<stop>');\n  text = text.replaceAll('<prd>', '.');\n\n  const split = text.split('<stop>');\n  text = text.replaceAll('<stop>', '');\n\n  const sentences: [string, number, number][] = [];\n  let buf = '';\n  let start = 0;\n  let end = 0;\n  for (const match of split) {\n    const sentence = match.trim();\n    if (!sentence) continue;\n\n    buf += ' ' + sentence;\n    end += match.length;\n    if (buf.length > minLength) {\n      sentences.push([buf.slice(1), start, end]);\n      start = end;\n      buf = '';\n    }\n  }\n\n  if (buf) {\n    sentences.push([buf.slice(1), start, text.length - 1]);\n  }\n\n  return sentences;\n};\n"],"mappings":"AAOO,MAAM,iBAAiB,CAAC,MAAc,YAAY,OAAmC;AAC1F,QAAM,YAAY;AAClB,QAAM,WAAW;AACjB,QAAM,WAAW;AACjB,QAAM,WACJ;AACF,QAAM,WAAW;AACjB,QAAM,WAAW;AACjB,QAAM,SAAS;AACf,QAAM,OAAO;AAEb,SAAO,KAAK,WAAW,MAAM,GAAG;AAChC,SAAO,KAAK,WAAW,UAAU,SAAS;AAC1C,SAAO,KAAK,WAAW,UAAU,SAAS;AAC1C,SAAO,KAAK,WAAW,IAAI,OAAO,GAAG,MAAM,MAAM,MAAM,IAAI,GAAG,GAAG,WAAW;AAC5E,SAAO,KAAK,WAAW,MAAM,CAAC,UAAU,QAAQ,OAAO,MAAM,MAAM,CAAC;AACpE,SAAO,KAAK,WAAW,SAAS,eAAe;AAC/C,SAAO,KAAK,WAAW,IAAI,OAAO,IAAK,SAAS,OAAO,GAAG,GAAG,SAAS;AACtE,SAAO,KAAK,WAAW,IAAI,OAAO,GAAG,QAAQ,IAAI,QAAQ,IAAI,GAAG,GAAG,aAAa;AAChF,SAAO,KAAK;AAAA,IACV,IAAI,OAAO,GAAG,SAAS,MAAM,SAAS,MAAM,SAAS,OAAO,GAAG;AAAA,IAC/D;AAAA,EACF;AACA,SAAO,KAAK,WAAW,IAAI,OAAO,GAAG,SAAS,MAAM,SAAS,OAAO,GAAG,GAAG,gBAAgB;AAC1F,SAAO,KAAK,WAAW,IAAI,OAAO,IAAI,QAAQ,OAAO,QAAQ,IAAI,GAAG,GAAG,aAAa;AACpF,SAAO,KAAK,WAAW,IAAI,OAAO,IAAI,QAAQ,OAAO,GAAG,GAAG,SAAS;AACpE,SAAO,KAAK,WAAW,IAAI,OAAO,IAAI,SAAS,OAAO,GAAG,GAAG,SAAS;AACrE,SAAO,KAAK,WAAW,WAAM,SAAI;AACjC,SAAO,KAAK,WAAW,MAAM,IAAI;AACjC,SAAO,KAAK,WAAW,MAAM,IAAI;AACjC,SAAO,KAAK,WAAW,MAAM,IAAI;AACjC,SAAO,KAAK,WAAW,KAAK,SAAS;AACrC,SAAO,KAAK,WAAW,KAAK,SAAS;AACrC,SAAO,KAAK,WAAW,KAAK,SAAS;AACrC,SAAO,KAAK,WAAW,SAAS,GAAG;AAEnC,QAAM,QAAQ,KAAK,MAAM,QAAQ;AACjC,SAAO,KAAK,WAAW,UAAU,EAAE;AAEnC,QAAM,YAAwC,CAAC;AAC/C,MAAI,MAAM;AACV,MAAI,QAAQ;AACZ,MAAI,MAAM;AACV,aAAW,SAAS,OAAO;AACzB,UAAM,WAAW,MAAM,KAAK;AAC5B,QAAI,CAAC,SAAU;AAEf,WAAO,MAAM;AACb,WAAO,MAAM;AACb,QAAI,IAAI,SAAS,WAAW;AAC1B,gBAAU,KAAK,CAAC,IAAI,MAAM,CAAC,GAAG,OAAO,GAAG,CAAC;AACzC,cAAQ;AACR,YAAM;AAAA,IACR;AAAA,EACF;AAEA,MAAI,KAAK;AACP,cAAU,KAAK,CAAC,IAAI,MAAM,CAAC,GAAG,OAAO,KAAK,SAAS,CAAC,CAAC;AAAA,EACvD;AAEA,SAAO;AACT;","names":[]}
+{"version":3,"sources":["../../../src/tokenize/basic/sentence.ts"],"sourcesContent":["// SPDX-FileCopyrightText: 2024 LiveKit, Inc.\n//\n// SPDX-License-Identifier: Apache-2.0\n\n/**\n * Split the text into sentences.\n */\nexport const splitSentences = (text: string, minLength = 20): [string, number, number][] => {\n  const alphabets = /([A-Za-z])/g;\n  const prefixes = /(Mr|St|Mrs|Ms|Dr)[.]/g;\n  const suffixes = /(Inc|Ltd|Jr|Sr|Co)/g;\n  const starters =\n    /(Mr|Mrs|Ms|Dr|Prof|Capt|Cpt|Lt|He\\s|She\\s|It\\s|They\\s|Their\\s|Our\\s|We\\s|But\\s|However\\s|That\\s|This\\s|Wherever)/g;\n  const acronyms = /([A-Z][.][A-Z][.](?:[A-Z][.])?)/g;\n  const websites = /[.](com|net|org|io|gov|edu|me)/g;\n  const digits = /([0-9])/g;\n  const dots = /\\.{2,}/g;\n\n  text = text.replaceAll('\\n', ' ');\n  text = text.replaceAll(prefixes, '$1<prd>');\n  text = text.replaceAll(websites, '<prd>$2');\n  text = text.replaceAll(new RegExp(`${digits.source}[.]${digits.source}`, 'g'), '$1<prd>$2');\n  text = text.replaceAll(dots, (match) => '<prd>'.repeat(match.length));\n  text = text.replaceAll('Ph.D.', 'Ph<prd>D<prd>');\n  text = text.replaceAll(new RegExp(`\\\\s${alphabets.source}[.] `, 'g'), ' $1<prd> ');\n  text = text.replaceAll(new RegExp(`${acronyms.source} ${starters.source}`, 'g'), '$1<stop> $2');\n  text = text.replaceAll(\n    new RegExp(`${alphabets.source}[.]${alphabets.source}[.]${alphabets.source}[.]`, 'g'),\n    '$1<prd>$2<prd>$3<prd>',\n  );\n  text = text.replaceAll(\n    new RegExp(`${alphabets.source}[.]${alphabets.source}[.]`, 'g'),\n    '$1<prd>$2<prd>',\n  );\n  text = text.replaceAll(\n    new RegExp(` ${suffixes.source}[.] ${starters.source}`, 'g'),\n    '$1<stop> $2',\n  );\n  text = text.replaceAll(new RegExp(` ${suffixes.source}[.]`, 'g'), '$1<prd>');\n  text = text.replaceAll(new RegExp(` ${alphabets.source}[.]`, 'g'), '$1<prd>');\n  text = text.replaceAll('.”', '”.');\n  text = text.replaceAll('.\"', '\".');\n  text = text.replaceAll('!\"', '\"!');\n  text = text.replaceAll('?\"', '\"?');\n  text = text.replaceAll('.', '.<stop>');\n  text = text.replaceAll('?', '?<stop>');\n  text = text.replaceAll('!', '!<stop>');\n  text = text.replaceAll('<prd>', '.');\n\n  const split = text.split('<stop>');\n  text = text.replaceAll('<stop>', '');\n\n  const sentences: [string, number, number][] = [];\n  let buf = '';\n  let start = 0;\n  let end = 0;\n  for (const match of split) {\n    const sentence = match.trim();\n    if (!sentence) continue;\n\n    buf += ' ' + sentence;\n    end += match.length;\n    if (buf.length > minLength) {\n      sentences.push([buf.slice(1), start, end]);\n      start = end;\n      buf = '';\n    }\n  }\n\n  if (buf) {\n    sentences.push([buf.slice(1), start, text.length - 1]);\n  }\n\n  return sentences;\n};\n"],"mappings":"AAOO,MAAM,iBAAiB,CAAC,MAAc,YAAY,OAAmC;AAC1F,QAAM,YAAY;AAClB,QAAM,WAAW;AACjB,QAAM,WAAW;AACjB,QAAM,WACJ;AACF,QAAM,WAAW;AACjB,QAAM,WAAW;AACjB,QAAM,SAAS;AACf,QAAM,OAAO;AAEb,SAAO,KAAK,WAAW,MAAM,GAAG;AAChC,SAAO,KAAK,WAAW,UAAU,SAAS;AAC1C,SAAO,KAAK,WAAW,UAAU,SAAS;AAC1C,SAAO,KAAK,WAAW,IAAI,OAAO,GAAG,OAAO,MAAM,MAAM,OAAO,MAAM,IAAI,GAAG,GAAG,WAAW;AAC1F,SAAO,KAAK,WAAW,MAAM,CAAC,UAAU,QAAQ,OAAO,MAAM,MAAM,CAAC;AACpE,SAAO,KAAK,WAAW,SAAS,eAAe;AAC/C,SAAO,KAAK,WAAW,IAAI,OAAO,MAAM,UAAU,MAAM,QAAQ,GAAG,GAAG,WAAW;AACjF,SAAO,KAAK,WAAW,IAAI,OAAO,GAAG,SAAS,MAAM,IAAI,SAAS,MAAM,IAAI,GAAG,GAAG,aAAa;AAC9F,SAAO,KAAK;AAAA,IACV,IAAI,OAAO,GAAG,UAAU,MAAM,MAAM,UAAU,MAAM,MAAM,UAAU,MAAM,OAAO,GAAG;AAAA,IACpF;AAAA,EACF;AACA,SAAO,KAAK;AAAA,IACV,IAAI,OAAO,GAAG,UAAU,MAAM,MAAM,UAAU,MAAM,OAAO,GAAG;AAAA,IAC9D;AAAA,EACF;AACA,SAAO,KAAK;AAAA,IACV,IAAI,OAAO,IAAI,SAAS,MAAM,OAAO,SAAS,MAAM,IAAI,GAAG;AAAA,IAC3D;AAAA,EACF;AACA,SAAO,KAAK,WAAW,IAAI,OAAO,IAAI,SAAS,MAAM,OAAO,GAAG,GAAG,SAAS;AAC3E,SAAO,KAAK,WAAW,IAAI,OAAO,IAAI,UAAU,MAAM,OAAO,GAAG,GAAG,SAAS;AAC5E,SAAO,KAAK,WAAW,WAAM,SAAI;AACjC,SAAO,KAAK,WAAW,MAAM,IAAI;AACjC,SAAO,KAAK,WAAW,MAAM,IAAI;AACjC,SAAO,KAAK,WAAW,MAAM,IAAI;AACjC,SAAO,KAAK,WAAW,KAAK,SAAS;AACrC,SAAO,KAAK,WAAW,KAAK,SAAS;AACrC,SAAO,KAAK,WAAW,KAAK,SAAS;AACrC,SAAO,KAAK,WAAW,SAAS,GAAG;AAEnC,QAAM,QAAQ,KAAK,MAAM,QAAQ;AACjC,SAAO,KAAK,WAAW,UAAU,EAAE;AAEnC,QAAM,YAAwC,CAAC;AAC/C,MAAI,MAAM;AACV,MAAI,QAAQ;AACZ,MAAI,MAAM;AACV,aAAW,SAAS,OAAO;AACzB,UAAM,WAAW,MAAM,KAAK;AAC5B,QAAI,CAAC,SAAU;AAEf,WAAO,MAAM;AACb,WAAO,MAAM;AACb,QAAI,IAAI,SAAS,WAAW;AAC1B,gBAAU,KAAK,CAAC,IAAI,MAAM,CAAC,GAAG,OAAO,GAAG,CAAC;AACzC,cAAQ;AACR,YAAM;AAAA,IACR;AAAA,EACF;AAEA,MAAI,KAAK;AACP,cAAU,KAAK,CAAC,IAAI,MAAM,CAAC,GAAG,OAAO,KAAK,SAAS,CAAC,CAAC;AAAA,EACvD;AAEA,SAAO;AACT;","names":[]}

package/dist/tokenize/tokenizer.test.cjs

@@ -0,0 +1,220 @@
+"use strict";
+var import_vitest = require("vitest");
+var import_basic = require("./basic/index.cjs");
+var import_paragraph = require("./basic/paragraph.cjs");
+const TEXT = "Hi! LiveKit is a platform for live audio and video applications and services. R.T.C stands for Real-Time Communication... again R.T.C. Mr. Theo is testing the sentence tokenizer. This is a test. Another test. A short sentence. A longer sentence that is longer than the previous sentence. f(x) = x * 2.54 + 42. Hey! Hi! Hello! ";
+const EXPECTED_MIN_20 = [
+  "Hi! LiveKit is a platform for live audio and video applications and services.",
+  "R.T.C stands for Real-Time Communication... again R.T.C.",
+  "Mr. Theo is testing the sentence tokenizer.",
+  "This is a test. Another test.",
+  "A short sentence. A longer sentence that is longer than the previous sentence.",
+  "f(x) = x * 2.54 + 42.",
+  "Hey! Hi! Hello!"
+];
+const WORDS_TEXT = "This is a test. Blabla another test! multiple consecutive spaces:     done";
+const WORDS_EXPECTED = [
+  "This",
+  "is",
+  "a",
+  "test",
+  "Blabla",
+  "another",
+  "test",
+  "multiple",
+  "consecutive",
+  "spaces",
+  "done"
+];
+const WORDS_PUNCT_TEXT = 'This is <phoneme alphabet="cmu-arpabet" ph="AE K CH UW AH L IY">actually</phoneme> tricky to handle.';
+const WORDS_PUNCT_EXPECTED = [
+  "This",
+  "is",
+  "<phoneme",
+  'alphabet="cmu-arpabet"',
+  'ph="AE',
+  "K",
+  "CH",
+  "UW",
+  "AH",
+  "L",
+  'IY">actually</phoneme>',
+  "tricky",
+  "to",
+  "handle."
+];
+const HYPHENATOR_TEXT = ["Segment", "expected", "communication", "window", "welcome", "bedroom"];
+const HYPHENATOR_EXPECTED = [
+  ["Seg", "ment"],
+  ["ex", "pect", "ed"],
+  ["com", "mu", "ni", "ca", "tion"],
+  ["win", "dow"],
+  ["wel", "come"],
+  ["bed", "room"]
+];
+const PARAGRAPH_TEST_CASES = [
+  ["Single paragraph.", [["Single paragraph.", 0, 17]]],
+  [
+    "Paragraph 1.\n\nParagraph 2.",
+    [
+      ["Paragraph 1.", 0, 12],
+      ["Paragraph 2.", 14, 26]
+    ]
+  ],
+  [
+    "Para 1.\n\nPara 2.\n\nPara 3.",
+    [
+      ["Para 1.", 0, 7],
+      ["Para 2.", 9, 16],
+      ["Para 3.", 18, 25]
+    ]
+  ],
+  ["\n\nParagraph with leading newlines.", [["Paragraph with leading newlines.", 2, 34]]],
+  ["Paragraph with trailing newlines.\n\n", [["Paragraph with trailing newlines.", 0, 33]]],
+  [
+    "\n\n  Paragraph with leading and trailing spaces.  \n\n",
+    [["Paragraph with leading and trailing spaces.", 4, 47]]
+  ],
+  [
+    "Para 1.\n\n\n\nPara 2.",
+    // Multiple newlines between paragraphs
+    [
+      ["Para 1.", 0, 7],
+      ["Para 2.", 11, 18]
+    ]
+  ],
+  [
+    "Para 1.\n \n \nPara 2.",
+    // Newlines with spaces between paragraphs
+    [
+      ["Para 1.", 0, 7],
+      ["Para 2.", 12, 19]
+    ]
+  ],
+  [
+    "",
+    // Empty string
+    []
+  ],
+  [
+    "\n\n\n",
+    // Only newlines
+    []
+  ],
+  [
+    "Line 1\nLine 2\nLine 3",
+    // Single paragraph with newlines
+    [["Line 1\nLine 2\nLine 3", 0, 20]]
+  ]
+];
+(0, import_vitest.describe)("tokenizer", () => {
+  (0, import_vitest.describe)("SentenceTokenizer", () => {
+    const tokenizer = new import_basic.SentenceTokenizer();
+    (0, import_vitest.it)("should tokenize sentences correctly", () => {
+      (0, import_vitest.expect)(tokenizer.tokenize(TEXT).every((x, i) => EXPECTED_MIN_20[i] === x)).toBeTruthy();
+    });
+    (0, import_vitest.it)("should stream tokenize sentences correctly", async () => {
+      const pattern = [1, 2, 4];
+      let text = TEXT;
+      const chunks = [];
+      const patternIter = Array(Math.ceil(text.length / pattern.reduce((sum, num) => sum + num, 0))).fill(pattern).flat()[Symbol.iterator]();
+      for (const size of patternIter) {
+        if (!text) break;
+        chunks.push(text.slice(void 0, size));
+        text = text.slice(size);
+      }
+      const stream = tokenizer.stream();
+      for (const chunk of chunks) {
+        stream.pushText(chunk);
+      }
+      stream.endInput();
+      stream.close();
+      for (const x of EXPECTED_MIN_20) {
+        await stream.next().then((value) => {
+          if (value.value) {
+            (0, import_vitest.expect)(value.value.token).toStrictEqual(x);
+          }
+        });
+      }
+    });
+  });
+  (0, import_vitest.describe)("WordTokenizer", () => {
+    const tokenizer = new import_basic.WordTokenizer();
+    (0, import_vitest.it)("should tokenize words correctly", () => {
+      (0, import_vitest.expect)(tokenizer.tokenize(WORDS_TEXT).every((x, i) => WORDS_EXPECTED[i] === x)).toBeTruthy();
+    });
+    (0, import_vitest.it)("should stream tokenize words correctly", async () => {
+      const pattern = [1, 2, 4];
+      let text = WORDS_TEXT;
+      const chunks = [];
+      const patternIter = Array(Math.ceil(text.length / pattern.reduce((sum, num) => sum + num, 0))).fill(pattern).flat()[Symbol.iterator]();
+      for (const size of patternIter) {
+        if (!text) break;
+        chunks.push(text.slice(void 0, size));
+        text = text.slice(size);
+      }
+      const stream = tokenizer.stream();
+      for (const chunk of chunks) {
+        stream.pushText(chunk);
+      }
+      stream.endInput();
+      stream.close();
+      for (const x of WORDS_EXPECTED) {
+        await stream.next().then((value) => {
+          if (value.value) {
+            (0, import_vitest.expect)(value.value.token).toStrictEqual(x);
+          }
+        });
+      }
+    });
+    (0, import_vitest.describe)("punctuation handling", () => {
+      const tokenizerPunct = new import_basic.WordTokenizer(false);
+      (0, import_vitest.it)("should tokenize words correctly", () => {
+        (0, import_vitest.expect)(
+          tokenizerPunct.tokenize(WORDS_PUNCT_TEXT).every((x, i) => WORDS_PUNCT_EXPECTED[i] === x)
+        ).toBeTruthy();
+      });
+      (0, import_vitest.it)("should stream tokenize words correctly", async () => {
+        const pattern = [1, 2, 4];
+        let text = WORDS_PUNCT_TEXT;
+        const chunks = [];
+        const patternIter = Array(
+          Math.ceil(text.length / pattern.reduce((sum, num) => sum + num, 0))
+        ).fill(pattern).flat()[Symbol.iterator]();
+        for (const size of patternIter) {
+          if (!text) break;
+          chunks.push(text.slice(void 0, size));
+          text = text.slice(size);
+        }
+        const stream = tokenizerPunct.stream();
+        for (const chunk of chunks) {
+          stream.pushText(chunk);
+        }
+        stream.endInput();
+        stream.close();
+        for (const x of WORDS_PUNCT_EXPECTED) {
+          await stream.next().then((value) => {
+            if (value.value) {
+              (0, import_vitest.expect)(value.value.token).toStrictEqual(x);
+            }
+          });
+        }
+      });
+    });
+  });
+  (0, import_vitest.describe)("hyphenateWord", () => {
+    (0, import_vitest.it)("should hyphenate correctly", () => {
+      HYPHENATOR_TEXT.forEach((x, i) => {
+        (0, import_vitest.expect)((0, import_basic.hyphenateWord)(x)).toStrictEqual(HYPHENATOR_EXPECTED[i]);
+      });
+    });
+  });
+  (0, import_vitest.describe)("splitParagraphs", () => {
+    (0, import_vitest.it)("should tokenize paragraphs correctly", () => {
+      PARAGRAPH_TEST_CASES.forEach(([a, b]) => {
+        (0, import_vitest.expect)((0, import_paragraph.splitParagraphs)(a)).toStrictEqual(b);
+      });
+    });
+  });
+});
+//# sourceMappingURL=tokenizer.test.cjs.map

package/dist/tokenize/tokenizer.test.cjs.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../../src/tokenize/tokenizer.test.ts"],"sourcesContent":["// SPDX-FileCopyrightText: 2024 LiveKit, Inc.\n//\n// SPDX-License-Identifier: Apache-2.0\nimport { describe, expect, it } from 'vitest';\nimport { SentenceTokenizer, WordTokenizer, hyphenateWord } from './basic/index.js';\nimport { splitParagraphs } from './basic/paragraph.js';\n\nconst TEXT =\n  'Hi! ' +\n  'LiveKit is a platform for live audio and video applications and services. ' +\n  'R.T.C stands for Real-Time Communication... again R.T.C. ' +\n  'Mr. Theo is testing the sentence tokenizer. ' +\n  'This is a test. Another test. ' +\n  'A short sentence. ' +\n  'A longer sentence that is longer than the previous sentence. ' +\n  'f(x) = x * 2.54 + 42. ' +\n  'Hey! Hi! Hello! ';\n\nconst EXPECTED_MIN_20 = [\n  'Hi! LiveKit is a platform for live audio and video applications and services.',\n  'R.T.C stands for Real-Time Communication... again R.T.C.',\n  'Mr. Theo is testing the sentence tokenizer.',\n  'This is a test. Another test.',\n  'A short sentence. A longer sentence that is longer than the previous sentence.',\n  'f(x) = x * 2.54 + 42.',\n  'Hey! Hi! Hello!',\n];\n\nconst WORDS_TEXT = 'This is a test. Blabla another test! multiple consecutive spaces:     done';\nconst WORDS_EXPECTED = [\n  'This',\n  'is',\n  'a',\n  'test',\n  'Blabla',\n  'another',\n  'test',\n  'multiple',\n  'consecutive',\n  'spaces',\n  'done',\n];\n\nconst WORDS_PUNCT_TEXT =\n  'This is <phoneme alphabet=\"cmu-arpabet\" ph=\"AE K CH UW AH L IY\">actually</phoneme> tricky to handle.';\nconst WORDS_PUNCT_EXPECTED = [\n  'This',\n  'is',\n  '<phoneme',\n  'alphabet=\"cmu-arpabet\"',\n  'ph=\"AE',\n  'K',\n  'CH',\n  'UW',\n  'AH',\n  'L',\n  'IY\">actually</phoneme>',\n  'tricky',\n  'to',\n  'handle.',\n];\n\nconst HYPHENATOR_TEXT = ['Segment', 'expected', 'communication', 'window', 'welcome', 'bedroom'];\nconst HYPHENATOR_EXPECTED = [\n  ['Seg', 'ment'],\n  ['ex', 'pect', 'ed'],\n  ['com', 'mu', 'ni', 'ca', 'tion'],\n  ['win', 'dow'],\n  ['wel', 'come'],\n  ['bed', 'room'],\n];\n\nconst PARAGRAPH_TEST_CASES: [string, [string, number, number][]][] = [\n  ['Single paragraph.', [['Single paragraph.', 0, 17]]],\n  [\n    'Paragraph 1.\\n\\nParagraph 2.',\n    [\n      ['Paragraph 1.', 0, 12],\n      ['Paragraph 2.', 14, 26],\n    ],\n  ],\n  [\n    'Para 1.\\n\\nPara 2.\\n\\nPara 3.',\n    [\n      ['Para 1.', 0, 7],\n      ['Para 2.', 9, 16],\n      ['Para 3.', 18, 25],\n    ],\n  ],\n  ['\\n\\nParagraph with leading newlines.', [['Paragraph with leading newlines.', 2, 34]]],\n  ['Paragraph with trailing newlines.\\n\\n', [['Paragraph with trailing newlines.', 0, 33]]],\n  [\n    '\\n\\n  Paragraph with leading and trailing spaces.  \\n\\n',\n    [['Paragraph with leading and trailing spaces.', 4, 47]],\n  ],\n  [\n    'Para 1.\\n\\n\\n\\nPara 2.', // Multiple newlines between paragraphs\n    [\n      ['Para 1.', 0, 7],\n      ['Para 2.', 11, 18],\n    ],\n  ],\n  [\n    'Para 1.\\n \\n \\nPara 2.', // Newlines with spaces between paragraphs\n    [\n      ['Para 1.', 0, 7],\n      ['Para 2.', 12, 19],\n    ],\n  ],\n  [\n    '', // Empty string\n    [],\n  ],\n  [\n    '\\n\\n\\n', // Only newlines\n    [],\n  ],\n  [\n    'Line 1\\nLine 2\\nLine 3', // Single paragraph with newlines\n    [['Line 1\\nLine 2\\nLine 3', 0, 20]],\n  ],\n];\n\ndescribe('tokenizer', () => {\n  describe('SentenceTokenizer', () => {\n    const tokenizer = new SentenceTokenizer();\n\n    it('should tokenize sentences correctly', () => {\n      expect(tokenizer.tokenize(TEXT).every((x, i) => EXPECTED_MIN_20[i] === x)).toBeTruthy();\n    });\n\n    it('should stream tokenize sentences correctly', async () => {\n      const pattern = [1, 2, 4];\n      let text = TEXT;\n      const chunks = [];\n      const patternIter = Array(Math.ceil(text.length / pattern.reduce((sum, num) => sum + num, 0)))\n        .fill(pattern)\n        .flat()\n        [Symbol.iterator]();\n\n      for (const size of patternIter) {\n        if (!text) break;\n        chunks.push(text.slice(undefined, size));\n        text = text.slice(size);\n      }\n      const stream = tokenizer.stream();\n      for (const chunk of chunks) {\n        stream.pushText(chunk);\n      }\n      stream.endInput();\n      stream.close();\n\n      for (const x of EXPECTED_MIN_20) {\n        await stream.next().then((value) => {\n          if (value.value) {\n            expect(value.value.token).toStrictEqual(x);\n          }\n        });\n      }\n    });\n  });\n  describe('WordTokenizer', () => {\n    const tokenizer = new WordTokenizer();\n\n    it('should tokenize words correctly', () => {\n      expect(tokenizer.tokenize(WORDS_TEXT).every((x, i) => WORDS_EXPECTED[i] === x)).toBeTruthy();\n    });\n\n    it('should stream tokenize words correctly', async () => {\n      const pattern = [1, 2, 4];\n      let text = WORDS_TEXT;\n      const chunks = [];\n      const patternIter = Array(Math.ceil(text.length / pattern.reduce((sum, num) => sum + num, 0)))\n        .fill(pattern)\n        .flat()\n        [Symbol.iterator]();\n\n      for (const size of patternIter) {\n        if (!text) break;\n        chunks.push(text.slice(undefined, size));\n        text = text.slice(size);\n      }\n      const stream = tokenizer.stream();\n      for (const chunk of chunks) {\n        stream.pushText(chunk);\n      }\n      stream.endInput();\n      stream.close();\n\n      for (const x of WORDS_EXPECTED) {\n        await stream.next().then((value) => {\n          if (value.value) {\n            expect(value.value.token).toStrictEqual(x);\n          }\n        });\n      }\n    });\n\n    describe('punctuation handling', () => {\n      const tokenizerPunct = new WordTokenizer(false);\n\n      it('should tokenize words correctly', () => {\n        expect(\n          tokenizerPunct.tokenize(WORDS_PUNCT_TEXT).every((x, i) => WORDS_PUNCT_EXPECTED[i] === x),\n        ).toBeTruthy();\n      });\n\n      it('should stream tokenize words correctly', async () => {\n        const pattern = [1, 2, 4];\n        let text = WORDS_PUNCT_TEXT;\n        const chunks = [];\n        const patternIter = Array(\n          Math.ceil(text.length / pattern.reduce((sum, num) => sum + num, 0)),\n        )\n          .fill(pattern)\n          .flat()\n          [Symbol.iterator]();\n\n        for (const size of patternIter) {\n          if (!text) break;\n          chunks.push(text.slice(undefined, size));\n          text = text.slice(size);\n        }\n        const stream = tokenizerPunct.stream();\n        for (const chunk of chunks) {\n          stream.pushText(chunk);\n        }\n        stream.endInput();\n        stream.close();\n\n        for (const x of WORDS_PUNCT_EXPECTED) {\n          await stream.next().then((value) => {\n            if (value.value) {\n              expect(value.value.token).toStrictEqual(x);\n            }\n          });\n        }\n      });\n    });\n  });\n  describe('hyphenateWord', () => {\n    it('should hyphenate correctly', () => {\n      HYPHENATOR_TEXT.forEach((x, i) => {\n        expect(hyphenateWord(x)).toStrictEqual(HYPHENATOR_EXPECTED[i]);\n      });\n    });\n  });\n  describe('splitParagraphs', () => {\n    it('should tokenize paragraphs correctly', () => {\n      PARAGRAPH_TEST_CASES.forEach(([a, b]) => {\n        expect(splitParagraphs(a)).toStrictEqual(b);\n      });\n    });\n  });\n});\n"],"mappings":";AAGA,oBAAqC;AACrC,mBAAgE;AAChE,uBAAgC;AAEhC,MAAM,OACJ;AAUF,MAAM,kBAAkB;AAAA,EACtB;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AACF;AAEA,MAAM,aAAa;AACnB,MAAM,iBAAiB;AAAA,EACrB;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AACF;AAEA,MAAM,mBACJ;AACF,MAAM,uBAAuB;AAAA,EAC3B;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AACF;AAEA,MAAM,kBAAkB,CAAC,WAAW,YAAY,iBAAiB,UAAU,WAAW,SAAS;AAC/F,MAAM,sBAAsB;AAAA,EAC1B,CAAC,OAAO,MAAM;AAAA,EACd,CAAC,MAAM,QAAQ,IAAI;AAAA,EACnB,CAAC,OAAO,MAAM,MAAM,MAAM,MAAM;AAAA,EAChC,CAAC,OAAO,KAAK;AAAA,EACb,CAAC,OAAO,MAAM;AAAA,EACd,CAAC,OAAO,MAAM;AAChB;AAEA,MAAM,uBAA+D;AAAA,EACnE,CAAC,qBAAqB,CAAC,CAAC,qBAAqB,GAAG,EAAE,CAAC,CAAC;AAAA,EACpD;AAAA,IACE;AAAA,IACA;AAAA,MACE,CAAC,gBAAgB,GAAG,EAAE;AAAA,MACtB,CAAC,gBAAgB,IAAI,EAAE;AAAA,IACzB;AAAA,EACF;AAAA,EACA;AAAA,IACE;AAAA,IACA;AAAA,MACE,CAAC,WAAW,GAAG,CAAC;AAAA,MAChB,CAAC,WAAW,GAAG,EAAE;AAAA,MACjB,CAAC,WAAW,IAAI,EAAE;AAAA,IACpB;AAAA,EACF;AAAA,EACA,CAAC,wCAAwC,CAAC,CAAC,oCAAoC,GAAG,EAAE,CAAC,CAAC;AAAA,EACtF,CAAC,yCAAyC,CAAC,CAAC,qCAAqC,GAAG,EAAE,CAAC,CAAC;AAAA,EACxF;AAAA,IACE;AAAA,IACA,CAAC,CAAC,+CAA+C,GAAG,EAAE,CAAC;AAAA,EACzD;AAAA,EACA;AAAA,IACE;AAAA;AAAA,IACA;AAAA,MACE,CAAC,WAAW,GAAG,CAAC;AAAA,MAChB,CAAC,WAAW,IAAI,EAAE;AAAA,IACpB;AAAA,EACF;AAAA,EACA;AAAA,IACE;AAAA;AAAA,IACA;AAAA,MACE,CAAC,WAAW,GAAG,CAAC;AAAA,MAChB,CAAC,WAAW,IAAI,EAAE;AAAA,IACpB;AAAA,EACF;AAAA,EACA;AAAA,IACE;AAAA;AAAA,IACA,CAAC;AAAA,EACH;AAAA,EACA;AAAA,IACE;AAAA;AAAA,IACA,CAAC;AAAA,EACH;AAAA,EACA;AAAA,IACE;AAAA;AAAA,IACA,CAAC,CAAC,0BAA0B,GAAG,EAAE,CAAC;AAAA,EACpC;AACF;AAAA,IAEA,wBAAS,aAAa,MAAM;AAC1B,8BAAS,qBAAqB,MAAM;AAClC,UAAM,YAAY,IAAI,+BAAkB;AAExC,0BAAG,uCAAuC,MAAM;AAC9C,gCAAO,UAAU,SAAS,IAAI,EAAE,MAAM,CAAC,GAAG,MAAM,gBAAgB,CAAC,MAAM,CAAC,CAAC,EAAE,WAAW;AAAA,IACxF,CAAC;AAED,0BAAG,8CAA8C,YAAY;AAC3D,YAAM,UAAU,CAAC,GAAG,GAAG,CAAC;AACxB,UAAI,OAAO;AACX,YAAM,SAAS,CAAC;AAChB,YAAM,cAAc,MAAM,KAAK,KAAK,KAAK,SAAS,QAAQ,OAAO,CAAC,KAAK,QAAQ,MAAM,KAAK,CAAC,CAAC,CAAC,EAC1F,KAAK,OAAO,EACZ,KAAK,EACL,OAAO,QAAQ,EAAE;AAEpB,iBAAW,QAAQ,aAAa;AAC9B,YAAI,CAAC,KAAM;AACX,eAAO,KAAK,KAAK,MAAM,QAAW,IAAI,CAAC;AACvC,eAAO,KAAK,MAAM,IAAI;AAAA,MACxB;AACA,YAAM,SAAS,UAAU,OAAO;AAChC,iBAAW,SAAS,QAAQ;AAC1B,eAAO,SAAS,KAAK;AAAA,MACvB;AACA,aAAO,SAAS;AAChB,aAAO,MAAM;AAEb,iBAAW,KAAK,iBAAiB;AAC/B,cAAM,OAAO,KAAK,EAAE,KAAK,CAAC,UAAU;AAClC,cAAI,MAAM,OAAO;AACf,sCAAO,MAAM,MAAM,KAAK,EAAE,cAAc,CAAC;AAAA,UAC3C;AAAA,QACF,CAAC;AAAA,MACH;AAAA,IACF,CAAC;AAAA,EACH,CAAC;AACD,8BAAS,iBAAiB,MAAM;AAC9B,UAAM,YAAY,IAAI,2BAAc;AAEpC,0BAAG,mCAAmC,MAAM;AAC1C,gCAAO,UAAU,SAAS,UAAU,EAAE,MAAM,CAAC,GAAG,MAAM,eAAe,CAAC,MAAM,CAAC,CAAC,EAAE,WAAW;AAAA,IAC7F,CAAC;AAED,0BAAG,0CAA0C,YAAY;AACvD,YAAM,UAAU,CAAC,GAAG,GAAG,CAAC;AACxB,UAAI,OAAO;AACX,YAAM,SAAS,CAAC;AAChB,YAAM,cAAc,MAAM,KAAK,KAAK,KAAK,SAAS,QAAQ,OAAO,CAAC,KAAK,QAAQ,MAAM,KAAK,CAAC,CAAC,CAAC,EAC1F,KAAK,OAAO,EACZ,KAAK,EACL,OAAO,QAAQ,EAAE;AAEpB,iBAAW,QAAQ,aAAa;AAC9B,YAAI,CAAC,KAAM;AACX,eAAO,KAAK,KAAK,MAAM,QAAW,IAAI,CAAC;AACvC,eAAO,KAAK,MAAM,IAAI;AAAA,MACxB;AACA,YAAM,SAAS,UAAU,OAAO;AAChC,iBAAW,SAAS,QAAQ;AAC1B,eAAO,SAAS,KAAK;AAAA,MACvB;AACA,aAAO,SAAS;AAChB,aAAO,MAAM;AAEb,iBAAW,KAAK,gBAAgB;AAC9B,cAAM,OAAO,KAAK,EAAE,KAAK,CAAC,UAAU;AAClC,cAAI,MAAM,OAAO;AACf,sCAAO,MAAM,MAAM,KAAK,EAAE,cAAc,CAAC;AAAA,UAC3C;AAAA,QACF,CAAC;AAAA,MACH;AAAA,IACF,CAAC;AAED,gCAAS,wBAAwB,MAAM;AACrC,YAAM,iBAAiB,IAAI,2BAAc,KAAK;AAE9C,4BAAG,mCAAmC,MAAM;AAC1C;AAAA,UACE,eAAe,SAAS,gBAAgB,EAAE,MAAM,CAAC,GAAG,MAAM,qBAAqB,CAAC,MAAM,CAAC;AAAA,QACzF,EAAE,WAAW;AAAA,MACf,CAAC;AAED,4BAAG,0CAA0C,YAAY;AACvD,cAAM,UAAU,CAAC,GAAG,GAAG,CAAC;AACxB,YAAI,OAAO;AACX,cAAM,SAAS,CAAC;AAChB,cAAM,cAAc;AAAA,UAClB,KAAK,KAAK,KAAK,SAAS,QAAQ,OAAO,CAAC,KAAK,QAAQ,MAAM,KAAK,CAAC,CAAC;AAAA,QACpE,EACG,KAAK,OAAO,EACZ,KAAK,EACL,OAAO,QAAQ,EAAE;AAEpB,mBAAW,QAAQ,aAAa;AAC9B,cAAI,CAAC,KAAM;AACX,iBAAO,KAAK,KAAK,MAAM,QAAW,IAAI,CAAC;AACvC,iBAAO,KAAK,MAAM,IAAI;AAAA,QACxB;AACA,cAAM,SAAS,eAAe,OAAO;AACrC,mBAAW,SAAS,QAAQ;AAC1B,iBAAO,SAAS,KAAK;AAAA,QACvB;AACA,eAAO,SAAS;AAChB,eAAO,MAAM;AAEb,mBAAW,KAAK,sBAAsB;AACpC,gBAAM,OAAO,KAAK,EAAE,KAAK,CAAC,UAAU;AAClC,gBAAI,MAAM,OAAO;AACf,wCAAO,MAAM,MAAM,KAAK,EAAE,cAAc,CAAC;AAAA,YAC3C;AAAA,UACF,CAAC;AAAA,QACH;AAAA,MACF,CAAC;AAAA,IACH,CAAC;AAAA,EACH,CAAC;AACD,8BAAS,iBAAiB,MAAM;AAC9B,0BAAG,8BAA8B,MAAM;AACrC,sBAAgB,QAAQ,CAAC,GAAG,MAAM;AAChC,sCAAO,4BAAc,CAAC,CAAC,EAAE,cAAc,oBAAoB,CAAC,CAAC;AAAA,MAC/D,CAAC;AAAA,IACH,CAAC;AAAA,EACH,CAAC;AACD,8BAAS,mBAAmB,MAAM;AAChC,0BAAG,wCAAwC,MAAM;AAC/C,2BAAqB,QAAQ,CAAC,CAAC,GAAG,CAAC,MAAM;AACvC,sCAAO,kCAAgB,CAAC,CAAC,EAAE,cAAc,CAAC;AAAA,MAC5C,CAAC;AAAA,IACH,CAAC;AAAA,EACH,CAAC;AACH,CAAC;","names":[]}

package/dist/tokenize/tokenizer.test.d.ts

@@ -0,0 +1,2 @@
+export {};
+//# sourceMappingURL=tokenizer.test.d.ts.map

package/dist/tokenize/tokenizer.test.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"tokenizer.test.d.ts","sourceRoot":"","sources":["../../src/tokenize/tokenizer.test.ts"],"names":[],"mappings":""}

package/dist/tokenize/tokenizer.test.js

@@ -0,0 +1,219 @@
+import { describe, expect, it } from "vitest";
+import { SentenceTokenizer, WordTokenizer, hyphenateWord } from "./basic/index.js";
+import { splitParagraphs } from "./basic/paragraph.js";
+const TEXT = "Hi! LiveKit is a platform for live audio and video applications and services. R.T.C stands for Real-Time Communication... again R.T.C. Mr. Theo is testing the sentence tokenizer. This is a test. Another test. A short sentence. A longer sentence that is longer than the previous sentence. f(x) = x * 2.54 + 42. Hey! Hi! Hello! ";
+const EXPECTED_MIN_20 = [
+  "Hi! LiveKit is a platform for live audio and video applications and services.",
+  "R.T.C stands for Real-Time Communication... again R.T.C.",
+  "Mr. Theo is testing the sentence tokenizer.",
+  "This is a test. Another test.",
+  "A short sentence. A longer sentence that is longer than the previous sentence.",
+  "f(x) = x * 2.54 + 42.",
+  "Hey! Hi! Hello!"
+];
+const WORDS_TEXT = "This is a test. Blabla another test! multiple consecutive spaces:     done";
+const WORDS_EXPECTED = [
+  "This",
+  "is",
+  "a",
+  "test",
+  "Blabla",
+  "another",
+  "test",
+  "multiple",
+  "consecutive",
+  "spaces",
+  "done"
+];
+const WORDS_PUNCT_TEXT = 'This is <phoneme alphabet="cmu-arpabet" ph="AE K CH UW AH L IY">actually</phoneme> tricky to handle.';
+const WORDS_PUNCT_EXPECTED = [
+  "This",
+  "is",
+  "<phoneme",
+  'alphabet="cmu-arpabet"',
+  'ph="AE',
+  "K",
+  "CH",
+  "UW",
+  "AH",
+  "L",
+  'IY">actually</phoneme>',
+  "tricky",
+  "to",
+  "handle."
+];
+const HYPHENATOR_TEXT = ["Segment", "expected", "communication", "window", "welcome", "bedroom"];
+const HYPHENATOR_EXPECTED = [
+  ["Seg", "ment"],
+  ["ex", "pect", "ed"],
+  ["com", "mu", "ni", "ca", "tion"],
+  ["win", "dow"],
+  ["wel", "come"],
+  ["bed", "room"]
+];
+const PARAGRAPH_TEST_CASES = [
+  ["Single paragraph.", [["Single paragraph.", 0, 17]]],
+  [
+    "Paragraph 1.\n\nParagraph 2.",
+    [
+      ["Paragraph 1.", 0, 12],
+      ["Paragraph 2.", 14, 26]
+    ]
+  ],
+  [
+    "Para 1.\n\nPara 2.\n\nPara 3.",
+    [
+      ["Para 1.", 0, 7],
+      ["Para 2.", 9, 16],
+      ["Para 3.", 18, 25]
+    ]
+  ],
+  ["\n\nParagraph with leading newlines.", [["Paragraph with leading newlines.", 2, 34]]],
+  ["Paragraph with trailing newlines.\n\n", [["Paragraph with trailing newlines.", 0, 33]]],
+  [
+    "\n\n  Paragraph with leading and trailing spaces.  \n\n",
+    [["Paragraph with leading and trailing spaces.", 4, 47]]
+  ],
+  [
+    "Para 1.\n\n\n\nPara 2.",
+    // Multiple newlines between paragraphs
+    [
+      ["Para 1.", 0, 7],
+      ["Para 2.", 11, 18]
+    ]
+  ],
+  [
+    "Para 1.\n \n \nPara 2.",
+    // Newlines with spaces between paragraphs
+    [
+      ["Para 1.", 0, 7],
+      ["Para 2.", 12, 19]
+    ]
+  ],
+  [
+    "",
+    // Empty string
+    []
+  ],
+  [
+    "\n\n\n",
+    // Only newlines
+    []
+  ],
+  [
+    "Line 1\nLine 2\nLine 3",
+    // Single paragraph with newlines
+    [["Line 1\nLine 2\nLine 3", 0, 20]]
+  ]
+];
+describe("tokenizer", () => {
+  describe("SentenceTokenizer", () => {
+    const tokenizer = new SentenceTokenizer();
+    it("should tokenize sentences correctly", () => {
+      expect(tokenizer.tokenize(TEXT).every((x, i) => EXPECTED_MIN_20[i] === x)).toBeTruthy();
+    });
+    it("should stream tokenize sentences correctly", async () => {
+      const pattern = [1, 2, 4];
+      let text = TEXT;
+      const chunks = [];
+      const patternIter = Array(Math.ceil(text.length / pattern.reduce((sum, num) => sum + num, 0))).fill(pattern).flat()[Symbol.iterator]();
+      for (const size of patternIter) {
+        if (!text) break;
+        chunks.push(text.slice(void 0, size));
+        text = text.slice(size);
+      }
+      const stream = tokenizer.stream();
+      for (const chunk of chunks) {
+        stream.pushText(chunk);
+      }
+      stream.endInput();
+      stream.close();
+      for (const x of EXPECTED_MIN_20) {
+        await stream.next().then((value) => {
+          if (value.value) {
+            expect(value.value.token).toStrictEqual(x);
+          }
+        });
+      }
+    });
+  });
+  describe("WordTokenizer", () => {
+    const tokenizer = new WordTokenizer();
+    it("should tokenize words correctly", () => {
+      expect(tokenizer.tokenize(WORDS_TEXT).every((x, i) => WORDS_EXPECTED[i] === x)).toBeTruthy();
+    });
+    it("should stream tokenize words correctly", async () => {
+      const pattern = [1, 2, 4];
+      let text = WORDS_TEXT;
+      const chunks = [];
+      const patternIter = Array(Math.ceil(text.length / pattern.reduce((sum, num) => sum + num, 0))).fill(pattern).flat()[Symbol.iterator]();
+      for (const size of patternIter) {
+        if (!text) break;
+        chunks.push(text.slice(void 0, size));
+        text = text.slice(size);
+      }
+      const stream = tokenizer.stream();
+      for (const chunk of chunks) {
+        stream.pushText(chunk);
+      }
+      stream.endInput();
+      stream.close();
+      for (const x of WORDS_EXPECTED) {
+        await stream.next().then((value) => {
+          if (value.value) {
+            expect(value.value.token).toStrictEqual(x);
+          }
+        });
+      }
+    });
+    describe("punctuation handling", () => {
+      const tokenizerPunct = new WordTokenizer(false);
+      it("should tokenize words correctly", () => {
+        expect(
+          tokenizerPunct.tokenize(WORDS_PUNCT_TEXT).every((x, i) => WORDS_PUNCT_EXPECTED[i] === x)
+        ).toBeTruthy();
+      });
+      it("should stream tokenize words correctly", async () => {
+        const pattern = [1, 2, 4];
+        let text = WORDS_PUNCT_TEXT;
+        const chunks = [];
+        const patternIter = Array(
+          Math.ceil(text.length / pattern.reduce((sum, num) => sum + num, 0))
+        ).fill(pattern).flat()[Symbol.iterator]();
+        for (const size of patternIter) {
+          if (!text) break;
+          chunks.push(text.slice(void 0, size));
+          text = text.slice(size);
+        }
+        const stream = tokenizerPunct.stream();
+        for (const chunk of chunks) {
+          stream.pushText(chunk);
+        }
+        stream.endInput();
+        stream.close();
+        for (const x of WORDS_PUNCT_EXPECTED) {
+          await stream.next().then((value) => {
+            if (value.value) {
+              expect(value.value.token).toStrictEqual(x);
+            }
+          });
+        }
+      });
+    });
+  });
+  describe("hyphenateWord", () => {
+    it("should hyphenate correctly", () => {
+      HYPHENATOR_TEXT.forEach((x, i) => {
+        expect(hyphenateWord(x)).toStrictEqual(HYPHENATOR_EXPECTED[i]);
+      });
+    });
+  });
+  describe("splitParagraphs", () => {
+    it("should tokenize paragraphs correctly", () => {
+      PARAGRAPH_TEST_CASES.forEach(([a, b]) => {
+        expect(splitParagraphs(a)).toStrictEqual(b);
+      });
+    });
+  });
+});
+//# sourceMappingURL=tokenizer.test.js.map

package/dist/tokenize/tokenizer.test.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../../src/tokenize/tokenizer.test.ts"],"sourcesContent":["// SPDX-FileCopyrightText: 2024 LiveKit, Inc.\n//\n// SPDX-License-Identifier: Apache-2.0\nimport { describe, expect, it } from 'vitest';\nimport { SentenceTokenizer, WordTokenizer, hyphenateWord } from './basic/index.js';\nimport { splitParagraphs } from './basic/paragraph.js';\n\nconst TEXT =\n  'Hi! ' +\n  'LiveKit is a platform for live audio and video applications and services. ' +\n  'R.T.C stands for Real-Time Communication... again R.T.C. ' +\n  'Mr. Theo is testing the sentence tokenizer. ' +\n  'This is a test. Another test. ' +\n  'A short sentence. ' +\n  'A longer sentence that is longer than the previous sentence. ' +\n  'f(x) = x * 2.54 + 42. ' +\n  'Hey! Hi! Hello! ';\n\nconst EXPECTED_MIN_20 = [\n  'Hi! LiveKit is a platform for live audio and video applications and services.',\n  'R.T.C stands for Real-Time Communication... again R.T.C.',\n  'Mr. Theo is testing the sentence tokenizer.',\n  'This is a test. Another test.',\n  'A short sentence. A longer sentence that is longer than the previous sentence.',\n  'f(x) = x * 2.54 + 42.',\n  'Hey! Hi! Hello!',\n];\n\nconst WORDS_TEXT = 'This is a test. Blabla another test! multiple consecutive spaces:     done';\nconst WORDS_EXPECTED = [\n  'This',\n  'is',\n  'a',\n  'test',\n  'Blabla',\n  'another',\n  'test',\n  'multiple',\n  'consecutive',\n  'spaces',\n  'done',\n];\n\nconst WORDS_PUNCT_TEXT =\n  'This is <phoneme alphabet=\"cmu-arpabet\" ph=\"AE K CH UW AH L IY\">actually</phoneme> tricky to handle.';\nconst WORDS_PUNCT_EXPECTED = [\n  'This',\n  'is',\n  '<phoneme',\n  'alphabet=\"cmu-arpabet\"',\n  'ph=\"AE',\n  'K',\n  'CH',\n  'UW',\n  'AH',\n  'L',\n  'IY\">actually</phoneme>',\n  'tricky',\n  'to',\n  'handle.',\n];\n\nconst HYPHENATOR_TEXT = ['Segment', 'expected', 'communication', 'window', 'welcome', 'bedroom'];\nconst HYPHENATOR_EXPECTED = [\n  ['Seg', 'ment'],\n  ['ex', 'pect', 'ed'],\n  ['com', 'mu', 'ni', 'ca', 'tion'],\n  ['win', 'dow'],\n  ['wel', 'come'],\n  ['bed', 'room'],\n];\n\nconst PARAGRAPH_TEST_CASES: [string, [string, number, number][]][] = [\n  ['Single paragraph.', [['Single paragraph.', 0, 17]]],\n  [\n    'Paragraph 1.\\n\\nParagraph 2.',\n    [\n      ['Paragraph 1.', 0, 12],\n      ['Paragraph 2.', 14, 26],\n    ],\n  ],\n  [\n    'Para 1.\\n\\nPara 2.\\n\\nPara 3.',\n    [\n      ['Para 1.', 0, 7],\n      ['Para 2.', 9, 16],\n      ['Para 3.', 18, 25],\n    ],\n  ],\n  ['\\n\\nParagraph with leading newlines.', [['Paragraph with leading newlines.', 2, 34]]],\n  ['Paragraph with trailing newlines.\\n\\n', [['Paragraph with trailing newlines.', 0, 33]]],\n  [\n    '\\n\\n  Paragraph with leading and trailing spaces.  \\n\\n',\n    [['Paragraph with leading and trailing spaces.', 4, 47]],\n  ],\n  [\n    'Para 1.\\n\\n\\n\\nPara 2.', // Multiple newlines between paragraphs\n    [\n      ['Para 1.', 0, 7],\n      ['Para 2.', 11, 18],\n    ],\n  ],\n  [\n    'Para 1.\\n \\n \\nPara 2.', // Newlines with spaces between paragraphs\n    [\n      ['Para 1.', 0, 7],\n      ['Para 2.', 12, 19],\n    ],\n  ],\n  [\n    '', // Empty string\n    [],\n  ],\n  [\n    '\\n\\n\\n', // Only newlines\n    [],\n  ],\n  [\n    'Line 1\\nLine 2\\nLine 3', // Single paragraph with newlines\n    [['Line 1\\nLine 2\\nLine 3', 0, 20]],\n  ],\n];\n\ndescribe('tokenizer', () => {\n  describe('SentenceTokenizer', () => {\n    const tokenizer = new SentenceTokenizer();\n\n    it('should tokenize sentences correctly', () => {\n      expect(tokenizer.tokenize(TEXT).every((x, i) => EXPECTED_MIN_20[i] === x)).toBeTruthy();\n    });\n\n    it('should stream tokenize sentences correctly', async () => {\n      const pattern = [1, 2, 4];\n      let text = TEXT;\n      const chunks = [];\n      const patternIter = Array(Math.ceil(text.length / pattern.reduce((sum, num) => sum + num, 0)))\n        .fill(pattern)\n        .flat()\n        [Symbol.iterator]();\n\n      for (const size of patternIter) {\n        if (!text) break;\n        chunks.push(text.slice(undefined, size));\n        text = text.slice(size);\n      }\n      const stream = tokenizer.stream();\n      for (const chunk of chunks) {\n        stream.pushText(chunk);\n      }\n      stream.endInput();\n      stream.close();\n\n      for (const x of EXPECTED_MIN_20) {\n        await stream.next().then((value) => {\n          if (value.value) {\n            expect(value.value.token).toStrictEqual(x);\n          }\n        });\n      }\n    });\n  });\n  describe('WordTokenizer', () => {\n    const tokenizer = new WordTokenizer();\n\n    it('should tokenize words correctly', () => {\n      expect(tokenizer.tokenize(WORDS_TEXT).every((x, i) => WORDS_EXPECTED[i] === x)).toBeTruthy();\n    });\n\n    it('should stream tokenize words correctly', async () => {\n      const pattern = [1, 2, 4];\n      let text = WORDS_TEXT;\n      const chunks = [];\n      const patternIter = Array(Math.ceil(text.length / pattern.reduce((sum, num) => sum + num, 0)))\n        .fill(pattern)\n        .flat()\n        [Symbol.iterator]();\n\n      for (const size of patternIter) {\n        if (!text) break;\n        chunks.push(text.slice(undefined, size));\n        text = text.slice(size);\n      }\n      const stream = tokenizer.stream();\n      for (const chunk of chunks) {\n        stream.pushText(chunk);\n      }\n      stream.endInput();\n      stream.close();\n\n      for (const x of WORDS_EXPECTED) {\n        await stream.next().then((value) => {\n          if (value.value) {\n            expect(value.value.token).toStrictEqual(x);\n          }\n        });\n      }\n    });\n\n    describe('punctuation handling', () => {\n      const tokenizerPunct = new WordTokenizer(false);\n\n      it('should tokenize words correctly', () => {\n        expect(\n          tokenizerPunct.tokenize(WORDS_PUNCT_TEXT).every((x, i) => WORDS_PUNCT_EXPECTED[i] === x),\n        ).toBeTruthy();\n      });\n\n      it('should stream tokenize words correctly', async () => {\n        const pattern = [1, 2, 4];\n        let text = WORDS_PUNCT_TEXT;\n        const chunks = [];\n        const patternIter = Array(\n          Math.ceil(text.length / pattern.reduce((sum, num) => sum + num, 0)),\n        )\n          .fill(pattern)\n          .flat()\n          [Symbol.iterator]();\n\n        for (const size of patternIter) {\n          if (!text) break;\n          chunks.push(text.slice(undefined, size));\n          text = text.slice(size);\n        }\n        const stream = tokenizerPunct.stream();\n        for (const chunk of chunks) {\n          stream.pushText(chunk);\n        }\n        stream.endInput();\n        stream.close();\n\n        for (const x of WORDS_PUNCT_EXPECTED) {\n          await stream.next().then((value) => {\n            if (value.value) {\n              expect(value.value.token).toStrictEqual(x);\n            }\n          });\n        }\n      });\n    });\n  });\n  describe('hyphenateWord', () => {\n    it('should hyphenate correctly', () => {\n      HYPHENATOR_TEXT.forEach((x, i) => {\n        expect(hyphenateWord(x)).toStrictEqual(HYPHENATOR_EXPECTED[i]);\n      });\n    });\n  });\n  describe('splitParagraphs', () => {\n    it('should tokenize paragraphs correctly', () => {\n      PARAGRAPH_TEST_CASES.forEach(([a, b]) => {\n        expect(splitParagraphs(a)).toStrictEqual(b);\n      });\n    });\n  });\n});\n"],"mappings":"AAGA,SAAS,UAAU,QAAQ,UAAU;AACrC,SAAS,mBAAmB,eAAe,qBAAqB;AAChE,SAAS,uBAAuB;AAEhC,MAAM,OACJ;AAUF,MAAM,kBAAkB;AAAA,EACtB;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AACF;AAEA,MAAM,aAAa;AACnB,MAAM,iBAAiB;AAAA,EACrB;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AACF;AAEA,MAAM,mBACJ;AACF,MAAM,uBAAuB;AAAA,EAC3B;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AACF;AAEA,MAAM,kBAAkB,CAAC,WAAW,YAAY,iBAAiB,UAAU,WAAW,SAAS;AAC/F,MAAM,sBAAsB;AAAA,EAC1B,CAAC,OAAO,MAAM;AAAA,EACd,CAAC,MAAM,QAAQ,IAAI;AAAA,EACnB,CAAC,OAAO,MAAM,MAAM,MAAM,MAAM;AAAA,EAChC,CAAC,OAAO,KAAK;AAAA,EACb,CAAC,OAAO,MAAM;AAAA,EACd,CAAC,OAAO,MAAM;AAChB;AAEA,MAAM,uBAA+D;AAAA,EACnE,CAAC,qBAAqB,CAAC,CAAC,qBAAqB,GAAG,EAAE,CAAC,CAAC;AAAA,EACpD;AAAA,IACE;AAAA,IACA;AAAA,MACE,CAAC,gBAAgB,GAAG,EAAE;AAAA,MACtB,CAAC,gBAAgB,IAAI,EAAE;AAAA,IACzB;AAAA,EACF;AAAA,EACA;AAAA,IACE;AAAA,IACA;AAAA,MACE,CAAC,WAAW,GAAG,CAAC;AAAA,MAChB,CAAC,WAAW,GAAG,EAAE;AAAA,MACjB,CAAC,WAAW,IAAI,EAAE;AAAA,IACpB;AAAA,EACF;AAAA,EACA,CAAC,wCAAwC,CAAC,CAAC,oCAAoC,GAAG,EAAE,CAAC,CAAC;AAAA,EACtF,CAAC,yCAAyC,CAAC,CAAC,qCAAqC,GAAG,EAAE,CAAC,CAAC;AAAA,EACxF;AAAA,IACE;AAAA,IACA,CAAC,CAAC,+CAA+C,GAAG,EAAE,CAAC;AAAA,EACzD;AAAA,EACA;AAAA,IACE;AAAA;AAAA,IACA;AAAA,MACE,CAAC,WAAW,GAAG,CAAC;AAAA,MAChB,CAAC,WAAW,IAAI,EAAE;AAAA,IACpB;AAAA,EACF;AAAA,EACA;AAAA,IACE;AAAA;AAAA,IACA;AAAA,MACE,CAAC,WAAW,GAAG,CAAC;AAAA,MAChB,CAAC,WAAW,IAAI,EAAE;AAAA,IACpB;AAAA,EACF;AAAA,EACA;AAAA,IACE;AAAA;AAAA,IACA,CAAC;AAAA,EACH;AAAA,EACA;AAAA,IACE;AAAA;AAAA,IACA,CAAC;AAAA,EACH;AAAA,EACA;AAAA,IACE;AAAA;AAAA,IACA,CAAC,CAAC,0BAA0B,GAAG,EAAE,CAAC;AAAA,EACpC;AACF;AAEA,SAAS,aAAa,MAAM;AAC1B,WAAS,qBAAqB,MAAM;AAClC,UAAM,YAAY,IAAI,kBAAkB;AAExC,OAAG,uCAAuC,MAAM;AAC9C,aAAO,UAAU,SAAS,IAAI,EAAE,MAAM,CAAC,GAAG,MAAM,gBAAgB,CAAC,MAAM,CAAC,CAAC,EAAE,WAAW;AAAA,IACxF,CAAC;AAED,OAAG,8CAA8C,YAAY;AAC3D,YAAM,UAAU,CAAC,GAAG,GAAG,CAAC;AACxB,UAAI,OAAO;AACX,YAAM,SAAS,CAAC;AAChB,YAAM,cAAc,MAAM,KAAK,KAAK,KAAK,SAAS,QAAQ,OAAO,CAAC,KAAK,QAAQ,MAAM,KAAK,CAAC,CAAC,CAAC,EAC1F,KAAK,OAAO,EACZ,KAAK,EACL,OAAO,QAAQ,EAAE;AAEpB,iBAAW,QAAQ,aAAa;AAC9B,YAAI,CAAC,KAAM;AACX,eAAO,KAAK,KAAK,MAAM,QAAW,IAAI,CAAC;AACvC,eAAO,KAAK,MAAM,IAAI;AAAA,MACxB;AACA,YAAM,SAAS,UAAU,OAAO;AAChC,iBAAW,SAAS,QAAQ;AAC1B,eAAO,SAAS,KAAK;AAAA,MACvB;AACA,aAAO,SAAS;AAChB,aAAO,MAAM;AAEb,iBAAW,KAAK,iBAAiB;AAC/B,cAAM,OAAO,KAAK,EAAE,KAAK,CAAC,UAAU;AAClC,cAAI,MAAM,OAAO;AACf,mBAAO,MAAM,MAAM,KAAK,EAAE,cAAc,CAAC;AAAA,UAC3C;AAAA,QACF,CAAC;AAAA,MACH;AAAA,IACF,CAAC;AAAA,EACH,CAAC;AACD,WAAS,iBAAiB,MAAM;AAC9B,UAAM,YAAY,IAAI,cAAc;AAEpC,OAAG,mCAAmC,MAAM;AAC1C,aAAO,UAAU,SAAS,UAAU,EAAE,MAAM,CAAC,GAAG,MAAM,eAAe,CAAC,MAAM,CAAC,CAAC,EAAE,WAAW;AAAA,IAC7F,CAAC;AAED,OAAG,0CAA0C,YAAY;AACvD,YAAM,UAAU,CAAC,GAAG,GAAG,CAAC;AACxB,UAAI,OAAO;AACX,YAAM,SAAS,CAAC;AAChB,YAAM,cAAc,MAAM,KAAK,KAAK,KAAK,SAAS,QAAQ,OAAO,CAAC,KAAK,QAAQ,MAAM,KAAK,CAAC,CAAC,CAAC,EAC1F,KAAK,OAAO,EACZ,KAAK,EACL,OAAO,QAAQ,EAAE;AAEpB,iBAAW,QAAQ,aAAa;AAC9B,YAAI,CAAC,KAAM;AACX,eAAO,KAAK,KAAK,MAAM,QAAW,IAAI,CAAC;AACvC,eAAO,KAAK,MAAM,IAAI;AAAA,MACxB;AACA,YAAM,SAAS,UAAU,OAAO;AAChC,iBAAW,SAAS,QAAQ;AAC1B,eAAO,SAAS,KAAK;AAAA,MACvB;AACA,aAAO,SAAS;AAChB,aAAO,MAAM;AAEb,iBAAW,KAAK,gBAAgB;AAC9B,cAAM,OAAO,KAAK,EAAE,KAAK,CAAC,UAAU;AAClC,cAAI,MAAM,OAAO;AACf,mBAAO,MAAM,MAAM,KAAK,EAAE,cAAc,CAAC;AAAA,UAC3C;AAAA,QACF,CAAC;AAAA,MACH;AAAA,IACF,CAAC;AAED,aAAS,wBAAwB,MAAM;AACrC,YAAM,iBAAiB,IAAI,cAAc,KAAK;AAE9C,SAAG,mCAAmC,MAAM;AAC1C;AAAA,UACE,eAAe,SAAS,gBAAgB,EAAE,MAAM,CAAC,GAAG,MAAM,qBAAqB,CAAC,MAAM,CAAC;AAAA,QACzF,EAAE,WAAW;AAAA,MACf,CAAC;AAED,SAAG,0CAA0C,YAAY;AACvD,cAAM,UAAU,CAAC,GAAG,GAAG,CAAC;AACxB,YAAI,OAAO;AACX,cAAM,SAAS,CAAC;AAChB,cAAM,cAAc;AAAA,UAClB,KAAK,KAAK,KAAK,SAAS,QAAQ,OAAO,CAAC,KAAK,QAAQ,MAAM,KAAK,CAAC,CAAC;AAAA,QACpE,EACG,KAAK,OAAO,EACZ,KAAK,EACL,OAAO,QAAQ,EAAE;AAEpB,mBAAW,QAAQ,aAAa;AAC9B,cAAI,CAAC,KAAM;AACX,iBAAO,KAAK,KAAK,MAAM,QAAW,IAAI,CAAC;AACvC,iBAAO,KAAK,MAAM,IAAI;AAAA,QACxB;AACA,cAAM,SAAS,eAAe,OAAO;AACrC,mBAAW,SAAS,QAAQ;AAC1B,iBAAO,SAAS,KAAK;AAAA,QACvB;AACA,eAAO,SAAS;AAChB,eAAO,MAAM;AAEb,mBAAW,KAAK,sBAAsB;AACpC,gBAAM,OAAO,KAAK,EAAE,KAAK,CAAC,UAAU;AAClC,gBAAI,MAAM,OAAO;AACf,qBAAO,MAAM,MAAM,KAAK,EAAE,cAAc,CAAC;AAAA,YAC3C;AAAA,UACF,CAAC;AAAA,QACH;AAAA,MACF,CAAC;AAAA,IACH,CAAC;AAAA,EACH,CAAC;AACD,WAAS,iBAAiB,MAAM;AAC9B,OAAG,8BAA8B,MAAM;AACrC,sBAAgB,QAAQ,CAAC,GAAG,MAAM;AAChC,eAAO,cAAc,CAAC,CAAC,EAAE,cAAc,oBAAoB,CAAC,CAAC;AAAA,MAC/D,CAAC;AAAA,IACH,CAAC;AAAA,EACH,CAAC;AACD,WAAS,mBAAmB,MAAM;AAChC,OAAG,wCAAwC,MAAM;AAC/C,2BAAqB,QAAQ,CAAC,CAAC,GAAG,CAAC,MAAM;AACvC,eAAO,gBAAgB,CAAC,CAAC,EAAE,cAAc,CAAC;AAAA,MAC5C,CAAC;AAAA,IACH,CAAC;AAAA,EACH,CAAC;AACH,CAAC;","names":[]}

package/dist/worker.cjs

@@ -500,7 +500,8 @@ class Worker {
               available: true,
               participantIdentity: args.identity,
               participantName: args.name,
-              participantMetadata: args.metadata
+              participantMetadata: args.metadata,
+              participantAttributes: args.attributes
             }
           }
         })