@livekit/agents 0.0.0-20260120144724

package/dist/tokenize/basic/paragraph.cjs.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../../../src/tokenize/basic/paragraph.ts"],"sourcesContent":["// SPDX-FileCopyrightText: 2024 LiveKit, Inc.\n//\n// SPDX-License-Identifier: Apache-2.0\n\n/**\n * Split the text into paragraphs.\n */\nexport const splitParagraphs = (text: string): [string, number, number][] => {\n  const re = /\\n\\s*\\n/g;\n  const splits = Array.from(text.matchAll(re));\n\n  const paragraphs: [string, number, number][] = [];\n  let start = 0;\n\n  // no splits (single paragraph)\n  if (splits.length === 0) {\n    const stripped = text.trim();\n    if (!stripped) return paragraphs;\n\n    const start = text.indexOf(stripped);\n    return [[stripped, start, start + stripped.length]];\n  }\n\n  for (const split of splits) {\n    const end = split.index!;\n    const paragraph = text.slice(start, end).trim();\n    if (paragraph) {\n      const paragraphStart = start + text.slice(start, end).indexOf(paragraph);\n      const paragraphEnd = paragraphStart + paragraph.length;\n      paragraphs.push([paragraph, paragraphStart, paragraphEnd]);\n    }\n    start = end + split[0].length;\n  }\n\n  const lastParagraph = text.slice(start).trim();\n  if (lastParagraph) {\n    const paragraphStart = start + text.slice(start).indexOf(lastParagraph);\n    const paragraphEnd = paragraphStart + lastParagraph.length;\n    paragraphs.push([lastParagraph, paragraphStart, paragraphEnd]);\n  }\n\n  return paragraphs;\n};\n"],"mappings":";;;;;;;;;;;;;;;;;;AAAA;AAAA;AAAA;AAAA;AAAA;AAOO,MAAM,kBAAkB,CAAC,SAA6C;AAC3E,QAAM,KAAK;AACX,QAAM,SAAS,MAAM,KAAK,KAAK,SAAS,EAAE,CAAC;AAE3C,QAAM,aAAyC,CAAC;AAChD,MAAI,QAAQ;AAGZ,MAAI,OAAO,WAAW,GAAG;AACvB,UAAM,WAAW,KAAK,KAAK;AAC3B,QAAI,CAAC,SAAU,QAAO;AAEtB,UAAMA,SAAQ,KAAK,QAAQ,QAAQ;AACnC,WAAO,CAAC,CAAC,UAAUA,QAAOA,SAAQ,SAAS,MAAM,CAAC;AAAA,EACpD;AAEA,aAAW,SAAS,QAAQ;AAC1B,UAAM,MAAM,MAAM;AAClB,UAAM,YAAY,KAAK,MAAM,OAAO,GAAG,EAAE,KAAK;AAC9C,QAAI,WAAW;AACb,YAAM,iBAAiB,QAAQ,KAAK,MAAM,OAAO,GAAG,EAAE,QAAQ,SAAS;AACvE,YAAM,eAAe,iBAAiB,UAAU;AAChD,iBAAW,KAAK,CAAC,WAAW,gBAAgB,YAAY,CAAC;AAAA,IAC3D;AACA,YAAQ,MAAM,MAAM,CAAC,EAAE;AAAA,EACzB;AAEA,QAAM,gBAAgB,KAAK,MAAM,KAAK,EAAE,KAAK;AAC7C,MAAI,eAAe;AACjB,UAAM,iBAAiB,QAAQ,KAAK,MAAM,KAAK,EAAE,QAAQ,aAAa;AACtE,UAAM,eAAe,iBAAiB,cAAc;AACpD,eAAW,KAAK,CAAC,eAAe,gBAAgB,YAAY,CAAC;AAAA,EAC/D;AAEA,SAAO;AACT;","names":["start"]}

package/dist/tokenize/basic/paragraph.d.cts

@@ -0,0 +1,5 @@
+/**
+ * Split the text into paragraphs.
+ */
+export declare const splitParagraphs: (text: string) => [string, number, number][];
+//# sourceMappingURL=paragraph.d.ts.map

package/dist/tokenize/basic/paragraph.d.ts

@@ -0,0 +1,5 @@
+/**
+ * Split the text into paragraphs.
+ */
+export declare const splitParagraphs: (text: string) => [string, number, number][];
+//# sourceMappingURL=paragraph.d.ts.map

package/dist/tokenize/basic/paragraph.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"paragraph.d.ts","sourceRoot":"","sources":["../../../src/tokenize/basic/paragraph.ts"],"names":[],"mappings":"AAIA;;GAEG;AACH,eAAO,MAAM,eAAe,SAAU,MAAM,KAAG,CAAC,MAAM,EAAE,MAAM,EAAE,MAAM,CAAC,EAmCtE,CAAC"}

package/dist/tokenize/basic/paragraph.js

@@ -0,0 +1,33 @@
+const splitParagraphs = (text) => {
+  const re = /\n\s*\n/g;
+  const splits = Array.from(text.matchAll(re));
+  const paragraphs = [];
+  let start = 0;
+  if (splits.length === 0) {
+    const stripped = text.trim();
+    if (!stripped) return paragraphs;
+    const start2 = text.indexOf(stripped);
+    return [[stripped, start2, start2 + stripped.length]];
+  }
+  for (const split of splits) {
+    const end = split.index;
+    const paragraph = text.slice(start, end).trim();
+    if (paragraph) {
+      const paragraphStart = start + text.slice(start, end).indexOf(paragraph);
+      const paragraphEnd = paragraphStart + paragraph.length;
+      paragraphs.push([paragraph, paragraphStart, paragraphEnd]);
+    }
+    start = end + split[0].length;
+  }
+  const lastParagraph = text.slice(start).trim();
+  if (lastParagraph) {
+    const paragraphStart = start + text.slice(start).indexOf(lastParagraph);
+    const paragraphEnd = paragraphStart + lastParagraph.length;
+    paragraphs.push([lastParagraph, paragraphStart, paragraphEnd]);
+  }
+  return paragraphs;
+};
+export {
+  splitParagraphs
+};
+//# sourceMappingURL=paragraph.js.map

package/dist/tokenize/basic/paragraph.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../../../src/tokenize/basic/paragraph.ts"],"sourcesContent":["// SPDX-FileCopyrightText: 2024 LiveKit, Inc.\n//\n// SPDX-License-Identifier: Apache-2.0\n\n/**\n * Split the text into paragraphs.\n */\nexport const splitParagraphs = (text: string): [string, number, number][] => {\n  const re = /\\n\\s*\\n/g;\n  const splits = Array.from(text.matchAll(re));\n\n  const paragraphs: [string, number, number][] = [];\n  let start = 0;\n\n  // no splits (single paragraph)\n  if (splits.length === 0) {\n    const stripped = text.trim();\n    if (!stripped) return paragraphs;\n\n    const start = text.indexOf(stripped);\n    return [[stripped, start, start + stripped.length]];\n  }\n\n  for (const split of splits) {\n    const end = split.index!;\n    const paragraph = text.slice(start, end).trim();\n    if (paragraph) {\n      const paragraphStart = start + text.slice(start, end).indexOf(paragraph);\n      const paragraphEnd = paragraphStart + paragraph.length;\n      paragraphs.push([paragraph, paragraphStart, paragraphEnd]);\n    }\n    start = end + split[0].length;\n  }\n\n  const lastParagraph = text.slice(start).trim();\n  if (lastParagraph) {\n    const paragraphStart = start + text.slice(start).indexOf(lastParagraph);\n    const paragraphEnd = paragraphStart + lastParagraph.length;\n    paragraphs.push([lastParagraph, paragraphStart, paragraphEnd]);\n  }\n\n  return paragraphs;\n};\n"],"mappings":"AAOO,MAAM,kBAAkB,CAAC,SAA6C;AAC3E,QAAM,KAAK;AACX,QAAM,SAAS,MAAM,KAAK,KAAK,SAAS,EAAE,CAAC;AAE3C,QAAM,aAAyC,CAAC;AAChD,MAAI,QAAQ;AAGZ,MAAI,OAAO,WAAW,GAAG;AACvB,UAAM,WAAW,KAAK,KAAK;AAC3B,QAAI,CAAC,SAAU,QAAO;AAEtB,UAAMA,SAAQ,KAAK,QAAQ,QAAQ;AACnC,WAAO,CAAC,CAAC,UAAUA,QAAOA,SAAQ,SAAS,MAAM,CAAC;AAAA,EACpD;AAEA,aAAW,SAAS,QAAQ;AAC1B,UAAM,MAAM,MAAM;AAClB,UAAM,YAAY,KAAK,MAAM,OAAO,GAAG,EAAE,KAAK;AAC9C,QAAI,WAAW;AACb,YAAM,iBAAiB,QAAQ,KAAK,MAAM,OAAO,GAAG,EAAE,QAAQ,SAAS;AACvE,YAAM,eAAe,iBAAiB,UAAU;AAChD,iBAAW,KAAK,CAAC,WAAW,gBAAgB,YAAY,CAAC;AAAA,IAC3D;AACA,YAAQ,MAAM,MAAM,CAAC,EAAE;AAAA,EACzB;AAEA,QAAM,gBAAgB,KAAK,MAAM,KAAK,EAAE,KAAK;AAC7C,MAAI,eAAe;AACjB,UAAM,iBAAiB,QAAQ,KAAK,MAAM,KAAK,EAAE,QAAQ,aAAa;AACtE,UAAM,eAAe,iBAAiB,cAAc;AACpD,eAAW,KAAK,CAAC,eAAe,gBAAgB,YAAY,CAAC;AAAA,EAC/D;AAEA,SAAO;AACT;","names":["start"]}

package/dist/tokenize/basic/sentence.cjs

@@ -0,0 +1,97 @@
+"use strict";
+var __defProp = Object.defineProperty;
+var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
+var __getOwnPropNames = Object.getOwnPropertyNames;
+var __hasOwnProp = Object.prototype.hasOwnProperty;
+var __export = (target, all) => {
+  for (var name in all)
+    __defProp(target, name, { get: all[name], enumerable: true });
+};
+var __copyProps = (to, from, except, desc) => {
+  if (from && typeof from === "object" || typeof from === "function") {
+    for (let key of __getOwnPropNames(from))
+      if (!__hasOwnProp.call(to, key) && key !== except)
+        __defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
+  }
+  return to;
+};
+var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
+var sentence_exports = {};
+__export(sentence_exports, {
+  splitSentences: () => splitSentences
+});
+module.exports = __toCommonJS(sentence_exports);
+const splitSentences = (text, minLength = 20, retainFormat = false) => {
+  const alphabets = /([A-Za-z])/g;
+  const prefixes = /(Mr|St|Mrs|Ms|Dr)[.]/g;
+  const suffixes = /(Inc|Ltd|Jr|Sr|Co)/g;
+  const starters = /(Mr|Mrs|Ms|Dr|Prof|Capt|Cpt|Lt|He\s|She\s|It\s|They\s|Their\s|Our\s|We\s|But\s|However\s|That\s|This\s|Wherever)/g;
+  const acronyms = /([A-Z][.][A-Z][.](?:[A-Z][.])?)/g;
+  const websites = /[.](com|net|org|io|gov|edu|me)/g;
+  const digits = /([0-9])/g;
+  const dots = /\.{2,}/g;
+  if (retainFormat) {
+    text = text.replaceAll("\n", "<nel><stop>");
+  } else {
+    text = text.replaceAll("\n", " ");
+  }
+  text = text.replaceAll(prefixes, "$1<prd>");
+  text = text.replaceAll(websites, "<prd>$2");
+  text = text.replaceAll(new RegExp(`${digits.source}[.]${digits.source}`, "g"), "$1<prd>$2");
+  text = text.replaceAll(dots, (match) => "<prd>".repeat(match.length));
+  text = text.replaceAll("Ph.D.", "Ph<prd>D<prd>");
+  text = text.replaceAll(new RegExp(`\\s${alphabets.source}[.] `, "g"), " $1<prd> ");
+  text = text.replaceAll(new RegExp(`${acronyms.source} ${starters.source}`, "g"), "$1<stop> $2");
+  text = text.replaceAll(
+    new RegExp(`${alphabets.source}[.]${alphabets.source}[.]${alphabets.source}[.]`, "g"),
+    "$1<prd>$2<prd>$3<prd>"
+  );
+  text = text.replaceAll(
+    new RegExp(`${alphabets.source}[.]${alphabets.source}[.]`, "g"),
+    "$1<prd>$2<prd>"
+  );
+  text = text.replaceAll(
+    new RegExp(` ${suffixes.source}[.] ${starters.source}`, "g"),
+    "$1<stop> $2"
+  );
+  text = text.replaceAll(new RegExp(` ${suffixes.source}[.]`, "g"), "$1<prd>");
+  text = text.replaceAll(new RegExp(` ${alphabets.source}[.]`, "g"), "$1<prd>");
+  text = text.replaceAll(".\u201D", "\u201D.");
+  text = text.replaceAll('."', '".');
+  text = text.replaceAll('!"', '"!');
+  text = text.replaceAll('?"', '"?');
+  text = text.replaceAll(".", ".<stop>");
+  text = text.replaceAll("?", "?<stop>");
+  text = text.replaceAll("!", "!<stop>");
+  text = text.replaceAll("<prd>", ".");
+  if (retainFormat) {
+    text = text.replaceAll("<nel>", "\n");
+  }
+  const split = text.split("<stop>");
+  text = text.replaceAll("<stop>", "");
+  const sentences = [];
+  let buf = "";
+  let start = 0;
+  let end = 0;
+  const prePad = retainFormat ? "" : " ";
+  for (const match of split) {
+    const sentence = retainFormat ? match : match.trim();
+    if (!sentence) continue;
+    buf += prePad + sentence;
+    end += match.length;
+    if (buf.length > minLength) {
+      sentences.push([buf.slice(prePad.length), start, end]);
+      start = end;
+      buf = "";
+    }
+  }
+  if (buf) {
+    sentences.push([buf.slice(prePad.length), start, text.length - 1]);
+  }
+  return sentences;
+};
+// Annotate the CommonJS export names for ESM import in node:
+0 && (module.exports = {
+  splitSentences
+});
+//# sourceMappingURL=sentence.cjs.map

package/dist/tokenize/basic/sentence.cjs.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../../../src/tokenize/basic/sentence.ts"],"sourcesContent":["// SPDX-FileCopyrightText: 2024 LiveKit, Inc.\n//\n// SPDX-License-Identifier: Apache-2.0\n\n/**\n * Split the text into sentences.\n */\nexport const splitSentences = (\n  text: string,\n  minLength = 20,\n  retainFormat: boolean = false,\n): [string, number, number][] => {\n  const alphabets = /([A-Za-z])/g;\n  const prefixes = /(Mr|St|Mrs|Ms|Dr)[.]/g;\n  const suffixes = /(Inc|Ltd|Jr|Sr|Co)/g;\n  const starters =\n    /(Mr|Mrs|Ms|Dr|Prof|Capt|Cpt|Lt|He\\s|She\\s|It\\s|They\\s|Their\\s|Our\\s|We\\s|But\\s|However\\s|That\\s|This\\s|Wherever)/g;\n  const acronyms = /([A-Z][.][A-Z][.](?:[A-Z][.])?)/g;\n  const websites = /[.](com|net|org|io|gov|edu|me)/g;\n  const digits = /([0-9])/g;\n  const dots = /\\.{2,}/g;\n\n  if (retainFormat) {\n    text = text.replaceAll('\\n', '<nel><stop>');\n  } else {\n    text = text.replaceAll('\\n', ' ');\n  }\n\n  text = text.replaceAll(prefixes, '$1<prd>');\n  text = text.replaceAll(websites, '<prd>$2');\n  text = text.replaceAll(new RegExp(`${digits.source}[.]${digits.source}`, 'g'), '$1<prd>$2');\n  text = text.replaceAll(dots, (match) => '<prd>'.repeat(match.length));\n  text = text.replaceAll('Ph.D.', 'Ph<prd>D<prd>');\n  text = text.replaceAll(new RegExp(`\\\\s${alphabets.source}[.] `, 'g'), ' $1<prd> ');\n  text = text.replaceAll(new RegExp(`${acronyms.source} ${starters.source}`, 'g'), '$1<stop> $2');\n  text = text.replaceAll(\n    new RegExp(`${alphabets.source}[.]${alphabets.source}[.]${alphabets.source}[.]`, 'g'),\n    '$1<prd>$2<prd>$3<prd>',\n  );\n  text = text.replaceAll(\n    new RegExp(`${alphabets.source}[.]${alphabets.source}[.]`, 'g'),\n    '$1<prd>$2<prd>',\n  );\n  text = text.replaceAll(\n    new RegExp(` ${suffixes.source}[.] ${starters.source}`, 'g'),\n    '$1<stop> $2',\n  );\n  text = text.replaceAll(new RegExp(` ${suffixes.source}[.]`, 'g'), '$1<prd>');\n  text = text.replaceAll(new RegExp(` ${alphabets.source}[.]`, 'g'), '$1<prd>');\n  text = text.replaceAll('.”', '”.');\n  text = text.replaceAll('.\"', '\".');\n  text = text.replaceAll('!\"', '\"!');\n  text = text.replaceAll('?\"', '\"?');\n  text = text.replaceAll('.', '.<stop>');\n  text = text.replaceAll('?', '?<stop>');\n  text = text.replaceAll('!', '!<stop>');\n  text = text.replaceAll('<prd>', '.');\n\n  if (retainFormat) {\n    text = text.replaceAll('<nel>', '\\n');\n  }\n\n  const split = text.split('<stop>');\n  text = text.replaceAll('<stop>', '');\n\n  const sentences: [string, number, number][] = [];\n  let buf = '';\n  let start = 0;\n  let end = 0;\n  const prePad = retainFormat ? '' : ' ';\n  for (const match of split) {\n    const sentence = retainFormat ? match : match.trim();\n    if (!sentence) continue;\n\n    buf += prePad + sentence;\n    end += match.length;\n    if (buf.length > minLength) {\n      sentences.push([buf.slice(prePad.length), start, end]);\n      start = end;\n      buf = '';\n    }\n  }\n\n  if (buf) {\n    sentences.push([buf.slice(prePad.length), start, text.length - 1]);\n  }\n\n  return sentences;\n};\n"],"mappings":";;;;;;;;;;;;;;;;;;AAAA;AAAA;AAAA;AAAA;AAAA;AAOO,MAAM,iBAAiB,CAC5B,MACA,YAAY,IACZ,eAAwB,UACO;AAC/B,QAAM,YAAY;AAClB,QAAM,WAAW;AACjB,QAAM,WAAW;AACjB,QAAM,WACJ;AACF,QAAM,WAAW;AACjB,QAAM,WAAW;AACjB,QAAM,SAAS;AACf,QAAM,OAAO;AAEb,MAAI,cAAc;AAChB,WAAO,KAAK,WAAW,MAAM,aAAa;AAAA,EAC5C,OAAO;AACL,WAAO,KAAK,WAAW,MAAM,GAAG;AAAA,EAClC;AAEA,SAAO,KAAK,WAAW,UAAU,SAAS;AAC1C,SAAO,KAAK,WAAW,UAAU,SAAS;AAC1C,SAAO,KAAK,WAAW,IAAI,OAAO,GAAG,OAAO,MAAM,MAAM,OAAO,MAAM,IAAI,GAAG,GAAG,WAAW;AAC1F,SAAO,KAAK,WAAW,MAAM,CAAC,UAAU,QAAQ,OAAO,MAAM,MAAM,CAAC;AACpE,SAAO,KAAK,WAAW,SAAS,eAAe;AAC/C,SAAO,KAAK,WAAW,IAAI,OAAO,MAAM,UAAU,MAAM,QAAQ,GAAG,GAAG,WAAW;AACjF,SAAO,KAAK,WAAW,IAAI,OAAO,GAAG,SAAS,MAAM,IAAI,SAAS,MAAM,IAAI,GAAG,GAAG,aAAa;AAC9F,SAAO,KAAK;AAAA,IACV,IAAI,OAAO,GAAG,UAAU,MAAM,MAAM,UAAU,MAAM,MAAM,UAAU,MAAM,OAAO,GAAG;AAAA,IACpF;AAAA,EACF;AACA,SAAO,KAAK;AAAA,IACV,IAAI,OAAO,GAAG,UAAU,MAAM,MAAM,UAAU,MAAM,OAAO,GAAG;AAAA,IAC9D;AAAA,EACF;AACA,SAAO,KAAK;AAAA,IACV,IAAI,OAAO,IAAI,SAAS,MAAM,OAAO,SAAS,MAAM,IAAI,GAAG;AAAA,IAC3D;AAAA,EACF;AACA,SAAO,KAAK,WAAW,IAAI,OAAO,IAAI,SAAS,MAAM,OAAO,GAAG,GAAG,SAAS;AAC3E,SAAO,KAAK,WAAW,IAAI,OAAO,IAAI,UAAU,MAAM,OAAO,GAAG,GAAG,SAAS;AAC5E,SAAO,KAAK,WAAW,WAAM,SAAI;AACjC,SAAO,KAAK,WAAW,MAAM,IAAI;AACjC,SAAO,KAAK,WAAW,MAAM,IAAI;AACjC,SAAO,KAAK,WAAW,MAAM,IAAI;AACjC,SAAO,KAAK,WAAW,KAAK,SAAS;AACrC,SAAO,KAAK,WAAW,KAAK,SAAS;AACrC,SAAO,KAAK,WAAW,KAAK,SAAS;AACrC,SAAO,KAAK,WAAW,SAAS,GAAG;AAEnC,MAAI,cAAc;AAChB,WAAO,KAAK,WAAW,SAAS,IAAI;AAAA,EACtC;AAEA,QAAM,QAAQ,KAAK,MAAM,QAAQ;AACjC,SAAO,KAAK,WAAW,UAAU,EAAE;AAEnC,QAAM,YAAwC,CAAC;AAC/C,MAAI,MAAM;AACV,MAAI,QAAQ;AACZ,MAAI,MAAM;AACV,QAAM,SAAS,eAAe,KAAK;AACnC,aAAW,SAAS,OAAO;AACzB,UAAM,WAAW,eAAe,QAAQ,MAAM,KAAK;AACnD,QAAI,CAAC,SAAU;AAEf,WAAO,SAAS;AAChB,WAAO,MAAM;AACb,QAAI,IAAI,SAAS,WAAW;AAC1B,gBAAU,KAAK,CAAC,IAAI,MAAM,OAAO,MAAM,GAAG,OAAO,GAAG,CAAC;AACrD,cAAQ;AACR,YAAM;AAAA,IACR;AAAA,EACF;AAEA,MAAI,KAAK;AACP,cAAU,KAAK,CAAC,IAAI,MAAM,OAAO,MAAM,GAAG,OAAO,KAAK,SAAS,CAAC,CAAC;AAAA,EACnE;AAEA,SAAO;AACT;","names":[]}

package/dist/tokenize/basic/sentence.d.cts

@@ -0,0 +1,5 @@
+/**
+ * Split the text into sentences.
+ */
+export declare const splitSentences: (text: string, minLength?: number, retainFormat?: boolean) => [string, number, number][];
+//# sourceMappingURL=sentence.d.ts.map

package/dist/tokenize/basic/sentence.d.ts

@@ -0,0 +1,5 @@
+/**
+ * Split the text into sentences.
+ */
+export declare const splitSentences: (text: string, minLength?: number, retainFormat?: boolean) => [string, number, number][];
+//# sourceMappingURL=sentence.d.ts.map

package/dist/tokenize/basic/sentence.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"sentence.d.ts","sourceRoot":"","sources":["../../../src/tokenize/basic/sentence.ts"],"names":[],"mappings":"AAIA;;GAEG;AACH,eAAO,MAAM,cAAc,SACnB,MAAM,qCAEE,OAAO,KACpB,CAAC,MAAM,EAAE,MAAM,EAAE,MAAM,CAAC,EA6E1B,CAAC"}

package/dist/tokenize/basic/sentence.js

@@ -0,0 +1,73 @@
+const splitSentences = (text, minLength = 20, retainFormat = false) => {
+  const alphabets = /([A-Za-z])/g;
+  const prefixes = /(Mr|St|Mrs|Ms|Dr)[.]/g;
+  const suffixes = /(Inc|Ltd|Jr|Sr|Co)/g;
+  const starters = /(Mr|Mrs|Ms|Dr|Prof|Capt|Cpt|Lt|He\s|She\s|It\s|They\s|Their\s|Our\s|We\s|But\s|However\s|That\s|This\s|Wherever)/g;
+  const acronyms = /([A-Z][.][A-Z][.](?:[A-Z][.])?)/g;
+  const websites = /[.](com|net|org|io|gov|edu|me)/g;
+  const digits = /([0-9])/g;
+  const dots = /\.{2,}/g;
+  if (retainFormat) {
+    text = text.replaceAll("\n", "<nel><stop>");
+  } else {
+    text = text.replaceAll("\n", " ");
+  }
+  text = text.replaceAll(prefixes, "$1<prd>");
+  text = text.replaceAll(websites, "<prd>$2");
+  text = text.replaceAll(new RegExp(`${digits.source}[.]${digits.source}`, "g"), "$1<prd>$2");
+  text = text.replaceAll(dots, (match) => "<prd>".repeat(match.length));
+  text = text.replaceAll("Ph.D.", "Ph<prd>D<prd>");
+  text = text.replaceAll(new RegExp(`\\s${alphabets.source}[.] `, "g"), " $1<prd> ");
+  text = text.replaceAll(new RegExp(`${acronyms.source} ${starters.source}`, "g"), "$1<stop> $2");
+  text = text.replaceAll(
+    new RegExp(`${alphabets.source}[.]${alphabets.source}[.]${alphabets.source}[.]`, "g"),
+    "$1<prd>$2<prd>$3<prd>"
+  );
+  text = text.replaceAll(
+    new RegExp(`${alphabets.source}[.]${alphabets.source}[.]`, "g"),
+    "$1<prd>$2<prd>"
+  );
+  text = text.replaceAll(
+    new RegExp(` ${suffixes.source}[.] ${starters.source}`, "g"),
+    "$1<stop> $2"
+  );
+  text = text.replaceAll(new RegExp(` ${suffixes.source}[.]`, "g"), "$1<prd>");
+  text = text.replaceAll(new RegExp(` ${alphabets.source}[.]`, "g"), "$1<prd>");
+  text = text.replaceAll(".\u201D", "\u201D.");
+  text = text.replaceAll('."', '".');
+  text = text.replaceAll('!"', '"!');
+  text = text.replaceAll('?"', '"?');
+  text = text.replaceAll(".", ".<stop>");
+  text = text.replaceAll("?", "?<stop>");
+  text = text.replaceAll("!", "!<stop>");
+  text = text.replaceAll("<prd>", ".");
+  if (retainFormat) {
+    text = text.replaceAll("<nel>", "\n");
+  }
+  const split = text.split("<stop>");
+  text = text.replaceAll("<stop>", "");
+  const sentences = [];
+  let buf = "";
+  let start = 0;
+  let end = 0;
+  const prePad = retainFormat ? "" : " ";
+  for (const match of split) {
+    const sentence = retainFormat ? match : match.trim();
+    if (!sentence) continue;
+    buf += prePad + sentence;
+    end += match.length;
+    if (buf.length > minLength) {
+      sentences.push([buf.slice(prePad.length), start, end]);
+      start = end;
+      buf = "";
+    }
+  }
+  if (buf) {
+    sentences.push([buf.slice(prePad.length), start, text.length - 1]);
+  }
+  return sentences;
+};
+export {
+  splitSentences
+};
+//# sourceMappingURL=sentence.js.map

package/dist/tokenize/basic/sentence.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../../../src/tokenize/basic/sentence.ts"],"sourcesContent":["// SPDX-FileCopyrightText: 2024 LiveKit, Inc.\n//\n// SPDX-License-Identifier: Apache-2.0\n\n/**\n * Split the text into sentences.\n */\nexport const splitSentences = (\n  text: string,\n  minLength = 20,\n  retainFormat: boolean = false,\n): [string, number, number][] => {\n  const alphabets = /([A-Za-z])/g;\n  const prefixes = /(Mr|St|Mrs|Ms|Dr)[.]/g;\n  const suffixes = /(Inc|Ltd|Jr|Sr|Co)/g;\n  const starters =\n    /(Mr|Mrs|Ms|Dr|Prof|Capt|Cpt|Lt|He\\s|She\\s|It\\s|They\\s|Their\\s|Our\\s|We\\s|But\\s|However\\s|That\\s|This\\s|Wherever)/g;\n  const acronyms = /([A-Z][.][A-Z][.](?:[A-Z][.])?)/g;\n  const websites = /[.](com|net|org|io|gov|edu|me)/g;\n  const digits = /([0-9])/g;\n  const dots = /\\.{2,}/g;\n\n  if (retainFormat) {\n    text = text.replaceAll('\\n', '<nel><stop>');\n  } else {\n    text = text.replaceAll('\\n', ' ');\n  }\n\n  text = text.replaceAll(prefixes, '$1<prd>');\n  text = text.replaceAll(websites, '<prd>$2');\n  text = text.replaceAll(new RegExp(`${digits.source}[.]${digits.source}`, 'g'), '$1<prd>$2');\n  text = text.replaceAll(dots, (match) => '<prd>'.repeat(match.length));\n  text = text.replaceAll('Ph.D.', 'Ph<prd>D<prd>');\n  text = text.replaceAll(new RegExp(`\\\\s${alphabets.source}[.] `, 'g'), ' $1<prd> ');\n  text = text.replaceAll(new RegExp(`${acronyms.source} ${starters.source}`, 'g'), '$1<stop> $2');\n  text = text.replaceAll(\n    new RegExp(`${alphabets.source}[.]${alphabets.source}[.]${alphabets.source}[.]`, 'g'),\n    '$1<prd>$2<prd>$3<prd>',\n  );\n  text = text.replaceAll(\n    new RegExp(`${alphabets.source}[.]${alphabets.source}[.]`, 'g'),\n    '$1<prd>$2<prd>',\n  );\n  text = text.replaceAll(\n    new RegExp(` ${suffixes.source}[.] ${starters.source}`, 'g'),\n    '$1<stop> $2',\n  );\n  text = text.replaceAll(new RegExp(` ${suffixes.source}[.]`, 'g'), '$1<prd>');\n  text = text.replaceAll(new RegExp(` ${alphabets.source}[.]`, 'g'), '$1<prd>');\n  text = text.replaceAll('.”', '”.');\n  text = text.replaceAll('.\"', '\".');\n  text = text.replaceAll('!\"', '\"!');\n  text = text.replaceAll('?\"', '\"?');\n  text = text.replaceAll('.', '.<stop>');\n  text = text.replaceAll('?', '?<stop>');\n  text = text.replaceAll('!', '!<stop>');\n  text = text.replaceAll('<prd>', '.');\n\n  if (retainFormat) {\n    text = text.replaceAll('<nel>', '\\n');\n  }\n\n  const split = text.split('<stop>');\n  text = text.replaceAll('<stop>', '');\n\n  const sentences: [string, number, number][] = [];\n  let buf = '';\n  let start = 0;\n  let end = 0;\n  const prePad = retainFormat ? '' : ' ';\n  for (const match of split) {\n    const sentence = retainFormat ? match : match.trim();\n    if (!sentence) continue;\n\n    buf += prePad + sentence;\n    end += match.length;\n    if (buf.length > minLength) {\n      sentences.push([buf.slice(prePad.length), start, end]);\n      start = end;\n      buf = '';\n    }\n  }\n\n  if (buf) {\n    sentences.push([buf.slice(prePad.length), start, text.length - 1]);\n  }\n\n  return sentences;\n};\n"],"mappings":"AAOO,MAAM,iBAAiB,CAC5B,MACA,YAAY,IACZ,eAAwB,UACO;AAC/B,QAAM,YAAY;AAClB,QAAM,WAAW;AACjB,QAAM,WAAW;AACjB,QAAM,WACJ;AACF,QAAM,WAAW;AACjB,QAAM,WAAW;AACjB,QAAM,SAAS;AACf,QAAM,OAAO;AAEb,MAAI,cAAc;AAChB,WAAO,KAAK,WAAW,MAAM,aAAa;AAAA,EAC5C,OAAO;AACL,WAAO,KAAK,WAAW,MAAM,GAAG;AAAA,EAClC;AAEA,SAAO,KAAK,WAAW,UAAU,SAAS;AAC1C,SAAO,KAAK,WAAW,UAAU,SAAS;AAC1C,SAAO,KAAK,WAAW,IAAI,OAAO,GAAG,OAAO,MAAM,MAAM,OAAO,MAAM,IAAI,GAAG,GAAG,WAAW;AAC1F,SAAO,KAAK,WAAW,MAAM,CAAC,UAAU,QAAQ,OAAO,MAAM,MAAM,CAAC;AACpE,SAAO,KAAK,WAAW,SAAS,eAAe;AAC/C,SAAO,KAAK,WAAW,IAAI,OAAO,MAAM,UAAU,MAAM,QAAQ,GAAG,GAAG,WAAW;AACjF,SAAO,KAAK,WAAW,IAAI,OAAO,GAAG,SAAS,MAAM,IAAI,SAAS,MAAM,IAAI,GAAG,GAAG,aAAa;AAC9F,SAAO,KAAK;AAAA,IACV,IAAI,OAAO,GAAG,UAAU,MAAM,MAAM,UAAU,MAAM,MAAM,UAAU,MAAM,OAAO,GAAG;AAAA,IACpF;AAAA,EACF;AACA,SAAO,KAAK;AAAA,IACV,IAAI,OAAO,GAAG,UAAU,MAAM,MAAM,UAAU,MAAM,OAAO,GAAG;AAAA,IAC9D;AAAA,EACF;AACA,SAAO,KAAK;AAAA,IACV,IAAI,OAAO,IAAI,SAAS,MAAM,OAAO,SAAS,MAAM,IAAI,GAAG;AAAA,IAC3D;AAAA,EACF;AACA,SAAO,KAAK,WAAW,IAAI,OAAO,IAAI,SAAS,MAAM,OAAO,GAAG,GAAG,SAAS;AAC3E,SAAO,KAAK,WAAW,IAAI,OAAO,IAAI,UAAU,MAAM,OAAO,GAAG,GAAG,SAAS;AAC5E,SAAO,KAAK,WAAW,WAAM,SAAI;AACjC,SAAO,KAAK,WAAW,MAAM,IAAI;AACjC,SAAO,KAAK,WAAW,MAAM,IAAI;AACjC,SAAO,KAAK,WAAW,MAAM,IAAI;AACjC,SAAO,KAAK,WAAW,KAAK,SAAS;AACrC,SAAO,KAAK,WAAW,KAAK,SAAS;AACrC,SAAO,KAAK,WAAW,KAAK,SAAS;AACrC,SAAO,KAAK,WAAW,SAAS,GAAG;AAEnC,MAAI,cAAc;AAChB,WAAO,KAAK,WAAW,SAAS,IAAI;AAAA,EACtC;AAEA,QAAM,QAAQ,KAAK,MAAM,QAAQ;AACjC,SAAO,KAAK,WAAW,UAAU,EAAE;AAEnC,QAAM,YAAwC,CAAC;AAC/C,MAAI,MAAM;AACV,MAAI,QAAQ;AACZ,MAAI,MAAM;AACV,QAAM,SAAS,eAAe,KAAK;AACnC,aAAW,SAAS,OAAO;AACzB,UAAM,WAAW,eAAe,QAAQ,MAAM,KAAK;AACnD,QAAI,CAAC,SAAU;AAEf,WAAO,SAAS;AAChB,WAAO,MAAM;AACb,QAAI,IAAI,SAAS,WAAW;AAC1B,gBAAU,KAAK,CAAC,IAAI,MAAM,OAAO,MAAM,GAAG,OAAO,GAAG,CAAC;AACrD,cAAQ;AACR,YAAM;AAAA,IACR;AAAA,EACF;AAEA,MAAI,KAAK;AACP,cAAU,KAAK,CAAC,IAAI,MAAM,OAAO,MAAM,GAAG,OAAO,KAAK,SAAS,CAAC,CAAC;AAAA,EACnE;AAEA,SAAO;AACT;","names":[]}

package/dist/tokenize/basic/word.cjs

@@ -0,0 +1,44 @@
+"use strict";
+var __defProp = Object.defineProperty;
+var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
+var __getOwnPropNames = Object.getOwnPropertyNames;
+var __hasOwnProp = Object.prototype.hasOwnProperty;
+var __export = (target, all) => {
+  for (var name in all)
+    __defProp(target, name, { get: all[name], enumerable: true });
+};
+var __copyProps = (to, from, except, desc) => {
+  if (from && typeof from === "object" || typeof from === "function") {
+    for (let key of __getOwnPropNames(from))
+      if (!__hasOwnProp.call(to, key) && key !== except)
+        __defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
+  }
+  return to;
+};
+var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
+var word_exports = {};
+__export(word_exports, {
+  splitWords: () => splitWords
+});
+module.exports = __toCommonJS(word_exports);
+var import_tokenizer = require("../tokenizer.cjs");
+const splitWords = (text, ignorePunctuation = true) => {
+  const re = /\S+/g;
+  const words = [];
+  let arr;
+  while ((arr = re.exec(text)) !== null) {
+    let word = arr[0];
+    const start = arr.index;
+    const end = start + word.length;
+    if (ignorePunctuation) {
+      word = word.replace(new RegExp(`[${import_tokenizer.PUNCTUATIONS.join("")}]`, "g"), "");
+    }
+    words.push([word, start, end]);
+  }
+  return words;
+};
+// Annotate the CommonJS export names for ESM import in node:
+0 && (module.exports = {
+  splitWords
+});
+//# sourceMappingURL=word.cjs.map

package/dist/tokenize/basic/word.cjs.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../../../src/tokenize/basic/word.ts"],"sourcesContent":["// SPDX-FileCopyrightText: 2024 LiveKit, Inc.\n//\n// SPDX-License-Identifier: Apache-2.0\nimport { PUNCTUATIONS } from '../tokenizer.js';\n\n/**\n * Split the text into words.\n */\nexport const splitWords = (text: string, ignorePunctuation = true): [string, number, number][] => {\n  const re = /\\S+/g;\n  const words: [string, number, number][] = [];\n\n  let arr;\n  while ((arr = re.exec(text)) !== null) {\n    let word = arr[0];\n    const start = arr.index;\n    const end = start + word.length;\n\n    if (ignorePunctuation) {\n      word = word.replace(new RegExp(`[${PUNCTUATIONS.join('')}]`, 'g'), '');\n    }\n\n    words.push([word, start, end]);\n  }\n\n  return words;\n};\n"],"mappings":";;;;;;;;;;;;;;;;;;AAAA;AAAA;AAAA;AAAA;AAAA;AAGA,uBAA6B;AAKtB,MAAM,aAAa,CAAC,MAAc,oBAAoB,SAAqC;AAChG,QAAM,KAAK;AACX,QAAM,QAAoC,CAAC;AAE3C,MAAI;AACJ,UAAQ,MAAM,GAAG,KAAK,IAAI,OAAO,MAAM;AACrC,QAAI,OAAO,IAAI,CAAC;AAChB,UAAM,QAAQ,IAAI;AAClB,UAAM,MAAM,QAAQ,KAAK;AAEzB,QAAI,mBAAmB;AACrB,aAAO,KAAK,QAAQ,IAAI,OAAO,IAAI,8BAAa,KAAK,EAAE,CAAC,KAAK,GAAG,GAAG,EAAE;AAAA,IACvE;AAEA,UAAM,KAAK,CAAC,MAAM,OAAO,GAAG,CAAC;AAAA,EAC/B;AAEA,SAAO;AACT;","names":[]}

package/dist/tokenize/basic/word.d.cts

@@ -0,0 +1,5 @@
+/**
+ * Split the text into words.
+ */
+export declare const splitWords: (text: string, ignorePunctuation?: boolean) => [string, number, number][];
+//# sourceMappingURL=word.d.ts.map

package/dist/tokenize/basic/word.d.ts

@@ -0,0 +1,5 @@
+/**
+ * Split the text into words.
+ */
+export declare const splitWords: (text: string, ignorePunctuation?: boolean) => [string, number, number][];
+//# sourceMappingURL=word.d.ts.map

package/dist/tokenize/basic/word.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"word.d.ts","sourceRoot":"","sources":["../../../src/tokenize/basic/word.ts"],"names":[],"mappings":"AAKA;;GAEG;AACH,eAAO,MAAM,UAAU,SAAU,MAAM,kCAA6B,CAAC,MAAM,EAAE,MAAM,EAAE,MAAM,CAAC,EAkB3F,CAAC"}

package/dist/tokenize/basic/word.js

@@ -0,0 +1,20 @@
+import { PUNCTUATIONS } from "../tokenizer.js";
+const splitWords = (text, ignorePunctuation = true) => {
+  const re = /\S+/g;
+  const words = [];
+  let arr;
+  while ((arr = re.exec(text)) !== null) {
+    let word = arr[0];
+    const start = arr.index;
+    const end = start + word.length;
+    if (ignorePunctuation) {
+      word = word.replace(new RegExp(`[${PUNCTUATIONS.join("")}]`, "g"), "");
+    }
+    words.push([word, start, end]);
+  }
+  return words;
+};
+export {
+  splitWords
+};
+//# sourceMappingURL=word.js.map

package/dist/tokenize/basic/word.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../../../src/tokenize/basic/word.ts"],"sourcesContent":["// SPDX-FileCopyrightText: 2024 LiveKit, Inc.\n//\n// SPDX-License-Identifier: Apache-2.0\nimport { PUNCTUATIONS } from '../tokenizer.js';\n\n/**\n * Split the text into words.\n */\nexport const splitWords = (text: string, ignorePunctuation = true): [string, number, number][] => {\n  const re = /\\S+/g;\n  const words: [string, number, number][] = [];\n\n  let arr;\n  while ((arr = re.exec(text)) !== null) {\n    let word = arr[0];\n    const start = arr.index;\n    const end = start + word.length;\n\n    if (ignorePunctuation) {\n      word = word.replace(new RegExp(`[${PUNCTUATIONS.join('')}]`, 'g'), '');\n    }\n\n    words.push([word, start, end]);\n  }\n\n  return words;\n};\n"],"mappings":"AAGA,SAAS,oBAAoB;AAKtB,MAAM,aAAa,CAAC,MAAc,oBAAoB,SAAqC;AAChG,QAAM,KAAK;AACX,QAAM,QAAoC,CAAC;AAE3C,MAAI;AACJ,UAAQ,MAAM,GAAG,KAAK,IAAI,OAAO,MAAM;AACrC,QAAI,OAAO,IAAI,CAAC;AAChB,UAAM,QAAQ,IAAI;AAClB,UAAM,MAAM,QAAQ,KAAK;AAEzB,QAAI,mBAAmB;AACrB,aAAO,KAAK,QAAQ,IAAI,OAAO,IAAI,aAAa,KAAK,EAAE,CAAC,KAAK,GAAG,GAAG,EAAE;AAAA,IACvE;AAEA,UAAM,KAAK,CAAC,MAAM,OAAO,GAAG,CAAC;AAAA,EAC/B;AAEA,SAAO;AACT;","names":[]}

package/dist/tokenize/index.cjs

@@ -0,0 +1,55 @@
+"use strict";
+var __create = Object.create;
+var __defProp = Object.defineProperty;
+var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
+var __getOwnPropNames = Object.getOwnPropertyNames;
+var __getProtoOf = Object.getPrototypeOf;
+var __hasOwnProp = Object.prototype.hasOwnProperty;
+var __export = (target, all) => {
+  for (var name in all)
+    __defProp(target, name, { get: all[name], enumerable: true });
+};
+var __copyProps = (to, from, except, desc) => {
+  if (from && typeof from === "object" || typeof from === "function") {
+    for (let key of __getOwnPropNames(from))
+      if (!__hasOwnProp.call(to, key) && key !== except)
+        __defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
+  }
+  return to;
+};
+var __toESM = (mod, isNodeMode, target) => (target = mod != null ? __create(__getProtoOf(mod)) : {}, __copyProps(
+  // If the importer is in node compatibility mode or this is not an ESM
+  // file that has been converted to a CommonJS file using a Babel-
+  // compatible transform (i.e. "__esModule" has not been set), then set
+  // "default" to the CommonJS "module.exports" for node compatibility.
+  isNodeMode || !mod || !mod.__esModule ? __defProp(target, "default", { value: mod, enumerable: true }) : target,
+  mod
+));
+var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
+var tokenize_exports = {};
+__export(tokenize_exports, {
+  BufferedSentenceStream: () => import_token_stream.BufferedSentenceStream,
+  BufferedTokenStream: () => import_token_stream.BufferedTokenStream,
+  BufferedWordStream: () => import_token_stream.BufferedWordStream,
+  SentenceStream: () => import_tokenizer.SentenceStream,
+  SentenceTokenizer: () => import_tokenizer.SentenceTokenizer,
+  WordStream: () => import_tokenizer.WordStream,
+  WordTokenizer: () => import_tokenizer.WordTokenizer,
+  basic: () => basic
+});
+module.exports = __toCommonJS(tokenize_exports);
+var basic = __toESM(require("./basic/index.cjs"), 1);
+var import_tokenizer = require("./tokenizer.cjs");
+var import_token_stream = require("./token_stream.cjs");
+// Annotate the CommonJS export names for ESM import in node:
+0 && (module.exports = {
+  BufferedSentenceStream,
+  BufferedTokenStream,
+  BufferedWordStream,
+  SentenceStream,
+  SentenceTokenizer,
+  WordStream,
+  WordTokenizer,
+  basic
+});
+//# sourceMappingURL=index.cjs.map

package/dist/tokenize/index.cjs.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../../src/tokenize/index.ts"],"sourcesContent":["// SPDX-FileCopyrightText: 2024 LiveKit, Inc.\n//\n// SPDX-License-Identifier: Apache-2.0\nimport * as basic from './basic/index.js';\n\nexport {\n  type TokenData,\n  SentenceTokenizer,\n  SentenceStream,\n  WordTokenizer,\n  WordStream,\n} from './tokenizer.js';\n\nexport { BufferedSentenceStream, BufferedTokenStream, BufferedWordStream } from './token_stream.js';\n\nexport { basic };\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAGA,YAAuB;AAEvB,uBAMO;AAEP,0BAAgF;","names":[]}

package/dist/tokenize/index.d.cts

@@ -0,0 +1,5 @@
+import * as basic from './basic/index.js';
+export { type TokenData, SentenceTokenizer, SentenceStream, WordTokenizer, WordStream, } from './tokenizer.js';
+export { BufferedSentenceStream, BufferedTokenStream, BufferedWordStream } from './token_stream.js';
+export { basic };
+//# sourceMappingURL=index.d.ts.map

package/dist/tokenize/index.d.ts

@@ -0,0 +1,5 @@
+import * as basic from './basic/index.js';
+export { type TokenData, SentenceTokenizer, SentenceStream, WordTokenizer, WordStream, } from './tokenizer.js';
+export { BufferedSentenceStream, BufferedTokenStream, BufferedWordStream } from './token_stream.js';
+export { basic };
+//# sourceMappingURL=index.d.ts.map

package/dist/tokenize/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/tokenize/index.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,KAAK,MAAM,kBAAkB,CAAC;AAE1C,OAAO,EACL,KAAK,SAAS,EACd,iBAAiB,EACjB,cAAc,EACd,aAAa,EACb,UAAU,GACX,MAAM,gBAAgB,CAAC;AAExB,OAAO,EAAE,sBAAsB,EAAE,mBAAmB,EAAE,kBAAkB,EAAE,MAAM,mBAAmB,CAAC;AAEpG,OAAO,EAAE,KAAK,EAAE,CAAC"}

package/dist/tokenize/index.js

@@ -0,0 +1,19 @@
+import * as basic from "./basic/index.js";
+import {
+  SentenceTokenizer,
+  SentenceStream,
+  WordTokenizer,
+  WordStream
+} from "./tokenizer.js";
+import { BufferedSentenceStream, BufferedTokenStream, BufferedWordStream } from "./token_stream.js";
+export {
+  BufferedSentenceStream,
+  BufferedTokenStream,
+  BufferedWordStream,
+  SentenceStream,
+  SentenceTokenizer,
+  WordStream,
+  WordTokenizer,
+  basic
+};
+//# sourceMappingURL=index.js.map

package/dist/tokenize/index.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../../src/tokenize/index.ts"],"sourcesContent":["// SPDX-FileCopyrightText: 2024 LiveKit, Inc.\n//\n// SPDX-License-Identifier: Apache-2.0\nimport * as basic from './basic/index.js';\n\nexport {\n  type TokenData,\n  SentenceTokenizer,\n  SentenceStream,\n  WordTokenizer,\n  WordStream,\n} from './tokenizer.js';\n\nexport { BufferedSentenceStream, BufferedTokenStream, BufferedWordStream } from './token_stream.js';\n\nexport { basic };\n"],"mappings":"AAGA,YAAY,WAAW;AAEvB;AAAA,EAEE;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,OACK;AAEP,SAAS,wBAAwB,qBAAqB,0BAA0B;","names":[]}

package/dist/tokenize/token_stream.cjs

@@ -0,0 +1,168 @@
+"use strict";
+var __defProp = Object.defineProperty;
+var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
+var __getOwnPropNames = Object.getOwnPropertyNames;
+var __hasOwnProp = Object.prototype.hasOwnProperty;
+var __export = (target, all) => {
+  for (var name in all)
+    __defProp(target, name, { get: all[name], enumerable: true });
+};
+var __copyProps = (to, from, except, desc) => {
+  if (from && typeof from === "object" || typeof from === "function") {
+    for (let key of __getOwnPropNames(from))
+      if (!__hasOwnProp.call(to, key) && key !== except)
+        __defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
+  }
+  return to;
+};
+var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
+var token_stream_exports = {};
+__export(token_stream_exports, {
+  BufferedSentenceStream: () => BufferedSentenceStream,
+  BufferedTokenStream: () => BufferedTokenStream,
+  BufferedWordStream: () => BufferedWordStream
+});
+module.exports = __toCommonJS(token_stream_exports);
+var import_utils = require("../utils.cjs");
+var import_tokenizer = require("./tokenizer.cjs");
+class BufferedTokenStream {
+  queue = new import_utils.AsyncIterableQueue();
+  closed = false;
+  #func;
+  #minTokenLength;
+  #minContextLength;
+  #bufTokens = [];
+  #inBuf = "";
+  #outBuf = "";
+  #currentSegmentId;
+  constructor(func, minTokenLength, minContextLength) {
+    this.#func = func;
+    this.#minTokenLength = minTokenLength;
+    this.#minContextLength = minContextLength;
+    this.#currentSegmentId = (0, import_utils.shortuuid)();
+  }
+  /** Push a string of text into the token stream */
+  pushText(text) {
+    if (this.closed) {
+      throw new Error("Stream is closed");
+    }
+    this.#inBuf += text;
+    if (this.#inBuf.length < this.#minContextLength) return;
+    while (true) {
+      const tokens = this.#func(this.#inBuf);
+      if (tokens.length <= 1) break;
+      if (this.#outBuf) this.#outBuf += " ";
+      const tok = tokens.shift();
+      let tokText;
+      if (Array.isArray(tok)) {
+        tokText = tok[0];
+      } else {
+        tokText = tok;
+      }
+      this.#outBuf += tokText;
+      if (this.#outBuf.length >= this.#minTokenLength) {
+        this.queue.put({ token: this.#outBuf, segmentId: this.#currentSegmentId });
+        this.#outBuf = "";
+      }
+      if (typeof tok !== "string") {
+        this.#inBuf = this.#inBuf.slice(tok[2]);
+      } else {
+        this.#inBuf = this.#inBuf.slice(Math.max(0, this.#inBuf.indexOf(tok)) + tok.length).trimStart();
+      }
+    }
+  }
+  /** Flush the stream, causing it to process all pending text */
+  flush() {
+    if (this.closed) {
+      throw new Error("Stream is closed");
+    }
+    if (this.#inBuf || this.#outBuf) {
+      const tokens = this.#func(this.#inBuf);
+      if (tokens) {
+        if (this.#outBuf) this.#outBuf += " ";
+        if (Array.isArray(tokens[0])) {
+          this.#outBuf += tokens.map((tok) => tok[0]).join(" ");
+        } else {
+          this.#outBuf += tokens.join(" ");
+        }
+      }
+      if (this.#outBuf) {
+        this.queue.put({ token: this.#outBuf, segmentId: this.#currentSegmentId });
+      }
+      this.#currentSegmentId = (0, import_utils.shortuuid)();
+    }
+    this.#inBuf = "";
+    this.#outBuf = "";
+  }
+  /** Mark the input as ended and forbid additional pushes */
+  endInput() {
+    if (this.closed) {
+      throw new Error("Stream is closed");
+    }
+    this.flush();
+    this.close();
+  }
+  next() {
+    return this.queue.next();
+  }
+  /** Close both the input and output of the token stream */
+  close() {
+    this.queue.close();
+    this.closed = true;
+  }
+  [Symbol.asyncIterator]() {
+    return this;
+  }
+}
+class BufferedSentenceStream extends import_tokenizer.SentenceStream {
+  #stream;
+  constructor(func, minTokenLength, minContextLength) {
+    super();
+    this.#stream = new BufferedTokenStream(func, minTokenLength, minContextLength);
+  }
+  pushText(text) {
+    this.#stream.pushText(text);
+  }
+  flush() {
+    this.#stream.flush();
+  }
+  close() {
+    super.close();
+    this.#stream.close();
+  }
+  endInput() {
+    this.#stream.endInput();
+  }
+  next() {
+    return this.#stream.next();
+  }
+}
+class BufferedWordStream extends import_tokenizer.WordStream {
+  #stream;
+  constructor(func, minTokenLength, minContextLength) {
+    super();
+    this.#stream = new BufferedTokenStream(func, minTokenLength, minContextLength);
+  }
+  pushText(text) {
+    this.#stream.pushText(text);
+  }
+  flush() {
+    this.#stream.flush();
+  }
+  endInput() {
+    this.#stream.endInput();
+  }
+  close() {
+    this.#stream.close();
+  }
+  next() {
+    return this.#stream.next();
+  }
+}
+// Annotate the CommonJS export names for ESM import in node:
+0 && (module.exports = {
+  BufferedSentenceStream,
+  BufferedTokenStream,
+  BufferedWordStream
+});
+//# sourceMappingURL=token_stream.cjs.map