@livekit/agents 1.1.0 → 1.2.1

package/dist/telemetry/utils.js

@@ -0,0 +1,51 @@
+import { SpanStatusCode, context as otelContext, trace } from "@opentelemetry/api";
+import * as traceTypes from "./trace_types.js";
+import { tracer } from "./traces.js";
+function recordException(span, error) {
+  span.recordException(error);
+  span.setStatus({
+    code: SpanStatusCode.ERROR,
+    message: error.message
+  });
+  span.setAttributes({
+    [traceTypes.ATTR_EXCEPTION_TYPE]: error.constructor.name,
+    [traceTypes.ATTR_EXCEPTION_MESSAGE]: error.message,
+    [traceTypes.ATTR_EXCEPTION_TRACE]: error.stack || ""
+  });
+}
+function recordRealtimeMetrics(span, metrics) {
+  const attrs = {
+    [traceTypes.ATTR_GEN_AI_REQUEST_MODEL]: metrics.label || "unknown",
+    [traceTypes.ATTR_REALTIME_MODEL_METRICS]: JSON.stringify(metrics),
+    [traceTypes.ATTR_GEN_AI_USAGE_INPUT_TOKENS]: metrics.inputTokens,
+    [traceTypes.ATTR_GEN_AI_USAGE_OUTPUT_TOKENS]: metrics.outputTokens,
+    [traceTypes.ATTR_GEN_AI_USAGE_INPUT_TEXT_TOKENS]: metrics.inputTokenDetails.textTokens,
+    [traceTypes.ATTR_GEN_AI_USAGE_INPUT_AUDIO_TOKENS]: metrics.inputTokenDetails.audioTokens,
+    [traceTypes.ATTR_GEN_AI_USAGE_INPUT_CACHED_TOKENS]: metrics.inputTokenDetails.cachedTokens,
+    [traceTypes.ATTR_GEN_AI_USAGE_OUTPUT_TEXT_TOKENS]: metrics.outputTokenDetails.textTokens,
+    [traceTypes.ATTR_GEN_AI_USAGE_OUTPUT_AUDIO_TOKENS]: metrics.outputTokenDetails.audioTokens
+  };
+  if (metrics.ttftMs !== void 0 && metrics.ttftMs !== -1) {
+    const completionStartTime = metrics.timestamp + metrics.ttftMs;
+    const completionStartTimeUtc = new Date(completionStartTime).toISOString();
+    attrs[traceTypes.ATTR_LANGFUSE_COMPLETION_START_TIME] = completionStartTimeUtc;
+  }
+  if (span.isRecording()) {
+    span.setAttributes(attrs);
+  } else {
+    const currentContext = otelContext.active();
+    const spanContext = trace.setSpan(currentContext, span);
+    tracer.getTracer().startActiveSpan("realtime_metrics", {}, spanContext, (child) => {
+      try {
+        child.setAttributes(attrs);
+      } finally {
+        child.end();
+      }
+    });
+  }
+}
+export {
+  recordException,
+  recordRealtimeMetrics
+};
+//# sourceMappingURL=utils.js.map

package/dist/telemetry/utils.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../../src/telemetry/utils.ts"],"sourcesContent":["// SPDX-FileCopyrightText: 2025 LiveKit, Inc.\n//\n// SPDX-License-Identifier: Apache-2.0\nimport { type Span, SpanStatusCode, context as otelContext, trace } from '@opentelemetry/api';\nimport type { RealtimeModelMetrics } from '../metrics/base.js';\nimport * as traceTypes from './trace_types.js';\nimport { tracer } from './traces.js';\n\nexport function recordException(span: Span, error: Error): void {\n  span.recordException(error);\n  span.setStatus({\n    code: SpanStatusCode.ERROR,\n    message: error.message,\n  });\n\n  // Set exception attributes for better visibility\n  // (in case the exception event is not rendered by the backend)\n  span.setAttributes({\n    [traceTypes.ATTR_EXCEPTION_TYPE]: error.constructor.name,\n    [traceTypes.ATTR_EXCEPTION_MESSAGE]: error.message,\n    [traceTypes.ATTR_EXCEPTION_TRACE]: error.stack || '',\n  });\n}\n\nexport function recordRealtimeMetrics(span: Span, metrics: RealtimeModelMetrics): void {\n  const attrs: Record<string, string | number> = {\n    [traceTypes.ATTR_GEN_AI_REQUEST_MODEL]: metrics.label || 'unknown',\n    [traceTypes.ATTR_REALTIME_MODEL_METRICS]: JSON.stringify(metrics),\n    [traceTypes.ATTR_GEN_AI_USAGE_INPUT_TOKENS]: metrics.inputTokens,\n    [traceTypes.ATTR_GEN_AI_USAGE_OUTPUT_TOKENS]: metrics.outputTokens,\n    [traceTypes.ATTR_GEN_AI_USAGE_INPUT_TEXT_TOKENS]: metrics.inputTokenDetails.textTokens,\n    [traceTypes.ATTR_GEN_AI_USAGE_INPUT_AUDIO_TOKENS]: metrics.inputTokenDetails.audioTokens,\n    [traceTypes.ATTR_GEN_AI_USAGE_INPUT_CACHED_TOKENS]: metrics.inputTokenDetails.cachedTokens,\n    [traceTypes.ATTR_GEN_AI_USAGE_OUTPUT_TEXT_TOKENS]: metrics.outputTokenDetails.textTokens,\n    [traceTypes.ATTR_GEN_AI_USAGE_OUTPUT_AUDIO_TOKENS]: metrics.outputTokenDetails.audioTokens,\n  };\n\n  // Add LangFuse-specific completion start time if TTFT is available\n  if (metrics.ttftMs !== undefined && metrics.ttftMs !== -1) {\n    const completionStartTime = metrics.timestamp + metrics.ttftMs;\n    // Convert to UTC ISO string for LangFuse compatibility\n    const completionStartTimeUtc = new Date(completionStartTime).toISOString();\n    attrs[traceTypes.ATTR_LANGFUSE_COMPLETION_START_TIME] = completionStartTimeUtc;\n  }\n\n  if (span.isRecording()) {\n    span.setAttributes(attrs);\n  } else {\n    const currentContext = otelContext.active();\n    const spanContext = trace.setSpan(currentContext, span);\n\n    // Create a dedicated child span for orphaned metrics\n    tracer.getTracer().startActiveSpan('realtime_metrics', {}, spanContext, (child) => {\n      try {\n        child.setAttributes(attrs);\n      } finally {\n        child.end();\n      }\n    });\n  }\n}\n"],"mappings":"AAGA,SAAoB,gBAAgB,WAAW,aAAa,aAAa;AAEzE,YAAY,gBAAgB;AAC5B,SAAS,cAAc;AAEhB,SAAS,gBAAgB,MAAY,OAAoB;AAC9D,OAAK,gBAAgB,KAAK;AAC1B,OAAK,UAAU;AAAA,IACb,MAAM,eAAe;AAAA,IACrB,SAAS,MAAM;AAAA,EACjB,CAAC;AAID,OAAK,cAAc;AAAA,IACjB,CAAC,WAAW,mBAAmB,GAAG,MAAM,YAAY;AAAA,IACpD,CAAC,WAAW,sBAAsB,GAAG,MAAM;AAAA,IAC3C,CAAC,WAAW,oBAAoB,GAAG,MAAM,SAAS;AAAA,EACpD,CAAC;AACH;AAEO,SAAS,sBAAsB,MAAY,SAAqC;AACrF,QAAM,QAAyC;AAAA,IAC7C,CAAC,WAAW,yBAAyB,GAAG,QAAQ,SAAS;AAAA,IACzD,CAAC,WAAW,2BAA2B,GAAG,KAAK,UAAU,OAAO;AAAA,IAChE,CAAC,WAAW,8BAA8B,GAAG,QAAQ;AAAA,IACrD,CAAC,WAAW,+BAA+B,GAAG,QAAQ;AAAA,IACtD,CAAC,WAAW,mCAAmC,GAAG,QAAQ,kBAAkB;AAAA,IAC5E,CAAC,WAAW,oCAAoC,GAAG,QAAQ,kBAAkB;AAAA,IAC7E,CAAC,WAAW,qCAAqC,GAAG,QAAQ,kBAAkB;AAAA,IAC9E,CAAC,WAAW,oCAAoC,GAAG,QAAQ,mBAAmB;AAAA,IAC9E,CAAC,WAAW,qCAAqC,GAAG,QAAQ,mBAAmB;AAAA,EACjF;AAGA,MAAI,QAAQ,WAAW,UAAa,QAAQ,WAAW,IAAI;AACzD,UAAM,sBAAsB,QAAQ,YAAY,QAAQ;AAExD,UAAM,yBAAyB,IAAI,KAAK,mBAAmB,EAAE,YAAY;AACzE,UAAM,WAAW,mCAAmC,IAAI;AAAA,EAC1D;AAEA,MAAI,KAAK,YAAY,GAAG;AACtB,SAAK,cAAc,KAAK;AAAA,EAC1B,OAAO;AACL,UAAM,iBAAiB,YAAY,OAAO;AAC1C,UAAM,cAAc,MAAM,QAAQ,gBAAgB,IAAI;AAGtD,WAAO,UAAU,EAAE,gBAAgB,oBAAoB,CAAC,GAAG,aAAa,CAAC,UAAU;AACjF,UAAI;AACF,cAAM,cAAc,KAAK;AAAA,MAC3B,UAAE;AACA,cAAM,IAAI;AAAA,MACZ;AAAA,IACF,CAAC;AAAA,EACH;AACF;","names":[]}

package/dist/tokenize/basic/sentence.cjs

@@ -27,7 +27,7 @@ const splitSentences = (text, minLength = 20, retainFormat = false) => {
   const suffixes = /(Inc|Ltd|Jr|Sr|Co)/g;
   const starters = /(Mr|Mrs|Ms|Dr|Prof|Capt|Cpt|Lt|He\s|She\s|It\s|They\s|Their\s|Our\s|We\s|But\s|However\s|That\s|This\s|Wherever)/g;
   const acronyms = /([A-Z][.][A-Z][.](?:[A-Z][.])?)/g;
-  const websites = /[.](com|net|org|io|gov|edu|me)/g;
+  const websites = /(\w+\.)+(com|net|org|io|gov|edu|me)/g;
   const digits = /([0-9])/g;
   const dots = /\.{2,}/g;
   if (retainFormat) {
@@ -36,7 +36,7 @@ const splitSentences = (text, minLength = 20, retainFormat = false) => {
     text = text.replaceAll("\n", " ");
   }
   text = text.replaceAll(prefixes, "$1<prd>");
-  text = text.replaceAll(websites, "<prd>$2");
+  text = text.replace(websites, (match) => match.replaceAll(".", "<prd>"));
   text = text.replaceAll(new RegExp(`${digits.source}[.]${digits.source}`, "g"), "$1<prd>$2");
   text = text.replaceAll(dots, (match) => "<prd>".repeat(match.length));
   text = text.replaceAll("Ph.D.", "Ph<prd>D<prd>");
@@ -60,7 +60,7 @@ const splitSentences = (text, minLength = 20, retainFormat = false) => {
   text = text.replaceAll('."', '".');
   text = text.replaceAll('!"', '"!');
   text = text.replaceAll('?"', '"?');
-  text = text.replaceAll(".", ".<stop>");
+  text = text.replace(/\.(?=\s|$)/g, ".<stop>");
   text = text.replaceAll("?", "?<stop>");
   text = text.replaceAll("!", "!<stop>");
   text = text.replaceAll("<prd>", ".");

package/dist/tokenize/basic/sentence.cjs.map

@@ -1 +1 @@
-{"version":3,"sources":["../../../src/tokenize/basic/sentence.ts"],"sourcesContent":["// SPDX-FileCopyrightText: 2024 LiveKit, Inc.\n//\n// SPDX-License-Identifier: Apache-2.0\n\n/**\n * Split the text into sentences.\n */\nexport const splitSentences = (\n  text: string,\n  minLength = 20,\n  retainFormat: boolean = false,\n): [string, number, number][] => {\n  const alphabets = /([A-Za-z])/g;\n  const prefixes = /(Mr|St|Mrs|Ms|Dr)[.]/g;\n  const suffixes = /(Inc|Ltd|Jr|Sr|Co)/g;\n  const starters =\n    /(Mr|Mrs|Ms|Dr|Prof|Capt|Cpt|Lt|He\\s|She\\s|It\\s|They\\s|Their\\s|Our\\s|We\\s|But\\s|However\\s|That\\s|This\\s|Wherever)/g;\n  const acronyms = /([A-Z][.][A-Z][.](?:[A-Z][.])?)/g;\n  const websites = /[.](com|net|org|io|gov|edu|me)/g;\n  const digits = /([0-9])/g;\n  const dots = /\\.{2,}/g;\n\n  if (retainFormat) {\n    text = text.replaceAll('\\n', '<nel><stop>');\n  } else {\n    text = text.replaceAll('\\n', ' ');\n  }\n\n  text = text.replaceAll(prefixes, '$1<prd>');\n  text = text.replaceAll(websites, '<prd>$2');\n  text = text.replaceAll(new RegExp(`${digits.source}[.]${digits.source}`, 'g'), '$1<prd>$2');\n  text = text.replaceAll(dots, (match) => '<prd>'.repeat(match.length));\n  text = text.replaceAll('Ph.D.', 'Ph<prd>D<prd>');\n  text = text.replaceAll(new RegExp(`\\\\s${alphabets.source}[.] `, 'g'), ' $1<prd> ');\n  text = text.replaceAll(new RegExp(`${acronyms.source} ${starters.source}`, 'g'), '$1<stop> $2');\n  text = text.replaceAll(\n    new RegExp(`${alphabets.source}[.]${alphabets.source}[.]${alphabets.source}[.]`, 'g'),\n    '$1<prd>$2<prd>$3<prd>',\n  );\n  text = text.replaceAll(\n    new RegExp(`${alphabets.source}[.]${alphabets.source}[.]`, 'g'),\n    '$1<prd>$2<prd>',\n  );\n  text = text.replaceAll(\n    new RegExp(` ${suffixes.source}[.] ${starters.source}`, 'g'),\n    '$1<stop> $2',\n  );\n  text = text.replaceAll(new RegExp(` ${suffixes.source}[.]`, 'g'), '$1<prd>');\n  text = text.replaceAll(new RegExp(` ${alphabets.source}[.]`, 'g'), '$1<prd>');\n  text = text.replaceAll('.”', '”.');\n  text = text.replaceAll('.\"', '\".');\n  text = text.replaceAll('!\"', '\"!');\n  text = text.replaceAll('?\"', '\"?');\n  text = text.replaceAll('.', '.<stop>');\n  text = text.replaceAll('?', '?<stop>');\n  text = text.replaceAll('!', '!<stop>');\n  text = text.replaceAll('<prd>', '.');\n\n  if (retainFormat) {\n    text = text.replaceAll('<nel>', '\\n');\n  }\n\n  const split = text.split('<stop>');\n  text = text.replaceAll('<stop>', '');\n\n  const sentences: [string, number, number][] = [];\n  let buf = '';\n  let start = 0;\n  let end = 0;\n  const prePad = retainFormat ? '' : ' ';\n  for (const match of split) {\n    const sentence = retainFormat ? match : match.trim();\n    if (!sentence) continue;\n\n    buf += prePad + sentence;\n    end += match.length;\n    if (buf.length > minLength) {\n      sentences.push([buf.slice(prePad.length), start, end]);\n      start = end;\n      buf = '';\n    }\n  }\n\n  if (buf) {\n    sentences.push([buf.slice(prePad.length), start, text.length - 1]);\n  }\n\n  return sentences;\n};\n"],"mappings":";;;;;;;;;;;;;;;;;;AAAA;AAAA;AAAA;AAAA;AAAA;AAOO,MAAM,iBAAiB,CAC5B,MACA,YAAY,IACZ,eAAwB,UACO;AAC/B,QAAM,YAAY;AAClB,QAAM,WAAW;AACjB,QAAM,WAAW;AACjB,QAAM,WACJ;AACF,QAAM,WAAW;AACjB,QAAM,WAAW;AACjB,QAAM,SAAS;AACf,QAAM,OAAO;AAEb,MAAI,cAAc;AAChB,WAAO,KAAK,WAAW,MAAM,aAAa;AAAA,EAC5C,OAAO;AACL,WAAO,KAAK,WAAW,MAAM,GAAG;AAAA,EAClC;AAEA,SAAO,KAAK,WAAW,UAAU,SAAS;AAC1C,SAAO,KAAK,WAAW,UAAU,SAAS;AAC1C,SAAO,KAAK,WAAW,IAAI,OAAO,GAAG,OAAO,MAAM,MAAM,OAAO,MAAM,IAAI,GAAG,GAAG,WAAW;AAC1F,SAAO,KAAK,WAAW,MAAM,CAAC,UAAU,QAAQ,OAAO,MAAM,MAAM,CAAC;AACpE,SAAO,KAAK,WAAW,SAAS,eAAe;AAC/C,SAAO,KAAK,WAAW,IAAI,OAAO,MAAM,UAAU,MAAM,QAAQ,GAAG,GAAG,WAAW;AACjF,SAAO,KAAK,WAAW,IAAI,OAAO,GAAG,SAAS,MAAM,IAAI,SAAS,MAAM,IAAI,GAAG,GAAG,aAAa;AAC9F,SAAO,KAAK;AAAA,IACV,IAAI,OAAO,GAAG,UAAU,MAAM,MAAM,UAAU,MAAM,MAAM,UAAU,MAAM,OAAO,GAAG;AAAA,IACpF;AAAA,EACF;AACA,SAAO,KAAK;AAAA,IACV,IAAI,OAAO,GAAG,UAAU,MAAM,MAAM,UAAU,MAAM,OAAO,GAAG;AAAA,IAC9D;AAAA,EACF;AACA,SAAO,KAAK;AAAA,IACV,IAAI,OAAO,IAAI,SAAS,MAAM,OAAO,SAAS,MAAM,IAAI,GAAG;AAAA,IAC3D;AAAA,EACF;AACA,SAAO,KAAK,WAAW,IAAI,OAAO,IAAI,SAAS,MAAM,OAAO,GAAG,GAAG,SAAS;AAC3E,SAAO,KAAK,WAAW,IAAI,OAAO,IAAI,UAAU,MAAM,OAAO,GAAG,GAAG,SAAS;AAC5E,SAAO,KAAK,WAAW,WAAM,SAAI;AACjC,SAAO,KAAK,WAAW,MAAM,IAAI;AACjC,SAAO,KAAK,WAAW,MAAM,IAAI;AACjC,SAAO,KAAK,WAAW,MAAM,IAAI;AACjC,SAAO,KAAK,WAAW,KAAK,SAAS;AACrC,SAAO,KAAK,WAAW,KAAK,SAAS;AACrC,SAAO,KAAK,WAAW,KAAK,SAAS;AACrC,SAAO,KAAK,WAAW,SAAS,GAAG;AAEnC,MAAI,cAAc;AAChB,WAAO,KAAK,WAAW,SAAS,IAAI;AAAA,EACtC;AAEA,QAAM,QAAQ,KAAK,MAAM,QAAQ;AACjC,SAAO,KAAK,WAAW,UAAU,EAAE;AAEnC,QAAM,YAAwC,CAAC;AAC/C,MAAI,MAAM;AACV,MAAI,QAAQ;AACZ,MAAI,MAAM;AACV,QAAM,SAAS,eAAe,KAAK;AACnC,aAAW,SAAS,OAAO;AACzB,UAAM,WAAW,eAAe,QAAQ,MAAM,KAAK;AACnD,QAAI,CAAC,SAAU;AAEf,WAAO,SAAS;AAChB,WAAO,MAAM;AACb,QAAI,IAAI,SAAS,WAAW;AAC1B,gBAAU,KAAK,CAAC,IAAI,MAAM,OAAO,MAAM,GAAG,OAAO,GAAG,CAAC;AACrD,cAAQ;AACR,YAAM;AAAA,IACR;AAAA,EACF;AAEA,MAAI,KAAK;AACP,cAAU,KAAK,CAAC,IAAI,MAAM,OAAO,MAAM,GAAG,OAAO,KAAK,SAAS,CAAC,CAAC;AAAA,EACnE;AAEA,SAAO;AACT;","names":[]}
+{"version":3,"sources":["../../../src/tokenize/basic/sentence.ts"],"sourcesContent":["// SPDX-FileCopyrightText: 2024 LiveKit, Inc.\n//\n// SPDX-License-Identifier: Apache-2.0\n\n/**\n * Split the text into sentences.\n */\nexport const splitSentences = (\n  text: string,\n  minLength = 20,\n  retainFormat: boolean = false,\n): [string, number, number][] => {\n  const alphabets = /([A-Za-z])/g;\n  const prefixes = /(Mr|St|Mrs|Ms|Dr)[.]/g;\n  const suffixes = /(Inc|Ltd|Jr|Sr|Co)/g;\n  const starters =\n    /(Mr|Mrs|Ms|Dr|Prof|Capt|Cpt|Lt|He\\s|She\\s|It\\s|They\\s|Their\\s|Our\\s|We\\s|But\\s|However\\s|That\\s|This\\s|Wherever)/g;\n  const acronyms = /([A-Z][.][A-Z][.](?:[A-Z][.])?)/g;\n  const websites = /(\\w+\\.)+(com|net|org|io|gov|edu|me)/g;\n  const digits = /([0-9])/g;\n  const dots = /\\.{2,}/g;\n\n  if (retainFormat) {\n    text = text.replaceAll('\\n', '<nel><stop>');\n  } else {\n    text = text.replaceAll('\\n', ' ');\n  }\n\n  text = text.replaceAll(prefixes, '$1<prd>');\n  text = text.replace(websites, (match) => match.replaceAll('.', '<prd>'));\n  text = text.replaceAll(new RegExp(`${digits.source}[.]${digits.source}`, 'g'), '$1<prd>$2');\n  text = text.replaceAll(dots, (match) => '<prd>'.repeat(match.length));\n  text = text.replaceAll('Ph.D.', 'Ph<prd>D<prd>');\n  text = text.replaceAll(new RegExp(`\\\\s${alphabets.source}[.] `, 'g'), ' $1<prd> ');\n  text = text.replaceAll(new RegExp(`${acronyms.source} ${starters.source}`, 'g'), '$1<stop> $2');\n  text = text.replaceAll(\n    new RegExp(`${alphabets.source}[.]${alphabets.source}[.]${alphabets.source}[.]`, 'g'),\n    '$1<prd>$2<prd>$3<prd>',\n  );\n  text = text.replaceAll(\n    new RegExp(`${alphabets.source}[.]${alphabets.source}[.]`, 'g'),\n    '$1<prd>$2<prd>',\n  );\n  text = text.replaceAll(\n    new RegExp(` ${suffixes.source}[.] ${starters.source}`, 'g'),\n    '$1<stop> $2',\n  );\n  text = text.replaceAll(new RegExp(` ${suffixes.source}[.]`, 'g'), '$1<prd>');\n  text = text.replaceAll(new RegExp(` ${alphabets.source}[.]`, 'g'), '$1<prd>');\n  text = text.replaceAll('.”', '”.');\n  text = text.replaceAll('.\"', '\".');\n  text = text.replaceAll('!\"', '\"!');\n  text = text.replaceAll('?\"', '\"?');\n  text = text.replace(/\\.(?=\\s|$)/g, '.<stop>');\n  text = text.replaceAll('?', '?<stop>');\n  text = text.replaceAll('!', '!<stop>');\n  text = text.replaceAll('<prd>', '.');\n\n  if (retainFormat) {\n    text = text.replaceAll('<nel>', '\\n');\n  }\n\n  const split = text.split('<stop>');\n  text = text.replaceAll('<stop>', '');\n\n  const sentences: [string, number, number][] = [];\n  let buf = '';\n  let start = 0;\n  let end = 0;\n  const prePad = retainFormat ? '' : ' ';\n  for (const match of split) {\n    const sentence = retainFormat ? match : match.trim();\n    if (!sentence) continue;\n\n    buf += prePad + sentence;\n    end += match.length;\n    if (buf.length > minLength) {\n      sentences.push([buf.slice(prePad.length), start, end]);\n      start = end;\n      buf = '';\n    }\n  }\n\n  if (buf) {\n    sentences.push([buf.slice(prePad.length), start, text.length - 1]);\n  }\n\n  return sentences;\n};\n"],"mappings":";;;;;;;;;;;;;;;;;;AAAA;AAAA;AAAA;AAAA;AAAA;AAOO,MAAM,iBAAiB,CAC5B,MACA,YAAY,IACZ,eAAwB,UACO;AAC/B,QAAM,YAAY;AAClB,QAAM,WAAW;AACjB,QAAM,WAAW;AACjB,QAAM,WACJ;AACF,QAAM,WAAW;AACjB,QAAM,WAAW;AACjB,QAAM,SAAS;AACf,QAAM,OAAO;AAEb,MAAI,cAAc;AAChB,WAAO,KAAK,WAAW,MAAM,aAAa;AAAA,EAC5C,OAAO;AACL,WAAO,KAAK,WAAW,MAAM,GAAG;AAAA,EAClC;AAEA,SAAO,KAAK,WAAW,UAAU,SAAS;AAC1C,SAAO,KAAK,QAAQ,UAAU,CAAC,UAAU,MAAM,WAAW,KAAK,OAAO,CAAC;AACvE,SAAO,KAAK,WAAW,IAAI,OAAO,GAAG,OAAO,MAAM,MAAM,OAAO,MAAM,IAAI,GAAG,GAAG,WAAW;AAC1F,SAAO,KAAK,WAAW,MAAM,CAAC,UAAU,QAAQ,OAAO,MAAM,MAAM,CAAC;AACpE,SAAO,KAAK,WAAW,SAAS,eAAe;AAC/C,SAAO,KAAK,WAAW,IAAI,OAAO,MAAM,UAAU,MAAM,QAAQ,GAAG,GAAG,WAAW;AACjF,SAAO,KAAK,WAAW,IAAI,OAAO,GAAG,SAAS,MAAM,IAAI,SAAS,MAAM,IAAI,GAAG,GAAG,aAAa;AAC9F,SAAO,KAAK;AAAA,IACV,IAAI,OAAO,GAAG,UAAU,MAAM,MAAM,UAAU,MAAM,MAAM,UAAU,MAAM,OAAO,GAAG;AAAA,IACpF;AAAA,EACF;AACA,SAAO,KAAK;AAAA,IACV,IAAI,OAAO,GAAG,UAAU,MAAM,MAAM,UAAU,MAAM,OAAO,GAAG;AAAA,IAC9D;AAAA,EACF;AACA,SAAO,KAAK;AAAA,IACV,IAAI,OAAO,IAAI,SAAS,MAAM,OAAO,SAAS,MAAM,IAAI,GAAG;AAAA,IAC3D;AAAA,EACF;AACA,SAAO,KAAK,WAAW,IAAI,OAAO,IAAI,SAAS,MAAM,OAAO,GAAG,GAAG,SAAS;AAC3E,SAAO,KAAK,WAAW,IAAI,OAAO,IAAI,UAAU,MAAM,OAAO,GAAG,GAAG,SAAS;AAC5E,SAAO,KAAK,WAAW,WAAM,SAAI;AACjC,SAAO,KAAK,WAAW,MAAM,IAAI;AACjC,SAAO,KAAK,WAAW,MAAM,IAAI;AACjC,SAAO,KAAK,WAAW,MAAM,IAAI;AACjC,SAAO,KAAK,QAAQ,eAAe,SAAS;AAC5C,SAAO,KAAK,WAAW,KAAK,SAAS;AACrC,SAAO,KAAK,WAAW,KAAK,SAAS;AACrC,SAAO,KAAK,WAAW,SAAS,GAAG;AAEnC,MAAI,cAAc;AAChB,WAAO,KAAK,WAAW,SAAS,IAAI;AAAA,EACtC;AAEA,QAAM,QAAQ,KAAK,MAAM,QAAQ;AACjC,SAAO,KAAK,WAAW,UAAU,EAAE;AAEnC,QAAM,YAAwC,CAAC;AAC/C,MAAI,MAAM;AACV,MAAI,QAAQ;AACZ,MAAI,MAAM;AACV,QAAM,SAAS,eAAe,KAAK;AACnC,aAAW,SAAS,OAAO;AACzB,UAAM,WAAW,eAAe,QAAQ,MAAM,KAAK;AACnD,QAAI,CAAC,SAAU;AAEf,WAAO,SAAS;AAChB,WAAO,MAAM;AACb,QAAI,IAAI,SAAS,WAAW;AAC1B,gBAAU,KAAK,CAAC,IAAI,MAAM,OAAO,MAAM,GAAG,OAAO,GAAG,CAAC;AACrD,cAAQ;AACR,YAAM;AAAA,IACR;AAAA,EACF;AAEA,MAAI,KAAK;AACP,cAAU,KAAK,CAAC,IAAI,MAAM,OAAO,MAAM,GAAG,OAAO,KAAK,SAAS,CAAC,CAAC;AAAA,EACnE;AAEA,SAAO;AACT;","names":[]}

package/dist/tokenize/basic/sentence.js

@@ -4,7 +4,7 @@ const splitSentences = (text, minLength = 20, retainFormat = false) => {
   const suffixes = /(Inc|Ltd|Jr|Sr|Co)/g;
   const starters = /(Mr|Mrs|Ms|Dr|Prof|Capt|Cpt|Lt|He\s|She\s|It\s|They\s|Their\s|Our\s|We\s|But\s|However\s|That\s|This\s|Wherever)/g;
   const acronyms = /([A-Z][.][A-Z][.](?:[A-Z][.])?)/g;
-  const websites = /[.](com|net|org|io|gov|edu|me)/g;
+  const websites = /(\w+\.)+(com|net|org|io|gov|edu|me)/g;
   const digits = /([0-9])/g;
   const dots = /\.{2,}/g;
   if (retainFormat) {
@@ -13,7 +13,7 @@ const splitSentences = (text, minLength = 20, retainFormat = false) => {
     text = text.replaceAll("\n", " ");
   }
   text = text.replaceAll(prefixes, "$1<prd>");
-  text = text.replaceAll(websites, "<prd>$2");
+  text = text.replace(websites, (match) => match.replaceAll(".", "<prd>"));
   text = text.replaceAll(new RegExp(`${digits.source}[.]${digits.source}`, "g"), "$1<prd>$2");
   text = text.replaceAll(dots, (match) => "<prd>".repeat(match.length));
   text = text.replaceAll("Ph.D.", "Ph<prd>D<prd>");
@@ -37,7 +37,7 @@ const splitSentences = (text, minLength = 20, retainFormat = false) => {
   text = text.replaceAll('."', '".');
   text = text.replaceAll('!"', '"!');
   text = text.replaceAll('?"', '"?');
-  text = text.replaceAll(".", ".<stop>");
+  text = text.replace(/\.(?=\s|$)/g, ".<stop>");
   text = text.replaceAll("?", "?<stop>");
   text = text.replaceAll("!", "!<stop>");
   text = text.replaceAll("<prd>", ".");

package/dist/tokenize/basic/sentence.js.map

@@ -1 +1 @@
-{"version":3,"sources":["../../../src/tokenize/basic/sentence.ts"],"sourcesContent":["// SPDX-FileCopyrightText: 2024 LiveKit, Inc.\n//\n// SPDX-License-Identifier: Apache-2.0\n\n/**\n * Split the text into sentences.\n */\nexport const splitSentences = (\n  text: string,\n  minLength = 20,\n  retainFormat: boolean = false,\n): [string, number, number][] => {\n  const alphabets = /([A-Za-z])/g;\n  const prefixes = /(Mr|St|Mrs|Ms|Dr)[.]/g;\n  const suffixes = /(Inc|Ltd|Jr|Sr|Co)/g;\n  const starters =\n    /(Mr|Mrs|Ms|Dr|Prof|Capt|Cpt|Lt|He\\s|She\\s|It\\s|They\\s|Their\\s|Our\\s|We\\s|But\\s|However\\s|That\\s|This\\s|Wherever)/g;\n  const acronyms = /([A-Z][.][A-Z][.](?:[A-Z][.])?)/g;\n  const websites = /[.](com|net|org|io|gov|edu|me)/g;\n  const digits = /([0-9])/g;\n  const dots = /\\.{2,}/g;\n\n  if (retainFormat) {\n    text = text.replaceAll('\\n', '<nel><stop>');\n  } else {\n    text = text.replaceAll('\\n', ' ');\n  }\n\n  text = text.replaceAll(prefixes, '$1<prd>');\n  text = text.replaceAll(websites, '<prd>$2');\n  text = text.replaceAll(new RegExp(`${digits.source}[.]${digits.source}`, 'g'), '$1<prd>$2');\n  text = text.replaceAll(dots, (match) => '<prd>'.repeat(match.length));\n  text = text.replaceAll('Ph.D.', 'Ph<prd>D<prd>');\n  text = text.replaceAll(new RegExp(`\\\\s${alphabets.source}[.] `, 'g'), ' $1<prd> ');\n  text = text.replaceAll(new RegExp(`${acronyms.source} ${starters.source}`, 'g'), '$1<stop> $2');\n  text = text.replaceAll(\n    new RegExp(`${alphabets.source}[.]${alphabets.source}[.]${alphabets.source}[.]`, 'g'),\n    '$1<prd>$2<prd>$3<prd>',\n  );\n  text = text.replaceAll(\n    new RegExp(`${alphabets.source}[.]${alphabets.source}[.]`, 'g'),\n    '$1<prd>$2<prd>',\n  );\n  text = text.replaceAll(\n    new RegExp(` ${suffixes.source}[.] ${starters.source}`, 'g'),\n    '$1<stop> $2',\n  );\n  text = text.replaceAll(new RegExp(` ${suffixes.source}[.]`, 'g'), '$1<prd>');\n  text = text.replaceAll(new RegExp(` ${alphabets.source}[.]`, 'g'), '$1<prd>');\n  text = text.replaceAll('.”', '”.');\n  text = text.replaceAll('.\"', '\".');\n  text = text.replaceAll('!\"', '\"!');\n  text = text.replaceAll('?\"', '\"?');\n  text = text.replaceAll('.', '.<stop>');\n  text = text.replaceAll('?', '?<stop>');\n  text = text.replaceAll('!', '!<stop>');\n  text = text.replaceAll('<prd>', '.');\n\n  if (retainFormat) {\n    text = text.replaceAll('<nel>', '\\n');\n  }\n\n  const split = text.split('<stop>');\n  text = text.replaceAll('<stop>', '');\n\n  const sentences: [string, number, number][] = [];\n  let buf = '';\n  let start = 0;\n  let end = 0;\n  const prePad = retainFormat ? '' : ' ';\n  for (const match of split) {\n    const sentence = retainFormat ? match : match.trim();\n    if (!sentence) continue;\n\n    buf += prePad + sentence;\n    end += match.length;\n    if (buf.length > minLength) {\n      sentences.push([buf.slice(prePad.length), start, end]);\n      start = end;\n      buf = '';\n    }\n  }\n\n  if (buf) {\n    sentences.push([buf.slice(prePad.length), start, text.length - 1]);\n  }\n\n  return sentences;\n};\n"],"mappings":"AAOO,MAAM,iBAAiB,CAC5B,MACA,YAAY,IACZ,eAAwB,UACO;AAC/B,QAAM,YAAY;AAClB,QAAM,WAAW;AACjB,QAAM,WAAW;AACjB,QAAM,WACJ;AACF,QAAM,WAAW;AACjB,QAAM,WAAW;AACjB,QAAM,SAAS;AACf,QAAM,OAAO;AAEb,MAAI,cAAc;AAChB,WAAO,KAAK,WAAW,MAAM,aAAa;AAAA,EAC5C,OAAO;AACL,WAAO,KAAK,WAAW,MAAM,GAAG;AAAA,EAClC;AAEA,SAAO,KAAK,WAAW,UAAU,SAAS;AAC1C,SAAO,KAAK,WAAW,UAAU,SAAS;AAC1C,SAAO,KAAK,WAAW,IAAI,OAAO,GAAG,OAAO,MAAM,MAAM,OAAO,MAAM,IAAI,GAAG,GAAG,WAAW;AAC1F,SAAO,KAAK,WAAW,MAAM,CAAC,UAAU,QAAQ,OAAO,MAAM,MAAM,CAAC;AACpE,SAAO,KAAK,WAAW,SAAS,eAAe;AAC/C,SAAO,KAAK,WAAW,IAAI,OAAO,MAAM,UAAU,MAAM,QAAQ,GAAG,GAAG,WAAW;AACjF,SAAO,KAAK,WAAW,IAAI,OAAO,GAAG,SAAS,MAAM,IAAI,SAAS,MAAM,IAAI,GAAG,GAAG,aAAa;AAC9F,SAAO,KAAK;AAAA,IACV,IAAI,OAAO,GAAG,UAAU,MAAM,MAAM,UAAU,MAAM,MAAM,UAAU,MAAM,OAAO,GAAG;AAAA,IACpF;AAAA,EACF;AACA,SAAO,KAAK;AAAA,IACV,IAAI,OAAO,GAAG,UAAU,MAAM,MAAM,UAAU,MAAM,OAAO,GAAG;AAAA,IAC9D;AAAA,EACF;AACA,SAAO,KAAK;AAAA,IACV,IAAI,OAAO,IAAI,SAAS,MAAM,OAAO,SAAS,MAAM,IAAI,GAAG;AAAA,IAC3D;AAAA,EACF;AACA,SAAO,KAAK,WAAW,IAAI,OAAO,IAAI,SAAS,MAAM,OAAO,GAAG,GAAG,SAAS;AAC3E,SAAO,KAAK,WAAW,IAAI,OAAO,IAAI,UAAU,MAAM,OAAO,GAAG,GAAG,SAAS;AAC5E,SAAO,KAAK,WAAW,WAAM,SAAI;AACjC,SAAO,KAAK,WAAW,MAAM,IAAI;AACjC,SAAO,KAAK,WAAW,MAAM,IAAI;AACjC,SAAO,KAAK,WAAW,MAAM,IAAI;AACjC,SAAO,KAAK,WAAW,KAAK,SAAS;AACrC,SAAO,KAAK,WAAW,KAAK,SAAS;AACrC,SAAO,KAAK,WAAW,KAAK,SAAS;AACrC,SAAO,KAAK,WAAW,SAAS,GAAG;AAEnC,MAAI,cAAc;AAChB,WAAO,KAAK,WAAW,SAAS,IAAI;AAAA,EACtC;AAEA,QAAM,QAAQ,KAAK,MAAM,QAAQ;AACjC,SAAO,KAAK,WAAW,UAAU,EAAE;AAEnC,QAAM,YAAwC,CAAC;AAC/C,MAAI,MAAM;AACV,MAAI,QAAQ;AACZ,MAAI,MAAM;AACV,QAAM,SAAS,eAAe,KAAK;AACnC,aAAW,SAAS,OAAO;AACzB,UAAM,WAAW,eAAe,QAAQ,MAAM,KAAK;AACnD,QAAI,CAAC,SAAU;AAEf,WAAO,SAAS;AAChB,WAAO,MAAM;AACb,QAAI,IAAI,SAAS,WAAW;AAC1B,gBAAU,KAAK,CAAC,IAAI,MAAM,OAAO,MAAM,GAAG,OAAO,GAAG,CAAC;AACrD,cAAQ;AACR,YAAM;AAAA,IACR;AAAA,EACF;AAEA,MAAI,KAAK;AACP,cAAU,KAAK,CAAC,IAAI,MAAM,OAAO,MAAM,GAAG,OAAO,KAAK,SAAS,CAAC,CAAC;AAAA,EACnE;AAEA,SAAO;AACT;","names":[]}
+{"version":3,"sources":["../../../src/tokenize/basic/sentence.ts"],"sourcesContent":["// SPDX-FileCopyrightText: 2024 LiveKit, Inc.\n//\n// SPDX-License-Identifier: Apache-2.0\n\n/**\n * Split the text into sentences.\n */\nexport const splitSentences = (\n  text: string,\n  minLength = 20,\n  retainFormat: boolean = false,\n): [string, number, number][] => {\n  const alphabets = /([A-Za-z])/g;\n  const prefixes = /(Mr|St|Mrs|Ms|Dr)[.]/g;\n  const suffixes = /(Inc|Ltd|Jr|Sr|Co)/g;\n  const starters =\n    /(Mr|Mrs|Ms|Dr|Prof|Capt|Cpt|Lt|He\\s|She\\s|It\\s|They\\s|Their\\s|Our\\s|We\\s|But\\s|However\\s|That\\s|This\\s|Wherever)/g;\n  const acronyms = /([A-Z][.][A-Z][.](?:[A-Z][.])?)/g;\n  const websites = /(\\w+\\.)+(com|net|org|io|gov|edu|me)/g;\n  const digits = /([0-9])/g;\n  const dots = /\\.{2,}/g;\n\n  if (retainFormat) {\n    text = text.replaceAll('\\n', '<nel><stop>');\n  } else {\n    text = text.replaceAll('\\n', ' ');\n  }\n\n  text = text.replaceAll(prefixes, '$1<prd>');\n  text = text.replace(websites, (match) => match.replaceAll('.', '<prd>'));\n  text = text.replaceAll(new RegExp(`${digits.source}[.]${digits.source}`, 'g'), '$1<prd>$2');\n  text = text.replaceAll(dots, (match) => '<prd>'.repeat(match.length));\n  text = text.replaceAll('Ph.D.', 'Ph<prd>D<prd>');\n  text = text.replaceAll(new RegExp(`\\\\s${alphabets.source}[.] `, 'g'), ' $1<prd> ');\n  text = text.replaceAll(new RegExp(`${acronyms.source} ${starters.source}`, 'g'), '$1<stop> $2');\n  text = text.replaceAll(\n    new RegExp(`${alphabets.source}[.]${alphabets.source}[.]${alphabets.source}[.]`, 'g'),\n    '$1<prd>$2<prd>$3<prd>',\n  );\n  text = text.replaceAll(\n    new RegExp(`${alphabets.source}[.]${alphabets.source}[.]`, 'g'),\n    '$1<prd>$2<prd>',\n  );\n  text = text.replaceAll(\n    new RegExp(` ${suffixes.source}[.] ${starters.source}`, 'g'),\n    '$1<stop> $2',\n  );\n  text = text.replaceAll(new RegExp(` ${suffixes.source}[.]`, 'g'), '$1<prd>');\n  text = text.replaceAll(new RegExp(` ${alphabets.source}[.]`, 'g'), '$1<prd>');\n  text = text.replaceAll('.”', '”.');\n  text = text.replaceAll('.\"', '\".');\n  text = text.replaceAll('!\"', '\"!');\n  text = text.replaceAll('?\"', '\"?');\n  text = text.replace(/\\.(?=\\s|$)/g, '.<stop>');\n  text = text.replaceAll('?', '?<stop>');\n  text = text.replaceAll('!', '!<stop>');\n  text = text.replaceAll('<prd>', '.');\n\n  if (retainFormat) {\n    text = text.replaceAll('<nel>', '\\n');\n  }\n\n  const split = text.split('<stop>');\n  text = text.replaceAll('<stop>', '');\n\n  const sentences: [string, number, number][] = [];\n  let buf = '';\n  let start = 0;\n  let end = 0;\n  const prePad = retainFormat ? '' : ' ';\n  for (const match of split) {\n    const sentence = retainFormat ? match : match.trim();\n    if (!sentence) continue;\n\n    buf += prePad + sentence;\n    end += match.length;\n    if (buf.length > minLength) {\n      sentences.push([buf.slice(prePad.length), start, end]);\n      start = end;\n      buf = '';\n    }\n  }\n\n  if (buf) {\n    sentences.push([buf.slice(prePad.length), start, text.length - 1]);\n  }\n\n  return sentences;\n};\n"],"mappings":"AAOO,MAAM,iBAAiB,CAC5B,MACA,YAAY,IACZ,eAAwB,UACO;AAC/B,QAAM,YAAY;AAClB,QAAM,WAAW;AACjB,QAAM,WAAW;AACjB,QAAM,WACJ;AACF,QAAM,WAAW;AACjB,QAAM,WAAW;AACjB,QAAM,SAAS;AACf,QAAM,OAAO;AAEb,MAAI,cAAc;AAChB,WAAO,KAAK,WAAW,MAAM,aAAa;AAAA,EAC5C,OAAO;AACL,WAAO,KAAK,WAAW,MAAM,GAAG;AAAA,EAClC;AAEA,SAAO,KAAK,WAAW,UAAU,SAAS;AAC1C,SAAO,KAAK,QAAQ,UAAU,CAAC,UAAU,MAAM,WAAW,KAAK,OAAO,CAAC;AACvE,SAAO,KAAK,WAAW,IAAI,OAAO,GAAG,OAAO,MAAM,MAAM,OAAO,MAAM,IAAI,GAAG,GAAG,WAAW;AAC1F,SAAO,KAAK,WAAW,MAAM,CAAC,UAAU,QAAQ,OAAO,MAAM,MAAM,CAAC;AACpE,SAAO,KAAK,WAAW,SAAS,eAAe;AAC/C,SAAO,KAAK,WAAW,IAAI,OAAO,MAAM,UAAU,MAAM,QAAQ,GAAG,GAAG,WAAW;AACjF,SAAO,KAAK,WAAW,IAAI,OAAO,GAAG,SAAS,MAAM,IAAI,SAAS,MAAM,IAAI,GAAG,GAAG,aAAa;AAC9F,SAAO,KAAK;AAAA,IACV,IAAI,OAAO,GAAG,UAAU,MAAM,MAAM,UAAU,MAAM,MAAM,UAAU,MAAM,OAAO,GAAG;AAAA,IACpF;AAAA,EACF;AACA,SAAO,KAAK;AAAA,IACV,IAAI,OAAO,GAAG,UAAU,MAAM,MAAM,UAAU,MAAM,OAAO,GAAG;AAAA,IAC9D;AAAA,EACF;AACA,SAAO,KAAK;AAAA,IACV,IAAI,OAAO,IAAI,SAAS,MAAM,OAAO,SAAS,MAAM,IAAI,GAAG;AAAA,IAC3D;AAAA,EACF;AACA,SAAO,KAAK,WAAW,IAAI,OAAO,IAAI,SAAS,MAAM,OAAO,GAAG,GAAG,SAAS;AAC3E,SAAO,KAAK,WAAW,IAAI,OAAO,IAAI,UAAU,MAAM,OAAO,GAAG,GAAG,SAAS;AAC5E,SAAO,KAAK,WAAW,WAAM,SAAI;AACjC,SAAO,KAAK,WAAW,MAAM,IAAI;AACjC,SAAO,KAAK,WAAW,MAAM,IAAI;AACjC,SAAO,KAAK,WAAW,MAAM,IAAI;AACjC,SAAO,KAAK,QAAQ,eAAe,SAAS;AAC5C,SAAO,KAAK,WAAW,KAAK,SAAS;AACrC,SAAO,KAAK,WAAW,KAAK,SAAS;AACrC,SAAO,KAAK,WAAW,SAAS,GAAG;AAEnC,MAAI,cAAc;AAChB,WAAO,KAAK,WAAW,SAAS,IAAI;AAAA,EACtC;AAEA,QAAM,QAAQ,KAAK,MAAM,QAAQ;AACjC,SAAO,KAAK,WAAW,UAAU,EAAE;AAEnC,QAAM,YAAwC,CAAC;AAC/C,MAAI,MAAM;AACV,MAAI,QAAQ;AACZ,MAAI,MAAM;AACV,QAAM,SAAS,eAAe,KAAK;AACnC,aAAW,SAAS,OAAO;AACzB,UAAM,WAAW,eAAe,QAAQ,MAAM,KAAK;AACnD,QAAI,CAAC,SAAU;AAEf,WAAO,SAAS;AAChB,WAAO,MAAM;AACb,QAAI,IAAI,SAAS,WAAW;AAC1B,gBAAU,KAAK,CAAC,IAAI,MAAM,OAAO,MAAM,GAAG,OAAO,GAAG,CAAC;AACrD,cAAQ;AACR,YAAM;AAAA,IACR;AAAA,EACF;AAEA,MAAI,KAAK;AACP,cAAU,KAAK,CAAC,IAAI,MAAM,OAAO,MAAM,GAAG,OAAO,KAAK,SAAS,CAAC,CAAC;AAAA,EACnE;AAEA,SAAO;AACT;","names":[]}

package/dist/tokenize/tokenizer.test.cjs

@@ -2,13 +2,15 @@
 var import_vitest = require("vitest");
 var import_basic = require("./basic/index.cjs");
 var import_paragraph = require("./basic/paragraph.cjs");
-const TEXT = "Hi! LiveKit is a platform for live audio and video applications and services. R.T.C stands for Real-Time Communication... again R.T.C. Mr. Theo is testing the sentence tokenizer. This is a test. Another test. A short sentence. A longer sentence that is longer than the previous sentence. f(x) = x * 2.54 + 42. Hey! Hi! Hello! ";
+const TEXT = "Hi! LiveKit is a platform for live audio and video applications and services. R.T.C stands for Real-Time Communication... again R.T.C. Mr. Theo is testing the sentence tokenizer. This is a test. Another test. A short sentence. A longer sentence that is longer than the previous sentence. Find additional resources on livekit.com. Find additional resources on docs.livekit.com. f(x) = x * 2.54 + 42. Hey! Hi! Hello! ";
 const EXPECTED_MIN_20 = [
   "Hi! LiveKit is a platform for live audio and video applications and services.",
   "R.T.C stands for Real-Time Communication... again R.T.C.",
   "Mr. Theo is testing the sentence tokenizer.",
   "This is a test. Another test.",
   "A short sentence. A longer sentence that is longer than the previous sentence.",
+  "Find additional resources on livekit.com.",
+  "Find additional resources on docs.livekit.com.",
   "f(x) = x * 2.54 + 42.",
   "Hey! Hi! Hello!"
 ];

package/dist/tokenize/tokenizer.test.cjs.map

@@ -1 +1 @@
-{"version":3,"sources":["../../src/tokenize/tokenizer.test.ts"],"sourcesContent":["// SPDX-FileCopyrightText: 2024 LiveKit, Inc.\n//\n// SPDX-License-Identifier: Apache-2.0\nimport { describe, expect, it } from 'vitest';\nimport { SentenceTokenizer, WordTokenizer, hyphenateWord } from './basic/index.js';\nimport { splitParagraphs } from './basic/paragraph.js';\n\nconst TEXT =\n  'Hi! ' +\n  'LiveKit is a platform for live audio and video applications and services. ' +\n  'R.T.C stands for Real-Time Communication... again R.T.C. ' +\n  'Mr. Theo is testing the sentence tokenizer. ' +\n  'This is a test. Another test. ' +\n  'A short sentence. ' +\n  'A longer sentence that is longer than the previous sentence. ' +\n  'f(x) = x * 2.54 + 42. ' +\n  'Hey! Hi! Hello! ';\n\nconst EXPECTED_MIN_20 = [\n  'Hi! LiveKit is a platform for live audio and video applications and services.',\n  'R.T.C stands for Real-Time Communication... again R.T.C.',\n  'Mr. Theo is testing the sentence tokenizer.',\n  'This is a test. Another test.',\n  'A short sentence. A longer sentence that is longer than the previous sentence.',\n  'f(x) = x * 2.54 + 42.',\n  'Hey! Hi! Hello!',\n];\n\nconst WORDS_TEXT = 'This is a test. Blabla another test! multiple consecutive spaces:     done';\nconst WORDS_EXPECTED = [\n  'This',\n  'is',\n  'a',\n  'test',\n  'Blabla',\n  'another',\n  'test',\n  'multiple',\n  'consecutive',\n  'spaces',\n  'done',\n];\n\nconst WORDS_PUNCT_TEXT =\n  'This is <phoneme alphabet=\"cmu-arpabet\" ph=\"AE K CH UW AH L IY\">actually</phoneme> tricky to handle.';\nconst WORDS_PUNCT_EXPECTED = [\n  'This',\n  'is',\n  '<phoneme',\n  'alphabet=\"cmu-arpabet\"',\n  'ph=\"AE',\n  'K',\n  'CH',\n  'UW',\n  'AH',\n  'L',\n  'IY\">actually</phoneme>',\n  'tricky',\n  'to',\n  'handle.',\n];\n\nconst HYPHENATOR_TEXT = ['Segment', 'expected', 'communication', 'window', 'welcome', 'bedroom'];\nconst HYPHENATOR_EXPECTED = [\n  ['Seg', 'ment'],\n  ['ex', 'pect', 'ed'],\n  ['com', 'mu', 'ni', 'ca', 'tion'],\n  ['win', 'dow'],\n  ['wel', 'come'],\n  ['bed', 'room'],\n];\n\nconst PARAGRAPH_TEST_CASES: [string, [string, number, number][]][] = [\n  ['Single paragraph.', [['Single paragraph.', 0, 17]]],\n  [\n    'Paragraph 1.\\n\\nParagraph 2.',\n    [\n      ['Paragraph 1.', 0, 12],\n      ['Paragraph 2.', 14, 26],\n    ],\n  ],\n  [\n    'Para 1.\\n\\nPara 2.\\n\\nPara 3.',\n    [\n      ['Para 1.', 0, 7],\n      ['Para 2.', 9, 16],\n      ['Para 3.', 18, 25],\n    ],\n  ],\n  ['\\n\\nParagraph with leading newlines.', [['Paragraph with leading newlines.', 2, 34]]],\n  ['Paragraph with trailing newlines.\\n\\n', [['Paragraph with trailing newlines.', 0, 33]]],\n  [\n    '\\n\\n  Paragraph with leading and trailing spaces.  \\n\\n',\n    [['Paragraph with leading and trailing spaces.', 4, 47]],\n  ],\n  [\n    'Para 1.\\n\\n\\n\\nPara 2.', // Multiple newlines between paragraphs\n    [\n      ['Para 1.', 0, 7],\n      ['Para 2.', 11, 18],\n    ],\n  ],\n  [\n    'Para 1.\\n \\n \\nPara 2.', // Newlines with spaces between paragraphs\n    [\n      ['Para 1.', 0, 7],\n      ['Para 2.', 12, 19],\n    ],\n  ],\n  [\n    '', // Empty string\n    [],\n  ],\n  [\n    '\\n\\n\\n', // Only newlines\n    [],\n  ],\n  [\n    'Line 1\\nLine 2\\nLine 3', // Single paragraph with newlines\n    [['Line 1\\nLine 2\\nLine 3', 0, 20]],\n  ],\n];\n\ndescribe('tokenizer', () => {\n  describe('SentenceTokenizer', () => {\n    const tokenizer = new SentenceTokenizer();\n\n    it('should tokenize sentences correctly', () => {\n      expect(tokenizer.tokenize(TEXT).every((x, i) => EXPECTED_MIN_20[i] === x)).toBeTruthy();\n    });\n\n    it('should stream tokenize sentences correctly', async () => {\n      const pattern = [1, 2, 4];\n      let text = TEXT;\n      const chunks = [];\n      const patternIter = Array(Math.ceil(text.length / pattern.reduce((sum, num) => sum + num, 0)))\n        .fill(pattern)\n        .flat()\n        [Symbol.iterator]();\n\n      for (const size of patternIter) {\n        if (!text) break;\n        chunks.push(text.slice(undefined, size));\n        text = text.slice(size);\n      }\n      const stream = tokenizer.stream();\n      for (const chunk of chunks) {\n        stream.pushText(chunk);\n      }\n      stream.endInput();\n      stream.close();\n\n      for (const x of EXPECTED_MIN_20) {\n        await stream.next().then((value) => {\n          if (value.value) {\n            expect(value.value.token).toStrictEqual(x);\n          }\n        });\n      }\n    });\n  });\n  describe('WordTokenizer', () => {\n    const tokenizer = new WordTokenizer();\n\n    it('should tokenize words correctly', () => {\n      expect(tokenizer.tokenize(WORDS_TEXT).every((x, i) => WORDS_EXPECTED[i] === x)).toBeTruthy();\n    });\n\n    it('should stream tokenize words correctly', async () => {\n      const pattern = [1, 2, 4];\n      let text = WORDS_TEXT;\n      const chunks = [];\n      const patternIter = Array(Math.ceil(text.length / pattern.reduce((sum, num) => sum + num, 0)))\n        .fill(pattern)\n        .flat()\n        [Symbol.iterator]();\n\n      for (const size of patternIter) {\n        if (!text) break;\n        chunks.push(text.slice(undefined, size));\n        text = text.slice(size);\n      }\n      const stream = tokenizer.stream();\n      for (const chunk of chunks) {\n        stream.pushText(chunk);\n      }\n      stream.endInput();\n      stream.close();\n\n      for (const x of WORDS_EXPECTED) {\n        await stream.next().then((value) => {\n          if (value.value) {\n            expect(value.value.token).toStrictEqual(x);\n          }\n        });\n      }\n    });\n\n    describe('punctuation handling', () => {\n      const tokenizerPunct = new WordTokenizer(false);\n\n      it('should tokenize words correctly', () => {\n        expect(\n          tokenizerPunct.tokenize(WORDS_PUNCT_TEXT).every((x, i) => WORDS_PUNCT_EXPECTED[i] === x),\n        ).toBeTruthy();\n      });\n\n      it('should stream tokenize words correctly', async () => {\n        const pattern = [1, 2, 4];\n        let text = WORDS_PUNCT_TEXT;\n        const chunks = [];\n        const patternIter = Array(\n          Math.ceil(text.length / pattern.reduce((sum, num) => sum + num, 0)),\n        )\n          .fill(pattern)\n          .flat()\n          [Symbol.iterator]();\n\n        for (const size of patternIter) {\n          if (!text) break;\n          chunks.push(text.slice(undefined, size));\n          text = text.slice(size);\n        }\n        const stream = tokenizerPunct.stream();\n        for (const chunk of chunks) {\n          stream.pushText(chunk);\n        }\n        stream.endInput();\n        stream.close();\n\n        for (const x of WORDS_PUNCT_EXPECTED) {\n          await stream.next().then((value) => {\n            if (value.value) {\n              expect(value.value.token).toStrictEqual(x);\n            }\n          });\n        }\n      });\n    });\n  });\n  describe('hyphenateWord', () => {\n    it('should hyphenate correctly', () => {\n      HYPHENATOR_TEXT.forEach((x, i) => {\n        expect(hyphenateWord(x)).toStrictEqual(HYPHENATOR_EXPECTED[i]);\n      });\n    });\n  });\n  describe('splitParagraphs', () => {\n    it('should tokenize paragraphs correctly', () => {\n      PARAGRAPH_TEST_CASES.forEach(([a, b]) => {\n        expect(splitParagraphs(a)).toStrictEqual(b);\n      });\n    });\n  });\n});\n"],"mappings":";AAGA,oBAAqC;AACrC,mBAAgE;AAChE,uBAAgC;AAEhC,MAAM,OACJ;AAUF,MAAM,kBAAkB;AAAA,EACtB;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AACF;AAEA,MAAM,aAAa;AACnB,MAAM,iBAAiB;AAAA,EACrB;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AACF;AAEA,MAAM,mBACJ;AACF,MAAM,uBAAuB;AAAA,EAC3B;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AACF;AAEA,MAAM,kBAAkB,CAAC,WAAW,YAAY,iBAAiB,UAAU,WAAW,SAAS;AAC/F,MAAM,sBAAsB;AAAA,EAC1B,CAAC,OAAO,MAAM;AAAA,EACd,CAAC,MAAM,QAAQ,IAAI;AAAA,EACnB,CAAC,OAAO,MAAM,MAAM,MAAM,MAAM;AAAA,EAChC,CAAC,OAAO,KAAK;AAAA,EACb,CAAC,OAAO,MAAM;AAAA,EACd,CAAC,OAAO,MAAM;AAChB;AAEA,MAAM,uBAA+D;AAAA,EACnE,CAAC,qBAAqB,CAAC,CAAC,qBAAqB,GAAG,EAAE,CAAC,CAAC;AAAA,EACpD;AAAA,IACE;AAAA,IACA;AAAA,MACE,CAAC,gBAAgB,GAAG,EAAE;AAAA,MACtB,CAAC,gBAAgB,IAAI,EAAE;AAAA,IACzB;AAAA,EACF;AAAA,EACA;AAAA,IACE;AAAA,IACA;AAAA,MACE,CAAC,WAAW,GAAG,CAAC;AAAA,MAChB,CAAC,WAAW,GAAG,EAAE;AAAA,MACjB,CAAC,WAAW,IAAI,EAAE;AAAA,IACpB;AAAA,EACF;AAAA,EACA,CAAC,wCAAwC,CAAC,CAAC,oCAAoC,GAAG,EAAE,CAAC,CAAC;AAAA,EACtF,CAAC,yCAAyC,CAAC,CAAC,qCAAqC,GAAG,EAAE,CAAC,CAAC;AAAA,EACxF;AAAA,IACE;AAAA,IACA,CAAC,CAAC,+CAA+C,GAAG,EAAE,CAAC;AAAA,EACzD;AAAA,EACA;AAAA,IACE;AAAA;AAAA,IACA;AAAA,MACE,CAAC,WAAW,GAAG,CAAC;AAAA,MAChB,CAAC,WAAW,IAAI,EAAE;AAAA,IACpB;AAAA,EACF;AAAA,EACA;AAAA,IACE;AAAA;AAAA,IACA;AAAA,MACE,CAAC,WAAW,GAAG,CAAC;AAAA,MAChB,CAAC,WAAW,IAAI,EAAE;AAAA,IACpB;AAAA,EACF;AAAA,EACA;AAAA,IACE;AAAA;AAAA,IACA,CAAC;AAAA,EACH;AAAA,EACA;AAAA,IACE;AAAA;AAAA,IACA,CAAC;AAAA,EACH;AAAA,EACA;AAAA,IACE;AAAA;AAAA,IACA,CAAC,CAAC,0BAA0B,GAAG,EAAE,CAAC;AAAA,EACpC;AACF;AAAA,IAEA,wBAAS,aAAa,MAAM;AAC1B,8BAAS,qBAAqB,MAAM;AAClC,UAAM,YAAY,IAAI,+BAAkB;AAExC,0BAAG,uCAAuC,MAAM;AAC9C,gCAAO,UAAU,SAAS,IAAI,EAAE,MAAM,CAAC,GAAG,MAAM,gBAAgB,CAAC,MAAM,CAAC,CAAC,EAAE,WAAW;AAAA,IACxF,CAAC;AAED,0BAAG,8CAA8C,YAAY;AAC3D,YAAM,UAAU,CAAC,GAAG,GAAG,CAAC;AACxB,UAAI,OAAO;AACX,YAAM,SAAS,CAAC;AAChB,YAAM,cAAc,MAAM,KAAK,KAAK,KAAK,SAAS,QAAQ,OAAO,CAAC,KAAK,QAAQ,MAAM,KAAK,CAAC,CAAC,CAAC,EAC1F,KAAK,OAAO,EACZ,KAAK,EACL,OAAO,QAAQ,EAAE;AAEpB,iBAAW,QAAQ,aAAa;AAC9B,YAAI,CAAC,KAAM;AACX,eAAO,KAAK,KAAK,MAAM,QAAW,IAAI,CAAC;AACvC,eAAO,KAAK,MAAM,IAAI;AAAA,MACxB;AACA,YAAM,SAAS,UAAU,OAAO;AAChC,iBAAW,SAAS,QAAQ;AAC1B,eAAO,SAAS,KAAK;AAAA,MACvB;AACA,aAAO,SAAS;AAChB,aAAO,MAAM;AAEb,iBAAW,KAAK,iBAAiB;AAC/B,cAAM,OAAO,KAAK,EAAE,KAAK,CAAC,UAAU;AAClC,cAAI,MAAM,OAAO;AACf,sCAAO,MAAM,MAAM,KAAK,EAAE,cAAc,CAAC;AAAA,UAC3C;AAAA,QACF,CAAC;AAAA,MACH;AAAA,IACF,CAAC;AAAA,EACH,CAAC;AACD,8BAAS,iBAAiB,MAAM;AAC9B,UAAM,YAAY,IAAI,2BAAc;AAEpC,0BAAG,mCAAmC,MAAM;AAC1C,gCAAO,UAAU,SAAS,UAAU,EAAE,MAAM,CAAC,GAAG,MAAM,eAAe,CAAC,MAAM,CAAC,CAAC,EAAE,WAAW;AAAA,IAC7F,CAAC;AAED,0BAAG,0CAA0C,YAAY;AACvD,YAAM,UAAU,CAAC,GAAG,GAAG,CAAC;AACxB,UAAI,OAAO;AACX,YAAM,SAAS,CAAC;AAChB,YAAM,cAAc,MAAM,KAAK,KAAK,KAAK,SAAS,QAAQ,OAAO,CAAC,KAAK,QAAQ,MAAM,KAAK,CAAC,CAAC,CAAC,EAC1F,KAAK,OAAO,EACZ,KAAK,EACL,OAAO,QAAQ,EAAE;AAEpB,iBAAW,QAAQ,aAAa;AAC9B,YAAI,CAAC,KAAM;AACX,eAAO,KAAK,KAAK,MAAM,QAAW,IAAI,CAAC;AACvC,eAAO,KAAK,MAAM,IAAI;AAAA,MACxB;AACA,YAAM,SAAS,UAAU,OAAO;AAChC,iBAAW,SAAS,QAAQ;AAC1B,eAAO,SAAS,KAAK;AAAA,MACvB;AACA,aAAO,SAAS;AAChB,aAAO,MAAM;AAEb,iBAAW,KAAK,gBAAgB;AAC9B,cAAM,OAAO,KAAK,EAAE,KAAK,CAAC,UAAU;AAClC,cAAI,MAAM,OAAO;AACf,sCAAO,MAAM,MAAM,KAAK,EAAE,cAAc,CAAC;AAAA,UAC3C;AAAA,QACF,CAAC;AAAA,MACH;AAAA,IACF,CAAC;AAED,gCAAS,wBAAwB,MAAM;AACrC,YAAM,iBAAiB,IAAI,2BAAc,KAAK;AAE9C,4BAAG,mCAAmC,MAAM;AAC1C;AAAA,UACE,eAAe,SAAS,gBAAgB,EAAE,MAAM,CAAC,GAAG,MAAM,qBAAqB,CAAC,MAAM,CAAC;AAAA,QACzF,EAAE,WAAW;AAAA,MACf,CAAC;AAED,4BAAG,0CAA0C,YAAY;AACvD,cAAM,UAAU,CAAC,GAAG,GAAG,CAAC;AACxB,YAAI,OAAO;AACX,cAAM,SAAS,CAAC;AAChB,cAAM,cAAc;AAAA,UAClB,KAAK,KAAK,KAAK,SAAS,QAAQ,OAAO,CAAC,KAAK,QAAQ,MAAM,KAAK,CAAC,CAAC;AAAA,QACpE,EACG,KAAK,OAAO,EACZ,KAAK,EACL,OAAO,QAAQ,EAAE;AAEpB,mBAAW,QAAQ,aAAa;AAC9B,cAAI,CAAC,KAAM;AACX,iBAAO,KAAK,KAAK,MAAM,QAAW,IAAI,CAAC;AACvC,iBAAO,KAAK,MAAM,IAAI;AAAA,QACxB;AACA,cAAM,SAAS,eAAe,OAAO;AACrC,mBAAW,SAAS,QAAQ;AAC1B,iBAAO,SAAS,KAAK;AAAA,QACvB;AACA,eAAO,SAAS;AAChB,eAAO,MAAM;AAEb,mBAAW,KAAK,sBAAsB;AACpC,gBAAM,OAAO,KAAK,EAAE,KAAK,CAAC,UAAU;AAClC,gBAAI,MAAM,OAAO;AACf,wCAAO,MAAM,MAAM,KAAK,EAAE,cAAc,CAAC;AAAA,YAC3C;AAAA,UACF,CAAC;AAAA,QACH;AAAA,MACF,CAAC;AAAA,IACH,CAAC;AAAA,EACH,CAAC;AACD,8BAAS,iBAAiB,MAAM;AAC9B,0BAAG,8BAA8B,MAAM;AACrC,sBAAgB,QAAQ,CAAC,GAAG,MAAM;AAChC,sCAAO,4BAAc,CAAC,CAAC,EAAE,cAAc,oBAAoB,CAAC,CAAC;AAAA,MAC/D,CAAC;AAAA,IACH,CAAC;AAAA,EACH,CAAC;AACD,8BAAS,mBAAmB,MAAM;AAChC,0BAAG,wCAAwC,MAAM;AAC/C,2BAAqB,QAAQ,CAAC,CAAC,GAAG,CAAC,MAAM;AACvC,sCAAO,kCAAgB,CAAC,CAAC,EAAE,cAAc,CAAC;AAAA,MAC5C,CAAC;AAAA,IACH,CAAC;AAAA,EACH,CAAC;AACH,CAAC;","names":[]}
+{"version":3,"sources":["../../src/tokenize/tokenizer.test.ts"],"sourcesContent":["// SPDX-FileCopyrightText: 2024 LiveKit, Inc.\n//\n// SPDX-License-Identifier: Apache-2.0\nimport { describe, expect, it } from 'vitest';\nimport { SentenceTokenizer, WordTokenizer, hyphenateWord } from './basic/index.js';\nimport { splitParagraphs } from './basic/paragraph.js';\n\nconst TEXT =\n  'Hi! ' +\n  'LiveKit is a platform for live audio and video applications and services. ' +\n  'R.T.C stands for Real-Time Communication... again R.T.C. ' +\n  'Mr. Theo is testing the sentence tokenizer. ' +\n  'This is a test. Another test. ' +\n  'A short sentence. ' +\n  'A longer sentence that is longer than the previous sentence. ' +\n  'Find additional resources on livekit.com. ' +\n  'Find additional resources on docs.livekit.com. ' +\n  'f(x) = x * 2.54 + 42. ' +\n  'Hey! Hi! Hello! ';\n\nconst EXPECTED_MIN_20 = [\n  'Hi! LiveKit is a platform for live audio and video applications and services.',\n  'R.T.C stands for Real-Time Communication... again R.T.C.',\n  'Mr. Theo is testing the sentence tokenizer.',\n  'This is a test. Another test.',\n  'A short sentence. A longer sentence that is longer than the previous sentence.',\n  'Find additional resources on livekit.com.',\n  'Find additional resources on docs.livekit.com.',\n  'f(x) = x * 2.54 + 42.',\n  'Hey! Hi! Hello!',\n];\n\nconst WORDS_TEXT = 'This is a test. Blabla another test! multiple consecutive spaces:     done';\nconst WORDS_EXPECTED = [\n  'This',\n  'is',\n  'a',\n  'test',\n  'Blabla',\n  'another',\n  'test',\n  'multiple',\n  'consecutive',\n  'spaces',\n  'done',\n];\n\nconst WORDS_PUNCT_TEXT =\n  'This is <phoneme alphabet=\"cmu-arpabet\" ph=\"AE K CH UW AH L IY\">actually</phoneme> tricky to handle.';\nconst WORDS_PUNCT_EXPECTED = [\n  'This',\n  'is',\n  '<phoneme',\n  'alphabet=\"cmu-arpabet\"',\n  'ph=\"AE',\n  'K',\n  'CH',\n  'UW',\n  'AH',\n  'L',\n  'IY\">actually</phoneme>',\n  'tricky',\n  'to',\n  'handle.',\n];\n\nconst HYPHENATOR_TEXT = ['Segment', 'expected', 'communication', 'window', 'welcome', 'bedroom'];\nconst HYPHENATOR_EXPECTED = [\n  ['Seg', 'ment'],\n  ['ex', 'pect', 'ed'],\n  ['com', 'mu', 'ni', 'ca', 'tion'],\n  ['win', 'dow'],\n  ['wel', 'come'],\n  ['bed', 'room'],\n];\n\nconst PARAGRAPH_TEST_CASES: [string, [string, number, number][]][] = [\n  ['Single paragraph.', [['Single paragraph.', 0, 17]]],\n  [\n    'Paragraph 1.\\n\\nParagraph 2.',\n    [\n      ['Paragraph 1.', 0, 12],\n      ['Paragraph 2.', 14, 26],\n    ],\n  ],\n  [\n    'Para 1.\\n\\nPara 2.\\n\\nPara 3.',\n    [\n      ['Para 1.', 0, 7],\n      ['Para 2.', 9, 16],\n      ['Para 3.', 18, 25],\n    ],\n  ],\n  ['\\n\\nParagraph with leading newlines.', [['Paragraph with leading newlines.', 2, 34]]],\n  ['Paragraph with trailing newlines.\\n\\n', [['Paragraph with trailing newlines.', 0, 33]]],\n  [\n    '\\n\\n  Paragraph with leading and trailing spaces.  \\n\\n',\n    [['Paragraph with leading and trailing spaces.', 4, 47]],\n  ],\n  [\n    'Para 1.\\n\\n\\n\\nPara 2.', // Multiple newlines between paragraphs\n    [\n      ['Para 1.', 0, 7],\n      ['Para 2.', 11, 18],\n    ],\n  ],\n  [\n    'Para 1.\\n \\n \\nPara 2.', // Newlines with spaces between paragraphs\n    [\n      ['Para 1.', 0, 7],\n      ['Para 2.', 12, 19],\n    ],\n  ],\n  [\n    '', // Empty string\n    [],\n  ],\n  [\n    '\\n\\n\\n', // Only newlines\n    [],\n  ],\n  [\n    'Line 1\\nLine 2\\nLine 3', // Single paragraph with newlines\n    [['Line 1\\nLine 2\\nLine 3', 0, 20]],\n  ],\n];\n\ndescribe('tokenizer', () => {\n  describe('SentenceTokenizer', () => {\n    const tokenizer = new SentenceTokenizer();\n\n    it('should tokenize sentences correctly', () => {\n      expect(tokenizer.tokenize(TEXT).every((x, i) => EXPECTED_MIN_20[i] === x)).toBeTruthy();\n    });\n\n    it('should stream tokenize sentences correctly', async () => {\n      const pattern = [1, 2, 4];\n      let text = TEXT;\n      const chunks = [];\n      const patternIter = Array(Math.ceil(text.length / pattern.reduce((sum, num) => sum + num, 0)))\n        .fill(pattern)\n        .flat()\n        [Symbol.iterator]();\n\n      for (const size of patternIter) {\n        if (!text) break;\n        chunks.push(text.slice(undefined, size));\n        text = text.slice(size);\n      }\n      const stream = tokenizer.stream();\n      for (const chunk of chunks) {\n        stream.pushText(chunk);\n      }\n      stream.endInput();\n      stream.close();\n\n      for (const x of EXPECTED_MIN_20) {\n        await stream.next().then((value) => {\n          if (value.value) {\n            expect(value.value.token).toStrictEqual(x);\n          }\n        });\n      }\n    });\n  });\n  describe('WordTokenizer', () => {\n    const tokenizer = new WordTokenizer();\n\n    it('should tokenize words correctly', () => {\n      expect(tokenizer.tokenize(WORDS_TEXT).every((x, i) => WORDS_EXPECTED[i] === x)).toBeTruthy();\n    });\n\n    it('should stream tokenize words correctly', async () => {\n      const pattern = [1, 2, 4];\n      let text = WORDS_TEXT;\n      const chunks = [];\n      const patternIter = Array(Math.ceil(text.length / pattern.reduce((sum, num) => sum + num, 0)))\n        .fill(pattern)\n        .flat()\n        [Symbol.iterator]();\n\n      for (const size of patternIter) {\n        if (!text) break;\n        chunks.push(text.slice(undefined, size));\n        text = text.slice(size);\n      }\n      const stream = tokenizer.stream();\n      for (const chunk of chunks) {\n        stream.pushText(chunk);\n      }\n      stream.endInput();\n      stream.close();\n\n      for (const x of WORDS_EXPECTED) {\n        await stream.next().then((value) => {\n          if (value.value) {\n            expect(value.value.token).toStrictEqual(x);\n          }\n        });\n      }\n    });\n\n    describe('punctuation handling', () => {\n      const tokenizerPunct = new WordTokenizer(false);\n\n      it('should tokenize words correctly', () => {\n        expect(\n          tokenizerPunct.tokenize(WORDS_PUNCT_TEXT).every((x, i) => WORDS_PUNCT_EXPECTED[i] === x),\n        ).toBeTruthy();\n      });\n\n      it('should stream tokenize words correctly', async () => {\n        const pattern = [1, 2, 4];\n        let text = WORDS_PUNCT_TEXT;\n        const chunks = [];\n        const patternIter = Array(\n          Math.ceil(text.length / pattern.reduce((sum, num) => sum + num, 0)),\n        )\n          .fill(pattern)\n          .flat()\n          [Symbol.iterator]();\n\n        for (const size of patternIter) {\n          if (!text) break;\n          chunks.push(text.slice(undefined, size));\n          text = text.slice(size);\n        }\n        const stream = tokenizerPunct.stream();\n        for (const chunk of chunks) {\n          stream.pushText(chunk);\n        }\n        stream.endInput();\n        stream.close();\n\n        for (const x of WORDS_PUNCT_EXPECTED) {\n          await stream.next().then((value) => {\n            if (value.value) {\n              expect(value.value.token).toStrictEqual(x);\n            }\n          });\n        }\n      });\n    });\n  });\n  describe('hyphenateWord', () => {\n    it('should hyphenate correctly', () => {\n      HYPHENATOR_TEXT.forEach((x, i) => {\n        expect(hyphenateWord(x)).toStrictEqual(HYPHENATOR_EXPECTED[i]);\n      });\n    });\n  });\n  describe('splitParagraphs', () => {\n    it('should tokenize paragraphs correctly', () => {\n      PARAGRAPH_TEST_CASES.forEach(([a, b]) => {\n        expect(splitParagraphs(a)).toStrictEqual(b);\n      });\n    });\n  });\n});\n"],"mappings":";AAGA,oBAAqC;AACrC,mBAAgE;AAChE,uBAAgC;AAEhC,MAAM,OACJ;AAYF,MAAM,kBAAkB;AAAA,EACtB;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AACF;AAEA,MAAM,aAAa;AACnB,MAAM,iBAAiB;AAAA,EACrB;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AACF;AAEA,MAAM,mBACJ;AACF,MAAM,uBAAuB;AAAA,EAC3B;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AACF;AAEA,MAAM,kBAAkB,CAAC,WAAW,YAAY,iBAAiB,UAAU,WAAW,SAAS;AAC/F,MAAM,sBAAsB;AAAA,EAC1B,CAAC,OAAO,MAAM;AAAA,EACd,CAAC,MAAM,QAAQ,IAAI;AAAA,EACnB,CAAC,OAAO,MAAM,MAAM,MAAM,MAAM;AAAA,EAChC,CAAC,OAAO,KAAK;AAAA,EACb,CAAC,OAAO,MAAM;AAAA,EACd,CAAC,OAAO,MAAM;AAChB;AAEA,MAAM,uBAA+D;AAAA,EACnE,CAAC,qBAAqB,CAAC,CAAC,qBAAqB,GAAG,EAAE,CAAC,CAAC;AAAA,EACpD;AAAA,IACE;AAAA,IACA;AAAA,MACE,CAAC,gBAAgB,GAAG,EAAE;AAAA,MACtB,CAAC,gBAAgB,IAAI,EAAE;AAAA,IACzB;AAAA,EACF;AAAA,EACA;AAAA,IACE;AAAA,IACA;AAAA,MACE,CAAC,WAAW,GAAG,CAAC;AAAA,MAChB,CAAC,WAAW,GAAG,EAAE;AAAA,MACjB,CAAC,WAAW,IAAI,EAAE;AAAA,IACpB;AAAA,EACF;AAAA,EACA,CAAC,wCAAwC,CAAC,CAAC,oCAAoC,GAAG,EAAE,CAAC,CAAC;AAAA,EACtF,CAAC,yCAAyC,CAAC,CAAC,qCAAqC,GAAG,EAAE,CAAC,CAAC;AAAA,EACxF;AAAA,IACE;AAAA,IACA,CAAC,CAAC,+CAA+C,GAAG,EAAE,CAAC;AAAA,EACzD;AAAA,EACA;AAAA,IACE;AAAA;AAAA,IACA;AAAA,MACE,CAAC,WAAW,GAAG,CAAC;AAAA,MAChB,CAAC,WAAW,IAAI,EAAE;AAAA,IACpB;AAAA,EACF;AAAA,EACA;AAAA,IACE;AAAA;AAAA,IACA;AAAA,MACE,CAAC,WAAW,GAAG,CAAC;AAAA,MAChB,CAAC,WAAW,IAAI,EAAE;AAAA,IACpB;AAAA,EACF;AAAA,EACA;AAAA,IACE;AAAA;AAAA,IACA,CAAC;AAAA,EACH;AAAA,EACA;AAAA,IACE;AAAA;AAAA,IACA,CAAC;AAAA,EACH;AAAA,EACA;AAAA,IACE;AAAA;AAAA,IACA,CAAC,CAAC,0BAA0B,GAAG,EAAE,CAAC;AAAA,EACpC;AACF;AAAA,IAEA,wBAAS,aAAa,MAAM;AAC1B,8BAAS,qBAAqB,MAAM;AAClC,UAAM,YAAY,IAAI,+BAAkB;AAExC,0BAAG,uCAAuC,MAAM;AAC9C,gCAAO,UAAU,SAAS,IAAI,EAAE,MAAM,CAAC,GAAG,MAAM,gBAAgB,CAAC,MAAM,CAAC,CAAC,EAAE,WAAW;AAAA,IACxF,CAAC;AAED,0BAAG,8CAA8C,YAAY;AAC3D,YAAM,UAAU,CAAC,GAAG,GAAG,CAAC;AACxB,UAAI,OAAO;AACX,YAAM,SAAS,CAAC;AAChB,YAAM,cAAc,MAAM,KAAK,KAAK,KAAK,SAAS,QAAQ,OAAO,CAAC,KAAK,QAAQ,MAAM,KAAK,CAAC,CAAC,CAAC,EAC1F,KAAK,OAAO,EACZ,KAAK,EACL,OAAO,QAAQ,EAAE;AAEpB,iBAAW,QAAQ,aAAa;AAC9B,YAAI,CAAC,KAAM;AACX,eAAO,KAAK,KAAK,MAAM,QAAW,IAAI,CAAC;AACvC,eAAO,KAAK,MAAM,IAAI;AAAA,MACxB;AACA,YAAM,SAAS,UAAU,OAAO;AAChC,iBAAW,SAAS,QAAQ;AAC1B,eAAO,SAAS,KAAK;AAAA,MACvB;AACA,aAAO,SAAS;AAChB,aAAO,MAAM;AAEb,iBAAW,KAAK,iBAAiB;AAC/B,cAAM,OAAO,KAAK,EAAE,KAAK,CAAC,UAAU;AAClC,cAAI,MAAM,OAAO;AACf,sCAAO,MAAM,MAAM,KAAK,EAAE,cAAc,CAAC;AAAA,UAC3C;AAAA,QACF,CAAC;AAAA,MACH;AAAA,IACF,CAAC;AAAA,EACH,CAAC;AACD,8BAAS,iBAAiB,MAAM;AAC9B,UAAM,YAAY,IAAI,2BAAc;AAEpC,0BAAG,mCAAmC,MAAM;AAC1C,gCAAO,UAAU,SAAS,UAAU,EAAE,MAAM,CAAC,GAAG,MAAM,eAAe,CAAC,MAAM,CAAC,CAAC,EAAE,WAAW;AAAA,IAC7F,CAAC;AAED,0BAAG,0CAA0C,YAAY;AACvD,YAAM,UAAU,CAAC,GAAG,GAAG,CAAC;AACxB,UAAI,OAAO;AACX,YAAM,SAAS,CAAC;AAChB,YAAM,cAAc,MAAM,KAAK,KAAK,KAAK,SAAS,QAAQ,OAAO,CAAC,KAAK,QAAQ,MAAM,KAAK,CAAC,CAAC,CAAC,EAC1F,KAAK,OAAO,EACZ,KAAK,EACL,OAAO,QAAQ,EAAE;AAEpB,iBAAW,QAAQ,aAAa;AAC9B,YAAI,CAAC,KAAM;AACX,eAAO,KAAK,KAAK,MAAM,QAAW,IAAI,CAAC;AACvC,eAAO,KAAK,MAAM,IAAI;AAAA,MACxB;AACA,YAAM,SAAS,UAAU,OAAO;AAChC,iBAAW,SAAS,QAAQ;AAC1B,eAAO,SAAS,KAAK;AAAA,MACvB;AACA,aAAO,SAAS;AAChB,aAAO,MAAM;AAEb,iBAAW,KAAK,gBAAgB;AAC9B,cAAM,OAAO,KAAK,EAAE,KAAK,CAAC,UAAU;AAClC,cAAI,MAAM,OAAO;AACf,sCAAO,MAAM,MAAM,KAAK,EAAE,cAAc,CAAC;AAAA,UAC3C;AAAA,QACF,CAAC;AAAA,MACH;AAAA,IACF,CAAC;AAED,gCAAS,wBAAwB,MAAM;AACrC,YAAM,iBAAiB,IAAI,2BAAc,KAAK;AAE9C,4BAAG,mCAAmC,MAAM;AAC1C;AAAA,UACE,eAAe,SAAS,gBAAgB,EAAE,MAAM,CAAC,GAAG,MAAM,qBAAqB,CAAC,MAAM,CAAC;AAAA,QACzF,EAAE,WAAW;AAAA,MACf,CAAC;AAED,4BAAG,0CAA0C,YAAY;AACvD,cAAM,UAAU,CAAC,GAAG,GAAG,CAAC;AACxB,YAAI,OAAO;AACX,cAAM,SAAS,CAAC;AAChB,cAAM,cAAc;AAAA,UAClB,KAAK,KAAK,KAAK,SAAS,QAAQ,OAAO,CAAC,KAAK,QAAQ,MAAM,KAAK,CAAC,CAAC;AAAA,QACpE,EACG,KAAK,OAAO,EACZ,KAAK,EACL,OAAO,QAAQ,EAAE;AAEpB,mBAAW,QAAQ,aAAa;AAC9B,cAAI,CAAC,KAAM;AACX,iBAAO,KAAK,KAAK,MAAM,QAAW,IAAI,CAAC;AACvC,iBAAO,KAAK,MAAM,IAAI;AAAA,QACxB;AACA,cAAM,SAAS,eAAe,OAAO;AACrC,mBAAW,SAAS,QAAQ;AAC1B,iBAAO,SAAS,KAAK;AAAA,QACvB;AACA,eAAO,SAAS;AAChB,eAAO,MAAM;AAEb,mBAAW,KAAK,sBAAsB;AACpC,gBAAM,OAAO,KAAK,EAAE,KAAK,CAAC,UAAU;AAClC,gBAAI,MAAM,OAAO;AACf,wCAAO,MAAM,MAAM,KAAK,EAAE,cAAc,CAAC;AAAA,YAC3C;AAAA,UACF,CAAC;AAAA,QACH;AAAA,MACF,CAAC;AAAA,IACH,CAAC;AAAA,EACH,CAAC;AACD,8BAAS,iBAAiB,MAAM;AAC9B,0BAAG,8BAA8B,MAAM;AACrC,sBAAgB,QAAQ,CAAC,GAAG,MAAM;AAChC,sCAAO,4BAAc,CAAC,CAAC,EAAE,cAAc,oBAAoB,CAAC,CAAC;AAAA,MAC/D,CAAC;AAAA,IACH,CAAC;AAAA,EACH,CAAC;AACD,8BAAS,mBAAmB,MAAM;AAChC,0BAAG,wCAAwC,MAAM;AAC/C,2BAAqB,QAAQ,CAAC,CAAC,GAAG,CAAC,MAAM;AACvC,sCAAO,kCAAgB,CAAC,CAAC,EAAE,cAAc,CAAC;AAAA,MAC5C,CAAC;AAAA,IACH,CAAC;AAAA,EACH,CAAC;AACH,CAAC;","names":[]}

package/dist/tokenize/tokenizer.test.js

@@ -1,13 +1,15 @@
 import { describe, expect, it } from "vitest";
 import { SentenceTokenizer, WordTokenizer, hyphenateWord } from "./basic/index.js";
 import { splitParagraphs } from "./basic/paragraph.js";
-const TEXT = "Hi! LiveKit is a platform for live audio and video applications and services. R.T.C stands for Real-Time Communication... again R.T.C. Mr. Theo is testing the sentence tokenizer. This is a test. Another test. A short sentence. A longer sentence that is longer than the previous sentence. f(x) = x * 2.54 + 42. Hey! Hi! Hello! ";
+const TEXT = "Hi! LiveKit is a platform for live audio and video applications and services. R.T.C stands for Real-Time Communication... again R.T.C. Mr. Theo is testing the sentence tokenizer. This is a test. Another test. A short sentence. A longer sentence that is longer than the previous sentence. Find additional resources on livekit.com. Find additional resources on docs.livekit.com. f(x) = x * 2.54 + 42. Hey! Hi! Hello! ";
 const EXPECTED_MIN_20 = [
   "Hi! LiveKit is a platform for live audio and video applications and services.",
   "R.T.C stands for Real-Time Communication... again R.T.C.",
   "Mr. Theo is testing the sentence tokenizer.",
   "This is a test. Another test.",
   "A short sentence. A longer sentence that is longer than the previous sentence.",
+  "Find additional resources on livekit.com.",
+  "Find additional resources on docs.livekit.com.",
   "f(x) = x * 2.54 + 42.",
   "Hey! Hi! Hello!"
 ];

package/dist/tokenize/tokenizer.test.js.map

@@ -1 +1 @@
-{"version":3,"sources":["../../src/tokenize/tokenizer.test.ts"],"sourcesContent":["// SPDX-FileCopyrightText: 2024 LiveKit, Inc.\n//\n// SPDX-License-Identifier: Apache-2.0\nimport { describe, expect, it } from 'vitest';\nimport { SentenceTokenizer, WordTokenizer, hyphenateWord } from './basic/index.js';\nimport { splitParagraphs } from './basic/paragraph.js';\n\nconst TEXT =\n  'Hi! ' +\n  'LiveKit is a platform for live audio and video applications and services. ' +\n  'R.T.C stands for Real-Time Communication... again R.T.C. ' +\n  'Mr. Theo is testing the sentence tokenizer. ' +\n  'This is a test. Another test. ' +\n  'A short sentence. ' +\n  'A longer sentence that is longer than the previous sentence. ' +\n  'f(x) = x * 2.54 + 42. ' +\n  'Hey! Hi! Hello! ';\n\nconst EXPECTED_MIN_20 = [\n  'Hi! LiveKit is a platform for live audio and video applications and services.',\n  'R.T.C stands for Real-Time Communication... again R.T.C.',\n  'Mr. Theo is testing the sentence tokenizer.',\n  'This is a test. Another test.',\n  'A short sentence. A longer sentence that is longer than the previous sentence.',\n  'f(x) = x * 2.54 + 42.',\n  'Hey! Hi! Hello!',\n];\n\nconst WORDS_TEXT = 'This is a test. Blabla another test! multiple consecutive spaces:     done';\nconst WORDS_EXPECTED = [\n  'This',\n  'is',\n  'a',\n  'test',\n  'Blabla',\n  'another',\n  'test',\n  'multiple',\n  'consecutive',\n  'spaces',\n  'done',\n];\n\nconst WORDS_PUNCT_TEXT =\n  'This is <phoneme alphabet=\"cmu-arpabet\" ph=\"AE K CH UW AH L IY\">actually</phoneme> tricky to handle.';\nconst WORDS_PUNCT_EXPECTED = [\n  'This',\n  'is',\n  '<phoneme',\n  'alphabet=\"cmu-arpabet\"',\n  'ph=\"AE',\n  'K',\n  'CH',\n  'UW',\n  'AH',\n  'L',\n  'IY\">actually</phoneme>',\n  'tricky',\n  'to',\n  'handle.',\n];\n\nconst HYPHENATOR_TEXT = ['Segment', 'expected', 'communication', 'window', 'welcome', 'bedroom'];\nconst HYPHENATOR_EXPECTED = [\n  ['Seg', 'ment'],\n  ['ex', 'pect', 'ed'],\n  ['com', 'mu', 'ni', 'ca', 'tion'],\n  ['win', 'dow'],\n  ['wel', 'come'],\n  ['bed', 'room'],\n];\n\nconst PARAGRAPH_TEST_CASES: [string, [string, number, number][]][] = [\n  ['Single paragraph.', [['Single paragraph.', 0, 17]]],\n  [\n    'Paragraph 1.\\n\\nParagraph 2.',\n    [\n      ['Paragraph 1.', 0, 12],\n      ['Paragraph 2.', 14, 26],\n    ],\n  ],\n  [\n    'Para 1.\\n\\nPara 2.\\n\\nPara 3.',\n    [\n      ['Para 1.', 0, 7],\n      ['Para 2.', 9, 16],\n      ['Para 3.', 18, 25],\n    ],\n  ],\n  ['\\n\\nParagraph with leading newlines.', [['Paragraph with leading newlines.', 2, 34]]],\n  ['Paragraph with trailing newlines.\\n\\n', [['Paragraph with trailing newlines.', 0, 33]]],\n  [\n    '\\n\\n  Paragraph with leading and trailing spaces.  \\n\\n',\n    [['Paragraph with leading and trailing spaces.', 4, 47]],\n  ],\n  [\n    'Para 1.\\n\\n\\n\\nPara 2.', // Multiple newlines between paragraphs\n    [\n      ['Para 1.', 0, 7],\n      ['Para 2.', 11, 18],\n    ],\n  ],\n  [\n    'Para 1.\\n \\n \\nPara 2.', // Newlines with spaces between paragraphs\n    [\n      ['Para 1.', 0, 7],\n      ['Para 2.', 12, 19],\n    ],\n  ],\n  [\n    '', // Empty string\n    [],\n  ],\n  [\n    '\\n\\n\\n', // Only newlines\n    [],\n  ],\n  [\n    'Line 1\\nLine 2\\nLine 3', // Single paragraph with newlines\n    [['Line 1\\nLine 2\\nLine 3', 0, 20]],\n  ],\n];\n\ndescribe('tokenizer', () => {\n  describe('SentenceTokenizer', () => {\n    const tokenizer = new SentenceTokenizer();\n\n    it('should tokenize sentences correctly', () => {\n      expect(tokenizer.tokenize(TEXT).every((x, i) => EXPECTED_MIN_20[i] === x)).toBeTruthy();\n    });\n\n    it('should stream tokenize sentences correctly', async () => {\n      const pattern = [1, 2, 4];\n      let text = TEXT;\n      const chunks = [];\n      const patternIter = Array(Math.ceil(text.length / pattern.reduce((sum, num) => sum + num, 0)))\n        .fill(pattern)\n        .flat()\n        [Symbol.iterator]();\n\n      for (const size of patternIter) {\n        if (!text) break;\n        chunks.push(text.slice(undefined, size));\n        text = text.slice(size);\n      }\n      const stream = tokenizer.stream();\n      for (const chunk of chunks) {\n        stream.pushText(chunk);\n      }\n      stream.endInput();\n      stream.close();\n\n      for (const x of EXPECTED_MIN_20) {\n        await stream.next().then((value) => {\n          if (value.value) {\n            expect(value.value.token).toStrictEqual(x);\n          }\n        });\n      }\n    });\n  });\n  describe('WordTokenizer', () => {\n    const tokenizer = new WordTokenizer();\n\n    it('should tokenize words correctly', () => {\n      expect(tokenizer.tokenize(WORDS_TEXT).every((x, i) => WORDS_EXPECTED[i] === x)).toBeTruthy();\n    });\n\n    it('should stream tokenize words correctly', async () => {\n      const pattern = [1, 2, 4];\n      let text = WORDS_TEXT;\n      const chunks = [];\n      const patternIter = Array(Math.ceil(text.length / pattern.reduce((sum, num) => sum + num, 0)))\n        .fill(pattern)\n        .flat()\n        [Symbol.iterator]();\n\n      for (const size of patternIter) {\n        if (!text) break;\n        chunks.push(text.slice(undefined, size));\n        text = text.slice(size);\n      }\n      const stream = tokenizer.stream();\n      for (const chunk of chunks) {\n        stream.pushText(chunk);\n      }\n      stream.endInput();\n      stream.close();\n\n      for (const x of WORDS_EXPECTED) {\n        await stream.next().then((value) => {\n          if (value.value) {\n            expect(value.value.token).toStrictEqual(x);\n          }\n        });\n      }\n    });\n\n    describe('punctuation handling', () => {\n      const tokenizerPunct = new WordTokenizer(false);\n\n      it('should tokenize words correctly', () => {\n        expect(\n          tokenizerPunct.tokenize(WORDS_PUNCT_TEXT).every((x, i) => WORDS_PUNCT_EXPECTED[i] === x),\n        ).toBeTruthy();\n      });\n\n      it('should stream tokenize words correctly', async () => {\n        const pattern = [1, 2, 4];\n        let text = WORDS_PUNCT_TEXT;\n        const chunks = [];\n        const patternIter = Array(\n          Math.ceil(text.length / pattern.reduce((sum, num) => sum + num, 0)),\n        )\n          .fill(pattern)\n          .flat()\n          [Symbol.iterator]();\n\n        for (const size of patternIter) {\n          if (!text) break;\n          chunks.push(text.slice(undefined, size));\n          text = text.slice(size);\n        }\n        const stream = tokenizerPunct.stream();\n        for (const chunk of chunks) {\n          stream.pushText(chunk);\n        }\n        stream.endInput();\n        stream.close();\n\n        for (const x of WORDS_PUNCT_EXPECTED) {\n          await stream.next().then((value) => {\n            if (value.value) {\n              expect(value.value.token).toStrictEqual(x);\n            }\n          });\n        }\n      });\n    });\n  });\n  describe('hyphenateWord', () => {\n    it('should hyphenate correctly', () => {\n      HYPHENATOR_TEXT.forEach((x, i) => {\n        expect(hyphenateWord(x)).toStrictEqual(HYPHENATOR_EXPECTED[i]);\n      });\n    });\n  });\n  describe('splitParagraphs', () => {\n    it('should tokenize paragraphs correctly', () => {\n      PARAGRAPH_TEST_CASES.forEach(([a, b]) => {\n        expect(splitParagraphs(a)).toStrictEqual(b);\n      });\n    });\n  });\n});\n"],"mappings":"AAGA,SAAS,UAAU,QAAQ,UAAU;AACrC,SAAS,mBAAmB,eAAe,qBAAqB;AAChE,SAAS,uBAAuB;AAEhC,MAAM,OACJ;AAUF,MAAM,kBAAkB;AAAA,EACtB;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AACF;AAEA,MAAM,aAAa;AACnB,MAAM,iBAAiB;AAAA,EACrB;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AACF;AAEA,MAAM,mBACJ;AACF,MAAM,uBAAuB;AAAA,EAC3B;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AACF;AAEA,MAAM,kBAAkB,CAAC,WAAW,YAAY,iBAAiB,UAAU,WAAW,SAAS;AAC/F,MAAM,sBAAsB;AAAA,EAC1B,CAAC,OAAO,MAAM;AAAA,EACd,CAAC,MAAM,QAAQ,IAAI;AAAA,EACnB,CAAC,OAAO,MAAM,MAAM,MAAM,MAAM;AAAA,EAChC,CAAC,OAAO,KAAK;AAAA,EACb,CAAC,OAAO,MAAM;AAAA,EACd,CAAC,OAAO,MAAM;AAChB;AAEA,MAAM,uBAA+D;AAAA,EACnE,CAAC,qBAAqB,CAAC,CAAC,qBAAqB,GAAG,EAAE,CAAC,CAAC;AAAA,EACpD;AAAA,IACE;AAAA,IACA;AAAA,MACE,CAAC,gBAAgB,GAAG,EAAE;AAAA,MACtB,CAAC,gBAAgB,IAAI,EAAE;AAAA,IACzB;AAAA,EACF;AAAA,EACA;AAAA,IACE;AAAA,IACA;AAAA,MACE,CAAC,WAAW,GAAG,CAAC;AAAA,MAChB,CAAC,WAAW,GAAG,EAAE;AAAA,MACjB,CAAC,WAAW,IAAI,EAAE;AAAA,IACpB;AAAA,EACF;AAAA,EACA,CAAC,wCAAwC,CAAC,CAAC,oCAAoC,GAAG,EAAE,CAAC,CAAC;AAAA,EACtF,CAAC,yCAAyC,CAAC,CAAC,qCAAqC,GAAG,EAAE,CAAC,CAAC;AAAA,EACxF;AAAA,IACE;AAAA,IACA,CAAC,CAAC,+CAA+C,GAAG,EAAE,CAAC;AAAA,EACzD;AAAA,EACA;AAAA,IACE;AAAA;AAAA,IACA;AAAA,MACE,CAAC,WAAW,GAAG,CAAC;AAAA,MAChB,CAAC,WAAW,IAAI,EAAE;AAAA,IACpB;AAAA,EACF;AAAA,EACA;AAAA,IACE;AAAA;AAAA,IACA;AAAA,MACE,CAAC,WAAW,GAAG,CAAC;AAAA,MAChB,CAAC,WAAW,IAAI,EAAE;AAAA,IACpB;AAAA,EACF;AAAA,EACA;AAAA,IACE;AAAA;AAAA,IACA,CAAC;AAAA,EACH;AAAA,EACA;AAAA,IACE;AAAA;AAAA,IACA,CAAC;AAAA,EACH;AAAA,EACA;AAAA,IACE;AAAA;AAAA,IACA,CAAC,CAAC,0BAA0B,GAAG,EAAE,CAAC;AAAA,EACpC;AACF;AAEA,SAAS,aAAa,MAAM;AAC1B,WAAS,qBAAqB,MAAM;AAClC,UAAM,YAAY,IAAI,kBAAkB;AAExC,OAAG,uCAAuC,MAAM;AAC9C,aAAO,UAAU,SAAS,IAAI,EAAE,MAAM,CAAC,GAAG,MAAM,gBAAgB,CAAC,MAAM,CAAC,CAAC,EAAE,WAAW;AAAA,IACxF,CAAC;AAED,OAAG,8CAA8C,YAAY;AAC3D,YAAM,UAAU,CAAC,GAAG,GAAG,CAAC;AACxB,UAAI,OAAO;AACX,YAAM,SAAS,CAAC;AAChB,YAAM,cAAc,MAAM,KAAK,KAAK,KAAK,SAAS,QAAQ,OAAO,CAAC,KAAK,QAAQ,MAAM,KAAK,CAAC,CAAC,CAAC,EAC1F,KAAK,OAAO,EACZ,KAAK,EACL,OAAO,QAAQ,EAAE;AAEpB,iBAAW,QAAQ,aAAa;AAC9B,YAAI,CAAC,KAAM;AACX,eAAO,KAAK,KAAK,MAAM,QAAW,IAAI,CAAC;AACvC,eAAO,KAAK,MAAM,IAAI;AAAA,MACxB;AACA,YAAM,SAAS,UAAU,OAAO;AAChC,iBAAW,SAAS,QAAQ;AAC1B,eAAO,SAAS,KAAK;AAAA,MACvB;AACA,aAAO,SAAS;AAChB,aAAO,MAAM;AAEb,iBAAW,KAAK,iBAAiB;AAC/B,cAAM,OAAO,KAAK,EAAE,KAAK,CAAC,UAAU;AAClC,cAAI,MAAM,OAAO;AACf,mBAAO,MAAM,MAAM,KAAK,EAAE,cAAc,CAAC;AAAA,UAC3C;AAAA,QACF,CAAC;AAAA,MACH;AAAA,IACF,CAAC;AAAA,EACH,CAAC;AACD,WAAS,iBAAiB,MAAM;AAC9B,UAAM,YAAY,IAAI,cAAc;AAEpC,OAAG,mCAAmC,MAAM;AAC1C,aAAO,UAAU,SAAS,UAAU,EAAE,MAAM,CAAC,GAAG,MAAM,eAAe,CAAC,MAAM,CAAC,CAAC,EAAE,WAAW;AAAA,IAC7F,CAAC;AAED,OAAG,0CAA0C,YAAY;AACvD,YAAM,UAAU,CAAC,GAAG,GAAG,CAAC;AACxB,UAAI,OAAO;AACX,YAAM,SAAS,CAAC;AAChB,YAAM,cAAc,MAAM,KAAK,KAAK,KAAK,SAAS,QAAQ,OAAO,CAAC,KAAK,QAAQ,MAAM,KAAK,CAAC,CAAC,CAAC,EAC1F,KAAK,OAAO,EACZ,KAAK,EACL,OAAO,QAAQ,EAAE;AAEpB,iBAAW,QAAQ,aAAa;AAC9B,YAAI,CAAC,KAAM;AACX,eAAO,KAAK,KAAK,MAAM,QAAW,IAAI,CAAC;AACvC,eAAO,KAAK,MAAM,IAAI;AAAA,MACxB;AACA,YAAM,SAAS,UAAU,OAAO;AAChC,iBAAW,SAAS,QAAQ;AAC1B,eAAO,SAAS,KAAK;AAAA,MACvB;AACA,aAAO,SAAS;AAChB,aAAO,MAAM;AAEb,iBAAW,KAAK,gBAAgB;AAC9B,cAAM,OAAO,KAAK,EAAE,KAAK,CAAC,UAAU;AAClC,cAAI,MAAM,OAAO;AACf,mBAAO,MAAM,MAAM,KAAK,EAAE,cAAc,CAAC;AAAA,UAC3C;AAAA,QACF,CAAC;AAAA,MACH;AAAA,IACF,CAAC;AAED,aAAS,wBAAwB,MAAM;AACrC,YAAM,iBAAiB,IAAI,cAAc,KAAK;AAE9C,SAAG,mCAAmC,MAAM;AAC1C;AAAA,UACE,eAAe,SAAS,gBAAgB,EAAE,MAAM,CAAC,GAAG,MAAM,qBAAqB,CAAC,MAAM,CAAC;AAAA,QACzF,EAAE,WAAW;AAAA,MACf,CAAC;AAED,SAAG,0CAA0C,YAAY;AACvD,cAAM,UAAU,CAAC,GAAG,GAAG,CAAC;AACxB,YAAI,OAAO;AACX,cAAM,SAAS,CAAC;AAChB,cAAM,cAAc;AAAA,UAClB,KAAK,KAAK,KAAK,SAAS,QAAQ,OAAO,CAAC,KAAK,QAAQ,MAAM,KAAK,CAAC,CAAC;AAAA,QACpE,EACG,KAAK,OAAO,EACZ,KAAK,EACL,OAAO,QAAQ,EAAE;AAEpB,mBAAW,QAAQ,aAAa;AAC9B,cAAI,CAAC,KAAM;AACX,iBAAO,KAAK,KAAK,MAAM,QAAW,IAAI,CAAC;AACvC,iBAAO,KAAK,MAAM,IAAI;AAAA,QACxB;AACA,cAAM,SAAS,eAAe,OAAO;AACrC,mBAAW,SAAS,QAAQ;AAC1B,iBAAO,SAAS,KAAK;AAAA,QACvB;AACA,eAAO,SAAS;AAChB,eAAO,MAAM;AAEb,mBAAW,KAAK,sBAAsB;AACpC,gBAAM,OAAO,KAAK,EAAE,KAAK,CAAC,UAAU;AAClC,gBAAI,MAAM,OAAO;AACf,qBAAO,MAAM,MAAM,KAAK,EAAE,cAAc,CAAC;AAAA,YAC3C;AAAA,UACF,CAAC;AAAA,QACH;AAAA,MACF,CAAC;AAAA,IACH,CAAC;AAAA,EACH,CAAC;AACD,WAAS,iBAAiB,MAAM;AAC9B,OAAG,8BAA8B,MAAM;AACrC,sBAAgB,QAAQ,CAAC,GAAG,MAAM;AAChC,eAAO,cAAc,CAAC,CAAC,EAAE,cAAc,oBAAoB,CAAC,CAAC;AAAA,MAC/D,CAAC;AAAA,IACH,CAAC;AAAA,EACH,CAAC;AACD,WAAS,mBAAmB,MAAM;AAChC,OAAG,wCAAwC,MAAM;AAC/C,2BAAqB,QAAQ,CAAC,CAAC,GAAG,CAAC,MAAM;AACvC,eAAO,gBAAgB,CAAC,CAAC,EAAE,cAAc,CAAC;AAAA,MAC5C,CAAC;AAAA,IACH,CAAC;AAAA,EACH,CAAC;AACH,CAAC;","names":[]}
+{"version":3,"sources":["../../src/tokenize/tokenizer.test.ts"],"sourcesContent":["// SPDX-FileCopyrightText: 2024 LiveKit, Inc.\n//\n// SPDX-License-Identifier: Apache-2.0\nimport { describe, expect, it } from 'vitest';\nimport { SentenceTokenizer, WordTokenizer, hyphenateWord } from './basic/index.js';\nimport { splitParagraphs } from './basic/paragraph.js';\n\nconst TEXT =\n  'Hi! ' +\n  'LiveKit is a platform for live audio and video applications and services. ' +\n  'R.T.C stands for Real-Time Communication... again R.T.C. ' +\n  'Mr. Theo is testing the sentence tokenizer. ' +\n  'This is a test. Another test. ' +\n  'A short sentence. ' +\n  'A longer sentence that is longer than the previous sentence. ' +\n  'Find additional resources on livekit.com. ' +\n  'Find additional resources on docs.livekit.com. ' +\n  'f(x) = x * 2.54 + 42. ' +\n  'Hey! Hi! Hello! ';\n\nconst EXPECTED_MIN_20 = [\n  'Hi! LiveKit is a platform for live audio and video applications and services.',\n  'R.T.C stands for Real-Time Communication... again R.T.C.',\n  'Mr. Theo is testing the sentence tokenizer.',\n  'This is a test. Another test.',\n  'A short sentence. A longer sentence that is longer than the previous sentence.',\n  'Find additional resources on livekit.com.',\n  'Find additional resources on docs.livekit.com.',\n  'f(x) = x * 2.54 + 42.',\n  'Hey! Hi! Hello!',\n];\n\nconst WORDS_TEXT = 'This is a test. Blabla another test! multiple consecutive spaces:     done';\nconst WORDS_EXPECTED = [\n  'This',\n  'is',\n  'a',\n  'test',\n  'Blabla',\n  'another',\n  'test',\n  'multiple',\n  'consecutive',\n  'spaces',\n  'done',\n];\n\nconst WORDS_PUNCT_TEXT =\n  'This is <phoneme alphabet=\"cmu-arpabet\" ph=\"AE K CH UW AH L IY\">actually</phoneme> tricky to handle.';\nconst WORDS_PUNCT_EXPECTED = [\n  'This',\n  'is',\n  '<phoneme',\n  'alphabet=\"cmu-arpabet\"',\n  'ph=\"AE',\n  'K',\n  'CH',\n  'UW',\n  'AH',\n  'L',\n  'IY\">actually</phoneme>',\n  'tricky',\n  'to',\n  'handle.',\n];\n\nconst HYPHENATOR_TEXT = ['Segment', 'expected', 'communication', 'window', 'welcome', 'bedroom'];\nconst HYPHENATOR_EXPECTED = [\n  ['Seg', 'ment'],\n  ['ex', 'pect', 'ed'],\n  ['com', 'mu', 'ni', 'ca', 'tion'],\n  ['win', 'dow'],\n  ['wel', 'come'],\n  ['bed', 'room'],\n];\n\nconst PARAGRAPH_TEST_CASES: [string, [string, number, number][]][] = [\n  ['Single paragraph.', [['Single paragraph.', 0, 17]]],\n  [\n    'Paragraph 1.\\n\\nParagraph 2.',\n    [\n      ['Paragraph 1.', 0, 12],\n      ['Paragraph 2.', 14, 26],\n    ],\n  ],\n  [\n    'Para 1.\\n\\nPara 2.\\n\\nPara 3.',\n    [\n      ['Para 1.', 0, 7],\n      ['Para 2.', 9, 16],\n      ['Para 3.', 18, 25],\n    ],\n  ],\n  ['\\n\\nParagraph with leading newlines.', [['Paragraph with leading newlines.', 2, 34]]],\n  ['Paragraph with trailing newlines.\\n\\n', [['Paragraph with trailing newlines.', 0, 33]]],\n  [\n    '\\n\\n  Paragraph with leading and trailing spaces.  \\n\\n',\n    [['Paragraph with leading and trailing spaces.', 4, 47]],\n  ],\n  [\n    'Para 1.\\n\\n\\n\\nPara 2.', // Multiple newlines between paragraphs\n    [\n      ['Para 1.', 0, 7],\n      ['Para 2.', 11, 18],\n    ],\n  ],\n  [\n    'Para 1.\\n \\n \\nPara 2.', // Newlines with spaces between paragraphs\n    [\n      ['Para 1.', 0, 7],\n      ['Para 2.', 12, 19],\n    ],\n  ],\n  [\n    '', // Empty string\n    [],\n  ],\n  [\n    '\\n\\n\\n', // Only newlines\n    [],\n  ],\n  [\n    'Line 1\\nLine 2\\nLine 3', // Single paragraph with newlines\n    [['Line 1\\nLine 2\\nLine 3', 0, 20]],\n  ],\n];\n\ndescribe('tokenizer', () => {\n  describe('SentenceTokenizer', () => {\n    const tokenizer = new SentenceTokenizer();\n\n    it('should tokenize sentences correctly', () => {\n      expect(tokenizer.tokenize(TEXT).every((x, i) => EXPECTED_MIN_20[i] === x)).toBeTruthy();\n    });\n\n    it('should stream tokenize sentences correctly', async () => {\n      const pattern = [1, 2, 4];\n      let text = TEXT;\n      const chunks = [];\n      const patternIter = Array(Math.ceil(text.length / pattern.reduce((sum, num) => sum + num, 0)))\n        .fill(pattern)\n        .flat()\n        [Symbol.iterator]();\n\n      for (const size of patternIter) {\n        if (!text) break;\n        chunks.push(text.slice(undefined, size));\n        text = text.slice(size);\n      }\n      const stream = tokenizer.stream();\n      for (const chunk of chunks) {\n        stream.pushText(chunk);\n      }\n      stream.endInput();\n      stream.close();\n\n      for (const x of EXPECTED_MIN_20) {\n        await stream.next().then((value) => {\n          if (value.value) {\n            expect(value.value.token).toStrictEqual(x);\n          }\n        });\n      }\n    });\n  });\n  describe('WordTokenizer', () => {\n    const tokenizer = new WordTokenizer();\n\n    it('should tokenize words correctly', () => {\n      expect(tokenizer.tokenize(WORDS_TEXT).every((x, i) => WORDS_EXPECTED[i] === x)).toBeTruthy();\n    });\n\n    it('should stream tokenize words correctly', async () => {\n      const pattern = [1, 2, 4];\n      let text = WORDS_TEXT;\n      const chunks = [];\n      const patternIter = Array(Math.ceil(text.length / pattern.reduce((sum, num) => sum + num, 0)))\n        .fill(pattern)\n        .flat()\n        [Symbol.iterator]();\n\n      for (const size of patternIter) {\n        if (!text) break;\n        chunks.push(text.slice(undefined, size));\n        text = text.slice(size);\n      }\n      const stream = tokenizer.stream();\n      for (const chunk of chunks) {\n        stream.pushText(chunk);\n      }\n      stream.endInput();\n      stream.close();\n\n      for (const x of WORDS_EXPECTED) {\n        await stream.next().then((value) => {\n          if (value.value) {\n            expect(value.value.token).toStrictEqual(x);\n          }\n        });\n      }\n    });\n\n    describe('punctuation handling', () => {\n      const tokenizerPunct = new WordTokenizer(false);\n\n      it('should tokenize words correctly', () => {\n        expect(\n          tokenizerPunct.tokenize(WORDS_PUNCT_TEXT).every((x, i) => WORDS_PUNCT_EXPECTED[i] === x),\n        ).toBeTruthy();\n      });\n\n      it('should stream tokenize words correctly', async () => {\n        const pattern = [1, 2, 4];\n        let text = WORDS_PUNCT_TEXT;\n        const chunks = [];\n        const patternIter = Array(\n          Math.ceil(text.length / pattern.reduce((sum, num) => sum + num, 0)),\n        )\n          .fill(pattern)\n          .flat()\n          [Symbol.iterator]();\n\n        for (const size of patternIter) {\n          if (!text) break;\n          chunks.push(text.slice(undefined, size));\n          text = text.slice(size);\n        }\n        const stream = tokenizerPunct.stream();\n        for (const chunk of chunks) {\n          stream.pushText(chunk);\n        }\n        stream.endInput();\n        stream.close();\n\n        for (const x of WORDS_PUNCT_EXPECTED) {\n          await stream.next().then((value) => {\n            if (value.value) {\n              expect(value.value.token).toStrictEqual(x);\n            }\n          });\n        }\n      });\n    });\n  });\n  describe('hyphenateWord', () => {\n    it('should hyphenate correctly', () => {\n      HYPHENATOR_TEXT.forEach((x, i) => {\n        expect(hyphenateWord(x)).toStrictEqual(HYPHENATOR_EXPECTED[i]);\n      });\n    });\n  });\n  describe('splitParagraphs', () => {\n    it('should tokenize paragraphs correctly', () => {\n      PARAGRAPH_TEST_CASES.forEach(([a, b]) => {\n        expect(splitParagraphs(a)).toStrictEqual(b);\n      });\n    });\n  });\n});\n"],"mappings":"AAGA,SAAS,UAAU,QAAQ,UAAU;AACrC,SAAS,mBAAmB,eAAe,qBAAqB;AAChE,SAAS,uBAAuB;AAEhC,MAAM,OACJ;AAYF,MAAM,kBAAkB;AAAA,EACtB;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AACF;AAEA,MAAM,aAAa;AACnB,MAAM,iBAAiB;AAAA,EACrB;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AACF;AAEA,MAAM,mBACJ;AACF,MAAM,uBAAuB;AAAA,EAC3B;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AACF;AAEA,MAAM,kBAAkB,CAAC,WAAW,YAAY,iBAAiB,UAAU,WAAW,SAAS;AAC/F,MAAM,sBAAsB;AAAA,EAC1B,CAAC,OAAO,MAAM;AAAA,EACd,CAAC,MAAM,QAAQ,IAAI;AAAA,EACnB,CAAC,OAAO,MAAM,MAAM,MAAM,MAAM;AAAA,EAChC,CAAC,OAAO,KAAK;AAAA,EACb,CAAC,OAAO,MAAM;AAAA,EACd,CAAC,OAAO,MAAM;AAChB;AAEA,MAAM,uBAA+D;AAAA,EACnE,CAAC,qBAAqB,CAAC,CAAC,qBAAqB,GAAG,EAAE,CAAC,CAAC;AAAA,EACpD;AAAA,IACE;AAAA,IACA;AAAA,MACE,CAAC,gBAAgB,GAAG,EAAE;AAAA,MACtB,CAAC,gBAAgB,IAAI,EAAE;AAAA,IACzB;AAAA,EACF;AAAA,EACA;AAAA,IACE;AAAA,IACA;AAAA,MACE,CAAC,WAAW,GAAG,CAAC;AAAA,MAChB,CAAC,WAAW,GAAG,EAAE;AAAA,MACjB,CAAC,WAAW,IAAI,EAAE;AAAA,IACpB;AAAA,EACF;AAAA,EACA,CAAC,wCAAwC,CAAC,CAAC,oCAAoC,GAAG,EAAE,CAAC,CAAC;AAAA,EACtF,CAAC,yCAAyC,CAAC,CAAC,qCAAqC,GAAG,EAAE,CAAC,CAAC;AAAA,EACxF;AAAA,IACE;AAAA,IACA,CAAC,CAAC,+CAA+C,GAAG,EAAE,CAAC;AAAA,EACzD;AAAA,EACA;AAAA,IACE;AAAA;AAAA,IACA;AAAA,MACE,CAAC,WAAW,GAAG,CAAC;AAAA,MAChB,CAAC,WAAW,IAAI,EAAE;AAAA,IACpB;AAAA,EACF;AAAA,EACA;AAAA,IACE;AAAA;AAAA,IACA;AAAA,MACE,CAAC,WAAW,GAAG,CAAC;AAAA,MAChB,CAAC,WAAW,IAAI,EAAE;AAAA,IACpB;AAAA,EACF;AAAA,EACA;AAAA,IACE;AAAA;AAAA,IACA,CAAC;AAAA,EACH;AAAA,EACA;AAAA,IACE;AAAA;AAAA,IACA,CAAC;AAAA,EACH;AAAA,EACA;AAAA,IACE;AAAA;AAAA,IACA,CAAC,CAAC,0BAA0B,GAAG,EAAE,CAAC;AAAA,EACpC;AACF;AAEA,SAAS,aAAa,MAAM;AAC1B,WAAS,qBAAqB,MAAM;AAClC,UAAM,YAAY,IAAI,kBAAkB;AAExC,OAAG,uCAAuC,MAAM;AAC9C,aAAO,UAAU,SAAS,IAAI,EAAE,MAAM,CAAC,GAAG,MAAM,gBAAgB,CAAC,MAAM,CAAC,CAAC,EAAE,WAAW;AAAA,IACxF,CAAC;AAED,OAAG,8CAA8C,YAAY;AAC3D,YAAM,UAAU,CAAC,GAAG,GAAG,CAAC;AACxB,UAAI,OAAO;AACX,YAAM,SAAS,CAAC;AAChB,YAAM,cAAc,MAAM,KAAK,KAAK,KAAK,SAAS,QAAQ,OAAO,CAAC,KAAK,QAAQ,MAAM,KAAK,CAAC,CAAC,CAAC,EAC1F,KAAK,OAAO,EACZ,KAAK,EACL,OAAO,QAAQ,EAAE;AAEpB,iBAAW,QAAQ,aAAa;AAC9B,YAAI,CAAC,KAAM;AACX,eAAO,KAAK,KAAK,MAAM,QAAW,IAAI,CAAC;AACvC,eAAO,KAAK,MAAM,IAAI;AAAA,MACxB;AACA,YAAM,SAAS,UAAU,OAAO;AAChC,iBAAW,SAAS,QAAQ;AAC1B,eAAO,SAAS,KAAK;AAAA,MACvB;AACA,aAAO,SAAS;AAChB,aAAO,MAAM;AAEb,iBAAW,KAAK,iBAAiB;AAC/B,cAAM,OAAO,KAAK,EAAE,KAAK,CAAC,UAAU;AAClC,cAAI,MAAM,OAAO;AACf,mBAAO,MAAM,MAAM,KAAK,EAAE,cAAc,CAAC;AAAA,UAC3C;AAAA,QACF,CAAC;AAAA,MACH;AAAA,IACF,CAAC;AAAA,EACH,CAAC;AACD,WAAS,iBAAiB,MAAM;AAC9B,UAAM,YAAY,IAAI,cAAc;AAEpC,OAAG,mCAAmC,MAAM;AAC1C,aAAO,UAAU,SAAS,UAAU,EAAE,MAAM,CAAC,GAAG,MAAM,eAAe,CAAC,MAAM,CAAC,CAAC,EAAE,WAAW;AAAA,IAC7F,CAAC;AAED,OAAG,0CAA0C,YAAY;AACvD,YAAM,UAAU,CAAC,GAAG,GAAG,CAAC;AACxB,UAAI,OAAO;AACX,YAAM,SAAS,CAAC;AAChB,YAAM,cAAc,MAAM,KAAK,KAAK,KAAK,SAAS,QAAQ,OAAO,CAAC,KAAK,QAAQ,MAAM,KAAK,CAAC,CAAC,CAAC,EAC1F,KAAK,OAAO,EACZ,KAAK,EACL,OAAO,QAAQ,EAAE;AAEpB,iBAAW,QAAQ,aAAa;AAC9B,YAAI,CAAC,KAAM;AACX,eAAO,KAAK,KAAK,MAAM,QAAW,IAAI,CAAC;AACvC,eAAO,KAAK,MAAM,IAAI;AAAA,MACxB;AACA,YAAM,SAAS,UAAU,OAAO;AAChC,iBAAW,SAAS,QAAQ;AAC1B,eAAO,SAAS,KAAK;AAAA,MACvB;AACA,aAAO,SAAS;AAChB,aAAO,MAAM;AAEb,iBAAW,KAAK,gBAAgB;AAC9B,cAAM,OAAO,KAAK,EAAE,KAAK,CAAC,UAAU;AAClC,cAAI,MAAM,OAAO;AACf,mBAAO,MAAM,MAAM,KAAK,EAAE,cAAc,CAAC;AAAA,UAC3C;AAAA,QACF,CAAC;AAAA,MACH;AAAA,IACF,CAAC;AAED,aAAS,wBAAwB,MAAM;AACrC,YAAM,iBAAiB,IAAI,cAAc,KAAK;AAE9C,SAAG,mCAAmC,MAAM;AAC1C;AAAA,UACE,eAAe,SAAS,gBAAgB,EAAE,MAAM,CAAC,GAAG,MAAM,qBAAqB,CAAC,MAAM,CAAC;AAAA,QACzF,EAAE,WAAW;AAAA,MACf,CAAC;AAED,SAAG,0CAA0C,YAAY;AACvD,cAAM,UAAU,CAAC,GAAG,GAAG,CAAC;AACxB,YAAI,OAAO;AACX,cAAM,SAAS,CAAC;AAChB,cAAM,cAAc;AAAA,UAClB,KAAK,KAAK,KAAK,SAAS,QAAQ,OAAO,CAAC,KAAK,QAAQ,MAAM,KAAK,CAAC,CAAC;AAAA,QACpE,EACG,KAAK,OAAO,EACZ,KAAK,EACL,OAAO,QAAQ,EAAE;AAEpB,mBAAW,QAAQ,aAAa;AAC9B,cAAI,CAAC,KAAM;AACX,iBAAO,KAAK,KAAK,MAAM,QAAW,IAAI,CAAC;AACvC,iBAAO,KAAK,MAAM,IAAI;AAAA,QACxB;AACA,cAAM,SAAS,eAAe,OAAO;AACrC,mBAAW,SAAS,QAAQ;AAC1B,iBAAO,SAAS,KAAK;AAAA,QACvB;AACA,eAAO,SAAS;AAChB,eAAO,MAAM;AAEb,mBAAW,KAAK,sBAAsB;AACpC,gBAAM,OAAO,KAAK,EAAE,KAAK,CAAC,UAAU;AAClC,gBAAI,MAAM,OAAO;AACf,qBAAO,MAAM,MAAM,KAAK,EAAE,cAAc,CAAC;AAAA,YAC3C;AAAA,UACF,CAAC;AAAA,QACH;AAAA,MACF,CAAC;AAAA,IACH,CAAC;AAAA,EACH,CAAC;AACD,WAAS,iBAAiB,MAAM;AAC9B,OAAG,8BAA8B,MAAM;AACrC,sBAAgB,QAAQ,CAAC,GAAG,MAAM;AAChC,eAAO,cAAc,CAAC,CAAC,EAAE,cAAc,oBAAoB,CAAC,CAAC;AAAA,MAC/D,CAAC;AAAA,IACH,CAAC;AAAA,EACH,CAAC;AACD,WAAS,mBAAmB,MAAM;AAChC,OAAG,wCAAwC,MAAM;AAC/C,2BAAqB,QAAQ,CAAC,CAAC,GAAG,CAAC,MAAM;AACvC,eAAO,gBAAgB,CAAC,CAAC,EAAE,cAAc,CAAC;AAAA,MAC5C,CAAC;AAAA,IACH,CAAC;AAAA,EACH,CAAC;AACH,CAAC;","names":[]}

package/dist/transcription.cjs.map

@@ -1 +1 @@
-{"version":3,"sources":["../src/transcription.ts"],"sourcesContent":["// SPDX-FileCopyrightText: 2024 LiveKit, Inc.\n//\n// SPDX-License-Identifier: Apache-2.0\nimport { TranscriptionSegment } from '@livekit/protocol';\nimport { AudioFrame } from '@livekit/rtc-node';\nimport type { TypedEventEmitter as TypedEmitter } from '@livekit/typed-emitter';\nimport { EventEmitter } from 'node:events';\nimport { basic } from './tokenize/index.js';\nimport type { SentenceStream, SentenceTokenizer } from './tokenize/tokenizer.js';\nimport { AsyncIterableQueue, Future, shortuuid } from './utils.js';\n\n// standard speech rate in hyphens/ms\nconst STANDARD_SPEECH_RATE = 3830;\n\nexport interface TextSyncOptions {\n  language: string;\n  speed: number;\n  newSentenceDelay: number;\n  sentenceTokenizer: SentenceTokenizer;\n  hyphenateWord: (word: string) => string[];\n  splitWords: (words: string) => [string, number, number][];\n}\n\nexport const defaultTextSyncOptions: TextSyncOptions = {\n  language: '',\n  speed: 1,\n  newSentenceDelay: 400,\n  sentenceTokenizer: new basic.SentenceTokenizer(),\n  hyphenateWord: basic.hyphenateWord,\n  splitWords: basic.splitWords,\n};\n\ninterface AudioData {\n  pushedDuration: number;\n  done: boolean;\n}\n\ninterface TextData {\n  sentenceStream: SentenceStream;\n  pushedText: string;\n  done: boolean;\n  forwardedHyphens: number;\n  forwardedSentences: number;\n}\n\ntype SyncCallbacks = {\n  textUpdated: (text: TranscriptionSegment) => void;\n};\n\nexport class TextAudioSynchronizer extends (EventEmitter as new () => TypedEmitter<SyncCallbacks>) {\n  #opts: TextSyncOptions;\n  #speed: number;\n\n  #closed = false;\n  #interrupted = false;\n  #closeFut = new Future();\n\n  #playingSegIndex = -1;\n  #finishedSegIndex = -1;\n\n  #textQChanged = new AsyncIterableQueue<number>();\n  #textQ: (TextData | undefined)[] = [];\n  #audioQChanged = new AsyncIterableQueue<number>();\n  #audioQ: (AudioData | undefined)[] = [];\n\n  #playedText = '';\n  #task?: Promise<void>;\n\n  #audioData?: AudioData;\n  #textData?: TextData;\n\n  constructor(opts: TextSyncOptions) {\n    super();\n\n    this.#opts = opts;\n    this.#speed = opts.speed * STANDARD_SPEECH_RATE;\n  }\n\n  pushAudio(frame: AudioFrame) {\n    this.#checkNotClosed();\n    if (!this.#audioData) {\n      this.#audioData = { pushedDuration: 0, done: false };\n      this.#audioQ.push(this.#audioData);\n      this.#audioQChanged.put(1);\n    }\n    this.#audioData.pushedDuration += frame.samplesPerChannel / frame.sampleRate;\n  }\n\n  pushText(text: string) {\n    this.#checkNotClosed();\n    if (!this.#textData) {\n      this.#textData = {\n        sentenceStream: this.#opts.sentenceTokenizer.stream(),\n        pushedText: '',\n        done: false,\n        forwardedHyphens: 0,\n        forwardedSentences: 0,\n      };\n      this.#textQ.push(this.#textData);\n      this.#textQChanged.put(1);\n    }\n\n    this.#textData.pushedText += text;\n    this.#textData.sentenceStream.pushText(text);\n  }\n\n  markAudioSegmentEnd() {\n    this.#checkNotClosed();\n\n    if (!this.#audioData) {\n      // create empty audio data if none exists\n      this.pushAudio(new AudioFrame(new Int16Array(), 24000, 1, 0));\n    }\n\n    this.#audioData!.done = true;\n    this.#audioData = undefined;\n  }\n\n  markTextSegmentEnd() {\n    this.#checkNotClosed();\n\n    if (!this.#textData) {\n      this.pushText('');\n    }\n\n    this.#textData!.done = true;\n    this.#textData?.sentenceStream.flush();\n    this.#textData?.sentenceStream.close();\n    this.#textData = undefined;\n  }\n\n  segmentPlayoutStarted() {\n    this.#checkNotClosed();\n    this.#playingSegIndex++;\n\n    if (!this.#task) {\n      this.#task = this.#mainLoop();\n    }\n  }\n\n  segmentPlayoutFinished() {\n    this.#checkNotClosed();\n    this.#finishedSegIndex++;\n  }\n\n  get playedText(): string {\n    return this.#playedText;\n  }\n\n  async close(interrupt: boolean) {\n    if (this.#closed) {\n      return;\n    }\n    this.#closed = true;\n    this.#interrupted = interrupt;\n    this.#closeFut.resolve();\n\n    for (const textData of this.#textQ) {\n      textData?.sentenceStream.close();\n    }\n\n    this.#textQ.push(undefined);\n    this.#audioQ.push(undefined);\n    this.#textQChanged.put(1);\n    this.#audioQChanged.put(1);\n\n    await this.#task;\n  }\n\n  async #mainLoop() {\n    let segIndex = 0;\n    let qDone = false;\n\n    while (!qDone) {\n      await this.#textQChanged.next();\n      await this.#audioQChanged.next();\n\n      while (this.#textQ.length && this.#audioQ.length) {\n        const textData = this.#textQ.pop();\n        const audioData = this.#audioQ.pop();\n\n        if (!(textData && audioData)) {\n          qDone = true;\n          break;\n        }\n\n        // wait for segment to start playing\n        while (!this.#closed) {\n          if (this.#playingSegIndex >= segIndex) break;\n          await this.#sleepIfNotClosed(125);\n        }\n\n        const sentenceStream = textData.sentenceStream;\n        const forwardStartTime = Date.now();\n\n        for await (const ev of sentenceStream) {\n          await this.#syncSentence(segIndex, forwardStartTime, textData, audioData, ev.token);\n        }\n\n        segIndex++;\n      }\n    }\n  }\n\n  async #syncSentence(\n    segIndex: number,\n    segStartTime: number,\n    textData: TextData,\n    audioData: AudioData,\n    sentence: string,\n  ) {\n    let realSpeed: number | undefined;\n    if (audioData.pushedDuration > 0 && audioData.done) {\n      realSpeed = this.#calcHyphens(textData.pushedText).length / audioData.pushedDuration;\n    }\n\n    const segId = shortuuid('SG_');\n    const words = this.#opts.splitWords(sentence);\n    const processedWords: string[] = [];\n\n    const ogText = this.#playedText;\n    // eslint-disable-next-line @typescript-eslint/no-unused-vars\n    for (const [word, _, end] of words) {\n      if (segIndex <= this.#finishedSegIndex) break;\n      if (this.#interrupted) return;\n\n      const wordHyphens = this.#opts.hyphenateWord(word).length;\n      processedWords.push(word);\n\n      const elapsed = Date.now() - segStartTime;\n      const text = sentence.slice(0, end); // TODO: rstrip punctuations\n\n      let speed = this.#speed;\n      let delay: number;\n      if (realSpeed) {\n        speed = realSpeed;\n        const estimatedPausesMs = textData.forwardedSentences * this.#opts.newSentenceDelay;\n        const hyphPauses = estimatedPausesMs * speed;\n        const targetHyphens = Math.round(speed * elapsed);\n        const dt = targetHyphens - textData.forwardedHyphens - hyphPauses;\n        const toWaitHyphens = Math.max(0, wordHyphens - dt);\n        delay = toWaitHyphens / speed;\n      } else {\n        delay = wordHyphens / speed;\n      }\n\n      const firstDelay = Math.min(delay / 2, 2 / speed);\n      await this.#sleepIfNotClosed(firstDelay * 1000000);\n\n      this.emit(\n        'textUpdated',\n        new TranscriptionSegment({\n          id: segId,\n          text: text,\n          startTime: BigInt(0),\n          endTime: BigInt(0),\n          final: false,\n          language: this.#opts.language,\n        }),\n      );\n\n      this.#playedText = `${ogText} ${text}`;\n      await this.#sleepIfNotClosed((delay - firstDelay) * 1000000);\n      textData.forwardedHyphens += wordHyphens;\n    }\n\n    this.emit(\n      'textUpdated',\n      new TranscriptionSegment({\n        id: segId,\n        text: sentence,\n        startTime: BigInt(0),\n        endTime: BigInt(0),\n        final: true,\n        language: this.#opts.language,\n      }),\n    );\n\n    this.#playedText = `${ogText} ${sentence}`;\n\n    await this.#sleepIfNotClosed(this.#opts.newSentenceDelay);\n    textData.forwardedSentences++;\n  }\n\n  async #sleepIfNotClosed(delay: number) {\n    await Promise.race([\n      this.#closeFut.await,\n      new Promise((resolve) => setTimeout(resolve, delay)),\n    ]);\n  }\n\n  #calcHyphens(text: string): string[] {\n    const hyphens: string[] = [];\n    const words = this.#opts.splitWords(text);\n    for (const word of words) {\n      const n = this.#opts.hyphenateWord(word[0]);\n      hyphens.push(...n);\n    }\n    return hyphens;\n  }\n\n  #checkNotClosed() {\n    if (this.#closed) {\n      throw new Error('TextAudioSynchronizer is closed');\n    }\n  }\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAGA,sBAAqC;AACrC,sBAA2B;AAE3B,yBAA6B;AAC7B,sBAAsB;AAEtB,mBAAsD;AAGtD,MAAM,uBAAuB;AAWtB,MAAM,yBAA0C;AAAA,EACrD,UAAU;AAAA,EACV,OAAO;AAAA,EACP,kBAAkB;AAAA,EAClB,mBAAmB,IAAI,sBAAM,kBAAkB;AAAA,EAC/C,eAAe,sBAAM;AAAA,EACrB,YAAY,sBAAM;AACpB;AAmBO,MAAM,8BAA+B,gCAAuD;AAAA,EACjG;AAAA,EACA;AAAA,EAEA,UAAU;AAAA,EACV,eAAe;AAAA,EACf,YAAY,IAAI,oBAAO;AAAA,EAEvB,mBAAmB;AAAA,EACnB,oBAAoB;AAAA,EAEpB,gBAAgB,IAAI,gCAA2B;AAAA,EAC/C,SAAmC,CAAC;AAAA,EACpC,iBAAiB,IAAI,gCAA2B;AAAA,EAChD,UAAqC,CAAC;AAAA,EAEtC,cAAc;AAAA,EACd;AAAA,EAEA;AAAA,EACA;AAAA,EAEA,YAAY,MAAuB;AACjC,UAAM;AAEN,SAAK,QAAQ;AACb,SAAK,SAAS,KAAK,QAAQ;AAAA,EAC7B;AAAA,EAEA,UAAU,OAAmB;AAC3B,SAAK,gBAAgB;AACrB,QAAI,CAAC,KAAK,YAAY;AACpB,WAAK,aAAa,EAAE,gBAAgB,GAAG,MAAM,MAAM;AACnD,WAAK,QAAQ,KAAK,KAAK,UAAU;AACjC,WAAK,eAAe,IAAI,CAAC;AAAA,IAC3B;AACA,SAAK,WAAW,kBAAkB,MAAM,oBAAoB,MAAM;AAAA,EACpE;AAAA,EAEA,SAAS,MAAc;AACrB,SAAK,gBAAgB;AACrB,QAAI,CAAC,KAAK,WAAW;AACnB,WAAK,YAAY;AAAA,QACf,gBAAgB,KAAK,MAAM,kBAAkB,OAAO;AAAA,QACpD,YAAY;AAAA,QACZ,MAAM;AAAA,QACN,kBAAkB;AAAA,QAClB,oBAAoB;AAAA,MACtB;AACA,WAAK,OAAO,KAAK,KAAK,SAAS;AAC/B,WAAK,cAAc,IAAI,CAAC;AAAA,IAC1B;AAEA,SAAK,UAAU,cAAc;AAC7B,SAAK,UAAU,eAAe,SAAS,IAAI;AAAA,EAC7C;AAAA,EAEA,sBAAsB;AACpB,SAAK,gBAAgB;AAErB,QAAI,CAAC,KAAK,YAAY;AAEpB,WAAK,UAAU,IAAI,2BAAW,IAAI,WAAW,GAAG,MAAO,GAAG,CAAC,CAAC;AAAA,IAC9D;AAEA,SAAK,WAAY,OAAO;AACxB,SAAK,aAAa;AAAA,EACpB;AAAA,EAEA,qBAAqB;AAtHvB;AAuHI,SAAK,gBAAgB;AAErB,QAAI,CAAC,KAAK,WAAW;AACnB,WAAK,SAAS,EAAE;AAAA,IAClB;AAEA,SAAK,UAAW,OAAO;AACvB,eAAK,cAAL,mBAAgB,eAAe;AAC/B,eAAK,cAAL,mBAAgB,eAAe;AAC/B,SAAK,YAAY;AAAA,EACnB;AAAA,EAEA,wBAAwB;AACtB,SAAK,gBAAgB;AACrB,SAAK;AAEL,QAAI,CAAC,KAAK,OAAO;AACf,WAAK,QAAQ,KAAK,UAAU;AAAA,IAC9B;AAAA,EACF;AAAA,EAEA,yBAAyB;AACvB,SAAK,gBAAgB;AACrB,SAAK;AAAA,EACP;AAAA,EAEA,IAAI,aAAqB;AACvB,WAAO,KAAK;AAAA,EACd;AAAA,EAEA,MAAM,MAAM,WAAoB;AAC9B,QAAI,KAAK,SAAS;AAChB;AAAA,IACF;AACA,SAAK,UAAU;AACf,SAAK,eAAe;AACpB,SAAK,UAAU,QAAQ;AAEvB,eAAW,YAAY,KAAK,QAAQ;AAClC,2CAAU,eAAe;AAAA,IAC3B;AAEA,SAAK,OAAO,KAAK,MAAS;AAC1B,SAAK,QAAQ,KAAK,MAAS;AAC3B,SAAK,cAAc,IAAI,CAAC;AACxB,SAAK,eAAe,IAAI,CAAC;AAEzB,UAAM,KAAK;AAAA,EACb;AAAA,EAEA,MAAM,YAAY;AAChB,QAAI,WAAW;AACf,QAAI,QAAQ;AAEZ,WAAO,CAAC,OAAO;AACb,YAAM,KAAK,cAAc,KAAK;AAC9B,YAAM,KAAK,eAAe,KAAK;AAE/B,aAAO,KAAK,OAAO,UAAU,KAAK,QAAQ,QAAQ;AAChD,cAAM,WAAW,KAAK,OAAO,IAAI;AACjC,cAAM,YAAY,KAAK,QAAQ,IAAI;AAEnC,YAAI,EAAE,YAAY,YAAY;AAC5B,kBAAQ;AACR;AAAA,QACF;AAGA,eAAO,CAAC,KAAK,SAAS;AACpB,cAAI,KAAK,oBAAoB,SAAU;AACvC,gBAAM,KAAK,kBAAkB,GAAG;AAAA,QAClC;AAEA,cAAM,iBAAiB,SAAS;AAChC,cAAM,mBAAmB,KAAK,IAAI;AAElC,yBAAiB,MAAM,gBAAgB;AACrC,gBAAM,KAAK,cAAc,UAAU,kBAAkB,UAAU,WAAW,GAAG,KAAK;AAAA,QACpF;AAEA;AAAA,MACF;AAAA,IACF;AAAA,EACF;AAAA,EAEA,MAAM,cACJ,UACA,cACA,UACA,WACA,UACA;AACA,QAAI;AACJ,QAAI,UAAU,iBAAiB,KAAK,UAAU,MAAM;AAClD,kBAAY,KAAK,aAAa,SAAS,UAAU,EAAE,SAAS,UAAU;AAAA,IACxE;AAEA,UAAM,YAAQ,wBAAU,KAAK;AAC7B,UAAM,QAAQ,KAAK,MAAM,WAAW,QAAQ;AAC5C,UAAM,iBAA2B,CAAC;AAElC,UAAM,SAAS,KAAK;AAEpB,eAAW,CAAC,MAAM,GAAG,GAAG,KAAK,OAAO;AAClC,UAAI,YAAY,KAAK,kBAAmB;AACxC,UAAI,KAAK,aAAc;AAEvB,YAAM,cAAc,KAAK,MAAM,cAAc,IAAI,EAAE;AACnD,qBAAe,KAAK,IAAI;AAExB,YAAM,UAAU,KAAK,IAAI,IAAI;AAC7B,YAAM,OAAO,SAAS,MAAM,GAAG,GAAG;AAElC,UAAI,QAAQ,KAAK;AACjB,UAAI;AACJ,UAAI,WAAW;AACb,gBAAQ;AACR,cAAM,oBAAoB,SAAS,qBAAqB,KAAK,MAAM;AACnE,cAAM,aAAa,oBAAoB;AACvC,cAAM,gBAAgB,KAAK,MAAM,QAAQ,OAAO;AAChD,cAAM,KAAK,gBAAgB,SAAS,mBAAmB;AACvD,cAAM,gBAAgB,KAAK,IAAI,GAAG,cAAc,EAAE;AAClD,gBAAQ,gBAAgB;AAAA,MAC1B,OAAO;AACL,gBAAQ,cAAc;AAAA,MACxB;AAEA,YAAM,aAAa,KAAK,IAAI,QAAQ,GAAG,IAAI,KAAK;AAChD,YAAM,KAAK,kBAAkB,aAAa,GAAO;AAEjD,WAAK;AAAA,QACH;AAAA,QACA,IAAI,qCAAqB;AAAA,UACvB,IAAI;AAAA,UACJ;AAAA,UACA,WAAW,OAAO,CAAC;AAAA,UACnB,SAAS,OAAO,CAAC;AAAA,UACjB,OAAO;AAAA,UACP,UAAU,KAAK,MAAM;AAAA,QACvB,CAAC;AAAA,MACH;AAEA,WAAK,cAAc,GAAG,MAAM,IAAI,IAAI;AACpC,YAAM,KAAK,mBAAmB,QAAQ,cAAc,GAAO;AAC3D,eAAS,oBAAoB;AAAA,IAC/B;AAEA,SAAK;AAAA,MACH;AAAA,MACA,IAAI,qCAAqB;AAAA,QACvB,IAAI;AAAA,QACJ,MAAM;AAAA,QACN,WAAW,OAAO,CAAC;AAAA,QACnB,SAAS,OAAO,CAAC;AAAA,QACjB,OAAO;AAAA,QACP,UAAU,KAAK,MAAM;AAAA,MACvB,CAAC;AAAA,IACH;AAEA,SAAK,cAAc,GAAG,MAAM,IAAI,QAAQ;AAExC,UAAM,KAAK,kBAAkB,KAAK,MAAM,gBAAgB;AACxD,aAAS;AAAA,EACX;AAAA,EAEA,MAAM,kBAAkB,OAAe;AACrC,UAAM,QAAQ,KAAK;AAAA,MACjB,KAAK,UAAU;AAAA,MACf,IAAI,QAAQ,CAAC,YAAY,WAAW,SAAS,KAAK,CAAC;AAAA,IACrD,CAAC;AAAA,EACH;AAAA,EAEA,aAAa,MAAwB;AACnC,UAAM,UAAoB,CAAC;AAC3B,UAAM,QAAQ,KAAK,MAAM,WAAW,IAAI;AACxC,eAAW,QAAQ,OAAO;AACxB,YAAM,IAAI,KAAK,MAAM,cAAc,KAAK,CAAC,CAAC;AAC1C,cAAQ,KAAK,GAAG,CAAC;AAAA,IACnB;AACA,WAAO;AAAA,EACT;AAAA,EAEA,kBAAkB;AAChB,QAAI,KAAK,SAAS;AAChB,YAAM,IAAI,MAAM,iCAAiC;AAAA,IACnD;AAAA,EACF;AACF;","names":[]}
+{"version":3,"sources":["../src/transcription.ts"],"sourcesContent":["// SPDX-FileCopyrightText: 2024 LiveKit, Inc.\n//\n// SPDX-License-Identifier: Apache-2.0\nimport { TranscriptionSegment } from '@livekit/protocol';\nimport { AudioFrame } from '@livekit/rtc-node';\nimport type { TypedEventEmitter as TypedEmitter } from '@livekit/typed-emitter';\nimport { EventEmitter } from 'node:events';\nimport { basic } from './tokenize/index.js';\nimport type { SentenceStream, SentenceTokenizer } from './tokenize/tokenizer.js';\nimport { AsyncIterableQueue, Future, shortuuid } from './utils.js';\n\n// standard speech rate in hyphens/ms\nconst STANDARD_SPEECH_RATE = 3830;\n\nexport interface TextSyncOptions {\n  /** Language code for transcription. */\n  language: string;\n  /** Speech speed multiplier. */\n  speed: number;\n  /** Delay between sentences in milliseconds. */\n  newSentenceDelay: number;\n  /** Tokenizer for splitting text into sentences. */\n  sentenceTokenizer: SentenceTokenizer;\n  /** Function to hyphenate words. */\n  hyphenateWord: (word: string) => string[];\n  /** Function to split text into words with positions. */\n  splitWords: (words: string) => [string, number, number][];\n}\n\nexport const defaultTextSyncOptions: TextSyncOptions = {\n  language: '',\n  speed: 1,\n  newSentenceDelay: 400,\n  sentenceTokenizer: new basic.SentenceTokenizer(),\n  hyphenateWord: basic.hyphenateWord,\n  splitWords: basic.splitWords,\n};\n\ninterface AudioData {\n  pushedDuration: number;\n  done: boolean;\n}\n\ninterface TextData {\n  sentenceStream: SentenceStream;\n  pushedText: string;\n  done: boolean;\n  forwardedHyphens: number;\n  forwardedSentences: number;\n}\n\ntype SyncCallbacks = {\n  textUpdated: (text: TranscriptionSegment) => void;\n};\n\nexport class TextAudioSynchronizer extends (EventEmitter as new () => TypedEmitter<SyncCallbacks>) {\n  #opts: TextSyncOptions;\n  #speed: number;\n\n  #closed = false;\n  #interrupted = false;\n  #closeFut = new Future();\n\n  #playingSegIndex = -1;\n  #finishedSegIndex = -1;\n\n  #textQChanged = new AsyncIterableQueue<number>();\n  #textQ: (TextData | undefined)[] = [];\n  #audioQChanged = new AsyncIterableQueue<number>();\n  #audioQ: (AudioData | undefined)[] = [];\n\n  #playedText = '';\n  #task?: Promise<void>;\n\n  #audioData?: AudioData;\n  #textData?: TextData;\n\n  constructor(opts: TextSyncOptions) {\n    super();\n\n    this.#opts = opts;\n    this.#speed = opts.speed * STANDARD_SPEECH_RATE;\n  }\n\n  pushAudio(frame: AudioFrame) {\n    this.#checkNotClosed();\n    if (!this.#audioData) {\n      this.#audioData = { pushedDuration: 0, done: false };\n      this.#audioQ.push(this.#audioData);\n      this.#audioQChanged.put(1);\n    }\n    this.#audioData.pushedDuration += frame.samplesPerChannel / frame.sampleRate;\n  }\n\n  pushText(text: string) {\n    this.#checkNotClosed();\n    if (!this.#textData) {\n      this.#textData = {\n        sentenceStream: this.#opts.sentenceTokenizer.stream(),\n        pushedText: '',\n        done: false,\n        forwardedHyphens: 0,\n        forwardedSentences: 0,\n      };\n      this.#textQ.push(this.#textData);\n      this.#textQChanged.put(1);\n    }\n\n    this.#textData.pushedText += text;\n    this.#textData.sentenceStream.pushText(text);\n  }\n\n  markAudioSegmentEnd() {\n    this.#checkNotClosed();\n\n    if (!this.#audioData) {\n      // create empty audio data if none exists\n      this.pushAudio(new AudioFrame(new Int16Array(), 24000, 1, 0));\n    }\n\n    this.#audioData!.done = true;\n    this.#audioData = undefined;\n  }\n\n  markTextSegmentEnd() {\n    this.#checkNotClosed();\n\n    if (!this.#textData) {\n      this.pushText('');\n    }\n\n    this.#textData!.done = true;\n    this.#textData?.sentenceStream.flush();\n    this.#textData?.sentenceStream.close();\n    this.#textData = undefined;\n  }\n\n  segmentPlayoutStarted() {\n    this.#checkNotClosed();\n    this.#playingSegIndex++;\n\n    if (!this.#task) {\n      this.#task = this.#mainLoop();\n    }\n  }\n\n  segmentPlayoutFinished() {\n    this.#checkNotClosed();\n    this.#finishedSegIndex++;\n  }\n\n  get playedText(): string {\n    return this.#playedText;\n  }\n\n  async close(interrupt: boolean) {\n    if (this.#closed) {\n      return;\n    }\n    this.#closed = true;\n    this.#interrupted = interrupt;\n    this.#closeFut.resolve();\n\n    for (const textData of this.#textQ) {\n      textData?.sentenceStream.close();\n    }\n\n    this.#textQ.push(undefined);\n    this.#audioQ.push(undefined);\n    this.#textQChanged.put(1);\n    this.#audioQChanged.put(1);\n\n    await this.#task;\n  }\n\n  async #mainLoop() {\n    let segIndex = 0;\n    let qDone = false;\n\n    while (!qDone) {\n      await this.#textQChanged.next();\n      await this.#audioQChanged.next();\n\n      while (this.#textQ.length && this.#audioQ.length) {\n        const textData = this.#textQ.pop();\n        const audioData = this.#audioQ.pop();\n\n        if (!(textData && audioData)) {\n          qDone = true;\n          break;\n        }\n\n        // wait for segment to start playing\n        while (!this.#closed) {\n          if (this.#playingSegIndex >= segIndex) break;\n          await this.#sleepIfNotClosed(125);\n        }\n\n        const sentenceStream = textData.sentenceStream;\n        const forwardStartTime = Date.now();\n\n        for await (const ev of sentenceStream) {\n          await this.#syncSentence(segIndex, forwardStartTime, textData, audioData, ev.token);\n        }\n\n        segIndex++;\n      }\n    }\n  }\n\n  async #syncSentence(\n    segIndex: number,\n    segStartTime: number,\n    textData: TextData,\n    audioData: AudioData,\n    sentence: string,\n  ) {\n    let realSpeed: number | undefined;\n    if (audioData.pushedDuration > 0 && audioData.done) {\n      realSpeed = this.#calcHyphens(textData.pushedText).length / audioData.pushedDuration;\n    }\n\n    const segId = shortuuid('SG_');\n    const words = this.#opts.splitWords(sentence);\n    const processedWords: string[] = [];\n\n    const ogText = this.#playedText;\n    // eslint-disable-next-line @typescript-eslint/no-unused-vars\n    for (const [word, _, end] of words) {\n      if (segIndex <= this.#finishedSegIndex) break;\n      if (this.#interrupted) return;\n\n      const wordHyphens = this.#opts.hyphenateWord(word).length;\n      processedWords.push(word);\n\n      const elapsed = Date.now() - segStartTime;\n      const text = sentence.slice(0, end); // TODO: rstrip punctuations\n\n      let speed = this.#speed;\n      let delay: number;\n      if (realSpeed) {\n        speed = realSpeed;\n        const estimatedPausesMs = textData.forwardedSentences * this.#opts.newSentenceDelay;\n        const hyphPauses = estimatedPausesMs * speed;\n        const targetHyphens = Math.round(speed * elapsed);\n        const dt = targetHyphens - textData.forwardedHyphens - hyphPauses;\n        const toWaitHyphens = Math.max(0, wordHyphens - dt);\n        delay = toWaitHyphens / speed;\n      } else {\n        delay = wordHyphens / speed;\n      }\n\n      const firstDelay = Math.min(delay / 2, 2 / speed);\n      await this.#sleepIfNotClosed(firstDelay * 1000000);\n\n      this.emit(\n        'textUpdated',\n        new TranscriptionSegment({\n          id: segId,\n          text: text,\n          startTime: BigInt(0),\n          endTime: BigInt(0),\n          final: false,\n          language: this.#opts.language,\n        }),\n      );\n\n      this.#playedText = `${ogText} ${text}`;\n      await this.#sleepIfNotClosed((delay - firstDelay) * 1000000);\n      textData.forwardedHyphens += wordHyphens;\n    }\n\n    this.emit(\n      'textUpdated',\n      new TranscriptionSegment({\n        id: segId,\n        text: sentence,\n        startTime: BigInt(0),\n        endTime: BigInt(0),\n        final: true,\n        language: this.#opts.language,\n      }),\n    );\n\n    this.#playedText = `${ogText} ${sentence}`;\n\n    await this.#sleepIfNotClosed(this.#opts.newSentenceDelay);\n    textData.forwardedSentences++;\n  }\n\n  async #sleepIfNotClosed(delay: number) {\n    await Promise.race([\n      this.#closeFut.await,\n      new Promise((resolve) => setTimeout(resolve, delay)),\n    ]);\n  }\n\n  #calcHyphens(text: string): string[] {\n    const hyphens: string[] = [];\n    const words = this.#opts.splitWords(text);\n    for (const word of words) {\n      const n = this.#opts.hyphenateWord(word[0]);\n      hyphens.push(...n);\n    }\n    return hyphens;\n  }\n\n  #checkNotClosed() {\n    if (this.#closed) {\n      throw new Error('TextAudioSynchronizer is closed');\n    }\n  }\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAGA,sBAAqC;AACrC,sBAA2B;AAE3B,yBAA6B;AAC7B,sBAAsB;AAEtB,mBAAsD;AAGtD,MAAM,uBAAuB;AAiBtB,MAAM,yBAA0C;AAAA,EACrD,UAAU;AAAA,EACV,OAAO;AAAA,EACP,kBAAkB;AAAA,EAClB,mBAAmB,IAAI,sBAAM,kBAAkB;AAAA,EAC/C,eAAe,sBAAM;AAAA,EACrB,YAAY,sBAAM;AACpB;AAmBO,MAAM,8BAA+B,gCAAuD;AAAA,EACjG;AAAA,EACA;AAAA,EAEA,UAAU;AAAA,EACV,eAAe;AAAA,EACf,YAAY,IAAI,oBAAO;AAAA,EAEvB,mBAAmB;AAAA,EACnB,oBAAoB;AAAA,EAEpB,gBAAgB,IAAI,gCAA2B;AAAA,EAC/C,SAAmC,CAAC;AAAA,EACpC,iBAAiB,IAAI,gCAA2B;AAAA,EAChD,UAAqC,CAAC;AAAA,EAEtC,cAAc;AAAA,EACd;AAAA,EAEA;AAAA,EACA;AAAA,EAEA,YAAY,MAAuB;AACjC,UAAM;AAEN,SAAK,QAAQ;AACb,SAAK,SAAS,KAAK,QAAQ;AAAA,EAC7B;AAAA,EAEA,UAAU,OAAmB;AAC3B,SAAK,gBAAgB;AACrB,QAAI,CAAC,KAAK,YAAY;AACpB,WAAK,aAAa,EAAE,gBAAgB,GAAG,MAAM,MAAM;AACnD,WAAK,QAAQ,KAAK,KAAK,UAAU;AACjC,WAAK,eAAe,IAAI,CAAC;AAAA,IAC3B;AACA,SAAK,WAAW,kBAAkB,MAAM,oBAAoB,MAAM;AAAA,EACpE;AAAA,EAEA,SAAS,MAAc;AACrB,SAAK,gBAAgB;AACrB,QAAI,CAAC,KAAK,WAAW;AACnB,WAAK,YAAY;AAAA,QACf,gBAAgB,KAAK,MAAM,kBAAkB,OAAO;AAAA,QACpD,YAAY;AAAA,QACZ,MAAM;AAAA,QACN,kBAAkB;AAAA,QAClB,oBAAoB;AAAA,MACtB;AACA,WAAK,OAAO,KAAK,KAAK,SAAS;AAC/B,WAAK,cAAc,IAAI,CAAC;AAAA,IAC1B;AAEA,SAAK,UAAU,cAAc;AAC7B,SAAK,UAAU,eAAe,SAAS,IAAI;AAAA,EAC7C;AAAA,EAEA,sBAAsB;AACpB,SAAK,gBAAgB;AAErB,QAAI,CAAC,KAAK,YAAY;AAEpB,WAAK,UAAU,IAAI,2BAAW,IAAI,WAAW,GAAG,MAAO,GAAG,CAAC,CAAC;AAAA,IAC9D;AAEA,SAAK,WAAY,OAAO;AACxB,SAAK,aAAa;AAAA,EACpB;AAAA,EAEA,qBAAqB;AA5HvB;AA6HI,SAAK,gBAAgB;AAErB,QAAI,CAAC,KAAK,WAAW;AACnB,WAAK,SAAS,EAAE;AAAA,IAClB;AAEA,SAAK,UAAW,OAAO;AACvB,eAAK,cAAL,mBAAgB,eAAe;AAC/B,eAAK,cAAL,mBAAgB,eAAe;AAC/B,SAAK,YAAY;AAAA,EACnB;AAAA,EAEA,wBAAwB;AACtB,SAAK,gBAAgB;AACrB,SAAK;AAEL,QAAI,CAAC,KAAK,OAAO;AACf,WAAK,QAAQ,KAAK,UAAU;AAAA,IAC9B;AAAA,EACF;AAAA,EAEA,yBAAyB;AACvB,SAAK,gBAAgB;AACrB,SAAK;AAAA,EACP;AAAA,EAEA,IAAI,aAAqB;AACvB,WAAO,KAAK;AAAA,EACd;AAAA,EAEA,MAAM,MAAM,WAAoB;AAC9B,QAAI,KAAK,SAAS;AAChB;AAAA,IACF;AACA,SAAK,UAAU;AACf,SAAK,eAAe;AACpB,SAAK,UAAU,QAAQ;AAEvB,eAAW,YAAY,KAAK,QAAQ;AAClC,2CAAU,eAAe;AAAA,IAC3B;AAEA,SAAK,OAAO,KAAK,MAAS;AAC1B,SAAK,QAAQ,KAAK,MAAS;AAC3B,SAAK,cAAc,IAAI,CAAC;AACxB,SAAK,eAAe,IAAI,CAAC;AAEzB,UAAM,KAAK;AAAA,EACb;AAAA,EAEA,MAAM,YAAY;AAChB,QAAI,WAAW;AACf,QAAI,QAAQ;AAEZ,WAAO,CAAC,OAAO;AACb,YAAM,KAAK,cAAc,KAAK;AAC9B,YAAM,KAAK,eAAe,KAAK;AAE/B,aAAO,KAAK,OAAO,UAAU,KAAK,QAAQ,QAAQ;AAChD,cAAM,WAAW,KAAK,OAAO,IAAI;AACjC,cAAM,YAAY,KAAK,QAAQ,IAAI;AAEnC,YAAI,EAAE,YAAY,YAAY;AAC5B,kBAAQ;AACR;AAAA,QACF;AAGA,eAAO,CAAC,KAAK,SAAS;AACpB,cAAI,KAAK,oBAAoB,SAAU;AACvC,gBAAM,KAAK,kBAAkB,GAAG;AAAA,QAClC;AAEA,cAAM,iBAAiB,SAAS;AAChC,cAAM,mBAAmB,KAAK,IAAI;AAElC,yBAAiB,MAAM,gBAAgB;AACrC,gBAAM,KAAK,cAAc,UAAU,kBAAkB,UAAU,WAAW,GAAG,KAAK;AAAA,QACpF;AAEA;AAAA,MACF;AAAA,IACF;AAAA,EACF;AAAA,EAEA,MAAM,cACJ,UACA,cACA,UACA,WACA,UACA;AACA,QAAI;AACJ,QAAI,UAAU,iBAAiB,KAAK,UAAU,MAAM;AAClD,kBAAY,KAAK,aAAa,SAAS,UAAU,EAAE,SAAS,UAAU;AAAA,IACxE;AAEA,UAAM,YAAQ,wBAAU,KAAK;AAC7B,UAAM,QAAQ,KAAK,MAAM,WAAW,QAAQ;AAC5C,UAAM,iBAA2B,CAAC;AAElC,UAAM,SAAS,KAAK;AAEpB,eAAW,CAAC,MAAM,GAAG,GAAG,KAAK,OAAO;AAClC,UAAI,YAAY,KAAK,kBAAmB;AACxC,UAAI,KAAK,aAAc;AAEvB,YAAM,cAAc,KAAK,MAAM,cAAc,IAAI,EAAE;AACnD,qBAAe,KAAK,IAAI;AAExB,YAAM,UAAU,KAAK,IAAI,IAAI;AAC7B,YAAM,OAAO,SAAS,MAAM,GAAG,GAAG;AAElC,UAAI,QAAQ,KAAK;AACjB,UAAI;AACJ,UAAI,WAAW;AACb,gBAAQ;AACR,cAAM,oBAAoB,SAAS,qBAAqB,KAAK,MAAM;AACnE,cAAM,aAAa,oBAAoB;AACvC,cAAM,gBAAgB,KAAK,MAAM,QAAQ,OAAO;AAChD,cAAM,KAAK,gBAAgB,SAAS,mBAAmB;AACvD,cAAM,gBAAgB,KAAK,IAAI,GAAG,cAAc,EAAE;AAClD,gBAAQ,gBAAgB;AAAA,MAC1B,OAAO;AACL,gBAAQ,cAAc;AAAA,MACxB;AAEA,YAAM,aAAa,KAAK,IAAI,QAAQ,GAAG,IAAI,KAAK;AAChD,YAAM,KAAK,kBAAkB,aAAa,GAAO;AAEjD,WAAK;AAAA,QACH;AAAA,QACA,IAAI,qCAAqB;AAAA,UACvB,IAAI;AAAA,UACJ;AAAA,UACA,WAAW,OAAO,CAAC;AAAA,UACnB,SAAS,OAAO,CAAC;AAAA,UACjB,OAAO;AAAA,UACP,UAAU,KAAK,MAAM;AAAA,QACvB,CAAC;AAAA,MACH;AAEA,WAAK,cAAc,GAAG,MAAM,IAAI,IAAI;AACpC,YAAM,KAAK,mBAAmB,QAAQ,cAAc,GAAO;AAC3D,eAAS,oBAAoB;AAAA,IAC/B;AAEA,SAAK;AAAA,MACH;AAAA,MACA,IAAI,qCAAqB;AAAA,QACvB,IAAI;AAAA,QACJ,MAAM;AAAA,QACN,WAAW,OAAO,CAAC;AAAA,QACnB,SAAS,OAAO,CAAC;AAAA,QACjB,OAAO;AAAA,QACP,UAAU,KAAK,MAAM;AAAA,MACvB,CAAC;AAAA,IACH;AAEA,SAAK,cAAc,GAAG,MAAM,IAAI,QAAQ;AAExC,UAAM,KAAK,kBAAkB,KAAK,MAAM,gBAAgB;AACxD,aAAS;AAAA,EACX;AAAA,EAEA,MAAM,kBAAkB,OAAe;AACrC,UAAM,QAAQ,KAAK;AAAA,MACjB,KAAK,UAAU;AAAA,MACf,IAAI,QAAQ,CAAC,YAAY,WAAW,SAAS,KAAK,CAAC;AAAA,IACrD,CAAC;AAAA,EACH;AAAA,EAEA,aAAa,MAAwB;AACnC,UAAM,UAAoB,CAAC;AAC3B,UAAM,QAAQ,KAAK,MAAM,WAAW,IAAI;AACxC,eAAW,QAAQ,OAAO;AACxB,YAAM,IAAI,KAAK,MAAM,cAAc,KAAK,CAAC,CAAC;AAC1C,cAAQ,KAAK,GAAG,CAAC;AAAA,IACnB;AACA,WAAO;AAAA,EACT;AAAA,EAEA,kBAAkB;AAChB,QAAI,KAAK,SAAS;AAChB,YAAM,IAAI,MAAM,iCAAiC;AAAA,IACnD;AAAA,EACF;AACF;","names":[]}

package/dist/transcription.d.cts

@@ -3,11 +3,17 @@ import { AudioFrame } from '@livekit/rtc-node';
 import type { TypedEventEmitter as TypedEmitter } from '@livekit/typed-emitter';
 import type { SentenceTokenizer } from './tokenize/tokenizer.js';
 export interface TextSyncOptions {
+    /** Language code for transcription. */
     language: string;
+    /** Speech speed multiplier. */
     speed: number;
+    /** Delay between sentences in milliseconds. */
     newSentenceDelay: number;
+    /** Tokenizer for splitting text into sentences. */
     sentenceTokenizer: SentenceTokenizer;
+    /** Function to hyphenate words. */
     hyphenateWord: (word: string) => string[];
+    /** Function to split text into words with positions. */
     splitWords: (words: string) => [string, number, number][];
 }
 export declare const defaultTextSyncOptions: TextSyncOptions;

package/dist/transcription.d.ts

@@ -3,11 +3,17 @@ import { AudioFrame } from '@livekit/rtc-node';
 import type { TypedEventEmitter as TypedEmitter } from '@livekit/typed-emitter';
 import type { SentenceTokenizer } from './tokenize/tokenizer.js';
 export interface TextSyncOptions {
+    /** Language code for transcription. */
     language: string;
+    /** Speech speed multiplier. */
     speed: number;
+    /** Delay between sentences in milliseconds. */
     newSentenceDelay: number;
+    /** Tokenizer for splitting text into sentences. */
     sentenceTokenizer: SentenceTokenizer;
+    /** Function to hyphenate words. */
     hyphenateWord: (word: string) => string[];
+    /** Function to split text into words with positions. */
     splitWords: (words: string) => [string, number, number][];
 }
 export declare const defaultTextSyncOptions: TextSyncOptions;

package/dist/transcription.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"transcription.d.ts","sourceRoot":"","sources":["../src/transcription.ts"],"names":[],"mappings":"AAGA,OAAO,EAAE,oBAAoB,EAAE,MAAM,mBAAmB,CAAC;AACzD,OAAO,EAAE,UAAU,EAAE,MAAM,mBAAmB,CAAC;AAC/C,OAAO,KAAK,EAAE,iBAAiB,IAAI,YAAY,EAAE,MAAM,wBAAwB,CAAC;AAGhF,OAAO,KAAK,EAAkB,iBAAiB,EAAE,MAAM,yBAAyB,CAAC;AAMjF,MAAM,WAAW,eAAe;IAC9B,QAAQ,EAAE,MAAM,CAAC;IACjB,KAAK,EAAE,MAAM,CAAC;IACd,gBAAgB,EAAE,MAAM,CAAC;IACzB,iBAAiB,EAAE,iBAAiB,CAAC;IACrC,aAAa,EAAE,CAAC,IAAI,EAAE,MAAM,KAAK,MAAM,EAAE,CAAC;IAC1C,UAAU,EAAE,CAAC,KAAK,EAAE,MAAM,KAAK,CAAC,MAAM,EAAE,MAAM,EAAE,MAAM,CAAC,EAAE,CAAC;CAC3D;AAED,eAAO,MAAM,sBAAsB,EAAE,eAOpC,CAAC;AAeF,KAAK,aAAa,GAAG;IACnB,WAAW,EAAE,CAAC,IAAI,EAAE,oBAAoB,KAAK,IAAI,CAAC;CACnD,CAAC;oDAEoE,aAAa,aAAa,CAAC;AAAjG,qBAAa,qBAAsB,SAAQ,0BAAuD;;gBAsBpF,IAAI,EAAE,eAAe;IAOjC,SAAS,CAAC,KAAK,EAAE,UAAU;IAU3B,QAAQ,CAAC,IAAI,EAAE,MAAM;IAkBrB,mBAAmB;IAYnB,kBAAkB;IAalB,qBAAqB;IASrB,sBAAsB;IAKtB,IAAI,UAAU,IAAI,MAAM,CAEvB;IAEK,KAAK,CAAC,SAAS,EAAE,OAAO;CA6J/B"}
+{"version":3,"file":"transcription.d.ts","sourceRoot":"","sources":["../src/transcription.ts"],"names":[],"mappings":"AAGA,OAAO,EAAE,oBAAoB,EAAE,MAAM,mBAAmB,CAAC;AACzD,OAAO,EAAE,UAAU,EAAE,MAAM,mBAAmB,CAAC;AAC/C,OAAO,KAAK,EAAE,iBAAiB,IAAI,YAAY,EAAE,MAAM,wBAAwB,CAAC;AAGhF,OAAO,KAAK,EAAkB,iBAAiB,EAAE,MAAM,yBAAyB,CAAC;AAMjF,MAAM,WAAW,eAAe;IAC9B,uCAAuC;IACvC,QAAQ,EAAE,MAAM,CAAC;IACjB,+BAA+B;IAC/B,KAAK,EAAE,MAAM,CAAC;IACd,+CAA+C;IAC/C,gBAAgB,EAAE,MAAM,CAAC;IACzB,mDAAmD;IACnD,iBAAiB,EAAE,iBAAiB,CAAC;IACrC,mCAAmC;IACnC,aAAa,EAAE,CAAC,IAAI,EAAE,MAAM,KAAK,MAAM,EAAE,CAAC;IAC1C,wDAAwD;IACxD,UAAU,EAAE,CAAC,KAAK,EAAE,MAAM,KAAK,CAAC,MAAM,EAAE,MAAM,EAAE,MAAM,CAAC,EAAE,CAAC;CAC3D;AAED,eAAO,MAAM,sBAAsB,EAAE,eAOpC,CAAC;AAeF,KAAK,aAAa,GAAG;IACnB,WAAW,EAAE,CAAC,IAAI,EAAE,oBAAoB,KAAK,IAAI,CAAC;CACnD,CAAC;oDAEoE,aAAa,aAAa,CAAC;AAAjG,qBAAa,qBAAsB,SAAQ,0BAAuD;;gBAsBpF,IAAI,EAAE,eAAe;IAOjC,SAAS,CAAC,KAAK,EAAE,UAAU;IAU3B,QAAQ,CAAC,IAAI,EAAE,MAAM;IAkBrB,mBAAmB;IAYnB,kBAAkB;IAalB,qBAAqB;IASrB,sBAAsB;IAKtB,IAAI,UAAU,IAAI,MAAM,CAEvB;IAEK,KAAK,CAAC,SAAS,EAAE,OAAO;CA6J/B"}

package/dist/transcription.js.map

@@ -1 +1 @@
-{"version":3,"sources":["../src/transcription.ts"],"sourcesContent":["// SPDX-FileCopyrightText: 2024 LiveKit, Inc.\n//\n// SPDX-License-Identifier: Apache-2.0\nimport { TranscriptionSegment } from '@livekit/protocol';\nimport { AudioFrame } from '@livekit/rtc-node';\nimport type { TypedEventEmitter as TypedEmitter } from '@livekit/typed-emitter';\nimport { EventEmitter } from 'node:events';\nimport { basic } from './tokenize/index.js';\nimport type { SentenceStream, SentenceTokenizer } from './tokenize/tokenizer.js';\nimport { AsyncIterableQueue, Future, shortuuid } from './utils.js';\n\n// standard speech rate in hyphens/ms\nconst STANDARD_SPEECH_RATE = 3830;\n\nexport interface TextSyncOptions {\n  language: string;\n  speed: number;\n  newSentenceDelay: number;\n  sentenceTokenizer: SentenceTokenizer;\n  hyphenateWord: (word: string) => string[];\n  splitWords: (words: string) => [string, number, number][];\n}\n\nexport const defaultTextSyncOptions: TextSyncOptions = {\n  language: '',\n  speed: 1,\n  newSentenceDelay: 400,\n  sentenceTokenizer: new basic.SentenceTokenizer(),\n  hyphenateWord: basic.hyphenateWord,\n  splitWords: basic.splitWords,\n};\n\ninterface AudioData {\n  pushedDuration: number;\n  done: boolean;\n}\n\ninterface TextData {\n  sentenceStream: SentenceStream;\n  pushedText: string;\n  done: boolean;\n  forwardedHyphens: number;\n  forwardedSentences: number;\n}\n\ntype SyncCallbacks = {\n  textUpdated: (text: TranscriptionSegment) => void;\n};\n\nexport class TextAudioSynchronizer extends (EventEmitter as new () => TypedEmitter<SyncCallbacks>) {\n  #opts: TextSyncOptions;\n  #speed: number;\n\n  #closed = false;\n  #interrupted = false;\n  #closeFut = new Future();\n\n  #playingSegIndex = -1;\n  #finishedSegIndex = -1;\n\n  #textQChanged = new AsyncIterableQueue<number>();\n  #textQ: (TextData | undefined)[] = [];\n  #audioQChanged = new AsyncIterableQueue<number>();\n  #audioQ: (AudioData | undefined)[] = [];\n\n  #playedText = '';\n  #task?: Promise<void>;\n\n  #audioData?: AudioData;\n  #textData?: TextData;\n\n  constructor(opts: TextSyncOptions) {\n    super();\n\n    this.#opts = opts;\n    this.#speed = opts.speed * STANDARD_SPEECH_RATE;\n  }\n\n  pushAudio(frame: AudioFrame) {\n    this.#checkNotClosed();\n    if (!this.#audioData) {\n      this.#audioData = { pushedDuration: 0, done: false };\n      this.#audioQ.push(this.#audioData);\n      this.#audioQChanged.put(1);\n    }\n    this.#audioData.pushedDuration += frame.samplesPerChannel / frame.sampleRate;\n  }\n\n  pushText(text: string) {\n    this.#checkNotClosed();\n    if (!this.#textData) {\n      this.#textData = {\n        sentenceStream: this.#opts.sentenceTokenizer.stream(),\n        pushedText: '',\n        done: false,\n        forwardedHyphens: 0,\n        forwardedSentences: 0,\n      };\n      this.#textQ.push(this.#textData);\n      this.#textQChanged.put(1);\n    }\n\n    this.#textData.pushedText += text;\n    this.#textData.sentenceStream.pushText(text);\n  }\n\n  markAudioSegmentEnd() {\n    this.#checkNotClosed();\n\n    if (!this.#audioData) {\n      // create empty audio data if none exists\n      this.pushAudio(new AudioFrame(new Int16Array(), 24000, 1, 0));\n    }\n\n    this.#audioData!.done = true;\n    this.#audioData = undefined;\n  }\n\n  markTextSegmentEnd() {\n    this.#checkNotClosed();\n\n    if (!this.#textData) {\n      this.pushText('');\n    }\n\n    this.#textData!.done = true;\n    this.#textData?.sentenceStream.flush();\n    this.#textData?.sentenceStream.close();\n    this.#textData = undefined;\n  }\n\n  segmentPlayoutStarted() {\n    this.#checkNotClosed();\n    this.#playingSegIndex++;\n\n    if (!this.#task) {\n      this.#task = this.#mainLoop();\n    }\n  }\n\n  segmentPlayoutFinished() {\n    this.#checkNotClosed();\n    this.#finishedSegIndex++;\n  }\n\n  get playedText(): string {\n    return this.#playedText;\n  }\n\n  async close(interrupt: boolean) {\n    if (this.#closed) {\n      return;\n    }\n    this.#closed = true;\n    this.#interrupted = interrupt;\n    this.#closeFut.resolve();\n\n    for (const textData of this.#textQ) {\n      textData?.sentenceStream.close();\n    }\n\n    this.#textQ.push(undefined);\n    this.#audioQ.push(undefined);\n    this.#textQChanged.put(1);\n    this.#audioQChanged.put(1);\n\n    await this.#task;\n  }\n\n  async #mainLoop() {\n    let segIndex = 0;\n    let qDone = false;\n\n    while (!qDone) {\n      await this.#textQChanged.next();\n      await this.#audioQChanged.next();\n\n      while (this.#textQ.length && this.#audioQ.length) {\n        const textData = this.#textQ.pop();\n        const audioData = this.#audioQ.pop();\n\n        if (!(textData && audioData)) {\n          qDone = true;\n          break;\n        }\n\n        // wait for segment to start playing\n        while (!this.#closed) {\n          if (this.#playingSegIndex >= segIndex) break;\n          await this.#sleepIfNotClosed(125);\n        }\n\n        const sentenceStream = textData.sentenceStream;\n        const forwardStartTime = Date.now();\n\n        for await (const ev of sentenceStream) {\n          await this.#syncSentence(segIndex, forwardStartTime, textData, audioData, ev.token);\n        }\n\n        segIndex++;\n      }\n    }\n  }\n\n  async #syncSentence(\n    segIndex: number,\n    segStartTime: number,\n    textData: TextData,\n    audioData: AudioData,\n    sentence: string,\n  ) {\n    let realSpeed: number | undefined;\n    if (audioData.pushedDuration > 0 && audioData.done) {\n      realSpeed = this.#calcHyphens(textData.pushedText).length / audioData.pushedDuration;\n    }\n\n    const segId = shortuuid('SG_');\n    const words = this.#opts.splitWords(sentence);\n    const processedWords: string[] = [];\n\n    const ogText = this.#playedText;\n    // eslint-disable-next-line @typescript-eslint/no-unused-vars\n    for (const [word, _, end] of words) {\n      if (segIndex <= this.#finishedSegIndex) break;\n      if (this.#interrupted) return;\n\n      const wordHyphens = this.#opts.hyphenateWord(word).length;\n      processedWords.push(word);\n\n      const elapsed = Date.now() - segStartTime;\n      const text = sentence.slice(0, end); // TODO: rstrip punctuations\n\n      let speed = this.#speed;\n      let delay: number;\n      if (realSpeed) {\n        speed = realSpeed;\n        const estimatedPausesMs = textData.forwardedSentences * this.#opts.newSentenceDelay;\n        const hyphPauses = estimatedPausesMs * speed;\n        const targetHyphens = Math.round(speed * elapsed);\n        const dt = targetHyphens - textData.forwardedHyphens - hyphPauses;\n        const toWaitHyphens = Math.max(0, wordHyphens - dt);\n        delay = toWaitHyphens / speed;\n      } else {\n        delay = wordHyphens / speed;\n      }\n\n      const firstDelay = Math.min(delay / 2, 2 / speed);\n      await this.#sleepIfNotClosed(firstDelay * 1000000);\n\n      this.emit(\n        'textUpdated',\n        new TranscriptionSegment({\n          id: segId,\n          text: text,\n          startTime: BigInt(0),\n          endTime: BigInt(0),\n          final: false,\n          language: this.#opts.language,\n        }),\n      );\n\n      this.#playedText = `${ogText} ${text}`;\n      await this.#sleepIfNotClosed((delay - firstDelay) * 1000000);\n      textData.forwardedHyphens += wordHyphens;\n    }\n\n    this.emit(\n      'textUpdated',\n      new TranscriptionSegment({\n        id: segId,\n        text: sentence,\n        startTime: BigInt(0),\n        endTime: BigInt(0),\n        final: true,\n        language: this.#opts.language,\n      }),\n    );\n\n    this.#playedText = `${ogText} ${sentence}`;\n\n    await this.#sleepIfNotClosed(this.#opts.newSentenceDelay);\n    textData.forwardedSentences++;\n  }\n\n  async #sleepIfNotClosed(delay: number) {\n    await Promise.race([\n      this.#closeFut.await,\n      new Promise((resolve) => setTimeout(resolve, delay)),\n    ]);\n  }\n\n  #calcHyphens(text: string): string[] {\n    const hyphens: string[] = [];\n    const words = this.#opts.splitWords(text);\n    for (const word of words) {\n      const n = this.#opts.hyphenateWord(word[0]);\n      hyphens.push(...n);\n    }\n    return hyphens;\n  }\n\n  #checkNotClosed() {\n    if (this.#closed) {\n      throw new Error('TextAudioSynchronizer is closed');\n    }\n  }\n}\n"],"mappings":"AAGA,SAAS,4BAA4B;AACrC,SAAS,kBAAkB;AAE3B,SAAS,oBAAoB;AAC7B,SAAS,aAAa;AAEtB,SAAS,oBAAoB,QAAQ,iBAAiB;AAGtD,MAAM,uBAAuB;AAWtB,MAAM,yBAA0C;AAAA,EACrD,UAAU;AAAA,EACV,OAAO;AAAA,EACP,kBAAkB;AAAA,EAClB,mBAAmB,IAAI,MAAM,kBAAkB;AAAA,EAC/C,eAAe,MAAM;AAAA,EACrB,YAAY,MAAM;AACpB;AAmBO,MAAM,8BAA+B,aAAuD;AAAA,EACjG;AAAA,EACA;AAAA,EAEA,UAAU;AAAA,EACV,eAAe;AAAA,EACf,YAAY,IAAI,OAAO;AAAA,EAEvB,mBAAmB;AAAA,EACnB,oBAAoB;AAAA,EAEpB,gBAAgB,IAAI,mBAA2B;AAAA,EAC/C,SAAmC,CAAC;AAAA,EACpC,iBAAiB,IAAI,mBAA2B;AAAA,EAChD,UAAqC,CAAC;AAAA,EAEtC,cAAc;AAAA,EACd;AAAA,EAEA;AAAA,EACA;AAAA,EAEA,YAAY,MAAuB;AACjC,UAAM;AAEN,SAAK,QAAQ;AACb,SAAK,SAAS,KAAK,QAAQ;AAAA,EAC7B;AAAA,EAEA,UAAU,OAAmB;AAC3B,SAAK,gBAAgB;AACrB,QAAI,CAAC,KAAK,YAAY;AACpB,WAAK,aAAa,EAAE,gBAAgB,GAAG,MAAM,MAAM;AACnD,WAAK,QAAQ,KAAK,KAAK,UAAU;AACjC,WAAK,eAAe,IAAI,CAAC;AAAA,IAC3B;AACA,SAAK,WAAW,kBAAkB,MAAM,oBAAoB,MAAM;AAAA,EACpE;AAAA,EAEA,SAAS,MAAc;AACrB,SAAK,gBAAgB;AACrB,QAAI,CAAC,KAAK,WAAW;AACnB,WAAK,YAAY;AAAA,QACf,gBAAgB,KAAK,MAAM,kBAAkB,OAAO;AAAA,QACpD,YAAY;AAAA,QACZ,MAAM;AAAA,QACN,kBAAkB;AAAA,QAClB,oBAAoB;AAAA,MACtB;AACA,WAAK,OAAO,KAAK,KAAK,SAAS;AAC/B,WAAK,cAAc,IAAI,CAAC;AAAA,IAC1B;AAEA,SAAK,UAAU,cAAc;AAC7B,SAAK,UAAU,eAAe,SAAS,IAAI;AAAA,EAC7C;AAAA,EAEA,sBAAsB;AACpB,SAAK,gBAAgB;AAErB,QAAI,CAAC,KAAK,YAAY;AAEpB,WAAK,UAAU,IAAI,WAAW,IAAI,WAAW,GAAG,MAAO,GAAG,CAAC,CAAC;AAAA,IAC9D;AAEA,SAAK,WAAY,OAAO;AACxB,SAAK,aAAa;AAAA,EACpB;AAAA,EAEA,qBAAqB;AAtHvB;AAuHI,SAAK,gBAAgB;AAErB,QAAI,CAAC,KAAK,WAAW;AACnB,WAAK,SAAS,EAAE;AAAA,IAClB;AAEA,SAAK,UAAW,OAAO;AACvB,eAAK,cAAL,mBAAgB,eAAe;AAC/B,eAAK,cAAL,mBAAgB,eAAe;AAC/B,SAAK,YAAY;AAAA,EACnB;AAAA,EAEA,wBAAwB;AACtB,SAAK,gBAAgB;AACrB,SAAK;AAEL,QAAI,CAAC,KAAK,OAAO;AACf,WAAK,QAAQ,KAAK,UAAU;AAAA,IAC9B;AAAA,EACF;AAAA,EAEA,yBAAyB;AACvB,SAAK,gBAAgB;AACrB,SAAK;AAAA,EACP;AAAA,EAEA,IAAI,aAAqB;AACvB,WAAO,KAAK;AAAA,EACd;AAAA,EAEA,MAAM,MAAM,WAAoB;AAC9B,QAAI,KAAK,SAAS;AAChB;AAAA,IACF;AACA,SAAK,UAAU;AACf,SAAK,eAAe;AACpB,SAAK,UAAU,QAAQ;AAEvB,eAAW,YAAY,KAAK,QAAQ;AAClC,2CAAU,eAAe;AAAA,IAC3B;AAEA,SAAK,OAAO,KAAK,MAAS;AAC1B,SAAK,QAAQ,KAAK,MAAS;AAC3B,SAAK,cAAc,IAAI,CAAC;AACxB,SAAK,eAAe,IAAI,CAAC;AAEzB,UAAM,KAAK;AAAA,EACb;AAAA,EAEA,MAAM,YAAY;AAChB,QAAI,WAAW;AACf,QAAI,QAAQ;AAEZ,WAAO,CAAC,OAAO;AACb,YAAM,KAAK,cAAc,KAAK;AAC9B,YAAM,KAAK,eAAe,KAAK;AAE/B,aAAO,KAAK,OAAO,UAAU,KAAK,QAAQ,QAAQ;AAChD,cAAM,WAAW,KAAK,OAAO,IAAI;AACjC,cAAM,YAAY,KAAK,QAAQ,IAAI;AAEnC,YAAI,EAAE,YAAY,YAAY;AAC5B,kBAAQ;AACR;AAAA,QACF;AAGA,eAAO,CAAC,KAAK,SAAS;AACpB,cAAI,KAAK,oBAAoB,SAAU;AACvC,gBAAM,KAAK,kBAAkB,GAAG;AAAA,QAClC;AAEA,cAAM,iBAAiB,SAAS;AAChC,cAAM,mBAAmB,KAAK,IAAI;AAElC,yBAAiB,MAAM,gBAAgB;AACrC,gBAAM,KAAK,cAAc,UAAU,kBAAkB,UAAU,WAAW,GAAG,KAAK;AAAA,QACpF;AAEA;AAAA,MACF;AAAA,IACF;AAAA,EACF;AAAA,EAEA,MAAM,cACJ,UACA,cACA,UACA,WACA,UACA;AACA,QAAI;AACJ,QAAI,UAAU,iBAAiB,KAAK,UAAU,MAAM;AAClD,kBAAY,KAAK,aAAa,SAAS,UAAU,EAAE,SAAS,UAAU;AAAA,IACxE;AAEA,UAAM,QAAQ,UAAU,KAAK;AAC7B,UAAM,QAAQ,KAAK,MAAM,WAAW,QAAQ;AAC5C,UAAM,iBAA2B,CAAC;AAElC,UAAM,SAAS,KAAK;AAEpB,eAAW,CAAC,MAAM,GAAG,GAAG,KAAK,OAAO;AAClC,UAAI,YAAY,KAAK,kBAAmB;AACxC,UAAI,KAAK,aAAc;AAEvB,YAAM,cAAc,KAAK,MAAM,cAAc,IAAI,EAAE;AACnD,qBAAe,KAAK,IAAI;AAExB,YAAM,UAAU,KAAK,IAAI,IAAI;AAC7B,YAAM,OAAO,SAAS,MAAM,GAAG,GAAG;AAElC,UAAI,QAAQ,KAAK;AACjB,UAAI;AACJ,UAAI,WAAW;AACb,gBAAQ;AACR,cAAM,oBAAoB,SAAS,qBAAqB,KAAK,MAAM;AACnE,cAAM,aAAa,oBAAoB;AACvC,cAAM,gBAAgB,KAAK,MAAM,QAAQ,OAAO;AAChD,cAAM,KAAK,gBAAgB,SAAS,mBAAmB;AACvD,cAAM,gBAAgB,KAAK,IAAI,GAAG,cAAc,EAAE;AAClD,gBAAQ,gBAAgB;AAAA,MAC1B,OAAO;AACL,gBAAQ,cAAc;AAAA,MACxB;AAEA,YAAM,aAAa,KAAK,IAAI,QAAQ,GAAG,IAAI,KAAK;AAChD,YAAM,KAAK,kBAAkB,aAAa,GAAO;AAEjD,WAAK;AAAA,QACH;AAAA,QACA,IAAI,qBAAqB;AAAA,UACvB,IAAI;AAAA,UACJ;AAAA,UACA,WAAW,OAAO,CAAC;AAAA,UACnB,SAAS,OAAO,CAAC;AAAA,UACjB,OAAO;AAAA,UACP,UAAU,KAAK,MAAM;AAAA,QACvB,CAAC;AAAA,MACH;AAEA,WAAK,cAAc,GAAG,MAAM,IAAI,IAAI;AACpC,YAAM,KAAK,mBAAmB,QAAQ,cAAc,GAAO;AAC3D,eAAS,oBAAoB;AAAA,IAC/B;AAEA,SAAK;AAAA,MACH;AAAA,MACA,IAAI,qBAAqB;AAAA,QACvB,IAAI;AAAA,QACJ,MAAM;AAAA,QACN,WAAW,OAAO,CAAC;AAAA,QACnB,SAAS,OAAO,CAAC;AAAA,QACjB,OAAO;AAAA,QACP,UAAU,KAAK,MAAM;AAAA,MACvB,CAAC;AAAA,IACH;AAEA,SAAK,cAAc,GAAG,MAAM,IAAI,QAAQ;AAExC,UAAM,KAAK,kBAAkB,KAAK,MAAM,gBAAgB;AACxD,aAAS;AAAA,EACX;AAAA,EAEA,MAAM,kBAAkB,OAAe;AACrC,UAAM,QAAQ,KAAK;AAAA,MACjB,KAAK,UAAU;AAAA,MACf,IAAI,QAAQ,CAAC,YAAY,WAAW,SAAS,KAAK,CAAC;AAAA,IACrD,CAAC;AAAA,EACH;AAAA,EAEA,aAAa,MAAwB;AACnC,UAAM,UAAoB,CAAC;AAC3B,UAAM,QAAQ,KAAK,MAAM,WAAW,IAAI;AACxC,eAAW,QAAQ,OAAO;AACxB,YAAM,IAAI,KAAK,MAAM,cAAc,KAAK,CAAC,CAAC;AAC1C,cAAQ,KAAK,GAAG,CAAC;AAAA,IACnB;AACA,WAAO;AAAA,EACT;AAAA,EAEA,kBAAkB;AAChB,QAAI,KAAK,SAAS;AAChB,YAAM,IAAI,MAAM,iCAAiC;AAAA,IACnD;AAAA,EACF;AACF;","names":[]}
+{"version":3,"sources":["../src/transcription.ts"],"sourcesContent":["// SPDX-FileCopyrightText: 2024 LiveKit, Inc.\n//\n// SPDX-License-Identifier: Apache-2.0\nimport { TranscriptionSegment } from '@livekit/protocol';\nimport { AudioFrame } from '@livekit/rtc-node';\nimport type { TypedEventEmitter as TypedEmitter } from '@livekit/typed-emitter';\nimport { EventEmitter } from 'node:events';\nimport { basic } from './tokenize/index.js';\nimport type { SentenceStream, SentenceTokenizer } from './tokenize/tokenizer.js';\nimport { AsyncIterableQueue, Future, shortuuid } from './utils.js';\n\n// standard speech rate in hyphens/ms\nconst STANDARD_SPEECH_RATE = 3830;\n\nexport interface TextSyncOptions {\n  /** Language code for transcription. */\n  language: string;\n  /** Speech speed multiplier. */\n  speed: number;\n  /** Delay between sentences in milliseconds. */\n  newSentenceDelay: number;\n  /** Tokenizer for splitting text into sentences. */\n  sentenceTokenizer: SentenceTokenizer;\n  /** Function to hyphenate words. */\n  hyphenateWord: (word: string) => string[];\n  /** Function to split text into words with positions. */\n  splitWords: (words: string) => [string, number, number][];\n}\n\nexport const defaultTextSyncOptions: TextSyncOptions = {\n  language: '',\n  speed: 1,\n  newSentenceDelay: 400,\n  sentenceTokenizer: new basic.SentenceTokenizer(),\n  hyphenateWord: basic.hyphenateWord,\n  splitWords: basic.splitWords,\n};\n\ninterface AudioData {\n  pushedDuration: number;\n  done: boolean;\n}\n\ninterface TextData {\n  sentenceStream: SentenceStream;\n  pushedText: string;\n  done: boolean;\n  forwardedHyphens: number;\n  forwardedSentences: number;\n}\n\ntype SyncCallbacks = {\n  textUpdated: (text: TranscriptionSegment) => void;\n};\n\nexport class TextAudioSynchronizer extends (EventEmitter as new () => TypedEmitter<SyncCallbacks>) {\n  #opts: TextSyncOptions;\n  #speed: number;\n\n  #closed = false;\n  #interrupted = false;\n  #closeFut = new Future();\n\n  #playingSegIndex = -1;\n  #finishedSegIndex = -1;\n\n  #textQChanged = new AsyncIterableQueue<number>();\n  #textQ: (TextData | undefined)[] = [];\n  #audioQChanged = new AsyncIterableQueue<number>();\n  #audioQ: (AudioData | undefined)[] = [];\n\n  #playedText = '';\n  #task?: Promise<void>;\n\n  #audioData?: AudioData;\n  #textData?: TextData;\n\n  constructor(opts: TextSyncOptions) {\n    super();\n\n    this.#opts = opts;\n    this.#speed = opts.speed * STANDARD_SPEECH_RATE;\n  }\n\n  pushAudio(frame: AudioFrame) {\n    this.#checkNotClosed();\n    if (!this.#audioData) {\n      this.#audioData = { pushedDuration: 0, done: false };\n      this.#audioQ.push(this.#audioData);\n      this.#audioQChanged.put(1);\n    }\n    this.#audioData.pushedDuration += frame.samplesPerChannel / frame.sampleRate;\n  }\n\n  pushText(text: string) {\n    this.#checkNotClosed();\n    if (!this.#textData) {\n      this.#textData = {\n        sentenceStream: this.#opts.sentenceTokenizer.stream(),\n        pushedText: '',\n        done: false,\n        forwardedHyphens: 0,\n        forwardedSentences: 0,\n      };\n      this.#textQ.push(this.#textData);\n      this.#textQChanged.put(1);\n    }\n\n    this.#textData.pushedText += text;\n    this.#textData.sentenceStream.pushText(text);\n  }\n\n  markAudioSegmentEnd() {\n    this.#checkNotClosed();\n\n    if (!this.#audioData) {\n      // create empty audio data if none exists\n      this.pushAudio(new AudioFrame(new Int16Array(), 24000, 1, 0));\n    }\n\n    this.#audioData!.done = true;\n    this.#audioData = undefined;\n  }\n\n  markTextSegmentEnd() {\n    this.#checkNotClosed();\n\n    if (!this.#textData) {\n      this.pushText('');\n    }\n\n    this.#textData!.done = true;\n    this.#textData?.sentenceStream.flush();\n    this.#textData?.sentenceStream.close();\n    this.#textData = undefined;\n  }\n\n  segmentPlayoutStarted() {\n    this.#checkNotClosed();\n    this.#playingSegIndex++;\n\n    if (!this.#task) {\n      this.#task = this.#mainLoop();\n    }\n  }\n\n  segmentPlayoutFinished() {\n    this.#checkNotClosed();\n    this.#finishedSegIndex++;\n  }\n\n  get playedText(): string {\n    return this.#playedText;\n  }\n\n  async close(interrupt: boolean) {\n    if (this.#closed) {\n      return;\n    }\n    this.#closed = true;\n    this.#interrupted = interrupt;\n    this.#closeFut.resolve();\n\n    for (const textData of this.#textQ) {\n      textData?.sentenceStream.close();\n    }\n\n    this.#textQ.push(undefined);\n    this.#audioQ.push(undefined);\n    this.#textQChanged.put(1);\n    this.#audioQChanged.put(1);\n\n    await this.#task;\n  }\n\n  async #mainLoop() {\n    let segIndex = 0;\n    let qDone = false;\n\n    while (!qDone) {\n      await this.#textQChanged.next();\n      await this.#audioQChanged.next();\n\n      while (this.#textQ.length && this.#audioQ.length) {\n        const textData = this.#textQ.pop();\n        const audioData = this.#audioQ.pop();\n\n        if (!(textData && audioData)) {\n          qDone = true;\n          break;\n        }\n\n        // wait for segment to start playing\n        while (!this.#closed) {\n          if (this.#playingSegIndex >= segIndex) break;\n          await this.#sleepIfNotClosed(125);\n        }\n\n        const sentenceStream = textData.sentenceStream;\n        const forwardStartTime = Date.now();\n\n        for await (const ev of sentenceStream) {\n          await this.#syncSentence(segIndex, forwardStartTime, textData, audioData, ev.token);\n        }\n\n        segIndex++;\n      }\n    }\n  }\n\n  async #syncSentence(\n    segIndex: number,\n    segStartTime: number,\n    textData: TextData,\n    audioData: AudioData,\n    sentence: string,\n  ) {\n    let realSpeed: number | undefined;\n    if (audioData.pushedDuration > 0 && audioData.done) {\n      realSpeed = this.#calcHyphens(textData.pushedText).length / audioData.pushedDuration;\n    }\n\n    const segId = shortuuid('SG_');\n    const words = this.#opts.splitWords(sentence);\n    const processedWords: string[] = [];\n\n    const ogText = this.#playedText;\n    // eslint-disable-next-line @typescript-eslint/no-unused-vars\n    for (const [word, _, end] of words) {\n      if (segIndex <= this.#finishedSegIndex) break;\n      if (this.#interrupted) return;\n\n      const wordHyphens = this.#opts.hyphenateWord(word).length;\n      processedWords.push(word);\n\n      const elapsed = Date.now() - segStartTime;\n      const text = sentence.slice(0, end); // TODO: rstrip punctuations\n\n      let speed = this.#speed;\n      let delay: number;\n      if (realSpeed) {\n        speed = realSpeed;\n        const estimatedPausesMs = textData.forwardedSentences * this.#opts.newSentenceDelay;\n        const hyphPauses = estimatedPausesMs * speed;\n        const targetHyphens = Math.round(speed * elapsed);\n        const dt = targetHyphens - textData.forwardedHyphens - hyphPauses;\n        const toWaitHyphens = Math.max(0, wordHyphens - dt);\n        delay = toWaitHyphens / speed;\n      } else {\n        delay = wordHyphens / speed;\n      }\n\n      const firstDelay = Math.min(delay / 2, 2 / speed);\n      await this.#sleepIfNotClosed(firstDelay * 1000000);\n\n      this.emit(\n        'textUpdated',\n        new TranscriptionSegment({\n          id: segId,\n          text: text,\n          startTime: BigInt(0),\n          endTime: BigInt(0),\n          final: false,\n          language: this.#opts.language,\n        }),\n      );\n\n      this.#playedText = `${ogText} ${text}`;\n      await this.#sleepIfNotClosed((delay - firstDelay) * 1000000);\n      textData.forwardedHyphens += wordHyphens;\n    }\n\n    this.emit(\n      'textUpdated',\n      new TranscriptionSegment({\n        id: segId,\n        text: sentence,\n        startTime: BigInt(0),\n        endTime: BigInt(0),\n        final: true,\n        language: this.#opts.language,\n      }),\n    );\n\n    this.#playedText = `${ogText} ${sentence}`;\n\n    await this.#sleepIfNotClosed(this.#opts.newSentenceDelay);\n    textData.forwardedSentences++;\n  }\n\n  async #sleepIfNotClosed(delay: number) {\n    await Promise.race([\n      this.#closeFut.await,\n      new Promise((resolve) => setTimeout(resolve, delay)),\n    ]);\n  }\n\n  #calcHyphens(text: string): string[] {\n    const hyphens: string[] = [];\n    const words = this.#opts.splitWords(text);\n    for (const word of words) {\n      const n = this.#opts.hyphenateWord(word[0]);\n      hyphens.push(...n);\n    }\n    return hyphens;\n  }\n\n  #checkNotClosed() {\n    if (this.#closed) {\n      throw new Error('TextAudioSynchronizer is closed');\n    }\n  }\n}\n"],"mappings":"AAGA,SAAS,4BAA4B;AACrC,SAAS,kBAAkB;AAE3B,SAAS,oBAAoB;AAC7B,SAAS,aAAa;AAEtB,SAAS,oBAAoB,QAAQ,iBAAiB;AAGtD,MAAM,uBAAuB;AAiBtB,MAAM,yBAA0C;AAAA,EACrD,UAAU;AAAA,EACV,OAAO;AAAA,EACP,kBAAkB;AAAA,EAClB,mBAAmB,IAAI,MAAM,kBAAkB;AAAA,EAC/C,eAAe,MAAM;AAAA,EACrB,YAAY,MAAM;AACpB;AAmBO,MAAM,8BAA+B,aAAuD;AAAA,EACjG;AAAA,EACA;AAAA,EAEA,UAAU;AAAA,EACV,eAAe;AAAA,EACf,YAAY,IAAI,OAAO;AAAA,EAEvB,mBAAmB;AAAA,EACnB,oBAAoB;AAAA,EAEpB,gBAAgB,IAAI,mBAA2B;AAAA,EAC/C,SAAmC,CAAC;AAAA,EACpC,iBAAiB,IAAI,mBAA2B;AAAA,EAChD,UAAqC,CAAC;AAAA,EAEtC,cAAc;AAAA,EACd;AAAA,EAEA;AAAA,EACA;AAAA,EAEA,YAAY,MAAuB;AACjC,UAAM;AAEN,SAAK,QAAQ;AACb,SAAK,SAAS,KAAK,QAAQ;AAAA,EAC7B;AAAA,EAEA,UAAU,OAAmB;AAC3B,SAAK,gBAAgB;AACrB,QAAI,CAAC,KAAK,YAAY;AACpB,WAAK,aAAa,EAAE,gBAAgB,GAAG,MAAM,MAAM;AACnD,WAAK,QAAQ,KAAK,KAAK,UAAU;AACjC,WAAK,eAAe,IAAI,CAAC;AAAA,IAC3B;AACA,SAAK,WAAW,kBAAkB,MAAM,oBAAoB,MAAM;AAAA,EACpE;AAAA,EAEA,SAAS,MAAc;AACrB,SAAK,gBAAgB;AACrB,QAAI,CAAC,KAAK,WAAW;AACnB,WAAK,YAAY;AAAA,QACf,gBAAgB,KAAK,MAAM,kBAAkB,OAAO;AAAA,QACpD,YAAY;AAAA,QACZ,MAAM;AAAA,QACN,kBAAkB;AAAA,QAClB,oBAAoB;AAAA,MACtB;AACA,WAAK,OAAO,KAAK,KAAK,SAAS;AAC/B,WAAK,cAAc,IAAI,CAAC;AAAA,IAC1B;AAEA,SAAK,UAAU,cAAc;AAC7B,SAAK,UAAU,eAAe,SAAS,IAAI;AAAA,EAC7C;AAAA,EAEA,sBAAsB;AACpB,SAAK,gBAAgB;AAErB,QAAI,CAAC,KAAK,YAAY;AAEpB,WAAK,UAAU,IAAI,WAAW,IAAI,WAAW,GAAG,MAAO,GAAG,CAAC,CAAC;AAAA,IAC9D;AAEA,SAAK,WAAY,OAAO;AACxB,SAAK,aAAa;AAAA,EACpB;AAAA,EAEA,qBAAqB;AA5HvB;AA6HI,SAAK,gBAAgB;AAErB,QAAI,CAAC,KAAK,WAAW;AACnB,WAAK,SAAS,EAAE;AAAA,IAClB;AAEA,SAAK,UAAW,OAAO;AACvB,eAAK,cAAL,mBAAgB,eAAe;AAC/B,eAAK,cAAL,mBAAgB,eAAe;AAC/B,SAAK,YAAY;AAAA,EACnB;AAAA,EAEA,wBAAwB;AACtB,SAAK,gBAAgB;AACrB,SAAK;AAEL,QAAI,CAAC,KAAK,OAAO;AACf,WAAK,QAAQ,KAAK,UAAU;AAAA,IAC9B;AAAA,EACF;AAAA,EAEA,yBAAyB;AACvB,SAAK,gBAAgB;AACrB,SAAK;AAAA,EACP;AAAA,EAEA,IAAI,aAAqB;AACvB,WAAO,KAAK;AAAA,EACd;AAAA,EAEA,MAAM,MAAM,WAAoB;AAC9B,QAAI,KAAK,SAAS;AAChB;AAAA,IACF;AACA,SAAK,UAAU;AACf,SAAK,eAAe;AACpB,SAAK,UAAU,QAAQ;AAEvB,eAAW,YAAY,KAAK,QAAQ;AAClC,2CAAU,eAAe;AAAA,IAC3B;AAEA,SAAK,OAAO,KAAK,MAAS;AAC1B,SAAK,QAAQ,KAAK,MAAS;AAC3B,SAAK,cAAc,IAAI,CAAC;AACxB,SAAK,eAAe,IAAI,CAAC;AAEzB,UAAM,KAAK;AAAA,EACb;AAAA,EAEA,MAAM,YAAY;AAChB,QAAI,WAAW;AACf,QAAI,QAAQ;AAEZ,WAAO,CAAC,OAAO;AACb,YAAM,KAAK,cAAc,KAAK;AAC9B,YAAM,KAAK,eAAe,KAAK;AAE/B,aAAO,KAAK,OAAO,UAAU,KAAK,QAAQ,QAAQ;AAChD,cAAM,WAAW,KAAK,OAAO,IAAI;AACjC,cAAM,YAAY,KAAK,QAAQ,IAAI;AAEnC,YAAI,EAAE,YAAY,YAAY;AAC5B,kBAAQ;AACR;AAAA,QACF;AAGA,eAAO,CAAC,KAAK,SAAS;AACpB,cAAI,KAAK,oBAAoB,SAAU;AACvC,gBAAM,KAAK,kBAAkB,GAAG;AAAA,QAClC;AAEA,cAAM,iBAAiB,SAAS;AAChC,cAAM,mBAAmB,KAAK,IAAI;AAElC,yBAAiB,MAAM,gBAAgB;AACrC,gBAAM,KAAK,cAAc,UAAU,kBAAkB,UAAU,WAAW,GAAG,KAAK;AAAA,QACpF;AAEA;AAAA,MACF;AAAA,IACF;AAAA,EACF;AAAA,EAEA,MAAM,cACJ,UACA,cACA,UACA,WACA,UACA;AACA,QAAI;AACJ,QAAI,UAAU,iBAAiB,KAAK,UAAU,MAAM;AAClD,kBAAY,KAAK,aAAa,SAAS,UAAU,EAAE,SAAS,UAAU;AAAA,IACxE;AAEA,UAAM,QAAQ,UAAU,KAAK;AAC7B,UAAM,QAAQ,KAAK,MAAM,WAAW,QAAQ;AAC5C,UAAM,iBAA2B,CAAC;AAElC,UAAM,SAAS,KAAK;AAEpB,eAAW,CAAC,MAAM,GAAG,GAAG,KAAK,OAAO;AAClC,UAAI,YAAY,KAAK,kBAAmB;AACxC,UAAI,KAAK,aAAc;AAEvB,YAAM,cAAc,KAAK,MAAM,cAAc,IAAI,EAAE;AACnD,qBAAe,KAAK,IAAI;AAExB,YAAM,UAAU,KAAK,IAAI,IAAI;AAC7B,YAAM,OAAO,SAAS,MAAM,GAAG,GAAG;AAElC,UAAI,QAAQ,KAAK;AACjB,UAAI;AACJ,UAAI,WAAW;AACb,gBAAQ;AACR,cAAM,oBAAoB,SAAS,qBAAqB,KAAK,MAAM;AACnE,cAAM,aAAa,oBAAoB;AACvC,cAAM,gBAAgB,KAAK,MAAM,QAAQ,OAAO;AAChD,cAAM,KAAK,gBAAgB,SAAS,mBAAmB;AACvD,cAAM,gBAAgB,KAAK,IAAI,GAAG,cAAc,EAAE;AAClD,gBAAQ,gBAAgB;AAAA,MAC1B,OAAO;AACL,gBAAQ,cAAc;AAAA,MACxB;AAEA,YAAM,aAAa,KAAK,IAAI,QAAQ,GAAG,IAAI,KAAK;AAChD,YAAM,KAAK,kBAAkB,aAAa,GAAO;AAEjD,WAAK;AAAA,QACH;AAAA,QACA,IAAI,qBAAqB;AAAA,UACvB,IAAI;AAAA,UACJ;AAAA,UACA,WAAW,OAAO,CAAC;AAAA,UACnB,SAAS,OAAO,CAAC;AAAA,UACjB,OAAO;AAAA,UACP,UAAU,KAAK,MAAM;AAAA,QACvB,CAAC;AAAA,MACH;AAEA,WAAK,cAAc,GAAG,MAAM,IAAI,IAAI;AACpC,YAAM,KAAK,mBAAmB,QAAQ,cAAc,GAAO;AAC3D,eAAS,oBAAoB;AAAA,IAC/B;AAEA,SAAK;AAAA,MACH;AAAA,MACA,IAAI,qBAAqB;AAAA,QACvB,IAAI;AAAA,QACJ,MAAM;AAAA,QACN,WAAW,OAAO,CAAC;AAAA,QACnB,SAAS,OAAO,CAAC;AAAA,QACjB,OAAO;AAAA,QACP,UAAU,KAAK,MAAM;AAAA,MACvB,CAAC;AAAA,IACH;AAEA,SAAK,cAAc,GAAG,MAAM,IAAI,QAAQ;AAExC,UAAM,KAAK,kBAAkB,KAAK,MAAM,gBAAgB;AACxD,aAAS;AAAA,EACX;AAAA,EAEA,MAAM,kBAAkB,OAAe;AACrC,UAAM,QAAQ,KAAK;AAAA,MACjB,KAAK,UAAU;AAAA,MACf,IAAI,QAAQ,CAAC,YAAY,WAAW,SAAS,KAAK,CAAC;AAAA,IACrD,CAAC;AAAA,EACH;AAAA,EAEA,aAAa,MAAwB;AACnC,UAAM,UAAoB,CAAC;AAC3B,UAAM,QAAQ,KAAK,MAAM,WAAW,IAAI;AACxC,eAAW,QAAQ,OAAO;AACxB,YAAM,IAAI,KAAK,MAAM,cAAc,KAAK,CAAC,CAAC;AAC1C,cAAQ,KAAK,GAAG,CAAC;AAAA,IACnB;AACA,WAAO;AAAA,EACT;AAAA,EAEA,kBAAkB;AAChB,QAAI,KAAK,SAAS;AAChB,YAAM,IAAI,MAAM,iCAAiC;AAAA,IACnD;AAAA,EACF;AACF;","names":[]}