@livekit/agents 0.5.2 → 0.6.1

package/dist/pipeline/speech_handle.cjs

@@ -31,16 +31,24 @@ class SpeechHandle {
   #userQuestion;
   #userCommitted = false;
   #initFut = new import_utils.Future();
+  #doneFut = new import_utils.Future();
   #speechCommitted = false;
   #source;
   #synthesisHandle;
   #initialized = false;
-  constructor(id, allowInterruptions, addToChatCtx, isReply, userQuestion) {
+  #fncNestedDepth;
+  #fncExtraToolsMesages;
+  #nestedSpeechHandles = [];
+  #nestedSpeechChanged = new import_utils.AsyncIterableQueue();
+  #nestedSpeechFinished = false;
+  constructor(id, allowInterruptions, addToChatCtx, isReply, userQuestion, fncNestedDepth = 0, extraToolsMessages = void 0) {
     this.#id = id;
     this.#allowInterruptions = allowInterruptions;
     this.#addToChatCtx = addToChatCtx;
     this.#isReply = isReply;
     this.#userQuestion = userQuestion;
+    this.#fncNestedDepth = fncNestedDepth;
+    this.#fncExtraToolsMesages = extraToolsMessages;
   }
   static createAssistantReply(allowInterruptions, addToChatCtx, userQuestion) {
     return new SpeechHandle((0, import_crypto.randomUUID)(), allowInterruptions, addToChatCtx, true, userQuestion);
@@ -48,6 +56,17 @@ class SpeechHandle {
   static createAssistantSpeech(allowInterruptions, addToChatCtx) {
     return new SpeechHandle((0, import_crypto.randomUUID)(), allowInterruptions, addToChatCtx, false, "");
   }
+  static createToolSpeech(allowInterruptions, addToChatCtx, fncNestedDepth, extraToolsMessages) {
+    return new SpeechHandle(
+      (0, import_crypto.randomUUID)(),
+      allowInterruptions,
+      addToChatCtx,
+      false,
+      "",
+      fncNestedDepth,
+      extraToolsMessages
+    );
+  }
   async waitForInitialization() {
     await this.#initFut.await;
   }
@@ -109,6 +128,34 @@ class SpeechHandle {
     var _a;
     return !!((_a = this.#synthesisHandle) == null ? void 0 : _a.interrupted);
   }
+  get fncNestedDepth() {
+    return this.#fncNestedDepth;
+  }
+  get extraToolsMessages() {
+    return this.#fncExtraToolsMesages;
+  }
+  addNestedSpeech(handle) {
+    this.#nestedSpeechHandles.push(handle);
+    this.#nestedSpeechChanged.put();
+  }
+  get nestedSpeechHandles() {
+    return this.#nestedSpeechHandles;
+  }
+  async nestedSpeechChanged() {
+    await this.#nestedSpeechChanged.next();
+  }
+  get nestedSpeechFinished() {
+    return this.#nestedSpeechFinished;
+  }
+  markNestedSpeechFinished() {
+    this.#nestedSpeechFinished = true;
+  }
+  join() {
+    return this.#doneFut.await;
+  }
+  setDone() {
+    this.#doneFut.resolve();
+  }
   interrupt() {
     if (!this.#allowInterruptions) {
       throw new Error("interruptions are not allowed");
@@ -118,6 +165,7 @@ class SpeechHandle {
   cancel() {
     var _a;
     this.#initFut.reject(new Error());
+    this.#nestedSpeechChanged.close();
     (_a = this.#synthesisHandle) == null ? void 0 : _a.interrupt();
   }
 }

package/dist/pipeline/speech_handle.cjs.map

@@ -1 +1 @@
-{"version":3,"sources":["../../src/pipeline/speech_handle.ts"],"sourcesContent":["// SPDX-FileCopyrightText: 2024 LiveKit, Inc.\n//\n// SPDX-License-Identifier: Apache-2.0\nimport { randomUUID } from 'crypto';\nimport type { LLMStream } from '../llm/index.js';\nimport { Future } from '../utils.js';\nimport type { SynthesisHandle } from './agent_output.js';\n\nexport class SpeechHandle {\n  #id: string;\n  #allowInterruptions: boolean;\n  #addToChatCtx: boolean;\n  #isReply: boolean;\n  #userQuestion: string;\n  #userCommitted = false;\n  #initFut = new Future();\n  #speechCommitted = false;\n  #source?: string | LLMStream | AsyncIterable<string>;\n  #synthesisHandle?: SynthesisHandle;\n  #initialized = false;\n\n  constructor(\n    id: string,\n    allowInterruptions: boolean,\n    addToChatCtx: boolean,\n    isReply: boolean,\n    userQuestion: string,\n  ) {\n    this.#id = id;\n    this.#allowInterruptions = allowInterruptions;\n    this.#addToChatCtx = addToChatCtx;\n    this.#isReply = isReply;\n    this.#userQuestion = userQuestion;\n  }\n\n  static createAssistantReply(\n    allowInterruptions: boolean,\n    addToChatCtx: boolean,\n    userQuestion: string,\n  ): SpeechHandle {\n    return new SpeechHandle(randomUUID(), allowInterruptions, addToChatCtx, true, userQuestion);\n  }\n\n  static createAssistantSpeech(allowInterruptions: boolean, addToChatCtx: boolean): SpeechHandle {\n    return new SpeechHandle(randomUUID(), allowInterruptions, addToChatCtx, false, '');\n  }\n\n  async waitForInitialization() {\n    await this.#initFut.await;\n  }\n\n  initialize(source: string | LLMStream | AsyncIterable<string>, synthesisHandle: SynthesisHandle) {\n    if (this.interrupted) {\n      throw new Error('speech was interrupted');\n    }\n\n    this.#source = source;\n    this.#synthesisHandle = synthesisHandle;\n    this.#initialized = true;\n    this.#initFut.resolve();\n  }\n\n  markUserCommitted() {\n    this.#userCommitted = true;\n  }\n\n  markSpeechCommitted() {\n    this.#speechCommitted = true;\n  }\n\n  get userCommitted(): boolean {\n    return this.#userCommitted;\n  }\n\n  get speechCommitted(): boolean {\n    return this.#speechCommitted;\n  }\n\n  get id(): string {\n    return this.#id;\n  }\n\n  get allowInterruptions(): boolean {\n    return this.#allowInterruptions;\n  }\n\n  get addToChatCtx(): boolean {\n    return this.#addToChatCtx;\n  }\n\n  get source(): string | LLMStream | AsyncIterable<string> {\n    if (!this.#source) {\n      throw new Error('speech not initialized');\n    }\n    return this.#source;\n  }\n\n  get synthesisHandle(): SynthesisHandle {\n    if (!this.#synthesisHandle) {\n      throw new Error('speech not initialized');\n    }\n    return this.#synthesisHandle;\n  }\n\n  set synthesisHandle(handle: SynthesisHandle) {\n    this.#synthesisHandle = handle;\n  }\n\n  get initialized(): boolean {\n    return this.#initialized;\n  }\n\n  get isReply(): boolean {\n    return this.#isReply;\n  }\n\n  get userQuestion(): string {\n    return this.#userQuestion;\n  }\n\n  get interrupted(): boolean {\n    return !!this.#synthesisHandle?.interrupted;\n  }\n\n  interrupt() {\n    if (!this.#allowInterruptions) {\n      throw new Error('interruptions are not allowed');\n    }\n    this.cancel();\n  }\n\n  cancel() {\n    this.#initFut.reject(new Error());\n    this.#synthesisHandle?.interrupt();\n  }\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;AAAA;AAAA;AAAA;AAAA;AAAA;AAGA,oBAA2B;AAE3B,mBAAuB;AAGhB,MAAM,aAAa;AAAA,EACxB;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA,iBAAiB;AAAA,EACjB,WAAW,IAAI,oBAAO;AAAA,EACtB,mBAAmB;AAAA,EACnB;AAAA,EACA;AAAA,EACA,eAAe;AAAA,EAEf,YACE,IACA,oBACA,cACA,SACA,cACA;AACA,SAAK,MAAM;AACX,SAAK,sBAAsB;AAC3B,SAAK,gBAAgB;AACrB,SAAK,WAAW;AAChB,SAAK,gBAAgB;AAAA,EACvB;AAAA,EAEA,OAAO,qBACL,oBACA,cACA,cACc;AACd,WAAO,IAAI,iBAAa,0BAAW,GAAG,oBAAoB,cAAc,MAAM,YAAY;AAAA,EAC5F;AAAA,EAEA,OAAO,sBAAsB,oBAA6B,cAAqC;AAC7F,WAAO,IAAI,iBAAa,0BAAW,GAAG,oBAAoB,cAAc,OAAO,EAAE;AAAA,EACnF;AAAA,EAEA,MAAM,wBAAwB;AAC5B,UAAM,KAAK,SAAS;AAAA,EACtB;AAAA,EAEA,WAAW,QAAoD,iBAAkC;AAC/F,QAAI,KAAK,aAAa;AACpB,YAAM,IAAI,MAAM,wBAAwB;AAAA,IAC1C;AAEA,SAAK,UAAU;AACf,SAAK,mBAAmB;AACxB,SAAK,eAAe;AACpB,SAAK,SAAS,QAAQ;AAAA,EACxB;AAAA,EAEA,oBAAoB;AAClB,SAAK,iBAAiB;AAAA,EACxB;AAAA,EAEA,sBAAsB;AACpB,SAAK,mBAAmB;AAAA,EAC1B;AAAA,EAEA,IAAI,gBAAyB;AAC3B,WAAO,KAAK;AAAA,EACd;AAAA,EAEA,IAAI,kBAA2B;AAC7B,WAAO,KAAK;AAAA,EACd;AAAA,EAEA,IAAI,KAAa;AACf,WAAO,KAAK;AAAA,EACd;AAAA,EAEA,IAAI,qBAA8B;AAChC,WAAO,KAAK;AAAA,EACd;AAAA,EAEA,IAAI,eAAwB;AAC1B,WAAO,KAAK;AAAA,EACd;AAAA,EAEA,IAAI,SAAqD;AACvD,QAAI,CAAC,KAAK,SAAS;AACjB,YAAM,IAAI,MAAM,wBAAwB;AAAA,IAC1C;AACA,WAAO,KAAK;AAAA,EACd;AAAA,EAEA,IAAI,kBAAmC;AACrC,QAAI,CAAC,KAAK,kBAAkB;AAC1B,YAAM,IAAI,MAAM,wBAAwB;AAAA,IAC1C;AACA,WAAO,KAAK;AAAA,EACd;AAAA,EAEA,IAAI,gBAAgB,QAAyB;AAC3C,SAAK,mBAAmB;AAAA,EAC1B;AAAA,EAEA,IAAI,cAAuB;AACzB,WAAO,KAAK;AAAA,EACd;AAAA,EAEA,IAAI,UAAmB;AACrB,WAAO,KAAK;AAAA,EACd;AAAA,EAEA,IAAI,eAAuB;AACzB,WAAO,KAAK;AAAA,EACd;AAAA,EAEA,IAAI,cAAuB;AAxH7B;AAyHI,WAAO,CAAC,GAAC,UAAK,qBAAL,mBAAuB;AAAA,EAClC;AAAA,EAEA,YAAY;AACV,QAAI,CAAC,KAAK,qBAAqB;AAC7B,YAAM,IAAI,MAAM,+BAA+B;AAAA,IACjD;AACA,SAAK,OAAO;AAAA,EACd;AAAA,EAEA,SAAS;AAnIX;AAoII,SAAK,SAAS,OAAO,IAAI,MAAM,CAAC;AAChC,eAAK,qBAAL,mBAAuB;AAAA,EACzB;AACF;","names":[]}
+{"version":3,"sources":["../../src/pipeline/speech_handle.ts"],"sourcesContent":["// SPDX-FileCopyrightText: 2024 LiveKit, Inc.\n//\n// SPDX-License-Identifier: Apache-2.0\nimport { randomUUID } from 'crypto';\nimport type { ChatMessage, LLMStream } from '../llm/index.js';\nimport { AsyncIterableQueue, Future } from '../utils.js';\nimport type { SynthesisHandle } from './agent_output.js';\n\nexport class SpeechHandle {\n  #id: string;\n  #allowInterruptions: boolean;\n  #addToChatCtx: boolean;\n  #isReply: boolean;\n  #userQuestion: string;\n  #userCommitted = false;\n  #initFut = new Future();\n  #doneFut = new Future();\n  #speechCommitted = false;\n  #source?: string | LLMStream | AsyncIterable<string>;\n  #synthesisHandle?: SynthesisHandle;\n  #initialized = false;\n  #fncNestedDepth: number;\n  #fncExtraToolsMesages?: ChatMessage[];\n  #nestedSpeechHandles: SpeechHandle[] = [];\n  #nestedSpeechChanged = new AsyncIterableQueue<void>();\n  #nestedSpeechFinished = false;\n\n  constructor(\n    id: string,\n    allowInterruptions: boolean,\n    addToChatCtx: boolean,\n    isReply: boolean,\n    userQuestion: string,\n    fncNestedDepth = 0,\n    extraToolsMessages: ChatMessage[] | undefined = undefined,\n  ) {\n    this.#id = id;\n    this.#allowInterruptions = allowInterruptions;\n    this.#addToChatCtx = addToChatCtx;\n    this.#isReply = isReply;\n    this.#userQuestion = userQuestion;\n    this.#fncNestedDepth = fncNestedDepth;\n    this.#fncExtraToolsMesages = extraToolsMessages;\n  }\n\n  static createAssistantReply(\n    allowInterruptions: boolean,\n    addToChatCtx: boolean,\n    userQuestion: string,\n  ): SpeechHandle {\n    return new SpeechHandle(randomUUID(), allowInterruptions, addToChatCtx, true, userQuestion);\n  }\n\n  static createAssistantSpeech(allowInterruptions: boolean, addToChatCtx: boolean): SpeechHandle {\n    return new SpeechHandle(randomUUID(), allowInterruptions, addToChatCtx, false, '');\n  }\n\n  static createToolSpeech(\n    allowInterruptions: boolean,\n    addToChatCtx: boolean,\n    fncNestedDepth: number,\n    extraToolsMessages: ChatMessage[],\n  ): SpeechHandle {\n    return new SpeechHandle(\n      randomUUID(),\n      allowInterruptions,\n      addToChatCtx,\n      false,\n      '',\n      fncNestedDepth,\n      extraToolsMessages,\n    );\n  }\n\n  async waitForInitialization() {\n    await this.#initFut.await;\n  }\n\n  initialize(source: string | LLMStream | AsyncIterable<string>, synthesisHandle: SynthesisHandle) {\n    if (this.interrupted) {\n      throw new Error('speech was interrupted');\n    }\n\n    this.#source = source;\n    this.#synthesisHandle = synthesisHandle;\n    this.#initialized = true;\n    this.#initFut.resolve();\n  }\n\n  markUserCommitted() {\n    this.#userCommitted = true;\n  }\n\n  markSpeechCommitted() {\n    this.#speechCommitted = true;\n  }\n\n  get userCommitted(): boolean {\n    return this.#userCommitted;\n  }\n\n  get speechCommitted(): boolean {\n    return this.#speechCommitted;\n  }\n\n  get id(): string {\n    return this.#id;\n  }\n\n  get allowInterruptions(): boolean {\n    return this.#allowInterruptions;\n  }\n\n  get addToChatCtx(): boolean {\n    return this.#addToChatCtx;\n  }\n\n  get source(): string | LLMStream | AsyncIterable<string> {\n    if (!this.#source) {\n      throw new Error('speech not initialized');\n    }\n    return this.#source;\n  }\n\n  get synthesisHandle(): SynthesisHandle {\n    if (!this.#synthesisHandle) {\n      throw new Error('speech not initialized');\n    }\n    return this.#synthesisHandle;\n  }\n\n  set synthesisHandle(handle: SynthesisHandle) {\n    this.#synthesisHandle = handle;\n  }\n\n  get initialized(): boolean {\n    return this.#initialized;\n  }\n\n  get isReply(): boolean {\n    return this.#isReply;\n  }\n\n  get userQuestion(): string {\n    return this.#userQuestion;\n  }\n\n  get interrupted(): boolean {\n    return !!this.#synthesisHandle?.interrupted;\n  }\n\n  get fncNestedDepth(): number {\n    return this.#fncNestedDepth;\n  }\n\n  get extraToolsMessages(): ChatMessage[] | undefined {\n    return this.#fncExtraToolsMesages;\n  }\n\n  addNestedSpeech(handle: SpeechHandle) {\n    this.#nestedSpeechHandles.push(handle);\n    this.#nestedSpeechChanged.put();\n  }\n\n  get nestedSpeechHandles(): SpeechHandle[] {\n    return this.#nestedSpeechHandles;\n  }\n\n  async nestedSpeechChanged() {\n    await this.#nestedSpeechChanged.next();\n  }\n\n  get nestedSpeechFinished(): boolean {\n    return this.#nestedSpeechFinished;\n  }\n\n  markNestedSpeechFinished() {\n    this.#nestedSpeechFinished = true;\n  }\n\n  join() {\n    return this.#doneFut.await;\n  }\n\n  setDone() {\n    this.#doneFut.resolve();\n  }\n\n  interrupt() {\n    if (!this.#allowInterruptions) {\n      throw new Error('interruptions are not allowed');\n    }\n    this.cancel();\n  }\n\n  cancel() {\n    this.#initFut.reject(new Error());\n    this.#nestedSpeechChanged.close();\n    this.#synthesisHandle?.interrupt();\n  }\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;AAAA;AAAA;AAAA;AAAA;AAAA;AAGA,oBAA2B;AAE3B,mBAA2C;AAGpC,MAAM,aAAa;AAAA,EACxB;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA,iBAAiB;AAAA,EACjB,WAAW,IAAI,oBAAO;AAAA,EACtB,WAAW,IAAI,oBAAO;AAAA,EACtB,mBAAmB;AAAA,EACnB;AAAA,EACA;AAAA,EACA,eAAe;AAAA,EACf;AAAA,EACA;AAAA,EACA,uBAAuC,CAAC;AAAA,EACxC,uBAAuB,IAAI,gCAAyB;AAAA,EACpD,wBAAwB;AAAA,EAExB,YACE,IACA,oBACA,cACA,SACA,cACA,iBAAiB,GACjB,qBAAgD,QAChD;AACA,SAAK,MAAM;AACX,SAAK,sBAAsB;AAC3B,SAAK,gBAAgB;AACrB,SAAK,WAAW;AAChB,SAAK,gBAAgB;AACrB,SAAK,kBAAkB;AACvB,SAAK,wBAAwB;AAAA,EAC/B;AAAA,EAEA,OAAO,qBACL,oBACA,cACA,cACc;AACd,WAAO,IAAI,iBAAa,0BAAW,GAAG,oBAAoB,cAAc,MAAM,YAAY;AAAA,EAC5F;AAAA,EAEA,OAAO,sBAAsB,oBAA6B,cAAqC;AAC7F,WAAO,IAAI,iBAAa,0BAAW,GAAG,oBAAoB,cAAc,OAAO,EAAE;AAAA,EACnF;AAAA,EAEA,OAAO,iBACL,oBACA,cACA,gBACA,oBACc;AACd,WAAO,IAAI;AAAA,UACT,0BAAW;AAAA,MACX;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,IACF;AAAA,EACF;AAAA,EAEA,MAAM,wBAAwB;AAC5B,UAAM,KAAK,SAAS;AAAA,EACtB;AAAA,EAEA,WAAW,QAAoD,iBAAkC;AAC/F,QAAI,KAAK,aAAa;AACpB,YAAM,IAAI,MAAM,wBAAwB;AAAA,IAC1C;AAEA,SAAK,UAAU;AACf,SAAK,mBAAmB;AACxB,SAAK,eAAe;AACpB,SAAK,SAAS,QAAQ;AAAA,EACxB;AAAA,EAEA,oBAAoB;AAClB,SAAK,iBAAiB;AAAA,EACxB;AAAA,EAEA,sBAAsB;AACpB,SAAK,mBAAmB;AAAA,EAC1B;AAAA,EAEA,IAAI,gBAAyB;AAC3B,WAAO,KAAK;AAAA,EACd;AAAA,EAEA,IAAI,kBAA2B;AAC7B,WAAO,KAAK;AAAA,EACd;AAAA,EAEA,IAAI,KAAa;AACf,WAAO,KAAK;AAAA,EACd;AAAA,EAEA,IAAI,qBAA8B;AAChC,WAAO,KAAK;AAAA,EACd;AAAA,EAEA,IAAI,eAAwB;AAC1B,WAAO,KAAK;AAAA,EACd;AAAA,EAEA,IAAI,SAAqD;AACvD,QAAI,CAAC,KAAK,SAAS;AACjB,YAAM,IAAI,MAAM,wBAAwB;AAAA,IAC1C;AACA,WAAO,KAAK;AAAA,EACd;AAAA,EAEA,IAAI,kBAAmC;AACrC,QAAI,CAAC,KAAK,kBAAkB;AAC1B,YAAM,IAAI,MAAM,wBAAwB;AAAA,IAC1C;AACA,WAAO,KAAK;AAAA,EACd;AAAA,EAEA,IAAI,gBAAgB,QAAyB;AAC3C,SAAK,mBAAmB;AAAA,EAC1B;AAAA,EAEA,IAAI,cAAuB;AACzB,WAAO,KAAK;AAAA,EACd;AAAA,EAEA,IAAI,UAAmB;AACrB,WAAO,KAAK;AAAA,EACd;AAAA,EAEA,IAAI,eAAuB;AACzB,WAAO,KAAK;AAAA,EACd;AAAA,EAEA,IAAI,cAAuB;AAnJ7B;AAoJI,WAAO,CAAC,GAAC,UAAK,qBAAL,mBAAuB;AAAA,EAClC;AAAA,EAEA,IAAI,iBAAyB;AAC3B,WAAO,KAAK;AAAA,EACd;AAAA,EAEA,IAAI,qBAAgD;AAClD,WAAO,KAAK;AAAA,EACd;AAAA,EAEA,gBAAgB,QAAsB;AACpC,SAAK,qBAAqB,KAAK,MAAM;AACrC,SAAK,qBAAqB,IAAI;AAAA,EAChC;AAAA,EAEA,IAAI,sBAAsC;AACxC,WAAO,KAAK;AAAA,EACd;AAAA,EAEA,MAAM,sBAAsB;AAC1B,UAAM,KAAK,qBAAqB,KAAK;AAAA,EACvC;AAAA,EAEA,IAAI,uBAAgC;AAClC,WAAO,KAAK;AAAA,EACd;AAAA,EAEA,2BAA2B;AACzB,SAAK,wBAAwB;AAAA,EAC/B;AAAA,EAEA,OAAO;AACL,WAAO,KAAK,SAAS;AAAA,EACvB;AAAA,EAEA,UAAU;AACR,SAAK,SAAS,QAAQ;AAAA,EACxB;AAAA,EAEA,YAAY;AACV,QAAI,CAAC,KAAK,qBAAqB;AAC7B,YAAM,IAAI,MAAM,+BAA+B;AAAA,IACjD;AACA,SAAK,OAAO;AAAA,EACd;AAAA,EAEA,SAAS;AAnMX;AAoMI,SAAK,SAAS,OAAO,IAAI,MAAM,CAAC;AAChC,SAAK,qBAAqB,MAAM;AAChC,eAAK,qBAAL,mBAAuB;AAAA,EACzB;AACF;","names":[]}

package/dist/pipeline/speech_handle.d.ts

@@ -1,10 +1,11 @@
-import type { LLMStream } from '../llm/index.js';
+import type { ChatMessage, LLMStream } from '../llm/index.js';
 import type { SynthesisHandle } from './agent_output.js';
 export declare class SpeechHandle {
     #private;
-    constructor(id: string, allowInterruptions: boolean, addToChatCtx: boolean, isReply: boolean, userQuestion: string);
+    constructor(id: string, allowInterruptions: boolean, addToChatCtx: boolean, isReply: boolean, userQuestion: string, fncNestedDepth?: number, extraToolsMessages?: ChatMessage[] | undefined);
     static createAssistantReply(allowInterruptions: boolean, addToChatCtx: boolean, userQuestion: string): SpeechHandle;
     static createAssistantSpeech(allowInterruptions: boolean, addToChatCtx: boolean): SpeechHandle;
+    static createToolSpeech(allowInterruptions: boolean, addToChatCtx: boolean, fncNestedDepth: number, extraToolsMessages: ChatMessage[]): SpeechHandle;
     waitForInitialization(): Promise<void>;
     initialize(source: string | LLMStream | AsyncIterable<string>, synthesisHandle: SynthesisHandle): void;
     markUserCommitted(): void;
@@ -21,6 +22,15 @@ export declare class SpeechHandle {
     get isReply(): boolean;
     get userQuestion(): string;
     get interrupted(): boolean;
+    get fncNestedDepth(): number;
+    get extraToolsMessages(): ChatMessage[] | undefined;
+    addNestedSpeech(handle: SpeechHandle): void;
+    get nestedSpeechHandles(): SpeechHandle[];
+    nestedSpeechChanged(): Promise<void>;
+    get nestedSpeechFinished(): boolean;
+    markNestedSpeechFinished(): void;
+    join(): Promise<void>;
+    setDone(): void;
     interrupt(): void;
     cancel(): void;
 }

package/dist/pipeline/speech_handle.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"speech_handle.d.ts","sourceRoot":"","sources":["../../src/pipeline/speech_handle.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,iBAAiB,CAAC;AAEjD,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,mBAAmB,CAAC;AAEzD,qBAAa,YAAY;;gBAcrB,EAAE,EAAE,MAAM,EACV,kBAAkB,EAAE,OAAO,EAC3B,YAAY,EAAE,OAAO,EACrB,OAAO,EAAE,OAAO,EAChB,YAAY,EAAE,MAAM;IAStB,MAAM,CAAC,oBAAoB,CACzB,kBAAkB,EAAE,OAAO,EAC3B,YAAY,EAAE,OAAO,EACrB,YAAY,EAAE,MAAM,GACnB,YAAY;IAIf,MAAM,CAAC,qBAAqB,CAAC,kBAAkB,EAAE,OAAO,EAAE,YAAY,EAAE,OAAO,GAAG,YAAY;IAIxF,qBAAqB;IAI3B,UAAU,CAAC,MAAM,EAAE,MAAM,GAAG,SAAS,GAAG,aAAa,CAAC,MAAM,CAAC,EAAE,eAAe,EAAE,eAAe;IAW/F,iBAAiB;IAIjB,mBAAmB;IAInB,IAAI,aAAa,IAAI,OAAO,CAE3B;IAED,IAAI,eAAe,IAAI,OAAO,CAE7B;IAED,IAAI,EAAE,IAAI,MAAM,CAEf;IAED,IAAI,kBAAkB,IAAI,OAAO,CAEhC;IAED,IAAI,YAAY,IAAI,OAAO,CAE1B;IAED,IAAI,MAAM,IAAI,MAAM,GAAG,SAAS,GAAG,aAAa,CAAC,MAAM,CAAC,CAKvD;IAED,IAAI,eAAe,IAAI,eAAe,CAKrC;IAED,IAAI,eAAe,CAAC,MAAM,EAAE,eAAe,EAE1C;IAED,IAAI,WAAW,IAAI,OAAO,CAEzB;IAED,IAAI,OAAO,IAAI,OAAO,CAErB;IAED,IAAI,YAAY,IAAI,MAAM,CAEzB;IAED,IAAI,WAAW,IAAI,OAAO,CAEzB;IAED,SAAS;IAOT,MAAM;CAIP"}
+{"version":3,"file":"speech_handle.d.ts","sourceRoot":"","sources":["../../src/pipeline/speech_handle.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EAAE,WAAW,EAAE,SAAS,EAAE,MAAM,iBAAiB,CAAC;AAE9D,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,mBAAmB,CAAC;AAEzD,qBAAa,YAAY;;gBAoBrB,EAAE,EAAE,MAAM,EACV,kBAAkB,EAAE,OAAO,EAC3B,YAAY,EAAE,OAAO,EACrB,OAAO,EAAE,OAAO,EAChB,YAAY,EAAE,MAAM,EACpB,cAAc,SAAI,EAClB,kBAAkB,GAAE,WAAW,EAAE,GAAG,SAAqB;IAW3D,MAAM,CAAC,oBAAoB,CACzB,kBAAkB,EAAE,OAAO,EAC3B,YAAY,EAAE,OAAO,EACrB,YAAY,EAAE,MAAM,GACnB,YAAY;IAIf,MAAM,CAAC,qBAAqB,CAAC,kBAAkB,EAAE,OAAO,EAAE,YAAY,EAAE,OAAO,GAAG,YAAY;IAI9F,MAAM,CAAC,gBAAgB,CACrB,kBAAkB,EAAE,OAAO,EAC3B,YAAY,EAAE,OAAO,EACrB,cAAc,EAAE,MAAM,EACtB,kBAAkB,EAAE,WAAW,EAAE,GAChC,YAAY;IAYT,qBAAqB;IAI3B,UAAU,CAAC,MAAM,EAAE,MAAM,GAAG,SAAS,GAAG,aAAa,CAAC,MAAM,CAAC,EAAE,eAAe,EAAE,eAAe;IAW/F,iBAAiB;IAIjB,mBAAmB;IAInB,IAAI,aAAa,IAAI,OAAO,CAE3B;IAED,IAAI,eAAe,IAAI,OAAO,CAE7B;IAED,IAAI,EAAE,IAAI,MAAM,CAEf;IAED,IAAI,kBAAkB,IAAI,OAAO,CAEhC;IAED,IAAI,YAAY,IAAI,OAAO,CAE1B;IAED,IAAI,MAAM,IAAI,MAAM,GAAG,SAAS,GAAG,aAAa,CAAC,MAAM,CAAC,CAKvD;IAED,IAAI,eAAe,IAAI,eAAe,CAKrC;IAED,IAAI,eAAe,CAAC,MAAM,EAAE,eAAe,EAE1C;IAED,IAAI,WAAW,IAAI,OAAO,CAEzB;IAED,IAAI,OAAO,IAAI,OAAO,CAErB;IAED,IAAI,YAAY,IAAI,MAAM,CAEzB;IAED,IAAI,WAAW,IAAI,OAAO,CAEzB;IAED,IAAI,cAAc,IAAI,MAAM,CAE3B;IAED,IAAI,kBAAkB,IAAI,WAAW,EAAE,GAAG,SAAS,CAElD;IAED,eAAe,CAAC,MAAM,EAAE,YAAY;IAKpC,IAAI,mBAAmB,IAAI,YAAY,EAAE,CAExC;IAEK,mBAAmB;IAIzB,IAAI,oBAAoB,IAAI,OAAO,CAElC;IAED,wBAAwB;IAIxB,IAAI;IAIJ,OAAO;IAIP,SAAS;IAOT,MAAM;CAKP"}

package/dist/pipeline/speech_handle.js

@@ -1,5 +1,5 @@
 import { randomUUID } from "crypto";
-import { Future } from "../utils.js";
+import { AsyncIterableQueue, Future } from "../utils.js";
 class SpeechHandle {
   #id;
   #allowInterruptions;
@@ -8,16 +8,24 @@ class SpeechHandle {
   #userQuestion;
   #userCommitted = false;
   #initFut = new Future();
+  #doneFut = new Future();
   #speechCommitted = false;
   #source;
   #synthesisHandle;
   #initialized = false;
-  constructor(id, allowInterruptions, addToChatCtx, isReply, userQuestion) {
+  #fncNestedDepth;
+  #fncExtraToolsMesages;
+  #nestedSpeechHandles = [];
+  #nestedSpeechChanged = new AsyncIterableQueue();
+  #nestedSpeechFinished = false;
+  constructor(id, allowInterruptions, addToChatCtx, isReply, userQuestion, fncNestedDepth = 0, extraToolsMessages = void 0) {
     this.#id = id;
     this.#allowInterruptions = allowInterruptions;
     this.#addToChatCtx = addToChatCtx;
     this.#isReply = isReply;
     this.#userQuestion = userQuestion;
+    this.#fncNestedDepth = fncNestedDepth;
+    this.#fncExtraToolsMesages = extraToolsMessages;
   }
   static createAssistantReply(allowInterruptions, addToChatCtx, userQuestion) {
     return new SpeechHandle(randomUUID(), allowInterruptions, addToChatCtx, true, userQuestion);
@@ -25,6 +33,17 @@ class SpeechHandle {
   static createAssistantSpeech(allowInterruptions, addToChatCtx) {
     return new SpeechHandle(randomUUID(), allowInterruptions, addToChatCtx, false, "");
   }
+  static createToolSpeech(allowInterruptions, addToChatCtx, fncNestedDepth, extraToolsMessages) {
+    return new SpeechHandle(
+      randomUUID(),
+      allowInterruptions,
+      addToChatCtx,
+      false,
+      "",
+      fncNestedDepth,
+      extraToolsMessages
+    );
+  }
   async waitForInitialization() {
     await this.#initFut.await;
   }
@@ -86,6 +105,34 @@ class SpeechHandle {
     var _a;
     return !!((_a = this.#synthesisHandle) == null ? void 0 : _a.interrupted);
   }
+  get fncNestedDepth() {
+    return this.#fncNestedDepth;
+  }
+  get extraToolsMessages() {
+    return this.#fncExtraToolsMesages;
+  }
+  addNestedSpeech(handle) {
+    this.#nestedSpeechHandles.push(handle);
+    this.#nestedSpeechChanged.put();
+  }
+  get nestedSpeechHandles() {
+    return this.#nestedSpeechHandles;
+  }
+  async nestedSpeechChanged() {
+    await this.#nestedSpeechChanged.next();
+  }
+  get nestedSpeechFinished() {
+    return this.#nestedSpeechFinished;
+  }
+  markNestedSpeechFinished() {
+    this.#nestedSpeechFinished = true;
+  }
+  join() {
+    return this.#doneFut.await;
+  }
+  setDone() {
+    this.#doneFut.resolve();
+  }
   interrupt() {
     if (!this.#allowInterruptions) {
       throw new Error("interruptions are not allowed");
@@ -95,6 +142,7 @@ class SpeechHandle {
   cancel() {
     var _a;
     this.#initFut.reject(new Error());
+    this.#nestedSpeechChanged.close();
     (_a = this.#synthesisHandle) == null ? void 0 : _a.interrupt();
   }
 }

package/dist/pipeline/speech_handle.js.map

@@ -1 +1 @@
-{"version":3,"sources":["../../src/pipeline/speech_handle.ts"],"sourcesContent":["// SPDX-FileCopyrightText: 2024 LiveKit, Inc.\n//\n// SPDX-License-Identifier: Apache-2.0\nimport { randomUUID } from 'crypto';\nimport type { LLMStream } from '../llm/index.js';\nimport { Future } from '../utils.js';\nimport type { SynthesisHandle } from './agent_output.js';\n\nexport class SpeechHandle {\n  #id: string;\n  #allowInterruptions: boolean;\n  #addToChatCtx: boolean;\n  #isReply: boolean;\n  #userQuestion: string;\n  #userCommitted = false;\n  #initFut = new Future();\n  #speechCommitted = false;\n  #source?: string | LLMStream | AsyncIterable<string>;\n  #synthesisHandle?: SynthesisHandle;\n  #initialized = false;\n\n  constructor(\n    id: string,\n    allowInterruptions: boolean,\n    addToChatCtx: boolean,\n    isReply: boolean,\n    userQuestion: string,\n  ) {\n    this.#id = id;\n    this.#allowInterruptions = allowInterruptions;\n    this.#addToChatCtx = addToChatCtx;\n    this.#isReply = isReply;\n    this.#userQuestion = userQuestion;\n  }\n\n  static createAssistantReply(\n    allowInterruptions: boolean,\n    addToChatCtx: boolean,\n    userQuestion: string,\n  ): SpeechHandle {\n    return new SpeechHandle(randomUUID(), allowInterruptions, addToChatCtx, true, userQuestion);\n  }\n\n  static createAssistantSpeech(allowInterruptions: boolean, addToChatCtx: boolean): SpeechHandle {\n    return new SpeechHandle(randomUUID(), allowInterruptions, addToChatCtx, false, '');\n  }\n\n  async waitForInitialization() {\n    await this.#initFut.await;\n  }\n\n  initialize(source: string | LLMStream | AsyncIterable<string>, synthesisHandle: SynthesisHandle) {\n    if (this.interrupted) {\n      throw new Error('speech was interrupted');\n    }\n\n    this.#source = source;\n    this.#synthesisHandle = synthesisHandle;\n    this.#initialized = true;\n    this.#initFut.resolve();\n  }\n\n  markUserCommitted() {\n    this.#userCommitted = true;\n  }\n\n  markSpeechCommitted() {\n    this.#speechCommitted = true;\n  }\n\n  get userCommitted(): boolean {\n    return this.#userCommitted;\n  }\n\n  get speechCommitted(): boolean {\n    return this.#speechCommitted;\n  }\n\n  get id(): string {\n    return this.#id;\n  }\n\n  get allowInterruptions(): boolean {\n    return this.#allowInterruptions;\n  }\n\n  get addToChatCtx(): boolean {\n    return this.#addToChatCtx;\n  }\n\n  get source(): string | LLMStream | AsyncIterable<string> {\n    if (!this.#source) {\n      throw new Error('speech not initialized');\n    }\n    return this.#source;\n  }\n\n  get synthesisHandle(): SynthesisHandle {\n    if (!this.#synthesisHandle) {\n      throw new Error('speech not initialized');\n    }\n    return this.#synthesisHandle;\n  }\n\n  set synthesisHandle(handle: SynthesisHandle) {\n    this.#synthesisHandle = handle;\n  }\n\n  get initialized(): boolean {\n    return this.#initialized;\n  }\n\n  get isReply(): boolean {\n    return this.#isReply;\n  }\n\n  get userQuestion(): string {\n    return this.#userQuestion;\n  }\n\n  get interrupted(): boolean {\n    return !!this.#synthesisHandle?.interrupted;\n  }\n\n  interrupt() {\n    if (!this.#allowInterruptions) {\n      throw new Error('interruptions are not allowed');\n    }\n    this.cancel();\n  }\n\n  cancel() {\n    this.#initFut.reject(new Error());\n    this.#synthesisHandle?.interrupt();\n  }\n}\n"],"mappings":"AAGA,SAAS,kBAAkB;AAE3B,SAAS,cAAc;AAGhB,MAAM,aAAa;AAAA,EACxB;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA,iBAAiB;AAAA,EACjB,WAAW,IAAI,OAAO;AAAA,EACtB,mBAAmB;AAAA,EACnB;AAAA,EACA;AAAA,EACA,eAAe;AAAA,EAEf,YACE,IACA,oBACA,cACA,SACA,cACA;AACA,SAAK,MAAM;AACX,SAAK,sBAAsB;AAC3B,SAAK,gBAAgB;AACrB,SAAK,WAAW;AAChB,SAAK,gBAAgB;AAAA,EACvB;AAAA,EAEA,OAAO,qBACL,oBACA,cACA,cACc;AACd,WAAO,IAAI,aAAa,WAAW,GAAG,oBAAoB,cAAc,MAAM,YAAY;AAAA,EAC5F;AAAA,EAEA,OAAO,sBAAsB,oBAA6B,cAAqC;AAC7F,WAAO,IAAI,aAAa,WAAW,GAAG,oBAAoB,cAAc,OAAO,EAAE;AAAA,EACnF;AAAA,EAEA,MAAM,wBAAwB;AAC5B,UAAM,KAAK,SAAS;AAAA,EACtB;AAAA,EAEA,WAAW,QAAoD,iBAAkC;AAC/F,QAAI,KAAK,aAAa;AACpB,YAAM,IAAI,MAAM,wBAAwB;AAAA,IAC1C;AAEA,SAAK,UAAU;AACf,SAAK,mBAAmB;AACxB,SAAK,eAAe;AACpB,SAAK,SAAS,QAAQ;AAAA,EACxB;AAAA,EAEA,oBAAoB;AAClB,SAAK,iBAAiB;AAAA,EACxB;AAAA,EAEA,sBAAsB;AACpB,SAAK,mBAAmB;AAAA,EAC1B;AAAA,EAEA,IAAI,gBAAyB;AAC3B,WAAO,KAAK;AAAA,EACd;AAAA,EAEA,IAAI,kBAA2B;AAC7B,WAAO,KAAK;AAAA,EACd;AAAA,EAEA,IAAI,KAAa;AACf,WAAO,KAAK;AAAA,EACd;AAAA,EAEA,IAAI,qBAA8B;AAChC,WAAO,KAAK;AAAA,EACd;AAAA,EAEA,IAAI,eAAwB;AAC1B,WAAO,KAAK;AAAA,EACd;AAAA,EAEA,IAAI,SAAqD;AACvD,QAAI,CAAC,KAAK,SAAS;AACjB,YAAM,IAAI,MAAM,wBAAwB;AAAA,IAC1C;AACA,WAAO,KAAK;AAAA,EACd;AAAA,EAEA,IAAI,kBAAmC;AACrC,QAAI,CAAC,KAAK,kBAAkB;AAC1B,YAAM,IAAI,MAAM,wBAAwB;AAAA,IAC1C;AACA,WAAO,KAAK;AAAA,EACd;AAAA,EAEA,IAAI,gBAAgB,QAAyB;AAC3C,SAAK,mBAAmB;AAAA,EAC1B;AAAA,EAEA,IAAI,cAAuB;AACzB,WAAO,KAAK;AAAA,EACd;AAAA,EAEA,IAAI,UAAmB;AACrB,WAAO,KAAK;AAAA,EACd;AAAA,EAEA,IAAI,eAAuB;AACzB,WAAO,KAAK;AAAA,EACd;AAAA,EAEA,IAAI,cAAuB;AAxH7B;AAyHI,WAAO,CAAC,GAAC,UAAK,qBAAL,mBAAuB;AAAA,EAClC;AAAA,EAEA,YAAY;AACV,QAAI,CAAC,KAAK,qBAAqB;AAC7B,YAAM,IAAI,MAAM,+BAA+B;AAAA,IACjD;AACA,SAAK,OAAO;AAAA,EACd;AAAA,EAEA,SAAS;AAnIX;AAoII,SAAK,SAAS,OAAO,IAAI,MAAM,CAAC;AAChC,eAAK,qBAAL,mBAAuB;AAAA,EACzB;AACF;","names":[]}
+{"version":3,"sources":["../../src/pipeline/speech_handle.ts"],"sourcesContent":["// SPDX-FileCopyrightText: 2024 LiveKit, Inc.\n//\n// SPDX-License-Identifier: Apache-2.0\nimport { randomUUID } from 'crypto';\nimport type { ChatMessage, LLMStream } from '../llm/index.js';\nimport { AsyncIterableQueue, Future } from '../utils.js';\nimport type { SynthesisHandle } from './agent_output.js';\n\nexport class SpeechHandle {\n  #id: string;\n  #allowInterruptions: boolean;\n  #addToChatCtx: boolean;\n  #isReply: boolean;\n  #userQuestion: string;\n  #userCommitted = false;\n  #initFut = new Future();\n  #doneFut = new Future();\n  #speechCommitted = false;\n  #source?: string | LLMStream | AsyncIterable<string>;\n  #synthesisHandle?: SynthesisHandle;\n  #initialized = false;\n  #fncNestedDepth: number;\n  #fncExtraToolsMesages?: ChatMessage[];\n  #nestedSpeechHandles: SpeechHandle[] = [];\n  #nestedSpeechChanged = new AsyncIterableQueue<void>();\n  #nestedSpeechFinished = false;\n\n  constructor(\n    id: string,\n    allowInterruptions: boolean,\n    addToChatCtx: boolean,\n    isReply: boolean,\n    userQuestion: string,\n    fncNestedDepth = 0,\n    extraToolsMessages: ChatMessage[] | undefined = undefined,\n  ) {\n    this.#id = id;\n    this.#allowInterruptions = allowInterruptions;\n    this.#addToChatCtx = addToChatCtx;\n    this.#isReply = isReply;\n    this.#userQuestion = userQuestion;\n    this.#fncNestedDepth = fncNestedDepth;\n    this.#fncExtraToolsMesages = extraToolsMessages;\n  }\n\n  static createAssistantReply(\n    allowInterruptions: boolean,\n    addToChatCtx: boolean,\n    userQuestion: string,\n  ): SpeechHandle {\n    return new SpeechHandle(randomUUID(), allowInterruptions, addToChatCtx, true, userQuestion);\n  }\n\n  static createAssistantSpeech(allowInterruptions: boolean, addToChatCtx: boolean): SpeechHandle {\n    return new SpeechHandle(randomUUID(), allowInterruptions, addToChatCtx, false, '');\n  }\n\n  static createToolSpeech(\n    allowInterruptions: boolean,\n    addToChatCtx: boolean,\n    fncNestedDepth: number,\n    extraToolsMessages: ChatMessage[],\n  ): SpeechHandle {\n    return new SpeechHandle(\n      randomUUID(),\n      allowInterruptions,\n      addToChatCtx,\n      false,\n      '',\n      fncNestedDepth,\n      extraToolsMessages,\n    );\n  }\n\n  async waitForInitialization() {\n    await this.#initFut.await;\n  }\n\n  initialize(source: string | LLMStream | AsyncIterable<string>, synthesisHandle: SynthesisHandle) {\n    if (this.interrupted) {\n      throw new Error('speech was interrupted');\n    }\n\n    this.#source = source;\n    this.#synthesisHandle = synthesisHandle;\n    this.#initialized = true;\n    this.#initFut.resolve();\n  }\n\n  markUserCommitted() {\n    this.#userCommitted = true;\n  }\n\n  markSpeechCommitted() {\n    this.#speechCommitted = true;\n  }\n\n  get userCommitted(): boolean {\n    return this.#userCommitted;\n  }\n\n  get speechCommitted(): boolean {\n    return this.#speechCommitted;\n  }\n\n  get id(): string {\n    return this.#id;\n  }\n\n  get allowInterruptions(): boolean {\n    return this.#allowInterruptions;\n  }\n\n  get addToChatCtx(): boolean {\n    return this.#addToChatCtx;\n  }\n\n  get source(): string | LLMStream | AsyncIterable<string> {\n    if (!this.#source) {\n      throw new Error('speech not initialized');\n    }\n    return this.#source;\n  }\n\n  get synthesisHandle(): SynthesisHandle {\n    if (!this.#synthesisHandle) {\n      throw new Error('speech not initialized');\n    }\n    return this.#synthesisHandle;\n  }\n\n  set synthesisHandle(handle: SynthesisHandle) {\n    this.#synthesisHandle = handle;\n  }\n\n  get initialized(): boolean {\n    return this.#initialized;\n  }\n\n  get isReply(): boolean {\n    return this.#isReply;\n  }\n\n  get userQuestion(): string {\n    return this.#userQuestion;\n  }\n\n  get interrupted(): boolean {\n    return !!this.#synthesisHandle?.interrupted;\n  }\n\n  get fncNestedDepth(): number {\n    return this.#fncNestedDepth;\n  }\n\n  get extraToolsMessages(): ChatMessage[] | undefined {\n    return this.#fncExtraToolsMesages;\n  }\n\n  addNestedSpeech(handle: SpeechHandle) {\n    this.#nestedSpeechHandles.push(handle);\n    this.#nestedSpeechChanged.put();\n  }\n\n  get nestedSpeechHandles(): SpeechHandle[] {\n    return this.#nestedSpeechHandles;\n  }\n\n  async nestedSpeechChanged() {\n    await this.#nestedSpeechChanged.next();\n  }\n\n  get nestedSpeechFinished(): boolean {\n    return this.#nestedSpeechFinished;\n  }\n\n  markNestedSpeechFinished() {\n    this.#nestedSpeechFinished = true;\n  }\n\n  join() {\n    return this.#doneFut.await;\n  }\n\n  setDone() {\n    this.#doneFut.resolve();\n  }\n\n  interrupt() {\n    if (!this.#allowInterruptions) {\n      throw new Error('interruptions are not allowed');\n    }\n    this.cancel();\n  }\n\n  cancel() {\n    this.#initFut.reject(new Error());\n    this.#nestedSpeechChanged.close();\n    this.#synthesisHandle?.interrupt();\n  }\n}\n"],"mappings":"AAGA,SAAS,kBAAkB;AAE3B,SAAS,oBAAoB,cAAc;AAGpC,MAAM,aAAa;AAAA,EACxB;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA,iBAAiB;AAAA,EACjB,WAAW,IAAI,OAAO;AAAA,EACtB,WAAW,IAAI,OAAO;AAAA,EACtB,mBAAmB;AAAA,EACnB;AAAA,EACA;AAAA,EACA,eAAe;AAAA,EACf;AAAA,EACA;AAAA,EACA,uBAAuC,CAAC;AAAA,EACxC,uBAAuB,IAAI,mBAAyB;AAAA,EACpD,wBAAwB;AAAA,EAExB,YACE,IACA,oBACA,cACA,SACA,cACA,iBAAiB,GACjB,qBAAgD,QAChD;AACA,SAAK,MAAM;AACX,SAAK,sBAAsB;AAC3B,SAAK,gBAAgB;AACrB,SAAK,WAAW;AAChB,SAAK,gBAAgB;AACrB,SAAK,kBAAkB;AACvB,SAAK,wBAAwB;AAAA,EAC/B;AAAA,EAEA,OAAO,qBACL,oBACA,cACA,cACc;AACd,WAAO,IAAI,aAAa,WAAW,GAAG,oBAAoB,cAAc,MAAM,YAAY;AAAA,EAC5F;AAAA,EAEA,OAAO,sBAAsB,oBAA6B,cAAqC;AAC7F,WAAO,IAAI,aAAa,WAAW,GAAG,oBAAoB,cAAc,OAAO,EAAE;AAAA,EACnF;AAAA,EAEA,OAAO,iBACL,oBACA,cACA,gBACA,oBACc;AACd,WAAO,IAAI;AAAA,MACT,WAAW;AAAA,MACX;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,IACF;AAAA,EACF;AAAA,EAEA,MAAM,wBAAwB;AAC5B,UAAM,KAAK,SAAS;AAAA,EACtB;AAAA,EAEA,WAAW,QAAoD,iBAAkC;AAC/F,QAAI,KAAK,aAAa;AACpB,YAAM,IAAI,MAAM,wBAAwB;AAAA,IAC1C;AAEA,SAAK,UAAU;AACf,SAAK,mBAAmB;AACxB,SAAK,eAAe;AACpB,SAAK,SAAS,QAAQ;AAAA,EACxB;AAAA,EAEA,oBAAoB;AAClB,SAAK,iBAAiB;AAAA,EACxB;AAAA,EAEA,sBAAsB;AACpB,SAAK,mBAAmB;AAAA,EAC1B;AAAA,EAEA,IAAI,gBAAyB;AAC3B,WAAO,KAAK;AAAA,EACd;AAAA,EAEA,IAAI,kBAA2B;AAC7B,WAAO,KAAK;AAAA,EACd;AAAA,EAEA,IAAI,KAAa;AACf,WAAO,KAAK;AAAA,EACd;AAAA,EAEA,IAAI,qBAA8B;AAChC,WAAO,KAAK;AAAA,EACd;AAAA,EAEA,IAAI,eAAwB;AAC1B,WAAO,KAAK;AAAA,EACd;AAAA,EAEA,IAAI,SAAqD;AACvD,QAAI,CAAC,KAAK,SAAS;AACjB,YAAM,IAAI,MAAM,wBAAwB;AAAA,IAC1C;AACA,WAAO,KAAK;AAAA,EACd;AAAA,EAEA,IAAI,kBAAmC;AACrC,QAAI,CAAC,KAAK,kBAAkB;AAC1B,YAAM,IAAI,MAAM,wBAAwB;AAAA,IAC1C;AACA,WAAO,KAAK;AAAA,EACd;AAAA,EAEA,IAAI,gBAAgB,QAAyB;AAC3C,SAAK,mBAAmB;AAAA,EAC1B;AAAA,EAEA,IAAI,cAAuB;AACzB,WAAO,KAAK;AAAA,EACd;AAAA,EAEA,IAAI,UAAmB;AACrB,WAAO,KAAK;AAAA,EACd;AAAA,EAEA,IAAI,eAAuB;AACzB,WAAO,KAAK;AAAA,EACd;AAAA,EAEA,IAAI,cAAuB;AAnJ7B;AAoJI,WAAO,CAAC,GAAC,UAAK,qBAAL,mBAAuB;AAAA,EAClC;AAAA,EAEA,IAAI,iBAAyB;AAC3B,WAAO,KAAK;AAAA,EACd;AAAA,EAEA,IAAI,qBAAgD;AAClD,WAAO,KAAK;AAAA,EACd;AAAA,EAEA,gBAAgB,QAAsB;AACpC,SAAK,qBAAqB,KAAK,MAAM;AACrC,SAAK,qBAAqB,IAAI;AAAA,EAChC;AAAA,EAEA,IAAI,sBAAsC;AACxC,WAAO,KAAK;AAAA,EACd;AAAA,EAEA,MAAM,sBAAsB;AAC1B,UAAM,KAAK,qBAAqB,KAAK;AAAA,EACvC;AAAA,EAEA,IAAI,uBAAgC;AAClC,WAAO,KAAK;AAAA,EACd;AAAA,EAEA,2BAA2B;AACzB,SAAK,wBAAwB;AAAA,EAC/B;AAAA,EAEA,OAAO;AACL,WAAO,KAAK,SAAS;AAAA,EACvB;AAAA,EAEA,UAAU;AACR,SAAK,SAAS,QAAQ;AAAA,EACxB;AAAA,EAEA,YAAY;AACV,QAAI,CAAC,KAAK,qBAAqB;AAC7B,YAAM,IAAI,MAAM,+BAA+B;AAAA,IACjD;AACA,SAAK,OAAO;AAAA,EACd;AAAA,EAEA,SAAS;AAnMX;AAoMI,SAAK,SAAS,OAAO,IAAI,MAAM,CAAC;AAChC,SAAK,qBAAqB,MAAM;AAChC,eAAK,qBAAL,mBAAuB;AAAA,EACzB;AACF;","names":[]}

package/dist/stt/index.cjs.map

@@ -1 +1 @@
-{"version":3,"sources":["../../src/stt/index.ts"],"sourcesContent":["// SPDX-FileCopyrightText: 2024 LiveKit, Inc.\n//\n// SPDX-License-Identifier: Apache-2.0\n\nexport {\n  type SpeechEvent,\n  type SpeechData,\n  type STTCapabilities,\n  SpeechEventType,\n  STT,\n  SpeechStream,\n} from './stt.js';\nexport { StreamAdapter, StreamAdapterWrapper } from './stream_adapter.js';\n"],"mappings":";;;;;;;;;;;;;;;;;;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAIA,iBAOO;AACP,4BAAoD;","names":[]}
+{"version":3,"sources":["../../src/stt/index.ts"],"sourcesContent":["// SPDX-FileCopyrightText: 2024 LiveKit, Inc.\n//\n// SPDX-License-Identifier: Apache-2.0\n\nexport {\n  type SpeechEvent,\n  type SpeechData,\n  type STTCapabilities,\n  type RecognitionUsage,\n  type STTCallbacks,\n  SpeechEventType,\n  STT,\n  SpeechStream,\n} from './stt.js';\nexport { StreamAdapter, StreamAdapterWrapper } from './stream_adapter.js';\n"],"mappings":";;;;;;;;;;;;;;;;;;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAIA,iBASO;AACP,4BAAoD;","names":[]}

package/dist/stt/index.d.ts

@@ -1,3 +1,3 @@
-export { type SpeechEvent, type SpeechData, type STTCapabilities, SpeechEventType, STT, SpeechStream, } from './stt.js';
+export { type SpeechEvent, type SpeechData, type STTCapabilities, type RecognitionUsage, type STTCallbacks, SpeechEventType, STT, SpeechStream, } from './stt.js';
 export { StreamAdapter, StreamAdapterWrapper } from './stream_adapter.js';
 //# sourceMappingURL=index.d.ts.map

package/dist/stt/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/stt/index.ts"],"names":[],"mappings":"AAIA,OAAO,EACL,KAAK,WAAW,EAChB,KAAK,UAAU,EACf,KAAK,eAAe,EACpB,eAAe,EACf,GAAG,EACH,YAAY,GACb,MAAM,UAAU,CAAC;AAClB,OAAO,EAAE,aAAa,EAAE,oBAAoB,EAAE,MAAM,qBAAqB,CAAC"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/stt/index.ts"],"names":[],"mappings":"AAIA,OAAO,EACL,KAAK,WAAW,EAChB,KAAK,UAAU,EACf,KAAK,eAAe,EACpB,KAAK,gBAAgB,EACrB,KAAK,YAAY,EACjB,eAAe,EACf,GAAG,EACH,YAAY,GACb,MAAM,UAAU,CAAC;AAClB,OAAO,EAAE,aAAa,EAAE,oBAAoB,EAAE,MAAM,qBAAqB,CAAC"}

package/dist/stt/index.js.map

@@ -1 +1 @@
-{"version":3,"sources":["../../src/stt/index.ts"],"sourcesContent":["// SPDX-FileCopyrightText: 2024 LiveKit, Inc.\n//\n// SPDX-License-Identifier: Apache-2.0\n\nexport {\n  type SpeechEvent,\n  type SpeechData,\n  type STTCapabilities,\n  SpeechEventType,\n  STT,\n  SpeechStream,\n} from './stt.js';\nexport { StreamAdapter, StreamAdapterWrapper } from './stream_adapter.js';\n"],"mappings":"AAIA;AAAA,EAIE;AAAA,EACA;AAAA,EACA;AAAA,OACK;AACP,SAAS,eAAe,4BAA4B;","names":[]}
+{"version":3,"sources":["../../src/stt/index.ts"],"sourcesContent":["// SPDX-FileCopyrightText: 2024 LiveKit, Inc.\n//\n// SPDX-License-Identifier: Apache-2.0\n\nexport {\n  type SpeechEvent,\n  type SpeechData,\n  type STTCapabilities,\n  type RecognitionUsage,\n  type STTCallbacks,\n  SpeechEventType,\n  STT,\n  SpeechStream,\n} from './stt.js';\nexport { StreamAdapter, StreamAdapterWrapper } from './stream_adapter.js';\n"],"mappings":"AAIA;AAAA,EAME;AAAA,EACA;AAAA,EACA;AAAA,OACK;AACP,SAAS,eAAe,4BAA4B;","names":[]}

package/dist/stt/stream_adapter.cjs

@@ -27,12 +27,17 @@ var import_stt = require("./stt.cjs");
 class StreamAdapter extends import_stt.STT {
   #stt;
   #vad;
+  label;
   constructor(stt, vad) {
     super({ streaming: true, interimResults: false });
     this.#stt = stt;
     this.#vad = vad;
+    this.label = `stt.StreamAdapter<${this.#stt.label}>`;
+    this.#stt.on(import_stt.SpeechEventType.METRICS_COLLECTED, (metrics) => {
+      this.emit(import_stt.SpeechEventType.METRICS_COLLECTED, metrics);
+    });
   }
-  recognize(frame) {
+  _recognize(frame) {
     return this.#stt.recognize(frame);
   }
   stream() {
@@ -42,12 +47,17 @@ class StreamAdapter extends import_stt.STT {
 class StreamAdapterWrapper extends import_stt.SpeechStream {
   #stt;
   #vadStream;
+  label;
   constructor(stt, vad) {
-    super();
+    super(stt);
     this.#stt = stt;
     this.#vadStream = vad.stream();
+    this.label = `stt.StreamAdapterWrapper<${this.#stt.label}>`;
     this.#run();
   }
+  async monitorMetrics() {
+    return;
+  }
   async #run() {
     const forwardInput = async () => {
       for await (const input of this.input) {
@@ -63,15 +73,15 @@ class StreamAdapterWrapper extends import_stt.SpeechStream {
       for await (const ev of this.#vadStream) {
         switch (ev.type) {
           case import_vad.VADEventType.START_OF_SPEECH:
-            this.queue.put({ type: import_stt.SpeechEventType.START_OF_SPEECH });
+            this.output.put({ type: import_stt.SpeechEventType.START_OF_SPEECH });
             break;
           case import_vad.VADEventType.END_OF_SPEECH:
-            this.queue.put({ type: import_stt.SpeechEventType.END_OF_SPEECH });
+            this.output.put({ type: import_stt.SpeechEventType.END_OF_SPEECH });
             const event = await this.#stt.recognize(ev.frames);
             if (!event.alternatives[0].text) {
               continue;
             }
-            this.queue.put(event);
+            this.output.put(event);
             break;
         }
       }

package/dist/stt/stream_adapter.cjs.map

@@ -1 +1 @@
-{"version":3,"sources":["../../src/stt/stream_adapter.ts"],"sourcesContent":["// SPDX-FileCopyrightText: 2024 LiveKit, Inc.\n//\n// SPDX-License-Identifier: Apache-2.0\nimport type { AudioFrame } from '@livekit/rtc-node';\nimport type { VAD, VADStream } from '../vad.js';\nimport { VADEventType } from '../vad.js';\nimport type { SpeechEvent } from './stt.js';\nimport { STT, SpeechEventType, SpeechStream } from './stt.js';\n\nexport class StreamAdapter extends STT {\n  #stt: STT;\n  #vad: VAD;\n\n  constructor(stt: STT, vad: VAD) {\n    super({ streaming: true, interimResults: false });\n    this.#stt = stt;\n    this.#vad = vad;\n  }\n\n  recognize(frame: AudioFrame): Promise<SpeechEvent> {\n    return this.#stt.recognize(frame);\n  }\n\n  stream(): StreamAdapterWrapper {\n    return new StreamAdapterWrapper(this.#stt, this.#vad);\n  }\n}\n\nexport class StreamAdapterWrapper extends SpeechStream {\n  #stt: STT;\n  #vadStream: VADStream;\n\n  constructor(stt: STT, vad: VAD) {\n    super();\n    this.#stt = stt;\n    this.#vadStream = vad.stream();\n\n    this.#run();\n  }\n\n  async #run() {\n    const forwardInput = async () => {\n      for await (const input of this.input) {\n        if (input === SpeechStream.FLUSH_SENTINEL) {\n          this.#vadStream.flush();\n        } else {\n          this.#vadStream.pushFrame(input);\n        }\n      }\n      this.#vadStream.endInput();\n    };\n\n    const recognize = async () => {\n      for await (const ev of this.#vadStream) {\n        switch (ev.type) {\n          case VADEventType.START_OF_SPEECH:\n            this.queue.put({ type: SpeechEventType.START_OF_SPEECH });\n            break;\n          case VADEventType.END_OF_SPEECH:\n            this.queue.put({ type: SpeechEventType.END_OF_SPEECH });\n\n            const event = await this.#stt.recognize(ev.frames);\n            if (!event.alternatives![0].text) {\n              continue;\n            }\n\n            this.queue.put(event);\n            break;\n        }\n      }\n    };\n\n    Promise.all([forwardInput(), recognize()]);\n  }\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAKA,iBAA6B;AAE7B,iBAAmD;AAE5C,MAAM,sBAAsB,eAAI;AAAA,EACrC;AAAA,EACA;AAAA,EAEA,YAAY,KAAU,KAAU;AAC9B,UAAM,EAAE,WAAW,MAAM,gBAAgB,MAAM,CAAC;AAChD,SAAK,OAAO;AACZ,SAAK,OAAO;AAAA,EACd;AAAA,EAEA,UAAU,OAAyC;AACjD,WAAO,KAAK,KAAK,UAAU,KAAK;AAAA,EAClC;AAAA,EAEA,SAA+B;AAC7B,WAAO,IAAI,qBAAqB,KAAK,MAAM,KAAK,IAAI;AAAA,EACtD;AACF;AAEO,MAAM,6BAA6B,wBAAa;AAAA,EACrD;AAAA,EACA;AAAA,EAEA,YAAY,KAAU,KAAU;AAC9B,UAAM;AACN,SAAK,OAAO;AACZ,SAAK,aAAa,IAAI,OAAO;AAE7B,SAAK,KAAK;AAAA,EACZ;AAAA,EAEA,MAAM,OAAO;AACX,UAAM,eAAe,YAAY;AAC/B,uBAAiB,SAAS,KAAK,OAAO;AACpC,YAAI,UAAU,wBAAa,gBAAgB;AACzC,eAAK,WAAW,MAAM;AAAA,QACxB,OAAO;AACL,eAAK,WAAW,UAAU,KAAK;AAAA,QACjC;AAAA,MACF;AACA,WAAK,WAAW,SAAS;AAAA,IAC3B;AAEA,UAAM,YAAY,YAAY;AAC5B,uBAAiB,MAAM,KAAK,YAAY;AACtC,gBAAQ,GAAG,MAAM;AAAA,UACf,KAAK,wBAAa;AAChB,iBAAK,MAAM,IAAI,EAAE,MAAM,2BAAgB,gBAAgB,CAAC;AACxD;AAAA,UACF,KAAK,wBAAa;AAChB,iBAAK,MAAM,IAAI,EAAE,MAAM,2BAAgB,cAAc,CAAC;AAEtD,kBAAM,QAAQ,MAAM,KAAK,KAAK,UAAU,GAAG,MAAM;AACjD,gBAAI,CAAC,MAAM,aAAc,CAAC,EAAE,MAAM;AAChC;AAAA,YACF;AAEA,iBAAK,MAAM,IAAI,KAAK;AACpB;AAAA,QACJ;AAAA,MACF;AAAA,IACF;AAEA,YAAQ,IAAI,CAAC,aAAa,GAAG,UAAU,CAAC,CAAC;AAAA,EAC3C;AACF;","names":[]}
+{"version":3,"sources":["../../src/stt/stream_adapter.ts"],"sourcesContent":["// SPDX-FileCopyrightText: 2024 LiveKit, Inc.\n//\n// SPDX-License-Identifier: Apache-2.0\nimport type { AudioFrame } from '@livekit/rtc-node';\nimport type { VAD, VADStream } from '../vad.js';\nimport { VADEventType } from '../vad.js';\nimport type { SpeechEvent } from './stt.js';\nimport { STT, SpeechEventType, SpeechStream } from './stt.js';\n\nexport class StreamAdapter extends STT {\n  #stt: STT;\n  #vad: VAD;\n  label: string;\n\n  constructor(stt: STT, vad: VAD) {\n    super({ streaming: true, interimResults: false });\n    this.#stt = stt;\n    this.#vad = vad;\n    this.label = `stt.StreamAdapter<${this.#stt.label}>`;\n\n    this.#stt.on(SpeechEventType.METRICS_COLLECTED, (metrics) => {\n      this.emit(SpeechEventType.METRICS_COLLECTED, metrics);\n    });\n  }\n\n  _recognize(frame: AudioFrame): Promise<SpeechEvent> {\n    return this.#stt.recognize(frame);\n  }\n\n  stream(): StreamAdapterWrapper {\n    return new StreamAdapterWrapper(this.#stt, this.#vad);\n  }\n}\n\nexport class StreamAdapterWrapper extends SpeechStream {\n  #stt: STT;\n  #vadStream: VADStream;\n  label: string;\n\n  constructor(stt: STT, vad: VAD) {\n    super(stt);\n    this.#stt = stt;\n    this.#vadStream = vad.stream();\n    this.label = `stt.StreamAdapterWrapper<${this.#stt.label}>`;\n\n    this.#run();\n  }\n\n  async monitorMetrics() {\n    return; // do nothing\n  }\n\n  async #run() {\n    const forwardInput = async () => {\n      for await (const input of this.input) {\n        if (input === SpeechStream.FLUSH_SENTINEL) {\n          this.#vadStream.flush();\n        } else {\n          this.#vadStream.pushFrame(input);\n        }\n      }\n      this.#vadStream.endInput();\n    };\n\n    const recognize = async () => {\n      for await (const ev of this.#vadStream) {\n        switch (ev.type) {\n          case VADEventType.START_OF_SPEECH:\n            this.output.put({ type: SpeechEventType.START_OF_SPEECH });\n            break;\n          case VADEventType.END_OF_SPEECH:\n            this.output.put({ type: SpeechEventType.END_OF_SPEECH });\n\n            const event = await this.#stt.recognize(ev.frames);\n            if (!event.alternatives![0].text) {\n              continue;\n            }\n\n            this.output.put(event);\n            break;\n        }\n      }\n    };\n\n    Promise.all([forwardInput(), recognize()]);\n  }\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAKA,iBAA6B;AAE7B,iBAAmD;AAE5C,MAAM,sBAAsB,eAAI;AAAA,EACrC;AAAA,EACA;AAAA,EACA;AAAA,EAEA,YAAY,KAAU,KAAU;AAC9B,UAAM,EAAE,WAAW,MAAM,gBAAgB,MAAM,CAAC;AAChD,SAAK,OAAO;AACZ,SAAK,OAAO;AACZ,SAAK,QAAQ,qBAAqB,KAAK,KAAK,KAAK;AAEjD,SAAK,KAAK,GAAG,2BAAgB,mBAAmB,CAAC,YAAY;AAC3D,WAAK,KAAK,2BAAgB,mBAAmB,OAAO;AAAA,IACtD,CAAC;AAAA,EACH;AAAA,EAEA,WAAW,OAAyC;AAClD,WAAO,KAAK,KAAK,UAAU,KAAK;AAAA,EAClC;AAAA,EAEA,SAA+B;AAC7B,WAAO,IAAI,qBAAqB,KAAK,MAAM,KAAK,IAAI;AAAA,EACtD;AACF;AAEO,MAAM,6BAA6B,wBAAa;AAAA,EACrD;AAAA,EACA;AAAA,EACA;AAAA,EAEA,YAAY,KAAU,KAAU;AAC9B,UAAM,GAAG;AACT,SAAK,OAAO;AACZ,SAAK,aAAa,IAAI,OAAO;AAC7B,SAAK,QAAQ,4BAA4B,KAAK,KAAK,KAAK;AAExD,SAAK,KAAK;AAAA,EACZ;AAAA,EAEA,MAAM,iBAAiB;AACrB;AAAA,EACF;AAAA,EAEA,MAAM,OAAO;AACX,UAAM,eAAe,YAAY;AAC/B,uBAAiB,SAAS,KAAK,OAAO;AACpC,YAAI,UAAU,wBAAa,gBAAgB;AACzC,eAAK,WAAW,MAAM;AAAA,QACxB,OAAO;AACL,eAAK,WAAW,UAAU,KAAK;AAAA,QACjC;AAAA,MACF;AACA,WAAK,WAAW,SAAS;AAAA,IAC3B;AAEA,UAAM,YAAY,YAAY;AAC5B,uBAAiB,MAAM,KAAK,YAAY;AACtC,gBAAQ,GAAG,MAAM;AAAA,UACf,KAAK,wBAAa;AAChB,iBAAK,OAAO,IAAI,EAAE,MAAM,2BAAgB,gBAAgB,CAAC;AACzD;AAAA,UACF,KAAK,wBAAa;AAChB,iBAAK,OAAO,IAAI,EAAE,MAAM,2BAAgB,cAAc,CAAC;AAEvD,kBAAM,QAAQ,MAAM,KAAK,KAAK,UAAU,GAAG,MAAM;AACjD,gBAAI,CAAC,MAAM,aAAc,CAAC,EAAE,MAAM;AAChC;AAAA,YACF;AAEA,iBAAK,OAAO,IAAI,KAAK;AACrB;AAAA,QACJ;AAAA,MACF;AAAA,IACF;AAEA,YAAQ,IAAI,CAAC,aAAa,GAAG,UAAU,CAAC,CAAC;AAAA,EAC3C;AACF;","names":[]}

package/dist/stt/stream_adapter.d.ts

@@ -4,12 +4,15 @@ import type { SpeechEvent } from './stt.js';
 import { STT, SpeechStream } from './stt.js';
 export declare class StreamAdapter extends STT {
     #private;
+    label: string;
     constructor(stt: STT, vad: VAD);
-    recognize(frame: AudioFrame): Promise<SpeechEvent>;
+    _recognize(frame: AudioFrame): Promise<SpeechEvent>;
     stream(): StreamAdapterWrapper;
 }
 export declare class StreamAdapterWrapper extends SpeechStream {
     #private;
+    label: string;
     constructor(stt: STT, vad: VAD);
+    monitorMetrics(): Promise<void>;
 }
 //# sourceMappingURL=stream_adapter.d.ts.map

package/dist/stt/stream_adapter.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"stream_adapter.d.ts","sourceRoot":"","sources":["../../src/stt/stream_adapter.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,mBAAmB,CAAC;AACpD,OAAO,KAAK,EAAE,GAAG,EAAa,MAAM,WAAW,CAAC;AAEhD,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,UAAU,CAAC;AAC5C,OAAO,EAAE,GAAG,EAAmB,YAAY,EAAE,MAAM,UAAU,CAAC;AAE9D,qBAAa,aAAc,SAAQ,GAAG;;gBAIxB,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG;IAM9B,SAAS,CAAC,KAAK,EAAE,UAAU,GAAG,OAAO,CAAC,WAAW,CAAC;IAIlD,MAAM,IAAI,oBAAoB;CAG/B;AAED,qBAAa,oBAAqB,SAAQ,YAAY;;gBAIxC,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG;CA0C/B"}
+{"version":3,"file":"stream_adapter.d.ts","sourceRoot":"","sources":["../../src/stt/stream_adapter.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,mBAAmB,CAAC;AACpD,OAAO,KAAK,EAAE,GAAG,EAAa,MAAM,WAAW,CAAC;AAEhD,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,UAAU,CAAC;AAC5C,OAAO,EAAE,GAAG,EAAmB,YAAY,EAAE,MAAM,UAAU,CAAC;AAE9D,qBAAa,aAAc,SAAQ,GAAG;;IAGpC,KAAK,EAAE,MAAM,CAAC;gBAEF,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG;IAW9B,UAAU,CAAC,KAAK,EAAE,UAAU,GAAG,OAAO,CAAC,WAAW,CAAC;IAInD,MAAM,IAAI,oBAAoB;CAG/B;AAED,qBAAa,oBAAqB,SAAQ,YAAY;;IAGpD,KAAK,EAAE,MAAM,CAAC;gBAEF,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG;IASxB,cAAc;CAsCrB"}

package/dist/stt/stream_adapter.js

@@ -3,12 +3,17 @@ import { STT, SpeechEventType, SpeechStream } from "./stt.js";
 class StreamAdapter extends STT {
   #stt;
   #vad;
+  label;
   constructor(stt, vad) {
     super({ streaming: true, interimResults: false });
     this.#stt = stt;
     this.#vad = vad;
+    this.label = `stt.StreamAdapter<${this.#stt.label}>`;
+    this.#stt.on(SpeechEventType.METRICS_COLLECTED, (metrics) => {
+      this.emit(SpeechEventType.METRICS_COLLECTED, metrics);
+    });
   }
-  recognize(frame) {
+  _recognize(frame) {
     return this.#stt.recognize(frame);
   }
   stream() {
@@ -18,12 +23,17 @@ class StreamAdapter extends STT {
 class StreamAdapterWrapper extends SpeechStream {
   #stt;
   #vadStream;
+  label;
   constructor(stt, vad) {
-    super();
+    super(stt);
     this.#stt = stt;
     this.#vadStream = vad.stream();
+    this.label = `stt.StreamAdapterWrapper<${this.#stt.label}>`;
     this.#run();
   }
+  async monitorMetrics() {
+    return;
+  }
   async #run() {
     const forwardInput = async () => {
       for await (const input of this.input) {
@@ -39,15 +49,15 @@ class StreamAdapterWrapper extends SpeechStream {
       for await (const ev of this.#vadStream) {
         switch (ev.type) {
           case VADEventType.START_OF_SPEECH:
-            this.queue.put({ type: SpeechEventType.START_OF_SPEECH });
+            this.output.put({ type: SpeechEventType.START_OF_SPEECH });
             break;
           case VADEventType.END_OF_SPEECH:
-            this.queue.put({ type: SpeechEventType.END_OF_SPEECH });
+            this.output.put({ type: SpeechEventType.END_OF_SPEECH });
             const event = await this.#stt.recognize(ev.frames);
             if (!event.alternatives[0].text) {
               continue;
             }
-            this.queue.put(event);
+            this.output.put(event);
             break;
         }
       }

package/dist/stt/stream_adapter.js.map

@@ -1 +1 @@
-{"version":3,"sources":["../../src/stt/stream_adapter.ts"],"sourcesContent":["// SPDX-FileCopyrightText: 2024 LiveKit, Inc.\n//\n// SPDX-License-Identifier: Apache-2.0\nimport type { AudioFrame } from '@livekit/rtc-node';\nimport type { VAD, VADStream } from '../vad.js';\nimport { VADEventType } from '../vad.js';\nimport type { SpeechEvent } from './stt.js';\nimport { STT, SpeechEventType, SpeechStream } from './stt.js';\n\nexport class StreamAdapter extends STT {\n  #stt: STT;\n  #vad: VAD;\n\n  constructor(stt: STT, vad: VAD) {\n    super({ streaming: true, interimResults: false });\n    this.#stt = stt;\n    this.#vad = vad;\n  }\n\n  recognize(frame: AudioFrame): Promise<SpeechEvent> {\n    return this.#stt.recognize(frame);\n  }\n\n  stream(): StreamAdapterWrapper {\n    return new StreamAdapterWrapper(this.#stt, this.#vad);\n  }\n}\n\nexport class StreamAdapterWrapper extends SpeechStream {\n  #stt: STT;\n  #vadStream: VADStream;\n\n  constructor(stt: STT, vad: VAD) {\n    super();\n    this.#stt = stt;\n    this.#vadStream = vad.stream();\n\n    this.#run();\n  }\n\n  async #run() {\n    const forwardInput = async () => {\n      for await (const input of this.input) {\n        if (input === SpeechStream.FLUSH_SENTINEL) {\n          this.#vadStream.flush();\n        } else {\n          this.#vadStream.pushFrame(input);\n        }\n      }\n      this.#vadStream.endInput();\n    };\n\n    const recognize = async () => {\n      for await (const ev of this.#vadStream) {\n        switch (ev.type) {\n          case VADEventType.START_OF_SPEECH:\n            this.queue.put({ type: SpeechEventType.START_OF_SPEECH });\n            break;\n          case VADEventType.END_OF_SPEECH:\n            this.queue.put({ type: SpeechEventType.END_OF_SPEECH });\n\n            const event = await this.#stt.recognize(ev.frames);\n            if (!event.alternatives![0].text) {\n              continue;\n            }\n\n            this.queue.put(event);\n            break;\n        }\n      }\n    };\n\n    Promise.all([forwardInput(), recognize()]);\n  }\n}\n"],"mappings":"AAKA,SAAS,oBAAoB;AAE7B,SAAS,KAAK,iBAAiB,oBAAoB;AAE5C,MAAM,sBAAsB,IAAI;AAAA,EACrC;AAAA,EACA;AAAA,EAEA,YAAY,KAAU,KAAU;AAC9B,UAAM,EAAE,WAAW,MAAM,gBAAgB,MAAM,CAAC;AAChD,SAAK,OAAO;AACZ,SAAK,OAAO;AAAA,EACd;AAAA,EAEA,UAAU,OAAyC;AACjD,WAAO,KAAK,KAAK,UAAU,KAAK;AAAA,EAClC;AAAA,EAEA,SAA+B;AAC7B,WAAO,IAAI,qBAAqB,KAAK,MAAM,KAAK,IAAI;AAAA,EACtD;AACF;AAEO,MAAM,6BAA6B,aAAa;AAAA,EACrD;AAAA,EACA;AAAA,EAEA,YAAY,KAAU,KAAU;AAC9B,UAAM;AACN,SAAK,OAAO;AACZ,SAAK,aAAa,IAAI,OAAO;AAE7B,SAAK,KAAK;AAAA,EACZ;AAAA,EAEA,MAAM,OAAO;AACX,UAAM,eAAe,YAAY;AAC/B,uBAAiB,SAAS,KAAK,OAAO;AACpC,YAAI,UAAU,aAAa,gBAAgB;AACzC,eAAK,WAAW,MAAM;AAAA,QACxB,OAAO;AACL,eAAK,WAAW,UAAU,KAAK;AAAA,QACjC;AAAA,MACF;AACA,WAAK,WAAW,SAAS;AAAA,IAC3B;AAEA,UAAM,YAAY,YAAY;AAC5B,uBAAiB,MAAM,KAAK,YAAY;AACtC,gBAAQ,GAAG,MAAM;AAAA,UACf,KAAK,aAAa;AAChB,iBAAK,MAAM,IAAI,EAAE,MAAM,gBAAgB,gBAAgB,CAAC;AACxD;AAAA,UACF,KAAK,aAAa;AAChB,iBAAK,MAAM,IAAI,EAAE,MAAM,gBAAgB,cAAc,CAAC;AAEtD,kBAAM,QAAQ,MAAM,KAAK,KAAK,UAAU,GAAG,MAAM;AACjD,gBAAI,CAAC,MAAM,aAAc,CAAC,EAAE,MAAM;AAChC;AAAA,YACF;AAEA,iBAAK,MAAM,IAAI,KAAK;AACpB;AAAA,QACJ;AAAA,MACF;AAAA,IACF;AAEA,YAAQ,IAAI,CAAC,aAAa,GAAG,UAAU,CAAC,CAAC;AAAA,EAC3C;AACF;","names":[]}
+{"version":3,"sources":["../../src/stt/stream_adapter.ts"],"sourcesContent":["// SPDX-FileCopyrightText: 2024 LiveKit, Inc.\n//\n// SPDX-License-Identifier: Apache-2.0\nimport type { AudioFrame } from '@livekit/rtc-node';\nimport type { VAD, VADStream } from '../vad.js';\nimport { VADEventType } from '../vad.js';\nimport type { SpeechEvent } from './stt.js';\nimport { STT, SpeechEventType, SpeechStream } from './stt.js';\n\nexport class StreamAdapter extends STT {\n  #stt: STT;\n  #vad: VAD;\n  label: string;\n\n  constructor(stt: STT, vad: VAD) {\n    super({ streaming: true, interimResults: false });\n    this.#stt = stt;\n    this.#vad = vad;\n    this.label = `stt.StreamAdapter<${this.#stt.label}>`;\n\n    this.#stt.on(SpeechEventType.METRICS_COLLECTED, (metrics) => {\n      this.emit(SpeechEventType.METRICS_COLLECTED, metrics);\n    });\n  }\n\n  _recognize(frame: AudioFrame): Promise<SpeechEvent> {\n    return this.#stt.recognize(frame);\n  }\n\n  stream(): StreamAdapterWrapper {\n    return new StreamAdapterWrapper(this.#stt, this.#vad);\n  }\n}\n\nexport class StreamAdapterWrapper extends SpeechStream {\n  #stt: STT;\n  #vadStream: VADStream;\n  label: string;\n\n  constructor(stt: STT, vad: VAD) {\n    super(stt);\n    this.#stt = stt;\n    this.#vadStream = vad.stream();\n    this.label = `stt.StreamAdapterWrapper<${this.#stt.label}>`;\n\n    this.#run();\n  }\n\n  async monitorMetrics() {\n    return; // do nothing\n  }\n\n  async #run() {\n    const forwardInput = async () => {\n      for await (const input of this.input) {\n        if (input === SpeechStream.FLUSH_SENTINEL) {\n          this.#vadStream.flush();\n        } else {\n          this.#vadStream.pushFrame(input);\n        }\n      }\n      this.#vadStream.endInput();\n    };\n\n    const recognize = async () => {\n      for await (const ev of this.#vadStream) {\n        switch (ev.type) {\n          case VADEventType.START_OF_SPEECH:\n            this.output.put({ type: SpeechEventType.START_OF_SPEECH });\n            break;\n          case VADEventType.END_OF_SPEECH:\n            this.output.put({ type: SpeechEventType.END_OF_SPEECH });\n\n            const event = await this.#stt.recognize(ev.frames);\n            if (!event.alternatives![0].text) {\n              continue;\n            }\n\n            this.output.put(event);\n            break;\n        }\n      }\n    };\n\n    Promise.all([forwardInput(), recognize()]);\n  }\n}\n"],"mappings":"AAKA,SAAS,oBAAoB;AAE7B,SAAS,KAAK,iBAAiB,oBAAoB;AAE5C,MAAM,sBAAsB,IAAI;AAAA,EACrC;AAAA,EACA;AAAA,EACA;AAAA,EAEA,YAAY,KAAU,KAAU;AAC9B,UAAM,EAAE,WAAW,MAAM,gBAAgB,MAAM,CAAC;AAChD,SAAK,OAAO;AACZ,SAAK,OAAO;AACZ,SAAK,QAAQ,qBAAqB,KAAK,KAAK,KAAK;AAEjD,SAAK,KAAK,GAAG,gBAAgB,mBAAmB,CAAC,YAAY;AAC3D,WAAK,KAAK,gBAAgB,mBAAmB,OAAO;AAAA,IACtD,CAAC;AAAA,EACH;AAAA,EAEA,WAAW,OAAyC;AAClD,WAAO,KAAK,KAAK,UAAU,KAAK;AAAA,EAClC;AAAA,EAEA,SAA+B;AAC7B,WAAO,IAAI,qBAAqB,KAAK,MAAM,KAAK,IAAI;AAAA,EACtD;AACF;AAEO,MAAM,6BAA6B,aAAa;AAAA,EACrD;AAAA,EACA;AAAA,EACA;AAAA,EAEA,YAAY,KAAU,KAAU;AAC9B,UAAM,GAAG;AACT,SAAK,OAAO;AACZ,SAAK,aAAa,IAAI,OAAO;AAC7B,SAAK,QAAQ,4BAA4B,KAAK,KAAK,KAAK;AAExD,SAAK,KAAK;AAAA,EACZ;AAAA,EAEA,MAAM,iBAAiB;AACrB;AAAA,EACF;AAAA,EAEA,MAAM,OAAO;AACX,UAAM,eAAe,YAAY;AAC/B,uBAAiB,SAAS,KAAK,OAAO;AACpC,YAAI,UAAU,aAAa,gBAAgB;AACzC,eAAK,WAAW,MAAM;AAAA,QACxB,OAAO;AACL,eAAK,WAAW,UAAU,KAAK;AAAA,QACjC;AAAA,MACF;AACA,WAAK,WAAW,SAAS;AAAA,IAC3B;AAEA,UAAM,YAAY,YAAY;AAC5B,uBAAiB,MAAM,KAAK,YAAY;AACtC,gBAAQ,GAAG,MAAM;AAAA,UACf,KAAK,aAAa;AAChB,iBAAK,OAAO,IAAI,EAAE,MAAM,gBAAgB,gBAAgB,CAAC;AACzD;AAAA,UACF,KAAK,aAAa;AAChB,iBAAK,OAAO,IAAI,EAAE,MAAM,gBAAgB,cAAc,CAAC;AAEvD,kBAAM,QAAQ,MAAM,KAAK,KAAK,UAAU,GAAG,MAAM;AACjD,gBAAI,CAAC,MAAM,aAAc,CAAC,EAAE,MAAM;AAChC;AAAA,YACF;AAEA,iBAAK,OAAO,IAAI,KAAK;AACrB;AAAA,QACJ;AAAA,MACF;AAAA,IACF;AAEA,YAAQ,IAAI,CAAC,aAAa,GAAG,UAAU,CAAC,CAAC;AAAA,EAC3C;AACF;","names":[]}

package/dist/stt/stt.cjs

@@ -23,29 +23,72 @@ __export(stt_exports, {
   SpeechStream: () => SpeechStream
 });
 module.exports = __toCommonJS(stt_exports);
+var import_node_events = require("node:events");
 var import_utils = require("../utils.cjs");
 var SpeechEventType = /* @__PURE__ */ ((SpeechEventType2) => {
   SpeechEventType2[SpeechEventType2["START_OF_SPEECH"] = 0] = "START_OF_SPEECH";
   SpeechEventType2[SpeechEventType2["INTERIM_TRANSCRIPT"] = 1] = "INTERIM_TRANSCRIPT";
   SpeechEventType2[SpeechEventType2["FINAL_TRANSCRIPT"] = 2] = "FINAL_TRANSCRIPT";
   SpeechEventType2[SpeechEventType2["END_OF_SPEECH"] = 3] = "END_OF_SPEECH";
+  SpeechEventType2[SpeechEventType2["RECOGNITION_USAGE"] = 4] = "RECOGNITION_USAGE";
+  SpeechEventType2[SpeechEventType2["METRICS_COLLECTED"] = 5] = "METRICS_COLLECTED";
   return SpeechEventType2;
 })(SpeechEventType || {});
-class STT {
+class STT extends import_node_events.EventEmitter {
   #capabilities;
   constructor(capabilities) {
+    super();
     this.#capabilities = capabilities;
   }
   /** Returns this STT's capabilities */
   get capabilities() {
     return this.#capabilities;
   }
+  /** Receives an audio buffer and returns transcription in the form of a {@link SpeechEvent} */
+  async recognize(frame) {
+    const startTime = process.hrtime.bigint();
+    const event = await this._recognize(frame);
+    const duration = Number((process.hrtime.bigint() - startTime) / BigInt(1e6));
+    this.emit(5 /* METRICS_COLLECTED */, {
+      requestId: event.requestId ?? "",
+      timestamp: Date.now(),
+      duration,
+      label: this.label,
+      audioDuration: Array.isArray(frame) ? frame.reduce((sum, a) => sum + a.samplesPerChannel / a.sampleRate, 0) : frame.samplesPerChannel / frame.sampleRate,
+      streamed: false
+    });
+    return event;
+  }
 }
 class SpeechStream {
   static FLUSH_SENTINEL = Symbol("FLUSH_SENTINEL");
   input = new import_utils.AsyncIterableQueue();
+  output = new import_utils.AsyncIterableQueue();
   queue = new import_utils.AsyncIterableQueue();
   closed = false;
+  #stt;
+  constructor(stt) {
+    this.#stt = stt;
+    this.monitorMetrics();
+  }
+  async monitorMetrics() {
+    const startTime = process.hrtime.bigint();
+    for await (const event of this.queue) {
+      this.output.put(event);
+      if (event.type !== 4 /* RECOGNITION_USAGE */) continue;
+      const duration = process.hrtime.bigint() - startTime;
+      const metrics = {
+        timestamp: Date.now(),
+        requestId: event.requestId,
+        duration: Math.trunc(Number(duration / BigInt(1e6))),
+        label: this.label,
+        audioDuration: event.recognitionUsage.audioDuration,
+        streamed: true
+      };
+      this.#stt.emit(5 /* METRICS_COLLECTED */, metrics);
+    }
+    this.output.close();
+  }
   /** Push an audio frame to the STT */
   pushFrame(frame) {
     if (this.input.closed) {
@@ -77,12 +120,13 @@ class SpeechStream {
     this.input.close();
   }
   next() {
-    return this.queue.next();
+    return this.output.next();
   }
   /** Close both the input and output of the STT stream */
   close() {
     this.input.close();
     this.queue.close();
+    this.output.close();
     this.closed = true;
   }
   [Symbol.asyncIterator]() {

package/dist/stt/stt.cjs.map

@@ -1 +1 @@
-{"version":3,"sources":["../../src/stt/stt.ts"],"sourcesContent":["// SPDX-FileCopyrightText: 2024 LiveKit, Inc.\n//\n// SPDX-License-Identifier: Apache-2.0\nimport type { AudioFrame } from '@livekit/rtc-node';\nimport type { AudioBuffer } from '../utils.js';\nimport { AsyncIterableQueue } from '../utils.js';\n\n/** Indicates start/middle/end of speech */\nexport enum SpeechEventType {\n  /**\n   * Indicate the start of speech.\n   * If the STT doesn't support this event, this will be emitted at the same time\n   * as the first INTERIM_TRANSCRIPT.\n   */\n  START_OF_SPEECH = 0,\n  /**\n   * Interim transcript, useful for real-time transcription.\n   */\n  INTERIM_TRANSCRIPT = 1,\n  /**\n   * Final transcript, emitted when the STT is confident enough that a certain\n   * portion of the speech will not change.\n   */\n  FINAL_TRANSCRIPT = 2,\n  /**\n   * Indicate the end of speech, emitted when the user stops speaking.\n   * The first alternative is a combination of all the previous FINAL_TRANSCRIPT events.\n   */\n  END_OF_SPEECH = 3,\n}\n\n/** SpeechData contains metadata about this {@link SpeechEvent}. */\nexport interface SpeechData {\n  language: string;\n  text: string;\n  startTime: number;\n  endTime: number;\n  confidence: number;\n}\n\n/** SpeechEvent is a packet of speech-to-text data. */\nexport interface SpeechEvent {\n  type: SpeechEventType;\n  alternatives?: [SpeechData, ...SpeechData[]];\n}\n\n/**\n * Describes the capabilities of the STT provider.\n *\n * @remarks\n * At present, the framework only supports providers that have a streaming endpoint.\n */\nexport interface STTCapabilities {\n  streaming: boolean;\n  interimResults: boolean;\n}\n\n/**\n * An instance of a speech-to-text adapter.\n *\n * @remarks\n * This class is abstract, and as such cannot be used directly. Instead, use a provider plugin that\n * exports its own child STT class, which inherits this class's methods.\n */\nexport abstract class STT {\n  #capabilities: STTCapabilities;\n\n  constructor(capabilities: STTCapabilities) {\n    this.#capabilities = capabilities;\n  }\n\n  /** Returns this STT's capabilities */\n  get capabilities(): STTCapabilities {\n    return this.#capabilities;\n  }\n\n  /** Receives an audio buffer and returns transcription in the form of a {@link SpeechEvent} */\n  abstract recognize(frame: AudioBuffer): Promise<SpeechEvent>;\n\n  /**\n   * Returns a {@link SpeechStream} that can be used to push audio frames and receive\n   * transcriptions\n   */\n  abstract stream(): SpeechStream;\n}\n\n/**\n * An instance of a speech-to-text stream, as an asynchronous iterable iterator.\n *\n * @example Looping through frames\n * ```ts\n * for await (const event of stream) {\n *   if (event.type === SpeechEventType.FINAL_TRANSCRIPT) {\n *     console.log(event.alternatives[0].text)\n *   }\n * }\n * ```\n *\n * @remarks\n * This class is abstract, and as such cannot be used directly. Instead, use a provider plugin that\n * exports its own child SpeechStream class, which inherits this class's methods.\n */\nexport abstract class SpeechStream implements AsyncIterableIterator<SpeechEvent> {\n  protected static readonly FLUSH_SENTINEL = Symbol('FLUSH_SENTINEL');\n  protected input = new AsyncIterableQueue<AudioFrame | typeof SpeechStream.FLUSH_SENTINEL>();\n  protected queue = new AsyncIterableQueue<SpeechEvent>();\n  protected closed = false;\n\n  /** Push an audio frame to the STT */\n  pushFrame(frame: AudioFrame) {\n    if (this.input.closed) {\n      throw new Error('Input is closed');\n    }\n    if (this.closed) {\n      throw new Error('Stream is closed');\n    }\n    this.input.put(frame);\n  }\n\n  /** Flush the STT, causing it to process all pending text */\n  flush() {\n    if (this.input.closed) {\n      throw new Error('Input is closed');\n    }\n    if (this.closed) {\n      throw new Error('Stream is closed');\n    }\n    this.input.put(SpeechStream.FLUSH_SENTINEL);\n  }\n\n  /** Mark the input as ended and forbid additional pushes */\n  endInput() {\n    if (this.input.closed) {\n      throw new Error('Input is closed');\n    }\n    if (this.closed) {\n      throw new Error('Stream is closed');\n    }\n    this.input.close();\n  }\n\n  next(): Promise<IteratorResult<SpeechEvent>> {\n    return this.queue.next();\n  }\n\n  /** Close both the input and output of the STT stream */\n  close() {\n    this.input.close();\n    this.queue.close();\n    this.closed = true;\n  }\n\n  [Symbol.asyncIterator](): SpeechStream {\n    return this;\n  }\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAKA,mBAAmC;AAG5B,IAAK,kBAAL,kBAAKA,qBAAL;AAML,EAAAA,kCAAA,qBAAkB,KAAlB;AAIA,EAAAA,kCAAA,wBAAqB,KAArB;AAKA,EAAAA,kCAAA,sBAAmB,KAAnB;AAKA,EAAAA,kCAAA,mBAAgB,KAAhB;AApBU,SAAAA;AAAA,GAAA;AAwDL,MAAe,IAAI;AAAA,EACxB;AAAA,EAEA,YAAY,cAA+B;AACzC,SAAK,gBAAgB;AAAA,EACvB;AAAA;AAAA,EAGA,IAAI,eAAgC;AAClC,WAAO,KAAK;AAAA,EACd;AAUF;AAkBO,MAAe,aAA2D;AAAA,EAC/E,OAA0B,iBAAiB,OAAO,gBAAgB;AAAA,EACxD,QAAQ,IAAI,gCAAoE;AAAA,EAChF,QAAQ,IAAI,gCAAgC;AAAA,EAC5C,SAAS;AAAA;AAAA,EAGnB,UAAU,OAAmB;AAC3B,QAAI,KAAK,MAAM,QAAQ;AACrB,YAAM,IAAI,MAAM,iBAAiB;AAAA,IACnC;AACA,QAAI,KAAK,QAAQ;AACf,YAAM,IAAI,MAAM,kBAAkB;AAAA,IACpC;AACA,SAAK,MAAM,IAAI,KAAK;AAAA,EACtB;AAAA;AAAA,EAGA,QAAQ;AACN,QAAI,KAAK,MAAM,QAAQ;AACrB,YAAM,IAAI,MAAM,iBAAiB;AAAA,IACnC;AACA,QAAI,KAAK,QAAQ;AACf,YAAM,IAAI,MAAM,kBAAkB;AAAA,IACpC;AACA,SAAK,MAAM,IAAI,aAAa,cAAc;AAAA,EAC5C;AAAA;AAAA,EAGA,WAAW;AACT,QAAI,KAAK,MAAM,QAAQ;AACrB,YAAM,IAAI,MAAM,iBAAiB;AAAA,IACnC;AACA,QAAI,KAAK,QAAQ;AACf,YAAM,IAAI,MAAM,kBAAkB;AAAA,IACpC;AACA,SAAK,MAAM,MAAM;AAAA,EACnB;AAAA,EAEA,OAA6C;AAC3C,WAAO,KAAK,MAAM,KAAK;AAAA,EACzB;AAAA;AAAA,EAGA,QAAQ;AACN,SAAK,MAAM,MAAM;AACjB,SAAK,MAAM,MAAM;AACjB,SAAK,SAAS;AAAA,EAChB;AAAA,EAEA,CAAC,OAAO,aAAa,IAAkB;AACrC,WAAO;AAAA,EACT;AACF;","names":["SpeechEventType"]}
+{"version":3,"sources":["../../src/stt/stt.ts"],"sourcesContent":["// SPDX-FileCopyrightText: 2024 LiveKit, Inc.\n//\n// SPDX-License-Identifier: Apache-2.0\nimport type { AudioFrame } from '@livekit/rtc-node';\nimport type { TypedEventEmitter as TypedEmitter } from '@livekit/typed-emitter';\nimport { EventEmitter } from 'node:events';\nimport type { STTMetrics } from '../metrics/base.js';\nimport type { AudioBuffer } from '../utils.js';\nimport { AsyncIterableQueue } from '../utils.js';\n\n/** Indicates start/middle/end of speech */\nexport enum SpeechEventType {\n  /**\n   * Indicate the start of speech.\n   * If the STT doesn't support this event, this will be emitted at the same time\n   * as the first INTERIM_TRANSCRIPT.\n   */\n  START_OF_SPEECH = 0,\n  /**\n   * Interim transcript, useful for real-time transcription.\n   */\n  INTERIM_TRANSCRIPT = 1,\n  /**\n   * Final transcript, emitted when the STT is confident enough that a certain\n   * portion of the speech will not change.\n   */\n  FINAL_TRANSCRIPT = 2,\n  /**\n   * Indicate the end of speech, emitted when the user stops speaking.\n   * The first alternative is a combination of all the previous FINAL_TRANSCRIPT events.\n   */\n  END_OF_SPEECH = 3,\n  /** Usage event, emitted periodically to indicate usage metrics. */\n  RECOGNITION_USAGE = 4,\n  METRICS_COLLECTED = 5,\n}\n\n/** SpeechData contains metadata about this {@link SpeechEvent}. */\nexport interface SpeechData {\n  language: string;\n  text: string;\n  startTime: number;\n  endTime: number;\n  confidence: number;\n}\n\nexport interface RecognitionUsage {\n  audioDuration: number;\n}\n\n/** SpeechEvent is a packet of speech-to-text data. */\nexport interface SpeechEvent {\n  type: SpeechEventType;\n  alternatives?: [SpeechData, ...SpeechData[]];\n  requestId?: string;\n  recognitionUsage?: RecognitionUsage;\n}\n\n/**\n * Describes the capabilities of the STT provider.\n *\n * @remarks\n * At present, the framework only supports providers that have a streaming endpoint.\n */\nexport interface STTCapabilities {\n  streaming: boolean;\n  interimResults: boolean;\n}\n\nexport type STTCallbacks = {\n  [SpeechEventType.METRICS_COLLECTED]: (metrics: STTMetrics) => void;\n};\n\n/**\n * An instance of a speech-to-text adapter.\n *\n * @remarks\n * This class is abstract, and as such cannot be used directly. Instead, use a provider plugin that\n * exports its own child STT class, which inherits this class's methods.\n */\nexport abstract class STT extends (EventEmitter as new () => TypedEmitter<STTCallbacks>) {\n  abstract label: string;\n  #capabilities: STTCapabilities;\n\n  constructor(capabilities: STTCapabilities) {\n    super();\n    this.#capabilities = capabilities;\n  }\n\n  /** Returns this STT's capabilities */\n  get capabilities(): STTCapabilities {\n    return this.#capabilities;\n  }\n\n  /** Receives an audio buffer and returns transcription in the form of a {@link SpeechEvent} */\n  async recognize(frame: AudioBuffer): Promise<SpeechEvent> {\n    const startTime = process.hrtime.bigint();\n    const event = await this._recognize(frame);\n    const duration = Number((process.hrtime.bigint() - startTime) / BigInt(1000000));\n    this.emit(SpeechEventType.METRICS_COLLECTED, {\n      requestId: event.requestId ?? '',\n      timestamp: Date.now(),\n      duration,\n      label: this.label,\n      audioDuration: Array.isArray(frame)\n        ? frame.reduce((sum, a) => sum + a.samplesPerChannel / a.sampleRate, 0)\n        : frame.samplesPerChannel / frame.sampleRate,\n      streamed: false,\n    });\n    return event;\n  }\n\n  protected abstract _recognize(frame: AudioBuffer): Promise<SpeechEvent>;\n\n  /**\n   * Returns a {@link SpeechStream} that can be used to push audio frames and receive\n   * transcriptions\n   */\n  abstract stream(): SpeechStream;\n}\n\n/**\n * An instance of a speech-to-text stream, as an asynchronous iterable iterator.\n *\n * @example Looping through frames\n * ```ts\n * for await (const event of stream) {\n *   if (event.type === SpeechEventType.FINAL_TRANSCRIPT) {\n *     console.log(event.alternatives[0].text)\n *   }\n * }\n * ```\n *\n * @remarks\n * This class is abstract, and as such cannot be used directly. Instead, use a provider plugin that\n * exports its own child SpeechStream class, which inherits this class's methods.\n */\nexport abstract class SpeechStream implements AsyncIterableIterator<SpeechEvent> {\n  protected static readonly FLUSH_SENTINEL = Symbol('FLUSH_SENTINEL');\n  protected input = new AsyncIterableQueue<AudioFrame | typeof SpeechStream.FLUSH_SENTINEL>();\n  protected output = new AsyncIterableQueue<SpeechEvent>();\n  protected queue = new AsyncIterableQueue<SpeechEvent>();\n  abstract label: string;\n  protected closed = false;\n  #stt: STT;\n\n  constructor(stt: STT) {\n    this.#stt = stt;\n    this.monitorMetrics();\n  }\n\n  protected async monitorMetrics() {\n    const startTime = process.hrtime.bigint();\n\n    for await (const event of this.queue) {\n      this.output.put(event);\n      if (event.type !== SpeechEventType.RECOGNITION_USAGE) continue;\n      const duration = process.hrtime.bigint() - startTime;\n      const metrics: STTMetrics = {\n        timestamp: Date.now(),\n        requestId: event.requestId!,\n        duration: Math.trunc(Number(duration / BigInt(1000000))),\n        label: this.label,\n        audioDuration: event.recognitionUsage!.audioDuration,\n        streamed: true,\n      };\n      this.#stt.emit(SpeechEventType.METRICS_COLLECTED, metrics);\n    }\n    this.output.close();\n  }\n\n  /** Push an audio frame to the STT */\n  pushFrame(frame: AudioFrame) {\n    if (this.input.closed) {\n      throw new Error('Input is closed');\n    }\n    if (this.closed) {\n      throw new Error('Stream is closed');\n    }\n    this.input.put(frame);\n  }\n\n  /** Flush the STT, causing it to process all pending text */\n  flush() {\n    if (this.input.closed) {\n      throw new Error('Input is closed');\n    }\n    if (this.closed) {\n      throw new Error('Stream is closed');\n    }\n    this.input.put(SpeechStream.FLUSH_SENTINEL);\n  }\n\n  /** Mark the input as ended and forbid additional pushes */\n  endInput() {\n    if (this.input.closed) {\n      throw new Error('Input is closed');\n    }\n    if (this.closed) {\n      throw new Error('Stream is closed');\n    }\n    this.input.close();\n  }\n\n  next(): Promise<IteratorResult<SpeechEvent>> {\n    return this.output.next();\n  }\n\n  /** Close both the input and output of the STT stream */\n  close() {\n    this.input.close();\n    this.queue.close();\n    this.output.close();\n    this.closed = true;\n  }\n\n  [Symbol.asyncIterator](): SpeechStream {\n    return this;\n  }\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAKA,yBAA6B;AAG7B,mBAAmC;AAG5B,IAAK,kBAAL,kBAAKA,qBAAL;AAML,EAAAA,kCAAA,qBAAkB,KAAlB;AAIA,EAAAA,kCAAA,wBAAqB,KAArB;AAKA,EAAAA,kCAAA,sBAAmB,KAAnB;AAKA,EAAAA,kCAAA,mBAAgB,KAAhB;AAEA,EAAAA,kCAAA,uBAAoB,KAApB;AACA,EAAAA,kCAAA,uBAAoB,KAApB;AAvBU,SAAAA;AAAA,GAAA;AAqEL,MAAe,YAAa,gCAAsD;AAAA,EAEvF;AAAA,EAEA,YAAY,cAA+B;AACzC,UAAM;AACN,SAAK,gBAAgB;AAAA,EACvB;AAAA;AAAA,EAGA,IAAI,eAAgC;AAClC,WAAO,KAAK;AAAA,EACd;AAAA;AAAA,EAGA,MAAM,UAAU,OAA0C;AACxD,UAAM,YAAY,QAAQ,OAAO,OAAO;AACxC,UAAM,QAAQ,MAAM,KAAK,WAAW,KAAK;AACzC,UAAM,WAAW,QAAQ,QAAQ,OAAO,OAAO,IAAI,aAAa,OAAO,GAAO,CAAC;AAC/E,SAAK,KAAK,2BAAmC;AAAA,MAC3C,WAAW,MAAM,aAAa;AAAA,MAC9B,WAAW,KAAK,IAAI;AAAA,MACpB;AAAA,MACA,OAAO,KAAK;AAAA,MACZ,eAAe,MAAM,QAAQ,KAAK,IAC9B,MAAM,OAAO,CAAC,KAAK,MAAM,MAAM,EAAE,oBAAoB,EAAE,YAAY,CAAC,IACpE,MAAM,oBAAoB,MAAM;AAAA,MACpC,UAAU;AAAA,IACZ,CAAC;AACD,WAAO;AAAA,EACT;AASF;AAkBO,MAAe,aAA2D;AAAA,EAC/E,OAA0B,iBAAiB,OAAO,gBAAgB;AAAA,EACxD,QAAQ,IAAI,gCAAoE;AAAA,EAChF,SAAS,IAAI,gCAAgC;AAAA,EAC7C,QAAQ,IAAI,gCAAgC;AAAA,EAE5C,SAAS;AAAA,EACnB;AAAA,EAEA,YAAY,KAAU;AACpB,SAAK,OAAO;AACZ,SAAK,eAAe;AAAA,EACtB;AAAA,EAEA,MAAgB,iBAAiB;AAC/B,UAAM,YAAY,QAAQ,OAAO,OAAO;AAExC,qBAAiB,SAAS,KAAK,OAAO;AACpC,WAAK,OAAO,IAAI,KAAK;AACrB,UAAI,MAAM,SAAS,0BAAmC;AACtD,YAAM,WAAW,QAAQ,OAAO,OAAO,IAAI;AAC3C,YAAM,UAAsB;AAAA,QAC1B,WAAW,KAAK,IAAI;AAAA,QACpB,WAAW,MAAM;AAAA,QACjB,UAAU,KAAK,MAAM,OAAO,WAAW,OAAO,GAAO,CAAC,CAAC;AAAA,QACvD,OAAO,KAAK;AAAA,QACZ,eAAe,MAAM,iBAAkB;AAAA,QACvC,UAAU;AAAA,MACZ;AACA,WAAK,KAAK,KAAK,2BAAmC,OAAO;AAAA,IAC3D;AACA,SAAK,OAAO,MAAM;AAAA,EACpB;AAAA;AAAA,EAGA,UAAU,OAAmB;AAC3B,QAAI,KAAK,MAAM,QAAQ;AACrB,YAAM,IAAI,MAAM,iBAAiB;AAAA,IACnC;AACA,QAAI,KAAK,QAAQ;AACf,YAAM,IAAI,MAAM,kBAAkB;AAAA,IACpC;AACA,SAAK,MAAM,IAAI,KAAK;AAAA,EACtB;AAAA;AAAA,EAGA,QAAQ;AACN,QAAI,KAAK,MAAM,QAAQ;AACrB,YAAM,IAAI,MAAM,iBAAiB;AAAA,IACnC;AACA,QAAI,KAAK,QAAQ;AACf,YAAM,IAAI,MAAM,kBAAkB;AAAA,IACpC;AACA,SAAK,MAAM,IAAI,aAAa,cAAc;AAAA,EAC5C;AAAA;AAAA,EAGA,WAAW;AACT,QAAI,KAAK,MAAM,QAAQ;AACrB,YAAM,IAAI,MAAM,iBAAiB;AAAA,IACnC;AACA,QAAI,KAAK,QAAQ;AACf,YAAM,IAAI,MAAM,kBAAkB;AAAA,IACpC;AACA,SAAK,MAAM,MAAM;AAAA,EACnB;AAAA,EAEA,OAA6C;AAC3C,WAAO,KAAK,OAAO,KAAK;AAAA,EAC1B;AAAA;AAAA,EAGA,QAAQ;AACN,SAAK,MAAM,MAAM;AACjB,SAAK,MAAM,MAAM;AACjB,SAAK,OAAO,MAAM;AAClB,SAAK,SAAS;AAAA,EAChB;AAAA,EAEA,CAAC,OAAO,aAAa,IAAkB;AACrC,WAAO;AAAA,EACT;AACF;","names":["SpeechEventType"]}