@livekit/agents 1.0.8 → 1.0.10

package/dist/voice/speech_handle.js.map

@@ -1 +1 @@
-{"version":3,"sources":["../../src/voice/speech_handle.ts"],"sourcesContent":["// SPDX-FileCopyrightText: 2024 LiveKit, Inc.\n//\n// SPDX-License-Identifier: Apache-2.0\nimport type { ChatItem } from '../llm/index.js';\nimport { Event, Future, shortuuid } from '../utils.js';\nimport type { Task } from '../utils.js';\nimport { asyncLocalStorage } from './agent.js';\n\nexport class SpeechHandle {\n  /** Priority for messages that should be played after all other messages in the queue */\n  static SPEECH_PRIORITY_LOW = 0;\n  /** Every speech generates by the VoiceAgent defaults to this priority. */\n  static SPEECH_PRIORITY_NORMAL = 5;\n  /** Priority for important messages that should be played before others. */\n  static SPEECH_PRIORITY_HIGH = 10;\n\n  private interruptFut = new Future<void>();\n  private authorizedEvent = new Event();\n  private scheduledFut = new Future<void>();\n  private doneFut = new Future<void>();\n\n  private generations: Future<void>[] = [];\n  /** @internal */\n  _tasks: Task<void>[] = [];\n  private _chatItems: ChatItem[] = [];\n  private _numSteps = 1;\n\n  private itemAddedCallbacks: Set<(item: ChatItem) => void> = new Set();\n  private doneCallbacks: Set<(sh: SpeechHandle) => void> = new Set();\n\n  constructor(\n    private _id: string,\n    private _allowInterruptions: boolean,\n    /** @internal */\n    public _stepIndex: number,\n    readonly parent?: SpeechHandle,\n  ) {\n    this.doneFut.await.finally(() => {\n      for (const callback of this.doneCallbacks) {\n        callback(this);\n      }\n    });\n  }\n\n  static create(options?: {\n    allowInterruptions?: boolean;\n    stepIndex?: number;\n    parent?: SpeechHandle;\n  }) {\n    const { allowInterruptions = true, stepIndex = 0, parent } = options ?? {};\n\n    return new SpeechHandle(shortuuid('speech_'), allowInterruptions, stepIndex, parent);\n  }\n\n  get interrupted(): boolean {\n    return this.interruptFut.done;\n  }\n\n  get numSteps(): number {\n    return this._numSteps;\n  }\n\n  get id(): string {\n    return this._id;\n  }\n\n  get scheduled(): boolean {\n    return this.scheduledFut.done;\n  }\n\n  get allowInterruptions(): boolean {\n    return this._allowInterruptions;\n  }\n\n  /**\n   * Allow or disallow interruptions on this SpeechHandle.\n   *\n   * When set to false, the SpeechHandle will no longer accept any incoming\n   * interruption requests until re-enabled. If the handle is already\n   * interrupted, clearing interruptions is not allowed.\n   *\n   * @param value - true to allow interruptions, false to disallow\n   * @throws Error If attempting to disable interruptions when already interrupted\n   */\n  set allowInterruptions(value: boolean) {\n    if (this.interrupted && !value) {\n      throw new Error(\n        'Cannot set allow_interruptions to False, the SpeechHandle is already interrupted',\n      );\n    }\n    this._allowInterruptions = value;\n  }\n\n  done(): boolean {\n    return this.doneFut.done;\n  }\n\n  get chatItems(): ChatItem[] {\n    return this._chatItems;\n  }\n\n  /**\n   * Interrupt the current speech generation.\n   *\n   * @throws Error If this speech handle does not allow interruptions.\n   *\n   * @returns The same speech handle that was interrupted.\n   */\n  interrupt(force: boolean = false): SpeechHandle {\n    if (!force && !this.allowInterruptions) {\n      throw new Error('This generation handle does not allow interruptions');\n    }\n\n    this._cancel();\n    return this;\n  }\n\n  /**\n   * Waits for the entire assistant turn to complete playback.\n   *\n   * This method waits until the assistant has fully finished speaking,\n   * including any finalization steps beyond initial response generation.\n   * This is appropriate to call when you want to ensure the speech output\n   * has entirely played out, including any tool calls and response follow-ups.\n   */\n  async waitForPlayout(): Promise<void> {\n    const store = asyncLocalStorage.getStore();\n    if (store && store?.functionCall) {\n      throw new Error(\n        `Cannot call 'SpeechHandle.waitForPlayout()' from inside the function tool '${store.functionCall.name}'. ` +\n          'This creates a circular wait: the speech handle is waiting for the function tool to complete, ' +\n          'while the function tool is simultaneously waiting for the speech handle.\\n' +\n          \"To wait for the assistant's spoken response prior to running this tool, use RunContext.wait_for_playout() instead.\",\n      );\n    }\n  }\n\n  async waitIfNotInterrupted(aw: Promise<unknown>[]): Promise<void> {\n    const allTasksPromise = Promise.all(aw);\n    const fs: Promise<unknown>[] = [allTasksPromise, this.interruptFut.await];\n    await Promise.race(fs);\n  }\n\n  addDoneCallback(callback: (sh: SpeechHandle) => void) {\n    this.doneCallbacks.add(callback);\n  }\n\n  removeDoneCallback(callback: (sh: SpeechHandle) => void) {\n    this.doneCallbacks.delete(callback);\n  }\n\n  /** @internal */\n  _cancel(): SpeechHandle {\n    if (this.done()) {\n      return this;\n    }\n\n    if (!this.interruptFut.done) {\n      this.interruptFut.resolve();\n    }\n\n    return this;\n  }\n\n  /** @internal */\n  _authorizeGeneration(): void {\n    const fut = new Future<void>();\n    this.generations.push(fut);\n    this.authorizedEvent.set();\n  }\n\n  /** @internal */\n  _clearAuthorization(): void {\n    this.authorizedEvent.clear();\n  }\n\n  /** @internal */\n  async _waitForAuthorization(): Promise<void> {\n    await this.authorizedEvent.wait();\n  }\n\n  /** @internal */\n  async _waitForGeneration(stepIdx: number = -1): Promise<void> {\n    if (this.generations.length === 0) {\n      throw new Error('cannot use wait_for_generation: no active generation is running.');\n    }\n\n    const index = stepIdx === -1 ? this.generations.length - 1 : stepIdx;\n    const generation = this.generations[index];\n    if (!generation) {\n      throw new Error(`Generation at index ${index} not found.`);\n    }\n    return generation.await;\n  }\n\n  /** @internal */\n  async _waitForScheduled(): Promise<void> {\n    return this.scheduledFut.await;\n  }\n\n  /** @internal */\n  _markGenerationDone(): void {\n    if (this.generations.length === 0) {\n      throw new Error('cannot use mark_generation_done: no active generation is running.');\n    }\n\n    const lastGeneration = this.generations[this.generations.length - 1];\n    if (lastGeneration && !lastGeneration.done) {\n      lastGeneration.resolve();\n    }\n  }\n\n  /** @internal */\n  _markDone(): void {\n    if (!this.doneFut.done) {\n      this.doneFut.resolve();\n      if (this.generations.length > 0) {\n        this._markGenerationDone(); // preemptive generation could be cancelled before being scheduled\n      }\n    }\n  }\n\n  /** @internal */\n  _markScheduled(): void {\n    if (!this.scheduledFut.done) {\n      this.scheduledFut.resolve();\n    }\n  }\n\n  /** @internal */\n  _addItemAddedCallback(callback: (item: ChatItem) => void): void {\n    this.itemAddedCallbacks.add(callback);\n  }\n\n  /** @internal */\n  _removeItemAddedCallback(callback: (item: ChatItem) => void): void {\n    this.itemAddedCallbacks.delete(callback);\n  }\n\n  /** @internal */\n  _itemAdded(items: ChatItem[]): void {\n    for (const item of items) {\n      for (const cb of this.itemAddedCallbacks) {\n        cb(item);\n      }\n      this._chatItems.push(item);\n    }\n  }\n}\n"],"mappings":"AAIA,SAAS,OAAO,QAAQ,iBAAiB;AAEzC,SAAS,yBAAyB;AAE3B,MAAM,aAAa;AAAA,EAsBxB,YACU,KACA,qBAED,YACE,QACT;AALQ;AACA;AAED;AACE;AAET,SAAK,QAAQ,MAAM,QAAQ,MAAM;AAC/B,iBAAW,YAAY,KAAK,eAAe;AACzC,iBAAS,IAAI;AAAA,MACf;AAAA,IACF,CAAC;AAAA,EACH;AAAA;AAAA,EAhCA,OAAO,sBAAsB;AAAA;AAAA,EAE7B,OAAO,yBAAyB;AAAA;AAAA,EAEhC,OAAO,uBAAuB;AAAA,EAEtB,eAAe,IAAI,OAAa;AAAA,EAChC,kBAAkB,IAAI,MAAM;AAAA,EAC5B,eAAe,IAAI,OAAa;AAAA,EAChC,UAAU,IAAI,OAAa;AAAA,EAE3B,cAA8B,CAAC;AAAA;AAAA,EAEvC,SAAuB,CAAC;AAAA,EAChB,aAAyB,CAAC;AAAA,EAC1B,YAAY;AAAA,EAEZ,qBAAoD,oBAAI,IAAI;AAAA,EAC5D,gBAAiD,oBAAI,IAAI;AAAA,EAgBjE,OAAO,OAAO,SAIX;AACD,UAAM,EAAE,qBAAqB,MAAM,YAAY,GAAG,OAAO,IAAI,WAAW,CAAC;AAEzE,WAAO,IAAI,aAAa,UAAU,SAAS,GAAG,oBAAoB,WAAW,MAAM;AAAA,EACrF;AAAA,EAEA,IAAI,cAAuB;AACzB,WAAO,KAAK,aAAa;AAAA,EAC3B;AAAA,EAEA,IAAI,WAAmB;AACrB,WAAO,KAAK;AAAA,EACd;AAAA,EAEA,IAAI,KAAa;AACf,WAAO,KAAK;AAAA,EACd;AAAA,EAEA,IAAI,YAAqB;AACvB,WAAO,KAAK,aAAa;AAAA,EAC3B;AAAA,EAEA,IAAI,qBAA8B;AAChC,WAAO,KAAK;AAAA,EACd;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAYA,IAAI,mBAAmB,OAAgB;AACrC,QAAI,KAAK,eAAe,CAAC,OAAO;AAC9B,YAAM,IAAI;AAAA,QACR;AAAA,MACF;AAAA,IACF;AACA,SAAK,sBAAsB;AAAA,EAC7B;AAAA,EAEA,OAAgB;AACd,WAAO,KAAK,QAAQ;AAAA,EACtB;AAAA,EAEA,IAAI,YAAwB;AAC1B,WAAO,KAAK;AAAA,EACd;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EASA,UAAU,QAAiB,OAAqB;AAC9C,QAAI,CAAC,SAAS,CAAC,KAAK,oBAAoB;AACtC,YAAM,IAAI,MAAM,qDAAqD;AAAA,IACvE;AAEA,SAAK,QAAQ;AACb,WAAO;AAAA,EACT;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAUA,MAAM,iBAAgC;AACpC,UAAM,QAAQ,kBAAkB,SAAS;AACzC,QAAI,UAAS,+BAAO,eAAc;AAChC,YAAM,IAAI;AAAA,QACR,8EAA8E,MAAM,aAAa,IAAI;AAAA;AAAA,MAIvG;AAAA,IACF;AAAA,EACF;AAAA,EAEA,MAAM,qBAAqB,IAAuC;AAChE,UAAM,kBAAkB,QAAQ,IAAI,EAAE;AACtC,UAAM,KAAyB,CAAC,iBAAiB,KAAK,aAAa,KAAK;AACxE,UAAM,QAAQ,KAAK,EAAE;AAAA,EACvB;AAAA,EAEA,gBAAgB,UAAsC;AACpD,SAAK,cAAc,IAAI,QAAQ;AAAA,EACjC;AAAA,EAEA,mBAAmB,UAAsC;AACvD,SAAK,cAAc,OAAO,QAAQ;AAAA,EACpC;AAAA;AAAA,EAGA,UAAwB;AACtB,QAAI,KAAK,KAAK,GAAG;AACf,aAAO;AAAA,IACT;AAEA,QAAI,CAAC,KAAK,aAAa,MAAM;AAC3B,WAAK,aAAa,QAAQ;AAAA,IAC5B;AAEA,WAAO;AAAA,EACT;AAAA;AAAA,EAGA,uBAA6B;AAC3B,UAAM,MAAM,IAAI,OAAa;AAC7B,SAAK,YAAY,KAAK,GAAG;AACzB,SAAK,gBAAgB,IAAI;AAAA,EAC3B;AAAA;AAAA,EAGA,sBAA4B;AAC1B,SAAK,gBAAgB,MAAM;AAAA,EAC7B;AAAA;AAAA,EAGA,MAAM,wBAAuC;AAC3C,UAAM,KAAK,gBAAgB,KAAK;AAAA,EAClC;AAAA;AAAA,EAGA,MAAM,mBAAmB,UAAkB,IAAmB;AAC5D,QAAI,KAAK,YAAY,WAAW,GAAG;AACjC,YAAM,IAAI,MAAM,kEAAkE;AAAA,IACpF;AAEA,UAAM,QAAQ,YAAY,KAAK,KAAK,YAAY,SAAS,IAAI;AAC7D,UAAM,aAAa,KAAK,YAAY,KAAK;AACzC,QAAI,CAAC,YAAY;AACf,YAAM,IAAI,MAAM,uBAAuB,KAAK,aAAa;AAAA,IAC3D;AACA,WAAO,WAAW;AAAA,EACpB;AAAA;AAAA,EAGA,MAAM,oBAAmC;AACvC,WAAO,KAAK,aAAa;AAAA,EAC3B;AAAA;AAAA,EAGA,sBAA4B;AAC1B,QAAI,KAAK,YAAY,WAAW,GAAG;AACjC,YAAM,IAAI,MAAM,mEAAmE;AAAA,IACrF;AAEA,UAAM,iBAAiB,KAAK,YAAY,KAAK,YAAY,SAAS,CAAC;AACnE,QAAI,kBAAkB,CAAC,eAAe,MAAM;AAC1C,qBAAe,QAAQ;AAAA,IACzB;AAAA,EACF;AAAA;AAAA,EAGA,YAAkB;AAChB,QAAI,CAAC,KAAK,QAAQ,MAAM;AACtB,WAAK,QAAQ,QAAQ;AACrB,UAAI,KAAK,YAAY,SAAS,GAAG;AAC/B,aAAK,oBAAoB;AAAA,MAC3B;AAAA,IACF;AAAA,EACF;AAAA;AAAA,EAGA,iBAAuB;AACrB,QAAI,CAAC,KAAK,aAAa,MAAM;AAC3B,WAAK,aAAa,QAAQ;AAAA,IAC5B;AAAA,EACF;AAAA;AAAA,EAGA,sBAAsB,UAA0C;AAC9D,SAAK,mBAAmB,IAAI,QAAQ;AAAA,EACtC;AAAA;AAAA,EAGA,yBAAyB,UAA0C;AACjE,SAAK,mBAAmB,OAAO,QAAQ;AAAA,EACzC;AAAA;AAAA,EAGA,WAAW,OAAyB;AAClC,eAAW,QAAQ,OAAO;AACxB,iBAAW,MAAM,KAAK,oBAAoB;AACxC,WAAG,IAAI;AAAA,MACT;AACA,WAAK,WAAW,KAAK,IAAI;AAAA,IAC3B;AAAA,EACF;AACF;","names":[]}
+{"version":3,"sources":["../../src/voice/speech_handle.ts"],"sourcesContent":["// SPDX-FileCopyrightText: 2024 LiveKit, Inc.\n//\n// SPDX-License-Identifier: Apache-2.0\nimport type { ChatItem } from '../llm/index.js';\nimport { Event, Future, shortuuid } from '../utils.js';\nimport type { Task } from '../utils.js';\nimport { asyncLocalStorage } from './agent.js';\n\nexport class SpeechHandle {\n  /** Priority for messages that should be played after all other messages in the queue */\n  static SPEECH_PRIORITY_LOW = 0;\n  /** Every speech generates by the VoiceAgent defaults to this priority. */\n  static SPEECH_PRIORITY_NORMAL = 5;\n  /** Priority for important messages that should be played before others. */\n  static SPEECH_PRIORITY_HIGH = 10;\n\n  private interruptFut = new Future<void>();\n  private authorizedEvent = new Event();\n  private scheduledFut = new Future<void>();\n  private doneFut = new Future<void>();\n\n  private generations: Future<void>[] = [];\n  /** @internal */\n  _tasks: Task<void>[] = [];\n  private _chatItems: ChatItem[] = [];\n  private _numSteps = 1;\n\n  private itemAddedCallbacks: Set<(item: ChatItem) => void> = new Set();\n  private doneCallbacks: Set<(sh: SpeechHandle) => void> = new Set();\n\n  constructor(\n    private _id: string,\n    private _allowInterruptions: boolean,\n    /** @internal */\n    public _stepIndex: number,\n    readonly parent?: SpeechHandle,\n  ) {\n    this.doneFut.await.finally(() => {\n      for (const callback of this.doneCallbacks) {\n        callback(this);\n      }\n    });\n  }\n\n  static create(options?: {\n    allowInterruptions?: boolean;\n    stepIndex?: number;\n    parent?: SpeechHandle;\n  }) {\n    const { allowInterruptions = true, stepIndex = 0, parent } = options ?? {};\n\n    return new SpeechHandle(shortuuid('speech_'), allowInterruptions, stepIndex, parent);\n  }\n\n  get interrupted(): boolean {\n    return this.interruptFut.done;\n  }\n\n  get numSteps(): number {\n    return this._numSteps;\n  }\n\n  get id(): string {\n    return this._id;\n  }\n\n  get scheduled(): boolean {\n    return this.scheduledFut.done;\n  }\n\n  get allowInterruptions(): boolean {\n    return this._allowInterruptions;\n  }\n\n  /**\n   * Allow or disallow interruptions on this SpeechHandle.\n   *\n   * When set to false, the SpeechHandle will no longer accept any incoming\n   * interruption requests until re-enabled. If the handle is already\n   * interrupted, clearing interruptions is not allowed.\n   *\n   * @param value - true to allow interruptions, false to disallow\n   * @throws Error If attempting to disable interruptions when already interrupted\n   */\n  set allowInterruptions(value: boolean) {\n    if (this.interrupted && !value) {\n      throw new Error(\n        'Cannot set allow_interruptions to False, the SpeechHandle is already interrupted',\n      );\n    }\n    this._allowInterruptions = value;\n  }\n\n  done(): boolean {\n    return this.doneFut.done;\n  }\n\n  get chatItems(): ChatItem[] {\n    return this._chatItems;\n  }\n\n  /**\n   * Interrupt the current speech generation.\n   *\n   * @throws Error If this speech handle does not allow interruptions.\n   *\n   * @returns The same speech handle that was interrupted.\n   */\n  interrupt(force: boolean = false): SpeechHandle {\n    if (!force && !this.allowInterruptions) {\n      throw new Error('This generation handle does not allow interruptions');\n    }\n\n    this._cancel();\n    return this;\n  }\n\n  /**\n   * Waits for the entire assistant turn to complete playback.\n   *\n   * This method waits until the assistant has fully finished speaking,\n   * including any finalization steps beyond initial response generation.\n   * This is appropriate to call when you want to ensure the speech output\n   * has entirely played out, including any tool calls and response follow-ups.\n   */\n  async waitForPlayout(): Promise<void> {\n    const store = asyncLocalStorage.getStore();\n    if (store && store?.functionCall) {\n      throw new Error(\n        `Cannot call 'SpeechHandle.waitForPlayout()' from inside the function tool '${store.functionCall.name}'. ` +\n          'This creates a circular wait: the speech handle is waiting for the function tool to complete, ' +\n          'while the function tool is simultaneously waiting for the speech handle.\\n' +\n          \"To wait for the assistant's spoken response prior to running this tool, use RunContext.wait_for_playout() instead.\",\n      );\n    }\n    await this.doneFut.await;\n  }\n\n  async waitIfNotInterrupted(aw: Promise<unknown>[]): Promise<void> {\n    const allTasksPromise = Promise.all(aw);\n    const fs: Promise<unknown>[] = [allTasksPromise, this.interruptFut.await];\n    await Promise.race(fs);\n  }\n\n  addDoneCallback(callback: (sh: SpeechHandle) => void) {\n    this.doneCallbacks.add(callback);\n  }\n\n  removeDoneCallback(callback: (sh: SpeechHandle) => void) {\n    this.doneCallbacks.delete(callback);\n  }\n\n  /** @internal */\n  _cancel(): SpeechHandle {\n    if (this.done()) {\n      return this;\n    }\n\n    if (!this.interruptFut.done) {\n      this.interruptFut.resolve();\n    }\n\n    return this;\n  }\n\n  /** @internal */\n  _authorizeGeneration(): void {\n    const fut = new Future<void>();\n    this.generations.push(fut);\n    this.authorizedEvent.set();\n  }\n\n  /** @internal */\n  _clearAuthorization(): void {\n    this.authorizedEvent.clear();\n  }\n\n  /** @internal */\n  async _waitForAuthorization(): Promise<void> {\n    await this.authorizedEvent.wait();\n  }\n\n  /** @internal */\n  async _waitForGeneration(stepIdx: number = -1): Promise<void> {\n    if (this.generations.length === 0) {\n      throw new Error('cannot use wait_for_generation: no active generation is running.');\n    }\n\n    const index = stepIdx === -1 ? this.generations.length - 1 : stepIdx;\n    const generation = this.generations[index];\n    if (!generation) {\n      throw new Error(`Generation at index ${index} not found.`);\n    }\n    return generation.await;\n  }\n\n  /** @internal */\n  async _waitForScheduled(): Promise<void> {\n    return this.scheduledFut.await;\n  }\n\n  /** @internal */\n  _markGenerationDone(): void {\n    if (this.generations.length === 0) {\n      throw new Error('cannot use mark_generation_done: no active generation is running.');\n    }\n\n    const lastGeneration = this.generations[this.generations.length - 1];\n    if (lastGeneration && !lastGeneration.done) {\n      lastGeneration.resolve();\n    }\n  }\n\n  /** @internal */\n  _markDone(): void {\n    if (!this.doneFut.done) {\n      this.doneFut.resolve();\n      if (this.generations.length > 0) {\n        this._markGenerationDone(); // preemptive generation could be cancelled before being scheduled\n      }\n    }\n  }\n\n  /** @internal */\n  _markScheduled(): void {\n    if (!this.scheduledFut.done) {\n      this.scheduledFut.resolve();\n    }\n  }\n\n  /** @internal */\n  _addItemAddedCallback(callback: (item: ChatItem) => void): void {\n    this.itemAddedCallbacks.add(callback);\n  }\n\n  /** @internal */\n  _removeItemAddedCallback(callback: (item: ChatItem) => void): void {\n    this.itemAddedCallbacks.delete(callback);\n  }\n\n  /** @internal */\n  _itemAdded(items: ChatItem[]): void {\n    for (const item of items) {\n      for (const cb of this.itemAddedCallbacks) {\n        cb(item);\n      }\n      this._chatItems.push(item);\n    }\n  }\n}\n"],"mappings":"AAIA,SAAS,OAAO,QAAQ,iBAAiB;AAEzC,SAAS,yBAAyB;AAE3B,MAAM,aAAa;AAAA,EAsBxB,YACU,KACA,qBAED,YACE,QACT;AALQ;AACA;AAED;AACE;AAET,SAAK,QAAQ,MAAM,QAAQ,MAAM;AAC/B,iBAAW,YAAY,KAAK,eAAe;AACzC,iBAAS,IAAI;AAAA,MACf;AAAA,IACF,CAAC;AAAA,EACH;AAAA;AAAA,EAhCA,OAAO,sBAAsB;AAAA;AAAA,EAE7B,OAAO,yBAAyB;AAAA;AAAA,EAEhC,OAAO,uBAAuB;AAAA,EAEtB,eAAe,IAAI,OAAa;AAAA,EAChC,kBAAkB,IAAI,MAAM;AAAA,EAC5B,eAAe,IAAI,OAAa;AAAA,EAChC,UAAU,IAAI,OAAa;AAAA,EAE3B,cAA8B,CAAC;AAAA;AAAA,EAEvC,SAAuB,CAAC;AAAA,EAChB,aAAyB,CAAC;AAAA,EAC1B,YAAY;AAAA,EAEZ,qBAAoD,oBAAI,IAAI;AAAA,EAC5D,gBAAiD,oBAAI,IAAI;AAAA,EAgBjE,OAAO,OAAO,SAIX;AACD,UAAM,EAAE,qBAAqB,MAAM,YAAY,GAAG,OAAO,IAAI,WAAW,CAAC;AAEzE,WAAO,IAAI,aAAa,UAAU,SAAS,GAAG,oBAAoB,WAAW,MAAM;AAAA,EACrF;AAAA,EAEA,IAAI,cAAuB;AACzB,WAAO,KAAK,aAAa;AAAA,EAC3B;AAAA,EAEA,IAAI,WAAmB;AACrB,WAAO,KAAK;AAAA,EACd;AAAA,EAEA,IAAI,KAAa;AACf,WAAO,KAAK;AAAA,EACd;AAAA,EAEA,IAAI,YAAqB;AACvB,WAAO,KAAK,aAAa;AAAA,EAC3B;AAAA,EAEA,IAAI,qBAA8B;AAChC,WAAO,KAAK;AAAA,EACd;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAYA,IAAI,mBAAmB,OAAgB;AACrC,QAAI,KAAK,eAAe,CAAC,OAAO;AAC9B,YAAM,IAAI;AAAA,QACR;AAAA,MACF;AAAA,IACF;AACA,SAAK,sBAAsB;AAAA,EAC7B;AAAA,EAEA,OAAgB;AACd,WAAO,KAAK,QAAQ;AAAA,EACtB;AAAA,EAEA,IAAI,YAAwB;AAC1B,WAAO,KAAK;AAAA,EACd;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EASA,UAAU,QAAiB,OAAqB;AAC9C,QAAI,CAAC,SAAS,CAAC,KAAK,oBAAoB;AACtC,YAAM,IAAI,MAAM,qDAAqD;AAAA,IACvE;AAEA,SAAK,QAAQ;AACb,WAAO;AAAA,EACT;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAUA,MAAM,iBAAgC;AACpC,UAAM,QAAQ,kBAAkB,SAAS;AACzC,QAAI,UAAS,+BAAO,eAAc;AAChC,YAAM,IAAI;AAAA,QACR,8EAA8E,MAAM,aAAa,IAAI;AAAA;AAAA,MAIvG;AAAA,IACF;AACA,UAAM,KAAK,QAAQ;AAAA,EACrB;AAAA,EAEA,MAAM,qBAAqB,IAAuC;AAChE,UAAM,kBAAkB,QAAQ,IAAI,EAAE;AACtC,UAAM,KAAyB,CAAC,iBAAiB,KAAK,aAAa,KAAK;AACxE,UAAM,QAAQ,KAAK,EAAE;AAAA,EACvB;AAAA,EAEA,gBAAgB,UAAsC;AACpD,SAAK,cAAc,IAAI,QAAQ;AAAA,EACjC;AAAA,EAEA,mBAAmB,UAAsC;AACvD,SAAK,cAAc,OAAO,QAAQ;AAAA,EACpC;AAAA;AAAA,EAGA,UAAwB;AACtB,QAAI,KAAK,KAAK,GAAG;AACf,aAAO;AAAA,IACT;AAEA,QAAI,CAAC,KAAK,aAAa,MAAM;AAC3B,WAAK,aAAa,QAAQ;AAAA,IAC5B;AAEA,WAAO;AAAA,EACT;AAAA;AAAA,EAGA,uBAA6B;AAC3B,UAAM,MAAM,IAAI,OAAa;AAC7B,SAAK,YAAY,KAAK,GAAG;AACzB,SAAK,gBAAgB,IAAI;AAAA,EAC3B;AAAA;AAAA,EAGA,sBAA4B;AAC1B,SAAK,gBAAgB,MAAM;AAAA,EAC7B;AAAA;AAAA,EAGA,MAAM,wBAAuC;AAC3C,UAAM,KAAK,gBAAgB,KAAK;AAAA,EAClC;AAAA;AAAA,EAGA,MAAM,mBAAmB,UAAkB,IAAmB;AAC5D,QAAI,KAAK,YAAY,WAAW,GAAG;AACjC,YAAM,IAAI,MAAM,kEAAkE;AAAA,IACpF;AAEA,UAAM,QAAQ,YAAY,KAAK,KAAK,YAAY,SAAS,IAAI;AAC7D,UAAM,aAAa,KAAK,YAAY,KAAK;AACzC,QAAI,CAAC,YAAY;AACf,YAAM,IAAI,MAAM,uBAAuB,KAAK,aAAa;AAAA,IAC3D;AACA,WAAO,WAAW;AAAA,EACpB;AAAA;AAAA,EAGA,MAAM,oBAAmC;AACvC,WAAO,KAAK,aAAa;AAAA,EAC3B;AAAA;AAAA,EAGA,sBAA4B;AAC1B,QAAI,KAAK,YAAY,WAAW,GAAG;AACjC,YAAM,IAAI,MAAM,mEAAmE;AAAA,IACrF;AAEA,UAAM,iBAAiB,KAAK,YAAY,KAAK,YAAY,SAAS,CAAC;AACnE,QAAI,kBAAkB,CAAC,eAAe,MAAM;AAC1C,qBAAe,QAAQ;AAAA,IACzB;AAAA,EACF;AAAA;AAAA,EAGA,YAAkB;AAChB,QAAI,CAAC,KAAK,QAAQ,MAAM;AACtB,WAAK,QAAQ,QAAQ;AACrB,UAAI,KAAK,YAAY,SAAS,GAAG;AAC/B,aAAK,oBAAoB;AAAA,MAC3B;AAAA,IACF;AAAA,EACF;AAAA;AAAA,EAGA,iBAAuB;AACrB,QAAI,CAAC,KAAK,aAAa,MAAM;AAC3B,WAAK,aAAa,QAAQ;AAAA,IAC5B;AAAA,EACF;AAAA;AAAA,EAGA,sBAAsB,UAA0C;AAC9D,SAAK,mBAAmB,IAAI,QAAQ;AAAA,EACtC;AAAA;AAAA,EAGA,yBAAyB,UAA0C;AACjE,SAAK,mBAAmB,OAAO,QAAQ;AAAA,EACzC;AAAA;AAAA,EAGA,WAAW,OAAyB;AAClC,eAAW,QAAQ,OAAO;AACxB,iBAAW,MAAM,KAAK,oBAAoB;AACxC,WAAG,IAAI;AAAA,MACT;AACA,WAAK,WAAW,KAAK,IAAI;AAAA,IAC3B;AAAA,EACF;AACF;","names":[]}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@livekit/agents",
-  "version": "1.0.8",
+  "version": "1.0.10",
   "description": "LiveKit Agents - Node.js",
   "main": "dist/index.js",
   "require": "dist/index.cjs",

package/src/voice/agent_activity.ts

@@ -1506,6 +1506,10 @@ export class AgentActivity implements RecognitionHooks {
       abortController: AbortController,
       outputs: Array<[string, _TextOut | null, _AudioOut | null]>,
     ) => {
+      replyAbortController.signal.addEventListener('abort', () => abortController.abort(), {
+        once: true,
+      });
+
       const forwardTasks: Array<Task<void>> = [];
       try {
         for await (const msg of ev.messageStream) {
@@ -1563,7 +1567,7 @@ export class AgentActivity implements RecognitionHooks {
     const tasks = [
       Task.from(
         (controller) => readMessages(controller, messageOutputs),
-        replyAbortController,
+        undefined,
         'AgentActivity.realtime_generation.read_messages',
       ),
     ];
@@ -1775,6 +1779,20 @@ export class AgentActivity implements RecognitionHooks {
     }
 
     if (functionToolsExecutedEvent.functionCallOutputs.length > 0) {
+      // wait all speeches played before updating the tool output and generating the response
+      // most realtime models dont support generating multiple responses at the same time
+      while (this.currentSpeech || this.speechQueue.size() > 0) {
+        if (
+          this.currentSpeech &&
+          !this.currentSpeech.done() &&
+          this.currentSpeech !== speechHandle
+        ) {
+          await this.currentSpeech.waitForPlayout();
+        } else {
+          // Don't block the event loop
+          await new Promise((resolve) => setImmediate(resolve));
+        }
+      }
       const chatCtx = this.realtimeSession.chatCtx.copy();
       chatCtx.items.push(...functionToolsExecutedEvent.functionCallOutputs);
       try {

package/src/voice/generation_tools.test.ts

@@ -0,0 +1,268 @@
+// SPDX-FileCopyrightText: 2025 LiveKit, Inc.
+//
+// SPDX-License-Identifier: Apache-2.0
+import { ReadableStream as NodeReadableStream } from 'stream/web';
+import { describe, expect, it } from 'vitest';
+import { z } from 'zod';
+import { FunctionCall, tool } from '../llm/index.js';
+import { initializeLogger } from '../log.js';
+import type { Task } from '../utils.js';
+import { cancelAndWait, delay } from '../utils.js';
+import { type _TextOut, performTextForwarding, performToolExecutions } from './generation.js';
+
+function createStringStream(chunks: string[], delayMs: number = 0): NodeReadableStream<string> {
+  return new NodeReadableStream<string>({
+    async start(controller) {
+      for (const c of chunks) {
+        if (delayMs > 0) {
+          await delay(delayMs);
+        }
+        controller.enqueue(c);
+      }
+      controller.close();
+    },
+  });
+}
+
+function createFunctionCallStream(fc: FunctionCall): NodeReadableStream<FunctionCall> {
+  return new NodeReadableStream<FunctionCall>({
+    start(controller) {
+      controller.enqueue(fc);
+      controller.close();
+    },
+  });
+}
+
+function createFunctionCallStreamFromArray(fcs: FunctionCall[]): NodeReadableStream<FunctionCall> {
+  return new NodeReadableStream<FunctionCall>({
+    start(controller) {
+      for (const fc of fcs) {
+        controller.enqueue(fc);
+      }
+      controller.close();
+    },
+  });
+}
+
+describe('Generation + Tool Execution', () => {
+  initializeLogger({ pretty: false, level: 'silent' });
+
+  it('should not abort tool when preamble forwarders are cleaned up', async () => {
+    const replyAbortController = new AbortController();
+    const forwarderController = new AbortController();
+
+    const chunks = Array.from({ length: 50 }, () => `Hi.`);
+    const fullPreambleText = chunks.join('');
+    const preamble = createStringStream(chunks, 20);
+    const [textForwardTask, textOut]: [Task<void>, _TextOut] = performTextForwarding(
+      preamble,
+      forwarderController,
+      null,
+    );
+
+    // Tool that takes > 5 seconds
+    let toolAborted = false;
+    const getWeather = tool({
+      description: 'weather',
+      parameters: z.object({ location: z.string() }),
+      execute: async ({ location }, { abortSignal }) => {
+        if (abortSignal) {
+          abortSignal.addEventListener('abort', () => {
+            toolAborted = true;
+          });
+        }
+        // 6s delay
+        await delay(6000);
+        return `Sunny in ${location}`;
+      },
+    });
+
+    const fc = FunctionCall.create({
+      callId: 'call_1',
+      name: 'getWeather',
+      args: JSON.stringify({ location: 'San Francisco' }),
+    });
+    const toolCallStream = createFunctionCallStream(fc);
+
+    const [execTask, toolOutput] = performToolExecutions({
+      session: {} as any,
+      speechHandle: { id: 'speech_test', _itemAdded: () => {} } as any,
+      toolCtx: { getWeather } as any,
+      toolCallStream,
+      controller: replyAbortController,
+      onToolExecutionStarted: () => {},
+      onToolExecutionCompleted: () => {},
+    });
+
+    // Ensure tool has started, then cancel forwarders mid-stream (without aborting parent AbortController)
+    await toolOutput.firstToolStartedFuture.await;
+    await delay(100);
+    await cancelAndWait([textForwardTask], 5000);
+
+    await execTask.result;
+
+    expect(toolOutput.output.length).toBe(1);
+    const out = toolOutput.output[0]!;
+    expect(out.toolCallOutput?.isError).toBe(false);
+    expect(out.toolCallOutput?.output).toContain('Sunny in San Francisco');
+    // Forwarder should have been cancelled before finishing all preamble chunks
+    expect(textOut.text).not.toBe(fullPreambleText);
+    // Tool's abort signal must not have fired
+    expect(toolAborted).toBe(false);
+  }, 30_000);
+
+  it('should return basic tool execution output', async () => {
+    const replyAbortController = new AbortController();
+
+    const echo = tool({
+      description: 'echo',
+      parameters: z.object({ msg: z.string() }),
+      execute: async ({ msg }) => `echo: ${msg}`,
+    });
+
+    const fc = FunctionCall.create({
+      callId: 'call_2',
+      name: 'echo',
+      args: JSON.stringify({ msg: 'hello' }),
+    });
+    const toolCallStream = createFunctionCallStream(fc);
+
+    const [execTask, toolOutput] = performToolExecutions({
+      session: {} as any,
+      speechHandle: { id: 'speech_test2', _itemAdded: () => {} } as any,
+      toolCtx: { echo } as any,
+      toolCallStream,
+      controller: replyAbortController,
+    });
+
+    await execTask.result;
+    expect(toolOutput.output.length).toBe(1);
+    const out = toolOutput.output[0];
+    expect(out?.toolCallOutput?.isError).toBe(false);
+    expect(out?.toolCallOutput?.output).toContain('echo: hello');
+  });
+
+  it('should abort tool when reply is aborted mid-execution', async () => {
+    const replyAbortController = new AbortController();
+
+    let aborted = false;
+    const longOp = tool({
+      description: 'longOp',
+      parameters: z.object({ ms: z.number() }),
+      execute: async ({ ms }, { abortSignal }) => {
+        if (abortSignal) {
+          abortSignal.addEventListener('abort', () => {
+            aborted = true;
+          });
+        }
+        await delay(ms);
+        return 'done';
+      },
+    });
+
+    const fc = FunctionCall.create({
+      callId: 'call_abort_1',
+      name: 'longOp',
+      args: JSON.stringify({ ms: 5000 }),
+    });
+    const toolCallStream = createFunctionCallStream(fc);
+
+    const [execTask, toolOutput] = performToolExecutions({
+      session: {} as any,
+      speechHandle: { id: 'speech_abort', _itemAdded: () => {} } as any,
+      toolCtx: { longOp } as any,
+      toolCallStream,
+      controller: replyAbortController,
+    });
+
+    await toolOutput.firstToolStartedFuture.await;
+    replyAbortController.abort();
+    await execTask.result;
+
+    expect(aborted).toBe(true);
+    expect(toolOutput.output.length).toBe(1);
+    const out = toolOutput.output[0];
+    expect(out?.toolCallOutput?.isError).toBe(true);
+  }, 20_000);
+
+  it('should return error output on invalid tool args (zod validation failure)', async () => {
+    const replyAbortController = new AbortController();
+
+    const echo = tool({
+      description: 'echo',
+      parameters: z.object({ msg: z.string() }),
+      execute: async ({ msg }) => `echo: ${msg}`,
+    });
+
+    // invalid: msg should be string
+    const fc = FunctionCall.create({
+      callId: 'call_invalid_args',
+      name: 'echo',
+      args: JSON.stringify({ msg: 123 }),
+    });
+    const toolCallStream = createFunctionCallStream(fc);
+
+    const [execTask, toolOutput] = performToolExecutions({
+      session: {} as any,
+      speechHandle: { id: 'speech_invalid', _itemAdded: () => {} } as any,
+      toolCtx: { echo } as any,
+      toolCallStream,
+      controller: replyAbortController,
+    });
+
+    await execTask.result;
+    expect(toolOutput.output.length).toBe(1);
+    const out = toolOutput.output[0];
+    expect(out?.toolCallOutput?.isError).toBe(true);
+  });
+
+  it('should handle multiple tool calls within a single stream', async () => {
+    const replyAbortController = new AbortController();
+
+    const sum = tool({
+      description: 'sum',
+      parameters: z.object({ a: z.number(), b: z.number() }),
+      execute: async ({ a, b }) => a + b,
+    });
+    const upper = tool({
+      description: 'upper',
+      parameters: z.object({ s: z.string() }),
+      execute: async ({ s }) => s.toUpperCase(),
+    });
+
+    const fc1 = FunctionCall.create({
+      callId: 'call_multi_1',
+      name: 'sum',
+      args: JSON.stringify({ a: 2, b: 3 }),
+    });
+    const fc2 = FunctionCall.create({
+      callId: 'call_multi_2',
+      name: 'upper',
+      args: JSON.stringify({ s: 'hey' }),
+    });
+    const toolCallStream = createFunctionCallStreamFromArray([fc1, fc2]);
+
+    const [execTask, toolOutput] = performToolExecutions({
+      session: {} as any,
+      speechHandle: { id: 'speech_multi', _itemAdded: () => {} } as any,
+      toolCtx: { sum, upper } as any,
+      toolCallStream,
+      controller: replyAbortController,
+    });
+
+    await execTask.result;
+    expect(toolOutput.output.length).toBe(2);
+
+    // sort by callId to assert deterministically
+    const sorted = [...toolOutput.output].sort((a, b) =>
+      a.toolCall.callId.localeCompare(b.toolCall.callId),
+    );
+
+    expect(sorted[0]?.toolCall.name).toBe('sum');
+    expect(sorted[0]?.toolCallOutput?.isError).toBe(false);
+    expect(sorted[0]?.toolCallOutput?.output).toBe('5');
+    expect(sorted[1]?.toolCall.name).toBe('upper');
+    expect(sorted[1]?.toolCallOutput?.isError).toBe(false);
+    expect(sorted[1]?.toolCallOutput?.output).toBe('"HEY"');
+  });
+});

package/src/voice/speech_handle.ts

@@ -133,6 +133,7 @@ export class SpeechHandle {
           "To wait for the assistant's spoken response prior to running this tool, use RunContext.wait_for_playout() instead.",
       );
     }
+    await this.doneFut.await;
   }
 
   async waitIfNotInterrupted(aw: Promise<unknown>[]): Promise<void> {