@livekit/agents 1.0.36-dev.0 → 1.0.37

package/dist/voice/agent_activity.d.ts

@@ -6,7 +6,6 @@ import { type GenerationCreatedEvent, type InputSpeechStartedEvent, type InputSp
 import { STT, type SpeechEvent } from '../stt/stt.js';
 import { TTS } from '../tts/tts.js';
 import { Future, Task } from '../utils.js';
-import type { InterruptionEvent } from '../inference/interruption/interruption.js';
 import { VAD, type VADEvent } from '../vad.js';
 import type { Agent } from './agent.js';
 import { type AgentSession, type TurnDetectionMode } from './agent_session.js';
@@ -34,18 +33,6 @@ export declare class AgentActivity implements RecognitionHooks {
     /** @internal */
     _mainTask?: Task<void>;
     _userTurnCompletedTask?: Promise<void>;
-    /**
-     * Notify that agent started speaking.
-     * This enables interruption detection in AudioRecognition.
-     * @internal
-     */
-    notifyAgentSpeechStarted(): void;
-    /**
-     * Notify that agent stopped speaking.
-     * This disables interruption detection in AudioRecognition.
-     * @internal
-     */
-    notifyAgentSpeechEnded(): void;
     constructor(agent: Agent, agentSession: AgentSession);
     start(): Promise<void>;
     get currentSpeech(): SpeechHandle | undefined;
@@ -81,7 +68,6 @@ export declare class AgentActivity implements RecognitionHooks {
     onStartOfSpeech(_ev: VADEvent): void;
     onEndOfSpeech(ev: VADEvent): void;
     onVADInferenceDone(ev: VADEvent): void;
-    onInterruption(ev: InterruptionEvent): void;
     onInterimTranscript(ev: SpeechEvent): void;
     onFinalTranscript(ev: SpeechEvent): void;
     onPreemptiveGeneration(info: PreemptiveGenerationInfo): void;

package/dist/voice/agent_activity.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"agent_activity.d.ts","sourceRoot":"","sources":["../../src/voice/agent_activity.ts"],"names":[],"mappings":";AAIA,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,mBAAmB,CAAC;AAKpD,OAAO,EAAE,cAAc,EAAE,MAAM,iBAAiB,CAAC;AACjD,OAAO,EAAE,KAAK,WAAW,EAAE,WAAW,EAAE,MAAM,wBAAwB,CAAC;AACvE,OAAO,EAIL,KAAK,sBAAsB,EAC3B,KAAK,uBAAuB,EAC5B,KAAK,uBAAuB,EAC5B,KAAK,2BAA2B,EAChC,GAAG,EACH,aAAa,EAEb,KAAK,eAAe,EACpB,KAAK,UAAU,EACf,KAAK,WAAW,EACjB,MAAM,iBAAiB,CAAC;AAazB,OAAO,EAAE,GAAG,EAAiB,KAAK,WAAW,EAAE,MAAM,eAAe,CAAC;AAGrE,OAAO,EAAE,GAAG,EAAiB,MAAM,eAAe,CAAC;AACnD,OAAO,EAAE,MAAM,EAAE,IAAI,EAA0B,MAAM,aAAa,CAAC;AACnE,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,2CAA2C,CAAC;AAEnF,OAAO,EAAE,GAAG,EAAE,KAAK,QAAQ,EAAE,MAAM,WAAW,CAAC;AAC/C,OAAO,KAAK,EAAE,KAAK,EAAiB,MAAM,YAAY,CAAC;AAEvD,OAAO,EAAE,KAAK,YAAY,EAAE,KAAK,iBAAiB,EAAE,MAAM,oBAAoB,CAAC;AAC/E,OAAO,EAEL,KAAK,aAAa,EAClB,KAAK,wBAAwB,EAC7B,KAAK,gBAAgB,EAEtB,MAAM,wBAAwB,CAAC;AAqBhC,OAAO,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC;AAclD,qBAAa,aAAc,YAAW,gBAAgB;IACpD,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,yBAAyB,CAAQ;IACzD,OAAO,CAAC,OAAO,CAAS;IACxB,OAAO,CAAC,gBAAgB,CAAC,CAAmB;IAC5C,OAAO,CAAC,eAAe,CAAC,CAAkB;IAC1C,OAAO,CAAC,aAAa,CAAC,CAAoB;IAC1C,OAAO,CAAC,iBAAiB,CAAC,CAA4C;IACtE,OAAO,CAAC,MAAM,CAAS;IACvB,OAAO,CAAC,SAAS,CAAS;IAC1B,OAAO,CAAC,cAAc,CAAC,CAAe;IACtC,OAAO,CAAC,WAAW,CAAuC;IAC1D,OAAO,CAAC,SAAS,CAAS;IAC1B,OAAO,CAAC,WAAW,CAA8B;IACjD,OAAO,CAAC,IAAI,CAAe;IAC3B,OAAO,CAAC,WAAW,CAA4C;IAE/D,OAAO,CAAC,UAAU,CAA2B;IAC7C,OAAO,CAAC,qBAAqB,CAAC,CAAuB;IAErD,KAAK,EAAE,KAAK,CAAC;IACb,YAAY,EAAE,YAAY,CAAC;IAE3B,gBAAgB;IAChB,SAAS,CAAC,EAAE,IAAI,CAAC,IAAI,CAAC,CAAC;IACvB,sBAAsB,CAAC,EAAE,OAAO,CAAC,IAAI,CAAC,CAAC;IAEvC;;;;OAIG;IACH,wBAAwB,IAAI,IAAI;IAIhC;;;;OAIG;IACH,sBAAsB,IAAI,IAAI;gBAIlB,KAAK,EAAE,KAAK,EAAE,YAAY,EAAE,YAAY;IA6F9C,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;IAqH5B,IAAI,aAAa,IAAI,YAAY,GAAG,SAAS,CAE5C;IAED,IAAI,GAAG,IAAI,GAAG,GAAG,SAAS,CAEzB;IAED,IAAI,GAAG,IAAI,GAAG,GAAG,SAAS,CAEzB;IAED,IAAI,GAAG,IAAI,GAAG,GAAG,aAAa,GAAG,SAAS,CAEzC;IAED,IAAI,GAAG,IAAI,GAAG,GAAG,SAAS,CAEzB;IAED,IAAI,KAAK,IAAI,WAAW,CAEvB;IAED,IAAI,QAAQ,IAAI,OAAO,CAEtB;IAED,IAAI,kBAAkB,IAAI,eAAe,GAAG,SAAS,CAEpD;IAED,IAAI,kBAAkB,IAAI,OAAO,CAGhC;IAED,IAAI,aAAa,IAAI,iBAAiB,GAAG,SAAS,CAGjD;IAED,IAAI,OAAO,IAAI,WAAW,CAEzB;IAEK,aAAa,CAAC,OAAO,EAAE,WAAW,GAAG,OAAO,CAAC,IAAI,CAAC;IAiBxD,aAAa,CAAC,EAAE,UAAU,EAAE,EAAE;QAAE,UAAU,CAAC,EAAE,UAAU,GAAG,IAAI,CAAA;KAAE,GAAG,IAAI;IAUvE,gBAAgB,CAAC,WAAW,EAAE,cAAc,CAAC,UAAU,CAAC,GAAG,IAAI;IAwB/D,gBAAgB,IAAI,IAAI;IAIxB,cAAc;IAUd,aAAa;IAKb,GAAG,CACD,IAAI,EAAE,MAAM,GAAG,cAAc,CAAC,MAAM,CAAC,EACrC,OAAO,CAAC,EAAE;QACR,KAAK,CAAC,EAAE,cAAc,CAAC,UAAU,CAAC,CAAC;QACnC,kBAAkB,CAAC,EAAE,OAAO,CAAC;QAC7B,YAAY,CAAC,EAAE,OAAO,CAAC;KACxB,GACA,YAAY;IAwDf,OAAO,CAAC,kBAAkB,CAqBxB;IAEF,OAAO,CAAC,OAAO;IAoBf,oBAAoB,CAAC,GAAG,EAAE,uBAAuB,GAAG,IAAI;IAmBxD,oBAAoB,CAAC,EAAE,EAAE,uBAAuB,GAAG,IAAI;IAkBvD,kCAAkC,CAAC,EAAE,EAAE,2BAA2B,GAAG,IAAI;IAoBzE,mBAAmB,CAAC,EAAE,EAAE,sBAAsB,GAAG,IAAI;IAsCrD,eAAe,CAAC,GAAG,EAAE,QAAQ,GAAG,IAAI;IAIpC,aAAa,CAAC,EAAE,EAAE,QAAQ,GAAG,IAAI;IAQjC,kBAAkB,CAAC,EAAE,EAAE,QAAQ,GAAG,IAAI;IA+CtC,cAAc,CAAC,EAAE,EAAE,iBAAiB,GAAG,IAAI;IAwC3C,mBAAmB,CAAC,EAAE,EAAE,WAAW,GAAG,IAAI;IAiB1C,iBAAiB,CAAC,EAAE,EAAE,WAAW,GAAG,IAAI;IAiBxC,sBAAsB,CAAC,IAAI,EAAE,wBAAwB,GAAG,IAAI;IA0C5D,OAAO,CAAC,0BAA0B;IAOlC,OAAO,CAAC,gBAAgB;IA4BlB,WAAW,CAAC,IAAI,EAAE,aAAa,GAAG,OAAO,CAAC,OAAO,CAAC;IA0CxD,eAAe,IAAI,WAAW;YAIhB,QAAQ;IAuCtB,OAAO,CAAC,cAAc;IAItB,aAAa,CAAC,OAAO,EAAE;QACrB,WAAW,CAAC,EAAE,WAAW,CAAC;QAC1B,OAAO,CAAC,EAAE,WAAW,CAAC;QACtB,YAAY,CAAC,EAAE,MAAM,CAAC;QACtB,UAAU,CAAC,EAAE,UAAU,GAAG,IAAI,CAAC;QAC/B,kBAAkB,CAAC,EAAE,OAAO,CAAC;QAC7B,cAAc,CAAC,EAAE,OAAO,CAAC;KAC1B,GAAG,YAAY;IAuGhB,SAAS,IAAI,MAAM,CAAC,IAAI,CAAC;IA0BzB,OAAO,CAAC,mBAAmB;YAMb,iBAAiB;YAyHjB,OAAO;IA0HrB,OAAO,CAAC,sBAAsB,CAuX5B;IAEF,OAAO,CAAC,iBAAiB,CA2BrB;YAEU,sBAAsB;YAsBtB,2BAA2B;YAkc3B,iBAAiB;IAqD/B,OAAO,CAAC,cAAc;IAiBhB,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;YAQd,UAAU;IA2BlB,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;CAyC7B"}
+{"version":3,"file":"agent_activity.d.ts","sourceRoot":"","sources":["../../src/voice/agent_activity.ts"],"names":[],"mappings":";AAIA,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,mBAAmB,CAAC;AAKpD,OAAO,EAAE,cAAc,EAAE,MAAM,iBAAiB,CAAC;AACjD,OAAO,EAAE,KAAK,WAAW,EAAE,WAAW,EAAE,MAAM,wBAAwB,CAAC;AACvE,OAAO,EAIL,KAAK,sBAAsB,EAC3B,KAAK,uBAAuB,EAC5B,KAAK,uBAAuB,EAC5B,KAAK,2BAA2B,EAChC,GAAG,EACH,aAAa,EAEb,KAAK,eAAe,EACpB,KAAK,UAAU,EACf,KAAK,WAAW,EACjB,MAAM,iBAAiB,CAAC;AAazB,OAAO,EAAE,GAAG,EAAiB,KAAK,WAAW,EAAE,MAAM,eAAe,CAAC;AAGrE,OAAO,EAAE,GAAG,EAAiB,MAAM,eAAe,CAAC;AACnD,OAAO,EAAE,MAAM,EAAE,IAAI,EAA0B,MAAM,aAAa,CAAC;AACnE,OAAO,EAAE,GAAG,EAAE,KAAK,QAAQ,EAAE,MAAM,WAAW,CAAC;AAC/C,OAAO,KAAK,EAAE,KAAK,EAAiB,MAAM,YAAY,CAAC;AAEvD,OAAO,EAAE,KAAK,YAAY,EAAE,KAAK,iBAAiB,EAAE,MAAM,oBAAoB,CAAC;AAC/E,OAAO,EAEL,KAAK,aAAa,EAClB,KAAK,wBAAwB,EAC7B,KAAK,gBAAgB,EAEtB,MAAM,wBAAwB,CAAC;AAqBhC,OAAO,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC;AAclD,qBAAa,aAAc,YAAW,gBAAgB;IACpD,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,yBAAyB,CAAQ;IACzD,OAAO,CAAC,OAAO,CAAS;IACxB,OAAO,CAAC,gBAAgB,CAAC,CAAmB;IAC5C,OAAO,CAAC,eAAe,CAAC,CAAkB;IAC1C,OAAO,CAAC,aAAa,CAAC,CAAoB;IAC1C,OAAO,CAAC,iBAAiB,CAAC,CAA4C;IACtE,OAAO,CAAC,MAAM,CAAS;IACvB,OAAO,CAAC,SAAS,CAAS;IAC1B,OAAO,CAAC,cAAc,CAAC,CAAe;IACtC,OAAO,CAAC,WAAW,CAAuC;IAC1D,OAAO,CAAC,SAAS,CAAS;IAC1B,OAAO,CAAC,WAAW,CAA8B;IACjD,OAAO,CAAC,IAAI,CAAe;IAC3B,OAAO,CAAC,WAAW,CAA4C;IAE/D,OAAO,CAAC,UAAU,CAA2B;IAC7C,OAAO,CAAC,qBAAqB,CAAC,CAAuB;IAErD,KAAK,EAAE,KAAK,CAAC;IACb,YAAY,EAAE,YAAY,CAAC;IAE3B,gBAAgB;IAChB,SAAS,CAAC,EAAE,IAAI,CAAC,IAAI,CAAC,CAAC;IACvB,sBAAsB,CAAC,EAAE,OAAO,CAAC,IAAI,CAAC,CAAC;gBAE3B,KAAK,EAAE,KAAK,EAAE,YAAY,EAAE,YAAY;IA6F9C,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;IAoH5B,IAAI,aAAa,IAAI,YAAY,GAAG,SAAS,CAE5C;IAED,IAAI,GAAG,IAAI,GAAG,GAAG,SAAS,CAEzB;IAED,IAAI,GAAG,IAAI,GAAG,GAAG,SAAS,CAEzB;IAED,IAAI,GAAG,IAAI,GAAG,GAAG,aAAa,GAAG,SAAS,CAEzC;IAED,IAAI,GAAG,IAAI,GAAG,GAAG,SAAS,CAEzB;IAED,IAAI,KAAK,IAAI,WAAW,CAEvB;IAED,IAAI,QAAQ,IAAI,OAAO,CAEtB;IAED,IAAI,kBAAkB,IAAI,eAAe,GAAG,SAAS,CAEpD;IAED,IAAI,kBAAkB,IAAI,OAAO,CAGhC;IAED,IAAI,aAAa,IAAI,iBAAiB,GAAG,SAAS,CAGjD;IAED,IAAI,OAAO,IAAI,WAAW,CAEzB;IAEK,aAAa,CAAC,OAAO,EAAE,WAAW,GAAG,OAAO,CAAC,IAAI,CAAC;IAiBxD,aAAa,CAAC,EAAE,UAAU,EAAE,EAAE;QAAE,UAAU,CAAC,EAAE,UAAU,GAAG,IAAI,CAAA;KAAE,GAAG,IAAI;IAUvE,gBAAgB,CAAC,WAAW,EAAE,cAAc,CAAC,UAAU,CAAC,GAAG,IAAI;IAwB/D,gBAAgB,IAAI,IAAI;IAIxB,cAAc;IAUd,aAAa;IAKb,GAAG,CACD,IAAI,EAAE,MAAM,GAAG,cAAc,CAAC,MAAM,CAAC,EACrC,OAAO,CAAC,EAAE;QACR,KAAK,CAAC,EAAE,cAAc,CAAC,UAAU,CAAC,CAAC;QACnC,kBAAkB,CAAC,EAAE,OAAO,CAAC;QAC7B,YAAY,CAAC,EAAE,OAAO,CAAC;KACxB,GACA,YAAY;IAwDf,OAAO,CAAC,kBAAkB,CAqBxB;IAEF,OAAO,CAAC,OAAO;IAoBf,oBAAoB,CAAC,GAAG,EAAE,uBAAuB,GAAG,IAAI;IAmBxD,oBAAoB,CAAC,EAAE,EAAE,uBAAuB,GAAG,IAAI;IAkBvD,kCAAkC,CAAC,EAAE,EAAE,2BAA2B,GAAG,IAAI;IAoBzE,mBAAmB,CAAC,EAAE,EAAE,sBAAsB,GAAG,IAAI;IAsCrD,eAAe,CAAC,GAAG,EAAE,QAAQ,GAAG,IAAI;IAIpC,aAAa,CAAC,EAAE,EAAE,QAAQ,GAAG,IAAI;IAQjC,kBAAkB,CAAC,EAAE,EAAE,QAAQ,GAAG,IAAI;IA+CtC,mBAAmB,CAAC,EAAE,EAAE,WAAW,GAAG,IAAI;IAiB1C,iBAAiB,CAAC,EAAE,EAAE,WAAW,GAAG,IAAI;IAiBxC,sBAAsB,CAAC,IAAI,EAAE,wBAAwB,GAAG,IAAI;IA0C5D,OAAO,CAAC,0BAA0B;IAOlC,OAAO,CAAC,gBAAgB;IA4BlB,WAAW,CAAC,IAAI,EAAE,aAAa,GAAG,OAAO,CAAC,OAAO,CAAC;IA0CxD,eAAe,IAAI,WAAW;YAIhB,QAAQ;IAuCtB,OAAO,CAAC,cAAc;IAItB,aAAa,CAAC,OAAO,EAAE;QACrB,WAAW,CAAC,EAAE,WAAW,CAAC;QAC1B,OAAO,CAAC,EAAE,WAAW,CAAC;QACtB,YAAY,CAAC,EAAE,MAAM,CAAC;QACtB,UAAU,CAAC,EAAE,UAAU,GAAG,IAAI,CAAC;QAC/B,kBAAkB,CAAC,EAAE,OAAO,CAAC;QAC7B,cAAc,CAAC,EAAE,OAAO,CAAC;KAC1B,GAAG,YAAY;IAuGhB,SAAS,IAAI,MAAM,CAAC,IAAI,CAAC;IA0BzB,OAAO,CAAC,mBAAmB;YAMb,iBAAiB;YAyHjB,OAAO;IA0HrB,OAAO,CAAC,sBAAsB,CAyX5B;IAEF,OAAO,CAAC,iBAAiB,CA2BrB;YAEU,sBAAsB;YAsBtB,2BAA2B;YAkc3B,iBAAiB;IAqD/B,OAAO,CAAC,cAAc;IAiBhB,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;YAQd,UAAU;IA2BlB,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;CAyC7B"}

package/dist/voice/agent_activity.js

@@ -16,7 +16,6 @@ import { recordRealtimeMetrics, traceTypes, tracer } from "../telemetry/index.js
 import { splitWords } from "../tokenize/basic/word.js";
 import { TTS } from "../tts/tts.js";
 import { Future, Task, cancelAndWait, waitFor } from "../utils.js";
-import { InterruptionEventType } from "../inference/interruption/interruption.js";
 import { VAD } from "../vad.js";
 import { StopResponse, asyncLocalStorage } from "./agent.js";
 import {} from "./agent_session.js";
@@ -67,24 +66,6 @@ class AgentActivity {
   /** @internal */
   _mainTask;
   _userTurnCompletedTask;
-  /**
-   * Notify that agent started speaking.
-   * This enables interruption detection in AudioRecognition.
-   * @internal
-   */
-  notifyAgentSpeechStarted() {
-    var _a;
-    (_a = this.audioRecognition) == null ? void 0 : _a.onStartOfAgentSpeech();
-  }
-  /**
-   * Notify that agent stopped speaking.
-   * This disables interruption detection in AudioRecognition.
-   * @internal
-   */
-  notifyAgentSpeechEnded() {
-    var _a;
-    (_a = this.audioRecognition) == null ? void 0 : _a.onEndOfAgentSpeech();
-  }
   constructor(agent, agentSession) {
     this.agent = agent;
     this.agentSession = agentSession;
@@ -217,7 +198,6 @@ class AgentActivity {
         // Disable stt node if stt is not provided
         stt: this.stt ? (...args) => this.agent.sttNode(...args) : void 0,
         vad: this.vad,
-        interruptionDetector: this.agentSession.interruptionDetector,
         turnDetector: typeof this.turnDetection === "string" ? void 0 : this.turnDetection,
         turnDetectionMode: this.turnDetectionMode,
         minEndpointingDelay: this.agentSession.options.minEndpointingDelay,
@@ -511,35 +491,6 @@ class AgentActivity {
       this._currentSpeech.interrupt();
     }
   }
-  onInterruption(ev) {
-    var _a, _b;
-    if (ev.type !== InterruptionEventType.INTERRUPTION) {
-      return;
-    }
-    this.logger.info(
-      {
-        probability: ev.probability,
-        detectionDelay: ev.detectionDelay,
-        totalDuration: ev.totalDuration
-      },
-      "adaptive interruption detected"
-    );
-    if (this.turnDetection === "manual" || this.turnDetection === "realtime_llm") {
-      return;
-    }
-    if (this.llm instanceof RealtimeModel && this.llm.capabilities.turnDetection) {
-      return;
-    }
-    (_a = this.realtimeSession) == null ? void 0 : _a.startUserActivity();
-    if (this._currentSpeech && !this._currentSpeech.interrupted && this._currentSpeech.allowInterruptions) {
-      this.logger.info(
-        { "speech id": this._currentSpeech.id },
-        "speech interrupted by adaptive interruption detector"
-      );
-      (_b = this.realtimeSession) == null ? void 0 : _b.interrupt();
-      this._currentSpeech.interrupt();
-    }
-  }
   onInterimTranscript(ev) {
     if (this.llm instanceof RealtimeModel && this.llm.capabilities.userTranscription) {
       return;
@@ -1020,10 +971,12 @@ ${instructions}` : instructions,
       replyAbortController
     );
     tasks.push(llmTask);
-    const [ttsTextInput, llmOutput] = llmGenData.textStream.tee();
     let ttsTask = null;
     let ttsStream = null;
+    let llmOutput;
     if (audioOutput) {
+      const [ttsTextInput, textOutput] = llmGenData.textStream.tee();
+      llmOutput = textOutput;
       [ttsTask, ttsStream] = performTTSInference(
         (...args) => this.agent.ttsNode(...args),
         ttsTextInput,
@@ -1031,6 +984,8 @@ ${instructions}` : instructions,
         replyAbortController
       );
       tasks.push(ttsTask);
+    } else {
+      llmOutput = llmGenData.textStream;
     }
     await speechHandle.waitIfNotInterrupted([speechHandle._waitForScheduled()]);
     if (newMessage && speechHandle.scheduled) {
@@ -1077,9 +1032,15 @@ ${instructions}` : instructions,
     } else {
       textOut == null ? void 0 : textOut.firstTextFut.await.finally(onFirstFrame);
     }
-    const onToolExecutionStarted = (_) => {
+    const onToolExecutionStarted = (f) => {
+      speechHandle._itemAdded([f]);
+      this.agent._chatCtx.items.push(f);
+      this.agentSession._toolItemsAdded([f]);
     };
-    const onToolExecutionCompleted = (_) => {
+    const onToolExecutionCompleted = (out) => {
+      if (out.toolCallOutput) {
+        speechHandle._itemAdded([out.toolCallOutput]);
+      }
     };
     const [executeToolsTask, toolOutput] = performToolExecutions({
       session: this.agentSession,
@@ -1139,6 +1100,7 @@ ${instructions}` : instructions,
         });
         chatCtx.insert(message);
         this.agent._chatCtx.insert(message);
+        speechHandle._itemAdded([message]);
         this.agentSession._conversationItemAdded(message);
       }
       if (this.agentSession.agentState === "speaking") {
@@ -1162,6 +1124,7 @@ ${instructions}` : instructions,
       });
       chatCtx.insert(message);
       this.agent._chatCtx.insert(message);
+      speechHandle._itemAdded([message]);
       this.agentSession._conversationItemAdded(message);
       this.logger.info(
         { speech_id: speechHandle.id, message: textOut.text },
@@ -1230,24 +1193,12 @@ ${instructions}` : instructions,
     ];
     if (shouldGenerateToolReply) {
       chatCtx.insert(toolMessages);
-      const handle = SpeechHandle.create({
-        allowInterruptions: speechHandle.allowInterruptions,
-        stepIndex: speechHandle._stepIndex + 1,
-        parent: speechHandle
-      });
-      this.agentSession.emit(
-        AgentSessionEventTypes.SpeechCreated,
-        createSpeechCreatedEvent({
-          userInitiated: false,
-          source: "tool_response",
-          speechHandle: handle
-        })
-      );
+      speechHandle._numSteps += 1;
       const respondToolChoice = draining || modelSettings.toolChoice === "none" ? "none" : "auto";
       const toolResponseTask = this.createSpeechTask({
         task: Task.from(
           () => this.pipelineReplyTask(
-            handle,
+            speechHandle,
             chatCtx,
             toolCtx,
             { toolChoice: respondToolChoice },
@@ -1257,11 +1208,11 @@ ${instructions}` : instructions,
             toolMessages
           )
         ),
-        ownedSpeechHandle: handle,
+        ownedSpeechHandle: speechHandle,
         name: "AgentActivity.pipelineReply"
       });
       toolResponseTask.finally(() => this.onPipelineReplyDone());
-      this.scheduleSpeech(handle, SpeechHandle.SPEECH_PRIORITY_NORMAL, true);
+      this.scheduleSpeech(speechHandle, SpeechHandle.SPEECH_PRIORITY_NORMAL, true);
     } else if (functionToolsExecutedEvent.functionCallOutputs.length > 0) {
       for (const msg of toolMessages) {
         msg.createdAt = replyStartedAt;