@livekit/agents 1.0.15 → 1.0.16

package/dist/voice/agent_activity.js

@@ -7,6 +7,7 @@ import {
   LLM,
   RealtimeModel
 } from "../llm/index.js";
+import { isSameToolChoice, isSameToolContext } from "../llm/tool_context.js";
 import { log } from "../log.js";
 import { DeferredReadableStream } from "../stream/deferred_stream.js";
 import { STT } from "../stt/stt.js";
@@ -55,6 +56,7 @@ class AgentActivity {
   audioStream = new DeferredReadableStream();
   // default to null as None, which maps to the default provider tool choice value
   toolChoice = null;
+  _preemptiveGeneration;
   agent;
   agentSession;
   /** @internal */
@@ -427,8 +429,12 @@ class AgentActivity {
   onStartOfSpeech(_ev) {
     this.agentSession._updateUserState("speaking");
   }
-  onEndOfSpeech(_ev) {
-    this.agentSession._updateUserState("listening");
+  onEndOfSpeech(ev) {
+    let speechEndTime = Date.now();
+    if (ev) {
+      speechEndTime = speechEndTime - ev.silenceDuration;
+    }
+    this.agentSession._updateUserState("listening", speechEndTime);
   }
   onVADInferenceDone(ev) {
     var _a, _b;
@@ -482,6 +488,44 @@ class AgentActivity {
       })
     );
   }
+  onPreemptiveGeneration(info) {
+    if (!this.agentSession.options.preemptiveGeneration || this.draining || this._currentSpeech !== void 0 && !this._currentSpeech.interrupted || !(this.llm instanceof LLM)) {
+      return;
+    }
+    this.cancelPreemptiveGeneration();
+    this.logger.info(
+      {
+        newTranscript: info.newTranscript,
+        transcriptConfidence: info.transcriptConfidence
+      },
+      "starting preemptive generation"
+    );
+    const userMessage = ChatMessage.create({
+      role: "user",
+      content: info.newTranscript
+    });
+    const chatCtx = this.agent.chatCtx.copy();
+    const speechHandle = this.generateReply({
+      userMessage,
+      chatCtx,
+      scheduleSpeech: false
+    });
+    this._preemptiveGeneration = {
+      speechHandle,
+      userMessage,
+      info,
+      chatCtx: chatCtx.copy(),
+      tools: { ...this.tools },
+      toolChoice: this.toolChoice,
+      createdAt: Date.now()
+    };
+  }
+  cancelPreemptiveGeneration() {
+    if (this._preemptiveGeneration !== void 0) {
+      this._preemptiveGeneration.speechHandle._cancel();
+      this._preemptiveGeneration = void 0;
+    }
+  }
   createSpeechTask(options) {
     const { task, ownedSpeechHandle } = options;
     this.speechTasks.add(task);
@@ -503,10 +547,12 @@ class AgentActivity {
   }
   async onEndOfTurn(info) {
     if (this.draining) {
+      this.cancelPreemptiveGeneration();
       this.logger.warn({ user_input: info.newTranscript }, "skipping user input, task is draining");
       return true;
     }
     if (this.stt && this.turnDetection !== "manual" && this._currentSpeech && this._currentSpeech.allowInterruptions && !this._currentSpeech.interrupted && this.agentSession.options.minInterruptionWords > 0 && info.newTranscript.split(" ").length < this.agentSession.options.minInterruptionWords) {
+      this.cancelPreemptiveGeneration();
       this.logger.info("skipping user input, new_transcript is too short");
       return false;
     }
@@ -560,7 +606,8 @@ class AgentActivity {
       chatCtx,
       instructions: defaultInstructions,
       toolChoice: defaultToolChoice,
-      allowInterruptions: defaultAllowInterruptions
+      allowInterruptions: defaultAllowInterruptions,
+      scheduleSpeech = true
     } = options;
     let instructions = defaultInstructions;
     let toolChoice = defaultToolChoice;
@@ -633,7 +680,9 @@ ${instructions}` : instructions,
       });
       task.finally(() => this.onPipelineReplyDone());
     }
-    this.scheduleSpeech(handle, SpeechHandle.SPEECH_PRIORITY_NORMAL);
+    if (scheduleSpeech) {
+      this.scheduleSpeech(handle, SpeechHandle.SPEECH_PRIORITY_NORMAL);
+    }
     return handle;
   }
   interrupt() {
@@ -706,13 +755,36 @@ ${instructions}` : instructions,
     } else if (this.llm === void 0) {
       return;
     }
-    const speechHandle = this.generateReply({ userMessage, chatCtx });
+    let speechHandle;
+    if (this._preemptiveGeneration !== void 0) {
+      const preemptive = this._preemptiveGeneration;
+      if (preemptive.info.newTranscript === (userMessage == null ? void 0 : userMessage.textContent) && preemptive.chatCtx.isEquivalent(chatCtx) && isSameToolContext(preemptive.tools, this.tools) && isSameToolChoice(preemptive.toolChoice, this.toolChoice)) {
+        speechHandle = preemptive.speechHandle;
+        this.scheduleSpeech(speechHandle, SpeechHandle.SPEECH_PRIORITY_NORMAL);
+        this.logger.debug(
+          {
+            preemptiveLeadTime: Date.now() - preemptive.createdAt
+          },
+          "using preemptive generation"
+        );
+      } else {
+        this.logger.warn(
+          "preemptive generation enabled but chat context or tools have changed after `onUserTurnCompleted`"
+        );
+        preemptive.speechHandle._cancel();
+      }
+      this._preemptiveGeneration = void 0;
+    }
+    if (speechHandle === void 0) {
+      speechHandle = this.generateReply({ userMessage, chatCtx });
+    }
     const eouMetrics = {
       type: "eou_metrics",
       timestamp: Date.now(),
       endOfUtteranceDelayMs: info.endOfUtteranceDelay,
       transcriptionDelayMs: info.transcriptionDelay,
       onUserTurnCompletedDelayMs: callbackDuration,
+      lastSpeakingTimeMs: info.stoppedSpeakingAt ?? 0,
       speechId: speechHandle.id
     };
     this.agentSession.emit(
@@ -820,8 +892,6 @@ ${instructions}` : instructions,
     chatCtx = chatCtx.copy();
     if (newMessage) {
       chatCtx.insert(newMessage);
-      this.agent._chatCtx.insert(newMessage);
-      this.agentSession._conversationItemAdded(newMessage);
     }
     if (instructions) {
       try {
@@ -834,7 +904,6 @@ ${instructions}` : instructions,
         this.logger.error({ error: e }, "error occurred during updateInstructions");
       }
     }
-    this.agentSession._updateAgentState("thinking");
     const tasks = [];
     const [llmTask, llmGenData] = performLLMInference(
       // preserve  `this` context in llmNode
@@ -858,6 +927,10 @@ ${instructions}` : instructions,
       tasks.push(ttsTask);
     }
     await speechHandle.waitIfNotInterrupted([speechHandle._waitForScheduled()]);
+    if (newMessage && speechHandle.scheduled) {
+      this.agent._chatCtx.insert(newMessage);
+      this.agentSession._conversationItemAdded(newMessage);
+    }
     if (speechHandle.interrupted) {
       replyAbortController.abort();
       await cancelAndWait(tasks, AgentActivity.REPLY_TASK_CANCEL_TIMEOUT);
@@ -1439,6 +1512,7 @@ ${instructions}` : instructions,
     const unlock = await this.lock.lock();
     try {
       if (this._draining) return;
+      this.cancelPreemptiveGeneration();
       this.createSpeechTask({
         task: Task.from(() => this.agent.onExit()),
         name: "AgentActivity_onExit"
@@ -1457,6 +1531,7 @@ ${instructions}` : instructions,
       if (!this._draining) {
         this.logger.warn("task closing without draining");
       }
+      this.cancelPreemptiveGeneration();
       if (this.llm instanceof LLM) {
         this.llm.off("metrics_collected", this.onMetricsCollected);
       }