@livekit/agents 1.0.15 → 1.0.17

package/dist/voice/agent_activity.cjs

@@ -27,6 +27,7 @@ var import_node_async_hooks = require("node:async_hooks");
 var import_web = require("node:stream/web");
 var import_chat_context = require("../llm/chat_context.cjs");
 var import_llm = require("../llm/index.cjs");
+var import_tool_context = require("../llm/tool_context.cjs");
 var import_log = require("../log.cjs");
 var import_deferred_stream = require("../stream/deferred_stream.cjs");
 var import_stt = require("../stt/stt.cjs");
@@ -58,6 +59,7 @@ class AgentActivity {
   audioStream = new import_deferred_stream.DeferredReadableStream();
   // default to null as None, which maps to the default provider tool choice value
   toolChoice = null;
+  _preemptiveGeneration;
   agent;
   agentSession;
   /** @internal */
@@ -430,8 +432,12 @@ class AgentActivity {
   onStartOfSpeech(_ev) {
     this.agentSession._updateUserState("speaking");
   }
-  onEndOfSpeech(_ev) {
-    this.agentSession._updateUserState("listening");
+  onEndOfSpeech(ev) {
+    let speechEndTime = Date.now();
+    if (ev) {
+      speechEndTime = speechEndTime - ev.silenceDuration;
+    }
+    this.agentSession._updateUserState("listening", speechEndTime);
   }
   onVADInferenceDone(ev) {
     var _a, _b;
@@ -485,6 +491,44 @@ class AgentActivity {
       })
     );
   }
+  onPreemptiveGeneration(info) {
+    if (!this.agentSession.options.preemptiveGeneration || this.draining || this._currentSpeech !== void 0 && !this._currentSpeech.interrupted || !(this.llm instanceof import_llm.LLM)) {
+      return;
+    }
+    this.cancelPreemptiveGeneration();
+    this.logger.info(
+      {
+        newTranscript: info.newTranscript,
+        transcriptConfidence: info.transcriptConfidence
+      },
+      "starting preemptive generation"
+    );
+    const userMessage = import_chat_context.ChatMessage.create({
+      role: "user",
+      content: info.newTranscript
+    });
+    const chatCtx = this.agent.chatCtx.copy();
+    const speechHandle = this.generateReply({
+      userMessage,
+      chatCtx,
+      scheduleSpeech: false
+    });
+    this._preemptiveGeneration = {
+      speechHandle,
+      userMessage,
+      info,
+      chatCtx: chatCtx.copy(),
+      tools: { ...this.tools },
+      toolChoice: this.toolChoice,
+      createdAt: Date.now()
+    };
+  }
+  cancelPreemptiveGeneration() {
+    if (this._preemptiveGeneration !== void 0) {
+      this._preemptiveGeneration.speechHandle._cancel();
+      this._preemptiveGeneration = void 0;
+    }
+  }
   createSpeechTask(options) {
     const { task, ownedSpeechHandle } = options;
     this.speechTasks.add(task);
@@ -506,10 +550,12 @@ class AgentActivity {
   }
   async onEndOfTurn(info) {
     if (this.draining) {
+      this.cancelPreemptiveGeneration();
       this.logger.warn({ user_input: info.newTranscript }, "skipping user input, task is draining");
       return true;
     }
     if (this.stt && this.turnDetection !== "manual" && this._currentSpeech && this._currentSpeech.allowInterruptions && !this._currentSpeech.interrupted && this.agentSession.options.minInterruptionWords > 0 && info.newTranscript.split(" ").length < this.agentSession.options.minInterruptionWords) {
+      this.cancelPreemptiveGeneration();
       this.logger.info("skipping user input, new_transcript is too short");
       return false;
     }
@@ -563,7 +609,8 @@ class AgentActivity {
       chatCtx,
       instructions: defaultInstructions,
       toolChoice: defaultToolChoice,
-      allowInterruptions: defaultAllowInterruptions
+      allowInterruptions: defaultAllowInterruptions,
+      scheduleSpeech = true
     } = options;
     let instructions = defaultInstructions;
     let toolChoice = defaultToolChoice;
@@ -636,7 +683,9 @@ ${instructions}` : instructions,
       });
       task.finally(() => this.onPipelineReplyDone());
     }
-    this.scheduleSpeech(handle, import_speech_handle.SpeechHandle.SPEECH_PRIORITY_NORMAL);
+    if (scheduleSpeech) {
+      this.scheduleSpeech(handle, import_speech_handle.SpeechHandle.SPEECH_PRIORITY_NORMAL);
+    }
     return handle;
   }
   interrupt() {
@@ -709,13 +758,36 @@ ${instructions}` : instructions,
     } else if (this.llm === void 0) {
       return;
     }
-    const speechHandle = this.generateReply({ userMessage, chatCtx });
+    let speechHandle;
+    if (this._preemptiveGeneration !== void 0) {
+      const preemptive = this._preemptiveGeneration;
+      if (preemptive.info.newTranscript === (userMessage == null ? void 0 : userMessage.textContent) && preemptive.chatCtx.isEquivalent(chatCtx) && (0, import_tool_context.isSameToolContext)(preemptive.tools, this.tools) && (0, import_tool_context.isSameToolChoice)(preemptive.toolChoice, this.toolChoice)) {
+        speechHandle = preemptive.speechHandle;
+        this.scheduleSpeech(speechHandle, import_speech_handle.SpeechHandle.SPEECH_PRIORITY_NORMAL);
+        this.logger.debug(
+          {
+            preemptiveLeadTime: Date.now() - preemptive.createdAt
+          },
+          "using preemptive generation"
+        );
+      } else {
+        this.logger.warn(
+          "preemptive generation enabled but chat context or tools have changed after `onUserTurnCompleted`"
+        );
+        preemptive.speechHandle._cancel();
+      }
+      this._preemptiveGeneration = void 0;
+    }
+    if (speechHandle === void 0) {
+      speechHandle = this.generateReply({ userMessage, chatCtx });
+    }
     const eouMetrics = {
       type: "eou_metrics",
       timestamp: Date.now(),
       endOfUtteranceDelayMs: info.endOfUtteranceDelay,
       transcriptionDelayMs: info.transcriptionDelay,
       onUserTurnCompletedDelayMs: callbackDuration,
+      lastSpeakingTimeMs: info.stoppedSpeakingAt ?? 0,
       speechId: speechHandle.id
     };
     this.agentSession.emit(
@@ -823,8 +895,6 @@ ${instructions}` : instructions,
     chatCtx = chatCtx.copy();
     if (newMessage) {
       chatCtx.insert(newMessage);
-      this.agent._chatCtx.insert(newMessage);
-      this.agentSession._conversationItemAdded(newMessage);
     }
     if (instructions) {
       try {
@@ -837,7 +907,6 @@ ${instructions}` : instructions,
         this.logger.error({ error: e }, "error occurred during updateInstructions");
       }
     }
-    this.agentSession._updateAgentState("thinking");
     const tasks = [];
     const [llmTask, llmGenData] = (0, import_generation.performLLMInference)(
       // preserve  `this` context in llmNode
@@ -861,6 +930,10 @@ ${instructions}` : instructions,
       tasks.push(ttsTask);
     }
     await speechHandle.waitIfNotInterrupted([speechHandle._waitForScheduled()]);
+    if (newMessage && speechHandle.scheduled) {
+      this.agent._chatCtx.insert(newMessage);
+      this.agentSession._conversationItemAdded(newMessage);
+    }
     if (speechHandle.interrupted) {
       replyAbortController.abort();
       await (0, import_utils.cancelAndWait)(tasks, AgentActivity.REPLY_TASK_CANCEL_TIMEOUT);
@@ -1442,6 +1515,7 @@ ${instructions}` : instructions,
     const unlock = await this.lock.lock();
     try {
       if (this._draining) return;
+      this.cancelPreemptiveGeneration();
       this.createSpeechTask({
         task: import_utils.Task.from(() => this.agent.onExit()),
         name: "AgentActivity_onExit"
@@ -1460,6 +1534,7 @@ ${instructions}` : instructions,
       if (!this._draining) {
         this.logger.warn("task closing without draining");
       }
+      this.cancelPreemptiveGeneration();
       if (this.llm instanceof import_llm.LLM) {
         this.llm.off("metrics_collected", this.onMetricsCollected);
       }