@livekit/agents 1.1.0 → 1.2.1

package/dist/voice/agent.d.ts

@@ -3,44 +3,71 @@
 import type { AudioFrame } from '@livekit/rtc-node';
 import { AsyncLocalStorage } from 'node:async_hooks';
 import { ReadableStream } from 'node:stream/web';
-import { type LLMModels, type STTModels, type TTSModels } from '../inference/index.js';
+import { type LLMModels, type STTModelString, type TTSModelString } from '../inference/index.js';
 import { ReadonlyChatContext } from '../llm/chat_context.js';
-import type { ChatMessage, FunctionCall, RealtimeModel } from '../llm/index.js';
-import { type ChatChunk, ChatContext, LLM, type ToolChoice, type ToolContext } from '../llm/index.js';
+import type { ChatMessage, FunctionCall } from '../llm/index.js';
+import { type ChatChunk, ChatContext, LLM, RealtimeModel, type ToolChoice, type ToolContext } from '../llm/index.js';
 import type { STT, SpeechEvent } from '../stt/index.js';
 import type { TTS } from '../tts/index.js';
+import { Task } from '../utils.js';
 import type { VAD } from '../vad.js';
-import type { AgentActivity } from './agent_activity.js';
+import { type AgentActivity } from './agent_activity.js';
 import type { AgentSession, TurnDetectionMode } from './agent_session.js';
-export declare const asyncLocalStorage: AsyncLocalStorage<{
+import type { TimedString } from './io.js';
+import type { SpeechHandle } from './speech_handle.js';
+import type { TurnHandlingOptions } from './turn_config/turn_handling.js';
+export declare const functionCallStorage: AsyncLocalStorage<{
     functionCall?: FunctionCall | undefined;
 }>;
+export declare const speechHandleStorage: AsyncLocalStorage<SpeechHandle>;
+type _ActivityTaskInfo = {
+    functionCall: FunctionCall | null;
+    speechHandle: SpeechHandle | null;
+    inlineTask: boolean;
+};
+/** @internal */
+export declare function _setActivityTaskInfo<T>(task: Task<T>, options: {
+    functionCall?: FunctionCall | null;
+    speechHandle?: SpeechHandle | null;
+    inlineTask?: boolean;
+}): void;
+/** @internal */
+export declare function _getActivityTaskInfo<T>(task: Task<T>): _ActivityTaskInfo | undefined;
 export declare const STOP_RESPONSE_SYMBOL: unique symbol;
 export declare class StopResponse extends Error {
     constructor();
 }
 export declare function isStopResponse(value: unknown): value is StopResponse;
 export interface ModelSettings {
+    /** The tool choice to use when calling the LLM. */
     toolChoice?: ToolChoice;
 }
 export interface AgentOptions<UserData> {
+    id?: string;
     instructions: string;
     chatCtx?: ChatContext;
     tools?: ToolContext<UserData>;
-    turnDetection?: TurnDetectionMode;
-    stt?: STT | STTModels;
+    stt?: STT | STTModelString;
     vad?: VAD;
     llm?: LLM | RealtimeModel | LLMModels;
-    tts?: TTS | TTSModels;
-    allowInterruptions?: boolean;
+    tts?: TTS | TTSModelString;
+    turnHandling?: TurnHandlingOptions;
     minConsecutiveSpeechDelay?: number;
+    useTtsAlignedTranscript?: boolean;
+    /** @deprecated use turnHandling.turnDetection instead */
+    turnDetection?: TurnDetectionMode;
+    /** @deprecated use turnHandling.interruption.enabled instead */
+    allowInterruptions?: boolean;
 }
 export declare class Agent<UserData = any> {
-    private turnDetection?;
+    private _id;
     private _stt?;
     private _vad?;
     private _llm?;
     private _tts?;
+    private _turnHandling?;
+    private _minConsecutiveSpeechDelay?;
+    private _useTtsAlignedTranscript?;
     /** @internal */
     _agentActivity?: AgentActivity;
     /** @internal */
@@ -49,18 +76,22 @@ export declare class Agent<UserData = any> {
     _instructions: string;
     /** @internal */
     _tools?: ToolContext<UserData>;
-    constructor({ instructions, chatCtx, tools, turnDetection, stt, vad, llm, tts, }: AgentOptions<UserData>);
+    constructor({ id, instructions, chatCtx, tools, turnDetection, stt, vad, llm, tts, allowInterruptions, turnHandling, minConsecutiveSpeechDelay, useTtsAlignedTranscript, }: AgentOptions<UserData>);
     get vad(): VAD | undefined;
     get stt(): STT | undefined;
     get llm(): LLM | RealtimeModel | undefined;
     get tts(): TTS | undefined;
+    get useTtsAlignedTranscript(): boolean | undefined;
     get chatCtx(): ReadonlyChatContext;
+    get id(): string;
     get instructions(): string;
     get toolCtx(): ToolContext<UserData>;
     get session(): AgentSession<UserData>;
+    get turnHandling(): Partial<TurnHandlingOptions> | undefined;
+    get minConsecutiveSpeechDelay(): number | undefined;
     onEnter(): Promise<void>;
     onExit(): Promise<void>;
-    transcriptionNode(text: ReadableStream<string>, modelSettings: ModelSettings): Promise<ReadableStream<string> | null>;
+    transcriptionNode(text: ReadableStream<string | TimedString>, modelSettings: ModelSettings): Promise<ReadableStream<string | TimedString> | null>;
     onUserTurnCompleted(_chatCtx: ChatContext, _newMessage: ChatMessage): Promise<void>;
     sttNode(audio: ReadableStream<AudioFrame>, modelSettings: ModelSettings): Promise<ReadableStream<SpeechEvent | string> | null>;
     llmNode(chatCtx: ChatContext, toolCtx: ToolContext, modelSettings: ModelSettings): Promise<ReadableStream<ChatChunk | string> | null>;
@@ -68,12 +99,22 @@ export declare class Agent<UserData = any> {
     realtimeAudioOutputNode(audio: ReadableStream<AudioFrame>, modelSettings: ModelSettings): Promise<ReadableStream<AudioFrame> | null>;
     getActivityOrThrow(): AgentActivity;
     updateChatCtx(chatCtx: ChatContext): Promise<void>;
+    updateTools(tools: ToolContext): Promise<void>;
     static default: {
         sttNode(agent: Agent, audio: ReadableStream<AudioFrame>, _modelSettings: ModelSettings): Promise<ReadableStream<SpeechEvent | string> | null>;
         llmNode(agent: Agent, chatCtx: ChatContext, toolCtx: ToolContext, modelSettings: ModelSettings): Promise<ReadableStream<ChatChunk | string> | null>;
         ttsNode(agent: Agent, text: ReadableStream<string>, _modelSettings: ModelSettings): Promise<ReadableStream<AudioFrame> | null>;
-        transcriptionNode(agent: Agent, text: ReadableStream<string>, _modelSettings: ModelSettings): Promise<ReadableStream<string> | null>;
+        transcriptionNode(agent: Agent, text: ReadableStream<string | TimedString>, _modelSettings: ModelSettings): Promise<ReadableStream<string | TimedString> | null>;
         realtimeAudioOutputNode(_agent: Agent, audio: ReadableStream<AudioFrame>, _modelSettings: ModelSettings): Promise<ReadableStream<AudioFrame> | null>;
     };
 }
+export declare class AgentTask<ResultT = unknown, UserData = any> extends Agent<UserData> {
+    #private;
+    private started;
+    private future;
+    get done(): boolean;
+    complete(result: ResultT | Error): void;
+    run(): Promise<ResultT>;
+}
+export {};
 //# sourceMappingURL=agent.d.ts.map

package/dist/voice/agent.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"agent.d.ts","sourceRoot":"","sources":["../../src/voice/agent.ts"],"names":[],"mappings":";;AAGA,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,mBAAmB,CAAC;AACpD,OAAO,EAAE,iBAAiB,EAAE,MAAM,kBAAkB,CAAC;AACrD,OAAO,EAAE,cAAc,EAAE,MAAM,iBAAiB,CAAC;AACjD,OAAO,EAIL,KAAK,SAAS,EACd,KAAK,SAAS,EACd,KAAK,SAAS,EACf,MAAM,uBAAuB,CAAC;AAC/B,OAAO,EAAE,mBAAmB,EAAE,MAAM,wBAAwB,CAAC;AAC7D,OAAO,KAAK,EAAE,WAAW,EAAE,YAAY,EAAE,aAAa,EAAE,MAAM,iBAAiB,CAAC;AAChF,OAAO,EACL,KAAK,SAAS,EACd,WAAW,EACX,GAAG,EACH,KAAK,UAAU,EACf,KAAK,WAAW,EACjB,MAAM,iBAAiB,CAAC;AACzB,OAAO,KAAK,EAAE,GAAG,EAAE,WAAW,EAAE,MAAM,iBAAiB,CAAC;AAGxD,OAAO,KAAK,EAAE,GAAG,EAAE,MAAM,iBAAiB,CAAC;AAE3C,OAAO,KAAK,EAAE,GAAG,EAAE,MAAM,WAAW,CAAC;AACrC,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,qBAAqB,CAAC;AACzD,OAAO,KAAK,EAAE,YAAY,EAAE,iBAAiB,EAAE,MAAM,oBAAoB,CAAC;AAE1E,eAAO,MAAM,iBAAiB;;EAA2D,CAAC;AAC1F,eAAO,MAAM,oBAAoB,eAAyB,CAAC;AAE3D,qBAAa,YAAa,SAAQ,KAAK;;CAStC;AAED,wBAAgB,cAAc,CAAC,KAAK,EAAE,OAAO,GAAG,KAAK,IAAI,YAAY,CAOpE;AAED,MAAM,WAAW,aAAa;IAE5B,UAAU,CAAC,EAAE,UAAU,CAAC;CACzB;AAED,MAAM,WAAW,YAAY,CAAC,QAAQ;IACpC,YAAY,EAAE,MAAM,CAAC;IACrB,OAAO,CAAC,EAAE,WAAW,CAAC;IACtB,KAAK,CAAC,EAAE,WAAW,CAAC,QAAQ,CAAC,CAAC;IAC9B,aAAa,CAAC,EAAE,iBAAiB,CAAC;IAClC,GAAG,CAAC,EAAE,GAAG,GAAG,SAAS,CAAC;IACtB,GAAG,CAAC,EAAE,GAAG,CAAC;IACV,GAAG,CAAC,EAAE,GAAG,GAAG,aAAa,GAAG,SAAS,CAAC;IACtC,GAAG,CAAC,EAAE,GAAG,GAAG,SAAS,CAAC;IACtB,kBAAkB,CAAC,EAAE,OAAO,CAAC;IAC7B,yBAAyB,CAAC,EAAE,MAAM,CAAC;CACpC;AAED,qBAAa,KAAK,CAAC,QAAQ,GAAG,GAAG;IAC/B,OAAO,CAAC,aAAa,CAAC,CAAoB;IAC1C,OAAO,CAAC,IAAI,CAAC,CAAM;IACnB,OAAO,CAAC,IAAI,CAAC,CAAM;IACnB,OAAO,CAAC,IAAI,CAAC,CAAsB;IACnC,OAAO,CAAC,IAAI,CAAC,CAAM;IAEnB,gBAAgB;IAChB,cAAc,CAAC,EAAE,aAAa,CAAC;IAE/B,gBAAgB;IAChB,QAAQ,EAAE,WAAW,CAAC;IAEtB,gBAAgB;IAChB,aAAa,EAAE,MAAM,CAAC;IAEtB,gBAAgB;IAChB,MAAM,CAAC,EAAE,WAAW,CAAC,QAAQ,CAAC,CAAC;gBAEnB,EACV,YAAY,EACZ,OAAO,EACP,KAAK,EACL,aAAa,EACb,GAAG,EACH,GAAG,EACH,GAAG,EACH,GAAG,GACJ,EAAE,YAAY,CAAC,QAAQ,CAAC;IAiCzB,IAAI,GAAG,IAAI,GAAG,GAAG,SAAS,CAEzB;IAED,IAAI,GAAG,IAAI,GAAG,GAAG,SAAS,CAEzB;IAED,IAAI,GAAG,IAAI,GAAG,GAAG,aAAa,GAAG,SAAS,CAEzC;IAED,IAAI,GAAG,IAAI,GAAG,GAAG,SAAS,CAEzB;IAED,IAAI,OAAO,IAAI,mBAAmB,CAEjC;IAED,IAAI,YAAY,IAAI,MAAM,CAEzB;IAED,IAAI,OAAO,IAAI,WAAW,CAAC,QAAQ,CAAC,CAEnC;IAED,IAAI,OAAO,IAAI,YAAY,CAAC,QAAQ,CAAC,CAEpC;IAEK,OAAO,IAAI,OAAO,CAAC,IAAI,CAAC;IAExB,MAAM,IAAI,OAAO,CAAC,IAAI,CAAC;IAEvB,iBAAiB,CACrB,IAAI,EAAE,cAAc,CAAC,MAAM,CAAC,EAC5B,aAAa,EAAE,aAAa,GAC3B,OAAO,CAAC,cAAc,CAAC,MAAM,CAAC,GAAG,IAAI,CAAC;IAInC,mBAAmB,CAAC,QAAQ,EAAE,WAAW,EAAE,WAAW,EAAE,WAAW,GAAG,OAAO,CAAC,IAAI,CAAC;IAEnF,OAAO,CACX,KAAK,EAAE,cAAc,CAAC,UAAU,CAAC,EACjC,aAAa,EAAE,aAAa,GAC3B,OAAO,CAAC,cAAc,CAAC,WAAW,GAAG,MAAM,CAAC,GAAG,IAAI,CAAC;IAIjD,OAAO,CACX,OAAO,EAAE,WAAW,EACpB,OAAO,EAAE,WAAW,EACpB,aAAa,EAAE,aAAa,GAC3B,OAAO,CAAC,cAAc,CAAC,SAAS,GAAG,MAAM,CAAC,GAAG,IAAI,CAAC;IAI/C,OAAO,CACX,IAAI,EAAE,cAAc,CAAC,MAAM,CAAC,EAC5B,aAAa,EAAE,aAAa,GAC3B,OAAO,CAAC,cAAc,CAAC,UAAU,CAAC,GAAG,IAAI,CAAC;IAIvC,uBAAuB,CAC3B,KAAK,EAAE,cAAc,CAAC,UAAU,CAAC,EACjC,aAAa,EAAE,aAAa,GAC3B,OAAO,CAAC,cAAc,CAAC,UAAU,CAAC,GAAG,IAAI,CAAC;IAM7C,kBAAkB,IAAI,aAAa;IAO7B,aAAa,CAAC,OAAO,EAAE,WAAW,GAAG,OAAO,CAAC,IAAI,CAAC;IASxD,MAAM,CAAC,OAAO;uBAEH,KAAK,SACL,eAAe,UAAU,CAAC,kBACjB,aAAa,GAC5B,QAAQ,eAAe,WAAW,GAAG,MAAM,CAAC,GAAG,IAAI,CAAC;uBAmC9C,KAAK,WACH,WAAW,WACX,WAAW,iBACL,aAAa,GAC3B,QAAQ,eAAe,SAAS,GAAG,MAAM,CAAC,GAAG,IAAI,CAAC;uBAmC5C,KAAK,QACN,eAAe,MAAM,CAAC,kBACZ,aAAa,GAC5B,QAAQ,eAAe,UAAU,CAAC,GAAG,IAAI,CAAC;iCAgCpC,KAAK,QACN,eAAe,MAAM,CAAC,kBACZ,aAAa,GAC5B,QAAQ,eAAe,MAAM,CAAC,GAAG,IAAI,CAAC;wCAK/B,KAAK,SACN,eAAe,UAAU,CAAC,kBACjB,aAAa,GAC5B,QAAQ,eAAe,UAAU,CAAC,GAAG,IAAI,CAAC;MAG7C;CACH"}
+{"version":3,"file":"agent.d.ts","sourceRoot":"","sources":["../../src/voice/agent.ts"],"names":[],"mappings":";;AAGA,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,mBAAmB,CAAC;AACpD,OAAO,EAAE,iBAAiB,EAAE,MAAM,kBAAkB,CAAC;AACrD,OAAO,EAAE,cAAc,EAAE,MAAM,iBAAiB,CAAC;AACjD,OAAO,EAIL,KAAK,SAAS,EACd,KAAK,cAAc,EACnB,KAAK,cAAc,EACpB,MAAM,uBAAuB,CAAC;AAC/B,OAAO,EAAE,mBAAmB,EAAE,MAAM,wBAAwB,CAAC;AAC7D,OAAO,KAAK,EAAE,WAAW,EAAE,YAAY,EAAE,MAAM,iBAAiB,CAAC;AACjE,OAAO,EACL,KAAK,SAAS,EACd,WAAW,EACX,GAAG,EACH,aAAa,EACb,KAAK,UAAU,EACf,KAAK,WAAW,EACjB,MAAM,iBAAiB,CAAC;AAEzB,OAAO,KAAK,EAAE,GAAG,EAAE,WAAW,EAAE,MAAM,iBAAiB,CAAC;AAGxD,OAAO,KAAK,EAAE,GAAG,EAAE,MAAM,iBAAiB,CAAC;AAG3C,OAAO,EAAU,IAAI,EAAE,MAAM,aAAa,CAAC;AAC3C,OAAO,KAAK,EAAE,GAAG,EAAE,MAAM,WAAW,CAAC;AACrC,OAAO,EAAE,KAAK,aAAa,EAAwB,MAAM,qBAAqB,CAAC;AAC/E,OAAO,KAAK,EAAE,YAAY,EAAE,iBAAiB,EAAE,MAAM,oBAAoB,CAAC;AAC1E,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,SAAS,CAAC;AAC3C,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC;AACvD,OAAO,KAAK,EAAE,mBAAmB,EAAE,MAAM,gCAAgC,CAAC;AAG1E,eAAO,MAAM,mBAAmB;;EAA2D,CAAC;AAC5F,eAAO,MAAM,mBAAmB,iCAAwC,CAAC;AAGzE,KAAK,iBAAiB,GAAG;IACvB,YAAY,EAAE,YAAY,GAAG,IAAI,CAAC;IAClC,YAAY,EAAE,YAAY,GAAG,IAAI,CAAC;IAClC,UAAU,EAAE,OAAO,CAAC;CACrB,CAAC;AAEF,gBAAgB;AAChB,wBAAgB,oBAAoB,CAAC,CAAC,EACpC,IAAI,EAAE,IAAI,CAAC,CAAC,CAAC,EACb,OAAO,EAAE;IACP,YAAY,CAAC,EAAE,YAAY,GAAG,IAAI,CAAC;IACnC,YAAY,CAAC,EAAE,YAAY,GAAG,IAAI,CAAC;IACnC,UAAU,CAAC,EAAE,OAAO,CAAC;CACtB,GACA,IAAI,CAkBN;AAED,gBAAgB;AAChB,wBAAgB,oBAAoB,CAAC,CAAC,EAAE,IAAI,EAAE,IAAI,CAAC,CAAC,CAAC,GAAG,iBAAiB,GAAG,SAAS,CAEpF;AACD,eAAO,MAAM,oBAAoB,eAAyB,CAAC;AAE3D,qBAAa,YAAa,SAAQ,KAAK;;CAStC;AAED,wBAAgB,cAAc,CAAC,KAAK,EAAE,OAAO,GAAG,KAAK,IAAI,YAAY,CAOpE;AAED,MAAM,WAAW,aAAa;IAC5B,mDAAmD;IACnD,UAAU,CAAC,EAAE,UAAU,CAAC;CACzB;AAED,MAAM,WAAW,YAAY,CAAC,QAAQ;IACpC,EAAE,CAAC,EAAE,MAAM,CAAC;IACZ,YAAY,EAAE,MAAM,CAAC;IACrB,OAAO,CAAC,EAAE,WAAW,CAAC;IACtB,KAAK,CAAC,EAAE,WAAW,CAAC,QAAQ,CAAC,CAAC;IAC9B,GAAG,CAAC,EAAE,GAAG,GAAG,cAAc,CAAC;IAC3B,GAAG,CAAC,EAAE,GAAG,CAAC;IACV,GAAG,CAAC,EAAE,GAAG,GAAG,aAAa,GAAG,SAAS,CAAC;IACtC,GAAG,CAAC,EAAE,GAAG,GAAG,cAAc,CAAC;IAC3B,YAAY,CAAC,EAAE,mBAAmB,CAAC;IACnC,yBAAyB,CAAC,EAAE,MAAM,CAAC;IACnC,uBAAuB,CAAC,EAAE,OAAO,CAAC;IAClC,yDAAyD;IACzD,aAAa,CAAC,EAAE,iBAAiB,CAAC;IAClC,gEAAgE;IAChE,kBAAkB,CAAC,EAAE,OAAO,CAAC;CAC9B;AAED,qBAAa,KAAK,CAAC,QAAQ,GAAG,GAAG;IAC/B,OAAO,CAAC,GAAG,CAAS;IACpB,OAAO,CAAC,IAAI,CAAC,CAAM;IACnB,OAAO,CAAC,IAAI,CAAC,CAAM;IACnB,OAAO,CAAC,IAAI,CAAC,CAAsB;IACnC,OAAO,CAAC,IAAI,CAAC,CAAM;IACnB,OAAO,CAAC,aAAa,CAAC,CAA+B;IAErD,OAAO,CAAC,0BAA0B,CAAC,CAAS;IAC5C,OAAO,CAAC,wBAAwB,CAAC,CAAU;IAE3C,gBAAgB;IAChB,cAAc,CAAC,EAAE,aAAa,CAAC;IAE/B,gBAAgB;IAChB,QAAQ,EAAE,WAAW,CAAC;IAEtB,gBAAgB;IAChB,aAAa,EAAE,MAAM,CAAC;IAEtB,gBAAgB;IAChB,MAAM,CAAC,EAAE,WAAW,CAAC,QAAQ,CAAC,CAAC;gBAEnB,EACV,EAAE,EACF,YAAY,EACZ,OAAO,EACP,KAAK,EACL,aAAa,EACb,GAAG,EACH,GAAG,EACH,GAAG,EACH,GAAG,EACH,kBAAkB,EAClB,YAAY,EACZ,yBAAyB,EACzB,uBAAuB,GACxB,EAAE,YAAY,CAAC,QAAQ,CAAC;IAyDzB,IAAI,GAAG,IAAI,GAAG,GAAG,SAAS,CAEzB;IAED,IAAI,GAAG,IAAI,GAAG,GAAG,SAAS,CAEzB;IAED,IAAI,GAAG,IAAI,GAAG,GAAG,aAAa,GAAG,SAAS,CAEzC;IAED,IAAI,GAAG,IAAI,GAAG,GAAG,SAAS,CAEzB;IAED,IAAI,uBAAuB,IAAI,OAAO,GAAG,SAAS,CAEjD;IAED,IAAI,OAAO,IAAI,mBAAmB,CAEjC;IAED,IAAI,EAAE,IAAI,MAAM,CAEf;IAED,IAAI,YAAY,IAAI,MAAM,CAEzB;IAED,IAAI,OAAO,IAAI,WAAW,CAAC,QAAQ,CAAC,CAEnC;IAED,IAAI,OAAO,IAAI,YAAY,CAAC,QAAQ,CAAC,CAEpC;IAED,IAAI,YAAY,IAAI,OAAO,CAAC,mBAAmB,CAAC,GAAG,SAAS,CAE3D;IAED,IAAI,yBAAyB,IAAI,MAAM,GAAG,SAAS,CAElD;IAEK,OAAO,IAAI,OAAO,CAAC,IAAI,CAAC;IAExB,MAAM,IAAI,OAAO,CAAC,IAAI,CAAC;IAEvB,iBAAiB,CACrB,IAAI,EAAE,cAAc,CAAC,MAAM,GAAG,WAAW,CAAC,EAC1C,aAAa,EAAE,aAAa,GAC3B,OAAO,CAAC,cAAc,CAAC,MAAM,GAAG,WAAW,CAAC,GAAG,IAAI,CAAC;IAIjD,mBAAmB,CAAC,QAAQ,EAAE,WAAW,EAAE,WAAW,EAAE,WAAW,GAAG,OAAO,CAAC,IAAI,CAAC;IAEnF,OAAO,CACX,KAAK,EAAE,cAAc,CAAC,UAAU,CAAC,EACjC,aAAa,EAAE,aAAa,GAC3B,OAAO,CAAC,cAAc,CAAC,WAAW,GAAG,MAAM,CAAC,GAAG,IAAI,CAAC;IAIjD,OAAO,CACX,OAAO,EAAE,WAAW,EACpB,OAAO,EAAE,WAAW,EACpB,aAAa,EAAE,aAAa,GAC3B,OAAO,CAAC,cAAc,CAAC,SAAS,GAAG,MAAM,CAAC,GAAG,IAAI,CAAC;IAI/C,OAAO,CACX,IAAI,EAAE,cAAc,CAAC,MAAM,CAAC,EAC5B,aAAa,EAAE,aAAa,GAC3B,OAAO,CAAC,cAAc,CAAC,UAAU,CAAC,GAAG,IAAI,CAAC;IAIvC,uBAAuB,CAC3B,KAAK,EAAE,cAAc,CAAC,UAAU,CAAC,EACjC,aAAa,EAAE,aAAa,GAC3B,OAAO,CAAC,cAAc,CAAC,UAAU,CAAC,GAAG,IAAI,CAAC;IAM7C,kBAAkB,IAAI,aAAa;IAO7B,aAAa,CAAC,OAAO,EAAE,WAAW,GAAG,OAAO,CAAC,IAAI,CAAC;IAUlD,WAAW,CAAC,KAAK,EAAE,WAAW,GAAG,OAAO,CAAC,IAAI,CAAC;IAUpD,MAAM,CAAC,OAAO;uBAEH,KAAK,SACL,eAAe,UAAU,CAAC,kBACjB,aAAa,GAC5B,QAAQ,eAAe,WAAW,GAAG,MAAM,CAAC,GAAG,IAAI,CAAC;uBA2D9C,KAAK,WACH,WAAW,WACX,WAAW,iBACL,aAAa,GAC3B,QAAQ,eAAe,SAAS,GAAG,MAAM,CAAC,GAAG,IAAI,CAAC;uBAiD5C,KAAK,QACN,eAAe,MAAM,CAAC,kBACZ,aAAa,GAC5B,QAAQ,eAAe,UAAU,CAAC,GAAG,IAAI,CAAC;iCAgDpC,KAAK,QACN,eAAe,MAAM,GAAG,WAAW,CAAC,kBAC1B,aAAa,GAC5B,QAAQ,eAAe,MAAM,GAAG,WAAW,CAAC,GAAG,IAAI,CAAC;wCAK7C,KAAK,SACN,eAAe,UAAU,CAAC,kBACjB,aAAa,GAC5B,QAAQ,eAAe,UAAU,CAAC,GAAG,IAAI,CAAC;MAG7C;CACH;AAED,qBAAa,SAAS,CAAC,OAAO,GAAG,OAAO,EAAE,QAAQ,GAAG,GAAG,CAAE,SAAQ,KAAK,CAAC,QAAQ,CAAC;;IAC/E,OAAO,CAAC,OAAO,CAAS;IACxB,OAAO,CAAC,MAAM,CAAyB;IAIvC,IAAI,IAAI,IAAI,OAAO,CAElB;IAED,QAAQ,CAAC,MAAM,EAAE,OAAO,GAAG,KAAK,GAAG,IAAI;IAiBjC,GAAG,IAAI,OAAO,CAAC,OAAO,CAAC;CAyG9B"}

package/dist/voice/agent.js

@@ -8,12 +8,40 @@ import {
 import { ReadonlyChatContext } from "../llm/chat_context.js";
 import {
   ChatContext,
-  LLM
+  LLM,
+  RealtimeModel
 } from "../llm/index.js";
+import { log } from "../log.js";
 import { StreamAdapter as STTStreamAdapter } from "../stt/index.js";
 import { SentenceTokenizer as BasicSentenceTokenizer } from "../tokenize/basic/index.js";
 import { SynthesizeStream, StreamAdapter as TTSStreamAdapter } from "../tts/index.js";
-const asyncLocalStorage = new AsyncLocalStorage();
+import { USERDATA_TIMED_TRANSCRIPT } from "../types.js";
+import { Future, Task } from "../utils.js";
+import { agentActivityStorage } from "./agent_activity.js";
+import { migrateTurnHandling } from "./turn_config/utils.js";
+const functionCallStorage = new AsyncLocalStorage();
+const speechHandleStorage = new AsyncLocalStorage();
+const activityTaskInfoStorage = /* @__PURE__ */ new WeakMap();
+function _setActivityTaskInfo(task, options) {
+  const info = activityTaskInfoStorage.get(task) ?? {
+    functionCall: null,
+    speechHandle: null,
+    inlineTask: false
+  };
+  if (Object.hasOwn(options, "functionCall")) {
+    info.functionCall = options.functionCall ?? null;
+  }
+  if (Object.hasOwn(options, "speechHandle")) {
+    info.speechHandle = options.speechHandle ?? null;
+  }
+  if (Object.hasOwn(options, "inlineTask")) {
+    info.inlineTask = options.inlineTask ?? false;
+  }
+  activityTaskInfoStorage.set(task, info);
+}
+function _getActivityTaskInfo(task) {
+  return activityTaskInfoStorage.get(task);
+}
 const STOP_RESPONSE_SYMBOL = Symbol("StopResponse");
 class StopResponse extends Error {
   constructor() {
@@ -28,11 +56,14 @@ function isStopResponse(value) {
   return value !== void 0 && value !== null && typeof value === "object" && STOP_RESPONSE_SYMBOL in value;
 }
 class Agent {
-  turnDetection;
+  _id;
   _stt;
   _vad;
   _llm;
   _tts;
+  _turnHandling;
+  _minConsecutiveSpeechDelay;
+  _useTtsAlignedTranscript;
   /** @internal */
   _agentActivity;
   /** @internal */
@@ -42,6 +73,7 @@ class Agent {
   /** @internal */
   _tools;
   constructor({
+    id,
     instructions,
     chatCtx,
     tools,
@@ -49,30 +81,51 @@ class Agent {
     stt,
     vad,
     llm,
-    tts
+    tts,
+    allowInterruptions,
+    turnHandling,
+    minConsecutiveSpeechDelay,
+    useTtsAlignedTranscript
   }) {
+    if (id) {
+      this._id = id;
+    } else {
+      const className = this.constructor.name;
+      if (className === "Agent") {
+        this._id = "default_agent";
+      } else {
+        this._id = className.replace(/([A-Z])/g, "_$1").toLowerCase().replace(/^_/, "");
+      }
+    }
     this._instructions = instructions;
     this._tools = { ...tools };
     this._chatCtx = chatCtx ? chatCtx.copy({
       toolCtx: this._tools
     }) : ChatContext.empty();
-    this.turnDetection = turnDetection;
+    const resolvedTurnHandling = migrateTurnHandling({
+      turnDetection,
+      allowInterruptions,
+      turnHandling
+    });
+    this._turnHandling = Object.keys(resolvedTurnHandling).length > 0 ? resolvedTurnHandling : void 0;
     this._vad = vad;
     if (typeof stt === "string") {
-      this._stt = new InferenceSTT({ model: stt });
+      this._stt = InferenceSTT.fromModelString(stt);
     } else {
       this._stt = stt;
     }
     if (typeof llm === "string") {
-      this._llm = new InferenceLLM({ model: llm });
+      this._llm = InferenceLLM.fromModelString(llm);
     } else {
       this._llm = llm;
     }
     if (typeof tts === "string") {
-      this._tts = new InferenceTTS({ model: tts });
+      this._tts = InferenceTTS.fromModelString(tts);
     } else {
       this._tts = tts;
     }
+    this._minConsecutiveSpeechDelay = minConsecutiveSpeechDelay;
+    this._useTtsAlignedTranscript = useTtsAlignedTranscript;
     this._agentActivity = void 0;
   }
   get vad() {
@@ -87,9 +140,15 @@ class Agent {
   get tts() {
     return this._tts;
   }
+  get useTtsAlignedTranscript() {
+    return this._useTtsAlignedTranscript;
+  }
   get chatCtx() {
     return new ReadonlyChatContext(this._chatCtx.items);
   }
+  get id() {
+    return this._id;
+  }
   get instructions() {
     return this._instructions;
   }
@@ -99,6 +158,12 @@ class Agent {
   get session() {
     return this.getActivityOrThrow().agentSession;
   }
+  get turnHandling() {
+    return this._turnHandling;
+  }
+  get minConsecutiveSpeechDelay() {
+    return this._minConsecutiveSpeechDelay;
+  }
   async onEnter() {
   }
   async onExit() {
@@ -134,33 +199,60 @@ class Agent {
     }
     this._agentActivity.updateChatCtx(chatCtx);
   }
+  // TODO(parity): Add when AgentConfigUpdate is ported to ChatContext.
+  async updateTools(tools) {
+    if (!this._agentActivity) {
+      this._tools = { ...tools };
+      this._chatCtx = this._chatCtx.copy({ toolCtx: this._tools });
+      return;
+    }
+    await this._agentActivity.updateTools(tools);
+  }
   static default = {
     async sttNode(agent, audio, _modelSettings) {
+      var _a;
       const activity = agent.getActivityOrThrow();
       if (!activity.stt) {
         throw new Error("sttNode called but no STT node is available");
       }
-      let wrapped_stt = activity.stt;
-      if (!wrapped_stt.capabilities.streaming) {
-        if (!agent.vad) {
+      let wrappedStt = activity.stt;
+      if (!wrappedStt.capabilities.streaming) {
+        const vad = agent.vad || activity.vad;
+        if (!vad) {
           throw new Error(
             "STT does not support streaming, add a VAD to the AgentTask/VoiceAgent to enable streaming"
           );
         }
-        wrapped_stt = new STTStreamAdapter(wrapped_stt, agent.vad);
+        wrappedStt = new STTStreamAdapter(wrappedStt, vad);
       }
-      const stream = wrapped_stt.stream();
+      const connOptions = activity.agentSession.connOptions.sttConnOptions;
+      const stream = wrappedStt.stream({ connOptions });
+      const audioInputStartedAt = activity.inputStartedAt ?? // Use input started at proxied from AudioRecognition if available
+      ((_a = activity.agentSession._recorderIO) == null ? void 0 : _a.recordingStartedAt) ?? // Fallback to recording start time if available
+      activity.agentSession._startedAt ?? // Fallback to session start time
+      Date.now();
+      stream.startTimeOffset = (Date.now() - audioInputStartedAt) / 1e3;
       stream.updateInputStream(audio);
+      let cleaned = false;
+      const cleanup = () => {
+        if (cleaned) return;
+        cleaned = true;
+        stream.detachInputStream();
+        stream.close();
+      };
       return new ReadableStream({
         async start(controller) {
-          for await (const event of stream) {
-            controller.enqueue(event);
+          try {
+            for await (const event of stream) {
+              controller.enqueue(event);
+            }
+            controller.close();
+          } finally {
+            cleanup();
           }
-          controller.close();
         },
         cancel() {
-          stream.detachInputStream();
-          stream.close();
+          cleanup();
         }
       });
     },
@@ -175,21 +267,32 @@ class Agent {
         );
       }
       const { toolChoice } = modelSettings;
+      const connOptions = activity.agentSession.connOptions.llmConnOptions;
       const stream = activity.llm.chat({
         chatCtx,
         toolCtx,
         toolChoice,
-        parallelToolCalls: true
+        connOptions
       });
+      let cleaned = false;
+      const cleanup = () => {
+        if (cleaned) return;
+        cleaned = true;
+        stream.close();
+      };
       return new ReadableStream({
         async start(controller) {
-          for await (const chunk of stream) {
-            controller.enqueue(chunk);
+          try {
+            for await (const chunk of stream) {
+              controller.enqueue(chunk);
+            }
+            controller.close();
+          } finally {
+            cleanup();
           }
-          controller.close();
         },
         cancel() {
-          stream.close();
+          cleanup();
         }
       });
     },
@@ -198,24 +301,38 @@ class Agent {
       if (!activity.tts) {
         throw new Error("ttsNode called but no TTS node is available");
       }
-      let wrapped_tts = activity.tts;
+      let wrappedTts = activity.tts;
       if (!activity.tts.capabilities.streaming) {
-        wrapped_tts = new TTSStreamAdapter(wrapped_tts, new BasicSentenceTokenizer());
+        wrappedTts = new TTSStreamAdapter(wrappedTts, new BasicSentenceTokenizer());
       }
-      const stream = wrapped_tts.stream();
+      const connOptions = activity.agentSession.connOptions.ttsConnOptions;
+      const stream = wrappedTts.stream({ connOptions });
       stream.updateInputStream(text);
+      let cleaned = false;
+      const cleanup = () => {
+        if (cleaned) return;
+        cleaned = true;
+        stream.close();
+      };
       return new ReadableStream({
         async start(controller) {
-          for await (const chunk of stream) {
-            if (chunk === SynthesizeStream.END_OF_STREAM) {
-              break;
+          try {
+            for await (const chunk of stream) {
+              if (chunk === SynthesizeStream.END_OF_STREAM) {
+                break;
+              }
+              if (chunk.timedTranscripts && chunk.timedTranscripts.length > 0) {
+                chunk.frame.userdata[USERDATA_TIMED_TRANSCRIPT] = chunk.timedTranscripts;
+              }
+              controller.enqueue(chunk.frame);
             }
-            controller.enqueue(chunk.frame);
+            controller.close();
+          } finally {
+            cleanup();
           }
-          controller.close();
         },
         cancel() {
-          stream.close();
+          cleanup();
         }
       });
     },
@@ -227,11 +344,114 @@ class Agent {
     }
   };
 }
+class AgentTask extends Agent {
+  started = false;
+  future = new Future();
+  #logger = log();
+  get done() {
+    return this.future.done;
+  }
+  complete(result) {
+    if (this.future.done) {
+      throw new Error(`${this.constructor.name} is already done`);
+    }
+    if (result instanceof Error) {
+      this.future.reject(result);
+    } else {
+      this.future.resolve(result);
+    }
+    const speechHandle = speechHandleStorage.getStore();
+    if (speechHandle) {
+      speechHandle._maybeRunFinalOutput = result;
+    }
+  }
+  async run() {
+    if (this.started) {
+      throw new Error(
+        `Task ${this.constructor.name} has already started and cannot be awaited multiple times`
+      );
+    }
+    this.started = true;
+    const currentTask = Task.current();
+    if (!currentTask) {
+      throw new Error(`${this.constructor.name} must be executed inside a Task context`);
+    }
+    const taskInfo = _getActivityTaskInfo(currentTask);
+    if (!taskInfo || !taskInfo.inlineTask) {
+      throw new Error(
+        `${this.constructor.name} should only be awaited inside function tools or the onEnter/onExit methods of an Agent`
+      );
+    }
+    const speechHandle = speechHandleStorage.getStore();
+    const oldActivity = agentActivityStorage.getStore();
+    if (!oldActivity) {
+      throw new Error(`${this.constructor.name} must be executed inside an AgentActivity context`);
+    }
+    currentTask.addDoneCallback(() => {
+      if (this.future.done) return;
+      this.#logger.error(`The Task finished before ${this.constructor.name} was completed.`);
+      this.complete(new Error(`The Task finished before ${this.constructor.name} was completed.`));
+    });
+    const oldAgent = oldActivity.agent;
+    const session = oldActivity.agentSession;
+    const blockedTasks = [currentTask];
+    const onEnterTask = oldActivity._onEnterTask;
+    if (onEnterTask && !onEnterTask.done && onEnterTask !== currentTask) {
+      blockedTasks.push(onEnterTask);
+    }
+    if (taskInfo.functionCall && oldActivity.llm instanceof RealtimeModel && !oldActivity.llm.capabilities.manualFunctionCalls) {
+      this.#logger.error(
+        `Realtime model does not support resuming function calls from chat context, using AgentTask inside a function tool may have unexpected behavior.`
+      );
+    }
+    await session._updateActivity(this, {
+      previousActivity: "pause",
+      newActivity: "start",
+      blockedTasks
+    });
+    let runState = session._globalRunState;
+    if (speechHandle && runState && !runState.done()) {
+      if (runState._watchedHandleCount() > 1) {
+        runState._unwatchHandle(speechHandle);
+      }
+      runState._markDoneIfNeeded();
+    }
+    try {
+      return await this.future.await;
+    } finally {
+      runState = session._globalRunState;
+      if (session.currentAgent !== this) {
+        this.#logger.warn(
+          `${this.constructor.name} completed, but the agent has changed in the meantime. Ignoring handoff to the previous agent, likely due to AgentSession.updateAgent being invoked.`
+        );
+        await oldActivity.close();
+      } else {
+        if (speechHandle && runState && !runState.done()) {
+          runState._watchHandle(speechHandle);
+        }
+        const mergedChatCtx = oldAgent._chatCtx.merge(this._chatCtx, {
+          excludeFunctionCall: true,
+          excludeInstructions: true
+        });
+        oldAgent._chatCtx.items = mergedChatCtx.items;
+        await session._updateActivity(oldAgent, {
+          previousActivity: "close",
+          newActivity: "resume",
+          waitOnEnter: false
+        });
+      }
+    }
+  }
+}
 export {
   Agent,
+  AgentTask,
   STOP_RESPONSE_SYMBOL,
   StopResponse,
-  asyncLocalStorage,
-  isStopResponse
+  _getActivityTaskInfo,
+  _setActivityTaskInfo,
+  functionCallStorage,
+  isStopResponse,
+  speechHandleStorage
 };
 //# sourceMappingURL=agent.js.map