@livekit/agents-plugin-openai 1.2.3 → 1.2.5

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@livekit/agents-plugin-openai",
-  "version": "1.2.3",
+  "version": "1.2.5",
   "description": "OpenAI plugin for LiveKit Node Agents",
   "main": "dist/index.js",
   "require": "dist/index.cjs",
@@ -25,14 +25,14 @@
     "README.md"
   ],
   "devDependencies": {
-    "@livekit/rtc-node": "^0.13.24",
+    "@livekit/rtc-node": "^0.13.25",
     "@microsoft/api-extractor": "^7.35.0",
     "@types/ws": "^8.5.10",
     "tsup": "^8.3.5",
     "typescript": "^5.0.0",
-    "@livekit/agents": "1.2.3",
-    "@livekit/agents-plugin-silero": "1.2.3",
-    "@livekit/agents-plugins-test": "1.2.3"
+    "@livekit/agents": "1.2.5",
+    "@livekit/agents-plugin-silero": "1.2.5",
+    "@livekit/agents-plugins-test": "1.2.5"
   },
   "dependencies": {
     "@livekit/mutex": "^1.1.1",
@@ -40,9 +40,9 @@
     "ws": "^8.18.0"
   },
   "peerDependencies": {
-    "@livekit/rtc-node": "^0.13.24",
+    "@livekit/rtc-node": "^0.13.25",
     "zod": "^3.25.76 || ^4.1.8",
-    "@livekit/agents": "1.2.3"
+    "@livekit/agents": "1.2.5"
   },
   "scripts": {
     "build": "tsup --onSuccess \"pnpm build:types\"",

package/src/realtime/realtime_model.ts

@@ -185,6 +185,10 @@ export class RealtimeModel extends llm.RealtimeModel {
       autoToolReplyGeneration: false,
       audioOutput: modalities.includes('audio'),
       manualFunctionCalls: true,
+      midSessionChatCtxUpdate: true,
+      midSessionInstructionsUpdate: true,
+      midSessionToolsUpdate: true,
+      perResponseToolChoice: true,
     });
 
     const isAzure = !!(options.apiVersion || options.entraToken || options.azureDeployment);
@@ -477,17 +481,75 @@ export class RealtimeSession extends llm.RealtimeSession {
   async updateChatCtx(_chatCtx: llm.ChatContext): Promise<void> {
     const unlock = await this.updateChatCtxLock.lock();
     try {
+      const validation = llm.validateChatContextStructure(_chatCtx);
+      const blockingErrors = validation.issues.filter(
+        (issue: llm.ChatContextValidationIssue) =>
+          issue.severity === 'error' && issue.code !== 'timestamp_order',
+      );
+      const timestampOrderIssue = validation.issues.find(
+        (issue: llm.ChatContextValidationIssue) => issue.code === 'timestamp_order',
+      );
+      if (blockingErrors.length > 0) {
+        this.#logger.error(
+          { issues: validation.issues, blockingErrors },
+          'Invalid chat context supplied to updateChatCtx',
+        );
+        throw new Error(
+          `Invalid chat context: ${validation.errors} errors, ${validation.warnings} warnings`,
+        );
+      }
+      if (timestampOrderIssue) {
+        this.#logger.warn(
+          { timestampOrderIssue },
+          'Proceeding with non-monotonic createdAt ordering in realtime chat context',
+        );
+      }
+      if (lkOaiDebug > 0 && validation.warnings > 0) {
+        this.#logger.debug(
+          {
+            warnings: validation.warnings,
+            issues: validation.issues,
+          },
+          'Chat context warnings detected before realtime update',
+        );
+      }
+
       const events = await this.createChatCtxUpdateEvents(_chatCtx);
       const futures: Future<void>[] = [];
+      const ownedCreateFutures: { [id: string]: Future<void> } = {};
+      const ownedDeleteFutures: { [id: string]: Future<void> } = {};
+
+      const cleanupTimedOutFutures = () => {
+        // remove timed-out entries so late server acks
+        // don't resolve stale futures from a previous updateChatCtx call.
+        for (const [itemId, future] of Object.entries(ownedDeleteFutures)) {
+          if (this.itemDeleteFutures[itemId] === future) {
+            delete this.itemDeleteFutures[itemId];
+          }
+        }
+        for (const [itemId, future] of Object.entries(ownedCreateFutures)) {
+          if (this.itemCreateFutures[itemId] === future) {
+            delete this.itemCreateFutures[itemId];
+          }
+        }
+      };
 
       for (const event of events) {
-        const future = new Future<void>();
-        futures.push(future);
-
         if (event.type === 'conversation.item.create') {
+          const future = new Future<void>();
+          futures.push(future);
           this.itemCreateFutures[event.item.id] = future;
+          ownedCreateFutures[event.item.id] = future;
         } else if (event.type == 'conversation.item.delete') {
+          const existingDeleteFuture = this.itemDeleteFutures[event.item_id];
+          if (existingDeleteFuture) {
+            futures.push(existingDeleteFuture);
+            continue;
+          }
+          const future = new Future<void>();
+          futures.push(future);
           this.itemDeleteFutures[event.item_id] = future;
+          ownedDeleteFutures[event.item_id] = future;
         }
 
         this.sendEvent(event);
@@ -497,13 +559,21 @@ export class RealtimeSession extends llm.RealtimeSession {
         return;
       }
 
-      // wait for futures to resolve or timeout
-      await Promise.race([
-        Promise.all(futures),
-        delay(5000).then(() => {
-          throw new Error('Chat ctx update events timed out');
-        }),
-      ]);
+      // wait for futures to resolve or timeout.
+      // Cancel the timeout branch once futures resolve to avoid stale cleanup.
+      const timeoutController = new AbortController();
+      const timeoutPromise = delay(5000, { signal: timeoutController.signal }).then(() => {
+        cleanupTimedOutFutures();
+        throw new Error('Chat ctx update events timed out');
+      });
+
+      try {
+        await Promise.race([Promise.all(futures), timeoutPromise]);
+      } finally {
+        if (!timeoutController.signal.aborted) {
+          timeoutController.abort();
+        }
+      }
     } catch (e) {
       this.#logger.error((e as Error).message);
       throw e;

package/src/realtime/realtime_model_beta.ts

@@ -177,6 +177,10 @@ export class RealtimeModel extends llm.RealtimeModel {
       autoToolReplyGeneration: false,
       audioOutput: modalities.includes('audio'),
       manualFunctionCalls: true,
+      midSessionChatCtxUpdate: true,
+      midSessionInstructionsUpdate: true,
+      midSessionToolsUpdate: true,
+      perResponseToolChoice: true,
     });
 
     const isAzure = !!(options.apiVersion || options.entraToken || options.azureDeployment);

package/src/responses/llm.ts

@@ -26,6 +26,8 @@ export interface LLMOptions {
   store?: boolean;
   metadata?: Record<string, string>;
   strictToolSchema?: boolean;
+  /** Specifies the processing tier (e.g. 'auto', 'default', 'priority', 'flex'). */
+  serviceTier?: string;
 
   /**
    * Whether to use the WebSocket API.
@@ -114,6 +116,10 @@ class ResponsesHttpLLM extends llm.LLM {
       modelOptions.metadata = this.#opts.metadata;
     }
 
+    if (this.#opts.serviceTier) {
+      modelOptions.service_tier = this.#opts.serviceTier;
+    }
+
     return new ResponsesHttpLLMStream(this, {
       model: this.#opts.model,
       client: this.#client,
@@ -333,6 +339,7 @@ class ResponsesHttpLLMStream extends llm.LLMStream {
           promptTokens: event.response.usage.input_tokens,
           promptCachedTokens: event.response.usage.input_tokens_details.cached_tokens,
           totalTokens: event.response.usage.total_tokens,
+          serviceTier: event.response.service_tier ?? undefined,
         },
       };
     }

package/src/ws/llm.ts

@@ -141,6 +141,8 @@ export interface WSLLMOptions {
   store?: boolean;
   metadata?: Record<string, string>;
   strictToolSchema?: boolean;
+  /** Specifies the processing tier (e.g. 'auto', 'default', 'priority', 'flex'). */
+  serviceTier?: string;
 }
 
 const defaultLLMOptions: WSLLMOptions = {
@@ -266,6 +268,10 @@ export class WSLLM extends llm.LLM {
       modelOptions.metadata = this.#opts.metadata;
     }
 
+    if (this.#opts.serviceTier) {
+      modelOptions.service_tier = this.#opts.serviceTier;
+    }
+
     let inputChatCtx = chatCtx;
     let prevResponseId: string | undefined;
     const canUseStoredResponse = modelOptions.store !== false;
@@ -584,6 +590,7 @@ export class WSLLMStream extends llm.LLMStream {
           promptTokens: event.response.usage.input_tokens,
           promptCachedTokens: event.response.usage.input_tokens_details.cached_tokens,
           totalTokens: event.response.usage.total_tokens,
+          serviceTier: event.response.service_tier ?? undefined,
         },
       };
     }

package/src/ws/types.ts

@@ -66,6 +66,7 @@ export const wsResponseCompletedEventSchema = z.object({
   response: z
     .object({
       id: z.string(),
+      service_tier: z.string().nullable().optional(),
       usage: z
         .object({
           output_tokens: z.number(),