@livekit/agents 1.0.30 → 1.0.32

package/src/ipc/supervised_proc.test.ts

@@ -0,0 +1,153 @@
+// SPDX-FileCopyrightText: 2024 LiveKit, Inc.
+//
+// SPDX-License-Identifier: Apache-2.0
+import { fork, spawn } from 'node:child_process';
+import { unlinkSync, writeFileSync } from 'node:fs';
+import { tmpdir } from 'node:os';
+import { join } from 'node:path';
+import pidusage from 'pidusage';
+import { afterAll, beforeAll, describe, expect, it } from 'vitest';
+
+const childScript = join(tmpdir(), 'test_child.mjs');
+
+beforeAll(() => {
+  writeFileSync(
+    childScript,
+    `process.on('message', (msg) => process.send?.({ echo: msg }));
+     setInterval(() => {}, 1000);`,
+  );
+});
+
+afterAll(() => {
+  try {
+    unlinkSync(childScript);
+  } catch {}
+});
+
+async function getChildMemoryUsageMB(pid: number | undefined): Promise<number> {
+  if (!pid) return 0;
+  try {
+    const stats = await pidusage(pid);
+    return stats.memory / (1024 * 1024);
+  } catch (err) {
+    const code = (err as NodeJS.ErrnoException).code;
+    if (code === 'ENOENT' || code === 'ESRCH') {
+      return 0;
+    }
+    throw err;
+  }
+}
+
+describe('pidusage on dead process', () => {
+  it('raw pidusage throws on dead pid', async () => {
+    const child = spawn('sleep', ['10']);
+    const pid = child.pid!;
+
+    child.kill('SIGKILL');
+    await new Promise<void>((r) => child.on('exit', r));
+
+    await expect(pidusage(pid)).rejects.toThrow();
+  });
+
+  it('fixed version returns 0 instead of crashing', async () => {
+    const child = spawn('sleep', ['10']);
+    const pid = child.pid!;
+
+    child.kill('SIGKILL');
+    await new Promise<void>((r) => child.on('exit', r));
+
+    const mem = await getChildMemoryUsageMB(pid);
+    expect(mem).toBe(0);
+  });
+
+  it('handles concurrent calls on dying process', async () => {
+    const child = spawn('sleep', ['10']);
+    const pid = child.pid!;
+    const exitPromise = new Promise<void>((r) => child.on('exit', r));
+
+    child.kill('SIGKILL');
+
+    const results = await Promise.all([
+      getChildMemoryUsageMB(pid),
+      getChildMemoryUsageMB(pid),
+      getChildMemoryUsageMB(pid),
+    ]);
+
+    await exitPromise;
+    expect(results.every((r) => r === 0)).toBe(true);
+  });
+});
+
+describe('IPC send on dead process', () => {
+  it('child.connected becomes false when child dies', async () => {
+    const child = fork(childScript, [], { stdio: ['pipe', 'pipe', 'pipe', 'ipc'] });
+    const exitPromise = new Promise<void>((r) => child.on('exit', r));
+
+    await new Promise((r) => setTimeout(r, 50));
+    expect(child.connected).toBe(true);
+
+    child.kill('SIGKILL');
+    await exitPromise;
+
+    expect(child.connected).toBe(false);
+  });
+
+  it('checking connected before send prevents crash', async () => {
+    const child = fork(childScript, [], { stdio: ['pipe', 'pipe', 'pipe', 'ipc'] });
+    const exitPromise = new Promise<void>((r) => child.on('exit', r));
+
+    // Suppress EPIPE errors that can occur due to race conditions between
+    // child.connected check and the actual pipe state
+    child.on('error', (err: NodeJS.ErrnoException) => {
+      if (err.code !== 'EPIPE') throw err;
+    });
+
+    let sent = 0;
+    let skipped = 0;
+
+    const interval = setInterval(() => {
+      if (child.connected) {
+        child.send({ ping: Date.now() });
+        sent++;
+      } else {
+        skipped++;
+      }
+    }, 20);
+
+    await new Promise((r) => setTimeout(r, 60));
+    child.kill('SIGKILL');
+    await exitPromise;
+    await new Promise((r) => setTimeout(r, 80));
+    clearInterval(interval);
+
+    expect(sent).toBeGreaterThan(0);
+    expect(skipped).toBeGreaterThan(0);
+  });
+});
+
+describe('timer cleanup', () => {
+  it('clearInterval stops the interval', async () => {
+    let count = 0;
+    const interval = setInterval(() => count++, 30);
+
+    await new Promise((r) => setTimeout(r, 80));
+    const countAtClear = count;
+    clearInterval(interval);
+
+    await new Promise((r) => setTimeout(r, 80));
+    expect(count).toBe(countAtClear);
+  });
+
+  it('double clear is safe', () => {
+    const interval = setInterval(() => {}, 100);
+    const timeout = setTimeout(() => {}, 1000);
+
+    clearInterval(interval);
+    clearTimeout(timeout);
+
+    expect(() => {
+      clearInterval(interval);
+      clearTimeout(timeout);
+    }).not.toThrow();
+  });
+});

package/src/ipc/supervised_proc.ts

@@ -80,7 +80,9 @@ export abstract class SupervisedProc {
     await this.init.await;
 
     this.#pingInterval = setInterval(() => {
-      this.proc!.send({ case: 'pingRequest', value: { timestamp: Date.now() } });
+      if (this.proc?.connected) {
+        this.proc.send({ case: 'pingRequest', value: { timestamp: Date.now() } });
+      }
     }, this.#opts.pingInterval);
 
     this.#pongTimeout = setTimeout(() => {
@@ -141,6 +143,7 @@ export abstract class SupervisedProc {
     });
 
     this.proc!.on('exit', () => {
+      this.clearTimers();
       this.#join.resolve();
     });
 
@@ -159,11 +162,13 @@ export abstract class SupervisedProc {
 
   async initialize() {
     const timer = setTimeout(() => {
-      const err = new Error('runner initialization timed out');
-      this.init.reject(err);
-      throw err;
+      this.init.reject(new Error('runner initialization timed out'));
     }, this.#opts.initializeTimeout);
-    this.proc!.send({
+    if (!this.proc?.connected) {
+      this.init.reject(new Error('process not connected'));
+      return;
+    }
+    this.proc.send({
       case: 'initializeRequest',
       value: {
         loggerOptions,
@@ -187,7 +192,9 @@ export abstract class SupervisedProc {
     }
     this.#closing = true;
 
-    this.proc!.send({ case: 'shutdownRequest' });
+    if (this.proc?.connected) {
+      this.proc.send({ case: 'shutdownRequest' });
+    }
 
     const timer = setTimeout(() => {
       this.#logger.error('job shutdown is taking too much time');
@@ -203,8 +210,11 @@ export abstract class SupervisedProc {
     if (this.#runningJob) {
       throw new Error('executor already has a running job');
     }
+    if (!this.proc?.connected) {
+      throw new Error('process not connected');
+    }
     this.#runningJob = info;
-    this.proc!.send({ case: 'startJobRequest', value: { runningJob: info } });
+    this.proc.send({ case: 'startJobRequest', value: { runningJob: info } });
   }
 
   private async getChildMemoryUsageMB(): Promise<number> {
@@ -212,8 +222,16 @@ export abstract class SupervisedProc {
     if (!pid) {
       return 0;
     }
-    const stats = await pidusage(pid);
-    return stats.memory / (1024 * 1024); // Convert bytes to MB
+    try {
+      const stats = await pidusage(pid);
+      return stats.memory / (1024 * 1024);
+    } catch (err) {
+      const code = (err as NodeJS.ErrnoException).code;
+      if (code === 'ENOENT' || code === 'ESRCH') {
+        return 0;
+      }
+      throw err;
+    }
   }
 
   private clearTimers() {

package/src/job.ts

@@ -126,7 +126,10 @@ export class JobContext {
     this.#onShutdown = onShutdown;
     this.onParticipantConnected = this.onParticipantConnected.bind(this);
     this.#room.on(RoomEvent.ParticipantConnected, this.onParticipantConnected);
-    this.#logger = log().child({ info: this.#info });
+    this.#logger = log().child({
+      jobId: this.#info.job.id,
+      roomName: this.#info.job.room?.name,
+    });
     this.#inferenceExecutor = inferenceExecutor;
     this._sessionDirectory = path.join(os.tmpdir(), 'livekit-agents', `job-${this.#info.job.id}`);
   }

package/src/llm/chat_context.ts

@@ -189,19 +189,35 @@ export class FunctionCall {
 
   createdAt: number;
 
+  /**
+   * Opaque signature for Gemini thinking mode.
+   * When using Gemini 3+ models with thinking enabled, this signature must be
+   * preserved and returned with function responses to maintain thought context.
+   */
+  thoughtSignature?: string;
+
   constructor(params: {
     callId: string;
     name: string;
     args: string;
     id?: string;
     createdAt?: number;
+    thoughtSignature?: string;
   }) {
-    const { callId, name, args, id = shortuuid('item_'), createdAt = Date.now() } = params;
+    const {
+      callId,
+      name,
+      args,
+      id = shortuuid('item_'),
+      createdAt = Date.now(),
+      thoughtSignature,
+    } = params;
     this.id = id;
     this.callId = callId;
     this.args = args;
     this.name = name;
     this.createdAt = createdAt;
+    this.thoughtSignature = thoughtSignature;
   }
 
   static create(params: {
@@ -210,6 +226,7 @@ export class FunctionCall {
     args: string;
     id?: string;
     createdAt?: number;
+    thoughtSignature?: string;
   }) {
     return new FunctionCall(params);
   }
@@ -224,6 +241,10 @@ export class FunctionCall {
       args: this.args,
     };
 
+    if (this.thoughtSignature) {
+      result.thoughtSignature = this.thoughtSignature;
+    }
+
     if (!excludeTimestamp) {
       result.createdAt = this.createdAt;
     }
@@ -602,7 +623,12 @@ export class ChatContext {
           return false;
         }
       } else if (a.type === 'function_call' && b.type === 'function_call') {
-        if (a.name !== b.name || a.callId !== b.callId || a.args !== b.args) {
+        if (
+          a.name !== b.name ||
+          a.callId !== b.callId ||
+          a.args !== b.args ||
+          a.thoughtSignature !== b.thoughtSignature
+        ) {
           return false;
         }
       } else if (a.type === 'function_call_output' && b.type === 'function_call_output') {

package/src/llm/provider_format/google.ts

@@ -67,13 +67,17 @@ export async function toChatCtx(
         }
       }
     } else if (msg.type === 'function_call') {
-      parts.push({
+      const functionCallPart: Record<string, unknown> = {
         functionCall: {
           id: msg.callId,
           name: msg.name,
           args: JSON.parse(msg.args || '{}'),
         },
-      });
+      };
+      if (msg.thoughtSignature) {
+        functionCallPart.thoughtSignature = msg.thoughtSignature;
+      }
+      parts.push(functionCallPart);
     } else if (msg.type === 'function_call_output') {
       const response = msg.isError ? { error: msg.output } : { output: msg.output };
       parts.push({

package/src/llm/realtime.ts

@@ -26,6 +26,8 @@ export interface GenerationCreatedEvent {
   messageStream: ReadableStream<MessageGeneration>;
   functionStream: ReadableStream<FunctionCall>;
   userInitiated: boolean;
+  /** Response ID for correlating metrics with spans */
+  responseId?: string;
 }
 
 export interface RealtimeModelError {
@@ -63,6 +65,9 @@ export abstract class RealtimeModel {
     return this._capabilities;
   }
 
+  /** The model name/identifier used by this realtime model */
+  abstract get model(): string;
+
   abstract session(): RealtimeSession;
 
   abstract close(): Promise<void>;

package/src/log.ts

@@ -62,6 +62,11 @@ class OtelDestination extends Writable {
  * Enable OTEL logging by reconfiguring the logger with multistream.
  * Uses a custom destination that receives full JSON logs (with msg, level, time).
  *
+ * The base logger level is set to 'debug' so all logs are generated,
+ * while each stream filters to its own level:
+ * - Terminal: user-specified level (default: 'info')
+ * - OTEL/Cloud: always 'debug' to capture all logs for observability
+ *
  * @internal
  */
 export const enableOtelLogging = () => {
@@ -73,11 +78,12 @@ export const enableOtelLogging = () => {
 
   const { pretty, level } = loggerOptions;
 
-  const logLevel = level || 'info';
+  const terminalLevel = level || 'info';
   const streams: { stream: DestinationStream; level: string }[] = [
-    { stream: pretty ? pinoPretty({ colorize: true }) : process.stdout, level: logLevel },
+    { stream: pretty ? pinoPretty({ colorize: true }) : process.stdout, level: terminalLevel },
     { stream: new OtelDestination(), level: 'debug' },
   ];
 
-  logger = pino({ level: logLevel }, multistream(streams));
+  // Base level must be 'debug' to generate all logs; each stream filters independently
+  logger = pino({ level: 'debug' }, multistream(streams));
 };

package/src/stream/stream_channel.ts

@@ -8,25 +8,33 @@ export interface StreamChannel<T> {
   write(chunk: T): Promise<void>;
   close(): Promise<void>;
   stream(): ReadableStream<T>;
+  readonly closed: boolean;
 }
 
 export function createStreamChannel<T>(): StreamChannel<T> {
   const transform = new IdentityTransform<T>();
   const writer = transform.writable.getWriter();
+  let isClosed = false;
 
   return {
     write: (chunk: T) => writer.write(chunk),
     stream: () => transform.readable,
     close: async () => {
       try {
-        return await writer.close();
+        const result = await writer.close();
+        isClosed = true;
+        return result;
       } catch (e) {
         if (e instanceof Error && e.name === 'TypeError') {
           // Ignore error if the stream is already closed
+          isClosed = true;
           return;
         }
         throw e;
       }
     },
+    get closed() {
+      return isClosed;
+    },
   };
 }

package/src/telemetry/otel_http_exporter.ts

@@ -122,16 +122,20 @@ export class SimpleOTLPHttpLogExporter {
         }))
       : [];
 
-    const logRecords = records.map((record) => ({
-      timeUnixNano: String(BigInt(Math.floor(record.timestampMs * 1_000_000))),
-      observedTimeUnixNano: String(BigInt(Date.now()) * BigInt(1_000_000)),
-      severityNumber: record.severityNumber ?? SeverityNumber.UNSPECIFIED,
-      severityText: record.severityText ?? 'unspecified',
-      body: { stringValue: record.body },
-      attributes: this.convertAttributes(record.attributes),
-      traceId: '',
-      spanId: '',
-    }));
+    const logRecords = records.map((record) => {
+      // Ensure timestampMs is a valid number, fallback to current time if NaN/undefined
+      const timestampMs = Number.isFinite(record.timestampMs) ? record.timestampMs : Date.now();
+      return {
+        timeUnixNano: String(BigInt(Math.floor(timestampMs * 1_000_000))),
+        observedTimeUnixNano: String(BigInt(Date.now()) * BigInt(1_000_000)),
+        severityNumber: record.severityNumber ?? SeverityNumber.UNSPECIFIED,
+        severityText: record.severityText ?? 'unspecified',
+        body: { stringValue: record.body },
+        attributes: this.convertAttributes(record.attributes),
+        traceId: '',
+        spanId: '',
+      };
+    });
 
     return {
       resourceLogs: [

package/src/telemetry/traces.ts

@@ -457,9 +457,15 @@ export async function uploadSessionReport(options: {
   // get reordered by the dashboard
   let lastTimestamp = 0;
   for (const item of report.chatHistory.items) {
+    // Skip null/undefined items
+    if (!item) continue;
+
     // Ensure monotonically increasing timestamps for proper ordering
     // Add 0.001ms (1 microsecond) offset when timestamps collide
-    let itemTimestamp = item.createdAt;
+    // Also handle undefined/NaN timestamps from realtime mode (defensive)
+    const hasValidTimestamp = Number.isFinite(item.createdAt);
+    let itemTimestamp = hasValidTimestamp ? item.createdAt : Date.now();
+
     if (itemTimestamp <= lastTimestamp) {
       itemTimestamp = lastTimestamp + 0.001; // Add 1 microsecond
     }
@@ -482,6 +488,7 @@ export async function uploadSessionReport(options: {
       severityText,
     });
   }
+
   await logExporter.export(logRecords);
 
   const apiKey = process.env.LIVEKIT_API_KEY;
@@ -574,13 +581,30 @@ export async function uploadSessionReport(options: {
         }
 
         if (res.statusCode && res.statusCode >= 400) {
-          reject(
-            new Error(`Failed to upload session report: ${res.statusCode} ${res.statusMessage}`),
-          );
+          // Read response body for error details
+          let body = '';
+          res.on('data', (chunk) => {
+            body += chunk.toString();
+          });
+          res.on('error', (readErr) => {
+            reject(
+              new Error(
+                `Failed to upload session report: ${res.statusCode} ${res.statusMessage} (body read error: ${readErr.message})`,
+              ),
+            );
+          });
+          res.on('end', () => {
+            reject(
+              new Error(
+                `Failed to upload session report: ${res.statusCode} ${res.statusMessage} - ${body}`,
+              ),
+            );
+          });
           return;
         }
 
         res.resume(); // Drain the response
+        res.on('error', (readErr) => reject(new Error(`Response read error: ${readErr.message}`)));
         res.on('end', () => resolve());
       },
     );

package/src/voice/agent_activity.ts

@@ -37,7 +37,7 @@ import type {
 } from '../metrics/base.js';
 import { DeferredReadableStream } from '../stream/deferred_stream.js';
 import { STT, type STTError, type SpeechEvent } from '../stt/stt.js';
-import { traceTypes, tracer } from '../telemetry/index.js';
+import { recordRealtimeMetrics, traceTypes, tracer } from '../telemetry/index.js';
 import { splitWords } from '../tokenize/basic/word.js';
 import { TTS, type TTSError } from '../tts/tts.js';
 import { Future, Task, cancelAndWait, waitFor } from '../utils.js';
@@ -91,6 +91,7 @@ export class AgentActivity implements RecognitionHooks {
   private started = false;
   private audioRecognition?: AudioRecognition;
   private realtimeSession?: RealtimeSession;
+  private realtimeSpans?: Map<string, Span>; // Maps response_id to OTEL span for metrics recording
   private turnDetectionMode?: Exclude<TurnDetectionMode, _TurnDetector>;
   private logger = log();
   private _draining = false;
@@ -218,6 +219,7 @@ export class AgentActivity implements RecognitionHooks {
 
       if (this.llm instanceof RealtimeModel) {
         this.realtimeSession = this.llm.session();
+        this.realtimeSpans = new Map<string, Span>();
         this.realtimeSession.on('generation_created', (ev) => this.onGenerationCreated(ev));
         this.realtimeSession.on('input_speech_started', (ev) => this.onInputSpeechStarted(ev));
         this.realtimeSession.on('input_speech_stopped', (ev) => this.onInputSpeechStopped(ev));
@@ -505,6 +507,16 @@ export class AgentActivity implements RecognitionHooks {
     if (speechHandle && (ev.type === 'llm_metrics' || ev.type === 'tts_metrics')) {
       ev.speechId = speechHandle.id;
     }
+
+    // Record realtime metrics on the associated span (if available)
+    if (ev.type === 'realtime_model_metrics' && this.realtimeSpans) {
+      const span = this.realtimeSpans.get(ev.requestId);
+      if (span) {
+        recordRealtimeMetrics(span, ev);
+        this.realtimeSpans.delete(ev.requestId);
+      }
+    }
+
     this.agentSession.emit(
       AgentSessionEventTypes.MetricsCollected,
       createMetricsCollectedEvent({ metrics: ev }),
@@ -1722,6 +1734,12 @@ export class AgentActivity implements RecognitionHooks {
       throw new Error('llm is not a realtime model');
     }
 
+    // Store span for metrics recording when they arrive later
+    span.setAttribute(traceTypes.ATTR_GEN_AI_REQUEST_MODEL, this.llm.model);
+    if (this.realtimeSpans && ev.responseId) {
+      this.realtimeSpans.set(ev.responseId, span);
+    }
+
     this.logger.debug(
       { speech_id: speechHandle.id, stepIndex: speechHandle.numSteps },
       'realtime generation started',
@@ -2009,7 +2027,13 @@ export class AgentActivity implements RecognitionHooks {
 
     await executeToolsTask.result;
 
-    if (toolOutput.output.length === 0) return;
+    if (toolOutput.output.length === 0) {
+      // return to listening state for thinking-only turns (no audio output, no tools)
+      if (!speechHandle.interrupted) {
+        this.agentSession._updateAgentState('listening');
+      }
+      return;
+    }
 
     // important: no agent ouput should be used after this point
     const { maxToolSteps } = this.agentSession.options;
@@ -2274,6 +2298,7 @@ export class AgentActivity implements RecognitionHooks {
       }
 
       this.detachAudioInput();
+      this.realtimeSpans?.clear();
       await this.realtimeSession?.close();
       await this.audioRecognition?.close();
       await this._mainTask?.cancelAndWait();

package/src/voice/generation.ts

@@ -442,6 +442,8 @@ export function performLLMInference(
                 callId: `${data.id}/fnc_${data.generatedToolCalls.length}`,
                 name: tool.name,
                 args: tool.args,
+                // Preserve thought signature for Gemini 3+ thinking mode
+                thoughtSignature: tool.thoughtSignature,
               });
 
               data.generatedToolCalls.push(toolCall);

package/src/llm/__snapshots__/utils.test.ts.snap

@@ -1,65 +0,0 @@
-// Vitest Snapshot v1, https://vitest.dev/guide/snapshot.html
-
-exports[`oaiParams > common types 1`] = `
-{
-  "additionalProperties": false,
-  "properties": {
-    "address": {
-      "additionalProperties": false,
-      "properties": {
-        "city": {
-          "type": "string",
-        },
-        "state": {
-          "type": "string",
-        },
-        "street": {
-          "type": "string",
-        },
-        "zip": {
-          "type": "string",
-        },
-      },
-      "required": [
-        "street",
-        "city",
-        "state",
-        "zip",
-      ],
-      "type": "object",
-    },
-    "age": {
-      "type": "number",
-    },
-    "favoriteColor": {
-      "enum": [
-        "red",
-        "green",
-        "blue",
-      ],
-      "type": "string",
-    },
-    "hobbies": {
-      "items": {
-        "type": "string",
-      },
-      "type": "array",
-    },
-    "isStudent": {
-      "type": "boolean",
-    },
-    "name": {
-      "type": "string",
-    },
-  },
-  "required": [
-    "name",
-    "age",
-    "isStudent",
-    "hobbies",
-    "favoriteColor",
-    "address",
-  ],
-  "type": "object",
-}
-`;