@livekit/agents 0.0.0-20260120144724

package/src/voice/report.ts

@@ -0,0 +1,100 @@
+// SPDX-FileCopyrightText: 2025 LiveKit, Inc.
+//
+// SPDX-License-Identifier: Apache-2.0
+import type { ChatContext } from '../llm/chat_context.js';
+import type { VoiceOptions } from './agent_session.js';
+import type { AgentEvent } from './events.js';
+
+export interface SessionReport {
+  jobId: string;
+  roomId: string;
+  room: string;
+  options: VoiceOptions;
+  events: AgentEvent[];
+  chatHistory: ChatContext;
+  enableRecording: boolean;
+  /** Timestamp when the session started (milliseconds) */
+  startedAt: number;
+  /** Timestamp when the session report was created (milliseconds), typically at the end of the session */
+  timestamp: number;
+  /** Path to the audio recording file (if recording was enabled) */
+  audioRecordingPath?: string;
+  /** Timestamp when the audio recording started (milliseconds) */
+  audioRecordingStartedAt?: number;
+  /** Duration of the session in milliseconds */
+  duration?: number;
+}
+
+export interface SessionReportOptions {
+  jobId: string;
+  roomId: string;
+  room: string;
+  options: VoiceOptions;
+  events: AgentEvent[];
+  chatHistory: ChatContext;
+  enableRecording?: boolean;
+  /** Timestamp when the session started (milliseconds) */
+  startedAt?: number;
+  /** Timestamp when the session report was created (milliseconds) */
+  timestamp?: number;
+  /** Path to the audio recording file (if recording was enabled) */
+  audioRecordingPath?: string;
+  /** Timestamp when the audio recording started (milliseconds) */
+  audioRecordingStartedAt?: number;
+}
+
+export function createSessionReport(opts: SessionReportOptions): SessionReport {
+  const timestamp = opts.timestamp ?? Date.now();
+  const audioRecordingStartedAt = opts.audioRecordingStartedAt;
+
+  return {
+    jobId: opts.jobId,
+    roomId: opts.roomId,
+    room: opts.room,
+    options: opts.options,
+    events: opts.events,
+    chatHistory: opts.chatHistory,
+    enableRecording: opts.enableRecording ?? false,
+    startedAt: opts.startedAt ?? Date.now(),
+    timestamp,
+    audioRecordingPath: opts.audioRecordingPath,
+    audioRecordingStartedAt,
+    duration:
+      audioRecordingStartedAt !== undefined ? timestamp - audioRecordingStartedAt : undefined,
+  };
+}
+
+//   - header: protobuf MetricsRecordingHeader (room_id, duration, start_time)
+//   - chat_history: JSON serialized chat history (use sessionReportToJSON)
+//   - audio: audio recording file if available (ogg format)
+//   - Uploads to LiveKit Cloud observability endpoint with JWT auth
+export function sessionReportToJSON(report: SessionReport): Record<string, unknown> {
+  const events: Record<string, unknown>[] = [];
+
+  for (const event of report.events) {
+    if (event.type === 'metrics_collected') {
+      continue; // metrics are too noisy, Cloud is using the chat_history as the source of truth
+    }
+
+    events.push({ ...event });
+  }
+
+  return {
+    job_id: report.jobId,
+    room_id: report.roomId,
+    room: report.room,
+    events,
+    options: {
+      allow_interruptions: report.options.allowInterruptions,
+      discard_audio_if_uninterruptible: report.options.discardAudioIfUninterruptible,
+      min_interruption_duration: report.options.minInterruptionDuration,
+      min_interruption_words: report.options.minInterruptionWords,
+      min_endpointing_delay: report.options.minEndpointingDelay,
+      max_endpointing_delay: report.options.maxEndpointingDelay,
+      max_tool_steps: report.options.maxToolSteps,
+    },
+    chat_history: report.chatHistory.toJSON({ excludeTimestamp: false }),
+    enable_user_data_training: report.enableRecording,
+    timestamp: report.timestamp,
+  };
+}

package/src/voice/room_io/_input.ts

@@ -0,0 +1,162 @@
+// SPDX-FileCopyrightText: 2025 LiveKit, Inc.
+//
+// SPDX-License-Identifier: Apache-2.0
+import type { AudioFrame } from '@livekit/rtc-node';
+import {
+  AudioStream,
+  type NoiseCancellationOptions,
+  RemoteParticipant,
+  type RemoteTrack,
+  type RemoteTrackPublication,
+  type Room,
+  RoomEvent,
+  TrackSource,
+} from '@livekit/rtc-node';
+import type { ReadableStream } from 'node:stream/web';
+import { log } from '../../log.js';
+import { resampleStream } from '../../utils.js';
+import { AudioInput } from '../io.js';
+
+export class ParticipantAudioInputStream extends AudioInput {
+  private room: Room;
+  private sampleRate: number;
+  private numChannels: number;
+  private noiseCancellation?: NoiseCancellationOptions;
+  private publication: RemoteTrackPublication | null = null;
+  private participantIdentity: string | null = null;
+  private logger = log();
+  constructor({
+    room,
+    sampleRate,
+    numChannels,
+    noiseCancellation,
+  }: {
+    room: Room;
+    sampleRate: number;
+    numChannels: number;
+    noiseCancellation?: NoiseCancellationOptions;
+  }) {
+    super();
+    this.room = room;
+    this.sampleRate = sampleRate;
+    this.numChannels = numChannels;
+    this.noiseCancellation = noiseCancellation;
+
+    this.room.on(RoomEvent.TrackSubscribed, this.onTrackSubscribed);
+    this.room.on(RoomEvent.TrackUnpublished, this.onTrackUnpublished);
+  }
+
+  setParticipant(participant: RemoteParticipant | string | null) {
+    this.logger.debug({ participant }, 'setting participant audio input');
+    const participantIdentity =
+      participant instanceof RemoteParticipant ? participant.identity : participant;
+
+    if (this.participantIdentity === participantIdentity) {
+      return;
+    }
+    this.participantIdentity = participantIdentity;
+    this.closeStream();
+
+    if (!participantIdentity) {
+      return;
+    }
+
+    const participantValue =
+      participant instanceof RemoteParticipant
+        ? participant
+        : this.room.remoteParticipants.get(participantIdentity);
+
+    // Convert Map iterator to array for Pino serialization
+    const trackPublicationsArray = Array.from(participantValue?.trackPublications.values() ?? []);
+
+    this.logger.info(
+      {
+        participantValue: participantValue?.identity,
+        trackPublications: trackPublicationsArray,
+        lengthOfTrackPublications: trackPublicationsArray.length,
+      },
+      'participantValue.trackPublications',
+    );
+    // We need to check if the participant has a microphone track and subscribe to it
+    // in case we miss the tracksubscribed event
+    if (participantValue) {
+      for (const publication of participantValue.trackPublications.values()) {
+        if (publication.track && publication.source === TrackSource.SOURCE_MICROPHONE) {
+          this.onTrackSubscribed(publication.track, publication, participantValue);
+          break;
+        }
+      }
+    }
+  }
+
+  private onTrackUnpublished = (
+    publication: RemoteTrackPublication,
+    participant: RemoteParticipant,
+  ) => {
+    if (
+      this.publication?.sid !== publication.sid ||
+      participant.identity !== this.participantIdentity
+    ) {
+      return;
+    }
+    this.closeStream();
+
+    // subscribe to the first available track
+    for (const publication of participant.trackPublications.values()) {
+      if (
+        publication.track &&
+        this.onTrackSubscribed(publication.track, publication, participant)
+      ) {
+        return;
+      }
+    }
+  };
+
+  private closeStream() {
+    if (this.deferredStream.isSourceSet) {
+      this.deferredStream.detachSource();
+    }
+    this.publication = null;
+  }
+
+  private onTrackSubscribed = (
+    track: RemoteTrack,
+    publication: RemoteTrackPublication,
+    participant: RemoteParticipant,
+  ): boolean => {
+    this.logger.debug({ participant: participant.identity }, 'onTrackSubscribed in _input');
+    if (
+      this.participantIdentity !== participant.identity ||
+      publication.source !== TrackSource.SOURCE_MICROPHONE ||
+      (this.publication && this.publication.sid === publication.sid)
+    ) {
+      return false;
+    }
+    this.closeStream();
+    this.publication = publication;
+    this.deferredStream.setSource(
+      resampleStream({
+        stream: this.createStream(track),
+        outputRate: this.sampleRate,
+      }),
+    );
+    return true;
+  };
+
+  private createStream(track: RemoteTrack): ReadableStream<AudioFrame> {
+    return new AudioStream(track, {
+      sampleRate: this.sampleRate,
+      numChannels: this.numChannels,
+      noiseCancellation: this.noiseCancellation,
+      // TODO(AJS-269): resolve compatibility issue with node-sdk to remove the forced type casting
+    }) as unknown as ReadableStream<AudioFrame>;
+  }
+
+  async close() {
+    this.room.off(RoomEvent.TrackSubscribed, this.onTrackSubscribed);
+    this.room.off(RoomEvent.TrackUnpublished, this.onTrackUnpublished);
+    this.closeStream();
+    // Ignore errors - stream may be locked by RecorderIO or already cancelled
+    await this.deferredStream.stream.cancel().catch(() => {});
+  }
+}

package/src/voice/room_io/_output.ts

@@ -0,0 +1,439 @@
+// SPDX-FileCopyrightText: 2025 LiveKit, Inc.
+//
+// SPDX-License-Identifier: Apache-2.0
+import type { RemoteParticipant } from '@livekit/rtc-node';
+import {
+  type AudioFrame,
+  AudioSource,
+  LocalAudioTrack,
+  type LocalTrackPublication,
+  type Participant,
+  type RemoteTrackPublication,
+  type Room,
+  RoomEvent,
+  type TextStreamWriter,
+  TrackPublishOptions,
+  TrackSource,
+} from '@livekit/rtc-node';
+import {
+  ATTRIBUTE_TRANSCRIPTION_FINAL,
+  ATTRIBUTE_TRANSCRIPTION_SEGMENT_ID,
+  ATTRIBUTE_TRANSCRIPTION_TRACK_ID,
+  TOPIC_TRANSCRIPTION,
+} from '../../constants.js';
+import { log } from '../../log.js';
+import { Future, Task, shortuuid } from '../../utils.js';
+import { AudioOutput, TextOutput } from '../io.js';
+import { findMicrophoneTrackId } from '../transcription/index.js';
+
+abstract class BaseParticipantTranscriptionOutput extends TextOutput {
+  protected room: Room;
+  protected isDeltaStream: boolean;
+  protected participantIdentity: string | null = null;
+  protected trackId?: string;
+  protected capturing: boolean = false;
+  protected latestText: string = '';
+  protected currentId: string = this.generateCurrentId();
+  protected logger = log();
+
+  constructor(room: Room, isDeltaStream: boolean, participant: Participant | string | null) {
+    super();
+    this.room = room;
+    this.isDeltaStream = isDeltaStream;
+
+    this.room.on(RoomEvent.TrackPublished, this.onTrackPublished);
+    this.room.on(RoomEvent.LocalTrackPublished, this.onLocalTrackPublished);
+
+    this.setParticipant(participant);
+  }
+
+  setParticipant(participant: Participant | string | null) {
+    if (typeof participant === 'string' || participant === null) {
+      this.participantIdentity = participant;
+    } else {
+      this.participantIdentity = participant.identity;
+    }
+
+    if (!this.participantIdentity) {
+      return;
+    }
+
+    try {
+      this.trackId = findMicrophoneTrackId(this.room, this.participantIdentity);
+    } catch (error) {
+      // track id is optional for TextStream when audio is not published
+    }
+
+    this.flush();
+    this.resetState();
+  }
+
+  protected onTrackPublished = (track: RemoteTrackPublication, participant: RemoteParticipant) => {
+    if (
+      !this.participantIdentity ||
+      participant.identity !== this.participantIdentity ||
+      track.source !== TrackSource.SOURCE_MICROPHONE
+    ) {
+      return;
+    }
+
+    this.trackId = track.sid;
+  };
+
+  protected onLocalTrackPublished = (track: LocalTrackPublication) => {
+    if (
+      !this.participantIdentity ||
+      this.participantIdentity !== this.room.localParticipant?.identity ||
+      track.source !== TrackSource.SOURCE_MICROPHONE
+    ) {
+      return;
+    }
+
+    this.trackId = track.sid;
+  };
+
+  protected generateCurrentId(): string {
+    return shortuuid('SG_');
+  }
+
+  protected resetState() {
+    this.currentId = this.generateCurrentId();
+    this.capturing = false;
+    this.latestText = '';
+  }
+
+  async captureText(text: string) {
+    if (!this.participantIdentity) {
+      return;
+    }
+
+    this.latestText = text;
+    await this.handleCaptureText(text);
+  }
+
+  flush() {
+    if (!this.participantIdentity || !this.capturing) {
+      return;
+    }
+
+    this.capturing = false;
+    this.handleFlush();
+  }
+
+  protected abstract handleCaptureText(text: string): Promise<void>;
+  protected abstract handleFlush(): void;
+}
+
+export class ParticipantTranscriptionOutput extends BaseParticipantTranscriptionOutput {
+  private writer: TextStreamWriter | null = null;
+  private flushTask: Task<void> | null = null;
+
+  protected async handleCaptureText(text: string): Promise<void> {
+    if (this.flushTask && !this.flushTask.done) {
+      await this.flushTask.result;
+    }
+
+    if (!this.capturing) {
+      this.resetState();
+      this.capturing = true;
+    }
+
+    try {
+      if (this.room.isConnected) {
+        if (this.isDeltaStream) {
+          // reuse the existing writer
+          if (this.writer === null) {
+            this.writer = await this.createTextWriter();
+          }
+          await this.writer.write(text);
+        } else {
+          const tmpWriter = await this.createTextWriter();
+          await tmpWriter.write(text);
+          await tmpWriter.close();
+        }
+      }
+    } catch (error) {
+      this.logger.error(error, 'failed to publish transcription');
+    }
+  }
+
+  protected handleFlush() {
+    const currWriter = this.writer;
+    this.writer = null;
+    this.flushTask = Task.from((controller) => this.flushTaskImpl(currWriter, controller.signal));
+  }
+
+  private async createTextWriter(attributes?: Record<string, string>): Promise<TextStreamWriter> {
+    if (!this.participantIdentity) {
+      throw new Error('participantIdentity not found');
+    }
+
+    if (!this.room.localParticipant) {
+      throw new Error('localParticipant not found');
+    }
+
+    if (!attributes) {
+      attributes = {
+        [ATTRIBUTE_TRANSCRIPTION_FINAL]: 'false',
+      };
+      if (this.trackId) {
+        attributes[ATTRIBUTE_TRANSCRIPTION_TRACK_ID] = this.trackId;
+      }
+    }
+    attributes[ATTRIBUTE_TRANSCRIPTION_SEGMENT_ID] = this.currentId;
+
+    return await this.room.localParticipant.streamText({
+      topic: TOPIC_TRANSCRIPTION,
+      senderIdentity: this.participantIdentity,
+      attributes,
+    });
+  }
+
+  private async flushTaskImpl(writer: TextStreamWriter | null, signal: AbortSignal): Promise<void> {
+    const attributes: Record<string, string> = {
+      [ATTRIBUTE_TRANSCRIPTION_FINAL]: 'true',
+    };
+    if (this.trackId) {
+      attributes[ATTRIBUTE_TRANSCRIPTION_TRACK_ID] = this.trackId;
+    }
+
+    const abortPromise = new Promise<void>((resolve) => {
+      signal.addEventListener('abort', () => resolve());
+    });
+
+    try {
+      if (this.room.isConnected) {
+        if (this.isDeltaStream) {
+          if (writer) {
+            await Promise.race([writer.close(), abortPromise]);
+          }
+        } else {
+          const tmpWriter = await Promise.race([this.createTextWriter(attributes), abortPromise]);
+          if (signal.aborted || !tmpWriter) {
+            return;
+          }
+          await Promise.race([tmpWriter.write(this.latestText), abortPromise]);
+          if (signal.aborted) {
+            return;
+          }
+          await Promise.race([tmpWriter.close(), abortPromise]);
+        }
+      }
+    } catch (error) {
+      this.logger.error(error, 'failed to publish transcription');
+    }
+  }
+}
+
+export class ParticipantLegacyTranscriptionOutput extends BaseParticipantTranscriptionOutput {
+  private pushedText: string = '';
+  private flushTask: Promise<void> | null = null;
+
+  protected async handleCaptureText(text: string): Promise<void> {
+    if (!this.trackId) {
+      return;
+    }
+
+    if (this.flushTask) {
+      await this.flushTask;
+    }
+
+    if (!this.capturing) {
+      this.resetState();
+      this.capturing = true;
+    }
+
+    if (this.isDeltaStream) {
+      this.pushedText += text;
+    } else {
+      this.pushedText = text;
+    }
+
+    await this.publishTranscription(this.currentId, this.pushedText, false);
+  }
+
+  protected handleFlush() {
+    if (!this.trackId) {
+      return;
+    }
+
+    this.flushTask = this.publishTranscription(this.currentId, this.pushedText, true);
+    this.resetState();
+  }
+
+  async publishTranscription(id: string, text: string, final: boolean, signal?: AbortSignal) {
+    if (!this.participantIdentity || !this.trackId) {
+      return;
+    }
+
+    try {
+      if (this.room.isConnected) {
+        if (signal?.aborted) {
+          return;
+        }
+
+        await this.room.localParticipant?.publishTranscription({
+          participantIdentity: this.participantIdentity,
+          trackSid: this.trackId,
+          segments: [{ id, text, final, startTime: BigInt(0), endTime: BigInt(0), language: '' }],
+        });
+      }
+    } catch (error) {
+      this.logger.error(error, 'failed to publish transcription');
+    }
+  }
+
+  protected resetState() {
+    super.resetState();
+    this.pushedText = '';
+  }
+}
+
+export class ParalellTextOutput extends TextOutput {
+  /** @internal */
+  _sinks: TextOutput[];
+
+  constructor(sinks: TextOutput[], nextInChain?: TextOutput) {
+    super(nextInChain);
+    this._sinks = sinks;
+  }
+
+  async captureText(text: string) {
+    await Promise.all(this._sinks.map((sink) => sink.captureText(text)));
+  }
+
+  flush() {
+    for (const sink of this._sinks) {
+      sink.flush();
+    }
+  }
+}
+
+export interface AudioOutputOptions {
+  sampleRate: number;
+  numChannels: number;
+  trackPublishOptions: TrackPublishOptions;
+  queueSizeMs?: number;
+}
+export class ParticipantAudioOutput extends AudioOutput {
+  private room: Room;
+  private options: AudioOutputOptions;
+  private audioSource: AudioSource;
+  private publication?: LocalTrackPublication;
+  private flushTask?: Task<void>;
+
+  /** Duration of audio pushed to the source, in seconds */
+  private pushedDuration: number = 0;
+  private startedFuture: Future<void> = new Future();
+  private interruptedFuture: Future<void> = new Future();
+
+  constructor(room: Room, options: AudioOutputOptions) {
+    super(options.sampleRate, undefined, { pause: true });
+    this.room = room;
+    this.options = options;
+    this.audioSource = new AudioSource(options.sampleRate, options.numChannels);
+  }
+
+  get subscribed(): boolean {
+    return this.startedFuture.done;
+  }
+
+  async start(signal: AbortSignal): Promise<void> {
+    await this.publishTrack(signal);
+  }
+
+  async captureFrame(frame: AudioFrame): Promise<void> {
+    await this.startedFuture.await;
+
+    super.captureFrame(frame);
+
+    // TODO(AJS-102): use frame.durationMs once available in rtc-node
+    this.pushedDuration += frame.samplesPerChannel / frame.sampleRate;
+    await this.audioSource.captureFrame(frame);
+  }
+
+  private async waitForPlayoutTask(abortController: AbortController): Promise<void> {
+    const abortFuture = new Future<boolean>();
+
+    const resolveAbort = () => {
+      if (!abortFuture.done) abortFuture.resolve(true);
+    };
+
+    abortController.signal.addEventListener('abort', resolveAbort);
+
+    this.audioSource.waitForPlayout().finally(() => {
+      abortController.signal.removeEventListener('abort', resolveAbort);
+      if (!abortFuture.done) abortFuture.resolve(false);
+    });
+
+    const interrupted = await Promise.race([
+      abortFuture.await,
+      this.interruptedFuture.await.then(() => true),
+    ]);
+
+    let pushedDuration = this.pushedDuration;
+
+    if (interrupted) {
+      // Calculate actual played duration accounting for queued audio
+      // Note: queuedDuration is in milliseconds, pushedDuration is in seconds
+      pushedDuration = Math.max(this.pushedDuration - this.audioSource.queuedDuration / 1000, 0);
+      this.audioSource.clearQueue();
+    }
+
+    this.pushedDuration = 0;
+    this.interruptedFuture = new Future();
+    this.onPlaybackFinished({
+      playbackPosition: pushedDuration,
+      interrupted,
+    });
+  }
+
+  /**
+   * Flush any buffered audio, marking the current playback/segment as complete
+   */
+  flush(): void {
+    super.flush();
+
+    if (!this.pushedDuration) {
+      return;
+    }
+
+    if (this.flushTask && !this.flushTask.done) {
+      this.logger.error('flush called while playback is in progress');
+      this.flushTask.cancel();
+    }
+
+    this.flushTask = Task.from((controller) => this.waitForPlayoutTask(controller));
+  }
+
+  clearBuffer(): void {
+    if (!this.pushedDuration) {
+      return;
+    }
+
+    this.interruptedFuture.resolve();
+  }
+
+  private async publishTrack(signal: AbortSignal) {
+    const track = LocalAudioTrack.createAudioTrack('roomio_audio', this.audioSource);
+    this.publication = await this.room.localParticipant?.publishTrack(
+      track,
+      new TrackPublishOptions({ source: TrackSource.SOURCE_MICROPHONE }),
+    );
+
+    if (signal.aborted) {
+      return;
+    }
+
+    await this.publication?.waitForSubscription();
+
+    if (!this.startedFuture.done) {
+      this.startedFuture.resolve();
+    }
+  }
+
+  async close() {
+    // TODO(AJS-106): add republish track
+    await this.audioSource.close();
+  }
+}

package/src/voice/room_io/index.ts

@@ -0,0 +1,5 @@
+// SPDX-FileCopyrightText: 2025 LiveKit, Inc.
+//
+// SPDX-License-Identifier: Apache-2.0
+export * from './_output.js';
+export * from './room_io.js';