@livekit/agents 1.1.0 → 1.2.0

package/dist/voice/audio_recognition_span.test.js

@@ -0,0 +1,299 @@
+import { ParticipantKind } from "@livekit/rtc-node";
+import { ROOT_CONTEXT, context as otelContext, trace } from "@opentelemetry/api";
+import {
+  InMemorySpanExporter,
+  SimpleSpanProcessor
+} from "@opentelemetry/sdk-trace-base";
+import { NodeTracerProvider } from "@opentelemetry/sdk-trace-node";
+import { ReadableStream } from "node:stream/web";
+import { describe, expect, it, vi } from "vitest";
+import { ChatContext } from "../llm/chat_context.js";
+import { initializeLogger } from "../log.js";
+import { SpeechEventType } from "../stt/stt.js";
+import { setTracerProvider, tracer } from "../telemetry/index.js";
+import { VAD, VADEventType } from "../vad.js";
+import { AgentSession } from "./agent_session.js";
+import {
+  AudioRecognition
+} from "./audio_recognition.js";
+function setupInMemoryTracing() {
+  const exporter = new InMemorySpanExporter();
+  const provider = new NodeTracerProvider();
+  provider.addSpanProcessor(new SimpleSpanProcessor(exporter));
+  provider.register();
+  setTracerProvider(provider);
+  return { exporter };
+}
+function spanByName(spans, name) {
+  return spans.find((s) => s.name === name);
+}
+function createFakeSession(rootSpanContext = ROOT_CONTEXT) {
+  return {
+    _agentState: "listening",
+    _roomIO: {
+      linkedParticipant: { sid: "p3", identity: "charlie", kind: ParticipantKind.AGENT }
+    },
+    _setUserAwayTimer: vi.fn(),
+    _cancelUserAwayTimer: vi.fn(),
+    _userSpeakingSpan: void 0,
+    _userState: "listening",
+    emit: vi.fn(),
+    rootSpanContext
+  };
+}
+class FakeVADStream extends Object {
+  // We intentionally avoid extending the real VADStream (it is not exported as a value in JS output
+  // in some bundling contexts). Instead we emulate the async iterator shape used by AudioRecognition.
+  events;
+  idx = 0;
+  constructor(events) {
+    super();
+    this.events = events;
+  }
+  updateInputStream() {
+  }
+  detachInputStream() {
+  }
+  close() {
+  }
+  [Symbol.asyncIterator]() {
+    return this;
+  }
+  async next() {
+    if (this.idx >= this.events.length) {
+      return { done: true, value: void 0 };
+    }
+    const value = this.events[this.idx++];
+    return { done: false, value };
+  }
+}
+class FakeVAD extends VAD {
+  label = "fake-vad";
+  events;
+  constructor(events) {
+    super({ updateInterval: 1 });
+    this.events = events;
+  }
+  stream() {
+    return new FakeVADStream(this.events);
+  }
+}
+const alwaysTrueTurnDetector = {
+  model: "test-turn-detector",
+  provider: "test-provider",
+  supportsLanguage: async () => true,
+  unlikelyThreshold: async () => void 0,
+  predictEndOfTurn: async () => 1
+};
+describe("AudioRecognition user_turn span parity", () => {
+  initializeLogger({ pretty: false, level: "silent" });
+  it("creates user_turn and parents eou_detection under it (stt mode)", async () => {
+    const { exporter } = setupInMemoryTracing();
+    const hooks = {
+      onInterruption: vi.fn(),
+      onStartOfSpeech: vi.fn(),
+      onVADInferenceDone: vi.fn(),
+      onEndOfSpeech: vi.fn(),
+      onInterimTranscript: vi.fn(),
+      onFinalTranscript: vi.fn(),
+      onPreemptiveGeneration: vi.fn(),
+      retrieveChatCtx: () => ChatContext.empty(),
+      onEndOfTurn: vi.fn(async () => true)
+    };
+    const sttEvents = [
+      { type: SpeechEventType.START_OF_SPEECH },
+      {
+        type: SpeechEventType.FINAL_TRANSCRIPT,
+        alternatives: [
+          {
+            language: "en",
+            text: "hello",
+            startTime: 0,
+            endTime: 0,
+            confidence: 0.9
+          }
+        ]
+      },
+      { type: SpeechEventType.END_OF_SPEECH }
+    ];
+    const sttNode = async () => new ReadableStream({
+      start(controller) {
+        for (const ev of sttEvents) controller.enqueue(ev);
+        controller.close();
+      }
+    });
+    const ar = new AudioRecognition({
+      recognitionHooks: hooks,
+      stt: sttNode,
+      vad: void 0,
+      turnDetector: alwaysTrueTurnDetector,
+      turnDetectionMode: "stt",
+      minEndpointingDelay: 0,
+      maxEndpointingDelay: 0,
+      sttModel: "deepgram-nova2",
+      sttProvider: "deepgram",
+      getLinkedParticipant: () => ({ sid: "p1", identity: "bob", kind: ParticipantKind.AGENT })
+    });
+    await ar.start();
+    await new Promise((r) => setTimeout(r, 20));
+    await ar.close();
+    const spans = exporter.getFinishedSpans();
+    const userTurn = spanByName(spans, "user_turn");
+    const eou = spanByName(spans, "eou_detection");
+    expect(userTurn, "user_turn span missing").toBeTruthy();
+    expect(eou, "eou_detection span missing").toBeTruthy();
+    if (!userTurn || !eou) {
+      throw new Error("expected user_turn and eou_detection spans");
+    }
+    expect(eou.parentSpanId).toBe(userTurn.spanContext().spanId);
+    expect(userTurn.attributes["lk.participant_id"]).toBe("p1");
+    expect(userTurn.attributes["lk.participant_identity"]).toBe("bob");
+    expect(userTurn.attributes["lk.participant_kind"]).toBe("AGENT");
+    expect(userTurn.attributes["gen_ai.request.model"]).toBe("deepgram-nova2");
+    expect(userTurn.attributes["gen_ai.provider.name"]).toBe("deepgram");
+    expect(userTurn.attributes["lk.user_transcript"]).toContain("hello");
+    expect(userTurn.attributes["lk.transcript_confidence"]).toBeGreaterThan(0);
+  });
+  it("creates user_turn from VAD startTime (vad mode) and keeps same parenting", async () => {
+    const { exporter } = setupInMemoryTracing();
+    const hooks = {
+      onInterruption: vi.fn(),
+      onStartOfSpeech: vi.fn(),
+      onVADInferenceDone: vi.fn(),
+      onEndOfSpeech: vi.fn(),
+      onInterimTranscript: vi.fn(),
+      onFinalTranscript: vi.fn(),
+      onPreemptiveGeneration: vi.fn(),
+      retrieveChatCtx: () => ChatContext.empty(),
+      onEndOfTurn: vi.fn(async () => true)
+    };
+    const now = Date.now();
+    const vadEvents = [
+      {
+        type: VADEventType.START_OF_SPEECH,
+        samplesIndex: 0,
+        timestamp: now,
+        speechDuration: 100,
+        silenceDuration: 0,
+        frames: [],
+        probability: 0,
+        inferenceDuration: 0,
+        speaking: true,
+        rawAccumulatedSilence: 0,
+        rawAccumulatedSpeech: 0
+      },
+      {
+        type: VADEventType.END_OF_SPEECH,
+        samplesIndex: 0,
+        timestamp: now + 200,
+        speechDuration: 100,
+        silenceDuration: 100,
+        frames: [],
+        probability: 0,
+        inferenceDuration: 0,
+        speaking: false,
+        rawAccumulatedSilence: 0,
+        rawAccumulatedSpeech: 0
+      }
+    ];
+    const sttEvents = [
+      {
+        type: SpeechEventType.FINAL_TRANSCRIPT,
+        alternatives: [
+          {
+            language: "en",
+            text: "test",
+            startTime: 0,
+            endTime: 0,
+            confidence: 0.8
+          }
+        ]
+      }
+    ];
+    const sttNode = async () => new ReadableStream({
+      start(controller) {
+        for (const ev of sttEvents) controller.enqueue(ev);
+        controller.close();
+      }
+    });
+    const ar = new AudioRecognition({
+      recognitionHooks: hooks,
+      stt: sttNode,
+      vad: new FakeVAD(vadEvents),
+      turnDetector: alwaysTrueTurnDetector,
+      turnDetectionMode: "vad",
+      minEndpointingDelay: 0,
+      maxEndpointingDelay: 0,
+      sttModel: "stt-model",
+      sttProvider: "stt-provider",
+      getLinkedParticipant: () => ({ sid: "p2", identity: "alice", kind: ParticipantKind.AGENT })
+    });
+    await ar.start();
+    await new Promise((r) => setTimeout(r, 20));
+    await ar.close();
+    const spans = exporter.getFinishedSpans();
+    const userTurn = spanByName(spans, "user_turn");
+    const eou = spanByName(spans, "eou_detection");
+    expect(userTurn).toBeTruthy();
+    expect(eou).toBeTruthy();
+    if (!userTurn || !eou) {
+      throw new Error("expected user_turn and eou_detection spans");
+    }
+    expect(eou.parentSpanId).toBe(userTurn.spanContext().spanId);
+    expect(hooks.onStartOfSpeech).toHaveBeenCalled();
+    expect(hooks.onEndOfSpeech).toHaveBeenCalled();
+  });
+  it("parents user_speaking under user_turn when an explicit speech context is provided", () => {
+    const { exporter } = setupInMemoryTracing();
+    const sessionSpan = tracer.startSpan({ name: "agent_session", context: ROOT_CONTEXT });
+    const sessionContext = trace.setSpan(ROOT_CONTEXT, sessionSpan);
+    const fakeSession = createFakeSession(sessionContext);
+    const userTurn = tracer.startSpan({ name: "user_turn", context: sessionContext });
+    const userTurnContext = trace.setSpan(sessionContext, userTurn);
+    const speakingStartedAt = Date.now() - 100;
+    const speakingEndedAt = Date.now();
+    otelContext.with(userTurnContext, () => {
+      AgentSession.prototype._updateUserState.call(fakeSession, "speaking", {
+        lastSpeakingTime: speakingStartedAt,
+        otelContext: otelContext.active()
+      });
+      AgentSession.prototype._updateUserState.call(fakeSession, "listening", {
+        lastSpeakingTime: speakingEndedAt,
+        otelContext: otelContext.active()
+      });
+    });
+    userTurn.end();
+    sessionSpan.end();
+    const spans = exporter.getFinishedSpans();
+    const userSpeaking = spanByName(spans, "user_speaking");
+    const exportedUserTurn = spanByName(spans, "user_turn");
+    expect(userSpeaking).toBeTruthy();
+    expect(exportedUserTurn).toBeTruthy();
+    if (!userSpeaking || !exportedUserTurn) {
+      throw new Error("expected user_speaking and user_turn spans");
+    }
+    expect(userSpeaking.parentSpanId).toBe(exportedUserTurn.spanContext().spanId);
+    expect(userSpeaking.attributes["lk.participant_id"]).toBe("p3");
+  });
+  it("keeps user_speaking attached to the session root without an explicit speech context", () => {
+    const { exporter } = setupInMemoryTracing();
+    const sessionSpan = tracer.startSpan({ name: "agent_session", context: ROOT_CONTEXT });
+    const sessionContext = trace.setSpan(ROOT_CONTEXT, sessionSpan);
+    const fakeSession = createFakeSession(sessionContext);
+    AgentSession.prototype._updateUserState.call(fakeSession, "speaking", {
+      lastSpeakingTime: Date.now() - 100
+    });
+    AgentSession.prototype._updateUserState.call(fakeSession, "listening", {
+      lastSpeakingTime: Date.now()
+    });
+    sessionSpan.end();
+    const spans = exporter.getFinishedSpans();
+    const userSpeaking = spanByName(spans, "user_speaking");
+    expect(userSpeaking).toBeTruthy();
+    if (!userSpeaking) {
+      throw new Error("expected user_speaking span");
+    }
+    expect(userSpeaking.parentSpanId).toBe(sessionSpan.spanContext().spanId);
+  });
+});
+//# sourceMappingURL=audio_recognition_span.test.js.map

package/dist/voice/audio_recognition_span.test.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../../src/voice/audio_recognition_span.test.ts"],"sourcesContent":["// SPDX-FileCopyrightText: 2026 LiveKit, Inc.\n//\n// SPDX-License-Identifier: Apache-2.0\nimport { ParticipantKind } from '@livekit/rtc-node';\nimport { ROOT_CONTEXT, context as otelContext, trace } from '@opentelemetry/api';\nimport {\n  InMemorySpanExporter,\n  type ReadableSpan,\n  SimpleSpanProcessor,\n} from '@opentelemetry/sdk-trace-base';\nimport { NodeTracerProvider } from '@opentelemetry/sdk-trace-node';\nimport { ReadableStream } from 'node:stream/web';\nimport { describe, expect, it, vi } from 'vitest';\nimport { ChatContext } from '../llm/chat_context.js';\nimport { initializeLogger } from '../log.js';\nimport { type SpeechEvent, SpeechEventType } from '../stt/stt.js';\nimport { setTracerProvider, tracer } from '../telemetry/index.js';\nimport { VAD, type VADEvent, VADEventType, type VADStream } from '../vad.js';\nimport { AgentSession } from './agent_session.js';\nimport {\n  AudioRecognition,\n  type RecognitionHooks,\n  type _TurnDetector,\n} from './audio_recognition.js';\nimport type { STTNode } from './io.js';\n\nfunction setupInMemoryTracing() {\n  const exporter = new InMemorySpanExporter();\n  const provider = new NodeTracerProvider();\n  provider.addSpanProcessor(new SimpleSpanProcessor(exporter));\n  provider.register();\n  setTracerProvider(provider);\n  return { exporter };\n}\n\nfunction spanByName(spans: ReadableSpan[], name: string) {\n  return spans.find((s) => s.name === name);\n}\n\nfunction createFakeSession(rootSpanContext = ROOT_CONTEXT): AgentSession {\n  return {\n    _agentState: 'listening',\n    _roomIO: {\n      linkedParticipant: { sid: 'p3', identity: 'charlie', kind: ParticipantKind.AGENT },\n    },\n    _setUserAwayTimer: vi.fn(),\n    _cancelUserAwayTimer: vi.fn(),\n    _userSpeakingSpan: undefined,\n    _userState: 'listening',\n    emit: vi.fn(),\n    rootSpanContext,\n  } as unknown as AgentSession;\n}\n\nclass FakeVADStream extends (Object as unknown as { new (): VADStream }) {\n  // We intentionally avoid extending the real VADStream (it is not exported as a value in JS output\n  // in some bundling contexts). Instead we emulate the async iterator shape used by AudioRecognition.\n  private events: VADEvent[];\n  private idx = 0;\n  constructor(events: VADEvent[]) {\n    super();\n    this.events = events;\n  }\n  updateInputStream() {}\n  detachInputStream() {}\n  close() {}\n  [Symbol.asyncIterator]() {\n    return this;\n  }\n  async next(): Promise<IteratorResult<VADEvent>> {\n    if (this.idx >= this.events.length) {\n      return { done: true, value: undefined };\n    }\n    const value = this.events[this.idx++]!;\n    return { done: false, value };\n  }\n}\n\nclass FakeVAD extends VAD {\n  label = 'fake-vad';\n  private events: VADEvent[];\n  constructor(events: VADEvent[]) {\n    super({ updateInterval: 1 });\n    this.events = events;\n  }\n  stream(): any {\n    return new FakeVADStream(this.events);\n  }\n}\n\nconst alwaysTrueTurnDetector: _TurnDetector = {\n  model: 'test-turn-detector',\n  provider: 'test-provider',\n  supportsLanguage: async () => true,\n  unlikelyThreshold: async () => undefined,\n  predictEndOfTurn: async () => 1.0,\n};\n\ndescribe('AudioRecognition user_turn span parity', () => {\n  initializeLogger({ pretty: false, level: 'silent' });\n\n  it('creates user_turn and parents eou_detection under it (stt mode)', async () => {\n    const { exporter } = setupInMemoryTracing();\n\n    const hooks: RecognitionHooks = {\n      onInterruption: vi.fn(),\n      onStartOfSpeech: vi.fn(),\n      onVADInferenceDone: vi.fn(),\n      onEndOfSpeech: vi.fn(),\n      onInterimTranscript: vi.fn(),\n      onFinalTranscript: vi.fn(),\n      onPreemptiveGeneration: vi.fn(),\n      retrieveChatCtx: () => ChatContext.empty(),\n      onEndOfTurn: vi.fn(async () => true),\n    };\n\n    const sttEvents: SpeechEvent[] = [\n      { type: SpeechEventType.START_OF_SPEECH },\n      {\n        type: SpeechEventType.FINAL_TRANSCRIPT,\n        alternatives: [\n          {\n            language: 'en',\n            text: 'hello',\n            startTime: 0,\n            endTime: 0,\n            confidence: 0.9,\n          },\n        ],\n      },\n      { type: SpeechEventType.END_OF_SPEECH },\n    ];\n\n    const sttNode: STTNode = async () =>\n      new ReadableStream<SpeechEvent | string>({\n        start(controller) {\n          for (const ev of sttEvents) controller.enqueue(ev);\n          controller.close();\n        },\n      });\n\n    const ar = new AudioRecognition({\n      recognitionHooks: hooks,\n      stt: sttNode,\n      vad: undefined,\n      turnDetector: alwaysTrueTurnDetector,\n      turnDetectionMode: 'stt',\n      minEndpointingDelay: 0,\n      maxEndpointingDelay: 0,\n      sttModel: 'deepgram-nova2',\n      sttProvider: 'deepgram',\n      getLinkedParticipant: () => ({ sid: 'p1', identity: 'bob', kind: ParticipantKind.AGENT }),\n    });\n\n    await ar.start();\n    // allow background task to drain\n    await new Promise((r) => setTimeout(r, 20));\n    await ar.close();\n\n    const spans = exporter.getFinishedSpans();\n    const userTurn = spanByName(spans, 'user_turn');\n    const eou = spanByName(spans, 'eou_detection');\n    expect(userTurn, 'user_turn span missing').toBeTruthy();\n    expect(eou, 'eou_detection span missing').toBeTruthy();\n    if (!userTurn || !eou) {\n      throw new Error('expected user_turn and eou_detection spans');\n    }\n\n    expect(eou.parentSpanId).toBe(userTurn.spanContext().spanId);\n\n    // creation-time attributes\n    expect(userTurn.attributes['lk.participant_id']).toBe('p1');\n    expect(userTurn.attributes['lk.participant_identity']).toBe('bob');\n    expect(userTurn.attributes['lk.participant_kind']).toBe('AGENT');\n    expect(userTurn.attributes['gen_ai.request.model']).toBe('deepgram-nova2');\n    expect(userTurn.attributes['gen_ai.provider.name']).toBe('deepgram');\n\n    // end-of-turn attributes\n    expect(userTurn.attributes['lk.user_transcript']).toContain('hello');\n    expect(userTurn.attributes['lk.transcript_confidence']).toBeGreaterThan(0);\n  });\n\n  it('creates user_turn from VAD startTime (vad mode) and keeps same parenting', async () => {\n    const { exporter } = setupInMemoryTracing();\n\n    const hooks: RecognitionHooks = {\n      onInterruption: vi.fn(),\n      onStartOfSpeech: vi.fn(),\n      onVADInferenceDone: vi.fn(),\n      onEndOfSpeech: vi.fn(),\n      onInterimTranscript: vi.fn(),\n      onFinalTranscript: vi.fn(),\n      onPreemptiveGeneration: vi.fn(),\n      retrieveChatCtx: () => ChatContext.empty(),\n      onEndOfTurn: vi.fn(async () => true),\n    };\n\n    const now = Date.now();\n    const vadEvents: VADEvent[] = [\n      {\n        type: VADEventType.START_OF_SPEECH,\n        samplesIndex: 0,\n        timestamp: now,\n        speechDuration: 100,\n        silenceDuration: 0,\n        frames: [],\n        probability: 0,\n        inferenceDuration: 0,\n        speaking: true,\n        rawAccumulatedSilence: 0,\n        rawAccumulatedSpeech: 0,\n      },\n      {\n        type: VADEventType.END_OF_SPEECH,\n        samplesIndex: 0,\n        timestamp: now + 200,\n        speechDuration: 100,\n        silenceDuration: 100,\n        frames: [],\n        probability: 0,\n        inferenceDuration: 0,\n        speaking: false,\n        rawAccumulatedSilence: 0,\n        rawAccumulatedSpeech: 0,\n      },\n    ];\n\n    const sttEvents: SpeechEvent[] = [\n      {\n        type: SpeechEventType.FINAL_TRANSCRIPT,\n        alternatives: [\n          {\n            language: 'en',\n            text: 'test',\n            startTime: 0,\n            endTime: 0,\n            confidence: 0.8,\n          },\n        ],\n      },\n    ];\n\n    const sttNode: STTNode = async () =>\n      new ReadableStream<SpeechEvent | string>({\n        start(controller) {\n          for (const ev of sttEvents) controller.enqueue(ev);\n          controller.close();\n        },\n      });\n\n    const ar = new AudioRecognition({\n      recognitionHooks: hooks,\n      stt: sttNode,\n      vad: new FakeVAD(vadEvents),\n      turnDetector: alwaysTrueTurnDetector,\n      turnDetectionMode: 'vad',\n      minEndpointingDelay: 0,\n      maxEndpointingDelay: 0,\n      sttModel: 'stt-model',\n      sttProvider: 'stt-provider',\n      getLinkedParticipant: () => ({ sid: 'p2', identity: 'alice', kind: ParticipantKind.AGENT }),\n    });\n\n    await ar.start();\n    await new Promise((r) => setTimeout(r, 20));\n    await ar.close();\n\n    const spans = exporter.getFinishedSpans();\n    const userTurn = spanByName(spans, 'user_turn');\n    const eou = spanByName(spans, 'eou_detection');\n    expect(userTurn).toBeTruthy();\n    expect(eou).toBeTruthy();\n    if (!userTurn || !eou) {\n      throw new Error('expected user_turn and eou_detection spans');\n    }\n    expect(eou.parentSpanId).toBe(userTurn.spanContext().spanId);\n\n    expect(hooks.onStartOfSpeech).toHaveBeenCalled();\n    expect(hooks.onEndOfSpeech).toHaveBeenCalled();\n  });\n\n  it('parents user_speaking under user_turn when an explicit speech context is provided', () => {\n    const { exporter } = setupInMemoryTracing();\n    const sessionSpan = tracer.startSpan({ name: 'agent_session', context: ROOT_CONTEXT });\n    const sessionContext = trace.setSpan(ROOT_CONTEXT, sessionSpan);\n    const fakeSession = createFakeSession(sessionContext);\n    const userTurn = tracer.startSpan({ name: 'user_turn', context: sessionContext });\n    const userTurnContext = trace.setSpan(sessionContext, userTurn);\n    const speakingStartedAt = Date.now() - 100;\n    const speakingEndedAt = Date.now();\n\n    otelContext.with(userTurnContext, () => {\n      AgentSession.prototype._updateUserState.call(fakeSession, 'speaking', {\n        lastSpeakingTime: speakingStartedAt,\n        otelContext: otelContext.active(),\n      });\n      AgentSession.prototype._updateUserState.call(fakeSession, 'listening', {\n        lastSpeakingTime: speakingEndedAt,\n        otelContext: otelContext.active(),\n      });\n    });\n\n    userTurn.end();\n    sessionSpan.end();\n\n    const spans = exporter.getFinishedSpans();\n    const userSpeaking = spanByName(spans, 'user_speaking');\n    const exportedUserTurn = spanByName(spans, 'user_turn');\n    expect(userSpeaking).toBeTruthy();\n    expect(exportedUserTurn).toBeTruthy();\n    if (!userSpeaking || !exportedUserTurn) {\n      throw new Error('expected user_speaking and user_turn spans');\n    }\n    expect(userSpeaking.parentSpanId).toBe(exportedUserTurn.spanContext().spanId);\n    expect(userSpeaking.attributes['lk.participant_id']).toBe('p3');\n  });\n\n  it('keeps user_speaking attached to the session root without an explicit speech context', () => {\n    const { exporter } = setupInMemoryTracing();\n    const sessionSpan = tracer.startSpan({ name: 'agent_session', context: ROOT_CONTEXT });\n    const sessionContext = trace.setSpan(ROOT_CONTEXT, sessionSpan);\n    const fakeSession = createFakeSession(sessionContext);\n\n    AgentSession.prototype._updateUserState.call(fakeSession, 'speaking', {\n      lastSpeakingTime: Date.now() - 100,\n    });\n    AgentSession.prototype._updateUserState.call(fakeSession, 'listening', {\n      lastSpeakingTime: Date.now(),\n    });\n\n    sessionSpan.end();\n\n    const spans = exporter.getFinishedSpans();\n    const userSpeaking = spanByName(spans, 'user_speaking');\n    expect(userSpeaking).toBeTruthy();\n    if (!userSpeaking) {\n      throw new Error('expected user_speaking span');\n    }\n    expect(userSpeaking.parentSpanId).toBe(sessionSpan.spanContext().spanId);\n  });\n});\n"],"mappings":"AAGA,SAAS,uBAAuB;AAChC,SAAS,cAAc,WAAW,aAAa,aAAa;AAC5D;AAAA,EACE;AAAA,EAEA;AAAA,OACK;AACP,SAAS,0BAA0B;AACnC,SAAS,sBAAsB;AAC/B,SAAS,UAAU,QAAQ,IAAI,UAAU;AACzC,SAAS,mBAAmB;AAC5B,SAAS,wBAAwB;AACjC,SAA2B,uBAAuB;AAClD,SAAS,mBAAmB,cAAc;AAC1C,SAAS,KAAoB,oBAAoC;AACjE,SAAS,oBAAoB;AAC7B;AAAA,EACE;AAAA,OAGK;AAGP,SAAS,uBAAuB;AAC9B,QAAM,WAAW,IAAI,qBAAqB;AAC1C,QAAM,WAAW,IAAI,mBAAmB;AACxC,WAAS,iBAAiB,IAAI,oBAAoB,QAAQ,CAAC;AAC3D,WAAS,SAAS;AAClB,oBAAkB,QAAQ;AAC1B,SAAO,EAAE,SAAS;AACpB;AAEA,SAAS,WAAW,OAAuB,MAAc;AACvD,SAAO,MAAM,KAAK,CAAC,MAAM,EAAE,SAAS,IAAI;AAC1C;AAEA,SAAS,kBAAkB,kBAAkB,cAA4B;AACvE,SAAO;AAAA,IACL,aAAa;AAAA,IACb,SAAS;AAAA,MACP,mBAAmB,EAAE,KAAK,MAAM,UAAU,WAAW,MAAM,gBAAgB,MAAM;AAAA,IACnF;AAAA,IACA,mBAAmB,GAAG,GAAG;AAAA,IACzB,sBAAsB,GAAG,GAAG;AAAA,IAC5B,mBAAmB;AAAA,IACnB,YAAY;AAAA,IACZ,MAAM,GAAG,GAAG;AAAA,IACZ;AAAA,EACF;AACF;AAEA,MAAM,sBAAuB,OAA4C;AAAA;AAAA;AAAA,EAG/D;AAAA,EACA,MAAM;AAAA,EACd,YAAY,QAAoB;AAC9B,UAAM;AACN,SAAK,SAAS;AAAA,EAChB;AAAA,EACA,oBAAoB;AAAA,EAAC;AAAA,EACrB,oBAAoB;AAAA,EAAC;AAAA,EACrB,QAAQ;AAAA,EAAC;AAAA,EACT,CAAC,OAAO,aAAa,IAAI;AACvB,WAAO;AAAA,EACT;AAAA,EACA,MAAM,OAA0C;AAC9C,QAAI,KAAK,OAAO,KAAK,OAAO,QAAQ;AAClC,aAAO,EAAE,MAAM,MAAM,OAAO,OAAU;AAAA,IACxC;AACA,UAAM,QAAQ,KAAK,OAAO,KAAK,KAAK;AACpC,WAAO,EAAE,MAAM,OAAO,MAAM;AAAA,EAC9B;AACF;AAEA,MAAM,gBAAgB,IAAI;AAAA,EACxB,QAAQ;AAAA,EACA;AAAA,EACR,YAAY,QAAoB;AAC9B,UAAM,EAAE,gBAAgB,EAAE,CAAC;AAC3B,SAAK,SAAS;AAAA,EAChB;AAAA,EACA,SAAc;AACZ,WAAO,IAAI,cAAc,KAAK,MAAM;AAAA,EACtC;AACF;AAEA,MAAM,yBAAwC;AAAA,EAC5C,OAAO;AAAA,EACP,UAAU;AAAA,EACV,kBAAkB,YAAY;AAAA,EAC9B,mBAAmB,YAAY;AAAA,EAC/B,kBAAkB,YAAY;AAChC;AAEA,SAAS,0CAA0C,MAAM;AACvD,mBAAiB,EAAE,QAAQ,OAAO,OAAO,SAAS,CAAC;AAEnD,KAAG,mEAAmE,YAAY;AAChF,UAAM,EAAE,SAAS,IAAI,qBAAqB;AAE1C,UAAM,QAA0B;AAAA,MAC9B,gBAAgB,GAAG,GAAG;AAAA,MACtB,iBAAiB,GAAG,GAAG;AAAA,MACvB,oBAAoB,GAAG,GAAG;AAAA,MAC1B,eAAe,GAAG,GAAG;AAAA,MACrB,qBAAqB,GAAG,GAAG;AAAA,MAC3B,mBAAmB,GAAG,GAAG;AAAA,MACzB,wBAAwB,GAAG,GAAG;AAAA,MAC9B,iBAAiB,MAAM,YAAY,MAAM;AAAA,MACzC,aAAa,GAAG,GAAG,YAAY,IAAI;AAAA,IACrC;AAEA,UAAM,YAA2B;AAAA,MAC/B,EAAE,MAAM,gBAAgB,gBAAgB;AAAA,MACxC;AAAA,QACE,MAAM,gBAAgB;AAAA,QACtB,cAAc;AAAA,UACZ;AAAA,YACE,UAAU;AAAA,YACV,MAAM;AAAA,YACN,WAAW;AAAA,YACX,SAAS;AAAA,YACT,YAAY;AAAA,UACd;AAAA,QACF;AAAA,MACF;AAAA,MACA,EAAE,MAAM,gBAAgB,cAAc;AAAA,IACxC;AAEA,UAAM,UAAmB,YACvB,IAAI,eAAqC;AAAA,MACvC,MAAM,YAAY;AAChB,mBAAW,MAAM,UAAW,YAAW,QAAQ,EAAE;AACjD,mBAAW,MAAM;AAAA,MACnB;AAAA,IACF,CAAC;AAEH,UAAM,KAAK,IAAI,iBAAiB;AAAA,MAC9B,kBAAkB;AAAA,MAClB,KAAK;AAAA,MACL,KAAK;AAAA,MACL,cAAc;AAAA,MACd,mBAAmB;AAAA,MACnB,qBAAqB;AAAA,MACrB,qBAAqB;AAAA,MACrB,UAAU;AAAA,MACV,aAAa;AAAA,MACb,sBAAsB,OAAO,EAAE,KAAK,MAAM,UAAU,OAAO,MAAM,gBAAgB,MAAM;AAAA,IACzF,CAAC;AAED,UAAM,GAAG,MAAM;AAEf,UAAM,IAAI,QAAQ,CAAC,MAAM,WAAW,GAAG,EAAE,CAAC;AAC1C,UAAM,GAAG,MAAM;AAEf,UAAM,QAAQ,SAAS,iBAAiB;AACxC,UAAM,WAAW,WAAW,OAAO,WAAW;AAC9C,UAAM,MAAM,WAAW,OAAO,eAAe;AAC7C,WAAO,UAAU,wBAAwB,EAAE,WAAW;AACtD,WAAO,KAAK,4BAA4B,EAAE,WAAW;AACrD,QAAI,CAAC,YAAY,CAAC,KAAK;AACrB,YAAM,IAAI,MAAM,4CAA4C;AAAA,IAC9D;AAEA,WAAO,IAAI,YAAY,EAAE,KAAK,SAAS,YAAY,EAAE,MAAM;AAG3D,WAAO,SAAS,WAAW,mBAAmB,CAAC,EAAE,KAAK,IAAI;AAC1D,WAAO,SAAS,WAAW,yBAAyB,CAAC,EAAE,KAAK,KAAK;AACjE,WAAO,SAAS,WAAW,qBAAqB,CAAC,EAAE,KAAK,OAAO;AAC/D,WAAO,SAAS,WAAW,sBAAsB,CAAC,EAAE,KAAK,gBAAgB;AACzE,WAAO,SAAS,WAAW,sBAAsB,CAAC,EAAE,KAAK,UAAU;AAGnE,WAAO,SAAS,WAAW,oBAAoB,CAAC,EAAE,UAAU,OAAO;AACnE,WAAO,SAAS,WAAW,0BAA0B,CAAC,EAAE,gBAAgB,CAAC;AAAA,EAC3E,CAAC;AAED,KAAG,4EAA4E,YAAY;AACzF,UAAM,EAAE,SAAS,IAAI,qBAAqB;AAE1C,UAAM,QAA0B;AAAA,MAC9B,gBAAgB,GAAG,GAAG;AAAA,MACtB,iBAAiB,GAAG,GAAG;AAAA,MACvB,oBAAoB,GAAG,GAAG;AAAA,MAC1B,eAAe,GAAG,GAAG;AAAA,MACrB,qBAAqB,GAAG,GAAG;AAAA,MAC3B,mBAAmB,GAAG,GAAG;AAAA,MACzB,wBAAwB,GAAG,GAAG;AAAA,MAC9B,iBAAiB,MAAM,YAAY,MAAM;AAAA,MACzC,aAAa,GAAG,GAAG,YAAY,IAAI;AAAA,IACrC;AAEA,UAAM,MAAM,KAAK,IAAI;AACrB,UAAM,YAAwB;AAAA,MAC5B;AAAA,QACE,MAAM,aAAa;AAAA,QACnB,cAAc;AAAA,QACd,WAAW;AAAA,QACX,gBAAgB;AAAA,QAChB,iBAAiB;AAAA,QACjB,QAAQ,CAAC;AAAA,QACT,aAAa;AAAA,QACb,mBAAmB;AAAA,QACnB,UAAU;AAAA,QACV,uBAAuB;AAAA,QACvB,sBAAsB;AAAA,MACxB;AAAA,MACA;AAAA,QACE,MAAM,aAAa;AAAA,QACnB,cAAc;AAAA,QACd,WAAW,MAAM;AAAA,QACjB,gBAAgB;AAAA,QAChB,iBAAiB;AAAA,QACjB,QAAQ,CAAC;AAAA,QACT,aAAa;AAAA,QACb,mBAAmB;AAAA,QACnB,UAAU;AAAA,QACV,uBAAuB;AAAA,QACvB,sBAAsB;AAAA,MACxB;AAAA,IACF;AAEA,UAAM,YAA2B;AAAA,MAC/B;AAAA,QACE,MAAM,gBAAgB;AAAA,QACtB,cAAc;AAAA,UACZ;AAAA,YACE,UAAU;AAAA,YACV,MAAM;AAAA,YACN,WAAW;AAAA,YACX,SAAS;AAAA,YACT,YAAY;AAAA,UACd;AAAA,QACF;AAAA,MACF;AAAA,IACF;AAEA,UAAM,UAAmB,YACvB,IAAI,eAAqC;AAAA,MACvC,MAAM,YAAY;AAChB,mBAAW,MAAM,UAAW,YAAW,QAAQ,EAAE;AACjD,mBAAW,MAAM;AAAA,MACnB;AAAA,IACF,CAAC;AAEH,UAAM,KAAK,IAAI,iBAAiB;AAAA,MAC9B,kBAAkB;AAAA,MAClB,KAAK;AAAA,MACL,KAAK,IAAI,QAAQ,SAAS;AAAA,MAC1B,cAAc;AAAA,MACd,mBAAmB;AAAA,MACnB,qBAAqB;AAAA,MACrB,qBAAqB;AAAA,MACrB,UAAU;AAAA,MACV,aAAa;AAAA,MACb,sBAAsB,OAAO,EAAE,KAAK,MAAM,UAAU,SAAS,MAAM,gBAAgB,MAAM;AAAA,IAC3F,CAAC;AAED,UAAM,GAAG,MAAM;AACf,UAAM,IAAI,QAAQ,CAAC,MAAM,WAAW,GAAG,EAAE,CAAC;AAC1C,UAAM,GAAG,MAAM;AAEf,UAAM,QAAQ,SAAS,iBAAiB;AACxC,UAAM,WAAW,WAAW,OAAO,WAAW;AAC9C,UAAM,MAAM,WAAW,OAAO,eAAe;AAC7C,WAAO,QAAQ,EAAE,WAAW;AAC5B,WAAO,GAAG,EAAE,WAAW;AACvB,QAAI,CAAC,YAAY,CAAC,KAAK;AACrB,YAAM,IAAI,MAAM,4CAA4C;AAAA,IAC9D;AACA,WAAO,IAAI,YAAY,EAAE,KAAK,SAAS,YAAY,EAAE,MAAM;AAE3D,WAAO,MAAM,eAAe,EAAE,iBAAiB;AAC/C,WAAO,MAAM,aAAa,EAAE,iBAAiB;AAAA,EAC/C,CAAC;AAED,KAAG,qFAAqF,MAAM;AAC5F,UAAM,EAAE,SAAS,IAAI,qBAAqB;AAC1C,UAAM,cAAc,OAAO,UAAU,EAAE,MAAM,iBAAiB,SAAS,aAAa,CAAC;AACrF,UAAM,iBAAiB,MAAM,QAAQ,cAAc,WAAW;AAC9D,UAAM,cAAc,kBAAkB,cAAc;AACpD,UAAM,WAAW,OAAO,UAAU,EAAE,MAAM,aAAa,SAAS,eAAe,CAAC;AAChF,UAAM,kBAAkB,MAAM,QAAQ,gBAAgB,QAAQ;AAC9D,UAAM,oBAAoB,KAAK,IAAI,IAAI;AACvC,UAAM,kBAAkB,KAAK,IAAI;AAEjC,gBAAY,KAAK,iBAAiB,MAAM;AACtC,mBAAa,UAAU,iBAAiB,KAAK,aAAa,YAAY;AAAA,QACpE,kBAAkB;AAAA,QAClB,aAAa,YAAY,OAAO;AAAA,MAClC,CAAC;AACD,mBAAa,UAAU,iBAAiB,KAAK,aAAa,aAAa;AAAA,QACrE,kBAAkB;AAAA,QAClB,aAAa,YAAY,OAAO;AAAA,MAClC,CAAC;AAAA,IACH,CAAC;AAED,aAAS,IAAI;AACb,gBAAY,IAAI;AAEhB,UAAM,QAAQ,SAAS,iBAAiB;AACxC,UAAM,eAAe,WAAW,OAAO,eAAe;AACtD,UAAM,mBAAmB,WAAW,OAAO,WAAW;AACtD,WAAO,YAAY,EAAE,WAAW;AAChC,WAAO,gBAAgB,EAAE,WAAW;AACpC,QAAI,CAAC,gBAAgB,CAAC,kBAAkB;AACtC,YAAM,IAAI,MAAM,4CAA4C;AAAA,IAC9D;AACA,WAAO,aAAa,YAAY,EAAE,KAAK,iBAAiB,YAAY,EAAE,MAAM;AAC5E,WAAO,aAAa,WAAW,mBAAmB,CAAC,EAAE,KAAK,IAAI;AAAA,EAChE,CAAC;AAED,KAAG,uFAAuF,MAAM;AAC9F,UAAM,EAAE,SAAS,IAAI,qBAAqB;AAC1C,UAAM,cAAc,OAAO,UAAU,EAAE,MAAM,iBAAiB,SAAS,aAAa,CAAC;AACrF,UAAM,iBAAiB,MAAM,QAAQ,cAAc,WAAW;AAC9D,UAAM,cAAc,kBAAkB,cAAc;AAEpD,iBAAa,UAAU,iBAAiB,KAAK,aAAa,YAAY;AAAA,MACpE,kBAAkB,KAAK,IAAI,IAAI;AAAA,IACjC,CAAC;AACD,iBAAa,UAAU,iBAAiB,KAAK,aAAa,aAAa;AAAA,MACrE,kBAAkB,KAAK,IAAI;AAAA,IAC7B,CAAC;AAED,gBAAY,IAAI;AAEhB,UAAM,QAAQ,SAAS,iBAAiB;AACxC,UAAM,eAAe,WAAW,OAAO,eAAe;AACtD,WAAO,YAAY,EAAE,WAAW;AAChC,QAAI,CAAC,cAAc;AACjB,YAAM,IAAI,MAAM,6BAA6B;AAAA,IAC/C;AACA,WAAO,aAAa,YAAY,EAAE,KAAK,YAAY,YAAY,EAAE,MAAM;AAAA,EACzE,CAAC;AACH,CAAC;","names":[]}

package/dist/voice/avatar/datastream_io.cjs

@@ -41,9 +41,10 @@ class DataStreamAudioOutput extends import_io.AudioOutput {
   started = false;
   lock = new import_mutex.Mutex();
   startTask;
+  firstFrameEmitted = false;
   #logger = (0, import_log.log)();
   constructor(opts) {
-    super(opts.sampleRate, void 0);
+    super(opts.sampleRate, void 0, { pause: false });
     const { room, destinationIdentity, sampleRate, waitRemoteTrack } = opts;
     this.room = room;
     this.destinationIdentity = destinationIdentity;
@@ -116,6 +117,10 @@ class DataStreamAudioOutput extends import_io.AudioOutput {
     }
     await this.startTask.result;
     await super.captureFrame(frame);
+    if (!this.firstFrameEmitted) {
+      this.firstFrameEmitted = true;
+      this.onPlaybackStarted(Date.now());
+    }
     if (!this.streamWriter) {
       this.streamWriter = await this.room.localParticipant.streamBytes({
         name: (0, import_utils.shortuuid)("AUDIO_"),
@@ -139,6 +144,7 @@ class DataStreamAudioOutput extends import_io.AudioOutput {
     this.streamWriter.close().finally(() => {
       this.streamWriter = void 0;
     });
+    this.firstFrameEmitted = false;
   }
   clearBuffer() {
     if (!this.started) return;

package/dist/voice/avatar/datastream_io.cjs.map

@@ -1 +1 @@
-{"version":3,"sources":["../../../src/voice/avatar/datastream_io.ts"],"sourcesContent":["// SPDX-FileCopyrightText: 2025 LiveKit, Inc.\n//\n// SPDX-License-Identifier: Apache-2.0\nimport { Mutex } from '@livekit/mutex';\nimport {\n  type AudioFrame,\n  type ByteStreamWriter,\n  type Room,\n  RoomEvent,\n  type RpcInvocationData,\n  type TrackKind,\n} from '@livekit/rtc-node';\nimport { log } from '../../log.js';\nimport {\n  Future,\n  Task,\n  shortuuid,\n  waitForParticipant,\n  waitForTrackPublication,\n} from '../../utils.js';\nimport { AudioOutput, type PlaybackFinishedEvent } from '../io.js';\n\nconst RPC_CLEAR_BUFFER = 'lk.clear_buffer';\nconst RPC_PLAYBACK_FINISHED = 'lk.playback_finished';\nconst AUDIO_STREAM_TOPIC = 'lk.audio_stream';\n\nexport interface DataStreamAudioOutputOptions {\n  room: Room;\n  destinationIdentity: string;\n  sampleRate?: number;\n  waitRemoteTrack?: TrackKind;\n}\n\n/**\n * AudioOutput implementation that streams audio to a remote avatar worker using LiveKit DataStream.\n */\nexport class DataStreamAudioOutput extends AudioOutput {\n  static _playbackFinishedRpcRegistered: boolean = false;\n  static _playbackFinishedHandlers: Record<string, (data: RpcInvocationData) => string> = {};\n\n  private room: Room;\n  private destinationIdentity: string;\n  private roomConnectedFuture: Future<void>;\n  private waitRemoteTrack?: TrackKind;\n  private streamWriter?: ByteStreamWriter;\n  private pushedDuration: number = 0;\n  private started: boolean = false;\n  private lock = new Mutex();\n  private startTask?: Task<void>;\n\n  #logger = log();\n\n  constructor(opts: DataStreamAudioOutputOptions) {\n    super(opts.sampleRate, undefined);\n\n    const { room, destinationIdentity, sampleRate, waitRemoteTrack } = opts;\n    this.room = room;\n    this.destinationIdentity = destinationIdentity;\n    this.sampleRate = sampleRate;\n    this.waitRemoteTrack = waitRemoteTrack;\n\n    const onRoomConnected = async () => {\n      if (this.startTask) return;\n\n      await this.roomConnectedFuture.await;\n\n      // register the rpc method right after the room is connected\n      DataStreamAudioOutput.registerPlaybackFinishedRpc({\n        room,\n        callerIdentity: this.destinationIdentity,\n        handler: (data) => this.handlePlaybackFinished(data),\n      });\n\n      this.startTask = Task.from(({ signal }) => this._start(signal));\n    };\n\n    this.roomConnectedFuture = new Future<void>();\n\n    this.room.on(RoomEvent.ConnectionStateChanged, (_) => {\n      if (room.isConnected && !this.roomConnectedFuture.done) {\n        this.roomConnectedFuture.resolve(undefined);\n      }\n    });\n\n    if (this.room.isConnected) {\n      this.roomConnectedFuture.resolve(undefined);\n    }\n\n    onRoomConnected();\n  }\n\n  private async _start(_abortSignal: AbortSignal) {\n    const unlock = await this.lock.lock();\n\n    try {\n      if (this.started) return;\n\n      await this.roomConnectedFuture.await;\n\n      this.#logger.debug(\n        {\n          identity: this.destinationIdentity,\n        },\n        'waiting for the remote participant',\n      );\n\n      await waitForParticipant({\n        room: this.room,\n        identity: this.destinationIdentity,\n      });\n\n      if (this.waitRemoteTrack) {\n        this.#logger.debug(\n          {\n            identity: this.destinationIdentity,\n            kind: this.waitRemoteTrack,\n          },\n          'waiting for the remote track',\n        );\n\n        await waitForTrackPublication({\n          room: this.room,\n          identity: this.destinationIdentity,\n          kind: this.waitRemoteTrack,\n        });\n      }\n\n      this.#logger.debug(\n        {\n          identity: this.destinationIdentity,\n        },\n        'remote participant ready',\n      );\n\n      this.started = true;\n    } finally {\n      unlock();\n    }\n  }\n\n  async captureFrame(frame: AudioFrame): Promise<void> {\n    if (!this.startTask) {\n      this.startTask = Task.from(({ signal }) => this._start(signal));\n    }\n\n    await this.startTask.result;\n    await super.captureFrame(frame);\n\n    if (!this.streamWriter) {\n      this.streamWriter = await this.room.localParticipant!.streamBytes({\n        name: shortuuid('AUDIO_'),\n        topic: AUDIO_STREAM_TOPIC,\n        destinationIdentities: [this.destinationIdentity],\n        attributes: {\n          sample_rate: frame.sampleRate.toString(),\n          num_channels: frame.channels.toString(),\n        },\n      });\n      this.pushedDuration = 0;\n    }\n\n    // frame.data is a Int16Array, write accepts a Uint8Array\n    await this.streamWriter.write(new Uint8Array(frame.data.buffer));\n    this.pushedDuration += frame.samplesPerChannel / frame.sampleRate;\n  }\n\n  flush(): void {\n    super.flush();\n\n    if (this.streamWriter === undefined || !this.started) {\n      return;\n    }\n\n    this.streamWriter.close().finally(() => {\n      this.streamWriter = undefined;\n    });\n  }\n\n  clearBuffer(): void {\n    if (!this.started) return;\n\n    this.room.localParticipant!.performRpc({\n      destinationIdentity: this.destinationIdentity,\n      method: RPC_CLEAR_BUFFER,\n      payload: '',\n    });\n  }\n\n  private handlePlaybackFinished(data: RpcInvocationData): string {\n    if (data.callerIdentity !== this.destinationIdentity) {\n      this.#logger.warn(\n        {\n          callerIdentity: data.callerIdentity,\n          destinationIdentity: this.destinationIdentity,\n        },\n        'playback finished event received from unexpected participant',\n      );\n      return 'reject';\n    }\n\n    this.#logger.info(\n      {\n        callerIdentity: data.callerIdentity,\n      },\n      'playback finished event received',\n    );\n\n    const playbackFinishedEvent = JSON.parse(data.payload) as PlaybackFinishedEvent;\n    this.onPlaybackFinished(playbackFinishedEvent);\n    return 'ok';\n  }\n\n  static registerPlaybackFinishedRpc({\n    room,\n    callerIdentity,\n    handler,\n  }: {\n    room: Room;\n    callerIdentity: string;\n    handler: (data: RpcInvocationData) => string;\n  }) {\n    DataStreamAudioOutput._playbackFinishedHandlers[callerIdentity] = handler;\n\n    if (DataStreamAudioOutput._playbackFinishedRpcRegistered) {\n      return;\n    }\n\n    const rpcHandler = async (data: RpcInvocationData): Promise<string> => {\n      const handler = DataStreamAudioOutput._playbackFinishedHandlers[data.callerIdentity];\n      if (!handler) {\n        log().warn(\n          {\n            callerIdentity: data.callerIdentity,\n            expectedIdentities: Object.keys(DataStreamAudioOutput._playbackFinishedHandlers),\n          },\n          'playback finished event received from unexpected participant',\n        );\n\n        return 'reject';\n      }\n      return handler(data);\n    };\n\n    room.localParticipant?.registerRpcMethod(RPC_PLAYBACK_FINISHED, rpcHandler);\n    DataStreamAudioOutput._playbackFinishedRpcRegistered = true;\n  }\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;AAAA;AAAA;AAAA;AAAA;AAAA;AAGA,mBAAsB;AACtB,sBAOO;AACP,iBAAoB;AACpB,mBAMO;AACP,gBAAwD;AAExD,MAAM,mBAAmB;AACzB,MAAM,wBAAwB;AAC9B,MAAM,qBAAqB;AAYpB,MAAM,8BAA8B,sBAAY;AAAA,EACrD,OAAO,iCAA0C;AAAA,EACjD,OAAO,4BAAiF,CAAC;AAAA,EAEjF;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA,iBAAyB;AAAA,EACzB,UAAmB;AAAA,EACnB,OAAO,IAAI,mBAAM;AAAA,EACjB;AAAA,EAER,cAAU,gBAAI;AAAA,EAEd,YAAY,MAAoC;AAC9C,UAAM,KAAK,YAAY,MAAS;AAEhC,UAAM,EAAE,MAAM,qBAAqB,YAAY,gBAAgB,IAAI;AACnE,SAAK,OAAO;AACZ,SAAK,sBAAsB;AAC3B,SAAK,aAAa;AAClB,SAAK,kBAAkB;AAEvB,UAAM,kBAAkB,YAAY;AAClC,UAAI,KAAK,UAAW;AAEpB,YAAM,KAAK,oBAAoB;AAG/B,4BAAsB,4BAA4B;AAAA,QAChD;AAAA,QACA,gBAAgB,KAAK;AAAA,QACrB,SAAS,CAAC,SAAS,KAAK,uBAAuB,IAAI;AAAA,MACrD,CAAC;AAED,WAAK,YAAY,kBAAK,KAAK,CAAC,EAAE,OAAO,MAAM,KAAK,OAAO,MAAM,CAAC;AAAA,IAChE;AAEA,SAAK,sBAAsB,IAAI,oBAAa;AAE5C,SAAK,KAAK,GAAG,0BAAU,wBAAwB,CAAC,MAAM;AACpD,UAAI,KAAK,eAAe,CAAC,KAAK,oBAAoB,MAAM;AACtD,aAAK,oBAAoB,QAAQ,MAAS;AAAA,MAC5C;AAAA,IACF,CAAC;AAED,QAAI,KAAK,KAAK,aAAa;AACzB,WAAK,oBAAoB,QAAQ,MAAS;AAAA,IAC5C;AAEA,oBAAgB;AAAA,EAClB;AAAA,EAEA,MAAc,OAAO,cAA2B;AAC9C,UAAM,SAAS,MAAM,KAAK,KAAK,KAAK;AAEpC,QAAI;AACF,UAAI,KAAK,QAAS;AAElB,YAAM,KAAK,oBAAoB;AAE/B,WAAK,QAAQ;AAAA,QACX;AAAA,UACE,UAAU,KAAK;AAAA,QACjB;AAAA,QACA;AAAA,MACF;AAEA,gBAAM,iCAAmB;AAAA,QACvB,MAAM,KAAK;AAAA,QACX,UAAU,KAAK;AAAA,MACjB,CAAC;AAED,UAAI,KAAK,iBAAiB;AACxB,aAAK,QAAQ;AAAA,UACX;AAAA,YACE,UAAU,KAAK;AAAA,YACf,MAAM,KAAK;AAAA,UACb;AAAA,UACA;AAAA,QACF;AAEA,kBAAM,sCAAwB;AAAA,UAC5B,MAAM,KAAK;AAAA,UACX,UAAU,KAAK;AAAA,UACf,MAAM,KAAK;AAAA,QACb,CAAC;AAAA,MACH;AAEA,WAAK,QAAQ;AAAA,QACX;AAAA,UACE,UAAU,KAAK;AAAA,QACjB;AAAA,QACA;AAAA,MACF;AAEA,WAAK,UAAU;AAAA,IACjB,UAAE;AACA,aAAO;AAAA,IACT;AAAA,EACF;AAAA,EAEA,MAAM,aAAa,OAAkC;AACnD,QAAI,CAAC,KAAK,WAAW;AACnB,WAAK,YAAY,kBAAK,KAAK,CAAC,EAAE,OAAO,MAAM,KAAK,OAAO,MAAM,CAAC;AAAA,IAChE;AAEA,UAAM,KAAK,UAAU;AACrB,UAAM,MAAM,aAAa,KAAK;AAE9B,QAAI,CAAC,KAAK,cAAc;AACtB,WAAK,eAAe,MAAM,KAAK,KAAK,iBAAkB,YAAY;AAAA,QAChE,UAAM,wBAAU,QAAQ;AAAA,QACxB,OAAO;AAAA,QACP,uBAAuB,CAAC,KAAK,mBAAmB;AAAA,QAChD,YAAY;AAAA,UACV,aAAa,MAAM,WAAW,SAAS;AAAA,UACvC,cAAc,MAAM,SAAS,SAAS;AAAA,QACxC;AAAA,MACF,CAAC;AACD,WAAK,iBAAiB;AAAA,IACxB;AAGA,UAAM,KAAK,aAAa,MAAM,IAAI,WAAW,MAAM,KAAK,MAAM,CAAC;AAC/D,SAAK,kBAAkB,MAAM,oBAAoB,MAAM;AAAA,EACzD;AAAA,EAEA,QAAc;AACZ,UAAM,MAAM;AAEZ,QAAI,KAAK,iBAAiB,UAAa,CAAC,KAAK,SAAS;AACpD;AAAA,IACF;AAEA,SAAK,aAAa,MAAM,EAAE,QAAQ,MAAM;AACtC,WAAK,eAAe;AAAA,IACtB,CAAC;AAAA,EACH;AAAA,EAEA,cAAoB;AAClB,QAAI,CAAC,KAAK,QAAS;AAEnB,SAAK,KAAK,iBAAkB,WAAW;AAAA,MACrC,qBAAqB,KAAK;AAAA,MAC1B,QAAQ;AAAA,MACR,SAAS;AAAA,IACX,CAAC;AAAA,EACH;AAAA,EAEQ,uBAAuB,MAAiC;AAC9D,QAAI,KAAK,mBAAmB,KAAK,qBAAqB;AACpD,WAAK,QAAQ;AAAA,QACX;AAAA,UACE,gBAAgB,KAAK;AAAA,UACrB,qBAAqB,KAAK;AAAA,QAC5B;AAAA,QACA;AAAA,MACF;AACA,aAAO;AAAA,IACT;AAEA,SAAK,QAAQ;AAAA,MACX;AAAA,QACE,gBAAgB,KAAK;AAAA,MACvB;AAAA,MACA;AAAA,IACF;AAEA,UAAM,wBAAwB,KAAK,MAAM,KAAK,OAAO;AACrD,SAAK,mBAAmB,qBAAqB;AAC7C,WAAO;AAAA,EACT;AAAA,EAEA,OAAO,4BAA4B;AAAA,IACjC;AAAA,IACA;AAAA,IACA;AAAA,EACF,GAIG;AA5NL;AA6NI,0BAAsB,0BAA0B,cAAc,IAAI;AAElE,QAAI,sBAAsB,gCAAgC;AACxD;AAAA,IACF;AAEA,UAAM,aAAa,OAAO,SAA6C;AACrE,YAAMA,WAAU,sBAAsB,0BAA0B,KAAK,cAAc;AACnF,UAAI,CAACA,UAAS;AACZ,4BAAI,EAAE;AAAA,UACJ;AAAA,YACE,gBAAgB,KAAK;AAAA,YACrB,oBAAoB,OAAO,KAAK,sBAAsB,yBAAyB;AAAA,UACjF;AAAA,UACA;AAAA,QACF;AAEA,eAAO;AAAA,MACT;AACA,aAAOA,SAAQ,IAAI;AAAA,IACrB;AAEA,eAAK,qBAAL,mBAAuB,kBAAkB,uBAAuB;AAChE,0BAAsB,iCAAiC;AAAA,EACzD;AACF;","names":["handler"]}
+{"version":3,"sources":["../../../src/voice/avatar/datastream_io.ts"],"sourcesContent":["// SPDX-FileCopyrightText: 2025 LiveKit, Inc.\n//\n// SPDX-License-Identifier: Apache-2.0\nimport { Mutex } from '@livekit/mutex';\nimport {\n  type AudioFrame,\n  type ByteStreamWriter,\n  type Room,\n  RoomEvent,\n  type RpcInvocationData,\n  type TrackKind,\n} from '@livekit/rtc-node';\nimport { log } from '../../log.js';\nimport {\n  Future,\n  Task,\n  shortuuid,\n  waitForParticipant,\n  waitForTrackPublication,\n} from '../../utils.js';\nimport { AudioOutput, type PlaybackFinishedEvent } from '../io.js';\n\nconst RPC_CLEAR_BUFFER = 'lk.clear_buffer';\nconst RPC_PLAYBACK_FINISHED = 'lk.playback_finished';\nconst AUDIO_STREAM_TOPIC = 'lk.audio_stream';\n\nexport interface DataStreamAudioOutputOptions {\n  room: Room;\n  destinationIdentity: string;\n  sampleRate?: number;\n  waitRemoteTrack?: TrackKind;\n}\n\n/**\n * AudioOutput implementation that streams audio to a remote avatar worker using LiveKit DataStream.\n */\nexport class DataStreamAudioOutput extends AudioOutput {\n  static _playbackFinishedRpcRegistered: boolean = false;\n  static _playbackFinishedHandlers: Record<string, (data: RpcInvocationData) => string> = {};\n\n  private room: Room;\n  private destinationIdentity: string;\n  private roomConnectedFuture: Future<void>;\n  private waitRemoteTrack?: TrackKind;\n  private streamWriter?: ByteStreamWriter;\n  private pushedDuration: number = 0;\n  private started: boolean = false;\n  private lock = new Mutex();\n  private startTask?: Task<void>;\n  private firstFrameEmitted: boolean = false;\n\n  #logger = log();\n\n  constructor(opts: DataStreamAudioOutputOptions) {\n    super(opts.sampleRate, undefined, { pause: false });\n\n    const { room, destinationIdentity, sampleRate, waitRemoteTrack } = opts;\n    this.room = room;\n    this.destinationIdentity = destinationIdentity;\n    this.sampleRate = sampleRate;\n    this.waitRemoteTrack = waitRemoteTrack;\n\n    const onRoomConnected = async () => {\n      if (this.startTask) return;\n\n      await this.roomConnectedFuture.await;\n\n      // register the rpc method right after the room is connected\n      DataStreamAudioOutput.registerPlaybackFinishedRpc({\n        room,\n        callerIdentity: this.destinationIdentity,\n        handler: (data) => this.handlePlaybackFinished(data),\n      });\n\n      this.startTask = Task.from(({ signal }) => this._start(signal));\n    };\n\n    this.roomConnectedFuture = new Future<void>();\n\n    this.room.on(RoomEvent.ConnectionStateChanged, (_) => {\n      if (room.isConnected && !this.roomConnectedFuture.done) {\n        this.roomConnectedFuture.resolve(undefined);\n      }\n    });\n\n    if (this.room.isConnected) {\n      this.roomConnectedFuture.resolve(undefined);\n    }\n\n    onRoomConnected();\n  }\n\n  private async _start(_abortSignal: AbortSignal) {\n    const unlock = await this.lock.lock();\n\n    try {\n      if (this.started) return;\n\n      await this.roomConnectedFuture.await;\n\n      this.#logger.debug(\n        {\n          identity: this.destinationIdentity,\n        },\n        'waiting for the remote participant',\n      );\n\n      await waitForParticipant({\n        room: this.room,\n        identity: this.destinationIdentity,\n      });\n\n      if (this.waitRemoteTrack) {\n        this.#logger.debug(\n          {\n            identity: this.destinationIdentity,\n            kind: this.waitRemoteTrack,\n          },\n          'waiting for the remote track',\n        );\n\n        await waitForTrackPublication({\n          room: this.room,\n          identity: this.destinationIdentity,\n          kind: this.waitRemoteTrack,\n        });\n      }\n\n      this.#logger.debug(\n        {\n          identity: this.destinationIdentity,\n        },\n        'remote participant ready',\n      );\n\n      this.started = true;\n    } finally {\n      unlock();\n    }\n  }\n\n  async captureFrame(frame: AudioFrame): Promise<void> {\n    if (!this.startTask) {\n      this.startTask = Task.from(({ signal }) => this._start(signal));\n    }\n\n    await this.startTask.result;\n    await super.captureFrame(frame);\n\n    if (!this.firstFrameEmitted) {\n      this.firstFrameEmitted = true;\n      this.onPlaybackStarted(Date.now());\n    }\n\n    if (!this.streamWriter) {\n      this.streamWriter = await this.room.localParticipant!.streamBytes({\n        name: shortuuid('AUDIO_'),\n        topic: AUDIO_STREAM_TOPIC,\n        destinationIdentities: [this.destinationIdentity],\n        attributes: {\n          sample_rate: frame.sampleRate.toString(),\n          num_channels: frame.channels.toString(),\n        },\n      });\n      this.pushedDuration = 0;\n    }\n\n    // frame.data is a Int16Array, write accepts a Uint8Array\n    await this.streamWriter.write(new Uint8Array(frame.data.buffer));\n    this.pushedDuration += frame.samplesPerChannel / frame.sampleRate;\n  }\n\n  flush(): void {\n    super.flush();\n\n    if (this.streamWriter === undefined || !this.started) {\n      return;\n    }\n\n    this.streamWriter.close().finally(() => {\n      this.streamWriter = undefined;\n    });\n\n    this.firstFrameEmitted = false;\n  }\n\n  clearBuffer(): void {\n    if (!this.started) return;\n\n    this.room.localParticipant!.performRpc({\n      destinationIdentity: this.destinationIdentity,\n      method: RPC_CLEAR_BUFFER,\n      payload: '',\n    });\n  }\n\n  private handlePlaybackFinished(data: RpcInvocationData): string {\n    if (data.callerIdentity !== this.destinationIdentity) {\n      this.#logger.warn(\n        {\n          callerIdentity: data.callerIdentity,\n          destinationIdentity: this.destinationIdentity,\n        },\n        'playback finished event received from unexpected participant',\n      );\n      return 'reject';\n    }\n\n    this.#logger.info(\n      {\n        callerIdentity: data.callerIdentity,\n      },\n      'playback finished event received',\n    );\n\n    const playbackFinishedEvent = JSON.parse(data.payload) as PlaybackFinishedEvent;\n    this.onPlaybackFinished(playbackFinishedEvent);\n    return 'ok';\n  }\n\n  static registerPlaybackFinishedRpc({\n    room,\n    callerIdentity,\n    handler,\n  }: {\n    room: Room;\n    callerIdentity: string;\n    handler: (data: RpcInvocationData) => string;\n  }) {\n    DataStreamAudioOutput._playbackFinishedHandlers[callerIdentity] = handler;\n\n    if (DataStreamAudioOutput._playbackFinishedRpcRegistered) {\n      return;\n    }\n\n    const rpcHandler = async (data: RpcInvocationData): Promise<string> => {\n      const handler = DataStreamAudioOutput._playbackFinishedHandlers[data.callerIdentity];\n      if (!handler) {\n        log().warn(\n          {\n            callerIdentity: data.callerIdentity,\n            expectedIdentities: Object.keys(DataStreamAudioOutput._playbackFinishedHandlers),\n          },\n          'playback finished event received from unexpected participant',\n        );\n\n        return 'reject';\n      }\n      return handler(data);\n    };\n\n    room.localParticipant?.registerRpcMethod(RPC_PLAYBACK_FINISHED, rpcHandler);\n    DataStreamAudioOutput._playbackFinishedRpcRegistered = true;\n  }\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;AAAA;AAAA;AAAA;AAAA;AAAA;AAGA,mBAAsB;AACtB,sBAOO;AACP,iBAAoB;AACpB,mBAMO;AACP,gBAAwD;AAExD,MAAM,mBAAmB;AACzB,MAAM,wBAAwB;AAC9B,MAAM,qBAAqB;AAYpB,MAAM,8BAA8B,sBAAY;AAAA,EACrD,OAAO,iCAA0C;AAAA,EACjD,OAAO,4BAAiF,CAAC;AAAA,EAEjF;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA,iBAAyB;AAAA,EACzB,UAAmB;AAAA,EACnB,OAAO,IAAI,mBAAM;AAAA,EACjB;AAAA,EACA,oBAA6B;AAAA,EAErC,cAAU,gBAAI;AAAA,EAEd,YAAY,MAAoC;AAC9C,UAAM,KAAK,YAAY,QAAW,EAAE,OAAO,MAAM,CAAC;AAElD,UAAM,EAAE,MAAM,qBAAqB,YAAY,gBAAgB,IAAI;AACnE,SAAK,OAAO;AACZ,SAAK,sBAAsB;AAC3B,SAAK,aAAa;AAClB,SAAK,kBAAkB;AAEvB,UAAM,kBAAkB,YAAY;AAClC,UAAI,KAAK,UAAW;AAEpB,YAAM,KAAK,oBAAoB;AAG/B,4BAAsB,4BAA4B;AAAA,QAChD;AAAA,QACA,gBAAgB,KAAK;AAAA,QACrB,SAAS,CAAC,SAAS,KAAK,uBAAuB,IAAI;AAAA,MACrD,CAAC;AAED,WAAK,YAAY,kBAAK,KAAK,CAAC,EAAE,OAAO,MAAM,KAAK,OAAO,MAAM,CAAC;AAAA,IAChE;AAEA,SAAK,sBAAsB,IAAI,oBAAa;AAE5C,SAAK,KAAK,GAAG,0BAAU,wBAAwB,CAAC,MAAM;AACpD,UAAI,KAAK,eAAe,CAAC,KAAK,oBAAoB,MAAM;AACtD,aAAK,oBAAoB,QAAQ,MAAS;AAAA,MAC5C;AAAA,IACF,CAAC;AAED,QAAI,KAAK,KAAK,aAAa;AACzB,WAAK,oBAAoB,QAAQ,MAAS;AAAA,IAC5C;AAEA,oBAAgB;AAAA,EAClB;AAAA,EAEA,MAAc,OAAO,cAA2B;AAC9C,UAAM,SAAS,MAAM,KAAK,KAAK,KAAK;AAEpC,QAAI;AACF,UAAI,KAAK,QAAS;AAElB,YAAM,KAAK,oBAAoB;AAE/B,WAAK,QAAQ;AAAA,QACX;AAAA,UACE,UAAU,KAAK;AAAA,QACjB;AAAA,QACA;AAAA,MACF;AAEA,gBAAM,iCAAmB;AAAA,QACvB,MAAM,KAAK;AAAA,QACX,UAAU,KAAK;AAAA,MACjB,CAAC;AAED,UAAI,KAAK,iBAAiB;AACxB,aAAK,QAAQ;AAAA,UACX;AAAA,YACE,UAAU,KAAK;AAAA,YACf,MAAM,KAAK;AAAA,UACb;AAAA,UACA;AAAA,QACF;AAEA,kBAAM,sCAAwB;AAAA,UAC5B,MAAM,KAAK;AAAA,UACX,UAAU,KAAK;AAAA,UACf,MAAM,KAAK;AAAA,QACb,CAAC;AAAA,MACH;AAEA,WAAK,QAAQ;AAAA,QACX;AAAA,UACE,UAAU,KAAK;AAAA,QACjB;AAAA,QACA;AAAA,MACF;AAEA,WAAK,UAAU;AAAA,IACjB,UAAE;AACA,aAAO;AAAA,IACT;AAAA,EACF;AAAA,EAEA,MAAM,aAAa,OAAkC;AACnD,QAAI,CAAC,KAAK,WAAW;AACnB,WAAK,YAAY,kBAAK,KAAK,CAAC,EAAE,OAAO,MAAM,KAAK,OAAO,MAAM,CAAC;AAAA,IAChE;AAEA,UAAM,KAAK,UAAU;AACrB,UAAM,MAAM,aAAa,KAAK;AAE9B,QAAI,CAAC,KAAK,mBAAmB;AAC3B,WAAK,oBAAoB;AACzB,WAAK,kBAAkB,KAAK,IAAI,CAAC;AAAA,IACnC;AAEA,QAAI,CAAC,KAAK,cAAc;AACtB,WAAK,eAAe,MAAM,KAAK,KAAK,iBAAkB,YAAY;AAAA,QAChE,UAAM,wBAAU,QAAQ;AAAA,QACxB,OAAO;AAAA,QACP,uBAAuB,CAAC,KAAK,mBAAmB;AAAA,QAChD,YAAY;AAAA,UACV,aAAa,MAAM,WAAW,SAAS;AAAA,UACvC,cAAc,MAAM,SAAS,SAAS;AAAA,QACxC;AAAA,MACF,CAAC;AACD,WAAK,iBAAiB;AAAA,IACxB;AAGA,UAAM,KAAK,aAAa,MAAM,IAAI,WAAW,MAAM,KAAK,MAAM,CAAC;AAC/D,SAAK,kBAAkB,MAAM,oBAAoB,MAAM;AAAA,EACzD;AAAA,EAEA,QAAc;AACZ,UAAM,MAAM;AAEZ,QAAI,KAAK,iBAAiB,UAAa,CAAC,KAAK,SAAS;AACpD;AAAA,IACF;AAEA,SAAK,aAAa,MAAM,EAAE,QAAQ,MAAM;AACtC,WAAK,eAAe;AAAA,IACtB,CAAC;AAED,SAAK,oBAAoB;AAAA,EAC3B;AAAA,EAEA,cAAoB;AAClB,QAAI,CAAC,KAAK,QAAS;AAEnB,SAAK,KAAK,iBAAkB,WAAW;AAAA,MACrC,qBAAqB,KAAK;AAAA,MAC1B,QAAQ;AAAA,MACR,SAAS;AAAA,IACX,CAAC;AAAA,EACH;AAAA,EAEQ,uBAAuB,MAAiC;AAC9D,QAAI,KAAK,mBAAmB,KAAK,qBAAqB;AACpD,WAAK,QAAQ;AAAA,QACX;AAAA,UACE,gBAAgB,KAAK;AAAA,UACrB,qBAAqB,KAAK;AAAA,QAC5B;AAAA,QACA;AAAA,MACF;AACA,aAAO;AAAA,IACT;AAEA,SAAK,QAAQ;AAAA,MACX;AAAA,QACE,gBAAgB,KAAK;AAAA,MACvB;AAAA,MACA;AAAA,IACF;AAEA,UAAM,wBAAwB,KAAK,MAAM,KAAK,OAAO;AACrD,SAAK,mBAAmB,qBAAqB;AAC7C,WAAO;AAAA,EACT;AAAA,EAEA,OAAO,4BAA4B;AAAA,IACjC;AAAA,IACA;AAAA,IACA;AAAA,EACF,GAIG;AApOL;AAqOI,0BAAsB,0BAA0B,cAAc,IAAI;AAElE,QAAI,sBAAsB,gCAAgC;AACxD;AAAA,IACF;AAEA,UAAM,aAAa,OAAO,SAA6C;AACrE,YAAMA,WAAU,sBAAsB,0BAA0B,KAAK,cAAc;AACnF,UAAI,CAACA,UAAS;AACZ,4BAAI,EAAE;AAAA,UACJ;AAAA,YACE,gBAAgB,KAAK;AAAA,YACrB,oBAAoB,OAAO,KAAK,sBAAsB,yBAAyB;AAAA,UACjF;AAAA,UACA;AAAA,QACF;AAEA,eAAO;AAAA,MACT;AACA,aAAOA,SAAQ,IAAI;AAAA,IACrB;AAEA,eAAK,qBAAL,mBAAuB,kBAAkB,uBAAuB;AAChE,0BAAsB,iCAAiC;AAAA,EACzD;AACF;","names":["handler"]}

package/dist/voice/avatar/datastream_io.d.cts

@@ -22,6 +22,7 @@ export declare class DataStreamAudioOutput extends AudioOutput {
     private started;
     private lock;
     private startTask?;
+    private firstFrameEmitted;
     constructor(opts: DataStreamAudioOutputOptions);
     private _start;
     captureFrame(frame: AudioFrame): Promise<void>;

package/dist/voice/avatar/datastream_io.d.ts

@@ -22,6 +22,7 @@ export declare class DataStreamAudioOutput extends AudioOutput {
     private started;
     private lock;
     private startTask?;
+    private firstFrameEmitted;
     constructor(opts: DataStreamAudioOutputOptions);
     private _start;
     captureFrame(frame: AudioFrame): Promise<void>;

package/dist/voice/avatar/datastream_io.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"datastream_io.d.ts","sourceRoot":"","sources":["../../../src/voice/avatar/datastream_io.ts"],"names":[],"mappings":"AAIA,OAAO,EACL,KAAK,UAAU,EAEf,KAAK,IAAI,EAET,KAAK,iBAAiB,EACtB,KAAK,SAAS,EACf,MAAM,mBAAmB,CAAC;AAS3B,OAAO,EAAE,WAAW,EAA8B,MAAM,UAAU,CAAC;AAMnE,MAAM,WAAW,4BAA4B;IAC3C,IAAI,EAAE,IAAI,CAAC;IACX,mBAAmB,EAAE,MAAM,CAAC;IAC5B,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,eAAe,CAAC,EAAE,SAAS,CAAC;CAC7B;AAED;;GAEG;AACH,qBAAa,qBAAsB,SAAQ,WAAW;;IACpD,MAAM,CAAC,8BAA8B,EAAE,OAAO,CAAS;IACvD,MAAM,CAAC,yBAAyB,EAAE,MAAM,CAAC,MAAM,EAAE,CAAC,IAAI,EAAE,iBAAiB,KAAK,MAAM,CAAC,CAAM;IAE3F,OAAO,CAAC,IAAI,CAAO;IACnB,OAAO,CAAC,mBAAmB,CAAS;IACpC,OAAO,CAAC,mBAAmB,CAAe;IAC1C,OAAO,CAAC,eAAe,CAAC,CAAY;IACpC,OAAO,CAAC,YAAY,CAAC,CAAmB;IACxC,OAAO,CAAC,cAAc,CAAa;IACnC,OAAO,CAAC,OAAO,CAAkB;IACjC,OAAO,CAAC,IAAI,CAAe;IAC3B,OAAO,CAAC,SAAS,CAAC,CAAa;gBAInB,IAAI,EAAE,4BAA4B;YAuChC,MAAM;IAiDd,YAAY,CAAC,KAAK,EAAE,UAAU,GAAG,OAAO,CAAC,IAAI,CAAC;IA0BpD,KAAK,IAAI,IAAI;IAYb,WAAW,IAAI,IAAI;IAUnB,OAAO,CAAC,sBAAsB;IAwB9B,MAAM,CAAC,2BAA2B,CAAC,EACjC,IAAI,EACJ,cAAc,EACd,OAAO,GACR,EAAE;QACD,IAAI,EAAE,IAAI,CAAC;QACX,cAAc,EAAE,MAAM,CAAC;QACvB,OAAO,EAAE,CAAC,IAAI,EAAE,iBAAiB,KAAK,MAAM,CAAC;KAC9C;CA0BF"}
+{"version":3,"file":"datastream_io.d.ts","sourceRoot":"","sources":["../../../src/voice/avatar/datastream_io.ts"],"names":[],"mappings":"AAIA,OAAO,EACL,KAAK,UAAU,EAEf,KAAK,IAAI,EAET,KAAK,iBAAiB,EACtB,KAAK,SAAS,EACf,MAAM,mBAAmB,CAAC;AAS3B,OAAO,EAAE,WAAW,EAA8B,MAAM,UAAU,CAAC;AAMnE,MAAM,WAAW,4BAA4B;IAC3C,IAAI,EAAE,IAAI,CAAC;IACX,mBAAmB,EAAE,MAAM,CAAC;IAC5B,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,eAAe,CAAC,EAAE,SAAS,CAAC;CAC7B;AAED;;GAEG;AACH,qBAAa,qBAAsB,SAAQ,WAAW;;IACpD,MAAM,CAAC,8BAA8B,EAAE,OAAO,CAAS;IACvD,MAAM,CAAC,yBAAyB,EAAE,MAAM,CAAC,MAAM,EAAE,CAAC,IAAI,EAAE,iBAAiB,KAAK,MAAM,CAAC,CAAM;IAE3F,OAAO,CAAC,IAAI,CAAO;IACnB,OAAO,CAAC,mBAAmB,CAAS;IACpC,OAAO,CAAC,mBAAmB,CAAe;IAC1C,OAAO,CAAC,eAAe,CAAC,CAAY;IACpC,OAAO,CAAC,YAAY,CAAC,CAAmB;IACxC,OAAO,CAAC,cAAc,CAAa;IACnC,OAAO,CAAC,OAAO,CAAkB;IACjC,OAAO,CAAC,IAAI,CAAe;IAC3B,OAAO,CAAC,SAAS,CAAC,CAAa;IAC/B,OAAO,CAAC,iBAAiB,CAAkB;gBAI/B,IAAI,EAAE,4BAA4B;YAuChC,MAAM;IAiDd,YAAY,CAAC,KAAK,EAAE,UAAU,GAAG,OAAO,CAAC,IAAI,CAAC;IA+BpD,KAAK,IAAI,IAAI;IAcb,WAAW,IAAI,IAAI;IAUnB,OAAO,CAAC,sBAAsB;IAwB9B,MAAM,CAAC,2BAA2B,CAAC,EACjC,IAAI,EACJ,cAAc,EACd,OAAO,GACR,EAAE;QACD,IAAI,EAAE,IAAI,CAAC;QACX,cAAc,EAAE,MAAM,CAAC;QACvB,OAAO,EAAE,CAAC,IAAI,EAAE,iBAAiB,KAAK,MAAM,CAAC;KAC9C;CA0BF"}

package/dist/voice/avatar/datastream_io.js

@@ -26,9 +26,10 @@ class DataStreamAudioOutput extends AudioOutput {
   started = false;
   lock = new Mutex();
   startTask;
+  firstFrameEmitted = false;
   #logger = log();
   constructor(opts) {
-    super(opts.sampleRate, void 0);
+    super(opts.sampleRate, void 0, { pause: false });
     const { room, destinationIdentity, sampleRate, waitRemoteTrack } = opts;
     this.room = room;
     this.destinationIdentity = destinationIdentity;
@@ -101,6 +102,10 @@ class DataStreamAudioOutput extends AudioOutput {
     }
     await this.startTask.result;
     await super.captureFrame(frame);
+    if (!this.firstFrameEmitted) {
+      this.firstFrameEmitted = true;
+      this.onPlaybackStarted(Date.now());
+    }
     if (!this.streamWriter) {
       this.streamWriter = await this.room.localParticipant.streamBytes({
         name: shortuuid("AUDIO_"),
@@ -124,6 +129,7 @@ class DataStreamAudioOutput extends AudioOutput {
     this.streamWriter.close().finally(() => {
       this.streamWriter = void 0;
     });
+    this.firstFrameEmitted = false;
   }
   clearBuffer() {
     if (!this.started) return;

package/dist/voice/avatar/datastream_io.js.map

@@ -1 +1 @@
-{"version":3,"sources":["../../../src/voice/avatar/datastream_io.ts"],"sourcesContent":["// SPDX-FileCopyrightText: 2025 LiveKit, Inc.\n//\n// SPDX-License-Identifier: Apache-2.0\nimport { Mutex } from '@livekit/mutex';\nimport {\n  type AudioFrame,\n  type ByteStreamWriter,\n  type Room,\n  RoomEvent,\n  type RpcInvocationData,\n  type TrackKind,\n} from '@livekit/rtc-node';\nimport { log } from '../../log.js';\nimport {\n  Future,\n  Task,\n  shortuuid,\n  waitForParticipant,\n  waitForTrackPublication,\n} from '../../utils.js';\nimport { AudioOutput, type PlaybackFinishedEvent } from '../io.js';\n\nconst RPC_CLEAR_BUFFER = 'lk.clear_buffer';\nconst RPC_PLAYBACK_FINISHED = 'lk.playback_finished';\nconst AUDIO_STREAM_TOPIC = 'lk.audio_stream';\n\nexport interface DataStreamAudioOutputOptions {\n  room: Room;\n  destinationIdentity: string;\n  sampleRate?: number;\n  waitRemoteTrack?: TrackKind;\n}\n\n/**\n * AudioOutput implementation that streams audio to a remote avatar worker using LiveKit DataStream.\n */\nexport class DataStreamAudioOutput extends AudioOutput {\n  static _playbackFinishedRpcRegistered: boolean = false;\n  static _playbackFinishedHandlers: Record<string, (data: RpcInvocationData) => string> = {};\n\n  private room: Room;\n  private destinationIdentity: string;\n  private roomConnectedFuture: Future<void>;\n  private waitRemoteTrack?: TrackKind;\n  private streamWriter?: ByteStreamWriter;\n  private pushedDuration: number = 0;\n  private started: boolean = false;\n  private lock = new Mutex();\n  private startTask?: Task<void>;\n\n  #logger = log();\n\n  constructor(opts: DataStreamAudioOutputOptions) {\n    super(opts.sampleRate, undefined);\n\n    const { room, destinationIdentity, sampleRate, waitRemoteTrack } = opts;\n    this.room = room;\n    this.destinationIdentity = destinationIdentity;\n    this.sampleRate = sampleRate;\n    this.waitRemoteTrack = waitRemoteTrack;\n\n    const onRoomConnected = async () => {\n      if (this.startTask) return;\n\n      await this.roomConnectedFuture.await;\n\n      // register the rpc method right after the room is connected\n      DataStreamAudioOutput.registerPlaybackFinishedRpc({\n        room,\n        callerIdentity: this.destinationIdentity,\n        handler: (data) => this.handlePlaybackFinished(data),\n      });\n\n      this.startTask = Task.from(({ signal }) => this._start(signal));\n    };\n\n    this.roomConnectedFuture = new Future<void>();\n\n    this.room.on(RoomEvent.ConnectionStateChanged, (_) => {\n      if (room.isConnected && !this.roomConnectedFuture.done) {\n        this.roomConnectedFuture.resolve(undefined);\n      }\n    });\n\n    if (this.room.isConnected) {\n      this.roomConnectedFuture.resolve(undefined);\n    }\n\n    onRoomConnected();\n  }\n\n  private async _start(_abortSignal: AbortSignal) {\n    const unlock = await this.lock.lock();\n\n    try {\n      if (this.started) return;\n\n      await this.roomConnectedFuture.await;\n\n      this.#logger.debug(\n        {\n          identity: this.destinationIdentity,\n        },\n        'waiting for the remote participant',\n      );\n\n      await waitForParticipant({\n        room: this.room,\n        identity: this.destinationIdentity,\n      });\n\n      if (this.waitRemoteTrack) {\n        this.#logger.debug(\n          {\n            identity: this.destinationIdentity,\n            kind: this.waitRemoteTrack,\n          },\n          'waiting for the remote track',\n        );\n\n        await waitForTrackPublication({\n          room: this.room,\n          identity: this.destinationIdentity,\n          kind: this.waitRemoteTrack,\n        });\n      }\n\n      this.#logger.debug(\n        {\n          identity: this.destinationIdentity,\n        },\n        'remote participant ready',\n      );\n\n      this.started = true;\n    } finally {\n      unlock();\n    }\n  }\n\n  async captureFrame(frame: AudioFrame): Promise<void> {\n    if (!this.startTask) {\n      this.startTask = Task.from(({ signal }) => this._start(signal));\n    }\n\n    await this.startTask.result;\n    await super.captureFrame(frame);\n\n    if (!this.streamWriter) {\n      this.streamWriter = await this.room.localParticipant!.streamBytes({\n        name: shortuuid('AUDIO_'),\n        topic: AUDIO_STREAM_TOPIC,\n        destinationIdentities: [this.destinationIdentity],\n        attributes: {\n          sample_rate: frame.sampleRate.toString(),\n          num_channels: frame.channels.toString(),\n        },\n      });\n      this.pushedDuration = 0;\n    }\n\n    // frame.data is a Int16Array, write accepts a Uint8Array\n    await this.streamWriter.write(new Uint8Array(frame.data.buffer));\n    this.pushedDuration += frame.samplesPerChannel / frame.sampleRate;\n  }\n\n  flush(): void {\n    super.flush();\n\n    if (this.streamWriter === undefined || !this.started) {\n      return;\n    }\n\n    this.streamWriter.close().finally(() => {\n      this.streamWriter = undefined;\n    });\n  }\n\n  clearBuffer(): void {\n    if (!this.started) return;\n\n    this.room.localParticipant!.performRpc({\n      destinationIdentity: this.destinationIdentity,\n      method: RPC_CLEAR_BUFFER,\n      payload: '',\n    });\n  }\n\n  private handlePlaybackFinished(data: RpcInvocationData): string {\n    if (data.callerIdentity !== this.destinationIdentity) {\n      this.#logger.warn(\n        {\n          callerIdentity: data.callerIdentity,\n          destinationIdentity: this.destinationIdentity,\n        },\n        'playback finished event received from unexpected participant',\n      );\n      return 'reject';\n    }\n\n    this.#logger.info(\n      {\n        callerIdentity: data.callerIdentity,\n      },\n      'playback finished event received',\n    );\n\n    const playbackFinishedEvent = JSON.parse(data.payload) as PlaybackFinishedEvent;\n    this.onPlaybackFinished(playbackFinishedEvent);\n    return 'ok';\n  }\n\n  static registerPlaybackFinishedRpc({\n    room,\n    callerIdentity,\n    handler,\n  }: {\n    room: Room;\n    callerIdentity: string;\n    handler: (data: RpcInvocationData) => string;\n  }) {\n    DataStreamAudioOutput._playbackFinishedHandlers[callerIdentity] = handler;\n\n    if (DataStreamAudioOutput._playbackFinishedRpcRegistered) {\n      return;\n    }\n\n    const rpcHandler = async (data: RpcInvocationData): Promise<string> => {\n      const handler = DataStreamAudioOutput._playbackFinishedHandlers[data.callerIdentity];\n      if (!handler) {\n        log().warn(\n          {\n            callerIdentity: data.callerIdentity,\n            expectedIdentities: Object.keys(DataStreamAudioOutput._playbackFinishedHandlers),\n          },\n          'playback finished event received from unexpected participant',\n        );\n\n        return 'reject';\n      }\n      return handler(data);\n    };\n\n    room.localParticipant?.registerRpcMethod(RPC_PLAYBACK_FINISHED, rpcHandler);\n    DataStreamAudioOutput._playbackFinishedRpcRegistered = true;\n  }\n}\n"],"mappings":"AAGA,SAAS,aAAa;AACtB;AAAA,EAIE;AAAA,OAGK;AACP,SAAS,WAAW;AACpB;AAAA,EACE;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,OACK;AACP,SAAS,mBAA+C;AAExD,MAAM,mBAAmB;AACzB,MAAM,wBAAwB;AAC9B,MAAM,qBAAqB;AAYpB,MAAM,8BAA8B,YAAY;AAAA,EACrD,OAAO,iCAA0C;AAAA,EACjD,OAAO,4BAAiF,CAAC;AAAA,EAEjF;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA,iBAAyB;AAAA,EACzB,UAAmB;AAAA,EACnB,OAAO,IAAI,MAAM;AAAA,EACjB;AAAA,EAER,UAAU,IAAI;AAAA,EAEd,YAAY,MAAoC;AAC9C,UAAM,KAAK,YAAY,MAAS;AAEhC,UAAM,EAAE,MAAM,qBAAqB,YAAY,gBAAgB,IAAI;AACnE,SAAK,OAAO;AACZ,SAAK,sBAAsB;AAC3B,SAAK,aAAa;AAClB,SAAK,kBAAkB;AAEvB,UAAM,kBAAkB,YAAY;AAClC,UAAI,KAAK,UAAW;AAEpB,YAAM,KAAK,oBAAoB;AAG/B,4BAAsB,4BAA4B;AAAA,QAChD;AAAA,QACA,gBAAgB,KAAK;AAAA,QACrB,SAAS,CAAC,SAAS,KAAK,uBAAuB,IAAI;AAAA,MACrD,CAAC;AAED,WAAK,YAAY,KAAK,KAAK,CAAC,EAAE,OAAO,MAAM,KAAK,OAAO,MAAM,CAAC;AAAA,IAChE;AAEA,SAAK,sBAAsB,IAAI,OAAa;AAE5C,SAAK,KAAK,GAAG,UAAU,wBAAwB,CAAC,MAAM;AACpD,UAAI,KAAK,eAAe,CAAC,KAAK,oBAAoB,MAAM;AACtD,aAAK,oBAAoB,QAAQ,MAAS;AAAA,MAC5C;AAAA,IACF,CAAC;AAED,QAAI,KAAK,KAAK,aAAa;AACzB,WAAK,oBAAoB,QAAQ,MAAS;AAAA,IAC5C;AAEA,oBAAgB;AAAA,EAClB;AAAA,EAEA,MAAc,OAAO,cAA2B;AAC9C,UAAM,SAAS,MAAM,KAAK,KAAK,KAAK;AAEpC,QAAI;AACF,UAAI,KAAK,QAAS;AAElB,YAAM,KAAK,oBAAoB;AAE/B,WAAK,QAAQ;AAAA,QACX;AAAA,UACE,UAAU,KAAK;AAAA,QACjB;AAAA,QACA;AAAA,MACF;AAEA,YAAM,mBAAmB;AAAA,QACvB,MAAM,KAAK;AAAA,QACX,UAAU,KAAK;AAAA,MACjB,CAAC;AAED,UAAI,KAAK,iBAAiB;AACxB,aAAK,QAAQ;AAAA,UACX;AAAA,YACE,UAAU,KAAK;AAAA,YACf,MAAM,KAAK;AAAA,UACb;AAAA,UACA;AAAA,QACF;AAEA,cAAM,wBAAwB;AAAA,UAC5B,MAAM,KAAK;AAAA,UACX,UAAU,KAAK;AAAA,UACf,MAAM,KAAK;AAAA,QACb,CAAC;AAAA,MACH;AAEA,WAAK,QAAQ;AAAA,QACX;AAAA,UACE,UAAU,KAAK;AAAA,QACjB;AAAA,QACA;AAAA,MACF;AAEA,WAAK,UAAU;AAAA,IACjB,UAAE;AACA,aAAO;AAAA,IACT;AAAA,EACF;AAAA,EAEA,MAAM,aAAa,OAAkC;AACnD,QAAI,CAAC,KAAK,WAAW;AACnB,WAAK,YAAY,KAAK,KAAK,CAAC,EAAE,OAAO,MAAM,KAAK,OAAO,MAAM,CAAC;AAAA,IAChE;AAEA,UAAM,KAAK,UAAU;AACrB,UAAM,MAAM,aAAa,KAAK;AAE9B,QAAI,CAAC,KAAK,cAAc;AACtB,WAAK,eAAe,MAAM,KAAK,KAAK,iBAAkB,YAAY;AAAA,QAChE,MAAM,UAAU,QAAQ;AAAA,QACxB,OAAO;AAAA,QACP,uBAAuB,CAAC,KAAK,mBAAmB;AAAA,QAChD,YAAY;AAAA,UACV,aAAa,MAAM,WAAW,SAAS;AAAA,UACvC,cAAc,MAAM,SAAS,SAAS;AAAA,QACxC;AAAA,MACF,CAAC;AACD,WAAK,iBAAiB;AAAA,IACxB;AAGA,UAAM,KAAK,aAAa,MAAM,IAAI,WAAW,MAAM,KAAK,MAAM,CAAC;AAC/D,SAAK,kBAAkB,MAAM,oBAAoB,MAAM;AAAA,EACzD;AAAA,EAEA,QAAc;AACZ,UAAM,MAAM;AAEZ,QAAI,KAAK,iBAAiB,UAAa,CAAC,KAAK,SAAS;AACpD;AAAA,IACF;AAEA,SAAK,aAAa,MAAM,EAAE,QAAQ,MAAM;AACtC,WAAK,eAAe;AAAA,IACtB,CAAC;AAAA,EACH;AAAA,EAEA,cAAoB;AAClB,QAAI,CAAC,KAAK,QAAS;AAEnB,SAAK,KAAK,iBAAkB,WAAW;AAAA,MACrC,qBAAqB,KAAK;AAAA,MAC1B,QAAQ;AAAA,MACR,SAAS;AAAA,IACX,CAAC;AAAA,EACH;AAAA,EAEQ,uBAAuB,MAAiC;AAC9D,QAAI,KAAK,mBAAmB,KAAK,qBAAqB;AACpD,WAAK,QAAQ;AAAA,QACX;AAAA,UACE,gBAAgB,KAAK;AAAA,UACrB,qBAAqB,KAAK;AAAA,QAC5B;AAAA,QACA;AAAA,MACF;AACA,aAAO;AAAA,IACT;AAEA,SAAK,QAAQ;AAAA,MACX;AAAA,QACE,gBAAgB,KAAK;AAAA,MACvB;AAAA,MACA;AAAA,IACF;AAEA,UAAM,wBAAwB,KAAK,MAAM,KAAK,OAAO;AACrD,SAAK,mBAAmB,qBAAqB;AAC7C,WAAO;AAAA,EACT;AAAA,EAEA,OAAO,4BAA4B;AAAA,IACjC;AAAA,IACA;AAAA,IACA;AAAA,EACF,GAIG;AA5NL;AA6NI,0BAAsB,0BAA0B,cAAc,IAAI;AAElE,QAAI,sBAAsB,gCAAgC;AACxD;AAAA,IACF;AAEA,UAAM,aAAa,OAAO,SAA6C;AACrE,YAAMA,WAAU,sBAAsB,0BAA0B,KAAK,cAAc;AACnF,UAAI,CAACA,UAAS;AACZ,YAAI,EAAE;AAAA,UACJ;AAAA,YACE,gBAAgB,KAAK;AAAA,YACrB,oBAAoB,OAAO,KAAK,sBAAsB,yBAAyB;AAAA,UACjF;AAAA,UACA;AAAA,QACF;AAEA,eAAO;AAAA,MACT;AACA,aAAOA,SAAQ,IAAI;AAAA,IACrB;AAEA,eAAK,qBAAL,mBAAuB,kBAAkB,uBAAuB;AAChE,0BAAsB,iCAAiC;AAAA,EACzD;AACF;","names":["handler"]}
+{"version":3,"sources":["../../../src/voice/avatar/datastream_io.ts"],"sourcesContent":["// SPDX-FileCopyrightText: 2025 LiveKit, Inc.\n//\n// SPDX-License-Identifier: Apache-2.0\nimport { Mutex } from '@livekit/mutex';\nimport {\n  type AudioFrame,\n  type ByteStreamWriter,\n  type Room,\n  RoomEvent,\n  type RpcInvocationData,\n  type TrackKind,\n} from '@livekit/rtc-node';\nimport { log } from '../../log.js';\nimport {\n  Future,\n  Task,\n  shortuuid,\n  waitForParticipant,\n  waitForTrackPublication,\n} from '../../utils.js';\nimport { AudioOutput, type PlaybackFinishedEvent } from '../io.js';\n\nconst RPC_CLEAR_BUFFER = 'lk.clear_buffer';\nconst RPC_PLAYBACK_FINISHED = 'lk.playback_finished';\nconst AUDIO_STREAM_TOPIC = 'lk.audio_stream';\n\nexport interface DataStreamAudioOutputOptions {\n  room: Room;\n  destinationIdentity: string;\n  sampleRate?: number;\n  waitRemoteTrack?: TrackKind;\n}\n\n/**\n * AudioOutput implementation that streams audio to a remote avatar worker using LiveKit DataStream.\n */\nexport class DataStreamAudioOutput extends AudioOutput {\n  static _playbackFinishedRpcRegistered: boolean = false;\n  static _playbackFinishedHandlers: Record<string, (data: RpcInvocationData) => string> = {};\n\n  private room: Room;\n  private destinationIdentity: string;\n  private roomConnectedFuture: Future<void>;\n  private waitRemoteTrack?: TrackKind;\n  private streamWriter?: ByteStreamWriter;\n  private pushedDuration: number = 0;\n  private started: boolean = false;\n  private lock = new Mutex();\n  private startTask?: Task<void>;\n  private firstFrameEmitted: boolean = false;\n\n  #logger = log();\n\n  constructor(opts: DataStreamAudioOutputOptions) {\n    super(opts.sampleRate, undefined, { pause: false });\n\n    const { room, destinationIdentity, sampleRate, waitRemoteTrack } = opts;\n    this.room = room;\n    this.destinationIdentity = destinationIdentity;\n    this.sampleRate = sampleRate;\n    this.waitRemoteTrack = waitRemoteTrack;\n\n    const onRoomConnected = async () => {\n      if (this.startTask) return;\n\n      await this.roomConnectedFuture.await;\n\n      // register the rpc method right after the room is connected\n      DataStreamAudioOutput.registerPlaybackFinishedRpc({\n        room,\n        callerIdentity: this.destinationIdentity,\n        handler: (data) => this.handlePlaybackFinished(data),\n      });\n\n      this.startTask = Task.from(({ signal }) => this._start(signal));\n    };\n\n    this.roomConnectedFuture = new Future<void>();\n\n    this.room.on(RoomEvent.ConnectionStateChanged, (_) => {\n      if (room.isConnected && !this.roomConnectedFuture.done) {\n        this.roomConnectedFuture.resolve(undefined);\n      }\n    });\n\n    if (this.room.isConnected) {\n      this.roomConnectedFuture.resolve(undefined);\n    }\n\n    onRoomConnected();\n  }\n\n  private async _start(_abortSignal: AbortSignal) {\n    const unlock = await this.lock.lock();\n\n    try {\n      if (this.started) return;\n\n      await this.roomConnectedFuture.await;\n\n      this.#logger.debug(\n        {\n          identity: this.destinationIdentity,\n        },\n        'waiting for the remote participant',\n      );\n\n      await waitForParticipant({\n        room: this.room,\n        identity: this.destinationIdentity,\n      });\n\n      if (this.waitRemoteTrack) {\n        this.#logger.debug(\n          {\n            identity: this.destinationIdentity,\n            kind: this.waitRemoteTrack,\n          },\n          'waiting for the remote track',\n        );\n\n        await waitForTrackPublication({\n          room: this.room,\n          identity: this.destinationIdentity,\n          kind: this.waitRemoteTrack,\n        });\n      }\n\n      this.#logger.debug(\n        {\n          identity: this.destinationIdentity,\n        },\n        'remote participant ready',\n      );\n\n      this.started = true;\n    } finally {\n      unlock();\n    }\n  }\n\n  async captureFrame(frame: AudioFrame): Promise<void> {\n    if (!this.startTask) {\n      this.startTask = Task.from(({ signal }) => this._start(signal));\n    }\n\n    await this.startTask.result;\n    await super.captureFrame(frame);\n\n    if (!this.firstFrameEmitted) {\n      this.firstFrameEmitted = true;\n      this.onPlaybackStarted(Date.now());\n    }\n\n    if (!this.streamWriter) {\n      this.streamWriter = await this.room.localParticipant!.streamBytes({\n        name: shortuuid('AUDIO_'),\n        topic: AUDIO_STREAM_TOPIC,\n        destinationIdentities: [this.destinationIdentity],\n        attributes: {\n          sample_rate: frame.sampleRate.toString(),\n          num_channels: frame.channels.toString(),\n        },\n      });\n      this.pushedDuration = 0;\n    }\n\n    // frame.data is a Int16Array, write accepts a Uint8Array\n    await this.streamWriter.write(new Uint8Array(frame.data.buffer));\n    this.pushedDuration += frame.samplesPerChannel / frame.sampleRate;\n  }\n\n  flush(): void {\n    super.flush();\n\n    if (this.streamWriter === undefined || !this.started) {\n      return;\n    }\n\n    this.streamWriter.close().finally(() => {\n      this.streamWriter = undefined;\n    });\n\n    this.firstFrameEmitted = false;\n  }\n\n  clearBuffer(): void {\n    if (!this.started) return;\n\n    this.room.localParticipant!.performRpc({\n      destinationIdentity: this.destinationIdentity,\n      method: RPC_CLEAR_BUFFER,\n      payload: '',\n    });\n  }\n\n  private handlePlaybackFinished(data: RpcInvocationData): string {\n    if (data.callerIdentity !== this.destinationIdentity) {\n      this.#logger.warn(\n        {\n          callerIdentity: data.callerIdentity,\n          destinationIdentity: this.destinationIdentity,\n        },\n        'playback finished event received from unexpected participant',\n      );\n      return 'reject';\n    }\n\n    this.#logger.info(\n      {\n        callerIdentity: data.callerIdentity,\n      },\n      'playback finished event received',\n    );\n\n    const playbackFinishedEvent = JSON.parse(data.payload) as PlaybackFinishedEvent;\n    this.onPlaybackFinished(playbackFinishedEvent);\n    return 'ok';\n  }\n\n  static registerPlaybackFinishedRpc({\n    room,\n    callerIdentity,\n    handler,\n  }: {\n    room: Room;\n    callerIdentity: string;\n    handler: (data: RpcInvocationData) => string;\n  }) {\n    DataStreamAudioOutput._playbackFinishedHandlers[callerIdentity] = handler;\n\n    if (DataStreamAudioOutput._playbackFinishedRpcRegistered) {\n      return;\n    }\n\n    const rpcHandler = async (data: RpcInvocationData): Promise<string> => {\n      const handler = DataStreamAudioOutput._playbackFinishedHandlers[data.callerIdentity];\n      if (!handler) {\n        log().warn(\n          {\n            callerIdentity: data.callerIdentity,\n            expectedIdentities: Object.keys(DataStreamAudioOutput._playbackFinishedHandlers),\n          },\n          'playback finished event received from unexpected participant',\n        );\n\n        return 'reject';\n      }\n      return handler(data);\n    };\n\n    room.localParticipant?.registerRpcMethod(RPC_PLAYBACK_FINISHED, rpcHandler);\n    DataStreamAudioOutput._playbackFinishedRpcRegistered = true;\n  }\n}\n"],"mappings":"AAGA,SAAS,aAAa;AACtB;AAAA,EAIE;AAAA,OAGK;AACP,SAAS,WAAW;AACpB;AAAA,EACE;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,OACK;AACP,SAAS,mBAA+C;AAExD,MAAM,mBAAmB;AACzB,MAAM,wBAAwB;AAC9B,MAAM,qBAAqB;AAYpB,MAAM,8BAA8B,YAAY;AAAA,EACrD,OAAO,iCAA0C;AAAA,EACjD,OAAO,4BAAiF,CAAC;AAAA,EAEjF;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA,iBAAyB;AAAA,EACzB,UAAmB;AAAA,EACnB,OAAO,IAAI,MAAM;AAAA,EACjB;AAAA,EACA,oBAA6B;AAAA,EAErC,UAAU,IAAI;AAAA,EAEd,YAAY,MAAoC;AAC9C,UAAM,KAAK,YAAY,QAAW,EAAE,OAAO,MAAM,CAAC;AAElD,UAAM,EAAE,MAAM,qBAAqB,YAAY,gBAAgB,IAAI;AACnE,SAAK,OAAO;AACZ,SAAK,sBAAsB;AAC3B,SAAK,aAAa;AAClB,SAAK,kBAAkB;AAEvB,UAAM,kBAAkB,YAAY;AAClC,UAAI,KAAK,UAAW;AAEpB,YAAM,KAAK,oBAAoB;AAG/B,4BAAsB,4BAA4B;AAAA,QAChD;AAAA,QACA,gBAAgB,KAAK;AAAA,QACrB,SAAS,CAAC,SAAS,KAAK,uBAAuB,IAAI;AAAA,MACrD,CAAC;AAED,WAAK,YAAY,KAAK,KAAK,CAAC,EAAE,OAAO,MAAM,KAAK,OAAO,MAAM,CAAC;AAAA,IAChE;AAEA,SAAK,sBAAsB,IAAI,OAAa;AAE5C,SAAK,KAAK,GAAG,UAAU,wBAAwB,CAAC,MAAM;AACpD,UAAI,KAAK,eAAe,CAAC,KAAK,oBAAoB,MAAM;AACtD,aAAK,oBAAoB,QAAQ,MAAS;AAAA,MAC5C;AAAA,IACF,CAAC;AAED,QAAI,KAAK,KAAK,aAAa;AACzB,WAAK,oBAAoB,QAAQ,MAAS;AAAA,IAC5C;AAEA,oBAAgB;AAAA,EAClB;AAAA,EAEA,MAAc,OAAO,cAA2B;AAC9C,UAAM,SAAS,MAAM,KAAK,KAAK,KAAK;AAEpC,QAAI;AACF,UAAI,KAAK,QAAS;AAElB,YAAM,KAAK,oBAAoB;AAE/B,WAAK,QAAQ;AAAA,QACX;AAAA,UACE,UAAU,KAAK;AAAA,QACjB;AAAA,QACA;AAAA,MACF;AAEA,YAAM,mBAAmB;AAAA,QACvB,MAAM,KAAK;AAAA,QACX,UAAU,KAAK;AAAA,MACjB,CAAC;AAED,UAAI,KAAK,iBAAiB;AACxB,aAAK,QAAQ;AAAA,UACX;AAAA,YACE,UAAU,KAAK;AAAA,YACf,MAAM,KAAK;AAAA,UACb;AAAA,UACA;AAAA,QACF;AAEA,cAAM,wBAAwB;AAAA,UAC5B,MAAM,KAAK;AAAA,UACX,UAAU,KAAK;AAAA,UACf,MAAM,KAAK;AAAA,QACb,CAAC;AAAA,MACH;AAEA,WAAK,QAAQ;AAAA,QACX;AAAA,UACE,UAAU,KAAK;AAAA,QACjB;AAAA,QACA;AAAA,MACF;AAEA,WAAK,UAAU;AAAA,IACjB,UAAE;AACA,aAAO;AAAA,IACT;AAAA,EACF;AAAA,EAEA,MAAM,aAAa,OAAkC;AACnD,QAAI,CAAC,KAAK,WAAW;AACnB,WAAK,YAAY,KAAK,KAAK,CAAC,EAAE,OAAO,MAAM,KAAK,OAAO,MAAM,CAAC;AAAA,IAChE;AAEA,UAAM,KAAK,UAAU;AACrB,UAAM,MAAM,aAAa,KAAK;AAE9B,QAAI,CAAC,KAAK,mBAAmB;AAC3B,WAAK,oBAAoB;AACzB,WAAK,kBAAkB,KAAK,IAAI,CAAC;AAAA,IACnC;AAEA,QAAI,CAAC,KAAK,cAAc;AACtB,WAAK,eAAe,MAAM,KAAK,KAAK,iBAAkB,YAAY;AAAA,QAChE,MAAM,UAAU,QAAQ;AAAA,QACxB,OAAO;AAAA,QACP,uBAAuB,CAAC,KAAK,mBAAmB;AAAA,QAChD,YAAY;AAAA,UACV,aAAa,MAAM,WAAW,SAAS;AAAA,UACvC,cAAc,MAAM,SAAS,SAAS;AAAA,QACxC;AAAA,MACF,CAAC;AACD,WAAK,iBAAiB;AAAA,IACxB;AAGA,UAAM,KAAK,aAAa,MAAM,IAAI,WAAW,MAAM,KAAK,MAAM,CAAC;AAC/D,SAAK,kBAAkB,MAAM,oBAAoB,MAAM;AAAA,EACzD;AAAA,EAEA,QAAc;AACZ,UAAM,MAAM;AAEZ,QAAI,KAAK,iBAAiB,UAAa,CAAC,KAAK,SAAS;AACpD;AAAA,IACF;AAEA,SAAK,aAAa,MAAM,EAAE,QAAQ,MAAM;AACtC,WAAK,eAAe;AAAA,IACtB,CAAC;AAED,SAAK,oBAAoB;AAAA,EAC3B;AAAA,EAEA,cAAoB;AAClB,QAAI,CAAC,KAAK,QAAS;AAEnB,SAAK,KAAK,iBAAkB,WAAW;AAAA,MACrC,qBAAqB,KAAK;AAAA,MAC1B,QAAQ;AAAA,MACR,SAAS;AAAA,IACX,CAAC;AAAA,EACH;AAAA,EAEQ,uBAAuB,MAAiC;AAC9D,QAAI,KAAK,mBAAmB,KAAK,qBAAqB;AACpD,WAAK,QAAQ;AAAA,QACX;AAAA,UACE,gBAAgB,KAAK;AAAA,UACrB,qBAAqB,KAAK;AAAA,QAC5B;AAAA,QACA;AAAA,MACF;AACA,aAAO;AAAA,IACT;AAEA,SAAK,QAAQ;AAAA,MACX;AAAA,QACE,gBAAgB,KAAK;AAAA,MACvB;AAAA,MACA;AAAA,IACF;AAEA,UAAM,wBAAwB,KAAK,MAAM,KAAK,OAAO;AACrD,SAAK,mBAAmB,qBAAqB;AAC7C,WAAO;AAAA,EACT;AAAA,EAEA,OAAO,4BAA4B;AAAA,IACjC;AAAA,IACA;AAAA,IACA;AAAA,EACF,GAIG;AApOL;AAqOI,0BAAsB,0BAA0B,cAAc,IAAI;AAElE,QAAI,sBAAsB,gCAAgC;AACxD;AAAA,IACF;AAEA,UAAM,aAAa,OAAO,SAA6C;AACrE,YAAMA,WAAU,sBAAsB,0BAA0B,KAAK,cAAc;AACnF,UAAI,CAACA,UAAS;AACZ,YAAI,EAAE;AAAA,UACJ;AAAA,YACE,gBAAgB,KAAK;AAAA,YACrB,oBAAoB,OAAO,KAAK,sBAAsB,yBAAyB;AAAA,UACjF;AAAA,UACA;AAAA,QACF;AAEA,eAAO;AAAA,MACT;AACA,aAAOA,SAAQ,IAAI;AAAA,IACrB;AAEA,eAAK,qBAAL,mBAAuB,kBAAkB,uBAAuB;AAChE,0BAAsB,iCAAiC;AAAA,EACzD;AACF;","names":["handler"]}