@livekit/rtc-node 0.13.2 → 0.13.4

package/src/participant.ts

@@ -1,33 +1,59 @@
 // SPDX-FileCopyrightText: 2024 LiveKit, Inc.
 //
 // SPDX-License-Identifier: Apache-2.0
+import { Mutex } from '@livekit/mutex';
+import type { PathLike } from 'node:fs';
+import { open, stat } from 'node:fs/promises';
+import {
+  type ByteStreamInfo,
+  type ByteStreamOptions,
+  ByteStreamWriter,
+  type TextStreamInfo,
+  TextStreamWriter,
+} from './data_streams/index.js';
 import { FfiClient, FfiHandle } from './ffi_client.js';
+import { log } from './log.js';
 import {
   DisconnectReason,
   type OwnedParticipant,
   type ParticipantInfo,
-  type ParticipantKind,
+  ParticipantKind,
 } from './proto/participant_pb.js';
-import type {
-  PublishDataCallback,
-  PublishDataResponse,
-  PublishSipDtmfCallback,
-  PublishSipDtmfResponse,
-  PublishTrackCallback,
-  PublishTrackResponse,
-  PublishTranscriptionCallback,
-  PublishTranscriptionResponse,
-  SendChatMessageCallback,
-  SendChatMessageResponse,
-  SetLocalAttributesCallback,
-  SetLocalAttributesResponse,
-  SetLocalMetadataCallback,
-  SetLocalMetadataResponse,
-  SetLocalNameCallback,
-  SetLocalNameResponse,
-  TrackPublishOptions,
-  UnpublishTrackCallback,
-  UnpublishTrackResponse,
+import {
+  DataStream_ByteHeader,
+  DataStream_Chunk,
+  DataStream_Header,
+  DataStream_OperationType,
+  DataStream_TextHeader,
+  DataStream_Trailer,
+  type PublishDataCallback,
+  type PublishDataResponse,
+  type PublishSipDtmfCallback,
+  type PublishSipDtmfResponse,
+  type PublishTrackCallback,
+  type PublishTrackResponse,
+  type PublishTranscriptionCallback,
+  type PublishTranscriptionResponse,
+  type SendChatMessageCallback,
+  type SendChatMessageResponse,
+  type SendStreamChunkCallback,
+  SendStreamChunkRequest,
+  type SendStreamChunkResponse,
+  type SendStreamHeaderCallback,
+  SendStreamHeaderRequest,
+  type SendStreamHeaderResponse,
+  type SendStreamTrailerCallback,
+  SendStreamTrailerRequest,
+  type SendStreamTrailerResponse,
+  type SetLocalAttributesCallback,
+  type SetLocalAttributesResponse,
+  type SetLocalMetadataCallback,
+  type SetLocalMetadataResponse,
+  type SetLocalNameCallback,
+  type SetLocalNameResponse,
+  type TrackPublishOptions,
+  type UnpublishTrackCallback,
+  type UnpublishTrackResponse,
 } from './proto/room_pb.js';
 import { ChatMessage as ChatMessageModel } from './proto/room_pb.js';
 import {
@@ -62,6 +88,9 @@ import type { RemoteTrackPublication, TrackPublication } from './track_publicati
 import { LocalTrackPublication } from './track_publication.js';
 import type { Transcription } from './transcription.js';
 import type { ChatMessage } from './types.js';
+import { numberToBigInt, splitUtf8 } from './utils.js';
+
+const STREAM_CHUNK_SIZE = 15_000;
 
 export abstract class Participant {
   /** @internal */
@@ -85,20 +114,20 @@ export abstract class Participant {
     return this.info.name;
   }
 
-  get identity(): string | undefined {
-    return this.info.identity;
+  get identity(): string {
+    return this.info.identity ?? '';
   }
 
-  get metadata(): string | undefined {
-    return this.info.metadata;
+  get metadata(): string {
+    return this.info.metadata ?? '';
   }
 
-  get attributes(): Record<string, string> | undefined {
-    return this.info.attributes;
+  get attributes(): Record<string, string> {
+    return this.info.attributes ?? {};
   }
 
-  get kind(): ParticipantKind | undefined {
-    return this.info.kind;
+  get kind(): ParticipantKind {
+    return this.info.kind ?? ParticipantKind.STANDARD;
   }
 
   get disconnectReason(): DisconnectReason | undefined {
@@ -219,6 +248,299 @@ export class LocalParticipant extends Participant {
     });
   }
 
+  /**
+   * Returns a `StreamWriter` instance that allows to write individual chunks of text to a stream.
+   * Well suited for TTS and/or streaming LLM output. If you want to simply send a text then use sendText() instead
+   */
+  async streamText(options?: {
+    topic?: string;
+    attributes?: Record<string, string>;
+    destinationIdentities?: Array<string>;
+    streamId?: string;
+  }): Promise<TextStreamWriter> {
+    const senderIdentity = this.identity;
+    const streamId = options?.streamId ?? crypto.randomUUID();
+    const destinationIdentities = options?.destinationIdentities;
+
+    const info: TextStreamInfo = {
+      streamId: streamId,
+      mimeType: 'text/plain',
+      topic: options?.topic ?? '',
+      timestamp: Date.now(),
+    };
+
+    const headerReq = new SendStreamHeaderRequest({
+      senderIdentity,
+      destinationIdentities,
+      localParticipantHandle: this.ffi_handle.handle,
+      header: new DataStream_Header({
+        streamId,
+        mimeType: info.mimeType,
+        topic: info.topic,
+        timestamp: numberToBigInt(info.timestamp),
+        attributes: options?.attributes,
+        contentHeader: {
+          case: 'textHeader',
+          value: new DataStream_TextHeader({
+            operationType: DataStream_OperationType.CREATE,
+            version: 0,
+            replyToStreamId: '',
+            generated: false,
+          }),
+        },
+      }),
+    });
+
+    await this.sendStreamHeader(headerReq);
+
+    let chunkId = 0;
+    const localHandle = this.ffi_handle.handle;
+    const sendTrailer = this.sendStreamTrailer;
+    const sendChunk = this.sendStreamChunk;
+
+    const writableStream = new WritableStream<[string, number?]>({
+      // Implement the sink
+      write([textChunk, overrideChunkId]) {
+        const textInBytes = new TextEncoder().encode(textChunk);
+
+        if (textInBytes.byteLength > STREAM_CHUNK_SIZE) {
+          this.abort?.();
+          throw new Error('chunk size too large');
+        }
+
+        return new Promise(async (resolve) => {
+          const chunkRequest = new SendStreamChunkRequest({
+            senderIdentity,
+            localParticipantHandle: localHandle,
+            destinationIdentities,
+            chunk: new DataStream_Chunk({
+              content: textInBytes,
+              streamId,
+              chunkIndex: numberToBigInt(overrideChunkId ?? chunkId),
+            }),
+          });
+
+          await sendChunk(chunkRequest);
+
+          if (!overrideChunkId) {
+            chunkId += 1;
+            resolve();
+          }
+        });
+      },
+      async close() {
+        const trailerReq = new SendStreamTrailerRequest({
+          senderIdentity,
+          localParticipantHandle: localHandle,
+          destinationIdentities,
+          trailer: new DataStream_Trailer({
+            streamId,
+            reason: '',
+          }),
+        });
+        await sendTrailer(trailerReq);
+      },
+      abort(err) {
+        log.error('Sink error:', err);
+      },
+    });
+
+    const writer = new TextStreamWriter(writableStream, info);
+
+    return writer;
+  }
+
+  async sendText(
+    text: string,
+    options?: {
+      topic?: string;
+      attributes?: Record<string, string>;
+      destinationIdentities?: Array<string>;
+      streamId?: string;
+    },
+  ) {
+    const writer = await this.streamText(options);
+
+    for (const chunk in splitUtf8(text, STREAM_CHUNK_SIZE)) {
+      await writer.write(chunk);
+    }
+    await writer.close();
+    return writer.info;
+  }
+
+  async streamBytes(options?: {
+    name?: string;
+    topic?: string;
+    attributes?: Record<string, string>;
+    destinationIdentities?: Array<string>;
+    streamId?: string;
+    mimeType?: string;
+    totalSize?: number;
+  }) {
+    const senderIdentity = this.identity;
+    const streamId = options?.streamId ?? crypto.randomUUID();
+    const destinationIdentities = options?.destinationIdentities;
+
+    const info: ByteStreamInfo = {
+      streamId: streamId,
+      mimeType: options?.mimeType ?? 'application/octet-stream',
+      topic: options?.topic ?? '',
+      timestamp: Date.now(),
+      attributes: options?.attributes,
+      totalSize: options?.totalSize,
+      name: options?.name ?? 'unknown',
+    };
+
+    const headerReq = new SendStreamHeaderRequest({
+      senderIdentity,
+      destinationIdentities,
+      localParticipantHandle: this.ffi_handle.handle,
+      header: new DataStream_Header({
+        streamId,
+        mimeType: info.mimeType,
+        topic: info.topic,
+        timestamp: numberToBigInt(info.timestamp),
+        attributes: info.attributes,
+        totalLength: numberToBigInt(info.totalSize),
+        contentHeader: {
+          case: 'byteHeader',
+          value: new DataStream_ByteHeader({
+            name: info.name,
+          }),
+        },
+      }),
+    });
+
+    await this.sendStreamHeader(headerReq);
+
+    let chunkId = 0;
+    const localHandle = this.ffi_handle.handle;
+    const sendTrailer = this.sendStreamTrailer;
+    const sendChunk = this.sendStreamChunk;
+    const writeMutex = new Mutex();
+
+    const writableStream = new WritableStream<[Uint8Array, number?]>({
+      async write([chunk]) {
+        const unlock = await writeMutex.lock();
+
+        let byteOffset = 0;
+        try {
+          while (byteOffset < chunk.byteLength) {
+            const subChunk = chunk.slice(byteOffset, byteOffset + STREAM_CHUNK_SIZE);
+            const chunkRequest = new SendStreamChunkRequest({
+              senderIdentity,
+              localParticipantHandle: localHandle,
+              destinationIdentities,
+              chunk: new DataStream_Chunk({
+                content: subChunk,
+                streamId,
+                chunkIndex: numberToBigInt(chunkId),
+              }),
+            });
+
+            await sendChunk(chunkRequest);
+            chunkId += 1;
+            byteOffset += subChunk.byteLength;
+          }
+        } finally {
+          unlock();
+        }
+      },
+      async close() {
+        const trailerReq = new SendStreamTrailerRequest({
+          senderIdentity,
+          localParticipantHandle: localHandle,
+          destinationIdentities,
+          trailer: new DataStream_Trailer({
+            streamId,
+            reason: '',
+          }),
+        });
+        await sendTrailer(trailerReq);
+      },
+      abort(err) {
+        log.error('Sink error:', err);
+      },
+    });
+
+    const byteWriter = new ByteStreamWriter(writableStream, info);
+
+    return byteWriter;
+  }
+
+  /** Sends a file provided as PathLike to specified recipients */
+  async sendFile(path: PathLike, options?: ByteStreamOptions) {
+    const fileStats = await stat(path);
+    const file = await open(path);
+    try {
+      const stream: ReadableStream<Uint8Array> = file.readableWebStream({ type: 'bytes' });
+      const streamId = crypto.randomUUID();
+      const destinationIdentities = options?.destinationIdentities;
+
+      const writer = await this.streamBytes({
+        streamId: streamId,
+        name: options?.name,
+        totalSize: fileStats.size,
+        destinationIdentities,
+        topic: options?.topic,
+        mimeType: options?.mimeType,
+        attributes: options?.attributes,
+      });
+
+      for await (const chunk of stream) {
+        await writer.write(chunk);
+      }
+      await writer.close();
+    } finally {
+      await file.close();
+    }
+  }
+
+  private async sendStreamHeader(req: SendStreamHeaderRequest) {
+    const type = 'sendStreamHeader';
+    const res = FfiClient.instance.request<SendStreamHeaderResponse>({
+      message: { case: type, value: req },
+    });
+
+    const cb = await FfiClient.instance.waitFor<SendStreamHeaderCallback>((ev) => {
+      return ev.message.case == type && ev.message.value.asyncId == res.asyncId;
+    });
+
+    if (cb.error) {
+      throw new Error(cb.error);
+    }
+  }
+
+  private async sendStreamChunk(req: SendStreamChunkRequest) {
+    const type = 'sendStreamChunk';
+    const res = FfiClient.instance.request<SendStreamChunkResponse>({
+      message: { case: type, value: req },
+    });
+
+    const cb = await FfiClient.instance.waitFor<SendStreamChunkCallback>((ev) => {
+      return ev.message.case == type && ev.message.value.asyncId == res.asyncId;
+    });
+
+    if (cb.error) {
+      throw new Error(cb.error);
+    }
+  }
+
+  private async sendStreamTrailer(req: SendStreamTrailerRequest) {
+    const type = 'sendStreamTrailer';
+    const res = FfiClient.instance.request<SendStreamTrailerResponse>({
+      message: { case: type, value: req },
+    });
+
+    const cb = await FfiClient.instance.waitFor<SendStreamTrailerCallback>((ev) => {
+      return ev.message.case == type && ev.message.value.asyncId == res.asyncId;
+    });
+
+    if (cb.error) {
+      throw new Error(cb.error);
+    }
+  }
+
   /**
    * Sends a chat message to participants in the room
    *

package/src/proto/ffi_pb.ts

@@ -19,7 +19,7 @@
 
 import type { BinaryReadOptions, FieldList, JsonReadOptions, JsonValue, PartialMessage, PlainMessage } from "@bufbuild/protobuf";
 import { Message, proto2 } from "@bufbuild/protobuf";
-import { ConnectCallback, ConnectRequest, ConnectResponse, DisconnectCallback, DisconnectRequest, DisconnectResponse, EditChatMessageRequest, GetSessionStatsCallback, GetSessionStatsRequest, GetSessionStatsResponse, PublishDataCallback, PublishDataRequest, PublishDataResponse, PublishSipDtmfCallback, PublishSipDtmfRequest, PublishSipDtmfResponse, PublishTrackCallback, PublishTrackRequest, PublishTrackResponse, PublishTranscriptionCallback, PublishTranscriptionRequest, PublishTranscriptionResponse, RoomEvent, SendChatMessageCallback, SendChatMessageRequest, SendChatMessageResponse, SetLocalAttributesCallback, SetLocalAttributesRequest, SetLocalAttributesResponse, SetLocalMetadataCallback, SetLocalMetadataRequest, SetLocalMetadataResponse, SetLocalNameCallback, SetLocalNameRequest, SetLocalNameResponse, SetSubscribedRequest, SetSubscribedResponse, UnpublishTrackCallback, UnpublishTrackRequest, UnpublishTrackResponse } from "./room_pb.js";
+import { ConnectCallback, ConnectRequest, ConnectResponse, DisconnectCallback, DisconnectRequest, DisconnectResponse, EditChatMessageRequest, GetSessionStatsCallback, GetSessionStatsRequest, GetSessionStatsResponse, PublishDataCallback, PublishDataRequest, PublishDataResponse, PublishSipDtmfCallback, PublishSipDtmfRequest, PublishSipDtmfResponse, PublishTrackCallback, PublishTrackRequest, PublishTrackResponse, PublishTranscriptionCallback, PublishTranscriptionRequest, PublishTranscriptionResponse, RoomEvent, SendChatMessageCallback, SendChatMessageRequest, SendChatMessageResponse, SendStreamChunkCallback, SendStreamChunkRequest, SendStreamChunkResponse, SendStreamHeaderCallback, SendStreamHeaderRequest, SendStreamHeaderResponse, SendStreamTrailerCallback, SendStreamTrailerRequest, SendStreamTrailerResponse, SetDataChannelBufferedAmountLowThresholdRequest, SetDataChannelBufferedAmountLowThresholdResponse, SetLocalAttributesCallback, SetLocalAttributesRequest, SetLocalAttributesResponse, SetLocalMetadataCallback, SetLocalMetadataRequest, SetLocalMetadataResponse, SetLocalNameCallback, SetLocalNameRequest, SetLocalNameResponse, SetSubscribedRequest, SetSubscribedResponse, UnpublishTrackCallback, UnpublishTrackRequest, UnpublishTrackResponse } from "./room_pb.js";
 import { CreateAudioTrackRequest, CreateAudioTrackResponse, CreateVideoTrackRequest, CreateVideoTrackResponse, EnableRemoteTrackRequest, EnableRemoteTrackResponse, GetStatsCallback, GetStatsRequest, GetStatsResponse, LocalTrackMuteRequest, LocalTrackMuteResponse, TrackEvent } from "./track_pb.js";
 import { CaptureVideoFrameRequest, CaptureVideoFrameResponse, NewVideoSourceRequest, NewVideoSourceResponse, NewVideoStreamRequest, NewVideoStreamResponse, VideoConvertRequest, VideoConvertResponse, VideoStreamEvent, VideoStreamFromParticipantRequest, VideoStreamFromParticipantResponse } from "./video_frame_pb.js";
 import { AudioStreamEvent, AudioStreamFromParticipantRequest, AudioStreamFromParticipantResponse, CaptureAudioFrameCallback, CaptureAudioFrameRequest, CaptureAudioFrameResponse, ClearAudioBufferRequest, ClearAudioBufferResponse, FlushSoxResamplerRequest, FlushSoxResamplerResponse, NewAudioResamplerRequest, NewAudioResamplerResponse, NewAudioSourceRequest, NewAudioSourceResponse, NewAudioStreamRequest, NewAudioStreamResponse, NewSoxResamplerRequest, NewSoxResamplerResponse, PushSoxResamplerRequest, PushSoxResamplerResponse, RemixAndResampleRequest, RemixAndResampleResponse } from "./audio_frame_pb.js";
@@ -339,6 +339,34 @@ export class FfiRequest extends Message<FfiRequest> {
      */
     value: UpdateRemoteTrackPublicationDimensionRequest;
     case: "updateRemoteTrackPublicationDimension";
+  } | {
+    /**
+     * Data Streams
+     *
+     * @generated from field: livekit.proto.SendStreamHeaderRequest send_stream_header = 44;
+     */
+    value: SendStreamHeaderRequest;
+    case: "sendStreamHeader";
+  } | {
+    /**
+     * @generated from field: livekit.proto.SendStreamChunkRequest send_stream_chunk = 45;
+     */
+    value: SendStreamChunkRequest;
+    case: "sendStreamChunk";
+  } | {
+    /**
+     * @generated from field: livekit.proto.SendStreamTrailerRequest send_stream_trailer = 46;
+     */
+    value: SendStreamTrailerRequest;
+    case: "sendStreamTrailer";
+  } | {
+    /**
+     * Data Channel
+     *
+     * @generated from field: livekit.proto.SetDataChannelBufferedAmountLowThresholdRequest set_data_channel_buffered_amount_low_threshold = 47;
+     */
+    value: SetDataChannelBufferedAmountLowThresholdRequest;
+    case: "setDataChannelBufferedAmountLowThreshold";
   } | { case: undefined; value?: undefined } = { case: undefined };
 
   constructor(data?: PartialMessage<FfiRequest>) {
@@ -391,6 +419,10 @@ export class FfiRequest extends Message<FfiRequest> {
     { no: 41, name: "rpc_method_invocation_response", kind: "message", T: RpcMethodInvocationResponseRequest, oneof: "message" },
     { no: 42, name: "enable_remote_track_publication", kind: "message", T: EnableRemoteTrackPublicationRequest, oneof: "message" },
     { no: 43, name: "update_remote_track_publication_dimension", kind: "message", T: UpdateRemoteTrackPublicationDimensionRequest, oneof: "message" },
+    { no: 44, name: "send_stream_header", kind: "message", T: SendStreamHeaderRequest, oneof: "message" },
+    { no: 45, name: "send_stream_chunk", kind: "message", T: SendStreamChunkRequest, oneof: "message" },
+    { no: 46, name: "send_stream_trailer", kind: "message", T: SendStreamTrailerRequest, oneof: "message" },
+    { no: 47, name: "set_data_channel_buffered_amount_low_threshold", kind: "message", T: SetDataChannelBufferedAmountLowThresholdRequest, oneof: "message" },
   ]);
 
   static fromBinary(bytes: Uint8Array, options?: Partial<BinaryReadOptions>): FfiRequest {
@@ -677,6 +709,34 @@ export class FfiResponse extends Message<FfiResponse> {
      */
     value: UpdateRemoteTrackPublicationDimensionResponse;
     case: "updateRemoteTrackPublicationDimension";
+  } | {
+    /**
+     * Data Streams
+     *
+     * @generated from field: livekit.proto.SendStreamHeaderResponse send_stream_header = 43;
+     */
+    value: SendStreamHeaderResponse;
+    case: "sendStreamHeader";
+  } | {
+    /**
+     * @generated from field: livekit.proto.SendStreamChunkResponse send_stream_chunk = 44;
+     */
+    value: SendStreamChunkResponse;
+    case: "sendStreamChunk";
+  } | {
+    /**
+     * @generated from field: livekit.proto.SendStreamTrailerResponse send_stream_trailer = 45;
+     */
+    value: SendStreamTrailerResponse;
+    case: "sendStreamTrailer";
+  } | {
+    /**
+     * Data Channel
+     *
+     * @generated from field: livekit.proto.SetDataChannelBufferedAmountLowThresholdResponse set_data_channel_buffered_amount_low_threshold = 46;
+     */
+    value: SetDataChannelBufferedAmountLowThresholdResponse;
+    case: "setDataChannelBufferedAmountLowThreshold";
   } | { case: undefined; value?: undefined } = { case: undefined };
 
   constructor(data?: PartialMessage<FfiResponse>) {
@@ -728,6 +788,10 @@ export class FfiResponse extends Message<FfiResponse> {
     { no: 40, name: "rpc_method_invocation_response", kind: "message", T: RpcMethodInvocationResponseResponse, oneof: "message" },
     { no: 41, name: "enable_remote_track_publication", kind: "message", T: EnableRemoteTrackPublicationResponse, oneof: "message" },
     { no: 42, name: "update_remote_track_publication_dimension", kind: "message", T: UpdateRemoteTrackPublicationDimensionResponse, oneof: "message" },
+    { no: 43, name: "send_stream_header", kind: "message", T: SendStreamHeaderResponse, oneof: "message" },
+    { no: 44, name: "send_stream_chunk", kind: "message", T: SendStreamChunkResponse, oneof: "message" },
+    { no: 45, name: "send_stream_trailer", kind: "message", T: SendStreamTrailerResponse, oneof: "message" },
+    { no: 46, name: "set_data_channel_buffered_amount_low_threshold", kind: "message", T: SetDataChannelBufferedAmountLowThresholdResponse, oneof: "message" },
   ]);
 
   static fromBinary(bytes: Uint8Array, options?: Partial<BinaryReadOptions>): FfiResponse {
@@ -896,6 +960,24 @@ export class FfiEvent extends Message<FfiEvent> {
      */
     value: RpcMethodInvocationEvent;
     case: "rpcMethodInvocation";
+  } | {
+    /**
+     * @generated from field: livekit.proto.SendStreamHeaderCallback send_stream_header = 25;
+     */
+    value: SendStreamHeaderCallback;
+    case: "sendStreamHeader";
+  } | {
+    /**
+     * @generated from field: livekit.proto.SendStreamChunkCallback send_stream_chunk = 26;
+     */
+    value: SendStreamChunkCallback;
+    case: "sendStreamChunk";
+  } | {
+    /**
+     * @generated from field: livekit.proto.SendStreamTrailerCallback send_stream_trailer = 27;
+     */
+    value: SendStreamTrailerCallback;
+    case: "sendStreamTrailer";
   } | { case: undefined; value?: undefined } = { case: undefined };
 
   constructor(data?: PartialMessage<FfiEvent>) {
@@ -929,6 +1011,9 @@ export class FfiEvent extends Message<FfiEvent> {
     { no: 22, name: "chat_message", kind: "message", T: SendChatMessageCallback, oneof: "message" },
     { no: 23, name: "perform_rpc", kind: "message", T: PerformRpcCallback, oneof: "message" },
     { no: 24, name: "rpc_method_invocation", kind: "message", T: RpcMethodInvocationEvent, oneof: "message" },
+    { no: 25, name: "send_stream_header", kind: "message", T: SendStreamHeaderCallback, oneof: "message" },
+    { no: 26, name: "send_stream_chunk", kind: "message", T: SendStreamChunkCallback, oneof: "message" },
+    { no: 27, name: "send_stream_trailer", kind: "message", T: SendStreamTrailerCallback, oneof: "message" },
   ]);
 
   static fromBinary(bytes: Uint8Array, options?: Partial<BinaryReadOptions>): FfiEvent {