assemblyai 4.12.2 → 4.13.0-beta.1

package/dist/workerd.mjs

@@ -15,7 +15,7 @@ if (typeof navigator !== "undefined" && navigator.userAgent) {
     defaultUserAgentString += navigator.userAgent;
 }
 const defaultUserAgent = {
-    sdk: { name: "JavaScript", version: "4.12.2" },
+    sdk: { name: "JavaScript", version: "4.13.0-beta.1" },
 };
 if (typeof process !== "undefined") {
     if (process.versions.node && defaultUserAgentString.indexOf("Node") === -1) {
@@ -197,9 +197,48 @@ const RealtimeErrorMessages = {
 class RealtimeError extends Error {
 }
 
+const StreamingErrorType = {
+    BadSampleRate: 4000,
+    AuthFailed: 4001,
+    InsufficientFunds: 4002,
+    FreeTierUser: 4003,
+    NonexistentSessionId: 4004,
+    SessionExpired: 4008,
+    ClosedSession: 4010,
+    RateLimited: 4029,
+    UniqueSessionViolation: 4030,
+    SessionTimeout: 4031,
+    AudioTooShort: 4032,
+    AudioTooLong: 4033,
+    AudioTooSmallToTranscode: 4034,
+    BadSchema: 4101,
+    TooManyStreams: 4102,
+    Reconnected: 4103,
+};
+const StreamingErrorMessages = {
+    [StreamingErrorType.BadSampleRate]: "Sample rate must be a positive integer",
+    [StreamingErrorType.AuthFailed]: "Not Authorized",
+    [StreamingErrorType.InsufficientFunds]: "Insufficient funds",
+    [StreamingErrorType.FreeTierUser]: "This feature is paid-only and requires you to add a credit card. Please visit https://app.assemblyai.com/ to add a credit card to your account.",
+    [StreamingErrorType.NonexistentSessionId]: "Session ID does not exist",
+    [StreamingErrorType.SessionExpired]: "Session has expired",
+    [StreamingErrorType.ClosedSession]: "Session is closed",
+    [StreamingErrorType.RateLimited]: "Rate limited",
+    [StreamingErrorType.UniqueSessionViolation]: "Unique session violation",
+    [StreamingErrorType.SessionTimeout]: "Session Timeout",
+    [StreamingErrorType.AudioTooShort]: "Audio too short",
+    [StreamingErrorType.AudioTooLong]: "Audio too long",
+    [StreamingErrorType.AudioTooSmallToTranscode]: "Audio too small to transcode",
+    [StreamingErrorType.BadSchema]: "Bad schema",
+    [StreamingErrorType.TooManyStreams]: "Too many streams",
+    [StreamingErrorType.Reconnected]: "This session has been reconnected. This WebSocket is no longer valid.",
+};
+class StreamingError extends Error {
+}
+
 const defaultRealtimeUrl = "wss://api.assemblyai.com/v2/realtime/ws";
 const forceEndOfUtteranceMessage = `{"force_end_utterance":true}`;
-const terminateSessionMessage = `{"terminate_session":true}`;
+const terminateSessionMessage$1 = `{"terminate_session":true}`;
 /**
  * RealtimeTranscriber connects to the Streaming Speech-to-Text API and lets you transcribe audio in real-time.
  */
@@ -388,11 +427,11 @@ class RealtimeTranscriber {
                     const sessionTerminatedPromise = new Promise((resolve) => {
                         this.sessionTerminatedResolve = resolve;
                     });
-                    this.socket.send(terminateSessionMessage);
+                    this.socket.send(terminateSessionMessage$1);
                     await sessionTerminatedPromise;
                 }
                 else {
-                    this.socket.send(terminateSessionMessage);
+                    this.socket.send(terminateSessionMessage$1);
                 }
             }
             if (this.socket?.removeAllListeners)
@@ -729,7 +768,166 @@ function dataUrlToBlob(dataUrl) {
     return new Blob([u8arr], { type: mime });
 }
 
+const defaultStreamingUrl$1 = "wss://streaming.assemblyai.com/v3/ws";
+const terminateSessionMessage = `{"type":"Terminate"}`;
+class StreamingTranscriber {
+    constructor(params) {
+        this.listeners = {};
+        this.params = {
+            ...params,
+            websocketBaseUrl: params.websocketBaseUrl || defaultStreamingUrl$1,
+        };
+        if ("token" in params && params.token)
+            this.token = params.token;
+        if ("apiKey" in params && params.apiKey)
+            this.apiKey = params.apiKey;
+        if (!(this.token || this.apiKey)) {
+            throw new Error("API key or temporary token is required.");
+        }
+    }
+    connectionUrl() {
+        const url = new URL(this.params.websocketBaseUrl ?? "");
+        if (url.protocol !== "wss:") {
+            throw new Error("Invalid protocol, must be wss");
+        }
+        const searchParams = new URLSearchParams();
+        searchParams.set("sample_rate", this.params.sampleRate.toString());
+        if (this.params.wordFinalizationMaxWaitTime) {
+            searchParams.set("word_finalization_max_wait_time", this.params.wordFinalizationMaxWaitTime.toString());
+        }
+        if (this.params.endOfTurnConfidenceThreshold) {
+            searchParams.set("end_of_turn_confidence_threshold", this.params.endOfTurnConfidenceThreshold.toString());
+        }
+        if (this.params.minEndOfTurnSilenceWhenConfident) {
+            searchParams.set("min_end_of_turn_silence_when_confident", this.params.minEndOfTurnSilenceWhenConfident.toString());
+        }
+        if (this.params.maxTurnSilence) {
+            searchParams.set("max_turn_silence", this.params.maxTurnSilence.toString());
+        }
+        if (this.params.formattedFinals) {
+            searchParams.set("formatted_finals", this.params.formattedFinals.toString());
+        }
+        url.search = searchParams.toString();
+        return url;
+    }
+    // eslint-disable-next-line @typescript-eslint/no-explicit-any
+    on(event, listener) {
+        this.listeners[event] = listener;
+    }
+    connect() {
+        return new Promise((resolve) => {
+            if (this.socket) {
+                throw new Error("Already connected");
+            }
+            const url = this.connectionUrl();
+            this.socket = factory(url.toString(), {
+                headers: { Authorization: this.token || this.apiKey },
+            });
+            this.socket.binaryType = "arraybuffer";
+            this.socket.onopen = () => { };
+            this.socket.onclose = ({ code, reason }) => {
+                if (!reason) {
+                    if (code in StreamingErrorMessages) {
+                        reason = StreamingErrorMessages[code];
+                    }
+                }
+                this.listeners.close?.(code, reason);
+            };
+            this.socket.onerror = (event) => {
+                if (event.error)
+                    this.listeners.error?.(event.error);
+                else
+                    this.listeners.error?.(new Error(event.message));
+            };
+            this.socket.onmessage = ({ data }) => {
+                const message = JSON.parse(data.toString());
+                if ("error" in message) {
+                    this.listeners.error?.(new StreamingError(message.error));
+                    return;
+                }
+                switch (message.type) {
+                    case "Begin": {
+                        resolve(message);
+                        this.listeners.open?.(message);
+                        break;
+                    }
+                    case "Turn": {
+                        this.listeners.turn?.(message);
+                        break;
+                    }
+                    case "Termination": {
+                        this.sessionTerminatedResolve?.();
+                        break;
+                    }
+                }
+            };
+        });
+    }
+    stream() {
+        return new WritableStream({
+            write: (chunk) => {
+                this.sendAudio(chunk);
+            },
+        });
+    }
+    sendAudio(audio) {
+        this.send(audio);
+    }
+    send(data) {
+        if (!this.socket || this.socket.readyState !== this.socket.OPEN) {
+            throw new Error("Socket is not open for communication");
+        }
+        this.socket.send(data);
+    }
+    async close(waitForSessionTermination = true) {
+        if (this.socket) {
+            if (this.socket.readyState === this.socket.OPEN) {
+                if (waitForSessionTermination) {
+                    const sessionTerminatedPromise = new Promise((resolve) => {
+                        this.sessionTerminatedResolve = resolve;
+                    });
+                    this.socket.send(terminateSessionMessage);
+                    await sessionTerminatedPromise;
+                }
+                else {
+                    this.socket.send(terminateSessionMessage);
+                }
+            }
+            if (this.socket?.removeAllListeners)
+                this.socket.removeAllListeners();
+            this.socket.close();
+        }
+        this.listeners = {};
+        this.socket = undefined;
+    }
+}
+
+class StreamingTranscriberFactory extends BaseService {
+    constructor(params) {
+        super(params);
+    }
+    transcriber(params) {
+        return new StreamingTranscriber(params);
+    }
+    async createTemporaryToken(params) {
+        const searchParams = new URLSearchParams();
+        // Add each param to the search params
+        Object.entries(params).forEach(([key, value]) => {
+            if (value !== undefined && value !== null) {
+                searchParams.append(key, String(value));
+            }
+        });
+        const queryString = searchParams.toString();
+        const url = queryString ? `/v3/token?${queryString}` : "/v3/token";
+        const data = await this.fetchJson(url, {
+            method: "GET",
+        });
+        return data.token;
+    }
+}
+
 const defaultBaseUrl = "https://api.assemblyai.com";
+const defaultStreamingUrl = "https://streaming.assemblyai.com";
 class AssemblyAI {
     /**
      * Create a new AssemblyAI client.
@@ -744,7 +942,11 @@ class AssemblyAI {
         this.transcripts = new TranscriptService(params, this.files);
         this.lemur = new LemurService(params);
         this.realtime = new RealtimeTranscriberFactory(params);
+        this.streaming = new StreamingTranscriberFactory({
+            ...params,
+            baseUrl: params.streamingBaseUrl || defaultStreamingUrl,
+        });
     }
 }
 
-export { AssemblyAI, FileService, LemurService, RealtimeService, RealtimeServiceFactory, RealtimeTranscriber, RealtimeTranscriberFactory, TranscriptService };
+export { AssemblyAI, FileService, LemurService, RealtimeService, RealtimeServiceFactory, RealtimeTranscriber, RealtimeTranscriberFactory, StreamingTranscriber, TranscriptService };

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "assemblyai",
-  "version": "4.12.2",
+  "version": "4.13.0-beta.1",
   "description": "The AssemblyAI JavaScript SDK provides an easy-to-use interface for interacting with the AssemblyAI API, which supports async and real-time transcription, as well as the latest LeMUR models.",
   "engines": {
     "node": ">=18"

package/src/services/index.ts

@@ -8,8 +8,10 @@ import {
 } from "./realtime";
 import { TranscriptService } from "./transcripts";
 import { FileService } from "./files";
+import { StreamingTranscriber, StreamingTranscriberFactory } from "./streaming";
 
 const defaultBaseUrl = "https://api.assemblyai.com";
+const defaultStreamingUrl = "https://streaming.assemblyai.com";
 
 class AssemblyAI {
   /**
@@ -32,6 +34,11 @@ class AssemblyAI {
    */
   public realtime: RealtimeTranscriberFactory;
 
+  /**
+   * The streaming service.
+   */
+  public streaming: StreamingTranscriberFactory;
+
   /**
    * Create a new AssemblyAI client.
    * @param params - The parameters for the service, including the API key and base URL, if any.
@@ -41,10 +48,16 @@ class AssemblyAI {
     if (params.baseUrl && params.baseUrl.endsWith("/")) {
       params.baseUrl = params.baseUrl.slice(0, -1);
     }
+
     this.files = new FileService(params);
     this.transcripts = new TranscriptService(params, this.files);
     this.lemur = new LemurService(params);
     this.realtime = new RealtimeTranscriberFactory(params);
+
+    this.streaming = new StreamingTranscriberFactory({
+      ...params,
+      baseUrl: params.streamingBaseUrl || defaultStreamingUrl,
+    });
   }
 }
 
@@ -57,4 +70,5 @@ export {
   RealtimeService,
   TranscriptService,
   FileService,
+  StreamingTranscriber,
 };

package/src/services/streaming/factory.ts

@@ -0,0 +1,39 @@
+import {
+  BaseServiceParams,
+  StreamingTokenParams,
+  StreamingTranscriberParams,
+  StreamingTemporaryTokenResponse,
+} from "../..";
+import { StreamingTranscriber } from "./service";
+import { BaseService } from "../base";
+
+export class StreamingTranscriberFactory extends BaseService {
+  constructor(params: BaseServiceParams) {
+    super(params);
+  }
+
+  transcriber(params: StreamingTranscriberParams): StreamingTranscriber {
+    return new StreamingTranscriber(params);
+  }
+
+  async createTemporaryToken(params: StreamingTokenParams) {
+    const searchParams = new URLSearchParams();
+
+    // Add each param to the search params
+    Object.entries(params).forEach(([key, value]) => {
+      if (value !== undefined && value !== null) {
+        searchParams.append(key, String(value));
+      }
+    });
+
+    const queryString = searchParams.toString();
+    const url = queryString ? `/v3/token?${queryString}` : "/v3/token";
+
+    const data = await this.fetchJson<StreamingTemporaryTokenResponse>(url, {
+      method: "GET",
+    });
+    return data.token;
+  }
+}
+
+export class StreamingServiceFactory extends StreamingTranscriberFactory {}

package/src/services/streaming/index.ts

@@ -0,0 +1,2 @@
+export * from "./factory";
+export * from "./service";

package/src/services/streaming/service.ts

@@ -0,0 +1,219 @@
+import { WritableStream } from "#streams";
+import {
+  PolyfillWebSocket,
+  factory as polyfillWebSocketFactory,
+} from "#websocket";
+import { ErrorEvent, MessageEvent, CloseEvent } from "ws";
+import {
+  StreamingEvents,
+  StreamingListeners,
+  StreamingTranscriberParams,
+  AudioData,
+  BeginEvent,
+  StreamingEventMessage,
+  TurnEvent,
+} from "../..";
+import { StreamingError, StreamingErrorMessages } from "../../utils/errors";
+import { StreamingErrorTypeCodes } from "../../utils/errors/streaming";
+
+const defaultStreamingUrl = "wss://streaming.assemblyai.com/v3/ws";
+const terminateSessionMessage = `{"type":"Terminate"}`;
+
+type BufferLike =
+  | string
+  | Buffer
+  | DataView
+  | number
+  | ArrayBufferView
+  | Uint8Array
+  | ArrayBuffer
+  | SharedArrayBuffer
+  | ReadonlyArray<unknown>
+  | ReadonlyArray<number>
+  | { valueOf(): ArrayBuffer }
+  | { valueOf(): SharedArrayBuffer }
+  | { valueOf(): Uint8Array }
+  | { valueOf(): ReadonlyArray<number> }
+  | { valueOf(): string }
+  | { [Symbol.toPrimitive](hint: string): string };
+
+export class StreamingTranscriber {
+  private apiKey?: string;
+  private token?: string;
+  private params: StreamingTranscriberParams;
+
+  private socket?: PolyfillWebSocket;
+  private listeners: StreamingListeners = {};
+  private sessionTerminatedResolve?: () => void;
+
+  constructor(params: StreamingTranscriberParams) {
+    this.params = {
+      ...params,
+      websocketBaseUrl: params.websocketBaseUrl || defaultStreamingUrl,
+    };
+
+    if ("token" in params && params.token) this.token = params.token;
+    if ("apiKey" in params && params.apiKey) this.apiKey = params.apiKey;
+
+    if (!(this.token || this.apiKey)) {
+      throw new Error("API key or temporary token is required.");
+    }
+  }
+
+  private connectionUrl(): URL {
+    const url = new URL(this.params.websocketBaseUrl ?? "");
+
+    if (url.protocol !== "wss:") {
+      throw new Error("Invalid protocol, must be wss");
+    }
+
+    const searchParams = new URLSearchParams();
+
+    searchParams.set("sample_rate", this.params.sampleRate.toString());
+
+    if (this.params.wordFinalizationMaxWaitTime) {
+      searchParams.set(
+        "word_finalization_max_wait_time",
+        this.params.wordFinalizationMaxWaitTime.toString(),
+      );
+    }
+
+    if (this.params.endOfTurnConfidenceThreshold) {
+      searchParams.set(
+        "end_of_turn_confidence_threshold",
+        this.params.endOfTurnConfidenceThreshold.toString(),
+      );
+    }
+
+    if (this.params.minEndOfTurnSilenceWhenConfident) {
+      searchParams.set(
+        "min_end_of_turn_silence_when_confident",
+        this.params.minEndOfTurnSilenceWhenConfident.toString(),
+      );
+    }
+
+    if (this.params.maxTurnSilence) {
+      searchParams.set(
+        "max_turn_silence",
+        this.params.maxTurnSilence.toString(),
+      );
+    }
+
+    if (this.params.formattedFinals) {
+      searchParams.set(
+        "formatted_finals",
+        this.params.formattedFinals.toString(),
+      );
+    }
+
+    url.search = searchParams.toString();
+
+    return url;
+  }
+
+  on(event: "open", listener: (event: BeginEvent) => void): void;
+  on(event: "turn", listener: (event: TurnEvent) => void): void;
+  on(event: "error", listener: (error: Error) => void): void;
+  on(event: "close", listener: (code: number, reason: string) => void): void;
+  // eslint-disable-next-line @typescript-eslint/no-explicit-any
+  on(event: StreamingEvents, listener: (...args: any[]) => void) {
+    this.listeners[event] = listener;
+  }
+
+  connect() {
+    return new Promise<BeginEvent>((resolve) => {
+      if (this.socket) {
+        throw new Error("Already connected");
+      }
+
+      const url = this.connectionUrl();
+
+      this.socket = polyfillWebSocketFactory(url.toString(), {
+        headers: { Authorization: this.token || this.apiKey },
+      });
+
+      this.socket.binaryType = "arraybuffer";
+
+      this.socket.onopen = () => {};
+
+      this.socket.onclose = ({ code, reason }: CloseEvent) => {
+        if (!reason) {
+          if (code in StreamingErrorMessages) {
+            reason = StreamingErrorMessages[code as StreamingErrorTypeCodes];
+          }
+        }
+        this.listeners.close?.(code, reason);
+      };
+
+      this.socket.onerror = (event: ErrorEvent) => {
+        if (event.error) this.listeners.error?.(event.error as Error);
+        else this.listeners.error?.(new Error(event.message));
+      };
+
+      this.socket.onmessage = ({ data }: MessageEvent) => {
+        const message = JSON.parse(data.toString()) as StreamingEventMessage;
+
+        if ("error" in message) {
+          this.listeners.error?.(new StreamingError(message.error));
+          return;
+        }
+
+        switch (message.type) {
+          case "Begin": {
+            resolve(message);
+            this.listeners.open?.(message);
+            break;
+          }
+          case "Turn": {
+            this.listeners.turn?.(message);
+            break;
+          }
+          case "Termination": {
+            this.sessionTerminatedResolve?.();
+            break;
+          }
+        }
+      };
+    });
+  }
+
+  stream(): WritableStream<AudioData> {
+    return new WritableStream<AudioData>({
+      write: (chunk: AudioData) => {
+        this.sendAudio(chunk);
+      },
+    });
+  }
+
+  sendAudio(audio: AudioData) {
+    this.send(audio);
+  }
+
+  private send(data: BufferLike) {
+    if (!this.socket || this.socket.readyState !== this.socket.OPEN) {
+      throw new Error("Socket is not open for communication");
+    }
+    this.socket.send(data);
+  }
+
+  async close(waitForSessionTermination = true) {
+    if (this.socket) {
+      if (this.socket.readyState === this.socket.OPEN) {
+        if (waitForSessionTermination) {
+          const sessionTerminatedPromise = new Promise<void>((resolve) => {
+            this.sessionTerminatedResolve = resolve;
+          });
+          this.socket.send(terminateSessionMessage);
+          await sessionTerminatedPromise;
+        } else {
+          this.socket.send(terminateSessionMessage);
+        }
+      }
+      if (this.socket?.removeAllListeners) this.socket.removeAllListeners();
+      this.socket.close();
+    }
+
+    this.listeners = {};
+    this.socket = undefined;
+  }
+}

package/src/types/index.ts

@@ -1,8 +1,9 @@
 export * from "./files";
 export * from "./transcripts";
 export * from "./realtime";
-export * from "./services";
 export * from "./asyncapi.generated";
+export * from "./streaming";
+export * from "./services";
 export * from "./openapi.generated";
 export * from "./deprecated";
 

package/src/types/services/index.ts

@@ -3,6 +3,7 @@ import { UserAgent } from "..";
 type BaseServiceParams = {
   apiKey: string;
   baseUrl?: string;
+  streamingBaseUrl?: string;
   /**
    * The AssemblyAI user agent to use for requests.
    * The provided components will be merged into the default AssemblyAI user agent.

package/src/types/streaming/index.ts

@@ -0,0 +1,94 @@
+export type StreamingTranscriberParams = {
+  websocketBaseUrl?: string;
+  apiKey?: string;
+  token?: string;
+  sampleRate: number;
+
+  wordFinalizationMaxWaitTime?: number;
+  endOfTurnConfidenceThreshold?: number;
+  minEndOfTurnSilenceWhenConfident?: number;
+  maxTurnSilence?: number;
+  formattedFinals?: boolean;
+};
+
+export type StreamingEvents = "open" | "close" | "turn" | "error";
+
+export type StreamingListeners = {
+  open?: (event: BeginEvent) => void;
+  close?: (code: number, reason: string) => void;
+  turn?: (event: TurnEvent) => void;
+  error?: (error: Error) => void;
+};
+
+export type StreamingTokenParams = {
+  expires_in_seconds: number;
+  max_session_duration_seconds: number;
+};
+
+export type StreamingTemporaryTokenResponse = {
+  token: string;
+};
+
+export type StreamingAudioData = ArrayBufferLike;
+
+export type BeginEvent = {
+  type: "Begin";
+  id: string;
+  expires_at: number;
+};
+
+export type TurnEvent = {
+  type: "Turn";
+  turn_order: number;
+  turn_is_formatted: boolean;
+  end_of_turn: boolean;
+  transcript: string;
+  end_of_turn_confidence: number;
+  words: StreamingWord[];
+};
+
+export type StreamingWord = {
+  start: number;
+  end: number;
+  confidence: number;
+  text: string;
+  word_is_final: boolean;
+};
+
+export type TerminationEvent = {
+  type: "Termination";
+  audio_duration_seconds: number;
+  session_duration_seconds: number;
+};
+
+export type StreamingTerminateSession = {
+  type: "Terminate";
+};
+
+export type StreamingUpdateConfiguration = {
+  type: "UpdateConfiguration";
+  word_finalization_max_wait_time?: number;
+  end_of_turn_confidence_threshold?: number;
+  min_end_of_turn_silence_when_confident?: number;
+  max_turn_silence?: number;
+  formatted_finals?: boolean;
+};
+
+export type StreamingForceEndpoint = {
+  type: "ForceEndpoint";
+};
+
+export type ErrorEvent = {
+  error: string;
+};
+
+export type StreamingEventMessage =
+  | BeginEvent
+  | TurnEvent
+  | TerminationEvent
+  | ErrorEvent;
+
+export type StreamingOperationMessage =
+  | StreamingUpdateConfiguration
+  | StreamingForceEndpoint
+  | StreamingTerminateSession;

package/src/utils/errors/index.ts

@@ -3,3 +3,9 @@ export {
   RealtimeErrorType,
   RealtimeErrorMessages,
 } from "./realtime";
+
+export {
+  StreamingError,
+  StreamingErrorType,
+  StreamingErrorMessages,
+} from "./streaming";