assemblyai 4.35.4 → 4.36.4

package/src/services/index.ts

@@ -1,4 +1,6 @@
 import { BaseServiceParams } from "..";
+import { SyncTranscriber } from "./sync";
+import { SyncTranscriptError } from "../utils/errors";
 import { LemurService } from "./lemur";
 import {
   RealtimeTranscriber,
@@ -23,6 +25,7 @@ import {
 
 const defaultBaseUrl = "https://api.assemblyai.com";
 const defaultStreamingUrl = "https://streaming.assemblyai.com";
+const defaultSyncUrl = "https://sync.assemblyai.com";
 
 class AssemblyAI {
   /**
@@ -50,6 +53,11 @@ class AssemblyAI {
    */
   public streaming: StreamingTranscriberFactory;
 
+  /**
+   * The synchronous transcription service.
+   */
+  public sync: SyncTranscriber;
+
   /**
    * Create a new AssemblyAI client.
    * @param params - The parameters for the service, including the API key and base URL, if any.
@@ -69,11 +77,22 @@ class AssemblyAI {
       ...params,
       baseUrl: params.streamingBaseUrl || defaultStreamingUrl,
     });
+
+    let syncBaseUrl = params.syncBaseUrl || defaultSyncUrl;
+    if (syncBaseUrl.endsWith("/")) {
+      syncBaseUrl = syncBaseUrl.slice(0, -1);
+    }
+    this.sync = new SyncTranscriber({
+      ...params,
+      baseUrl: syncBaseUrl,
+    });
   }
 }
 
 export {
   AssemblyAI,
+  SyncTranscriber,
+  SyncTranscriptError,
   LemurService,
   RealtimeTranscriberFactory,
   RealtimeTranscriber,

package/src/services/streaming/service.ts

@@ -20,6 +20,7 @@ import {
   StreamingForceEndpoint,
   StreamingKeepAlive,
   WarningEvent,
+  HeartbeatEvent,
 } from "../..";
 import type { VadDetector, VadFrame } from "../../types/streaming/dual-channel";
 import { EnergyVad } from "./energy-vad";
@@ -164,10 +165,16 @@ export class StreamingTranscriber {
       throw new Error("API key or temporary token is required.");
     }
 
-    const isOpus = params.encoding === "opus" || params.encoding === "ogg_opus";
-    if (params.sampleRate === undefined && (!isOpus || params.channels)) {
+    const isSelfDescribing =
+      params.encoding === "opus" ||
+      params.encoding === "ogg_opus" ||
+      params.encoding === "aac";
+    if (
+      params.sampleRate === undefined &&
+      (!isSelfDescribing || params.channels)
+    ) {
       throw new Error(
-        '`sampleRate` is required; it may only be omitted when `encoding` is "opus" or "ogg_opus" (the Opus stream is self-describing) and dual-channel mode is not used.',
+        '`sampleRate` is required; it may only be omitted when `encoding` is "opus", "ogg_opus", or "aac" (these streams are self-describing) and dual-channel mode is not used.',
       );
     }
 
@@ -283,6 +290,13 @@ export class StreamingTranscriber {
       searchParams.set("format_turns", this.params.formatTurns.toString());
     }
 
+    if (this.params.sessionHeartbeat !== undefined) {
+      searchParams.set(
+        "session_heartbeat",
+        this.params.sessionHeartbeat.toString(),
+      );
+    }
+
     if (this.params.encoding) {
       searchParams.set("encoding", this.params.encoding.toString());
     }
@@ -474,6 +488,7 @@ export class StreamingTranscriber {
     listener: (event: SpeakerRevisionEvent) => void,
   ): void;
   on(event: "warning", listener: (event: WarningEvent) => void): void;
+  on(event: "heartbeat", listener: (event: HeartbeatEvent) => void): void;
   on(event: "vad", listener: (event: VadFrame) => void): void;
   on(event: "error", listener: (error: Error) => void): void;
   on(event: "close", listener: (code: number, reason: string) => void): void;
@@ -693,6 +708,10 @@ Learn more at https://github.com/AssemblyAI/assemblyai-node-sdk/blob/main/docs/c
             this.listeners.warning?.(warning);
             break;
           }
+          case "Heartbeat": {
+            this.listeners.heartbeat?.(message);
+            break;
+          }
           case "Termination": {
             this.sessionTerminatedResolve?.();
             break;

package/src/services/sync/index.ts

@@ -0,0 +1 @@
+export * from "./service";

package/src/services/sync/service.ts

@@ -0,0 +1,369 @@
+import { readFile } from "#fs";
+import { BaseService } from "../base";
+import {
+  BaseServiceParams,
+  SyncAudioInput,
+  SyncTranscribeOptions,
+  SyncTranscriptResponse,
+  SyncTranscriptionConfig,
+} from "../..";
+import { defaultSyncSpeechModel } from "../../types/sync";
+import { SyncTranscriptError } from "../../utils/errors/sync";
+import { getPath } from "../../utils/path";
+
+// Canonical paths since the sync API gained a /v1 prefix (#18103); the
+// unprefixed routes remain served for SDK versions that predate it.
+const transcribeEndpoint = "/v1/transcribe";
+const warmEndpoint = "/v1/warm";
+const modelHeader = "X-AAI-Model";
+// Kept above the server's 30 s deadline so the client doesn't race it.
+const defaultTimeoutMs = 60_000;
+const warmTimeoutMs = 10_000;
+const maxPromptLength = 4096;
+const maxKeytermsPromptLength = 2048;
+const maxContextTurns = 100;
+const maxContextLength = 4096;
+// Extensions that signal raw S16LE PCM rather than a WAV container.
+const pcmSuffixes = [".pcm", ".raw"];
+
+/**
+ * The synchronous transcription service: audio in, transcript out,
+ * one request.
+ *
+ * Unlike `client.transcripts` (which submits a job to the async API and
+ * polls for completion), `SyncTranscriber` posts the audio to the sync
+ * API and returns the finished transcript in the HTTP response. There is no
+ * job id or status to poll. Accepts a local file path, raw audio bytes, a
+ * Blob, or a readable stream — but not a URL.
+ */
+export class SyncTranscriber extends BaseService {
+  /**
+   * Create a new synchronous transcription service.
+   * @param params - The parameters to use for the service.
+   */
+  constructor(params: BaseServiceParams) {
+    super(params);
+  }
+
+  /**
+   * Transcribe audio and return the finished transcript in one request.
+   * @param audio - A local file path, raw audio bytes, a Blob, or a readable
+   * stream. Raw PCM also requires `sample_rate` and `channels` on the config.
+   * @param config - Options for this transcription request.
+   * @param options - Client-side options, such as the request timeout.
+   * @returns A promise that resolves to the finished transcript.
+   * @throws SyncTranscriptError when the request fails.
+   */
+  async transcribe(
+    audio: SyncAudioInput,
+    config: SyncTranscriptionConfig = {},
+    options: SyncTranscribeOptions = {},
+  ): Promise<SyncTranscriptResponse> {
+    const { bytes, filename, contentType } = await resolveAudio(audio, config);
+
+    const body = new FormData();
+    body.append(
+      "audio",
+      new Blob([bytes as BlobPart], { type: contentType }),
+      filename,
+    );
+    const configJson = buildConfigJson(config);
+    if (configJson) {
+      body.append(
+        "config",
+        new Blob([JSON.stringify(configJson)], { type: "application/json" }),
+      );
+    }
+
+    const response = await this.fetchResponse(transcribeEndpoint, {
+      method: "POST",
+      body,
+      headers: { [modelHeader]: config.model ?? defaultSyncSpeechModel },
+      signal: AbortSignal.timeout(options.timeout ?? defaultTimeoutMs),
+    });
+    if (response.status !== 200) throw await errorFromResponse(response);
+    return (await response.json()) as SyncTranscriptResponse;
+  }
+
+  /**
+   * Open the connection to the sync API ahead of time.
+   *
+   * The sync API is a single request/response, so a `transcribe()` that
+   * opens its connection on demand pays the full DNS + TCP + TLS handshake
+   * on the critical path. Call `warm()` as soon as you know audio is coming —
+   * typically while the clip is still being recorded — so the next
+   * `transcribe()` reuses the already-open connection. `warm()` is idempotent
+   * and cheap; call it shortly before `transcribe()` so the pooled connection
+   * hasn't idled out.
+   * @param params - Optionally the model to route the probe to, so the warmed
+   * connection lands on the same backend as the eventual transcription.
+   * @returns A promise that resolves to `true` once the connection is open
+   * (any HTTP response — even a non-200 — means the socket is
+   * established), or `false` if the connection could not be opened.
+   */
+  async warm(params?: { model?: string }): Promise<boolean> {
+    try {
+      await this.fetchResponse(warmEndpoint, {
+        method: "GET",
+        headers: { [modelHeader]: params?.model ?? defaultSyncSpeechModel },
+        signal: AbortSignal.timeout(warmTimeoutMs),
+      });
+      return true;
+    } catch {
+      return false;
+    }
+  }
+}
+
+type ResolvedAudio = {
+  bytes: Uint8Array;
+  filename: string;
+  contentType: string;
+};
+
+/**
+ * Read the audio input into bytes and decide its multipart content type.
+ *
+ * PCM is selected when the source has a `.pcm`/`.raw` extension or when
+ * `sample_rate`/`channels` are set on the config (the fields the sync API
+ * requires only for raw PCM) — and both must then be present. Everything
+ * else is treated as a WAV container. URLs are rejected — the sync API has
+ * no URL ingestion.
+ */
+async function resolveAudio(
+  input: SyncAudioInput,
+  config: SyncTranscriptionConfig,
+): Promise<ResolvedAudio> {
+  let bytes: Uint8Array;
+  let filename: string | undefined;
+  let suffix = "";
+
+  if (typeof input === "string") {
+    if (/^https?:\/\//i.test(input)) {
+      throw new Error(
+        "SyncTranscriber does not accept URLs. Pass a local file path or " +
+          "audio bytes, or use client.transcripts for URL/async transcription.",
+      );
+    }
+    if (input.startsWith("data:")) {
+      bytes = dataUrlToBytes(input);
+    } else {
+      const path = getPath(input) ?? input;
+      bytes = await readStream(await readFile(path));
+      filename = basename(path);
+      suffix = extname(filename);
+    }
+  } else if (input instanceof Uint8Array) {
+    bytes = input;
+  } else if (input instanceof ArrayBuffer) {
+    bytes = new Uint8Array(input);
+  } else if (input instanceof Blob) {
+    bytes = new Uint8Array(await input.arrayBuffer());
+    // File instances carry a name; the File global itself needs Node >= 20.
+    const name = (input as { name?: string }).name;
+    if (name) {
+      filename = basename(name);
+      suffix = extname(filename);
+    }
+  } else if (isWebReadableStream(input)) {
+    bytes = await readStream(input);
+  } else if (isAsyncIterable(input)) {
+    bytes = await readAsyncIterable(input);
+    // fs.ReadStream carries the path it was opened from.
+    const path = (input as { path?: string | Buffer }).path;
+    if (typeof path === "string") {
+      filename = basename(path);
+      suffix = extname(filename);
+    }
+  } else {
+    throw new TypeError("unsupported audio input type");
+  }
+
+  const wantsPcm =
+    config.sample_rate !== undefined || config.channels !== undefined;
+  const isPcm = pcmSuffixes.includes(suffix) || wantsPcm;
+  if (
+    isPcm &&
+    (config.sample_rate === undefined || config.channels === undefined)
+  ) {
+    throw new Error(
+      "raw PCM audio requires both sample_rate and channels in the config",
+    );
+  }
+
+  const contentType = isPcm ? "audio/pcm" : "audio/wav";
+  if (!filename) filename = isPcm ? "audio.pcm" : "audio.wav";
+
+  return { bytes, filename, contentType };
+}
+
+/**
+ * Serialize the config to the JSON `config` part, validating and normalizing
+ * field values to match the server's caps. The routing `model` is never
+ * included — it travels in the `X-AAI-Model` header. Returns `undefined`
+ * when there is nothing to send, so the part can be omitted entirely.
+ */
+function buildConfigJson(
+  config: SyncTranscriptionConfig,
+): Record<string, unknown> | undefined {
+  if (config.prompt !== undefined && config.prompt.length > maxPromptLength) {
+    throw new Error(
+      `prompt exceeds ${maxPromptLength} characters (got ${config.prompt.length})`,
+    );
+  }
+
+  const json: Record<string, unknown> = {};
+  if (config.prompt !== undefined) json["prompt"] = config.prompt;
+  const keytermsPrompt = normalizeKeytermsPrompt(config.keyterms_prompt);
+  if (keytermsPrompt) json["keyterms_prompt"] = keytermsPrompt;
+  const context = normalizeConversationContext(config.conversation_context);
+  if (context) json["conversation_context"] = context;
+  if (config.language_codes !== undefined)
+    json["language_codes"] = config.language_codes;
+  if (config.sample_rate !== undefined)
+    json["sample_rate"] = config.sample_rate;
+  if (config.channels !== undefined) json["channels"] = config.channels;
+  if (config.timestamps !== undefined) json["timestamps"] = config.timestamps;
+
+  return Object.keys(json).length > 0 ? json : undefined;
+}
+
+function normalizeKeytermsPrompt(
+  keytermsPrompt?: string[],
+): string[] | undefined {
+  if (!keytermsPrompt) return undefined;
+  const terms = keytermsPrompt
+    .map((term) => term.trim())
+    .filter((term) => term.length > 0);
+  const total = terms.reduce((sum, term) => sum + term.length, 0);
+  if (total > maxKeytermsPromptLength) {
+    throw new Error(
+      `keyterms_prompt exceeds ${maxKeytermsPromptLength} characters (got ${total})`,
+    );
+  }
+  return terms.length > 0 ? terms : undefined;
+}
+
+function normalizeConversationContext(
+  context?: string | string[],
+): string[] | undefined {
+  if (context === undefined) return undefined;
+  let turns = (typeof context === "string" ? [context] : context)
+    .map((turn) => turn.trim())
+    .filter((turn) => turn.length > 0);
+  let total = turns.reduce((sum, turn) => sum + turn.length, 0);
+  // Over-cap context is trimmed oldest-first, never rejected.
+  while (
+    turns.length > 0 &&
+    (turns.length > maxContextTurns || total > maxContextLength)
+  ) {
+    total -= turns[0].length;
+    turns = turns.slice(1);
+  }
+  return turns.length > 0 ? turns : undefined;
+}
+
+/**
+ * Build a SyncTranscriptError from a non-200 response. The primary format
+ * is an RFC 9457 problem-details body (`status`/`title`/`detail`); legacy
+ * `{error_code, message}` and `{detail}`-only bodies are also accepted.
+ */
+async function errorFromResponse(
+  response: Response,
+): Promise<SyncTranscriptError> {
+  let errorCode: string | undefined;
+  let message: string | undefined;
+
+  const text = await response.text();
+  try {
+    const body = JSON.parse(text);
+    if (body && typeof body === "object" && !Array.isArray(body)) {
+      if (typeof body.error_code === "string") errorCode = body.error_code;
+      if (errorCode === undefined && typeof body.title === "string") {
+        errorCode = body.title.toLowerCase().replace(/ /g, "_");
+      }
+      if (typeof body.detail === "string") message = body.detail;
+      else if (typeof body.message === "string") message = body.message;
+    }
+  } catch {
+    if (text) message = text;
+  }
+  if (!message) {
+    message = `sync transcription failed with status ${response.status}`;
+  }
+
+  const retryHeader = response.headers.get("retry-after");
+  const retryAfter =
+    retryHeader && /^\d+$/.test(retryHeader)
+      ? parseInt(retryHeader, 10)
+      : undefined;
+
+  return new SyncTranscriptError(
+    message,
+    response.status,
+    errorCode,
+    retryAfter,
+  );
+}
+
+function basename(path: string): string {
+  return path.split(/[\\/]/).pop() ?? path;
+}
+
+function extname(filename: string): string {
+  const dotIndex = filename.lastIndexOf(".");
+  return dotIndex > 0 ? filename.slice(dotIndex).toLowerCase() : "";
+}
+
+function dataUrlToBytes(dataUrl: string): Uint8Array {
+  const base64 = dataUrl.split(",")[1];
+  const binary = atob(base64);
+  const bytes = new Uint8Array(binary.length);
+  for (let i = 0; i < binary.length; i++) bytes[i] = binary.charCodeAt(i);
+  return bytes;
+}
+
+function isWebReadableStream(
+  input: unknown,
+): input is ReadableStream<Uint8Array> {
+  return typeof (input as ReadableStream<Uint8Array>)?.getReader === "function";
+}
+
+function isAsyncIterable(input: unknown): input is AsyncIterable<Uint8Array> {
+  return (
+    typeof (input as AsyncIterable<Uint8Array>)?.[Symbol.asyncIterator] ===
+    "function"
+  );
+}
+
+async function readStream(
+  stream: ReadableStream<Uint8Array>,
+): Promise<Uint8Array> {
+  const chunks: Uint8Array[] = [];
+  const reader = stream.getReader();
+  for (;;) {
+    const { done, value } = await reader.read();
+    if (done) break;
+    chunks.push(value);
+  }
+  return concatChunks(chunks);
+}
+
+async function readAsyncIterable(
+  iterable: AsyncIterable<Uint8Array>,
+): Promise<Uint8Array> {
+  const chunks: Uint8Array[] = [];
+  for await (const chunk of iterable) chunks.push(chunk);
+  return concatChunks(chunks);
+}
+
+function concatChunks(chunks: Uint8Array[]): Uint8Array {
+  const total = chunks.reduce((sum, chunk) => sum + chunk.length, 0);
+  const bytes = new Uint8Array(total);
+  let offset = 0;
+  for (const chunk of chunks) {
+    bytes.set(chunk, offset);
+    offset += chunk.length;
+  }
+  return bytes;
+}

package/src/types/asyncapi.generated.ts

@@ -25,7 +25,12 @@ export type AudioData = ArrayBufferLike;
  * The encoding of the audio data
  * @defaultValue "pcm_s16"le
  */
-export type AudioEncoding = "pcm_s16le" | "pcm_mulaw" | "opus" | "ogg_opus";
+export type AudioEncoding =
+  | "pcm_s16le"
+  | "pcm_mulaw"
+  | "opus"
+  | "ogg_opus"
+  | "aac";
 
 /**
  * Configure the threshold for how long to wait before ending an utterance. Default is 700ms.

package/src/types/index.ts

@@ -1,3 +1,4 @@
+export * from "./sync";
 export * from "./files";
 export * from "./transcripts";
 export * from "./realtime";

package/src/types/services/index.ts

@@ -4,6 +4,7 @@ type BaseServiceParams = {
   apiKey: string;
   baseUrl?: string;
   streamingBaseUrl?: string;
+  syncBaseUrl?: string;
   /**
    * The AssemblyAI user agent to use for requests.
    * The provided components will be merged into the default AssemblyAI user agent.

package/src/types/streaming/index.ts

@@ -94,8 +94,8 @@ export type StreamingTranscriberParams = {
   connectionRetryDelay?: number;
   /**
    * Required for PCM encodings (and for dual-channel mode). May be omitted
-   * for Opus encodings (`opus`, `ogg_opus`) — the stream is self-describing
-   * and the server ignores the value.
+   * for self-describing encodings (`opus`, `ogg_opus`, `aac`) — the stream
+   * carries its own rate and the server ignores the value.
    */
   sampleRate?: number;
   encoding?: AudioEncoding;
@@ -108,6 +108,7 @@ export type StreamingTranscriberParams = {
   maxTurnSilence?: number;
   vadThreshold?: number;
   formatTurns?: boolean;
+  sessionHeartbeat?: boolean;
   filterProfanity?: boolean;
   keyterms?: string[];
   keytermsPrompt?: string[];
@@ -184,6 +185,7 @@ export type StreamingEvents =
   | "llmGatewayResponse"
   | "speakerRevision"
   | "warning"
+  | "heartbeat"
   | "vad"
   | "error";
 
@@ -195,6 +197,7 @@ export type StreamingListeners = {
   llmGatewayResponse?: (event: LLMGatewayResponseEvent) => void;
   speakerRevision?: (event: SpeakerRevisionEvent) => void;
   warning?: (event: WarningEvent) => void;
+  heartbeat?: (event: HeartbeatEvent) => void;
   vad?: (event: VadFrame) => void;
   error?: (error: Error) => void;
 };
@@ -370,6 +373,7 @@ export type StreamingUpdateConfiguration = {
   max_turn_silence?: number;
   vad_threshold?: number;
   format_turns?: boolean;
+  session_heartbeat?: boolean;
   keyterms_prompt?: string[];
   prompt?: string;
   agent_context?: string;
@@ -404,6 +408,14 @@ export type WarningEvent = {
   warning: string;
 };
 
+export type HeartbeatEvent = {
+  type: "Heartbeat";
+  total_audio_received_ms: number;
+  total_duration_ms: number;
+  realtime_factor: number;
+  max_speech_probability: number;
+};
+
 export type LLMGatewayResponseEvent = {
   type: "LLMGatewayResponse";
   turn_order: number;
@@ -443,7 +455,8 @@ export type StreamingEventMessage =
   | LLMGatewayResponseEvent
   | SpeakerRevisionEvent
   | ErrorEvent
-  | WarningEvent;
+  | WarningEvent
+  | HeartbeatEvent;
 
 export type StreamingOperationMessage =
   | StreamingUpdateConfiguration