assemblyai 1.0.1 → 2.0.0-beta

package/src/services/realtime/index.ts

@@ -0,0 +1,106 @@
+import { BaseServiceParams } from "@/services/base";
+import {
+  Connectable,
+  RealtimeConnectionParams,
+  RealtimeListener,
+  RealtimeListeners,
+  RealtimeTranscript,
+} from "@/types";
+import { RealtimeError, RealtimeErrorType } from "@/utils/errors";
+import WebSocket, { type CloseEvent, type MessageEvent } from "ws";
+
+export default class RealtimeService
+  implements Connectable<RealtimeTranscript, RealtimeConnectionParams>
+{
+  private params: BaseServiceParams;
+  private socket?: WebSocket;
+  private listeners: RealtimeListeners = {};
+
+  constructor(params: BaseServiceParams) {
+    this.params = params;
+  }
+
+  private connectionUrl(
+    connectionParams: Required<RealtimeConnectionParams>
+  ): URL {
+    if (!this.params.baseUrl) {
+      throw new Error("Missing base URL");
+    }
+
+    const url = new URL(this.params.baseUrl);
+
+    if (url.protocol !== "wss:") {
+      throw new Error("Invalid protocol, must be wss");
+    }
+
+    const searchParams = new URLSearchParams();
+    searchParams.set("sample_rate", connectionParams.sampleRate.toString());
+    searchParams.set("word_boost", connectionParams.wordBoost.join(","));
+    url.search = searchParams.toString();
+
+    return url;
+  }
+
+  on(event: "data", listener: (transcript: RealtimeTranscript) => void): void;
+  on(event: "error", listener: (error: Error) => void): void;
+  on(event: "open", listener: () => void): void;
+  on(event: "close", listener: (code: number, reason: string) => void): void;
+  on(event: RealtimeListener, listener: (...args: any[]) => void) {
+    this.listeners[event] = listener;
+  }
+
+  connect(connectionParams: RealtimeConnectionParams) {
+    if (this.socket) {
+      throw new Error("Already connected");
+    }
+
+    const url = this.connectionUrl({
+      ...connectionParams,
+      sampleRate: connectionParams.sampleRate || 16000,
+      wordBoost: connectionParams.wordBoost || [],
+    });
+
+    this.socket = new WebSocket(url.toString(), {
+      headers: {
+        Authorization: this.params.token,
+      },
+    });
+
+    this.socket.onopen = () => this.listeners.open?.();
+    this.socket.onclose = ({ code, reason }: CloseEvent) =>
+      this.listeners.close?.(code, reason);
+
+    this.socket.onerror = (error) => {
+      const code = error.message as unknown as RealtimeErrorType;
+      const realtimeError = new RealtimeError(code);
+      this.listeners.error?.(realtimeError);
+    };
+
+    this.socket.onmessage = ({ data }: MessageEvent) => {
+      const message = JSON.parse(data.toString()) as RealtimeTranscript;
+      this.listeners.data?.(message);
+    };
+  }
+
+  send(data: ArrayBuffer) {
+    if (!this.socket || this.socket.readyState !== WebSocket.OPEN) {
+      throw new Error("Socket is not open for communication");
+    }
+
+    const payload = {
+      audio_data: Buffer.from(data).toString("base64"),
+    };
+
+    this.socket.send(JSON.stringify(payload));
+  }
+
+  disconnect() {
+    if (this.socket) {
+      this.socket.removeAllListeners();
+      this.socket.close();
+    }
+
+    this.listeners = {};
+    this.socket = undefined;
+  }
+}

package/src/services/transcripts/index.ts

@@ -0,0 +1,156 @@
+import BaseService from "@/services/base";
+import TranscriptSubtitleService from "./subtitles";
+import {
+  ParagraphsBody,
+  SegmentType,
+  SentencesBody,
+  Transcript,
+  TranscriptList,
+  TranscriptParams,
+  TranscriptPollingSettings,
+  Createable,
+  Deletable,
+  Listable,
+  Retrieveable,
+  TranscriptListInternal,
+  TranscriptListItemInternal,
+  TranscriptListItem,
+} from "@/types";
+import TranscriptRedactionService from "./redactions";
+import { AxiosInstance } from "axios";
+import FileService from "../files";
+
+export default class TranscriptService
+  extends BaseService
+  implements
+    Createable<Transcript, TranscriptParams, TranscriptPollingSettings>,
+    Retrieveable<Transcript>,
+    Deletable<Transcript>,
+    Listable<TranscriptList>
+{
+  /**
+   * The subtitles service.
+   */
+  public subtitles: TranscriptSubtitleService;
+
+  /**
+   * The redactions service.
+   */
+  public redactions: TranscriptRedactionService;
+
+  constructor(client: AxiosInstance, private files: FileService) {
+    super(client);
+    this.subtitles = new TranscriptSubtitleService(client);
+    this.redactions = new TranscriptRedactionService(client);
+  }
+
+  async create(
+    params: TranscriptParams,
+    settings: TranscriptPollingSettings | null = {
+      pollingInterval: 3000,
+      pollingTimeout: 180000,
+    }
+  ): Promise<Transcript> {
+    const path = getPath(params.audio_url);
+    if (path !== null) {
+      const uploadUrl = await this.files.upload(path);
+      params.audio_url = uploadUrl;
+    }
+
+    const res = await this.client.post<Transcript>("/v2/transcript", params);
+
+    if (settings && settings.pollingInterval) {
+      return await this.poll(res.data.id, settings);
+    }
+
+    return res.data;
+  }
+
+  private async poll(
+    transcriptId: string,
+    settings: TranscriptPollingSettings
+  ): Promise<Transcript> {
+    const {
+      pollingInterval,
+      pollingTimeout,
+      resolver = (t) => t.status === "completed" || t.status === "error",
+    } = settings;
+    const startTime = Date.now();
+
+    while (true) {
+      const transcript = await this.retrieve(transcriptId);
+      if (resolver(transcript)) {
+        return transcript;
+      } else if (Date.now() - startTime < pollingTimeout) {
+        await new Promise((resolve) => setTimeout(resolve, pollingInterval));
+      } else {
+        throw new Error("Polling timeout");
+      }
+    }
+  }
+
+  async retrieve(id: string): Promise<Transcript> {
+    const res = await this.client.get<Transcript>(`/v2/transcript/${id}`);
+    return res.data;
+  }
+
+  async list(nextUrl?: string): Promise<TranscriptList> {
+    const { data } = await this.client.get<TranscriptListInternal>(
+      nextUrl || "/v2/transcript"
+    );
+    for (const transcriptListItem of data.transcripts as unknown as (
+      | TranscriptListItemInternal
+      | TranscriptListItem
+    )[]) {
+      transcriptListItem.created = new Date(transcriptListItem.created);
+      transcriptListItem.completed = new Date(transcriptListItem.completed);
+    }
+
+    return data as unknown as TranscriptList;
+  }
+
+  async delete(id: string): Promise<Transcript> {
+    const res = await this.client.delete<Transcript>(`/v2/transcript/${id}`);
+    return res.data;
+  }
+
+  /**
+   * Retrieve all segments of a transcript.
+   * @param id The identifier of the transcript.
+   * @param type The type of segment to retrieve.
+   * @return A promise that resolves to the retrieved resource.
+   */
+  private async segments<T>(id: string, type: SegmentType): Promise<T> {
+    const { data } = await this.client.get<T>(`/v2/transcript/${id}/${type}`);
+    return data;
+  }
+
+  /**
+   * Retrieve all sentences of a transcript.
+   * @param id The identifier of the transcript.
+   * @return A promise that resolves to the sentences.
+   */
+  async sentences(id: string): Promise<SentencesBody> {
+    return this.segments<SentencesBody>(id, "sentences");
+  }
+
+  /**
+   * Retrieve all paragraphs of a transcript.
+   * @param id The identifier of the transcript.
+   * @return A promise that resolves to the paragraphs.
+   */
+  async paragraphs(id: string): Promise<ParagraphsBody> {
+    return this.segments<ParagraphsBody>(id, "paragraphs");
+  }
+}
+
+function getPath(path: string) {
+  let url: URL;
+  try {
+    url = new URL(path);
+    if (url.protocol === "file:") return url.pathname;
+    else return null;
+  } catch {
+    return path;
+  }
+}

package/src/services/transcripts/redactions.ts

@@ -0,0 +1,27 @@
+import BaseService from "@/services/base";
+import { Retrieveable } from "@/types";
+
+type TranscriptRedactionParams = {
+  transcript_id: string;
+  path: string;
+};
+
+type TranscriptRedactionResponse = {
+  status: "redacted_audio_ready";
+  redacted_audio_url: string;
+};
+
+export default class TranscriptRedactionService
+  extends BaseService
+  implements
+    Retrieveable<TranscriptRedactionResponse, TranscriptRedactionParams>
+{
+  async retrieve(
+    params: TranscriptRedactionParams
+  ): Promise<TranscriptRedactionResponse> {
+    const { data } = await this.client.get<TranscriptRedactionResponse>(
+      `/v2/transcript/${params.transcript_id}/redacted-audio`
+    );
+    return data;
+  }
+}

package/src/services/transcripts/subtitles.ts

@@ -0,0 +1,26 @@
+import BaseService from "@/services/base";
+import { Retrieveable } from "@/types";
+
+type Subtitle = string;
+type SubtitleFormat = "vtt" | "srt";
+
+type TranscriptionSubtitleParams = {
+  transcript_id: string;
+  format: SubtitleFormat;
+};
+
+export default class TranscriptSubtitleService
+  extends BaseService
+  implements Retrieveable<Subtitle, TranscriptionSubtitleParams>
+{
+  async retrieve({
+    transcript_id,
+    format,
+  }: TranscriptionSubtitleParams): Promise<Subtitle> {
+    const { data } = await this.client.get<Subtitle>(
+      `/v2/transcript/${transcript_id}/${format}`
+    );
+
+    return data;
+  }
+}

package/src/types/core/index.ts

@@ -0,0 +1,2 @@
+export type * from "./transcript";
+export type * from "./segments";

package/src/types/core/params.ts

@@ -0,0 +1,3 @@
+type WordBoostWeight = "low" | "default" | "high";
+
+export type { WordBoostWeight };

package/src/types/core/segments.ts

@@ -0,0 +1,29 @@
+type SegmentType = "paragraphs" | "sentences";
+
+type Segment<T> = T & {
+  id: string;
+  confidence: number;
+  audio_duration: number;
+};
+
+type ParagraphsBody = Segment<{ paragraphs: Utterance[] }>;
+type SentencesBody = Segment<{ sentences: Utterance[] }>;
+
+type Word = {
+  text: string;
+  start: number;
+  end: number;
+  confidence: number;
+  speaker?: string | null;
+};
+
+type Utterance = {
+  start: number;
+  end: number;
+  confidence: number;
+  speaker?: string;
+  text: string;
+  words: Word[];
+};
+
+export type { Word, Utterance, ParagraphsBody, SentencesBody, SegmentType };

package/src/types/core/transcript.ts

@@ -0,0 +1,159 @@
+import {
+  AutoChapter,
+  AutoHighlights,
+  ContentSafetyLabels,
+  Entity,
+  PIIAudioQuality,
+  PIIRedactionPolicies,
+  SentimentAnalysis,
+  SummaryModel,
+  SummaryType,
+  TopicDetection,
+} from "@/types/models";
+import { PageDetails } from "@/types/shared";
+import { Utterance, Word } from "./segments";
+import { WordBoostWeight } from "./params";
+
+type TranscriptStatus = "queued" | "processing" | "completed" | "error";
+
+type TranscriptBase = {
+  status?: TranscriptStatus | null;
+
+  // Core Transcription
+  audio_url: string;
+  audio_start_from?: number | null;
+  audio_end_at?: number | null;
+  filter_profanity?: boolean;
+  custom_spelling?: null;
+  disfluencies?: boolean;
+  dual_channel?: boolean | null;
+  punctuate?: boolean;
+  speaker_labels?: boolean;
+  format_text?: boolean;
+
+  // Audio Intelligence
+  auto_chapters?: boolean;
+  content_safety?: boolean;
+  entity_detection?: boolean;
+  language_code?: string;
+  language_detection?: boolean;
+  iab_categories?: boolean;
+  auto_highlights?: boolean;
+  summarization?: boolean;
+  summary_type?: SummaryType;
+  summary_model?: SummaryModel;
+  redact_pii?: boolean;
+  redact_pii_sub?: string;
+  redact_pii_audio_quality?: PIIAudioQuality;
+  redact_pii_audio?: boolean;
+  redact_pii_policies?: PIIRedactionPolicies[];
+  sentiment_analysis?: boolean;
+  webhook_auth_header_name?: null;
+  webhook_auth?: boolean;
+  webhook_status_code?: null | number;
+  webhook_url?: null | string;
+
+  word_boost?: string[];
+  boost_param?: WordBoostWeight;
+};
+
+type TranscriptParams = Omit<TranscriptBase, "status">;
+
+type TranscriptQueued = TranscriptBase & {
+  id: string;
+  status: "queued";
+};
+
+type TranscriptProcessing = TranscriptBase & {
+  id: string;
+  status: "processing";
+};
+
+type TranscriptCompleted = TranscriptBase & {
+  status: "completed";
+  id: string;
+  audio_duration?: number;
+  words?: Word[];
+  utterances?: Utterance[];
+  text?: string;
+  confidence?: number;
+  chapters?: AutoChapter[] | null;
+  content_safety_labels?: ContentSafetyLabels;
+  entities?: Entity[];
+  iab_categories_result?: TopicDetection;
+  auto_highlights_result?: AutoHighlights;
+  summary?: string;
+  sentiment_analysis_results?: SentimentAnalysis[];
+
+  /**
+   * @deprecated Acoustic Model is deprecated and will be removed in a future release
+   */
+  acoustic_model?: unknown;
+
+  /**
+   * @deprecated Speed Boost is deprecated and will be removed in a future release
+   */
+  speed_boost?: boolean;
+
+  /**
+   * @deprecated Language Model is deprecated and will be removed in a future release
+   */
+  language_model?: unknown;
+};
+
+type TranscriptFailed = TranscriptBase & {
+  status: "error";
+  id: string;
+  error?: string;
+};
+
+type Transcript =
+  | Readonly<TranscriptQueued>
+  | Readonly<TranscriptProcessing>
+  | Readonly<TranscriptCompleted>
+  | Readonly<TranscriptFailed>;
+
+type TranscriptPollingSettings = {
+  pollingInterval: number;
+  pollingTimeout: number;
+  resolver?: (transcript: Transcript) => boolean;
+};
+
+type TranscriptListInternal = {
+  page_details: PageDetails;
+  transcripts: TranscriptListItemInternal[];
+};
+
+type TranscriptListItemInternal = {
+  id: string;
+  resource_url: string;
+  status: TranscriptStatus;
+  created: string;
+  completed: string;
+  audio_url: string;
+};
+
+type TranscriptList = {
+  page_details: PageDetails;
+  transcripts: TranscriptListItem[];
+};
+
+type TranscriptListItem = {
+  id: string;
+  resource_url: string;
+  status: TranscriptStatus;
+  created: Date;
+  completed: Date;
+  audio_url: string;
+};
+
+export type {
+  Transcript,
+  TranscriptStatus,
+  TranscriptParams,
+  TranscriptList,
+  TranscriptListItem,
+  TranscriptListInternal,
+  TranscriptListItemInternal,
+  TranscriptPollingSettings,
+};

package/src/types/index.ts

@@ -0,0 +1,6 @@
+export type * from "./core";
+export type * from "./lemur";
+export type * from "./models";
+export type * from "./realtime";
+export type * from "./services";
+export type * from "./shared";

package/src/types/lemur/index.ts

@@ -0,0 +1,97 @@
+type LeMURModel = "default" | "basic";
+type LeMURContext = string | object | LeMURContext[];
+
+type LeMURBaseServiceParams = {
+  /**
+   * A list of completed transcripts with text. Up to 100 files max, or 100 hours max. Whichever is lower.
+   */
+  transcript_ids: string[];
+  /**
+   * The model that is used for the final prompt after compression is performed (options: "basic" and "default").
+   */
+  final_model?: LeMURModel;
+  /**
+   * Max output size in tokens. Up to 4000 allowed.
+   */
+  max_output_size?: number;
+};
+
+type SummaryParams = LeMURBaseServiceParams & {
+  /**
+   * How you want the summary to be returned. This can be any text. Examples: "TLDR", "bullet points"
+   */
+  answer_format?: string;
+  /**
+   * Any context about the transcripts you wish to provide. This can be a string, or free-form JSON.
+   */
+  context?: LeMURContext;
+};
+
+type Question = {
+  /**
+   * The question you wish to ask. For more complex questions use default model.
+   */
+  question: string;
+  /**
+   * Any context about the transcripts you wish to provide. This can be a string, or free-form JSON.
+   */
+  context?: LeMURContext;
+  /**
+   * How you want the answer to be returned. This can be any text. Cannot be used with answer_options. Examples: "short sentence", "bullet points"
+   */
+  answer_format?: string;
+  /**
+   * What discrete options to return. Useful for precise responses. Cannot be used with answer_format. Example: ["Yes", "No"]
+   */
+  answer_options?: string[];
+};
+
+type QuestionAnswerParams = LeMURBaseServiceParams & {
+  /**
+   * Any context about the transcripts you wish to provide. This can be a string, or free-form JSON.
+   */
+  context?: LeMURContext;
+  /**
+   * 	A list of questions to ask. Question format listed below.
+   */
+  questions: Question[];
+};
+
+type ActionItemsParams = LeMURBaseServiceParams & {
+  /**
+   * Any context about the transcripts you wish to provide. This can be a string, or free-form JSON.
+   */
+  context?: LeMURContext;
+};
+
+type TaskParams = LeMURBaseServiceParams & {
+  /**
+   * Your text to prompt the model to produce a desired output, including any context you want to pass into the model.
+   */
+  prompt: string;
+};
+
+type LeMURResponse = {
+  response: string;
+  request_id: string;
+};
+
+type QuestionAnswer = {
+  question: string;
+  answer: string;
+};
+
+type LeMURQuestionAnswerResponse = {
+  response: QuestionAnswer[];
+  request_id: string;
+};
+
+export type {
+  SummaryParams,
+  QuestionAnswerParams,
+  ActionItemsParams,
+  TaskParams,
+  LeMURResponse,
+  LeMURQuestionAnswerResponse,
+  LeMURBaseServiceParams,
+};

package/src/types/models/auto_chapter.ts

@@ -0,0 +1,9 @@
+type AutoChapter = {
+  summary: string;
+  headline: string;
+  gist: string;
+  start: number;
+  end: number;
+};
+
+export type { AutoChapter };

package/src/types/models/auto_highlights.ts

@@ -0,0 +1,14 @@
+import { ModelResultStatus } from "./shared";
+
+type AutoHighlightsResult = {
+  text: string;
+  count: number;
+  rank: number;
+};
+
+type AutoHighlights = {
+  status: ModelResultStatus;
+  results: AutoHighlightsResult[];
+};
+
+export type { AutoHighlights };

package/src/types/models/content_safety.ts

@@ -0,0 +1,34 @@
+import { ModelResultStatus, ModelSummary } from "./shared";
+import { Timestamp } from "@/types/shared";
+
+type ContentSafetyLabel = {
+  label: string;
+  confidence: number;
+  severity: number | null;
+};
+
+type ContentSafetyResult = {
+  text: string;
+  labels: ContentSafetyLabel[];
+  timestamp: Timestamp;
+};
+
+type ContentSafetySeverityScore = {
+  low: number;
+  medium: number;
+  high: number;
+};
+
+type ContentSafetySeverityScoreSummary = Record<
+  string,
+  ContentSafetySeverityScore
+>;
+
+type ContentSafetyLabels = {
+  status: ModelResultStatus;
+  results: ContentSafetyResult[];
+  summary: ModelSummary;
+  severity_score_summary: ContentSafetySeverityScoreSummary;
+};
+
+export type { ContentSafetyLabels };

package/src/types/models/entity_detection.ts

@@ -0,0 +1,8 @@
+type Entity = {
+  entity_type: string;
+  text: string;
+  start: number;
+  end: number;
+};
+
+export type { Entity };

package/src/types/models/index.ts

@@ -0,0 +1,8 @@
+export type * from "./auto_chapter";
+export type * from "./auto_highlights";
+export type * from "./content_safety";
+export type * from "./entity_detection";
+export type * from "./pii";
+export type * from "./sentiment_analysis";
+export type * from "./summarization";
+export type * from "./topic_detection";