assemblyai 4.5.0-beta.0 → 4.6.0

package/src/types/openapi.generated.ts

@@ -497,35 +497,50 @@ export type Entity = {
  * The type of entity for the detected entity
  */
 export type EntityType =
+  | "account_number"
   | "banking_information"
   | "blood_type"
   | "credit_card_cvv"
   | "credit_card_expiration"
   | "credit_card_number"
   | "date"
+  | "date_interval"
   | "date_of_birth"
   | "drivers_license"
   | "drug"
+  | "duration"
   | "email_address"
   | "event"
+  | "filename"
+  | "gender_sexuality"
+  | "healthcare_number"
   | "injury"
+  | "ip_address"
   | "language"
   | "location"
+  | "marital_status"
   | "medical_condition"
   | "medical_process"
   | "money_amount"
   | "nationality"
+  | "number_sequence"
   | "occupation"
   | "organization"
+  | "passport_number"
   | "password"
   | "person_age"
   | "person_name"
   | "phone_number"
+  | "physical_attribute"
   | "political_affiliation"
   | "religion"
+  | "statistics"
   | "time"
   | "url"
-  | "us_social_security_number";
+  | "us_social_security_number"
+  | "username"
+  | "vehicle_id"
+  | "zodiac_sign";
 
 /**
  * @example
@@ -573,7 +588,11 @@ export type LemurActionItemsParams = LemurBaseParams & {
  * ```js
  * {
  *   "request_id": "5"e1b27c2-691f-4414-8bc5-f14678442f9e",
- *   "response": "Here are some potential action items based on the transcript:\n\n- Monitor air quality levels in affected areas and issue warnings as needed.\n\n- Advise vulnerable populations like children, the elderly, and those with respiratory conditions to limit time outdoors.\n\n- Have schools cancel outdoor activities when air quality is poor.\n\n- Educate the public on health impacts of smoke inhalation and precautions to take.\n\n- Track progression of smoke plumes using weather and air quality monitoring systems.\n\n- Coordinate cross-regionally to manage smoke exposure as air masses shift.\n\n- Plan for likely increase in such events due to climate change. Expand monitoring and forecasting capabilities.\n\n- Conduct research to better understand health impacts of wildfire smoke and mitigation strategies.\n\n- Develop strategies to prevent and manage wildfires to limit air quality impacts.\n"
+ *   "response": "Here are some potential action items based on the transcript:\n\n- Monitor air quality levels in affected areas and issue warnings as needed.\n\n- Advise vulnerable populations like children, the elderly, and those with respiratory conditions to limit time outdoors.\n\n- Have schools cancel outdoor activities when air quality is poor.\n\n- Educate the public on health impacts of smoke inhalation and precautions to take.\n\n- Track progression of smoke plumes using weather and air quality monitoring systems.\n\n- Coordinate cross-regionally to manage smoke exposure as air masses shift.\n\n- Plan for likely increase in such events due to climate change. Expand monitoring and forecasting capabilities.\n\n- Conduct research to better understand health impacts of wildfire smoke and mitigation strategies.\n\n- Develop strategies to prevent and manage wildfires to limit air quality impacts.\n",
+ *   "usage": {
+ *     "input_tokens": 27,
+ *     "output_tokens": 3
+ *   }
  * }
  * ```
  */
@@ -613,7 +632,6 @@ export type LemurBaseParams = {
   >;
   /**
    * The model that is used for the final prompt after compression is performed.
-   * Defaults to "default".
    *
    * @defaultValue "default
    */
@@ -644,7 +662,11 @@ export type LemurBaseParams = {
  * @example
  * ```js
  * {
- *   "request_id": "5e1b27c2-691f-4414-8bc5-f14678442f9e"
+ *   "request_id": "5e1b27c2-691f-4414-8bc5-f14678442f9e",
+ *   "usage": {
+ *     "input_tokens": 27,
+ *     "output_tokens": 3
+ *   }
  * }
  * ```
  */
@@ -653,6 +675,10 @@ export type LemurBaseResponse = {
    * The ID of the LeMUR request
    */
   request_id: string;
+  /**
+   * The usage numbers for the LeMUR request
+   */
+  usage: LemurUsage;
 };
 
 /**
@@ -660,10 +686,16 @@ export type LemurBaseResponse = {
  *
  */
 export type LemurModel =
+  | "anthropic/claude-3-5-sonnet"
+  | "anthropic/claude-3-opus"
+  | "anthropic/claude-3-haiku"
+  | "anthropic/claude-3-sonnet"
+  | "anthropic/claude-2-1"
+  | "anthropic/claude-2"
   | "default"
+  | "anthropic/claude-instant-1-2"
   | "basic"
-  | "assemblyai/mistral-7b"
-  | "anthropic/claude-2-1";
+  | "assemblyai/mistral-7b";
 
 /**
  * @example
@@ -773,7 +805,11 @@ export type LemurQuestionAnswerParams = LemurBaseParams & {
  *       "answer": "yes",
  *       "question": "Is global warming affecting wildfires?"
  *     }
- *   ]
+ *   ],
+ *   "usage": {
+ *     "input_tokens": 27,
+ *     "output_tokens": 3
+ *   }
  * }
  * ```
  */
@@ -784,12 +820,18 @@ export type LemurQuestionAnswerResponse = LemurBaseResponse & {
   response: LemurQuestionAnswer[];
 };
 
+export type LemurResponse =
+  | LemurTaskResponse
+  | LemurSummaryResponse
+  | LemurQuestionAnswerResponse
+  | LemurActionItemsResponse;
+
 /**
  * @example
  * ```js
  * {
  *   "transcript_ids": [
- *     "64nygnr62k-405c-4ae8-8a6b-d90b40ff3cce"
+ *     "47b95ba5-8889-44d8-bc80-5de38306e582"
  *   ],
  *   "context": "This is an interview about wildfires.",
  *   "final_model": "default",
@@ -810,7 +852,11 @@ export type LemurSummaryParams = LemurBaseParams & {
  * ```js
  * {
  *   "request_id": "5e1b27c2-691f-4414-8bc5-f14678442f9e",
- *   "response": "- Wildfires in Canada are sending smoke and air pollution across parts of the US, triggering air quality alerts from Maine to Minnesota. Concentrations of particulate matter have exceeded safety levels.\n\n- Weather systems are channeling the smoke through Pennsylvania into the Mid-Atlantic and Northeast regions. New York City has canceled outdoor activities to keep children and vulnerable groups indoors.\n\n- Very small particulate matter can enter the lungs and impact respiratory, cardiovascular and neurological health. Young children, the elderly and those with preexisting conditions are most at risk.\n\n- The conditions causing the poor air quality could get worse or shift to different areas in coming days depending on weather patterns. More wildfires may also contribute to higher concentrations.\n\n- Climate change is leading to longer and more severe fire seasons. Events of smoke traveling long distances and affecting air quality over wide areas will likely become more common in the future.\"\n"
+ *   "response": "- Wildfires in Canada are sending smoke and air pollution across parts of the US, triggering air quality alerts from Maine to Minnesota. Concentrations of particulate matter have exceeded safety levels.\n\n- Weather systems are channeling the smoke through Pennsylvania into the Mid-Atlantic and Northeast regions. New York City has canceled outdoor activities to keep children and vulnerable groups indoors.\n\n- Very small particulate matter can enter the lungs and impact respiratory, cardiovascular and neurological health. Young children, the elderly and those with preexisting conditions are most at risk.\n\n- The conditions causing the poor air quality could get worse or shift to different areas in coming days depending on weather patterns. More wildfires may also contribute to higher concentrations.\n\n- Climate change is leading to longer and more severe fire seasons. Events of smoke traveling long distances and affecting air quality over wide areas will likely become more common in the future.\"\n",
+ *   "usage": {
+ *     "input_tokens": 27,
+ *     "output_tokens": 3
+ *   }
  * }
  * ```
  */
@@ -848,7 +894,11 @@ export type LemurTaskParams = {
  * ```js
  * {
  *   "request_id": "5e1b27c2-691f-4414-8bc5-f14678442f9e",
- *   "response": "Based on the transcript, the following locations were mentioned as being affected by wildfire smoke from Canada:\n\n- Maine\n- Maryland\n- Minnesota\n- Mid Atlantic region\n- Northeast region\n- New York City\n- Baltimore\n"
+ *   "response": "Based on the transcript, the following locations were mentioned as being affected by wildfire smoke from Canada:\n\n- Maine\n- Maryland\n- Minnesota\n- Mid Atlantic region\n- Northeast region\n- New York City\n- Baltimore\n",
+ *   "usage": {
+ *     "input_tokens": 27,
+ *     "output_tokens": 3
+ *   }
  * }
  * ```
  */
@@ -859,6 +909,20 @@ export type LemurTaskResponse = {
   response: string;
 } & LemurBaseResponse;
 
+/**
+ * The usage numbers for the LeMUR request
+ */
+export type LemurUsage = {
+  /**
+   * The number of input tokens used by the model
+   */
+  input_tokens: number;
+  /**
+   * The number of output tokens generated by the model
+   */
+  output_tokens: number;
+};
+
 /**
  * @example
  * ```js
@@ -901,7 +965,6 @@ export type ListTranscriptParams = {
 };
 
 /**
- * Details of the transcript page.
  * Details of the transcript page. Transcripts are sorted from newest to oldest. The previous URL always points to a page with older transcripts.
  * @example
  * ```js
@@ -924,12 +987,10 @@ export type PageDetails = {
    */
   limit: number;
   /**
-   * The URL to the next page of transcripts
    * The URL to the next page of transcripts. The next URL always points to a page with newer transcripts.
    */
   next_url: string | null;
   /**
-   * The URL to the previous page of transcripts
    * The URL to the next page of transcripts. The previous URL always points to a page with older transcripts.
    */
   prev_url: string | null;
@@ -1023,34 +1084,54 @@ export type ParagraphsResponse = {
   paragraphs: TranscriptParagraph[];
 };
 
+/**
+ * The type of PII to redact
+ */
 export type PiiPolicy =
-  | "medical_process"
-  | "medical_condition"
+  | "account_number"
+  | "banking_information"
   | "blood_type"
-  | "drug"
-  | "injury"
-  | "number_sequence"
-  | "email_address"
-  | "date_of_birth"
-  | "phone_number"
-  | "us_social_security_number"
-  | "credit_card_number"
-  | "credit_card_expiration"
   | "credit_card_cvv"
+  | "credit_card_expiration"
+  | "credit_card_number"
   | "date"
-  | "nationality"
+  | "date_interval"
+  | "date_of_birth"
+  | "drivers_license"
+  | "drug"
+  | "duration"
+  | "email_address"
   | "event"
+  | "filename"
+  | "gender_sexuality"
+  | "healthcare_number"
+  | "injury"
+  | "ip_address"
   | "language"
   | "location"
+  | "marital_status"
+  | "medical_condition"
+  | "medical_process"
   | "money_amount"
-  | "person_name"
-  | "person_age"
+  | "nationality"
+  | "number_sequence"
+  | "occupation"
   | "organization"
+  | "passport_number"
+  | "password"
+  | "person_age"
+  | "person_name"
+  | "phone_number"
+  | "physical_attribute"
   | "political_affiliation"
-  | "occupation"
   | "religion"
-  | "drivers_license"
-  | "banking_information";
+  | "statistics"
+  | "time"
+  | "url"
+  | "us_social_security_number"
+  | "username"
+  | "vehicle_id"
+  | "zodiac_sign";
 
 /**
  * @example
@@ -1092,6 +1173,11 @@ export type RealtimeTemporaryTokenResponse = {
   token: string;
 };
 
+/**
+ * The notification when the redacted audio is ready.
+ */
+export type RedactedAudioNotification = RedactedAudioResponse;
+
 /**
  * @example
  * ```js
@@ -1234,7 +1320,7 @@ export type SentencesResponse = {
 export type Sentiment = "POSITIVE" | "NEUTRAL" | "NEGATIVE";
 
 /**
- * The result of the sentiment analysis model
+ * The result of the Sentiment Analysis model
  * @example
  * ```js
  * {
@@ -2239,7 +2325,7 @@ export type Transcript = {
   auto_highlights: boolean;
   /**
    * An array of results for the Key Phrases model, if it is enabled.
-   * See {@link https://www.assemblyai.com/docs/models/key-phrases | Key phrases } for more information.
+   * See {@link https://www.assemblyai.com/docs/models/key-phrases | Key Phrases } for more information.
    */
   auto_highlights_result?: AutoHighlightsResult | null;
   /**
@@ -2361,7 +2447,7 @@ export type Transcript = {
   sentiment_analysis?: boolean | null;
   /**
    * An array of results for the Sentiment Analysis model, if it is enabled.
-   * See {@link https://www.assemblyai.com/docs/models/sentiment-analysis | Sentiment analysis } for more information.
+   * See {@link https://www.assemblyai.com/docs/models/sentiment-analysis | Sentiment Analysis } for more information.
    */
   sentiment_analysis_results?: SentimentAnalysisResult[] | null;
   /**
@@ -2430,15 +2516,17 @@ export type Transcript = {
    */
   webhook_auth: boolean;
   /**
-   * The header name which should be sent back with webhook calls
+   * The header name to be sent with the transcript completed or failed webhook requests
    */
   webhook_auth_header_name?: string | null;
   /**
-   * The status code we received from your server when delivering your webhook, if a webhook URL was provided
+   * The status code we received from your server when delivering the transcript completed or failed webhook request, if a webhook URL was provided
    */
   webhook_status_code?: number | null;
   /**
-   * The URL to which we send webhooks upon trancription completion
+   * The URL to which we send webhook requests.
+   * We sends two different types of webhook requests.
+   * One request when a transcript is completed or failed, and one request when the redacted audio is ready if redact_pii_audio is enabled.
    */
   webhook_url?: string | null;
   /**
@@ -2497,19 +2585,100 @@ export type TranscriptLanguageCode =
   | "it"
   | "pt"
   | "nl"
-  | "hi"
-  | "ja"
+  | "af"
+  | "sq"
+  | "am"
+  | "ar"
+  | "hy"
+  | "as"
+  | "az"
+  | "ba"
+  | "eu"
+  | "be"
+  | "bn"
+  | "bs"
+  | "br"
+  | "bg"
+  | "my"
+  | "ca"
   | "zh"
+  | "hr"
+  | "cs"
+  | "da"
+  | "et"
+  | "fo"
   | "fi"
+  | "gl"
+  | "ka"
+  | "el"
+  | "gu"
+  | "ht"
+  | "ha"
+  | "haw"
+  | "he"
+  | "hi"
+  | "hu"
+  | "is"
+  | "id"
+  | "ja"
+  | "jw"
+  | "kn"
+  | "kk"
+  | "km"
   | "ko"
+  | "lo"
+  | "la"
+  | "lv"
+  | "ln"
+  | "lt"
+  | "lb"
+  | "mk"
+  | "mg"
+  | "ms"
+  | "ml"
+  | "mt"
+  | "mi"
+  | "mr"
+  | "mn"
+  | "ne"
+  | "no"
+  | "nn"
+  | "oc"
+  | "pa"
+  | "ps"
+  | "fa"
   | "pl"
+  | "ro"
   | "ru"
+  | "sa"
+  | "sr"
+  | "sn"
+  | "sd"
+  | "si"
+  | "sk"
+  | "sl"
+  | "so"
+  | "su"
+  | "sw"
+  | "sv"
+  | "tl"
+  | "tg"
+  | "ta"
+  | "tt"
+  | "te"
+  | "th"
+  | "bo"
   | "tr"
+  | "tk"
   | "uk"
-  | "vi";
+  | "ur"
+  | "uz"
+  | "vi"
+  | "cy"
+  | "yi"
+  | "yo";
 
 /**
- * A list of transcripts
  * A list of transcripts. Transcripts are sorted from newest to oldest. The previous URL always points to a page with older transcripts.
  * @example
  * ```js
@@ -2649,7 +2818,7 @@ export type TranscriptOptionalParams = {
    */
   auto_chapters?: boolean;
   /**
-   * Whether Key Phrases is enabled, either true or false
+   * Enable Key Phrases, either true or false
    */
   auto_highlights?: boolean;
   /**
@@ -2661,7 +2830,7 @@ export type TranscriptOptionalParams = {
    */
   content_safety?: boolean;
   /**
-   * The confidence threshold for content moderation. Values must be between 25 and 100.
+   * The confidence threshold for the Content Moderation model. Values must be between 25 and 100.
    */
   content_safety_confidence?: number;
   /**
@@ -2669,7 +2838,7 @@ export type TranscriptOptionalParams = {
    */
   custom_spelling?: TranscriptCustomSpelling[];
   /**
-   * Whether custom topics is enabled, either true or false
+   * Enable custom topics, either true or false
    */
   custom_topics?: boolean;
   /**
@@ -2702,7 +2871,7 @@ export type TranscriptOptionalParams = {
    */
   language_code?: LiteralUnion<TranscriptLanguageCode, string> | null;
   /**
-   * Whether {@link https://www.assemblyai.com/docs/models/speech-recognition#automatic-language-detection | Automatic language detection } was enabled in the transcription request, either true or false.
+   * Enable {@link https://www.assemblyai.com/docs/models/speech-recognition#automatic-language-detection | Automatic language detection }, either true or false.
    */
   language_detection?: boolean;
   /**
@@ -2770,21 +2939,21 @@ export type TranscriptOptionalParams = {
    */
   summary_type?: SummaryType;
   /**
-   * The list of custom topics provided, if custom topics is enabled
+   * The list of custom topics
    */
   topics?: string[];
   /**
-   * The header name which should be sent back with webhook calls
+   * The header name to be sent with the transcript completed or failed webhook requests
    * @defaultValue null
    */
   webhook_auth_header_name?: string | null;
   /**
-   * Specify a header name and value to send back with a webhook call for added security
+   * The header value to send back with the transcript completed or failed webhook requests for added security
    * @defaultValue null
    */
   webhook_auth_header_value?: string | null;
   /**
-   * The URL to which AssemblyAI send webhooks upon trancription completion
+   * The URL to which we send webhook requests. We sends two different types of webhook requests. One request when a transcript is completed or failed, and one request when the redacted audio is ready if redact_pii_audio is enabled.
    */
   webhook_url?: string;
   /**
@@ -3135,6 +3304,13 @@ export type TranscriptUtterance = {
   words: TranscriptWord[];
 };
 
+/**
+ * The notifications sent to the webhook URL.
+ */
+export type TranscriptWebhookNotification =
+  | TranscriptReadyNotification
+  | RedactedAudioNotification;
+
 /**
  * @example
  * ```js

package/src/types/services/index.ts

@@ -1,6 +1,14 @@
+import { UserAgent } from "..";
+
 type BaseServiceParams = {
   apiKey: string;
   baseUrl?: string;
+  /**
+   * The AssemblyAI user agent to use for requests.
+   * The provided components will be merged into the default AssemblyAI user agent.
+   * If `false`, the AssemblyAI user agent will be removed.
+   */
+  userAgent?: UserAgent | false;
 };
 
 export type { BaseServiceParams };

package/src/types/transcripts/index.ts

@@ -54,3 +54,13 @@ export type SubmitParams = TranscribeParams;
  * The options to transcribe an audio file, including polling options.
  */
 export type TranscribeOptions = PollingOptions;
+
+/**
+ * The PII redacted audio file, transmitted over the network.
+ */
+export type RedactedAudioFile = {
+  arrayBuffer: () => Promise<ArrayBuffer>;
+  blob: () => Promise<Blob>;
+  body: ReadableStream<Uint8Array> | null;
+  bodyUsed: boolean;
+};

package/src/utils/path.ts

@@ -1,6 +1,7 @@
 export function getPath(path: string) {
   if (path.startsWith("http")) return null;
   if (path.startsWith("https")) return null;
+  if (path.startsWith("data:")) return null;
   if (path.startsWith("file://")) return path.substring(7);
   if (path.startsWith("file:")) return path.substring(5);
   return path;

package/src/utils/userAgent.ts

@@ -0,0 +1,51 @@
+import { UserAgent } from "../types";
+
+export const buildUserAgent = (userAgent: UserAgent | false) =>
+  defaultUserAgentString +
+  (userAgent === false
+    ? ""
+    : " AssemblyAI/1.0 (" +
+      Object.entries({ ...defaultUserAgent, ...userAgent })
+        .map(([key, item]) =>
+          item ? `${key}=${item.name}/${item.version}` : "",
+        )
+        .join(" ") +
+      ")");
+
+let defaultUserAgentString = "";
+if (typeof navigator !== "undefined" && navigator.userAgent) {
+  defaultUserAgentString += navigator.userAgent;
+}
+
+const defaultUserAgent: UserAgent = {
+  sdk: { name: "JavaScript", version: "__SDK_VERSION__" },
+};
+
+if (typeof process !== "undefined") {
+  if (process.versions.node && defaultUserAgentString.indexOf("Node") === -1) {
+    defaultUserAgent.runtime_env = {
+      name: "Node",
+      version: process.versions.node,
+    };
+  }
+  if (process.versions.bun && defaultUserAgentString.indexOf("Bun") === -1) {
+    defaultUserAgent.runtime_env = {
+      name: "Bun",
+      version: process.versions.bun,
+    };
+  }
+}
+
+declare const Deno:
+  | {
+      version: {
+        deno: string;
+      };
+    }
+  | undefined;
+
+if (typeof Deno !== "undefined") {
+  if (process.versions.bun && defaultUserAgentString.indexOf("Deno") === -1) {
+    defaultUserAgent.runtime_env = { name: "Deno", version: Deno.version.deno };
+  }
+}