@mediapipe/tasks-audio 1.2.0-rc.20261021 → 1.2.0-rc.20261024

package/audio.d.ts

@@ -89,6 +89,143 @@ declare abstract class AudioTaskRunner<T> extends TaskRunner {
     setDefaultSampleRate(sampleRate: number): void;
 }
 
+/**
+ * MediaPipe Audio Task that performs automatic speech recognition (ASR) in
+ * unary and streaming modes via WASM SIMD or WebGPU.
+ */
+export declare class AudioTranscriber extends TaskRunner {
+    /**
+     * Initializes the Wasm runtime and creates a new `AudioTranscriber` based on
+     * the provided options.
+     *
+     * @export
+     * @param wasmFileset A configuration object that provides the location of
+     *     the Wasm binary and its loader.
+     * @param options The options for the AudioTranscriber. Note that either a
+     *     path to the model or the model buffer itself must be provided via
+     *     `baseOptions`.
+     */
+    static createFromOptions(wasmFileset: WasmFileset, options: AudioTranscriberOptions): Promise<AudioTranscriber>;
+    /**
+     * Initializes the Wasm runtime and creates a new `AudioTranscriber` from a
+     * model asset path.
+     *
+     * @export
+     * @param wasmFileset A configuration object that provides the location of
+     *     the Wasm binary and its loader.
+     * @param modelAssetPath The path or URL to the `.litertlm` model asset.
+     */
+    static createFromModelPath(wasmFileset: WasmFileset, modelAssetPath: string | string): Promise<AudioTranscriber>;
+    /**
+     * Initializes the Wasm runtime and creates a new `AudioTranscriber` from a
+     * model asset buffer or stream.
+     *
+     * @export
+     * @param wasmFileset A configuration object that provides the location of
+     *     the Wasm binary and its loader.
+     * @param modelAssetBuffer A `Uint8Array` or `ReadableStreamDefaultReader`
+     *     containing the `.litertlm` model bytes.
+     */
+    static createFromModelBuffer(wasmFileset: WasmFileset, modelAssetBuffer: Uint8Array | ReadableStreamDefaultReader): Promise<AudioTranscriber>;
+    private constructor();
+    /**
+     * Sets the options for the `AudioTranscriber` and initializes the underlying
+     * engine. Options cannot be updated once the engine has been created.
+     *
+     * @export
+     * @param options The options for the AudioTranscriber.
+     */
+    setOptions(options: AudioTranscriberOptions): Promise<void>;
+    /**
+     * Transcribes mono PCM float audio samples in `[-1.0, 1.0]`.
+     *
+     * @export
+     * @param audioSamples Mono PCM float audio samples at `sampleRateHz`.
+     * @param endOfStream Whether this chunk marks the end of the audio stream.
+     *     Defaults to `true` (unary single-utterance mode).
+     * @return The transcription result containing segments and concatenated text.
+     */
+    transcribe(audioSamples: Float32Array | number[], endOfStream?: boolean): Promise<AudioTranscriberResult>;
+    /**
+     * Feeds a streaming chunk of mono PCM float audio samples (`endOfStream = false`)
+     * and returns any newly emitted interim or final segments.
+     *
+     * @export
+     * @param audioSamples Mono PCM float audio samples at `sampleRateHz`.
+     * @return Newly emitted segments from the streaming session.
+     */
+    transcribeStream(audioSamples: Float32Array | number[]): Promise<AudioTranscriberResult>;
+    /**
+     * Flushes any remaining buffered audio and finalizes the current stream.
+     *
+     * @export
+     * @return Finalized transcription segments.
+     */
+    finishStream(): Promise<AudioTranscriberResult>;
+    /**
+     * Prewarms the underlying inference engine to reduce first-chunk latency.
+     *
+     * @export
+     */
+    prewarm(): Promise<void>;
+    /**
+     * Closes the `AudioTranscriber` and releases native engine and Wasm memory.
+     *
+     * @export
+     */
+    close(): void;
+}
+
+/**
+ * Model architecture family for AudioTranscriber.
+ */
+export declare type AudioTranscriberModelArchitecture = "AUTO" | "OMNI" | "TINY_GEMMA";
+
+/**
+ * Options to configure the MediaPipe AudioTranscriber Task on the Web.
+ */
+export declare interface AudioTranscriberOptions extends TaskRunnerOptions {
+    /**
+     * Input audio sample rate in Hz. Defaults to 16000 Hz when unset or 0.
+     */
+    sampleRateHz?: number;
+    /**
+     * Whether to emit interim (non-final) transcription segments during
+     * streaming. Defaults to false.
+     */
+    enableInterimResults?: boolean;
+    /**
+     * Model architecture backend family ('AUTO', 'OMNI', or 'TINY_GEMMA', or
+     * numeric code 0=AUTO, 1=OMNI, 2=TINY_GEMMA). Defaults to 'AUTO'.
+     */
+    modelArchitecture?: AudioTranscriberModelArchitecture | number;
+    /**
+     * Optional system or user prompt override for generative ASR models.
+     */
+    prompt?: string;
+    /**
+     * Optional directory path in the Wasm virtual filesystem for caching
+     * compiled model artifacts.
+     */
+    cacheDir?: string;
+    /**
+     * Optional byte length of `baseOptions.modelAssetBuffer` when provided as a
+     * `ReadableStreamDefaultReader`, enabling single-pass zero-copy streaming
+     * allocation directly in the Wasm heap.
+     */
+    modelAssetSize?: number;
+}
+
+/**
+ * Result of an AudioTranscriber invocation.
+ */
+export declare interface AudioTranscriberResult {
+    /** Ordered list of transcribed segments. */
+    segments: TranscriptionSegment[];
+    /** Concatenated full transcript across all segments in this result. */
+    text: string;
+}
+
 /**
  * Copyright 2022 The MediaPipe Authors.
  *
@@ -325,6 +462,17 @@ export declare class FilesetResolver {
     static forVisionTasks(basePath?: string, useModule?: boolean): Promise<WasmFileset>;
 }
 
+/**
+ * Per-utterance synthesis overrides passed to `TextToSpeech.synthesize()` and
+ * `TextToSpeech.synthesizeStream()`.
+ */
+export declare interface SynthesisOptions {
+    /** Optional voice identifier override (e.g. "af_heart", "ef_dora"). */
+    voiceId?: string;
+    /** Optional BCP-47 locale override (e.g. "en-US", "es-ES", "ja-JP"). */
+    locale?: string;
+}
+
 /** Base class for all MediaPipe Tasks. */
 declare abstract class TaskRunner {
     protected constructor();
@@ -344,6 +492,259 @@ declare interface TaskRunnerOptions {
     baseOptions?: BaseOptions_2;
 }
 
+/**
+ * MediaPipe Audio Task that performs neural text-to-speech synthesis in unary
+ * and streaming modes via WASM SIMD or WebGPU.
+ */
+export declare class TextToSpeech extends TaskRunner {
+    /**
+     * Initializes the Wasm runtime and creates a new `TextToSpeech` instance
+     * based on the provided options.
+     *
+     * @export
+     * @param wasmFileset A configuration object that provides the location of
+     *     the Wasm binary and its loader.
+     * @param options The options for TextToSpeech. Note that either a path to the
+     *     model or the model buffer itself must be provided via `baseOptions`.
+     */
+    static createFromOptions(wasmFileset: WasmFileset, options: TextToSpeechOptions): Promise<TextToSpeech>;
+    /**
+     * Initializes the Wasm runtime and creates a new `TextToSpeech` instance
+     * from a model asset path.
+     *
+     * @export
+     * @param wasmFileset A configuration object that provides the location of
+     *     the Wasm binary and its loader.
+     * @param modelAssetPath The path or URL to the `.litertlm` model asset.
+     */
+    static createFromModelPath(wasmFileset: WasmFileset, modelAssetPath: string | string): Promise<TextToSpeech>;
+    /**
+     * Initializes the Wasm runtime and creates a new `TextToSpeech` instance
+     * from a model asset buffer or stream.
+     *
+     * @export
+     * @param wasmFileset A configuration object that provides the location of
+     *     the Wasm binary and its loader.
+     * @param modelAssetBuffer A `Uint8Array` or `ReadableStreamDefaultReader`
+     *     containing the `.litertlm` model bytes.
+     */
+    static createFromModelBuffer(wasmFileset: WasmFileset, modelAssetBuffer: Uint8Array | ReadableStreamDefaultReader): Promise<TextToSpeech>;
+    private constructor();
+    /**
+     * Sets the options for `TextToSpeech` and initializes the underlying engine.
+     * Options cannot be updated once the engine has been created.
+     *
+     * @export
+     * @param options The options for TextToSpeech.
+     */
+    setOptions(options: TextToSpeechOptions): Promise<void>;
+    /**
+     * Queries available voices bundled in the initialized model.
+     *
+     * @export
+     */
+    listVoices(): Promise<VoiceInfo[]>;
+    /**
+     * Synthesizes a complete utterance and returns the full PCM audio buffer.
+     *
+     * @export
+     * @param text The text to synthesize.
+     * @param options Optional per-utterance voice or locale overrides.
+     */
+    synthesize(text: string, options?: SynthesisOptions): Promise<TextToSpeechResult>;
+    /**
+     * Starts a streaming synthesis session returning a `TextToSpeechStream`
+     * (supporting both `readNext()` and `for await (const chunk of stream)`).
+     *
+     * @export
+     * @param initialText Initial text to synthesize (pass `""` when streaming
+     *     incremental LLM text deltas via `stream.pushText()`).
+     * @param options Optional per-utterance voice or locale overrides.
+     * @param finishInput Pass `true` (default) for single-utterance streaming, or
+     *     `false` to keep the input stream open for `stream.pushText()` and
+     *     `stream.finishInput()`.
+     */
+    synthesizeStream(initialText?: string, options?: SynthesisOptions, finishInput?: boolean): Promise<TextToSpeechStream>;
+    /**
+     * Cancels any in-flight unary or streaming synthesis on this `TextToSpeech`
+     * instance.
+     *
+     * @export
+     */
+    cancel(): void;
+    /**
+     * Closes the `TextToSpeech` instance and releases native engine and Wasm
+     * memory.
+     *
+     * @export
+     */
+    close(): void;
+}
+
+/**
+ * A single streaming PCM audio chunk emitted by `TextToSpeechStream`.
+ */
+export declare interface TextToSpeechAudioChunk {
+    /** Synthesized 32-bit mono float PCM audio samples in `[-1.0, 1.0]`. */
+    samples: Float32Array;
+    /** Audio sample rate in Hz (e.g. 24000). */
+    sampleRateHz: number;
+    /** 0-based index of this chunk within the streaming session. */
+    chunkIndex: number;
+    /** Whether this is the final chunk in the stream. */
+    isFinal: boolean;
+}
+
+/**
+ * Options to configure the MediaPipe TextToSpeech Task on the Web.
+ */
+export declare interface TextToSpeechOptions extends TaskRunnerOptions {
+    /**
+     * Optional default voice ID (e.g. "af_heart"). When unset, the engine's
+     * default voice is used.
+     */
+    defaultVoiceId?: string;
+    /**
+     * Optional default BCP-47 locale (e.g. "en-US"). When unset, inferred from
+     * the selected voice metadata.
+     */
+    defaultLocale?: string;
+    /**
+     * Speaking rate multiplier in `(0.0, 4.0]`. Defaults to `1.0` (normal speed).
+     */
+    speakingRate?: number;
+    /**
+     * Target sequence length bucket for static-shape Kokoro models (`0` = auto).
+     */
+    targetBucket?: number;
+    /**
+     * Optional directory path in the Wasm virtual filesystem for caching unpacked
+     * `.litertlm` assets and compiled weights.
+     */
+    cacheDirectory?: string;
+    /**
+     * Optional byte length of `baseOptions.modelAssetBuffer` when provided as a
+     * `ReadableStreamDefaultReader`, enabling single-pass zero-copy streaming
+     * allocation directly in the Wasm heap.
+     */
+    modelAssetSize?: number;
+}
+
+/**
+ * Result of a unary `TextToSpeech.synthesize()` invocation.
+ */
+export declare interface TextToSpeechResult {
+    /** Synthesized 32-bit mono float PCM audio samples in `[-1.0, 1.0]`. */
+    samples: Float32Array;
+    /** Audio sample rate in Hz (e.g. 24000). */
+    sampleRateHz: number;
+}
+
+/**
+ * Active streaming synthesis session returned by
+ * `TextToSpeech.synthesizeStream()`. Supports both pull-based `readNext()`
+ * and `for await (const chunk of stream)` iteration, as well as incremental
+ * text input via `pushText()` and `finishInput()`.
+ */
+export declare class TextToSpeechStream implements AsyncIterable<TextToSpeechAudioChunk> {
+    private constructor();
+    /**
+     * Pulls the next synthesized PCM audio chunk, or returns `null` when the
+     * stream reaches end-of-stream.
+     *
+     * @export
+     */
+    readNext(): Promise<TextToSpeechAudioChunk | null>;
+    /**
+     * Appends an incremental text fragment (e.g. from an LLM token stream) to an
+     * open stream created with `finishInput = false`.
+     *
+     * @export
+     * @param textDelta The incremental text delta to append.
+     */
+    pushText(textDelta: string): Promise<void>;
+    /**
+     * Marks the end of incremental text input on this stream, flushing any
+     * remaining partial sentence into the synthesis pipeline.
+     *
+     * @export
+     */
+    finishInput(): Promise<void>;
+    /**
+     * Cancels this streaming synthesis session immediately.
+     *
+     * @export
+     */
+    cancel(): void;
+    /**
+     * Releases native stream resources.
+     *
+     * @export
+     */
+    close(): void;
+    [Symbol.asyncIterator](): AsyncIterator<TextToSpeechAudioChunk>;
+}
+
+/**
+ * Copyright 2026 The MediaPipe Authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+/**
+ * A single transcribed segment emitted by AudioTranscriber.
+ */
+export declare interface TranscriptionSegment {
+    /** The transcribed text for this segment. */
+    transcript: string;
+    /** Whether this segment is finalized (as opposed to an interim hypothesis). */
+    isFinal: boolean;
+    /** Start timestamp of the segment in milliseconds. */
+    fromTimestampMs: number;
+    /** End timestamp of the segment in milliseconds. */
+    toTimestampMs: number;
+}
+
+/**
+ * Copyright 2026 The MediaPipe Authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+/**
+ * Voice metadata descriptor returned by `TextToSpeech.listVoices()`.
+ */
+export declare interface VoiceInfo {
+    /** Unique voice identifier (e.g. "af_heart", "bf_emma", "jf_alpha"). */
+    voiceId: string;
+    /** Human-readable display name for the voice. */
+    displayName: string;
+    /** BCP-47 locale tag for the voice (e.g. "en-US", "en-GB", "ja-JP"). */
+    locale: string;
+    /** Output PCM audio sample rate in Hz (e.g. 24000). */
+    sampleRateHz: number;
+    /** Whether the voice embedding is bundled and ready for offline use. */
+    isDownloaded: boolean;
+}
+
 /**
  * Copyright 2022 The MediaPipe Authors.
  *