@onjmin/koe 1.0.7 → 1.0.9

package/dist/index.d.ts

@@ -401,15 +401,38 @@ interface WorldlineLoadOptions {
  * function you give it once per frame.
  */
 type CurveInput = number | ((tMs: number, totalMs: number) => number);
+/**
+ * One sample placed on a phrase timeline (mirrors worldline's
+ * `PhraseSynth::AddRequest` + the UTAU `SynthRequest` fields that matter for
+ * speech). All times are ms.
+ */
 interface PhraseUnit {
     pcm: Float64Array;
+    /** Where the (offset-trimmed) sample head is placed on the phrase timeline. */
     posMs: number;
+    /** Leading part of the resampled sample to skip before `posMs`. */
     skipMs: number;
+    /** How much of the resampled sample is used from `posMs`. */
     lengthMs: number;
     fadeInMs: number;
     fadeOutMs: number;
+    /** Fixed (unstretched) consonant region of the sample. */
     consonantMs: number;
+    /** Trim from the sample tail; defaults to two WORLD frames. */
     cutMs?: number;
+    /**
+     * UTAU "required length": the resampler stretches the sample to this total
+     * length (consonant kept, vowel stretched). Defaults to `lengthMs`. UtauTTS
+     * rounds it up to 50 ms steps so the vowel loop has slack for the release.
+     */
+    requiredLengthMs?: number;
+    /** UTAU volume, 100 = unity. */
+    volume?: number;
+    /**
+     * MIDI note the resampler targets before the F0 curve is applied. Pick the
+     * note closest to the sample's own pitch to keep the formant shift minimal.
+     */
+    tone?: number;
 }
 interface RenderPhraseParams {
     units: PhraseUnit[];
@@ -574,7 +597,7 @@ interface Frames {
     peakDb: number;
 }
 /** Extract every feature the mora estimator needs from one mono signal. */
-declare function analyze(wav: WavData): Frames;
+declare function analyze$1(wav: WavData): Frames;
 /** Convenience wrapper: decode a WAV buffer and analyse it. */
 declare function analyzeWav(buf: ArrayBuffer): Frames;
 
@@ -830,4 +853,440 @@ declare function formatOto(entries: readonly OtoEntry[]): string;
 /** Render entries as Shift-JIS oto.ini bytes, ready to write to disk. */
 declare function encodeOto(entries: readonly OtoEntry[]): Uint8Array;
 
-export { type ConsonantClass, type FileResult, type Frames, type FrqData, type GenerateOptions, type GenerateResult, type Grid, KoeEngine, type KoeEngineOptions, MIN_WORLDLINE_SAMPLES, type Manifest, type MoraPosition, type NoteEvent, type OtoEntry, type PackInput, type PackOutput, type PhonemeEntry, type PlayOptions, type RenderNoteParams, type SkippedFile, type Syllable, type TrimmedPhoneme, VoiceBank, WORLDLINE_SAMPLE_RATE, type WavData, type WavInput, Worldline, type WorldlineLoadOptions, type ZipFile, analyze, analyzeWav, detectF0, detectGrid, encodeOto, encodeShiftJis, estimateSequence, estimateSolo, estimateVowelJoin, formatOto, frqAverageF0InRange, frqFileName, generateOto, generateOtoForFile, leadInFromEntry, locateMora, normalizePcm, noteNameToHz, otoRegion, pack, packKoe, parseFrq, parseFrqAverageF0, parseKoeHeader, parseOto, parseWav, pcmBase, pitchFromAliasSuffix, readWavPcm48k, resample, samplesToMs, splitKana, suffixFromFolderName, summarise, toHiragana, toInt16, toMono, transcribe, trimToOto, unzipToFileMap, zipFiles };
+/**
+ * Asset loading for the TTS stack: the Go UtauTTS wasm (~10 MB), the
+ * jpreprocess wasm (~2 MB) and its naist-jdic dictionary (~29 MB gzipped,
+ * ~80 MB inflated) and the TCN prosody model (~1 MB).
+ *
+ * Everything goes through {@link fetchAsset}, which streams the download with
+ * progress callbacks and stores the raw response in the Cache API so the second
+ * visit skips the network entirely (the browser HTTP cache alone is not
+ * reliable for files this size).
+ */
+interface AssetProgress {
+    url: string;
+    /** Bytes received so far (compressed bytes for gzipped files). */
+    loaded: number;
+    /** Total bytes when the server sent Content-Length, otherwise 0. */
+    total: number;
+    /** True when served from the Cache API (no network). */
+    fromCache: boolean;
+}
+interface AssetFetchOptions {
+    /**
+     * Cache API bucket name. `null` disables caching. Defaults to
+     * `"koe-tts-assets-v1"`.
+     */
+    cacheName?: string | null;
+    onProgress?: (progress: AssetProgress) => void;
+    signal?: AbortSignal;
+    /**
+     * Check the cached copy against the server with a HEAD request (ETag,
+     * Last-Modified or Content-Length) so a redeployed asset is refetched.
+     * Offline or on error the cached copy is used. Default true.
+     */
+    revalidate?: boolean;
+}
+/**
+ * Fetch a static asset with download progress, backed by the Cache API.
+ *
+ * The returned Response has a fully buffered body, so it can be handed to
+ * `WebAssembly.instantiateStreaming` / wasm-bindgen `init()` as-is.
+ */
+declare function fetchAsset(url: string, options?: AssetFetchOptions): Promise<Response>;
+/** {@link fetchAsset} and return the body bytes. */
+declare function fetchAssetBytes(url: string, options?: AssetFetchOptions): Promise<Uint8Array>;
+/** {@link fetchAsset} and return the body as text. */
+declare function fetchAssetText(url: string, options?: AssetFetchOptions): Promise<string>;
+/** The naist-jdic files produced by jpreprocess's dictionary build, in `init_dictionary` order. */
+declare const NAIST_JDIC_FILES: readonly ["metadata.json", "char_def.bin", "matrix.mtx", "dict.da", "dict.vals", "unk.bin", "dict.wordsidx", "dict.words"];
+type NaistJdicFile = (typeof NAIST_JDIC_FILES)[number];
+type NaistJdicData = Record<NaistJdicFile, Uint8Array>;
+/** The subset of the wasm-bindgen module surface used by koe. */
+interface JpreprocessModule {
+    init_dictionary(metadata: Uint8Array, charDef: Uint8Array, matrix: Uint8Array, dictDa: Uint8Array, dictVals: Uint8Array, unk: Uint8Array, wordsIdx: Uint8Array, words: Uint8Array): void;
+    analyze_text(text: string): string;
+    is_ready(): boolean;
+    /** Load an HTS voice (.htsvoice bytes) for `analyze_prosody`. */
+    init_voice?(htsvoice: Uint8Array): void;
+    is_voice_ready?(): boolean;
+    /** HTS phoneme durations + F0 for the text (JSON, see `HtsProsodyFrames`). */
+    analyze_prosody?(text: string, speed: number): string;
+}
+interface LoadNaistJdicOptions extends AssetFetchOptions {
+    /** Files are `<name>.gz` and inflated in the browser. Default true. */
+    compressed?: boolean;
+}
+/**
+ * Download (or read from the Cache API) the naist-jdic dictionary files for
+ * jpreprocess. `baseUrl` is the directory holding `dict.da.gz` etc.
+ *
+ * Progress is reported per file via `onProgress`; sum `loaded`/`total` across
+ * the {@link NAIST_JDIC_FILES} URLs for an aggregate bar.
+ */
+declare function loadNaistJdic(baseUrl: string, options?: LoadNaistJdicOptions): Promise<NaistJdicData>;
+/** Hand the loaded dictionary to the jpreprocess wasm module (once per page). */
+declare function initJpreprocessDictionary(module: JpreprocessModule, data: NaistJdicData): void;
+
+interface NjdNode {
+    string: string;
+    pos: string;
+    pos_group1: string;
+    pron: string;
+    read: string;
+    acc: number;
+    mora_size: number;
+    chain_flag: number;
+}
+interface FeatureFrame {
+    mora: string;
+    pause: boolean;
+    accent_phrase_position?: number;
+    accent_phrase_length?: number;
+    accent_nucleus?: number;
+    accent_high?: boolean;
+    accent_phrase_start?: boolean;
+    accent_phrase_end?: boolean;
+    word_start?: boolean;
+    word_end?: boolean;
+    pos?: string;
+    pos_group1?: string;
+}
+declare function analyze(nodes: NjdNode[]): {
+    reading: string;
+    features: FeatureFrame[];
+};
+declare function sparse_features(token: FeatureFrame): Record<string, number>;
+
+/**
+ * Prosody transplant from an HTS voice (Open JTalk's statistical model, run by
+ * jbonsai inside the jpreprocess wasm): HTS decides *how long* each mora is and
+ * *what pitch contour* the sentence has; the UTAU voice bank only supplies the
+ * timbre. This is the same division of labour as Cantari (VOICEVOX prosody +
+ * worldline) but with a 1-2 MB model that runs in the browser.
+ */
+/** One phoneme of the HTS timeline (`analyze_prosody` output). */
+interface HtsPhoneme {
+    phone: string;
+    start_ms: number;
+    duration_ms: number;
+}
+/** Raw `analyze_prosody` result. */
+interface HtsProsodyFrames {
+    frame_ms: number;
+    sample_rate: number;
+    phonemes: HtsPhoneme[];
+    /** F0 in Hz per frame; 0 = unvoiced. */
+    f0_hz: number[];
+}
+/** HTS prosody aligned to the UtauTTS mora sequence, ready for `plan()`. */
+interface HtsProsody {
+    /** Per feature-frame (mora or pause) duration in ms, same order as the features. */
+    moraDurationsMs: number[];
+    /** Cents relative to the speaker median, on the plan time axis (0 = first mora onset). */
+    pitchCurve: {
+        frame_ms: number;
+        cents: number[];
+    };
+    /** Median voiced F0 of the HTS utterance in Hz. */
+    medianHz: number;
+    /** Total planned duration in ms (sum of `moraDurationsMs`). */
+    durationMs: number;
+    /**
+     * Per feature-frame flag: HTS predicted the mora nucleus as a devoiced vowel
+     * (Open JTalk's upper-case `I`/`U`, e.g. the "su" in "desu"). Pauses are false.
+     */
+    devoiced: boolean[];
+    /**
+     * Optional per feature-frame linear gain (1 = unity) applied to the units of
+     * that mora on top of UtauTTS's own volume; filled by `shapeProsody`.
+     */
+    moraGains?: number[];
+}
+interface AlignHtsOptions {
+    /** Multiplier on the cents curve. 1 = HTS as-is. Default 1. */
+    intonationStrength?: number;
+    /** Output frame period for the pitch curve in ms. Default 10 (worldline's frame). */
+    frameMs?: number;
+    /** Pause length used when the features have a pause HTS did not produce. Default 150. */
+    fallbackPauseMs?: number;
+    /**
+     * Soft limit for the cents curve: excursions beyond `kneeCents` are halved,
+     * then hard-clamped at `maxCents`. HTS voices end phrases with a creaky
+     * drop of close to an octave, which WORLD resynthesis of a singing sample
+     * cannot follow cleanly. Defaults: knee 400, max 700.
+     */
+    kneeCents?: number;
+    maxCents?: number;
+}
+/**
+ * Align HTS phoneme timing and F0 to the mora/pause sequence in `features`
+ * (from `openjtalkAnalyze`). Returns null when the two mora sequences cannot
+ * be matched, in which case the caller should fall back to the TCN contour.
+ */
+declare function alignHtsProsody(frames: HtsProsodyFrames, features: FeatureFrame[], options?: AlignHtsOptions): HtsProsody | null;
+
+/**
+ * Rule-based residuals on top of the HTS prosody (a cut-down Fujisaki-style
+ * layer): the statistical model gives the accent shape, these rules add what
+ * the HTS voices were not trained for and what a concatenative voice bank
+ * lacks on its own.
+ *
+ *  - question rise: a sentence ending in 「？」 gets at least a minimum rise
+ *    on its last mora; Open JTalk's interrogative label flag already makes
+ *    tohoku-f01 rise, so this only tops up voices/sentences that do not
+ *  - energy envelope: loudness follows pitch (accented morae louder, phrase
+ *    ends softer), which a plain unit concatenation renders flat
+ *  - devoiced vowels: morae HTS predicted as devoiced (「です」「ます」…) are
+ *    attenuated so the voiced sample does not shout a vowel the speaker
+ *    would whisper
+ *
+ * Zero assets, negligible compute; everything is mora-level arithmetic.
+ */
+interface ShapeProsodyOptions {
+    /** Treat the utterance as a question (see {@link isQuestion}). Default false. */
+    question?: boolean;
+    /**
+     * Minimum rise across the final mora of a question, in cents. Open JTalk
+     * labels carry an interrogative flag and HTS voices trained with it (e.g.
+     * tohoku-f01) already rise; the rule only tops up whatever is missing, so
+     * it is a safety net for voices/sentences where the rise does not appear.
+     * Default 350.
+     */
+    questionRiseCents?: number;
+    /**
+     * Loudness coupling: dB per 100 cents of pitch relative to the speaker
+     * median. 0 disables the energy envelope. Default 0.5 (≈ +2 dB at +400 cent).
+     */
+    energyDbPerSemitone?: number;
+    /** Linear gain for morae with a devoiced nucleus. Default 0.5 (−6 dB). */
+    devoicedGain?: number;
+    /** Clamp for the per-mora gain. Defaults 0.35 … 1.8. */
+    minGain?: number;
+    maxGain?: number;
+}
+/** True when the text ends with a question mark (「？」 or "?"), ignoring closing quotes. */
+declare function isQuestion(text: string): boolean;
+/**
+ * Apply the residual rules to an aligned HTS prosody. Returns a new object;
+ * `prosody` is not modified. Pass the result to `plan({ prosody })`: the pitch
+ * curve carries the question rise, `moraGains` the energy envelope.
+ */
+declare function shapeProsody(prosody: HtsProsody, features: FeatureFrame[], options?: ShapeProsodyOptions): HtsProsody;
+
+/** One unit of the UtauTTS synthesis plan (`internal/plan.Unit`). */
+interface UtauTTSUnit {
+    position: number;
+    role: string;
+    mora: string;
+    alias: string;
+    note_start_ms: number;
+    duration_ms: number;
+    offset_ms: number;
+    consonant_ms: number;
+    cutoff_ms: number;
+    preutterance_ms: number;
+    overlap_ms: number;
+    silent?: boolean;
+    pitch_factor: number;
+    energy_factor: number;
+    effective_preutterance_ms: number;
+    effective_consonant_ms: number;
+    effective_overlap_ms: number;
+    source_f0_hz?: number;
+    target_f0_hz?: number;
+    [extra: string]: unknown;
+}
+/** Placement of one unit on the worldline phrase timeline (`render.WorldlineTimelineUnit`). */
+interface UtauTTSTimelineUnit {
+    index: number;
+    position: number;
+    role: string;
+    mora: string;
+    alias: string;
+    note_start_ms: number;
+    duration_ms: number;
+    position_ms: number;
+    skip_ms: number;
+    length_ms: number;
+    fade_in_ms: number;
+    fade_out_ms: number;
+    offset_ms: number;
+    required_length_ms: number;
+    consonant_ms: number;
+    cutoff_ms: number;
+    tone: number;
+    consonant_velocity: number;
+    volume: number;
+    energy_factor: number;
+    source_f0_hz: number;
+    target_f0_hz: number;
+    envelope?: {
+        x_ms: number;
+        y: number;
+    }[];
+}
+/** Whole-phrase placement + F0 curve (`render.WorldlineTimeline`). */
+interface UtauTTSTimeline {
+    frame_ms: number;
+    leading_ms: number;
+    /** Timeline length in ms (plan duration + release + leading margin). */
+    duration_ms: number;
+    reference_hz: number;
+    /** Target F0 in Hz per `frame_ms` frame, starting at timeline 0. */
+    f0_curve: number[];
+    units: UtauTTSTimelineUnit[];
+}
+interface UtauTTSPlan {
+    reading: string;
+    language: string;
+    morae: {
+        Text: string;
+        Consonant: string;
+        Vowel: string;
+        Pause: boolean;
+    }[];
+    plan: {
+        duration_ms: number;
+        single_cv?: boolean;
+        leading_margin_ms?: number;
+        units: UtauTTSUnit[];
+        [extra: string]: unknown;
+    };
+    /** Frame pitch curve in cents relative to each unit's own pitch (plan time base). */
+    pitch_curve?: {
+        frame_ms: number;
+        cents: number[];
+    };
+    mora_timings: {
+        StartMS: number;
+        DurationMS: number;
+    }[];
+    timeline: UtauTTSTimeline;
+    /** @deprecated use `plan.duration_ms` */
+    duration_ms: number;
+    /** @deprecated use `plan.units` */
+    units: UtauTTSUnit[];
+    /** @deprecated use `pitch_curve` */
+    pitch_cents?: number[];
+    /** @deprecated use `pitch_curve` */
+    pitch_frame_ms?: number;
+}
+/** Synthesis parameters, same meaning and defaults as `utautts-cli` / the UtauTTS GUI. */
+interface UtauTTSOptions {
+    /** Voicebank tone for prefix.map lookups. Default "C4". */
+    tone?: string;
+    /** Base mora length in ms (0 = UtauTTS default 140). */
+    moraDurationMs?: number;
+    /** Pause length for punctuation in ms (0 = UtauTTS default 180). */
+    pauseDurationMs?: number;
+    /** Release envelope in ms. Default 20. */
+    releaseMs?: number;
+    /** Cap on the leading preutterance margin before the first mora (0 = no cap). */
+    leadingPreutteranceMs?: number;
+    /** Apply the TCN frame pitch contour and source pitch stabilisation. Default true. */
+    applyPitch?: boolean;
+    /** Contour strength 0..4. Default 1. */
+    intonationStrength?: number;
+    /** UtauTTS experimental speech timing (voicebank calibration). Default false. */
+    speechTiming?: boolean;
+    wordBoundaryEnvelope?: boolean;
+    /**
+     * External prosody (mora durations + cents curve) from `alignHtsProsody`.
+     * When set, UtauTTS uses these instead of its own durations and TCN contour;
+     * `intonationStrength` is applied by `alignHtsProsody`, not here.
+     */
+    prosody?: HtsProsody;
+}
+/** One rendered piece of audio. Sum overlapping chunks: seams are equal-power crossfades. */
+interface UtauTTSChunk {
+    /** Float32 PCM at 48 kHz. */
+    pcm: Float32Array;
+    /** Where the chunk starts on the timeline (ms from timeline 0). */
+    startMs: number;
+    index: number;
+    /** Timeline units rendered into this chunk (context units for seams excluded). */
+    units: UtauTTSTimelineUnit[];
+}
+interface UtauTTSRenderOptions {
+    /** Units in the first chunk (small → audio starts sooner). Default 3. */
+    firstChunkUnits?: number;
+    /** Units per later chunk. Default 6. */
+    chunkUnits?: number;
+    /** Crossfade length at a mid-phrase seam in ms. Default 20. */
+    seamCrossfadeMs?: number;
+    signal?: AbortSignal;
+    gender?: number;
+    tension?: number;
+    breathiness?: number;
+    voicing?: number;
+}
+/**
+ * Build the kana reading UtauTTS parses from the jpreprocess token features,
+ * so the mora sequence and the feature frames line up 1:1 by construction.
+ */
+declare function readingFromFeatures(features: FeatureFrame[]): string;
+/**
+ * Text-to-speech with UtauTTS's planner (Go wasm) and koe's worldline renderer.
+ *
+ * Pipeline per utterance:
+ *  1. jpreprocess (caller) → NJD nodes → {@link FeatureFrame}s via `openjtalkAnalyze`
+ *  2. {@link plan}: UtauTTS resolves units (Viterbi), builds the timing plan,
+ *     predicts the TCN pitch contour and lays the units on the worldline timeline
+ *     exactly as its native worldline bridge would
+ *  3. {@link renderChunks}: worldline renders the timeline in small chunks so
+ *     playback can start after the first few morae
+ *
+ * Load order: `initializeWasm()` once per page, `setModel()` once per model,
+ * `setBank()` once per voice bank.
+ */
+declare class UtauTTSAdapter {
+    private worldline;
+    private static modelId;
+    private bankAliases;
+    private currentBank;
+    private pcmCache;
+    constructor(worldline: Worldline);
+    /**
+     * Load the UtauTTS Go wasm. `wasm_exec.js` must already be on the page.
+     * Pass `fetch` (e.g. koe's `fetchAsset`) to stream from the Cache API.
+     */
+    static initializeWasm(wasmUrl?: string, options?: {
+        fetch?: (url: string) => Promise<Response>;
+    }): Promise<void>;
+    static get ready(): boolean;
+    /** Parse and cache a prosody model (e.g. `frame-intonation-v8.json`) inside the wasm. */
+    static setModel(modelJSON: string): string | null;
+    static get currentModelId(): string | null;
+    private static assertReady;
+    /**
+     * Register a koe voice bank with the planner: the manifest becomes a virtual
+     * oto.ini (koe PCM is pre-trimmed, so offset is 0) plus each sample's
+     * recorded pitch. Called automatically by {@link plan}; cheap when unchanged.
+     */
+    setBank(bank: VoiceBank): void;
+    /**
+     * Plan an utterance: unit selection, timing, pitch contour and worldline
+     * placement. `features` are the mora-level frames from `openjtalkAnalyze`
+     * (pauses included); the kana reading is derived from them.
+     */
+    plan(bank: VoiceBank, text: string, features: FeatureFrame[], options?: UtauTTSOptions): UtauTTSPlan;
+    private getPcm;
+    /**
+     * Render a plan chunk by chunk. Each chunk is independent audio positioned
+     * at `startMs`; schedule them as they arrive (see the demo) or sum them.
+     *
+     * Chunk breaks fall on pauses when possible. Inside a phrase a break renders
+     * one neighbouring unit of context on each side so the unit crossfade stays
+     * WORLD's spectral one, then the two renders are joined with a short
+     * equal-power crossfade in the following vowel.
+     */
+    renderChunks(bank: VoiceBank, plan: UtauTTSPlan, options?: UtauTTSRenderOptions): AsyncGenerator<UtauTTSChunk>;
+    /**
+     * Plan + render an utterance into one buffer (Float32, 48 kHz) covering the
+     * whole timeline. Use {@link plan} + {@link renderChunks} for streaming.
+     */
+    synthesizeText(bank: VoiceBank, text: string, features: FeatureFrame[], options?: UtauTTSOptions & UtauTTSRenderOptions): Promise<Float32Array | null>;
+}
+
+export { type AlignHtsOptions, type AssetFetchOptions, type AssetProgress, type ConsonantClass, type FeatureFrame, type FileResult, type Frames, type FrqData, type GenerateOptions, type GenerateResult, type Grid, type HtsPhoneme, type HtsProsody, type HtsProsodyFrames, type JpreprocessModule, KoeEngine, type KoeEngineOptions, MIN_WORLDLINE_SAMPLES, type Manifest, type MoraPosition, NAIST_JDIC_FILES, type NaistJdicData, type NjdNode, type NoteEvent, type OtoEntry, type PackInput, type PackOutput, type PhonemeEntry, type PlayOptions, type RenderNoteParams, type ShapeProsodyOptions, type SkippedFile, type Syllable, type TrimmedPhoneme, UtauTTSAdapter, type UtauTTSChunk, type UtauTTSOptions, type UtauTTSPlan, type UtauTTSRenderOptions, type UtauTTSTimeline, type UtauTTSTimelineUnit, type UtauTTSUnit, VoiceBank, WORLDLINE_SAMPLE_RATE, type WavData, type WavInput, Worldline, type WorldlineLoadOptions, type ZipFile, alignHtsProsody, analyze$1 as analyze, analyzeWav, detectF0, detectGrid, encodeOto, encodeShiftJis, estimateSequence, estimateSolo, estimateVowelJoin, fetchAsset, fetchAssetBytes, fetchAssetText, formatOto, frqAverageF0InRange, frqFileName, generateOto, generateOtoForFile, initJpreprocessDictionary, isQuestion, leadInFromEntry, loadNaistJdic, locateMora, normalizePcm, noteNameToHz, analyze as openjtalkAnalyze, otoRegion, pack, packKoe, parseFrq, parseFrqAverageF0, parseKoeHeader, parseOto, parseWav, pcmBase, pitchFromAliasSuffix, readWavPcm48k, readingFromFeatures, resample, samplesToMs, shapeProsody, sparse_features, splitKana, suffixFromFolderName, summarise, toHiragana, toInt16, toMono, transcribe, trimToOto, unzipToFileMap, zipFiles };