@bendyline/gezel-inference 0.0.0-stage → 1.0.0

package/dist/providers/audio/managed-speech-model.js

@@ -0,0 +1,86 @@
+import { randomUUID } from 'node:crypto';
+import { mkdir, rename, stat, writeFile } from 'node:fs/promises';
+import { join } from 'node:path';
+import { downloadBytes, verifyFile, } from '@bendyline/gezel/speech-models';
+/** Inventory reads never download or publish an installation. */
+export async function sharedSpeechModelInfo(assets, entry, root) {
+    let installedAt = '';
+    for (const file of entry.files) {
+        const local = join(root, entry.id, ...file.name.split('/'));
+        const source = (await assets.find(entry, file)) ??
+            ((await verifyFile(local, file.sha256, file.size)) ? local : null);
+        if (!source)
+            return null;
+        installedAt ||= (await stat(source)).mtime.toISOString();
+    }
+    return { id: entry.id, name: entry.label, approxSizeBytes: downloadBytes(entry), installedAt };
+}
+/** Acquire this app's references and publish a Gezel-compatible manifest last. */
+export async function ensureSpeechModel(assets, entry, root, download, onProgress) {
+    if (!download && !(await sharedSpeechModelInfo(assets, entry, root)))
+        return false;
+    let completed = 0;
+    const total = downloadBytes(entry);
+    for (const file of entry.files) {
+        if (!(await assets.materialize(entry, file, join(root, entry.id, ...file.name.split('/')), {
+            download,
+            onProgress: (received) => onProgress?.(completed + received, total),
+        })))
+            return false;
+        completed += file.size;
+    }
+    const directory = join(root, entry.id);
+    await mkdir(directory, { recursive: true });
+    const manifest = {
+        id: entry.id,
+        name: entry.label,
+        approxSizeBytes: total,
+        installedAt: new Date().toISOString(),
+        files: entry.files.map((file) => ({
+            role: entry.kind === 'stt' ? 'weights' : file.name,
+            filename: file.name,
+            sha256: file.sha256,
+        })),
+        fileSha256: Object.fromEntries(entry.files.map((file) => [file.name, file.sha256])),
+    };
+    await writeSpeechMetadata(directory, 'manifest.json', JSON.stringify(manifest, null, 2));
+    return true;
+}
+export async function writeSpeechMetadata(directory, name, bytes) {
+    const temporary = join(directory, `${name}.${randomUUID()}.tmp`);
+    await writeFile(temporary, bytes, { flag: 'wx' });
+    await rename(temporary, join(directory, name));
+}
+/** Preserve byte-level progress while the shared installer coordinates another process. */
+export async function* speechPullEvents(id, work) {
+    let latest;
+    let done = false;
+    let error;
+    let wake;
+    const pending = work((bytesWritten, totalBytes) => {
+        latest = { type: 'progress', bytesWritten, totalBytes };
+        wake?.();
+    })
+        .catch((failure) => {
+        error = failure;
+    })
+        .finally(() => {
+        done = true;
+        wake?.();
+    });
+    while (!done || latest) {
+        if (latest) {
+            const event = latest;
+            latest = undefined;
+            yield event;
+        }
+        else
+            await new Promise((resolve) => {
+                wake = resolve;
+            });
+    }
+    await pending;
+    if (error)
+        yield { type: 'error', error: error instanceof Error ? error.message : String(error) };
+    yield { type: 'done', id };
+}

package/dist/providers/audio/types.d.ts

@@ -0,0 +1,202 @@
+/**
+ * AudioProvider — service-side abstractions for speech-to-text (STT)
+ * and text-to-speech (TTS).
+ *
+ * Two distinct provider shapes because the two capabilities have
+ * different surface areas and different engines back them: STT is
+ * implemented by `WhisperCppProvider`, TTS by `KokoroProvider`. They
+ * share the model-pull / installed-list / health envelope conventions
+ * with `ImageProvider` so the Settings UI looks symmetric.
+ *
+ * Streaming methods are defined here but **not implemented** for the
+ * narration MVP — both providers throw "not yet supported" if called.
+ * They live on the interface so an audio-chat feature can fill them in
+ * without retrofitting the type. See the plan's "Architectural gaps to
+ * design around now" section for the rationale.
+ */
+export interface AudioInputBytes {
+    /** Raw audio bytes — typically a WAV file the route resolved from disk. */
+    data: Buffer;
+    mimeType: string;
+}
+export interface TranscribeInput {
+    /**
+     * The audio to transcribe. Either pre-resolved bytes or an absolute
+     * filesystem path (the provider opens it locally — used by sd-server-
+     * style flows where bytes already sit in the artifact tree).
+     */
+    audio: AudioInputBytes | {
+        path: string;
+    };
+    /**
+     * Manifest id of the STT model to use. Provider falls back to the
+     * first installed model when omitted.
+     */
+    model?: string;
+    /**
+     * BCP-47 language hint (e.g. "en", "es"). When omitted, whisper.cpp
+     * runs its language-detection step on the first 30 seconds — slower
+     * cold-start but free-form.
+     */
+    language?: string;
+    /**
+     * Recent transcript supplied as Whisper's initial prompt. This biases the
+     * current audio window toward the established wording and sequence while
+     * the returned text still contains only newly recognized speech.
+     */
+    prompt?: string;
+}
+export interface TranscribeSegment {
+    /** Segment start time in seconds. */
+    start: number;
+    end: number;
+    text: string;
+}
+export interface TranscribeOutput {
+    text: string;
+    /** Per-segment timestamps when the engine reports them. */
+    segments?: TranscribeSegment[];
+    /** Detected (or supplied) language code. */
+    language?: string;
+    /** Wall-clock duration of the transcription itself. */
+    durationMs: number;
+    /**
+     * Manifest id of the model that actually transcribed — the explicit
+     * request, else the provider's resolved fallback. Recorded so consumers
+     * (shadow sidecars, gates) can stamp provenance instead of NULL.
+     */
+    model?: string;
+}
+export interface SynthesizeInput {
+    text: string;
+    /** Stop after the current inference chunk when the caller disconnects or cancels. */
+    signal?: AbortSignal;
+    /** Reports sentence-level work when the active engine exposes it. */
+    onProgress?: (progress: SynthesizeProgress) => void | Promise<void>;
+    /** Delivers independently playable sentence audio as soon as it is available. */
+    onChunk?: (chunk: SynthesizeChunk) => void | Promise<void>;
+    /**
+     * Voice identifier. Kokoro ships 54+ named voices (`af_heart`,
+     * `am_adam`, …); falls back to the provider's default when omitted.
+     */
+    voice?: string;
+    /**
+     * Manifest id of the TTS model. The narration MVP ships Kokoro v1.0
+     * as the only model so this is informational today.
+     */
+    model?: string;
+    /**
+     * Playback speed multiplier. 1.0 = engine default. Kokoro's natural
+     * range is 0.5–2.0; values outside clamp at the engine.
+     */
+    speed?: number;
+}
+export interface SynthesizeProgress {
+    phase: 'loading' | 'synthesizing' | 'encoding';
+    completedCharacters: number;
+    totalCharacters: number;
+    completedChunks: number;
+}
+export interface SynthesizeChunk {
+    index: number;
+    wav: Buffer;
+    sampleRate: number;
+    durationSeconds: number;
+}
+export interface SynthesizeOutput {
+    /**
+     * Generated WAV (RIFF/PCM) bytes. Routes write this under
+     * `artifacts/audio/<sessionId>/…wav` and surface a thumbnail-style
+     * audio-row in chat the same way generated images render today.
+     */
+    wav: Buffer;
+    meta: {
+        voice: string;
+        model: string;
+        sampleRate: number;
+        durationSeconds: number;
+        durationMs: number;
+    };
+}
+export interface InstalledAudioModelInfo {
+    id: string;
+    name: string;
+    approxSizeBytes: number;
+    installedAt: string;
+}
+export type AudioModelPullEvent = {
+    type: 'progress';
+    bytesWritten: number;
+    totalBytes?: number;
+} | {
+    type: 'error';
+    error: string;
+} | {
+    type: 'done';
+    id: string;
+};
+export interface AudioModelFile {
+    /**
+     * Role of the file inside the manifest directory. The provider
+     * decides what each role means — e.g. whisper.cpp uses `weights`
+     * only; Kokoro uses `weights` (ONNX) plus per-voice `voice:<id>`
+     * entries.
+     */
+    role: string;
+    downloadUrl: string;
+    sha256: string;
+    approxSizeBytes: number;
+}
+export interface AudioModelPullSpec {
+    name: string;
+    files: AudioModelFile[];
+}
+export interface AudioEngineHealth {
+    /**
+     * `ok` — engine reachable and at least one model installed.
+     * `unreachable` — configured engine URL is not responding.
+     * `not-configured` — no engine binary or runtime wired up.
+     * `no-model` — engine reachable but the user hasn't pulled a model
+     *   yet. Distinct UI state so we can guide them straight to the
+     *   model picker.
+     */
+    status: 'ok' | 'unreachable' | 'not-configured' | 'no-model';
+    baseUrl?: string;
+    error?: string;
+}
+/** Speech-to-text provider. Implemented today by `WhisperCppProvider`. */
+export interface SpeechToTextProvider {
+    readonly name: string;
+    transcribe(input: TranscribeInput): Promise<TranscribeOutput>;
+    listInstalledModels(): Promise<InstalledAudioModelInfo[]>;
+    pullModel(id: string, spec: AudioModelPullSpec): AsyncIterable<AudioModelPullEvent>;
+    deleteModel(id: string): Promise<void>;
+    health(): Promise<AudioEngineHealth>;
+    /** Best-effort shutdown of any backing engine. Called when the service stops. */
+    shutdown?(): Promise<void>;
+}
+/** Text-to-speech provider. Implemented today by `KokoroProvider`. */
+export interface TextToSpeechProvider {
+    readonly name: string;
+    synthesize(input: SynthesizeInput): Promise<SynthesizeOutput>;
+    listInstalledModels(): Promise<InstalledAudioModelInfo[]>;
+    pullModel(id: string, spec: AudioModelPullSpec): AsyncIterable<AudioModelPullEvent>;
+    deleteModel(id: string): Promise<void>;
+    /** All voices currently available, across installed models. */
+    listVoices(): Promise<AudioVoiceInfo[]>;
+    health(): Promise<AudioEngineHealth>;
+    /** Best-effort shutdown of any backing engine. Called when the service stops. */
+    shutdown?(): Promise<void>;
+}
+export interface AudioVoiceInfo {
+    /** Stable id passed back to `synthesize({ voice })`. */
+    id: string;
+    /** Human-friendly label for the picker. */
+    name: string;
+    /** BCP-47 language tag (e.g. `en-US`). */
+    language: string;
+    /** `'female' | 'male' | 'neutral'` — informational only. */
+    gender?: 'female' | 'male' | 'neutral';
+    /** Manifest id of the model that provides this voice. */
+    modelId: string;
+}

package/dist/providers/audio/types.js

@@ -0,0 +1,17 @@
+/**
+ * AudioProvider — service-side abstractions for speech-to-text (STT)
+ * and text-to-speech (TTS).
+ *
+ * Two distinct provider shapes because the two capabilities have
+ * different surface areas and different engines back them: STT is
+ * implemented by `WhisperCppProvider`, TTS by `KokoroProvider`. They
+ * share the model-pull / installed-list / health envelope conventions
+ * with `ImageProvider` so the Settings UI looks symmetric.
+ *
+ * Streaming methods are defined here but **not implemented** for the
+ * narration MVP — both providers throw "not yet supported" if called.
+ * They live on the interface so an audio-chat feature can fill them in
+ * without retrofitting the type. See the plan's "Architectural gaps to
+ * design around now" section for the rationale.
+ */
+export {};

package/dist/providers/audio/whisper-cpp.d.ts

@@ -0,0 +1,99 @@
+/**
+ * WhisperCppProvider — talks to a local `whisper-server` binary over
+ * HTTP. The server speaks `POST /inference` with a multipart form
+ * upload (`file=@audio.wav`) plus optional `language`, `prompt`,
+ * `temperature`, `response_format`. We always request `json` so we
+ * get `{ text, segments, language }` back in one parse.
+ *
+ * Lifecycle of the binary itself is the supervisor's concern — this
+ * provider just calls `ensureRunning()` before each transcription and
+ * marks activity afterwards, identical to how
+ * `StableDiffusionCppProvider` delegates to `NativeEngineSupervisor`.
+ *
+ * Model management mirrors sd-cpp: weights live under
+ * `<modelsRoot>/<id>/<filename>` alongside a `manifest.json`. The
+ * narration MVP hard-codes a small catalog of Whisper GGML
+ * conversions hosted on Hugging Face under `ggerganov/whisper.cpp`
+ * (see `WHISPER_MODEL_CATALOG`); a future revision can route through
+ * the proper catalog system instead.
+ */
+import { type SpeechAssetOptions } from '@bendyline/gezel/speech-models';
+import { type ModelStorageRoots } from '../../models/storage-roots.js';
+import type { NativeEngineSupervisor } from '../native/supervisor.js';
+import type { AudioEngineHealth, AudioModelPullEvent, AudioModelPullSpec, InstalledAudioModelInfo, SpeechToTextProvider, TranscribeInput, TranscribeOutput } from './types.js';
+export interface WhisperCppProviderOptions {
+    baseUrl: string;
+    modelsRoot: string;
+    storageRoots?: ModelStorageRoots;
+    assets?: SpeechAssetOptions;
+    /** Per-request timeout in ms. Defaults to 5 minutes — long enough for
+     *  small models on a 30-minute audio file on weak hardware. */
+    timeoutMs?: number;
+    fetchImpl?: typeof fetch;
+    supervisor?: NativeEngineSupervisor;
+    configured?: boolean;
+}
+/**
+ * Whisper GGML conversions on Hugging Face. Sizes are approximate —
+ * the actual bytes are verified by sha256 at pull time. SHA values
+ * below are the published Hugging Face values; bumping a model means
+ * editing both the URL/sha together.
+ *
+ * Why hard-code instead of going through `@bendyline/gezel-catalog`:
+ * the audio MVP's three models don't justify the catalog manifest
+ * round-trip (registering a new `audio-stt-model` kind, plumbing it
+ * through `CatalogService`, etc.). When TTS / STT grow more variants,
+ * promote this list into the catalog same way image-models are.
+ */
+export declare const WHISPER_MODEL_CATALOG: ReadonlyArray<{
+    id: string;
+    name: string;
+    description: string;
+    approxSizeBytes: number;
+    downloadUrl: string;
+    sha256: string;
+    filename: string;
+}>;
+export declare class WhisperCppProvider implements SpeechToTextProvider {
+    readonly name = "whisper-cpp";
+    private readonly baseUrl;
+    private readonly modelsRoot;
+    private readonly assets?;
+    private readonly storageRoots;
+    private readonly timeoutMs;
+    private readonly fetchImpl;
+    private readonly supervisor?;
+    private readonly configured;
+    constructor(opts: WhisperCppProviderOptions);
+    transcribe(input: TranscribeInput): Promise<TranscribeOutput>;
+    private transcribeInner;
+    listInstalledModels(): Promise<InstalledAudioModelInfo[]>;
+    pullModel(id: string, spec: AudioModelPullSpec): AsyncIterable<AudioModelPullEvent>;
+    deleteModel(id: string): Promise<void>;
+    health(): Promise<AudioEngineHealth>;
+    shutdown(): Promise<void>;
+}
+/** whisper.cpp emits this sentinel for valid audio containing no speech. */
+export declare function normalizeWhisperTranscript(value: string): string;
+/** MediaRecorder MIME values often carry codec parameters; filenames cannot. */
+export declare function inlineAudioFilename(mimeType: string): string;
+/**
+ * Stream a URL to disk with sha256 verification, yielding cumulative
+ * progress events. Shared with KokoroProvider via a sibling import —
+ * exported so we don't duplicate the logic.
+ */
+export declare function downloadWithSha256(fetchImpl: typeof fetch, opts: {
+    url: string;
+    destPath: string;
+    expectedSha256: string;
+    approxSizeBytes: number;
+    writtenSoFar: number;
+    totalAllBytes: number;
+}): AsyncGenerator<AudioModelPullEvent, {
+    kind: 'ok';
+    writtenAll: number;
+    actualSize: number;
+} | {
+    kind: 'error';
+    error: string;
+}>;