@elizaos/plugin-local-inference 2.0.0-beta.1 → 2.0.3-beta.2

package/src/services/voice/kokoro/runtime-selection.d.ts

@@ -0,0 +1,92 @@
+/**
+ * Voice backend selection — picks between OmniVoice and Kokoro at engine
+ * arm time.
+ *
+ * The scheduler is backend-agnostic — both implementations satisfy
+ * `OmniVoiceBackend + StreamingTtsBackend`. This module isolates the
+ * decision logic so the engine layer (or a test) can ask "which backend
+ * should I instantiate?" and get a single, auditable answer.
+ *
+ * Decision modes:
+ *
+ *   - `omnivoice` — force OmniVoice. Used when the caller needs voice
+ *      cloning (Kokoro v1.0 has fixed voice packs, no per-user cloning).
+ *   - `kokoro`    — force Kokoro. Used when the caller cares about
+ *      first-audio latency over voice fidelity (Kokoro ≈ 97ms CPU TTFB,
+ *      OmniVoice ≈ 200ms on the fused build).
+ *   - `auto`      — apply the documented heuristic below.
+ *
+ * Mobile precedence: when `mobile === true` the selector returns Kokoro
+ * unconditionally (it is smaller + faster and the only backend shipped in
+ * mobile-class bundles). This short-circuits every mode and heuristic below.
+ *
+ * `auto` heuristic (deterministic, no model probes — those go through the
+ * autotune layer in `voice/scheduler.ts`):
+ *
+ *   1. If `requireVoiceCloning === true` → OmniVoice.
+ *   2. If `targetTtfaMs` is set and < 200 → Kokoro.
+ *   3. If a Kokoro RTF measurement is available and OmniVoice RTF is not,
+ *      or Kokoro RTF beats OmniVoice by ≥ 10% → Kokoro.
+ *   4. Else → first entry of `tierVoiceBackends` if provided (the
+ *      catalog's declared per-tier default). Falls back to OmniVoice
+ *      only when no tier policy is supplied.
+ *
+ * Tier policy comes from `ELIZA_1_VOICE_BACKENDS` in
+ * `packages/shared/src/local-inference/catalog.ts`. Callers should pass
+ * the active bundle's `voiceBackends` array so the selection is
+ * data-driven (small tiers → Kokoro default; large tiers → OmniVoice).
+ *
+ * The decision returns a tagged discriminated union, not a backend
+ * instance, so the engine layer can instantiate the chosen backend with
+ * its own dependencies (FFI handle / Kokoro runtime / etc.). This keeps
+ * the selection logic unit-testable without dragging the ORT or FFI
+ * surfaces into the test graph.
+ */
+export type VoiceBackendChoice = "omnivoice" | "kokoro";
+export type VoiceBackendMode = VoiceBackendChoice | "auto";
+export interface VoiceBackendInputs {
+    /** Caller-set mode. Defaults to `auto`. */
+    mode?: VoiceBackendMode;
+    /** Time-to-first-audio target (ms). Lower → prefer Kokoro. */
+    targetTtfaMs?: number;
+    /** Whether the caller needs per-user voice cloning. */
+    requireVoiceCloning?: boolean;
+    /** Latest measured RTF for Kokoro on this device (audio_seconds / wall_seconds). */
+    kokoroRtf?: number | null;
+    /** Latest measured RTF for OmniVoice on this device. */
+    omnivoiceRtf?: number | null;
+    /** Whether Kokoro model artifacts are present on disk. The selector
+     *  never returns Kokoro when this is `false` — no silent downgrade. */
+    kokoroAvailable: boolean;
+    /** Whether the OmniVoice FFI library is present on disk. */
+    omnivoiceAvailable: boolean;
+    /**
+     * True on mobile (iOS / Android) builds. Mobile uses Kokoro exclusively —
+     * it is smaller and faster than OmniVoice and is the only TTS backend
+     * shipped in mobile-class bundles. When set, the selector returns Kokoro
+     * unconditionally (ignoring `mode`, RTF, and TTFA heuristics) and throws
+     * if Kokoro artifacts are missing rather than falling back to OmniVoice.
+     */
+    mobile?: boolean;
+    /**
+     * The active bundle's per-tier voice backend policy, as declared in
+     * `ELIZA_1_VOICE_BACKENDS`. First entry is the catalog default for
+     * the tier; later entries are also bundled. The selector reads this
+     * to make the `auto` default tier-aware rather than hard-coding a
+     * single backend.
+     *
+     * Omit when called outside the Eliza-1 catalog context (e.g. ad-hoc
+     * smoke benches) — the selector falls back to OmniVoice as the
+     * historical default in that case.
+     */
+    tierVoiceBackends?: ReadonlyArray<VoiceBackendChoice>;
+}
+export interface VoiceBackendDecision {
+    backend: VoiceBackendChoice;
+    /** One-line reason — surfaced to telemetry. */
+    reason: string;
+}
+/** Resolve the env override (`ELIZA_TTS_BACKEND=kokoro|omnivoice|auto`). */
+export declare function readVoiceBackendModeFromEnv(env?: NodeJS.ProcessEnv): VoiceBackendMode | undefined;
+export declare function selectVoiceBackend(inputs: VoiceBackendInputs): VoiceBackendDecision;
+//# sourceMappingURL=runtime-selection.d.ts.map

package/src/services/voice/kokoro/runtime-selection.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"runtime-selection.d.ts","sourceRoot":"","sources":["runtime-selection.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA2CG;AAEH,MAAM,MAAM,kBAAkB,GAAG,WAAW,GAAG,QAAQ,CAAC;AAExD,MAAM,MAAM,gBAAgB,GAAG,kBAAkB,GAAG,MAAM,CAAC;AAE3D,MAAM,WAAW,kBAAkB;IAClC,2CAA2C;IAC3C,IAAI,CAAC,EAAE,gBAAgB,CAAC;IACxB,8DAA8D;IAC9D,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,uDAAuD;IACvD,mBAAmB,CAAC,EAAE,OAAO,CAAC;IAC9B,oFAAoF;IACpF,SAAS,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAC1B,wDAAwD;IACxD,YAAY,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAC7B;2EACuE;IACvE,eAAe,EAAE,OAAO,CAAC;IACzB,4DAA4D;IAC5D,kBAAkB,EAAE,OAAO,CAAC;IAC5B;;;;;;OAMG;IACH,MAAM,CAAC,EAAE,OAAO,CAAC;IACjB;;;;;;;;;;OAUG;IACH,iBAAiB,CAAC,EAAE,aAAa,CAAC,kBAAkB,CAAC,CAAC;CACtD;AAED,MAAM,WAAW,oBAAoB;IACpC,OAAO,EAAE,kBAAkB,CAAC;IAC5B,+CAA+C;IAC/C,MAAM,EAAE,MAAM,CAAC;CACf;AAKD,4EAA4E;AAC5E,wBAAgB,2BAA2B,CAC1C,GAAG,GAAE,MAAM,CAAC,UAAwB,GAClC,gBAAgB,GAAG,SAAS,CAO9B;AAED,wBAAgB,kBAAkB,CACjC,MAAM,EAAE,kBAAkB,GACxB,oBAAoB,CA8HtB"}

package/src/services/voice/kokoro/runtime-selection.ts

@@ -0,0 +1,237 @@
+/**
+ * Voice backend selection — picks between OmniVoice and Kokoro at engine
+ * arm time.
+ *
+ * The scheduler is backend-agnostic — both implementations satisfy
+ * `OmniVoiceBackend + StreamingTtsBackend`. This module isolates the
+ * decision logic so the engine layer (or a test) can ask "which backend
+ * should I instantiate?" and get a single, auditable answer.
+ *
+ * Decision modes:
+ *
+ *   - `omnivoice` — force OmniVoice. Used when the caller needs voice
+ *      cloning (Kokoro v1.0 has fixed voice packs, no per-user cloning).
+ *   - `kokoro`    — force Kokoro. Used when the caller cares about
+ *      first-audio latency over voice fidelity (Kokoro ≈ 97ms CPU TTFB,
+ *      OmniVoice ≈ 200ms on the fused build).
+ *   - `auto`      — apply the documented heuristic below.
+ *
+ * Mobile precedence: when `mobile === true` the selector returns Kokoro
+ * unconditionally (it is smaller + faster and the only backend shipped in
+ * mobile-class bundles). This short-circuits every mode and heuristic below.
+ *
+ * `auto` heuristic (deterministic, no model probes — those go through the
+ * autotune layer in `voice/scheduler.ts`):
+ *
+ *   1. If `requireVoiceCloning === true` → OmniVoice.
+ *   2. If `targetTtfaMs` is set and < 200 → Kokoro.
+ *   3. If a Kokoro RTF measurement is available and OmniVoice RTF is not,
+ *      or Kokoro RTF beats OmniVoice by ≥ 10% → Kokoro.
+ *   4. Else → first entry of `tierVoiceBackends` if provided (the
+ *      catalog's declared per-tier default). Falls back to OmniVoice
+ *      only when no tier policy is supplied.
+ *
+ * Tier policy comes from `ELIZA_1_VOICE_BACKENDS` in
+ * `packages/shared/src/local-inference/catalog.ts`. Callers should pass
+ * the active bundle's `voiceBackends` array so the selection is
+ * data-driven (small tiers → Kokoro default; large tiers → OmniVoice).
+ *
+ * The decision returns a tagged discriminated union, not a backend
+ * instance, so the engine layer can instantiate the chosen backend with
+ * its own dependencies (FFI handle / Kokoro runtime / etc.). This keeps
+ * the selection logic unit-testable without dragging the ORT or FFI
+ * surfaces into the test graph.
+ */
+
+export type VoiceBackendChoice = "omnivoice" | "kokoro";
+
+export type VoiceBackendMode = VoiceBackendChoice | "auto";
+
+export interface VoiceBackendInputs {
+	/** Caller-set mode. Defaults to `auto`. */
+	mode?: VoiceBackendMode;
+	/** Time-to-first-audio target (ms). Lower → prefer Kokoro. */
+	targetTtfaMs?: number;
+	/** Whether the caller needs per-user voice cloning. */
+	requireVoiceCloning?: boolean;
+	/** Latest measured RTF for Kokoro on this device (audio_seconds / wall_seconds). */
+	kokoroRtf?: number | null;
+	/** Latest measured RTF for OmniVoice on this device. */
+	omnivoiceRtf?: number | null;
+	/** Whether Kokoro model artifacts are present on disk. The selector
+	 *  never returns Kokoro when this is `false` — no silent downgrade. */
+	kokoroAvailable: boolean;
+	/** Whether the OmniVoice FFI library is present on disk. */
+	omnivoiceAvailable: boolean;
+	/**
+	 * True on mobile (iOS / Android) builds. Mobile uses Kokoro exclusively —
+	 * it is smaller and faster than OmniVoice and is the only TTS backend
+	 * shipped in mobile-class bundles. When set, the selector returns Kokoro
+	 * unconditionally (ignoring `mode`, RTF, and TTFA heuristics) and throws
+	 * if Kokoro artifacts are missing rather than falling back to OmniVoice.
+	 */
+	mobile?: boolean;
+	/**
+	 * The active bundle's per-tier voice backend policy, as declared in
+	 * `ELIZA_1_VOICE_BACKENDS`. First entry is the catalog default for
+	 * the tier; later entries are also bundled. The selector reads this
+	 * to make the `auto` default tier-aware rather than hard-coding a
+	 * single backend.
+	 *
+	 * Omit when called outside the Eliza-1 catalog context (e.g. ad-hoc
+	 * smoke benches) — the selector falls back to OmniVoice as the
+	 * historical default in that case.
+	 */
+	tierVoiceBackends?: ReadonlyArray<VoiceBackendChoice>;
+}
+
+export interface VoiceBackendDecision {
+	backend: VoiceBackendChoice;
+	/** One-line reason — surfaced to telemetry. */
+	reason: string;
+}
+
+const TTFA_CUTOFF_MS = 200;
+const RTF_MARGIN = 1.1; // Kokoro must beat OmniVoice by 10% to win on RTF.
+
+/** Resolve the env override (`ELIZA_TTS_BACKEND=kokoro|omnivoice|auto`). */
+export function readVoiceBackendModeFromEnv(
+	env: NodeJS.ProcessEnv = process.env,
+): VoiceBackendMode | undefined {
+	const raw = env.ELIZA_TTS_BACKEND?.trim().toLowerCase();
+	if (!raw) return undefined;
+	if (raw === "kokoro" || raw === "omnivoice" || raw === "auto") return raw;
+	throw new Error(
+		`[voice] ELIZA_TTS_BACKEND must be one of 'kokoro', 'omnivoice', 'auto' (got '${raw}')`,
+	);
+}
+
+export function selectVoiceBackend(
+	inputs: VoiceBackendInputs,
+): VoiceBackendDecision {
+	// Mobile is Kokoro-exclusive: it is smaller + faster and is the only TTS
+	// backend shipped in mobile-class bundles. This wins over every mode and
+	// heuristic below — no OmniVoice fallback on phones.
+	if (inputs.mobile) {
+		if (!inputs.kokoroAvailable) {
+			throw new Error(
+				"[voice] mobile builds use Kokoro exclusively but its model artifacts are not present on disk",
+			);
+		}
+		return {
+			backend: "kokoro",
+			reason:
+				"mobile platform — Kokoro exclusively (OmniVoice not shipped on mobile)",
+		};
+	}
+
+	const mode = inputs.mode ?? "auto";
+
+	if (mode === "kokoro") {
+		if (!inputs.kokoroAvailable) {
+			throw new Error(
+				"[voice] ELIZA_TTS_BACKEND=kokoro but Kokoro model artifacts are not present on disk",
+			);
+		}
+		return { backend: "kokoro", reason: "forced via mode=kokoro" };
+	}
+
+	if (mode === "omnivoice") {
+		if (!inputs.omnivoiceAvailable) {
+			throw new Error(
+				"[voice] ELIZA_TTS_BACKEND=omnivoice but the OmniVoice FFI library is not present",
+			);
+		}
+		return { backend: "omnivoice", reason: "forced via mode=omnivoice" };
+	}
+
+	// `auto` — apply heuristics.
+	if (inputs.requireVoiceCloning) {
+		if (!inputs.omnivoiceAvailable) {
+			throw new Error(
+				"[voice] voice cloning required but OmniVoice FFI library is not available; Kokoro v1.0 has no per-user cloning",
+			);
+		}
+		return {
+			backend: "omnivoice",
+			reason: "voice cloning required (Kokoro v1.0 cannot clone)",
+		};
+	}
+
+	if (
+		inputs.targetTtfaMs !== undefined &&
+		inputs.targetTtfaMs < TTFA_CUTOFF_MS
+	) {
+		if (inputs.kokoroAvailable) {
+			return {
+				backend: "kokoro",
+				reason: `targetTtfaMs=${inputs.targetTtfaMs} < ${TTFA_CUTOFF_MS} → Kokoro (~97ms CPU TTFB)`,
+			};
+		}
+		if (!inputs.omnivoiceAvailable) {
+			throw new Error(
+				"[voice] no TTS backend available (neither Kokoro model nor OmniVoice FFI library on disk)",
+			);
+		}
+		return {
+			backend: "omnivoice",
+			reason: "targetTtfaMs requested but Kokoro artifacts missing",
+		};
+	}
+
+	if (
+		inputs.kokoroAvailable &&
+		inputs.kokoroRtf !== null &&
+		inputs.kokoroRtf !== undefined &&
+		inputs.kokoroRtf > 0
+	) {
+		if (
+			inputs.omnivoiceRtf === null ||
+			inputs.omnivoiceRtf === undefined ||
+			inputs.omnivoiceRtf <= 0
+		) {
+			return {
+				backend: "kokoro",
+				reason: `Kokoro RTF=${inputs.kokoroRtf.toFixed(2)} measured; OmniVoice RTF unknown`,
+			};
+		}
+		if (inputs.kokoroRtf >= inputs.omnivoiceRtf * RTF_MARGIN) {
+			return {
+				backend: "kokoro",
+				reason: `Kokoro RTF=${inputs.kokoroRtf.toFixed(2)} beats OmniVoice RTF=${inputs.omnivoiceRtf.toFixed(2)} by ≥10%`,
+			};
+		}
+	}
+
+	if (!inputs.omnivoiceAvailable && inputs.kokoroAvailable) {
+		return {
+			backend: "kokoro",
+			reason: "OmniVoice FFI library not available; Kokoro is the only option",
+		};
+	}
+	if (!inputs.omnivoiceAvailable && !inputs.kokoroAvailable) {
+		throw new Error(
+			"[voice] no TTS backend available (neither Kokoro model nor OmniVoice FFI library on disk)",
+		);
+	}
+	// Both backends available, no override. Honor the tier's declared
+	// default if the caller supplied one (catalog-driven), else fall back
+	// to OmniVoice to preserve historical behavior for non-Eliza-1 contexts.
+	const tierDefault = inputs.tierVoiceBackends?.[0];
+	if (tierDefault === "kokoro") {
+		return {
+			backend: "kokoro",
+			reason: "tier default — kokoro per ELIZA_1_VOICE_BACKENDS",
+		};
+	}
+	if (tierDefault === "omnivoice") {
+		return {
+			backend: "omnivoice",
+			reason: "tier default — omnivoice per ELIZA_1_VOICE_BACKENDS",
+		};
+	}
+	return {
+		backend: "omnivoice",
+		reason: "default — OmniVoice on the fused build (no tier policy supplied)",
+	};
+}

package/src/services/voice/kokoro/types.d.ts

@@ -0,0 +1,82 @@
+/**
+ * Public types for the Kokoro-82M TTS backend (Apache-2.0, hexgrad/Kokoro-82M
+ * upstream).
+ *
+ * Kokoro is a small (~82M-param) StyleTTS-2 derivative that ships with a set
+ * of "voice packs" — pre-baked 256-dim style vectors (one .bin per voice).
+ * Adding voices is a cheap (~512KB) extra download.
+ */
+/** Canonical voice-pack id. Convention: `<lang>_<name>` (af_bella, am_michael). */
+export type KokoroVoiceId = string;
+/** One bundled voice pack — small fp32 style tensor on disk. */
+export interface KokoroVoicePack {
+    /** `af_bella`, `af_sarah`, `am_michael`, ... */
+    id: KokoroVoiceId;
+    /** Human-readable name shown in UI. */
+    displayName: string;
+    /** Two-letter language tag (`a` = American English, `b` = British English, etc. per Kokoro convention). */
+    lang: string;
+    /** Filename inside the voices/ directory, relative to the Kokoro model root. */
+    file: string;
+    /** Style-vector dim (256 for v1.0). */
+    dim: number;
+    /** Genre/voice tags for picker UIs. */
+    tags?: ReadonlyArray<string>;
+}
+/** Where the runtime expects to find Kokoro on disk. */
+export interface KokoroModelLayout {
+    /** Directory under `<stateDir>/local-inference/models/kokoro/`. */
+    root: string;
+    /** Model file — the Kokoro GGUF carried by our llama.cpp fork. */
+    modelFile: string;
+    /** Directory containing the per-voice style tensors. */
+    voicesDir: string;
+    /** Model output sample rate (Kokoro v1.0 = 24000). */
+    sampleRate: number;
+}
+/** Construction-time configuration for `KokoroTtsBackend`. */
+export interface KokoroBackendOptions {
+    /** Resolved on-disk layout. Required — the backend never guesses paths. */
+    layout: KokoroModelLayout;
+    /**
+     * Voice id to use when the caller's `SpeakerPreset.voiceId` is not in the
+     * voice-pack registry. The named voice MUST be present in `layout.voicesDir`.
+     */
+    defaultVoiceId: KokoroVoiceId;
+    /**
+     * Optional phonemizer override. Defaults to the bundled lazy phonemizer,
+     * which uses `phonemize` if installed and falls back to a deterministic
+     * grapheme-to-phoneme adapter otherwise (documented tradeoff in README).
+     */
+    phonemizer?: KokoroPhonemizer;
+    /**
+     * Max samples emitted in a single streaming chunk. Defaults to a quarter-
+     * second at 24kHz (6000) so the scheduler ring buffer sees a continuous
+     * trickle and TTFB stays close to the first inference completion.
+     */
+    streamingChunkSamples?: number;
+}
+/** A pure (or async-pure) text → phoneme-id sequence converter. */
+export interface KokoroPhonemizer {
+    /** Phonemize a single utterance into a sequence of integer ids. */
+    phonemize(text: string, lang: string): Promise<KokoroPhonemeSequence>;
+    /** Human-facing id (`"phonemize"`, `"espeak-ng"`, `"fallback-g2p"`, ...). */
+    readonly id: string;
+}
+export interface KokoroPhonemeSequence {
+    /** Token ids for the model's `input_ids` tensor. */
+    ids: Int32Array;
+    /** Original phoneme string, for debugging and tests. */
+    phonemes: string;
+}
+/** Raised when the on-disk model layout is missing or malformed. */
+export declare class KokoroModelMissingError extends Error {
+    readonly code: "kokoro-model-missing";
+    constructor(message: string);
+}
+/** Raised when phonemization cannot proceed (no phonemizer + non-ASCII text). */
+export declare class KokoroPhonemizerError extends Error {
+    readonly code: "kokoro-phonemizer-error";
+    constructor(message: string);
+}
+//# sourceMappingURL=types.d.ts.map

package/src/services/voice/kokoro/types.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["types.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AAEH,mFAAmF;AACnF,MAAM,MAAM,aAAa,GAAG,MAAM,CAAC;AAEnC,gEAAgE;AAChE,MAAM,WAAW,eAAe;IAC/B,gDAAgD;IAChD,EAAE,EAAE,aAAa,CAAC;IAClB,uCAAuC;IACvC,WAAW,EAAE,MAAM,CAAC;IACpB,2GAA2G;IAC3G,IAAI,EAAE,MAAM,CAAC;IACb,gFAAgF;IAChF,IAAI,EAAE,MAAM,CAAC;IACb,uCAAuC;IACvC,GAAG,EAAE,MAAM,CAAC;IACZ,uCAAuC;IACvC,IAAI,CAAC,EAAE,aAAa,CAAC,MAAM,CAAC,CAAC;CAC7B;AAED,wDAAwD;AACxD,MAAM,WAAW,iBAAiB;IACjC,mEAAmE;IACnE,IAAI,EAAE,MAAM,CAAC;IACb,kEAAkE;IAClE,SAAS,EAAE,MAAM,CAAC;IAClB,wDAAwD;IACxD,SAAS,EAAE,MAAM,CAAC;IAClB,sDAAsD;IACtD,UAAU,EAAE,MAAM,CAAC;CACnB;AAED,8DAA8D;AAC9D,MAAM,WAAW,oBAAoB;IACpC,2EAA2E;IAC3E,MAAM,EAAE,iBAAiB,CAAC;IAC1B;;;OAGG;IACH,cAAc,EAAE,aAAa,CAAC;IAC9B;;;;OAIG;IACH,UAAU,CAAC,EAAE,gBAAgB,CAAC;IAC9B;;;;OAIG;IACH,qBAAqB,CAAC,EAAE,MAAM,CAAC;CAC/B;AAED,mEAAmE;AACnE,MAAM,WAAW,gBAAgB;IAChC,mEAAmE;IACnE,SAAS,CAAC,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,qBAAqB,CAAC,CAAC;IACtE,6EAA6E;IAC7E,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAC;CACpB;AAED,MAAM,WAAW,qBAAqB;IACrC,oDAAoD;IACpD,GAAG,EAAE,UAAU,CAAC;IAChB,wDAAwD;IACxD,QAAQ,EAAE,MAAM,CAAC;CACjB;AAED,oEAAoE;AACpE,qBAAa,uBAAwB,SAAQ,KAAK;IACjD,QAAQ,CAAC,IAAI,EAAG,sBAAsB,CAAU;gBACpC,OAAO,EAAE,MAAM;CAI3B;AAED,iFAAiF;AACjF,qBAAa,qBAAsB,SAAQ,KAAK;IAC/C,QAAQ,CAAC,IAAI,EAAG,yBAAyB,CAAU;gBACvC,OAAO,EAAE,MAAM;CAI3B"}

package/src/services/voice/kokoro/types.ts

@@ -0,0 +1,95 @@
+/**
+ * Public types for the Kokoro-82M TTS backend (Apache-2.0, hexgrad/Kokoro-82M
+ * upstream).
+ *
+ * Kokoro is a small (~82M-param) StyleTTS-2 derivative that ships with a set
+ * of "voice packs" — pre-baked 256-dim style vectors (one .bin per voice).
+ * Adding voices is a cheap (~512KB) extra download.
+ */
+
+/** Canonical voice-pack id. Convention: `<lang>_<name>` (af_bella, am_michael). */
+export type KokoroVoiceId = string;
+
+/** One bundled voice pack — small fp32 style tensor on disk. */
+export interface KokoroVoicePack {
+	/** `af_bella`, `af_sarah`, `am_michael`, ... */
+	id: KokoroVoiceId;
+	/** Human-readable name shown in UI. */
+	displayName: string;
+	/** Two-letter language tag (`a` = American English, `b` = British English, etc. per Kokoro convention). */
+	lang: string;
+	/** Filename inside the voices/ directory, relative to the Kokoro model root. */
+	file: string;
+	/** Style-vector dim (256 for v1.0). */
+	dim: number;
+	/** Genre/voice tags for picker UIs. */
+	tags?: ReadonlyArray<string>;
+}
+
+/** Where the runtime expects to find Kokoro on disk. */
+export interface KokoroModelLayout {
+	/** Directory under `<stateDir>/local-inference/models/kokoro/`. */
+	root: string;
+	/** Model file — the Kokoro GGUF carried by our llama.cpp fork. */
+	modelFile: string;
+	/** Directory containing the per-voice style tensors. */
+	voicesDir: string;
+	/** Model output sample rate (Kokoro v1.0 = 24000). */
+	sampleRate: number;
+}
+
+/** Construction-time configuration for `KokoroTtsBackend`. */
+export interface KokoroBackendOptions {
+	/** Resolved on-disk layout. Required — the backend never guesses paths. */
+	layout: KokoroModelLayout;
+	/**
+	 * Voice id to use when the caller's `SpeakerPreset.voiceId` is not in the
+	 * voice-pack registry. The named voice MUST be present in `layout.voicesDir`.
+	 */
+	defaultVoiceId: KokoroVoiceId;
+	/**
+	 * Optional phonemizer override. Defaults to the bundled lazy phonemizer,
+	 * which uses `phonemize` if installed and falls back to a deterministic
+	 * grapheme-to-phoneme adapter otherwise (documented tradeoff in README).
+	 */
+	phonemizer?: KokoroPhonemizer;
+	/**
+	 * Max samples emitted in a single streaming chunk. Defaults to a quarter-
+	 * second at 24kHz (6000) so the scheduler ring buffer sees a continuous
+	 * trickle and TTFB stays close to the first inference completion.
+	 */
+	streamingChunkSamples?: number;
+}
+
+/** A pure (or async-pure) text → phoneme-id sequence converter. */
+export interface KokoroPhonemizer {
+	/** Phonemize a single utterance into a sequence of integer ids. */
+	phonemize(text: string, lang: string): Promise<KokoroPhonemeSequence>;
+	/** Human-facing id (`"phonemize"`, `"espeak-ng"`, `"fallback-g2p"`, ...). */
+	readonly id: string;
+}
+
+export interface KokoroPhonemeSequence {
+	/** Token ids for the model's `input_ids` tensor. */
+	ids: Int32Array;
+	/** Original phoneme string, for debugging and tests. */
+	phonemes: string;
+}
+
+/** Raised when the on-disk model layout is missing or malformed. */
+export class KokoroModelMissingError extends Error {
+	readonly code = "kokoro-model-missing" as const;
+	constructor(message: string) {
+		super(message);
+		this.name = "KokoroModelMissingError";
+	}
+}
+
+/** Raised when phonemization cannot proceed (no phonemizer + non-ASCII text). */
+export class KokoroPhonemizerError extends Error {
+	readonly code = "kokoro-phonemizer-error" as const;
+	constructor(message: string) {
+		super(message);
+		this.name = "KokoroPhonemizerError";
+	}
+}

package/src/services/voice/kokoro/voice-presets.d.ts

@@ -0,0 +1,23 @@
+/**
+ * Registry of bundled Kokoro voice packs (upstream: hexgrad/Kokoro-82M).
+ *
+ * Each entry maps a stable `KokoroVoiceId` (the `voices/<id>.bin` filename
+ * Kokoro ships) onto display metadata. The runtime resolves a caller's
+ * `SpeakerPreset.voiceId` against this table; an unknown id falls through to
+ * the backend's `defaultVoiceId`.
+ *
+ * The actual style tensor (256 fp32 values) lives at
+ * `<modelRoot>/voices/<file>` and is loaded lazily on first use.
+ *
+ * Reference: https://huggingface.co/hexgrad/Kokoro-82M
+ */
+import type { KokoroVoicePack } from "./types";
+export declare const KOKORO_VOICE_PACKS: ReadonlyArray<KokoroVoicePack>;
+/** Look up a voice pack by id. Returns `undefined` for unknown ids — the
+ *  backend chooses how to fall back (typically `defaultVoiceId`). */
+export declare function findKokoroVoice(id: string): KokoroVoicePack | undefined;
+/** The voice the runtime selects when nothing is configured. */
+export declare const KOKORO_DEFAULT_VOICE_ID = "af_same";
+/** Conservative fallback voice when a configured/default preset is not staged. */
+export declare const KOKORO_FALLBACK_VOICE_ID = "af_bella";
+//# sourceMappingURL=voice-presets.d.ts.map

package/src/services/voice/kokoro/voice-presets.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"voice-presets.d.ts","sourceRoot":"","sources":["voice-presets.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;GAYG;AAEH,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,SAAS,CAAC;AAE/C,eAAO,MAAM,kBAAkB,EAAE,aAAa,CAAC,eAAe,CAkG7D,CAAC;AAIF;qEACqE;AACrE,wBAAgB,eAAe,CAAC,EAAE,EAAE,MAAM,GAAG,eAAe,GAAG,SAAS,CAEvE;AAED,gEAAgE;AAChE,eAAO,MAAM,uBAAuB,YAAY,CAAC;AAEjD,kFAAkF;AAClF,eAAO,MAAM,wBAAwB,aAAa,CAAC"}

package/src/services/voice/kokoro/voice-presets.ts

@@ -0,0 +1,129 @@
+/**
+ * Registry of bundled Kokoro voice packs (upstream: hexgrad/Kokoro-82M).
+ *
+ * Each entry maps a stable `KokoroVoiceId` (the `voices/<id>.bin` filename
+ * Kokoro ships) onto display metadata. The runtime resolves a caller's
+ * `SpeakerPreset.voiceId` against this table; an unknown id falls through to
+ * the backend's `defaultVoiceId`.
+ *
+ * The actual style tensor (256 fp32 values) lives at
+ * `<modelRoot>/voices/<file>` and is loaded lazily on first use.
+ *
+ * Reference: https://huggingface.co/hexgrad/Kokoro-82M
+ */
+
+import type { KokoroVoicePack } from "./types";
+
+export const KOKORO_VOICE_PACKS: ReadonlyArray<KokoroVoicePack> = [
+	// American English — female
+	{
+		id: "af_bella",
+		displayName: "Bella (US English)",
+		lang: "a",
+		file: "af_bella.bin",
+		dim: 256,
+		tags: ["female", "warm", "default"],
+	},
+	{
+		id: "af_sarah",
+		displayName: "Sarah (US English)",
+		lang: "a",
+		file: "af_sarah.bin",
+		dim: 256,
+		tags: ["female", "professional"],
+	},
+	{
+		id: "af_nicole",
+		displayName: "Nicole (US English, breathy)",
+		lang: "a",
+		file: "af_nicole.bin",
+		dim: 256,
+		tags: ["female", "breathy"],
+	},
+	{
+		id: "af_sky",
+		displayName: "Sky (US English)",
+		lang: "a",
+		file: "af_sky.bin",
+		dim: 256,
+		tags: ["female", "young"],
+	},
+	// American English — male
+	{
+		id: "am_michael",
+		displayName: "Michael (US English)",
+		lang: "a",
+		file: "am_michael.bin",
+		dim: 256,
+		tags: ["male", "warm"],
+	},
+	{
+		id: "am_adam",
+		displayName: "Adam (US English)",
+		lang: "a",
+		file: "am_adam.bin",
+		dim: 256,
+		tags: ["male", "neutral"],
+	},
+	// British English
+	{
+		id: "bf_emma",
+		displayName: "Emma (British English)",
+		lang: "b",
+		file: "bf_emma.bin",
+		dim: 256,
+		tags: ["female", "british"],
+	},
+	{
+		id: "bf_isabella",
+		displayName: "Isabella (British English)",
+		lang: "b",
+		file: "bf_isabella.bin",
+		dim: 256,
+		tags: ["female", "british"],
+	},
+	{
+		id: "bm_george",
+		displayName: "George (British English)",
+		lang: "b",
+		file: "bm_george.bin",
+		dim: 256,
+		tags: ["male", "british"],
+	},
+	{
+		id: "bm_lewis",
+		displayName: "Lewis (British English)",
+		lang: "b",
+		file: "bm_lewis.bin",
+		dim: 256,
+		tags: ["male", "british"],
+	},
+	// Eliza-1 fine-tuned voice — same (research-only, derivative of *Her* 2013).
+	// Voice pack lives at `elizaos/eliza-1` under `voice/kokoro/voices/af_same.bin`
+	// (first push is private; do not promote to default without a public-release sign-off).
+	// Source corpus: `lalalune/ai_voices/sam` upstream subset, landed locally as
+	// `same` (58 clips, 3.51 min, research-only).
+	// Voice id obeys the Kokoro `<lang><sex>_<name>` convention (US English, female).
+	{
+		id: "af_same",
+		displayName: "Same (Eliza-1, US English)",
+		lang: "a",
+		file: "af_same.bin",
+		dim: 256,
+		tags: ["female", "same", "eliza-1-voice", "research-only"],
+	},
+];
+
+const VOICE_BY_ID = new Map(KOKORO_VOICE_PACKS.map((v) => [v.id, v] as const));
+
+/** Look up a voice pack by id. Returns `undefined` for unknown ids — the
+ *  backend chooses how to fall back (typically `defaultVoiceId`). */
+export function findKokoroVoice(id: string): KokoroVoicePack | undefined {
+	return VOICE_BY_ID.get(id);
+}
+
+/** The voice the runtime selects when nothing is configured. */
+export const KOKORO_DEFAULT_VOICE_ID = "af_same";
+
+/** Conservative fallback voice when a configured/default preset is not staged. */
+export const KOKORO_FALLBACK_VOICE_ID = "af_bella";