@elizaos/plugin-local-inference 2.0.0-beta.1 → 2.0.11-beta.7

package/src/services/voice/index.ts

@@ -0,0 +1,481 @@
+export {
+	type AttributedTurn,
+	type AttributedTurnListener,
+	type AttributionPipelineLike,
+	AUDIO_FRAME_PIPELINE_SAMPLE_RATE,
+	AudioFrameConsumer,
+	type AudioFrameConsumerConfig,
+	type AudioFrameConsumerDeps,
+	AudioFrameDecodeError,
+	type AudioFrameEvent,
+	decodeAudioFramePcm,
+	type RuntimeEventSink,
+	type VadSegmenter,
+} from "./audio-frame-consumer";
+export {
+	BargeInController,
+	type BargeInControllerConfig,
+	type BargeInListener,
+	type CancelSignal,
+} from "./barge-in";
+export {
+	type CoordinatorRuntime,
+	VoiceCancellationCoordinator,
+	type VoiceCancellationCoordinatorOptions,
+} from "./cancellation-coordinator";
+export {
+	type CheckpointHandle,
+	CheckpointHandleInvalidError,
+	CheckpointManager,
+	type CheckpointManagerLike,
+	type CheckpointManagerOptions,
+	MockCheckpointManager,
+	type MockCheckpointSnapshot,
+	type MockSnapshotSource,
+} from "./checkpoint-manager";
+export {
+	type BuildDeterministicFn,
+	type BuildMessageDependentFn,
+	type ContextPartial,
+	EagerContextBuilder,
+	type EagerContextBuilderOptions,
+	type FullContext,
+	mergeContext,
+} from "./eager-context-builder";
+export type {
+	LlamaContextLike as Eliza1EotLlamaContext,
+	LlamaContextSequenceLike as Eliza1EotLlamaSequence,
+	LlamaModelLike as Eliza1EotLlamaModel,
+} from "./eliza1-eot-scorer";
+export {
+	Eliza1EotScorer,
+	formatEotPrompt as formatEliza1EotPrompt,
+} from "./eliza1-eot-scorer";
+export {
+	buildLocalEmbeddingRoute,
+	EMBEDDING_DIR_REL_PATH,
+	EMBEDDING_FULL_DIM,
+	EMBEDDING_MATRYOSHKA_DIMS,
+	isValidEmbeddingDim,
+	type LocalEmbeddingRoute,
+	type LocalEmbeddingSource,
+	POOLED_TEXT_EMBEDDING_TIERS,
+	resolveLocalEmbeddingSource,
+	truncateMatryoshka,
+} from "./embedding";
+export {
+	attributeVoiceEmotion,
+	type VoiceEmotionAsrFeatures,
+	type VoiceEmotionAttribution,
+	type VoiceEmotionAttributionInput,
+	type VoiceEmotionAttributionMethod,
+	type VoiceEmotionAudioFeatures,
+	type VoiceEmotionEvidence,
+} from "./emotion-attribution";
+export {
+	decodeMonoPcm16Wav,
+	defaultLifecycleLoaders,
+	EngineVoiceBridge,
+	type EngineVoiceBridgeOptions,
+	encodeMonoPcm16Wav,
+	FfiOmniVoiceBackend,
+	StubOmniVoiceBackend,
+	type VoiceTurnEvents,
+} from "./engine-bridge";
+export {
+	Eliza1EotClassifier,
+	type Eliza1EotScoreResult,
+	type Eliza1EotScorerOptions,
+	EOT_COMMIT_SILENCE_MS,
+	EOT_COMMIT_THRESHOLD,
+	EOT_HANGOVER_EXTENSION_MS,
+	EOT_MID_CLAUSE_THRESHOLD,
+	EOT_TENTATIVE_SILENCE_MS,
+	EOT_TENTATIVE_THRESHOLD,
+	type EotClassifier,
+	HeuristicEotClassifier,
+	LIVEKIT_TURN_DETECTOR_EN_REVISION,
+	LIVEKIT_TURN_DETECTOR_INTL_REVISION,
+	RemoteEotClassifier,
+	type RemoteEotClassifierOptions,
+	turnDetectorRevisionForTier,
+	turnSignalFromProbability,
+	type VoiceNextSpeaker,
+	type VoiceTurnSignal,
+} from "./eot-classifier";
+export {
+	applyQwenUserTemplate,
+	createBundledLiveKitGgmlTurnDetector,
+	DEFAULT_LIVEKIT_TURN_DETECTOR_GGML_DIR,
+	DEFAULT_LIVEKIT_TURN_DETECTOR_GGUF_EN,
+	DEFAULT_LIVEKIT_TURN_DETECTOR_GGUF_INTL,
+	EotGgmlUnavailableError,
+	LIVEKIT_IM_END_TOKEN,
+	LiveKitGgmlTurnDetector,
+	type LiveKitGgmlTurnDetectorOptions,
+	turnDetectorGgufForTier,
+} from "./eot-classifier-ggml";
+export { VoiceStartupError } from "./errors";
+export * from "./ffi-bindings";
+export {
+	_resetSharedFirstLineCacheForTesting,
+	_resetVoiceRevisionMemoForTesting,
+	computeLocalVoiceRevision,
+	FIRST_SENTENCE_MAX_WORDS,
+	FIRST_SENTENCE_SNIP_VERSION,
+	FirstLineCache,
+	type FirstLineCacheEntry,
+	type FirstLineCacheKey,
+	type FirstLineCacheOptions,
+	type FirstLineCacheStats,
+	type FirstSentenceSnipResult,
+	fingerprintVoiceSettings,
+	firstLineCacheBypassFromEnv,
+	firstSentenceSnip,
+	getSharedFirstLineCache,
+	hashCacheKey,
+	type PutInput,
+	wordCount,
+} from "./first-line-cache";
+export {
+	type ArmedResources,
+	type TextResources,
+	VoiceLifecycle,
+	VoiceLifecycleError,
+	type VoiceLifecycleEvents,
+	type VoiceLifecycleLoaders,
+	type VoiceLifecycleState,
+} from "./lifecycle";
+export {
+	DesktopMicSource,
+	type DesktopMicSourceOptions,
+	PushMicSource,
+	pipeMicToRingBuffer,
+	resolveDesktopRecorder,
+} from "./mic-source";
+export {
+	DEFAULT_OPTIMISTIC_EOT_THRESHOLD,
+	OptimisticGenerationPolicy,
+	type OptimisticPolicyOptions,
+	type PowerSourceState,
+	type ResolveOptimisticPolicyArgs,
+	resolveOptimisticPolicyEnabled,
+	resolvePowerSourceState,
+} from "./optimistic-policy";
+export {
+	createDefaultPhonemeTokenizer,
+	type Phoneme,
+	type PhonemeTokenizer,
+	RuleBasedEnglishPhonemeTokenizer,
+} from "./phoneme-tokenizer";
+export {
+	type CachedPhraseAudio,
+	canonicalizePhraseText,
+	DEFAULT_PHRASE_CACHE_SEED,
+	FIRST_AUDIO_FILLERS,
+	PhraseCache,
+} from "./phrase-cache";
+export { chunkTokens, PhraseChunker } from "./phrase-chunker";
+export {
+	type DraftProposer,
+	splitTranscriptToTokens,
+	type TargetVerifier,
+	VoicePipeline,
+	type VoicePipelineConfig,
+	type VoicePipelineDeps,
+	type VoicePipelineEvents,
+} from "./pipeline";
+export {
+	MissingAsrTranscriber,
+	MtpDraftProposer,
+	MtpTargetVerifier,
+	type MtpTextRunner,
+	mtpTextRunner,
+} from "./pipeline-impls";
+export {
+	type PrefillOptimisticArgs,
+	type PrefillOptimisticOptions,
+	type PrefillOptimisticResult,
+	prefillOptimistic,
+} from "./prefill-client";
+export {
+	PrefixPreservingQueue,
+	type RollbackResult,
+	type TaggedAudioChunk,
+} from "./prefix-preserving-queue";
+export {
+	isOutlier,
+	VOICE_PROFILE_RECORD_SCHEMA_VERSION,
+	type VoiceImprintMatchHandle,
+	type VoiceProfileAudioRef,
+	type VoiceProfileConsentState,
+	type VoiceProfileObservation,
+	type VoiceProfileRecord,
+	VoiceProfileStore,
+	type VoiceProfileStoreOptions,
+	welfordUpdate,
+	welfordVariance,
+} from "./profile-store";
+export { InMemoryAudioSink, PcmRingBuffer } from "./ring-buffer";
+export { type RollbackEvent, RollbackQueue } from "./rollback-queue";
+export {
+	type SchedulerDeps,
+	type SchedulerEvents,
+	VoiceScheduler,
+} from "./scheduler";
+export {
+	createMtpDraftHandle,
+	type KernelSet,
+	type MmapRegionHandle,
+	type MtpDraftHandle,
+	type RefCountedResource,
+	type SchedulerSlot,
+	SharedResourceRegistry,
+	type SharedTokenizer,
+} from "./shared-resources";
+export {
+	type VoiceAttributionOutput,
+	VoiceAttributionPipeline,
+	type VoiceAttributionPipelineDeps,
+	type VoiceAttributionRequest,
+} from "./speaker/attribution-pipeline";
+export {
+	classifyFramesToSegments,
+	type Diarizer,
+	type DiarizerOutput,
+	DiarizerUnavailableError,
+	type LocalSpeakerSegment,
+	PYANNOTE_CLASS_COUNT,
+	PYANNOTE_CLASS_TO_SPEAKERS,
+	PYANNOTE_FRAME_STRIDE_MS,
+	PYANNOTE_FRAMES_PER_WINDOW,
+	PYANNOTE_SAMPLE_RATE,
+	PYANNOTE_SEGMENTATION_3_FP32_MODEL_ID,
+	PYANNOTE_SEGMENTATION_3_INT8_MODEL_ID,
+	PYANNOTE_WINDOW_SECONDS,
+	type PyannoteDiarizerModelId,
+} from "./speaker/diarizer";
+export {
+	FusedDiarizer,
+	type FusedDiarizerOptions,
+} from "./speaker/diarizer-fused";
+export {
+	averageEmbeddings,
+	type SpeakerEncoder,
+	SpeakerEncoderUnavailableError,
+	WESPEAKER_EMBEDDING_DIM,
+	WESPEAKER_MIN_SAMPLES,
+	WESPEAKER_RESNET34_LM_FP32_MODEL_ID,
+	WESPEAKER_RESNET34_LM_INT8_MODEL_ID,
+	WESPEAKER_SAMPLE_RATE,
+	type WespeakerModelId,
+} from "./speaker/encoder";
+export {
+	FusedSpeakerEncoder,
+	type FusedSpeakerEncoderOptions,
+} from "./speaker/encoder-fused";
+export {
+	SPEAKER_GGML_EMBEDDING_DIM,
+	SPEAKER_GGML_MIN_SAMPLES,
+	SPEAKER_GGML_SAMPLE_RATE,
+	SpeakerEncoderGgmlUnavailableError,
+	voiceSpeakerDistance,
+} from "./speaker/encoder-ggml";
+export {
+	type AttributedVoiceObservation,
+	attributeVoiceImprintObservations,
+	cosineSimilarity,
+	DEFAULT_VOICE_IMPRINT_MATCH_THRESHOLD,
+	matchVoiceImprint,
+	normalizeVoiceEmbedding,
+	type SpeakerAttributionResult,
+	updateVoiceImprintCentroid,
+	type VoiceImprintCentroidUpdate,
+	type VoiceImprintMatch,
+	type VoiceImprintObservationInput,
+	type VoiceImprintProfile,
+	voiceSpeakerFromImprintMatch,
+} from "./speaker-imprint";
+export {
+	DEFAULT_VOICE_ID,
+	DEFAULT_VOICE_PRESET_REL_PATH,
+	type LoadedPresetBundle,
+	type PresetBundlePaths,
+	SpeakerPresetCache,
+	type SpeakerPresetCacheOptions,
+	voicePresetPath,
+} from "./speaker-preset-cache";
+export {
+	SystemAudioSink,
+	type SystemAudioSinkOptions,
+	WavFileAudioSink,
+	type WavFileAudioSinkOptions,
+} from "./system-audio-sink";
+export {
+	ASR_SAMPLE_RATE,
+	AsrUnavailableError,
+	BaseStreamingTranscriber,
+	type CreateStreamingTranscriberOptions,
+	createStreamingTranscriber,
+	FfiBatchTranscriber,
+	type FfiBatchTranscriberOptions,
+	FfiStreamingTranscriber,
+	ffiSupportsStreamingAsr,
+	resampleLinear,
+} from "./transcriber";
+export {
+	type VoiceGenerateRequest,
+	VoiceTurnController,
+	type VoiceTurnControllerConfig,
+	type VoiceTurnControllerDeps,
+	type VoiceTurnControllerEvents,
+	type VoiceTurnOutcome,
+} from "./turn-controller";
+export * from "./types";
+export {
+	createSileroVadDetector,
+	createVadDetector,
+	GgmlSileroVad,
+	NativeSileroVad,
+	type QwenToolkitVadAdapter,
+	type ResolvedVadProvider,
+	RmsEnergyGate,
+	type RmsEnergyGateConfig,
+	resolveSileroVadPath,
+	resolveVadProvider,
+	rms,
+	VadDetector,
+	type VadDetectorConfig,
+	type VadLike,
+	type VadProviderId,
+	type VadProviderPreference,
+	VadUnavailableError,
+	vadProviderOrder,
+} from "./vad";
+export {
+	type AllocationPriority,
+	assessVoiceBundleFits,
+	BudgetExhaustedError,
+	type BudgetReservation,
+	createVoiceBudget,
+	createVoiceBudgetForTest,
+	DEFAULT_VOICE_BUNDLE_RESERVE_MB,
+	pickVoiceTierSlot,
+	priorityClassForRole,
+	type ReservationSnapshot,
+	VOICE_ENSEMBLE_BUDGETS,
+	type VoiceBudget,
+	type VoiceBundleFitDecision,
+	type VoiceEnsembleBudget,
+	type VoiceTierSlot,
+	voiceEnsemblePeakMb,
+	voiceEnsembleSteadyStateMb,
+} from "./voice-budget";
+export {
+	readVoicePresetFile,
+	VOICE_PRESET_MAGIC,
+	type VoicePresetFile,
+	VoicePresetFormatError,
+	type VoicePresetSeedPhrase,
+	writeVoicePresetFile,
+} from "./voice-preset-format";
+export {
+	analyzeVoiceProfileWav,
+	canonicalVoiceProfileJson,
+	createVoiceProfileArtifact,
+	VOICE_PROFILE_ARTIFACT_SCHEMA_VERSION,
+	VOICE_PROFILE_FEATURE_EMBEDDING_MODEL,
+	type VoiceProfileArtifact,
+	type VoiceProfileArtifactSample,
+	type VoiceProfileArtifactStatus,
+	type VoiceProfileArtifactVerification,
+	type VoiceProfileAudioFeatures,
+	type VoiceProfileConsent,
+	type VoiceProfileReferenceMetadata,
+	type VoiceProfileSampleInput,
+	verifyVoiceProfileArtifact,
+} from "./voice-profile-artifact";
+export {
+	DEFAULT_VOICE_SETTINGS,
+	effectiveBackendMode,
+	qualityPresetQuantizationRanking,
+	resolveVoiceSettings,
+	type VoiceBackendMode,
+	type VoiceModelQualityPreset,
+	type VoiceSettings,
+} from "./voice-settings";
+export {
+	type DrafterAbortReason,
+	type DrafterHandle,
+	type StartDrafterFn,
+	type VoiceState,
+	type VoiceStateEvent,
+	VoiceStateMachine,
+	type VoiceStateMachineEvents,
+	type VoiceStateMachineOptions,
+} from "./voice-state-machine";
+export {
+	GgmlWakeWordModel,
+	isPlaceholderWakeWordHead,
+	loadBundledWakeWordModel,
+	OPENWAKEWORD_DEFAULT_HEAD,
+	OPENWAKEWORD_DIR_REL_PATH,
+	OPENWAKEWORD_GGUF_REL_PATH,
+	OPENWAKEWORD_PLACEHOLDER_HEADS,
+	OpenWakeWordDetector,
+	resolveWakeWordModel,
+	type WakeWordConfig,
+	type WakeWordModel,
+	type WakeWordModelPaths,
+	WakeWordUnavailableError,
+} from "./wake-word";
+export {
+	type TtsBytes,
+	type TtsHandler,
+	type TtsHandlerInput,
+	type TtsHandlerOutput,
+	type TtsResolvedContext,
+	type WrapOptions,
+	wrapWithFirstLineCache,
+} from "./wrap-with-first-line-cache";
+
+/**
+ * Voice on/off invariants (binding for every consumer of this module):
+ *
+ * 1. Voice is OFF by default — text + drafter only. Before
+ *    `EngineVoiceBridge.start()` there are no voice resources in RAM.
+ *    After `start()` but before `VoiceLifecycle.arm()`, only the tiny
+ *    default speaker preset, phrase seed metadata, and scheduler
+ *    scaffolding are live. TTS/ASR weight regions are NOT mapped or
+ *    re-paged until `VoiceLifecycle.arm()` calls the fused ABI's
+ *    `mmap_acquire`.
+ *
+ * 2. Shared resources between text and voice (one instance each per
+ *    engine, refcounted by `SharedResourceRegistry`):
+ *      - tokenizer (Eliza-1/OmniVoice share a vocabulary)
+ *      - mmap regions for weights (deduplicated by absolute path)
+ *      - the fused kernel set (TurboQuant/QJL/Polar live in the
+ *        same shipped llama.cpp library after the fusion build)
+ *      - the scheduler queue (one queue, prioritised across surfaces)
+ *      - the MTP drafter (always wired — see AGENTS.md §3 #4)
+ *
+ *    Text and voice keep SEPARATE KV caches (different layer counts,
+ *    different head configs, different quantizations — AGENTS.md §4
+ *    "shared KV cache scheduling, not shared KV memory").
+ *
+ * 3. `arm()` lazily loads TTS + ASR via mmap; `disarm()` issues a real
+ *    page-eviction call (`madvise(MADV_DONTNEED)` on Linux/Android,
+ *    `madvise(MADV_FREE_REUSABLE)` on Apple, `VirtualUnlock` +
+ *    `OfferVirtualMemory` on Windows). The speaker preset and phrase
+ *    cache stay in a small LRU after disarm — they're KB-scale.
+ *
+ * 4. Hardware-resource exhaustion (RAM pressure, OS page eviction
+ *    refusal, mmap fail, kernel missing) MUST surface as a
+ *    `VoiceLifecycleError` with a structured `code`. There is NO
+ *    silent fallback to text-only and NO automatic downgrade to a
+ *    smaller voice model — see AGENTS.md §3.
+ *
+ * 5. Illegal lifecycle transitions throw `VoiceLifecycleError` with
+ *    code `"illegal-transition"`. The state is a discriminated
+ *    union, never a string.
+ */

package/src/services/voice/kokoro/__tests__/kokoro-backend.test.ts

@@ -0,0 +1,151 @@
+import { describe, expect, it } from "vitest";
+import type {
+	AudioSink,
+	Phrase,
+	SpeakerPreset,
+	TtsPcmChunk,
+} from "../../types";
+import { KokoroTtsBackend } from "../kokoro-backend";
+import { KokoroMockRuntime } from "../kokoro-runtime";
+import type { KokoroPhonemizer } from "../types";
+import { KOKORO_DEFAULT_VOICE_ID } from "../voices";
+
+function fixedPhonemizer(): KokoroPhonemizer {
+	return {
+		id: "fixed",
+		async phonemize() {
+			return { ids: Int32Array.from([1, 43, 60, 2]), phonemes: "ab" };
+		},
+	};
+}
+
+function makePreset(voiceId: string): SpeakerPreset {
+	return {
+		voiceId,
+		embedding: new Float32Array(8),
+		bytes: new Uint8Array(8),
+	};
+}
+
+function makePhrase(text: string): Phrase {
+	return {
+		id: 1,
+		text,
+		fromIndex: 0,
+		toIndex: text.length - 1,
+		terminator: "punctuation",
+	};
+}
+
+function makeBackend(opts?: { totalSamples?: number; chunkCount?: number }): {
+	backend: KokoroTtsBackend;
+	runtime: KokoroMockRuntime;
+} {
+	const runtime = new KokoroMockRuntime({
+		sampleRate: 24000,
+		totalSamples: opts?.totalSamples ?? 9600, // 0.4s
+		chunkCount: opts?.chunkCount ?? 4,
+	});
+	const backend = new KokoroTtsBackend({
+		runtime,
+		layout: {
+			root: "/tmp/kokoro",
+			modelFile: "kokoro-82m-v1_0.gguf",
+			voicesDir: "/tmp/kokoro/voices",
+			sampleRate: 24000,
+		},
+		defaultVoiceId: KOKORO_DEFAULT_VOICE_ID,
+		phonemizer: fixedPhonemizer(),
+		streamingChunkSamples: 1200, // 50ms — re-chunks the mock output
+	});
+	return { backend, runtime };
+}
+
+describe("KokoroTtsBackend", () => {
+	it("streams PCM chunks and emits a zero-length final tail", async () => {
+		const { backend, runtime } = makeBackend({
+			totalSamples: 9600,
+			chunkCount: 4,
+		});
+		const chunks: TtsPcmChunk[] = [];
+		const result = await backend.synthesizeStream({
+			phrase: makePhrase("hello"),
+			preset: makePreset(KOKORO_DEFAULT_VOICE_ID),
+			cancelSignal: { cancelled: false },
+			onChunk: (c) => {
+				chunks.push({
+					pcm: new Float32Array(c.pcm),
+					sampleRate: c.sampleRate,
+					isFinal: c.isFinal,
+				});
+				return undefined;
+			},
+		});
+		expect(result.cancelled).toBe(false);
+		expect(runtime.calls).toBe(1);
+		expect(chunks.length).toBeGreaterThan(1);
+		const last = chunks[chunks.length - 1];
+		expect(last).toBeDefined();
+		expect(last?.isFinal).toBe(true);
+		expect(last?.pcm.length).toBe(0);
+		const bodyChunks = chunks.slice(0, -1);
+		expect(bodyChunks.every((c) => !c.isFinal)).toBe(true);
+		expect(bodyChunks.every((c) => c.sampleRate === 24000)).toBe(true);
+		// Re-chunking honoured: every body chunk respects the 1200-sample cap.
+		expect(bodyChunks.every((c) => c.pcm.length <= 1200)).toBe(true);
+		const totalBody = bodyChunks.reduce((n, c) => n + c.pcm.length, 0);
+		expect(totalBody).toBe(9600);
+	});
+
+	it("propagates cancelSignal at chunk boundaries", async () => {
+		const { backend } = makeBackend({ totalSamples: 24000, chunkCount: 8 });
+		const cancelSignal = { cancelled: false };
+		let received = 0;
+		const result = await backend.synthesizeStream({
+			phrase: makePhrase("a longer line"),
+			preset: makePreset(KOKORO_DEFAULT_VOICE_ID),
+			cancelSignal,
+			onChunk: (c) => {
+				if (!c.isFinal) {
+					received++;
+					if (received === 2) cancelSignal.cancelled = true;
+				}
+				return undefined;
+			},
+		});
+		expect(result.cancelled).toBe(true);
+		// Final tail is always emitted, even on cancel.
+		// received counts body chunks only; the runtime stops once cancelled.
+		expect(received).toBeLessThanOrEqual(3);
+	});
+
+	it("synthesize() concatenates streamed chunks into one AudioChunk", async () => {
+		const { backend } = makeBackend({ totalSamples: 9600, chunkCount: 4 });
+		const chunk = await backend.synthesize({
+			phrase: makePhrase("hi"),
+			preset: makePreset(KOKORO_DEFAULT_VOICE_ID),
+			cancelSignal: { cancelled: false },
+		});
+		expect(chunk.sampleRate).toBe(24000);
+		expect(chunk.pcm.length).toBe(9600);
+		expect(chunk.phraseId).toBe(1);
+	});
+
+	it("falls back to the default voice when preset.voiceId is unknown", async () => {
+		const { backend } = makeBackend();
+		const chunk = await backend.synthesize({
+			phrase: makePhrase("hi"),
+			preset: makePreset("does_not_exist"),
+			cancelSignal: { cancelled: false },
+		});
+		expect(chunk.pcm.length).toBeGreaterThan(0);
+	});
+
+	it("supportsStreamingTts() returns true (satisfies the streaming seam)", () => {
+		const { backend } = makeBackend();
+		expect(backend.supportsStreamingTts()).toBe(true);
+	});
+});
+
+// Local declaration so the test file does not import the audio sink (unused).
+void (null as unknown as AudioSink);