@elizaos/plugin-local-inference 2.0.0-beta.1 → 2.0.3-beta.3

package/dist/services/voice/speaker-imprint.d.ts

@@ -0,0 +1,83 @@
+import type { VoiceInputSource, VoiceSegment, VoiceSpeaker } from "./types";
+export declare const DEFAULT_VOICE_IMPRINT_MATCH_THRESHOLD = 0.78;
+export interface VoiceImprintProfile {
+    id: string;
+    centroidEmbedding: ArrayLike<number> | null | undefined;
+    embeddingModel?: string | null;
+    sampleCount?: number | null;
+    confidence?: number | null;
+    label?: string | null;
+    displayName?: string | null;
+    entityId?: string | null;
+    sourceKind?: string | null;
+    sourceScopeId?: string | null;
+    metadata?: Record<string, unknown> | null;
+}
+export interface VoiceImprintMatch {
+    profile: VoiceImprintProfile;
+    similarity: number;
+    confidence: number;
+}
+export interface VoiceImprintCentroidUpdate {
+    centroidEmbedding: number[];
+    sampleCount: number;
+    confidence: number;
+}
+export interface VoiceImprintObservationInput {
+    id: string;
+    segmentId?: string;
+    text: string;
+    startMs: number;
+    endMs: number;
+    embedding: ArrayLike<number>;
+    embeddingModel?: string | null;
+    confidence?: number | null;
+    source?: VoiceInputSource;
+    metadata?: Record<string, unknown> | null;
+}
+export interface AttributedVoiceObservation {
+    observation: VoiceImprintObservationInput;
+    match: VoiceImprintMatch | null;
+    speaker: VoiceSpeaker | null;
+    segment: VoiceSegment;
+}
+export interface SpeakerAttributionResult {
+    observations: AttributedVoiceObservation[];
+    segments: VoiceSegment[];
+    primarySpeaker?: VoiceSpeaker;
+    summary: {
+        totalObservations: number;
+        matchedObservations: number;
+        unmatchedObservations: number;
+        meanConfidence: number;
+        meanSimilarity: number | null;
+    };
+}
+export declare function normalizeVoiceEmbedding(embedding: ArrayLike<number>): number[];
+export declare function cosineSimilarity(left: ArrayLike<number>, right: ArrayLike<number>): number;
+export declare function updateVoiceImprintCentroid(args: {
+    centroidEmbedding?: ArrayLike<number> | null;
+    sampleCount?: number | null;
+    confidence?: number | null;
+    observationEmbedding: ArrayLike<number>;
+    observationConfidence?: number | null;
+    maxObservationWeight?: number;
+}): VoiceImprintCentroidUpdate;
+export declare function matchVoiceImprint(args: {
+    embedding: ArrayLike<number>;
+    profiles: readonly VoiceImprintProfile[];
+    embeddingModel?: string | null;
+    threshold?: number;
+}): VoiceImprintMatch | null;
+export declare function voiceSpeakerFromImprintMatch(args: {
+    match: VoiceImprintMatch;
+    source?: VoiceInputSource;
+    observationId?: string;
+}): VoiceSpeaker;
+export declare function attributeVoiceImprintObservations(args: {
+    observations: readonly VoiceImprintObservationInput[];
+    profiles: readonly VoiceImprintProfile[];
+    threshold?: number;
+    defaultSource?: VoiceInputSource;
+}): SpeakerAttributionResult;
+//# sourceMappingURL=speaker-imprint.d.ts.map

package/dist/services/voice/speaker-imprint.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"speaker-imprint.d.ts","sourceRoot":"","sources":["../../../src/services/voice/speaker-imprint.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,gBAAgB,EAAE,YAAY,EAAE,YAAY,EAAE,MAAM,SAAS,CAAC;AAE5E,eAAO,MAAM,qCAAqC,OAAO,CAAC;AAE1D,MAAM,WAAW,mBAAmB;IACnC,EAAE,EAAE,MAAM,CAAC;IACX,iBAAiB,EAAE,SAAS,CAAC,MAAM,CAAC,GAAG,IAAI,GAAG,SAAS,CAAC;IACxD,cAAc,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAC/B,WAAW,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAC5B,UAAU,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAC3B,KAAK,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IACtB,WAAW,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAC5B,QAAQ,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IACzB,UAAU,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAC3B,aAAa,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAC9B,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,IAAI,CAAC;CAC1C;AAED,MAAM,WAAW,iBAAiB;IACjC,OAAO,EAAE,mBAAmB,CAAC;IAC7B,UAAU,EAAE,MAAM,CAAC;IACnB,UAAU,EAAE,MAAM,CAAC;CACnB;AAED,MAAM,WAAW,0BAA0B;IAC1C,iBAAiB,EAAE,MAAM,EAAE,CAAC;IAC5B,WAAW,EAAE,MAAM,CAAC;IACpB,UAAU,EAAE,MAAM,CAAC;CACnB;AAED,MAAM,WAAW,4BAA4B;IAC5C,EAAE,EAAE,MAAM,CAAC;IACX,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,IAAI,EAAE,MAAM,CAAC;IACb,OAAO,EAAE,MAAM,CAAC;IAChB,KAAK,EAAE,MAAM,CAAC;IACd,SAAS,EAAE,SAAS,CAAC,MAAM,CAAC,CAAC;IAC7B,cAAc,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAC/B,UAAU,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAC3B,MAAM,CAAC,EAAE,gBAAgB,CAAC;IAC1B,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,IAAI,CAAC;CAC1C;AAED,MAAM,WAAW,0BAA0B;IAC1C,WAAW,EAAE,4BAA4B,CAAC;IAC1C,KAAK,EAAE,iBAAiB,GAAG,IAAI,CAAC;IAChC,OAAO,EAAE,YAAY,GAAG,IAAI,CAAC;IAC7B,OAAO,EAAE,YAAY,CAAC;CACtB;AAED,MAAM,WAAW,wBAAwB;IACxC,YAAY,EAAE,0BAA0B,EAAE,CAAC;IAC3C,QAAQ,EAAE,YAAY,EAAE,CAAC;IACzB,cAAc,CAAC,EAAE,YAAY,CAAC;IAC9B,OAAO,EAAE;QACR,iBAAiB,EAAE,MAAM,CAAC;QAC1B,mBAAmB,EAAE,MAAM,CAAC;QAC5B,qBAAqB,EAAE,MAAM,CAAC;QAC9B,cAAc,EAAE,MAAM,CAAC;QACvB,cAAc,EAAE,MAAM,GAAG,IAAI,CAAC;KAC9B,CAAC;CACF;AAOD,wBAAgB,uBAAuB,CACtC,SAAS,EAAE,SAAS,CAAC,MAAM,CAAC,GAC1B,MAAM,EAAE,CAiBV;AAED,wBAAgB,gBAAgB,CAC/B,IAAI,EAAE,SAAS,CAAC,MAAM,CAAC,EACvB,KAAK,EAAE,SAAS,CAAC,MAAM,CAAC,GACtB,MAAM,CAWR;AAED,wBAAgB,0BAA0B,CAAC,IAAI,EAAE;IAChD,iBAAiB,CAAC,EAAE,SAAS,CAAC,MAAM,CAAC,GAAG,IAAI,CAAC;IAC7C,WAAW,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAC5B,UAAU,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAC3B,oBAAoB,EAAE,SAAS,CAAC,MAAM,CAAC,CAAC;IACxC,qBAAqB,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IACtC,oBAAoB,CAAC,EAAE,MAAM,CAAC;CAC9B,GAAG,0BAA0B,CA+B7B;AAED,wBAAgB,iBAAiB,CAAC,IAAI,EAAE;IACvC,SAAS,EAAE,SAAS,CAAC,MAAM,CAAC,CAAC;IAC7B,QAAQ,EAAE,SAAS,mBAAmB,EAAE,CAAC;IACzC,cAAc,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAC/B,SAAS,CAAC,EAAE,MAAM,CAAC;CACnB,GAAG,iBAAiB,GAAG,IAAI,CA+B3B;AAED,wBAAgB,4BAA4B,CAAC,IAAI,EAAE;IAClD,KAAK,EAAE,iBAAiB,CAAC;IACzB,MAAM,CAAC,EAAE,gBAAgB,CAAC;IAC1B,aAAa,CAAC,EAAE,MAAM,CAAC;CACvB,GAAG,YAAY,CAqBf;AAED,wBAAgB,iCAAiC,CAAC,IAAI,EAAE;IACvD,YAAY,EAAE,SAAS,4BAA4B,EAAE,CAAC;IACtD,QAAQ,EAAE,SAAS,mBAAmB,EAAE,CAAC;IACzC,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,aAAa,CAAC,EAAE,gBAAgB,CAAC;CACjC,GAAG,wBAAwB,CA2E3B"}

package/dist/services/voice/speaker-preset-cache.d.ts

@@ -0,0 +1,77 @@
+import type { SpeakerPreset } from "./types";
+import { type VoicePresetSeedPhrase } from "./voice-preset-format";
+export interface PresetBundlePaths {
+    bundleRoot: string;
+    /** Relative path to the *default* voice preset inside the bundle. */
+    cacheRelPath?: string;
+}
+export interface LoadedPresetBundle {
+    preset: SpeakerPreset;
+    /** Phrase-cache seed entries parsed alongside the embedding. The engine
+     *  bridge feeds these into a `PhraseCache.seed(...)` call before the
+     *  scheduler is constructed. */
+    phrases: ReadonlyArray<VoicePresetSeedPhrase>;
+}
+export interface SpeakerPresetCacheOptions {
+    /**
+     * Maximum number of distinct voices held in RAM at once. Speaker presets
+     * are KB-scale (an embedding vector + the seed-phrase PCM), so the default
+     * is generous; the bound exists so a connector that switches voices per
+     * speaker (e.g. a Discord room with many users) does not grow unbounded.
+     * On insert past the bound the least-recently-used voice is evicted.
+     *
+     * Defaults to 8 hot voices (R6 §1: "LRU 8 hot, mmap-backed").
+     */
+    maxVoices?: number;
+}
+export declare const DEFAULT_VOICE_ID = "default";
+export declare const DEFAULT_VOICE_PRESET_REL_PATH: string;
+/**
+ * Resolve the on-disk path of a voice preset inside a bundle. The default
+ * voice lives at `cache/voice-preset-default.bin`; additional voices ship as
+ * `cache/voice-preset-<voiceId>.bin`. Throws on a `voiceId` that is not a safe
+ * single path segment (no `/`, no `..`).
+ */
+export declare function voicePresetPath(bundleRoot: string, voiceId: string): string;
+/**
+ * LRU cache of parsed speaker presets keyed by `voiceId`. Holds the speaker
+ * embedding, the raw preset bytes (for FFI handoff), and the phrase-cache seed
+ * list parsed from the preset file. Multi-voice: `load(bundleRoot, voiceId)`
+ * reads `cache/voice-preset-<voiceId>.bin` from the bundle on a miss.
+ *
+ * v2 preset fields (`refAudioTokens`, `refText`, `instruct`) are surfaced
+ * on the `SpeakerPreset` shape so the FFI bridge can pass them through to
+ * `ov_tts_params` without going through the legacy "instruct == voiceId"
+ * misreading.
+ */
+export declare class SpeakerPresetCache {
+    private readonly entries;
+    private readonly maxVoices;
+    constructor(opts?: SpeakerPresetCacheOptions);
+    /**
+     * Load the bundle's default voice preset (`cache/voice-preset-default.bin`,
+     * or `paths.cacheRelPath` if overridden) and return both the speaker
+     * embedding and the phrase-cache seed entries. Cached for subsequent
+     * `get("default")` lookups (and marked most-recently-used).
+     */
+    loadFromBundle(paths: PresetBundlePaths, voiceId?: string): LoadedPresetBundle;
+    /**
+     * Load an arbitrary voice by id from a bundle root, reading
+     * `cache/voice-preset-<voiceId>.bin` (or `cache/voice-preset-default.bin`
+     * for `"default"`). Returns the cached entry on a hit (marked MRU).
+     */
+    load(bundleRoot: string, voiceId: string): LoadedPresetBundle;
+    /** True if `voiceId` is currently resident in the cache. */
+    has(voiceId: string): boolean;
+    put(preset: SpeakerPreset): void;
+    get(voiceId: string): SpeakerPreset | undefined;
+    /** Seed entries previously loaded for a voice, if any (does not touch LRU order). */
+    getSeed(voiceId: string): ReadonlyArray<VoicePresetSeedPhrase>;
+    /** Number of voices currently resident. */
+    size(): number;
+    /** Drop every cached preset. */
+    clear(): void;
+    private loadFile;
+    private evictOverflow;
+}
+//# sourceMappingURL=speaker-preset-cache.d.ts.map

package/dist/services/voice/speaker-preset-cache.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"speaker-preset-cache.d.ts","sourceRoot":"","sources":["../../../src/services/voice/speaker-preset-cache.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,SAAS,CAAC;AAC7C,OAAO,EAGN,KAAK,qBAAqB,EAC1B,MAAM,uBAAuB,CAAC;AAE/B,MAAM,WAAW,iBAAiB;IACjC,UAAU,EAAE,MAAM,CAAC;IACnB,qEAAqE;IACrE,YAAY,CAAC,EAAE,MAAM,CAAC;CACtB;AAED,MAAM,WAAW,kBAAkB;IAClC,MAAM,EAAE,aAAa,CAAC;IACtB;;oCAEgC;IAChC,OAAO,EAAE,aAAa,CAAC,qBAAqB,CAAC,CAAC;CAC9C;AAED,MAAM,WAAW,yBAAyB;IACzC;;;;;;;;OAQG;IACH,SAAS,CAAC,EAAE,MAAM,CAAC;CACnB;AAED,eAAO,MAAM,gBAAgB,YAAY,CAAC;AAG1C,eAAO,MAAM,6BAA6B,QAGzC,CAAC;AAEF;;;;;GAKG;AACH,wBAAgB,eAAe,CAAC,UAAU,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,GAAG,MAAM,CAU3E;AAOD;;;;;;;;;;GAUG;AACH,qBAAa,kBAAkB;IAG9B,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAiC;IACzD,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAS;gBAEvB,IAAI,GAAE,yBAA8B;IAOhD;;;;;OAKG;IACH,cAAc,CACb,KAAK,EAAE,iBAAiB,EACxB,OAAO,SAAmB,GACxB,kBAAkB;IASrB;;;;OAIG;IACH,IAAI,CAAC,UAAU,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,GAAG,kBAAkB;IAI7D,4DAA4D;IAC5D,GAAG,CAAC,OAAO,EAAE,MAAM,GAAG,OAAO;IAI7B,GAAG,CAAC,MAAM,EAAE,aAAa,GAAG,IAAI;IAUhC,GAAG,CAAC,OAAO,EAAE,MAAM,GAAG,aAAa,GAAG,SAAS;IAQ/C,qFAAqF;IACrF,OAAO,CAAC,OAAO,EAAE,MAAM,GAAG,aAAa,CAAC,qBAAqB,CAAC;IAI9D,2CAA2C;IAC3C,IAAI,IAAI,MAAM;IAId,gCAAgC;IAChC,KAAK,IAAI,IAAI;IAIb,OAAO,CAAC,QAAQ;IA+BhB,OAAO,CAAC,aAAa;CAOrB"}

package/dist/services/voice/streaming-asr/streaming-pipeline-adapter.d.ts

@@ -0,0 +1,160 @@
+/**
+ * Streaming-ASR pipeline adapter (item A1 / W7).
+ *
+ * The base `VoicePipeline` in `voice/pipeline.ts` drives a `StreamingTranscriber`
+ * as a batch: it pushes the WHOLE VAD-gated utterance buffer in a single
+ * `feed()` call, awaits `flush()`, and only then splits the final transcript
+ * into tokens for the drafter/verifier loop. That works for the fused
+ * batch decoder, but it leaves the biggest
+ * H2 UX seam (incremental partials → planner / barge-in word-confirm /
+ * speculative-on-pause) untapped when the fused build is in streaming mode.
+ *
+ * Rather than rewrite `pipeline.ts` (the Phase 0/1 agent owns it), this
+ * module is a small WRAPPER the engine bridge can use to deliver PCM
+ * chunks to a `StreamingTranscriber` as they arrive from the mic / VAD,
+ * surface every `partial` event to the turn controller via `onPartial`,
+ * and only finalize on `speech-end`. Behind a flag — when the fused
+ * library advertises `asrStreamSupported() === false` or the flag is off
+ * the caller keeps using `VoicePipeline.transcribeAll` exactly as today.
+ *
+ * Integration point (documented for the Phase 0/1 agent so they can wire
+ * it without merge friction):
+ *
+ *   1. `EngineVoiceBridge` decides whether streaming is available via
+ *      `pickStreamingMode({ ffi, asrBundlePresent, flag })`. When the
+ *      mode is `"streaming"`, the bridge constructs `StreamingAsrFeeder`
+ *      once per turn (passing the same transcriber that would have been
+ *      handed to `VoicePipeline`) and routes mic PCM frames through
+ *      `feeder.feedFrame(frame)` instead of buffering them.
+ *   2. The feeder forwards every transcriber `partial` event to
+ *      `onPartial(update)`. When VAD reports `speech-end` the caller
+ *      calls `await feeder.finalize()`; the returned `TranscriptUpdate`
+ *      is the final and is used to seed the drafter/verifier loop exactly
+ *      as before (`splitTranscriptToTokens(final.partial, 0, final.tokens)`).
+ *   3. The batch path (`VoicePipeline.transcribeAll`) is unchanged for
+ *      every other adapter — there is no fork in `pipeline.ts` itself.
+ *
+ * This file is intentionally small and side-effect-free so it can land
+ * during the merge window without touching files other agents own.
+ */
+import type { PcmFrame, StreamingTranscriber, TextToken, TranscriptUpdate } from "../types";
+/**
+ * LocalAgreement-n word-level partial stabilizer.
+ *
+ * Usage:
+ *   const buf = new LocalAgreementBuffer();
+ *   const stable = buf.stable(["hello", "there", "world"]);
+ *   // → [] on first call (need n=2 consecutive identical prefix)
+ *   const stable2 = buf.stable(["hello", "there", "how"]);
+ *   // → ["hello", "there"] (matched across two consecutive hypotheses)
+ */
+export declare class LocalAgreementBuffer {
+    private readonly n;
+    /** Rolling window of the last `n` hypotheses, oldest first. */
+    private window;
+    /** Monotonically growing committed word list. */
+    private committed;
+    constructor(n?: number);
+    /**
+     * Feed the latest word-level hypothesis. Returns the stable committed
+     * prefix — the longest leading word sequence that has appeared
+     * identically in `n` consecutive calls. Monotonically non-decreasing.
+     *
+     * A rolling window of the last `n` hypotheses is maintained. Once the
+     * window is full, the agreed prefix is the intersection across all `n`
+     * entries — word i is in the agreed prefix only if it is identical in
+     * every hypothesis in the window.
+     */
+    stable(current: string[]): string[];
+    /** Clear all state. Call at utterance boundaries. */
+    reset(): void;
+    /** The current committed stable word list (read-only view). */
+    getCommitted(): string[];
+}
+/** Available transcription drive modes. */
+export type StreamingPipelineMode = "streaming" | "batch";
+export interface PickStreamingModeArgs {
+    /** True only when the loaded fused library advertises a working streaming decoder. */
+    ffiSupportsStreaming: boolean;
+    /** True only when the bundled ASR model is present on disk. */
+    asrBundlePresent: boolean;
+    /**
+     * Feature flag — defaults to FALSE so the streaming path stays opt-in
+     * until the Phase 0/1 partial-stabilizer wiring lands. Once that lands
+     * the engine bridge flips this default to true.
+     */
+    enableStreaming: boolean;
+}
+/**
+ * Choose the transcription drive mode. Streaming is selected only when:
+ *   - the loaded fused library advertises a working streaming decoder
+ *     (`asr_stream_supported() === 1`), AND
+ *   - the bundled ASR model is present, AND
+ *   - the engine bridge has opted in via `enableStreaming`.
+ *
+ * Any other combination falls back to the existing batch path
+ * (`VoicePipeline.transcribeAll`).
+ */
+export declare function pickStreamingMode(args: PickStreamingModeArgs): StreamingPipelineMode;
+export interface StreamingAsrFeederEvents {
+    /**
+     * Called for every transcriber `partial` event the feeder observes
+     * BEFORE the segment is finalized. Includes the running `partial`
+     * text, `isFinal: false`, and (when the fused build supplied them)
+     * the shared text-model token ids.
+     */
+    onPartial?(update: TranscriptUpdate): void;
+    /**
+     * Called the first time ≥1 real word is recognized in the segment.
+     * Wired into the turn controller's word-confirm gate so the agent
+     * only barge-in-cancels on real speech, not blips.
+     */
+    onWords?(words: ReadonlyArray<string>): void;
+    /**
+     * Called once, after `finalize()` returns, with the final transcript
+     * split into contiguous text tokens (`splitTranscriptToTokens`). The
+     * batch path delivers the same shape via `transcribeAll`, so the
+     * downstream drafter/verifier loop sees an identical signal.
+     */
+    onFinalTokens?(tokens: ReadonlyArray<TextToken>, final: TranscriptUpdate): void;
+}
+/**
+ * Drives a `StreamingTranscriber` chunk-by-chunk on behalf of the engine
+ * bridge / turn controller. One instance per active speech segment;
+ * `finalize()` returns the final transcript and the feeder is disposed.
+ *
+ * Construction takes a `StreamingTranscriber` (already constructed via
+ * `createStreamingTranscriber` with the same options used for batch).
+ * The feeder does NOT own the transcriber's lifecycle — disposal still
+ * runs through the engine bridge so the same path is used when the
+ * batch fallback is taken.
+ */
+export declare class StreamingAsrFeeder {
+    private readonly transcriber;
+    private readonly events;
+    private latestPartial;
+    private finalized;
+    private unsubscribe;
+    constructor(args: {
+        transcriber: StreamingTranscriber;
+        events?: StreamingAsrFeederEvents;
+    });
+    /**
+     * Feed one PCM frame as it arrives from the mic / connector. Drops
+     * frames received after `finalize()` (the segment is over).
+     */
+    feedFrame(frame: PcmFrame): void;
+    /**
+     * Force-finalize on `speech-end`. Resolves with the final transcript
+     * and emits `onFinalTokens` so the caller can seed the drafter /
+     * verifier loop without re-running the surface split itself.
+     *
+     * Calling `finalize()` twice is a hard error — the segment is over.
+     */
+    finalize(): Promise<TranscriptUpdate>;
+    /** The most recent `partial` snapshot observed, or `null` until the first decode lands. */
+    getLatestPartial(): TranscriptUpdate | null;
+    /** Detach the transcriber subscription. Does NOT dispose the transcriber itself. */
+    dispose(): void;
+}
+//# sourceMappingURL=streaming-pipeline-adapter.d.ts.map

package/dist/services/voice/streaming-asr/streaming-pipeline-adapter.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"streaming-pipeline-adapter.d.ts","sourceRoot":"","sources":["../../../../src/services/voice/streaming-asr/streaming-pipeline-adapter.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAsCG;AAGH,OAAO,KAAK,EACX,QAAQ,EACR,oBAAoB,EACpB,SAAS,EACT,gBAAgB,EAChB,MAAM,UAAU,CAAC;AAkBlB;;;;;;;;;GASG;AACH,qBAAa,oBAAoB;IAChC,OAAO,CAAC,QAAQ,CAAC,CAAC,CAAS;IAC3B,+DAA+D;IAC/D,OAAO,CAAC,MAAM,CAAkB;IAChC,iDAAiD;IACjD,OAAO,CAAC,SAAS,CAAgB;gBAErB,CAAC,SAAI;IASjB;;;;;;;;;OASG;IACH,MAAM,CAAC,OAAO,EAAE,MAAM,EAAE,GAAG,MAAM,EAAE;IAqCnC,qDAAqD;IACrD,KAAK,IAAI,IAAI;IAKb,+DAA+D;IAC/D,YAAY,IAAI,MAAM,EAAE;CAGxB;AAED,2CAA2C;AAC3C,MAAM,MAAM,qBAAqB,GAAG,WAAW,GAAG,OAAO,CAAC;AAE1D,MAAM,WAAW,qBAAqB;IACrC,sFAAsF;IACtF,oBAAoB,EAAE,OAAO,CAAC;IAC9B,+DAA+D;IAC/D,gBAAgB,EAAE,OAAO,CAAC;IAC1B;;;;OAIG;IACH,eAAe,EAAE,OAAO,CAAC;CACzB;AAED;;;;;;;;;GASG;AACH,wBAAgB,iBAAiB,CAChC,IAAI,EAAE,qBAAqB,GACzB,qBAAqB,CAKvB;AAED,MAAM,WAAW,wBAAwB;IACxC;;;;;OAKG;IACH,SAAS,CAAC,CAAC,MAAM,EAAE,gBAAgB,GAAG,IAAI,CAAC;IAC3C;;;;OAIG;IACH,OAAO,CAAC,CAAC,KAAK,EAAE,aAAa,CAAC,MAAM,CAAC,GAAG,IAAI,CAAC;IAC7C;;;;;OAKG;IACH,aAAa,CAAC,CACb,MAAM,EAAE,aAAa,CAAC,SAAS,CAAC,EAChC,KAAK,EAAE,gBAAgB,GACrB,IAAI,CAAC;CACR;AAED;;;;;;;;;;GAUG;AACH,qBAAa,kBAAkB;IAC9B,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAuB;IACnD,OAAO,CAAC,QAAQ,CAAC,MAAM,CAA2B;IAClD,OAAO,CAAC,aAAa,CAAiC;IACtD,OAAO,CAAC,SAAS,CAAS;IAC1B,OAAO,CAAC,WAAW,CAA6B;gBAEpC,IAAI,EAAE;QACjB,WAAW,EAAE,oBAAoB,CAAC;QAClC,MAAM,CAAC,EAAE,wBAAwB,CAAC;KAClC;IAqBD;;;OAGG;IACH,SAAS,CAAC,KAAK,EAAE,QAAQ,GAAG,IAAI;IAKhC;;;;;;OAMG;IACG,QAAQ,IAAI,OAAO,CAAC,gBAAgB,CAAC;IAa3C,2FAA2F;IAC3F,gBAAgB,IAAI,gBAAgB,GAAG,IAAI;IAI3C,oFAAoF;IACpF,OAAO,IAAI,IAAI;CAIf"}

package/dist/services/voice/system-audio-sink.d.ts

@@ -0,0 +1,73 @@
+/**
+ * System audio sinks for the interactive voice harness.
+ *
+ * The voice scheduler writes synthesized PCM (`Float32Array` mono in
+ * [-1, 1] at the bridge sample rate) into an {@link AudioSink}. For tests
+ * and headless runs `InMemoryAudioSink` (in `./ring-buffer`) captures the
+ * samples; for an interactive session the harness needs the audio to
+ * actually come out of the speakers.
+ *
+ * `SystemAudioSink` shells out to a long-lived player that reads raw 16-bit
+ * signed-LE PCM on stdin. Per-platform selection (priority order):
+ *   - Linux:   `aplay` (alsa-utils), else `paplay` (PulseAudio), else
+ *              `play`/`sox` (sox), else `ffplay` (ffmpeg).
+ *   - macOS:   `play`/`sox` (sox), else `ffplay` (ffmpeg). `afplay` needs a
+ *              file (no stdin) so it cannot be used for streaming.
+ *   - Windows: `ffplay` (ffmpeg), else `play`/`sox` if installed. PowerShell's
+ *              `Media.SoundPlayer` also needs a file, not a stream, so it's not
+ *              used here — the renderer's `AudioContext` path (feeding nothing
+ *              here; the renderer plays directly) is the no-ffmpeg route.
+ * If no player is on `PATH`, `available()` returns false and the harness
+ * falls back to `WavFileAudioSink` (writes a rolling WAV) — never silence.
+ *
+ * `WavFileAudioSink` accumulates everything written and serializes a
+ * single mono PCM16 WAV on `finalize()` — used by `--no-audio` and as the
+ * no-player fallback.
+ */
+import type { AudioSink } from "./types";
+/**
+ * Exported view of {@link resolvePlayer} for the cross-platform preflight
+ * (`voice:interactive --platform-report`). Returns the bare program name
+ * (no args) the host would stream synthesized audio to, or `null`.
+ */
+export declare function resolveSystemPlayerName(sampleRate?: number): string | null;
+export interface SystemAudioSinkOptions {
+    sampleRate: number;
+}
+export declare class SystemAudioSink implements AudioSink {
+    private readonly sampleRate;
+    private readonly playerSpec;
+    private proc;
+    private buffered;
+    constructor(opts: SystemAudioSinkOptions);
+    available(): boolean;
+    player(): string;
+    private ensureProc;
+    write(pcm: Float32Array, _sampleRate: number): void;
+    drain(): void;
+    bufferedSamples(): number;
+    /** Flush + close the player. Idempotent. */
+    dispose(): Promise<void>;
+}
+export interface WavFileAudioSinkOptions {
+    sampleRate: number;
+    filePath: string;
+}
+/**
+ * Accumulates all written PCM and serializes a single mono PCM16 WAV on
+ * {@link finalize}. Used by `--no-audio` and as the no-player fallback so
+ * a headless run still produces an inspectable artifact (never silence).
+ */
+export declare class WavFileAudioSink implements AudioSink {
+    private readonly sampleRate;
+    private readonly filePath;
+    private readonly chunks;
+    private buffered;
+    constructor(opts: WavFileAudioSinkOptions);
+    write(pcm: Float32Array, _sampleRate: number): void;
+    drain(): void;
+    bufferedSamples(): number;
+    totalSamples(): number;
+    finalize(): Promise<string>;
+}
+//# sourceMappingURL=system-audio-sink.d.ts.map

package/dist/services/voice/system-audio-sink.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"system-audio-sink.d.ts","sourceRoot":"","sources":["../../../src/services/voice/system-audio-sink.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;GAyBG;AAQH,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,SAAS,CAAC;AAoIzC;;;;GAIG;AACH,wBAAgB,uBAAuB,CAAC,UAAU,SAAS,GAAG,MAAM,GAAG,IAAI,CAG1E;AAED,MAAM,WAAW,sBAAsB;IACtC,UAAU,EAAE,MAAM,CAAC;CACnB;AAWD,qBAAa,eAAgB,YAAW,SAAS;IAChD,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAS;IACpC,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAyC;IACpE,OAAO,CAAC,IAAI,CAAiC;IAC7C,OAAO,CAAC,QAAQ,CAAK;gBAET,IAAI,EAAE,sBAAsB;IAKxC,SAAS,IAAI,OAAO;IAIpB,MAAM,IAAI,MAAM;IAIhB,OAAO,CAAC,UAAU;IAmBlB,KAAK,CAAC,GAAG,EAAE,YAAY,EAAE,WAAW,EAAE,MAAM,GAAG,IAAI;IAUnD,KAAK,IAAI,IAAI;IAmBb,eAAe,IAAI,MAAM;IAIzB,4CAA4C;IACtC,OAAO,IAAI,OAAO,CAAC,IAAI,CAAC;CA6B9B;AAED,MAAM,WAAW,uBAAuB;IACvC,UAAU,EAAE,MAAM,CAAC;IACnB,QAAQ,EAAE,MAAM,CAAC;CACjB;AAED;;;;GAIG;AACH,qBAAa,gBAAiB,YAAW,SAAS;IACjD,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAS;IACpC,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAS;IAClC,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAsB;IAC7C,OAAO,CAAC,QAAQ,CAAK;gBAET,IAAI,EAAE,uBAAuB;IAKzC,KAAK,CAAC,GAAG,EAAE,YAAY,EAAE,WAAW,EAAE,MAAM,GAAG,IAAI;IAKnD,KAAK,IAAI,IAAI;IAMb,eAAe,IAAI,MAAM;IAIzB,YAAY,IAAI,MAAM;IAMhB,QAAQ,IAAI,OAAO,CAAC,MAAM,CAAC;CAgCjC"}

package/dist/services/voice/transcriber.d.ts

@@ -0,0 +1,244 @@
+/**
+ * Streaming ASR adapters for the local voice pipeline.
+ *
+ * Implements the `StreamingTranscriber` contract from `voice/types.ts`:
+ * PCM frames in (`feed`), running partial-transcript events out, `flush()`
+ * to force-finalize on `speech-end`. Two adapters, resolved in priority
+ * order by `createStreamingTranscriber()` — both backed by the single fused
+ * `libelizainference` build (the SOLE on-device ASR runtime):
+ *
+ *   1. `FfiStreamingTranscriber` — the FINAL path. Drives the fused
+ *      `libelizainference` streaming ASR ABI (`eliza_inference_asr_stream_*`,
+ *      ABI v2 — declared in `packages/app-core/scripts/omnivoice-fuse/ffi.h`,
+ *      bound in `voice/ffi-bindings.ts`). The C side is W7's job; until the
+ *      real fused build advertises streaming ASR the binding's `mmap`/`asr`
+ *      calls return `ELIZA_ERR_NOT_IMPLEMENTED`, which surfaces as a thrown
+ *      error here. Selected only when `ffi.asrStreamSupported()` is true.
+ *
+ *   2. `FfiBatchTranscriber` — the contract-clean INTERIM path. Runs the
+ *      fused build's *batch* decoder (`eliza_inference_asr_transcribe`, ABI
+ *      v1) over a sliding window with overlap, so each call covers ≤ ~6–7 s
+ *      of audio — incremental, not "buffer the whole utterance, one giant
+ *      decode". It lives inside the single shipped llama.cpp/GGML build and
+ *      emits Gemma text-vocabulary tokens, so it does not vendor a second
+ *      ggml or introduce a tokenizer-family mismatch.
+ *      Selected whenever a `libelizainference` handle + bundled ASR model are
+ *      present (which is always true when the fused build is loaded).
+ *
+ * If no fused ASR backend can be resolved, `createStreamingTranscriber()`
+ * throws `AsrUnavailableError` — a real failure, never a silent
+ * empty-transcript degrade and never a fall back to a second ASR runtime
+ * (AGENTS.md §3 + §9).
+ */
+import type { ElizaInferenceContextHandle, ElizaInferenceFfi } from "./ffi-bindings";
+import type { PcmFrame, StreamingTranscriber, TranscriberEventListener, TranscriptUpdate, VadEventSource, VoiceInputSource, VoiceSpeaker, VoiceTurnMetadata } from "./types";
+/** The local voice runtime resamples mic input to 16 kHz mono for ASR. */
+export declare const ASR_SAMPLE_RATE = 16000;
+/**
+ * Raised when no ASR backend can be resolved. Distinct error class so the
+ * caller (engine, `TRANSCRIPTION` model handler) can surface "ASR is not
+ * installed" with an actionable message rather than treating an empty
+ * string as a successful transcription.
+ */
+export declare class AsrUnavailableError extends Error {
+    constructor(message: string);
+}
+export type AsrBackendPreference = "auto" | "fused" | "ffi-batch";
+export declare function normalizeAsrBackendPreference(value: string | null | undefined): AsrBackendPreference | null;
+export declare function readAsrBackendPreferenceFromEnv(env?: NodeJS.ProcessEnv): AsrBackendPreference | null;
+/**
+ * Linear-interpolation resample of mono fp32 PCM. Used to coerce mic
+ * frames (commonly 16 / 24 / 48 kHz) to the ASR rate. Not a polyphase
+ * filter — adequate for speech ASR; the fused build does its own
+ * resampling so this is interim-batch only.
+ */
+export declare function resampleLinear(pcm: Float32Array, fromRate: number, toRate: number): Float32Array;
+/**
+ * Base implementing the boilerplate every adapter shares: listener
+ * fan-out, the `words`-once-per-segment latch, and (optional) VAD-event
+ * gating. Subclasses implement `onFrame` / `onFlush` / `onDispose` and
+ * call `emitPartial` / `emitFinal`.
+ */
+export declare abstract class BaseStreamingTranscriber implements StreamingTranscriber {
+    private readonly listeners;
+    private metadata;
+    /** True between `speech-start`/first-frame and the next `flush()`. */
+    protected segmentOpen: boolean;
+    /** Latched once `words` is emitted for the current segment. */
+    private wordsEmitted;
+    /** When set, frames are only forwarded while the VAD is in an active speech window. */
+    private vadActive;
+    private readonly vadPrerollFrames;
+    private vadUnsub;
+    private disposed;
+    constructor(vad?: VadEventSource, metadata?: TranscriptMetadataDefaults);
+    on(listener: TranscriberEventListener): () => void;
+    /**
+     * Update the metadata defaults that `withMetadata()` merges into every
+     * partial / final emission. The voice pipeline calls this once the
+     * async speaker-ID / diarizer lookup resolves, so the speaker /
+     * segments are attached to the rest of the turn without buffering all
+     * partials for the lookup.
+     */
+    setMetadataDefaults(metadata: TranscriptMetadataDefaults): void;
+    feed(frame: PcmFrame): void;
+    flush(): Promise<TranscriptUpdate>;
+    dispose(): void;
+    /** Subclass hook: a (VAD-gated) PCM frame for the current speech segment. */
+    protected abstract onFrame(frame: PcmFrame): void;
+    /** Subclass hook: drain buffered audio, run a final decode, return the final transcript. */
+    protected abstract onFlush(): Promise<TranscriptUpdate>;
+    /** Subclass hook: release native resources. */
+    protected abstract onDispose(): void;
+    private rememberVadPreroll;
+    private drainVadPreroll;
+    /** Emit a running-partial event and (the first time it has words) a `words` event. */
+    protected emitPartial(update: TranscriptUpdate): void;
+    private withMetadata;
+    private emit;
+    private onVadEvent;
+}
+export interface TranscriptMetadataDefaults {
+    source?: VoiceInputSource;
+    speaker?: VoiceSpeaker;
+    turn?: VoiceTurnMetadata;
+}
+/**
+ * True when the loaded fused library has a working streaming ASR decoder
+ * (not just the v2 symbols — an ABI-only build exports them but `asrStreamSupported`
+ * returns false). This is the gate `createStreamingTranscriber` uses to
+ * pick the fused path over the fused-batch interim adapter.
+ */
+export declare function ffiSupportsStreamingAsr(ffi: ElizaInferenceFfi | null | undefined): boolean;
+/**
+ * `StreamingTranscriber` over the fused `libelizainference` streaming ASR
+ * ABI. Each `feed()` forwards the (resampled) PCM into `asrStreamFeed`;
+ * after a feed it reads the running partial via `asrStreamPartial`.
+ * `flush()` calls `asrStreamFinish` then re-opens a fresh stream for the
+ * next segment. Token ids, when the library returns them, are surfaced in
+ * `TranscriptUpdate.tokens` — the fused build shares the text vocabulary
+ * (AGENTS.md §1) so they feed STT-finish token injection directly.
+ *
+ * The C side is owned by W7; until the fused build implements these
+ * symbols every call throws (the binding maps `ELIZA_ERR_NOT_IMPLEMENTED`
+ * to a `VoiceLifecycleError`). That is intentional — no fake transcripts.
+ */
+export declare class FfiStreamingTranscriber extends BaseStreamingTranscriber {
+    private readonly ffi;
+    private readonly getContext;
+    /** Token count to ask the library for per partial; 0 = don't request tokens. */
+    private readonly maxTokens;
+    private stream;
+    constructor(args: {
+        ffi: ElizaInferenceFfi;
+        getContext: () => ElizaInferenceContextHandle;
+        vad?: VadEventSource;
+        metadata?: TranscriptMetadataDefaults;
+        source?: VoiceInputSource;
+        /** Cap on token ids read back per transcript snapshot. Default 256. */
+        maxTokens?: number;
+    });
+    private ensureStream;
+    protected onFrame(frame: PcmFrame): void;
+    protected onFlush(): Promise<TranscriptUpdate>;
+    protected onDispose(): void;
+}
+export interface FfiBatchTranscriberOptions {
+    ffi: ElizaInferenceFfi;
+    getContext: () => ElizaInferenceContextHandle;
+    vad?: VadEventSource;
+    metadata?: TranscriptMetadataDefaults;
+    source?: VoiceInputSource;
+    /** Sliding-window length, seconds. Each batch decode covers ≤ this + overlap. Default 6.0. */
+    windowSeconds?: number;
+    /** Trailing overlap kept when committing a prefix chunk, seconds. Default 1.0. */
+    overlapSeconds?: number;
+    /** Minimum new audio (seconds) accumulated before the next decode pass. Default 1.2. */
+    stepSeconds?: number;
+}
+/**
+ * Interim streaming-ASR adapter over the fused `libelizainference` **batch**
+ * decoder (`eliza_inference_asr_transcribe`, ABI v1). The fused build's true
+ * streaming decoder (`eliza_inference_asr_stream_*`, ABI v2) reports unsupported
+ * until its runtime lands; this adapter is the contract-clean interim — it runs
+ * inside the one shipped llama.cpp/GGML build and emits
+ * Gemma token-vocab text, so no second ggml is vendored and no
+ * tokenizer-family mismatch is introduced.
+ *
+ * It runs a *windowed re-transcription with overlap* strategy: a prefix older
+ * than `windowSeconds` is committed (decoded once, in window-sized chunks
+ * with `overlapSeconds` carry-over) and only the tail window is re-decoded
+ * each step. So each `asr_transcribe` call is bounded by `windowSeconds +
+ * overlap` of audio (≈6–7 s) — incremental, not "buffer the whole utterance,
+ * run one giant batch decode". Decodes run serially on the shared ASR mutex
+ * (the fused context's ASR region is single-threaded).
+ *
+ * Requires `ffi.mmapAcquire(ctx, "asr")` to have been called on `getContext()`
+ * — the `EngineVoiceBridge` lifecycle does this when voice input is armed.
+ */
+export declare class FfiBatchTranscriber extends BaseStreamingTranscriber {
+    private readonly ffi;
+    private readonly getContext;
+    private readonly windowSamples;
+    private readonly overlapSamples;
+    private readonly stepSamples;
+    /** All 16 kHz samples accumulated for the current speech segment. */
+    private buf;
+    /** Samples in `buf` already folded into `committed`. */
+    private committedSamples;
+    /** Text decoded from `buf[0 .. committedSamples)`. */
+    private committed;
+    /** `buf.length` at the last decode pass — throttles to `stepSamples`. */
+    private lastDecodeAt;
+    /** Decode chain — `asr_transcribe` calls serialize on the native ASR mutex anyway. */
+    private decodeChain;
+    constructor(opts: FfiBatchTranscriberOptions);
+    private decodeWindow;
+    protected onFrame(frame: PcmFrame): void;
+    protected onFlush(): Promise<TranscriptUpdate>;
+    protected onDispose(): void;
+    private resetSegment;
+    private scheduleDecode;
+    private runDecode;
+}
+export interface CreateStreamingTranscriberOptions {
+    /** Fused FFI handle (when a `libelizainference` build is loaded), else null. */
+    ffi?: ElizaInferenceFfi | null;
+    /** Provider for the fused context pointer (the bridge owns the lazy create). */
+    getContext?: () => ElizaInferenceContextHandle;
+    /**
+     * Whether a bundled ASR model directory is present. The fused path is
+     * only chosen when this is true AND the library advertises streaming
+     * ASR.
+     */
+    asrBundlePresent?: boolean;
+    /** VAD event stream to gate decoding (W1). */
+    vad?: VadEventSource;
+    /** Optional attribution metadata stamped onto emitted transcript updates. */
+    metadata?: TranscriptMetadataDefaults;
+    /** Convenience shorthand for `metadata.source`. */
+    source?: VoiceInputSource;
+    /** Fused-batch-interim window/step overrides (see `FfiBatchTranscriber`). */
+    ffiBatch?: Omit<FfiBatchTranscriberOptions, "ffi" | "getContext">;
+    /**
+     * Force a specific fused backend.
+     *   `"fused"`     → fused streaming ASR only (throws if unavailable),
+     *   `"ffi-batch"` → fused batch (interim) only (throws if unavailable),
+     *   `"auto"`      (default) → fused streaming → fused batch → throw.
+     */
+    prefer?: AsrBackendPreference;
+}
+/**
+ * Resolve the fused ASR adapter chain:
+ *   1. fused streaming ASR (`eliza_inference_asr_stream_*`, ABI v2 — the FINAL
+ *      path, W7),
+ *   2. fused batch (interim) — windowed `eliza_inference_asr_transcribe` (ABI
+ *      v1); contract-clean (one ggml, shared text vocab) and available now.
+ *
+ * The fused `libelizainference` build is the SOLE on-device ASR runtime. There
+ * is no whisper.cpp (or other second-runtime) fallback: if no fused decoder is
+ * available the caller gets a hard, actionable failure (AGENTS.md §3 + §9) —
+ * never a silent empty transcript.
+ */
+export declare function createStreamingTranscriber(opts?: CreateStreamingTranscriberOptions): StreamingTranscriber;
+//# sourceMappingURL=transcriber.d.ts.map