@elizaos/plugin-local-inference 2.0.0-beta.1 → 2.0.3-beta.3

package/dist/services/voice/audio-frame-consumer.d.ts

@@ -0,0 +1,294 @@
+/**
+ * AudioFrameConsumer — turn the Android `audioFrame` PCM stream into live,
+ * VAD-segmented, speaker-attributed voice turns.
+ *
+ * The Android native capture path (`plugin-native-talkmode`) streams an
+ * `audioFrame` Capacitor event: base64 little-endian s16 mono PCM at 16 kHz,
+ * 20 ms per frame, plus `{ sampleRate, channels, samples, rms, timestamp,
+ * frameIndex }`. This module is the platform-agnostic consumer that subscribes
+ * to that stream (wherever the bun:ffi voice libs are present) and runs:
+ *
+ *   audioFrame (base64 LE-s16) → decode → VadDetector (turn segmentation)
+ *     → on speech-end: VoiceAttributionPipeline.attribute(turn PCM)
+ *       → handleLiveVoiceAttribution → VOICE_TURN_OBSERVED + voiceTurnSignal
+ *
+ * Design notes:
+ *  - It does NOT reinvent VAD: it drives the existing `VadDetector` state
+ *    machine (`speech-start` / `speech-pause` / `speech-end`), reusing its
+ *    Silero onset/offset/hangover logic. It buffers the turn's PCM between
+ *    `speech-start` and `speech-end`, then attributes the whole utterance.
+ *  - Every native dependency (`VadDetector`, `VoiceAttributionPipeline`, the
+ *    runtime) is INJECTED, so the consumer is fully unit-testable with fakes
+ *    and has no static import of bun:ffi. A `build*` factory in the smoke
+ *    harness wires the real ggml-backed deps.
+ *  - The decode boundary (`decodeAudioFramePcm`) is the ONLY place that knows
+ *    the wire format (base64 LE-s16). Internally everything is Float32 [-1,1].
+ *
+ * What this module does NOT do: it does not transcribe (ASR text is the
+ * separate streaming-ASR path) and it does not own the WebView→agent
+ * transport — see `android/AUDIO_FRAMES.md` and `LIVE_PIPELINE.md` for the
+ * remaining device wiring.
+ */
+import { type EmitVoiceTurnObservedArgs, type HandleLiveVoiceAttributionOptions } from "../../runtime/voice-entity-binding.js";
+import type { VoiceTurnSignal } from "./eot-classifier.js";
+import { type ResidualSuppressionOptions } from "./nlms-echo-canceller.js";
+import type { VoiceAttributionOutput, VoiceAttributionPipeline } from "./speaker/attribution-pipeline.js";
+import type { PcmFrame, VadEvent, VoiceInputSource } from "./types.js";
+/**
+ * The `audioFrame` event payload, mirroring `TalkModeAudioFrameEvent` in
+ * `@elizaos/capacitor-talkmode`. Re-declared structurally here so this
+ * package does not take a build dep on the Capacitor plugin.
+ */
+export interface AudioFrameEvent {
+    /** Base64-encoded little-endian signed 16-bit mono PCM for this frame. */
+    pcm16: string;
+    /** Sample rate of the captured PCM in Hz (e.g. 16000). */
+    sampleRate: number;
+    /** Channel count (always 1 — mono). */
+    channels: number;
+    /** Number of PCM samples in this frame (`pcm16` byte length / 2). */
+    samples: number;
+    /** RMS amplitude of this frame, normalized 0..1. */
+    rms: number;
+    /** Monotonic capture timestamp for this frame, ms. */
+    timestamp: number;
+    /** Running index of this frame since capture started (0-based). */
+    frameIndex: number;
+}
+/** The sample rate every voice model in this pipeline is dimensioned for. */
+export declare const AUDIO_FRAME_PIPELINE_SAMPLE_RATE = 16000;
+export declare class AudioFrameDecodeError extends Error {
+    constructor(message: string);
+}
+/**
+ * Decode an `audioFrame` payload into a Float32 [-1, 1] window. This is the
+ * single boundary that understands the base64 LE-s16 wire format.
+ *
+ * The native capture path only ever produces 16 kHz mono; this asserts that
+ * invariant rather than resampling silently (the downstream Silero/WeSpeaker
+ * graphs are 16 kHz-only — a wrong rate is a bug to surface, not paper over).
+ */
+export declare function decodeAudioFramePcm(frame: AudioFrameEvent): Float32Array;
+/**
+ * The structural slice of `VadDetector` the consumer needs. Taking the
+ * structural view (not the concrete class) keeps the consumer testable with a
+ * fake VAD and avoids pulling the optional native VAD surface into callers
+ * that only want to feed frames.
+ */
+export interface VadSegmenter {
+    /** True while a speech segment (incl. its pause hangover) is open. */
+    readonly inSpeech: boolean;
+    /** Subscribe to the authoritative VAD timeline. Returns an unsubscribe fn. */
+    onVadEvent(listener: (event: VadEvent) => void): () => void;
+    /** Feed one mic frame; resolves once its windows are processed. */
+    pushFrame(frame: PcmFrame): Promise<void>;
+    /** Flush trailing samples and finalize any open segment. */
+    flush(): Promise<void>;
+    /** Clear all state at a hard boundary. */
+    reset(): void;
+}
+/**
+ * The structural slice of `VoiceAttributionPipeline` the consumer needs.
+ */
+export interface AttributionPipelineLike {
+    attribute(req: Parameters<VoiceAttributionPipeline["attribute"]>[0]): Promise<VoiceAttributionOutput>;
+}
+/**
+ * The structural slice of `IAgentRuntime` the consumer needs:
+ * `handleLiveVoiceAttribution` calls `emitEvent`.
+ */
+export interface RuntimeEventSink {
+    emitEvent(type: unknown, payload: Record<string, unknown>): Promise<void>;
+    /**
+     * Optional host-supplied far-end (agent TTS playback) reference for the live
+     * AEC path (#9583). When a host wires this, the live diarization route threads
+     * it into the session's NLMS echo canceller instead of relying on the
+     * playback-frames ingest route. Absent on headless/core runtimes.
+     */
+    voiceEchoReferenceProvider?: EchoReferenceProvider;
+}
+/**
+ * Transcribe a finalized turn's buffered PCM to text (#8786). When injected, the
+ * consumer joins the ASR transcript into the diarization attribution so
+ * `VOICE_TURN_OBSERVED` carries the real text — previously the live audio-frame
+ * path attributed *who* spoke but always emitted `text: ""`, so name/partner
+ * extraction (`VoiceObserver.ingestTurn`) could never fire from live audio.
+ *
+ * Returns the transcript, or `null`/empty for silence / no decode. Best-effort:
+ * the consumer swallows a rejection (counted in `transcriptionErrors`) and falls
+ * back to a transcript-less turn rather than dropping the diarized turn.
+ */
+export type TurnTranscriber = (pcm: Float32Array, sampleRate: number) => Promise<string | null> | string | null;
+export type SelfVoiceSimilarityResolver = (embedding: Float32Array, output: VoiceAttributionOutput) => Promise<number | null | undefined> | number | null | undefined;
+export interface AudioFrameConsumerDeps {
+    /** Turn-segmentation VAD (drives speech-start/pause/end). */
+    vad: VadSegmenter;
+    /** Diarization + speaker-attribution pipeline. */
+    pipeline: AttributionPipelineLike;
+    /** Runtime event sink for VOICE_TURN_OBSERVED. */
+    runtime: RuntimeEventSink;
+    /**
+     * Optional ASR for the finalized turn's PCM (#8786). When present, its text
+     * rides on `VOICE_TURN_OBSERVED` so live name/entity extraction runs. When
+     * absent the path stays diarization-only (transcript `""`, as before).
+     */
+    transcribe?: TurnTranscriber;
+    /**
+     * Optional live acoustic self-voice resolver. When wired, the consumer passes
+     * the turn's WeSpeaker embedding to the host's agent-TTS centroid matcher and
+     * forwards the resulting cosine into the ambient gate.
+     */
+    resolveSelfVoiceSimilarity?: SelfVoiceSimilarityResolver;
+    /**
+     * Optional agent-playback (far-end) reference for acoustic echo cancellation
+     * (#9455). Given a mic frame's clock timestamp and sample count, returns the
+     * agent's TTS playback PCM for that exact window (Float32 16 kHz), or null
+     * when the agent is not playing. When wired, the consumer runs an NLMS echo
+     * canceller on every mic frame BEFORE VAD/attribution so the agent never
+     * transcribes its own TTS. Absent → no AEC (unchanged behavior). The caller
+     * owns the playback capture + the playback→mic delay calibration.
+     */
+    echoReference?: EchoReferenceProvider;
+}
+/**
+ * Returns the agent's TTS playback PCM (the far-end echo reference) aligned to a
+ * mic frame's time window, or null when the agent is silent. See #9455.
+ */
+export type EchoReferenceProvider = (timestampMs: number, samples: number) => Float32Array | null;
+export interface AudioFrameConsumerConfig {
+    /** Source metadata stamped onto every attributed turn. */
+    source?: VoiceInputSource;
+    /** Gating options forwarded to `handleLiveVoiceAttribution` per turn. */
+    attributionOptions?: HandleLiveVoiceAttributionOptions;
+    /**
+     * Hard cap on a single buffered turn, in seconds. A speaker who never
+     * triggers `speech-end` (e.g. continuous noise) must not grow the buffer
+     * without bound. When exceeded the turn is force-finalized. Default 30 s.
+     */
+    maxTurnSeconds?: number;
+    /**
+     * Pre-roll seconds of audio kept before `speech-start` so the onset of the
+     * first word (which the VAD only confirms a window or two in) is not clipped
+     * out of the attribution buffer. Default 0.3 s.
+     */
+    preRollSeconds?: number;
+    /**
+     * Opt-in nonlinear residual-echo suppressor forwarded to the NLMS canceller
+     * (#9583/#9649). Default-off; only meaningful when an `echoReference` is wired
+     * (no canceller exists otherwise). See {@link NlmsEchoCancellerOptions.residualSuppression}.
+     */
+    residualSuppression?: boolean | ResidualSuppressionOptions;
+}
+/** A finalized, attributed turn the consumer surfaces to its caller. */
+export interface AttributedTurn {
+    turnId: string;
+    output: VoiceAttributionOutput;
+    signal: VoiceTurnSignal;
+    /** Turn span in the mic-clock (frame `timestamp`) domain. */
+    startedAtMs: number;
+    endedAtMs: number;
+    /** Total buffered turn samples that were attributed. */
+    samples: number;
+}
+export type AttributedTurnListener = (turn: AttributedTurn) => void;
+/**
+ * Drives the `audioFrame` → VAD turn-segmentation → attribution → signal
+ * pipeline. One instance per capture session.
+ *
+ * Frame ingestion is serialized through the injected VAD's `pushFrame`
+ * (which itself serializes the Silero forward pass), so `onAudioFrame` is
+ * safe to fire-and-forget from a Capacitor event listener; turns surface in
+ * order via `onTurn`.
+ */
+export declare class AudioFrameConsumer {
+    private readonly vad;
+    private readonly pipeline;
+    private readonly runtime;
+    private readonly transcribe;
+    private readonly resolveSelfVoiceSimilarity;
+    private readonly echoReference;
+    /** NLMS echo canceller, instantiated only when an `echoReference` is wired. */
+    private readonly echoCanceller;
+    private readonly source;
+    private readonly attributionOptions;
+    private readonly maxTurnSamples;
+    private readonly preRollSamples;
+    private readonly unsubscribeVad;
+    private readonly turnListeners;
+    /** Float32 chunks of the in-flight turn, oldest first. */
+    private turnChunks;
+    private turnSamples;
+    /** Rolling pre-roll ring (frames captured before speech-start). */
+    private preRoll;
+    private preRollSampleCount;
+    private capturing;
+    private turnSeq;
+    private turnStartedAtMs;
+    private lastFrameEndMs;
+    /** Serialized attribution chain so turns finalize one at a time, in order. */
+    private attributing;
+    private closed;
+    /** Count of frames that failed to decode (surfaced via getters, not thrown). */
+    droppedFrames: number;
+    /** Count of turns whose ASR transcribe threw (degraded to a transcript-less
+     *  turn rather than dropping the diarized turn). */
+    transcriptionErrors: number;
+    /** Count of mic frames the echo canceller actually processed (i.e. the agent
+     *  was playing). Frames skipped while the agent is silent do not count, so
+     *  this also measures how often AEC took the cheap passthrough path. */
+    echoFramesCancelled: number;
+    constructor(deps: AudioFrameConsumerDeps, config?: AudioFrameConsumerConfig);
+    /** True while a turn is being buffered (between speech-start and speech-end). */
+    get inTurn(): boolean;
+    /** Subscribe to finalized attributed turns. Returns an unsubscribe fn. */
+    onTurn(listener: AttributedTurnListener): () => void;
+    /**
+     * Feed one decoded-or-raw `audioFrame`. Accepts either the wire-format
+     * `AudioFrameEvent` (decoded here) or a pre-decoded Float32 window with the
+     * frame's mic-clock timestamp. Resolves once the frame's VAD windows are
+     * processed.
+     */
+    onAudioFrame(frame: AudioFrameEvent): Promise<void>;
+    /**
+     * Feed a pre-decoded Float32 16 kHz window with its mic-clock timestamp
+     * (ms). The decode boundary already ran; used by transports that decode
+     * upstream and by the host harness.
+     */
+    pushDecodedFrame(pcm: Float32Array, timestampMs: number): Promise<void>;
+    /**
+     * Run the echo canceller on one mic frame when (and only when) the agent is
+     * playing. The reference provider returns null while the agent is silent, in
+     * which case the mic frame is passed through verbatim and the FIR
+     * `process()` loop is not invoked. The canceller still observes the silent
+     * far-end so stale playback history is cleared before playback resumes.
+     * Returns the echo-cancelled (or untouched) mic frame.
+     */
+    private cancelEcho;
+    /**
+     * Flush the VAD (finalize any open segment) and await all pending
+     * attribution. Call at end-of-capture so a trailing utterance is not lost.
+     */
+    flush(): Promise<void>;
+    /** Release listeners and clear all buffers. Idempotent. */
+    close(): Promise<void>;
+    private onVadEvent;
+    private beginTurn;
+    private finalizeTurn;
+    private attributeTurn;
+    /**
+     * Merge the per-turn ASR transcript into the attribution options. Returns the
+     * base options unchanged when no transcriber is wired or the decode yields no
+     * text; a thrown decode is swallowed (counted in `transcriptionErrors`) so a
+     * diarized turn is never dropped over an ASR failure.
+     */
+    private resolveTurnOptions;
+    private appendTurnChunk;
+    private appendPreRoll;
+}
+/**
+ * Re-export of the producer's emit args, so a consumer caller can construct a
+ * VOICE_TURN_OBSERVED payload directly when wiring a custom transport without
+ * importing the runtime subpath twice.
+ */
+export type { EmitVoiceTurnObservedArgs };
+//# sourceMappingURL=audio-frame-consumer.d.ts.map

package/dist/services/voice/audio-frame-consumer.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"audio-frame-consumer.d.ts","sourceRoot":"","sources":["../../../src/services/voice/audio-frame-consumer.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA8BG;AAEH,OAAO,EACN,KAAK,yBAAyB,EAC9B,KAAK,iCAAiC,EAEtC,MAAM,uCAAuC,CAAC;AAC/C,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,qBAAqB,CAAC;AAC3D,OAAO,EAEN,KAAK,0BAA0B,EAC/B,MAAM,0BAA0B,CAAC;AAClC,OAAO,KAAK,EACX,sBAAsB,EACtB,wBAAwB,EACxB,MAAM,mCAAmC,CAAC;AAC3C,OAAO,KAAK,EAAE,QAAQ,EAAE,QAAQ,EAAE,gBAAgB,EAAE,MAAM,YAAY,CAAC;AAMvE;;;;GAIG;AACH,MAAM,WAAW,eAAe;IAC/B,0EAA0E;IAC1E,KAAK,EAAE,MAAM,CAAC;IACd,0DAA0D;IAC1D,UAAU,EAAE,MAAM,CAAC;IACnB,uCAAuC;IACvC,QAAQ,EAAE,MAAM,CAAC;IACjB,qEAAqE;IACrE,OAAO,EAAE,MAAM,CAAC;IAChB,oDAAoD;IACpD,GAAG,EAAE,MAAM,CAAC;IACZ,sDAAsD;IACtD,SAAS,EAAE,MAAM,CAAC;IAClB,mEAAmE;IACnE,UAAU,EAAE,MAAM,CAAC;CACnB;AAED,6EAA6E;AAC7E,eAAO,MAAM,gCAAgC,QAAS,CAAC;AAEvD,qBAAa,qBAAsB,SAAQ,KAAK;gBACnC,OAAO,EAAE,MAAM;CAI3B;AAED;;;;;;;GAOG;AACH,wBAAgB,mBAAmB,CAAC,KAAK,EAAE,eAAe,GAAG,YAAY,CA0BxE;AA8BD;;;;;GAKG;AACH,MAAM,WAAW,YAAY;IAC5B,sEAAsE;IACtE,QAAQ,CAAC,QAAQ,EAAE,OAAO,CAAC;IAC3B,8EAA8E;IAC9E,UAAU,CAAC,QAAQ,EAAE,CAAC,KAAK,EAAE,QAAQ,KAAK,IAAI,GAAG,MAAM,IAAI,CAAC;IAC5D,mEAAmE;IACnE,SAAS,CAAC,KAAK,EAAE,QAAQ,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IAC1C,4DAA4D;IAC5D,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;IACvB,0CAA0C;IAC1C,KAAK,IAAI,IAAI,CAAC;CACd;AAED;;GAEG;AACH,MAAM,WAAW,uBAAuB;IACvC,SAAS,CACR,GAAG,EAAE,UAAU,CAAC,wBAAwB,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC,CAAC,GACvD,OAAO,CAAC,sBAAsB,CAAC,CAAC;CACnC;AAED;;;GAGG;AACH,MAAM,WAAW,gBAAgB;IAChC,SAAS,CAAC,IAAI,EAAE,OAAO,EAAE,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IAC1E;;;;;OAKG;IACH,0BAA0B,CAAC,EAAE,qBAAqB,CAAC;CACnD;AAED;;;;;;;;;;GAUG;AACH,MAAM,MAAM,eAAe,GAAG,CAC7B,GAAG,EAAE,YAAY,EACjB,UAAU,EAAE,MAAM,KACd,OAAO,CAAC,MAAM,GAAG,IAAI,CAAC,GAAG,MAAM,GAAG,IAAI,CAAC;AAE5C,MAAM,MAAM,2BAA2B,GAAG,CACzC,SAAS,EAAE,YAAY,EACvB,MAAM,EAAE,sBAAsB,KAC1B,OAAO,CAAC,MAAM,GAAG,IAAI,GAAG,SAAS,CAAC,GAAG,MAAM,GAAG,IAAI,GAAG,SAAS,CAAC;AAMpE,MAAM,WAAW,sBAAsB;IACtC,6DAA6D;IAC7D,GAAG,EAAE,YAAY,CAAC;IAClB,kDAAkD;IAClD,QAAQ,EAAE,uBAAuB,CAAC;IAClC,kDAAkD;IAClD,OAAO,EAAE,gBAAgB,CAAC;IAC1B;;;;OAIG;IACH,UAAU,CAAC,EAAE,eAAe,CAAC;IAC7B;;;;OAIG;IACH,0BAA0B,CAAC,EAAE,2BAA2B,CAAC;IACzD;;;;;;;;OAQG;IACH,aAAa,CAAC,EAAE,qBAAqB,CAAC;CACtC;AAED;;;GAGG;AACH,MAAM,MAAM,qBAAqB,GAAG,CACnC,WAAW,EAAE,MAAM,EACnB,OAAO,EAAE,MAAM,KACX,YAAY,GAAG,IAAI,CAAC;AAEzB,MAAM,WAAW,wBAAwB;IACxC,0DAA0D;IAC1D,MAAM,CAAC,EAAE,gBAAgB,CAAC;IAC1B,yEAAyE;IACzE,kBAAkB,CAAC,EAAE,iCAAiC,CAAC;IACvD;;;;OAIG;IACH,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB;;;;OAIG;IACH,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB;;;;OAIG;IACH,mBAAmB,CAAC,EAAE,OAAO,GAAG,0BAA0B,CAAC;CAC3D;AAED,wEAAwE;AACxE,MAAM,WAAW,cAAc;IAC9B,MAAM,EAAE,MAAM,CAAC;IACf,MAAM,EAAE,sBAAsB,CAAC;IAC/B,MAAM,EAAE,eAAe,CAAC;IACxB,6DAA6D;IAC7D,WAAW,EAAE,MAAM,CAAC;IACpB,SAAS,EAAE,MAAM,CAAC;IAClB,wDAAwD;IACxD,OAAO,EAAE,MAAM,CAAC;CAChB;AAED,MAAM,MAAM,sBAAsB,GAAG,CAAC,IAAI,EAAE,cAAc,KAAK,IAAI,CAAC;AAEpE;;;;;;;;GAQG;AACH,qBAAa,kBAAkB;IAC9B,OAAO,CAAC,QAAQ,CAAC,GAAG,CAAe;IACnC,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAA0B;IACnD,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAmB;IAC3C,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAyB;IACpD,OAAO,CAAC,QAAQ,CAAC,0BAA0B,CAAqC;IAChF,OAAO,CAAC,QAAQ,CAAC,aAAa,CAA+B;IAC7D,+EAA+E;IAC/E,OAAO,CAAC,QAAQ,CAAC,aAAa,CAA2B;IACzD,OAAO,CAAC,QAAQ,CAAC,MAAM,CAA+B;IACtD,OAAO,CAAC,QAAQ,CAAC,kBAAkB,CAAoC;IACvE,OAAO,CAAC,QAAQ,CAAC,cAAc,CAAS;IACxC,OAAO,CAAC,QAAQ,CAAC,cAAc,CAAS;IACxC,OAAO,CAAC,QAAQ,CAAC,cAAc,CAAa;IAC5C,OAAO,CAAC,QAAQ,CAAC,aAAa,CAAqC;IAEnE,0DAA0D;IAC1D,OAAO,CAAC,UAAU,CAAsB;IACxC,OAAO,CAAC,WAAW,CAAK;IACxB,mEAAmE;IACnE,OAAO,CAAC,OAAO,CAAsB;IACrC,OAAO,CAAC,kBAAkB,CAAK;IAC/B,OAAO,CAAC,SAAS,CAAS;IAC1B,OAAO,CAAC,OAAO,CAAK;IACpB,OAAO,CAAC,eAAe,CAAK;IAC5B,OAAO,CAAC,cAAc,CAAK;IAC3B,8EAA8E;IAC9E,OAAO,CAAC,WAAW,CAAoC;IACvD,OAAO,CAAC,MAAM,CAAS;IAEvB,gFAAgF;IAChF,aAAa,SAAK;IAElB;wDACoD;IACpD,mBAAmB,SAAK;IAExB;;4EAEwE;IACxE,mBAAmB,SAAK;gBAGvB,IAAI,EAAE,sBAAsB,EAC5B,MAAM,GAAE,wBAA6B;IA+BtC,iFAAiF;IACjF,IAAI,MAAM,IAAI,OAAO,CAEpB;IAED,0EAA0E;IAC1E,MAAM,CAAC,QAAQ,EAAE,sBAAsB,GAAG,MAAM,IAAI;IAKpD;;;;;OAKG;IACG,YAAY,CAAC,KAAK,EAAE,eAAe,GAAG,OAAO,CAAC,IAAI,CAAC;IAkBzD;;;;OAIG;IACG,gBAAgB,CACrB,GAAG,EAAE,YAAY,EACjB,WAAW,EAAE,MAAM,GACjB,OAAO,CAAC,IAAI,CAAC;IAqBhB;;;;;;;OAOG;IACH,OAAO,CAAC,UAAU;IAWlB;;;OAGG;IACG,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;IAM5B,2DAA2D;IACrD,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;IAc5B,OAAO,CAAC,UAAU;IAgBlB,OAAO,CAAC,SAAS;IAWjB,OAAO,CAAC,YAAY;YAkBN,aAAa;IAkC3B;;;;;OAKG;YACW,kBAAkB;IAkChC,OAAO,CAAC,eAAe;IAUvB,OAAO,CAAC,aAAa;CAarB;AAgBD;;;;GAIG;AACH,YAAY,EAAE,yBAAyB,EAAE,CAAC"}

package/dist/services/voice/barge-in.d.ts

@@ -0,0 +1,112 @@
+/**
+ * Barge-in controller — distinguishes a blip from real speech while the
+ * agent is talking, and turns that into TTS pause/resume/hard-stop plus an
+ * LLM-generation abort.
+ *
+ * Inputs:
+ *   - the `VadEvent` stream from `VadDetector` (subscribe via `bindVad()`),
+ *   - W2's ASR word-confirm callback (`onWordsDetected()` — the
+ *     `WordsDetectedSink` contract).
+ *
+ * Behaviour while the agent is speaking (`agentSpeaking === true`):
+ *   - `speech-active`  → emit `pause-tts`. (Provisional — could still be a
+ *                        blip; the energy-duration heuristic guesses, ASR
+ *                        confirms.)
+ *   - `blip` (or a short `speech-end` before any words)
+ *                      → emit `resume-tts`. The agent keeps talking.
+ *   - `onWordsDetected({wordCount ≥ 1})` → emit `hard-stop` with a fresh
+ *                        `BargeInCancelToken`. Hard-stop means: cancel TTS
+ *                        *and* abort the in-flight LLM / MTP drafter
+ *                        generation. The engine layer (W9) threads
+ *                        `token.signal` into `dispatcher.generate` and polls
+ *                        `token.cancelled` at kernel boundaries.
+ *   - `speech-end` with a long-enough segment but no ASR words yet →
+ *                        treated as words-pending: emit `hard-stop` only
+ *                        once ASR confirms; if ASR never confirms within
+ *                        `wordsGraceMs`, resume TTS (it was non-speech the
+ *                        Silero VAD let through).
+ *
+ * Legacy API (still used by `VoiceScheduler` and `EngineVoiceBridge`):
+ *   `attach({onCancel})`, `onMicActive()`, `cancelSignal()`, `reset()` — a
+ *   thin "everything cancelled" path. `onMicActive()` is now equivalent to
+ *   `hardStop("manual")`.
+ *
+ * No fallback sludge: a `hard-stop` always carries a real `AbortSignal`; the
+ * controller never swallows a VAD event.
+ */
+import type { BargeInCancelToken, BargeInSignalListener, VadEventListener, WordsDetectedSink } from "./types";
+/** Minimal structural view of `VadDetector` — avoids a module dependency on
+ *  `vad.ts` (which pulls in the fused `libelizainference` VAD FFI surface). */
+interface VadEventSource {
+    onVadEvent(listener: VadEventListener): () => void;
+}
+export interface BargeInListener {
+    onCancel(): void;
+}
+export interface CancelSignal {
+    cancelled: boolean;
+}
+export interface BargeInControllerConfig {
+    /**
+     * After a `speech-active` (TTS paused) with no ASR word confirmation,
+     * resume TTS if ASR has not reported ≥1 word within this window. Default
+     * 600 ms. Long enough for a streaming ASR partial; short enough that a
+     * cough doesn't keep the agent muted.
+     */
+    wordsGraceMs?: number;
+}
+export declare class BargeInController implements WordsDetectedSink {
+    private readonly listeners;
+    private readonly signalListeners;
+    private readonly wordsGraceMs;
+    /** Legacy single-shot cancel flag, reset by `reset()`. */
+    private signal;
+    /** True while the agent's TTS is playing. The turn controller / scheduler
+     *  flips this via `setAgentSpeaking()`. Barge-in logic only acts while
+     *  this is true. */
+    private agentSpeaking;
+    /** True while we have emitted `pause-tts` and are waiting on the
+     *  blip-vs-words decision. */
+    private awaitingWordConfirm;
+    private wordConfirmDeadlineTimer;
+    private wordConfirmExpiresAtMs;
+    private lastEventTimestampMs;
+    private vadUnsub;
+    constructor(config?: BargeInControllerConfig);
+    /** Subscribe to `pause-tts` / `resume-tts` / `hard-stop`. */
+    onSignal(listener: BargeInSignalListener): () => void;
+    /** Wire this controller to a `VadDetector`. Returns an unsubscribe fn. */
+    bindVad(detector: VadEventSource): () => void;
+    unbindVad(): void;
+    /** The turn controller flips this when TTS starts/stops playing. */
+    setAgentSpeaking(speaking: boolean): void;
+    get isAgentSpeaking(): boolean;
+    private onVadEvent;
+    onWordsDetected(args: {
+        wordCount: number;
+        partialText: string;
+        timestampMs: number;
+    }): void;
+    /**
+     * Cancel TTS + abort the in-flight LLM / drafter generation. Returns the
+     * `BargeInCancelToken` whose `signal` the engine layer aborts on. Idempotent
+     * within a single barge-in episode — calling it again returns the same
+     * token until `reset()`.
+     */
+    hardStop(reason?: NonNullable<BargeInCancelToken["reason"]>, timestampMs?: number): BargeInCancelToken;
+    private activeToken;
+    /** The cancel token for the current barge-in episode (null until a
+     *  `hard-stop`). The engine threads `.signal` into generation. */
+    currentCancelToken(): BargeInCancelToken | null;
+    /** @deprecated Use `currentCancelToken()`; kept for `VoiceScheduler`. */
+    cancelSignal(): CancelSignal;
+    attach(listener: BargeInListener): () => void;
+    /** @deprecated Equivalent to `hardStop("manual")`; kept for the bridge. */
+    onMicActive(): void;
+    reset(): void;
+    private emitSignal;
+    private armWordConfirmDeadline;
+    private clearWordConfirm;
+}
+export {};
+//# sourceMappingURL=barge-in.d.ts.map

package/dist/services/voice/barge-in.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"barge-in.d.ts","sourceRoot":"","sources":["../../../src/services/voice/barge-in.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAmCG;AAEH,OAAO,KAAK,EACX,kBAAkB,EAElB,qBAAqB,EAErB,gBAAgB,EAChB,iBAAiB,EACjB,MAAM,SAAS,CAAC;AAEjB;+EAC+E;AAC/E,UAAU,cAAc;IACvB,UAAU,CAAC,QAAQ,EAAE,gBAAgB,GAAG,MAAM,IAAI,CAAC;CACnD;AAID,MAAM,WAAW,eAAe;IAC/B,QAAQ,IAAI,IAAI,CAAC;CACjB;AAED,MAAM,WAAW,YAAY;IAC5B,SAAS,EAAE,OAAO,CAAC;CACnB;AAkCD,MAAM,WAAW,uBAAuB;IACvC;;;;;OAKG;IACH,YAAY,CAAC,EAAE,MAAM,CAAC;CACtB;AAED,qBAAa,iBAAkB,YAAW,iBAAiB;IAC1D,OAAO,CAAC,QAAQ,CAAC,SAAS,CAA8B;IACxD,OAAO,CAAC,QAAQ,CAAC,eAAe,CAAoC;IACpE,OAAO,CAAC,QAAQ,CAAC,YAAY,CAAS;IAEtC,0DAA0D;IAC1D,OAAO,CAAC,MAAM,CAAsC;IAEpD;;wBAEoB;IACpB,OAAO,CAAC,aAAa,CAAS;IAC9B;kCAC8B;IAC9B,OAAO,CAAC,mBAAmB,CAAS;IACpC,OAAO,CAAC,wBAAwB,CAA8C;IAC9E,OAAO,CAAC,sBAAsB,CAAuB;IACrD,OAAO,CAAC,oBAAoB,CAAK;IACjC,OAAO,CAAC,QAAQ,CAA6B;gBAEjC,MAAM,GAAE,uBAA4B;IAMhD,6DAA6D;IAC7D,QAAQ,CAAC,QAAQ,EAAE,qBAAqB,GAAG,MAAM,IAAI;IAKrD,0EAA0E;IAC1E,OAAO,CAAC,QAAQ,EAAE,cAAc,GAAG,MAAM,IAAI;IAM7C,SAAS,IAAI,IAAI;IAOjB,oEAAoE;IACpE,gBAAgB,CAAC,QAAQ,EAAE,OAAO,GAAG,IAAI;IAUzC,IAAI,eAAe,IAAI,OAAO,CAE7B;IAID,OAAO,CAAC,UAAU;IA8ClB,eAAe,CAAC,IAAI,EAAE;QACrB,SAAS,EAAE,MAAM,CAAC;QAClB,WAAW,EAAE,MAAM,CAAC;QACpB,WAAW,EAAE,MAAM,CAAC;KACpB,GAAG,IAAI;IAiBR;;;;;OAKG;IACH,QAAQ,CACP,MAAM,GAAE,WAAW,CAAC,kBAAkB,CAAC,QAAQ,CAAC,CAAY,EAC5D,WAAW,GAAE,MAAgD,GAC3D,kBAAkB;IAkBrB,OAAO,CAAC,WAAW,CAAmC;IAEtD;sEACkE;IAClE,kBAAkB,IAAI,kBAAkB,GAAG,IAAI;IAM/C,yEAAyE;IACzE,YAAY,IAAI,YAAY;IAI5B,MAAM,CAAC,QAAQ,EAAE,eAAe,GAAG,MAAM,IAAI;IAK7C,2EAA2E;IAC3E,WAAW,IAAI,IAAI;IAInB,KAAK,IAAI,IAAI;IASb,OAAO,CAAC,UAAU;IAIlB,OAAO,CAAC,sBAAsB;IA0B9B,OAAO,CAAC,gBAAgB;CASxB"}

package/dist/services/voice/cancellation-coordinator.d.ts

@@ -0,0 +1,127 @@
+/**
+ * Voice cancellation coordinator — Wave 3 W3-9.
+ *
+ * Single brain that owns one `VoiceCancellationToken` per active voice turn
+ * and binds every cancellation source into it:
+ *
+ *   1. VAD start-of-speech while the agent is speaking (barge-in).
+ *   2. `BargeInController.hardStop` (ASR-confirmed barge-in words).
+ *   3. Turn-detector EOT revocation (user resumed mid-tentative-pause).
+ *   4. Runtime turn abort (`TurnControllerRegistry` "aborted" event).
+ *
+ * On any cancel, it fans out to:
+ *
+ *   1. The voice token's `AbortSignal` — every fetch / model call wired to
+ *      `signal` aborts at the next yield point.
+ *   2. `runtime.turnControllers.abortTurn(roomId, reason)` — the runtime's
+ *      planner-loop / action handlers see the abort within one tick
+ *      (between model calls / between actions / between provider calls).
+ *   3. Optional `slotAbort(slotId)` — invokes the registered LM
+ *      slot-abort callback (typically `MtpLlamaServer.abortSlot` which
+ *      either aborts in-flight HTTP fetches against that slot or, on a
+ *      capable fork, calls the slot-cancel REST route).
+ *   4. Optional `ttsStop()` — invokes the registered TTS-stop callback
+ *      (typically `EngineVoiceBridge.triggerBargeIn` which drains the
+ *      audio sink + cancels the FFI/HTTP synthesis path).
+ *
+ * The coordinator is intentionally a plain class — no engine coupling. The
+ * engine bridge (and tests) construct one with the structural runtime + the
+ * appropriate callbacks.
+ */
+import { type VoiceCancellationReason, VoiceCancellationRegistry, type VoiceCancellationToken } from "@elizaos/shared";
+/**
+ * Minimum runtime surface this coordinator needs. Matches a subset of
+ * `AgentRuntime.turnControllers`. Structural so unit tests can pass a fake.
+ */
+export interface CoordinatorRuntime {
+    turnControllers: {
+        abortTurn(roomId: string, reason: string): boolean;
+        onEvent(listener: (event: {
+            type: "started" | "completed" | "errored" | "aborted" | "aborted-cleanup";
+            roomId: string;
+            reason?: string;
+        }) => void): () => void;
+    };
+}
+export interface VoiceCancellationCoordinatorOptions {
+    /** The runtime to bind to. */
+    runtime: CoordinatorRuntime;
+    /**
+     * Abort the inference server slot. Wired to `MtpLlamaServer.abortSlot`
+     * in production. Async — the coordinator does NOT await it (the slot
+     * abort path is best-effort; the AbortSignal closure on the fetch is the
+     * authoritative cancel).
+     */
+    slotAbort?: (slotId: number, reason: VoiceCancellationReason) => void;
+    /**
+     * Hard-stop the TTS pipeline (audio sink drain + FFI/HTTP synthesis
+     * cancel). Wired to `EngineVoiceBridge.triggerBargeIn`. Synchronous —
+     * the audio sink drain MUST happen within one tick of `abort()`.
+     */
+    ttsStop?: (reason: VoiceCancellationReason) => void;
+    /**
+     * Optional pre-existing registry. Tests inject one to inspect token
+     * lifecycle directly. Production creates a fresh registry per session.
+     */
+    registry?: VoiceCancellationRegistry;
+}
+export declare class VoiceCancellationCoordinator {
+    private readonly runtime;
+    private readonly slotAbort;
+    private readonly ttsStop;
+    private readonly registry;
+    /** Active turns keyed by roomId. One per room. */
+    private readonly armed;
+    constructor(opts: VoiceCancellationCoordinatorOptions);
+    /**
+     * Begin a new voice turn for `roomId`. If a previous turn was active,
+     * it is aborted with `"external"` (the regular replace-on-arm semantics
+     * inherited from `VoiceCancellationRegistry`).
+     */
+    armTurn(args: {
+        roomId: string;
+        runId: string;
+        slot?: number;
+    }): VoiceCancellationToken;
+    /** Fetch the current voice token for `roomId`, or null. */
+    current(roomId: string): VoiceCancellationToken | null;
+    /** Snapshot of armed room ids. */
+    armedRoomIds(): string[];
+    /**
+     * Abort the active turn for `roomId` with the given reason. Idempotent.
+     * Returns true when a live token was aborted.
+     */
+    abort(roomId: string, reason: VoiceCancellationReason): boolean;
+    /**
+     * Trip the active token because VAD reported start-of-speech while the
+     * agent was speaking. Equivalent to `abort(roomId, "barge-in")` but
+     * keeps the call-site grep-able as the canonical barge-in entry point.
+     */
+    bargeIn(roomId: string): boolean;
+    /**
+     * Trip the active token because the turn detector revoked the previous
+     * EOT decision (user resumed within the rollback window).
+     */
+    revokeEot(roomId: string): boolean;
+    /**
+     * Wire a `BargeInController.onSignal` listener into this coordinator.
+     * The controller emits `hard-stop` when ASR confirms barge-in words;
+     * this glue translates it into `coordinator.bargeIn(roomId)` so the
+     * canonical token (and every downstream consumer) sees the abort.
+     *
+     * Returns the unsubscribe function from `onSignal`. Production callers
+     * (the engine bridge) call this once per `BargeInController` per
+     * room and keep the handle until session teardown.
+     */
+    bindBargeInController(roomId: string, controller: {
+        onSignal(listener: (signal: {
+            type: string;
+        }) => void): () => void;
+    }): () => void;
+    /**
+     * Tear down. Cancels every armed turn and unsubscribes from the
+     * runtime. Safe to call multiple times.
+     */
+    dispose(): void;
+}
+//# sourceMappingURL=cancellation-coordinator.d.ts.map

package/dist/services/voice/cancellation-coordinator.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"cancellation-coordinator.d.ts","sourceRoot":"","sources":["../../../src/services/voice/cancellation-coordinator.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA6BG;AAEH,OAAO,EACN,KAAK,uBAAuB,EAC5B,yBAAyB,EACzB,KAAK,sBAAsB,EAC3B,MAAM,iBAAiB,CAAC;AAEzB;;;GAGG;AACH,MAAM,WAAW,kBAAkB;IAClC,eAAe,EAAE;QAChB,SAAS,CAAC,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC;QACnD,OAAO,CACN,QAAQ,EAAE,CAAC,KAAK,EAAE;YACjB,IAAI,EACD,SAAS,GACT,WAAW,GACX,SAAS,GACT,SAAS,GACT,iBAAiB,CAAC;YACrB,MAAM,EAAE,MAAM,CAAC;YACf,MAAM,CAAC,EAAE,MAAM,CAAC;SAChB,KAAK,IAAI,GACR,MAAM,IAAI,CAAC;KACd,CAAC;CACF;AAED,MAAM,WAAW,mCAAmC;IACnD,8BAA8B;IAC9B,OAAO,EAAE,kBAAkB,CAAC;IAC5B;;;;;OAKG;IACH,SAAS,CAAC,EAAE,CAAC,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,uBAAuB,KAAK,IAAI,CAAC;IACtE;;;;OAIG;IACH,OAAO,CAAC,EAAE,CAAC,MAAM,EAAE,uBAAuB,KAAK,IAAI,CAAC;IACpD;;;OAGG;IACH,QAAQ,CAAC,EAAE,yBAAyB,CAAC;CACrC;AAgBD,qBAAa,4BAA4B;IACxC,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAqB;IAC7C,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAmD;IAC7E,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAiD;IACzE,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAA4B;IACrD,kDAAkD;IAClD,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAgC;gBAE1C,IAAI,EAAE,mCAAmC;IAOrD;;;;OAIG;IACH,OAAO,CAAC,IAAI,EAAE;QACb,MAAM,EAAE,MAAM,CAAC;QACf,KAAK,EAAE,MAAM,CAAC;QACd,IAAI,CAAC,EAAE,MAAM,CAAC;KACd,GAAG,sBAAsB;IA8D1B,2DAA2D;IAC3D,OAAO,CAAC,MAAM,EAAE,MAAM,GAAG,sBAAsB,GAAG,IAAI;IAItD,kCAAkC;IAClC,YAAY,IAAI,MAAM,EAAE;IAIxB;;;OAGG;IACH,KAAK,CAAC,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,uBAAuB,GAAG,OAAO;IAI/D;;;;OAIG;IACH,OAAO,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO;IAIhC;;;OAGG;IACH,SAAS,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO;IAIlC;;;;;;;;;OASG;IACH,qBAAqB,CACpB,MAAM,EAAE,MAAM,EACd,UAAU,EAAE;QACX,QAAQ,CAAC,QAAQ,EAAE,CAAC,MAAM,EAAE;YAAE,IAAI,EAAE,MAAM,CAAA;SAAE,KAAK,IAAI,GAAG,MAAM,IAAI,CAAC;KACnE,GACC,MAAM,IAAI;IAQb;;;OAGG;IACH,OAAO,IAAI,IAAI;CASf"}