@elizaos/plugin-local-inference 2.0.0-beta.1 → 2.0.3-beta.2

package/src/services/voice/mic-source.d.ts

@@ -0,0 +1,136 @@
+/**
+ * Mic capture → `PcmRingBuffer` + VAD tee.
+ *
+ * The `MicSource` interface (see `./types`) is the only seam the rest of the
+ * voice loop sees. The first concrete implementation, `DesktopMicSource`,
+ * shells out to the platform's standard PCM-capable recorder, emits 16 kHz
+ * mono `PcmFrame`s, and lets callers tee them anywhere:
+ *
+ *   mic → DesktopMicSource ─┬─→ PcmRingBuffer  (ASR reads PCM from here)
+ *                           └─→ VadDetector    (speech / barge-in signals)
+ *
+ * Per-platform recorder selection (in priority order):
+ *   - Linux:   `arecord` (alsa-utils), else `parec` (PulseAudio), else `sox`/`rec`.
+ *   - macOS:   `sox`/`rec` (`sox -d`), else `ffmpeg -f avfoundation`.
+ *   - Windows: `ffmpeg -f dshow` (DirectShow capture — bundled with most
+ *              Windows installs of ffmpeg; the renderer's `getUserMedia` path
+ *              feeding `PushMicSource` is the no-ffmpeg route).
+ *
+ * Connectors that already have a decoded PCM stream (Discord voice, the
+ * Electrobun renderer's `getUserMedia` path, a mobile capture callback —
+ * the Capacitor `Microphone` plugin on iOS/Android) implement `MicSource`
+ * over `PushMicSource` instead of spawning a process.
+ *
+ * No fallback sludge: if no recorder binary is on PATH (and no override
+ * argv was given), `start()` throws — the caller surfaces "no mic backend
+ * available", it does not pretend to capture silence.
+ */
+import { PcmRingBuffer } from "./ring-buffer";
+import type { AudioSink, MicSource, PcmFrame } from "./types";
+/**
+ * Pick the recorder that streams raw signed 16-bit LE mono PCM at
+ * `sampleRate` on stdout for the host platform. Returns `null` when none is
+ * available — the caller throws an actionable error or uses `PushMicSource`.
+ * Exported so the cross-platform preflight (`voice:interactive
+ * --platform-report`) can report which recorder the host would use.
+ */
+export declare function resolveDesktopRecorder(sampleRate: number, device?: string): {
+    program: string;
+    argv: string[];
+} | null;
+declare abstract class BaseMicSource implements MicSource {
+    readonly sampleRate: number;
+    readonly frameSamples: number;
+    protected readonly frameListeners: Set<(frame: PcmFrame) => void>;
+    protected readonly errorListeners: Set<(error: Error) => void>;
+    protected _running: boolean;
+    constructor(sampleRate: number, frameSamples: number);
+    get running(): boolean;
+    abstract start(): Promise<void>;
+    abstract stop(): Promise<void>;
+    onFrame(listener: (frame: PcmFrame) => void): () => void;
+    onError(listener: (error: Error) => void): () => void;
+    protected emitFrame(pcm: Float32Array, timestampMs: number): void;
+    protected emitError(error: Error): void;
+}
+export interface DesktopMicSourceOptions {
+    /** Capture sample rate. Default 16 kHz (what the VAD + Qwen3-ASR expect). */
+    sampleRate?: number;
+    /** Frame duration in ms. Default 32 ms (one Silero window @ 16 kHz). */
+    frameMs?: number;
+    /** Recorder program. Default: auto-resolved per platform (see module doc). */
+    program?: string;
+    /** Override the recorder argv. When set, `program` is the executable and
+     *  these are the args (must produce raw little-endian signed 16-bit mono
+     *  PCM at `sampleRate` on stdout). */
+    argv?: string[];
+    /** Capture device — ALSA name (Linux `arecord -D`), avfoundation index
+     *  (macOS, e.g. `:0`), or DirectShow friendly name (Windows). */
+    device?: string;
+}
+/**
+ * `MicSource` backed by a recorder subprocess. The recorder is auto-resolved
+ * per platform (Linux: `arecord`/`parec`/`sox`; macOS: `sox -d`/`ffmpeg -f
+ * avfoundation`; Windows: `ffmpeg -f dshow`); all stream raw PCM16 mono to
+ * stdout, which this class re-frames into fixed-size `Float32Array` frames
+ * in [-1, 1].
+ *
+ * When no recorder is available `start()` throws — the renderer's
+ * `getUserMedia` path (or the Capacitor `Microphone` plugin on mobile),
+ * both feeding a `PushMicSource`, are the no-CLI route.
+ */
+export declare class DesktopMicSource extends BaseMicSource {
+    private readonly program;
+    private readonly argv;
+    private proc;
+    private readonly carry;
+    private readonly bytesPerFrame;
+    constructor(opts?: DesktopMicSourceOptions);
+    start(): Promise<void>;
+    stop(): Promise<void>;
+    private ingest;
+}
+/**
+ * `MicSource` driven by an external producer (Discord opus-decoded PCM, the
+ * Electrobun renderer's `getUserMedia` chunks, a mobile capture callback,
+ * or a test). The producer calls `push(pcm)` (any sample count, mono,
+ * already at `sampleRate`); this class re-frames it to `frameSamples`-long
+ * frames and emits them. `start()` / `stop()` just toggle the gate.
+ */
+export declare class PushMicSource extends BaseMicSource {
+    private pending;
+    private pendingStartTimestampMs;
+    constructor(opts?: {
+        sampleRate?: number;
+        frameMs?: number;
+        frameSamples?: number;
+    });
+    start(): Promise<void>;
+    stop(): Promise<void>;
+    /**
+     * Feed mono PCM in [-1, 1] at `sampleRate`. Re-frames and emits. The
+     * timestamp is the first sample's timestamp; emitted frames advance by
+     * their sample offset so a large pushed buffer still presents a real audio
+     * timeline to VAD/ASR.
+     */
+    push(pcm: Float32Array, timestampMs?: number): void;
+    /** Feed mono PCM16 little-endian bytes (Discord / browser path). */
+    pushPcm16(bytes: Uint8Array, timestampMs?: number): void;
+    /** Surface a fatal producer-side error to subscribers. */
+    fail(error: Error): void;
+}
+/**
+ * Wire a `MicSource` to a `PcmRingBuffer` (the buffer the ASR reads PCM
+ * from). Returns the ring buffer and an unsubscribe function. The ring
+ * buffer's `onOverflow` is forwarded so callers can apply backpressure.
+ */
+export declare function pipeMicToRingBuffer(source: MicSource, sink: AudioSink, opts?: {
+    /** Ring buffer capacity in samples. Default 8 s at the source rate. */
+    capacitySamples?: number;
+    onOverflow?: (droppedSamples: number) => void;
+}): {
+    ringBuffer: PcmRingBuffer;
+    unsubscribe: () => void;
+};
+export {};
+//# sourceMappingURL=mic-source.d.ts.map

package/src/services/voice/mic-source.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"mic-source.d.ts","sourceRoot":"","sources":["mic-source.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;GA0BG;AAMH,OAAO,EAAE,aAAa,EAAE,MAAM,eAAe,CAAC;AAC9C,OAAO,KAAK,EAAE,SAAS,EAAE,SAAS,EAAE,QAAQ,EAAE,MAAM,SAAS,CAAC;AAiB9D;;;;;;GAMG;AACH,wBAAgB,sBAAsB,CACrC,UAAU,EAAE,MAAM,EAClB,MAAM,CAAC,EAAE,MAAM,GACb;IAAE,OAAO,EAAE,MAAM,CAAC;IAAC,IAAI,EAAE,MAAM,EAAE,CAAA;CAAE,GAAG,IAAI,CAiI5C;AAeD,uBAAe,aAAc,YAAW,SAAS;IAChD,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,YAAY,EAAE,MAAM,CAAC;IAC9B,SAAS,CAAC,QAAQ,CAAC,cAAc,cAAmB,QAAQ,KAAK,IAAI,EAAI;IACzE,SAAS,CAAC,QAAQ,CAAC,cAAc,cAAmB,KAAK,KAAK,IAAI,EAAI;IACtE,SAAS,CAAC,QAAQ,UAAS;gBAEf,UAAU,EAAE,MAAM,EAAE,YAAY,EAAE,MAAM;IAKpD,IAAI,OAAO,IAAI,OAAO,CAErB;IAED,QAAQ,CAAC,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;IAC/B,QAAQ,CAAC,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC;IAE9B,OAAO,CAAC,QAAQ,EAAE,CAAC,KAAK,EAAE,QAAQ,KAAK,IAAI,GAAG,MAAM,IAAI;IAKxD,OAAO,CAAC,QAAQ,EAAE,CAAC,KAAK,EAAE,KAAK,KAAK,IAAI,GAAG,MAAM,IAAI;IAKrD,SAAS,CAAC,SAAS,CAAC,GAAG,EAAE,YAAY,EAAE,WAAW,EAAE,MAAM,GAAG,IAAI;IAKjE,SAAS,CAAC,SAAS,CAAC,KAAK,EAAE,KAAK,GAAG,IAAI;CAIvC;AAED,MAAM,WAAW,uBAAuB;IACvC,6EAA6E;IAC7E,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,wEAAwE;IACxE,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,8EAA8E;IAC9E,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB;;0CAEsC;IACtC,IAAI,CAAC,EAAE,MAAM,EAAE,CAAC;IAChB;qEACiE;IACjE,MAAM,CAAC,EAAE,MAAM,CAAC;CAChB;AAED;;;;;;;;;;GAUG;AACH,qBAAa,gBAAiB,SAAQ,aAAa;IAClD,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAS;IACjC,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAW;IAChC,OAAO,CAAC,IAAI,CAA6B;IAEzC,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAgB;IACtC,OAAO,CAAC,QAAQ,CAAC,aAAa,CAAS;gBAE3B,IAAI,GAAE,uBAA4B;IA6BxC,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;IA2DtB,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC;IAqB3B,OAAO,CAAC,MAAM;CAiBd;AAED;;;;;;GAMG;AACH,qBAAa,aAAc,SAAQ,aAAa;IAE/C,OAAO,CAAC,OAAO,CAAqC;IACpD,OAAO,CAAC,uBAAuB,CAAK;gBAGnC,IAAI,GAAE;QAAE,UAAU,CAAC,EAAE,MAAM,CAAC;QAAC,OAAO,CAAC,EAAE,MAAM,CAAC;QAAC,YAAY,CAAC,EAAE,MAAM,CAAA;KAAO;IAStE,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;IAItB,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC;IAM3B;;;;;OAKG;IACH,IAAI,CAAC,GAAG,EAAE,YAAY,EAAE,WAAW,SAAQ,GAAG,IAAI;IAsBlD,oEAAoE;IACpE,SAAS,CAAC,KAAK,EAAE,UAAU,EAAE,WAAW,SAAQ,GAAG,IAAI;IAQvD,0DAA0D;IAC1D,IAAI,CAAC,KAAK,EAAE,KAAK,GAAG,IAAI;CAGxB;AAED;;;;GAIG;AACH,wBAAgB,mBAAmB,CAClC,MAAM,EAAE,SAAS,EACjB,IAAI,EAAE,SAAS,EACf,IAAI,GAAE;IACL,uEAAuE;IACvE,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,UAAU,CAAC,EAAE,CAAC,cAAc,EAAE,MAAM,KAAK,IAAI,CAAC;CACzC,GACJ;IAAE,UAAU,EAAE,aAAa,CAAC;IAAC,WAAW,EAAE,MAAM,IAAI,CAAA;CAAE,CAOxD"}

package/src/services/voice/mic-source.test.ts

@@ -0,0 +1,210 @@
+/**
+ * MicSource tests.
+ *
+ *   - `PushMicSource`: re-frames arbitrarily-sized pushes into fixed frames,
+ *     handles PCM16 byte input, gates on start/stop, surfaces errors.
+ *   - `pipeMicToRingBuffer`: PCM flows from a source into a `PcmRingBuffer`.
+ *   - `DesktopMicSource`: argv shape per platform; injected fake recorder
+ *     (`cat` of a raw PCM file) drives the byte→frame path; missing-binary
+ *     and dead-process paths surface as errors (no silent capture).
+ *
+ * No real microphone is touched — `DesktopMicSource`'s recorder is overridden
+ * with a synthetic process.
+ */
+
+import { mkdtempSync, writeFileSync } from "node:fs";
+import { tmpdir } from "node:os";
+import path from "node:path";
+import { describe, expect, it } from "vitest";
+import {
+	DesktopMicSource,
+	PushMicSource,
+	pipeMicToRingBuffer,
+	resolveDesktopRecorder,
+} from "./mic-source";
+import { InMemoryAudioSink } from "./ring-buffer";
+import type { PcmFrame } from "./types";
+
+const SR = 16_000;
+
+function int16le(samples: number[]): Buffer {
+	const buf = Buffer.alloc(samples.length * 2);
+	for (let i = 0; i < samples.length; i++) buf.writeInt16LE(samples[i], i * 2);
+	return buf;
+}
+
+describe("PushMicSource", () => {
+	it("re-frames pushes into fixed-size frames and carries the remainder", async () => {
+		const src = new PushMicSource({ sampleRate: SR, frameSamples: 100 });
+		const frames: PcmFrame[] = [];
+		src.onFrame((f) => frames.push(f));
+		await src.start();
+		src.push(new Float32Array(250).fill(0.5)); // → 2 frames, 50 carried
+		expect(frames).toHaveLength(2);
+		expect(frames[0].pcm.length).toBe(100);
+		src.push(new Float32Array(60).fill(0.25)); // 50 + 60 = 110 → 1 frame, 10 carried
+		expect(frames).toHaveLength(3);
+		expect(frames[2].pcm[0]).toBeCloseTo(0.5); // first 50 from prior push
+		expect(frames[2].pcm[99]).toBeCloseTo(0.25); // last 50 from this push
+		await src.stop();
+	});
+
+	it("does not emit frames before start() or after stop()", async () => {
+		const src = new PushMicSource({ frameSamples: 10 });
+		const frames: PcmFrame[] = [];
+		src.onFrame((f) => frames.push(f));
+		src.push(new Float32Array(30)); // before start — ignored
+		expect(frames).toHaveLength(0);
+		await src.start();
+		src.push(new Float32Array(30));
+		expect(frames).toHaveLength(3);
+		await src.stop();
+		src.push(new Float32Array(30)); // after stop — ignored
+		expect(frames).toHaveLength(3);
+		expect(src.running).toBe(false);
+	});
+
+	it("decodes PCM16 little-endian bytes", async () => {
+		const src = new PushMicSource({ frameSamples: 4 });
+		const frames: PcmFrame[] = [];
+		src.onFrame((f) => frames.push(f));
+		await src.start();
+		src.pushPcm16(
+			new Uint8Array(int16le([0, 16384, -16384, 32767, 0, 0, 0, 0])),
+		);
+		expect(frames).toHaveLength(2);
+		expect(frames[0].pcm[0]).toBeCloseTo(0);
+		expect(frames[0].pcm[1]).toBeCloseTo(0.5, 2);
+		expect(frames[0].pcm[2]).toBeCloseTo(-0.5, 2);
+		expect(frames[0].pcm[3]).toBeCloseTo(1, 1);
+		await src.stop();
+	});
+
+	it("surfaces a fatal producer error", async () => {
+		const src = new PushMicSource({ frameSamples: 8 });
+		const errors: Error[] = [];
+		src.onError((e) => errors.push(e));
+		await src.start();
+		src.fail(new Error("device lost"));
+		expect(errors).toHaveLength(1);
+		expect(src.running).toBe(false);
+	});
+});
+
+describe("pipeMicToRingBuffer", () => {
+	it("streams PCM frames into a ring buffer", async () => {
+		const src = new PushMicSource({ sampleRate: SR, frameSamples: 64 });
+		const sink = new InMemoryAudioSink();
+		const { ringBuffer, unsubscribe } = pipeMicToRingBuffer(src, sink, {
+			capacitySamples: 4096,
+		});
+		await src.start();
+		src.push(new Float32Array(64 * 5).fill(0.1));
+		expect(ringBuffer.size()).toBe(64 * 5);
+		unsubscribe();
+		src.push(new Float32Array(64).fill(0.1));
+		expect(ringBuffer.size()).toBe(64 * 5); // unsubscribed
+		await src.stop();
+	});
+});
+
+describe("DesktopMicSource", () => {
+	it("builds the arecord argv on Linux / sox on macOS", () => {
+		const src = new DesktopMicSource({ sampleRate: 16_000 });
+		const resolved = resolveDesktopRecorder(16_000);
+		const program = Reflect.get(src, "program") as string;
+		const argv = Reflect.get(src, "argv") as string[];
+		expect(program).toBe(resolved?.program ?? "");
+		expect(argv).toEqual(resolved?.argv ?? []);
+		if (process.platform === "linux" && program) {
+			expect(["arecord", "parec", "rec", "sox"]).toContain(program);
+			if (program === "arecord") {
+				expect(argv).toContain("S16_LE");
+			}
+			expect(argv.join(" ")).toContain("16000");
+		} else if (process.platform === "darwin") {
+			expect(["sox", "rec", "ffmpeg", ""]).toContain(program);
+			if (program === "ffmpeg") {
+				expect(argv).toContain("avfoundation");
+			} else if (program === "sox") {
+				expect(argv).toContain("-d");
+			}
+		}
+		expect(src.frameSamples).toBe(512); // 32 ms @ 16 kHz
+	});
+
+	it("re-frames raw PCM16 from the recorder subprocess into Float32 frames", async () => {
+		// Synthetic 'recorder': `cat` a raw PCM file → stdout. 256 samples per
+		// frame so the assertions are quick.
+		const dir = mkdtempSync(path.join(tmpdir(), "eliza-mic-"));
+		const rawPath = path.join(dir, "pcm.raw");
+		// 1024 samples = 4 frames of 256 @ 16 kHz/16 ms. A small recognizable
+		// pattern so we can verify byte ordering after the recorder->frame path.
+		const pattern = [0, 16384, -16384, 32767];
+		const raw: number[] = [];
+		for (let i = 0; i < 1024; i++) raw.push(pattern[i % 4]);
+		writeFileSync(rawPath, int16le(raw));
+
+		const src = new DesktopMicSource({
+			sampleRate: 16_000,
+			frameMs: 16, // 256 samples
+			program: "cat",
+			argv: [rawPath],
+		});
+		const frames: PcmFrame[] = [];
+		src.onFrame((f) => frames.push(f));
+		const errors: Error[] = [];
+		src.onError((e) => errors.push(e));
+		await src.start();
+		// `cat` writes the file to stdout then exits. (A real `arecord`/`sox`
+		// streams forever, so the source treats that exit as an error — fine
+		// for this test, which only cares that the byte→frame path works.)
+		// Wait for cat's exit to propagate: the exit event fires *after* all
+		// stdout data events, so once the error listener fires we know every
+		// frame has been emitted. A 2 s safety-net timeout prevents a hang if
+		// cat somehow never exits.
+		await Promise.race([
+			new Promise<void>((resolve) => {
+				const unsub = src.onError(() => {
+					unsub();
+					resolve();
+				});
+			}),
+			new Promise<void>((resolve) => setTimeout(resolve, 2000)),
+		]);
+		await src.stop();
+
+		expect(frames.length).toBeGreaterThanOrEqual(3);
+		expect(frames[0].pcm.length).toBe(256);
+		expect(frames[0].pcm[0]).toBeCloseTo(0);
+		expect(frames[0].pcm[1]).toBeCloseTo(0.5, 2);
+		expect(frames[0].pcm[2]).toBeCloseTo(-0.5, 2);
+		expect(frames[0].pcm[3]).toBeCloseTo(1, 1);
+		expect(frames[0].sampleRate).toBe(16_000);
+	});
+
+	it("surfaces a missing recorder binary as an error (no silent capture)", async () => {
+		const src = new DesktopMicSource({
+			program: "definitely-not-a-real-binary-xyz",
+			argv: [],
+		});
+		const errors: Error[] = [];
+		src.onError((e) => errors.push(e));
+		await src.start();
+		await new Promise((r) => setTimeout(r, 150));
+		await src.stop();
+		expect(errors.length).toBeGreaterThanOrEqual(1);
+		expect(errors.some((e) => /recorder/i.test(e.message))).toBe(true);
+	});
+
+	it("throws from start() when no CLI recorder is available for the platform", async () => {
+		// Simulate an unsupported platform by clearing program/argv.
+		const src = new DesktopMicSource({ program: "", argv: [] });
+		// On Linux/macOS the constructor still picks a default; force the empty
+		// path by overriding the private fields the way an unsupported platform
+		// would leave them.
+		(src as unknown as { program: string }).program = "";
+		(src as unknown as { argv: string[] }).argv = [];
+		await expect(src.start()).rejects.toThrow(/PushMicSource/);
+	});
+});