@elizaos/plugin-local-inference 2.0.0-beta.1 → 2.0.3-beta.3

package/src/services/voice/echo-reference-buffer.ts

@@ -0,0 +1,165 @@
+/**
+ * Echo-reference alignment buffer (#9583, follow-up to #9455/#9586).
+ *
+ * The NLMS echo canceller's `process(nearEnd, farEnd)` needs the far-end
+ * (the agent's TTS playback) **time-aligned** to the current mic frame: the
+ * echo in `nearEnd[t]` is the room-filtered playback from `t − delay`, where
+ * `delay` is the bulk playback→mic transport delay (estimated by
+ * {@link estimateEchoDelaySamples} in `echo-delay.ts`).
+ *
+ * The caller renders playback PCM in real time and `push()`es it here as it goes;
+ * per mic frame it then asks for the aligned far-end slice. This is the
+ * "caller must supply the reference" primitive the consumer seam was missing —
+ * a fixed-capacity delay line, pure logic (no FFI, no device, no audio I/O).
+ * Samples not yet rendered, or already evicted past capacity, are zero-filled
+ * (no echo reference ⇒ the adaptive filter simply has nothing to cancel there).
+ */
+
+export interface EchoReferenceBufferOptions {
+	/**
+	 * Ring-buffer capacity in samples. Must comfortably exceed
+	 * `maxDelaySamples + frameLength`. Default 24000 (1.5 s @ 16 kHz).
+	 */
+	capacitySamples?: number;
+	/** Sample rate for timestamp-based push/read helpers. Default 16000. */
+	sampleRateHz?: number;
+}
+
+export class EchoReferenceBuffer {
+	private readonly buffer: Float32Array;
+	private readonly valid: Uint8Array;
+	private readonly capacity: number;
+	private readonly sampleRateHz: number;
+	/** Total samples ever pushed (monotonic); the logical "now" cursor. */
+	private pushed = 0;
+	/** Timestamp mapped to absolute sample 0 for timestamp-aware playback. */
+	private originMs: number | null = null;
+
+	constructor(options: EchoReferenceBufferOptions = {}) {
+		this.capacity = Math.max(1, Math.floor(options.capacitySamples ?? 24000));
+		this.sampleRateHz = Math.max(1, Math.floor(options.sampleRateHz ?? 16_000));
+		this.buffer = new Float32Array(this.capacity);
+		this.valid = new Uint8Array(this.capacity);
+	}
+
+	/** Append rendered playback (far-end) PCM as it is produced. */
+	push(playback: Float32Array): void {
+		this.writeAt(this.pushed, playback);
+		this.pushed += playback.length;
+	}
+
+	/**
+	 * Store rendered playback at its capture/render timestamp. This preserves
+	 * real gaps between playback bursts instead of treating chunks as contiguous.
+	 */
+	pushAt(timestampMs: number, playback: Float32Array): void {
+		if (!Number.isFinite(timestampMs)) {
+			this.push(playback);
+			return;
+		}
+		if (this.originMs === null) this.originMs = timestampMs;
+		let start = this.sampleIndexFor(timestampMs);
+		if (start < 0) {
+			this.reset();
+			this.originMs = timestampMs;
+			start = 0;
+		}
+		this.writeAt(start, playback);
+		this.pushed = Math.max(this.pushed, start + playback.length);
+	}
+
+	/**
+	 * The far-end reference frame aligned to a mic frame of `length` samples
+	 * captured `delaySamples` after the corresponding playback. Returns the
+	 * playback window `[pushed − delaySamples − length, pushed − delaySamples)`.
+	 * Indices before the retained window (not yet pushed, or evicted past
+	 * capacity) are zero-filled.
+	 */
+	referenceFor(length: number, delaySamples: number): Float32Array {
+		const out = new Float32Array(Math.max(0, Math.floor(length)));
+		const delay = Math.max(0, Math.floor(delaySamples));
+		// Absolute index (in the monotonic stream) of the first output sample.
+		const start = this.pushed - delay - out.length;
+		return this.readWindow(start, out.length);
+	}
+
+	/**
+	 * Reference aligned to a mic frame starting at `timestampMs`.
+	 * Returns playback `[timestampMs - delay, timestampMs - delay + length)`.
+	 */
+	referenceAt(
+		timestampMs: number,
+		length: number,
+		delaySamples: number,
+	): Float32Array {
+		const outLength = Math.max(0, Math.floor(length));
+		if (this.originMs === null || !Number.isFinite(timestampMs)) {
+			return new Float32Array(outLength);
+		}
+		const delay = Math.max(0, Math.floor(delaySamples));
+		const start = this.sampleIndexFor(timestampMs) - delay;
+		return this.readWindow(start, outLength);
+	}
+
+	/** Samples pushed so far (the monotonic stream position). */
+	get position(): number {
+		return this.pushed;
+	}
+
+	/** Drop all buffered playback (e.g. on a new turn / barge-in flush). */
+	reset(): void {
+		this.buffer.fill(0);
+		this.valid.fill(0);
+		this.pushed = 0;
+		this.originMs = null;
+	}
+
+	private sampleIndexFor(timestampMs: number): number {
+		if (this.originMs === null) return 0;
+		return Math.round(
+			((timestampMs - this.originMs) / 1000) * this.sampleRateHz,
+		);
+	}
+
+	private writeAt(start: number, playback: Float32Array): void {
+		const safeStart = Math.max(0, Math.floor(start));
+		const end = safeStart + playback.length;
+		if (safeStart > this.pushed) this.clearRange(this.pushed, safeStart);
+		const oldest = Math.max(0, this.pushed - this.capacity);
+		if (end <= oldest) return;
+		for (let i = 0; i < playback.length; i++) {
+			const abs = safeStart + i;
+			if (abs < oldest) continue;
+			const slot = abs % this.capacity;
+			this.buffer[slot] = playback[i] ?? 0;
+			this.valid[slot] = 1;
+		}
+	}
+
+	private clearRange(from: number, to: number): void {
+		if (to <= from) return;
+		if (to - from >= this.capacity) {
+			this.buffer.fill(0);
+			this.valid.fill(0);
+			return;
+		}
+		for (let abs = Math.max(0, Math.floor(from)); abs < to; abs++) {
+			const slot = abs % this.capacity;
+			this.buffer[slot] = 0;
+			this.valid[slot] = 0;
+		}
+	}
+
+	private readWindow(start: number, length: number): Float32Array {
+		const out = new Float32Array(length);
+		const oldest = Math.max(0, this.pushed - this.capacity);
+		for (let i = 0; i < out.length; i++) {
+			const abs = start + i;
+			if (abs >= oldest && abs >= 0 && abs < this.pushed) {
+				const slot = abs % this.capacity;
+				if (this.valid[slot] === 1) out[i] = this.buffer[slot];
+			}
+		}
+		return out;
+	}
+}

package/src/services/voice/eliza1-eot-scorer.ts

@@ -0,0 +1,242 @@
+/**
+ * Eliza-1 EOT scorer — reuses the already-loaded text model to compute
+ * P(`<end_of_turn>` next | conversation_so_far) without shipping a separate
+ * detector ONNX.
+ *
+ * The runtime keeps a `LlamaModel` resident for chat generation. Voice
+ * EOT scoring is a single forward pass over the formatted Gemma chat
+ * prompt with the trailing `<end_of_turn>` removed. `capacitor-llama`'s
+ * `LlamaContextSequence.controlledEvaluate()` returns the next-token
+ * probability distribution, so we simply read the entry for the
+ * `<end_of_turn>` token id — no sampling loop, no KV-cache growth on the
+ * chat session.
+ *
+ * A dedicated `LlamaContext` is held just for this scorer so we do not
+ * fight the chat session pool for sequence slots. The context is small
+ * (single sequence, ≤512 tokens) and shares the model weights with the
+ * chat path; only the per-sequence KV cache is duplicated.
+ *
+ * Optionally a LoRA adapter is attached to that context — that is how
+ * a fine-tuned EOT head (trained per
+ * `packages/training/scripts/turn_detector/`) layers onto the same base
+ * weights without a separate GGUF.
+ */
+
+import path from "node:path";
+
+// `capacitor-llama` 3.18.1 surface we depend on. We avoid importing the
+// type directly so the binding stays an optional peer dep — callers pass
+// the model through a structural type.
+export interface LlamaModelLike {
+	tokenize(text: string, specialTokens?: boolean): readonly number[];
+	createContext(options: {
+		contextSize?: number;
+		sequences?: number;
+		flashAttention?: boolean;
+		lora?: string | { adapters: Array<{ filePath: string; scale?: number }> };
+	}): Promise<LlamaContextLike>;
+}
+
+export interface LlamaContextLike {
+	getSequence(options?: object): LlamaContextSequenceLike;
+	dispose(): Promise<void>;
+}
+
+export interface LlamaContextSequenceLike {
+	clearHistory(): Promise<void>;
+	controlledEvaluate(
+		input: ControlledEvaluateInputLike[],
+		options?: { evaluationPriority?: number },
+	): Promise<Array<ControlledEvaluateOutputLike | undefined>>;
+}
+
+export type ControlledEvaluateInputLike =
+	| number
+	| [
+			token: number,
+			options: {
+				generateNext?: {
+					probabilities?: boolean;
+					confidence?: boolean;
+				};
+			},
+	  ];
+
+export interface ControlledEvaluateOutputLike {
+	next: {
+		token?: number | null;
+		confidence?: number;
+		probabilities?: Map<number, number>;
+	};
+}
+
+const END_OF_TURN_TOKEN = "<end_of_turn>";
+const START_OF_TURN_USER_PREFIX = "<start_of_turn>user\n";
+
+export interface Eliza1EotScorerOptions {
+	/** The already-loaded text model (eliza-1 drafter). */
+	model: LlamaModelLike;
+	/** Optional EOT LoRA adapter path (absolute). When set, applied to the dedicated EOT context. */
+	loraPath?: string;
+	/** Adapter scale (default 1.0). Only meaningful when `loraPath` is set. */
+	loraScale?: number;
+	/** Max history tokens to keep in the prompt window. LiveKit uses 128. */
+	maxHistoryTokens?: number;
+	/** Context size for the dedicated EOT context. Default 512. */
+	contextSize?: number;
+	/** Model label for telemetry. */
+	modelLabel?: string;
+}
+
+export interface Eliza1EotScoreResult {
+	/** Probability of `<end_of_turn>` as the next token, ∈ [0, 1]. */
+	probability: number;
+	/** Wall-clock model latency for this scoring call. */
+	latencyMs: number;
+	/** Number of prompt tokens evaluated. */
+	promptTokens: number;
+}
+
+/**
+ * Stateful EOT scorer bound to a single loaded text model. Lazily
+ * allocates its own dedicated `LlamaContext` on first call. Safe to
+ * keep across many voice turns; call `dispose()` on shutdown.
+ */
+export class Eliza1EotScorer {
+	private readonly model: LlamaModelLike;
+	private readonly loraPath: string | undefined;
+	private readonly loraScale: number | undefined;
+	private readonly maxHistoryTokens: number;
+	private readonly contextSize: number;
+	readonly modelLabel: string;
+
+	private context: LlamaContextLike | null = null;
+	private sequence: LlamaContextSequenceLike | null = null;
+	private endOfTurnTokenId: number | null = null;
+	private initPromise: Promise<void> | null = null;
+	/** Serializes concurrent calls — controlledEvaluate is not thread-safe per-sequence. */
+	private inflight: Promise<unknown> = Promise.resolve();
+
+	constructor(options: Eliza1EotScorerOptions) {
+		this.model = options.model;
+		this.loraPath = options.loraPath;
+		this.loraScale = options.loraScale;
+		this.maxHistoryTokens = options.maxHistoryTokens ?? 128;
+		this.contextSize = options.contextSize ?? 512;
+		this.modelLabel =
+			options.modelLabel ??
+			(this.loraPath
+				? `eliza-1-drafter+eot-lora:${path.basename(this.loraPath)}`
+				: "eliza-1-drafter");
+	}
+
+	async score(partialTranscript: string): Promise<Eliza1EotScoreResult> {
+		await this.ensureReady();
+		const sequence = this.sequence;
+		const endOfTurnId = this.endOfTurnTokenId;
+		if (!sequence || endOfTurnId === null) {
+			throw new Error("[voice] Eliza1EotScorer not initialized.");
+		}
+
+		const tokens = this.tokenizePrompt(partialTranscript);
+		const start = performance.now();
+		const next = this.inflight.then(() =>
+			this.runOnce(sequence, tokens, endOfTurnId),
+		);
+		this.inflight = next.catch(() => undefined);
+		const probability = await next;
+		return {
+			probability,
+			latencyMs: performance.now() - start,
+			promptTokens: tokens.length,
+		};
+	}
+
+	async dispose(): Promise<void> {
+		const ctx = this.context;
+		this.context = null;
+		this.sequence = null;
+		this.endOfTurnTokenId = null;
+		this.initPromise = null;
+		if (ctx) await ctx.dispose();
+	}
+
+	private async ensureReady(): Promise<void> {
+		if (this.context && this.sequence && this.endOfTurnTokenId !== null) return;
+		if (!this.initPromise) this.initPromise = this.initialize();
+		await this.initPromise;
+	}
+
+	private async initialize(): Promise<void> {
+		const endOfTurnIds = this.model.tokenize(END_OF_TURN_TOKEN, true);
+		if (endOfTurnIds.length !== 1 || !Number.isInteger(endOfTurnIds[0])) {
+			throw new Error(
+				`[voice] Eliza1EotScorer: model tokenizer did not resolve <end_of_turn> to a single special token (got ${JSON.stringify(endOfTurnIds)}). The base model must be Gemma-template compatible.`,
+			);
+		}
+		this.endOfTurnTokenId = endOfTurnIds[0];
+
+		const contextOptions: Parameters<LlamaModelLike["createContext"]>[0] = {
+			contextSize: this.contextSize,
+			sequences: 1,
+			flashAttention: true,
+		};
+		if (this.loraPath) {
+			contextOptions.lora = {
+				adapters: [
+					{
+						filePath: this.loraPath,
+						...(this.loraScale !== undefined ? { scale: this.loraScale } : {}),
+					},
+				],
+			};
+		}
+
+		this.context = await this.model.createContext(contextOptions);
+		this.sequence = this.context.getSequence();
+	}
+
+	private tokenizePrompt(transcript: string): number[] {
+		const formatted = formatEotPrompt(transcript);
+		const ids = this.model.tokenize(formatted, true);
+		if (ids.length <= this.maxHistoryTokens) return [...ids];
+		return [...ids.slice(ids.length - this.maxHistoryTokens)];
+	}
+
+	private async runOnce(
+		sequence: LlamaContextSequenceLike,
+		tokens: number[],
+		endOfTurnId: number,
+	): Promise<number> {
+		if (tokens.length === 0) return 0.5;
+		await sequence.clearHistory();
+		const input: ControlledEvaluateInputLike[] = tokens.map((tok, i) =>
+			i === tokens.length - 1
+				? [tok, { generateNext: { probabilities: true } }]
+				: tok,
+		);
+		const out = await sequence.controlledEvaluate(input);
+		const last = out[tokens.length - 1];
+		const probs = last?.next.probabilities;
+		if (!probs) return 0.5;
+		const p = probs.get(endOfTurnId);
+		if (typeof p !== "number" || !Number.isFinite(p)) return 0.5;
+		return Math.max(0, Math.min(1, p));
+	}
+}
+
+/**
+ * Format the partial transcript using the Gemma chat template, with the
+ * trailing `<end_of_turn>` removed so the next predicted token *is* the
+ * EOT signal we want to measure.
+ *
+ * Matches the formatting LiveKit's turn-detector uses (single user turn,
+ * no system prompt, no generation prefix). When upstream history is
+ * available we can stack turns here, but the LiveKit recipe truncates
+ * to the last 128 tokens regardless, so a single user turn captures the
+ * relevant context for tier-1 EOT.
+ */
+export function formatEotPrompt(transcript: string): string {
+	const cleaned = transcript.trim();
+	return `${START_OF_TURN_USER_PREFIX}${cleaned}`;
+}

package/src/services/voice/embedding-server.ts

@@ -0,0 +1,200 @@
+/**
+ * Lazy embedding `llama-server` sidecar for Eliza-1 bundles.
+ *
+ * The route resolver decides which GGUF backs embeddings. This class owns the
+ * process boundary: validate the GGUF at construction, start the sidecar only
+ * on first non-empty `embed()`, and return Matryoshka-truncated vectors.
+ */
+
+import { type ChildProcess, spawn } from "node:child_process";
+import { existsSync } from "node:fs";
+import net from "node:net";
+import os from "node:os";
+import {
+	isValidEmbeddingDim,
+	type LocalEmbeddingRoute,
+	truncateMatryoshka,
+} from "./embedding";
+import { VoiceStartupError } from "./errors";
+
+interface EmbeddingServerConfig {
+	/** GGUF the sidecar mmaps. For the dedicated-region mode this is the `embedding/` file. */
+	modelPath: string;
+	/** Extra `llama-server` flags — the route's `embeddingServerFlags` (`--embeddings --pooling last`). */
+	serverFlags: ReadonlyArray<string>;
+	/** GPU offload: `"auto"` (= all layers) for CPU/Vulkan/CUDA hosts, `0` to force CPU. */
+	gpuLayers?: number | "auto";
+	/** Thread count for the embedding forward pass. Defaults to the host's logical core count. */
+	threads?: number;
+}
+
+export class EmbeddingServer {
+	private readonly config: EmbeddingServerConfig;
+	private child: ChildProcess | null = null;
+	private baseUrl: string | null = null;
+	private starting: Promise<void> | null = null;
+
+	constructor(config: EmbeddingServerConfig) {
+		if (!existsSync(config.modelPath)) {
+			throw new VoiceStartupError(
+				"missing-bundle-root",
+				`[embedding-server] model GGUF not found at ${config.modelPath}`,
+			);
+		}
+		this.config = config;
+	}
+
+	isRunning(): boolean {
+		return this.child !== null && this.child.exitCode === null;
+	}
+
+	async embed(texts: string[], dim = 1024): Promise<number[][]> {
+		if (texts.length === 0) return [];
+		if (!isValidEmbeddingDim(dim)) {
+			throw new Error(`[embedding] dim ${dim} is not a valid Matryoshka width`);
+		}
+		await this.ensureStarted();
+		const response = await fetch(`${this.baseUrl}/v1/embeddings`, {
+			method: "POST",
+			headers: { "content-type": "application/json" },
+			body: JSON.stringify({ input: texts }),
+		});
+		if (!response.ok) {
+			const body = await response.text().catch(() => "");
+			throw new Error(
+				`[embedding-server] /v1/embeddings returned ${response.status}: ${body.slice(0, 200)}`,
+			);
+		}
+		const payload = (await response.json()) as {
+			data?: Array<{ embedding?: number[] }>;
+		};
+		const rows = payload.data ?? [];
+		if (rows.length !== texts.length) {
+			throw new Error(
+				`[embedding-server] expected ${texts.length} embedding rows, got ${rows.length}`,
+			);
+		}
+		return rows.map((row, index) => {
+			if (!Array.isArray(row.embedding)) {
+				throw new Error(
+					`[embedding-server] response row ${index} missing embedding vector`,
+				);
+			}
+			return truncateMatryoshka(row.embedding, dim);
+		});
+	}
+
+	private async ensureStarted(): Promise<void> {
+		if (this.isRunning()) return;
+		this.starting ??= this.start().finally(() => {
+			this.starting = null;
+		});
+		await this.starting;
+	}
+
+	private async start(): Promise<void> {
+		const port = await reserveTcpPort();
+		const binary =
+			process.env.ELIZA_LLAMA_SERVER_PATH?.trim() || "llama-server";
+		const args = [
+			"-m",
+			this.config.modelPath,
+			...this.config.serverFlags,
+			"--host",
+			"127.0.0.1",
+			"--port",
+			String(port),
+			"--threads",
+			String(this.config.threads ?? Math.max(1, os.cpus().length)),
+		];
+		if (typeof this.config.gpuLayers === "number") {
+			args.push("--n-gpu-layers", String(this.config.gpuLayers));
+		}
+
+		const child = spawn(binary, args, {
+			stdio: ["ignore", "pipe", "pipe"],
+		});
+		this.child = child;
+		this.baseUrl = `http://127.0.0.1:${port}`;
+
+		child.once("exit", (code, signal) => {
+			if (this.child === child) this.child = null;
+			if (code !== 0 && code !== null) {
+				console.warn(
+					`[embedding-server] llama-server exited with code ${code}`,
+				);
+			} else if (signal) {
+				console.warn(`[embedding-server] llama-server exited on ${signal}`);
+			}
+		});
+
+		await this.waitUntilReady();
+	}
+
+	private async waitUntilReady(): Promise<void> {
+		const deadline = Date.now() + 20_000;
+		while (Date.now() < deadline) {
+			if (!this.isRunning()) {
+				throw new VoiceStartupError(
+					"missing-fused-build",
+					"[embedding-server] llama-server exited before /health became ready",
+				);
+			}
+			try {
+				const response = await fetch(`${this.baseUrl}/health`);
+				if (response.ok) return;
+			} catch {
+				// Server socket is not open yet.
+			}
+			await new Promise((resolve) => setTimeout(resolve, 100));
+		}
+		throw new VoiceStartupError(
+			"missing-fused-build",
+			"[embedding-server] timed out waiting for llama-server /health",
+		);
+	}
+
+	async stop(): Promise<void> {
+		const child = this.child;
+		if (!child) return;
+		this.child = null;
+		await new Promise<void>((resolve) => {
+			child.once("exit", () => resolve());
+			child.kill("SIGTERM");
+			setTimeout(() => {
+				if (child.exitCode === null) child.kill("SIGKILL");
+				resolve();
+			}, 2_000).unref();
+		});
+	}
+}
+
+export function embeddingServerForRoute(
+	route: LocalEmbeddingRoute,
+	opts: { gpuLayers?: number | "auto"; threads?: number } = {},
+): EmbeddingServer {
+	const modelPath =
+		route.source.kind === "pooled-text"
+			? route.source.textModelPath
+			: route.source.embeddingModelPath;
+	return new EmbeddingServer({
+		modelPath,
+		serverFlags: route.serverFlags,
+		gpuLayers: opts.gpuLayers,
+		threads: opts.threads,
+	});
+}
+
+async function reserveTcpPort(): Promise<number> {
+	return new Promise((resolve, reject) => {
+		const server = net.createServer();
+		server.once("error", reject);
+		server.listen(0, "127.0.0.1", () => {
+			const address = server.address();
+			server.close(() => {
+				if (address && typeof address === "object") resolve(address.port);
+				else reject(new Error("[embedding-server] failed to reserve TCP port"));
+			});
+		});
+	});
+}