@elizaos/plugin-local-inference 2.0.0-beta.1 → 2.0.3-beta.3

package/src/services/voice/embedding.test.ts

@@ -0,0 +1,131 @@
+/**
+ * Local-embedding route resolution + Matryoshka truncation (`embedding.ts`):
+ *   - `2b` (entry tier) → pooled-text source on the text backbone with
+ *     `--pooling last` (no separate GGUF)
+ *   - `4b` / `9b` / `27b` / `27b-256k` → dedicated `embedding/`
+ *     region; hard-fails when that region is missing (AGENTS.md §1 — do NOT
+ *     collapse to pooled text on the larger tiers; that breaks the 1024-dim
+ *     Matryoshka contract)
+ *   - every route guarantees 1024 dimensions and a sidecar embedding server
+ *     launched with `--embeddings --pooling last`
+ *   - `truncateMatryoshka` truncates 1024 → {64,128,256,512,768} and
+ *     L2-renormalizes; rejects invalid widths
+ */
+
+import { mkdirSync, mkdtempSync, writeFileSync } from "node:fs";
+import { tmpdir } from "node:os";
+import path from "node:path";
+import { describe, expect, it } from "vitest";
+import {
+	buildLocalEmbeddingRoute,
+	EMBEDDING_MATRYOSHKA_DIMS,
+	isValidEmbeddingDim,
+	resolveLocalEmbeddingSource,
+	truncateMatryoshka,
+} from "./embedding";
+
+function tmpBundle(): string {
+	return mkdtempSync(path.join(tmpdir(), "eliza-emb-"));
+}
+
+describe("resolveLocalEmbeddingSource", () => {
+	it("2b: reuses the text backbone with --pooling last (no duplicate GGUF)", () => {
+		const bundleRoot = tmpBundle();
+		const textPath = path.join(bundleRoot, "text", "eliza-1-2b-128k.gguf");
+		mkdirSync(path.dirname(textPath), { recursive: true });
+		writeFileSync(textPath, "gguf");
+		const src = resolveLocalEmbeddingSource({
+			bundleRoot,
+			tierId: "eliza-1-2b",
+			textModelPath: textPath,
+		});
+		expect(src.kind).toBe("pooled-text");
+		if (src.kind === "pooled-text") {
+			expect(src.textModelPath).toBe(textPath);
+			expect(src.poolingType).toBe("last");
+		}
+	});
+});
+
+describe("buildLocalEmbeddingRoute", () => {
+	it("2b route emits --embeddings --pooling last and guarantees 1024 dims", () => {
+		const bundleRoot = tmpBundle();
+		const textPath = path.join(bundleRoot, "text", "t.gguf");
+		mkdirSync(path.dirname(textPath), { recursive: true });
+		writeFileSync(textPath, "gguf");
+		const route = buildLocalEmbeddingRoute({
+			bundleRoot,
+			tierId: "eliza-1-2b",
+			textModelPath: textPath,
+		});
+		expect(route.dimensions).toBe(1024);
+		expect(route.defaultDim).toBe(1024);
+		expect(route.matryoshkaDims).toEqual(EMBEDDING_MATRYOSHKA_DIMS);
+		expect(route.serverFlags).toEqual(["--embeddings", "--pooling", "last"]);
+	});
+	it("accepts a smaller defaultDim and rejects invalid widths", () => {
+		const bundleRoot = tmpBundle();
+		const textPath = path.join(bundleRoot, "text", "t.gguf");
+		mkdirSync(path.dirname(textPath), { recursive: true });
+		writeFileSync(textPath, "gguf");
+		const route = buildLocalEmbeddingRoute({
+			bundleRoot,
+			tierId: "eliza-1-2b",
+			textModelPath: textPath,
+			defaultDim: 256,
+		});
+		expect(route.defaultDim).toBe(256);
+		expect(() =>
+			buildLocalEmbeddingRoute({
+				bundleRoot,
+				tierId: "eliza-1-2b",
+				textModelPath: textPath,
+				defaultDim: 300,
+			}),
+		).toThrow();
+	});
+});
+
+describe("truncateMatryoshka", () => {
+	it("EMBEDDING_MATRYOSHKA_DIMS = {64,128,256,512,768,1024}", () => {
+		expect([...EMBEDDING_MATRYOSHKA_DIMS]).toEqual([
+			64, 128, 256, 512, 768, 1024,
+		]);
+		expect(isValidEmbeddingDim(256)).toBe(true);
+		expect(isValidEmbeddingDim(1000)).toBe(false);
+	});
+
+	it("truncates to the leading slice and L2-renormalizes", () => {
+		// A 1024-dim vector that is unit-norm overall but whose leading 512
+		// components are not unit-norm on their own.
+		const full = new Array(1024).fill(0);
+		for (let i = 0; i < 1024; i += 1) full[i] = 1 / Math.sqrt(1024);
+		const half = truncateMatryoshka(full, 512);
+		expect(half).toHaveLength(512);
+		const norm = Math.sqrt(half.reduce((s, x) => s + x * x, 0));
+		expect(norm).toBeCloseTo(1, 6);
+		// Leading components preserved (up to the renormalization scale).
+		expect(half[0]).toBeCloseTo(1 / Math.sqrt(512), 6);
+	});
+
+	it("renormalizes even when dim == vec.length (raw last-token state may not be unit-norm)", () => {
+		const v = [3, 4]; // norm 5, not a valid Matryoshka width, but exercise the equal-length path via 64
+		const v64 = new Array(64)
+			.fill(0)
+			.map((_, i) => (i === 0 ? 3 : i === 1 ? 4 : 0));
+		const out = truncateMatryoshka(v64, 64);
+		const norm = Math.sqrt(out.reduce((s, x) => s + x * x, 0));
+		expect(norm).toBeCloseTo(1, 6);
+		void v;
+	});
+
+	it("rejects an invalid width or a too-short vector", () => {
+		expect(() => truncateMatryoshka(new Array(1024).fill(0.1), 300)).toThrow();
+		expect(() => truncateMatryoshka(new Array(128).fill(0.1), 256)).toThrow();
+	});
+
+	it("passes a zero vector through unchanged", () => {
+		const z = truncateMatryoshka(new Array(1024).fill(0), 64);
+		expect(z.every((x) => x === 0)).toBe(true);
+	});
+});

package/src/services/voice/embedding.ts

@@ -0,0 +1,242 @@
+/**
+ * Local embedding wiring for Eliza-1 bundles.
+ *
+ * Per `packages/inference/AGENTS.md` §1:
+ *   - On the `2b` tier (the entry/floor tier) the **embedding model IS the
+ *     text backbone**, served with `--pooling last` — there is no separate
+ *     `embedding/` GGUF and no duplicate parameters on the entry tier.
+ *   - On `4b` and larger tiers, a dedicated
+ *     `embedding/` GGUF region (Apache-2.0,
+ *     1024-dim Matryoshka, 32k ctx) is acquired lazily through the same
+ *     engine / `SharedResourceRegistry`. **Do not collapse it to pooled
+ *     text on the larger tiers** — that breaks the 1024-dim Matryoshka
+ *     contract (B1's verdict).
+ *
+ * This module is a pure resolver: given a bundle root + tier id it
+ * describes *where* embeddings come from (the text GGUF with a pooling
+ * flag, or a separate region file) without doing any I/O beyond an
+ * `existsSync`. The engine consumes the descriptor to mount the region
+ * and the local-embedding route. It also owns the Matryoshka-truncation
+ * helper that callers / the vector store use to trade dimensionality for
+ * storage (see `EMBEDDING_MATRYOSHKA_DIMS` + `truncateMatryoshka`).
+ */
+
+import { existsSync, readdirSync } from "node:fs";
+import path from "node:path";
+import type { Eliza1TierId } from "../catalog";
+import { VoiceStartupError } from "./errors";
+
+/** Bundle-relative directory holding a dedicated embedding GGUF (larger tiers). */
+export const EMBEDDING_DIR_REL_PATH = "embedding";
+
+/** Full output dimensionality of the Eliza-1 embedding model. */
+export const EMBEDDING_FULL_DIM = 1024 as const;
+
+/**
+ * Valid Matryoshka truncation points for the Eliza-1 embedding region. The model
+ * is trained so that the leading N components of the 1024-dim vector are
+ * themselves a usable embedding at these widths; quality degrades
+ * gracefully as N shrinks (see the tradeoff table in
+ * `reports/porting/2026-05-11/embedding-model-review.md`).
+ *
+ * 1024 (full) → 768 → 512 → 256 → 128 → 64. Smaller widths than 64 are
+ * not part of the published contract.
+ */
+export const EMBEDDING_MATRYOSHKA_DIMS: readonly number[] = [
+	64, 128, 256, 512, 768, 1024,
+];
+
+/** Type-narrow guard for `EMBEDDING_MATRYOSHKA_DIMS`. */
+export function isValidEmbeddingDim(dim: number): boolean {
+	return EMBEDDING_MATRYOSHKA_DIMS.includes(dim);
+}
+
+/**
+ * Truncate a full 1024-dim embedding to one of the Matryoshka widths and
+ * L2-renormalize. Renormalization matters: the dedicated embedding outputs are
+ * unit-norm at 1024 dims, but the leading slice is *not* unit-norm, and
+ * downstream cosine-similarity / dot-product retrieval assumes unit
+ * vectors.
+ *
+ * Throws on an invalid `dim` (must be one of `EMBEDDING_MATRYOSHKA_DIMS`)
+ * or when `vec` is shorter than `dim` — no silent truncation-to-whatever
+ * or zero-padding (Commandment 8: don't hide a broken pipeline).
+ */
+export function truncateMatryoshka(
+	vec: readonly number[],
+	dim: number,
+): number[] {
+	if (!isValidEmbeddingDim(dim)) {
+		throw new Error(
+			`[embedding] dim ${dim} is not a valid Matryoshka width; expected one of ${EMBEDDING_MATRYOSHKA_DIMS.join(", ")}`,
+		);
+	}
+	if (vec.length < dim) {
+		throw new Error(
+			`[embedding] cannot truncate a ${vec.length}-dim vector to ${dim} dims`,
+		);
+	}
+	if (vec.length === dim) {
+		// Already the requested width; still renormalize so a caller passing a
+		// raw last-token state (which may not be unit-norm) gets a clean vec.
+		return l2Normalize(vec.slice());
+	}
+	return l2Normalize(vec.slice(0, dim));
+}
+
+/** L2-normalize in place; returns the same array. Zero vectors pass through. */
+function l2Normalize(vec: number[]): number[] {
+	let sumSq = 0;
+	for (const x of vec) sumSq += x * x;
+	if (sumSq === 0) return vec;
+	const inv = 1 / Math.sqrt(sumSq);
+	for (let i = 0; i < vec.length; i += 1) vec[i] *= inv;
+	return vec;
+}
+
+export type LocalEmbeddingSource =
+	| {
+			/** `2b` (entry tier): reuse the text backbone GGUF; serve with `--pooling last`. */
+			readonly kind: "pooled-text";
+			readonly textModelPath: string;
+			readonly poolingType: "last";
+	  }
+	| {
+			/** Larger tiers: a dedicated `embedding/<name>.gguf` region. */
+			readonly kind: "dedicated-region";
+			readonly embeddingModelPath: string;
+			/** 1024-dim Matryoshka (the published Eliza-1 embedding contract). */
+			readonly dimensions: typeof EMBEDDING_FULL_DIM;
+			/**
+			 * The dedicated model already ships a contrastive `last`-token
+			 * pooling head — `--pooling last` is still passed so llama-server
+			 * doesn't fall back to the GGUF's metadata default. The model's own
+			 * pooling layer dominates; this just pins the read.
+			 */
+			readonly poolingType: "last";
+	  };
+
+/** First regular `.gguf` file under `dir`, or null. */
+function firstGguf(dir: string): string | null {
+	if (!existsSync(dir)) return null;
+	for (const entry of readdirSync(dir, { withFileTypes: true })) {
+		if (entry.isFile() && /\.gguf$/i.test(entry.name)) {
+			return path.join(dir, entry.name);
+		}
+	}
+	return null;
+}
+
+/**
+ * Tiers whose embedding model is the text backbone with `--pooling last`
+ * (no separate GGUF). The default mobile tiers deliberately avoid duplicate
+ * parameters; larger tiers may use a dedicated embedding region.
+ */
+export const POOLED_TEXT_EMBEDDING_TIERS: ReadonlySet<Eliza1TierId> = new Set([
+	"eliza-1-2b",
+]);
+
+/**
+ * Resolve the embedding source for an activated Eliza-1 bundle.
+ *
+ * @param bundleRoot   Bundle directory on disk.
+ * @param tierId       The Eliza-1 tier id (`eliza-1-2b`, ...).
+ * @param textModelPath Absolute path of the activated text GGUF (needed for
+ *                      the `pooled-text` case).
+ *
+ * Hard-fails (AGENTS.md §3) when a larger tier is missing its
+ * `embedding/` region — no silent fallback to pooled text, which would
+ * regress dimensions from 1024 to whatever the text model emits.
+ */
+export function resolveLocalEmbeddingSource(args: {
+	bundleRoot: string;
+	tierId: Eliza1TierId;
+	textModelPath: string;
+}): LocalEmbeddingSource {
+	if (POOLED_TEXT_EMBEDDING_TIERS.has(args.tierId)) {
+		if (!existsSync(args.textModelPath)) {
+			throw new VoiceStartupError(
+				"missing-bundle-root",
+				`[embedding] ${args.tierId}: text model not found at ${args.textModelPath} — cannot serve pooled-text embeddings.`,
+			);
+		}
+		return {
+			kind: "pooled-text",
+			textModelPath: args.textModelPath,
+			poolingType: "last",
+		};
+	}
+	const dir = path.join(args.bundleRoot, EMBEDDING_DIR_REL_PATH);
+	const gguf = firstGguf(dir);
+	if (!gguf) {
+		throw new VoiceStartupError(
+			"missing-bundle-root",
+			`[embedding] ${args.tierId}: required dedicated embedding region missing under ${dir}. Tiers above 2b ship a separate 1024-dim Matryoshka embedding GGUF (AGENTS.md §1) — do not fall back to pooled text.`,
+		);
+	}
+	return {
+		kind: "dedicated-region",
+		embeddingModelPath: gguf,
+		dimensions: EMBEDDING_FULL_DIM,
+		poolingType: "last",
+	};
+}
+
+/**
+ * Descriptor for the local-embedding route the engine exposes. The
+ * route's job is `text[] → number[dim][]`; the runtime mounts the source
+ * (pooled text or dedicated region) and forwards. Kept as a plain data
+ * shape so both the API layer and tests can assert it without standing up
+ * a server.
+ */
+export interface LocalEmbeddingRoute {
+	readonly tierId: Eliza1TierId;
+	readonly source: LocalEmbeddingSource;
+	/** Full output dimensionality the route produces before truncation. 1024 on every tier. */
+	readonly dimensions: typeof EMBEDDING_FULL_DIM;
+	/**
+	 * Default Matryoshka width the route returns when a caller does not ask
+	 * for a smaller `dim`. Always 1024 (= `dimensions`) — callers/the vector
+	 * store opt into a smaller width for storage savings.
+	 */
+	readonly defaultDim: number;
+	/** The Matryoshka widths a caller may request. */
+	readonly matryoshkaDims: readonly number[];
+	/**
+	 * `llama-server` flags for the embedding server process — always
+	 * `--embeddings --pooling last`. The embedding server is a lazily-started
+	 * sidecar over the route's GGUF (the text backbone on the `2b` entry tier,
+	 * the `embedding/` GGUF on larger tiers); see `embedding-server.ts`. The
+	 * chat `llama-server` is left untouched (completions-only) — these flags
+	 * do NOT go on it.
+	 */
+	readonly serverFlags: ReadonlyArray<string>;
+}
+
+export function buildLocalEmbeddingRoute(args: {
+	bundleRoot: string;
+	tierId: Eliza1TierId;
+	textModelPath: string;
+	/** Default output width; must be one of `EMBEDDING_MATRYOSHKA_DIMS`. Defaults to 1024. */
+	defaultDim?: number;
+}): LocalEmbeddingRoute {
+	const source = resolveLocalEmbeddingSource(args);
+	const defaultDim = args.defaultDim ?? EMBEDDING_FULL_DIM;
+	if (!isValidEmbeddingDim(defaultDim)) {
+		throw new Error(
+			`[embedding] defaultDim ${defaultDim} is not a valid Matryoshka width; expected one of ${EMBEDDING_MATRYOSHKA_DIMS.join(", ")}`,
+		);
+	}
+	// Both modes serve through a sidecar `llama-server --embeddings --pooling
+	// last` (over the text GGUF on the 2b entry tier, over the embedding/ GGUF
+	// on larger tiers). The chat server is never given these flags.
+	const serverFlags = ["--embeddings", "--pooling", source.poolingType];
+	return {
+		tierId: args.tierId,
+		source,
+		dimensions: EMBEDDING_FULL_DIM,
+		defaultDim,
+		matryoshkaDims: EMBEDDING_MATRYOSHKA_DIMS,
+		serverFlags,
+	};
+}

package/src/services/voice/emotion-attribution.test.ts

@@ -0,0 +1,129 @@
+import { describe, expect, it } from "vitest";
+import { attributeVoiceEmotion } from "./emotion-attribution";
+import { WAV2SMALL_INT8_MODEL_ID } from "./voice-emotion-classifier";
+
+describe("emotion-attribution", () => {
+	it("prefers explicit expressive text tags and marks attribution as non-native", () => {
+		const result = attributeVoiceEmotion({
+			text: "[excited] That is amazing news!",
+			audio: { durationMs: 1200, rms: 0.2, zeroCrossingRate: 0.08 },
+		});
+
+		expect(result.emotion).toBe("excited");
+		expect(result.method).toBe("text_tag");
+		expect(result.modelNativeEmotion).toBe(false);
+		expect(result.evidence[0]).toMatchObject({
+			source: "text_expressive_tag",
+			detail: "[excited]",
+		});
+	});
+
+	it("uses ASR transcript and audio features without claiming model-native emotion labels", () => {
+		const result = attributeVoiceEmotion({
+			asr: {
+				transcript: "I am worried this might break",
+				confidence: 0.91,
+				emotionLabel: "anger",
+				emotionLabelSupported: false,
+			},
+			audio: {
+				durationMs: 1100,
+				rms: 0.22,
+				zeroCrossingRate: 0.11,
+				speechRateWpm: 180,
+			},
+		});
+
+		expect(result.emotion).toBe("nervous");
+		expect(result.method).toBe("text_audio_heuristic");
+		expect(result.modelNativeEmotion).toBe(false);
+		expect(result.evidence).toContainEqual({
+			source: "asr_emotion_metadata_ignored",
+			detail: "anger",
+			confidence: 0,
+		});
+		expect(result.evidence.some((row) => row.source === "asr_transcript")).toBe(
+			true,
+		);
+	});
+
+	it("can use explicitly supported ASR emotion metadata but still labels it as metadata", () => {
+		const result = attributeVoiceEmotion({
+			asr: {
+				transcript: "I am okay",
+				emotionLabel: "happiness",
+				emotionLabelSupported: true,
+			},
+		});
+
+		expect(result.emotion).toBe("happy");
+		expect(result.method).toBe("explicit_asr_metadata");
+		expect(result.modelNativeEmotion).toBe(false);
+		expect(
+			result.evidence.some((row) => row.source === "asr_emotion_metadata"),
+		).toBe(true);
+	});
+
+	it("fuses acoustic classifier output with text evidence and reports vad + acoustic_text_fused", () => {
+		const result = attributeVoiceEmotion({
+			text: "I am furious about this!",
+			model: {
+				output: {
+					vad: { valence: 0.15, arousal: 0.85, dominance: 0.8 },
+					emotion: "angry",
+					confidence: 0.82,
+					scores: {
+						happy: 0.05,
+						sad: 0.1,
+						angry: 0.82,
+						nervous: 0.2,
+						calm: 0.0,
+						excited: 0.3,
+						whisper: 0.0,
+					},
+					modelId: WAV2SMALL_INT8_MODEL_ID,
+					latencyMs: 4,
+				},
+			},
+		});
+
+		expect(result.emotion).toBe("angry");
+		expect(result.method).toBe("acoustic_text_fused");
+		expect(result.vad).toMatchObject({
+			valence: 0.15,
+			arousal: 0.85,
+			dominance: 0.8,
+		});
+		expect(result.evidence.some((row) => row.source === "acoustic_model")).toBe(
+			true,
+		);
+	});
+
+	it("attributes acoustic_model alone when only the classifier output is provided", () => {
+		const result = attributeVoiceEmotion({
+			model: {
+				output: {
+					vad: { valence: 0.1, arousal: 0.1, dominance: 0.2 },
+					emotion: "sad",
+					confidence: 0.7,
+					scores: {
+						happy: 0.0,
+						sad: 0.7,
+						angry: 0.05,
+						nervous: 0.1,
+						calm: 0.1,
+						excited: 0.0,
+						whisper: 0.05,
+					},
+					modelId: WAV2SMALL_INT8_MODEL_ID,
+					latencyMs: 3,
+				},
+			},
+		});
+
+		expect(result.emotion).toBe("sad");
+		expect(result.method).toBe("acoustic_model");
+		expect(result.modelNativeEmotion).toBe(false);
+		expect(result.vad).toBeDefined();
+	});
+});