@elizaos/plugin-local-inference 2.0.0-beta.1 → 2.0.3-beta.2

package/src/services/voice/voice-preset-generator.test.ts

@@ -0,0 +1,89 @@
+import { execFileSync } from "node:child_process";
+import { existsSync, mkdtempSync, readFileSync, rmSync } from "node:fs";
+import { tmpdir } from "node:os";
+import path from "node:path";
+import { fileURLToPath } from "node:url";
+import { afterEach, beforeEach, describe, expect, it } from "vitest";
+import {
+	readVoicePresetFile,
+	VoicePresetFormatError,
+} from "./voice-preset-format";
+
+const HERE = path.dirname(fileURLToPath(import.meta.url));
+// .../plugins/plugin-local-inference/src/services/voice -> repo root
+const REPO_ROOT = path.resolve(HERE, "../../../../..");
+const APP_CORE_ROOT = path.join(REPO_ROOT, "packages", "app-core");
+const SCRIPT = path.join(
+	APP_CORE_ROOT,
+	"scripts",
+	"voice-preset",
+	"build-default-voice-preset.mjs",
+);
+
+function runGenerator(args: string[]): string {
+	return execFileSync("bun", [SCRIPT, ...args], {
+		cwd: APP_CORE_ROOT,
+		encoding: "utf8",
+		stdio: ["ignore", "pipe", "pipe"],
+	});
+}
+
+describe("build-default-voice-preset.mjs", () => {
+	let dir: string;
+
+	beforeEach(() => {
+		dir = mkdtempSync(path.join(tmpdir(), "eliza-voice-preset-gen-"));
+	});
+
+	afterEach(() => {
+		rmSync(dir, { recursive: true, force: true });
+	});
+
+	it("--placeholder writes a format-valid .bin that round-trips through readVoicePresetFile", () => {
+		expect(existsSync(SCRIPT)).toBe(true);
+		const out = path.join(dir, "voice-preset-default.bin");
+		const stdout = runGenerator(["--placeholder", "--out", out]);
+		expect(stdout).toMatch(/PLACEHOLDER/);
+		expect(existsSync(out)).toBe(true);
+
+		const parsed = readVoicePresetFile(new Uint8Array(readFileSync(out)));
+		expect(parsed.version).toBe(1);
+		// Default placeholder embedding dim is 256, all zeros.
+		expect(parsed.embedding.length).toBe(256);
+		expect(parsed.embedding.every((x) => x === 0)).toBe(true);
+		// Placeholder carries no audio.
+		expect(parsed.phrases).toHaveLength(0);
+	});
+
+	it("--placeholder --dim N honours the embedding dimension", () => {
+		const out = path.join(dir, "p.bin");
+		runGenerator(["--placeholder", "--dim", "64", "--out", out]);
+		const parsed = readVoicePresetFile(new Uint8Array(readFileSync(out)));
+		expect(parsed.embedding.length).toBe(64);
+	});
+
+	it("refuses to build a real preset without an embedding (exit 2, guidance message)", () => {
+		let threw = false;
+		try {
+			execFileSync("bun", [SCRIPT], {
+				cwd: APP_CORE_ROOT,
+				encoding: "utf8",
+				stdio: ["ignore", "pipe", "pipe"],
+			});
+		} catch (err) {
+			threw = true;
+			const e = err as { status?: number; stderr?: string };
+			expect(e.status).toBe(2);
+			expect(e.stderr ?? "").toMatch(/--embedding/);
+			expect(e.stderr ?? "").toMatch(/--placeholder/);
+		}
+		expect(threw).toBe(true);
+	});
+
+	// Sanity: a truncated file is rejected by the parser the generator uses.
+	it("the parser the generator targets rejects a truncated blob", () => {
+		expect(() => readVoicePresetFile(new Uint8Array([1, 2, 3]))).toThrow(
+			VoicePresetFormatError,
+		);
+	});
+});

package/src/services/voice/voice-profile-artifact.d.ts

@@ -0,0 +1,116 @@
+import type { VoiceInputSource } from "./types";
+export declare const VOICE_PROFILE_ARTIFACT_SCHEMA_VERSION: "eliza.voice_profile.v1";
+export declare const VOICE_PROFILE_FEATURE_EMBEDDING_MODEL: "eliza-voice-profile-features-v1";
+export type VoiceProfileArtifactStatus = "ready" | "needs_review" | "invalid";
+export interface VoiceProfileConsent {
+    attribution: boolean;
+    synthesis: boolean;
+    grantedBy?: string;
+    grantedAt?: string;
+    expiresAt?: string;
+    evidenceId?: string;
+}
+export interface VoiceProfileReferenceMetadata {
+    speakerId?: string;
+    label?: string;
+    displayName?: string;
+    referenceText?: string;
+    language?: string;
+    locale?: string;
+    source?: VoiceInputSource;
+    consent?: VoiceProfileConsent;
+    metadata?: Record<string, unknown>;
+}
+export interface VoiceProfileSampleInput {
+    id?: string;
+    wavBytes: ArrayBuffer | ArrayBufferView;
+    referenceText?: string;
+    recordedAt?: string;
+    source?: VoiceInputSource;
+    metadata?: Record<string, unknown>;
+}
+export interface VoiceProfileAudioFeatures {
+    sha256: string;
+    byteLength: number;
+    format: "wav/pcm_s16le";
+    channels: number;
+    sampleRateHz: number;
+    bitsPerSample: 16;
+    dataBytes: number;
+    samplesPerChannel: number;
+    durationMs: number;
+    peakAbs: number;
+    rms: number;
+    zeroCrossingRate: number;
+    silenceRatio: number;
+}
+export interface VoiceProfileArtifactSample {
+    id: string;
+    wavSha256: string;
+    referenceText: string | null;
+    recordedAt: string | null;
+    source: VoiceInputSource | null;
+    metadata: Record<string, unknown> | null;
+    audio: VoiceProfileAudioFeatures;
+    featureEmbedding: number[];
+}
+export interface VoiceProfileArtifact {
+    schemaVersion: typeof VOICE_PROFILE_ARTIFACT_SCHEMA_VERSION;
+    artifactId: string;
+    embeddingModel: typeof VOICE_PROFILE_FEATURE_EMBEDDING_MODEL;
+    centroidEmbedding: number[];
+    sampleCount: number;
+    totalDurationMs: number;
+    confidence: number;
+    reference: {
+        speakerId: string | null;
+        label: string | null;
+        displayName: string | null;
+        referenceText: string | null;
+        language: string | null;
+        locale: string | null;
+        source: VoiceInputSource | null;
+        consent: VoiceProfileConsent;
+        metadata: Record<string, unknown> | null;
+    };
+    samples: VoiceProfileArtifactSample[];
+    usage: {
+        attributionAuthorized: boolean;
+        synthesisAuthorized: boolean;
+        authorizationSource: "reference_metadata";
+    };
+    provenance: {
+        createdAt: string | null;
+        deterministic: true;
+        generator: "app-core.voice-profile-artifact";
+    };
+}
+export interface VoiceProfileArtifactVerification {
+    status: VoiceProfileArtifactStatus;
+    artifactId: string;
+    expectedArtifactId: string;
+    artifactIdMatches: boolean;
+    sampleCount: number;
+    totalDurationMs: number;
+    attributionStatus: "ready" | "missing_consent" | "invalid";
+    synthesisStatus: "authorized_by_metadata" | "not_authorized" | "insufficient_audio" | "invalid";
+    issues: string[];
+    samples: Array<{
+        id: string;
+        status: "pass" | "fail";
+        issues: string[];
+        wavSha256Matches?: boolean;
+    }>;
+}
+export declare function canonicalVoiceProfileJson(value: unknown): string;
+export declare function analyzeVoiceProfileWav(wavBytes: ArrayBuffer | ArrayBufferView): VoiceProfileAudioFeatures;
+export declare function createVoiceProfileArtifact(args: {
+    samples: readonly VoiceProfileSampleInput[];
+    reference?: VoiceProfileReferenceMetadata;
+    createdAt?: string | null;
+}): VoiceProfileArtifact;
+export declare function verifyVoiceProfileArtifact(args: {
+    artifact: VoiceProfileArtifact;
+    sampleWavs?: Readonly<Record<string, ArrayBuffer | ArrayBufferView>>;
+}): VoiceProfileArtifactVerification;
+//# sourceMappingURL=voice-profile-artifact.d.ts.map

package/src/services/voice/voice-profile-artifact.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"voice-profile-artifact.d.ts","sourceRoot":"","sources":["voice-profile-artifact.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,SAAS,CAAC;AAEhD,eAAO,MAAM,qCAAqC,EACjD,wBAAiC,CAAC;AAEnC,eAAO,MAAM,qCAAqC,EACjD,iCAA0C,CAAC;AAE5C,MAAM,MAAM,0BAA0B,GAAG,OAAO,GAAG,cAAc,GAAG,SAAS,CAAC;AAE9E,MAAM,WAAW,mBAAmB;IACnC,WAAW,EAAE,OAAO,CAAC;IACrB,SAAS,EAAE,OAAO,CAAC;IACnB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,UAAU,CAAC,EAAE,MAAM,CAAC;CACpB;AAED,MAAM,WAAW,6BAA6B;IAC7C,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,MAAM,CAAC,EAAE,gBAAgB,CAAC;IAC1B,OAAO,CAAC,EAAE,mBAAmB,CAAC;IAC9B,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CACnC;AAED,MAAM,WAAW,uBAAuB;IACvC,EAAE,CAAC,EAAE,MAAM,CAAC;IACZ,QAAQ,EAAE,WAAW,GAAG,eAAe,CAAC;IACxC,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,MAAM,CAAC,EAAE,gBAAgB,CAAC;IAC1B,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CACnC;AAED,MAAM,WAAW,yBAAyB;IACzC,MAAM,EAAE,MAAM,CAAC;IACf,UAAU,EAAE,MAAM,CAAC;IACnB,MAAM,EAAE,eAAe,CAAC;IACxB,QAAQ,EAAE,MAAM,CAAC;IACjB,YAAY,EAAE,MAAM,CAAC;IACrB,aAAa,EAAE,EAAE,CAAC;IAClB,SAAS,EAAE,MAAM,CAAC;IAClB,iBAAiB,EAAE,MAAM,CAAC;IAC1B,UAAU,EAAE,MAAM,CAAC;IACnB,OAAO,EAAE,MAAM,CAAC;IAChB,GAAG,EAAE,MAAM,CAAC;IACZ,gBAAgB,EAAE,MAAM,CAAC;IACzB,YAAY,EAAE,MAAM,CAAC;CACrB;AAED,MAAM,WAAW,0BAA0B;IAC1C,EAAE,EAAE,MAAM,CAAC;IACX,SAAS,EAAE,MAAM,CAAC;IAClB,aAAa,EAAE,MAAM,GAAG,IAAI,CAAC;IAC7B,UAAU,EAAE,MAAM,GAAG,IAAI,CAAC;IAC1B,MAAM,EAAE,gBAAgB,GAAG,IAAI,CAAC;IAChC,QAAQ,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,IAAI,CAAC;IACzC,KAAK,EAAE,yBAAyB,CAAC;IACjC,gBAAgB,EAAE,MAAM,EAAE,CAAC;CAC3B;AAED,MAAM,WAAW,oBAAoB;IACpC,aAAa,EAAE,OAAO,qCAAqC,CAAC;IAC5D,UAAU,EAAE,MAAM,CAAC;IACnB,cAAc,EAAE,OAAO,qCAAqC,CAAC;IAC7D,iBAAiB,EAAE,MAAM,EAAE,CAAC;IAC5B,WAAW,EAAE,MAAM,CAAC;IACpB,eAAe,EAAE,MAAM,CAAC;IACxB,UAAU,EAAE,MAAM,CAAC;IACnB,SAAS,EAAE;QACV,SAAS,EAAE,MAAM,GAAG,IAAI,CAAC;QACzB,KAAK,EAAE,MAAM,GAAG,IAAI,CAAC;QACrB,WAAW,EAAE,MAAM,GAAG,IAAI,CAAC;QAC3B,aAAa,EAAE,MAAM,GAAG,IAAI,CAAC;QAC7B,QAAQ,EAAE,MAAM,GAAG,IAAI,CAAC;QACxB,MAAM,EAAE,MAAM,GAAG,IAAI,CAAC;QACtB,MAAM,EAAE,gBAAgB,GAAG,IAAI,CAAC;QAChC,OAAO,EAAE,mBAAmB,CAAC;QAC7B,QAAQ,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,IAAI,CAAC;KACzC,CAAC;IACF,OAAO,EAAE,0BAA0B,EAAE,CAAC;IACtC,KAAK,EAAE;QACN,qBAAqB,EAAE,OAAO,CAAC;QAC/B,mBAAmB,EAAE,OAAO,CAAC;QAC7B,mBAAmB,EAAE,oBAAoB,CAAC;KAC1C,CAAC;IACF,UAAU,EAAE;QACX,SAAS,EAAE,MAAM,GAAG,IAAI,CAAC;QACzB,aAAa,EAAE,IAAI,CAAC;QACpB,SAAS,EAAE,iCAAiC,CAAC;KAC7C,CAAC;CACF;AAED,MAAM,WAAW,gCAAgC;IAChD,MAAM,EAAE,0BAA0B,CAAC;IACnC,UAAU,EAAE,MAAM,CAAC;IACnB,kBAAkB,EAAE,MAAM,CAAC;IAC3B,iBAAiB,EAAE,OAAO,CAAC;IAC3B,WAAW,EAAE,MAAM,CAAC;IACpB,eAAe,EAAE,MAAM,CAAC;IACxB,iBAAiB,EAAE,OAAO,GAAG,iBAAiB,GAAG,SAAS,CAAC;IAC3D,eAAe,EACZ,wBAAwB,GACxB,gBAAgB,GAChB,oBAAoB,GACpB,SAAS,CAAC;IACb,MAAM,EAAE,MAAM,EAAE,CAAC;IACjB,OAAO,EAAE,KAAK,CAAC;QACd,EAAE,EAAE,MAAM,CAAC;QACX,MAAM,EAAE,MAAM,GAAG,MAAM,CAAC;QACxB,MAAM,EAAE,MAAM,EAAE,CAAC;QACjB,gBAAgB,CAAC,EAAE,OAAO,CAAC;KAC3B,CAAC,CAAC;CACH;AAgED,wBAAgB,yBAAyB,CAAC,KAAK,EAAE,OAAO,GAAG,MAAM,CAEhE;AAUD,wBAAgB,sBAAsB,CACrC,QAAQ,EAAE,WAAW,GAAG,eAAe,GACrC,yBAAyB,CA2F3B;AAyCD,wBAAgB,0BAA0B,CAAC,IAAI,EAAE;IAChD,OAAO,EAAE,SAAS,uBAAuB,EAAE,CAAC;IAC5C,SAAS,CAAC,EAAE,6BAA6B,CAAC;IAC1C,SAAS,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;CAC1B,GAAG,oBAAoB,CA8FvB;AAED,wBAAgB,0BAA0B,CAAC,IAAI,EAAE;IAChD,QAAQ,EAAE,oBAAoB,CAAC;IAC/B,UAAU,CAAC,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,WAAW,GAAG,eAAe,CAAC,CAAC,CAAC;CACrE,GAAG,gCAAgC,CAoFnC"}

package/src/services/voice/voice-profile-artifact.test.ts

@@ -0,0 +1,138 @@
+import { describe, expect, it } from "vitest";
+import {
+	analyzeVoiceProfileWav,
+	createVoiceProfileArtifact,
+	verifyVoiceProfileArtifact,
+} from "./voice-profile-artifact";
+
+function sineWav(args: {
+	sampleRate?: number;
+	durationMs?: number;
+	frequencyHz?: number;
+	amplitude?: number;
+}): Uint8Array {
+	const sampleRate = args.sampleRate ?? 16_000;
+	const frames = Math.floor(sampleRate * ((args.durationMs ?? 1200) / 1000));
+	const dataBytes = frames * 2;
+	const out = new Uint8Array(44 + dataBytes);
+	const view = new DataView(out.buffer);
+	const writeAscii = (offset: number, value: string) => {
+		for (let i = 0; i < value.length; i += 1) {
+			out[offset + i] = value.charCodeAt(i);
+		}
+	};
+	writeAscii(0, "RIFF");
+	view.setUint32(4, 36 + dataBytes, true);
+	writeAscii(8, "WAVE");
+	writeAscii(12, "fmt ");
+	view.setUint32(16, 16, true);
+	view.setUint16(20, 1, true);
+	view.setUint16(22, 1, true);
+	view.setUint32(24, sampleRate, true);
+	view.setUint32(28, sampleRate * 2, true);
+	view.setUint16(32, 2, true);
+	view.setUint16(34, 16, true);
+	writeAscii(36, "data");
+	view.setUint32(40, dataBytes, true);
+	for (let i = 0; i < frames; i += 1) {
+		const value =
+			Math.sin((2 * Math.PI * (args.frequencyHz ?? 220) * i) / sampleRate) *
+			(args.amplitude ?? 0.35);
+		view.setInt16(44 + i * 2, Math.round(value * 32767), true);
+	}
+	return out;
+}
+
+describe("voice-profile-artifact", () => {
+	it("creates deterministic profile artifacts from WAV samples and reference metadata", () => {
+		const wav = sineWav({ durationMs: 1600, frequencyHz: 210 });
+		const args = {
+			samples: [
+				{
+					id: "owner-sample",
+					wavBytes: wav,
+					referenceText: "This is my reference phrase.",
+					recordedAt: "2026-05-12T12:00:00.000Z",
+				},
+			],
+			reference: {
+				speakerId: "entity-owner",
+				label: "Owner",
+				referenceText: "This is my reference phrase.",
+				locale: "en-US",
+				consent: {
+					attribution: true,
+					synthesis: false,
+					grantedAt: "2026-05-12T12:00:00.000Z",
+					evidenceId: "consent-1",
+				},
+			},
+			createdAt: "2026-05-12T12:00:01.000Z",
+		};
+
+		const first = createVoiceProfileArtifact(args);
+		const second = createVoiceProfileArtifact(args);
+
+		expect(first).toEqual(second);
+		expect(first.artifactId).toMatch(/^vpa_[a-f0-9]{32}$/);
+		expect(first.embeddingModel).toBe("eliza-voice-profile-features-v1");
+		expect(first.centroidEmbedding.length).toBeGreaterThan(8);
+		expect(first.usage).toMatchObject({
+			attributionAuthorized: true,
+			synthesisAuthorized: false,
+		});
+
+		const verification = verifyVoiceProfileArtifact({
+			artifact: first,
+			sampleWavs: { "owner-sample": wav },
+		});
+		expect(verification.status).toBe("ready");
+		expect(verification.artifactIdMatches).toBe(true);
+		expect(verification.attributionStatus).toBe("ready");
+		expect(verification.synthesisStatus).toBe("not_authorized");
+		expect(verification.samples[0]).toMatchObject({
+			status: "pass",
+			wavSha256Matches: true,
+		});
+	});
+
+	it("reports invalid status for tampered profile content or sample bytes", () => {
+		const wav = sineWav({ durationMs: 1300, frequencyHz: 220 });
+		const artifact = createVoiceProfileArtifact({
+			samples: [{ id: "sample", wavBytes: wav, referenceText: "hello world" }],
+			reference: {
+				consent: { attribution: true, synthesis: true },
+			},
+		});
+		const tampered = {
+			...artifact,
+			totalDurationMs: artifact.totalDurationMs + 10,
+		};
+		const verification = verifyVoiceProfileArtifact({
+			artifact: tampered,
+			sampleWavs: {
+				sample: sineWav({ durationMs: 1300, frequencyHz: 440 }),
+			},
+		});
+
+		expect(verification.status).toBe("invalid");
+		expect(verification.artifactIdMatches).toBe(false);
+		expect(verification.samples[0].wavSha256Matches).toBe(false);
+	});
+
+	it("extracts stable PCM16 WAV audio features", () => {
+		const wav = sineWav({ durationMs: 1000, frequencyHz: 300 });
+		const features = analyzeVoiceProfileWav(wav);
+
+		expect(features).toMatchObject({
+			format: "wav/pcm_s16le",
+			channels: 1,
+			sampleRateHz: 16000,
+			bitsPerSample: 16,
+			samplesPerChannel: 16000,
+			durationMs: 1000,
+		});
+		expect(features.rms).toBeGreaterThan(0.1);
+		expect(features.peakAbs).toBeGreaterThan(0.3);
+	});
+});