@elizaos/plugin-local-inference 2.0.0-beta.1 → 2.0.3-beta.3

package/src/services/voice/self-voice-imprint.test.ts

@@ -0,0 +1,59 @@
+import { describe, expect, it } from "vitest";
+import { AgentSelfVoiceImprint } from "./self-voice-imprint";
+import type { SpeakerEncoder } from "./speaker/encoder";
+
+function unitEmbedding(index: number): Float32Array {
+	const out = new Float32Array(256);
+	out[index] = 1;
+	return out;
+}
+
+function queuedEncoder(
+	embeddings: readonly Float32Array[],
+	seen: Float32Array[] = [],
+): SpeakerEncoder {
+	let cursor = 0;
+	return {
+		embeddingDim: 256,
+		sampleRate: 16_000,
+		async encode(pcm: Float32Array): Promise<Float32Array> {
+			seen.push(new Float32Array(pcm));
+			return embeddings[cursor++] ?? embeddings.at(-1) ?? unitEmbedding(0);
+		},
+		async dispose(): Promise<void> {},
+	};
+}
+
+describe("AgentSelfVoiceImprint", () => {
+	it("buffers TTS audio, resamples to WeSpeaker rate, and scores similarity", async () => {
+		const encodedWindows: Float32Array[] = [];
+		const imprint = new AgentSelfVoiceImprint({
+			encoder: queuedEncoder([unitEmbedding(0)], encodedWindows),
+		});
+
+		await imprint.observeAudio(new Float32Array(12_000).fill(0.1), 24_000);
+		expect(await imprint.similarity(unitEmbedding(0))).toBeNull();
+
+		await imprint.observeAudio(new Float32Array(12_000).fill(0.1), 24_000);
+
+		expect(encodedWindows).toHaveLength(1);
+		expect(encodedWindows[0]?.length).toBe(16_000);
+		await expect(imprint.similarity(unitEmbedding(0))).resolves.toBeCloseTo(1);
+	});
+
+	it("maintains a normalized rolling centroid of recent agent voice embeddings", async () => {
+		const imprint = new AgentSelfVoiceImprint({
+			encoder: queuedEncoder([unitEmbedding(0), unitEmbedding(1)]),
+			minSamples: 16_000,
+			maxEmbeddings: 2,
+		});
+
+		await imprint.observeAudio(new Float32Array(16_000).fill(0.1), 16_000);
+		await imprint.observeAudio(new Float32Array(16_000).fill(0.2), 16_000);
+
+		const diagonal = new Float32Array(256);
+		diagonal[0] = Math.SQRT1_2;
+		diagonal[1] = Math.SQRT1_2;
+		await expect(imprint.similarity(diagonal)).resolves.toBeCloseTo(1);
+	});
+});

package/src/services/voice/self-voice-imprint.ts

@@ -0,0 +1,102 @@
+import { averageEmbeddings, type SpeakerEncoder } from "./speaker/encoder";
+import {
+	SPEAKER_GGML_MIN_SAMPLES,
+	SPEAKER_GGML_SAMPLE_RATE,
+} from "./speaker/encoder-ggml";
+import { cosineSimilarity } from "./speaker-imprint";
+import { resampleLinear } from "./transcriber";
+
+export interface AgentSelfVoiceImprintOptions {
+	encoder: SpeakerEncoder;
+	/** Minimum 16 kHz samples before one self-voice embedding is encoded. */
+	minSamples?: number;
+	/** Maximum 16 kHz samples to encode for one centroid update. */
+	maxSamples?: number;
+	/** Number of recent agent-TTS embeddings retained in the centroid. */
+	maxEmbeddings?: number;
+}
+
+function concatSegments(
+	segments: readonly Float32Array[],
+	totalSamples: number,
+): Float32Array {
+	const out = new Float32Array(totalSamples);
+	let offset = 0;
+	for (const segment of segments) {
+		out.set(segment, offset);
+		offset += segment.length;
+	}
+	return out;
+}
+
+/**
+ * Maintains a bounded centroid of the agent's own synthesized voice.
+ *
+ * The live mic path already gets WeSpeaker embeddings from attribution. This
+ * helper observes the PCM that the scheduler hands to audio output, embeds that
+ * actual TTS audio, and exposes cosine similarity against future mic turns.
+ */
+export class AgentSelfVoiceImprint {
+	private readonly encoder: SpeakerEncoder;
+	private readonly minSamples: number;
+	private readonly maxSamples: number;
+	private readonly maxEmbeddings: number;
+	private readonly pendingSegments: Float32Array[] = [];
+	private pendingSamples = 0;
+	private readonly embeddings: Float32Array[] = [];
+	private centroid: Float32Array | null = null;
+	private queue: Promise<void> = Promise.resolve();
+
+	constructor(options: AgentSelfVoiceImprintOptions) {
+		this.encoder = options.encoder;
+		this.minSamples = options.minSamples ?? SPEAKER_GGML_MIN_SAMPLES;
+		this.maxSamples = options.maxSamples ?? SPEAKER_GGML_SAMPLE_RATE * 6;
+		this.maxEmbeddings = Math.max(1, options.maxEmbeddings ?? 8);
+	}
+
+	observeAudio(pcm: Float32Array, sampleRate: number): Promise<void> {
+		const work = this.queue.then(() =>
+			this.observeAudioLocked(pcm, sampleRate),
+		);
+		this.queue = work.catch(() => {});
+		return work;
+	}
+
+	async similarity(embedding: Float32Array): Promise<number | null> {
+		await this.queue;
+		if (!this.centroid) return null;
+		if (embedding.length !== this.centroid.length) return null;
+		return cosineSimilarity(embedding, this.centroid);
+	}
+
+	private async observeAudioLocked(
+		pcm: Float32Array,
+		sampleRate: number,
+	): Promise<void> {
+		if (pcm.length === 0 || sampleRate <= 0) return;
+		const speakerPcm =
+			sampleRate === SPEAKER_GGML_SAMPLE_RATE
+				? new Float32Array(pcm)
+				: resampleLinear(pcm, sampleRate, SPEAKER_GGML_SAMPLE_RATE);
+		if (speakerPcm.length === 0) return;
+
+		this.pendingSegments.push(speakerPcm);
+		this.pendingSamples += speakerPcm.length;
+		if (this.pendingSamples < this.minSamples) return;
+
+		const toEncode = concatSegments(this.pendingSegments, this.pendingSamples);
+		this.pendingSegments.length = 0;
+		this.pendingSamples = 0;
+
+		const window =
+			toEncode.length > this.maxSamples
+				? toEncode.subarray(0, this.maxSamples)
+				: toEncode;
+		const embedding = await this.encoder.encode(window);
+		this.embeddings.push(embedding);
+		while (this.embeddings.length > this.maxEmbeddings) {
+			this.embeddings.shift();
+		}
+		this.centroid = averageEmbeddings(this.embeddings);
+	}
+}

package/src/services/voice/shared-resources.ts

@@ -0,0 +1,343 @@
+/**
+ * Cross-cut resource sharing between the text + voice surfaces of a
+ * single Eliza-1 bundle.
+ *
+ * Per `packages/inference/AGENTS.md` §4 ("shared KV cache scheduling,
+ * not shared KV memory" + "one process, one llama.cpp build, one GGML
+ * pin"), text and voice MUST share:
+ *   - the tokenizer (Eliza-1/OmniVoice share a vocabulary in this lineage),
+ *   - the mmap regions for weights (deduplicated by absolute path),
+ *   - the kernel set (same shipped llama.cpp library after fusion),
+ *   - the scheduler queue (one queue, prioritised),
+ *   - the native MTP draft path (always wired for Eliza-1).
+ *
+ * What they do NOT share:
+ *   - KV cache memory (different layer counts, different head configs,
+ *     different quantizations — separate caches, shared scheduler).
+ *
+ * This module owns reference counts on each shared resource and is the
+ * single arbiter of when a voice-only region can be released. It does
+ * NOT do any I/O itself — the actual mmap, madvise, or full model-unload
+ * behavior lives behind the `MmapRegionHandle` interface so platform
+ * bindings can choose the right memory policy.
+ */
+
+/** Minimal structural logger — keeps this module free of upstream deps. */
+interface Logger {
+	debug?(message: string): void;
+	warn?(message: string): void;
+	info?(message: string): void;
+}
+
+/**
+ * The model roles that can be resident at once on the local-inference
+ * path. The `MemoryMonitor` evicts them in *ascending priority* under RAM
+ * pressure (lowest first): low-cost voice auxiliaries are cheapest to drop,
+ * the text target is the last thing to go. Voice TTS/ASR weights are evicted
+ * via `MmapRegionHandle.evictPages()`; the embedding model is unloaded by
+ * its owner.
+ */
+export type ResidentModelRole =
+	| "drafter"
+	| "emotion"
+	| "speaker-id"
+	| "vision"
+	| "embedding"
+	| "vad"
+	| "asr"
+	| "tts"
+	| "text-target";
+
+/**
+ * Eviction priority by role — lower evicts first. Matches the brief's
+ * `emotion < speaker-id < vision/mmproj < embedding < vad < ASR <
+ * TTS < text-target`. The cold-3 set (`emotion`, `speaker-id`) is cheap to
+ * load on demand, so evicting them is the first reclamation step under
+ * sustained pressure. See `.swarm/research/R9-memory.md` §4.1.
+ */
+export const RESIDENT_ROLE_PRIORITY: Readonly<
+	Record<ResidentModelRole, number>
+> = {
+	drafter: 10,
+	emotion: 15,
+	"speaker-id": 18,
+	vision: 20,
+	embedding: 25,
+	vad: 35,
+	asr: 40,
+	tts: 50,
+	"text-target": 100,
+};
+
+/**
+ * An evictable resident model role. The registry walks these in ascending
+ * `evictionPriority` under memory pressure and calls `evict()` until enough
+ * RAM has been reclaimed. `evict()` MUST be idempotent (a no-op when already
+ * evicted) and the role MUST re-load lazily on next use — the monitor only
+ * frees memory, it never re-loads.
+ */
+export interface EvictableModelRole extends RefCountedResource {
+	readonly role: ResidentModelRole;
+	/** Lower evicts first. Defaults to `RESIDENT_ROLE_PRIORITY[role]`. */
+	readonly evictionPriority: number;
+	/** True while the underlying weights/pages are still resident. */
+	isResident(): boolean;
+	/** Drop the resident weights/pages. Idempotent; re-loads lazily on demand. */
+	evict(): Promise<void>;
+	/** Best-effort estimate of RAM (MB) reclaimed by `evict()`. 0 when unknown. */
+	estimatedResidentMb(): number;
+}
+
+function isEvictableModelRole(
+	value: RefCountedResource,
+): value is EvictableModelRole {
+	const candidate = value as Partial<EvictableModelRole>;
+	return (
+		typeof candidate.role === "string" &&
+		typeof candidate.evictionPriority === "number" &&
+		typeof candidate.isResident === "function" &&
+		typeof candidate.evict === "function" &&
+		typeof candidate.estimatedResidentMb === "function"
+	);
+}
+
+/**
+ * Build an `EvictableModelRole` from a role + an `evict` callback. `release()`
+ * defaults to a no-op (the registry's refcount, not `release`, gates eviction
+ * for these); pass one if the role owns disposable state. `estimatedMb` lets
+ * the monitor know roughly how much it will reclaim — pass 0 when unknown.
+ */
+export function createEvictableModelRole(args: {
+	id?: string;
+	role: ResidentModelRole;
+	evictionPriority?: number;
+	estimatedMb?: number;
+	isResident: () => boolean;
+	evict: () => Promise<void>;
+	release?: () => Promise<void>;
+}): EvictableModelRole {
+	const id = args.id ?? `model-role:${args.role}`;
+	const priority = args.evictionPriority ?? RESIDENT_ROLE_PRIORITY[args.role];
+	const estimatedMb = args.estimatedMb ?? 0;
+	return {
+		id,
+		role: args.role,
+		evictionPriority: priority,
+		isResident: args.isResident,
+		estimatedResidentMb: () => (args.isResident() ? estimatedMb : 0),
+		async evict(): Promise<void> {
+			if (!args.isResident()) return;
+			await args.evict();
+		},
+		async release(): Promise<void> {
+			await args.release?.();
+		},
+	};
+}
+
+/**
+ * Anything ref-counted by the registry implements this. The caller of
+ * `release()` MUST guarantee that no further reads happen on the
+ * underlying resource — for mmap regions that means no kernel call has
+ * a pointer into the freed range.
+ */
+export interface RefCountedResource {
+	readonly id: string;
+	/** Released for real when the last ref drops. Idempotent. */
+	release(): Promise<void>;
+}
+
+/**
+ * mmap region handle. The fused omnivoice/llama.cpp build owns the real
+ * mmap call (it happens inside the FFI) — this interface is the JS-side
+ * proxy for it, so the lifecycle code can request page eviction without
+ * binding to a specific backend.
+ */
+export interface MmapRegionHandle extends RefCountedResource {
+	/** Absolute path of the file backing the mmap region. */
+	readonly path: string;
+	/** Byte size of the mapped region. */
+	readonly sizeBytes: number;
+	/**
+	 * Release memory pressure for this region. Backends may implement this
+	 * as a page hint or as a full voice-runtime unload. Common mappings:
+	 *   - POSIX (Linux/Android/macOS-bg): `madvise(addr, len, MADV_DONTNEED)`
+	 *   - macOS (foreground / iOS):        `madvise(addr, len, MADV_FREE_REUSABLE)`
+	 *   - Windows:                         `VirtualUnlock` + `OfferVirtualMemory`
+	 *
+	 * The lifecycle test mocks this to assert the call happened.
+	 */
+	evictPages(): Promise<void>;
+}
+
+/** Minimal tokenizer surface text + voice both consume. */
+export interface SharedTokenizer extends RefCountedResource {
+	readonly vocabSize: number;
+}
+
+/**
+ * Kernel set descriptor. The actual kernels are inside the fused
+ * llama.cpp build; this is the metadata the runtime reads at startup
+ * (AGENTS.md §3 #5: "the runtime MUST log the kernel set on startup").
+ */
+export interface KernelSet extends RefCountedResource {
+	readonly kernels: ReadonlyArray<string>;
+}
+
+/** Scheduler graph slot. One per active engine, refcounted by surface. */
+export interface SchedulerSlot extends RefCountedResource {
+	/** Surface (text/voice) currently holding a ref. */
+	surfaces(): ReadonlyArray<"text" | "voice">;
+}
+
+/** Native MTP draft state is shared between text-only and voice modes. */
+export interface MtpDraftHandle extends RefCountedResource {
+	readonly modelId: string;
+}
+
+export function createMtpDraftHandle(args: {
+	modelId: string;
+}): MtpDraftHandle {
+	return {
+		id: `mtp:${args.modelId}`,
+		modelId: args.modelId,
+		async release(): Promise<void> {
+			// MTP state lifetime is owned by the active native text runtime.
+		},
+	};
+}
+
+interface RegistryEntry<T extends RefCountedResource> {
+	readonly resource: T;
+	refCount: number;
+}
+
+/**
+ * Owns the shared resources for one engine. Voice + text both `acquire`
+ * and `release` against the same registry; the registry only releases
+ * the underlying resource when refcount hits zero.
+ *
+ * Thread-safety: all methods run on the single Node event loop; no
+ * locks needed. Promises returned from `release()` MUST be awaited so
+ * the lifecycle state machine can observe completion.
+ */
+export class SharedResourceRegistry {
+	private readonly entries = new Map<
+		string,
+		RegistryEntry<RefCountedResource>
+	>();
+	private readonly log?: Logger;
+	/**
+	 * When a higher-level component (the `MemoryArbiter`) owns the eviction
+	 * decision for this registry, it claims ownership here so the simpler
+	 * `MemoryMonitor` poll defers instead of evicting in parallel — a single
+	 * eviction decision point, no double-eviction on one pressure event
+	 * (#8809 AC#2). Null = no external owner, the monitor evicts itself.
+	 */
+	private evictionOwner: string | null = null;
+
+	constructor(opts: { logger?: Logger } = {}) {
+		this.log = opts.logger;
+	}
+
+	/** Claim the single eviction-decision ownership for this registry. */
+	claimEvictionOwnership(owner: string): void {
+		this.evictionOwner = owner;
+	}
+
+	/** Release ownership (only the current owner may release it). */
+	releaseEvictionOwnership(owner: string): void {
+		if (this.evictionOwner === owner) this.evictionOwner = null;
+	}
+
+	/** True when an external component owns the eviction decision. */
+	hasExternalEvictionOwner(): boolean {
+		return this.evictionOwner !== null;
+	}
+
+	/**
+	 * Register a resource if absent, increment refcount otherwise. Returns
+	 * the canonical instance — callers MUST use the returned value, not the
+	 * one passed in, so a second registration with the same id resolves to
+	 * the original (deduplication by id).
+	 */
+	acquire<T extends RefCountedResource>(resource: T): T {
+		const existing = this.entries.get(resource.id);
+		if (existing) {
+			existing.refCount++;
+			return existing.resource as T;
+		}
+		this.entries.set(resource.id, { resource, refCount: 1 });
+		return resource;
+	}
+
+	/**
+	 * Decrement refcount; release the resource when it hits zero. Throws
+	 * on unknown id — silent no-ops would hide leaks.
+	 */
+	async release(id: string): Promise<void> {
+		const entry = this.entries.get(id);
+		if (!entry) {
+			throw new Error(
+				`[shared-resources] release(${id}): unknown resource — possible double release or registry desync`,
+			);
+		}
+		entry.refCount--;
+		if (entry.refCount > 0) return;
+		this.entries.delete(id);
+		await entry.resource.release();
+		this.log?.debug?.(`[SharedResourceRegistry] released ${id}`);
+	}
+
+	/** Diagnostic: current refcount, or 0 when not present. */
+	refCount(id: string): number {
+		return this.entries.get(id)?.refCount ?? 0;
+	}
+
+	/** Diagnostic: snapshot of currently-tracked resource ids. */
+	ids(): ReadonlyArray<string> {
+		return Array.from(this.entries.keys());
+	}
+
+	/** Total tracked resources. */
+	size(): number {
+		return this.entries.size;
+	}
+
+	/**
+	 * Currently-resident evictable model roles, ascending by eviction
+	 * priority (cheapest-to-evict first). Used by `MemoryMonitor` to walk
+	 * roles under RAM pressure. Non-resident roles are filtered out — there's
+	 * nothing to reclaim.
+	 */
+	evictableRoles(): ReadonlyArray<EvictableModelRole> {
+		const out: EvictableModelRole[] = [];
+		for (const entry of this.entries.values()) {
+			if (isEvictableModelRole(entry.resource) && entry.resource.isResident()) {
+				out.push(entry.resource);
+			}
+		}
+		return out.sort((a, b) => a.evictionPriority - b.evictionPriority);
+	}
+
+	/**
+	 * Evict the lowest-priority resident role and return its `id`, or `null`
+	 * when nothing is evictable. Observable: emits an `info` log line so the
+	 * eviction is visible in the dev console. The role re-loads lazily on
+	 * next use — this only frees memory.
+	 */
+	async evictLowestPriorityRole(): Promise<{
+		id: string;
+		role: ResidentModelRole;
+		estimatedMb: number;
+	} | null> {
+		const [target] = this.evictableRoles();
+		if (!target) return null;
+		const estimatedMb = target.estimatedResidentMb();
+		await target.evict();
+		this.log?.info?.(
+			`[SharedResourceRegistry] evicted role ${target.role} (${target.id}); reclaimed ~${estimatedMb} MB`,
+		);
+		return { id: target.id, role: target.role, estimatedMb };
+	}
+}