@elizaos/plugin-local-inference 2.0.0-beta.1 → 2.0.3-beta.2

package/src/services/voice/optimistic-policy.test.ts

@@ -0,0 +1,101 @@
+import { describe, expect, it } from "vitest";
+import {
+	DEFAULT_OPTIMISTIC_EOT_THRESHOLD,
+	OptimisticGenerationPolicy,
+	resolveOptimisticPolicyEnabled,
+} from "./optimistic-policy";
+
+describe("resolveOptimisticPolicyEnabled — pure", () => {
+	it("override:true wins regardless of power source", () => {
+		expect(
+			resolveOptimisticPolicyEnabled({
+				powerSource: "battery",
+				override: true,
+			}),
+		).toBe(true);
+	});
+
+	it("override:false wins regardless of power source", () => {
+		expect(
+			resolveOptimisticPolicyEnabled({
+				powerSource: "plugged-in",
+				override: false,
+			}),
+		).toBe(false);
+	});
+
+	it("battery without override → false", () => {
+		expect(resolveOptimisticPolicyEnabled({ powerSource: "battery" })).toBe(
+			false,
+		);
+	});
+
+	it("plugged-in without override → true", () => {
+		expect(resolveOptimisticPolicyEnabled({ powerSource: "plugged-in" })).toBe(
+			true,
+		);
+	});
+
+	it("unknown power source defaults to true", () => {
+		expect(resolveOptimisticPolicyEnabled({ powerSource: "unknown" })).toBe(
+			true,
+		);
+		expect(resolveOptimisticPolicyEnabled({})).toBe(true);
+	});
+});
+
+describe("OptimisticGenerationPolicy", () => {
+	it("defaults: enabled, threshold = DEFAULT", () => {
+		const p = new OptimisticGenerationPolicy();
+		expect(p.enabled()).toBe(true);
+		expect(p.threshold).toBe(DEFAULT_OPTIMISTIC_EOT_THRESHOLD);
+	});
+
+	it("battery flips enabled() to false", () => {
+		const p = new OptimisticGenerationPolicy();
+		p.setPowerSource("battery");
+		expect(p.enabled()).toBe(false);
+	});
+
+	it("explicit override true wins on battery", () => {
+		const p = new OptimisticGenerationPolicy();
+		p.setPowerSource("battery");
+		p.setOverride(true);
+		expect(p.enabled()).toBe(true);
+	});
+
+	it("override:false wins on plugged-in", () => {
+		const p = new OptimisticGenerationPolicy();
+		p.setPowerSource("plugged-in");
+		p.setOverride(false);
+		expect(p.enabled()).toBe(false);
+	});
+
+	it("shouldStartOptimisticLm gates on EOT threshold", () => {
+		const p = new OptimisticGenerationPolicy({ eotThreshold: 0.7 });
+		expect(p.shouldStartOptimisticLm(0.5)).toBe(false);
+		expect(p.shouldStartOptimisticLm(0.7)).toBe(true);
+		expect(p.shouldStartOptimisticLm(0.95)).toBe(true);
+	});
+
+	it("shouldStartOptimisticLm returns false when disabled regardless of probability", () => {
+		const p = new OptimisticGenerationPolicy();
+		p.setPowerSource("battery");
+		expect(p.shouldStartOptimisticLm(0.99)).toBe(false);
+	});
+
+	it("override clears back to default with undefined", () => {
+		const p = new OptimisticGenerationPolicy();
+		p.setOverride(false);
+		expect(p.enabled()).toBe(false);
+		p.setOverride(undefined);
+		expect(p.enabled()).toBe(true);
+	});
+
+	it("constructor defaultEnabled=false flips the base policy", () => {
+		const p = new OptimisticGenerationPolicy({ defaultEnabled: false });
+		expect(p.enabled()).toBe(false);
+		p.setOverride(true);
+		expect(p.enabled()).toBe(true);
+	});
+});

package/src/services/voice/optimistic-policy.ts

@@ -0,0 +1,192 @@
+/**
+ * Optimistic-generation policy — Wave 3 W3-9.
+ *
+ * Decides whether the LM should fire optimistically the moment the turn
+ * detector crosses the EOT threshold (before the audio buffer is fully
+ * drained), or wait for the full hangover. Per the user spec:
+ *
+ *   > "We want to optimize to start running and checking basically the
+ *    second there is no audio / voice detected, and then we start
+ *    optimistically processing, and if the user starts talking again we
+ *    kill and add to the conversation and start reprocessing."
+ *
+ * Default heuristic:
+ *   - Plugged-in (or power state unknown): optimistic = true.
+ *   - On battery: optimistic = false (the extra forward pass burns
+ *     joules every false-positive EOT).
+ *
+ * Callers (notably `VoiceStateMachine`) read `shouldStartOptimisticLm()`
+ * each time they consider firing the speculative drafter. The policy can be
+ * mutated at runtime via `setOptimisticOverride()` so a Settings UI toggle
+ * or a per-session env var can pin the value without restarting the engine.
+ */
+
+/** Power-source signal. `unknown` is treated as "plugged in" for the
+ *  default policy (a desktop without battery telemetry). */
+export type PowerSourceState = "plugged-in" | "battery" | "unknown";
+
+/**
+ * Resolve the current power-source state.
+ *
+ * Production callers (engine bridge) prime the optimistic policy with this
+ * once at session start. There is no canonical power-source accessor in
+ * `device-tier.ts` (the hardware probe records RAM / GPU / cores but no
+ * battery telemetry), so this resolver consults two cheap signals:
+ *
+ *   1. `ELIZA_VOICE_POWER_SOURCE` env var (`"plugged-in" | "battery" |
+ *      "unknown"`). Wins outright when set — operators / Settings can pin
+ *      the policy without restarting the engine.
+ *   2. Linux `/sys/class/power_supply` AC online status when available
+ *      synchronously. Returns `"plugged-in"` when any AC adapter reports
+ *      `online`, `"battery"` when only batteries are present and none
+ *      report online, `"unknown"` otherwise.
+ *
+ * macOS / Windows / mobile go through `"unknown"` here — the desktop probe
+ * surfaces those via the Electrobun side and Settings overrides it through
+ * (1). The default heuristic (`unknown → enabled`) treats those as
+ * plugged-in for the optimistic gate, which is the right default for
+ * desktop / dev (battery-aware mobile builds set the override explicitly).
+ */
+export function resolvePowerSourceState(): PowerSourceState {
+	const fromEnv = process.env.ELIZA_VOICE_POWER_SOURCE?.trim().toLowerCase();
+	if (
+		fromEnv === "plugged-in" ||
+		fromEnv === "battery" ||
+		fromEnv === "unknown"
+	) {
+		return fromEnv;
+	}
+	if (process.platform === "linux") {
+		try {
+			// Lazy require so this module stays free of node-fs imports for
+			// non-linux platforms / browser bundlers.
+			// eslint-disable-next-line @typescript-eslint/no-require-imports
+			const fs = require("node:fs") as typeof import("node:fs");
+			const base = "/sys/class/power_supply";
+			if (!fs.existsSync(base)) return "unknown";
+			const entries = fs.readdirSync(base);
+			let sawBattery = false;
+			for (const entry of entries) {
+				const typePath = `${base}/${entry}/type`;
+				if (!fs.existsSync(typePath)) continue;
+				const type = fs.readFileSync(typePath, "utf8").trim();
+				if (type === "Mains") {
+					const onlinePath = `${base}/${entry}/online`;
+					if (
+						fs.existsSync(onlinePath) &&
+						fs.readFileSync(onlinePath, "utf8").trim() === "1"
+					) {
+						return "plugged-in";
+					}
+				} else if (type === "Battery") {
+					sawBattery = true;
+				}
+			}
+			return sawBattery ? "battery" : "unknown";
+		} catch {
+			return "unknown";
+		}
+	}
+	return "unknown";
+}
+
+export interface OptimisticPolicyOptions {
+	/**
+	 * Default value when no override is set. Resolved from the device tier
+	 * at construction time (per the user spec: true on plugged-in,
+	 * false on battery). Defaults to `true` when no power signal is given.
+	 */
+	defaultEnabled?: boolean;
+	/**
+	 * The threshold the turn detector's EOT probability must clear before
+	 * the optimistic LM start fires. Mirrors `EOT_TENTATIVE_THRESHOLD`
+	 * (the existing default) but is configurable per-policy so tier-aware
+	 * deployments can tighten it on slower devices.
+	 */
+	eotThreshold?: number;
+}
+
+export interface ResolveOptimisticPolicyArgs {
+	powerSource?: PowerSourceState;
+	/**
+	 * Explicit override. When set, wins over both the default and the
+	 * power-source heuristic. Sourced from `voice.optimisticGenerationOnEot`
+	 * in user settings (Wave 3C).
+	 */
+	override?: boolean;
+}
+
+export const DEFAULT_OPTIMISTIC_EOT_THRESHOLD = 0.6;
+
+/**
+ * Pure resolver. Takes the power source + an optional override and returns
+ * whether optimistic LM start should fire.
+ */
+export function resolveOptimisticPolicyEnabled(
+	args: ResolveOptimisticPolicyArgs,
+): boolean {
+	if (typeof args.override === "boolean") return args.override;
+	if (args.powerSource === "battery") return false;
+	// plugged-in OR unknown OR undefined → enabled (safer default for
+	// desktop / dev where power telemetry isn't surfaced).
+	return true;
+}
+
+/**
+ * Mutable runtime policy. The voice state machine asks
+ * `policy.shouldStartOptimisticLm(eotProb)` before firing the drafter.
+ *
+ * Reasoning for a class rather than a free function: at runtime we need
+ * the override + the power source to be hot-swappable from Settings or a
+ * device-event listener (battery state change) without re-plumbing the
+ * machine.
+ */
+export class OptimisticGenerationPolicy {
+	private overrideValue: boolean | undefined;
+	private powerSource: PowerSourceState;
+	private readonly defaultEnabled: boolean;
+	private readonly eotThreshold: number;
+
+	constructor(opts: OptimisticPolicyOptions = {}) {
+		this.defaultEnabled = opts.defaultEnabled ?? true;
+		this.eotThreshold = opts.eotThreshold ?? DEFAULT_OPTIMISTIC_EOT_THRESHOLD;
+		this.powerSource = "unknown";
+	}
+
+	/** Update the power source (called by the device-tier observer). */
+	setPowerSource(state: PowerSourceState): void {
+		this.powerSource = state;
+	}
+
+	/** Set / clear the user override. */
+	setOverride(value: boolean | undefined): void {
+		this.overrideValue = value;
+	}
+
+	/** Resolve "should we be optimistic right now?". */
+	enabled(): boolean {
+		// Explicit user override wins outright.
+		if (typeof this.overrideValue === "boolean") return this.overrideValue;
+		// No override: apply the battery heuristic on top of `defaultEnabled`.
+		// `defaultEnabled=false` pins the policy off regardless of power source;
+		// `defaultEnabled=true` enables it on plugged-in / unknown and disables
+		// it on battery.
+		if (!this.defaultEnabled) return false;
+		return resolveOptimisticPolicyEnabled({ powerSource: this.powerSource });
+	}
+
+	/**
+	 * Combined gate: the policy must be enabled AND the EOT probability
+	 * must clear the policy's threshold. This is the canonical check the
+	 * voice state machine calls before firing the drafter on a partial
+	 * transcript.
+	 */
+	shouldStartOptimisticLm(eotProb: number): boolean {
+		if (!this.enabled()) return false;
+		return eotProb >= this.eotThreshold;
+	}
+
+	get threshold(): number {
+		return this.eotThreshold;
+	}
+}

package/src/services/voice/optimistic-rollback.ts

@@ -0,0 +1,343 @@
+/**
+ * Optimistic-decode-with-rollback controller for the voice loop.
+ *
+ *   IDLE
+ *     │ speech-start
+ *     ▼
+ *   LISTENING
+ *     │ speech-pause
+ *     ▼
+ *   PAUSE_TENTATIVE  ── speech-active (within rollback window) ──▶ LISTENING
+ *     │                                                              ▲
+ *     │ speech-end (or pause > rollback window)                       │
+ *     ▼                                                               │
+ *   DRAFT_RESPONSE  ──── speech-active (within rollback window) ──────┘
+ *     │                  (restore checkpoint, abort drafter)
+ *     │ commit
+ *     ▼
+ *   COMMITTED → IDLE
+ *
+ * On `speech-pause` we snapshot the slot's KV state (`C1-<turn-id>`) and
+ * speculatively start the drafter against the partial transcript so that
+ * when the user IS done, the response is already underway. If they resume
+ * within the rollback window (~2× pause hangover), we restore the snapshot
+ * and abort the speculative draft — net cost is one checkpoint write/read.
+ *
+ * This module deliberately does NOT modify `turn-controller.ts`,
+ * `vad.ts`, `scheduler.ts`, `phrase-chunker.ts`, `barge-in.ts`, `pipeline.ts`,
+ * `pipeline-impls.ts`, or `transcriber.ts` — those are owned elsewhere.
+ * Instead, the turn controller composes this controller after the upstream
+ * llama.cpp merge lands and the feature flag flips.
+ */
+
+import type { CheckpointClient, CheckpointHandle } from "../checkpoint-client";
+import type { VadEvent, VadEventSource } from "./types";
+
+/**
+ * State of the optimistic-rollback controller. Mirrors the comment-block
+ * state machine above; exported so tests and telemetry can assert on it.
+ */
+export type OptimisticRollbackState =
+	| "idle"
+	| "listening"
+	| "pause-tentative"
+	| "draft-response"
+	| "committed";
+
+/**
+ * Telemetry event stream. Stays narrow — the controller emits exactly four
+ * lifecycle events per turn (save, restore-or-commit, draft-start,
+ * draft-abort). Consumers wire these into the existing voice-bench
+ * trajectory captures.
+ */
+export interface OptimisticRollbackTelemetry {
+	onCheckpointSaved?: (handle: CheckpointHandle, turnId: string) => void;
+	onCheckpointRestored?: (handle: CheckpointHandle, turnId: string) => void;
+	onSpeculativeDraftStarted?: (turnId: string) => void;
+	onSpeculativeDraftAborted?: (turnId: string, reason: AbortReason) => void;
+	/**
+	 * Fired when the controller silently swallows an error from the
+	 * checkpoint client — surfacing it via telemetry rather than rethrowing
+	 * keeps a failing checkpoint endpoint from breaking the voice loop. The
+	 * caller decides whether to flip the feature flag off.
+	 */
+	onCheckpointError?: (
+		op: "save" | "restore" | "cancel",
+		error: unknown,
+		turnId: string,
+	) => void;
+}
+
+export type AbortReason = "resumed" | "committed" | "shutdown";
+
+/**
+ * Speculative drafter handle. The voice turn controller hands one of these
+ * in when starting the drafter so this module can abort it without
+ * importing the drafter's internals. `abort()` MUST be idempotent — the
+ * controller may call it again on shutdown.
+ */
+export interface SpeculativeDraftHandle {
+	abort(): void;
+}
+
+/**
+ * Caller-supplied drafter starter. Returns a handle the controller can
+ * abort. Called on entry to `pause-tentative`. The promise must resolve
+ * synchronously (or near-synchronously) — the speculative draft runs in
+ * the background.
+ */
+export type StartSpeculativeDraft = (
+	partialTranscript: string,
+	turnId: string,
+) => SpeculativeDraftHandle;
+
+/**
+ * Caller-supplied source for the partial transcript captured at the
+ * `speech-pause` instant. Kept as a function so the controller doesn't
+ * have to hold a reference to the transcriber.
+ */
+export type ReadPartialTranscript = () => string;
+
+export interface OptimisticRollbackControllerOptions {
+	/** The slot id the voice loop is pinned to for this turn. */
+	slotId: number;
+	/**
+	 * Per-process feature flag. Defaults to `false` — flip on once the
+	 * upstream `--ctx-checkpoints` merge lands AND the rollout plan in
+	 * `docs/eliza-1-optimistic-rollback.md` reaches the desired bucket.
+	 * Forwards every VAD event to the wrapped state machine when off but
+	 * never makes a checkpoint REST call.
+	 */
+	enableOptimisticRollback?: boolean;
+	/**
+	 * VAD pause hangover (ms). Default 100 ms (lowered from 220ms; further
+	 * reduction gated on semantic EOT classifier V2) — matches the voice
+	 * loop's standard hangover. The rollback window is `2 ×` this value.
+	 */
+	pauseHangoverMs?: number;
+	/** Source of VAD events; usually `VadDetector`. */
+	vadSource: VadEventSource;
+	client: CheckpointClient;
+	startSpeculativeDraft: StartSpeculativeDraft;
+	readPartialTranscript: ReadPartialTranscript;
+	telemetry?: OptimisticRollbackTelemetry;
+	/**
+	 * Wall-clock function. Injected for tests; defaults to `Date.now`. The
+	 * controller uses this to enforce the rollback window — if a
+	 * `speech-active` arrives more than `2 × pauseHangoverMs` after the
+	 * `speech-pause`, the controller commits rather than restores.
+	 */
+	now?: () => number;
+}
+
+// Lowered from 220ms; further reduction gated on semantic EOT classifier (V2).
+const DEFAULT_PAUSE_HANGOVER_MS = 100;
+/** Rollback window = ROLLBACK_WINDOW_MULTIPLIER × pauseHangoverMs. */
+const ROLLBACK_WINDOW_MULTIPLIER = 2;
+
+/**
+ * Optimistic-rollback controller. Subscribes to a `VadEventSource` and
+ * drives the checkpoint REST client + a caller-supplied speculative
+ * drafter. Idempotent `dispose()` for clean shutdown.
+ */
+export class OptimisticRollbackController {
+	private state: OptimisticRollbackState = "idle";
+	private currentTurnId = 0;
+	private pauseTimestampMs: number | null = null;
+	private currentCheckpoint: CheckpointHandle | null = null;
+	private currentDraft: SpeculativeDraftHandle | null = null;
+	private readonly unsubscribe: () => void;
+	private readonly enabled: boolean;
+	private readonly pauseHangoverMs: number;
+	private readonly slotId: number;
+	private readonly client: CheckpointClient;
+	private readonly startSpeculativeDraft: StartSpeculativeDraft;
+	private readonly readPartialTranscript: ReadPartialTranscript;
+	private readonly telemetry: OptimisticRollbackTelemetry;
+	private readonly now: () => number;
+	private disposed = false;
+
+	constructor(opts: OptimisticRollbackControllerOptions) {
+		this.slotId = opts.slotId;
+		this.enabled = opts.enableOptimisticRollback ?? false;
+		this.pauseHangoverMs = opts.pauseHangoverMs ?? DEFAULT_PAUSE_HANGOVER_MS;
+		this.client = opts.client;
+		this.startSpeculativeDraft = opts.startSpeculativeDraft;
+		this.readPartialTranscript = opts.readPartialTranscript;
+		this.telemetry = opts.telemetry ?? {};
+		this.now = opts.now ?? Date.now;
+		this.unsubscribe = opts.vadSource.onVadEvent((event) =>
+			this.handleVadEvent(event),
+		);
+	}
+
+	/** Current state — read-only view for tests / telemetry. */
+	getState(): OptimisticRollbackState {
+		return this.state;
+	}
+
+	/**
+	 * Detach from the VAD source and abort any in-flight speculative draft.
+	 * Safe to call multiple times.
+	 */
+	dispose(): void {
+		if (this.disposed) return;
+		this.disposed = true;
+		this.unsubscribe();
+		if (this.currentDraft) {
+			this.currentDraft.abort();
+			this.telemetry.onSpeculativeDraftAborted?.(
+				this.turnIdString(),
+				"shutdown",
+			);
+			this.currentDraft = null;
+		}
+		if (this.currentCheckpoint && this.enabled) {
+			// Best-effort cancellation of any in-flight decode on the slot. We do
+			// NOT delete the snapshot file — upstream evicts on slot reuse, and a
+			// dangling snapshot is cheap. Errors here are swallowed (the loop is
+			// shutting down) but surfaced via telemetry.
+			this.client
+				.cancelSlot(this.slotId)
+				.catch((error: unknown) =>
+					this.telemetry.onCheckpointError?.(
+						"cancel",
+						error,
+						this.turnIdString(),
+					),
+				);
+		}
+		this.currentCheckpoint = null;
+		this.state = "idle";
+	}
+
+	private handleVadEvent(event: VadEvent): void {
+		if (this.disposed) return;
+		switch (event.type) {
+			case "speech-start":
+				this.transitionToListening();
+				return;
+			case "speech-pause":
+				this.handleSpeechPause(event.timestampMs);
+				return;
+			case "speech-active":
+				this.handleSpeechActive(event.timestampMs);
+				return;
+			case "speech-end":
+				this.handleSpeechEnd();
+				return;
+			case "blip":
+				return;
+			default: {
+				// exhaustive — VadEvent is a closed union; if it gains a variant
+				// the compiler will flag this cast.
+				const _exhaustive: never = event;
+				void _exhaustive;
+			}
+		}
+	}
+
+	private transitionToListening(): void {
+		if (this.state === "idle") {
+			this.currentTurnId += 1;
+		}
+		this.state = "listening";
+	}
+
+	private handleSpeechPause(timestampMs: number): void {
+		if (this.state !== "listening") return;
+		this.pauseTimestampMs = timestampMs;
+		if (!this.enabled) {
+			this.state = "pause-tentative";
+			return;
+		}
+		const turnId = this.turnIdString();
+		const filename = `C1-${turnId}`;
+		this.state = "pause-tentative";
+		void this.client
+			.saveCheckpoint(this.slotId, filename)
+			.then((handle) => {
+				if (this.state !== "pause-tentative") return;
+				this.currentCheckpoint = handle;
+				this.telemetry.onCheckpointSaved?.(handle, turnId);
+				this.startDraftIfStillPaused(turnId);
+			})
+			.catch((error: unknown) => {
+				this.telemetry.onCheckpointError?.("save", error, turnId);
+			});
+	}
+
+	private startDraftIfStillPaused(turnId: string): void {
+		if (this.state !== "pause-tentative") return;
+		const partial = this.readPartialTranscript();
+		this.currentDraft = this.startSpeculativeDraft(partial, turnId);
+		this.state = "draft-response";
+		this.telemetry.onSpeculativeDraftStarted?.(turnId);
+	}
+
+	private handleSpeechActive(timestampMs: number): void {
+		if (this.state !== "pause-tentative" && this.state !== "draft-response") {
+			// `listening → speech-active` is a heartbeat; ignore.
+			return;
+		}
+		const pauseAt = this.pauseTimestampMs;
+		if (pauseAt === null) {
+			this.state = "listening";
+			return;
+		}
+		const elapsed = timestampMs - pauseAt;
+		const rollbackWindowMs = this.pauseHangoverMs * ROLLBACK_WINDOW_MULTIPLIER;
+		if (elapsed > rollbackWindowMs) {
+			// Resumed too late to roll back — commit and let the drafter's
+			// output flow as the response.
+			this.commit();
+			return;
+		}
+		this.rollback("resumed");
+	}
+
+	private handleSpeechEnd(): void {
+		if (this.state === "pause-tentative" || this.state === "draft-response") {
+			this.commit();
+			return;
+		}
+		this.state = "idle";
+	}
+
+	private rollback(reason: AbortReason): void {
+		const turnId = this.turnIdString();
+		if (this.currentDraft) {
+			this.currentDraft.abort();
+			this.telemetry.onSpeculativeDraftAborted?.(turnId, reason);
+			this.currentDraft = null;
+		}
+		if (this.enabled && this.currentCheckpoint) {
+			const handle = this.currentCheckpoint;
+			void this.client
+				.restoreCheckpoint(this.slotId, handle.filename)
+				.then(() => this.telemetry.onCheckpointRestored?.(handle, turnId))
+				.catch((error: unknown) => {
+					this.telemetry.onCheckpointError?.("restore", error, turnId);
+				});
+		}
+		this.currentCheckpoint = null;
+		this.pauseTimestampMs = null;
+		this.state = "listening";
+	}
+
+	private commit(): void {
+		// Commit drops the snapshot reference and leaves the drafter's
+		// in-flight decode to flow into the response. We do NOT abort the
+		// drafter — its output IS the response.
+		this.currentCheckpoint = null;
+		this.currentDraft = null;
+		this.pauseTimestampMs = null;
+		this.state = "committed";
+		// Auto-return to idle so the next `speech-start` opens a new turn.
+		this.state = "idle";
+	}
+
+	private turnIdString(): string {
+		return `turn-${this.currentTurnId.toString(36)}`;
+	}
+}