@elizaos/plugin-local-inference 2.0.0-beta.1 → 2.0.3-beta.3

package/src/services/index.ts

@@ -0,0 +1,229 @@
+export type { LocalInferenceLoader } from "./active-model";
+export {
+	assertVoiceBundleFitsHost,
+	VoiceBundleDoesNotFitError,
+} from "./active-model";
+export {
+	type BackendDecision,
+	BackendDispatcher,
+	type BackendId,
+	type BackendOverride,
+	type BackendPlan,
+	decideBackend,
+	ELIZA_LLM_BACKEND_ENV,
+	type GenerateArgs as BackendGenerateArgs,
+	type GenerateResult,
+	type LocalInferenceBackend,
+	litertBackendSupported,
+	readBackendOverride,
+	resolveCatalogForPlan,
+} from "./backend";
+export {
+	ELIZA_1_PLACEHOLDER_IDS,
+	FIRST_RUN_DEFAULT_MODEL_ID,
+	findCatalogModel,
+	MODEL_CATALOG,
+} from "./catalog";
+export {
+	type CloudCandidate,
+	type CloudFallbackOptions,
+	classifyLocalError,
+	type FallbackReason,
+	findCloudCandidate,
+	type LocalGenerateOutcome,
+	makeCloudFallbackHandler,
+} from "./cloud-fallback";
+export {
+	computeRuntimeContextFit,
+	type RuntimeContextFit,
+	type RuntimeContextFitInput,
+} from "./context-fit";
+export {
+	buildDeviceResourceMetricsDevPayload,
+	type DeviceBridgeStatus,
+	type DeviceGenerationMetrics,
+	type DeviceResourceMetricsDevPayload,
+	deviceBridge,
+} from "./device-bridge";
+export {
+	type BatterySummary,
+	DeviceResourceMetrics,
+	type DeviceResourceSummary,
+	type DeviceThermalState,
+	type GenerationObservation,
+	type LowPowerSummary,
+	type ResourceSample,
+	type RssSummary,
+	type ThermalSummary,
+	type ThermalTransition,
+} from "./device-resource-metrics";
+export {
+	classifyDeviceTier,
+	DEVICE_TIER_ORDER,
+	DEVICE_TIER_THRESHOLDS,
+	type DeviceTier,
+	type DeviceTierAssessment,
+	effectiveModelMemoryGb,
+	type RecommendedMode,
+	TIER_WARNING_COPY,
+	totalRamMb,
+} from "./device-tier";
+export {
+	LocalInferenceEngine,
+	localInferenceEngine,
+	resolveIdleUnloadMs,
+	resolveMaxConcurrentSpeculativeResponses,
+} from "./engine";
+export {
+	type HandlerRegistration,
+	handlerRegistry,
+	type PublicRegistration,
+	toPublicRegistration,
+} from "./handler-registry";
+export { assessFit, probeHardware } from "./hardware";
+export {
+	createImageDescriptionRuntime,
+	type ImageDescriptionRuntime,
+	type ImageDescriptionRuntimeOptions,
+} from "./image-description-runtime";
+export {
+	type CapabilityProbes,
+	defaultsForNoBinding,
+	type InferenceCapabilities,
+	probeCapabilities,
+	type ResourceSnapshot,
+	type ThermalState,
+	type ThermalThrottleDecision,
+	thermalThrottleDecision,
+} from "./inference-capabilities";
+export {
+	InferenceTelemetry,
+	inferenceTelemetry,
+	type TelemetryTags,
+} from "./inference-telemetry";
+export {
+	estimateQuantizedKvBytesPerToken,
+	KV_SPILL_MIN_CONTEXT,
+	type KvGeometry,
+	type KvRestoreClass,
+	type KvSpillPlan,
+	KvSpillUnsupportedError,
+	planKvSpill,
+} from "./kv-spill";
+export {
+	buildVoiceLatencyDevPayload,
+	EndToEndLatencyTracer,
+	endVoiceLatencyTurn,
+	type HistogramSummary,
+	LATENCY_DERIVED_KEYS,
+	type LatencyCheckpoint,
+	type LatencyDerived,
+	type LatencyDerivedKey,
+	type LatencyTrace,
+	markVoiceLatency,
+	type TracerOptions,
+	VOICE_CHECKPOINTS,
+	type VoiceCheckpoint,
+	type VoiceLatencyDevPayload,
+	voiceLatencyTracer,
+} from "./latency-trace";
+export * from "./manifest/index";
+export {
+	type ArbiterCapability,
+	type ArbiterEvent,
+	type ArbiterEventListener,
+	type ArbiterHandle,
+	type CapabilityRegistration,
+	getMemoryArbiter,
+	MemoryArbiter,
+	type MemoryArbiterOptions,
+	setMemoryArbiter,
+	tryGetMemoryArbiter,
+} from "./memory-arbiter";
+export {
+	buildMemoryBenchmarkPlan,
+	buildMemoryBenchmarkReport,
+	type MemoryBenchmarkLoadResult,
+	type MemoryBenchmarkModelPlan,
+	type MemoryBenchmarkOptions,
+	type MemoryBenchmarkReport,
+	runMemoryBenchmark,
+	summarizeMemoryBenchmark,
+} from "./memory-benchmark";
+export {
+	type CapacitorPressureSource,
+	capacitorPressureSource,
+	compositePressureSource,
+	type MemoryPressureEvent,
+	type MemoryPressureLevel,
+	type MemoryPressureListener,
+	type MemoryPressureSource,
+	nodeOsPressureSource,
+} from "./memory-pressure";
+export {
+	type MtpDoctorCheck,
+	type MtpDoctorCheckStatus,
+	type MtpDoctorReport,
+	runMtpDoctor,
+} from "./mtp-doctor";
+export {
+	buildPlanActionsSkeleton,
+	buildPlannerGuidedDecode,
+	type PlannerAction,
+	type PlannerGuidedDecode,
+	planActionParameterSchema,
+} from "./planner-skeleton";
+export { buildTextGenerationReadiness } from "./readiness";
+export {
+	assessCatalogModelFit,
+	type BundleDefaultEligibility,
+	canBundleBeDefaultOnDevice,
+	catalogDownloadSizeBytes,
+	catalogDownloadSizeGb,
+	chooseSmallerFallbackModel,
+	classifyRecommendationPlatform,
+	deviceCapsFromProbe,
+	type RecommendationPlatformClass,
+	type RecommendedModelSelection,
+	recommendForFirstRun,
+	selectBestQuantizationVariant,
+	selectRecommendedModelForSlot,
+	selectRecommendedModels,
+} from "./recommendation";
+export {
+	type InferenceRuntimeMode,
+	type InferenceRuntimeModeInput,
+	inferencePlatformClass,
+	inferenceRuntimeMode,
+	isCapacitorNativeRuntime,
+	readRuntimeModeEnvOverride,
+	type SupportedHostPlatform,
+} from "./runtime-target";
+export { LocalInferenceService, localInferenceService } from "./service";
+export type {
+	ActiveModelState,
+	CatalogModel,
+	DownloadEvent,
+	DownloadJob,
+	DownloadState,
+	HardwareFitLevel,
+	HardwareProbe,
+	InstalledModel,
+	LocalInferenceDownloadStatus,
+	LocalInferenceReadiness,
+	LocalInferenceSlotReadiness,
+	LocalRuntimeAcceleration,
+	LocalRuntimeBackend,
+	LocalRuntimeKernel,
+	LocalRuntimeOptimizations,
+	ModelBucket,
+	ModelCategory,
+	ModelHubSnapshot,
+	TextGenerationSlot,
+} from "./types";
+export {
+	VisionEmbeddingCache,
+	type VisionEmbeddingCacheConfig,
+	type VisionEmbeddingEntry,
+} from "./vision-embedding-cache";
+export * from "./voice/index";

package/src/services/inference-capabilities.test.ts

@@ -0,0 +1,75 @@
+import { describe, expect, it } from "vitest";
+import {
+	defaultsForNoBinding,
+	probeCapabilities,
+	thermalThrottleDecision,
+} from "./inference-capabilities";
+
+describe("thermalThrottleDecision", () => {
+	it("does not throttle a cool device", () => {
+		const d = thermalThrottleDecision({ thermalState: "nominal" });
+		expect(d.throttleSpeculativeDecode).toBe(false);
+		expect(d.reduceLoad).toBe(false);
+	});
+
+	it("throttles speculative decode at serious and sheds load at critical", () => {
+		const serious = thermalThrottleDecision({ thermalState: "serious" });
+		expect(serious.throttleSpeculativeDecode).toBe(true);
+		expect(serious.reduceLoad).toBe(false);
+
+		const critical = thermalThrottleDecision({ thermalState: "critical" });
+		expect(critical.throttleSpeculativeDecode).toBe(true);
+		expect(critical.reduceLoad).toBe(true);
+	});
+
+	it("throttles under low-power mode even when cool", () => {
+		const d = thermalThrottleDecision({
+			thermalState: "nominal",
+			lowPowerMode: true,
+		});
+		expect(d.throttleSpeculativeDecode).toBe(true);
+		expect(d.reduceLoad).toBe(false);
+	});
+
+	it("does not throttle on an unknown thermal state without a low-power signal", () => {
+		const d = thermalThrottleDecision({ thermalState: "unknown" });
+		expect(d.throttleSpeculativeDecode).toBe(false);
+		expect(d.reduceLoad).toBe(false);
+	});
+
+	it("still honours low-power mode when thermal state is unknown", () => {
+		const d = thermalThrottleDecision({
+			thermalState: "unknown",
+			lowPowerMode: true,
+		});
+		expect(d.throttleSpeculativeDecode).toBe(true);
+	});
+});
+
+describe("probeCapabilities thermal gating (regression)", () => {
+	const probes = {
+		llmStreamSupported: () => true,
+		ttsStreamSupported: () => false,
+		mtpResident: () => true,
+		mmprojResident: () => false,
+		platform: () => "ios" as const,
+	};
+
+	it("keeps MTP on while cool but disables it under serious heat", () => {
+		expect(
+			probeCapabilities({ ...probes, thermalState: () => "fair" }).mtpSupported,
+		).toBe(true);
+		expect(
+			probeCapabilities({ ...probes, thermalState: () => "serious" })
+				.mtpSupported,
+		).toBe(false);
+	});
+
+	it("defaultsForNoBinding reports a safe all-off struct", () => {
+		const d = defaultsForNoBinding();
+		expect(d.streamingLlm).toBe(false);
+		expect(d.mtpSupported).toBe(false);
+		expect(d.thermalState).toBe("nominal");
+		expect(d.platform).toBe("unknown");
+	});
+});

package/src/services/inference-capabilities.ts

@@ -0,0 +1,204 @@
+/**
+ * Inference capability detection.
+ *
+ * Centralises "what does this device's local-inference stack expose"
+ * into one struct the runtime can read at startup.  The shape mirrors
+ * the per-platform binding probes (Android + iOS + desktop FFI) so the
+ * runtime doesn't have to import each platform's adapter just to
+ * surface the bits.
+ *
+ * Consumed by:
+ *   - the AOSP local-inference bootstrap, to choose the in-process FFI
+ *     streaming path,
+ *   - the desktop voice lifecycle service, to decide whether to wire the
+ *     FFI streaming runner factory,
+ *   - UI surfaces (model picker, voice toggle) that hide options the
+ *     loaded build cannot honour.
+ *
+ * Naming:
+ *   - `streamingLlm` — `eliza_inference_llm_stream_*` symbols are
+ *     resolved and the build reports `_supported() === 1`.
+ *   - `mtpSupported` — native MTP speculative decoding can actually run.
+ *   - `omnivoiceStreaming` — `eliza_inference_tts_synthesize_stream` is
+ *     present and supported.
+ *   - `mmprojSupported` — the build carries the multi-modal projector
+ *     and the device has the headroom to keep it resident.
+ *   - `thermalState` — best-effort current thermal snapshot from the
+ *     platform (`ProcessInfo.thermalState` on iOS,
+ *     `PowerManager.getCurrentThermalStatus` on Android).
+ *
+ * All fields are read-only snapshots; the runtime re-probes on resume.
+ */
+
+export type ThermalState = "nominal" | "fair" | "serious" | "critical";
+
+export interface InferenceCapabilities {
+	streamingLlm: boolean;
+	mtpSupported: boolean;
+	omnivoiceStreaming: boolean;
+	mmprojSupported: boolean;
+	thermalState: ThermalState;
+	/** Platform tag for diagnostics + routing. */
+	platform: "android" | "ios" | "desktop" | "unknown";
+}
+
+/** Minimal probe surface — what the caller hands in. */
+export interface CapabilityProbes {
+	/** True only when `eliza_inference_llm_stream_supported()` returns 1. */
+	llmStreamSupported(): boolean;
+	/** True only when `eliza_inference_tts_stream_supported()` returns 1. */
+	ttsStreamSupported(): boolean;
+	/** True only when the native MTP path is available for the loaded model. */
+	mtpResident(): boolean;
+	/** True only when the mmproj weights are present in the bundle. */
+	mmprojResident(): boolean;
+	/** Current thermal snapshot.  May return `nominal` on platforms without a thermal API. */
+	thermalState(): ThermalState;
+	/** Platform tag. */
+	platform(): "android" | "ios" | "desktop" | "unknown";
+}
+
+/**
+ * Build a capability struct from a set of probes.
+ *
+ * Policy decisions encoded here:
+ *   - Native MTP only fires when `llmStreamSupported` and the thermal state is
+ *     at most `fair`.
+ *   - mmproj is gated entirely on the bundle carrying it.  Devices
+ *     short on RAM can still load the chat model — they just lose the
+ *     vision path; the picker UI uses this bit to grey out vision
+ *     uploads.
+ *   - omnivoice streaming is gated entirely on the FFI build: the JS
+ *     side has no fallback path for streaming TTS, only for batch.
+ */
+export function probeCapabilities(
+	probes: CapabilityProbes,
+): InferenceCapabilities {
+	const streamingLlm = probes.llmStreamSupported();
+	const omnivoiceStreaming = probes.ttsStreamSupported();
+	const mtpResident = probes.mtpResident();
+	const mmprojResident = probes.mmprojResident();
+	const thermalState = probes.thermalState();
+	const platform = probes.platform();
+
+	const thermalBlocksMtp =
+		thermalState === "serious" || thermalState === "critical";
+
+	const mtpSupported = streamingLlm && mtpResident && !thermalBlocksMtp;
+
+	return {
+		streamingLlm,
+		mtpSupported,
+		omnivoiceStreaming,
+		mmprojSupported: mmprojResident,
+		thermalState,
+		platform,
+	};
+}
+
+/**
+ * Defaults probe: every flag off, platform `unknown`, thermal `nominal`.
+ * Used by the runtime when no FFI binding could be loaded (cloud-only
+ * fallback path).  Surfaces as a single struct the UI can render
+ * without branching on "no probe registered".
+ */
+export function defaultsForNoBinding(): InferenceCapabilities {
+	return {
+		streamingLlm: false,
+		mtpSupported: false,
+		omnivoiceStreaming: false,
+		mmprojSupported: false,
+		thermalState: "nominal",
+		platform: "unknown",
+	};
+}
+
+// ---------------------------------------------------------------------------
+// Sampled resource snapshot + thermal-throttle decision
+// ---------------------------------------------------------------------------
+
+/**
+ * A live, *sampled* device-resource snapshot from the native probe
+ * (iOS `getResourceSnapshot`, Android `ResourceProbe.getResourceSnapshot`) — as
+ * opposed to the one-shot `InferenceCapabilities` probe. The Mobile Resource
+ * Workbench (issue #8800) samples these on an interval across a sustained
+ * workload to build a thermal/RSS/battery timeline. Every numeric field is
+ * `null` when the platform could not measure it — never a fabricated zero.
+ */
+export interface ResourceSnapshot {
+	/** Current thermal state; `"unknown"` on platforms without a thermal API. */
+	thermalState: ThermalState | "unknown";
+	/** Whether the OS low-power / battery-saver mode is engaged, or null. */
+	lowPowerMode: boolean | null;
+	/** Process resident set size in MB, or null. */
+	residentMemoryMb: number | null;
+	/** Device-wide available RAM in MB, or null. */
+	availableRamMb: number | null;
+	/** Cumulative process CPU time in ms, or null. */
+	cpuTimeMs: number | null;
+	/** Battery level 0..100, or null. */
+	batteryLevelPct: number | null;
+	/** Sample timestamp in ms (epoch). */
+	capturedAtMs: number;
+}
+
+export interface ThermalThrottleDecision {
+	/**
+	 * Whether speculative decoding (MTP) should be disabled for the next step.
+	 * MTP burns extra compute for a latency win; under heat that trade flips.
+	 */
+	throttleSpeculativeDecode: boolean;
+	/**
+	 * Whether to proactively shed load (shrink batch / context, pause warmups)
+	 * because the device is at the top of the thermal range.
+	 */
+	reduceLoad: boolean;
+	reason: string;
+}
+
+const THROTTLE_SEVERITY: Record<ThermalState, number> = {
+	nominal: 0,
+	fair: 1,
+	serious: 2,
+	critical: 3,
+};
+
+/**
+ * Decide whether to throttle on-device inference for the current thermal /
+ * power state. Pure and synchronous so the streaming path can call it per token
+ * (the `ProcessInfo.thermalState` throttle hook the iOS streaming bridge still
+ * lists as a TODO) and the workbench can assert the policy without a device.
+ *
+ *   - `serious` / `critical` thermal → stop speculative decoding (matches the
+ *     existing one-shot MTP gate in `probeCapabilities`).
+ *   - `critical` thermal → additionally shed load.
+ *   - low-power mode → stop speculative decoding (honour the user's power intent).
+ *   - `unknown` thermal with no low-power signal → do not throttle (don't
+ *     penalise a device that simply lacks a thermal API).
+ */
+export function thermalThrottleDecision(input: {
+	thermalState: ThermalState | "unknown";
+	lowPowerMode?: boolean | null;
+}): ThermalThrottleDecision {
+	const lowPower = input.lowPowerMode === true;
+	if (input.thermalState === "unknown") {
+		return {
+			throttleSpeculativeDecode: lowPower,
+			reduceLoad: false,
+			reason: lowPower
+				? "low-power mode (thermal state unknown)"
+				: "thermal state unknown — no throttle",
+		};
+	}
+	const severity = THROTTLE_SEVERITY[input.thermalState];
+	const thermalThrottles = severity >= THROTTLE_SEVERITY.serious;
+	const reduceLoad = severity >= THROTTLE_SEVERITY.critical;
+	const throttleSpeculativeDecode = thermalThrottles || lowPower;
+	let reason: string;
+	if (reduceLoad) reason = `thermal ${input.thermalState} — shed load`;
+	else if (thermalThrottles)
+		reason = `thermal ${input.thermalState} — throttle speculative decode`;
+	else if (lowPower) reason = "low-power mode — throttle speculative decode";
+	else reason = `thermal ${input.thermalState} — nominal, no throttle`;
+	return { throttleSpeculativeDecode, reduceLoad, reason };
+}

package/src/services/inference-telemetry.ts

@@ -0,0 +1,143 @@
+/**
+ * Lightweight structured-metrics sink for local-inference telemetry.
+ *
+ * This module is intentionally small: it is a logger-backed histogram with a
+ * stable, call-site-friendly `record(name, value, tags?)` API so individual
+ * backends (FFI runtime, voice scheduler, …) can emit point-in-time
+ * observations without importing a heavy metrics framework.
+ *
+ * Design constraints
+ * ------------------
+ * - No external dependencies beyond `@elizaos/core` logger (already a dep).
+ * - `record()` is synchronous and must never throw into the call site.
+ * - Each named metric keeps a bounded ring of recent samples (default 256) for
+ *   in-process p50/p95 queries via `summary(name)`. This mirrors the
+ *   `BoundedHistogram` pattern already used by `latency-trace.ts`.
+ * - The module exports a process-wide singleton (`inferenceTelemetry`) that
+ *   all backends share, plus the class for test injection.
+ *
+ * Metric names used today
+ * -----------------------
+ *   inference.ttfa_ms        — time from fetch() to first HTTP chunk (L5)
+ *   inference.first_token_ms — time from fetch() to first decoded token (L5)
+ *   tts.chunk_size_ms        — duration of one PCM chunk in ms (T2)
+ *   tts.chunk_size_bytes     — byte size of one PCM chunk (T2)
+ */
+
+import { logger } from "@elizaos/core";
+
+// ---------------------------------------------------------------------------
+// Bounded histogram (copied structure from latency-trace.ts — same project)
+// ---------------------------------------------------------------------------
+
+interface HistogramSummary {
+	count: number;
+	p50: number | null;
+	p95: number | null;
+	mean: number | null;
+}
+
+class BoundedRing {
+	private readonly buf: number[];
+	private head = 0;
+	private filled = 0;
+
+	constructor(private readonly cap: number) {
+		this.buf = new Array<number>(cap).fill(0);
+	}
+
+	push(v: number): void {
+		if (!Number.isFinite(v)) return;
+		this.buf[this.head] = v;
+		this.head = (this.head + 1) % this.cap;
+		if (this.filled < this.cap) this.filled += 1;
+	}
+
+	summary(): HistogramSummary {
+		if (this.filled === 0)
+			return { count: 0, p50: null, p95: null, mean: null };
+		const slice = this.buf.slice(0, this.filled).sort((a, b) => a - b);
+		const pct = (q: number): number => {
+			const rank = Math.ceil((q / 100) * slice.length);
+			return slice[Math.min(slice.length - 1, Math.max(0, rank - 1))] as number;
+		};
+		const sum = slice.reduce((a, b) => a + b, 0);
+		return {
+			count: this.filled,
+			p50: pct(50),
+			p95: pct(95),
+			mean: sum / this.filled,
+		};
+	}
+}
+
+// ---------------------------------------------------------------------------
+// Public API
+// ---------------------------------------------------------------------------
+
+export type TelemetryTags = Record<string, string | number | boolean>;
+
+export class InferenceTelemetry {
+	private readonly rings = new Map<string, BoundedRing>();
+	private readonly capacity: number;
+
+	constructor(capacity = 256) {
+		this.capacity = capacity;
+	}
+
+	/**
+	 * Record a scalar observation. Never throws — telemetry must be
+	 * instrumentation, never a fault path.
+	 *
+	 * @param name  Dot-separated metric name, e.g. `"inference.ttfa_ms"`.
+	 * @param value Numeric value (non-finite values are silently dropped).
+	 * @param tags  Optional key/value labels emitted in the log line.
+	 */
+	record(name: string, value: number, tags?: TelemetryTags): void {
+		try {
+			let ring = this.rings.get(name);
+			if (!ring) {
+				ring = new BoundedRing(this.capacity);
+				this.rings.set(name, ring);
+			}
+			ring.push(value);
+			const tagStr = tags
+				? " " +
+					Object.entries(tags)
+						.map(([k, v]) => `${k}=${v}`)
+						.join(" ")
+				: "";
+			logger.debug(`[InferenceTelemetry] ${name}=${value}${tagStr}`);
+		} catch {
+			// Swallow — telemetry must never surface as a runtime error.
+		}
+	}
+
+	/**
+	 * Summary statistics for a named metric over the retained ring of samples.
+	 * Returns `{ count: 0, ... }` when the metric has never been recorded.
+	 */
+	summary(name: string): HistogramSummary {
+		return (
+			this.rings.get(name)?.summary() ?? {
+				count: 0,
+				p50: null,
+				p95: null,
+				mean: null,
+			}
+		);
+	}
+
+	/** Names of all metrics that have received at least one sample. */
+	metricNames(): string[] {
+		return [...this.rings.keys()];
+	}
+
+	/** Reset all retained samples. Useful in tests. */
+	reset(): void {
+		this.rings.clear();
+	}
+}
+
+/** Process-wide singleton used by FFI runtime and voice scheduler. */
+export const inferenceTelemetry = new InferenceTelemetry();