@elizaos/plugin-local-inference 2.0.3-beta.2 → 2.0.3-beta.3

package/src/services/live-signals.ts

@@ -0,0 +1,177 @@
+/**
+ * Live device signals for the capability-driven `auto` routing policy.
+ *
+ * `classifyDeviceTier()` is a *static* assessment — it scores the hardware once
+ * and caches the result. But a device that classifies MAX/GOOD at session start
+ * can still become a poor place to run local inference *right now*: a sustained
+ * workload pushes the SoC into thermal throttling, or decode throughput collapses
+ * under memory pressure. Those are live, time-varying signals the static tier
+ * cannot see.
+ *
+ * This module exposes the two live signals the `auto` branch demotes on:
+ *
+ *   - **thermal state** — read from the device bridge's most recent device
+ *     capabilities (`nominal | fair | serious | critical | unknown`). `serious`
+ *     or `critical` means the OS is actively throttling clocks; forcing local
+ *     inference there makes turns slower than cloud and worsens the throttle.
+ *   - **decode throughput** — the recent p50 decode tokens/sec from
+ *     `inferenceTelemetry`. When it falls below a usable budget the on-device
+ *     path is no longer competitive and `auto` routes to cloud.
+ *
+ * Both signals are *optional*: a quantity the host could not measure is reported
+ * as `null` (never a fabricated `0`), and a `null` signal never demotes — the
+ * static tier decision stands. The default source reads the real subsystems; a
+ * test or alternate host can inject its own via `setLiveDeviceSignalsSource`.
+ */
+
+import { deviceBridge } from "./device-bridge";
+import { inferenceTelemetry } from "./inference-telemetry";
+
+/** Thermal pressure reported by the OS. Mirrors the device-bridge wire enum. */
+export type ThermalState =
+	| "nominal"
+	| "fair"
+	| "serious"
+	| "critical"
+	| "unknown";
+
+/**
+ * The telemetry metric name carrying recent on-device decode throughput
+ * (tokens/sec). The backend records this after a generation completes; until
+ * then `summary()` returns a `count: 0` row and the signal is `null`.
+ */
+export const DECODE_TPS_METRIC = "inference.decode_tps";
+
+/**
+ * Minimum decode throughput (tokens/sec) below which the local path is no longer
+ * worth keeping under `auto`. Roughly the floor where on-device generation feels
+ * slower than a cloud round-trip on a constrained mobile SoC.
+ */
+export const MIN_DECODE_TPS_BUDGET = 6;
+
+/**
+ * Minimum decoded-token count a generation must produce before its throughput is
+ * trusted as a routing signal. A one- or two-token reply is dominated by prompt
+ * prefill + sampling fixed costs, so its tok/s is noise — recording it would
+ * smear the p50 the router demotes on. Real turns clear this floor easily.
+ */
+const MIN_TOKENS_FOR_TPS_SAMPLE = 4;
+
+/**
+ * Approximate decoded-token count for a generated string when an exact tokenizer
+ * count is not reachable from the call site. English averages ~4 characters per
+ * token; this is deliberately coarse — it only feeds a p50 the router compares
+ * against {@link MIN_DECODE_TPS_BUDGET}, never a billing or context-window count.
+ */
+export function estimateDecodeTokens(text: string): number {
+	return Math.ceil(text.length / 4);
+}
+
+/**
+ * Record one on-device decode-throughput observation into the telemetry ring the
+ * `auto` router reads via {@link DECODE_TPS_METRIC}. This is the producer that
+ * activates the dormant TPS demotion signal: backends call it after a real local
+ * generation completes with a known decoded-token count and wall-clock decode
+ * span.
+ *
+ * Drops the sample (records nothing) when it would be noise rather than signal:
+ * an empty/aborted generation, a sub-{@link MIN_TOKENS_FOR_TPS_SAMPLE} reply, or
+ * a non-positive elapsed time (a clock that did not advance). Never throws —
+ * `inferenceTelemetry.record` already guards the call site.
+ */
+export function recordDecodeThroughput(sample: {
+	tokens: number;
+	elapsedMs: number;
+}): void {
+	const { tokens, elapsedMs } = sample;
+	if (
+		!Number.isFinite(tokens) ||
+		tokens < MIN_TOKENS_FOR_TPS_SAMPLE ||
+		!Number.isFinite(elapsedMs) ||
+		elapsedMs <= 0
+	) {
+		return;
+	}
+	const tokPerSec = tokens / (elapsedMs / 1000);
+	if (!Number.isFinite(tokPerSec) || tokPerSec <= 0) {
+		return;
+	}
+	inferenceTelemetry.record(DECODE_TPS_METRIC, tokPerSec);
+}
+
+/** A point-in-time snapshot of the live device signals. */
+export interface LiveDeviceSignals {
+	/** Current OS thermal pressure, or `null` when no device reported it. */
+	thermalState: ThermalState | null;
+	/** Recent p50 decode throughput in tokens/sec, or `null` when unmeasured. */
+	decodeTokensPerSecond: number | null;
+}
+
+/** A source of live device signals — swappable for tests and alternate hosts. */
+export type LiveDeviceSignalsSource = () => LiveDeviceSignals;
+
+const VALID_THERMAL_STATES: ReadonlySet<string> = new Set([
+	"nominal",
+	"fair",
+	"serious",
+	"critical",
+	"unknown",
+]);
+
+function normalizeThermalState(value: unknown): ThermalState | null {
+	return typeof value === "string" && VALID_THERMAL_STATES.has(value)
+		? (value as ThermalState)
+		: null;
+}
+
+/**
+ * The default source: thermal from the device bridge's primary device, decode
+ * tok/s from the telemetry ring. Pure reads — never throws into the router.
+ */
+const defaultSource: LiveDeviceSignalsSource = () => {
+	const thermalState = normalizeThermalState(
+		deviceBridge.status().capabilities?.thermalState,
+	);
+	const decodeTokensPerSecond =
+		inferenceTelemetry.summary(DECODE_TPS_METRIC).p50;
+	return { thermalState, decodeTokensPerSecond };
+};
+
+let activeSource: LiveDeviceSignalsSource = defaultSource;
+
+/** Read the current live device signals. */
+export function readLiveDeviceSignals(): LiveDeviceSignals {
+	return activeSource();
+}
+
+/**
+ * Override the live-signals source. Pass `null` to restore the default
+ * (device-bridge + telemetry) source. Intended for tests and alternate hosts.
+ */
+export function setLiveDeviceSignalsSource(
+	source: LiveDeviceSignalsSource | null,
+): void {
+	activeSource = source ?? defaultSource;
+}
+
+/**
+ * Whether the live signals say the device should be demoted off local for now,
+ * even though the static tier favours local. True when the OS is throttling
+ * (`serious`/`critical`) or measured decode throughput sits below the budget.
+ * A `null` signal never demotes.
+ */
+export function liveSignalsDemoteLocal(signals: LiveDeviceSignals): boolean {
+	if (
+		signals.thermalState === "serious" ||
+		signals.thermalState === "critical"
+	) {
+		return true;
+	}
+	if (
+		signals.decodeTokensPerSecond !== null &&
+		signals.decodeTokensPerSecond < MIN_DECODE_TPS_BUDGET
+	) {
+		return true;
+	}
+	return false;
+}

package/src/services/llama-server-metrics.test.ts

@@ -0,0 +1,168 @@
+import { describe, expect, it } from "vitest";
+import {
+	diffSnapshots,
+	type LlamaServerMetricSnapshot,
+	parsePrometheusMetrics,
+} from "./llama-server-metrics";
+
+/**
+ * llama-server `/metrics` → Anthropic-shape usage accounting (#8848).
+ *
+ * `parsePrometheusMetrics` turns the Prometheus exposition payload into a
+ * counter snapshot, and `diffSnapshots` differences two snapshots into the
+ * cache-aware usage block callers bill against. Both are pure; a regression in
+ * the label-summing, the generation-counter flag, the negative-delta clamp, or
+ * the cache-creation bound silently corrupts every local-inference usage
+ * report. These pin that math.
+ */
+
+const snap = (
+	o: Partial<LlamaServerMetricSnapshot>,
+): LlamaServerMetricSnapshot => ({
+	takenAtMs: 0,
+	promptTokensTotal: 0,
+	predictedTokensTotal: 0,
+	promptTokensProcessedTotal: 0,
+	draftedTotal: 0,
+	acceptedTotal: 0,
+	kvCacheTokens: 0,
+	kvCacheUsedCells: 0,
+	...o,
+});
+
+describe("parsePrometheusMetrics", () => {
+	it("parses an unlabelled counter and flags generation counters", () => {
+		const out = parsePrometheusMetrics(
+			"llamacpp:prompt_tokens_total 1234",
+			1000,
+		);
+		expect(out.promptTokensTotal).toBe(1234);
+		expect(out.scrapeOk).toBe(true);
+		expect(out.hasGenerationCounters).toBe(true);
+		expect(out.takenAtMs).toBe(1000);
+	});
+
+	it("sums per-slot labelled samples for the same canonical field", () => {
+		const body =
+			'llamacpp:n_drafted_accepted_total{slot_id="0"} 12\n' +
+			'llamacpp:n_drafted_accepted_total{slot_id="1"} 8';
+		expect(parsePrometheusMetrics(body, 0).acceptedTotal).toBe(20);
+	});
+
+	it("prefers an unlabelled total over the labelled sum of the same field", () => {
+		const body =
+			'llamacpp:n_drafted_accepted_total{slot_id="0"} 99\n' +
+			"llamacpp:n_drafted_accepted_total 5";
+		expect(parsePrometheusMetrics(body, 0).acceptedTotal).toBe(5);
+	});
+
+	it("skips comments, malformed lines, and unknown counters", () => {
+		const body = [
+			"# HELP llamacpp:prompt_tokens_total total",
+			"# TYPE llamacpp:prompt_tokens_total counter",
+			"garbage line here",
+			"llamacpp:unknown_counter 7",
+		].join("\n");
+		const out = parsePrometheusMetrics(body, 0);
+		expect(out.promptTokensTotal).toBe(0);
+		expect(out.hasGenerationCounters).toBe(false);
+		expect(out.scrapeOk).toBe(true);
+	});
+
+	it("does not flag generation counters for a gauge-only scrape", () => {
+		const out = parsePrometheusMetrics("llamacpp:kv_cache_tokens 512", 0);
+		expect(out.kvCacheTokens).toBe(512);
+		expect(out.hasGenerationCounters).toBe(false);
+	});
+
+	it("returns an all-zero snapshot for an empty body", () => {
+		const out = parsePrometheusMetrics("", 0);
+		expect(out).toMatchObject({
+			promptTokensTotal: 0,
+			predictedTokensTotal: 0,
+			acceptedTotal: 0,
+			hasGenerationCounters: false,
+			scrapeOk: true,
+		});
+	});
+});
+
+describe("diffSnapshots", () => {
+	it("computes input/output and the cache creation/read split", () => {
+		const block = diffSnapshots(
+			snap({
+				promptTokensTotal: 100,
+				predictedTokensTotal: 10,
+				promptTokensProcessedTotal: 40,
+			}),
+			snap({
+				promptTokensTotal: 200,
+				predictedTokensTotal: 30,
+				promptTokensProcessedTotal: 90,
+			}),
+		);
+		expect(block).toMatchObject({
+			input_tokens: 100,
+			output_tokens: 20,
+			cache_creation_input_tokens: 50, // 90-40 processed
+			cache_read_input_tokens: 50, // 100 input - 50 fresh
+			cache_hit_rate: 0.5,
+		});
+	});
+
+	it("clamps negative deltas (server restart) to zero and omits cache_hit_rate", () => {
+		const block = diffSnapshots(
+			snap({ promptTokensTotal: 200, predictedTokensTotal: 50 }),
+			snap({ promptTokensTotal: 100, predictedTokensTotal: 10 }),
+		);
+		expect(block.input_tokens).toBe(0);
+		expect(block.output_tokens).toBe(0);
+		expect(block.cache_hit_rate).toBeUndefined();
+	});
+
+	it("lets responseUsage override the metric delta", () => {
+		const block = diffSnapshots(
+			snap({}),
+			snap({ promptTokensTotal: 1000, predictedTokensTotal: 500 }),
+			{ prompt_tokens: 7, completion_tokens: 3 },
+		);
+		expect(block.input_tokens).toBe(7);
+		expect(block.output_tokens).toBe(3);
+		expect(block.cache_read_input_tokens).toBe(7); // no fresh prefill → all cached
+		expect(block.cache_hit_rate).toBe(1);
+	});
+
+	it("bounds cache creation by the per-call input count", () => {
+		const block = diffSnapshots(
+			snap({}),
+			snap({ promptTokensTotal: 100, promptTokensProcessedTotal: 300 }),
+		);
+		expect(block.cache_creation_input_tokens).toBe(100); // min(300, 100)
+		expect(block.cache_read_input_tokens).toBe(0);
+		expect(block.cache_hit_rate).toBe(0);
+	});
+
+	it("adds MTP fields only when speculative tokens were drafted", () => {
+		const withMtp = diffSnapshots(
+			snap({}),
+			snap({
+				promptTokensTotal: 50,
+				predictedTokensTotal: 50,
+				draftedTotal: 10,
+				acceptedTotal: 6,
+			}),
+		);
+		expect(withMtp).toMatchObject({
+			mtp_drafted_tokens: 10,
+			mtp_accepted_tokens: 6,
+			mtp_acceptance_rate: 0.6,
+		});
+
+		const noMtp = diffSnapshots(
+			snap({}),
+			snap({ promptTokensTotal: 10, predictedTokensTotal: 10 }),
+		);
+		expect(noMtp.mtp_drafted_tokens).toBeUndefined();
+		expect(noMtp.mtp_acceptance_rate).toBeUndefined();
+	});
+});

package/src/services/manifest/eliza-1.manifest.v1.json

@@ -228,8 +228,37 @@
 						}
 					}
 				},
+				"specDecode": {
+					"description": "Optional speculative-decode capability metadata (drafter-agnostic: MTP separate drafter, embedded draft head, or EAGLE3 head). Independent from required tier kernel names.",
+					"type": "object",
+					"properties": {
+						"enabled": {
+							"type": "boolean"
+						},
+						"capability": {
+							"type": "string",
+							"minLength": 1
+						},
+						"specType": {
+							"type": "string",
+							"minLength": 1
+						},
+						"model": {
+							"type": "string",
+							"minLength": 1
+						},
+						"maxDraftTokens": {
+							"type": "integer",
+							"minimum": 1
+						},
+						"failure": {
+							"type": "string",
+							"minLength": 1
+						}
+					}
+				},
 				"eagle3": {
-					"description": "Optional EAGLE3 speculative-decoding capability metadata. This is independent from required tier kernel names.",
+					"description": "Back-compat alias for specDecode accepted for one release; same shape as specDecode.",
 					"type": "object",
 					"properties": {
 						"enabled": {
@@ -402,8 +431,34 @@
 						}
 					}
 				},
+				"specDecode": {
+					"description": "Optional speculative-decode eval metadata (drafter-agnostic). Passing claims require measured acceptanceRate and speedup in runtime validators.",
+					"type": "object",
+					"additionalProperties": false,
+					"properties": {
+						"acceptanceRate": {
+							"type": ["number", "null"],
+							"minimum": 0,
+							"maximum": 1
+						},
+						"speedup": {
+							"type": ["number", "null"],
+							"minimum": 0
+						},
+						"passed": {
+							"type": "boolean"
+						},
+						"pass": {
+							"type": "boolean"
+						},
+						"failure": {
+							"type": "string",
+							"minLength": 1
+						}
+					}
+				},
 				"eagle3": {
-					"description": "Optional EAGLE3 speculative-decoding eval metadata. Passing claims require measured acceptanceRate and speedup in runtime validators.",
+					"description": "Back-compat alias for specDecode eval metadata accepted for one release; same shape as specDecode.",
 					"type": "object",
 					"additionalProperties": false,
 					"properties": {
@@ -588,6 +643,28 @@
 		},
 		"defaultEligible": {
 			"type": "boolean"
+		},
+		"schemaVersion": {
+			"description": "Manifest format version. \"2\" adds the Gemma 4 tokenizer/kv/mtp identity block; absent/\"1\" is the pre-cutover shape.",
+			"oneOf": [{ "type": "string" }, { "type": "integer" }]
+		},
+		"tokenizer": {
+			"description": "Shared Gemma 4 tokenizer identity (schemaVersion 2).",
+			"type": "object",
+			"additionalProperties": false,
+			"required": ["family", "vocabSize"],
+			"properties": {
+				"family": { "type": "string" },
+				"vocabSize": { "type": "integer", "minimum": 1 }
+			}
+		},
+		"kv": {
+			"description": "KV-cache mode the bundle ships (e.g. \"stock-q8_0\").",
+			"type": "string"
+		},
+		"mtp": {
+			"description": "Speculative-decode MTP mode. Gemma 4 tiers ship a separate drafter GGUF (mtp/drafter-<tier>.gguf).",
+			"enum": ["separate-drafter", "embedded-draft-head"]
 		}
 	},
 	"allOf": [
@@ -664,6 +741,11 @@
 				"ctx": {
 					"type": "integer",
 					"minimum": 131072
+				},
+				"runtimeClass": {
+					"description": "Optional runtime-class discriminator for a files.text entry. Absent defaults to 'llama-cpp' (GGUF, the default path). A '.litertlm' text artifact declares 'litert-lm' so the loader routes it to the in-process LiteRT-LM backend (Android NPU/GPU delegate) instead of the fused llama.cpp path.",
+					"type": "string",
+					"enum": ["llama-cpp", "litert-lm"]
 				}
 			}
 		},

package/src/services/manifest/index.ts

@@ -10,6 +10,7 @@ export {
 	ELIZA_1_PROVENANCE_SLOTS,
 	ELIZA_1_RELEASE_CHANNELS,
 	ELIZA_1_RELEASE_STATES,
+	ELIZA_1_TEXT_RUNTIME_CLASSES,
 	ELIZA_1_TIERS,
 	ELIZA_1_TOKENIZER_FAMILY,
 	ELIZA_1_TOKENIZER_VOCAB_SIZE,
@@ -22,13 +23,16 @@ export {
 	Eliza1FilesSchema,
 	Eliza1KernelEnumSchema,
 	Eliza1KernelsSchema,
+	Eliza1LibFileEntrySchema,
 	Eliza1LineageSchema,
 	Eliza1ManifestSchema,
 	Eliza1ProvenanceSchema,
 	Eliza1RamBudgetSchema,
+	Eliza1TextRuntimeClassEnumSchema,
 	Eliza1TierEnumSchema,
 	Eliza1VerifiedBackendStatusSchema,
 	Eliza1VoiceSchema,
+	OPTIONAL_KERNELS_BY_TIER,
 	REQUIRED_KERNELS_BY_TIER,
 	RUNTIME_TO_ELIZA1_KERNEL,
 	SUPPORTED_BACKENDS_BY_TIER,
@@ -43,6 +47,7 @@ export type {
 	Eliza1Files,
 	Eliza1Kernel,
 	Eliza1Kernels,
+	Eliza1LibFileEntry,
 	Eliza1Lineage,
 	Eliza1Manifest,
 	Eliza1Provenance,
@@ -50,6 +55,7 @@ export type {
 	Eliza1RamBudget,
 	Eliza1ReleaseChannel,
 	Eliza1ReleaseState,
+	Eliza1TextRuntimeClass,
 	Eliza1Tier,
 	Eliza1VerifiedBackendStatus,
 	Eliza1Voice,