@elizaos/plugin-local-inference 2.0.0-beta.1 → 2.0.11-beta.7

package/src/services/manifest/validator.ts

@@ -0,0 +1,569 @@
+// Runtime validator and capability-check helpers for Eliza-1 manifests.
+//
+// Two layers of validation:
+//
+//   1. Schema validation (Zod)        — shape + types + per-field invariants.
+//   2. Contract validation (this file) — cross-field rules from
+//                                        packages/inference/AGENTS.md §3 + §6:
+//        - required-kernel set per tier is satisfied,
+//        - long-context bundles (ctx > 64k) require `turbo3_tcq`,
+//        - structural bundle invariants (voice-preset cache present, lineage
+//          ↔ files consistency, base-v1 provenance coverage),
+//        - and — for a *production* release only (`base-v1` / `finetuned-v2` /
+//          `final`, or any `defaultEligible: true` manifest) — every supported
+//          backend kernel-verified `pass` and every eval green. A
+//          candidate/staging release (`base-v1-candidate` / `local-standin` /
+//          `upload-candidate`) is publishable + installable on a device whose
+//          backend it verified, but is not held to the full bar; its
+//          `defaultEligible` must stay false.
+//
+// `defaultEligible: true` is the strongest claim a manifest can make. The
+// validator REFUSES the combination of `defaultEligible: true` and any
+// failing contract rule. This mirrors the publish-side gate in
+// `packages/training/scripts/manifest/eliza1_manifest.py`.
+
+import {
+	Eliza1ManifestSchema,
+	EMOTION_CLASSIFIER_IEMOCAP_F1_THRESHOLD,
+	EMOTION_CLASSIFIER_MEAN_LATENCY_MS_LIMIT,
+	EMOTION_CLASSIFIER_MELD_F1_THRESHOLD,
+	REQUIRED_KERNELS_BY_TIER,
+	SUPPORTED_BACKENDS_BY_TIER,
+	TURN_DETECTOR_F1_THRESHOLD,
+	TURN_DETECTOR_MEAN_LATENCY_MS_LIMIT,
+	VOICE_PRESET_CACHE_PATH,
+} from "./schema";
+import type {
+	Eliza1Backend,
+	Eliza1DeviceCaps,
+	Eliza1Kernel,
+	Eliza1Manifest,
+	Eliza1Tier,
+} from "./types";
+
+export interface ValidationOk {
+	ok: true;
+	manifest: Eliza1Manifest;
+}
+
+export interface ValidationErr {
+	ok: false;
+	errors: ReadonlyArray<string>;
+}
+
+export type ValidationResult = ValidationOk | ValidationErr;
+
+/**
+ * Schema + contract validation. Returns a Result-shaped object so callers
+ * can inspect every error rather than catching the first thrown one.
+ *
+ * Throws nothing for invalid input — invalid manifests are reported via
+ * `{ ok: false, errors }`. Truly exceptional cases (non-object input)
+ * surface as Zod issues, not exceptions.
+ */
+export function validateManifest(input: unknown): ValidationResult {
+	const parsed = Eliza1ManifestSchema.safeParse(input);
+	if (!parsed.success) {
+		return {
+			ok: false,
+			errors: parsed.error.issues.map(
+				(i) => `${i.path.join(".") || "<root>"}: ${i.message}`,
+			),
+		};
+	}
+
+	const errors = collectContractErrors(parsed.data, {
+		allowVersionStaging: true,
+	});
+	if (errors.length > 0) {
+		return { ok: false, errors };
+	}
+	return { ok: true, manifest: parsed.data };
+}
+
+/**
+ * Throws on invalid input. Use this from boot paths where a structured
+ * error is already attached at the boundary. Internal use only — UI
+ * code should prefer `validateManifest`.
+ */
+export function parseManifestOrThrow(input: unknown): Eliza1Manifest {
+	const result = validateManifest(input);
+	if (result.ok === false) {
+		throw new Error(
+			`Invalid Eliza-1 manifest:\n  - ${result.errors.join("\n  - ")}`,
+		);
+	}
+	return result.manifest;
+}
+
+/**
+ * `canSetAsDefault` is the recommendation-engine gate. A manifest that
+ * passes this is allowed to fill an empty default slot for the device:
+ *
+ *   - the manifest is contract-valid (every required kernel declared, every
+ *     required eval green for a strict release, lineage/files consistent),
+ *   - the device RAM meets the manifest's `ramBudgetMb.min` floor,
+ *   - the device exposes at least one backend the manifest verified `pass`
+ *     on out of the tier's supported set.
+ *
+ * A `defaultEligible: true` manifest is the strict release: every supported
+ * backend kernel-verified `pass`, every required eval green. A
+ * `defaultEligible: false` manifest with an explicit candidate/staging
+ * `releaseState` (`base-v1-candidate`, `local-standin`, `upload-candidate`)
+ * is still permitted to fill an empty default slot **when this device can
+ * run it** — the recommender prefers a strict release over a candidate when
+ * both are installed (see `isStrictReleaseManifest`). Version-only staging
+ * stamps such as `1.0.0-weights-staged.2` are accepted by the install parser
+ * so QA bundles can be materialized, but they do not get this auto-default
+ * relaxation unless the manifest also carries an explicit staging
+ * `releaseState`.
+ *
+ * The device-caps check rejects "this device has Vulkan only but the
+ * manifest only verified Metal/CUDA" — a manifest may be contract-valid
+ * but not runnable on this device.
+ */
+export function canSetAsDefault(
+	manifest: Eliza1Manifest,
+	device: Eliza1DeviceCaps,
+): boolean {
+	if (
+		collectContractErrors(manifest, { allowVersionStaging: false }).length > 0
+	) {
+		return false;
+	}
+	if (manifest.ramBudgetMb.min > device.ramMb) return false;
+
+	// The device must expose at least one backend that the manifest verified
+	// pass on. Pre-check against the tier's supported set so we don't accept
+	// a tier-server bundle on a Mac via the cpu fallback alone.
+	const supported = new Set<Eliza1Backend>(
+		SUPPORTED_BACKENDS_BY_TIER[manifest.tier],
+	);
+	const overlapping = device.availableBackends.filter(
+		(b) =>
+			supported.has(b) &&
+			manifest.kernels.verifiedBackends[b].status === "pass",
+	);
+	return overlapping.length > 0;
+}
+
+/**
+ * Strict release identifier: a `defaultEligible: true` manifest. The
+ * recommender uses this to prefer a strict release over a candidate
+ * bundle when both are installed and contract-valid. Mirrors the
+ * publish-side `eliza1_gates.yaml` strict bar.
+ */
+export function isStrictReleaseManifest(manifest: Eliza1Manifest): boolean {
+	return manifest.defaultEligible === true;
+}
+
+// ---------------------------------------------------------------------------
+// Internal: contract rules from AGENTS.md §3 + §6
+// ---------------------------------------------------------------------------
+
+// Release states that make the full "production" claim: every supported
+// backend kernel-verified `pass`, every eval green. `base-v1` / `finetuned-v2`
+// / `final` are published releases; `defaultEligible: true` always implies it
+// (it is the device auto-default). A manifest with no `provenance` block is
+// treated as production too — that was the only behaviour before this guard,
+// so back-compat holds. `base-v1-candidate` / `local-standin` /
+// `upload-candidate` are publishable + installable on a device whose backend
+// they *did* verify, but are not held to the full bar — their `defaultEligible`
+// must stay false (the schema's `releaseChannel=base-v1 → defaultEligible:false`
+// refinement already enforces that for the base-v1 channel; the validator now
+// honours the release-state vocabulary instead of applying the auto-default
+// bar to every manifest).
+const STRICT_RELEASE_STATES: ReadonlySet<string> = new Set([
+	"base-v1",
+	"finetuned-v2",
+	"final",
+]);
+
+const VISION_TIERS: ReadonlySet<Eliza1Tier> = new Set([
+	"0_8b",
+	"2b",
+	"4b",
+	"9b",
+	"27b",
+	"27b-256k",
+]);
+
+const MTP_TIERS: ReadonlySet<Eliza1Tier> = new Set([
+	"0_8b",
+	"2b",
+	"4b",
+	"9b",
+	"27b",
+	"27b-256k",
+]);
+
+const MIN_TEXT_CONTEXT = 131072;
+
+const STAGING_VERSION_TOKENS: ReadonlySet<string> = new Set([
+	"candidate",
+	"staged",
+	"dev",
+	"local",
+]);
+
+function isStagingManifestVersion(version: string): boolean {
+	const prerelease = version.match(
+		/^[0-9]+\.[0-9]+\.[0-9]+-([^+]+)(?:\+.*)?$/,
+	)?.[1];
+	if (!prerelease) return false;
+	return prerelease
+		.split(/[.-]/)
+		.some((token) => STAGING_VERSION_TOKENS.has(token.toLowerCase()));
+}
+
+function collectContractErrors(
+	m: Eliza1Manifest,
+	options: { allowVersionStaging?: boolean } = {},
+): string[] {
+	const errors: string[] = [];
+
+	const releaseState = m.provenance?.releaseState;
+	const strictRelease =
+		m.defaultEligible === true ||
+		(releaseState === undefined &&
+			!(
+				options.allowVersionStaging === true &&
+				isStagingManifestVersion(m.version)
+			)) ||
+		(releaseState !== undefined && STRICT_RELEASE_STATES.has(releaseState));
+
+	// Required-kernel coverage.
+	const declaredRequired = new Set<Eliza1Kernel>(m.kernels.required);
+	const tierRequired = REQUIRED_KERNELS_BY_TIER[m.tier];
+	for (const k of tierRequired) {
+		if (!declaredRequired.has(k)) {
+			errors.push(
+				`kernels.required: missing required kernel for tier ${m.tier}: ${k}`,
+			);
+		}
+	}
+
+	for (const [i, entry] of m.files.text.entries()) {
+		if (typeof entry.ctx !== "number") {
+			errors.push(`files.text[${i}].ctx: required for text GGUFs`);
+		} else if (entry.ctx < MIN_TEXT_CONTEXT) {
+			errors.push(
+				`files.text[${i}].ctx: ${entry.ctx} is below the 128k text GGUF floor`,
+			);
+		}
+		if (/-(32k|64k)\.gguf$/i.test(entry.path)) {
+			errors.push(
+				`files.text[${i}].path: 32k/64k text GGUFs are below the Eliza-1 release floor`,
+			);
+		}
+	}
+
+	// Long-context tiers MUST require turbo3_tcq once any text variant has
+	// ctx > 64k. AGENTS.md §3 Required for desktop/pro/server (#6).
+	const hasLongContextVariant = m.files.text.some(
+		(f) => typeof f.ctx === "number" && f.ctx > 65536,
+	);
+	if (hasLongContextVariant) {
+		if (!declaredRequired.has("turbo3_tcq")) {
+			errors.push(
+				"kernels.required: text variant with ctx > 64k requires turbo3_tcq",
+			);
+		}
+	}
+
+	const visionEnabled = VISION_TIERS.has(m.tier);
+	if (visionEnabled) {
+		if (m.files.vision.length === 0) {
+			errors.push(`files.vision: required for vision-enabled tier ${m.tier}`);
+		}
+	} else if (m.files.vision.length > 0) {
+		errors.push(`files.vision: unsupported for non-vision tier ${m.tier}`);
+	}
+
+	const mtpEnabled = MTP_TIERS.has(m.tier);
+	if (mtpEnabled) {
+		if (m.files.mtp.length === 0) {
+			errors.push(`files.mtp: required for MTP-enabled tier ${m.tier}`);
+		}
+		if (!m.lineage.drafter) {
+			errors.push(`lineage.drafter: required for MTP-enabled tier ${m.tier}`);
+		}
+		if (!m.evals.mtp) {
+			errors.push(`evals.mtp: required for MTP-enabled tier ${m.tier}`);
+		} else {
+			if (
+				m.evals.mtp.passed &&
+				(m.evals.mtp.acceptanceRate == null || m.evals.mtp.speedup == null)
+			) {
+				errors.push(
+					"evals.mtp.passed: cannot be true when acceptanceRate or speedup is null (needs-hardware bench)",
+				);
+			}
+			if (strictRelease && !m.evals.mtp.passed) {
+				errors.push("evals.mtp.passed: false");
+			}
+		}
+	} else if (m.files.mtp.length > 0) {
+		errors.push(`files.mtp: unsupported for non-MTP tier ${m.tier}`);
+	}
+
+	// Backend kernel-verify coverage. A production release must verify every
+	// backend the tier supports; a candidate/staging bundle need only verify at
+	// least one supported backend (the device-side `canSetAsDefault` /
+	// installability check then matches the device's available backends against
+	// the verified-`pass` set, so a CUDA-only candidate installs on CUDA hosts
+	// and is rejected on a Mac whose Metal it never verified).
+	const supportedBackends = SUPPORTED_BACKENDS_BY_TIER[m.tier];
+	if (strictRelease) {
+		for (const b of supportedBackends) {
+			const status = m.kernels.verifiedBackends[b].status;
+			if (status !== "pass") {
+				errors.push(
+					`kernels.verifiedBackends.${b}: status is "${status}", expected "pass" for tier ${m.tier}`,
+				);
+			}
+		}
+	} else if (
+		!supportedBackends.some(
+			(b) => m.kernels.verifiedBackends[b].status === "pass",
+		)
+	) {
+		errors.push(
+			`kernels.verifiedBackends: a publishable bundle must report status="pass" on at least one supported backend for tier ${m.tier} (got [${supportedBackends
+				.map((b) => `${b}:${m.kernels.verifiedBackends[b].status}`)
+				.join(", ")}])`,
+		);
+	}
+
+	// The precomputed default-voice speaker preset (`cache/voice-preset-default.bin`)
+	// is a mandatory bundle artifact — `EngineVoiceBridge.start()` hard-fails
+	// without it (AGENTS.md §4 / inference/AGENTS.md §2). It must be listed in
+	// `files.cache` so the downloader fetches it, and when the manifest declares
+	// a `voice` block its `cache.speakerPreset` must point at the same path.
+	if (!m.files.cache.some((f) => f.path === VOICE_PRESET_CACHE_PATH)) {
+		errors.push(`files.cache: missing required ${VOICE_PRESET_CACHE_PATH}`);
+	}
+	if (m.voice && m.voice.cache.speakerPreset !== VOICE_PRESET_CACHE_PATH) {
+		errors.push(
+			`voice.cache.speakerPreset: must be ${VOICE_PRESET_CACHE_PATH}, got ${m.voice.cache.speakerPreset}`,
+		);
+	}
+
+	// Eval gates. Enforced as pass/fail only for a production release; a
+	// candidate/staging bundle still carries the eval blobs (Zod-shape-checked,
+	// measured-or-`not-run`) but a non-green eval does not block publish/install
+	// — only `defaultEligible` promotion (which requires `strictRelease`).
+	if (strictRelease) {
+		if (!m.evals.textEval.passed) errors.push("evals.textEval.passed: false");
+		if (!m.evals.voiceRtf.passed) errors.push("evals.voiceRtf.passed: false");
+		if (!m.evals.e2eLoopOk) errors.push("evals.e2eLoopOk: false");
+		if (!m.evals.thirtyTurnOk) errors.push("evals.thirtyTurnOk: false");
+	}
+
+	// Optional component slots must be internally consistent: a shipped
+	// component needs auditable lineage, and lineage may not point at a
+	// component absent from the bundle. Components that affect runtime quality
+	// also require their own publish gate to pass.
+	if (m.defaultEligible) {
+		if (m.files.asr.length === 0) {
+			errors.push(
+				"files.asr: required for defaultEligible local voice bundles",
+			);
+		}
+		if ((m.files.vad ?? []).length === 0) {
+			errors.push(
+				"files.vad: required for defaultEligible local voice bundles",
+			);
+		}
+	}
+
+	for (const slot of [
+		"asr",
+		"embedding",
+		"imagegen",
+		"vision",
+		"vad",
+		"wakeword",
+		"turn",
+		"emotion",
+	] as const) {
+		const files = m.files[slot] ?? [];
+		const lineage = m.lineage[slot];
+		if (files.length > 0 && !lineage) {
+			errors.push(`lineage.${slot}: required when files.${slot} is non-empty`);
+		}
+		if (lineage && files.length === 0) {
+			errors.push(`files.${slot}: required when lineage.${slot} is present`);
+		}
+	}
+	if (m.lineage.drafter && m.files.mtp.length === 0) {
+		errors.push("files.mtp: required when lineage.drafter is present");
+	}
+
+	if (m.files.asr.length > 0) {
+		if (!m.evals.asrWer) {
+			errors.push("evals.asrWer: required when files.asr is non-empty");
+		} else if (strictRelease && !m.evals.asrWer.passed) {
+			errors.push("evals.asrWer.passed: false");
+		}
+	}
+	if ((m.files.embedding ?? []).length > 0) {
+		if (!m.evals.embedMteb) {
+			errors.push(
+				"evals.embedMteb: required when files.embedding is non-empty",
+			);
+		} else if (strictRelease && !m.evals.embedMteb.passed) {
+			errors.push("evals.embedMteb.passed: false");
+		}
+	}
+	if ((m.files.vad ?? []).length > 0) {
+		if (!m.evals.vadLatencyMs) {
+			errors.push("evals.vadLatencyMs: required when files.vad is non-empty");
+		} else if (strictRelease && !m.evals.vadLatencyMs.passed) {
+			errors.push("evals.vadLatencyMs.passed: false");
+		}
+	}
+	// Voice Wave 2 (2026-05-14): turn-detector eval gate. When the bundle
+	// ships `files.turn` (the LiveKit/Turnsense ONNX) the manifest MUST
+	// declare a `turnDetector` eval block; a strict release additionally
+	// requires `passed=true` AND the precomputed `passed` field to be
+	// internally consistent with the threshold constants.
+	if ((m.files.turn ?? []).length > 0) {
+		const td = m.evals.turnDetector;
+		if (!td) {
+			errors.push("evals.turnDetector: required when files.turn is non-empty");
+		} else {
+			const gateMet =
+				td.f1 >= TURN_DETECTOR_F1_THRESHOLD &&
+				td.meanLatencyMs <= TURN_DETECTOR_MEAN_LATENCY_MS_LIMIT;
+			if (td.passed !== gateMet) {
+				errors.push(
+					`evals.turnDetector.passed: ${td.passed} disagrees with measured gate (f1=${td.f1} ≥ ${TURN_DETECTOR_F1_THRESHOLD} && meanLatencyMs=${td.meanLatencyMs} ≤ ${TURN_DETECTOR_MEAN_LATENCY_MS_LIMIT} → ${gateMet})`,
+				);
+			}
+			if (strictRelease && !td.passed) {
+				errors.push("evals.turnDetector.passed: false");
+			}
+		}
+	}
+	const expressiveVoice =
+		m.voice?.capabilities.includes("emotion-tags") ||
+		m.voice?.capabilities.includes("singing");
+	if (expressiveVoice) {
+		if (!m.evals.expressive) {
+			errors.push(
+				"evals.expressive: required when voice capabilities include emotion-tags or singing",
+			);
+		} else if (strictRelease && !m.evals.expressive.passed) {
+			errors.push("evals.expressive.passed: false");
+		}
+	}
+
+	// Voice Wave 2 (2026-05-14): acoustic-emotion classifier eval gate. Same
+	// shape as `turnDetector`: a bundle that ships `files.emotion` MUST
+	// declare a precomputed `emotionClassifier` block; a strict release
+	// additionally requires `passed=true` and internal consistency with the
+	// threshold constants. The MELD bar is intentionally low (~0.35) per
+	// R3-emotion §6 — refusing to publish a real improvement is worse than
+	// admitting 7-class conversational SER is hard.
+	if ((m.files.emotion ?? []).length > 0) {
+		const ec = m.evals.emotionClassifier;
+		if (!ec) {
+			errors.push(
+				"evals.emotionClassifier: required when files.emotion is non-empty",
+			);
+		} else {
+			const gateMet =
+				ec.macroF1Meld >= EMOTION_CLASSIFIER_MELD_F1_THRESHOLD &&
+				ec.macroF1Iemocap >= EMOTION_CLASSIFIER_IEMOCAP_F1_THRESHOLD &&
+				ec.meanLatencyMs <= EMOTION_CLASSIFIER_MEAN_LATENCY_MS_LIMIT;
+			if (ec.passed !== gateMet) {
+				errors.push(
+					`evals.emotionClassifier.passed: ${ec.passed} disagrees with measured gate (` +
+						`macroF1Meld=${ec.macroF1Meld} ≥ ${EMOTION_CLASSIFIER_MELD_F1_THRESHOLD} && ` +
+						`macroF1Iemocap=${ec.macroF1Iemocap} ≥ ${EMOTION_CLASSIFIER_IEMOCAP_F1_THRESHOLD} && ` +
+						`meanLatencyMs=${ec.meanLatencyMs} ≤ ${EMOTION_CLASSIFIER_MEAN_LATENCY_MS_LIMIT} → ${gateMet})`,
+				);
+			}
+			if (strictRelease && !ec.passed) {
+				errors.push("evals.emotionClassifier.passed: false");
+			}
+		}
+	}
+
+	// base-v1 provenance coverage. A `base-v1` manifest (the upstream base
+	// models, GGUF-converted + fully optimized, NOT fine-tuned) MUST record
+	// where every shipped component comes from — that is the whole point of
+	// the release state.
+	if (m.provenance) {
+		if (
+			m.provenance.releaseState === "base-v1" &&
+			m.provenance.finetuned !== false
+		) {
+			errors.push(
+				"provenance.finetuned: must be false for releaseState=base-v1",
+			);
+		}
+		if (m.provenance.releaseState === "base-v1") {
+			const requiredSlots: Array<keyof typeof m.provenance.sourceModels> = [
+				"text",
+				"voice",
+			];
+			for (const slot of ["asr", "vad", "embedding", "vision"] as const) {
+				if ((m.files[slot] ?? []).length > 0) requiredSlots.push(slot);
+			}
+			if (m.files.mtp.length > 0) {
+				requiredSlots.push("drafter");
+			}
+			if ((m.files.imagegen ?? []).length > 0) {
+				requiredSlots.push("imagegen");
+			}
+			for (const slot of requiredSlots) {
+				if (!m.provenance.sourceModels[slot]) {
+					errors.push(
+						`provenance.sourceModels.${slot}: required for releaseState=base-v1 (component is in files.${slot})`,
+					);
+				}
+			}
+		}
+	}
+
+	// EAGLE3 bench metadata is always optional. When
+	// present, it may record a not-run/failure state; only a passing claim must
+	// include measured acceptance/speedup values.
+	if (m.evals.eagle3) {
+		const eagle3Passed = m.evals.eagle3.passed ?? m.evals.eagle3.pass;
+		if (
+			eagle3Passed === true &&
+			(m.evals.eagle3.acceptanceRate == null || m.evals.eagle3.speedup == null)
+		) {
+			errors.push(
+				"evals.eagle3: passed=true requires measured acceptanceRate and speedup",
+			);
+		}
+	}
+
+	// The strongest claim: defaultEligible. If anything above failed, this
+	// flag must be false. (Contract errors are already accumulated; we add
+	// an explicit message so callers can identify the violation cleanly.)
+	if (m.defaultEligible && errors.length > 0) {
+		errors.unshift(
+			"defaultEligible: true requires all required kernels, supported backends, and evals to pass",
+		);
+	}
+
+	return errors;
+}
+
+/**
+ * Convenience: list missing required kernels for a tier without doing
+ * full validation. Used by the recommendation engine when surfacing
+ * "this bundle is broken" diagnostics.
+ */
+export function missingRequiredKernels(
+	tier: Eliza1Tier,
+	declaredRequired: ReadonlyArray<Eliza1Kernel>,
+): ReadonlyArray<Eliza1Kernel> {
+	const declared = new Set(declaredRequired);
+	return REQUIRED_KERNELS_BY_TIER[tier].filter((k) => !declared.has(k));
+}