@elizaos/plugin-local-inference 2.0.0-beta.1 → 2.0.3-beta.2

package/src/services/imagegen/backend-selector.ts

@@ -0,0 +1,277 @@
+/**
+ * Image-gen backend-pick logic (WS3).
+ *
+ * Given a runtime platform + accelerator hint, return the ordered list
+ * of backend ids to try. The arbiter walks this list at load time and
+ * stops at the first backend that resolves to a live handle. A backend
+ * that throws `ImageGenBackendUnavailableError` lets the selector fall
+ * through to the next.
+ *
+ * Per-platform policy:
+ *
+ *   - **macOS Apple Silicon**: `mflux` (MLX) → `sd-cpp` (CPU fallback).
+ *     mflux is Metal-accelerated through MLX; sd-cpp-CPU is the
+ *     functional last-resort if the venv isn't installed.
+ *   - **macOS Intel**: `sd-cpp` (Metal/CPU). MLX requires Apple Silicon;
+ *     mflux is skipped on Intel Macs.
+ *   - **iOS**: `coreml` only. There is no sd-cpp on iOS — Core ML is
+ *     the only sanctioned acceleration path.
+ *   - **Linux + NVIDIA**: `sd-cpp` CUDA only when the sd-cpp probe,
+ *     binary manifest, help, or version output proves CUDA support.
+ *     Otherwise use `sd-cpp` CPU and let the loader emit an explicit
+ *     CUDA-missing fallback error if CUDA was requested elsewhere.
+ *   - **Linux + AMD/Intel**: `sd-cpp --vulkan` → `sd-cpp` (CPU fallback).
+ *   - **Linux + CPU**: `sd-cpp` (CPU).
+ *   - **Windows + NVIDIA**: `tensorrt` → `sd-cpp` (CUDA) → `sd-cpp` (CPU).
+ *   - **Windows + AMD/Intel**: `sd-cpp --vulkan` → `sd-cpp` (CPU).
+ *   - **Android (AOSP)**: `aosp` (libeliza-llama-shim with Vulkan/OpenCL,
+ *     handled by `plugin-aosp-local-inference`). No second backend on
+ *     mobile — falling back to CPU sd-cpp on a phone is too slow to be
+ *     useful and would just block the UI.
+ *
+ * The selector intentionally does NOT account for tier or model
+ * compatibility — that's the catalog's job. A 27B-tier user on a
+ * Vulkan-only Linux box doesn't change the selector; it just changes
+ * which model the chosen backend loads.
+ */
+
+import type { ImageGenBackend } from "./types";
+
+/** Per-platform runtime fingerprint the selector consumes. */
+export interface ImageGenRuntimeProfile {
+	/** Node-style `process.platform`: `"linux" | "darwin" | "win32" | "android"`. */
+	platform: NodeJS.Platform | "android" | "ios";
+	/**
+	 * `process.arch` (`"x64"` | `"arm64"`). Apple Silicon = `"arm64"` on
+	 * `"darwin"`. Intel Mac = `"x64"` on `"darwin"`.
+	 */
+	arch: NodeJS.Architecture | string;
+	/**
+	 * GPU vendor hint, when detected. `"nvidia"` enables CUDA / TensorRT
+	 * paths; `"amd"` and `"intel"` favour Vulkan. `"apple"` is implied by
+	 * platform=darwin+arch=arm64.
+	 */
+	gpu?: "nvidia" | "amd" | "intel" | "apple" | "none";
+	/**
+	 * sd-cpp binary capability evidence gathered from the first-run probe,
+	 * bundle manifest, or a loader-side help/version check. GPU vendor alone
+	 * is not evidence that the installed `sd` binary was built with CUDA.
+	 */
+	sdCpp?: {
+		accelerators?: readonly string[];
+		cudaCapable?: boolean;
+	};
+	/**
+	 * Explicit accelerator requirement for release verification and operator
+	 * smoke tests. When set, the selector returns only backends that can
+	 * satisfy that accelerator, so CUDA/Vulkan evidence cannot be produced by
+	 * silently falling through to CPU.
+	 */
+	requiredAccelerator?:
+		| "cpu"
+		| "cuda"
+		| "vulkan"
+		| "metal"
+		| "coreml"
+		| "tensorrt";
+	/**
+	 * True when running inside the iOS Capacitor shell. We can't infer
+	 * from `process.platform` alone because Capacitor reports `"ios"`
+	 * but Node's typings only know `darwin`/`linux`/etc.
+	 */
+	isIos?: boolean;
+	/**
+	 * True when running inside the Android Capacitor / `plugin-aosp-local-inference`
+	 * shell. Node reports `"linux"` on AOSP, so we need an explicit signal.
+	 */
+	isAndroid?: boolean;
+}
+
+export type ImageGenBackendId = ImageGenBackend["id"];
+
+/**
+ * Ordered list of `(backendId, accelerator)` pairs to try. The caller
+ * picks the first one whose backend loader returns a live handle.
+ *
+ * Accelerator strings match `ImageGenLoadArgs.accelerator`.
+ */
+export interface ImageGenBackendChoice {
+	backendId: ImageGenBackendId;
+	accelerator?:
+		| "auto"
+		| "cpu"
+		| "cuda"
+		| "vulkan"
+		| "metal"
+		| "coreml"
+		| "tensorrt";
+}
+
+export function selectImageGenBackends(
+	profile: ImageGenRuntimeProfile,
+): readonly ImageGenBackendChoice[] {
+	if (profile.requiredAccelerator) {
+		switch (profile.requiredAccelerator) {
+			case "coreml":
+				return [{ backendId: "coreml", accelerator: "coreml" }];
+			case "tensorrt":
+				return [{ backendId: "tensorrt", accelerator: "tensorrt" }];
+			case "metal":
+				return profile.platform === "darwin" && profile.arch === "arm64"
+					? [{ backendId: "mflux", accelerator: "metal" }]
+					: [{ backendId: "sd-cpp", accelerator: "metal" }];
+			case "cuda":
+				return [{ backendId: "sd-cpp", accelerator: "cuda" }];
+			case "vulkan":
+				return [{ backendId: "sd-cpp", accelerator: "vulkan" }];
+			case "cpu":
+				return [{ backendId: "sd-cpp", accelerator: "cpu" }];
+		}
+	}
+	if (profile.isIos) {
+		// iOS Capacitor: Core ML only. No fallback — sd-cpp is not
+		// shipped on iOS and falling back to nothing surfaces a clean
+		// "unavailable on this device" error in the UI.
+		return [{ backendId: "coreml", accelerator: "coreml" }];
+	}
+	if (profile.isAndroid) {
+		// AOSP: libeliza-llama-shim with Vulkan/OpenCL. The JNI handles
+		// the acceleration path; from the JS side we just request "auto"
+		// and let the shim pick.
+		return [{ backendId: "aosp", accelerator: "auto" }];
+	}
+
+	if (profile.platform === "darwin") {
+		if (profile.arch === "arm64") {
+			// Apple Silicon: mflux (MLX/Metal) first; sd-cpp CPU as
+			// last resort if the venv isn't installed.
+			return [
+				{ backendId: "mflux", accelerator: "metal" },
+				{ backendId: "sd-cpp", accelerator: "cpu" },
+			];
+		}
+		// Intel Mac. mflux requires Apple Silicon; sd-cpp Metal is the
+		// reasonable fast path. Builds for Intel Mac sd-cpp are sparse;
+		// CPU is the safe default.
+		return [
+			{ backendId: "sd-cpp", accelerator: "auto" },
+			{ backendId: "sd-cpp", accelerator: "cpu" },
+		];
+	}
+
+	if (profile.platform === "win32") {
+		if (profile.gpu === "nvidia") {
+			return [
+				{ backendId: "tensorrt", accelerator: "tensorrt" },
+				{ backendId: "sd-cpp", accelerator: "cuda" },
+				{ backendId: "sd-cpp", accelerator: "cpu" },
+			];
+		}
+		if (profile.gpu === "amd" || profile.gpu === "intel") {
+			return [
+				{ backendId: "sd-cpp", accelerator: "vulkan" },
+				{ backendId: "sd-cpp", accelerator: "cpu" },
+			];
+		}
+		return [{ backendId: "sd-cpp", accelerator: "cpu" }];
+	}
+
+	// Linux (and any other Unix the runtime reports). sd-cpp covers
+	// CUDA, Vulkan, and CPU through the same binary; we just pick the
+	// accelerator order.
+	if (profile.platform === "linux") {
+		if (profile.gpu === "nvidia") {
+			const sdCppCudaCapable =
+				profile.sdCpp?.cudaCapable === true ||
+				profile.sdCpp?.accelerators?.includes("cuda") === true;
+			return sdCppCudaCapable
+				? [
+						{ backendId: "sd-cpp", accelerator: "cuda" },
+						{ backendId: "sd-cpp", accelerator: "cpu" },
+					]
+				: [{ backendId: "sd-cpp", accelerator: "cpu" }];
+		}
+		if (profile.gpu === "amd" || profile.gpu === "intel") {
+			return [
+				{ backendId: "sd-cpp", accelerator: "vulkan" },
+				{ backendId: "sd-cpp", accelerator: "cpu" },
+			];
+		}
+		return [{ backendId: "sd-cpp", accelerator: "cpu" }];
+	}
+
+	// Unknown platform — try sd-cpp CPU and fail cleanly if it isn't
+	// installed.
+	return [{ backendId: "sd-cpp", accelerator: "cpu" }];
+}
+
+/**
+ * Tier → default image-gen model id. Mirrors
+ * `eliza/packages/chip/ELIZA_1_BUNDLE_EXTRAS.json` and the WS10 golden test's
+ * `PER_TIER_DEFAULT` map. Used by the WS3 capability registration to
+ * resolve the catalog's bare tier id into a concrete diffusion file.
+ */
+export const TIER_TO_DEFAULT_IMAGE_MODEL: Readonly<
+	Record<
+		string,
+		{
+			modelId: string;
+			file: string;
+			splitDiffusionModel?: boolean;
+			vae?: string;
+			llm?: string;
+		}
+	>
+> = {
+	"eliza-1-2b": {
+		modelId: "imagegen-sd-1_5-q5_0",
+		file: "imagegen/sd-1.5-Q5_0.gguf",
+	},
+	"eliza-1-4b": {
+		modelId: "imagegen-sd-1_5-q5_0",
+		file: "imagegen/sd-1.5-Q5_0.gguf",
+	},
+	"eliza-1-9b": {
+		modelId: "imagegen-z-image-turbo-q4_k_m",
+		file: "imagegen/z-image-turbo-Q4_K_M.gguf",
+		splitDiffusionModel: true,
+		vae: "imagegen/vae/ae.safetensors",
+		llm: "imagegen/text-encoders/Qwen3-4B-Instruct-2507-Q4_K_M.gguf",
+	},
+	"eliza-1-27b": {
+		modelId: "imagegen-z-image-turbo-q4_k_m",
+		file: "imagegen/z-image-turbo-Q4_K_M.gguf",
+		splitDiffusionModel: true,
+		vae: "imagegen/vae/ae.safetensors",
+		llm: "imagegen/text-encoders/Qwen3-4B-Instruct-2507-Q4_K_M.gguf",
+	},
+	"eliza-1-27b-256k": {
+		modelId: "imagegen-z-image-turbo-q4_k_m",
+		file: "imagegen/z-image-turbo-Q4_K_M.gguf",
+		splitDiffusionModel: true,
+		vae: "imagegen/vae/ae.safetensors",
+		llm: "imagegen/text-encoders/Qwen3-4B-Instruct-2507-Q4_K_M.gguf",
+	},
+};
+
+/**
+ * Resolve a tier id (or raw model id) to its default image-gen model.
+ * Returns null when the input doesn't match any known tier — caller
+ * surfaces a clear error.
+ */
+export function resolveDefaultImageGenModel(keyOrTier: string): {
+	modelId: string;
+	file: string;
+	splitDiffusionModel?: boolean;
+	vae?: string;
+	llm?: string;
+} | null {
+	const direct = TIER_TO_DEFAULT_IMAGE_MODEL[keyOrTier];
+	if (direct) return direct;
+	// Allow callers to pass the bare model id straight through; if it
+	// matches a known model file in the per-tier map we echo it back.
+	for (const entry of Object.values(TIER_TO_DEFAULT_IMAGE_MODEL)) {
+		if (entry.modelId === keyOrTier) return entry;
+	}
+	return null;
+}

package/src/services/imagegen/coreml-unavailable.d.ts

@@ -0,0 +1,105 @@
+/**
+ * Core ML image-gen backend contract (WS3) — iOS via Capacitor bridge.
+ *
+ * The contract for the Swift side. The real implementation lives in
+ * `eliza/packages/app-core/platforms/ios/App/App/ImageGenBridge.swift`
+ * (skeleton written under `// MARK: - Contract`). At runtime the iOS
+ * plugin (`@elizaos/plugin-ios-local-inference`) registers a Capacitor
+ * service that exposes:
+ *
+ *   await Capacitor.Plugins.ElizaImageGen.generateImage({
+ *     modelKey: "imagegen-coreml-sd-1_5",
+ *     prompt: "<text>",
+ *     negativePrompt: "<text>",   // optional
+ *     width: 512,
+ *     height: 512,
+ *     steps: 20,
+ *     guidanceScale: 7.5,
+ *     seed: 42                    // -1 = random
+ *   })
+ *     -> { png: "<base64>", seed: number, inferenceTimeMs: number }
+ *
+ * Swift side uses `apple/ml-stable-diffusion` Swift package directly;
+ * the `.mlpackage` directories are dropped into the app's Documents
+ * folder by the bundle installer.
+ *
+ * Until the Swift bridge is present and the Capacitor plugin exposes the
+ * binding, `loadCoreMlImageGenBackend` throws a structured
+ * `ImageGenBackendUnavailableError` so the selector can fall through
+ * — but on iOS there is no fall-through (sd-cpp doesn't run on iOS,
+ * mflux is macOS-only). A `coreml_unavailable` error there means
+ * "this device does not support image-gen yet."
+ *
+ * Publishing pipeline (iOS 17+ / iOS 26+):
+ *
+ *   Build (.mlpackage per tier):
+ *     git clone https://github.com/apple/ml-stable-diffusion && cd ml-stable-diffusion
+ *     python3 -m venv venv && ./venv/bin/pip install -e .
+ *     # SD 1.5 — 512x512, attention=SPLIT_EINSUM_V2 for ANE on iPhone 15+
+ *     ./venv/bin/python -m python_coreml_stable_diffusion.torch2coreml \
+ *       --convert-unet --convert-vae-encoder --convert-vae-decoder \
+ *       --convert-text-encoder --bundle-resources-for-swift-cli \
+ *       --attention-implementation SPLIT_EINSUM_V2 \
+ *       --model-version runwayml/stable-diffusion-v1-5 \
+ *       --output-dir build/sd-1.5-coreml-512
+ *     # SDXL — 1024x1024, same pipeline at --model-version
+ *     # stabilityai/sdxl-turbo for the iPhone 16 Pro / iPad M-class path.
+ *     xcrun coremlc compile build/sd-1.5-coreml-512 build/sd-1.5-coreml-512-compiled
+ *   Sign:
+ *     codesign --force --options runtime --timestamp \
+ *       --sign "Apple Distribution: Eliza Labs Inc." \
+ *       build/sd-1.5-coreml-512/*.mlpackage
+ *   Drop (into the Capacitor app bundle's on-demand resources):
+ *     Tag with `nameOfResourcesODR=ImageGenSD15` so the App Store delivers
+ *     the package only when the runtime requests it via `NSBundleResource
+ *     Request`. Total compressed .mlpackage size is ~600 MB for SD 1.5
+ *     at SPLIT_EINSUM_V2 and ~3 GB for SDXL-turbo.
+ *   Notarize / submit:
+ *     The .mlpackages ship inside the signed IPA; notarization is the
+ *     standard `altool notarytool submit` step for the host IPA.
+ *   iOS 26 (forward-compat note):
+ *     iOS 26 adds the public `MLTensor` API and an updated ANE driver;
+ *     re-build the .mlpackage with `--attention-implementation
+ *     SPLIT_EINSUM_V2` on the iOS 26 SDK to pick up the latest ANE
+ *     scheduling improvements. The Swift bridge code in
+ *     `ImageGenBridge.swift` is the same across iOS 17/18/26.
+ */
+import type { ImageGenBackend, ImageGenLoadArgs } from "./types";
+/**
+ * The Capacitor bridge shape. The iOS plugin registers an instance
+ * under the runtime service name `"capacitor-image-gen"` once the
+ * Swift side ships.
+ */
+export interface CoreMlImageGenBridge {
+    /**
+     * True when the Swift `ImageGenBridge.swift` is present AND a
+     * `.mlpackage` has been resolved for the active tier. False when
+     * either is missing — the backend throws on `generate` in
+     * that case rather than producing a synthetic PNG.
+     */
+    isAvailable(): boolean;
+    generateImage(args: {
+        modelKey: string;
+        prompt: string;
+        negativePrompt?: string;
+        width: number;
+        height: number;
+        steps: number;
+        guidanceScale: number;
+        seed: number;
+        signal?: AbortSignal;
+    }): Promise<{
+        /** Base64-encoded PNG. */
+        png: string;
+        seed: number;
+        inferenceTimeMs: number;
+    }>;
+}
+export interface LoadCoreMlImageGenBackendOptions {
+    loadArgs: ImageGenLoadArgs;
+    modelKey: string;
+    bridge?: CoreMlImageGenBridge;
+    now?: () => number;
+}
+export declare function loadCoreMlImageGenBackend(opts: LoadCoreMlImageGenBackendOptions): Promise<ImageGenBackend>;
+//# sourceMappingURL=coreml-unavailable.d.ts.map

package/src/services/imagegen/coreml-unavailable.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"coreml-unavailable.d.ts","sourceRoot":"","sources":["coreml-unavailable.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAiEG;AAIH,OAAO,KAAK,EACX,eAAe,EACf,gBAAgB,EAGhB,MAAM,SAAS,CAAC;AAEjB;;;;GAIG;AACH,MAAM,WAAW,oBAAoB;IACpC;;;;;OAKG;IACH,WAAW,IAAI,OAAO,CAAC;IACvB,aAAa,CAAC,IAAI,EAAE;QACnB,QAAQ,EAAE,MAAM,CAAC;QACjB,MAAM,EAAE,MAAM,CAAC;QACf,cAAc,CAAC,EAAE,MAAM,CAAC;QACxB,KAAK,EAAE,MAAM,CAAC;QACd,MAAM,EAAE,MAAM,CAAC;QACf,KAAK,EAAE,MAAM,CAAC;QACd,aAAa,EAAE,MAAM,CAAC;QACtB,IAAI,EAAE,MAAM,CAAC;QACb,MAAM,CAAC,EAAE,WAAW,CAAC;KACrB,GAAG,OAAO,CAAC;QACX,0BAA0B;QAC1B,GAAG,EAAE,MAAM,CAAC;QACZ,IAAI,EAAE,MAAM,CAAC;QACb,eAAe,EAAE,MAAM,CAAC;KACxB,CAAC,CAAC;CACH;AAED,MAAM,WAAW,gCAAgC;IAChD,QAAQ,EAAE,gBAAgB,CAAC;IAC3B,QAAQ,EAAE,MAAM,CAAC;IACjB,MAAM,CAAC,EAAE,oBAAoB,CAAC;IAC9B,GAAG,CAAC,EAAE,MAAM,MAAM,CAAC;CACnB;AAED,wBAAsB,yBAAyB,CAC9C,IAAI,EAAE,gCAAgC,GACpC,OAAO,CAAC,eAAe,CAAC,CA4F1B"}

package/src/services/imagegen/coreml-unavailable.ts

@@ -0,0 +1,237 @@
+/**
+ * Core ML image-gen backend contract (WS3) — iOS via Capacitor bridge.
+ *
+ * The contract for the Swift side. The real implementation lives in
+ * `eliza/packages/app-core/platforms/ios/App/App/ImageGenBridge.swift`
+ * (skeleton written under `// MARK: - Contract`). At runtime the iOS
+ * plugin (`@elizaos/plugin-ios-local-inference`) registers a Capacitor
+ * service that exposes:
+ *
+ *   await Capacitor.Plugins.ElizaImageGen.generateImage({
+ *     modelKey: "imagegen-coreml-sd-1_5",
+ *     prompt: "<text>",
+ *     negativePrompt: "<text>",   // optional
+ *     width: 512,
+ *     height: 512,
+ *     steps: 20,
+ *     guidanceScale: 7.5,
+ *     seed: 42                    // -1 = random
+ *   })
+ *     -> { png: "<base64>", seed: number, inferenceTimeMs: number }
+ *
+ * Swift side uses `apple/ml-stable-diffusion` Swift package directly;
+ * the `.mlpackage` directories are dropped into the app's Documents
+ * folder by the bundle installer.
+ *
+ * Until the Swift bridge is present and the Capacitor plugin exposes the
+ * binding, `loadCoreMlImageGenBackend` throws a structured
+ * `ImageGenBackendUnavailableError` so the selector can fall through
+ * — but on iOS there is no fall-through (sd-cpp doesn't run on iOS,
+ * mflux is macOS-only). A `coreml_unavailable` error there means
+ * "this device does not support image-gen yet."
+ *
+ * Publishing pipeline (iOS 17+ / iOS 26+):
+ *
+ *   Build (.mlpackage per tier):
+ *     git clone https://github.com/apple/ml-stable-diffusion && cd ml-stable-diffusion
+ *     python3 -m venv venv && ./venv/bin/pip install -e .
+ *     # SD 1.5 — 512x512, attention=SPLIT_EINSUM_V2 for ANE on iPhone 15+
+ *     ./venv/bin/python -m python_coreml_stable_diffusion.torch2coreml \
+ *       --convert-unet --convert-vae-encoder --convert-vae-decoder \
+ *       --convert-text-encoder --bundle-resources-for-swift-cli \
+ *       --attention-implementation SPLIT_EINSUM_V2 \
+ *       --model-version runwayml/stable-diffusion-v1-5 \
+ *       --output-dir build/sd-1.5-coreml-512
+ *     # SDXL — 1024x1024, same pipeline at --model-version
+ *     # stabilityai/sdxl-turbo for the iPhone 16 Pro / iPad M-class path.
+ *     xcrun coremlc compile build/sd-1.5-coreml-512 build/sd-1.5-coreml-512-compiled
+ *   Sign:
+ *     codesign --force --options runtime --timestamp \
+ *       --sign "Apple Distribution: Eliza Labs Inc." \
+ *       build/sd-1.5-coreml-512/*.mlpackage
+ *   Drop (into the Capacitor app bundle's on-demand resources):
+ *     Tag with `nameOfResourcesODR=ImageGenSD15` so the App Store delivers
+ *     the package only when the runtime requests it via `NSBundleResource
+ *     Request`. Total compressed .mlpackage size is ~600 MB for SD 1.5
+ *     at SPLIT_EINSUM_V2 and ~3 GB for SDXL-turbo.
+ *   Notarize / submit:
+ *     The .mlpackages ship inside the signed IPA; notarization is the
+ *     standard `altool notarytool submit` step for the host IPA.
+ *   iOS 26 (forward-compat note):
+ *     iOS 26 adds the public `MLTensor` API and an updated ANE driver;
+ *     re-build the .mlpackage with `--attention-implementation
+ *     SPLIT_EINSUM_V2` on the iOS 26 SDK to pick up the latest ANE
+ *     scheduling improvements. The Swift bridge code in
+ *     `ImageGenBridge.swift` is the same across iOS 17/18/26.
+ */
+
+import { ImageGenBackendUnavailableError } from "./errors";
+import { PNG_SIGNATURE, resolveSeed } from "./sd-cpp";
+import type {
+	ImageGenBackend,
+	ImageGenLoadArgs,
+	ImageGenRequest,
+	ImageGenResult,
+} from "./types";
+
+/**
+ * The Capacitor bridge shape. The iOS plugin registers an instance
+ * under the runtime service name `"capacitor-image-gen"` once the
+ * Swift side ships.
+ */
+export interface CoreMlImageGenBridge {
+	/**
+	 * True when the Swift `ImageGenBridge.swift` is present AND a
+	 * `.mlpackage` has been resolved for the active tier. False when
+	 * either is missing — the backend throws on `generate` in
+	 * that case rather than producing a synthetic PNG.
+	 */
+	isAvailable(): boolean;
+	generateImage(args: {
+		modelKey: string;
+		prompt: string;
+		negativePrompt?: string;
+		width: number;
+		height: number;
+		steps: number;
+		guidanceScale: number;
+		seed: number;
+		signal?: AbortSignal;
+	}): Promise<{
+		/** Base64-encoded PNG. */
+		png: string;
+		seed: number;
+		inferenceTimeMs: number;
+	}>;
+}
+
+export interface LoadCoreMlImageGenBackendOptions {
+	loadArgs: ImageGenLoadArgs;
+	modelKey: string;
+	bridge?: CoreMlImageGenBridge;
+	now?: () => number;
+}
+
+export async function loadCoreMlImageGenBackend(
+	opts: LoadCoreMlImageGenBackendOptions,
+): Promise<ImageGenBackend> {
+	const { bridge, modelKey } = opts;
+	const now = opts.now ?? Date.now;
+
+	if (!bridge?.isAvailable()) {
+		throw new ImageGenBackendUnavailableError(
+			"coreml",
+			"binding_unavailable",
+			"[imagegen/coreml] Capacitor ElizaImageGen plugin is not available. Wire ImageGenBridge.swift to apple/ml-stable-diffusion and ship a tier-matched .mlpackage. Until then, iOS image-gen is unavailable.",
+		);
+	}
+
+	let disposed = false;
+
+	return {
+		id: "coreml",
+		supports(req: ImageGenRequest) {
+			if (disposed) return false;
+			// Core ML compiles to a fixed input shape per `.mlpackage`.
+			// SD 1.5 stock packages target 512×512; SDXL packages target
+			// 1024×1024. We accept the catalog defaults; explicit asks
+			// outside the package's shape are rejected so the caller can
+			// see a clear error rather than the runtime cropping/upsampling.
+			const w = req.width ?? 512;
+			const h = req.height ?? 512;
+			if (w !== 512 && w !== 768 && w !== 1024) return false;
+			if (h !== 512 && h !== 768 && h !== 1024) return false;
+			return true;
+		},
+		async generate(req: ImageGenRequest): Promise<ImageGenResult> {
+			if (disposed) {
+				throw new ImageGenBackendUnavailableError(
+					"coreml",
+					"binding_unavailable",
+					"[imagegen/coreml] generate called after dispose()",
+				);
+			}
+			if (!req.prompt.trim()) {
+				throw new ImageGenBackendUnavailableError(
+					"coreml",
+					"unsupported_request",
+					"[imagegen/coreml] prompt is empty",
+				);
+			}
+			const seed = resolveSeed(req.seed);
+			const width = req.width ?? 512;
+			const height = req.height ?? 512;
+			const steps = req.steps ?? 20;
+			const guidanceScale = req.guidanceScale ?? 7.5;
+			const startMs = now();
+			const result = await bridge.generateImage({
+				modelKey,
+				prompt: req.prompt,
+				negativePrompt: req.negativePrompt,
+				width,
+				height,
+				steps,
+				guidanceScale,
+				seed,
+				signal: req.signal,
+			});
+			const elapsed =
+				typeof result.inferenceTimeMs === "number" && result.inferenceTimeMs > 0
+					? result.inferenceTimeMs
+					: Math.max(1, now() - startMs);
+			const bytes = decodeBase64Png(result.png);
+			// Core ML batch path doesn't surface per-step progress; emit a
+			// single completion event when the caller asked for one.
+			if (req.onProgressChunk) {
+				req.onProgressChunk({ step: steps, total: steps });
+			}
+			return {
+				image: bytes,
+				mime: "image/png",
+				seed: typeof result.seed === "number" ? result.seed : seed,
+				metadata: {
+					model: modelKey,
+					prompt: req.prompt,
+					steps,
+					guidanceScale,
+					inferenceTimeMs: elapsed,
+				},
+			};
+		},
+		async dispose() {
+			if (disposed) return;
+			disposed = true;
+			// Capacitor plugin owns the Swift-side handle's lifetime; nothing
+			// to free from JS. The bundle installer is responsible for the
+			// `.mlpackage` cleanup if the user deletes a tier.
+		},
+	};
+}
+
+function decodeBase64Png(base64: string): Uint8Array {
+	if (typeof base64 !== "string" || !base64) {
+		throw new ImageGenBackendUnavailableError(
+			"coreml",
+			"unsupported_request",
+			"[imagegen/coreml] Capacitor bridge returned empty base64 payload",
+		);
+	}
+	const buf = Buffer.from(base64, "base64");
+	if (buf.length < PNG_SIGNATURE.length) {
+		throw new ImageGenBackendUnavailableError(
+			"coreml",
+			"unsupported_request",
+			`[imagegen/coreml] base64 payload too short (${buf.length} bytes); not a PNG`,
+		);
+	}
+	for (let i = 0; i < PNG_SIGNATURE.length; i += 1) {
+		if (buf[i] !== PNG_SIGNATURE[i]) {
+			throw new ImageGenBackendUnavailableError(
+				"coreml",
+				"unsupported_request",
+				"[imagegen/coreml] base64 payload missing PNG signature",
+			);
+		}
+	}
+	return new Uint8Array(buf);
+}

package/src/services/imagegen/errors.d.ts

@@ -0,0 +1,16 @@
+/**
+ * Structured error every WS3 backend throws when it can't serve a
+ * request. Mirrors `VisionBackendUnavailableError` so the arbiter / WS3
+ * provider handler can surface a single typed failure mode upward.
+ */
+export declare class ImageGenBackendUnavailableError extends Error {
+    readonly backendId: string;
+    readonly reason: "binary_missing" | "binary_version_mismatch" | "cuda_binary_missing" | "vulkan_binary_missing" | "metal_binary_missing" | "cuda_unavailable" | "model_missing" | "binding_unavailable" | "unsupported_runtime" | "unsupported_request" | "subprocess_failed";
+    readonly code = "IMAGE_GEN_BACKEND_UNAVAILABLE";
+    constructor(backendId: string, reason: "binary_missing" | "binary_version_mismatch" | "cuda_binary_missing" | "vulkan_binary_missing" | "metal_binary_missing" | "cuda_unavailable" | "model_missing" | "binding_unavailable" | "unsupported_runtime" | "unsupported_request" | "subprocess_failed", message: string, options?: {
+        cause?: unknown;
+    });
+}
+/** Tells callers whether a thrown error came from a backend availability check. */
+export declare function isImageGenUnavailable(err: unknown): err is ImageGenBackendUnavailableError;
+//# sourceMappingURL=errors.d.ts.map

package/src/services/imagegen/errors.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"errors.d.ts","sourceRoot":"","sources":["errors.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AACH,qBAAa,+BAAgC,SAAQ,KAAK;IAGxD,QAAQ,CAAC,SAAS,EAAE,MAAM;IAC1B,QAAQ,CAAC,MAAM,EACZ,gBAAgB,GAChB,yBAAyB,GACzB,qBAAqB,GACrB,uBAAuB,GACvB,sBAAsB,GACtB,kBAAkB,GAClB,eAAe,GACf,qBAAqB,GACrB,qBAAqB,GACrB,qBAAqB,GACrB,mBAAmB;IAdvB,QAAQ,CAAC,IAAI,mCAAmC;gBAEtC,SAAS,EAAE,MAAM,EACjB,MAAM,EACZ,gBAAgB,GAChB,yBAAyB,GACzB,qBAAqB,GACrB,uBAAuB,GACvB,sBAAsB,GACtB,kBAAkB,GAClB,eAAe,GACf,qBAAqB,GACrB,qBAAqB,GACrB,qBAAqB,GACrB,mBAAmB,EACtB,OAAO,EAAE,MAAM,EACf,OAAO,CAAC,EAAE;QAAE,KAAK,CAAC,EAAE,OAAO,CAAA;KAAE;CAK9B;AAED,mFAAmF;AACnF,wBAAgB,qBAAqB,CACpC,GAAG,EAAE,OAAO,GACV,GAAG,IAAI,+BAA+B,CAOxC"}

package/src/services/imagegen/errors.ts

@@ -0,0 +1,40 @@
+/**
+ * Structured error every WS3 backend throws when it can't serve a
+ * request. Mirrors `VisionBackendUnavailableError` so the arbiter / WS3
+ * provider handler can surface a single typed failure mode upward.
+ */
+export class ImageGenBackendUnavailableError extends Error {
+	readonly code = "IMAGE_GEN_BACKEND_UNAVAILABLE";
+	constructor(
+		readonly backendId: string,
+		readonly reason:
+			| "binary_missing"
+			| "binary_version_mismatch"
+			| "cuda_binary_missing"
+			| "vulkan_binary_missing"
+			| "metal_binary_missing"
+			| "cuda_unavailable"
+			| "model_missing"
+			| "binding_unavailable"
+			| "unsupported_runtime"
+			| "unsupported_request"
+			| "subprocess_failed",
+		message: string,
+		options?: { cause?: unknown },
+	) {
+		super(message, options);
+		this.name = "ImageGenBackendUnavailableError";
+	}
+}
+
+/** Tells callers whether a thrown error came from a backend availability check. */
+export function isImageGenUnavailable(
+	err: unknown,
+): err is ImageGenBackendUnavailableError {
+	return (
+		err instanceof ImageGenBackendUnavailableError ||
+		(typeof err === "object" &&
+			err !== null &&
+			(err as { code?: unknown }).code === "IMAGE_GEN_BACKEND_UNAVAILABLE")
+	);
+}