@elizaos/plugin-local-inference 2.0.0-beta.1 → 2.0.3-beta.2

package/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Shaw Walters and elizaOS Contributors
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

package/README.md

@@ -0,0 +1,83 @@
+# @elizaos/plugin-local-inference
+
+Eliza-1 local inference provider for elizaOS. Serves text generation, embeddings, text-to-speech, ASR, image generation, and image description entirely on-device — no network required after model download.
+
+## What it does
+
+- **Text generation** (`TEXT_SMALL`, `TEXT_LARGE`) via an in-process llama.cpp FFI binding (`node-llama-cpp`), with optional MTP speculative decoding, GPU offload, and KV-cache persistence across turns.
+- **Text embeddings** (`TEXT_EMBEDDING`) via a dedicated embedding GGUF loaded separately from the chat model.
+- **Text-to-speech** (`TEXT_TO_SPEECH`) via the Kokoro TTS engine (ONNX-based, runs locally).
+- **Automatic speech recognition** (`TRANSCRIPTION`) via whisper.cpp (GGML-based).
+- **Image generation** (`IMAGE`) via sd.cpp, CoreML (Apple Silicon), mflux, TensorRT, or AOSP backends; selected by hardware and catalog entry.
+- **Image description / vision** (`IMAGE_DESCRIPTION`) via the Qwen3-VL multimodal projector attached to the active text model.
+- **Model catalog, download management, and hardware-fit recommendation** exposed as HTTP routes for the elizaOS dashboard.
+- **Voice pipeline**: barge-in, VAD, speaker imprint, phrase streaming, voice profiles, and first-run onboarding.
+
+## Capabilities added to an Eliza agent
+
+| Capability | How it appears |
+|---|---|
+| `GENERATE_MEDIA` action | Agent responds to "draw me a ...", "say ...", "speak ...", etc. by calling the local image or TTS backend. |
+| `TEXT_SMALL` / `TEXT_LARGE` handler | Agent uses the active Eliza-1 text model for all reasoning and response generation. |
+| `TEXT_EMBEDDING` handler | Agent embeds memories using the local embedding GGUF; avoids cloud API calls for RAG. |
+| `TEXT_TO_SPEECH` handler | Agent converts text to audio using Kokoro (or another registered TTS backend). |
+| `TRANSCRIPTION` handler | Agent transcribes audio using whisper.cpp. |
+| `IMAGE` handler | Agent generates images using the active local diffusion backend. |
+| `IMAGE_DESCRIPTION` handler | Agent describes images using the active multimodal model. |
+
+## Requirements
+
+- Node.js 20+ or Bun runtime.
+- `node-llama-cpp` (optional peer dependency) — required for the desktop/server llama.cpp path. Install it in the elizaOS agent package; it will not be automatically installed.
+- Native binaries for optional capabilities: `sd.cpp` for image-gen on Linux/Windows, `mflux` for Apple Silicon image-gen, `whisper.cpp` built with the shared library flag for ASR.
+- An Eliza-1 GGUF bundle downloaded via the model catalog (dashboard → Models, or `POST /api/local-inference/downloads`).
+
+## Enabling the plugin
+
+Add `@elizaos/plugin-local-inference` to the `plugins` array in your elizaOS agent character or bootstrap configuration:
+
+```ts
+import localInferencePlugin from "@elizaos/plugin-local-inference";
+
+const agent = new AgentRuntime({
+  plugins: [localInferencePlugin],
+  // ...
+});
+```
+
+The plugin registers its model handlers at priority `−100`. The routing-policy layer (not raw priority) controls whether a given request is served locally or by a cloud provider. Users configure this in the dashboard under Settings → Model Routing.
+
+## Configuration
+
+Key environment variables (all optional unless noted):
+
+| Variable | Purpose |
+|---|---|
+| `MODELS_DIR` | Override the GGUF model directory (default: `~/.eliza/models`) |
+| `LOCAL_SMALL_MODEL` | Small model filename (mobile/Capacitor adapter) |
+| `LOCAL_LARGE_MODEL` | Large model filename (mobile/Capacitor adapter) |
+| `ELIZA_DEFER_LOCAL_EMBEDDING_WARMUP` | Set truthy to defer startup GGUF embedding prefetch until the dev/runtime server is ready |
+| `ELIZA_SKIP_LOCAL_EMBEDDING_WARMUP` | Set truthy to skip GGUF embedding prefetch entirely while leaving local embedding settings intact |
+| `ELIZA_ENABLE_STARTUP_LOCAL_EMBEDDING_WARMUP` | Desktop startup opt-in that starts GGUF embedding warmup during runtime bootstrap when no skip/defer override is set |
+| `ELIZA_DISABLE_LOCAL_EMBEDDINGS` | Set `1` to disable local `TEXT_EMBEDDING` registration entirely |
+| `ELIZA_IMAGEGEN_ACCELERATOR` | Force image-gen backend: `coreml`, `mflux`, `sd-cpp`, `tensorrt` |
+| `ELIZA_DEVICE_BRIDGE_ENABLED` | Enable iOS/AOSP physical device bridge |
+| `HF_TOKEN` | HuggingFace token for gated model downloads |
+| `SD_CPP_BIN` | Absolute path to sd.cpp binary |
+| `MFLUX_BIN` | Absolute path to mflux binary |
+| `ELIZA_KOKORO_DEFAULT_VOICE_ID` | Default Kokoro TTS voice |
+| `ELIZA_WHISPER_USE_GPU` | Enable GPU for Whisper ASR |
+
+## Architecture notes
+
+The plugin exposes these subpath exports (see `package.json` `exports`):
+
+- `@elizaos/plugin-local-inference` — plugin object, `GENERATE_MEDIA` action, `handleLocalInferenceRoutes`, embedding presets.
+- `@elizaos/plugin-local-inference/runtime` — boot-time handler registration (`ensureLocalInferenceHandler`), embedding warm-up policy, mobile gate.
+- `@elizaos/plugin-local-inference/runtime/embedding-presets` — `detectEmbeddingPreset`, `EMBEDDING_PRESETS`.
+- `@elizaos/plugin-local-inference/routes` — HTTP route handlers (`handleLocalInferenceCompatRoutes`, TTS/ASR, voice) mounted by app-core.
+- `@elizaos/plugin-local-inference/services` — full service surfaces (engine, arbiter, catalog, recommendation, voice) for deep integrations.
+
+The **MemoryArbiter** (`services/memory-arbiter.ts`) is the single coordination point for all model handles across modalities. On memory-constrained devices (mobile, low-RAM desktop), the arbiter evicts models by priority before loading a new one. Cross-plugin consumers (vision, image-gen) register capabilities via `arbiter.registerCapability(...)` rather than loading models independently.
+
+For agent-facing documentation see `CLAUDE.md` / `AGENTS.md` in this directory.

package/package.json

@@ -1,45 +1,112 @@
 {
   "name": "@elizaos/plugin-local-inference",
-  "version": "2.0.0-beta.1",
-  "description": "Local inference HTTP routes for the Eliza agent: model catalog, downloads, hardware detection, and chat-command surface.",
+  "version": "2.0.3-beta.2",
+  "description": "Eliza-1 local provider plus HTTP routes for local model catalog, downloads, hardware detection, and chat commands.",
   "type": "module",
   "main": "dist/index.js",
-  "types": "dist/index.d.ts",
+  "types": "src/index.ts",
   "exports": {
     ".": {
-      "types": "./dist/index.d.ts",
+      "types": "./src/index.ts",
+      "eliza-source": {
+        "types": "./src/index.ts",
+        "import": "./src/index.ts",
+        "default": "./src/index.ts"
+      },
       "import": "./dist/index.js",
       "default": "./dist/index.js"
+    },
+    "./runtime": {
+      "types": "./src/runtime/index.ts",
+      "eliza-source": {
+        "types": "./src/runtime/index.ts",
+        "import": "./src/runtime/index.ts",
+        "default": "./src/runtime/index.ts"
+      },
+      "import": "./dist/runtime/index.js",
+      "default": "./dist/runtime/index.js"
+    },
+    "./runtime/embedding-presets": {
+      "types": "./src/runtime/embedding-presets.ts",
+      "eliza-source": {
+        "types": "./src/runtime/embedding-presets.ts",
+        "import": "./src/runtime/embedding-presets.ts",
+        "default": "./src/runtime/embedding-presets.ts"
+      },
+      "import": "./dist/runtime/index.js",
+      "default": "./dist/runtime/index.js"
+    },
+    "./routes": {
+      "types": "./src/routes/index.ts",
+      "eliza-source": {
+        "types": "./src/routes/index.ts",
+        "import": "./src/routes/index.ts",
+        "default": "./src/routes/index.ts"
+      },
+      "import": "./dist/routes/index.js",
+      "default": "./dist/routes/index.js"
+    },
+    "./services": {
+      "types": "./src/services/index.ts",
+      "eliza-source": {
+        "types": "./src/services/index.ts",
+        "import": "./src/services/index.ts",
+        "default": "./src/services/index.ts"
+      },
+      "import": "./dist/services/index.js",
+      "default": "./dist/services/index.js"
+    },
+    "./*.css": "./dist/*.css",
+    "./*": {
+      "types": "./src/*.ts",
+      "eliza-source": {
+        "types": "./src/*.ts",
+        "import": "./src/*.ts",
+        "default": "./src/*.ts"
+      },
+      "import": "./dist/*.js",
+      "default": "./dist/*.js"
     }
   },
   "files": [
-    "dist"
+    "dist",
+    "src"
   ],
   "scripts": {
-    "build": "tsup src/index.ts --format esm --dts --clean",
-    "dev": "tsup src/index.ts --format esm --dts --watch",
+    "build": "bun run build.ts",
+    "dev": "bun run build.ts",
+    "test": "NODE_OPTIONS='--experimental-sqlite' vitest run",
+    "test:asr:real": "NODE_OPTIONS='--experimental-sqlite' bun scripts/asr-real-smoke.ts",
+    "probe:sd-cpp": "node scripts/probe-sd-cpp.mjs --human",
     "clean": "rm -rf dist .turbo node_modules",
     "lint": "bunx @biomejs/biome check --write --unsafe .",
     "lint:check": "bunx @biomejs/biome check .",
     "format": "bunx @biomejs/biome format --write .",
     "format:check": "bunx @biomejs/biome format .",
-    "typecheck": "tsc --noEmit"
+    "typecheck": "tsgo --noEmit"
   },
   "dependencies": {
-    "@elizaos/core": "2.0.0-beta.1",
-    "@elizaos/plugin-capacitor-bridge": "2.0.0-beta.1"
+    "@elizaos/core": "2.0.3-beta.2",
+    "@elizaos/plugin-aosp-local-inference": "2.0.3-beta.2",
+    "@elizaos/plugin-capacitor-bridge": "2.0.3-beta.2",
+    "@elizaos/plugin-computeruse": "2.0.3-beta.2",
+    "@elizaos/shared": "2.0.3-beta.2",
+    "phonemizer": "^1.2.1",
+    "ws": "^8.20.1"
   },
   "peerDependencies": {
-    "@elizaos/core": "2.0.0-beta.1",
-    "@elizaos/plugin-capacitor-bridge": "2.0.0-beta.1"
+    "@elizaos/core": "2.0.3-beta.2",
+    "@elizaos/shared": "2.0.3-beta.2"
   },
   "devDependencies": {
     "@biomejs/biome": "^2.4.14",
     "@types/node": "24.12.2",
-    "tsup": "8.5.1",
-    "typescript": "^6.0.3"
+    "@types/ws": "^8.18.1",
+    "typescript": "^6.0.3",
+    "vitest": "^4.1.4"
   },
   "publishConfig": {
     "access": "public"
-  }
+  },
+  "gitHead": "82fe0f44215954c2417328203f5bd6510985c1fc"
 }

package/src/actions/generate-media.d.ts

@@ -0,0 +1,59 @@
+/**
+ * @module plugin-local-inference/actions/generate-media
+ *
+ * Unified `GENERATE_MEDIA` agent action.
+ *
+ * Routes a single user-facing intent to the appropriate `runtime.useModel(...)`
+ * call:
+ *
+ *   - image  → `ModelType.IMAGE`            (WS3 arbiter, returns PNG bytes)
+ *   - audio  → `ModelType.TEXT_TO_SPEECH`   (Eliza-1 / local TTS, returns PCM/WAV/MP3)
+ *   - video  → unavailable in the local backend; refuses with a clean message
+ *
+ * Intent classification is keyword-first (cheap, deterministic) with an
+ * optional `ModelType.TEXT_SMALL` JSON fallback for ambiguous prompts. The
+ * prompt body is extracted by stripping any leading imperative ("draw me a ",
+ * "say ", "speak in spanish: ") so that downstream models see a clean
+ * description.
+ *
+ * Trajectory hook: result `data.computerUseAction` is set to a stable
+ * marker (`GENERATE_MEDIA_IMAGE` / `GENERATE_MEDIA_AUDIO`) so the trajectory
+ * logger picks the action up exactly the way it picks up
+ * `plugin-computeruse` actions.
+ */
+import { type Action, type ActionResult, type HandlerCallback, type IAgentRuntime, type Memory } from "@elizaos/core";
+export type MediaKind = "image" | "audio" | "video";
+interface IntentDetection {
+    kind: MediaKind;
+    prompt: string;
+    source: "keyword" | "classifier";
+}
+type ClassifierFn = (prompt: string) => Promise<MediaKind | null>;
+export interface IntentDetectorOptions {
+    /**
+     * Optional override for the text-classifier fallback. Tests inject a
+     * deterministic classifier; in production this is bound to
+     * `runtime.useModel(ModelType.TEXT_SMALL, ...)`.
+     */
+    classifier?: ClassifierFn;
+}
+/**
+ * Detect the media intent from a user message.
+ *
+ * Algorithm:
+ *   1. Try keyword rules first (cheap, deterministic).
+ *   2. If nothing matched and a classifier is provided, ask it for a JSON
+ *      label. Trust the classifier only when it returns one of our three
+ *      kinds; otherwise return `null` so the caller can decline.
+ */
+export declare function detectMediaIntent(text: string, options?: IntentDetectorOptions): Promise<IntentDetection | null>;
+interface BuildHandlerOptions {
+    /** Test seam: override the intent detector. */
+    detectIntent: typeof detectMediaIntent;
+    /** Test seam: override the classifier resolver. */
+    classifierFactory: (runtime: IAgentRuntime) => ClassifierFn;
+}
+export declare function buildGenerateMediaHandler(opts?: Partial<BuildHandlerOptions>): (runtime: IAgentRuntime, message: Memory, _state?: unknown, _options?: unknown, callback?: HandlerCallback) => Promise<ActionResult>;
+export declare const generateMediaAction: Action;
+export default generateMediaAction;
+//# sourceMappingURL=generate-media.d.ts.map

package/src/actions/generate-media.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"generate-media.d.ts","sourceRoot":"","sources":["generate-media.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;GAsBG;AAEH,OAAO,EACN,KAAK,MAAM,EACX,KAAK,YAAY,EAGjB,KAAK,eAAe,EACpB,KAAK,aAAa,EAIlB,KAAK,MAAM,EAEX,MAAM,eAAe,CAAC;AAMvB,MAAM,MAAM,SAAS,GAAG,OAAO,GAAG,OAAO,GAAG,OAAO,CAAC;AAEpD,UAAU,eAAe;IACxB,IAAI,EAAE,SAAS,CAAC;IAChB,MAAM,EAAE,MAAM,CAAC;IACf,MAAM,EAAE,SAAS,GAAG,YAAY,CAAC;CACjC;AAoED,KAAK,YAAY,GAAG,CAAC,MAAM,EAAE,MAAM,KAAK,OAAO,CAAC,SAAS,GAAG,IAAI,CAAC,CAAC;AAElE,MAAM,WAAW,qBAAqB;IACrC;;;;OAIG;IACH,UAAU,CAAC,EAAE,YAAY,CAAC;CAC1B;AAsBD;;;;;;;;GAQG;AACH,wBAAsB,iBAAiB,CACtC,IAAI,EAAE,MAAM,EACZ,OAAO,GAAE,qBAA0B,GACjC,OAAO,CAAC,eAAe,GAAG,IAAI,CAAC,CASjC;AAoND,UAAU,mBAAmB;IAC5B,+CAA+C;IAC/C,YAAY,EAAE,OAAO,iBAAiB,CAAC;IACvC,mDAAmD;IACnD,iBAAiB,EAAE,CAAC,OAAO,EAAE,aAAa,KAAK,YAAY,CAAC;CAC5D;AAED,wBAAgB,yBAAyB,CACxC,IAAI,GAAE,OAAO,CAAC,mBAAmB,CAAM,IAKtC,SAAS,aAAa,EACtB,SAAS,MAAM,EACf,SAAS,OAAO,EAChB,WAAW,OAAO,EAClB,WAAW,eAAe,KACxB,OAAO,CAAC,YAAY,CAAC,CAsJxB;AAoBD,eAAO,MAAM,mBAAmB,EAAE,MA4EjC,CAAC;AAEF,eAAe,mBAAmB,CAAC"}