@elizaos/plugin-local-inference 2.0.0-beta.1 → 2.0.3-beta.3

package/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Shaw Walters and elizaOS Contributors
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

package/README.md

@@ -0,0 +1,157 @@
+# @elizaos/plugin-local-inference
+
+Eliza-1 local inference provider for elizaOS. Serves text generation, embeddings, text-to-speech, ASR, image generation, and image description entirely on-device — no network required after model download.
+
+## What it does
+
+- **Text generation** (`TEXT_SMALL`, `TEXT_LARGE`) via an in-process llama.cpp FFI binding. There are two text runtime classes, picked per model by the dispatcher (`services/backend.ts`):
+  - **fused Eliza-1 bundles** (`runtimeClass: "fused-eliza1"`) run through the fused `libelizainference` (`desktop-fused-ffi-backend-runtime.ts`) — the full local pipeline: manifest-gated MTP speculative decoding, fork kernels where applicable, native tokenization over the active Eliza-1 bundle tokenizer, and fused voice/vision/ASR. This is the default/recommended path.
+  - **generic single-file GGUF** (`runtimeClass: "generic-gguf"`) — a model you downloaded/scanned (Hugging Face / ModelScope / LM Studio / Ollama) loaded from an explicit `modelPath` with stock f16 KV and *reduced optimizations* (no MTP, no fork kernels, no fused voice/vision). The explicit-`modelPath` binding ships on mobile (`llama-cpp-capacitor`); on desktop it is not yet built into the shipping `libelizainference`, so an assigned generic model is rejected at the assignment boundary with a typed reason rather than failing silently at load.
+- `node-llama-cpp` has been retired; there is no node-llama-cpp fallback.
+- **Text embeddings** (`TEXT_EMBEDDING`) via a dedicated embedding GGUF loaded separately from the chat model.
+- **Text-to-speech** (`TEXT_TO_SPEECH`) via the local Kokoro/OmniVoice runtime selected by tier and policy.
+- **Automatic speech recognition** (`TRANSCRIPTION`) via the eligible bundled local ASR head in fused `libelizainference`; there is no whisper.cpp fallback.
+- **Image generation** (`IMAGE`) via sd.cpp, CoreML (Apple Silicon), mflux, TensorRT, or AOSP backends; selected by hardware and catalog entry.
+- **Image description / vision** (`IMAGE_DESCRIPTION`) via the tier-matched Eliza-1 multimodal projector attached to the active text model.
+- **Model catalog, download management, and hardware-fit recommendation** exposed as HTTP routes for the elizaOS dashboard.
+- **Voice pipeline**: barge-in, VAD, speaker imprint, phrase streaming, voice profiles, and first-run onboarding.
+
+## Capabilities added to an Eliza agent
+
+| Capability | How it appears |
+|---|---|
+| `GENERATE_MEDIA` action | Agent responds to "draw me a ...", "say ...", "speak ...", etc. by calling the local image or TTS backend. |
+| `TEXT_SMALL` / `TEXT_LARGE` handler | Agent uses the active Eliza-1 text model for all reasoning and response generation. |
+| `TEXT_EMBEDDING` handler | Agent embeds memories using the local embedding GGUF; avoids cloud API calls for RAG. |
+| `TEXT_TO_SPEECH` handler | Agent converts text to audio using the selected local TTS backend. |
+| `TRANSCRIPTION` handler | Agent transcribes audio using the eligible bundled local ASR runtime. |
+| `IMAGE` handler | Agent generates images using the active local diffusion backend. |
+| `IMAGE_DESCRIPTION` handler | Agent describes images using the active multimodal model. |
+
+## Requirements
+
+- Node.js 20+ or Bun runtime.
+- The fused `libelizainference` native library for the desktop text/voice/vision path (built from `tools/omnivoice`; resolved via `ELIZA_INFERENCE_LIBRARY` / `ELIZA_INFERENCE_LIB_DIR` or the bundle's `lib/` dir). Generic single-file GGUF additionally needs the explicit-`modelPath` binding (`llama-cpp-capacitor` on mobile).
+- Native binaries for optional capabilities: `sd.cpp` for image-gen on Linux/Windows and `mflux` for Apple Silicon image-gen.
+- An Eliza-1 GGUF bundle downloaded via the model catalog (dashboard → Models, or `POST /api/local-inference/downloads`).
+
+## Enabling the plugin
+
+Add `@elizaos/plugin-local-inference` to the `plugins` array in your elizaOS agent character or bootstrap configuration:
+
+```ts
+import localInferencePlugin from "@elizaos/plugin-local-inference";
+
+const agent = new AgentRuntime({
+  plugins: [localInferencePlugin],
+  // ...
+});
+```
+
+The plugin registers its model handlers at priority `−100`. The routing-policy layer (not raw priority) controls whether a given request is served locally or by a cloud provider. Users configure this in the dashboard under Settings → Model Routing.
+
+## Configuration
+
+Key environment variables (all optional unless noted):
+
+| Variable | Purpose |
+|---|---|
+| `MODELS_DIR` | Override the GGUF model directory (default: `~/.eliza/models`) |
+| `LOCAL_SMALL_MODEL` | Small model filename (mobile/Capacitor adapter) |
+| `LOCAL_LARGE_MODEL` | Large model filename (mobile/Capacitor adapter) |
+| `ELIZA_DEFER_LOCAL_EMBEDDING_WARMUP` | Set truthy to defer startup GGUF embedding prefetch until the dev/runtime server is ready |
+| `ELIZA_SKIP_LOCAL_EMBEDDING_WARMUP` | Set truthy to skip GGUF embedding prefetch entirely while leaving local embedding settings intact |
+| `ELIZA_ENABLE_STARTUP_LOCAL_EMBEDDING_WARMUP` | Desktop startup opt-in that starts GGUF embedding warmup during runtime bootstrap when no skip/defer override is set |
+| `ELIZA_DISABLE_LOCAL_EMBEDDINGS` | Set `1` to disable local `TEXT_EMBEDDING` registration entirely |
+| `ELIZA_IMAGEGEN_ACCELERATOR` | Force image-gen backend: `coreml`, `mflux`, `sd-cpp`, `tensorrt` |
+| `ELIZA_DEVICE_BRIDGE_ENABLED` | Enable iOS/AOSP physical device bridge |
+| `SD_CPP_BIN` | Absolute path to sd.cpp binary |
+| `MFLUX_BIN` | Absolute path to mflux binary |
+| `ELIZA_KOKORO_DEFAULT_VOICE_ID` | Default Kokoro TTS voice |
+
+## Architecture notes
+
+The plugin exposes these subpath exports (see `package.json` `exports`):
+
+- `@elizaos/plugin-local-inference` — plugin object, `GENERATE_MEDIA` action, `handleLocalInferenceRoutes`, embedding presets.
+- `@elizaos/plugin-local-inference/runtime` — boot-time handler registration (`ensureLocalInferenceHandler`), embedding warm-up policy, mobile gate.
+- `@elizaos/plugin-local-inference/runtime/embedding-presets` — `detectEmbeddingPreset`, `EMBEDDING_PRESETS`.
+- `@elizaos/plugin-local-inference/routes` — HTTP route handlers (`handleLocalInferenceCompatRoutes`, TTS/ASR, voice) mounted by app-core.
+- `@elizaos/plugin-local-inference/services` — full service surfaces (engine, arbiter, catalog, recommendation, voice) for deep integrations.
+
+The **MemoryArbiter** (`services/memory-arbiter.ts`) is the single coordination point for all model handles across modalities. On memory-constrained devices (mobile, low-RAM desktop), the arbiter evicts models by priority before loading a new one. Cross-plugin consumers (vision, image-gen) register capabilities via `arbiter.registerCapability(...)` rather than loading models independently.
+
+## Voice Workbench (#8785)
+
+This package hosts the **Voice Workbench** — one scenario/corpus/benchmark
+harness that unifies what used to be five disjoint voice-test families behind a
+single entrypoint, scenario format, and metric module.
+
+### Entrypoint
+
+```bash
+bun run --cwd plugins/plugin-local-inference voice:workbench \
+  [--mock|--logic|--real] [--out <dir>] [--baseline <report.json>]
+```
+
+- `--mock` (default) — ground-truth mock services; the CI plumbing lane (no
+  model, no network). Runs + passes.
+- `--logic` — the shipped decision logic (EOT / respond / echo / bystander /
+  wake-word gate + name extraction) over the corpus, without acoustic models.
+  CI-runnable; catches a regression in the decision layer.
+- `--real` — real local acoustic backend; any scenario without a provisioned
+  real service reports `skipped`, never a false `pass` (the honesty contract).
+- `--baseline <path>` — compare metrics against a golden report; exit 1 on any
+  metric regression past tolerance (the regression gate).
+
+It writes one machine-readable `report.json` plus a Markdown rendering — WER,
+EOT latency + false-trigger rate, diarization DER, respond accuracy,
+entity-match, first-audio/TTFT latency — via `workbench-entrypoint.ts` +
+`voice-workbench-report.ts`.
+
+### Pieces
+
+- **Scenario schema** — `services/voice/voice-scenario.ts`: a declarative
+  `VoiceScenario` (participants + ordered turns + assertions) that both the
+  headless runner and the headful player execute and the benchmark layer scores,
+  covering every class: multi-voice, pauses, respond/no-respond, multi-speaker,
+  diarization, entity-extraction, voice→entity match, EOT, transcription-mode,
+  multi-agent room, and the long-form monologue.
+- **Corpus generator** — `corpus:generate` (`scripts/generate-voice-corpus.ts`)
+  TTS-synthesizes each turn, splices pauses, and mixes multi-speaker streams into
+  a versioned labeled corpus (PCM + ground-truth JSON). Robustness DSP
+  (noise / reverb / gain / low-quality-line) lives in `corpus-augment.ts`.
+- **Single scoring module** — `services/voice/e2e-harness.ts` is the one shared
+  source of truth for voice scoring (`scoreTtsAsrRoundTrip`, `scoreEotDecision`,
+  `scoreDiarization`, `scoreRespondDecision`, `scoreEntityExtraction`,
+  `scoreVoiceEntityMatch`, `scoreEchoRejection`, `scoreOwnerSecurity`, …); WER
+  itself is `@elizaos/shared/voice-wer`. The duplicate `wordErrorRate` that used
+  to live in the headful `voice-selftest-harness.ts` is gone — it imports the
+  shared one.
+- **Headless runner** — `workbench-headless-runner.ts` drives each scenario class
+  through the real services and scores it; an absent corpus/backend yields
+  `skipped`, never `pass`.
+- **scenario-runner audio turn** — `packages/scenario-runner/src/voice-turn.ts`
+  adds a `voice` turn kind so voice scenarios are first-class `.scenario.ts`
+  files over a real `AgentRuntime`.
+- **Headful specs** — `packages/app/test/ui-smoke/voice-workbench-*.spec.ts` (one
+  per scenario class) drive the real frontend client pipeline with a per-turn
+  DOM-mirrored verdict.
+- **Real-model lanes** (dev hosts with a built engine + staged models):
+  `roundtrip:real` (cloud-TTS → local ASR → fast cloud LLM → cloud-TTS),
+  `robustness:real` (WER under noise / reverb / far-field / telephone),
+  `voicestack:real` (speaker recognition / diarization / VAD / local TTS),
+  `agentvoice:real` (agent-self-voice rejection + overlapping speakers).
+- **CI** — `.github/workflows/voice-workbench.yml` runs the `--logic` lane plus
+  the regression baseline on every change to the voice surface.
+
+### Legacy harnesses it absorbs
+
+The workbench is the single home for what was previously fragmented across a pure
+scoring lib (`e2e-harness.ts`, now promoted to the source of truth), a two-agent
+`voice:duet` harness, native `packages/benchmarks/voice/*.mjs` scenarios, Python
+benches, and the single-turn headful self-test (`voice-selftest`). Those remain
+runnable, but **new** voice coverage should be authored as a `VoiceScenario` +
+corpus and scored through `e2e-harness.ts`, not as a new bespoke harness.
+
+For agent-facing documentation see `CLAUDE.md` / `AGENTS.md` in this directory.

package/dist/actions/generate-media.d.ts

@@ -0,0 +1,59 @@
+/**
+ * @module plugin-local-inference/actions/generate-media
+ *
+ * Unified `GENERATE_MEDIA` agent action.
+ *
+ * Routes a single user-facing intent to the appropriate `runtime.useModel(...)`
+ * call:
+ *
+ *   - image  → `ModelType.IMAGE`            (WS3 arbiter, returns PNG bytes)
+ *   - audio  → `ModelType.TEXT_TO_SPEECH`   (Eliza-1 / local TTS, returns PCM/WAV/MP3)
+ *   - video  → unavailable in the local backend; refuses with a clean message
+ *
+ * Intent classification is keyword-first (cheap, deterministic) with an
+ * optional `ModelType.TEXT_SMALL` JSON fallback for ambiguous prompts. The
+ * prompt body is extracted by stripping any leading imperative ("draw me a ",
+ * "say ", "speak in spanish: ") so that downstream models see a clean
+ * description.
+ *
+ * Trajectory hook: result `data.computerUseAction` is set to a stable
+ * marker (`GENERATE_MEDIA_IMAGE` / `GENERATE_MEDIA_AUDIO`) so the trajectory
+ * logger picks the action up exactly the way it picks up
+ * `plugin-computeruse` actions.
+ */
+import { type Action, type ActionResult, type HandlerCallback, type IAgentRuntime, type Memory } from "@elizaos/core";
+export type MediaKind = "image" | "audio" | "video";
+interface IntentDetection {
+    kind: MediaKind;
+    prompt: string;
+    source: "keyword" | "classifier";
+}
+type ClassifierFn = (prompt: string) => Promise<MediaKind | null>;
+export interface IntentDetectorOptions {
+    /**
+     * Optional override for the text-classifier fallback. Tests inject a
+     * deterministic classifier; in production this is bound to
+     * `runtime.useModel(ModelType.TEXT_SMALL, ...)`.
+     */
+    classifier?: ClassifierFn;
+}
+/**
+ * Detect the media intent from a user message.
+ *
+ * Algorithm:
+ *   1. Try keyword rules first (cheap, deterministic).
+ *   2. If nothing matched and a classifier is provided, ask it for a JSON
+ *      label. Trust the classifier only when it returns one of our three
+ *      kinds; otherwise return `null` so the caller can decline.
+ */
+export declare function detectMediaIntent(text: string, options?: IntentDetectorOptions): Promise<IntentDetection | null>;
+interface BuildHandlerOptions {
+    /** Test seam: override the intent detector. */
+    detectIntent: typeof detectMediaIntent;
+    /** Test seam: override the classifier resolver. */
+    classifierFactory: (runtime: IAgentRuntime) => ClassifierFn;
+}
+export declare function buildGenerateMediaHandler(opts?: Partial<BuildHandlerOptions>): (runtime: IAgentRuntime, message: Memory, _state?: unknown, _options?: unknown, callback?: HandlerCallback) => Promise<ActionResult>;
+export declare const generateMediaAction: Action;
+export default generateMediaAction;
+//# sourceMappingURL=generate-media.d.ts.map

package/dist/actions/generate-media.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"generate-media.d.ts","sourceRoot":"","sources":["../../src/actions/generate-media.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;GAsBG;AAEH,OAAO,EACN,KAAK,MAAM,EACX,KAAK,YAAY,EAGjB,KAAK,eAAe,EACpB,KAAK,aAAa,EAIlB,KAAK,MAAM,EAEX,MAAM,eAAe,CAAC;AAMvB,MAAM,MAAM,SAAS,GAAG,OAAO,GAAG,OAAO,GAAG,OAAO,CAAC;AAEpD,UAAU,eAAe;IACxB,IAAI,EAAE,SAAS,CAAC;IAChB,MAAM,EAAE,MAAM,CAAC;IACf,MAAM,EAAE,SAAS,GAAG,YAAY,CAAC;CACjC;AAoED,KAAK,YAAY,GAAG,CAAC,MAAM,EAAE,MAAM,KAAK,OAAO,CAAC,SAAS,GAAG,IAAI,CAAC,CAAC;AAElE,MAAM,WAAW,qBAAqB;IACrC;;;;OAIG;IACH,UAAU,CAAC,EAAE,YAAY,CAAC;CAC1B;AAsBD;;;;;;;;GAQG;AACH,wBAAsB,iBAAiB,CACtC,IAAI,EAAE,MAAM,EACZ,OAAO,GAAE,qBAA0B,GACjC,OAAO,CAAC,eAAe,GAAG,IAAI,CAAC,CASjC;AAoND,UAAU,mBAAmB;IAC5B,+CAA+C;IAC/C,YAAY,EAAE,OAAO,iBAAiB,CAAC;IACvC,mDAAmD;IACnD,iBAAiB,EAAE,CAAC,OAAO,EAAE,aAAa,KAAK,YAAY,CAAC;CAC5D;AAED,wBAAgB,yBAAyB,CACxC,IAAI,GAAE,OAAO,CAAC,mBAAmB,CAAM,IAKtC,SAAS,aAAa,EACtB,SAAS,MAAM,EACf,SAAS,OAAO,EAChB,WAAW,OAAO,EAClB,WAAW,eAAe,KACxB,OAAO,CAAC,YAAY,CAAC,CAsJxB;AAoBD,eAAO,MAAM,mBAAmB,EAAE,MA4EjC,CAAC;AAEF,eAAe,mBAAmB,CAAC"}

package/dist/actions/identify-speaker.d.ts

@@ -0,0 +1,23 @@
+/**
+ * @module plugin-local-inference/actions/identify-speaker
+ *
+ * `IDENTIFY_SPEAKER` agent action — the explicit, user-driven half of the
+ * voice → entity binding (issue #8234, shape #2).
+ *
+ * When the OWNER names a voice the agent just heard but hasn't identified
+ * ("that was Jill", "this is my friend Sam"), this action binds the most
+ * recently observed *unidentified* speaker profile to a named person. It
+ * does not touch the entity graph directly — it emits `VOICE_TURN_OBSERVED`
+ * so the merge engine (plugin-lifeops) creates/merges the Entity, then the
+ * round-trip `VOICE_ENTITY_BOUND` handler persists `entityId` back onto the
+ * voice profile. If no merge-engine plugin is loaded the action is inert
+ * beyond logging intent.
+ *
+ * Target selection: an explicit `profileId` option wins; otherwise the
+ * single most-recently-observed profile whose `entityId` is still `null`
+ * (i.e. "the person who just spoke and isn't known yet").
+ */
+import { type Action } from "@elizaos/core";
+export declare function extractSpeakerName(text: string): string | null;
+export declare const identifySpeakerAction: Action;
+//# sourceMappingURL=identify-speaker.d.ts.map

package/dist/actions/identify-speaker.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"identify-speaker.d.ts","sourceRoot":"","sources":["../../src/actions/identify-speaker.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;GAkBG;AAEH,OAAO,EACN,KAAK,MAAM,EAMX,MAAM,eAAe,CAAC;AA8BvB,wBAAgB,kBAAkB,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,GAAG,IAAI,CAS9D;AA8FD,eAAO,MAAM,qBAAqB,EAAE,MAUnC,CAAC"}

package/dist/actions/transcription-control.d.ts

@@ -0,0 +1,29 @@
+/**
+ * START_TRANSCRIPTION / STOP_TRANSCRIPTION agent actions (#8789).
+ *
+ * The agent can turn long-form voice transcription on/off on the user's device.
+ * Transcription CAPTURE is client-side (the mic lives in the renderer shell), so
+ * the action can't toggle it directly — it emits a one-way `voice-control`
+ * command on the AgentEventService bus. That stream is forwarded to every
+ * connected client as an `agent_event`; the renderer re-dispatches it to the
+ * shell's capture toggle (the same server→client pattern as `shell:navigate`).
+ * Best-effort + non-blocking: if no client is connected the event is simply
+ * dropped, and the action reports intent, not capture success.
+ */
+import { type Action, type IAgentRuntime } from "@elizaos/core";
+/** Bus stream the renderer subscribes to for mic/transcription control. */
+export declare const VOICE_CONTROL_STREAM = "voice-control";
+export type VoiceControlCommand = "start" | "stop";
+/** The payload shape the client matches on. */
+export interface VoiceControlEvent {
+    type: "voice-control";
+    command: VoiceControlCommand;
+}
+/**
+ * Emit a transcription control command to connected clients. Returns false when
+ * no event bus is available (e.g. headless) so callers can report honestly.
+ */
+export declare function emitVoiceControl(runtime: IAgentRuntime, command: VoiceControlCommand): boolean;
+export declare const startTranscriptionAction: Action;
+export declare const stopTranscriptionAction: Action;
+//# sourceMappingURL=transcription-control.d.ts.map

package/dist/actions/transcription-control.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"transcription-control.d.ts","sourceRoot":"","sources":["../../src/actions/transcription-control.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;GAWG;AAEH,OAAO,EACN,KAAK,MAAM,EAGX,KAAK,aAAa,EAKlB,MAAM,eAAe,CAAC;AAEvB,2EAA2E;AAC3E,eAAO,MAAM,oBAAoB,kBAAkB,CAAC;AAEpD,MAAM,MAAM,mBAAmB,GAAG,OAAO,GAAG,MAAM,CAAC;AAEnD,+CAA+C;AAC/C,MAAM,WAAW,iBAAiB;IACjC,IAAI,EAAE,eAAe,CAAC;IACtB,OAAO,EAAE,mBAAmB,CAAC;CAC7B;AAYD;;;GAGG;AACH,wBAAgB,gBAAgB,CAC/B,OAAO,EAAE,aAAa,EACtB,OAAO,EAAE,mBAAmB,GAC1B,OAAO,CAWT;AAgCD,eAAO,MAAM,wBAAwB,EAAE,MAetC,CAAC;AAEF,eAAO,MAAM,uBAAuB,EAAE,MAcrC,CAAC"}

package/dist/adapters/capacitor-llama/environment.d.ts

@@ -0,0 +1,12 @@
+import { z } from "zod";
+export declare const configSchema: z.ZodObject<{
+    LOCAL_SMALL_MODEL: z.ZodDefault<z.ZodOptional<z.ZodString>>;
+    LOCAL_LARGE_MODEL: z.ZodDefault<z.ZodOptional<z.ZodString>>;
+    LOCAL_EMBEDDING_MODEL: z.ZodDefault<z.ZodOptional<z.ZodString>>;
+    MODELS_DIR: z.ZodOptional<z.ZodString>;
+    CACHE_DIR: z.ZodOptional<z.ZodString>;
+    LOCAL_EMBEDDING_DIMENSIONS: z.ZodPipe<z.ZodDefault<z.ZodOptional<z.ZodString>>, z.ZodTransform<number, string>>;
+}, z.core.$strip>;
+export type Config = z.infer<typeof configSchema>;
+export declare function validateConfig(): Config;
+//# sourceMappingURL=environment.d.ts.map

package/dist/adapters/capacitor-llama/environment.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"environment.d.ts","sourceRoot":"","sources":["../../../src/adapters/capacitor-llama/environment.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAMxB,eAAO,MAAM,YAAY;;;;;;;iBAWvB,CAAC;AAEH,MAAM,MAAM,MAAM,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,YAAY,CAAC,CAAC;AAElD,wBAAgB,cAAc,IAAI,MAAM,CAgDvC"}

package/dist/adapters/capacitor-llama/index.browser.d.ts

@@ -0,0 +1,9 @@
+/**
+ * Browser-unavailable Capacitor-llama adapter. Browsers cannot run
+ * `libllama.so` directly; callers should proxy through a server or switch
+ * providers. All model handlers throw with a clear message.
+ */
+import type { Plugin } from "@elizaos/core";
+export declare const localAiPlugin: Plugin;
+export default localAiPlugin;
+//# sourceMappingURL=index.browser.d.ts.map

package/dist/adapters/capacitor-llama/index.browser.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.browser.d.ts","sourceRoot":"","sources":["../../../src/adapters/capacitor-llama/index.browser.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,OAAO,KAAK,EAIX,MAAM,EAEN,MAAM,eAAe,CAAC;AAoCvB,eAAO,MAAM,aAAa,EAAE,MAgC3B,CAAC;AAEF,eAAe,aAAa,CAAC"}

package/dist/adapters/capacitor-llama/index.d.ts

@@ -0,0 +1,18 @@
+/**
+ * Local-AI plugin backed by the Capacitor-llama adapter.
+ *
+ * This is the unified replacement for the legacy capacitor-llama adapter.
+ * It registers a `Plugin` that handles `ModelType.TEXT_SMALL/LARGE/EMBEDDING`
+ * via `CapacitorLlamaContext` — a single contract that resolves to
+ * `llama-cpp-capacitor` on mobile and the desktop bun:ffi adapter on
+ * desktop. Both load THE vendored llama.cpp at
+ * `plugins/plugin-local-inference/native/llama.cpp/` (Wave 2's cross-compiles).
+ *
+ * `capacitor-llama` is NEVER imported here.
+ */
+import { type Plugin } from "@elizaos/core";
+export declare const localAiPlugin: Plugin;
+export default localAiPlugin;
+export { type NativePcmVoiceTurn, NativePcmVoiceTurnCoordinator, type NativePcmVoiceTurnCoordinatorOptions, type NativePcmVoiceTurnResult, } from "./native-voice-capture";
+export { type CapacitorTextRunnerOptions, createCapacitorMtpTextRunner, type DeviceVoiceEngine, type RunDeviceVoiceTurnArgs, runDeviceVoiceTurn, type VoiceTurnExitReason, } from "./voice-turn";
+//# sourceMappingURL=index.d.ts.map

package/dist/adapters/capacitor-llama/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/adapters/capacitor-llama/index.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;GAWG;AAqBH,OAAO,EAKN,KAAK,MAAM,EAEX,MAAM,eAAe,CAAC;AAilBvB,eAAO,MAAM,aAAa,EAAE,MA+K3B,CAAC;AAEF,eAAe,aAAa,CAAC;AAQ7B,OAAO,EACN,KAAK,kBAAkB,EACvB,6BAA6B,EAC7B,KAAK,oCAAoC,EACzC,KAAK,wBAAwB,GAC7B,MAAM,wBAAwB,CAAC;AAChC,OAAO,EACN,KAAK,0BAA0B,EAC/B,4BAA4B,EAC5B,KAAK,iBAAiB,EACtB,KAAK,sBAAsB,EAC3B,kBAAkB,EAClB,KAAK,mBAAmB,GACxB,MAAM,cAAc,CAAC"}

package/dist/adapters/capacitor-llama/loader.d.ts

@@ -0,0 +1,35 @@
+/**
+ * Canonical Capacitor-llama loader — MOBILE ONLY.
+ *
+ * `initCapacitorLlama(params)` returns a context that implements
+ * `CapacitorLlamaContext` by dynamic-importing `llama-cpp-capacitor` on
+ * `ELIZA_PLATFORM=android|ios`. The package's `LlamaContext` already implements
+ * the canonical shape verbatim, so we return it directly with an `as` cast (the
+ * d.ts overlaps by construction).
+ *
+ * The Capacitor llama API is mobile-only. libllama has been retired, so there
+ * is no desktop façade: desktop / server inference runs through the fused
+ * `libelizainference` engine (`LocalInferenceEngine` /
+ * `desktopFusedFfiBackendRuntime`), not this loader. `initCapacitorLlama` on a
+ * non-mobile platform throws `CapacitorLlamaUnsupportedError`.
+ */
+import { type CapacitorLlamaContext, type CapacitorLlamaContextParams } from "./types";
+export interface InitCapacitorLlamaOptions extends CapacitorLlamaContextParams {
+    /**
+     * Force a specific backend. Only `mobile` is supported — it requires the
+     * `llama-cpp-capacitor` package. The desktop façade was removed when
+     * libllama was retired; desktop inference runs through the fused engine.
+     */
+    backend?: "mobile";
+}
+/**
+ * Load a Capacitor-shaped llama.cpp context. Mobile-only: on
+ * `ELIZA_PLATFORM=android|ios` it resolves `llama-cpp-capacitor`; on any other
+ * platform it throws `CapacitorLlamaUnsupportedError` (libllama is retired —
+ * desktop/server inference runs through the fused `libelizainference` engine,
+ * not this loader). Callers MUST handle the throw explicitly.
+ */
+export declare function initCapacitorLlama(opts: InitCapacitorLlamaOptions): Promise<CapacitorLlamaContext>;
+/** Mobile-only: release every context. No-op on desktop. */
+export declare function releaseAllCapacitorLlama(): Promise<void>;
+//# sourceMappingURL=loader.d.ts.map

package/dist/adapters/capacitor-llama/loader.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"loader.d.ts","sourceRoot":"","sources":["../../../src/adapters/capacitor-llama/loader.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;GAcG;AAGH,OAAO,EACN,KAAK,qBAAqB,EAC1B,KAAK,2BAA2B,EAEhC,MAAM,SAAS,CAAC;AAiCjB,MAAM,WAAW,yBAA0B,SAAQ,2BAA2B;IAC7E;;;;OAIG;IACH,OAAO,CAAC,EAAE,QAAQ,CAAC;CACnB;AAQD;;;;;;GAMG;AACH,wBAAsB,kBAAkB,CACvC,IAAI,EAAE,yBAAyB,GAC7B,OAAO,CAAC,qBAAqB,CAAC,CAgBhC;AAED,4DAA4D;AAC5D,wBAAsB,wBAAwB,IAAI,OAAO,CAAC,IAAI,CAAC,CAW9D"}

package/dist/adapters/capacitor-llama/native-voice-capture.d.ts

@@ -0,0 +1,70 @@
+/**
+ * Native PCM capture coordinator for the Capacitor / AOSP voice path.
+ *
+ * Platform mic front-ends own capture and VAD/turn segmentation. This class is
+ * the TypeScript bridge they call with a completed PCM turn: it serializes the
+ * turn through {@link runDeviceVoiceTurn}, which joins ASR, speaker attribution,
+ * response policy, text generation, and TTS in the fused voice path.
+ */
+import type { VoicePipelineEvents } from "../../services/voice/pipeline";
+import type { TranscriptionAudio } from "../../services/voice/types";
+import type { VoicePreloadPredictor } from "../../services/voice/voice-preload-predictor";
+import type { CapacitorLlamaContext } from "./types";
+import { type CapacitorTextRunnerOptions, type DeviceVoiceEngine, type VoiceTurnExitReason } from "./voice-turn";
+export interface NativePcmVoiceTurn {
+    /** Captured mic PCM for one VAD-completed native turn. */
+    audio: TranscriptionAudio;
+    /** Optional host turn id, useful for traces and logs. */
+    turnId?: string;
+    /** Per-turn event hooks. Falls back to coordinator defaults when omitted. */
+    events?: VoicePipelineEvents;
+    /** Per-turn generated-token cap. Falls back to coordinator defaults. */
+    maxGeneratedTokens?: number;
+    /** Per-turn sampling overrides. Falls back to coordinator defaults. */
+    generation?: CapacitorTextRunnerOptions;
+    /** Per-turn preload predictor. Falls back to coordinator defaults. */
+    preloadPredictor?: VoicePreloadPredictor;
+}
+export interface NativePcmVoiceTurnCoordinatorOptions {
+    /** The engine that owns the armed voice bridge (`startVoice` + `armVoice`). */
+    engine: DeviceVoiceEngine;
+    /** Loaded on-device text model context for the MTP target verifier. */
+    context: CapacitorLlamaContext;
+    /** Default pipeline events for every accepted native turn. */
+    events?: VoicePipelineEvents;
+    /** Default generated-token cap for every accepted native turn. */
+    maxGeneratedTokens?: number;
+    /** Default sampling overrides for the on-device text runner. */
+    generation?: CapacitorTextRunnerOptions;
+    /** Default next-stage preload predictor. */
+    preloadPredictor?: VoicePreloadPredictor;
+}
+export interface NativePcmVoiceTurnResult {
+    turnId?: string;
+    exitReason: VoiceTurnExitReason;
+}
+/**
+ * Serializes native completed PCM turns into the fused device voice engine.
+ *
+ * Native capture can emit turns quickly, especially when a flush returns more
+ * than one segment. The fused voice bridge only allows one active turn; this
+ * coordinator preserves order by queueing each accepted turn after the previous
+ * `runDeviceVoiceTurn` settles.
+ */
+export declare class NativePcmVoiceTurnCoordinator {
+    private readonly options;
+    private queue;
+    private running;
+    constructor(options: NativePcmVoiceTurnCoordinatorOptions);
+    get isRunning(): boolean;
+    start(): void;
+    stop(): Promise<void>;
+    /**
+     * Accept one VAD-completed native PCM turn and run it through the fused
+     * device voice path. Throws when the coordinator has not been started so
+     * callers cannot accidentally bypass the capture lifecycle.
+     */
+    acceptTurn(turn: NativePcmVoiceTurn): Promise<NativePcmVoiceTurnResult>;
+    private resolveTurnOptions;
+}
+//# sourceMappingURL=native-voice-capture.d.ts.map

package/dist/adapters/capacitor-llama/native-voice-capture.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"native-voice-capture.d.ts","sourceRoot":"","sources":["../../../src/adapters/capacitor-llama/native-voice-capture.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AAEH,OAAO,KAAK,EAAE,mBAAmB,EAAE,MAAM,+BAA+B,CAAC;AACzE,OAAO,KAAK,EAAE,kBAAkB,EAAE,MAAM,4BAA4B,CAAC;AACrE,OAAO,KAAK,EAAE,qBAAqB,EAAE,MAAM,8CAA8C,CAAC;AAC1F,OAAO,KAAK,EAAE,qBAAqB,EAAE,MAAM,SAAS,CAAC;AACrD,OAAO,EACN,KAAK,0BAA0B,EAC/B,KAAK,iBAAiB,EAEtB,KAAK,mBAAmB,EACxB,MAAM,cAAc,CAAC;AAEtB,MAAM,WAAW,kBAAkB;IAClC,0DAA0D;IAC1D,KAAK,EAAE,kBAAkB,CAAC;IAC1B,yDAAyD;IACzD,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,6EAA6E;IAC7E,MAAM,CAAC,EAAE,mBAAmB,CAAC;IAC7B,wEAAwE;IACxE,kBAAkB,CAAC,EAAE,MAAM,CAAC;IAC5B,uEAAuE;IACvE,UAAU,CAAC,EAAE,0BAA0B,CAAC;IACxC,sEAAsE;IACtE,gBAAgB,CAAC,EAAE,qBAAqB,CAAC;CACzC;AAED,MAAM,WAAW,oCAAoC;IACpD,+EAA+E;IAC/E,MAAM,EAAE,iBAAiB,CAAC;IAC1B,uEAAuE;IACvE,OAAO,EAAE,qBAAqB,CAAC;IAC/B,8DAA8D;IAC9D,MAAM,CAAC,EAAE,mBAAmB,CAAC;IAC7B,kEAAkE;IAClE,kBAAkB,CAAC,EAAE,MAAM,CAAC;IAC5B,gEAAgE;IAChE,UAAU,CAAC,EAAE,0BAA0B,CAAC;IACxC,4CAA4C;IAC5C,gBAAgB,CAAC,EAAE,qBAAqB,CAAC;CACzC;AAED,MAAM,WAAW,wBAAwB;IACxC,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,UAAU,EAAE,mBAAmB,CAAC;CAChC;AAED;;;;;;;GAOG;AACH,qBAAa,6BAA6B;IACzC,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAuC;IAC/D,OAAO,CAAC,KAAK,CAAoC;IACjD,OAAO,CAAC,OAAO,CAAS;gBAEZ,OAAO,EAAE,oCAAoC;IAIzD,IAAI,SAAS,IAAI,OAAO,CAEvB;IAED,KAAK,IAAI,IAAI;IAIP,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC;IAK3B;;;;OAIG;IACH,UAAU,CAAC,IAAI,EAAE,kBAAkB,GAAG,OAAO,CAAC,wBAAwB,CAAC;IA6BvE,OAAO,CAAC,kBAAkB;CAoB1B"}

package/dist/adapters/capacitor-llama/structured-output.d.ts

@@ -0,0 +1,62 @@
+/**
+ * Structured-output planning on the Capacitor surface.
+ *
+ * llama.cpp itself supports GBNF grammars and JSON-Schema-as-grammar natively;
+ * the Capacitor binding exposes both via `CompletionParams.grammar` (raw GBNF
+ * source) and `CompletionParams.response_format` (JSON object or JSON schema).
+ *
+ * Function-calling is exposed differently than in node-llama-cpp: the
+ * Capacitor binding takes a raw `tools` object plus optional `tool_choice` /
+ * `parallel_tool_calls` and the model emits an OpenAI-shaped
+ * `tool_calls[]` array inside the completion result. We project this back
+ * into the elizaOS `ToolCallResult` shape so downstream callers do not care
+ * which backend produced the call.
+ */
+import type { JSONSchema, ToolDefinition } from "@elizaos/core";
+import type { CapacitorLlamaCompletionParams, CapacitorLlamaCompletionResult } from "./types";
+export interface ToolCallResult {
+    id: string;
+    name: string;
+    arguments: Record<string, unknown>;
+    type: "function";
+}
+export interface StructuredRequestPlan {
+    kind: "text" | "tools" | "schema" | "json_object";
+    /** Forwarded to `CapacitorLlamaCompletionParams.tools`. */
+    tools?: object;
+    /** Forwarded to `CapacitorLlamaCompletionParams.response_format`. */
+    responseFormat?: CapacitorLlamaCompletionParams["response_format"];
+    /** Optional `tool_choice` directive. */
+    toolChoice?: string;
+}
+/**
+ * Build the OpenAI-style `tools` object the Capacitor binding accepts. Each
+ * elizaOS `ToolDefinition` becomes one entry of:
+ *   { type: "function", function: { name, description, parameters } }
+ */
+export declare function buildCapacitorTools(tools: readonly ToolDefinition[]): object;
+/**
+ * Extract tool calls from the Capacitor completion result. The binding emits
+ * them on `result.tool_calls`; we parse `arguments` JSON and surface the
+ * elizaOS shape.
+ */
+export declare function extractToolCalls(result: CapacitorLlamaCompletionResult): ToolCallResult[];
+/**
+ * Decide which structured-output mode applies. Mirrors the legacy
+ * `planStructuredRequest` semantics: tools > schema > generic json > text.
+ */
+export declare function planStructuredRequest(params: {
+    tools?: readonly ToolDefinition[];
+    responseSchema?: JSONSchema;
+    responseFormat?: {
+        type: "json_object" | "text";
+    } | string | undefined;
+    toolChoice?: string;
+}): StructuredRequestPlan;
+/**
+ * Apply a `StructuredRequestPlan` onto a `CapacitorLlamaCompletionParams`
+ * object, returning a new merged object. Used by the index module right
+ * before dispatching the completion.
+ */
+export declare function applyStructuredPlan(params: CapacitorLlamaCompletionParams, plan: StructuredRequestPlan): CapacitorLlamaCompletionParams;
+//# sourceMappingURL=structured-output.d.ts.map

package/dist/adapters/capacitor-llama/structured-output.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"structured-output.d.ts","sourceRoot":"","sources":["../../../src/adapters/capacitor-llama/structured-output.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;GAaG;AAEH,OAAO,KAAK,EAAE,UAAU,EAAE,cAAc,EAAE,MAAM,eAAe,CAAC;AAChE,OAAO,KAAK,EACX,8BAA8B,EAC9B,8BAA8B,EAE9B,MAAM,SAAS,CAAC;AAEjB,MAAM,WAAW,cAAc;IAC9B,EAAE,EAAE,MAAM,CAAC;IACX,IAAI,EAAE,MAAM,CAAC;IACb,SAAS,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IACnC,IAAI,EAAE,UAAU,CAAC;CACjB;AAED,MAAM,WAAW,qBAAqB;IACrC,IAAI,EAAE,MAAM,GAAG,OAAO,GAAG,QAAQ,GAAG,aAAa,CAAC;IAClD,2DAA2D;IAC3D,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,qEAAqE;IACrE,cAAc,CAAC,EAAE,8BAA8B,CAAC,iBAAiB,CAAC,CAAC;IACnE,wCAAwC;IACxC,UAAU,CAAC,EAAE,MAAM,CAAC;CACpB;AAED;;;;GAIG;AACH,wBAAgB,mBAAmB,CAAC,KAAK,EAAE,SAAS,cAAc,EAAE,GAAG,MAAM,CAqB5E;AAED;;;;GAIG;AACH,wBAAgB,gBAAgB,CAC/B,MAAM,EAAE,8BAA8B,GACpC,cAAc,EAAE,CAalB;AAoBD;;;GAGG;AACH,wBAAgB,qBAAqB,CAAC,MAAM,EAAE;IAC7C,KAAK,CAAC,EAAE,SAAS,cAAc,EAAE,CAAC;IAClC,cAAc,CAAC,EAAE,UAAU,CAAC;IAC5B,cAAc,CAAC,EAAE;QAAE,IAAI,EAAE,aAAa,GAAG,MAAM,CAAA;KAAE,GAAG,MAAM,GAAG,SAAS,CAAC;IACvE,UAAU,CAAC,EAAE,MAAM,CAAC;CACpB,GAAG,qBAAqB,CA+BxB;AAED;;;;GAIG;AACH,wBAAgB,mBAAmB,CAClC,MAAM,EAAE,8BAA8B,EACtC,IAAI,EAAE,qBAAqB,GACzB,8BAA8B,CAOhC"}

package/dist/adapters/capacitor-llama/text-streaming.d.ts

@@ -0,0 +1,24 @@
+/**
+ * Capacitor → `TextStreamResult` bridge.
+ *
+ * The Capacitor `LlamaContext.completion(params, callback)` API is push-based:
+ * `callback` fires once per token. The elizaOS runtime consumes a pull-based
+ * `TextStreamResult` (`{ textStream, text, usage, finishReason }`). We bridge
+ * the two with a queue-backed async iterator, identical in shape to the
+ * legacy node-llama-cpp bridge.
+ */
+import type { TextStreamResult, TokenUsage } from "@elizaos/core";
+import type { CapacitorLlamaCompletionParams, CapacitorLlamaContext } from "./types";
+export interface StreamCapacitorPromptArgs {
+    ctx: CapacitorLlamaContext;
+    params: CapacitorLlamaCompletionParams;
+    estimateUsage: (prompt: string, fullText: string) => TokenUsage;
+    onChunk?: (delta: string) => void;
+    onComplete?: (info: {
+        fullText: string;
+        usage: TokenUsage;
+    }) => void;
+    postProcess?: (raw: string) => string;
+}
+export declare function streamCapacitorPrompt(args: StreamCapacitorPromptArgs): TextStreamResult;
+//# sourceMappingURL=text-streaming.d.ts.map

package/dist/adapters/capacitor-llama/text-streaming.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"text-streaming.d.ts","sourceRoot":"","sources":["../../../src/adapters/capacitor-llama/text-streaming.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AAEH,OAAO,KAAK,EAAE,gBAAgB,EAAE,UAAU,EAAE,MAAM,eAAe,CAAC;AAClE,OAAO,KAAK,EACX,8BAA8B,EAE9B,qBAAqB,EACrB,MAAM,SAAS,CAAC;AAwEjB,MAAM,WAAW,yBAAyB;IACzC,GAAG,EAAE,qBAAqB,CAAC;IAC3B,MAAM,EAAE,8BAA8B,CAAC;IACvC,aAAa,EAAE,CAAC,MAAM,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,KAAK,UAAU,CAAC;IAChE,OAAO,CAAC,EAAE,CAAC,KAAK,EAAE,MAAM,KAAK,IAAI,CAAC;IAClC,UAAU,CAAC,EAAE,CAAC,IAAI,EAAE;QAAE,QAAQ,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,UAAU,CAAA;KAAE,KAAK,IAAI,CAAC;IACrE,WAAW,CAAC,EAAE,CAAC,GAAG,EAAE,MAAM,KAAK,MAAM,CAAC;CACtC;AAED,wBAAgB,qBAAqB,CACpC,IAAI,EAAE,yBAAyB,GAC7B,gBAAgB,CAgIlB"}