@elizaos/plugin-local-inference 2.0.0-beta.1 → 2.0.11-beta.7

package/src/services/voice/eot-classifier-ggml.d.ts

@@ -0,0 +1,179 @@
+/**
+ * LiveKit turn-detector — GGUF-backed binding (J1.d).
+ *
+ * The text-side turn-completion classifier formats the latest partial
+ * user transcript with the Qwen chat template, strips the trailing
+ * `<|im_end|>`, and reads `P(<|im_end|>)` from the next-token
+ * distribution. The upstream `livekit/turn-detector` ships an ONNX
+ * graph; this binding consumes the **GGUF** export published at
+ * `elizaos/eliza-1` under `voice/turn-detector/onnx/turn-detector-en-q8.gguf`
+ * (and the multilingual variant at
+ * `voice/turn/intl/turn-detector-intl-q8.gguf`), running through the
+ * canonical fork wrapper `capacitor-llama` (per
+ * `plugins/plugin-local-inference/native/AGENTS.md` §1 — the npm
+ * `capacitor-llama` dep is the elizaOS fork's republish, binding to the
+ * fork's `libllama`).
+ *
+ * Why this exists: per I1's single-runtime audit
+ * (`.swarm/impl/I1-single-runtime.md` §B), the turn-detector was the
+ * cheapest of the four remaining ONNX surfaces to retire — the GGUF
+ * artifact was already published by H4 (see commit history), and the
+ * detector's architecture (Qwen2-style small decoder + classification
+ * head on the `<|im_end|>` logit) is exactly what `LLM_ARCH_QWEN2`
+ * already implements in the fork. The work is wiring, not porting.
+ *
+ * No silent fallback (AGENTS.md §3): when `capacitor-llama` is
+ * unavailable, the GGUF is missing, or the model load fails, this
+ * class throws `EotGgmlUnavailableError` with a structured code. The
+ * resolver above this binding picks `HeuristicEotClassifier` or the
+ * legacy ONNX path; the binding itself never fabricates a probability.
+ *
+ * Tokenizer ownership: the GGUF carries its own tokenizer (BPE +
+ * special tokens, including `<|im_end|>`); this binding does NOT
+ * import `@huggingface/transformers`. The `apply_chat_template`
+ * formatting is re-implemented here using the same template upstream
+ * uses (single-turn user message wrapped in `<|im_start|>user\n... \n`)
+ * — see `applyQwenUserTemplate` below.
+ *
+ * --- Planned LoRA hot-swap path ---
+ *
+ * The LiveKit GGUF is a separate 66 MB (EN) or 396 MB (intl) resident
+ * model. The chat target model (eliza-1-{0_8b,2b,4b}) is already
+ * loaded for conversation — its next-token distribution after the
+ * chat-template-formatted partial transcript provides exactly the
+ * same `P(<|im_end|>)` signal. A LoRA adapter (rank 8, ~5-10 MB)
+ * trained on `(transcript, eot_label)` pairs can shape that signal
+ * to match or beat the LiveKit baseline.
+ *
+ * The training pipeline lives at
+ * `packages/training/scripts/voice/eot/RUNBOOK.md`. The publish gates
+ * (AUROC ≥ 0.85, ECE ≤ 0.05, p95 ≤ 50 ms) are in
+ * `packages/training/benchmarks/eot_gates.md`.
+ *
+ * Integration sketch (deferred — implement when an adapter ships):
+ *
+ *   1. Bundle ships `voice/eot-lora/eliza-1-<tier>-eot-lora.bin`
+ *      alongside the chat target GGUF, with a manifest sidecar binding
+ *      the adapter to the target's SHA256 (refuse to load if mismatched).
+ *   2. Runtime detects the adapter via the bundle catalog. When present
+ *      AND the chat target is already loaded, the EOT resolver prefers
+ *      LoRA hot-swap over standing up the LiveKit GGUF process.
+ *   3. The hot-swap path uses llama.cpp's `--lora` flag on the chat
+ *      target. A single forward pass against the chat-template-formatted
+ *      transcript yields the next-token logits; read `<|im_end|>`'s
+ *      probability and return it.
+ *   4. Fail-closed: if adapter load fails or the SHA binding mismatches,
+ *      throw `EotGgmlUnavailableError("model-load-failed", ...)`. No
+ *      silent fallback to the un-adapted target (the un-adapted logits
+ *      would be the chat-model's own prior, not a calibrated EOT
+ *      classifier).
+ *
+ * Until that lands, this binding remains the canonical EOT path.
+ */
+import type { EotClassifier, VoiceTurnSignal } from "./eot-classifier";
+/** Raised when the GGUF binding cannot be loaded or scored. */
+export declare class EotGgmlUnavailableError extends Error {
+    readonly code: "native-missing" | "model-missing" | "model-load-failed" | "tokenizer-missing-im-end" | "evaluate-failed" | "invalid-input";
+    constructor(code: EotGgmlUnavailableError["code"], message: string);
+}
+/** HF mono-repo holding every voice sub-model. */
+export declare const LIVEKIT_TURN_DETECTOR_HF_REPO = "elizaos/eliza-1";
+/** Canonical English GGUF asset path inside the bundle. */
+export declare const DEFAULT_LIVEKIT_TURN_DETECTOR_GGUF_EN = "voice/turn-detector/onnx/turn-detector-en-q8.gguf";
+/** Canonical multilingual GGUF asset path inside the bundle. */
+export declare const DEFAULT_LIVEKIT_TURN_DETECTOR_GGUF_INTL = "voice/turn/intl/turn-detector-intl-q8.gguf";
+/** Special-token literal the detector reads the probability of. */
+export declare const LIVEKIT_IM_END_TOKEN = "<|im_end|>";
+/**
+ * Default on-disk location for the staged GGUF. The bundle downloader
+ * materializes assets under `<stateDir>/local-inference/models/...`; this
+ * matches the bundle downloader's staged-asset path layout.
+ */
+export declare const DEFAULT_LIVEKIT_TURN_DETECTOR_GGML_DIR: string;
+/**
+ * Resolve which GGUF variant a given Eliza-1 tier should bundle.
+ * Mirrors `turnDetectorRevisionForTier` in `eot-classifier.ts` —
+ * mobile tiers get the English-only ~40 MB Q8 GGUF, desktop tiers
+ * get the multilingual ~280 MB Q8 GGUF.
+ *
+ * Accepts both bare tier ids (`"4b"`) and prefixed catalog ids
+ * (`"eliza-1-4b"`).
+ */
+export declare function turnDetectorGgufForTier(tierId: string): {
+    hfPath: string;
+    variant: "en" | "intl";
+};
+export interface LiveKitGgmlTurnDetectorOptions {
+    /** Absolute path to the GGUF file. Required. */
+    ggufPath: string;
+    /**
+     * Upstream revision tag for telemetry only (`"v1.2.2-en"` /
+     * `"v0.4.1-intl"`). Does not affect inference.
+     */
+    revision?: string;
+    /** Max history tokens after Qwen-template wrapping. Default: 128. */
+    maxHistoryTokens?: number;
+    /** Optional model label for telemetry. */
+    model?: string;
+    /** Optional thread count for the context. Default: 2. */
+    threads?: number;
+}
+/**
+ * Local GGUF-backed LiveKit turn-detector. Uses a `capacitor-llama`
+ * evaluation of the Qwen2-style decoder, reading `P(<|im_end|>)` from the
+ * next-token distribution after the truncated user-template prefix.
+ *
+ * One detector instance owns one `LlamaModel` + one `LlamaContext` +
+ * one `LlamaSequence`. `score()` resets the sequence between calls —
+ * the detector is intentionally stateless (no carried context across
+ * different user transcripts).
+ */
+export declare class LiveKitGgmlTurnDetector implements EotClassifier {
+    readonly ggufPath: string;
+    private readonly maxHistoryTokens;
+    private readonly model;
+    private readonly revision;
+    private readonly threads;
+    private ready;
+    constructor(opts: LiveKitGgmlTurnDetectorOptions);
+    score(partialTranscript: string): Promise<number>;
+    signal(partialTranscript: string): Promise<VoiceTurnSignal>;
+    /** Release the underlying GGUF + context. Idempotent. */
+    dispose(): Promise<void>;
+    private load;
+    private loadInner;
+}
+/**
+ * Resolve a `LiveKitGgmlTurnDetector` against the bundle's on-disk
+ * staging directory. Search order:
+ *
+ *   1. Explicit `opts.ggufPath`.
+ *   2. `ELIZA_TURN_DETECTOR_GGUF` env var.
+ *   3. `<modelDir>/<en GGUF name>` (the canonical layout for the
+ *      bundle's English variant).
+ *   4. `<modelDir>/<intl GGUF name>` (multilingual).
+ *
+ * Returns `null` if no GGUF is found alongside the directory — the
+ * caller falls back to `HeuristicEotClassifier`.
+ */
+export declare function createBundledLiveKitGgmlTurnDetector(opts?: {
+    ggufPath?: string;
+    modelDir?: string;
+    revision?: string;
+    maxHistoryTokens?: number;
+    threads?: number;
+}): Promise<LiveKitGgmlTurnDetector | null>;
+/**
+ * Apply the single-turn user Qwen chat template, omitting the trailing
+ * `<|im_end|>` so the detector head scores it as the next token.
+ *
+ * Upstream `livekit/turn-detector` formats:
+ *
+ *   <|im_start|>user\n{transcript}<|im_end|>\n
+ *
+ * The detector strips the trailing `<|im_end|>\n` and reads
+ * `P(<|im_end|>)` after the user content. We emit the prefix exactly,
+ * stopping where the `<|im_end|>` would go.
+ */
+export declare function applyQwenUserTemplate(transcript: string): string;
+//# sourceMappingURL=eot-classifier-ggml.d.ts.map

package/src/services/voice/eot-classifier-ggml.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"eot-classifier-ggml.d.ts","sourceRoot":"","sources":["eot-classifier-ggml.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAuEG;AAKH,OAAO,KAAK,EAAE,aAAa,EAAE,eAAe,EAAE,MAAM,kBAAkB,CAAC;AAOvE,+DAA+D;AAC/D,qBAAa,uBAAwB,SAAQ,KAAK;IACjD,QAAQ,CAAC,IAAI,EACV,gBAAgB,GAChB,eAAe,GACf,mBAAmB,GACnB,0BAA0B,GAC1B,iBAAiB,GACjB,eAAe,CAAC;gBACP,IAAI,EAAE,uBAAuB,CAAC,MAAM,CAAC,EAAE,OAAO,EAAE,MAAM;CAKlE;AAMD,kDAAkD;AAClD,eAAO,MAAM,6BAA6B,oBAAoB,CAAC;AAE/D,2DAA2D;AAC3D,eAAO,MAAM,qCAAqC,sDACE,CAAC;AAErD,gEAAgE;AAChE,eAAO,MAAM,uCAAuC,+CACP,CAAC;AAE9C,mEAAmE;AACnE,eAAO,MAAM,oBAAoB,eAAe,CAAC;AAEjD;;;;GAIG;AACH,eAAO,MAAM,sCAAsC,QAMlD,CAAC;AAEF;;;;;;;;GAQG;AACH,wBAAgB,uBAAuB,CAAC,MAAM,EAAE,MAAM,GAAG;IACxD,MAAM,EAAE,MAAM,CAAC;IACf,OAAO,EAAE,IAAI,GAAG,MAAM,CAAC;CACvB,CAcA;AAqGD,MAAM,WAAW,8BAA8B;IAC9C,gDAAgD;IAChD,QAAQ,EAAE,MAAM,CAAC;IACjB;;;OAGG;IACH,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,qEAAqE;IACrE,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B,0CAA0C;IAC1C,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,yDAAyD;IACzD,OAAO,CAAC,EAAE,MAAM,CAAC;CACjB;AAMD;;;;;;;;;GASG;AACH,qBAAa,uBAAwB,YAAW,aAAa;IAC5D,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;IAC1B,OAAO,CAAC,QAAQ,CAAC,gBAAgB,CAAS;IAC1C,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAS;IAC/B,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAqB;IAC9C,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAS;IACjC,OAAO,CAAC,KAAK,CAKI;gBAEL,IAAI,EAAE,8BAA8B;IAkB1C,KAAK,CAAC,iBAAiB,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC;IAIjD,MAAM,CAAC,iBAAiB,EAAE,MAAM,GAAG,OAAO,CAAC,eAAe,CAAC;IA8DjE,yDAAyD;IACnD,OAAO,IAAI,OAAO,CAAC,IAAI,CAAC;IAW9B,OAAO,CAAC,IAAI;YAUE,SAAS;CA2DvB;AAMD;;;;;;;;;;;;GAYG;AACH,wBAAsB,oCAAoC,CACzD,IAAI,GAAE;IACL,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B,OAAO,CAAC,EAAE,MAAM,CAAC;CACZ,GACJ,OAAO,CAAC,uBAAuB,GAAG,IAAI,CAAC,CAsCzC;AAqBD;;;;;;;;;;;GAWG;AACH,wBAAgB,qBAAqB,CAAC,UAAU,EAAE,MAAM,GAAG,MAAM,CAEhE"}