@elizaos/plugin-local-inference 2.0.0-beta.1 → 2.0.11-beta.7

package/src/services/voice/eot-classifier.d.ts

@@ -0,0 +1,214 @@
+/**
+ * Semantic end-of-turn (EOT) classifier — Tier 3 of the three-tier VAD.
+ *
+ * Tier 1: RMS energy gate (~10 ms)
+ * Tier 2: Silero VAD (~32 ms hop)
+ * Tier 3: Semantic EOT classifier — P(turn_complete | transcript_so_far)
+ *
+ * The classifier operates on the partial transcript text emitted by streaming
+ * ASR, not on audio. It returns P(done) ∈ [0, 1]. The voice state machine
+ * uses it to:
+ *
+ *   P(done) ≥ 0.9 AND silence ≥ 50 ms  → commit immediately, skip hangover
+ *   P(done) ≥ 0.6 AND silence ≥ 20 ms  → enter PAUSE_TENTATIVE early (start drafter)
+ *   P(done) < 0.4                        → extend hangover by 50 ms (mid-clause)
+ *
+ * Three implementations ship:
+ *
+ *   `HeuristicEotClassifier` — deterministic, zero-latency, no model load.
+ *     This is the baseline; it is always available.
+ *
+ *   `RemoteEotClassifier` — fail-closed HTTP adapter for a real model server.
+ *     It throws on network/parse errors so callers never mistake a synthetic
+ *     fallback for a measured turn signal.
+ *
+ *   `Eliza1EotClassifier` — uses the already-loaded text model to compute
+ *     P(`<|im_end|>` | partial transcript). Zero additional model weights.
+ *
+ *   The GGUF-backed LiveKit detector lives in `eot-classifier-ggml.ts`.
+ *
+ * Cancellation contract (handshake with VoiceTurnController / R11): the
+ * classifier emits a `VoiceTurnSignal` per partial transcript. It NEVER
+ * aborts a turn directly — `signal()` is data, not a cancellation. The
+ * controller layer above consumes the signal and decides whether to
+ * suppress (via `BargeInCancelToken.signal` with reason `"turn-suppressed"`).
+ */
+import type { Eliza1EotScoreResult, Eliza1EotScorerOptions } from "./eliza1-eot-scorer";
+import { Eliza1EotScorer } from "./eliza1-eot-scorer";
+import { FfiEotScorer, type FfiEotScorerOptions } from "./fused-eot-scorer";
+export type VoiceNextSpeaker = "agent" | "user" | "unknown";
+export interface VoiceTurnSignal {
+    /** P(user turn complete | transcript/history). */
+    endOfTurnProbability: number;
+    /**
+     * The best turn-taking read from this signal. Text-only EOU models infer
+     * this from end-of-turn probability; audio/prosody models can set it
+     * directly.
+     */
+    nextSpeaker: VoiceNextSpeaker;
+    /** Whether the agent should begin a response now. */
+    agentShouldSpeak: boolean | null;
+    /** Implementation/source name for telemetry and trace records. */
+    source: "heuristic" | "livekit-turn-detector" | "eliza-1-drafter" | "remote" | "custom";
+    /** Optional model/version identifier for telemetry. */
+    model?: string;
+    /** Text actually scored after normalization/template truncation. */
+    transcript: string;
+    /** Wall-clock model latency, excluding caller queueing. */
+    latencyMs?: number;
+}
+/**
+ * End-of-turn classifier interface. Both implementations satisfy this contract
+ * so callers are backend-agnostic.
+ */
+export interface EotClassifier {
+    /** Return P(turn_complete) ∈ [0, 1] for `partialTranscript`. */
+    score(partialTranscript: string): Promise<number>;
+    /** Return the structured turn signal when the implementation can provide it. */
+    signal?(partialTranscript: string): Promise<VoiceTurnSignal>;
+}
+export declare function clampProbability(value: number): number;
+export declare function turnSignalFromProbability(args: {
+    probability: number;
+    transcript: string;
+    source: VoiceTurnSignal["source"];
+    model?: string;
+    latencyMs?: number;
+}): VoiceTurnSignal;
+/**
+ * Rules-of-thumb EOT classifier. The rules fire in priority order; the first
+ * match wins.
+ *
+ * Priority  Signal                                       P(done)
+ * --------  -------------------------------------------  -------
+ *   1       Sentence-final punctuation (. ! ?)            0.95
+ *   2       Question-tag words ("right?", "yeah?", "ok?") 0.85
+ *   3       Short utterance (< 3 words)                   0.70
+ *   4       Trailing conjunction (and/but/or/because/…)   0.15
+ *   5       Last word is a preposition or article         0.20
+ *   6       No signal                                     0.50
+ */
+export declare class HeuristicEotClassifier implements EotClassifier {
+    /** Conjunctions that strongly suggest the user is mid-clause. */
+    private static readonly TRAILING_CONJUNCTIONS;
+    /** Prepositions and articles that suggest an incomplete NP follows. */
+    private static readonly TRAILING_INCOMPLETE;
+    /** Question-tag suffixes that end an utterance (case-insensitive). */
+    private static readonly QUESTION_TAGS;
+    score(partialTranscript: string): Promise<number>;
+    signal(partialTranscript: string): Promise<VoiceTurnSignal>;
+}
+/**
+ * Resolve which upstream revision a given Eliza-1 tier should bundle.
+ * Mobile/small tiers (`0_8b`, `2b`) get the English-only variant;
+ * desktop/server tiers (`4b`+) get the multilingual variant.
+ *
+ * Accepts both bare tier ids (`"4b"`) and prefixed catalog ids
+ * (`"eliza-1-4b"`).
+ */
+export declare const LIVEKIT_TURN_DETECTOR_EN_REVISION = "v1.2.2-en";
+export declare const LIVEKIT_TURN_DETECTOR_INTL_REVISION = "v0.4.1-intl";
+export declare function turnDetectorRevisionForTier(tierId: string): typeof LIVEKIT_TURN_DETECTOR_EN_REVISION | typeof LIVEKIT_TURN_DETECTOR_INTL_REVISION;
+export interface RemoteEotClassifierOptions {
+    /**
+     * HTTP endpoint to POST the partial transcript to. Expected to return JSON
+     * with a `p_done` field: `{ "p_done": 0.92 }`.
+     *
+     * Example: LiveKit turn-detector inference endpoint or a custom model server.
+     */
+    endpoint: string;
+    /**
+     * Timeout in milliseconds for each HTTP request. Default 200 ms — the
+     * classifier must be faster than the silence hangover it's trying to beat.
+     */
+    timeoutMs?: number;
+    /** Optional model label for telemetry. */
+    model?: string;
+}
+/**
+ * Remote EOT classifier. POSTs `{ transcript: string }` to `endpoint`
+ * and expects `{ p_done: number }` back.
+ *
+ * Intended to be wired to a real LiveKit turn-detector HTTP API or a custom
+ * model inference server. This adapter fails closed: no fallback score is
+ * manufactured on network or parse errors.
+ */
+export declare class RemoteEotClassifier implements EotClassifier {
+    private readonly endpoint;
+    private readonly timeoutMs;
+    private readonly model;
+    constructor(opts: RemoteEotClassifierOptions);
+    score(partialTranscript: string): Promise<number>;
+    signal(partialTranscript: string): Promise<VoiceTurnSignal>;
+}
+/** P(done) ≥ this AND silence ≥ EOT_COMMIT_SILENCE_MS → commit immediately. */
+export declare const EOT_COMMIT_THRESHOLD = 0.9;
+/** P(done) ≥ this AND silence ≥ EOT_TENTATIVE_SILENCE_MS → enter PAUSE_TENTATIVE early. */
+export declare const EOT_TENTATIVE_THRESHOLD = 0.6;
+/** P(done) < this → extend hangover by EOT_HANGOVER_EXTENSION_MS. */
+export declare const EOT_MID_CLAUSE_THRESHOLD = 0.4;
+/** Minimum silence (ms) required alongside P ≥ EOT_COMMIT_THRESHOLD to commit. */
+export declare const EOT_COMMIT_SILENCE_MS = 50;
+/** Minimum silence (ms) required alongside P ≥ EOT_TENTATIVE_THRESHOLD to start drafter. */
+export declare const EOT_TENTATIVE_SILENCE_MS = 20;
+/** How many ms to add to the pause hangover when P < EOT_MID_CLAUSE_THRESHOLD. */
+export declare const EOT_HANGOVER_EXTENSION_MS = 50;
+export type { Eliza1EotScoreResult, Eliza1EotScorerOptions };
+/**
+ * Eliza-1 EOT classifier. Reuses the already-loaded text model (typically
+ * the eliza-1 drafter — same model MTP keeps warm for speculative
+ * decoding) to compute P(`<|im_end|>` | partial transcript). Optionally
+ * loads a fine-tuned EOT LoRA adapter on top of the base weights — see
+ * `packages/training/scripts/turn_detector/` for the training recipe.
+ *
+ * Unlike the GGUF-backed `LiveKitGgmlTurnDetector`, this classifier ships
+ * zero additional model weights — it leans on what's already in RAM.
+ */
+export declare class Eliza1EotClassifier implements EotClassifier {
+    private readonly scorer;
+    constructor(options: Eliza1EotScorerOptions | {
+        scorer: Eliza1EotScorer;
+    });
+    score(partialTranscript: string): Promise<number>;
+    signal(partialTranscript: string): Promise<VoiceTurnSignal>;
+    dispose(): Promise<void>;
+}
+/**
+ * Above this heuristic confidence the heuristic's high-precision syntactic
+ * verdict is trusted outright and the model forward pass is skipped. Sentence-
+ * final punctuation, question tags, trailing conjunctions, and dangling
+ * prepositions all clear this bar (P ≤ 0.2 or ≥ 0.8 → confidence ≥ 0.6); short
+ * utterances and the no-signal case fall below it and defer to the model.
+ */
+export declare const COMPOSITE_HEURISTIC_CONFIDENCE_CUTOFF = 0.6;
+/**
+ * End-of-turn classifier that blends the fused semantic scorer
+ * (P(`<|im_end|>`) over the loaded text model) with the heuristic syntactic
+ * rules. The heuristic is NOT a fallback — it is a tuned co-signal: when it is
+ * confident (clear punctuation / mid-clause conjunction / dangling preposition)
+ * its verdict wins outright and the model pass is skipped; in the ambiguous
+ * middle (short utterance, no syntactic cue) the model's semantic judgment
+ * dominates, blended by the heuristic's residual confidence. Acoustic
+ * silence/VAD timing lives a tier below this (the VAD), so this layer is the
+ * pure text-completion read.
+ */
+export declare class CompositeEotClassifier implements EotClassifier {
+    private readonly model;
+    private readonly heuristic;
+    private readonly confidenceCutoff;
+    constructor(options: {
+        model: FfiEotScorer;
+        heuristic?: HeuristicEotClassifier;
+        confidenceCutoff?: number;
+    });
+    private blend;
+    score(partialTranscript: string): Promise<number>;
+    signal(partialTranscript: string): Promise<VoiceTurnSignal>;
+}
+/**
+ * Build a composite EOT classifier backed by the fused FFI scorer, or null when
+ * the loaded fused build does not wire the v11 EOT symbol (a pre-v11 library) —
+ * the caller then falls back to a heuristic-only classifier.
+ */
+export declare function tryBuildFusedEotClassifier(options: FfiEotScorerOptions): CompositeEotClassifier | null;
+//# sourceMappingURL=eot-classifier.d.ts.map

package/src/services/voice/eot-classifier.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"eot-classifier.d.ts","sourceRoot":"","sources":["eot-classifier.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAkCG;AAEH,OAAO,KAAK,EACX,oBAAoB,EACpB,sBAAsB,EACtB,MAAM,qBAAqB,CAAC;AAC7B,OAAO,EAAE,eAAe,EAAE,MAAM,qBAAqB,CAAC;AACtD,OAAO,EAAE,YAAY,EAAE,KAAK,mBAAmB,EAAE,MAAM,oBAAoB,CAAC;AAM5E,MAAM,MAAM,gBAAgB,GAAG,OAAO,GAAG,MAAM,GAAG,SAAS,CAAC;AAE5D,MAAM,WAAW,eAAe;IAC/B,kDAAkD;IAClD,oBAAoB,EAAE,MAAM,CAAC;IAC7B;;;;OAIG;IACH,WAAW,EAAE,gBAAgB,CAAC;IAC9B,qDAAqD;IACrD,gBAAgB,EAAE,OAAO,GAAG,IAAI,CAAC;IACjC,kEAAkE;IAClE,MAAM,EACH,WAAW,GACX,uBAAuB,GACvB,iBAAiB,GACjB,QAAQ,GACR,QAAQ,CAAC;IACZ,uDAAuD;IACvD,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,oEAAoE;IACpE,UAAU,EAAE,MAAM,CAAC;IACnB,2DAA2D;IAC3D,SAAS,CAAC,EAAE,MAAM,CAAC;CACnB;AAED;;;GAGG;AACH,MAAM,WAAW,aAAa;IAC7B,gEAAgE;IAChE,KAAK,CAAC,iBAAiB,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC;IAClD,gFAAgF;IAChF,MAAM,CAAC,CAAC,iBAAiB,EAAE,MAAM,GAAG,OAAO,CAAC,eAAe,CAAC,CAAC;CAC7D;AAED,wBAAgB,gBAAgB,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM,CAGtD;AAED,wBAAgB,yBAAyB,CAAC,IAAI,EAAE;IAC/C,WAAW,EAAE,MAAM,CAAC;IACpB,UAAU,EAAE,MAAM,CAAC;IACnB,MAAM,EAAE,eAAe,CAAC,QAAQ,CAAC,CAAC;IAClC,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,SAAS,CAAC,EAAE,MAAM,CAAC;CACnB,GAAG,eAAe,CAkBlB;AAMD;;;;;;;;;;;;GAYG;AACH,qBAAa,sBAAuB,YAAW,aAAa;IAC3D,iEAAiE;IACjE,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,qBAAqB,CAuB1C;IAEH,uEAAuE;IACvE,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,mBAAmB,CAiCxC;IAEH,sEAAsE;IACtE,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,aAAa,CAYnC;IAEF,KAAK,CAAC,iBAAiB,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC;IA0C3C,MAAM,CAAC,iBAAiB,EAAE,MAAM,GAAG,OAAO,CAAC,eAAe,CAAC;CAQjE;AAMD;;;;;;;GAOG;AACH,eAAO,MAAM,iCAAiC,cAAc,CAAC;AAC7D,eAAO,MAAM,mCAAmC,gBAAgB,CAAC;AAEjE,wBAAgB,2BAA2B,CAC1C,MAAM,EAAE,MAAM,GAEZ,OAAO,iCAAiC,GACxC,OAAO,mCAAmC,CAQ5C;AAMD,MAAM,WAAW,0BAA0B;IAC1C;;;;;OAKG;IACH,QAAQ,EAAE,MAAM,CAAC;IACjB;;;OAGG;IACH,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,0CAA0C;IAC1C,KAAK,CAAC,EAAE,MAAM,CAAC;CACf;AAED;;;;;;;GAOG;AACH,qBAAa,mBAAoB,YAAW,aAAa;IACxD,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAS;IAClC,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAS;IACnC,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAS;gBAEnB,IAAI,EAAE,0BAA0B;IAMtC,KAAK,CAAC,iBAAiB,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC;IAIjD,MAAM,CAAC,iBAAiB,EAAE,MAAM,GAAG,OAAO,CAAC,eAAe,CAAC;CAuCjE;AAMD,+EAA+E;AAC/E,eAAO,MAAM,oBAAoB,MAAM,CAAC;AAExC,2FAA2F;AAC3F,eAAO,MAAM,uBAAuB,MAAM,CAAC;AAE3C,qEAAqE;AACrE,eAAO,MAAM,wBAAwB,MAAM,CAAC;AAE5C,kFAAkF;AAClF,eAAO,MAAM,qBAAqB,KAAK,CAAC;AAExC,4FAA4F;AAC5F,eAAO,MAAM,wBAAwB,KAAK,CAAC;AAE3C,kFAAkF;AAClF,eAAO,MAAM,yBAAyB,KAAK,CAAC;AAM5C,YAAY,EAAE,oBAAoB,EAAE,sBAAsB,EAAE,CAAC;AAE7D;;;;;;;;;GASG;AACH,qBAAa,mBAAoB,YAAW,aAAa;IACxD,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAkB;gBAE7B,OAAO,EAAE,sBAAsB,GAAG;QAAE,MAAM,EAAE,eAAe,CAAA;KAAE;IAKnE,KAAK,CAAC,iBAAiB,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC;IAKjD,MAAM,CAAC,iBAAiB,EAAE,MAAM,GAAG,OAAO,CAAC,eAAe,CAAC;IAW3D,OAAO,IAAI,OAAO,CAAC,IAAI,CAAC;CAG9B;AAMD;;;;;;GAMG;AACH,eAAO,MAAM,qCAAqC,MAAM,CAAC;AAEzD;;;;;;;;;;GAUG;AACH,qBAAa,sBAAuB,YAAW,aAAa;IAC3D,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAe;IACrC,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAyB;IACnD,OAAO,CAAC,QAAQ,CAAC,gBAAgB,CAAS;gBAE9B,OAAO,EAAE;QACpB,KAAK,EAAE,YAAY,CAAC;QACpB,SAAS,CAAC,EAAE,sBAAsB,CAAC;QACnC,gBAAgB,CAAC,EAAE,MAAM,CAAC;KAC1B;YAOa,KAAK;IAoBb,KAAK,CAAC,iBAAiB,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC;IAIjD,MAAM,CAAC,iBAAiB,EAAE,MAAM,GAAG,OAAO,CAAC,eAAe,CAAC;CAWjE;AAED;;;;GAIG;AACH,wBAAgB,0BAA0B,CACzC,OAAO,EAAE,mBAAmB,GAC1B,sBAAsB,GAAG,IAAI,CAG/B"}