@elizaos/plugin-local-inference 2.0.0-beta.1 → 2.0.3-beta.3

package/dist/services/vision/types.d.ts

@@ -0,0 +1,162 @@
+/**
+ * Vision-language describe-image types (WS2).
+ *
+ * Two layers live here:
+ *
+ *   1. The **request/result** contract every WS2 backend implements
+ *      (`VisionDescribeRequest`, `VisionDescribeResult`). Callers pass
+ *      raw image bytes + a prompt; backends return a title+description.
+ *
+ *   2. The **backend** interface (`VisionDescribeBackend`) that the
+ *      `MemoryArbiter` registers as a capability handler. One backend
+ *      per binding family (node-llama-cpp / llama-server / AOSP libllama
+ *      shim). All three implement the same `load → describe → unload`
+ *      shape so the arbiter can swap between them without caring how
+ *      the projector is wired underneath.
+ *
+ * Why a separate file: the arbiter's `CapabilityRegistration<TBackend,
+ * TRequest, TResult>` is generic; pinning concrete shapes here keeps
+ * the registration sites short and removes a dozen casts at the
+ * call-site.
+ */
+/**
+ * Channel order for the raw pixel buffer. Most platforms hand us RGBA
+ * (HTMLCanvasElement, Capacitor `Camera`, the desktop `puppeteer-core`
+ * screenshot pipeline). The encoder normalizes internally; this enum
+ * stays so the hash step can pick a stable byte layout that doesn't
+ * depend on the platform-provided buffer order.
+ */
+export type VisionImageChannelOrder = "rgba" | "rgb" | "bgra" | "bgr";
+/**
+ * The raw image data the backend will encode. The arbiter does not see
+ * this — it gets handed straight to the backend's `run()`. The reason
+ * we accept multiple wrappers (URL / base64 / bytes) is that the three
+ * upstream entry points (HTTP route, agent runtime model handler,
+ * computer-use frame loop) each prefer a different shape. The backend
+ * resolves to bytes once.
+ */
+export type VisionImageInput = {
+    kind: "bytes";
+    bytes: Uint8Array;
+    mimeType?: string;
+} | {
+    kind: "base64";
+    base64: string;
+    mimeType?: string;
+} | {
+    kind: "dataUrl";
+    dataUrl: string;
+} | {
+    kind: "url";
+    url: string;
+    mimeType?: string;
+};
+/**
+ * Caller request to `describeImage`. The `modelFamily` distinguishes
+ * projected-token cache entries from different VL families that share
+ * the same hash space — Gemma-VL tokens are not interchangeable with
+ * Florence-2 tokens. Default is `gemma-vl` (the WS2 deliverable);
+ * each additional family registers under its own identifier.
+ */
+export interface VisionDescribeRequest {
+    image: VisionImageInput;
+    prompt?: string;
+    /**
+     * The model family identifier. Used to namespace the projector cache
+     * so swapping the backend's model family invalidates cached tokens.
+     * Defaults to `"gemma-vl"` when omitted.
+     */
+    modelFamily?: string;
+    /** Max output tokens; defaults to 256 (description-length budget). */
+    maxTokens?: number;
+    /** 0..1, default 0.2 (descriptions should be deterministic-ish). */
+    temperature?: number;
+    signal?: AbortSignal;
+    /**
+     * Per-token callback. When set and the backend exposes streaming vision
+     * (the fused ABI-v13 path), the description is decoded token-by-token and
+     * each piece is delivered here as it generates — the same pipe as chat text.
+     * Backends without streaming describe ignore it and return the final result.
+     */
+    onTextChunk?: (chunk: string) => void | Promise<void>;
+    /** Per-step token cap for streaming describe (smaller = finer-grained UI). */
+    maxTokensPerStep?: number;
+}
+/** Backend response — same shape that ImageDescriptionResult expects. */
+export interface VisionDescribeResult {
+    title: string;
+    description: string;
+    /** Best-effort: ms spent in the projector (for arbiter telemetry). */
+    projectorMs?: number;
+    /** Best-effort: ms spent in the decoder. */
+    decodeMs?: number;
+    /** Whether the projected tokens came from the WS1 vision cache. */
+    cacheHit?: boolean;
+}
+/**
+ * Per-load arguments for a vision-describe backend. The arbiter's
+ * `load(modelKey)` only carries an opaque key; the binding resolves
+ * that key to real model+mmproj paths through this struct, which
+ * `createVisionCapabilityRegistration` populates from the catalog.
+ */
+export interface VisionDescribeLoadArgs {
+    /** Absolute path to the text decoder GGUF (the "main" model). */
+    modelPath: string;
+    /** Absolute path to the matching mmproj projector GGUF. */
+    mmprojPath: string;
+    /**
+     * GPU offload preference. The backend translates this to its native
+     * knob: node-llama-cpp `gpuLayers`, llama-server `--n-gpu-layers`,
+     * AOSP libllama shim `eliza_llama_model_params_set_n_gpu_layers`.
+     * `"auto"` lets the binding decide; numeric is honoured verbatim.
+     */
+    gpuLayers?: number | "auto" | "max";
+    /** Max sampled context window in tokens. Defaults to 4096. */
+    contextSize?: number;
+}
+/**
+ * The contract every WS2 backend implements. The shape is intentionally
+ * narrow: the arbiter only ever calls `describe`. `dispose` is wrapped
+ * by the arbiter's `unload` so the backend can free GPU/VRAM and drop
+ * file descriptors on eviction.
+ */
+export interface VisionDescribeBackend {
+    /** Stable identifier — `"capacitor-llama"`, `"llama-server"`, `"aosp"`, or `"fake"` (tests). */
+    readonly id: "capacitor-llama" | "llama-server" | "aosp" | "fake";
+    /**
+     * Run a describe pass. Backends MAY consult an injected projector cache
+     * via `args.projectedTokens` (when the caller's hash already produced
+     * a cache hit) instead of running the projector again; backends that
+     * don't implement projector-token reuse ignore the field.
+     */
+    describe(request: VisionDescribeRequest, args?: VisionDescribeBackendOptions): Promise<VisionDescribeResult>;
+    /** Release the loaded weights. Idempotent. */
+    dispose(): Promise<void>;
+}
+/**
+ * Per-call options the arbiter wrapper passes into the backend. Lives
+ * here (rather than on `VisionDescribeRequest`) so the caller-facing
+ * request type stays free of arbiter implementation details.
+ */
+export interface VisionDescribeBackendOptions {
+    /**
+     * Pre-computed projected tokens from the WS1 vision-embedding cache.
+     * When present the backend SHOULD skip its own projector step and
+     * decode against these tokens directly. Backends that can't do this
+     * still produce a correct result by ignoring the field; the arbiter's
+     * wrapper will measure `cacheHit: false` in that case.
+     */
+    projectedTokens?: {
+        tokens: Float32Array;
+        tokenCount: number;
+        hiddenSize: number;
+    };
+}
+/**
+ * Capability handler load function. The arbiter calls it with a model
+ * key (e.g. `"gemma-vl-4b"`); the implementation resolves to a real
+ * `(modelPath, mmprojPath)` pair from the catalog + installed registry
+ * and returns a live backend.
+ */
+export type VisionDescribeBackendLoader = (modelKey: string) => Promise<VisionDescribeBackend>;
+//# sourceMappingURL=types.d.ts.map

package/dist/services/vision/types.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../../src/services/vision/types.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;GAoBG;AAEH;;;;;;GAMG;AACH,MAAM,MAAM,uBAAuB,GAAG,MAAM,GAAG,KAAK,GAAG,MAAM,GAAG,KAAK,CAAC;AAEtE;;;;;;;GAOG;AACH,MAAM,MAAM,gBAAgB,GACzB;IAAE,IAAI,EAAE,OAAO,CAAC;IAAC,KAAK,EAAE,UAAU,CAAC;IAAC,QAAQ,CAAC,EAAE,MAAM,CAAA;CAAE,GACvD;IAAE,IAAI,EAAE,QAAQ,CAAC;IAAC,MAAM,EAAE,MAAM,CAAC;IAAC,QAAQ,CAAC,EAAE,MAAM,CAAA;CAAE,GACrD;IAAE,IAAI,EAAE,SAAS,CAAC;IAAC,OAAO,EAAE,MAAM,CAAA;CAAE,GACpC;IAAE,IAAI,EAAE,KAAK,CAAC;IAAC,GAAG,EAAE,MAAM,CAAC;IAAC,QAAQ,CAAC,EAAE,MAAM,CAAA;CAAE,CAAC;AAEnD;;;;;;GAMG;AACH,MAAM,WAAW,qBAAqB;IACrC,KAAK,EAAE,gBAAgB,CAAC;IACxB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB;;;;OAIG;IACH,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,sEAAsE;IACtE,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,oEAAoE;IACpE,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,MAAM,CAAC,EAAE,WAAW,CAAC;IACrB;;;;;OAKG;IACH,WAAW,CAAC,EAAE,CAAC,KAAK,EAAE,MAAM,KAAK,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IACtD,8EAA8E;IAC9E,gBAAgB,CAAC,EAAE,MAAM,CAAC;CAC1B;AAED,yEAAyE;AACzE,MAAM,WAAW,oBAAoB;IACpC,KAAK,EAAE,MAAM,CAAC;IACd,WAAW,EAAE,MAAM,CAAC;IACpB,sEAAsE;IACtE,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,4CAA4C;IAC5C,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,mEAAmE;IACnE,QAAQ,CAAC,EAAE,OAAO,CAAC;CACnB;AAED;;;;;GAKG;AACH,MAAM,WAAW,sBAAsB;IACtC,iEAAiE;IACjE,SAAS,EAAE,MAAM,CAAC;IAClB,2DAA2D;IAC3D,UAAU,EAAE,MAAM,CAAC;IACnB;;;;;OAKG;IACH,SAAS,CAAC,EAAE,MAAM,GAAG,MAAM,GAAG,KAAK,CAAC;IACpC,8DAA8D;IAC9D,WAAW,CAAC,EAAE,MAAM,CAAC;CACrB;AAED;;;;;GAKG;AACH,MAAM,WAAW,qBAAqB;IACrC,gGAAgG;IAChG,QAAQ,CAAC,EAAE,EAAE,iBAAiB,GAAG,cAAc,GAAG,MAAM,GAAG,MAAM,CAAC;IAClE;;;;;OAKG;IACH,QAAQ,CACP,OAAO,EAAE,qBAAqB,EAC9B,IAAI,CAAC,EAAE,4BAA4B,GACjC,OAAO,CAAC,oBAAoB,CAAC,CAAC;IACjC,8CAA8C;IAC9C,OAAO,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;CACzB;AAED;;;;GAIG;AACH,MAAM,WAAW,4BAA4B;IAC5C;;;;;;OAMG;IACH,eAAe,CAAC,EAAE;QACjB,MAAM,EAAE,YAAY,CAAC;QACrB,UAAU,EAAE,MAAM,CAAC;QACnB,UAAU,EAAE,MAAM,CAAC;KACnB,CAAC;CACF;AAED;;;;;GAKG;AACH,MAAM,MAAM,2BAA2B,GAAG,CACzC,QAAQ,EAAE,MAAM,KACZ,OAAO,CAAC,qBAAqB,CAAC,CAAC"}

package/dist/services/vision/vast-fallback.d.ts

@@ -0,0 +1,18 @@
+/**
+ * Final optional IMAGE_DESCRIPTION fallback layer.
+ *
+ * This mirrors the cloud wrapper shape but only runs when the previous
+ * handler explicitly returned a typed fallback outcome.
+ */
+import type { ImageDescriptionParams } from "@elizaos/core";
+import { type LocalVisionOutcome, type VisionFallbackReason, type WrappedImageDescriptionHandler } from "./cloud-fallback";
+export interface VisionVastFallbackOptions {
+    enabled?: boolean;
+    apiKey?: string;
+    baseUrl?: string;
+    fetch?: typeof fetch;
+    handler?: (params: ImageDescriptionParams | string, reason: VisionFallbackReason) => Promise<LocalVisionOutcome>;
+    log?: (message: string, detail?: Record<string, unknown>) => void;
+}
+export declare function wrapImageDescriptionHandlerWithVastFallback(previous: WrappedImageDescriptionHandler, options?: VisionVastFallbackOptions): WrappedImageDescriptionHandler;
+//# sourceMappingURL=vast-fallback.d.ts.map

package/dist/services/vision/vast-fallback.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"vast-fallback.d.ts","sourceRoot":"","sources":["../../../src/services/vision/vast-fallback.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH,OAAO,KAAK,EACX,sBAAsB,EAEtB,MAAM,eAAe,CAAC;AACvB,OAAO,EAEN,KAAK,kBAAkB,EAGvB,KAAK,oBAAoB,EACzB,KAAK,8BAA8B,EACnC,MAAM,kBAAkB,CAAC;AAE1B,MAAM,WAAW,yBAAyB;IACzC,OAAO,CAAC,EAAE,OAAO,CAAC;IAClB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,KAAK,CAAC,EAAE,OAAO,KAAK,CAAC;IACrB,OAAO,CAAC,EAAE,CACT,MAAM,EAAE,sBAAsB,GAAG,MAAM,EACvC,MAAM,EAAE,oBAAoB,KACxB,OAAO,CAAC,kBAAkB,CAAC,CAAC;IACjC,GAAG,CAAC,EAAE,CAAC,OAAO,EAAE,MAAM,EAAE,MAAM,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,KAAK,IAAI,CAAC;CAClE;AA+DD,wBAAgB,2CAA2C,CAC1D,QAAQ,EAAE,8BAA8B,EACxC,OAAO,GAAE,yBAA8B,GACrC,8BAA8B,CA8BhC"}

package/dist/services/vision-embedding-cache.d.ts

@@ -0,0 +1,98 @@
+/**
+ * Content-hashed cache for projected vision-language tokens (WS1 deliverable).
+ *
+ * Vision models in the Eliza-1 stack (Gemma-VL, Florence-2, Apothic-VL) all
+ * go through the same expensive projector step: raw pixel
+ * bytes → patch embeddings → projector → tokens that the text decoder
+ * actually consumes. When the user pastes the same screenshot three times
+ * in a row, or when computer-use takes near-duplicate frames of an idle
+ * screen, we want to skip the projector entirely and reuse the cached
+ * tokens.
+ *
+ * Contract:
+ *   - Caller computes a stable hash of the *normalized* input bytes
+ *     (downscaled to the model's input resolution, then SHA-256 of the
+ *     packed pixels). The hash is the cache key.
+ *   - Caller pairs the hash with the projected token tensor (a flat
+ *     `Float32Array` of length `tokens * hiddenSize`) AND the geometry
+ *     `{ tokens, hiddenSize }` so a reader can reshape on the way out.
+ *   - `get(hash)` returns `null` on miss or expiry, the entry on hit.
+ *     A hit also "touches" the entry to keep it warm under LRU.
+ *   - `set(hash, entry, ttlMs?)` inserts with a TTL (default 5 min); if
+ *     the LRU is full, the coldest entry is evicted.
+ *
+ * Why a separate module:
+ *   - The arbiter owns the *model handle*; the cache holds *per-input
+ *     projected weights* that survive across model loads/unloads of the
+ *     same family. Keeping the cache in a sibling module lets the vision
+ *     plugin reuse it even when the arbiter swapped the underlying model
+ *     for memory pressure (the projector tokens are still valid as long
+ *     as the model family + hash match — we encode the family in the key
+ *     to be safe).
+ *
+ * What this is NOT:
+ *   - A blob cache for the encoder *weights*. Those live in mmap regions
+ *     owned by the arbiter / SharedResourceRegistry and are evicted via
+ *     `MmapRegionHandle.evictPages()`.
+ *   - A cache for downstream LLM generations. Prefix-cache for text is
+ *     handled by `cache-bridge.ts` and the backend session pool.
+ */
+export interface VisionEmbeddingEntry {
+    /** Flat row-major buffer: `tokenCount * hiddenSize` floats. */
+    tokens: Float32Array;
+    tokenCount: number;
+    hiddenSize: number;
+    /** True when this entry is still within its TTL. */
+    live: boolean;
+}
+export interface VisionEmbeddingCacheConfig {
+    /** Max entries retained. LRU evicts beyond this. Default 32. */
+    maxEntries: number;
+    /** Default TTL when `set()` is called without one. Default 5 min. */
+    defaultTtlMs: number;
+}
+export declare class VisionEmbeddingCache {
+    private readonly config;
+    /**
+     * `Map` preserves insertion order; we re-insert on hit to bubble entries
+     * to the back, so the first key in iteration order is the LRU candidate.
+     */
+    private readonly entries;
+    private readonly now;
+    constructor(opts?: {
+        config?: Partial<VisionEmbeddingCacheConfig>;
+        now?: () => number;
+    });
+    /**
+     * Lookup. Returns the entry on hit (and refreshes LRU position), or null
+     * on miss / expiry. Expired entries are deleted on read so they don't
+     * silently consume the LRU budget.
+     */
+    get(hash: string): VisionEmbeddingEntry | null;
+    /**
+     * Insert. Replaces any existing entry under the same hash. Evicts the
+     * coldest entry if we're at capacity. `ttlMs` overrides the configured
+     * default; pass 0 to use the default.
+     */
+    set(hash: string, entry: {
+        tokens: Float32Array;
+        tokenCount: number;
+        hiddenSize: number;
+    }, ttlMs?: number): void;
+    /** Diagnostic: current entry count. */
+    size(): number;
+    /** Diagnostic: snapshot of (hash, byteSize, expiresAtMs) for each entry. */
+    snapshot(): ReadonlyArray<{
+        hash: string;
+        bytes: number;
+        expiresAtMs: number;
+    }>;
+    /** Drop everything. Cheap; only releases JS-side refs to the Float32Arrays. */
+    clear(): void;
+    /**
+     * Drop entries whose TTL has expired. Returns the number removed. Cheap
+     * to call from the arbiter's pressure tick.
+     */
+    purgeExpired(nowMs?: number): number;
+}
+//# sourceMappingURL=vision-embedding-cache.d.ts.map

package/dist/services/vision-embedding-cache.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"vision-embedding-cache.d.ts","sourceRoot":"","sources":["../../src/services/vision-embedding-cache.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAsCG;AASH,MAAM,WAAW,oBAAoB;IACpC,+DAA+D;IAC/D,MAAM,EAAE,YAAY,CAAC;IACrB,UAAU,EAAE,MAAM,CAAC;IACnB,UAAU,EAAE,MAAM,CAAC;IACnB,oDAAoD;IACpD,IAAI,EAAE,OAAO,CAAC;CACd;AAED,MAAM,WAAW,0BAA0B;IAC1C,gEAAgE;IAChE,UAAU,EAAE,MAAM,CAAC;IACnB,qEAAqE;IACrE,YAAY,EAAE,MAAM,CAAC;CACrB;AAOD,qBAAa,oBAAoB;IAChC,OAAO,CAAC,QAAQ,CAAC,MAAM,CAA6B;IACpD;;;OAGG;IACH,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAiC;IACzD,OAAO,CAAC,QAAQ,CAAC,GAAG,CAAe;gBAGlC,IAAI,GAAE;QACL,MAAM,CAAC,EAAE,OAAO,CAAC,0BAA0B,CAAC,CAAC;QAC7C,GAAG,CAAC,EAAE,MAAM,MAAM,CAAC;KACd;IAYP;;;;OAIG;IACH,GAAG,CAAC,IAAI,EAAE,MAAM,GAAG,oBAAoB,GAAG,IAAI;IAkB9C;;;;OAIG;IACH,GAAG,CACF,IAAI,EAAE,MAAM,EACZ,KAAK,EAAE;QAAE,MAAM,EAAE,YAAY,CAAC;QAAC,UAAU,EAAE,MAAM,CAAC;QAAC,UAAU,EAAE,MAAM,CAAA;KAAE,EACvE,KAAK,CAAC,EAAE,MAAM,GACZ,IAAI;IAsBP,uCAAuC;IACvC,IAAI,IAAI,MAAM;IAId,4EAA4E;IAC5E,QAAQ,IAAI,aAAa,CAAC;QACzB,IAAI,EAAE,MAAM,CAAC;QACb,KAAK,EAAE,MAAM,CAAC;QACd,WAAW,EAAE,MAAM,CAAC;KACpB,CAAC;IAYF,+EAA+E;IAC/E,KAAK,IAAI,IAAI;IAIb;;;OAGG;IACH,YAAY,CAAC,KAAK,GAAE,MAAmB,GAAG,MAAM;CAUhD"}

package/dist/services/voice/__test-helpers__/fake-ffi.d.ts

@@ -0,0 +1,27 @@
+/**
+ * Test-only `ElizaInferenceFfi` stand-in. Only the methods the voice
+ * pipeline exercises are non-trivial: `asrTranscribe` returns the supplied
+ * fixed transcript; `ttsSynthesize` writes a constant number of samples;
+ * `ttsSynthesizeStream` emits the same PCM as two chunks (one body + one
+ * `isFinal` tail) and honours `onChunk` returning `true` as a cancel.
+ * The ABI-v2 streaming-ASR symbols report "no working decoder" by
+ * default (the same as the C unsupported-build path) so the pipeline routes through the v1
+ * batch path unless a test opts into `asrStreamSupported`. Everything
+ * else is a no-op / identity so a test can wire a "fused" FFI without a
+ * real `.dylib`.
+ */
+import type { ElizaInferenceFfi } from "../ffi-bindings";
+export declare function fakeFfi(transcript: string, opts?: {
+    ttsSamples?: number;
+    ttsStreamSupported?: boolean;
+    asrStreamSupported?: boolean;
+    vadSupported?: boolean;
+    vadProbs?: readonly number[];
+    speakerSupported?: boolean;
+    /** The 256-float embedding `speakerEmbed` returns (defaults to zeros). */
+    speakerEmbedding?: Float32Array;
+    diarizSupported?: boolean;
+    /** The per-frame label sequence `diarizSegment` returns. */
+    diarizLabels?: Int8Array;
+}): ElizaInferenceFfi;
+//# sourceMappingURL=fake-ffi.d.ts.map

package/dist/services/voice/__test-helpers__/fake-ffi.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"fake-ffi.d.ts","sourceRoot":"","sources":["../../../../src/services/voice/__test-helpers__/fake-ffi.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;GAWG;AACH,OAAO,KAAK,EAAE,iBAAiB,EAAkB,MAAM,iBAAiB,CAAC;AAEzE,wBAAgB,OAAO,CACtB,UAAU,EAAE,MAAM,EAClB,IAAI,GAAE;IACL,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,kBAAkB,CAAC,EAAE,OAAO,CAAC;IAC7B,kBAAkB,CAAC,EAAE,OAAO,CAAC;IAC7B,YAAY,CAAC,EAAE,OAAO,CAAC;IACvB,QAAQ,CAAC,EAAE,SAAS,MAAM,EAAE,CAAC;IAC7B,gBAAgB,CAAC,EAAE,OAAO,CAAC;IAC3B,0EAA0E;IAC1E,gBAAgB,CAAC,EAAE,YAAY,CAAC;IAChC,eAAe,CAAC,EAAE,OAAO,CAAC;IAC1B,4DAA4D;IAC5D,YAAY,CAAC,EAAE,SAAS,CAAC;CACpB,GACJ,iBAAiB,CAgEnB"}

package/dist/services/voice/__test-helpers__/synthetic-speech.d.ts

@@ -0,0 +1,66 @@
+/**
+ * Deterministic-ish speech-like audio generator for VAD/wake-word smoke
+ * tests. Pure synthesis (glottal pulse train through a three-formant
+ * resonator bank with a syllable-rate amplitude envelope and mild f0
+ * jitter) — close enough to real speech in the time/frequency domain that
+ * the Silero VAD reads it as speech, without shipping a recorded WAV.
+ *
+ * `silence + speech + silence` is the canonical smoke fixture: the VAD
+ * should detect exactly one speech segment whose boundaries land inside
+ * the voiced region, and `VadDetector` should drop the leading/trailing
+ * silence windows from its speech-state timeline.
+ */
+export interface SpeechFixtureOptions {
+    sampleRate?: number;
+    /** Seconds of leading silence. */
+    leadSilenceSec?: number;
+    /** Seconds of synthesized speech. */
+    speechSec?: number;
+    /** Seconds of trailing silence. */
+    tailSilenceSec?: number;
+    /** Deterministic seed for the f0 jitter. */
+    seed?: number;
+    /**
+     * Per-speaker voice colour. Two speakers with different timbres have
+     * measurably different spectral envelopes, so an acoustic diarizer can tell
+     * them apart from the audio alone. Omit for the default (shared) voice — the
+     * VAD/wake-word smoke fixtures don't care who is speaking.
+     */
+    timbre?: SpeakerTimbre;
+}
+/** A speaker's voice colour: fundamental frequency + vocal-tract formants. */
+export interface SpeakerTimbre {
+    /** Base fundamental frequency (Hz) — speaker pitch. */
+    f0Hz: number;
+    /** Three `[centerHz, bandwidthHz]` formants — the vocal-tract resonances. */
+    formants: ReadonlyArray<readonly [number, number]>;
+}
+export interface SpeechFixture {
+    pcm: Float32Array;
+    sampleRate: number;
+    speechStartSample: number;
+    speechEndSample: number;
+}
+/** The shared (speaker-agnostic) voice used when no `timbre` is supplied. */
+export declare const DEFAULT_SPEAKER_TIMBRE: SpeakerTimbre;
+/**
+ * Deterministic, distinct voice colour for participant `index` of `count`
+ * speakers in a scenario. The speakers are spread EVENLY across a wide
+ * vocal-tract-length (formant-scaling) and pitch range, so every pair in a
+ * scenario is acoustically far apart — a blind acoustic diarizer can split them
+ * from the audio alone — while one participant always gets one timbre, so the
+ * same speaker clusters together. Spreading by position (not by a label hash)
+ * guarantees the separation; two labels could otherwise hash to near-identical
+ * voices and merge.
+ */
+export declare function speakerTimbreForIndex(index: number, count: number): SpeakerTimbre;
+/**
+ * The agent's own synthetic TTS voice — a fixed timbre, deliberately placed
+ * outside the speaker-seed range so it is acoustically distinct from every
+ * scenario participant. The corpus synthesizes agent-echo turns with this voice,
+ * and the acoustic self-voice gate enrolls it as the agent's imprint.
+ */
+export declare const AGENT_VOICE_TIMBRE: SpeakerTimbre;
+/** Build a `silence + synthesized speech + silence` PCM buffer. */
+export declare function makeSpeechWithSilenceFixture(opts?: SpeechFixtureOptions): SpeechFixture;
+//# sourceMappingURL=synthetic-speech.d.ts.map

package/dist/services/voice/__test-helpers__/synthetic-speech.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"synthetic-speech.d.ts","sourceRoot":"","sources":["../../../../src/services/voice/__test-helpers__/synthetic-speech.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;GAWG;AAEH,MAAM,WAAW,oBAAoB;IACpC,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,kCAAkC;IAClC,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,qCAAqC;IACrC,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,mCAAmC;IACnC,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,4CAA4C;IAC5C,IAAI,CAAC,EAAE,MAAM,CAAC;IACd;;;;;OAKG;IACH,MAAM,CAAC,EAAE,aAAa,CAAC;CACvB;AAED,8EAA8E;AAC9E,MAAM,WAAW,aAAa;IAC7B,uDAAuD;IACvD,IAAI,EAAE,MAAM,CAAC;IACb,6EAA6E;IAC7E,QAAQ,EAAE,aAAa,CAAC,SAAS,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC,CAAC;CACnD;AAED,MAAM,WAAW,aAAa;IAC7B,GAAG,EAAE,YAAY,CAAC;IAClB,UAAU,EAAE,MAAM,CAAC;IACnB,iBAAiB,EAAE,MAAM,CAAC;IAC1B,eAAe,EAAE,MAAM,CAAC;CACxB;AAyDD,6EAA6E;AAC7E,eAAO,MAAM,sBAAsB,EAAE,aAGpC,CAAC;AAEF;;;;;;;;;GASG;AACH,wBAAgB,qBAAqB,CACpC,KAAK,EAAE,MAAM,EACb,KAAK,EAAE,MAAM,GACX,aAAa,CAef;AAED;;;;;GAKG;AACH,eAAO,MAAM,kBAAkB,EAAE,aAOhC,CAAC;AAEF,mEAAmE;AACnE,wBAAgB,4BAA4B,CAC3C,IAAI,GAAE,oBAAyB,GAC7B,aAAa,CAqCf"}

package/dist/services/voice/acoustic-speaker-attribution.d.ts

@@ -0,0 +1,61 @@
+/**
+ * Model-free acoustic speaker attribution for the Voice Workbench (#9147, #9427).
+ *
+ * The headless decision-logic lane used to copy the ground-truth speaker label
+ * straight into `predictedSpeakerLabel`, so the DER gate compared ground truth
+ * to itself and could never fail (#9427). This module is the real thing it
+ * needed: it derives a speaker label from the AUDIO and nothing else.
+ *
+ *   extractTimbreEmbedding  — a deterministic mean-MFCC voice embedding (the
+ *                             speaker's timbre), via a dependency-free FFT +
+ *                             mel filterbank + DCT. No model, no network.
+ *   OnlineSpeakerClusterer  — blind clustering: each turn is matched to the
+ *                             nearest running speaker centroid by cosine and
+ *                             takes that cluster's id, or seeds a new cluster.
+ *                             It never sees ground truth, so a wrong cluster
+ *                             surfaces as real Diarization Error Rate.
+ *   selfVoiceSimilarity     — cosine of a turn against the agent's own TTS-voice
+ *                             imprint — the acoustic self-echo signal that used
+ *                             to be a hardcoded `0.9`.
+ *
+ * Pure + deterministic (no `Date.now`/`Math.random`), so the gate is byte-stable
+ * in CI yet genuinely able to fail on a misattribution.
+ */
+/**
+ * Cosine at/above which a turn joins an existing speaker cluster. Tuned so that
+ * the same synthetic speaker (even degraded by the corpus's room noise/reverb)
+ * stays in one cluster, while two distinct timbres split — see
+ * `acoustic-speaker-attribution.test.ts`.
+ */
+export declare const DEFAULT_SPEAKER_CLUSTER_THRESHOLD = 0.5;
+/**
+ * Mean-MFCC voice embedding of a mono PCM clip: the average cepstrum over its
+ * voiced frames, L2-normalized. Captures vocal-tract timbre (the speaker), not
+ * the words. Returns a zero vector for silence / too-short audio.
+ */
+export declare function extractTimbreEmbedding(pcm: Float32Array, sampleRate: number): number[];
+/**
+ * Blind online speaker clustering. Each turn embedding is matched to the nearest
+ * running centroid; if the best cosine clears the threshold it joins that
+ * cluster (and updates its centroid), otherwise it seeds a new one. The cluster
+ * ids it returns (`spk0`, `spk1`, …) are label-agnostic — DER maps them onto the
+ * ground-truth speakers optimally — so the clusterer never needs (and never
+ * sees) the true labels.
+ */
+export declare class OnlineSpeakerClusterer {
+    private readonly clusters;
+    private readonly threshold;
+    constructor(threshold?: number);
+    /** Assign an embedding to a cluster id, or `null` if it carries no signal. */
+    assign(embedding: ArrayLike<number>): string | null;
+    /** Embed `pcm` and assign it to a cluster id (or `null` for silence). */
+    assignAudio(pcm: Float32Array, sampleRate: number): string | null;
+}
+/**
+ * Cosine similarity (clamped to 0..1) of a turn's audio against the agent's own
+ * synthetic-voice imprint. High ⇒ the agent is hearing ITSELF (its TTS bled back
+ * into the mic). This is the acoustic self-echo signal the respond-gate consumes
+ * as `selfVoiceSimilarity` — a real measurement, not a constant.
+ */
+export declare function selfVoiceSimilarity(pcm: Float32Array, sampleRate: number): number;
+//# sourceMappingURL=acoustic-speaker-attribution.d.ts.map

package/dist/services/voice/acoustic-speaker-attribution.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"acoustic-speaker-attribution.d.ts","sourceRoot":"","sources":["../../../src/services/voice/acoustic-speaker-attribution.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;GAsBG;AAoBH;;;;;GAKG;AACH,eAAO,MAAM,iCAAiC,MAAM,CAAC;AA0HrD;;;;GAIG;AACH,wBAAgB,sBAAsB,CACrC,GAAG,EAAE,YAAY,EACjB,UAAU,EAAE,MAAM,GAChB,MAAM,EAAE,CA8DV;AAaD;;;;;;;GAOG;AACH,qBAAa,sBAAsB;IAClC,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAwB;IACjD,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAS;gBAEvB,SAAS,GAAE,MAA0C;IAIjE,8EAA8E;IAC9E,MAAM,CAAC,SAAS,EAAE,SAAS,CAAC,MAAM,CAAC,GAAG,MAAM,GAAG,IAAI;IA2BnD,yEAAyE;IACzE,WAAW,CAAC,GAAG,EAAE,YAAY,EAAE,UAAU,EAAE,MAAM,GAAG,MAAM,GAAG,IAAI;CAGjE;AAqBD;;;;;GAKG;AACH,wBAAgB,mBAAmB,CAClC,GAAG,EAAE,YAAY,EACjB,UAAU,EAAE,MAAM,GAChB,MAAM,CAIR"}