@elizaos/plugin-local-inference 2.0.0-beta.1 → 2.0.3-beta.2

package/src/services/voice/profile-store.d.ts

@@ -0,0 +1,248 @@
+/**
+ * Content-addressed voice-profile store with hot LRU + cold disk tiers.
+ *
+ * Each profile is one WeSpeaker ResNet34-LM centroid plus running
+ * variance (Welford), consent flags, and an entity binding. Profiles
+ * are content-addressed by `sha256(centroid_bytes)` so duplicate
+ * captures collapse and entity merges are safe.
+ *
+ * Layout under `$ELIZA_STATE_DIR/voice-profiles/`:
+ *
+ *   index.json                   — entityId/cluster index + LRU order
+ *   profiles/vp_<sha>.json       — one record per profile
+ *   audio/vp_<sha>/sample-*.wav  — optional, consent-gated
+ *
+ * The contract:
+ *  - **Hot LRU 30** in-memory records (default `hotCacheSize`).
+ *  - **Cold disk cap 200** (default `coldDiskMax`).
+ *  - `beginMatch()` starts at speech-start and resolves once minSpeechMs
+ *    of audio has been encoded — runs in parallel with ASR.
+ *  - `refine()` uses online running-mean + Welford variance.
+ *  - Profiles with a non-null `entityId` are never auto-evicted.
+ */
+import { type VoiceImprintMatch } from "./speaker-imprint";
+import type { VoiceInputSource } from "./types";
+/** Canonical schema version for `vp_*.json` records. */
+export declare const VOICE_PROFILE_RECORD_SCHEMA_VERSION: "eliza.voice_profile_record.v1";
+export interface VoiceProfileConsentState {
+    attributionAuthorized: boolean;
+    synthesisAuthorized: boolean;
+    grantedAt?: string;
+    grantedBy?: string;
+}
+export interface VoiceProfileAudioRef {
+    sampleId: string;
+    wavSha256: string;
+    durationMs: number;
+    recordedAt: string;
+    referenceText?: string;
+}
+export interface VoiceProfileRecord {
+    schemaVersion: typeof VOICE_PROFILE_RECORD_SCHEMA_VERSION;
+    profileId: string;
+    embeddingModel: string;
+    embeddingDim: number;
+    /** L2-normalized centroid; cosine == dot. */
+    centroid: number[];
+    /** Welford per-dim variance accumulator (M2 / max(1, n-1)). */
+    variance: number[];
+    /** Welford `M2` running sum (per-dim squared diff from running mean). */
+    welfordM2: number[];
+    sampleCount: number;
+    totalDurationMs: number;
+    firstObservedAt: string;
+    lastObservedAt: string;
+    lastRefinedAt: string;
+    entityId: string | null;
+    imprintClusterId: string;
+    confidence: number;
+    consent: VoiceProfileConsentState;
+    audioRefs?: VoiceProfileAudioRef[];
+    metadata?: Record<string, unknown>;
+}
+export interface VoiceProfileStoreOptions {
+    rootDir: string;
+    hotCacheSize?: number;
+    coldDiskMax?: number;
+    matchThreshold?: number;
+    /** Below this we open a new cluster instead of attributing. */
+    unmatchedClusterThreshold?: number;
+}
+export interface VoiceImprintMatchHandle {
+    /** Resolves once minSpeechMs of audio is encoded, or `null` if no match. */
+    result: Promise<VoiceImprintMatch | null>;
+    /** Synchronous polling for the latest match — null until first resolve. */
+    current(): VoiceImprintMatch | null;
+    cancel(): void;
+}
+/**
+ * Welford online variance update. Returns the new (mean, M2, count)
+ * triple so a caller can persist the M2 and derive variance lazily.
+ * `prevMean` is the running mean *before* the observation; the caller
+ * passes the L2-normalized embedding as the observation.
+ */
+export declare function welfordUpdate(args: {
+    count: number;
+    mean: readonly number[];
+    m2: readonly number[];
+    observation: ReadonlyArray<number>;
+}): {
+    mean: number[];
+    m2: number[];
+    count: number;
+};
+export declare function welfordVariance(m2: readonly number[], count: number): number[];
+/**
+ * Reject an observation if its per-dim distance from the centroid is
+ * more than `sigmaThreshold` standard deviations on more than half the
+ * dimensions. Used to drop cough / cross-talk samples that would
+ * corrupt the centroid.
+ */
+export declare function isOutlier(args: {
+    centroid: readonly number[];
+    variance: readonly number[];
+    observation: readonly number[];
+    sigmaThreshold?: number;
+}): boolean;
+export declare class VoiceProfileStore {
+    private readonly hotCacheSize;
+    private readonly coldDiskMax;
+    private readonly matchThreshold;
+    private readonly unmatchedThreshold;
+    private readonly rootDir;
+    private readonly profilesDir;
+    private readonly indexPath;
+    /** Hot cache: profileId → record. Insertion order = LRU order. */
+    private hot;
+    private indexCache;
+    constructor(options: VoiceProfileStoreOptions);
+    get matchThresholdValue(): number;
+    get unmatchedClusterThresholdValue(): number;
+    /** Public for tests / management UI. */
+    get hotCacheSizeValue(): number;
+    get coldDiskMaxValue(): number;
+    init(): Promise<void>;
+    private readIndex;
+    private writeIndex;
+    private profilePath;
+    private readProfileFromDisk;
+    private writeProfileToDisk;
+    /** Touch the LRU order: re-insert at the end. */
+    private touchHot;
+    private upsertIndexEntry;
+    private enforceColdLimit;
+    private ensureLoaded;
+    /** Walk profiles + return the best match above `matchThreshold` (or null). */
+    findBestMatch(args: {
+        embedding: Float32Array;
+        embeddingModel: string;
+    }): Promise<VoiceImprintMatch | null>;
+    private recordToImprintProfile;
+    /**
+     * Speculative match handle. The caller supplies a function that
+     * resolves to a single embedding once `minSpeechMs` of audio is
+     * available. The handle starts the lookup the moment it's
+     * constructed — there is no awaitable for "the encoder finished
+     * before we wanted it to".
+     */
+    beginMatch(args: {
+        embed: () => Promise<{
+            embedding: Float32Array;
+            embeddingModel: string;
+        } | null>;
+        signal?: AbortSignal;
+    }): VoiceImprintMatchHandle;
+    /** Create a new profile from a single capture. */
+    createProfile(args: {
+        centroid: Float32Array;
+        embeddingModel: string;
+        entityId?: string | null;
+        imprintClusterId?: string;
+        confidence: number;
+        durationMs: number;
+        consent?: Partial<VoiceProfileConsentState>;
+        audioRef?: VoiceProfileAudioRef;
+        metadata?: Record<string, unknown>;
+    }): Promise<VoiceProfileRecord>;
+    /**
+     * Fold one new embedding into the existing profile via the online
+     * running mean (sampleCount-weighted) and update Welford variance.
+     * Rejects outliers if `dropOutliers` is true (default).
+     */
+    refine(args: {
+        profileId: string;
+        embedding: Float32Array;
+        durationMs: number;
+        confidence: number;
+        audioRef?: VoiceProfileAudioRef;
+        dropOutliers?: boolean;
+    }): Promise<VoiceProfileRecord | null>;
+    bindEntity(args: {
+        profileId: string;
+        entityId: string;
+        label?: string;
+    }): Promise<VoiceProfileRecord | null>;
+    /**
+     * Merge a metadata patch onto a profile. Keys mapped to `null` are
+     * deleted; other keys overwrite. Used by the management routes for
+     * rename / relationship / retention edits.
+     */
+    updateMetadata(profileId: string, patch: Record<string, unknown>): Promise<VoiceProfileRecord | null>;
+    unbindEntity(profileId: string): Promise<VoiceProfileRecord | null>;
+    get(profileId: string): Promise<VoiceProfileRecord | null>;
+    list(): Promise<VoiceProfileRecord[]>;
+    /** For tests / management — drops a profile. Refuses if entityId is set. */
+    deleteProfile(args: {
+        profileId: string;
+        allowBoundEntity?: boolean;
+    }): Promise<boolean>;
+    /**
+     * Merge `sourceId` into `targetId`: a sample-count-weighted centroid
+     * combine (with the Chan parallel-variance update for Welford M2), union
+     * of audio refs, summed counts/durations, and confidence average. The
+     * target's metadata + entity binding win; an unbound target inherits the
+     * source's `entityId`. The source profile is deleted. Returns the merged
+     * target, or `null` if either profile is missing.
+     *
+     * Refuses when both carry a *different* `entityId` unless
+     * `allowEntityOverwrite` is set — merging two bound identities is a
+     * destructive operation the caller must opt into.
+     */
+    mergeProfiles(args: {
+        sourceId: string;
+        targetId: string;
+        allowEntityOverwrite?: boolean;
+    }): Promise<VoiceProfileRecord | null>;
+    /**
+     * Split the audio samples named by `sampleIds` out of `profileId` into a
+     * new profile. Returns the updated original plus the new split profile,
+     * or `null` if the profile is missing.
+     *
+     * Limitation: per-utterance embeddings are not retained (only the running
+     * centroid + Welford accumulators), so the split cannot re-cluster — the
+     * new profile copies the parent centroid and the split is by *audio sample
+     * assignment* only. Both profiles should be re-refined from fresh captures
+     * to diverge. The new profile is unbound (`entityId: null`) and gets a
+     * fresh imprint cluster.
+     */
+    splitProfile(args: {
+        profileId: string;
+        sampleIds: string[];
+    }): Promise<{
+        original: VoiceProfileRecord;
+        split: VoiceProfileRecord;
+    } | null>;
+}
+/** Snapshot of one observation for downstream attribution code. */
+export interface VoiceProfileObservation {
+    profileId: string;
+    imprintClusterId: string;
+    entityId: string | null;
+    embedding: Float32Array;
+    embeddingModel: string;
+    confidence: number;
+    source?: VoiceInputSource;
+    startMs?: number;
+    endMs?: number;
+}
+//# sourceMappingURL=profile-store.d.ts.map

package/src/services/voice/profile-store.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"profile-store.d.ts","sourceRoot":"","sources":["profile-store.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;GAqBG;AAMH,OAAO,EAGN,KAAK,iBAAiB,EAEtB,MAAM,mBAAmB,CAAC;AAC3B,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,SAAS,CAAC;AAEhD,wDAAwD;AACxD,eAAO,MAAM,mCAAmC,EAC/C,+BAAwC,CAAC;AAE1C,MAAM,WAAW,wBAAwB;IACxC,qBAAqB,EAAE,OAAO,CAAC;IAC/B,mBAAmB,EAAE,OAAO,CAAC;IAC7B,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,SAAS,CAAC,EAAE,MAAM,CAAC;CACnB;AAED,MAAM,WAAW,oBAAoB;IACpC,QAAQ,EAAE,MAAM,CAAC;IACjB,SAAS,EAAE,MAAM,CAAC;IAClB,UAAU,EAAE,MAAM,CAAC;IACnB,UAAU,EAAE,MAAM,CAAC;IACnB,aAAa,CAAC,EAAE,MAAM,CAAC;CACvB;AAED,MAAM,WAAW,kBAAkB;IAClC,aAAa,EAAE,OAAO,mCAAmC,CAAC;IAC1D,SAAS,EAAE,MAAM,CAAC;IAClB,cAAc,EAAE,MAAM,CAAC;IACvB,YAAY,EAAE,MAAM,CAAC;IACrB,6CAA6C;IAC7C,QAAQ,EAAE,MAAM,EAAE,CAAC;IACnB,+DAA+D;IAC/D,QAAQ,EAAE,MAAM,EAAE,CAAC;IACnB,yEAAyE;IACzE,SAAS,EAAE,MAAM,EAAE,CAAC;IACpB,WAAW,EAAE,MAAM,CAAC;IACpB,eAAe,EAAE,MAAM,CAAC;IACxB,eAAe,EAAE,MAAM,CAAC;IACxB,cAAc,EAAE,MAAM,CAAC;IACvB,aAAa,EAAE,MAAM,CAAC;IACtB,QAAQ,EAAE,MAAM,GAAG,IAAI,CAAC;IACxB,gBAAgB,EAAE,MAAM,CAAC;IACzB,UAAU,EAAE,MAAM,CAAC;IACnB,OAAO,EAAE,wBAAwB,CAAC;IAClC,SAAS,CAAC,EAAE,oBAAoB,EAAE,CAAC;IACnC,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CACnC;AAED,MAAM,WAAW,wBAAwB;IACxC,OAAO,EAAE,MAAM,CAAC;IAChB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,+DAA+D;IAC/D,yBAAyB,CAAC,EAAE,MAAM,CAAC;CACnC;AAED,MAAM,WAAW,uBAAuB;IACvC,4EAA4E;IAC5E,MAAM,EAAE,OAAO,CAAC,iBAAiB,GAAG,IAAI,CAAC,CAAC;IAC1C,2EAA2E;IAC3E,OAAO,IAAI,iBAAiB,GAAG,IAAI,CAAC;IACpC,MAAM,IAAI,IAAI,CAAC;CACf;AAiDD;;;;;GAKG;AACH,wBAAgB,aAAa,CAAC,IAAI,EAAE;IACnC,KAAK,EAAE,MAAM,CAAC;IACd,IAAI,EAAE,SAAS,MAAM,EAAE,CAAC;IACxB,EAAE,EAAE,SAAS,MAAM,EAAE,CAAC;IACtB,WAAW,EAAE,aAAa,CAAC,MAAM,CAAC,CAAC;CACnC,GAAG;IAAE,IAAI,EAAE,MAAM,EAAE,CAAC;IAAC,EAAE,EAAE,MAAM,EAAE,CAAC;IAAC,KAAK,EAAE,MAAM,CAAA;CAAE,CAoBlD;AAED,wBAAgB,eAAe,CAC9B,EAAE,EAAE,SAAS,MAAM,EAAE,EACrB,KAAK,EAAE,MAAM,GACX,MAAM,EAAE,CAGV;AAED;;;;;GAKG;AACH,wBAAgB,SAAS,CAAC,IAAI,EAAE;IAC/B,QAAQ,EAAE,SAAS,MAAM,EAAE,CAAC;IAC5B,QAAQ,EAAE,SAAS,MAAM,EAAE,CAAC;IAC5B,WAAW,EAAE,SAAS,MAAM,EAAE,CAAC;IAC/B,cAAc,CAAC,EAAE,MAAM,CAAC;CACxB,GAAG,OAAO,CAYV;AAED,qBAAa,iBAAiB;IAC7B,OAAO,CAAC,QAAQ,CAAC,YAAY,CAAS;IACtC,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAS;IACrC,OAAO,CAAC,QAAQ,CAAC,cAAc,CAAS;IACxC,OAAO,CAAC,QAAQ,CAAC,kBAAkB,CAAS;IAC5C,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAS;IACjC,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAS;IACrC,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAS;IACnC,kEAAkE;IAClE,OAAO,CAAC,GAAG,CAAyC;IACpD,OAAO,CAAC,UAAU,CAA0B;gBAEhC,OAAO,EAAE,wBAAwB;IAe7C,IAAI,mBAAmB,IAAI,MAAM,CAEhC;IAED,IAAI,8BAA8B,IAAI,MAAM,CAE3C;IAED,wCAAwC;IACxC,IAAI,iBAAiB,IAAI,MAAM,CAE9B;IAED,IAAI,gBAAgB,IAAI,MAAM,CAE7B;IAEK,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC;YAQb,SAAS;YAgBT,UAAU;IAOxB,OAAO,CAAC,WAAW;YAKL,mBAAmB;YAWnB,kBAAkB;IAMhC,iDAAiD;IACjD,OAAO,CAAC,QAAQ;YAaF,gBAAgB;YA0BhB,gBAAgB;YA0BhB,YAAY;IAa1B,8EAA8E;IACxE,aAAa,CAAC,IAAI,EAAE;QACzB,SAAS,EAAE,YAAY,CAAC;QACxB,cAAc,EAAE,MAAM,CAAC;KACvB,GAAG,OAAO,CAAC,iBAAiB,GAAG,IAAI,CAAC;IA8BrC,OAAO,CAAC,sBAAsB;IAkB9B;;;;;;OAMG;IACH,UAAU,CAAC,IAAI,EAAE;QAChB,KAAK,EAAE,MAAM,OAAO,CAAC;YACpB,SAAS,EAAE,YAAY,CAAC;YACxB,cAAc,EAAE,MAAM,CAAC;SACvB,GAAG,IAAI,CAAC,CAAC;QACV,MAAM,CAAC,EAAE,WAAW,CAAC;KACrB,GAAG,uBAAuB;IA+B3B,kDAAkD;IAC5C,aAAa,CAAC,IAAI,EAAE;QACzB,QAAQ,EAAE,YAAY,CAAC;QACvB,cAAc,EAAE,MAAM,CAAC;QACvB,QAAQ,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;QACzB,gBAAgB,CAAC,EAAE,MAAM,CAAC;QAC1B,UAAU,EAAE,MAAM,CAAC;QACnB,UAAU,EAAE,MAAM,CAAC;QACnB,OAAO,CAAC,EAAE,OAAO,CAAC,wBAAwB,CAAC,CAAC;QAC5C,QAAQ,CAAC,EAAE,oBAAoB,CAAC;QAChC,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;KACnC,GAAG,OAAO,CAAC,kBAAkB,CAAC;IAwC/B;;;;OAIG;IACG,MAAM,CAAC,IAAI,EAAE;QAClB,SAAS,EAAE,MAAM,CAAC;QAClB,SAAS,EAAE,YAAY,CAAC;QACxB,UAAU,EAAE,MAAM,CAAC;QACnB,UAAU,EAAE,MAAM,CAAC;QACnB,QAAQ,CAAC,EAAE,oBAAoB,CAAC;QAChC,YAAY,CAAC,EAAE,OAAO,CAAC;KACvB,GAAG,OAAO,CAAC,kBAAkB,GAAG,IAAI,CAAC;IA6DhC,UAAU,CAAC,IAAI,EAAE;QACtB,SAAS,EAAE,MAAM,CAAC;QAClB,QAAQ,EAAE,MAAM,CAAC;QACjB,KAAK,CAAC,EAAE,MAAM,CAAC;KACf,GAAG,OAAO,CAAC,kBAAkB,GAAG,IAAI,CAAC;IAkBtC;;;;OAIG;IACG,cAAc,CACnB,SAAS,EAAE,MAAM,EACjB,KAAK,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAC5B,OAAO,CAAC,kBAAkB,GAAG,IAAI,CAAC;IAmB/B,YAAY,CAAC,SAAS,EAAE,MAAM,GAAG,OAAO,CAAC,kBAAkB,GAAG,IAAI,CAAC;IAcnE,GAAG,CAAC,SAAS,EAAE,MAAM,GAAG,OAAO,CAAC,kBAAkB,GAAG,IAAI,CAAC;IAI1D,IAAI,IAAI,OAAO,CAAC,kBAAkB,EAAE,CAAC;IAU3C,4EAA4E;IACtE,aAAa,CAAC,IAAI,EAAE;QACzB,SAAS,EAAE,MAAM,CAAC;QAClB,gBAAgB,CAAC,EAAE,OAAO,CAAC;KAC3B,GAAG,OAAO,CAAC,OAAO,CAAC;IAgBpB;;;;;;;;;;;OAWG;IACG,aAAa,CAAC,IAAI,EAAE;QACzB,QAAQ,EAAE,MAAM,CAAC;QACjB,QAAQ,EAAE,MAAM,CAAC;QACjB,oBAAoB,CAAC,EAAE,OAAO,CAAC;KAC/B,GAAG,OAAO,CAAC,kBAAkB,GAAG,IAAI,CAAC;IAuFtC;;;;;;;;;;;OAWG;IACG,YAAY,CAAC,IAAI,EAAE;QACxB,SAAS,EAAE,MAAM,CAAC;QAClB,SAAS,EAAE,MAAM,EAAE,CAAC;KACpB,GAAG,OAAO,CAAC;QACX,QAAQ,EAAE,kBAAkB,CAAC;QAC7B,KAAK,EAAE,kBAAkB,CAAC;KAC1B,GAAG,IAAI,CAAC;CAmDT;AAED,mEAAmE;AACnE,MAAM,WAAW,uBAAuB;IACvC,SAAS,EAAE,MAAM,CAAC;IAClB,gBAAgB,EAAE,MAAM,CAAC;IACzB,QAAQ,EAAE,MAAM,GAAG,IAAI,CAAC;IACxB,SAAS,EAAE,YAAY,CAAC;IACxB,cAAc,EAAE,MAAM,CAAC;IACvB,UAAU,EAAE,MAAM,CAAC;IACnB,MAAM,CAAC,EAAE,gBAAgB,CAAC;IAC1B,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,KAAK,CAAC,EAAE,MAAM,CAAC;CACf"}