@elizaos/plugin-local-inference 2.0.0-beta.1 → 2.0.11-beta.7

package/src/services/device-bridge.d.ts

@@ -0,0 +1,188 @@
+/**
+ * Device-bridge: agent-side half of the "inference on the user's phone,
+ * agent in a container" architecture.
+ *
+ * Multi-device aware. Any number of devices can dial in; each `generate`
+ * is routed to the highest-scoring connected device at call time. A phone
+ * and a Mac paired to the same agent → requests go to the Mac; when the
+ * Mac disconnects, new requests fall through to the phone automatically.
+ *
+ * Scoring (higher = preferred):
+ *   - desktop / electrobun: 100 base
+ *   - ios / android:        10 base
+ *   - per GB of total RAM:  +2
+ *   - per GB of VRAM:       +5 (dedicated GPU wins big)
+ *   - has loaded the right model already: +50 (avoid a swap)
+ *
+ * Disconnect tolerance
+ * --------------------
+ * A pending request stays in `pendingGenerates` until either (a) a device
+ * (same or different) returns a matching correlation-id, or (b) the
+ * timeout fires. On any device (re)connect we re-route orphaned
+ * generates to the new best device.
+ *
+ * Durability
+ * ----------
+ * Pending requests are best-effort persisted to a JSON log under
+ * `$ELIZA_STATE_DIR/local-inference/pending-requests.json` so a brief
+ * agent restart doesn't lose the queue. Persistence is async and
+ * non-blocking — failures fall back to in-memory only.
+ */
+import type { Server as HttpServer } from "node:http";
+import type { AgentRuntime } from "@elizaos/core";
+import { type GenerationThroughput } from "@elizaos/shared/local-inference";
+import type { LocalInferenceLoadArgs } from "./active-model";
+interface DeviceCapabilities {
+    platform: "ios" | "android" | "web" | "electrobun" | "desktop";
+    deviceModel: string;
+    machineId?: string;
+    osVersion?: string;
+    isSimulator?: boolean;
+    totalRamGb: number;
+    availableRamGb?: number | null;
+    freeStorageGb?: number | null;
+    cpuCores: number;
+    gpu: {
+        backend: "metal" | "vulkan" | "gpu-delegate" | "cuda";
+        available: boolean;
+        totalVramGb?: number;
+    } | null;
+    gpuSupported?: boolean;
+    lowPowerMode?: boolean;
+    thermalState?: "nominal" | "fair" | "serious" | "critical" | "unknown";
+    mtpSupported?: boolean;
+    mtpReason?: string;
+}
+export interface DeviceSummary {
+    deviceId: string;
+    capabilities: DeviceCapabilities;
+    loadedPath: string | null;
+    connectedSince: string;
+    score: number;
+    activeRequests: number;
+    isPrimary: boolean;
+}
+export interface DeviceBridgeStatus {
+    /** True if any device is currently connected. */
+    connected: boolean;
+    devices: DeviceSummary[];
+    /** Device id of the current best-score device, or null when none. */
+    primaryDeviceId: string | null;
+    /** Total generates/loads/unloads queued (either in-flight or awaiting a device). */
+    pendingRequests: number;
+    deviceId: string | null;
+    capabilities: DeviceCapabilities | null;
+    loadedPath: string | null;
+    connectedSince: string | null;
+}
+/**
+ * One on-device generation's measured resource signal, emitted to
+ * `subscribeGenerationMetrics` listeners after every successful `generateResult`.
+ * The Mobile Resource Workbench folds these into a `DeviceResourceMetrics`
+ * accumulator (prefill/decode tok/s, TTFT, per-tier aggregation). All
+ * throughput fields are `null` when the device could not measure the inputs.
+ */
+export interface DeviceGenerationMetrics {
+    deviceId: string;
+    platform: DeviceCapabilities["platform"] | null;
+    /** Device model identifier (e.g. `iPhone17,2`) for per-device baselines. */
+    deviceModel: string | null;
+    promptTokens: number;
+    outputTokens: number;
+    durationMs: number;
+    ttftMs: number | null;
+    throughput: GenerationThroughput;
+}
+export declare class DeviceBridge {
+    private readonly devices;
+    private wss;
+    private restored;
+    private readonly pendingLoads;
+    private readonly pendingUnloads;
+    private readonly pendingGenerates;
+    private readonly pendingEmbeds;
+    private readonly statusListeners;
+    private readonly generationMetricsListeners;
+    /** The most recent successful generation's metrics, or null. */
+    private lastGenerationMetrics;
+    /** Bounded ring buffer of recent generation metrics for the dev endpoint. */
+    private readonly recentGenerations;
+    private static readonly RECENT_GENERATIONS_CAP;
+    private readonly expectedPairingToken;
+    status(): DeviceBridgeStatus;
+    private countRouted;
+    subscribeStatus(listener: (status: DeviceBridgeStatus) => void): () => void;
+    private emitStatus;
+    /**
+     * Subscribe to per-generation throughput metrics. Fires once per successful
+     * on-device generation with the differenced prefill/decode tok/s. Returns an
+     * unsubscribe function.
+     */
+    subscribeGenerationMetrics(listener: (metrics: DeviceGenerationMetrics) => void): () => void;
+    /** The most recent successful generation's measured metrics, or null. */
+    latestGenerationMetrics(): DeviceGenerationMetrics | null;
+    /** Most recent generation metrics (newest last), capped at `limit`. */
+    recentGenerationMetrics(limit?: number): DeviceGenerationMetrics[];
+    private emitGenerationMetrics;
+    attachToHttpServer(server: HttpServer): Promise<void>;
+    private handleConnection;
+    private onDeviceRegistered;
+    private onDeviceDisconnected;
+    private handleDeviceMessage;
+    private sendToDevice;
+    /** Highest-scoring connected device, optionally boosted for an already-loaded model. */
+    private pickBestDevice;
+    loadModel(args: LocalInferenceLoadArgs): Promise<void>;
+    unloadModel(): Promise<void>;
+    currentModelPath(): string | null;
+    embed(args: {
+        input: string;
+    }): Promise<{
+        embedding: number[];
+        tokens: number;
+    }>;
+    generate(args: {
+        prompt: string;
+        stopSequences?: string[];
+        maxTokens?: number;
+        temperature?: number;
+        cacheKey?: string;
+    }): Promise<string>;
+    private pendingLogPath;
+    /**
+     * Rewrite the pending-generate log. Called after every mutation to the
+     * pendingGenerates map. We only persist `generate` — loads/unloads are
+     * bound to a specific device's current state and aren't safely replayable
+     * across restart.
+     */
+    private persistPendingGenerates;
+    /**
+     * On startup, read persisted pending requests back into memory. Their
+     * promises are gone (the original caller's process is dead) so they can
+     * only be resolved externally, so we re-queue them with a fresh timeout.
+     * The first connected device that can handle generation will process them.
+     * If nothing consumes them within the timeout they reject quietly.
+     *
+     * Stale entries older than 24h are purged rather than resurrected.
+     */
+    private restorePendingGenerates;
+}
+export declare const deviceBridge: DeviceBridge;
+/** Shape returned by `GET /api/dev/device-resource-metrics`. */
+export interface DeviceResourceMetricsDevPayload {
+    generatedAtEpochMs: number;
+    status: DeviceBridgeStatus;
+    latest: DeviceGenerationMetrics | null;
+    recentGenerations: DeviceGenerationMetrics[];
+}
+/**
+ * Build the JSON body for `GET /api/dev/device-resource-metrics` — the Mobile
+ * Resource Workbench reads this to harvest per-generation prefill/decode tok/s
+ * (already differenced by the bridge) without driving the device WebView.
+ */
+export declare function buildDeviceResourceMetricsDevPayload(bridge?: DeviceBridge, limit?: number): DeviceResourceMetricsDevPayload;
+export declare function registerDeviceBridgeLoader(runtime: AgentRuntime & {
+    registerService?: (name: string, impl: unknown) => unknown;
+}): void;
+export {};
+//# sourceMappingURL=device-bridge.d.ts.map

package/src/services/device-bridge.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"device-bridge.d.ts","sourceRoot":"","sources":["device-bridge.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA6BG;AAIH,OAAO,KAAK,EAAE,MAAM,IAAI,UAAU,EAAmB,MAAM,WAAW,CAAC;AAGvE,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,eAAe,CAAC;AAElD,OAAO,EAEN,KAAK,oBAAoB,EACzB,MAAM,iCAAiC,CAAC;AACzC,OAAO,KAAK,EACX,sBAAsB,EAEtB,MAAM,gBAAgB,CAAC;AAQxB,UAAU,kBAAkB;IAC3B,QAAQ,EAAE,KAAK,GAAG,SAAS,GAAG,KAAK,GAAG,YAAY,GAAG,SAAS,CAAC;IAC/D,WAAW,EAAE,MAAM,CAAC;IACpB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,WAAW,CAAC,EAAE,OAAO,CAAC;IACtB,UAAU,EAAE,MAAM,CAAC;IACnB,cAAc,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAC/B,aAAa,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAC9B,QAAQ,EAAE,MAAM,CAAC;IACjB,GAAG,EAAE;QACJ,OAAO,EAAE,OAAO,GAAG,QAAQ,GAAG,cAAc,GAAG,MAAM,CAAC;QACtD,SAAS,EAAE,OAAO,CAAC;QACnB,WAAW,CAAC,EAAE,MAAM,CAAC;KACrB,GAAG,IAAI,CAAC;IACT,YAAY,CAAC,EAAE,OAAO,CAAC;IACvB,YAAY,CAAC,EAAE,OAAO,CAAC;IACvB,YAAY,CAAC,EAAE,SAAS,GAAG,MAAM,GAAG,SAAS,GAAG,UAAU,GAAG,SAAS,CAAC;IACvE,YAAY,CAAC,EAAE,OAAO,CAAC;IACvB,SAAS,CAAC,EAAE,MAAM,CAAC;CACnB;AAwLD,MAAM,WAAW,aAAa;IAC7B,QAAQ,EAAE,MAAM,CAAC;IACjB,YAAY,EAAE,kBAAkB,CAAC;IACjC,UAAU,EAAE,MAAM,GAAG,IAAI,CAAC;IAC1B,cAAc,EAAE,MAAM,CAAC;IACvB,KAAK,EAAE,MAAM,CAAC;IACd,cAAc,EAAE,MAAM,CAAC;IACvB,SAAS,EAAE,OAAO,CAAC;CACnB;AAED,MAAM,WAAW,kBAAkB;IAClC,iDAAiD;IACjD,SAAS,EAAE,OAAO,CAAC;IACnB,OAAO,EAAE,aAAa,EAAE,CAAC;IACzB,qEAAqE;IACrE,eAAe,EAAE,MAAM,GAAG,IAAI,CAAC;IAC/B,oFAAoF;IACpF,eAAe,EAAE,MAAM,CAAC;IAGxB,QAAQ,EAAE,MAAM,GAAG,IAAI,CAAC;IACxB,YAAY,EAAE,kBAAkB,GAAG,IAAI,CAAC;IACxC,UAAU,EAAE,MAAM,GAAG,IAAI,CAAC;IAC1B,cAAc,EAAE,MAAM,GAAG,IAAI,CAAC;CAC9B;AAQD;;;;;;GAMG;AACH,MAAM,WAAW,uBAAuB;IACvC,QAAQ,EAAE,MAAM,CAAC;IACjB,QAAQ,EAAE,kBAAkB,CAAC,UAAU,CAAC,GAAG,IAAI,CAAC;IAChD,4EAA4E;IAC5E,WAAW,EAAE,MAAM,GAAG,IAAI,CAAC;IAC3B,YAAY,EAAE,MAAM,CAAC;IACrB,YAAY,EAAE,MAAM,CAAC;IACrB,UAAU,EAAE,MAAM,CAAC;IACnB,MAAM,EAAE,MAAM,GAAG,IAAI,CAAC;IACtB,UAAU,EAAE,oBAAoB,CAAC;CACjC;AAyCD,qBAAa,YAAY;IACxB,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAsC;IAC9D,OAAO,CAAC,GAAG,CAA4B;IACvC,OAAO,CAAC,QAAQ,CAAS;IAEzB,OAAO,CAAC,QAAQ,CAAC,YAAY,CAAkC;IAC/D,OAAO,CAAC,QAAQ,CAAC,cAAc,CAAoC;IACnE,OAAO,CAAC,QAAQ,CAAC,gBAAgB,CAAsC;IACvE,OAAO,CAAC,QAAQ,CAAC,aAAa,CAAmC;IAEjE,OAAO,CAAC,QAAQ,CAAC,eAAe,CAE5B;IAEJ,OAAO,CAAC,QAAQ,CAAC,0BAA0B,CAEvC;IAEJ,gEAAgE;IAChE,OAAO,CAAC,qBAAqB,CAAwC;IAErE,6EAA6E;IAC7E,OAAO,CAAC,QAAQ,CAAC,iBAAiB,CAAiC;IACnE,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,sBAAsB,CAAO;IAErD,OAAO,CAAC,QAAQ,CAAC,oBAAoB,CACmB;IAExD,MAAM,IAAI,kBAAkB;IA0C5B,OAAO,CAAC,WAAW;IAWnB,eAAe,CAAC,QAAQ,EAAE,CAAC,MAAM,EAAE,kBAAkB,KAAK,IAAI,GAAG,MAAM,IAAI;IAO3E,OAAO,CAAC,UAAU;IAWlB;;;;OAIG;IACH,0BAA0B,CACzB,QAAQ,EAAE,CAAC,OAAO,EAAE,uBAAuB,KAAK,IAAI,GAClD,MAAM,IAAI;IAOb,yEAAyE;IACzE,uBAAuB,IAAI,uBAAuB,GAAG,IAAI;IAIzD,uEAAuE;IACvE,uBAAuB,CAAC,KAAK,SAAK,GAAG,uBAAuB,EAAE;IAK9D,OAAO,CAAC,qBAAqB;IAevB,kBAAkB,CAAC,MAAM,EAAE,UAAU,GAAG,OAAO,CAAC,IAAI,CAAC;IA8B3D,OAAO,CAAC,gBAAgB;IAiExB,OAAO,CAAC,kBAAkB;IA6G1B,OAAO,CAAC,oBAAoB;IA2D5B,OAAO,CAAC,mBAAmB;IA2F3B,OAAO,CAAC,YAAY;IAMpB,wFAAwF;IACxF,OAAO,CAAC,cAAc;IAiBhB,SAAS,CAAC,IAAI,EAAE,sBAAsB,GAAG,OAAO,CAAC,IAAI,CAAC;IAsCtD,WAAW,IAAI,OAAO,CAAC,IAAI,CAAC;IAuClC,gBAAgB,IAAI,MAAM,GAAG,IAAI;IAO3B,KAAK,CAAC,IAAI,EAAE;QACjB,KAAK,EAAE,MAAM,CAAC;KACd,GAAG,OAAO,CAAC;QAAE,SAAS,EAAE,MAAM,EAAE,CAAC;QAAC,MAAM,EAAE,MAAM,CAAA;KAAE,CAAC;IA6D9C,QAAQ,CAAC,IAAI,EAAE;QACpB,MAAM,EAAE,MAAM,CAAC;QACf,aAAa,CAAC,EAAE,MAAM,EAAE,CAAC;QACzB,SAAS,CAAC,EAAE,MAAM,CAAC;QACnB,WAAW,CAAC,EAAE,MAAM,CAAC;QACrB,QAAQ,CAAC,EAAE,MAAM,CAAC;KAClB,GAAG,OAAO,CAAC,MAAM,CAAC;IAgEnB,OAAO,CAAC,cAAc;IAItB;;;;;OAKG;YACW,uBAAuB;IAqBrC;;;;;;;;OAQG;YACW,uBAAuB;CA2DrC;AAED,eAAO,MAAM,YAAY,cAAqB,CAAC;AAE/C,gEAAgE;AAChE,MAAM,WAAW,+BAA+B;IAC/C,kBAAkB,EAAE,MAAM,CAAC;IAC3B,MAAM,EAAE,kBAAkB,CAAC;IAC3B,MAAM,EAAE,uBAAuB,GAAG,IAAI,CAAC;IACvC,iBAAiB,EAAE,uBAAuB,EAAE,CAAC;CAC7C;AAED;;;;GAIG;AACH,wBAAgB,oCAAoC,CACnD,MAAM,GAAE,YAA2B,EACnC,KAAK,SAAK,GACR,+BAA+B,CAOjC;AAED,wBAAgB,0BAA0B,CACzC,OAAO,EAAE,YAAY,GAAG;IACvB,eAAe,CAAC,EAAE,CAAC,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,OAAO,KAAK,OAAO,CAAC;CAC3D,GACC,IAAI,CAuBN"}