@elizaos/ui 2.0.0-alpha.85 → 2.0.0-beta.2

package/services/local-inference/dflash-doctor.d.ts

@@ -0,0 +1,22 @@
+export type DflashDoctorStatus = "pass" | "warn" | "fail";
+export interface DflashDoctorCheck {
+    id: string;
+    status: DflashDoctorStatus;
+    label: string;
+    detail: string;
+    fix?: string;
+}
+export interface DflashDoctorReport {
+    ok: boolean;
+    checks: DflashDoctorCheck[];
+}
+/**
+ * Run the DFlash health check. Reports pass/warn/fail across:
+ *   - DFlash runtime + llama-server binary discovery
+ *   - per-pair installation state (target + drafter)
+ *   - per-pair tokenizer-family parity (drafter must share the target's vocab)
+ *   - if a server is currently loaded, the most recent acceptance-rate
+ *     readout from `/metrics`
+ */
+export declare function runDflashDoctor(): Promise<DflashDoctorReport>;
+//# sourceMappingURL=dflash-doctor.d.ts.map

package/services/local-inference/dflash-doctor.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"dflash-doctor.d.ts","sourceRoot":"","sources":["../../../../../../src/services/local-inference/dflash-doctor.ts"],"names":[],"mappings":"AASA,MAAM,MAAM,kBAAkB,GAAG,MAAM,GAAG,MAAM,GAAG,MAAM,CAAC;AAE1D,MAAM,WAAW,iBAAiB;IAChC,EAAE,EAAE,MAAM,CAAC;IACX,MAAM,EAAE,kBAAkB,CAAC;IAC3B,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,MAAM,CAAC;IACf,GAAG,CAAC,EAAE,MAAM,CAAC;CACd;AAED,MAAM,WAAW,kBAAkB;IACjC,EAAE,EAAE,OAAO,CAAC;IACZ,MAAM,EAAE,iBAAiB,EAAE,CAAC;CAC7B;AAMD;;;;;;;GAOG;AACH,wBAAsB,eAAe,IAAI,OAAO,CAAC,kBAAkB,CAAC,CA8HnE"}

package/services/local-inference/dflash-doctor.js

@@ -0,0 +1,139 @@
+import fs from "node:fs";
+import { findCatalogModel, MODEL_CATALOG } from "./catalog.js";
+import { dflashLlamaServer, getDflashRuntimeStatus, } from "./dflash-server.js";
+import { listInstalledModels } from "./registry.js";
+const DFLASH_TARGET_IDS = MODEL_CATALOG.filter((model) => model.runtime?.dflash).map((model) => model.id);
+/**
+ * Run the DFlash health check. Reports pass/warn/fail across:
+ *   - DFlash runtime + llama-server binary discovery
+ *   - per-pair installation state (target + drafter)
+ *   - per-pair tokenizer-family parity (drafter must share the target's vocab)
+ *   - if a server is currently loaded, the most recent acceptance-rate
+ *     readout from `/metrics`
+ */
+export async function runDflashDoctor() {
+    const checks = [];
+    const runtime = getDflashRuntimeStatus();
+    checks.push({
+        id: "dflash-enabled",
+        status: runtime.enabled ? "pass" : runtime.required ? "fail" : "warn",
+        label: "DFlash runtime",
+        detail: runtime.reason,
+        fix: runtime.enabled
+            ? undefined
+            : "node packages/app-core/scripts/build-llama-cpp-dflash.mjs",
+    });
+    checks.push({
+        id: "llama-server-binary",
+        status: runtime.binaryPath && fs.existsSync(runtime.binaryPath) ? "pass" : "fail",
+        label: "llama-server",
+        detail: runtime.binaryPath ?? "No compatible binary found",
+        fix: "ELIZA_DFLASH_LLAMA_SERVER=/path/to/llama-server",
+    });
+    const installed = await listInstalledModels();
+    const installedIds = new Set(installed.map((model) => model.id));
+    for (const targetId of DFLASH_TARGET_IDS) {
+        const target = findCatalogModel(targetId);
+        const dflash = target?.runtime?.dflash;
+        if (!target || !dflash)
+            continue;
+        const drafter = findCatalogModel(dflash.drafterModelId);
+        const targetInstalled = installedIds.has(target.id);
+        const drafterInstalled = installedIds.has(dflash.drafterModelId);
+        checks.push({
+            id: `${target.id}:target`,
+            status: targetInstalled ? "pass" : "warn",
+            label: `${target.displayName} target`,
+            detail: targetInstalled ? "Installed" : `Download ${target.id}`,
+            fix: `eliza local-inference download ${target.id}`,
+        });
+        checks.push({
+            id: `${target.id}:drafter`,
+            status: drafterInstalled ? "pass" : targetInstalled ? "fail" : "warn",
+            label: `${target.displayName} drafter`,
+            detail: drafterInstalled
+                ? "Installed"
+                : `Missing companion ${dflash.drafterModelId}`,
+            fix: `eliza local-inference download ${dflash.drafterModelId}`,
+        });
+        const targetFamily = target.tokenizerFamily;
+        const drafterFamily = drafter?.tokenizerFamily;
+        if (!drafter) {
+            checks.push({
+                id: `${target.id}:tokenizer`,
+                status: "fail",
+                label: `${target.displayName} tokenizer parity`,
+                detail: `Drafter ${dflash.drafterModelId} not in catalog`,
+                fix: "Add drafter entry to catalog.ts",
+            });
+        }
+        else if (!targetFamily || !drafterFamily) {
+            checks.push({
+                id: `${target.id}:tokenizer`,
+                status: "fail",
+                label: `${target.displayName} tokenizer parity`,
+                detail: `Missing tokenizerFamily — target=${targetFamily ?? "<unset>"} drafter=${drafterFamily ?? "<unset>"}`,
+                fix: "Set tokenizerFamily on both catalog entries",
+            });
+        }
+        else if (targetFamily !== drafterFamily) {
+            checks.push({
+                id: `${target.id}:tokenizer`,
+                status: "fail",
+                label: `${target.displayName} tokenizer parity`,
+                detail: `Mismatch — target=${targetFamily} drafter=${drafterFamily}. Speculative decode requires shared vocab.`,
+                fix: `Repair catalog so drafter ${dflash.drafterModelId} uses tokenizerFamily=${targetFamily}`,
+            });
+        }
+        else {
+            checks.push({
+                id: `${target.id}:tokenizer`,
+                status: "pass",
+                label: `${target.displayName} tokenizer parity`,
+                detail: `Both target and drafter use tokenizerFamily=${targetFamily}`,
+            });
+        }
+    }
+    if (runtime.enabled && dflashLlamaServer.hasLoadedModel()) {
+        let metrics = null;
+        try {
+            metrics = await dflashLlamaServer.getMetrics();
+        }
+        catch {
+            metrics = null;
+        }
+        if (!metrics) {
+            checks.push({
+                id: "acceptance-rate",
+                status: "warn",
+                label: "Acceptance rate",
+                detail: "Server loaded but /metrics returned no speculative counters yet (no recent traffic).",
+            });
+        }
+        else if (metrics.drafted === 0) {
+            checks.push({
+                id: "acceptance-rate",
+                status: "warn",
+                label: "Acceptance rate",
+                detail: "Server loaded but drafter has not produced any tokens yet.",
+            });
+        }
+        else {
+            const rate = metrics.acceptanceRate;
+            const status = rate >= 0.5 ? "pass" : rate >= 0.25 ? "warn" : "fail";
+            checks.push({
+                id: "acceptance-rate",
+                status,
+                label: "Acceptance rate",
+                detail: `${(rate * 100).toFixed(1)}% (drafted=${metrics.drafted}, accepted=${metrics.accepted}, decoded=${metrics.decoded})`,
+                fix: status === "fail"
+                    ? "Acceptance below 25% — verify drafter and target share a tokenizer family and that the drafter is the matched DFlash distill."
+                    : undefined,
+            });
+        }
+    }
+    return {
+        ok: !checks.some((check) => check.status === "fail"),
+        checks,
+    };
+}

package/services/local-inference/dflash-server.d.ts

@@ -0,0 +1,76 @@
+/**
+ * Out-of-process llama-server backend for DFlash speculative decoding.
+ *
+ * DFlash needs llama-server flags (`-md`, `--spec-type dflash`) that the
+ * in-process node-llama-cpp API does not expose. This backend is deliberately
+ * small: spawn a compatible llama-server, wait for health, and use the
+ * OpenAI-compatible chat endpoint so llama-server applies the model chat
+ * template and reasoning controls consistently with LlamaChatSession.
+ */
+export interface DflashServerPlan {
+    targetModelPath: string;
+    drafterModelPath: string;
+    contextSize: number;
+    draftContextSize: number;
+    draftMin: number;
+    draftMax: number;
+    gpuLayers: number | "auto";
+    draftGpuLayers: number | "auto";
+    disableThinking: boolean;
+}
+export interface DflashGenerateArgs {
+    prompt: string;
+    stopSequences?: string[];
+    maxTokens?: number;
+    temperature?: number;
+    topP?: number;
+}
+export interface DflashRuntimeStatus {
+    enabled: boolean;
+    required: boolean;
+    binaryPath: string | null;
+    reason: string;
+}
+export declare function dflashEnabled(): boolean;
+export declare function dflashRequired(): boolean;
+export declare function resolveDflashBinary(): string | null;
+export declare function getDflashRuntimeStatus(): DflashRuntimeStatus;
+/** Cumulative speculative-decoding counters scraped from llama-server `/metrics`. */
+export interface DflashMetricsSnapshot {
+    drafted: number;
+    accepted: number;
+    decoded: number;
+    acceptanceRate: number;
+}
+/**
+ * Parse the cumulative speculative-decoding counters from llama-server's
+ * Prometheus-format `/metrics` endpoint. Returns null when none of the
+ * expected counters are present (older builds, server started without
+ * `--metrics`, drafter not yet engaged).
+ */
+export declare function parseDflashMetrics(text: string): DflashMetricsSnapshot | null;
+export declare class DflashLlamaServer {
+    private child;
+    private baseUrl;
+    private stderrTail;
+    private loadedPlan;
+    /** Last cumulative metrics scraped from `/metrics` for delta-rate logging. */
+    private lastMetrics;
+    hasLoadedModel(): boolean;
+    currentModelPath(): string | null;
+    start(plan: DflashServerPlan): Promise<void>;
+    stop(): Promise<void>;
+    generate(args: DflashGenerateArgs): Promise<string>;
+    /**
+     * Scrape llama-server's `/metrics` endpoint and return the current
+     * cumulative speculative-decoding counters. Returns null when the
+     * server isn't running, the endpoint isn't reachable, or the response
+     * doesn't contain the expected counters.
+     */
+    getMetrics(): Promise<DflashMetricsSnapshot | null>;
+    private scrapeAndLogAcceptance;
+    private captureLog;
+    private waitUntilReady;
+}
+export declare const dflashLlamaServer: DflashLlamaServer;
+//# sourceMappingURL=dflash-server.d.ts.map

package/services/local-inference/dflash-server.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"dflash-server.d.ts","sourceRoot":"","sources":["../../../../../../src/services/local-inference/dflash-server.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AASH,MAAM,WAAW,gBAAgB;IAC/B,eAAe,EAAE,MAAM,CAAC;IACxB,gBAAgB,EAAE,MAAM,CAAC;IACzB,WAAW,EAAE,MAAM,CAAC;IACpB,gBAAgB,EAAE,MAAM,CAAC;IACzB,QAAQ,EAAE,MAAM,CAAC;IACjB,QAAQ,EAAE,MAAM,CAAC;IACjB,SAAS,EAAE,MAAM,GAAG,MAAM,CAAC;IAC3B,cAAc,EAAE,MAAM,GAAG,MAAM,CAAC;IAChC,eAAe,EAAE,OAAO,CAAC;CAC1B;AAED,MAAM,WAAW,kBAAkB;IACjC,MAAM,EAAE,MAAM,CAAC;IACf,aAAa,CAAC,EAAE,MAAM,EAAE,CAAC;IACzB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,IAAI,CAAC,EAAE,MAAM,CAAC;CACf;AAED,MAAM,WAAW,mBAAmB;IAClC,OAAO,EAAE,OAAO,CAAC;IACjB,QAAQ,EAAE,OAAO,CAAC;IAClB,UAAU,EAAE,MAAM,GAAG,IAAI,CAAC;IAC1B,MAAM,EAAE,MAAM,CAAC;CAChB;AAoDD,wBAAgB,aAAa,IAAI,OAAO,CAMvC;AAED,wBAAgB,cAAc,IAAI,OAAO,CAExC;AAyBD,wBAAgB,mBAAmB,IAAI,MAAM,GAAG,IAAI,CAanD;AAED,wBAAgB,sBAAsB,IAAI,mBAAmB,CA8B5D;AAyED,qFAAqF;AACrF,MAAM,WAAW,qBAAqB;IACpC,OAAO,EAAE,MAAM,CAAC;IAChB,QAAQ,EAAE,MAAM,CAAC;IACjB,OAAO,EAAE,MAAM,CAAC;IAChB,cAAc,EAAE,MAAM,CAAC;CACxB;AAED;;;;;GAKG;AACH,wBAAgB,kBAAkB,CAChC,IAAI,EAAE,MAAM,GACX,qBAAqB,GAAG,IAAI,CAsC9B;AAED,qBAAa,iBAAiB;IAC5B,OAAO,CAAC,KAAK,CAA6B;IAC1C,OAAO,CAAC,OAAO,CAAuB;IACtC,OAAO,CAAC,UAAU,CAAgB;IAClC,OAAO,CAAC,UAAU,CAAiC;IACnD,8EAA8E;IAC9E,OAAO,CAAC,WAAW,CAAsC;IAEzD,cAAc,IAAI,OAAO;IAIzB,gBAAgB,IAAI,MAAM,GAAG,IAAI;IAI3B,KAAK,CAAC,IAAI,EAAE,gBAAgB,GAAG,OAAO,CAAC,IAAI,CAAC;IAyG5C,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC;IAgBrB,QAAQ,CAAC,IAAI,EAAE,kBAAkB,GAAG,OAAO,CAAC,MAAM,CAAC;IAoCzD;;;;;OAKG;IACG,UAAU,IAAI,OAAO,CAAC,qBAAqB,GAAG,IAAI,CAAC;YAO3C,sBAAsB;IAgBpC,OAAO,CAAC,UAAU;YAUJ,cAAc;CAgC7B;AAED,eAAO,MAAM,iBAAiB,mBAA0B,CAAC"}