@elizaos/ui 2.0.0-alpha.85 → 2.0.0-beta.2

package/services/local-inference/recommendation.js

@@ -0,0 +1,260 @@
+import { DEFAULT_ELIGIBLE_MODEL_IDS, FIRST_RUN_DEFAULT_MODEL_ID, MODEL_CATALOG, } from "./catalog.js";
+import { assessFit } from "./hardware.js";
+const BYTES_PER_GB = 1024 ** 3;
+/**
+ * Per-platform slot ladders. Every default-recommended entry is an
+ * Eliza-1 tier (the only default-eligible line — see catalog.ts and
+ * `packages/inference/AGENTS.md` §2). Ladders bias toward the smallest
+ * tier that fits the platform; desktops/servers pick larger tiers
+ * first when memory headroom allows.
+ */
+const SLOT_LADDERS = {
+    mobile: {
+        TEXT_SMALL: ["eliza-1-lite-0_6b", "eliza-1-mobile-1_7b"],
+        TEXT_LARGE: ["eliza-1-mobile-1_7b", "eliza-1-lite-0_6b"],
+    },
+    "apple-silicon": {
+        TEXT_SMALL: ["eliza-1-mobile-1_7b", "eliza-1-lite-0_6b"],
+        TEXT_LARGE: [
+            "eliza-1-pro-27b",
+            "eliza-1-desktop-9b",
+            "eliza-1-mobile-1_7b",
+        ],
+    },
+    "linux-gpu": {
+        TEXT_SMALL: ["eliza-1-mobile-1_7b", "eliza-1-lite-0_6b"],
+        TEXT_LARGE: [
+            "eliza-1-server-h200",
+            "eliza-1-pro-27b",
+            "eliza-1-desktop-9b",
+            "eliza-1-mobile-1_7b",
+        ],
+    },
+    "linux-cpu": {
+        TEXT_SMALL: ["eliza-1-mobile-1_7b", "eliza-1-lite-0_6b"],
+        TEXT_LARGE: ["eliza-1-desktop-9b", "eliza-1-mobile-1_7b"],
+    },
+    "desktop-gpu": {
+        TEXT_SMALL: ["eliza-1-mobile-1_7b", "eliza-1-lite-0_6b"],
+        TEXT_LARGE: [
+            "eliza-1-server-h200",
+            "eliza-1-pro-27b",
+            "eliza-1-desktop-9b",
+            "eliza-1-mobile-1_7b",
+        ],
+    },
+    "desktop-cpu": {
+        TEXT_SMALL: ["eliza-1-mobile-1_7b", "eliza-1-lite-0_6b"],
+        TEXT_LARGE: ["eliza-1-desktop-9b", "eliza-1-mobile-1_7b"],
+    },
+};
+function catalogById(catalog) {
+    return new Map(catalog.map((model) => [model.id, model]));
+}
+function chatCandidates(catalog) {
+    return catalog.filter((model) => !model.hiddenFromCatalog && model.runtimeRole !== "dflash-drafter");
+}
+export function classifyRecommendationPlatform(hardware) {
+    const platform = hardware.mobile?.platform ?? hardware.platform;
+    if (platform === "android" || platform === "ios")
+        return "mobile";
+    if (hardware.appleSilicon)
+        return "apple-silicon";
+    if (platform === "linux" && hardware.gpu)
+        return "linux-gpu";
+    if (platform === "linux")
+        return "linux-cpu";
+    if (hardware.gpu)
+        return "desktop-gpu";
+    return "desktop-cpu";
+}
+export function catalogDownloadSizeGb(model, catalog = MODEL_CATALOG) {
+    const byId = catalogById(catalog);
+    return (model.companionModelIds ?? []).reduce((total, companionId) => {
+        const companion = byId.get(companionId);
+        return total + (companion?.sizeGb ?? 0);
+    }, model.sizeGb);
+}
+export function catalogDownloadSizeBytes(model, catalog = MODEL_CATALOG) {
+    return Math.round(catalogDownloadSizeGb(model, catalog) * BYTES_PER_GB);
+}
+function mobileFit(hardware, model, catalog) {
+    const sizeGb = catalogDownloadSizeGb(model, catalog);
+    if (hardware.totalRamGb < model.minRamGb)
+        return "wontfit";
+    if (sizeGb > hardware.totalRamGb * 0.8)
+        return "wontfit";
+    if (sizeGb > hardware.totalRamGb * 0.65)
+        return "tight";
+    return "fits";
+}
+export function assessCatalogModelFit(hardware, model, catalog = MODEL_CATALOG) {
+    if (model.runtime?.dflash) {
+        const byId = catalogById(catalog);
+        if (!byId.has(model.runtime.dflash.drafterModelId))
+            return "wontfit";
+    }
+    if (classifyRecommendationPlatform(hardware) === "mobile") {
+        return mobileFit(hardware, model, catalog);
+    }
+    return assessFit(hardware, catalogDownloadSizeGb(model, catalog), model.minRamGb);
+}
+function canFit(hardware, model, catalog) {
+    return assessCatalogModelFit(hardware, model, catalog) !== "wontfit";
+}
+/**
+ * True when every kernel listed in `model.runtime.optimizations.requiresKernel`
+ * is advertised as `true` in the binary's CAPABILITIES.json kernels map.
+ *
+ * `binaryKernels === null` means we have no probe (older binary, or
+ * llama-server isn't installed). In that case we trust the catalog —
+ * filtering would hide every kernel-required model and the dispatcher's
+ * load-time check will surface the real error if/when the user tries to
+ * activate it.
+ */
+function kernelRequirementsSatisfied(model, binaryKernels) {
+    const required = model.runtime?.optimizations?.requiresKernel ?? [];
+    if (required.length === 0)
+        return true;
+    if (!binaryKernels)
+        return true;
+    return required.every((k) => binaryKernels[k] === true);
+}
+function modelsFromLadder(ids, catalog) {
+    const byId = catalogById(catalog);
+    return ids.flatMap((id) => {
+        const model = byId.get(id);
+        return model ? [model] : [];
+    });
+}
+/**
+ * True when this host has enough memory headroom to serve the long-context
+ * KV cache for a 64k+ window. Threshold mirrors the "16 GB workstation"
+ * line from the porting plan — a 64k context for an 8B model at fp16 KV
+ * occupies ~4 GB; with TurboQuant compression it fits inside 1 GB. Below
+ * 16 GB total we keep the historical short-context preference.
+ *
+ * For GPU hosts we look at total VRAM, since the KV cache lives wherever
+ * the layers do; for CPU-only hosts we look at total RAM.
+ */
+const LONG_CONTEXT_RAM_BUMP_THRESHOLD_GB = 16;
+const LONG_CONTEXT_MIN_LENGTH = 65536;
+function hasLongContextHeadroom(hardware) {
+    const vramGb = hardware.gpu?.totalVramGb ?? 0;
+    if (vramGb >= LONG_CONTEXT_RAM_BUMP_THRESHOLD_GB)
+        return true;
+    return hardware.totalRamGb >= LONG_CONTEXT_RAM_BUMP_THRESHOLD_GB;
+}
+function isLongContextModel(model) {
+    return (typeof model.contextLength === "number" &&
+        model.contextLength >= LONG_CONTEXT_MIN_LENGTH);
+}
+function fallbackCandidates(slot, hardware, catalog) {
+    const candidates = chatCandidates(catalog).filter((model) => DEFAULT_ELIGIBLE_MODEL_IDS.has(model.id) &&
+        canFit(hardware, model, catalog));
+    const preferLongContext = hasLongContextHeadroom(hardware);
+    return candidates.sort((left, right) => {
+        if (preferLongContext) {
+            const leftLong = isLongContextModel(left) ? 1 : 0;
+            const rightLong = isLongContextModel(right) ? 1 : 0;
+            if (leftLong !== rightLong)
+                return rightLong - leftLong;
+        }
+        const sizeDelta = catalogDownloadSizeGb(right, catalog) -
+            catalogDownloadSizeGb(left, catalog);
+        return slot === "TEXT_LARGE" ? sizeDelta : -sizeDelta;
+    });
+}
+export function selectRecommendedModelForSlot(slot, hardware, catalog = MODEL_CATALOG, options = {}) {
+    const platformClass = classifyRecommendationPlatform(hardware);
+    const ladder = modelsFromLadder(SLOT_LADDERS[platformClass][slot], catalog);
+    const binaryKernels = options.binaryKernels ?? null;
+    const eligible = ladder.filter((model) => canFit(hardware, model, catalog) &&
+        kernelRequirementsSatisfied(model, binaryKernels));
+    // On hosts with >= 16 GB RAM/VRAM, give long-context (>= 64k) ladder
+    // entries a small bump so we surface 128k models when they fit. The
+    // ladder order still wins when long-context availability is the same
+    // for every entry (or when the host doesn't have the headroom).
+    const ranked = eligible.length > 0 && hasLongContextHeadroom(hardware)
+        ? rankLadderByLongContext(eligible)
+        : eligible;
+    const alternatives = ranked.length > 0
+        ? ranked
+        : fallbackCandidates(slot, hardware, catalog).filter((model) => kernelRequirementsSatisfied(model, binaryKernels));
+    const model = alternatives[0] ?? null;
+    const fit = model ? assessCatalogModelFit(hardware, model, catalog) : null;
+    return {
+        slot,
+        platformClass,
+        model,
+        fit,
+        reason: model
+            ? `${platformClass} ${slot} ladder selected ${model.id}`
+            : `${platformClass} ${slot} ladder has no fitting catalog model`,
+        alternatives,
+    };
+}
+/**
+ * Stable sort that pulls long-context models toward the front while
+ * preserving relative order within each group. Used only on hosts with
+ * the long-context RAM/VRAM headroom — the ladder order remains the
+ * tie-breaker so DFlash-first preferences survive.
+ */
+function rankLadderByLongContext(ladder) {
+    return ladder
+        .map((model, idx) => ({ model, idx, long: isLongContextModel(model) }))
+        .sort((left, right) => {
+        if (left.long !== right.long)
+            return right.long ? 1 : -1;
+        return left.idx - right.idx;
+    })
+        .map((entry) => entry.model);
+}
+export function selectRecommendedModels(hardware, catalog = MODEL_CATALOG, options = {}) {
+    return {
+        TEXT_SMALL: selectRecommendedModelForSlot("TEXT_SMALL", hardware, catalog, options),
+        TEXT_LARGE: selectRecommendedModelForSlot("TEXT_LARGE", hardware, catalog, options),
+    };
+}
+/**
+ * Pick the model the engine should auto-load on first run when no user
+ * preference exists. Always resolves to an Eliza-1 default-eligible
+ * tier — never a non-Eliza catalog entry, never a HF-search result.
+ *
+ * Resolution order:
+ *   1. `FIRST_RUN_DEFAULT_MODEL_ID` when present in the catalog and in
+ *      the default-eligible set.
+ *   2. The first default-eligible chat entry in the catalog as a
+ *      defensive fallback if the default id is somehow missing
+ *      (catalog lint should prevent this; see catalog.test.ts).
+ *
+ * Returns null only when no default-eligible entry exists at all —
+ * which means the catalog is misconfigured and the caller should
+ * surface a hard error rather than degrade silently.
+ */
+export function recommendForFirstRun(catalog = MODEL_CATALOG) {
+    const byId = catalogById(catalog);
+    const preferred = byId.get(FIRST_RUN_DEFAULT_MODEL_ID);
+    if (preferred && DEFAULT_ELIGIBLE_MODEL_IDS.has(preferred.id))
+        return preferred;
+    return (catalog.find((model) => !model.hiddenFromCatalog &&
+        model.runtimeRole !== "dflash-drafter" &&
+        DEFAULT_ELIGIBLE_MODEL_IDS.has(model.id)) ?? null);
+}
+export function chooseSmallerFallbackModel(currentModelId, hardware, slot = "TEXT_LARGE", catalog = MODEL_CATALOG) {
+    const byId = catalogById(catalog);
+    const current = byId.get(currentModelId);
+    const currentSize = current
+        ? catalogDownloadSizeGb(current, catalog)
+        : Number.POSITIVE_INFINITY;
+    const platformClass = classifyRecommendationPlatform(hardware);
+    const ladderFallback = modelsFromLadder(SLOT_LADDERS[platformClass][slot], catalog)
+        .filter((model) => model.id !== currentModelId)
+        .filter((model) => catalogDownloadSizeGb(model, catalog) < currentSize)
+        .filter((model) => canFit(hardware, model, catalog))[0];
+    if (ladderFallback)
+        return ladderFallback;
+    return (fallbackCandidates(slot, hardware, catalog)
+        .filter((model) => model.id !== currentModelId)
+        .filter((model) => catalogDownloadSizeGb(model, catalog) < currentSize)[0] ?? null);
+}

package/services/local-inference/registry.d.ts

@@ -0,0 +1,35 @@
+/**
+ * On-disk registry of installed models.
+ *
+ * Two sources feed the registry:
+ *   1. Eliza-owned downloads (source: "eliza-download") — written on
+ *      successful completion by the downloader.
+ *   2. External scans (source: "external-scan") — merged in at read time
+ *      from `scanExternalModels()`. These are never persisted to the
+ *      registry file; a rescan runs whenever we read.
+ *
+ * The JSON file only holds Eliza-owned entries. That way, if a user
+ * cleans up LM Studio models we don't show stale ghosts.
+ */
+import type { InstalledModel } from "./types";
+/**
+ * Return all models currently usable: persisted Eliza downloads plus a
+ * fresh external-tool scan. External duplicates of Eliza-owned files are
+ * filtered out by path.
+ */
+export declare function listInstalledModels(): Promise<InstalledModel[]>;
+/** Add or update a Eliza-owned entry. External entries are rejected. */
+export declare function upsertElizaModel(model: InstalledModel): Promise<void>;
+/** Mark an existing Eliza-owned model as most-recently-used. */
+export declare function touchElizaModel(id: string): Promise<void>;
+/**
+ * Delete a Eliza-owned model from the registry and from disk.
+ *
+ * Refuses if the model was discovered from another tool — Eliza must not
+ * touch files it doesn't own. Callers surface that refusal as a 4xx.
+ */
+export declare function removeElizaModel(id: string): Promise<{
+    removed: boolean;
+    reason?: "external" | "not-found";
+}>;
+//# sourceMappingURL=registry.d.ts.map

package/services/local-inference/registry.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"registry.d.ts","sourceRoot":"","sources":["../../../../../../src/services/local-inference/registry.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;GAYG;AAMH,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,SAAS,CAAC;AAmC9C;;;;GAIG;AACH,wBAAsB,mBAAmB,IAAI,OAAO,CAAC,cAAc,EAAE,CAAC,CAcrE;AAED,wEAAwE;AACxE,wBAAsB,gBAAgB,CAAC,KAAK,EAAE,cAAc,GAAG,OAAO,CAAC,IAAI,CAAC,CAyB3E;AAED,gEAAgE;AAChE,wBAAsB,eAAe,CAAC,EAAE,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAM/D;AAED;;;;;GAKG;AACH,wBAAsB,gBAAgB,CAAC,EAAE,EAAE,MAAM,GAAG,OAAO,CAAC;IAC1D,OAAO,EAAE,OAAO,CAAC;IACjB,MAAM,CAAC,EAAE,UAAU,GAAG,WAAW,CAAC;CACnC,CAAC,CA6BD"}

package/services/local-inference/registry.js

@@ -0,0 +1,117 @@
+/**
+ * On-disk registry of installed models.
+ *
+ * Two sources feed the registry:
+ *   1. Eliza-owned downloads (source: "eliza-download") — written on
+ *      successful completion by the downloader.
+ *   2. External scans (source: "external-scan") — merged in at read time
+ *      from `scanExternalModels()`. These are never persisted to the
+ *      registry file; a rescan runs whenever we read.
+ *
+ * The JSON file only holds Eliza-owned entries. That way, if a user
+ * cleans up LM Studio models we don't show stale ghosts.
+ */
+import fs from "node:fs/promises";
+import path from "node:path";
+import { scanExternalModels } from "./external-scanner.js";
+import { isWithinElizaRoot, localInferenceRoot, registryPath } from "./paths.js";
+async function ensureRootDir() {
+    await fs.mkdir(localInferenceRoot(), { recursive: true });
+}
+async function readElizaOwned() {
+    try {
+        const raw = await fs.readFile(registryPath(), "utf8");
+        const parsed = JSON.parse(raw);
+        if (!parsed || parsed.version !== 1 || !Array.isArray(parsed.models)) {
+            return [];
+        }
+        return parsed.models.filter((m) => m && typeof m === "object" && m.source === "eliza-download");
+    }
+    catch {
+        return [];
+    }
+}
+async function writeElizaOwned(models) {
+    await ensureRootDir();
+    const tmp = `${registryPath()}.tmp`;
+    const payload = { version: 1, models };
+    await fs.writeFile(tmp, JSON.stringify(payload, null, 2), "utf8");
+    await fs.rename(tmp, registryPath());
+}
+/**
+ * Return all models currently usable: persisted Eliza downloads plus a
+ * fresh external-tool scan. External duplicates of Eliza-owned files are
+ * filtered out by path.
+ */
+export async function listInstalledModels() {
+    const [owned, external] = await Promise.all([
+        readElizaOwned(),
+        scanExternalModels(),
+    ]);
+    // Filter out Eliza-owned files that also survived a reboot of the local
+    // file and got re-detected by the scanner.
+    const ownedPaths = new Set(owned.map((m) => path.resolve(m.path)));
+    const dedupedExternal = external.filter((m) => !ownedPaths.has(path.resolve(m.path)));
+    return [...owned, ...dedupedExternal];
+}
+/** Add or update a Eliza-owned entry. External entries are rejected. */
+export async function upsertElizaModel(model) {
+    if (model.source !== "eliza-download") {
+        throw new Error("[local-inference] registry only accepts Eliza-owned models");
+    }
+    if (!isWithinElizaRoot(model.path)) {
+        throw new Error("[local-inference] Eliza-owned models must live under the local-inference root");
+    }
+    if (model.bundleRoot && !isWithinElizaRoot(model.bundleRoot)) {
+        throw new Error("[local-inference] Eliza-owned bundle roots must live under the local-inference root");
+    }
+    if (model.manifestPath && !isWithinElizaRoot(model.manifestPath)) {
+        throw new Error("[local-inference] Eliza-owned manifests must live under the local-inference root");
+    }
+    const owned = await readElizaOwned();
+    const withoutCurrent = owned.filter((m) => m.id !== model.id);
+    withoutCurrent.push(model);
+    await writeElizaOwned(withoutCurrent);
+}
+/** Mark an existing Eliza-owned model as most-recently-used. */
+export async function touchElizaModel(id) {
+    const owned = await readElizaOwned();
+    const target = owned.find((m) => m.id === id);
+    if (!target)
+        return;
+    target.lastUsedAt = new Date().toISOString();
+    await writeElizaOwned(owned);
+}
+/**
+ * Delete a Eliza-owned model from the registry and from disk.
+ *
+ * Refuses if the model was discovered from another tool — Eliza must not
+ * touch files it doesn't own. Callers surface that refusal as a 4xx.
+ */
+export async function removeElizaModel(id) {
+    const owned = await readElizaOwned();
+    const target = owned.find((m) => m.id === id);
+    if (!target) {
+        // Check whether it's a known external entry so we can return a
+        // helpful error message instead of 404.
+        const external = await scanExternalModels();
+        if (external.some((m) => m.id === id)) {
+            return { removed: false, reason: "external" };
+        }
+        return { removed: false, reason: "not-found" };
+    }
+    if (!isWithinElizaRoot(target.path)) {
+        return { removed: false, reason: "external" };
+    }
+    const removePath = target.bundleRoot && isWithinElizaRoot(target.bundleRoot)
+        ? target.bundleRoot
+        : target.path;
+    try {
+        await fs.rm(removePath, { recursive: true, force: true });
+    }
+    catch {
+        // If the file was already gone we still want to clear the registry entry.
+    }
+    await writeElizaOwned(owned.filter((m) => m.id !== id));
+    return { removed: true };
+}

package/services/local-inference/routing-policy.d.ts

@@ -0,0 +1,55 @@
+/**
+ * Policy engine + latency + cost tracking for cross-provider routing.
+ *
+ * The policy engine sits on top of the `handlerRegistry` and, given a
+ * model type and a user-selected policy, decides which provider's handler
+ * should serve the next request. The router-handler (registered at top
+ * priority) calls `pickProvider` to make that decision.
+ *
+ * Policies:
+ *   - manual       — honour `preferredProvider`; when no pref set, fall
+ *                    through to the runtime's native priority order
+ *                    (highest registered priority wins).
+ *   - cheapest     — pick the provider with the lowest per-token cost.
+ *   - fastest      — pick the provider with the lowest tracked p50 latency
+ *                    (needs at least a few samples; falls back to native).
+ *   - prefer-local — try local first; if it fails or has no handler,
+ *                    fall through to the next-best non-local.
+ *   - round-robin  — distribute load evenly across eligible providers.
+ *
+ * Latency is tracked in a ring buffer per provider per model type. Cost
+ * is a static table of published per-million-token rates; local providers
+ * are $0. Neither is exact — the goal is "good enough to discriminate"
+ * rather than dollar-accurate billing.
+ */
+import type { HandlerRegistration } from "./handler-registry";
+import type { RoutingPolicy } from "./routing-preferences";
+declare class PolicyEngine {
+    private stats;
+    private statsFor;
+    recordLatency(provider: string, modelType: string, durationMs: number): void;
+    recordPick(provider: string, modelType: string): void;
+    p50(provider: string, modelType: string): number | null;
+    lastPicked(provider: string, modelType: string): number | null;
+    costOf(provider: string): number | null;
+    /**
+     * Pick a provider for this (modelType, policy) given the registry.
+     * Returns the HandlerRegistration whose handler the router-handler
+     * should dispatch to, or null if no eligible handler exists.
+     *
+     * `preferredProvider` is only honoured for policy === "manual".
+     */
+    pickProvider(args: {
+        modelType: string;
+        policy: RoutingPolicy;
+        preferredProvider: string | null;
+        candidates: HandlerRegistration[];
+        /** Provider ID of the router itself — always excluded from candidates. */
+        selfProvider: string;
+    }): HandlerRegistration | null;
+    /** For tests and diagnostics. */
+    snapshot(): Record<string, Record<string, number | null>>;
+}
+export declare const policyEngine: PolicyEngine;
+export {};
+//# sourceMappingURL=routing-policy.d.ts.map

package/services/local-inference/routing-policy.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"routing-policy.d.ts","sourceRoot":"","sources":["../../../../../../src/services/local-inference/routing-policy.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;GAuBG;AAEH,OAAO,KAAK,EAAE,mBAAmB,EAAE,MAAM,oBAAoB,CAAC;AAC9D,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,uBAAuB,CAAC;AA4D3D,cAAM,YAAY;IAChB,OAAO,CAAC,KAAK,CAAmD;IAEhE,OAAO,CAAC,QAAQ;IAShB,aAAa,CAAC,QAAQ,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,GAAG,IAAI;IAU5E,UAAU,CAAC,QAAQ,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,GAAG,IAAI;IAIrD,GAAG,CAAC,QAAQ,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,GAAG,MAAM,GAAG,IAAI;IAIvD,UAAU,CAAC,QAAQ,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,GAAG,MAAM,GAAG,IAAI;IAI9D,MAAM,CAAC,QAAQ,EAAE,MAAM,GAAG,MAAM,GAAG,IAAI;IAQvC;;;;;;OAMG;IACH,YAAY,CAAC,IAAI,EAAE;QACjB,SAAS,EAAE,MAAM,CAAC;QAClB,MAAM,EAAE,aAAa,CAAC;QACtB,iBAAiB,EAAE,MAAM,GAAG,IAAI,CAAC;QACjC,UAAU,EAAE,mBAAmB,EAAE,CAAC;QAClC,0EAA0E;QAC1E,YAAY,EAAE,MAAM,CAAC;KACtB,GAAG,mBAAmB,GAAG,IAAI;IAsE9B,iCAAiC;IACjC,QAAQ,IAAI,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,GAAG,IAAI,CAAC,CAAC;CAW1D;AAED,eAAO,MAAM,YAAY,cAAqB,CAAC"}

package/services/local-inference/routing-policy.js

@@ -0,0 +1,195 @@
+/**
+ * Policy engine + latency + cost tracking for cross-provider routing.
+ *
+ * The policy engine sits on top of the `handlerRegistry` and, given a
+ * model type and a user-selected policy, decides which provider's handler
+ * should serve the next request. The router-handler (registered at top
+ * priority) calls `pickProvider` to make that decision.
+ *
+ * Policies:
+ *   - manual       — honour `preferredProvider`; when no pref set, fall
+ *                    through to the runtime's native priority order
+ *                    (highest registered priority wins).
+ *   - cheapest     — pick the provider with the lowest per-token cost.
+ *   - fastest      — pick the provider with the lowest tracked p50 latency
+ *                    (needs at least a few samples; falls back to native).
+ *   - prefer-local — try local first; if it fails or has no handler,
+ *                    fall through to the next-best non-local.
+ *   - round-robin  — distribute load evenly across eligible providers.
+ *
+ * Latency is tracked in a ring buffer per provider per model type. Cost
+ * is a static table of published per-million-token rates; local providers
+ * are $0. Neither is exact — the goal is "good enough to discriminate"
+ * rather than dollar-accurate billing.
+ */
+const RING_SIZE = 32;
+class RingBuffer {
+    buf = [];
+    push(sample) {
+        this.buf.push(sample);
+        if (this.buf.length > RING_SIZE)
+            this.buf.shift();
+    }
+    p50() {
+        if (this.buf.length === 0)
+            return null;
+        const sorted = [...this.buf].map((s) => s.durationMs).sort((a, b) => a - b);
+        return sorted[Math.floor(sorted.length / 2)] ?? null;
+    }
+    size() {
+        return this.buf.length;
+    }
+}
+/**
+ * Relative per-million-token costs. Keep conservative: the policy only
+ * needs the order to be right for product defaults.
+ * Local / device-bridge = 0 because the user already paid for the hardware.
+ * Subscriptions get a small marginal cost, direct APIs sit above that,
+ * and Eliza Cloud is last because managed fallback is the most expensive
+ * path for the user.
+ */
+const COST_PER_MILLION_TOKENS = {
+    "eliza-local-inference": { input: 0, output: 0 },
+    "eliza-device-bridge": { input: 0, output: 0 },
+    "capacitor-llama": { input: 0, output: 0 },
+    "anthropic-subscription": { input: 0.1, output: 0.1 },
+    "openai-codex": { input: 0.1, output: 0.1 },
+    "openai-subscription": { input: 0.1, output: 0.1 },
+    anthropic: { input: 3, output: 15 },
+    openai: { input: 2.5, output: 10 },
+    grok: { input: 5, output: 15 },
+    google: { input: 1.25, output: 5 },
+    "google-genai": { input: 1.25, output: 5 },
+    moonshot: { input: 1.25, output: 5 },
+    kimi: { input: 1.25, output: 5 },
+    zai: { input: 1.25, output: 5 },
+    glm: { input: 1.25, output: 5 },
+    mistral: { input: 2, output: 6 },
+    elizacloud: { input: 30, output: 60 },
+};
+class PolicyEngine {
+    stats = new Map();
+    statsFor(provider) {
+        let s = this.stats.get(provider);
+        if (!s) {
+            s = { latency: new Map(), lastPicked: new Map() };
+            this.stats.set(provider, s);
+        }
+        return s;
+    }
+    recordLatency(provider, modelType, durationMs) {
+        const s = this.statsFor(provider);
+        let buf = s.latency.get(modelType);
+        if (!buf) {
+            buf = new RingBuffer();
+            s.latency.set(modelType, buf);
+        }
+        buf.push({ durationMs, at: Date.now() });
+    }
+    recordPick(provider, modelType) {
+        this.statsFor(provider).lastPicked.set(modelType, Date.now());
+    }
+    p50(provider, modelType) {
+        return this.statsFor(provider).latency.get(modelType)?.p50() ?? null;
+    }
+    lastPicked(provider, modelType) {
+        return this.statsFor(provider).lastPicked.get(modelType) ?? null;
+    }
+    costOf(provider) {
+        const c = COST_PER_MILLION_TOKENS[provider];
+        if (!c)
+            return null;
+        // Weighted sum (3:1 output:input is a typical chat ratio). Treat missing
+        // output pricing as same as input.
+        return c.input * 0.25 + c.output * 0.75;
+    }
+    /**
+     * Pick a provider for this (modelType, policy) given the registry.
+     * Returns the HandlerRegistration whose handler the router-handler
+     * should dispatch to, or null if no eligible handler exists.
+     *
+     * `preferredProvider` is only honoured for policy === "manual".
+     */
+    pickProvider(args) {
+        const eligible = args.candidates
+            .filter((c) => c.provider !== args.selfProvider)
+            .slice()
+            // Defensive sort — real callers already sort, but test fixtures and
+            // non-registry callers might not, and a silent "pick-wrong" would be
+            // worse than the extra O(n log n).
+            .sort((a, b) => b.priority - a.priority);
+        if (eligible.length === 0)
+            return null;
+        switch (args.policy) {
+            case "manual": {
+                if (args.preferredProvider) {
+                    const match = eligible.find((c) => c.provider === args.preferredProvider);
+                    if (match)
+                        return match;
+                }
+                // Fallback: highest native priority.
+                return eligible[0] ?? null;
+            }
+            case "cheapest": {
+                const ranked = [...eligible].sort((a, b) => {
+                    const ca = this.costOf(a.provider) ?? Number.POSITIVE_INFINITY;
+                    const cb = this.costOf(b.provider) ?? Number.POSITIVE_INFINITY;
+                    if (ca !== cb)
+                        return ca - cb;
+                    return b.priority - a.priority;
+                });
+                return ranked[0] ?? null;
+            }
+            case "fastest": {
+                const ranked = [...eligible].sort((a, b) => {
+                    const la = this.p50(a.provider, args.modelType);
+                    const lb = this.p50(b.provider, args.modelType);
+                    // Untracked providers get Infinity → deprioritised until we
+                    // have samples. First call always falls through to native
+                    // priority via the tie-break.
+                    const va = la ?? Number.POSITIVE_INFINITY;
+                    const vb = lb ?? Number.POSITIVE_INFINITY;
+                    if (va !== vb)
+                        return va - vb;
+                    return b.priority - a.priority;
+                });
+                return ranked[0] ?? null;
+            }
+            case "prefer-local": {
+                const local = eligible.find((c) => c.provider === "eliza-local-inference" ||
+                    c.provider === "capacitor-llama");
+                if (local)
+                    return local;
+                const bridge = eligible.find((c) => c.provider === "eliza-device-bridge");
+                if (bridge)
+                    return bridge;
+                return eligible[0] ?? null;
+            }
+            case "round-robin": {
+                // Pick the one least-recently-picked. Ties broken by priority.
+                const ranked = [...eligible].sort((a, b) => {
+                    const la = this.lastPicked(a.provider, args.modelType) ?? 0;
+                    const lb = this.lastPicked(b.provider, args.modelType) ?? 0;
+                    if (la !== lb)
+                        return la - lb;
+                    return b.priority - a.priority;
+                });
+                return ranked[0] ?? null;
+            }
+        }
+    }
+    /** For tests and diagnostics. */
+    snapshot() {
+        const out = {};
+        for (const [provider, stats] of this.stats) {
+            out[provider] = {};
+            for (const [modelType, buf] of stats.latency) {
+                const row = out[provider];
+                if (row)
+                    row[modelType] = buf.p50();
+            }
+        }
+        return out;
+    }
+}
+export const policyEngine = new PolicyEngine();

package/services/local-inference/routing-preferences.d.ts

@@ -0,0 +1,8 @@
+/**
+ * Re-export of the shared routing-preferences module. The canonical
+ * implementation lives in `@elizaos/shared/local-inference` because both
+ * the server (`@elizaos/app-core`) and the UI client (`@elizaos/ui`)
+ * read/write the same routing.json with identical semantics.
+ */
+export { DEFAULT_ROUTING_POLICY, type RoutingPolicy, type RoutingPreferences, readRoutingPreferences, setPolicy, setPreferredProvider, writeRoutingPreferences, } from "@elizaos/shared";
+//# sourceMappingURL=routing-preferences.d.ts.map