@elizaos/app-core 2.0.0-alpha.85 → 2.0.0-beta.2

package/services/local-inference/hf-search.d.ts

@@ -0,0 +1,19 @@
+/**
+ * HuggingFace Hub search for GGUF models.
+ *
+ * Calls `https://huggingface.co/api/models` with `filter=gguf` to narrow
+ * results to repos that actually ship GGUF quantisations. Each matching
+ * repo is expanded with `/api/models/<repo>` to pick a representative
+ * quant file (preferring Q4_K_M when present). Results are shaped like
+ * `CatalogModel` so the existing ModelCard renders them directly.
+ *
+ * We deliberately do not persist these — they're dynamic, and a curated
+ * entry with the same hfRepo always takes precedence in the UI.
+ */
+import type { CatalogModel } from "./types";
+/**
+ * Search HuggingFace for GGUF repos matching `query`, returning
+ * catalog-shaped entries ready for the Model Hub UI.
+ */
+export declare function searchHuggingFaceGguf(query: string, limit?: number): Promise<CatalogModel[]>;
+//# sourceMappingURL=hf-search.d.ts.map

package/services/local-inference/hf-search.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"hf-search.d.ts","sourceRoot":"","sources":["../../../../../../src/services/local-inference/hf-search.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;GAWG;AAEH,OAAO,KAAK,EAAE,YAAY,EAAe,MAAM,SAAS,CAAC;AAsIzD;;;GAGG;AACH,wBAAsB,qBAAqB,CACzC,KAAK,EAAE,MAAM,EACb,KAAK,SAAK,GACT,OAAO,CAAC,YAAY,EAAE,CAAC,CA+EzB"}

package/services/local-inference/hf-search.js

@@ -0,0 +1,169 @@
+/**
+ * HuggingFace Hub search for GGUF models.
+ *
+ * Calls `https://huggingface.co/api/models` with `filter=gguf` to narrow
+ * results to repos that actually ship GGUF quantisations. Each matching
+ * repo is expanded with `/api/models/<repo>` to pick a representative
+ * quant file (preferring Q4_K_M when present). Results are shaped like
+ * `CatalogModel` so the existing ModelCard renders them directly.
+ *
+ * We deliberately do not persist these — they're dynamic, and a curated
+ * entry with the same hfRepo always takes precedence in the UI.
+ */
+const HF_API = "https://huggingface.co/api";
+const SEARCH_TIMEOUT_MS = 10_000;
+const QUANT_PREFERENCE = [
+    "Q4_K_M",
+    "Q5_K_M",
+    "Q4_0",
+    "Q5_0",
+    "Q3_K_M",
+    "Q8_0",
+    "Q2_K",
+];
+function pickQuantFile(siblings) {
+    const ggufs = siblings.filter((s) => s.rfilename?.toLowerCase().endsWith(".gguf"));
+    if (ggufs.length === 0)
+        return null;
+    for (const quant of QUANT_PREFERENCE) {
+        const match = ggufs.find((s) => s.rfilename?.toUpperCase().includes(quant));
+        if (match)
+            return match;
+    }
+    // No preferred quant — pick the smallest GGUF so we don't surprise the
+    // user with a 40 GB file.
+    return [...ggufs].sort((a, b) => (a.size ?? 0) - (b.size ?? 0))[0] ?? null;
+}
+function extractQuantLabel(filename) {
+    for (const quant of QUANT_PREFERENCE) {
+        if (filename.toUpperCase().includes(quant))
+            return quant;
+    }
+    return "GGUF";
+}
+/**
+ * Very rough parameter-count inference from model name / tags. We use this
+ * only to pick a bucket label — not for any hard memory check.
+ */
+function inferParams(name, tags) {
+    const lower = `${name} ${tags.join(" ")}`.toLowerCase();
+    const sizes = [
+        [/\b32b\b/, "32B", "xl"],
+        [/\b27b\b/, "27B", "large"],
+        [/\b24b\b/, "24B", "large"],
+        [/\b22b\b/, "22B", "large"],
+        [/\b16b\b/, "16B", "large"],
+        [/\b14b\b/, "14B", "large"],
+        [/\b13b\b/, "14B", "large"],
+        [/\b9b\b/, "9B", "mid"],
+        [/\b8b\b/, "8B", "mid"],
+        [/\b7b\b/, "7B", "mid"],
+        [/\b3b\b/, "3B", "small"],
+        [/\b1\.7b\b/, "1.7B", "small"],
+        [/\b1b\b/, "1B", "small"],
+    ];
+    for (const [re, params, bucket] of sizes) {
+        if (re.test(lower))
+            return { params, bucket };
+    }
+    return { params: "7B", bucket: "mid" };
+}
+function inferCategory(tags, pipelineTag) {
+    const lowerTags = tags.map((t) => t.toLowerCase());
+    if (lowerTags.some((t) => t.includes("code") || t.includes("coder"))) {
+        return "code";
+    }
+    if (lowerTags.some((t) => t.includes("reasoning") || t === "math" || t.includes("r1"))) {
+        return "reasoning";
+    }
+    if (lowerTags.some((t) => t.includes("function") || t.includes("tool") || t.includes("hermes"))) {
+        return "tools";
+    }
+    if (pipelineTag === "text-generation") {
+        return "chat";
+    }
+    return "chat";
+}
+async function fetchWithTimeout(url, init) {
+    const controller = new AbortController();
+    const timer = setTimeout(() => controller.abort(), SEARCH_TIMEOUT_MS);
+    try {
+        return await fetch(url, { ...init, signal: controller.signal });
+    }
+    finally {
+        clearTimeout(timer);
+    }
+}
+/**
+ * Search HuggingFace for GGUF repos matching `query`, returning
+ * catalog-shaped entries ready for the Model Hub UI.
+ */
+export async function searchHuggingFaceGguf(query, limit = 12) {
+    const trimmed = query.trim();
+    if (trimmed.length === 0)
+        return [];
+    const searchUrl = new URL(`${HF_API}/models`);
+    searchUrl.searchParams.set("search", trimmed);
+    searchUrl.searchParams.set("filter", "gguf");
+    searchUrl.searchParams.set("limit", String(Math.min(50, Math.max(1, limit * 2))));
+    searchUrl.searchParams.set("sort", "downloads");
+    searchUrl.searchParams.set("direction", "-1");
+    const searchRes = await fetchWithTimeout(searchUrl.toString(), {
+        headers: { accept: "application/json" },
+    });
+    if (!searchRes.ok) {
+        throw new Error(`HuggingFace search failed: HTTP ${searchRes.status}`);
+    }
+    const rawList = (await searchRes.json());
+    const candidates = rawList
+        .map((r) => r.id ?? r.modelId)
+        .filter((id) => typeof id === "string" && id.length > 0)
+        .slice(0, limit);
+    // Parallel detail fetches; failures drop quietly so one bad repo doesn't
+    // take down the whole search.
+    const details = await Promise.all(candidates.map(async (id) => {
+        try {
+            const res = await fetchWithTimeout(`${HF_API}/models/${encodeURIComponent(id)}`, { headers: { accept: "application/json" } });
+            if (!res.ok)
+                return null;
+            return (await res.json());
+        }
+        catch {
+            return null;
+        }
+    }));
+    const results = [];
+    for (let i = 0; i < candidates.length; i++) {
+        const id = candidates[i];
+        const detail = details[i];
+        if (!id || !detail?.siblings)
+            continue;
+        const sibling = pickQuantFile(detail.siblings);
+        if (!sibling?.rfilename)
+            continue;
+        const sizeBytes = sibling.size ?? 0;
+        const sizeGb = sizeBytes > 0 ? sizeBytes / 1024 ** 3 : 4;
+        const { params, bucket } = inferParams(id, detail.tags ?? []);
+        const quant = extractQuantLabel(sibling.rfilename);
+        const category = inferCategory(detail.tags ?? [], detail.pipeline_tag);
+        const displayName = id.split("/").pop() ?? id;
+        // minRam heuristic: 2x the file size, minimum 4 GB. Close enough for
+        // the fit badge; the ModelCard's fit computation still does the work.
+        const minRamGb = Math.max(4, Math.round(sizeGb * 2));
+        results.push({
+            id: `hf:${id}::${sibling.rfilename}`,
+            displayName,
+            hfRepo: id,
+            ggufFile: sibling.rfilename,
+            params,
+            quant,
+            sizeGb: Math.round(sizeGb * 10) / 10,
+            minRamGb,
+            category,
+            bucket,
+            blurb: (detail.tags ?? []).slice(0, 4).join(" · ") ||
+                `${detail.downloads ?? 0} downloads · ${detail.likes ?? 0} likes`,
+        });
+    }
+    return results;
+}

package/services/local-inference/index.d.ts

@@ -0,0 +1,10 @@
+export type { LocalInferenceLoader } from "./active-model";
+export { type BackendDecision, BackendDispatcher, type BackendOverride, type BackendPlan, decideBackend, type EmbedArgs, type EmbedResult, type GenerateArgs as BackendGenerateArgs, type GenerateResult, type LocalInferenceBackend, readBackendOverride, resolveCatalogForPlan, } from "./backend";
+export { ELIZA_1_PLACEHOLDER_IDS, FIRST_RUN_DEFAULT_MODEL_ID, findCatalogModel, MODEL_CATALOG, } from "./catalog";
+export { getDflashRuntimeStatus } from "./dflash-server";
+export { assessFit, probeHardware } from "./hardware";
+export { buildTextGenerationReadiness } from "./readiness";
+export { assessCatalogModelFit, catalogDownloadSizeBytes, catalogDownloadSizeGb, chooseSmallerFallbackModel, classifyRecommendationPlatform, type RecommendationPlatformClass, type RecommendedModelSelection, recommendForFirstRun, selectRecommendedModelForSlot, selectRecommendedModels, } from "./recommendation";
+export { LocalInferenceService, localInferenceService } from "./service";
+export type { ActiveModelState, CatalogModel, DownloadEvent, DownloadJob, DownloadState, HardwareFitLevel, HardwareProbe, InstalledModel, LocalInferenceDownloadStatus, LocalInferenceReadiness, LocalInferenceSlotReadiness, LocalRuntimeAcceleration, LocalRuntimeBackend, LocalRuntimeKernel, LocalRuntimeOptimizations, ModelBucket, ModelCategory, ModelHubSnapshot, TextGenerationSlot, } from "./types";
+//# sourceMappingURL=index.d.ts.map

package/services/local-inference/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../../../../src/services/local-inference/index.ts"],"names":[],"mappings":"AAAA,YAAY,EAAE,oBAAoB,EAAE,MAAM,gBAAgB,CAAC;AAC3D,OAAO,EACL,KAAK,eAAe,EACpB,iBAAiB,EACjB,KAAK,eAAe,EACpB,KAAK,WAAW,EAChB,aAAa,EACb,KAAK,SAAS,EACd,KAAK,WAAW,EAChB,KAAK,YAAY,IAAI,mBAAmB,EACxC,KAAK,cAAc,EACnB,KAAK,qBAAqB,EAC1B,mBAAmB,EACnB,qBAAqB,GACtB,MAAM,WAAW,CAAC;AACnB,OAAO,EACL,uBAAuB,EACvB,0BAA0B,EAC1B,gBAAgB,EAChB,aAAa,GACd,MAAM,WAAW,CAAC;AACnB,OAAO,EAAE,sBAAsB,EAAE,MAAM,iBAAiB,CAAC;AACzD,OAAO,EAAE,SAAS,EAAE,aAAa,EAAE,MAAM,YAAY,CAAC;AACtD,OAAO,EAAE,4BAA4B,EAAE,MAAM,aAAa,CAAC;AAC3D,OAAO,EACL,qBAAqB,EACrB,wBAAwB,EACxB,qBAAqB,EACrB,0BAA0B,EAC1B,8BAA8B,EAC9B,KAAK,2BAA2B,EAChC,KAAK,yBAAyB,EAC9B,oBAAoB,EACpB,6BAA6B,EAC7B,uBAAuB,GACxB,MAAM,kBAAkB,CAAC;AAC1B,OAAO,EAAE,qBAAqB,EAAE,qBAAqB,EAAE,MAAM,WAAW,CAAC;AACzE,YAAY,EACV,gBAAgB,EAChB,YAAY,EACZ,aAAa,EACb,WAAW,EACX,aAAa,EACb,gBAAgB,EAChB,aAAa,EACb,cAAc,EACd,4BAA4B,EAC5B,uBAAuB,EACvB,2BAA2B,EAC3B,wBAAwB,EACxB,mBAAmB,EACnB,kBAAkB,EAClB,yBAAyB,EACzB,WAAW,EACX,aAAa,EACb,gBAAgB,EAChB,kBAAkB,GACnB,MAAM,SAAS,CAAC"}

package/services/local-inference/index.js

@@ -0,0 +1,7 @@
+export { BackendDispatcher, decideBackend, readBackendOverride, resolveCatalogForPlan, } from "./backend";
+export { ELIZA_1_PLACEHOLDER_IDS, FIRST_RUN_DEFAULT_MODEL_ID, findCatalogModel, MODEL_CATALOG, } from "./catalog";
+export { getDflashRuntimeStatus } from "./dflash-server";
+export { assessFit, probeHardware } from "./hardware";
+export { buildTextGenerationReadiness } from "./readiness";
+export { assessCatalogModelFit, catalogDownloadSizeBytes, catalogDownloadSizeGb, chooseSmallerFallbackModel, classifyRecommendationPlatform, recommendForFirstRun, selectRecommendedModelForSlot, selectRecommendedModels, } from "./recommendation";
+export { LocalInferenceService, localInferenceService } from "./service";

package/services/local-inference/llama-server-metrics.d.ts

@@ -0,0 +1,108 @@
+/**
+ * Scraper for llama-server's `/metrics` (Prometheus exposition format)
+ * endpoint. Translates the running counters into the
+ * Anthropic-SDK-shaped `usage` block that callers already know how to
+ * consume from the cloud Anthropic plugin.
+ *
+ * llama-server publishes the following counters (per-process, monotonic):
+ *
+ *   llamacpp:n_decode_total           — context tokens decoded (prefill + gen)
+ *   llamacpp:n_tokens_predicted_total — output tokens
+ *   llamacpp:prompt_tokens_total      — total input tokens accepted
+ *   llamacpp:n_past_max               — high-water mark of cached past-tokens
+ *   llamacpp:n_prompt_tokens_processed_total — fresh tokens prefilled
+ *                                       (i.e. cache MISS), excludes cache hits
+ *   llamacpp:kv_cache_tokens          — current size of KV cache (gauge)
+ *   llamacpp:kv_cache_used_cells      — slots with active KV (gauge)
+ *
+ * For DFlash speculative decoding, the fork additionally publishes:
+ *
+ *   llamacpp:n_drafted_total          — drafter-emitted tokens
+ *   llamacpp:n_accepted_total         — accepted speculative tokens
+ *
+ * The mapping into Anthropic shape:
+ *
+ *   prompt_tokens_total                              → input_tokens
+ *   n_tokens_predicted_total                         → output_tokens
+ *   n_prompt_tokens_processed_total                  → cache_creation_input_tokens
+ *   prompt_tokens_total - n_prompt_tokens_processed_total → cache_read_input_tokens
+ *   n_drafted_total / n_accepted_total               → DFlash extension fields
+ *
+ * Counters are taken as deltas across two snapshots: take one before
+ * `generate`, one after, and subtract. Losing a few samples to process
+ * restart is acceptable — the deltas are useful for the call's own
+ * usage accounting, not for global monitoring.
+ */
+export interface LlamaServerMetricSnapshot {
+    /** Wall-clock ms when the snapshot was taken; useful for diagnostics. */
+    takenAtMs: number;
+    promptTokensTotal: number;
+    predictedTokensTotal: number;
+    /** Tokens that had to be freshly prefilled — i.e. cache MISS this turn. */
+    promptTokensProcessedTotal: number;
+    draftedTotal: number;
+    acceptedTotal: number;
+    /** Current size of the KV cache (gauge). */
+    kvCacheTokens: number;
+    /** Number of slots currently holding active KV (gauge). */
+    kvCacheUsedCells: number;
+}
+/**
+ * Parse a Prometheus exposition-format payload into a metric snapshot.
+ * Unknown or malformed lines are silently skipped — counters we don't
+ * recognise are not interesting and metric exporters add new ones over
+ * time.
+ *
+ * llama-server exposes one sample per metric (no labels), e.g.
+ *   `llamacpp:prompt_tokens_total 1234`
+ * So we only care about the simple `metric_name <number>` form.
+ */
+export declare function parsePrometheusMetrics(body: string, takenAtMs?: number): LlamaServerMetricSnapshot;
+/**
+ * Anthropic-SDK-shaped usage block, optionally extended with DFlash
+ * speculative-decoding metrics. The cloud plugin (plugin-anthropic)
+ * emits the first three fields verbatim; local inference adds the
+ * `dflash_*` fields when speculative decoding is active. Callers that
+ * already handle the cloud `usage` shape need no change.
+ */
+export interface LocalUsageBlock {
+    input_tokens: number;
+    output_tokens: number;
+    cache_creation_input_tokens: number;
+    cache_read_input_tokens: number;
+    dflash_drafted_tokens?: number;
+    dflash_accepted_tokens?: number;
+    /** 0..1 — proportion of drafted tokens that were accepted. */
+    dflash_acceptance_rate?: number;
+    /** 0..1 — proportion of input tokens that hit a warm slot (cache reuse). */
+    cache_hit_rate?: number;
+}
+/**
+ * Compute the Anthropic-shape usage block for a single generation by
+ * differencing two snapshots. `before` is taken just before the request,
+ * `after` just after the response was received. Negative deltas (caused
+ * by a metric reset between snapshots, e.g. server restart) are clamped
+ * to 0 — losing the sample is preferable to surfacing nonsense to the
+ * caller.
+ *
+ * Pass `responseUsage` to override input/output counts when the response
+ * payload itself reports per-call counters that are more accurate than
+ * the metric delta — llama-server's chat completion response includes
+ * `usage.{prompt,completion}_tokens` per request, which is exact while
+ * the metric delta is "everything that happened during the wall-clock
+ * window of the request."
+ */
+export declare function diffSnapshots(before: LlamaServerMetricSnapshot, after: LlamaServerMetricSnapshot, responseUsage?: {
+    prompt_tokens?: number;
+    completion_tokens?: number;
+}): LocalUsageBlock;
+/**
+ * GET `/metrics` from a running llama-server and parse it. Errors fall
+ * back to a zero-valued snapshot rather than throwing — observability
+ * MUST NOT break generation. Callers that want to detect scrape failures
+ * should compare `before.takenAtMs` against `after.takenAtMs`: a zero
+ * snapshot pair (both fields all-zero) means scraping returned nothing
+ * useful.
+ */
+export declare function fetchMetricsSnapshot(baseUrl: string, signal?: AbortSignal): Promise<LlamaServerMetricSnapshot>;
+//# sourceMappingURL=llama-server-metrics.d.ts.map

package/services/local-inference/llama-server-metrics.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"llama-server-metrics.d.ts","sourceRoot":"","sources":["../../../../../../src/services/local-inference/llama-server-metrics.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAkCG;AAEH,MAAM,WAAW,yBAAyB;IACxC,yEAAyE;IACzE,SAAS,EAAE,MAAM,CAAC;IAClB,iBAAiB,EAAE,MAAM,CAAC;IAC1B,oBAAoB,EAAE,MAAM,CAAC;IAC7B,2EAA2E;IAC3E,0BAA0B,EAAE,MAAM,CAAC;IACnC,YAAY,EAAE,MAAM,CAAC;IACrB,aAAa,EAAE,MAAM,CAAC;IACtB,4CAA4C;IAC5C,aAAa,EAAE,MAAM,CAAC;IACtB,2DAA2D;IAC3D,gBAAgB,EAAE,MAAM,CAAC;CAC1B;AAYD;;;;;;;;;GASG;AACH,wBAAgB,sBAAsB,CACpC,IAAI,EAAE,MAAM,EACZ,SAAS,GAAE,MAAmB,GAC7B,yBAAyB,CA8B3B;AAED;;;;;;GAMG;AACH,MAAM,WAAW,eAAe;IAC9B,YAAY,EAAE,MAAM,CAAC;IACrB,aAAa,EAAE,MAAM,CAAC;IACtB,2BAA2B,EAAE,MAAM,CAAC;IACpC,uBAAuB,EAAE,MAAM,CAAC;IAChC,qBAAqB,CAAC,EAAE,MAAM,CAAC;IAC/B,sBAAsB,CAAC,EAAE,MAAM,CAAC;IAChC,8DAA8D;IAC9D,sBAAsB,CAAC,EAAE,MAAM,CAAC;IAChC,4EAA4E;IAC5E,cAAc,CAAC,EAAE,MAAM,CAAC;CACzB;AAED;;;;;;;;;;;;;;GAcG;AACH,wBAAgB,aAAa,CAC3B,MAAM,EAAE,yBAAyB,EACjC,KAAK,EAAE,yBAAyB,EAChC,aAAa,CAAC,EAAE;IAAE,aAAa,CAAC,EAAE,MAAM,CAAC;IAAC,iBAAiB,CAAC,EAAE,MAAM,CAAA;CAAE,GACrE,eAAe,CA2CjB;AAOD;;;;;;;GAOG;AACH,wBAAsB,oBAAoB,CACxC,OAAO,EAAE,MAAM,EACf,MAAM,CAAC,EAAE,WAAW,GACnB,OAAO,CAAC,yBAAyB,CAAC,CA0BpC"}

package/services/local-inference/llama-server-metrics.js

@@ -0,0 +1,175 @@
+/**
+ * Scraper for llama-server's `/metrics` (Prometheus exposition format)
+ * endpoint. Translates the running counters into the
+ * Anthropic-SDK-shaped `usage` block that callers already know how to
+ * consume from the cloud Anthropic plugin.
+ *
+ * llama-server publishes the following counters (per-process, monotonic):
+ *
+ *   llamacpp:n_decode_total           — context tokens decoded (prefill + gen)
+ *   llamacpp:n_tokens_predicted_total — output tokens
+ *   llamacpp:prompt_tokens_total      — total input tokens accepted
+ *   llamacpp:n_past_max               — high-water mark of cached past-tokens
+ *   llamacpp:n_prompt_tokens_processed_total — fresh tokens prefilled
+ *                                       (i.e. cache MISS), excludes cache hits
+ *   llamacpp:kv_cache_tokens          — current size of KV cache (gauge)
+ *   llamacpp:kv_cache_used_cells      — slots with active KV (gauge)
+ *
+ * For DFlash speculative decoding, the fork additionally publishes:
+ *
+ *   llamacpp:n_drafted_total          — drafter-emitted tokens
+ *   llamacpp:n_accepted_total         — accepted speculative tokens
+ *
+ * The mapping into Anthropic shape:
+ *
+ *   prompt_tokens_total                              → input_tokens
+ *   n_tokens_predicted_total                         → output_tokens
+ *   n_prompt_tokens_processed_total                  → cache_creation_input_tokens
+ *   prompt_tokens_total - n_prompt_tokens_processed_total → cache_read_input_tokens
+ *   n_drafted_total / n_accepted_total               → DFlash extension fields
+ *
+ * Counters are taken as deltas across two snapshots: take one before
+ * `generate`, one after, and subtract. Losing a few samples to process
+ * restart is acceptable — the deltas are useful for the call's own
+ * usage accounting, not for global monitoring.
+ */
+const METRIC_KEYS = {
+    "llamacpp:prompt_tokens_total": "promptTokensTotal",
+    "llamacpp:n_tokens_predicted_total": "predictedTokensTotal",
+    "llamacpp:n_prompt_tokens_processed_total": "promptTokensProcessedTotal",
+    "llamacpp:n_drafted_total": "draftedTotal",
+    "llamacpp:n_accepted_total": "acceptedTotal",
+    "llamacpp:kv_cache_tokens": "kvCacheTokens",
+    "llamacpp:kv_cache_used_cells": "kvCacheUsedCells",
+};
+/**
+ * Parse a Prometheus exposition-format payload into a metric snapshot.
+ * Unknown or malformed lines are silently skipped — counters we don't
+ * recognise are not interesting and metric exporters add new ones over
+ * time.
+ *
+ * llama-server exposes one sample per metric (no labels), e.g.
+ *   `llamacpp:prompt_tokens_total 1234`
+ * So we only care about the simple `metric_name <number>` form.
+ */
+export function parsePrometheusMetrics(body, takenAtMs = Date.now()) {
+    const snapshot = {
+        takenAtMs,
+        promptTokensTotal: 0,
+        predictedTokensTotal: 0,
+        promptTokensProcessedTotal: 0,
+        draftedTotal: 0,
+        acceptedTotal: 0,
+        kvCacheTokens: 0,
+        kvCacheUsedCells: 0,
+    };
+    for (const rawLine of body.split(/\r?\n/)) {
+        const line = rawLine.trim();
+        if (!line || line.startsWith("#"))
+            continue;
+        // Prometheus line format: `name{labels?} value [timestamp]`. We
+        // accept the unlabelled form llama-server actually emits. Labelled
+        // forms are skipped — there's no per-slot label exposed today, and
+        // if one is added later we want the maintainer to opt in here.
+        const match = line.match(/^([a-zA-Z_:][\w:]*)\s+([+-]?\d+(?:\.\d+)?(?:e[+-]?\d+)?)/);
+        if (!match)
+            continue;
+        const name = match[1];
+        const value = Number(match[2]);
+        if (!Number.isFinite(value) || name === undefined)
+            continue;
+        const field = METRIC_KEYS[name];
+        if (!field)
+            continue;
+        snapshot[field] = value;
+    }
+    return snapshot;
+}
+/**
+ * Compute the Anthropic-shape usage block for a single generation by
+ * differencing two snapshots. `before` is taken just before the request,
+ * `after` just after the response was received. Negative deltas (caused
+ * by a metric reset between snapshots, e.g. server restart) are clamped
+ * to 0 — losing the sample is preferable to surfacing nonsense to the
+ * caller.
+ *
+ * Pass `responseUsage` to override input/output counts when the response
+ * payload itself reports per-call counters that are more accurate than
+ * the metric delta — llama-server's chat completion response includes
+ * `usage.{prompt,completion}_tokens` per request, which is exact while
+ * the metric delta is "everything that happened during the wall-clock
+ * window of the request."
+ */
+export function diffSnapshots(before, after, responseUsage) {
+    const promptDelta = clampNonNegative(after.promptTokensTotal - before.promptTokensTotal);
+    const predictedDelta = clampNonNegative(after.predictedTokensTotal - before.predictedTokensTotal);
+    const processedDelta = clampNonNegative(after.promptTokensProcessedTotal - before.promptTokensProcessedTotal);
+    const draftedDelta = clampNonNegative(after.draftedTotal - before.draftedTotal);
+    const acceptedDelta = clampNonNegative(after.acceptedTotal - before.acceptedTotal);
+    const responsePrompt = responseUsage?.prompt_tokens ?? promptDelta;
+    const responseCompletion = responseUsage?.completion_tokens ?? predictedDelta;
+    const inputTokens = responsePrompt;
+    const outputTokens = responseCompletion;
+    // Tokens that had to be freshly prefilled this call. Bounded above by
+    // the per-call input count — a metric-delta wider than the call's own
+    // input is a sampling artifact.
+    const cacheCreation = Math.min(processedDelta, inputTokens);
+    const cacheRead = Math.max(0, inputTokens - cacheCreation);
+    const block = {
+        input_tokens: inputTokens,
+        output_tokens: outputTokens,
+        cache_creation_input_tokens: cacheCreation,
+        cache_read_input_tokens: cacheRead,
+    };
+    if (inputTokens > 0) {
+        block.cache_hit_rate = cacheRead / inputTokens;
+    }
+    if (draftedDelta > 0) {
+        block.dflash_drafted_tokens = draftedDelta;
+        block.dflash_accepted_tokens = acceptedDelta;
+        block.dflash_acceptance_rate = acceptedDelta / draftedDelta;
+    }
+    return block;
+}
+function clampNonNegative(value) {
+    if (!Number.isFinite(value))
+        return 0;
+    return value < 0 ? 0 : value;
+}
+/**
+ * GET `/metrics` from a running llama-server and parse it. Errors fall
+ * back to a zero-valued snapshot rather than throwing — observability
+ * MUST NOT break generation. Callers that want to detect scrape failures
+ * should compare `before.takenAtMs` against `after.takenAtMs`: a zero
+ * snapshot pair (both fields all-zero) means scraping returned nothing
+ * useful.
+ */
+export async function fetchMetricsSnapshot(baseUrl, signal) {
+    const takenAtMs = Date.now();
+    const empty = {
+        takenAtMs,
+        promptTokensTotal: 0,
+        predictedTokensTotal: 0,
+        promptTokensProcessedTotal: 0,
+        draftedTotal: 0,
+        acceptedTotal: 0,
+        kvCacheTokens: 0,
+        kvCacheUsedCells: 0,
+    };
+    try {
+        const res = await fetch(`${baseUrl.replace(/\/$/, "")}/metrics`, {
+            method: "GET",
+            signal,
+        });
+        if (!res.ok)
+            return empty;
+        const body = await res.text();
+        return parsePrometheusMetrics(body, takenAtMs);
+    }
+    catch {
+        // Best effort: a metrics scrape failure must not abort the response
+        // path. Returning an empty snapshot causes diffSnapshots to surface
+        // zero deltas; the caller still sees the response payload usage.
+        return empty;
+    }
+}

package/services/local-inference/manifest/index.d.ts

@@ -0,0 +1,4 @@
+export { ELIZA_1_BACKENDS, ELIZA_1_KERNELS, ELIZA_1_MANIFEST_SCHEMA_URL, ELIZA_1_MANIFEST_SCHEMA_VERSION, ELIZA_1_TIERS, Eliza1BackendEnumSchema, Eliza1EvalsSchema, Eliza1FileEntrySchema, Eliza1FilesSchema, Eliza1KernelEnumSchema, Eliza1KernelsSchema, Eliza1LineageSchema, Eliza1ManifestSchema, Eliza1RamBudgetSchema, Eliza1TierEnumSchema, Eliza1VerifiedBackendStatusSchema, REQUIRED_KERNELS_BY_TIER, SUPPORTED_BACKENDS_BY_TIER, } from "./schema";
+export type { Eliza1Backend, Eliza1DeviceCaps, Eliza1Evals, Eliza1FileEntry, Eliza1Files, Eliza1Kernel, Eliza1Kernels, Eliza1Lineage, Eliza1Manifest, Eliza1RamBudget, Eliza1Tier, Eliza1VerifiedBackendStatus, } from "./types";
+export { canSetAsDefault, missingRequiredKernels, parseManifestOrThrow, validateManifest, type ValidationErr, type ValidationOk, type ValidationResult, } from "./validator";
+//# sourceMappingURL=index.d.ts.map

package/services/local-inference/manifest/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../../../../../src/services/local-inference/manifest/index.ts"],"names":[],"mappings":"AAIA,OAAO,EACL,gBAAgB,EAChB,eAAe,EACf,2BAA2B,EAC3B,+BAA+B,EAC/B,aAAa,EACb,uBAAuB,EACvB,iBAAiB,EACjB,qBAAqB,EACrB,iBAAiB,EACjB,sBAAsB,EACtB,mBAAmB,EACnB,mBAAmB,EACnB,oBAAoB,EACpB,qBAAqB,EACrB,oBAAoB,EACpB,iCAAiC,EACjC,wBAAwB,EACxB,0BAA0B,GAC3B,MAAM,UAAU,CAAC;AAElB,YAAY,EACV,aAAa,EACb,gBAAgB,EAChB,WAAW,EACX,eAAe,EACf,WAAW,EACX,YAAY,EACZ,aAAa,EACb,aAAa,EACb,cAAc,EACd,eAAe,EACf,UAAU,EACV,2BAA2B,GAC5B,MAAM,SAAS,CAAC;AAEjB,OAAO,EACL,eAAe,EACf,sBAAsB,EACtB,oBAAoB,EACpB,gBAAgB,EAChB,KAAK,aAAa,EAClB,KAAK,YAAY,EACjB,KAAK,gBAAgB,GACtB,MAAM,aAAa,CAAC"}

package/services/local-inference/manifest/index.js

@@ -0,0 +1,5 @@
+// Public entry point for the Eliza-1 manifest module.
+// Catalogs / downloader / recommendation should import from here, not
+// from the schema/types/validator files individually.
+export { ELIZA_1_BACKENDS, ELIZA_1_KERNELS, ELIZA_1_MANIFEST_SCHEMA_URL, ELIZA_1_MANIFEST_SCHEMA_VERSION, ELIZA_1_TIERS, Eliza1BackendEnumSchema, Eliza1EvalsSchema, Eliza1FileEntrySchema, Eliza1FilesSchema, Eliza1KernelEnumSchema, Eliza1KernelsSchema, Eliza1LineageSchema, Eliza1ManifestSchema, Eliza1RamBudgetSchema, Eliza1TierEnumSchema, Eliza1VerifiedBackendStatusSchema, REQUIRED_KERNELS_BY_TIER, SUPPORTED_BACKENDS_BY_TIER, } from "./schema";
+export { canSetAsDefault, missingRequiredKernels, parseManifestOrThrow, validateManifest, } from "./validator";