@elizaos/app-core 2.0.0-beta.2 → 2.0.11-beta.5

package/api/training-benchmarks.d.ts

@@ -0,0 +1,97 @@
+/**
+ * Read-only API for the benchmark trending DB scaffolded in W0-X5 (gap M2).
+ *
+ * The producers — benchmark adapters, the promotion gate, the trajectory-replay
+ * harness — are Python (W1-B*). They write to a small SQLite database via
+ * `eliza/packages/benchmarks/lib/results_store.py`. This module is the dashboard
+ * read-side: it exposes two endpoints that surface per-model history and pairwise
+ * comparisons to the Training view in the dashboard without re-running the
+ * benchmark.
+ *
+ * Endpoints
+ * =========
+ *
+ *   GET /api/training/benchmarks/scores?model_id=&benchmark=[&limit=]
+ *     Returns `{ runs: BenchmarkRunDTO[] }` — newest-first history for one
+ *     (model, benchmark) pair. `limit` defaults to 100, max 1000.
+ *
+ *   GET /api/training/benchmarks/compare?a=&b=&benchmark=
+ *     Returns the latest run for each side plus `delta = a.score - b.score`
+ *     (or `null` if either side is missing).
+ *
+ * Storage location
+ * ================
+ *
+ * The SQLite path follows the Python store's default:
+ *   - `ELIZA_BENCHMARK_RESULTS_DB` (env override) if set.
+ *   - else `<stateDir>/benchmarks/results.db`.
+ *
+ * Mode notes
+ * ==========
+ *
+ * - The DB file may not exist when no benchmarks have run yet. In that case
+ *   every endpoint returns an empty result — *not* a 5xx. Producers in W1-B*
+ *   will populate it.
+ * - The schema is locked to v1 in the Python module. Schema migration is a
+ *   coordinated change across the Python store and these routes.
+ */
+import type http from "node:http";
+import type { CompatRuntimeState } from "./compat-route-shared";
+export declare const ELIZA_BENCHMARK_SCORES_SCHEMA: "elizaos.training.benchmark-scores/v1";
+export declare const ELIZA_BENCHMARK_COMPARE_SCHEMA: "elizaos.training.benchmark-compare/v1";
+export interface BenchmarkRunDTO {
+    id: number;
+    modelId: string;
+    benchmark: string;
+    score: number;
+    /** Unix milliseconds, UTC. */
+    ts: number;
+    datasetVersion: string;
+    codeCommit: string;
+}
+export interface BenchmarkScoresResponse {
+    schema: typeof ELIZA_BENCHMARK_SCORES_SCHEMA;
+    modelId: string;
+    benchmark: string;
+    /** Whether the underlying SQLite database file exists. */
+    dbReady: boolean;
+    runs: BenchmarkRunDTO[];
+}
+export interface BenchmarkCompareResponse {
+    schema: typeof ELIZA_BENCHMARK_COMPARE_SCHEMA;
+    benchmark: string;
+    modelA: string;
+    modelB: string;
+    dbReady: boolean;
+    a: BenchmarkRunDTO | null;
+    b: BenchmarkRunDTO | null;
+    /** `a.score - b.score` when both sides have runs; otherwise `null`. */
+    delta: number | null;
+}
+export declare function resolveBenchmarkResultsDbPath(env?: NodeJS.ProcessEnv): string;
+interface BenchmarkResultsReader {
+    ready: boolean;
+    getHistory(args: {
+        modelId: string;
+        benchmark: string;
+        limit: number;
+    }): BenchmarkRunDTO[];
+    getLatest(args: {
+        modelId: string;
+        benchmark: string;
+    }): BenchmarkRunDTO | null;
+    close(): void;
+}
+export declare function openBenchmarkResultsReader(dbPath: string): BenchmarkResultsReader;
+export interface TrainingBenchmarksRouteOptions {
+    /** Override the DB path. Tests use this. */
+    dbPath?: string;
+    /**
+     * Open a reader. Tests can inject a fake to avoid touching disk.
+     * Defaults to {@link openBenchmarkResultsReader}.
+     */
+    openReader?: (dbPath: string) => BenchmarkResultsReader;
+}
+export declare function handleTrainingBenchmarksRoute(req: http.IncomingMessage, res: http.ServerResponse, state: CompatRuntimeState, options?: TrainingBenchmarksRouteOptions): Promise<boolean>;
+export {};
+//# sourceMappingURL=training-benchmarks.d.ts.map

package/api/training-benchmarks.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"training-benchmarks.d.ts","sourceRoot":"","sources":["../../../../../src/api/training-benchmarks.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAoCG;AAGH,OAAO,KAAK,IAAI,MAAM,WAAW,CAAC;AAgFlC,OAAO,KAAK,EAAE,kBAAkB,EAAE,MAAM,uBAAuB,CAAC;AAmBhE,eAAO,MAAM,6BAA6B,EACxC,sCAA+C,CAAC;AAClD,eAAO,MAAM,8BAA8B,EACzC,uCAAgD,CAAC;AAEnD,MAAM,WAAW,eAAe;IAC9B,EAAE,EAAE,MAAM,CAAC;IACX,OAAO,EAAE,MAAM,CAAC;IAChB,SAAS,EAAE,MAAM,CAAC;IAClB,KAAK,EAAE,MAAM,CAAC;IACd,8BAA8B;IAC9B,EAAE,EAAE,MAAM,CAAC;IACX,cAAc,EAAE,MAAM,CAAC;IACvB,UAAU,EAAE,MAAM,CAAC;CACpB;AAED,MAAM,WAAW,uBAAuB;IACtC,MAAM,EAAE,OAAO,6BAA6B,CAAC;IAC7C,OAAO,EAAE,MAAM,CAAC;IAChB,SAAS,EAAE,MAAM,CAAC;IAClB,0DAA0D;IAC1D,OAAO,EAAE,OAAO,CAAC;IACjB,IAAI,EAAE,eAAe,EAAE,CAAC;CACzB;AAED,MAAM,WAAW,wBAAwB;IACvC,MAAM,EAAE,OAAO,8BAA8B,CAAC;IAC9C,SAAS,EAAE,MAAM,CAAC;IAClB,MAAM,EAAE,MAAM,CAAC;IACf,MAAM,EAAE,MAAM,CAAC;IACf,OAAO,EAAE,OAAO,CAAC;IACjB,CAAC,EAAE,eAAe,GAAG,IAAI,CAAC;IAC1B,CAAC,EAAE,eAAe,GAAG,IAAI,CAAC;IAC1B,uEAAuE;IACvE,KAAK,EAAE,MAAM,GAAG,IAAI,CAAC;CACtB;AAMD,wBAAgB,6BAA6B,CAC3C,GAAG,GAAE,MAAM,CAAC,UAAwB,GACnC,MAAM,CAMR;AAmCD,UAAU,sBAAsB;IAC9B,KAAK,EAAE,OAAO,CAAC;IACf,UAAU,CAAC,IAAI,EAAE;QACf,OAAO,EAAE,MAAM,CAAC;QAChB,SAAS,EAAE,MAAM,CAAC;QAClB,KAAK,EAAE,MAAM,CAAC;KACf,GAAG,eAAe,EAAE,CAAC;IACtB,SAAS,CAAC,IAAI,EAAE;QACd,OAAO,EAAE,MAAM,CAAC;QAChB,SAAS,EAAE,MAAM,CAAC;KACnB,GAAG,eAAe,GAAG,IAAI,CAAC;IAC3B,KAAK,IAAI,IAAI,CAAC;CACf;AAED,wBAAgB,0BAA0B,CACxC,MAAM,EAAE,MAAM,GACb,sBAAsB,CAwDxB;AAgDD,MAAM,WAAW,8BAA8B;IAC7C,4CAA4C;IAC5C,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB;;;OAGG;IACH,UAAU,CAAC,EAAE,CAAC,MAAM,EAAE,MAAM,KAAK,sBAAsB,CAAC;CACzD;AAED,wBAAsB,6BAA6B,CACjD,GAAG,EAAE,IAAI,CAAC,eAAe,EACzB,GAAG,EAAE,IAAI,CAAC,cAAc,EACxB,KAAK,EAAE,kBAAkB,EACzB,OAAO,GAAE,8BAAmC,GAC3C,OAAO,CAAC,OAAO,CAAC,CA6BlB"}

package/api/training-benchmarks.js

@@ -0,0 +1,307 @@
+/**
+ * Read-only API for the benchmark trending DB scaffolded in W0-X5 (gap M2).
+ *
+ * The producers — benchmark adapters, the promotion gate, the trajectory-replay
+ * harness — are Python (W1-B*). They write to a small SQLite database via
+ * `eliza/packages/benchmarks/lib/results_store.py`. This module is the dashboard
+ * read-side: it exposes two endpoints that surface per-model history and pairwise
+ * comparisons to the Training view in the dashboard without re-running the
+ * benchmark.
+ *
+ * Endpoints
+ * =========
+ *
+ *   GET /api/training/benchmarks/scores?model_id=&benchmark=[&limit=]
+ *     Returns `{ runs: BenchmarkRunDTO[] }` — newest-first history for one
+ *     (model, benchmark) pair. `limit` defaults to 100, max 1000.
+ *
+ *   GET /api/training/benchmarks/compare?a=&b=&benchmark=
+ *     Returns the latest run for each side plus `delta = a.score - b.score`
+ *     (or `null` if either side is missing).
+ *
+ * Storage location
+ * ================
+ *
+ * The SQLite path follows the Python store's default:
+ *   - `ELIZA_BENCHMARK_RESULTS_DB` (env override) if set.
+ *   - else `<stateDir>/benchmarks/results.db`.
+ *
+ * Mode notes
+ * ==========
+ *
+ * - The DB file may not exist when no benchmarks have run yet. In that case
+ *   every endpoint returns an empty result — *not* a 5xx. Producers in W1-B*
+ *   will populate it.
+ * - The schema is locked to v1 in the Python module. Schema migration is a
+ *   coordinated change across the Python store and these routes.
+ */
+import fs from "node:fs";
+import { createRequire } from "node:module";
+import os from "node:os";
+import path from "node:path";
+import { ensureRouteAuthorized } from "./auth.js";
+const requireFromHere = createRequire(import.meta.url);
+let DatabaseSyncCached;
+function loadDatabaseSync() {
+    if (DatabaseSyncCached !== undefined)
+        return DatabaseSyncCached;
+    // Node ≥22.5 built-in.
+    try {
+        const mod = requireFromHere("node:sqlite");
+        if (mod.DatabaseSync) {
+            DatabaseSyncCached = mod.DatabaseSync;
+            return DatabaseSyncCached;
+        }
+    }
+    catch {
+        // Not Node — fall through to Bun.
+    }
+    // Bun built-in. Bun spells the read-only flag `readonly` and throws on
+    // Node's `readOnly`, so adapt the constructor to the shared signature.
+    try {
+        const { Database } = requireFromHere("bun:sqlite");
+        class BunDatabaseSync {
+            db;
+            constructor(filename, options) {
+                // Bun rejects `{ readonly: false }` (SQLITE_MISUSE — flags include
+                // neither READONLY nor READWRITE). Pass the flag only when read-only
+                // is requested; otherwise let Bun default to readwrite+create.
+                this.db = options?.readOnly
+                    ? new Database(filename, { readonly: true })
+                    : new Database(filename);
+            }
+            prepare(sql) {
+                return this.db.prepare(sql);
+            }
+            close() {
+                this.db.close();
+            }
+        }
+        DatabaseSyncCached = BunDatabaseSync;
+        return DatabaseSyncCached;
+    }
+    catch {
+        DatabaseSyncCached = null;
+    }
+    return DatabaseSyncCached;
+}
+import { sendJson, sendJsonError } from "./response.js";
+const ROUTE_PREFIX = "/api/training/benchmarks";
+const ROUTE_SCORES = `${ROUTE_PREFIX}/scores`;
+const ROUTE_COMPARE = `${ROUTE_PREFIX}/compare`;
+const DEFAULT_HISTORY_LIMIT = 100;
+const MAX_HISTORY_LIMIT = 1000;
+// Match the Python validator: [A-Za-z0-9._:/-], 1..256 chars. Wider than secret
+// keys because model ids contain slashes and colons in some catalogs.
+const SAFE_ID_RE = /^[A-Za-z0-9._:/-]{1,256}$/;
+const SAFE_BENCHMARK_RE = /^[A-Za-z0-9_.-]{1,128}$/;
+// ---------------------------------------------------------------------------
+// DTOs (response shapes)
+// ---------------------------------------------------------------------------
+export const ELIZA_BENCHMARK_SCORES_SCHEMA = "elizaos.training.benchmark-scores/v1";
+export const ELIZA_BENCHMARK_COMPARE_SCHEMA = "elizaos.training.benchmark-compare/v1";
+// ---------------------------------------------------------------------------
+// Storage location resolver
+// ---------------------------------------------------------------------------
+export function resolveBenchmarkResultsDbPath(env = process.env) {
+    const override = env.ELIZA_BENCHMARK_RESULTS_DB?.trim();
+    if (override && override.length > 0) {
+        return path.resolve(expandUserHome(override));
+    }
+    return path.join(resolveBenchmarkStateDir(env), "benchmarks", "results.db");
+}
+function expandUserHome(p) {
+    if (p === "~")
+        return os.homedir();
+    if (p.startsWith("~/"))
+        return path.join(os.homedir(), p.slice(2));
+    return p;
+}
+function resolveBenchmarkStateDir(env) {
+    const explicit = env.ELIZA_STATE_DIR?.trim();
+    if (explicit)
+        return path.resolve(expandUserHome(explicit));
+    const namespace = env.ELIZA_NAMESPACE?.trim() || "eliza";
+    const xdgStateHome = env.XDG_STATE_HOME?.trim();
+    const stateHome = xdgStateHome
+        ? path.isAbsolute(xdgStateHome)
+            ? xdgStateHome
+            : path.join(os.homedir(), xdgStateHome)
+        : path.join(os.homedir(), ".local", "state");
+    return path.join(stateHome, namespace);
+}
+export function openBenchmarkResultsReader(dbPath) {
+    if (!fs.existsSync(dbPath)) {
+        return {
+            ready: false,
+            getHistory: () => [],
+            getLatest: () => null,
+            close: () => { },
+        };
+    }
+    const DatabaseSync = loadDatabaseSync();
+    if (!DatabaseSync) {
+        // Runtime exposes neither node:sqlite nor bun:sqlite. Surface "not ready"
+        // instead of throwing — callers treat this as no benchmark data.
+        return {
+            ready: false,
+            getHistory: () => [],
+            getLatest: () => null,
+            close: () => { },
+        };
+    }
+    const db = new DatabaseSync(dbPath, { readOnly: true });
+    const historyStmt = db.prepare(`SELECT id, model_id, benchmark, score, ts, dataset_version, code_commit
+       FROM benchmark_runs
+      WHERE model_id = ? AND benchmark = ?
+      ORDER BY ts DESC, id DESC
+      LIMIT ?`);
+    const latestStmt = db.prepare(`SELECT id, model_id, benchmark, score, ts, dataset_version, code_commit
+       FROM benchmark_runs
+      WHERE model_id = ? AND benchmark = ?
+      ORDER BY ts DESC, id DESC
+      LIMIT 1`);
+    return {
+        ready: true,
+        getHistory({ modelId, benchmark, limit }) {
+            const rows = historyStmt.all(modelId, benchmark, limit);
+            return rows.map(rowToDto);
+        },
+        getLatest({ modelId, benchmark }) {
+            const rows = latestStmt.all(modelId, benchmark);
+            if (rows.length === 0)
+                return null;
+            return rowToDto(rows[0]);
+        },
+        close() {
+            db.close();
+        },
+    };
+}
+function rowToDto(row) {
+    return {
+        id: Number(row.id),
+        modelId: row.model_id,
+        benchmark: row.benchmark,
+        score: row.score,
+        ts: Number(row.ts),
+        datasetVersion: row.dataset_version,
+        codeCommit: row.code_commit,
+    };
+}
+// ---------------------------------------------------------------------------
+// Query-parameter helpers
+// ---------------------------------------------------------------------------
+function readModelId(url, name) {
+    const raw = url.searchParams.get(name);
+    if (raw == null)
+        return null;
+    const trimmed = raw.trim();
+    if (trimmed.length === 0)
+        return null;
+    return SAFE_ID_RE.test(trimmed) ? trimmed : null;
+}
+function readBenchmark(url) {
+    const raw = url.searchParams.get("benchmark");
+    if (raw == null)
+        return null;
+    const trimmed = raw.trim();
+    if (trimmed.length === 0)
+        return null;
+    return SAFE_BENCHMARK_RE.test(trimmed) ? trimmed : null;
+}
+function readLimit(url) {
+    const raw = url.searchParams.get("limit");
+    if (raw == null || raw.trim().length === 0)
+        return DEFAULT_HISTORY_LIMIT;
+    const n = Number(raw);
+    if (!Number.isFinite(n) || !Number.isInteger(n) || n <= 0) {
+        return DEFAULT_HISTORY_LIMIT;
+    }
+    return Math.min(n, MAX_HISTORY_LIMIT);
+}
+export async function handleTrainingBenchmarksRoute(req, res, state, options = {}) {
+    const url = new URL(req.url ?? "/", "http://localhost");
+    if (!url.pathname.startsWith(ROUTE_PREFIX)) {
+        return false;
+    }
+    const method = (req.method ?? "GET").toUpperCase();
+    if (method !== "GET") {
+        sendJsonError(res, 405, "method not allowed");
+        return true;
+    }
+    if (!(await ensureRouteAuthorized(req, res, state))) {
+        return true;
+    }
+    const dbPath = options.dbPath ?? resolveBenchmarkResultsDbPath();
+    const openReader = options.openReader ?? openBenchmarkResultsReader;
+    if (url.pathname === ROUTE_SCORES) {
+        return handleScores(res, url, dbPath, openReader);
+    }
+    if (url.pathname === ROUTE_COMPARE) {
+        return handleCompare(res, url, dbPath, openReader);
+    }
+    sendJsonError(res, 404, "not found");
+    return true;
+}
+function handleScores(res, url, dbPath, openReader) {
+    const modelId = readModelId(url, "model_id");
+    const benchmark = readBenchmark(url);
+    if (!modelId) {
+        sendJsonError(res, 400, "missing or invalid model_id");
+        return true;
+    }
+    if (!benchmark) {
+        sendJsonError(res, 400, "missing or invalid benchmark");
+        return true;
+    }
+    const limit = readLimit(url);
+    const reader = openReader(dbPath);
+    const runs = reader.getHistory({ modelId, benchmark, limit });
+    const dbReady = reader.ready;
+    reader.close();
+    const body = {
+        schema: ELIZA_BENCHMARK_SCORES_SCHEMA,
+        modelId,
+        benchmark,
+        dbReady,
+        runs,
+    };
+    sendJson(res, 200, body);
+    return true;
+}
+function handleCompare(res, url, dbPath, openReader) {
+    const modelA = readModelId(url, "a");
+    const modelB = readModelId(url, "b");
+    const benchmark = readBenchmark(url);
+    if (!modelA) {
+        sendJsonError(res, 400, "missing or invalid a");
+        return true;
+    }
+    if (!modelB) {
+        sendJsonError(res, 400, "missing or invalid b");
+        return true;
+    }
+    if (!benchmark) {
+        sendJsonError(res, 400, "missing or invalid benchmark");
+        return true;
+    }
+    const reader = openReader(dbPath);
+    const aRun = reader.getLatest({ modelId: modelA, benchmark });
+    const bRun = reader.getLatest({ modelId: modelB, benchmark });
+    const dbReady = reader.ready;
+    reader.close();
+    const delta = aRun !== null && bRun !== null ? aRun.score - bRun.score : null;
+    const body = {
+        schema: ELIZA_BENCHMARK_COMPARE_SCHEMA,
+        benchmark,
+        modelA,
+        modelB,
+        dbReady,
+        a: aRun,
+        b: bRun,
+        delta,
+    };
+    sendJson(res, 200, body);
+    return true;
+}

package/api/workbench-compat-routes.js

@@ -1,7 +1,7 @@
 import { logger } from "@elizaos/core";
 import { ensureRouteAuthorized } from "./auth.js";
-import { readCompatJsonBody, } from "./compat-route-shared";
-import { sendJsonError as sendJsonErrorResponse, sendJson as sendJsonResponse, } from "./response";
+import { readCompatJsonBody, } from "./compat-route-shared.js";
+import { sendJsonError as sendJsonErrorResponse, sendJson as sendJsonResponse, } from "./response.js";
 // ---------------------------------------------------------------------------
 // Helpers (only used by workbench/todos routes)
 // ---------------------------------------------------------------------------

package/benchmark/cerebras-autowire.d.ts

@@ -0,0 +1,28 @@
+/**
+ * Auto-wire the `@elizaos/plugin-openai` env keys from a `CEREBRAS_API_KEY`
+ * when no competing OpenAI-compat configuration is present.
+ *
+ * The `.env` in this repo typically defines `CEREBRAS_API_KEY` /
+ * `CEREBRAS_BASE_URL` / `CEREBRAS_MODEL` but does NOT set `OPENAI_BASE_URL` /
+ * `OPENAI_API_KEY` / `ELIZA_PROVIDER` — the keys the plugin code path
+ * actually reads. Without those, the openai plugin is skipped at load time,
+ * no TEXT_LARGE / TEXT_SMALL handler is registered, and every benchmark turn
+ * falls back to the "no LLM provider configured" unavailable-provider path.
+ *
+ * Promotes Cerebras config to OpenAI-compat keys when ALL of these hold:
+ *   - `CEREBRAS_API_KEY` is set
+ *   - no competing OpenAI-compat key is present (`OPENAI_API_KEY`,
+ *     `OPENAI_BASE_URL`, and `ELIZA_PROVIDER` are all unset)
+ *
+ * Never overwrites an existing OPENAI_* or ELIZA_PROVIDER value unless the
+ * benchmark run explicitly selected Cerebras.
+ *
+ * The resulting `OPENAI_BASE_URL=https://api.cerebras.ai/v1` is the V1 root
+ * onto which `@ai-sdk/openai`'s `openai.chat(model)` appends
+ * `/chat/completions` (Cerebras's only supported endpoint). The Responses API
+ * (`/v1/responses`) does not exist on Cerebras, so it is critical that
+ * `plugin-openai/models/text.ts` uses `openai.chat()` (not
+ * `openai.responses()` / `openai.languageModel()`).
+ */
+export declare function autoWireCerebras(): void;
+//# sourceMappingURL=cerebras-autowire.d.ts.map

package/benchmark/cerebras-autowire.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"cerebras-autowire.d.ts","sourceRoot":"","sources":["../../../../../src/benchmark/cerebras-autowire.ts"],"names":[],"mappings":"AAEA;;;;;;;;;;;;;;;;;;;;;;;;;GAyBG;AACH,wBAAgB,gBAAgB,IAAI,IAAI,CA4CvC"}

package/benchmark/cerebras-autowire.js

@@ -0,0 +1,62 @@
+import { elizaLogger } from "@elizaos/core";
+/**
+ * Auto-wire the `@elizaos/plugin-openai` env keys from a `CEREBRAS_API_KEY`
+ * when no competing OpenAI-compat configuration is present.
+ *
+ * The `.env` in this repo typically defines `CEREBRAS_API_KEY` /
+ * `CEREBRAS_BASE_URL` / `CEREBRAS_MODEL` but does NOT set `OPENAI_BASE_URL` /
+ * `OPENAI_API_KEY` / `ELIZA_PROVIDER` — the keys the plugin code path
+ * actually reads. Without those, the openai plugin is skipped at load time,
+ * no TEXT_LARGE / TEXT_SMALL handler is registered, and every benchmark turn
+ * falls back to the "no LLM provider configured" unavailable-provider path.
+ *
+ * Promotes Cerebras config to OpenAI-compat keys when ALL of these hold:
+ *   - `CEREBRAS_API_KEY` is set
+ *   - no competing OpenAI-compat key is present (`OPENAI_API_KEY`,
+ *     `OPENAI_BASE_URL`, and `ELIZA_PROVIDER` are all unset)
+ *
+ * Never overwrites an existing OPENAI_* or ELIZA_PROVIDER value unless the
+ * benchmark run explicitly selected Cerebras.
+ *
+ * The resulting `OPENAI_BASE_URL=https://api.cerebras.ai/v1` is the V1 root
+ * onto which `@ai-sdk/openai`'s `openai.chat(model)` appends
+ * `/chat/completions` (Cerebras's only supported endpoint). The Responses API
+ * (`/v1/responses`) does not exist on Cerebras, so it is critical that
+ * `plugin-openai/models/text.ts` uses `openai.chat()` (not
+ * `openai.responses()` / `openai.languageModel()`).
+ */
+export function autoWireCerebras() {
+    const cerebrasKey = process.env.CEREBRAS_API_KEY?.trim();
+    if (!cerebrasKey)
+        return;
+    const cerebrasBase = process.env.CEREBRAS_BASE_URL?.trim() || "https://api.cerebras.ai/v1";
+    const selectedModel = process.env.BENCHMARK_MODEL_NAME?.trim() ||
+        process.env.CEREBRAS_MODEL?.trim() ||
+        "";
+    const modelIsCerebras = selectedModel === "gpt-oss-120b" ||
+        selectedModel === "qwen-3-235b-a22b-instruct-2507";
+    const explicitCerebras = process.env.BENCHMARK_MODEL_PROVIDER?.trim().toLowerCase() === "cerebras" ||
+        process.env.ELIZA_PROVIDER?.trim().toLowerCase() === "cerebras" ||
+        process.env.OPENAI_BASE_URL?.includes("cerebras.ai") === true ||
+        modelIsCerebras;
+    const hasOpenAiKey = !!process.env.OPENAI_API_KEY?.trim();
+    const hasOpenAiBase = !!process.env.OPENAI_BASE_URL?.trim();
+    const hasElizaProvider = !!process.env.ELIZA_PROVIDER?.trim();
+    if (!explicitCerebras &&
+        (hasOpenAiKey || hasOpenAiBase || hasElizaProvider)) {
+        return;
+    }
+    process.env.OPENAI_BASE_URL = cerebrasBase;
+    process.env.OPENAI_API_KEY = cerebrasKey;
+    process.env.ELIZA_PROVIDER = "cerebras";
+    const cerebrasModel = process.env.CEREBRAS_MODEL?.trim() || "gpt-oss-120b";
+    if (!process.env.OPENAI_LARGE_MODEL?.trim()) {
+        process.env.OPENAI_LARGE_MODEL = cerebrasModel;
+    }
+    if (!process.env.OPENAI_SMALL_MODEL?.trim()) {
+        process.env.OPENAI_SMALL_MODEL = cerebrasModel;
+    }
+    elizaLogger.info(`[bench] Auto-wired Cerebras: OPENAI_BASE_URL=${cerebrasBase}, ` +
+        `ELIZA_PROVIDER=cerebras, OPENAI_LARGE_MODEL=${process.env.OPENAI_LARGE_MODEL}, ` +
+        `OPENAI_SMALL_MODEL=${process.env.OPENAI_SMALL_MODEL}`);
+}

package/benchmark/lifeops-bench-handler.d.ts

@@ -39,6 +39,20 @@ interface LifeOpsBenchSession {
     createdAtMs: number;
     /** Last assistant text returned, kept for diagnostics. */
     lastAssistantText: string;
+    turns: LifeOpsBenchTurnRecord[];
+}
+export interface LifeOpsBenchTurnRecord {
+    userText: string;
+    assistantText: string;
+    toolCalls: ToolCallRecord[];
+}
+interface ToolCallRecord {
+    id: string;
+    name: string;
+    arguments: Record<string, unknown>;
+    result: unknown;
+    ok: boolean;
+    error?: string;
 }
 /** Shape of the assistant planner result we extract tool_calls from. */
 export interface PlannerInvocationResult {
@@ -52,6 +66,16 @@ export interface PlannerInvocationResult {
         promptTokens?: number;
         completionTokens?: number;
         totalTokens?: number;
+        /**
+         * Provider-reported prompt-cache reads (Anthropic
+         * ``cache_read_input_tokens`` / OpenAI + Cerebras
+         * ``prompt_tokens_details.cached_tokens``). Optional because not every
+         * provider supports prompt caching; nullable upstream stays nullable
+         * here — no silent 0 fallback, per AGENTS.md Cmd #8.
+         */
+        cacheReadInputTokens?: number;
+        /** Anthropic-only ``cache_creation_input_tokens``. */
+        cacheCreationInputTokens?: number;
     };
 }
 /**
@@ -66,6 +90,7 @@ export type LifeOpsPlannerInvocation = (args: {
     userText: string;
     toolManifest: unknown;
     backend: LifeOpsFakeBackend;
+    previousTurns: LifeOpsBenchTurnRecord[];
 }) => Promise<PlannerInvocationResult>;
 export interface LifeOpsBenchHandlerOptions {
     /** Provided by the bench server: invokes Eliza's planner against `userText`. */
@@ -75,6 +100,17 @@ export interface LifeOpsBenchHandlerOptions {
     /** Maximum body size in bytes for POST routes. */
     maxBodyBytes?: number;
 }
+/**
+ * Unwraps an umbrella tool call (currently CALENDAR) into the dotted granular
+ * form the fake backend dispatches directly, stripping `subaction` from the
+ * forwarded kwargs. When no mapping applies the original `{ name, kwargs }`
+ * is returned unchanged so the backend's umbrella router (or its
+ * `LifeOpsBackendUnsupportedError`) still owns the response.
+ */
+export declare function translateUmbrellaAction(name: string, kwargs: Record<string, unknown>): {
+    name: string;
+    kwargs: Record<string, unknown>;
+};
 export declare class LifeOpsBenchHandler {
     private readonly sessions;
     private readonly invokePlanner;

package/benchmark/lifeops-bench-handler.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"lifeops-bench-handler.d.ts","sourceRoot":"","sources":["../../../../../src/benchmark/lifeops-bench-handler.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAgCG;AAEH,OAAO,KAAK,IAAI,MAAM,WAAW,CAAC;AAClC,OAAO,EAGL,kBAAkB,EACnB,MAAM,2BAA2B,CAAC;AAEnC,UAAU,mBAAmB;IAC3B,MAAM,EAAE,MAAM,CAAC;IACf,OAAO,EAAE,kBAAkB,CAAC;IAC5B,WAAW,EAAE,MAAM,CAAC;IACpB,0DAA0D;IAC1D,iBAAiB,EAAE,MAAM,CAAC;CAC3B;AAWD,wEAAwE;AACxE,MAAM,WAAW,uBAAuB;IACtC,IAAI,EAAE,MAAM,CAAC;IACb,SAAS,EAAE,KAAK,CAAC;QACf,EAAE,CAAC,EAAE,MAAM,CAAC;QACZ,IAAI,EAAE,MAAM,CAAC;QACb,SAAS,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;KACpC,CAAC,CAAC;IACH,KAAK,CAAC,EAAE;QACN,YAAY,CAAC,EAAE,MAAM,CAAC;QACtB,gBAAgB,CAAC,EAAE,MAAM,CAAC;QAC1B,WAAW,CAAC,EAAE,MAAM,CAAC;KACtB,CAAC;CACH;AAED;;;;;;GAMG;AACH,MAAM,MAAM,wBAAwB,GAAG,CAAC,IAAI,EAAE;IAC5C,MAAM,EAAE,MAAM,CAAC;IACf,QAAQ,EAAE,MAAM,CAAC;IACjB,YAAY,EAAE,OAAO,CAAC;IACtB,OAAO,EAAE,kBAAkB,CAAC;CAC7B,KAAK,OAAO,CAAC,uBAAuB,CAAC,CAAC;AAEvC,MAAM,WAAW,0BAA0B;IACzC,gFAAgF;IAChF,aAAa,EAAE,wBAAwB,CAAC;IACxC,+EAA+E;IAC/E,SAAS,CAAC,EAAE,CAAC,GAAG,EAAE,IAAI,CAAC,eAAe,EAAE,GAAG,EAAE,IAAI,CAAC,cAAc,KAAK,OAAO,CAAC;IAC7E,kDAAkD;IAClD,YAAY,CAAC,EAAE,MAAM,CAAC;CACvB;AAKD,qBAAa,mBAAmB;IAC9B,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAA0C;IACnE,OAAO,CAAC,QAAQ,CAAC,aAAa,CAA2B;IACzD,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAC,CAGd;IACb,OAAO,CAAC,QAAQ,CAAC,YAAY,CAAS;gBAE1B,OAAO,EAAE,0BAA0B;IAM/C,gFAAgF;IAC1E,SAAS,CACb,GAAG,EAAE,IAAI,CAAC,eAAe,EACzB,GAAG,EAAE,IAAI,CAAC,cAAc,EACxB,QAAQ,EAAE,MAAM,GACf,OAAO,CAAC,OAAO,CAAC;IAuCnB,0CAA0C;IAC1C,UAAU,CAAC,MAAM,EAAE,MAAM,GAAG,mBAAmB,GAAG,SAAS;YAM7C,WAAW;YAqDX,aAAa;IA0F3B,OAAO,CAAC,gBAAgB;YAgBV,cAAc;CAmB7B"}
+{"version":3,"file":"lifeops-bench-handler.d.ts","sourceRoot":"","sources":["../../../../../src/benchmark/lifeops-bench-handler.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAgCG;AAEH,OAAO,KAAK,IAAI,MAAM,WAAW,CAAC;AAClC,OAAO,EAGL,kBAAkB,EACnB,MAAM,2BAA2B,CAAC;AAEnC,UAAU,mBAAmB;IAC3B,MAAM,EAAE,MAAM,CAAC;IACf,OAAO,EAAE,kBAAkB,CAAC;IAC5B,WAAW,EAAE,MAAM,CAAC;IACpB,0DAA0D;IAC1D,iBAAiB,EAAE,MAAM,CAAC;IAC1B,KAAK,EAAE,sBAAsB,EAAE,CAAC;CACjC;AAED,MAAM,WAAW,sBAAsB;IACrC,QAAQ,EAAE,MAAM,CAAC;IACjB,aAAa,EAAE,MAAM,CAAC;IACtB,SAAS,EAAE,cAAc,EAAE,CAAC;CAC7B;AAED,UAAU,cAAc;IACtB,EAAE,EAAE,MAAM,CAAC;IACX,IAAI,EAAE,MAAM,CAAC;IACb,SAAS,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IACnC,MAAM,EAAE,OAAO,CAAC;IAChB,EAAE,EAAE,OAAO,CAAC;IACZ,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB;AAED,wEAAwE;AACxE,MAAM,WAAW,uBAAuB;IACtC,IAAI,EAAE,MAAM,CAAC;IACb,SAAS,EAAE,KAAK,CAAC;QACf,EAAE,CAAC,EAAE,MAAM,CAAC;QACZ,IAAI,EAAE,MAAM,CAAC;QACb,SAAS,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;KACpC,CAAC,CAAC;IACH,KAAK,CAAC,EAAE;QACN,YAAY,CAAC,EAAE,MAAM,CAAC;QACtB,gBAAgB,CAAC,EAAE,MAAM,CAAC;QAC1B,WAAW,CAAC,EAAE,MAAM,CAAC;QACrB;;;;;;WAMG;QACH,oBAAoB,CAAC,EAAE,MAAM,CAAC;QAC9B,sDAAsD;QACtD,wBAAwB,CAAC,EAAE,MAAM,CAAC;KACnC,CAAC;CACH;AAED;;;;;;GAMG;AACH,MAAM,MAAM,wBAAwB,GAAG,CAAC,IAAI,EAAE;IAC5C,MAAM,EAAE,MAAM,CAAC;IACf,QAAQ,EAAE,MAAM,CAAC;IACjB,YAAY,EAAE,OAAO,CAAC;IACtB,OAAO,EAAE,kBAAkB,CAAC;IAC5B,aAAa,EAAE,sBAAsB,EAAE,CAAC;CACzC,KAAK,OAAO,CAAC,uBAAuB,CAAC,CAAC;AAEvC,MAAM,WAAW,0BAA0B;IACzC,gFAAgF;IAChF,aAAa,EAAE,wBAAwB,CAAC;IACxC,+EAA+E;IAC/E,SAAS,CAAC,EAAE,CAAC,GAAG,EAAE,IAAI,CAAC,eAAe,EAAE,GAAG,EAAE,IAAI,CAAC,cAAc,KAAK,OAAO,CAAC;IAC7E,kDAAkD;IAClD,YAAY,CAAC,EAAE,MAAM,CAAC;CACvB;AA6BD;;;;;;GAMG;AACH,wBAAgB,uBAAuB,CACrC,IAAI,EAAE,MAAM,EACZ,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAC9B;IAAE,IAAI,EAAE,MAAM,CAAC;IAAC,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAA;CAAE,CAqBnD;AAED,qBAAa,mBAAmB;IAC9B,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAA0C;IACnE,OAAO,CAAC,QAAQ,CAAC,aAAa,CAA2B;IACzD,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAC,CAGd;IACb,OAAO,CAAC,QAAQ,CAAC,YAAY,CAAS;gBAE1B,OAAO,EAAE,0BAA0B;IAM/C,gFAAgF;IAC1E,SAAS,CACb,GAAG,EAAE,IAAI,CAAC,eAAe,EACzB,GAAG,EAAE,IAAI,CAAC,cAAc,EACxB,QAAQ,EAAE,MAAM,GACf,OAAO,CAAC,OAAO,CAAC;IAuCnB,0CAA0C;IAC1C,UAAU,CAAC,MAAM,EAAE,MAAM,GAAG,mBAAmB,GAAG,SAAS;YAM7C,WAAW;YAsDX,aAAa;IAiG3B,OAAO,CAAC,gBAAgB;YAgBV,cAAc;CAmB7B"}