@elizaos/app-core 2.0.0-beta.2 → 2.0.11-beta.5

package/benchmark/server.js

@@ -10,17 +10,892 @@ import crypto from "node:crypto";
 import http from "node:http";
 import path from "node:path";
 import { pathToFileURL } from "node:url";
-import { CORE_PLUGINS, createElizaPlugin } from "@elizaos/agent";
-import { AgentRuntime, elizaLogger, stringToUuid, } from "@elizaos/core";
+import { AgentRuntime, elizaLogger, ModelType, stringToUuid, } from "@elizaos/core";
 import dotenv from "dotenv";
-import { LifeOpsBenchHandler } from "./lifeops-bench-handler.js";
-import { clearCapturedAction, createBenchmarkPlugin, getCapturedAction, setBenchmarkContext, } from "./plugin";
-import { capturedActionToParams, coerceActions, coerceParams, composeBenchmarkPrompt, createSession, ensureBenchmarkSessionContext, extractBenchmarkName, extractRecord, extractTaskId, formatUnknownError, normalizeBenchmarkContext, resolveHost, resolvePort, sessionKey, toPlugin, } from "./server-utils.js";
-// Load environment variables BEFORE anything else
-// This ensures API keys are available when plugins initialize
-dotenv.config({ path: path.resolve(process.cwd(), ".env") });
+import { CORE_PLUGINS } from "../../../agent/src/runtime/core-plugins.js";
+import { createElizaPlugin } from "../../../agent/src/runtime/eliza-plugin.js";
+import { autoWireCerebras } from "./cerebras-autowire.js";
+import { LifeOpsBenchHandler, } from "./lifeops-bench-handler.js";
+import { clearCapturedAction, createBenchmarkPlugin, getCapturedAction, getCapturedActions, setBenchmarkContext, } from "./plugin.js";
+import { benchmarkTurnMetadata, capturedActionsToToolCalls, capturedActionToParams, coerceActions, coerceParams, composeBenchmarkPrompt, createSession, ensureBenchmarkSessionContext, extractBenchmarkName, extractRecord, extractTaskId, formatUnknownError, normalizeBenchmarkContext, normalizeBenchmarkModelUsage, resolveHost, resolvePort, sessionKey, summarizeBenchmarkTurnUsage, toPlugin, } from "./server-utils.js";
+// `dotenv.config({ path: cwd/.env })` only finds the file when the bench server
+// is started from the repo root. When `ElizaServerManager` spawns us with
+// `cwd=packages/app-core`, there is no `.env` next to that directory — so the
+// repo-root `.env` is invisible and `CEREBRAS_API_KEY` arrives unset. Walk
+// upward looking for the first `.env` so the bench server works regardless of
+// where the parent process happened to anchor cwd.
+function loadEnvFromAncestors(startDir) {
+    let current = path.resolve(startDir);
+    for (let i = 0; i < 8; i += 1) {
+        const candidate = path.join(current, ".env");
+        if (
+        // node:fs is heavy at top-level for a single existence check; use dotenv's
+        // own behavior — it returns no parsed data for missing files. We still need to
+        // know *which* path matched so we can log it and stop walking.
+        dotenv.config({ path: candidate, override: false }).parsed !== undefined) {
+            return candidate;
+        }
+        const parent = path.dirname(current);
+        if (parent === current)
+            break;
+        current = parent;
+    }
+    return null;
+}
+const _loadedEnvPath = loadEnvFromAncestors(process.cwd());
+if (_loadedEnvPath) {
+    elizaLogger.debug(`[bench] Loaded env from ${_loadedEnvPath}`);
+}
+// Cerebras auto-wiring. See `./cerebras-autowire.ts` for the rationale and
+// the rules under which `CEREBRAS_API_KEY` / `CEREBRAS_BASE_URL` /
+// `CEREBRAS_MODEL` are promoted to OpenAI-compat env keys.
+autoWireCerebras();
 const BENCH_TOKEN = process.env.ELIZA_BENCH_TOKEN?.trim() || null;
 const OPENROUTER_PLUGIN_MODULE = "@elizaos/plugin-openrouter";
+const OPENAI_COMPAT_MAX_ATTEMPTS = envPositiveInt("CEREBRAS_BENCH_MAX_ATTEMPTS", 4);
+const OPENAI_COMPAT_RETRY_BASE_MS = envPositiveInt("CEREBRAS_BENCH_RETRY_BASE_MS", 4000);
+const OPENAI_COMPAT_RETRY_MAX_MS = envPositiveInt("CEREBRAS_BENCH_RETRY_MAX_MS", 30000);
+function envPositiveInt(name, fallback) {
+    const raw = process.env[name];
+    if (!raw)
+        return fallback;
+    const parsed = Number.parseInt(raw, 10);
+    return Number.isFinite(parsed) && parsed > 0 ? parsed : fallback;
+}
+function sleep(ms) {
+    return new Promise((resolve) => setTimeout(resolve, ms));
+}
+function isRetryableOpenAiCompatibleStatus(status) {
+    return status === 408 || status === 409 || status === 429 || status >= 500;
+}
+function openAiCompatibleRetryDelayMs(response, attempt) {
+    const retryAfter = response.headers.get("retry-after");
+    if (retryAfter) {
+        const seconds = Number.parseFloat(retryAfter);
+        if (Number.isFinite(seconds) && seconds > 0) {
+            return Math.min(Math.ceil(seconds * 1000), OPENAI_COMPAT_RETRY_MAX_MS);
+        }
+        const timestamp = Date.parse(retryAfter);
+        if (Number.isFinite(timestamp)) {
+            return Math.min(Math.max(timestamp - Date.now(), 0), OPENAI_COMPAT_RETRY_MAX_MS);
+        }
+    }
+    return (Math.min(OPENAI_COMPAT_RETRY_BASE_MS * 2 ** Math.max(attempt - 1, 0), OPENAI_COMPAT_RETRY_MAX_MS) + Math.floor(Math.random() * 250));
+}
+function normalizeBenchmarkTaskAgentEnv() {
+    const benchmarkRequested = process.env.BENCHMARK_TASK_AGENT?.trim();
+    const requested = benchmarkRequested ||
+        process.env.ELIZA_ACP_DEFAULT_AGENT?.trim() ||
+        process.env.ELIZA_DEFAULT_AGENT_TYPE?.trim();
+    if (!requested)
+        return;
+    const normalized = requested.toLowerCase().replace(/_/g, "-");
+    const acpAgent = normalized === "elizaos" ||
+        normalized === "eliza-os" ||
+        normalized === "pi-agent" ||
+        normalized === "pi agent"
+        ? "opencode"
+        : normalized === "claude-code" || normalized === "claude code"
+            ? "claude"
+            : normalized === "openai" ||
+                normalized === "openai-codex" ||
+                normalized === "openai codex"
+                ? "codex"
+                : normalized === "open-code" || normalized === "open code"
+                    ? "opencode"
+                    : normalized;
+    process.env.BENCHMARK_TASK_AGENT ??= requested;
+    process.env.ELIZA_AGENT_ORCHESTRATOR ??= "1";
+    process.env.ELIZA_AGENT_SELECTION_STRATEGY ??= "fixed";
+    if (benchmarkRequested) {
+        process.env.ELIZA_AGENT_SELECTION_STRATEGY = "fixed";
+        process.env.ELIZA_ACP_DEFAULT_AGENT = acpAgent;
+        process.env.ELIZA_DEFAULT_AGENT_TYPE = acpAgent;
+    }
+    else {
+        process.env.ELIZA_ACP_DEFAULT_AGENT ??= acpAgent;
+        process.env.ELIZA_DEFAULT_AGENT_TYPE ??= acpAgent;
+    }
+    elizaLogger.info(`[bench] Benchmark task-agent ${requested} mapped to ACP adapter ${acpAgent}`);
+}
+normalizeBenchmarkTaskAgentEnv();
+function isLocaBenchmarkName(benchmark) {
+    const normalized = benchmark.trim().toLowerCase();
+    return normalized === "loca_bench" || normalized === "loca-bench";
+}
+function isBfclBenchmarkName(benchmark) {
+    return benchmark.trim().toLowerCase() === "bfcl";
+}
+function _isTauBenchmarkName(benchmark) {
+    const normalized = benchmark.trim().toLowerCase();
+    return normalized === "tau_bench" || normalized === "tau-bench";
+}
+function isTerminalBenchmarkName(benchmark) {
+    const normalized = benchmark.trim().toLowerCase();
+    return normalized === "terminal-bench" || normalized === "terminal_bench";
+}
+function isSweBenchmarkName(benchmark) {
+    const normalized = benchmark.trim().toLowerCase();
+    return normalized === "swe-bench" || normalized === "swe_bench";
+}
+function isVisualWebBenchmarkName(benchmark) {
+    const normalized = benchmark.trim().toLowerCase();
+    return normalized === "visualwebbench" || normalized === "visual-web-bench";
+}
+function isWebShopBenchmarkName(benchmark) {
+    const normalized = benchmark.trim().toLowerCase();
+    return normalized === "webshop" || normalized === "web-shop";
+}
+function isOsworldBenchmarkName(benchmark) {
+    const normalized = benchmark.trim().toLowerCase();
+    return normalized === "osworld" || normalized === "os-world";
+}
+function isHermesNativeEnvProxyName(benchmark) {
+    const normalized = benchmark.trim().toLowerCase();
+    return (normalized === "hermes_native_env" || normalized === "hermes-native-env");
+}
+function isWooBenchName(benchmark) {
+    const normalized = benchmark.trim().toLowerCase();
+    return normalized === "woobench" || normalized === "woo-bench";
+}
+function isActionCallingBenchmarkName(benchmark) {
+    const normalized = benchmark.trim().toLowerCase();
+    return (normalized === "action-calling" ||
+        normalized === "action_calling" ||
+        normalized === "vending-bench" ||
+        normalized === "vending_bench" ||
+        normalized === "tau_bench" ||
+        normalized === "tau-bench");
+}
+function isVendingBenchmarkName(benchmark) {
+    const normalized = benchmark.trim().toLowerCase();
+    return normalized === "vending-bench" || normalized === "vending_bench";
+}
+// ---------------------------------------------------------------------------
+// Boundary adapters: the benchmark harness and the message normalizers below
+// build OpenAI chat-completion *wire* objects (snake_case `tool_calls` /
+// `tool_call_id`, free-form tool defs) which the direct HTTP path
+// (`callOpenAiCompatibleActionCalling`) forwards verbatim. `runtime.useModel`
+// instead consumes the typed `@elizaos/core` contracts (`ChatMessage[]` /
+// `ToolDefinition[]` / `ToolChoice`). These converters validate the loosely
+// typed wire data at that boundary and return genuinely well-formed core
+// objects, mapping snake_case wire keys onto the camelCase fields the runtime
+// reads.
+// ---------------------------------------------------------------------------
+const CHAT_MESSAGE_ROLES = new Set([
+    "system",
+    "developer",
+    "user",
+    "assistant",
+    "tool",
+]);
+function asChatMessageRole(value) {
+    return typeof value === "string" &&
+        CHAT_MESSAGE_ROLES.has(value)
+        ? value
+        : "user";
+}
+function wireToolCallToToolCall(value) {
+    if (!value || typeof value !== "object" || Array.isArray(value))
+        return null;
+    const call = value;
+    const fn = call.function && typeof call.function === "object"
+        ? call.function
+        : undefined;
+    const name = typeof call.name === "string"
+        ? call.name
+        : typeof fn?.name === "string"
+            ? fn.name
+            : "";
+    if (!name)
+        return null;
+    const rawArgs = call.arguments ?? fn?.arguments ?? {};
+    // OpenAI wire format carries tool-call arguments as a JSON string; match the
+    // file-wide convention (see `normalizeLocaIncomingToolCall`) and keep
+    // `ToolCall.arguments` as a string so no value-shape cast is needed.
+    const args = typeof rawArgs === "string" ? rawArgs : JSON.stringify(rawArgs);
+    const id = typeof call.id === "string" ? call.id : name;
+    return { id, name, arguments: args, type: "function" };
+}
+/**
+ * Convert OpenAI-wire chat messages into `ChatMessage[]` for `useModel`,
+ * mapping snake_case tool fields (`tool_calls`/`tool_call_id`) onto the
+ * camelCase `ChatMessage` fields the runtime reads.
+ */
+function toChatMessages(wire) {
+    return wire.map((message) => {
+        const role = asChatMessageRole(message.role);
+        const content = typeof message.content === "string" ? message.content : null;
+        const chatMessage = { role, content };
+        if (typeof message.name === "string")
+            chatMessage.name = message.name;
+        const toolCallId = message.tool_call_id ?? message.toolCallId;
+        if (typeof toolCallId === "string")
+            chatMessage.toolCallId = toolCallId;
+        const rawToolCalls = Array.isArray(message.tool_calls)
+            ? message.tool_calls
+            : Array.isArray(message.toolCalls)
+                ? message.toolCalls
+                : [];
+        const toolCalls = rawToolCalls
+            .map(wireToolCallToToolCall)
+            .filter((call) => call !== null);
+        if (toolCalls.length > 0)
+            chatMessage.toolCalls = toolCalls;
+        return chatMessage;
+    });
+}
+/**
+ * Convert harness-supplied tool definitions (OpenAI `{ type, function: {...} }`
+ * or flat `{ name, ... }`) into `ToolDefinition[]`. Entries without a usable
+ * name are dropped rather than fabricated.
+ */
+function toToolDefinitions(raw) {
+    const tools = [];
+    for (const entry of raw) {
+        const fn = entry.function && typeof entry.function === "object"
+            ? entry.function
+            : undefined;
+        const name = typeof entry.name === "string"
+            ? entry.name
+            : typeof fn?.name === "string"
+                ? fn.name
+                : "";
+        if (!name)
+            continue;
+        const description = typeof entry.description === "string"
+            ? entry.description
+            : typeof fn?.description === "string"
+                ? fn.description
+                : undefined;
+        const rawParameters = entry.parameters ?? fn?.parameters;
+        const parameters = rawParameters && typeof rawParameters === "object"
+            ? rawParameters
+            : undefined;
+        const tool = { name };
+        if (description !== undefined)
+            tool.description = description;
+        if (parameters !== undefined)
+            tool.parameters = parameters;
+        tools.push(tool);
+    }
+    return tools;
+}
+/** Narrow a benchmark-supplied tool-choice string to a `ToolChoice`. */
+function toToolChoice(value) {
+    return value === "none" || value === "auto" || value === "required"
+        ? value
+        : "required";
+}
+function normalizeActionCallingNativeMessages(text, context) {
+    const rawMessages = Array.isArray(context.messages) ? context.messages : [];
+    const messages = normalizeLocaNativeMessages(rawMessages);
+    messages[0] = {
+        role: "system",
+        content: "You are running an action-calling benchmark through the Eliza benchmark server. " +
+            "Use native tool calls only. Do not serialize tool calls in prose, XML, markdown, or JSON text. " +
+            "If the user asks for multiple operations, emit every required tool call.",
+    };
+    if (messages.length > 1)
+        return messages;
+    return [
+        messages[0],
+        {
+            role: "user",
+            content: text,
+        },
+    ];
+}
+function normalizeActionCallingOpenAiMessages(text, context) {
+    const rawMessages = Array.isArray(context.messages) ? context.messages : [];
+    const messages = rawMessages
+        .map((message) => message && typeof message === "object" && !Array.isArray(message)
+        ? { ...message }
+        : null)
+        .filter((message) => message !== null)
+        .filter((message) => typeof message.role === "string");
+    const systemMessage = {
+        role: "system",
+        content: "Use native function/tool calls for any requested operation. If several operations are required, call every required tool; after a tool result, continue with the remaining required tool calls. Do not serialize tool calls in text, XML, markdown, or JSON. Return assistant text only when no tool call is needed.",
+    };
+    if (messages.length > 0 && messages[0]?.role === "system") {
+        messages[0] = systemMessage;
+    }
+    else {
+        messages.unshift(systemMessage);
+    }
+    if (messages.some((message) => message.role === "user")) {
+        return messages;
+    }
+    messages.push({ role: "user", content: text });
+    return messages;
+}
+function normalizeWooBenchNativeMessages(text, context) {
+    const rawMessages = Array.isArray(context.messages) ? context.messages : [];
+    const messages = rawMessages
+        .map((message) => message && typeof message === "object" && !Array.isArray(message)
+        ? { ...message }
+        : null)
+        .filter((message) => message !== null)
+        .filter((message) => typeof message.role === "string");
+    const systemPrompt = typeof context.system_prompt === "string" && context.system_prompt.trim()
+        ? context.system_prompt.trim()
+        : "You are running WooBench. Respond naturally, and use payment tools when charging or checking payment.";
+    const systemMessage = {
+        role: "system",
+        content: systemPrompt +
+            "\n\nUse native tool calls for CREATE_APP_CHARGE and CHECK_PAYMENT. " +
+            "When you charge or check payment, include a short conversational message in assistant text. " +
+            "Do not serialize tool calls in JSON, XML, markdown, or prose.",
+    };
+    if (messages.length > 0 && messages[0]?.role === "system") {
+        messages[0] = systemMessage;
+    }
+    else {
+        messages.unshift(systemMessage);
+    }
+    if (messages.some((message) => message.role === "user")) {
+        return messages;
+    }
+    messages.push({ role: "user", content: text });
+    return messages;
+}
+function resolveOpenAiCompatibleActionCallingConfig() {
+    const provider = (process.env.BENCHMARK_MODEL_PROVIDER ||
+        process.env.ELIZA_PROVIDER ||
+        "")
+        .trim()
+        .toLowerCase();
+    const model = process.env.BENCHMARK_MODEL_NAME?.trim() ||
+        process.env.OPENAI_LARGE_MODEL?.trim() ||
+        process.env.LARGE_MODEL?.trim() ||
+        process.env.CEREBRAS_MODEL?.trim() ||
+        "";
+    const baseUrl = process.env.OPENAI_BASE_URL?.trim() ||
+        process.env.CEREBRAS_BASE_URL?.trim() ||
+        (provider === "cerebras" ? "https://api.cerebras.ai/v1" : "");
+    const baseUrlIsCerebras = /(^|\.)cerebras\.ai(\/|$)/i.test(baseUrl);
+    const apiKey = baseUrlIsCerebras || provider === "cerebras"
+        ? process.env.CEREBRAS_API_KEY?.trim() ||
+            process.env.OPENAI_API_KEY?.trim() ||
+            ""
+        : process.env.OPENAI_API_KEY?.trim() || "";
+    if (!model || !baseUrl || !apiKey)
+        return null;
+    return {
+        baseUrl,
+        apiKey,
+        model,
+        provider: provider || (baseUrlIsCerebras ? "cerebras" : "openai"),
+    };
+}
+function chatCompletionsUrl(baseUrl) {
+    const trimmed = baseUrl.replace(/\/+$/, "");
+    return trimmed.endsWith("/chat/completions")
+        ? trimmed
+        : `${trimmed}/chat/completions`;
+}
+function pickUsageNumber(source, ...keys) {
+    if (!source)
+        return undefined;
+    for (const key of keys) {
+        const value = source[key];
+        if (typeof value === "number" && Number.isFinite(value))
+            return value;
+        if (typeof value === "string" && value.trim()) {
+            const parsed = Number(value);
+            if (Number.isFinite(parsed))
+                return parsed;
+        }
+    }
+    return undefined;
+}
+function normalizeOpenAiCompatibleUsage(usage, provider) {
+    if (!usage || typeof usage !== "object" || Array.isArray(usage)) {
+        return null;
+    }
+    const record = usage;
+    const promptDetails = record.prompt_tokens_details &&
+        typeof record.prompt_tokens_details === "object" &&
+        !Array.isArray(record.prompt_tokens_details)
+        ? record.prompt_tokens_details
+        : undefined;
+    const inputDetails = record.input_tokens_details &&
+        typeof record.input_tokens_details === "object" &&
+        !Array.isArray(record.input_tokens_details)
+        ? record.input_tokens_details
+        : undefined;
+    const promptTokens = pickUsageNumber(record, "prompt_tokens", "input_tokens", "promptTokens") ??
+        0;
+    const completionTokens = pickUsageNumber(record, "completion_tokens", "output_tokens", "completionTokens") ?? 0;
+    const totalTokens = pickUsageNumber(record, "total_tokens", "totalTokens") ??
+        promptTokens + completionTokens;
+    const cacheReadInputTokens = pickUsageNumber(record, "cache_read_input_tokens", "cached_tokens", "cachedInputTokens", "cacheReadInputTokens") ??
+        pickUsageNumber(promptDetails, "cached_tokens", "cache_read_input_tokens") ??
+        pickUsageNumber(inputDetails, "cached_tokens", "cache_read_input_tokens");
+    const cacheCreationInputTokens = pickUsageNumber(record, "cache_creation_input_tokens", "cacheCreationInputTokens") ??
+        pickUsageNumber(promptDetails, "cache_creation_input_tokens", "cacheCreationInputTokens") ??
+        pickUsageNumber(inputDetails, "cache_creation_input_tokens", "cacheCreationInputTokens");
+    return {
+        modelType: ModelType.TEXT_LARGE,
+        provider,
+        source: "openai-compatible-chat-completions",
+        promptTokens,
+        completionTokens,
+        totalTokens,
+        ...(cacheReadInputTokens !== undefined
+            ? { cachedTokens: cacheReadInputTokens, cacheReadInputTokens }
+            : {}),
+        ...(cacheCreationInputTokens !== undefined
+            ? { cacheCreationInputTokens }
+            : {}),
+    };
+}
+async function callOpenAiCompatibleActionCalling(params) {
+    const config = resolveOpenAiCompatibleActionCallingConfig();
+    if (!config)
+        return null;
+    const requestPayload = {
+        model: config.model,
+        messages: params.messages,
+        max_tokens: params.maxTokens,
+        temperature: params.temperature,
+    };
+    if (params.tools.length > 0) {
+        requestPayload.tools = params.tools;
+        requestPayload.tool_choice =
+            params.toolChoice === "none"
+                ? "none"
+                : params.toolChoice === "auto"
+                    ? "auto"
+                    : params.toolChoice || "required";
+    }
+    const requestBody = JSON.stringify(requestPayload);
+    let response = null;
+    for (let attempt = 1; attempt <= OPENAI_COMPAT_MAX_ATTEMPTS; attempt += 1) {
+        response = await fetch(chatCompletionsUrl(config.baseUrl), {
+            method: "POST",
+            headers: {
+                Authorization: `Bearer ${config.apiKey}`,
+                "Content-Type": "application/json",
+            },
+            body: requestBody,
+        });
+        if (response.ok ||
+            !isRetryableOpenAiCompatibleStatus(response.status) ||
+            attempt >= OPENAI_COMPAT_MAX_ATTEMPTS) {
+            break;
+        }
+        const delayMs = openAiCompatibleRetryDelayMs(response, attempt);
+        elizaLogger.warn(`[bench] OpenAI-compatible action-calling request failed (${response.status}); retrying in ${delayMs}ms (attempt ${attempt}/${OPENAI_COMPAT_MAX_ATTEMPTS})`);
+        await sleep(delayMs);
+    }
+    if (!response) {
+        throw new Error("OpenAI-compatible action-calling request was not sent");
+    }
+    if (!response.ok) {
+        const body = await response.text().catch(() => "");
+        throw new Error(`OpenAI-compatible action-calling request failed (${response.status}): ${body.slice(0, 500)}`);
+    }
+    const payload = (await response.json());
+    const choice = Array.isArray(payload.choices)
+        ? payload.choices[0]
+        : undefined;
+    const message = choice?.message &&
+        typeof choice.message === "object" &&
+        !Array.isArray(choice.message)
+        ? choice.message
+        : {};
+    return {
+        text: typeof message.content === "string" ? message.content : "",
+        toolCalls: normalizeLocaNativeToolCalls(message.tool_calls),
+        usage: normalizeOpenAiCompatibleUsage(payload.usage, config.provider),
+    };
+}
+async function callOpenAiCompatibleText(params) {
+    const config = resolveOpenAiCompatibleActionCallingConfig();
+    if (!config)
+        return null;
+    const requestBody = JSON.stringify({
+        model: config.model,
+        messages: [{ role: "user", content: params.prompt }],
+        max_tokens: params.maxTokens,
+        temperature: params.temperature,
+        ...(config.provider === "cerebras" ? { reasoning_effort: "low" } : {}),
+    });
+    let response = null;
+    for (let attempt = 1; attempt <= OPENAI_COMPAT_MAX_ATTEMPTS; attempt += 1) {
+        response = await fetch(chatCompletionsUrl(config.baseUrl), {
+            method: "POST",
+            headers: {
+                Authorization: `Bearer ${config.apiKey}`,
+                "Content-Type": "application/json",
+            },
+            body: requestBody,
+        });
+        if (response.ok ||
+            !isRetryableOpenAiCompatibleStatus(response.status) ||
+            attempt >= OPENAI_COMPAT_MAX_ATTEMPTS) {
+            break;
+        }
+        const delayMs = openAiCompatibleRetryDelayMs(response, attempt);
+        elizaLogger.warn(`[bench] OpenAI-compatible text request failed (${response.status}); retrying in ${delayMs}ms (attempt ${attempt}/${OPENAI_COMPAT_MAX_ATTEMPTS})`);
+        await sleep(delayMs);
+    }
+    if (!response) {
+        throw new Error("OpenAI-compatible text request was not sent");
+    }
+    if (!response.ok) {
+        const body = await response.text().catch(() => "");
+        throw new Error(`OpenAI-compatible text request failed (${response.status}): ${body.slice(0, 500)}`);
+    }
+    const payload = (await response.json());
+    const choice = Array.isArray(payload.choices)
+        ? payload.choices[0]
+        : undefined;
+    const message = choice?.message &&
+        typeof choice.message === "object" &&
+        !Array.isArray(choice.message)
+        ? choice.message
+        : {};
+    return {
+        text: typeof message.content === "string" ? message.content : "",
+        usage: normalizeOpenAiCompatibleUsage(payload.usage, config.provider),
+    };
+}
+function normalizeBfclNativeMessages(text, context) {
+    const question = typeof context.question === "string" && context.question.trim()
+        ? context.question.trim()
+        : text;
+    return [
+        {
+            role: "system",
+            content: "You are running BFCL through the Eliza benchmark server. Use native " +
+                "tool calls only. If the query asks for multiple or parallel calls, " +
+                "emit one tool call for each requested operation in the same assistant " +
+                "turn. Preserve schema field names and defaults exactly.",
+        },
+        {
+            role: "user",
+            content: question,
+        },
+    ];
+}
+function _normalizeTauNativeMessages(text, context) {
+    const rawMessages = Array.isArray(context.messages) ? context.messages : [];
+    const messages = normalizeLocaNativeMessages(rawMessages);
+    messages[0] = {
+        role: "system",
+        content: "You are running TauBench through the Eliza benchmark server. Use " +
+            "native tool calls for TauBench tools. Do not describe a tool call in " +
+            "prose. Use ordinary assistant text only for required customer " +
+            "confirmation or final task completion.",
+    };
+    if (messages.length > 1)
+        return messages;
+    return [
+        messages[0],
+        {
+            role: "user",
+            content: text,
+        },
+    ];
+}
+function normalizeLocaNativeMessages(rawMessages) {
+    const input = Array.isArray(rawMessages) ? rawMessages : [];
+    const toolNamesById = new Map();
+    const normalized = [
+        {
+            role: "system",
+            content: "You are running LOCA-bench through the Eliza benchmark server. " +
+                "Use native tool calls, not progress text. If work remains, call " +
+                "exactly one available filesystem or memory tool. Existing CSV rows " +
+                "may be examples; derive final rows from source_data/local_db and " +
+                "source_data/files, then write the requested CSV files.",
+        },
+    ];
+    for (const item of input) {
+        if (!item || typeof item !== "object" || Array.isArray(item))
+            continue;
+        const message = item;
+        const role = typeof message.role === "string" ? message.role : "user";
+        if (role === "assistant") {
+            const rawToolCalls = Array.isArray(message.tool_calls)
+                ? message.tool_calls
+                : Array.isArray(message.toolCalls)
+                    ? message.toolCalls
+                    : [];
+            const toolCalls = rawToolCalls
+                .map((call) => normalizeLocaIncomingToolCall(call))
+                .filter((call) => Boolean(call));
+            for (const call of toolCalls) {
+                const id = typeof call.id === "string" ? call.id : "";
+                const fn = call.function && typeof call.function === "object"
+                    ? call.function
+                    : {};
+                const name = typeof fn.name === "string" ? fn.name : "";
+                if (id && name)
+                    toolNamesById.set(id, name);
+            }
+            normalized.push({
+                role: "assistant",
+                content: typeof message.content === "string" ? message.content : "",
+                ...(toolCalls.length > 0 ? { tool_calls: toolCalls } : {}),
+            });
+            continue;
+        }
+        if (role === "tool") {
+            const toolCallId = typeof message.tool_call_id === "string"
+                ? message.tool_call_id
+                : typeof message.toolCallId === "string"
+                    ? message.toolCallId
+                    : typeof message.id === "string"
+                        ? message.id
+                        : "tool-call";
+            const toolName = typeof message.name === "string"
+                ? message.name
+                : typeof message.toolName === "string"
+                    ? message.toolName
+                    : toolNamesById.get(toolCallId) || "tool";
+            normalized.push({
+                role: "tool",
+                tool_call_id: toolCallId,
+                name: toolName,
+                content: typeof message.content === "string"
+                    ? message.content
+                    : JSON.stringify(message.content ?? ""),
+            });
+            continue;
+        }
+        normalized.push({
+            role: role === "system" ? "system" : "user",
+            content: typeof message.content === "string"
+                ? message.content
+                : JSON.stringify(message.content ?? ""),
+        });
+    }
+    return normalized;
+}
+function normalizeGenericToolMessages(rawMessages, fallbackText) {
+    const input = Array.isArray(rawMessages) ? rawMessages : [];
+    const toolNamesById = new Map();
+    const normalized = [];
+    for (const item of input) {
+        if (!item || typeof item !== "object" || Array.isArray(item))
+            continue;
+        const message = item;
+        const role = typeof message.role === "string" ? message.role : "user";
+        if (role === "assistant") {
+            const rawToolCalls = Array.isArray(message.tool_calls)
+                ? message.tool_calls
+                : Array.isArray(message.toolCalls)
+                    ? message.toolCalls
+                    : [];
+            const toolCalls = rawToolCalls
+                .map((call) => normalizeLocaIncomingToolCall(call))
+                .filter((call) => Boolean(call));
+            for (const call of toolCalls) {
+                const id = typeof call.id === "string" ? call.id : "";
+                const fn = call.function && typeof call.function === "object"
+                    ? call.function
+                    : {};
+                const name = typeof fn.name === "string" ? fn.name : "";
+                if (id && name)
+                    toolNamesById.set(id, name);
+            }
+            normalized.push({
+                role: "assistant",
+                content: typeof message.content === "string" ? message.content : "",
+                ...(toolCalls.length > 0 ? { tool_calls: toolCalls } : {}),
+            });
+            continue;
+        }
+        if (role === "tool") {
+            const toolCallId = typeof message.tool_call_id === "string"
+                ? message.tool_call_id
+                : typeof message.toolCallId === "string"
+                    ? message.toolCallId
+                    : typeof message.id === "string"
+                        ? message.id
+                        : "tool-call";
+            const toolName = typeof message.name === "string"
+                ? message.name
+                : typeof message.toolName === "string"
+                    ? message.toolName
+                    : toolNamesById.get(toolCallId) || "tool";
+            normalized.push({
+                role: "tool",
+                tool_call_id: toolCallId,
+                name: toolName,
+                content: typeof message.content === "string"
+                    ? message.content
+                    : JSON.stringify(message.content ?? ""),
+            });
+            continue;
+        }
+        normalized.push({
+            role: role === "system" ? "system" : "user",
+            content: typeof message.content === "string"
+                ? message.content
+                : JSON.stringify(message.content ?? ""),
+        });
+    }
+    if (normalized.length === 0) {
+        normalized.push({ role: "user", content: fallbackText });
+    }
+    return normalized;
+}
+function normalizeLocaIncomingToolCall(raw) {
+    if (!raw || typeof raw !== "object" || Array.isArray(raw))
+        return null;
+    const call = raw;
+    const fn = call.function && typeof call.function === "object"
+        ? call.function
+        : {};
+    const name = typeof fn.name === "string"
+        ? fn.name
+        : typeof call.name === "string"
+            ? call.name
+            : typeof call.toolName === "string"
+                ? call.toolName
+                : "";
+    if (!name)
+        return null;
+    const args = fn.arguments ?? call.arguments ?? call.input ?? {};
+    return {
+        id: typeof call.id === "string"
+            ? call.id
+            : typeof call.toolCallId === "string"
+                ? call.toolCallId
+                : `call_loca_${Math.random().toString(16).slice(2)}`,
+        type: "function",
+        function: {
+            name,
+            arguments: typeof args === "string" ? args : JSON.stringify(args),
+        },
+    };
+}
+function normalizeLocaNativeToolCalls(rawToolCalls) {
+    if (!Array.isArray(rawToolCalls))
+        return [];
+    const calls = [];
+    for (const raw of rawToolCalls) {
+        if (!raw || typeof raw !== "object" || Array.isArray(raw))
+            continue;
+        const call = raw;
+        const fn = call.function && typeof call.function === "object"
+            ? call.function
+            : {};
+        const name = typeof call.toolName === "string"
+            ? call.toolName
+            : typeof call.name === "string"
+                ? call.name
+                : typeof fn.name === "string"
+                    ? fn.name
+                    : "";
+        if (!name)
+            continue;
+        const args = call.input ?? call.args ?? call.arguments ?? fn.arguments ?? {};
+        calls.push({
+            id: typeof call.toolCallId === "string"
+                ? call.toolCallId
+                : typeof call.id === "string"
+                    ? call.id
+                    : `call_loca_native_${calls.length}`,
+            type: "function",
+            function: {
+                name,
+                arguments: typeof args === "string" ? args : JSON.stringify(args),
+            },
+        });
+    }
+    return calls;
+}
+function firstLocaBenchmarkActionFromToolCalls(toolCalls) {
+    const first = toolCalls[0];
+    if (!first)
+        return null;
+    let args = {};
+    try {
+        args = JSON.parse(first.function.arguments || "{}");
+    }
+    catch {
+        args = { _raw: first.function.arguments };
+    }
+    return {
+        tool_name: first.function.name,
+        arguments: args,
+    };
+}
+function firstWooBenchActionFromToolCalls(toolCalls) {
+    const first = toolCalls[0];
+    if (!first)
+        return null;
+    const command = first.function.name.trim().toUpperCase();
+    if (command !== "CREATE_APP_CHARGE" && command !== "CHECK_PAYMENT") {
+        return null;
+    }
+    let args = {};
+    try {
+        args = JSON.parse(first.function.arguments || "{}");
+    }
+    catch {
+        args = {};
+    }
+    const payload = args && typeof args === "object" && !Array.isArray(args)
+        ? { ...args }
+        : {};
+    return { command, ...payload };
+}
+function bfclBenchmarkActionFromToolCalls(toolCalls) {
+    if (toolCalls.length === 0)
+        return null;
+    const calls = toolCalls.map((call) => {
+        let args = {};
+        try {
+            args = JSON.parse(call.function.arguments || "{}");
+        }
+        catch {
+            args = { _raw: call.function.arguments };
+        }
+        return {
+            name: call.function.name,
+            arguments: args,
+        };
+    });
+    return {
+        calls,
+        arguments: { calls },
+    };
+}
+function webshopBenchmarkActionFromToolCalls(toolCalls) {
+    for (const call of toolCalls) {
+        const name = call.function.name.toLowerCase();
+        if (name !== "webshop_action" && name !== "benchmark_action") {
+            continue;
+        }
+        let args = {};
+        try {
+            args = JSON.parse(call.function.arguments || "{}");
+        }
+        catch {
+            args = { _raw: call.function.arguments };
+        }
+        if (!args || typeof args !== "object" || Array.isArray(args)) {
+            continue;
+        }
+        const record = args;
+        const command = typeof record.command === "string"
+            ? record.command.trim()
+            : typeof record.action === "string"
+                ? record.action.trim()
+                : "";
+        if (command) {
+            return { command };
+        }
+    }
+    return null;
+}
 // ---------------------------------------------------------------------------
 // Security: authentication + CORS
 // ---------------------------------------------------------------------------
@@ -31,6 +906,127 @@ const MAX_BODY_BYTES = Number.isFinite(configuredMaxBodyBytes) && configuredMaxB
     : DEFAULT_MAX_BODY_BYTES;
 /** Allowed CORS origins — only localhost variants. */
 const LOCALHOST_ORIGINS = new Set(["http://localhost", "https://localhost"]);
+function buildLifeOpsBenchmarkContext(backend, previousTurns) {
+    const world = backend.toDocument();
+    const nowIso = backend.getNow();
+    const nowMs = Date.parse(nowIso);
+    const calendarEvents = Object.values(world.stores.calendar_event)
+        .filter((event) => event.status !== "cancelled")
+        .sort((a, b) => {
+        const aDistance = Number.isFinite(nowMs)
+            ? Math.abs(Date.parse(a.start) - nowMs)
+            : 0;
+        const bDistance = Number.isFinite(nowMs)
+            ? Math.abs(Date.parse(b.start) - nowMs)
+            : 0;
+        if (aDistance !== bDistance)
+            return aDistance - bDistance;
+        return a.id.localeCompare(b.id);
+    })
+        .slice(0, 80)
+        .map((event) => ({
+        id: event.id,
+        calendarId: event.calendar_id,
+        title: event.title,
+        start: event.start,
+        end: event.end,
+        status: event.status,
+        source: event.source,
+    }));
+    const previousToolResults = previousTurns
+        .flatMap((turn) => turn.toolCalls.map((call) => ({
+        userText: turn.userText,
+        assistantText: turn.assistantText,
+        tool: call.name,
+        arguments: call.arguments,
+        ok: call.ok,
+        result: call.result,
+        error: call.error,
+    })))
+        .slice(-12);
+    return {
+        nowIso,
+        today: nowIso.slice(0, 10),
+        seed: backend.getSeed(),
+        calendarEvents,
+        previousToolResults,
+    };
+}
+function buildLifeOpsActionCallingMessages(params) {
+    const contextJson = JSON.stringify(params.lifeopsContext, null, 2);
+    return [
+        {
+            role: "system",
+            content: "You are running LifeOpsBench through the Eliza benchmark server. " +
+                "Use native tool calls for calendar, mail, message, task, and related LifeOps operations. " +
+                "For free/busy or availability questions, call CALENDAR with action and subaction exactly " +
+                "check_availability and provide top-level startAt/endAt ISO timestamps; do not use search_events. " +
+                "Do not serialize tool calls in text, XML, markdown, or JSON. " +
+                "After a tool call, the benchmark backend will execute it and feed back the result on the next turn. " +
+                "Return assistant text only when no tool call is needed.\n\n" +
+                `LifeOps benchmark context:\n${contextJson}`,
+        },
+        {
+            role: "user",
+            content: params.userText,
+        },
+    ];
+}
+function lifeOpsToolCallsFromNativeToolCalls(toolCalls) {
+    return toolCalls.map((call, index) => {
+        let parsedArgs = {};
+        try {
+            parsedArgs = JSON.parse(call.function.arguments || "{}");
+        }
+        catch {
+            parsedArgs = {};
+        }
+        return {
+            id: call.id || `call_${index}`,
+            name: call.function.name,
+            arguments: parsedArgs &&
+                typeof parsedArgs === "object" &&
+                !Array.isArray(parsedArgs)
+                ? parsedArgs
+                : {},
+        };
+    });
+}
+function shouldDropLifeOpsReadOnlyFollowupToolCalls(params) {
+    if (params.userText.trim() || !params.responseText.trim())
+        return false;
+    if (params.toolCalls.length === 0)
+        return false;
+    const onlyReminderCreates = params.toolCalls.every((call) => {
+        const name = call.name.trim().toUpperCase();
+        const action = String(call.arguments.action ?? "").toLowerCase();
+        return ((name === "SCHEDULED_TASKS" || name === "REMINDERS") &&
+            (action === "create" || action === "add" || action === ""));
+    });
+    if (!onlyReminderCreates)
+        return false;
+    const previousToolResults = params.lifeopsContext.previousToolResults;
+    if (!Array.isArray(previousToolResults))
+        return false;
+    return previousToolResults.some((entry) => {
+        if (!entry || typeof entry !== "object")
+            return false;
+        const record = entry;
+        if (record.ok !== true)
+            return false;
+        const tool = String(record.tool ?? "").toUpperCase();
+        if (!tool.startsWith("CALENDAR"))
+            return false;
+        const args = record.arguments && typeof record.arguments === "object"
+            ? record.arguments
+            : {};
+        const result = record.result && typeof record.result === "object"
+            ? record.result
+            : {};
+        const subaction = String(args.subaction ?? args.action ?? result.subaction ?? "").toLowerCase();
+        return subaction === "check_availability";
+    });
+}
 function isAllowedOrigin(origin) {
     if (!origin)
         return false;
@@ -97,7 +1093,7 @@ function disableManualCompactionAction(runtime) {
     if (!Array.isArray(runtimeWithActions.actions)) {
         return;
     }
-    const compactSessionIndex = runtimeWithActions.actions.findIndex((action) => action?.name?.toUpperCase() === "COMPACT_SESSION");
+    const compactSessionIndex = runtimeWithActions.actions.findIndex((action) => action.name.toUpperCase() === "COMPACT_SESSION");
     if (compactSessionIndex === -1) {
         return;
     }
@@ -137,7 +1133,7 @@ async function collectSessionDiagnostics(runtime, session) {
         }),
     ]);
     const compactionSummaries = allMessages
-        .filter((m) => m.content?.source === "compaction")
+        .filter((m) => m.content.source === "compaction")
         .sort((a, b) => (a.createdAt ?? 0) - (b.createdAt ?? 0));
     const latestCompactionSummary = compactionSummaries.at(-1) ?? null;
     const latestSummaryText = typeof latestCompactionSummary?.content?.text === "string"
@@ -146,10 +1142,10 @@ async function collectSessionDiagnostics(runtime, session) {
     const summaryPreview = latestSummaryText.slice(0, 400);
     const providerNames = runtime.providers.map((provider) => provider.name);
     const evaluatorNames = runtime.evaluators
-        ?.map((evaluator) => evaluator?.name ?? "")
+        ?.map((evaluator) => evaluator.name ?? "")
         .filter((name) => name.length > 0) ?? [];
     const actionNames = runtime.actions
-        ?.map((action) => action?.name?.toUpperCase() ?? "")
+        ?.map((action) => action.name?.toUpperCase() ?? "")
         .filter((name) => name.length > 0) ?? [];
     return {
         benchmark: session.benchmark,
@@ -188,10 +1184,20 @@ async function collectSessionDiagnostics(runtime, session) {
         actions: actionNames,
     };
 }
-// Proper robust server implementation
 export async function startBenchmarkServer() {
     const port = resolvePort();
     elizaLogger.info(`[bench] Initializing eliza benchmark runtime on port ${port}...`);
+    // Force the v5 planner to require a structured tool call on every benchmark
+    // turn (unless explicitly disabled). Without this, the planner often picks
+    // `REPLY` and emits the answer as prose, which scores 0 against harnesses
+    // like LifeOpsBench that judge on tool calls (`MESSAGE.triage`,
+    // `CALENDAR.create_event`, etc.). The core gate in `services/message.ts`
+    // (see `isBenchmarkForcingToolCall`) honors this env var ONLY for messages
+    // whose `content.source === "benchmark"` or whose `content.metadata.benchmark`
+    // is set, so a co-resident chat process is unaffected.
+    if (process.env.ELIZA_BENCH_FORCE_TOOL_CALL === undefined) {
+        process.env.ELIZA_BENCH_FORCE_TOOL_CALL = "1";
+    }
     // ═══════════════════════════════════════════════════════════════════════════
     // PLUGIN LOADING — Use full CORE_PLUGINS to test with realistic context
     // ═══════════════════════════════════════════════════════════════════════════
@@ -208,13 +1214,45 @@ export async function startBenchmarkServer() {
     const skipPlugins = new Set([
         "@elizaos/plugin-elizacloud", // Requires elizaOS cloud auth, conflicts with local LLM
     ]);
+    const initialOpenAiBaseUrl = process.env.OPENAI_BASE_URL?.trim();
+    const initialElizaProvider = process.env.ELIZA_PROVIDER?.trim().toLowerCase();
+    const initialBenchProvider = process.env.BENCHMARK_MODEL_PROVIDER?.trim().toLowerCase();
+    const initialCerebrasIntent = (!!initialOpenAiBaseUrl &&
+        /(^|\.)cerebras\.ai(\/|$)/i.test(initialOpenAiBaseUrl)) ||
+        initialElizaProvider === "cerebras" ||
+        initialBenchProvider === "cerebras";
+    // Local-inference stays enabled by default in benchmark mode so embedding,
+    // memory, and retrieval behavior remain representative of the Eliza-1 stack.
+    // A zero-vector stand-in is allowed only as an explicit diagnostic escape
+    // hatch, and logs loudly because those runs are not release evidence.
+    const skipEmbeddingPlugin = process.env.ELIZA_BENCH_ALLOW_STUB_EMBEDDING === "1" ||
+        process.env.ELIZA_BENCH_SKIP_EMBEDDING === "1";
+    if (skipEmbeddingPlugin) {
+        skipPlugins.add("@elizaos/plugin-local-inference");
+    }
+    if (initialCerebrasIntent && !skipEmbeddingPlugin) {
+        skipPlugins.add("@elizaos/plugin-local-inference");
+        elizaLogger.info("[bench] Cerebras benchmark mode: using @elizaos/plugin-openai's deterministic local TEXT_EMBEDDING fallback instead of @elizaos/plugin-local-inference without an active backend.");
+    }
     const skipCorePlugins = process.env.ELIZA_BENCH_SKIP_CORE_PLUGINS === "true";
-    const corePluginsToLoad = skipCorePlugins
+    const corePluginsToLoadBase = skipCorePlugins
         ? ["@elizaos/plugin-sql"]
         : CORE_PLUGINS;
+    const shouldLoadTaskAgentPlugin = Boolean(process.env.BENCHMARK_TASK_AGENT?.trim() ||
+        process.env.ELIZA_ACP_DEFAULT_AGENT?.trim() ||
+        process.env.ELIZA_DEFAULT_AGENT_TYPE?.trim());
+    const corePluginsToLoad = shouldLoadTaskAgentPlugin
+        ? Array.from(new Set([
+            ...corePluginsToLoadBase,
+            "@elizaos/plugin-agent-orchestrator",
+        ]))
+        : corePluginsToLoadBase;
     if (skipCorePlugins) {
         elizaLogger.info("[bench] Loading minimal core plugins for benchmark smoke run");
     }
+    if (shouldLoadTaskAgentPlugin) {
+        elizaLogger.info("[bench] Loading @elizaos/plugin-agent-orchestrator for benchmark task-agent routing");
+    }
     // Load all CORE_PLUGINS by default; smoke runs can opt into the minimal
     // required set so credential-free bridge checks start quickly.
     for (const pluginName of corePluginsToLoad) {
@@ -274,8 +1312,29 @@ export async function startBenchmarkServer() {
     catch (error) {
         elizaLogger.error(`[bench] Failed to load benchmark plugin: ${formatUnknownError(error)}`);
     }
-    // Trust is now a built-in core capability — enable via ENABLE_TRUST character setting.
-    // No need to load as a separate plugin.
+    // Register a zero-vector TEXT_EMBEDDING stand-in only when explicitly
+    // requested. The runtime calls `useModel(TEXT_EMBEDDING, ...)` for every
+    // persisted memory; without ANY handler, those calls throw and abort the
+    // turn. This path is diagnostic-only because it does not measure real
+    // Eliza-1 retrieval behavior.
+    if (skipEmbeddingPlugin) {
+        const EMBEDDING_DIMENSIONS = 1024;
+        const benchEmbeddingPlugin = {
+            name: "@elizaos/bench-stub-embedding",
+            description: "Benchmark-mode zero-vector TEXT_EMBEDDING handler. Replaces " +
+                "@elizaos/plugin-local-inference only when " +
+                "ELIZA_BENCH_ALLOW_STUB_EMBEDDING=1 is set.",
+            // Higher than local-embedding's `priority: 10` so we win even if a
+            // CORE_PLUGINS race were to register a competing handler later.
+            priority: 100,
+            models: {
+                TEXT_EMBEDDING: async () => new Array(EMBEDDING_DIMENSIONS).fill(0),
+            },
+        };
+        plugins.push(toPlugin(benchEmbeddingPlugin, "bench-stub-embedding"));
+        elizaLogger.warn(`[bench] Registered zero-vector TEXT_EMBEDDING stand-in (dim=${EMBEDDING_DIMENSIONS}, standIn=true); ` +
+            "this run is not valid release evidence. Unset ELIZA_BENCH_ALLOW_STUB_EMBEDDING and ELIZA_BENCH_SKIP_EMBEDDING to use @elizaos/plugin-local-inference.");
+    }
     // Load LLM provider plugins based on environment.
     //
     // Multi-plugin guard: when both Groq and another OpenAI-compatible
@@ -288,10 +1347,9 @@ export async function startBenchmarkServer() {
     // wrong on my end. Please try again."). Suppress Groq when the
     // explicit intent is a different provider.
     const groqApiKey = process.env.GROQ_API_KEY?.trim();
-    const _openAiBaseUrl = process.env.OPENAI_BASE_URL?.trim();
-    const _cerebrasIntent = !!_openAiBaseUrl && /(^|\.)cerebras\.ai(\/|$)/i.test(_openAiBaseUrl);
-    const _explicitProvider = process.env.MILADY_PROVIDER?.trim().toLowerCase();
-    const _benchProvider = process.env.BENCHMARK_MODEL_PROVIDER?.trim().toLowerCase();
+    const _cerebrasIntent = initialCerebrasIntent;
+    const _explicitProvider = initialElizaProvider;
+    const _benchProvider = initialBenchProvider;
     const _suppressGroqForOtherProvider = _cerebrasIntent ||
         (_explicitProvider !== undefined &&
             _explicitProvider !== "" &&
@@ -312,7 +1370,7 @@ export async function startBenchmarkServer() {
     }
     else if (groqApiKey && _suppressGroqForOtherProvider) {
         elizaLogger.info("[bench] Skipping @elizaos/plugin-groq: another provider is the explicit intent " +
-            `(cerebras=${_cerebrasIntent}, MILADY_PROVIDER=${_explicitProvider ?? ""}, BENCHMARK_MODEL_PROVIDER=${_benchProvider ?? ""})`);
+            `(cerebras=${_cerebrasIntent}, ELIZA_PROVIDER=${_explicitProvider ?? ""}, BENCHMARK_MODEL_PROVIDER=${_benchProvider ?? ""})`);
     }
     // Load the OpenAI plugin when either:
     //   - OPENAI_API_KEY is set (and is not actually a Groq key, prefix `gsk_`), or
@@ -323,9 +1381,9 @@ export async function startBenchmarkServer() {
     const openAiApiKey = process.env.OPENAI_API_KEY?.trim();
     const openAiBaseURL = process.env.OPENAI_BASE_URL?.trim();
     const cerebrasApiKey = process.env.CEREBRAS_API_KEY?.trim();
-    const miladyProvider = process.env.MILADY_PROVIDER?.trim().toLowerCase();
+    const elizaProvider = process.env.ELIZA_PROVIDER?.trim().toLowerCase();
     const baseUrlIsCerebras = !!openAiBaseURL && /(^|\.)cerebras\.ai(\/|$)/i.test(openAiBaseURL);
-    const providerIsCerebras = miladyProvider === "cerebras";
+    const providerIsCerebras = elizaProvider === "cerebras";
     const hasOpenAiCompatibleKey = (openAiApiKey && !openAiApiKey.startsWith("gsk_")) ||
         ((baseUrlIsCerebras || providerIsCerebras) && !!cerebrasApiKey);
     if (hasOpenAiCompatibleKey) {
@@ -335,34 +1393,14 @@ export async function startBenchmarkServer() {
         try {
             const { default: openaiPlugin } = await import("@elizaos/plugin-openai");
             const openaiPluginResolved = toPlugin(openaiPlugin, "@elizaos/plugin-openai");
-            // Cerebras has no /v1/embeddings endpoint. The openai plugin's
-            // TEXT_EMBEDDING handler will 404 against api.cerebras.ai and stall
-            // Stage 1 of the message pipeline before the planner picks an action.
-            // Strip TEXT_EMBEDDING when cerebras is the explicit intent so
-            // plugin-local-embedding (loaded via CORE_PLUGINS) wins for embeddings
-            // while the openai plugin still serves TEXT_LARGE / TEXT_SMALL.
-            let strippedEmbedding = false;
-            if ((baseUrlIsCerebras || providerIsCerebras) &&
-                openaiPluginResolved.models &&
-                "TEXT_EMBEDDING" in openaiPluginResolved.models) {
-                const filteredModels = { ...openaiPluginResolved.models };
-                delete filteredModels.TEXT_EMBEDDING;
-                plugins.push({
-                    ...openaiPluginResolved,
-                    models: filteredModels,
-                });
-                strippedEmbedding = true;
-            }
-            else {
-                plugins.push(openaiPluginResolved);
-            }
+            plugins.push(openaiPluginResolved);
             elizaLogger.info(`[bench] Loaded LLM plugin: @elizaos/plugin-openai (baseURL=${openAiBaseURL ?? "default"}, key=${openAiApiKey
                 ? "OPENAI_API_KEY"
                 : cerebrasApiKey
                     ? "CEREBRAS_API_KEY"
-                    : "none"}${strippedEmbedding ? ", TEXT_EMBEDDING stripped (cerebras)" : ""})`);
-            if (strippedEmbedding) {
-                elizaLogger.info("[bench] Cerebras detected: removed openai plugin's TEXT_EMBEDDING handler so @elizaos/plugin-local-embedding serves embeddings.");
+                    : "none"}${baseUrlIsCerebras || providerIsCerebras ? ", TEXT_EMBEDDING local fallback (cerebras)" : ""})`);
+            if (baseUrlIsCerebras || providerIsCerebras) {
+                elizaLogger.info("[bench] Cerebras detected: keeping openai plugin's deterministic local TEXT_EMBEDDING fallback because Cerebras does not expose /v1/embeddings.");
             }
         }
         catch (error) {
@@ -400,10 +1438,10 @@ export async function startBenchmarkServer() {
             elizaLogger.debug(`[bench] Anthropic plugin not available: ${formatUnknownError(error)}`);
         }
     }
-    // Load computer use plugin if enabled
-    if (process.env.ELIZA_ENABLE_COMPUTERUSE) {
+    // Load computer use plugin if enabled.
+    if (process.env.COMPUTER_USE_ENABLED === "1") {
         try {
-            process.env.COMPUTERUSE_ENABLED ??= "true";
+            process.env.COMPUTER_USE_ENABLED ??= "1";
             process.env.COMPUTERUSE_MODE ??= "local";
             const localComputerusePath = "../../../../plugins/plugin-computeruse/src/index.ts";
             const computeruseModule = (await import(__rewriteRelativeImportExtension(localComputerusePath)));
@@ -419,13 +1457,15 @@ export async function startBenchmarkServer() {
             elizaLogger.debug(`[bench] Computer use plugin not available: ${formatUnknownError(error)}`);
         }
     }
-    // Load mock plugin for testing (file is gitignored for local-only use)
-    if (process.env.ELIZA_BENCH_MOCK === "true") {
+    const mockBenchmarkEnabled = process.env.ELIZA_BENCH_MOCK === "true";
+    // Load mock plugin for testing. Mock runs are diagnostic only and must not be
+    // treated as release evidence.
+    if (mockBenchmarkEnabled) {
         try {
             const mockLocation = "./mock-plugin.ts";
             const { mockPlugin } = await import(__rewriteRelativeImportExtension(mockLocation));
             plugins.push(toPlugin(mockPlugin, mockLocation));
-            elizaLogger.info("[bench] Loaded mock benchmark plugin");
+            elizaLogger.warn("[bench] Loaded mock benchmark plugin (mock=true, standIn=true); this run is not valid release evidence.");
         }
         catch (error) {
             elizaLogger.error(`[bench] Failed to load mock benchmark plugin: ${formatUnknownError(error)}`);
@@ -467,6 +1507,7 @@ export async function startBenchmarkServer() {
     const envKeys = [
         "GROQ_API_KEY",
         "OPENAI_API_KEY",
+        "CEREBRAS_API_KEY",
         "ANTHROPIC_API_KEY",
         "OPENROUTER_API_KEY",
         "GOOGLE_GENERATIVE_AI_API_KEY",
@@ -511,6 +1552,11 @@ export async function startBenchmarkServer() {
         "OPENAI_PLANNER_MODEL",
         "OPENAI_RESPONSE_HANDLER_MODEL",
         "OPENAI_SHOULD_RESPOND_MODEL",
+        "CEREBRAS_MODEL",
+        "BENCHMARK_TASK_AGENT",
+        "ELIZA_ACP_DEFAULT_AGENT",
+        "ELIZA_DEFAULT_AGENT_TYPE",
+        "ELIZA_AGENT_SELECTION_STRATEGY",
         "OPENROUTER_SMALL_MODEL",
         "OPENROUTER_LARGE_MODEL",
         "OPENROUTER_NANO_MODEL",
@@ -550,6 +1596,29 @@ export async function startBenchmarkServer() {
         plugins,
     });
     await runtime.initialize();
+    // Wire the local-inference loader subsystem the same way the main app boot
+    // does (eliza/packages/app-core/src/runtime/eliza.ts). Without this, the
+    // bench-server's @elizaos/plugin-local-inference Plugin.init() never
+    // registers a `localInferenceLoader` service, so its TEXT_EMBEDDING handler
+    // falls all the way through to the zero-vector path even when an Eliza-1
+    // bundle is installed locally. Calling it here makes the bench-server use
+    // the eliza-1 embedding model (text/eliza-1-0_8b-32k.gguf) when present,
+    // and harmlessly skips handler upgrades when no backend is available —
+    // matching the main app's behavior so benchmark runs reflect real
+    // retrieval semantics.
+    if (!skipEmbeddingPlugin) {
+        try {
+            const { ensureLocalInferenceHandler } = await import("@elizaos/plugin-local-inference/runtime");
+            await ensureLocalInferenceHandler(runtime);
+            elizaLogger.info("[bench] Wired @elizaos/plugin-local-inference loader (embedding + voice handlers)");
+        }
+        catch (err) {
+            elizaLogger.warn(`[bench] Could not wire @elizaos/plugin-local-inference runtime: ${formatUnknownError(err)}`);
+        }
+    }
+    else {
+        elizaLogger.info("[bench] Skipping @elizaos/plugin-local-inference runtime wiring because benchmark embedding skip is enabled");
+    }
     disableManualCompactionAction(runtime);
     const modelHandlers = runtime.models;
     const modelHandlerSummary = Object.fromEntries([...(modelHandlers?.entries() ?? [])].map(([modelType, handlers]) => [
@@ -576,25 +1645,10 @@ export async function startBenchmarkServer() {
             registerEvent("MODEL_USED", (payload) => {
                 if (!activeUsageBuffer)
                     return;
-                if (!payload || typeof payload !== "object")
-                    return;
-                const p = payload;
-                const tokens = p.tokens ?? {};
-                const promptTokens = typeof tokens.prompt === "number" ? tokens.prompt : 0;
-                const completionTokens = typeof tokens.completion === "number" ? tokens.completion : 0;
-                const totalTokens = typeof tokens.total === "number"
-                    ? tokens.total
-                    : promptTokens + completionTokens;
-                const cachedTokens = typeof tokens.cached === "number" ? tokens.cached : undefined;
-                activeUsageBuffer.push({
-                    modelType: typeof p.type === "string" ? p.type : "unknown",
-                    provider: typeof p.provider === "string" ? p.provider : undefined,
-                    source: typeof p.source === "string" ? p.source : undefined,
-                    promptTokens,
-                    completionTokens,
-                    totalTokens,
-                    ...(cachedTokens !== undefined ? { cachedTokens } : {}),
-                });
+                const normalizedUsage = normalizeBenchmarkModelUsage(payload);
+                if (normalizedUsage) {
+                    activeUsageBuffer.push(normalizedUsage);
+                }
             });
             elizaLogger.info("[bench] Registered MODEL_USED listener for trajectory usage capture");
         }
@@ -605,30 +1659,6 @@ export async function startBenchmarkServer() {
     catch (err) {
         elizaLogger.warn(`[bench] Could not register MODEL_USED listener: ${formatUnknownError(err)}`);
     }
-    const summarizeUsage = (calls) => {
-        let promptTokens = 0;
-        let completionTokens = 0;
-        let totalTokens = 0;
-        let cachedTokens = 0;
-        for (const call of calls) {
-            promptTokens += call.promptTokens;
-            completionTokens += call.completionTokens;
-            totalTokens += call.totalTokens;
-            if (typeof call.cachedTokens === "number") {
-                cachedTokens += call.cachedTokens;
-            }
-        }
-        const cacheHitRatio = promptTokens > 0 ? cachedTokens / promptTokens : 0;
-        return {
-            promptTokens,
-            completionTokens,
-            totalTokens,
-            cachedTokens,
-            cacheHitRatio,
-            callCount: calls.length,
-            calls,
-        };
-    };
     const roomToSession = new Map();
     const entityToSession = new Map();
     const trajectoriesBySession = new Map();
@@ -677,7 +1707,6 @@ export async function startBenchmarkServer() {
     };
     const sessions = new Map();
     let lastSessionKey = null;
-    // Session TTL eviction (R4)
     const SESSION_TTL_MS = 24 * 60 * 60 * 1000;
     const SESSION_SWEEP_INTERVAL_MS = 60_000;
     const sessionCreatedAt = new Map();
@@ -732,20 +1761,54 @@ export async function startBenchmarkServer() {
     // ────────────────────────────────────────────────────────────────────────
     const lifeopsBenchHandler = new LifeOpsBenchHandler({
         checkAuth: checkBenchAuth,
-        invokePlanner: async ({ taskId, userText, toolManifest, backend }) => {
+        invokePlanner: async ({ taskId, userText, toolManifest, backend, previousTurns, }) => {
             const session = resolveSession(taskId, "lifeops_bench", true);
             if (!session)
                 throw new Error("Failed to resolve lifeops_bench session");
             await ensureBenchmarkSessionContext(runtime, session);
+            const lifeopsContext = buildLifeOpsBenchmarkContext(backend, previousTurns);
             const benchmarkContext = normalizeBenchmarkContext(session, {
                 benchmark: "lifeops_bench",
                 task_id: taskId,
                 ...(Array.isArray(toolManifest) ? { tools: toolManifest } : {}),
+                lifeops: lifeopsContext,
             });
-            const composedPrompt = composeBenchmarkPrompt({
-                text: userText,
-                context: benchmarkContext,
-            });
+            if (Array.isArray(toolManifest) && toolManifest.length > 0) {
+                const directUsageBuffer = [];
+                activeUsageBuffer = directUsageBuffer;
+                try {
+                    const directResult = await callOpenAiCompatibleActionCalling({
+                        messages: buildLifeOpsActionCallingMessages({
+                            userText,
+                            lifeopsContext,
+                        }),
+                        tools: toolManifest,
+                        toolChoice: "required",
+                        maxTokens: 1024,
+                        temperature: 0,
+                    });
+                    if (directResult) {
+                        if (directResult.usage) {
+                            directUsageBuffer.push(directResult.usage);
+                        }
+                        const toolCalls = lifeOpsToolCallsFromNativeToolCalls(directResult.toolCalls);
+                        if (toolCalls.length > 0) {
+                            const usage = summarizeBenchmarkTurnUsage(directUsageBuffer);
+                            return { text: directResult.text, toolCalls, usage };
+                        }
+                    }
+                }
+                finally {
+                    activeUsageBuffer = null;
+                }
+            }
+            // The ELIZA_BENCHMARK provider already renders the full LifeOps clock,
+            // world snapshot, tool manifest, and previous tool results. Duplicating
+            // that JSON into the user message balloons Cerebras prompts and can leave
+            // the TS bridge waiting on a huge outbound model call. Keep the message
+            // itself to the user's benchmark instruction and let the provider carry
+            // the structured context.
+            const composedPrompt = userText.trim();
             const incomingMessage = {
                 id: stringToUuid(`lifeops-msg:${Date.now()}:${Math.random()}`),
                 content: {
@@ -789,34 +1852,67 @@ export async function startBenchmarkServer() {
                 : callbackTexts.join("\n\n");
             const actions = coerceActions(result.responseContent?.actions);
             const params = coerceParams(result.responseContent?.params);
+            const capturedAction = getCapturedAction();
             // Map captured Eliza actions into lifeops_bench tool calls.
             // Strategy: each action name in `actions` is treated as a tool name;
             // its arguments come from `params[actionName]` when present, otherwise
             // an empty object. This matches how OpenClaw/Hermes adapters expose
             // their tool-call traces. The fake-backend rejects unsupported names
             // with a clear error so scenario authors learn about gaps quickly.
-            const toolCalls = actions.map((name, index) => {
+            const toolCalls = [];
+            // BENCHMARK_ACTION unwrap: when the planner picks BENCHMARK_ACTION, the
+            // bench plugin captures the underlying tool name + arguments (tau-bench
+            // shape: `{tool_name, arguments}`). Unwrap that capture into a real tool
+            // call against the LifeOps fake backend instead of forwarding the
+            // generic BENCHMARK_ACTION sentinel (which the fake backend rejects).
+            if (capturedAction &&
+                typeof capturedAction.toolName === "string" &&
+                capturedAction.toolName.trim().length > 0) {
+                toolCalls.push({
+                    id: "call_0",
+                    name: capturedAction.toolName,
+                    arguments: capturedAction.arguments &&
+                        typeof capturedAction.arguments === "object"
+                        ? capturedAction.arguments
+                        : {},
+                });
+            }
+            // Also pass through any directly-named actions (e.g. when the planner
+            // emits MESSAGE/CALENDAR directly without the BENCHMARK_ACTION wrapper),
+            // skipping the BENCHMARK_ACTION sentinel itself which has already been
+            // unwrapped above. REPLY/RESPOND are terminal assistant messages, not
+            // LifeOps backend tools; forwarding them as tool calls makes the Python
+            // runner keep looping after a finished response.
+            for (const name of actions) {
+                if (name === "BENCHMARK_ACTION" ||
+                    name === "REPLY" ||
+                    name === "RESPOND")
+                    continue;
+                if (capturedAction &&
+                    typeof capturedAction.toolName === "string" &&
+                    capturedAction.toolName === name)
+                    continue;
                 const paramsForAction = params[name];
                 const argumentsObj = paramsForAction &&
                     typeof paramsForAction === "object" &&
                     !Array.isArray(paramsForAction)
                     ? paramsForAction
                     : {};
-                return {
-                    id: `call_${index}`,
+                toolCalls.push({
+                    id: `call_${toolCalls.length}`,
                     name,
                     arguments: argumentsObj,
-                };
-            });
-            const usage = {
-                promptTokens: turnUsageBuffer.reduce((s, c) => s + c.promptTokens, 0),
-                completionTokens: turnUsageBuffer.reduce((s, c) => s + c.completionTokens, 0),
-                totalTokens: turnUsageBuffer.reduce((s, c) => s + c.totalTokens, 0),
-            };
-            // Touch the backend so unused-import linters do not strip the
-            // LifeOpsFakeBackend type — and so future planner integrations can
-            // pre-warm the backend before action execution.
-            void backend;
+                });
+            }
+            if (shouldDropLifeOpsReadOnlyFollowupToolCalls({
+                userText,
+                responseText,
+                lifeopsContext,
+                toolCalls,
+            })) {
+                toolCalls.length = 0;
+            }
+            const usage = summarizeBenchmarkTurnUsage(turnUsageBuffer);
             return { text: responseText, toolCalls, usage };
         },
     });
@@ -844,6 +1940,10 @@ export async function startBenchmarkServer() {
                 status: "ready",
                 agent_name: runtime.character.name ?? "Eliza",
                 plugins: plugins.length,
+                standIn: skipEmbeddingPlugin || mockBenchmarkEnabled,
+                mock: mockBenchmarkEnabled,
+                stubEmbedding: skipEmbeddingPlugin,
+                releaseEvidence: !(skipEmbeddingPlugin || mockBenchmarkEnabled),
                 active_session: activeSession
                     ? {
                         benchmark: activeSession.benchmark,
@@ -859,15 +1959,23 @@ export async function startBenchmarkServer() {
                 return;
             let body = "";
             let bodyBytes = 0;
+            let bodyTooLarge = false;
             req.on("data", (chunk) => {
                 bodyBytes += chunk.length;
                 if (bodyBytes > MAX_BODY_BYTES) {
+                    bodyTooLarge = true;
+                    res.writeHead(413, { "Content-Type": "application/json" });
+                    res.end(JSON.stringify({
+                        error: `Request body exceeded max size ${MAX_BODY_BYTES} bytes`,
+                    }));
                     req.destroy();
                     return;
                 }
                 body += chunk;
             });
             req.on("end", async () => {
+                if (bodyTooLarge)
+                    return;
                 try {
                     const parsed = body.trim()
                         ? JSON.parse(body)
@@ -1000,15 +2108,23 @@ export async function startBenchmarkServer() {
                 return;
             let body = "";
             let bodyBytes = 0;
+            let bodyTooLarge = false;
             req.on("data", (chunk) => {
                 bodyBytes += chunk.length;
                 if (bodyBytes > MAX_BODY_BYTES) {
+                    bodyTooLarge = true;
+                    res.writeHead(413, { "Content-Type": "application/json" });
+                    res.end(JSON.stringify({
+                        error: `Request body exceeded max size ${MAX_BODY_BYTES} bytes`,
+                    }));
                     req.destroy();
                     return;
                 }
                 body += chunk;
             });
             req.on("end", async () => {
+                if (bodyTooLarge)
+                    return;
                 try {
                     let parsed;
                     try {
@@ -1042,6 +2158,718 @@ export async function startBenchmarkServer() {
                         context: benchmarkContext,
                         image: parsed.image,
                     });
+                    if (isWooBenchName(session.benchmark)) {
+                        const messages = normalizeWooBenchNativeMessages(text, benchmarkContext);
+                        const tools = Array.isArray(benchmarkContext.tools)
+                            ? benchmarkContext.tools
+                            : [];
+                        const maxTokens = typeof benchmarkContext.max_tokens === "number"
+                            ? benchmarkContext.max_tokens
+                            : 2048;
+                        const temperature = typeof benchmarkContext.temperature === "number"
+                            ? benchmarkContext.temperature
+                            : 0;
+                        const toolChoice = tools.length === 0
+                            ? "none"
+                            : typeof benchmarkContext.tool_choice === "string"
+                                ? benchmarkContext.tool_choice
+                                : "auto";
+                        const turnUsageBuffer = [];
+                        activeUsageBuffer = turnUsageBuffer;
+                        let nativeResult;
+                        try {
+                            const directResult = await callOpenAiCompatibleActionCalling({
+                                messages,
+                                tools,
+                                toolChoice,
+                                maxTokens,
+                                temperature,
+                            });
+                            if (directResult) {
+                                if (directResult.usage) {
+                                    turnUsageBuffer.push(directResult.usage);
+                                }
+                                nativeResult = {
+                                    text: directResult.text,
+                                    toolCalls: directResult.toolCalls,
+                                };
+                            }
+                            else {
+                                const modelRequest = {
+                                    messages,
+                                    maxTokens,
+                                    temperature,
+                                };
+                                if (tools.length > 0) {
+                                    modelRequest.tools = tools;
+                                    modelRequest.toolChoice = toolChoice;
+                                }
+                                nativeResult = await runtime.useModel(ModelType.TEXT_LARGE, modelRequest);
+                            }
+                        }
+                        finally {
+                            activeUsageBuffer = null;
+                        }
+                        const turnUsage = summarizeBenchmarkTurnUsage(turnUsageBuffer);
+                        const nativeRecord = nativeResult && typeof nativeResult === "object"
+                            ? nativeResult
+                            : {};
+                        const toolCalls = normalizeLocaNativeToolCalls(nativeRecord.toolCalls);
+                        const responseText = typeof nativeRecord.text === "string"
+                            ? nativeRecord.text
+                            : typeof nativeResult === "string"
+                                ? nativeResult
+                                : "";
+                        const params = {};
+                        const benchmarkAction = firstWooBenchActionFromToolCalls(toolCalls);
+                        if (benchmarkAction) {
+                            params.BENCHMARK_ACTION = benchmarkAction;
+                            params.tool_calls = toolCalls;
+                        }
+                        const actions = benchmarkAction !== null
+                            ? ["BENCHMARK_ACTION"]
+                            : responseText.trim()
+                                ? ["REPLY"]
+                                : [];
+                        const finishedAt = Date.now();
+                        trajectory.push({
+                            step: trajectory.length + 1,
+                            startedAt,
+                            finishedAt,
+                            inputText: text,
+                            promptText: composedPrompt,
+                            context,
+                            thought: null,
+                            responseText,
+                            actions,
+                            params,
+                            usage: turnUsage,
+                        });
+                        trajectoriesBySession.set(key, trajectory);
+                        const metadata = benchmarkTurnMetadata({
+                            session,
+                            step: trajectory.length,
+                            context: benchmarkContext,
+                        });
+                        res.writeHead(200, { "Content-Type": "application/json" });
+                        res.end(JSON.stringify({
+                            text: responseText,
+                            thought: null,
+                            actions,
+                            params,
+                            captured_actions: [],
+                            tool_calls: toolCalls,
+                            usage: turnUsage,
+                            metadata,
+                            benchmark: session.benchmark,
+                            task_id: session.taskId,
+                            room_id: session.roomId,
+                            trajectory_step: trajectory.length,
+                        }));
+                        return;
+                    }
+                    if (isActionCallingBenchmarkName(session.benchmark) &&
+                        Array.isArray(benchmarkContext.tools) &&
+                        benchmarkContext.tools.length > 0) {
+                        const nativeMessages = _isTauBenchmarkName(session.benchmark)
+                            ? _normalizeTauNativeMessages(text, benchmarkContext)
+                            : isVendingBenchmarkName(session.benchmark)
+                                ? normalizeLocaNativeMessages(benchmarkContext.messages)
+                                : normalizeActionCallingNativeMessages(text, benchmarkContext);
+                        const openAiMessages = _isTauBenchmarkName(session.benchmark)
+                            ? nativeMessages
+                            : isVendingBenchmarkName(session.benchmark)
+                                ? nativeMessages
+                                : normalizeActionCallingOpenAiMessages(text, benchmarkContext);
+                        const maxTokens = typeof benchmarkContext.max_tokens === "number"
+                            ? benchmarkContext.max_tokens
+                            : 2048;
+                        const temperature = typeof benchmarkContext.temperature === "number"
+                            ? benchmarkContext.temperature
+                            : 0;
+                        const toolChoice = typeof benchmarkContext.tool_choice === "string"
+                            ? benchmarkContext.tool_choice
+                            : "required";
+                        const turnUsageBuffer = [];
+                        activeUsageBuffer = turnUsageBuffer;
+                        let nativeResult;
+                        try {
+                            const directResult = await callOpenAiCompatibleActionCalling({
+                                messages: openAiMessages,
+                                tools: benchmarkContext.tools,
+                                toolChoice,
+                                maxTokens,
+                                temperature,
+                            });
+                            if (directResult) {
+                                if (directResult.usage) {
+                                    turnUsageBuffer.push(directResult.usage);
+                                }
+                                nativeResult = {
+                                    text: directResult.text,
+                                    toolCalls: directResult.toolCalls,
+                                };
+                            }
+                            else {
+                                nativeResult = await runtime.useModel(ModelType.TEXT_LARGE, {
+                                    messages: toChatMessages(nativeMessages),
+                                    tools: toToolDefinitions(benchmarkContext.tools),
+                                    toolChoice: toToolChoice(toolChoice),
+                                    maxTokens,
+                                    temperature,
+                                });
+                            }
+                        }
+                        finally {
+                            activeUsageBuffer = null;
+                        }
+                        const turnUsage = summarizeBenchmarkTurnUsage(turnUsageBuffer);
+                        const nativeRecord = nativeResult && typeof nativeResult === "object"
+                            ? nativeResult
+                            : {};
+                        const toolCalls = normalizeLocaNativeToolCalls(nativeRecord.toolCalls);
+                        const responseText = typeof nativeRecord.text === "string"
+                            ? nativeRecord.text
+                            : typeof nativeResult === "string"
+                                ? nativeResult
+                                : "";
+                        const params = {};
+                        const benchmarkAction = firstLocaBenchmarkActionFromToolCalls(toolCalls);
+                        if (benchmarkAction) {
+                            params.BENCHMARK_ACTION = benchmarkAction;
+                            params.tool_calls = toolCalls;
+                        }
+                        const actions = toolCalls.length > 0
+                            ? ["BENCHMARK_ACTION"]
+                            : responseText.trim()
+                                ? ["REPLY"]
+                                : [];
+                        const finishedAt = Date.now();
+                        trajectory.push({
+                            step: trajectory.length + 1,
+                            startedAt,
+                            finishedAt,
+                            inputText: text,
+                            promptText: composedPrompt,
+                            context,
+                            thought: null,
+                            responseText,
+                            actions,
+                            params,
+                            usage: turnUsage,
+                        });
+                        trajectoriesBySession.set(key, trajectory);
+                        const metadata = benchmarkTurnMetadata({
+                            session,
+                            step: trajectory.length,
+                            context: benchmarkContext,
+                        });
+                        res.writeHead(200, { "Content-Type": "application/json" });
+                        res.end(JSON.stringify({
+                            text: responseText,
+                            thought: null,
+                            actions,
+                            params,
+                            captured_actions: [],
+                            tool_calls: toolCalls,
+                            usage: turnUsage,
+                            metadata,
+                            benchmark: session.benchmark,
+                            task_id: session.taskId,
+                            room_id: session.roomId,
+                            trajectory_step: trajectory.length,
+                        }));
+                        return;
+                    }
+                    if (isLocaBenchmarkName(session.benchmark) &&
+                        Array.isArray(benchmarkContext.tools) &&
+                        benchmarkContext.tools.length > 0) {
+                        const nativeMessages = normalizeLocaNativeMessages(benchmarkContext.messages);
+                        const maxTokens = typeof benchmarkContext.max_tokens === "number"
+                            ? benchmarkContext.max_tokens
+                            : 2048;
+                        const temperature = typeof benchmarkContext.temperature === "number"
+                            ? benchmarkContext.temperature
+                            : 0;
+                        const turnUsageBuffer = [];
+                        activeUsageBuffer = turnUsageBuffer;
+                        let nativeResult;
+                        try {
+                            const directResult = await callOpenAiCompatibleActionCalling({
+                                messages: nativeMessages,
+                                tools: benchmarkContext.tools,
+                                toolChoice: "required",
+                                maxTokens,
+                                temperature,
+                            });
+                            if (directResult) {
+                                if (directResult.usage) {
+                                    turnUsageBuffer.push(directResult.usage);
+                                }
+                                nativeResult = {
+                                    text: directResult.text,
+                                    toolCalls: directResult.toolCalls,
+                                };
+                            }
+                            else {
+                                nativeResult = await runtime.useModel(ModelType.TEXT_LARGE, {
+                                    messages: toChatMessages(nativeMessages),
+                                    tools: toToolDefinitions(benchmarkContext.tools),
+                                    toolChoice: "required",
+                                    maxTokens,
+                                    temperature,
+                                });
+                            }
+                        }
+                        finally {
+                            activeUsageBuffer = null;
+                        }
+                        const turnUsage = summarizeBenchmarkTurnUsage(turnUsageBuffer);
+                        const nativeRecord = nativeResult && typeof nativeResult === "object"
+                            ? nativeResult
+                            : {};
+                        const toolCalls = normalizeLocaNativeToolCalls(nativeRecord.toolCalls);
+                        const responseText = typeof nativeRecord.text === "string"
+                            ? nativeRecord.text
+                            : typeof nativeResult === "string"
+                                ? nativeResult
+                                : "";
+                        const params = {};
+                        const benchmarkAction = firstLocaBenchmarkActionFromToolCalls(toolCalls);
+                        if (benchmarkAction) {
+                            params.BENCHMARK_ACTION = benchmarkAction;
+                            params.tool_calls = toolCalls;
+                        }
+                        const actions = toolCalls.length > 0
+                            ? ["BENCHMARK_ACTION"]
+                            : responseText.trim()
+                                ? ["REPLY"]
+                                : [];
+                        const finishedAt = Date.now();
+                        trajectory.push({
+                            step: trajectory.length + 1,
+                            startedAt,
+                            finishedAt,
+                            inputText: text,
+                            promptText: composedPrompt,
+                            context,
+                            thought: null,
+                            responseText,
+                            actions,
+                            params,
+                            usage: turnUsage,
+                        });
+                        trajectoriesBySession.set(key, trajectory);
+                        const metadata = benchmarkTurnMetadata({
+                            session,
+                            step: trajectory.length,
+                            context: benchmarkContext,
+                        });
+                        res.writeHead(200, { "Content-Type": "application/json" });
+                        res.end(JSON.stringify({
+                            text: responseText,
+                            thought: null,
+                            actions,
+                            params,
+                            captured_actions: [],
+                            tool_calls: toolCalls,
+                            usage: turnUsage,
+                            metadata,
+                            benchmark: session.benchmark,
+                            task_id: session.taskId,
+                            room_id: session.roomId,
+                            trajectory_step: trajectory.length,
+                        }));
+                        return;
+                    }
+                    if (isBfclBenchmarkName(session.benchmark) &&
+                        Array.isArray(benchmarkContext.tools) &&
+                        benchmarkContext.tools.length > 0) {
+                        const messages = normalizeBfclNativeMessages(text, benchmarkContext);
+                        const toolChoice = benchmarkContext.is_relevant === false ? "none" : "required";
+                        const maxTokens = typeof benchmarkContext.max_tokens === "number"
+                            ? benchmarkContext.max_tokens
+                            : 2048;
+                        const temperature = typeof benchmarkContext.temperature === "number"
+                            ? benchmarkContext.temperature
+                            : 0;
+                        const turnUsageBuffer = [];
+                        activeUsageBuffer = turnUsageBuffer;
+                        let nativeResult;
+                        try {
+                            const directResult = await callOpenAiCompatibleActionCalling({
+                                messages,
+                                tools: benchmarkContext.tools,
+                                toolChoice,
+                                maxTokens,
+                                temperature,
+                            }).catch((err) => {
+                                elizaLogger.warn(`[bench] BFCL direct native tool call failed; falling back to runtime model path: ${formatUnknownError(err)}`);
+                                return null;
+                            });
+                            if (directResult) {
+                                if (directResult.usage) {
+                                    turnUsageBuffer.push(directResult.usage);
+                                }
+                                nativeResult = {
+                                    text: directResult.text,
+                                    toolCalls: directResult.toolCalls,
+                                };
+                            }
+                            else {
+                                nativeResult = await runtime.useModel(ModelType.TEXT_LARGE, {
+                                    messages: toChatMessages(messages),
+                                    tools: toToolDefinitions(benchmarkContext.tools),
+                                    toolChoice: toToolChoice(toolChoice),
+                                    maxTokens,
+                                    temperature,
+                                });
+                            }
+                        }
+                        finally {
+                            activeUsageBuffer = null;
+                        }
+                        const turnUsage = summarizeBenchmarkTurnUsage(turnUsageBuffer);
+                        const nativeRecord = nativeResult && typeof nativeResult === "object"
+                            ? nativeResult
+                            : {};
+                        const toolCalls = normalizeLocaNativeToolCalls(nativeRecord.toolCalls);
+                        const responseText = typeof nativeRecord.text === "string"
+                            ? nativeRecord.text
+                            : typeof nativeResult === "string"
+                                ? nativeResult
+                                : "";
+                        const params = {};
+                        const benchmarkAction = bfclBenchmarkActionFromToolCalls(toolCalls);
+                        if (benchmarkAction) {
+                            params.BENCHMARK_ACTION = benchmarkAction;
+                            params.tool_calls = toolCalls;
+                        }
+                        const actions = toolCalls.length > 0
+                            ? ["BENCHMARK_ACTION"]
+                            : responseText.trim()
+                                ? ["REPLY"]
+                                : [];
+                        const finishedAt = Date.now();
+                        trajectory.push({
+                            step: trajectory.length + 1,
+                            startedAt,
+                            finishedAt,
+                            inputText: text,
+                            promptText: composedPrompt,
+                            context,
+                            thought: null,
+                            responseText,
+                            actions,
+                            params,
+                            usage: turnUsage,
+                        });
+                        trajectoriesBySession.set(key, trajectory);
+                        const metadata = benchmarkTurnMetadata({
+                            session,
+                            step: trajectory.length,
+                            context: benchmarkContext,
+                        });
+                        res.writeHead(200, { "Content-Type": "application/json" });
+                        res.end(JSON.stringify({
+                            text: responseText,
+                            thought: null,
+                            actions,
+                            params,
+                            captured_actions: [],
+                            tool_calls: toolCalls,
+                            usage: turnUsage,
+                            metadata,
+                            benchmark: session.benchmark,
+                            task_id: session.taskId,
+                            room_id: session.roomId,
+                            trajectory_step: trajectory.length,
+                        }));
+                        return;
+                    }
+                    if (isWebShopBenchmarkName(session.benchmark) &&
+                        Array.isArray(benchmarkContext.tools) &&
+                        benchmarkContext.tools.length > 0) {
+                        const messages = [
+                            {
+                                role: "system",
+                                content: "You are running WebShop through the Eliza benchmark server. Use the webshop_action tool exactly once with command set to one valid command from the current available actions. Do not answer in prose.",
+                            },
+                            { role: "user", content: text },
+                        ];
+                        const turnUsageBuffer = [];
+                        activeUsageBuffer = turnUsageBuffer;
+                        let nativeResult;
+                        try {
+                            const directResult = await callOpenAiCompatibleActionCalling({
+                                messages,
+                                tools: benchmarkContext.tools,
+                                toolChoice: "required",
+                                maxTokens: 256,
+                                temperature: typeof benchmarkContext.temperature === "number"
+                                    ? benchmarkContext.temperature
+                                    : 0,
+                            }).catch((err) => {
+                                elizaLogger.warn(`[bench] WebShop direct native tool call failed; falling back to runtime model path: ${formatUnknownError(err)}`);
+                                return null;
+                            });
+                            if (directResult) {
+                                if (directResult.usage) {
+                                    turnUsageBuffer.push(directResult.usage);
+                                }
+                                nativeResult = {
+                                    text: directResult.text,
+                                    toolCalls: directResult.toolCalls,
+                                };
+                            }
+                            else {
+                                nativeResult = await runtime.useModel(ModelType.TEXT_LARGE, {
+                                    messages: toChatMessages(messages),
+                                    tools: toToolDefinitions(benchmarkContext.tools),
+                                    toolChoice: "required",
+                                    maxTokens: 256,
+                                    temperature: typeof benchmarkContext.temperature === "number"
+                                        ? benchmarkContext.temperature
+                                        : 0,
+                                });
+                            }
+                        }
+                        finally {
+                            activeUsageBuffer = null;
+                        }
+                        const turnUsage = summarizeBenchmarkTurnUsage(turnUsageBuffer);
+                        const nativeRecord = nativeResult && typeof nativeResult === "object"
+                            ? nativeResult
+                            : {};
+                        const toolCalls = normalizeLocaNativeToolCalls(nativeRecord.toolCalls);
+                        const responseText = typeof nativeRecord.text === "string"
+                            ? nativeRecord.text
+                            : typeof nativeResult === "string"
+                                ? nativeResult
+                                : "";
+                        const params = {};
+                        const benchmarkAction = webshopBenchmarkActionFromToolCalls(toolCalls);
+                        if (benchmarkAction) {
+                            params.BENCHMARK_ACTION = benchmarkAction;
+                            params.tool_calls = toolCalls;
+                        }
+                        const actions = benchmarkAction !== null
+                            ? ["BENCHMARK_ACTION"]
+                            : responseText.trim()
+                                ? ["REPLY"]
+                                : [];
+                        const finishedAt = Date.now();
+                        trajectory.push({
+                            step: trajectory.length + 1,
+                            startedAt,
+                            finishedAt,
+                            inputText: text,
+                            promptText: composedPrompt,
+                            context,
+                            thought: null,
+                            responseText,
+                            actions,
+                            params,
+                            usage: turnUsage,
+                        });
+                        trajectoriesBySession.set(key, trajectory);
+                        const metadata = benchmarkTurnMetadata({
+                            session,
+                            step: trajectory.length,
+                            context: benchmarkContext,
+                        });
+                        res.writeHead(200, { "Content-Type": "application/json" });
+                        res.end(JSON.stringify({
+                            text: responseText,
+                            thought: null,
+                            actions,
+                            params,
+                            captured_actions: [],
+                            tool_calls: toolCalls,
+                            usage: turnUsage,
+                            metadata,
+                            benchmark: session.benchmark,
+                            task_id: session.taskId,
+                            room_id: session.roomId,
+                            trajectory_step: trajectory.length,
+                        }));
+                        return;
+                    }
+                    if (isHermesNativeEnvProxyName(session.benchmark) &&
+                        Array.isArray(benchmarkContext.tools) &&
+                        benchmarkContext.tools.length > 0) {
+                        const messages = normalizeGenericToolMessages(benchmarkContext.messages, text);
+                        const maxTokens = typeof benchmarkContext.max_tokens === "number"
+                            ? benchmarkContext.max_tokens
+                            : 4096;
+                        const temperature = typeof benchmarkContext.temperature === "number"
+                            ? benchmarkContext.temperature
+                            : 0;
+                        const toolChoice = typeof benchmarkContext.tool_choice === "string"
+                            ? benchmarkContext.tool_choice
+                            : "auto";
+                        const turnUsageBuffer = [];
+                        activeUsageBuffer = turnUsageBuffer;
+                        let nativeResult;
+                        try {
+                            const directResult = await callOpenAiCompatibleActionCalling({
+                                messages,
+                                tools: benchmarkContext.tools,
+                                toolChoice,
+                                maxTokens,
+                                temperature,
+                            });
+                            if (directResult) {
+                                if (directResult.usage) {
+                                    turnUsageBuffer.push(directResult.usage);
+                                }
+                                nativeResult = {
+                                    text: directResult.text,
+                                    toolCalls: directResult.toolCalls,
+                                };
+                            }
+                            else {
+                                nativeResult = await runtime.useModel(ModelType.TEXT_LARGE, {
+                                    messages: toChatMessages(messages),
+                                    tools: toToolDefinitions(benchmarkContext.tools),
+                                    toolChoice: toToolChoice(toolChoice),
+                                    maxTokens,
+                                    temperature,
+                                });
+                            }
+                        }
+                        finally {
+                            activeUsageBuffer = null;
+                        }
+                        const turnUsage = summarizeBenchmarkTurnUsage(turnUsageBuffer);
+                        const nativeRecord = nativeResult && typeof nativeResult === "object"
+                            ? nativeResult
+                            : {};
+                        const toolCalls = normalizeLocaNativeToolCalls(nativeRecord.toolCalls);
+                        const responseText = typeof nativeRecord.text === "string"
+                            ? nativeRecord.text
+                            : typeof nativeResult === "string"
+                                ? nativeResult
+                                : "";
+                        const finishedAt = Date.now();
+                        trajectory.push({
+                            step: trajectory.length + 1,
+                            startedAt,
+                            finishedAt,
+                            inputText: text,
+                            promptText: composedPrompt,
+                            context,
+                            thought: null,
+                            responseText,
+                            actions: toolCalls.length > 0 ? ["BENCHMARK_ACTION"] : [],
+                            params: toolCalls.length > 0 ? { tool_calls: toolCalls } : {},
+                            usage: turnUsage,
+                        });
+                        trajectoriesBySession.set(key, trajectory);
+                        const metadata = benchmarkTurnMetadata({
+                            session,
+                            step: trajectory.length,
+                            context: benchmarkContext,
+                        });
+                        res.writeHead(200, { "Content-Type": "application/json" });
+                        res.end(JSON.stringify({
+                            text: responseText,
+                            thought: null,
+                            actions: toolCalls.length > 0 ? ["BENCHMARK_ACTION"] : [],
+                            params: toolCalls.length > 0 ? { tool_calls: toolCalls } : {},
+                            captured_actions: [],
+                            tool_calls: toolCalls,
+                            usage: turnUsage,
+                            metadata,
+                            benchmark: session.benchmark,
+                            task_id: session.taskId,
+                            room_id: session.roomId,
+                            trajectory_step: trajectory.length,
+                        }));
+                        return;
+                    }
+                    if (isTerminalBenchmarkName(session.benchmark) ||
+                        isSweBenchmarkName(session.benchmark) ||
+                        isVisualWebBenchmarkName(session.benchmark) ||
+                        isOsworldBenchmarkName(session.benchmark)) {
+                        const maxTokens = typeof benchmarkContext.max_tokens === "number"
+                            ? benchmarkContext.max_tokens
+                            : 4096;
+                        const temperature = typeof benchmarkContext.temperature === "number"
+                            ? benchmarkContext.temperature
+                            : 0;
+                        const turnUsageBuffer = [];
+                        activeUsageBuffer = turnUsageBuffer;
+                        let nativeResult;
+                        try {
+                            const directResult = await callOpenAiCompatibleText({
+                                prompt: composedPrompt,
+                                maxTokens,
+                                temperature,
+                            });
+                            if (directResult) {
+                                if (directResult.usage) {
+                                    turnUsageBuffer.push(directResult.usage);
+                                }
+                                nativeResult = directResult.text;
+                            }
+                            else {
+                                nativeResult = await runtime.useModel(ModelType.TEXT_LARGE, {
+                                    prompt: composedPrompt,
+                                    maxTokens,
+                                    temperature,
+                                });
+                            }
+                        }
+                        finally {
+                            activeUsageBuffer = null;
+                        }
+                        const turnUsage = summarizeBenchmarkTurnUsage(turnUsageBuffer);
+                        const nativeRecord = nativeResult && typeof nativeResult === "object"
+                            ? nativeResult
+                            : {};
+                        const responseText = typeof nativeRecord.text === "string"
+                            ? nativeRecord.text
+                            : typeof nativeResult === "string"
+                                ? nativeResult
+                                : "";
+                        const finishedAt = Date.now();
+                        trajectory.push({
+                            step: trajectory.length + 1,
+                            startedAt,
+                            finishedAt,
+                            inputText: text,
+                            promptText: composedPrompt,
+                            context,
+                            thought: null,
+                            responseText,
+                            actions: responseText.trim() ? ["REPLY"] : [],
+                            params: {},
+                            usage: turnUsage,
+                        });
+                        trajectoriesBySession.set(key, trajectory);
+                        const metadata = benchmarkTurnMetadata({
+                            session,
+                            step: trajectory.length,
+                            context: benchmarkContext,
+                        });
+                        res.writeHead(200, { "Content-Type": "application/json" });
+                        res.end(JSON.stringify({
+                            text: responseText,
+                            thought: null,
+                            actions: responseText.trim() ? ["REPLY"] : [],
+                            params: {},
+                            captured_actions: [],
+                            tool_calls: [],
+                            usage: turnUsage,
+                            metadata,
+                            benchmark: session.benchmark,
+                            task_id: session.taskId,
+                            room_id: session.roomId,
+                            trajectory_step: trajectory.length,
+                        }));
+                        return;
+                    }
                     const incomingMessage = {
                         id: stringToUuid(`benchmark-msg:${Date.now()}:${Math.random()}`),
                         content: {
@@ -1083,8 +2911,9 @@ export async function startBenchmarkServer() {
                             activeUsageBuffer = null;
                         }
                     })();
-                    const turnUsage = summarizeUsage(turnUsageBuffer);
+                    const turnUsage = summarizeBenchmarkTurnUsage(turnUsageBuffer);
                     const capturedAction = getCapturedAction();
+                    const capturedActions = getCapturedActions();
                     const responseText = typeof result.responseContent?.text === "string"
                         ? result.responseContent.text
                         : callbackTexts.join("\n\n");
@@ -1101,6 +2930,12 @@ export async function startBenchmarkServer() {
                     const params = Object.keys(parsedParams).length > 0
                         ? parsedParams
                         : capturedActionToParams(capturedAction);
+                    if (capturedActions.length > 1) {
+                        params.BENCHMARK_ACTIONS = capturedActions
+                            .map((action) => capturedActionToParams(action).BENCHMARK_ACTION)
+                            .filter(Boolean);
+                    }
+                    const toolCalls = capturedActionsToToolCalls(capturedActions);
                     const finishedAt = Date.now();
                     trajectory.push({
                         step: trajectory.length + 1,
@@ -1116,12 +2951,21 @@ export async function startBenchmarkServer() {
                         usage: turnUsage,
                     });
                     trajectoriesBySession.set(key, trajectory);
+                    const metadata = benchmarkTurnMetadata({
+                        session,
+                        step: trajectory.length,
+                        context: benchmarkContext,
+                    });
                     res.writeHead(200, { "Content-Type": "application/json" });
                     res.end(JSON.stringify({
                         text: responseText,
                         thought,
                         actions,
                         params,
+                        captured_actions: capturedActions,
+                        tool_calls: toolCalls,
+                        usage: turnUsage,
+                        metadata,
                         benchmark: session.benchmark,
                         task_id: session.taskId,
                         room_id: session.roomId,
@@ -1171,6 +3015,6 @@ export async function startBenchmarkServer() {
     });
 }
 startBenchmarkServer().catch((err) => {
-    console.error("Failed to start benchmark server:", err);
+    elizaLogger.error(`[bench] Failed to start benchmark server: ${formatUnknownError(err)}`);
     process.exit(1);
 });