@elizaos/agent 0.25.8 → 2.0.0-alpha.83

package/packages/agent/src/benchmark-server.js

@@ -0,0 +1,773 @@
+import http from "node:http";
+import path from "node:path";
+import { AgentRuntime, ChannelType, elizaLogger, InMemoryDatabaseAdapter, ModelType, stringToUuid, } from "@elizaos/core";
+import dotenv from "dotenv";
+import { createElizaPlugin } from "./runtime/eliza-plugin.js";
+// Load environment variables BEFORE anything else
+// This ensures API keys are available when plugins initialize
+dotenv.config({ path: path.resolve(process.cwd(), ".env") });
+const DEFAULT_PORT = 3939;
+const BENCHMARK_WORLD_ID = stringToUuid("autonomous-benchmark-world");
+const BENCHMARK_MESSAGE_SERVER_ID = stringToUuid("autonomous-benchmark-message-server");
+function formatUnknownError(error) {
+    if (error instanceof Error) {
+        return `${error.name}: ${error.message}`;
+    }
+    return String(error);
+}
+function toPlugin(candidate, source) {
+    if (!candidate || typeof candidate !== "object") {
+        throw new Error(`Plugin from ${source} was not an object`);
+    }
+    const pluginLike = candidate;
+    if (typeof pluginLike.name !== "string" || pluginLike.name.length === 0) {
+        throw new Error(`Plugin from ${source} was missing a valid name`);
+    }
+    return candidate;
+}
+function resolvePort() {
+    const raw = process.env.AUTONOMOUS_BENCH_PORT;
+    if (!raw)
+        return DEFAULT_PORT;
+    const parsed = Number(raw);
+    if (!Number.isFinite(parsed) || parsed < 1 || parsed > 65535) {
+        elizaLogger.warn(`[bench] Invalid AUTONOMOUS_BENCH_PORT="${raw}"; using ${DEFAULT_PORT}`);
+        return DEFAULT_PORT;
+    }
+    return Math.floor(parsed);
+}
+function extractRecord(value) {
+    if (!value || typeof value !== "object" || Array.isArray(value)) {
+        return undefined;
+    }
+    return value;
+}
+function extractTaskId(context) {
+    const bySnake = context?.task_id;
+    if (typeof bySnake === "string" && bySnake.trim())
+        return bySnake.trim();
+    const byCamel = context?.taskId;
+    if (typeof byCamel === "string" && byCamel.trim())
+        return byCamel.trim();
+    return "default-task";
+}
+function extractBenchmarkName(context) {
+    const benchmark = context?.benchmark;
+    if (typeof benchmark === "string" && benchmark.trim()) {
+        return benchmark.trim();
+    }
+    return "unknown";
+}
+function composeBenchmarkPrompt(params) {
+    const segments = [params.text.trim()];
+    if (params.context && Object.keys(params.context).length > 0) {
+        segments.push([
+            "BENCHMARK CONTEXT (authoritative):",
+            JSON.stringify(params.context, null, 2),
+        ].join("\n"));
+    }
+    if (params.image !== undefined) {
+        segments.push(["IMAGE PAYLOAD:", JSON.stringify(params.image, null, 2)].join("\n"));
+    }
+    if (params.mode === "message-service") {
+        segments.push("Respond using normal Eliza action output so actions/params can be executed and evaluated.");
+    }
+    else {
+        segments.push("Reply directly to the benchmark prompt. If it asks for JSON, return JSON only with no extra narration.");
+    }
+    return segments.join("\n\n");
+}
+function resolveBenchmarkMessageMode() {
+    const raw = process.env.AUTONOMOUS_BENCH_MESSAGE_MODE?.trim().toLowerCase();
+    if (raw === "message-service")
+        return "message-service";
+    return "direct-model";
+}
+function coerceActions(value) {
+    if (!Array.isArray(value))
+        return [];
+    return value.filter((entry) => typeof entry === "string");
+}
+function coerceParams(value) {
+    if (value && typeof value === "object" && !Array.isArray(value)) {
+        return value;
+    }
+    if (typeof value === "string") {
+        try {
+            const parsed = JSON.parse(value);
+            if (parsed && typeof parsed === "object" && !Array.isArray(parsed)) {
+                return parsed;
+            }
+        }
+        catch {
+            // ignore malformed param strings
+        }
+    }
+    return {};
+}
+function sessionKey(session) {
+    return `${session.benchmark}:${session.taskId}`;
+}
+async function ensureBenchmarkSessionContext(runtime, session) {
+    await runtime.ensureWorldExists({
+        id: BENCHMARK_WORLD_ID,
+        name: "Autonomous Benchmark World",
+        agentId: runtime.agentId,
+        messageServerId: BENCHMARK_MESSAGE_SERVER_ID,
+        metadata: {
+            type: "benchmark",
+            description: "World used for benchmark sessions",
+            extra: {
+                benchmark: session.benchmark,
+            },
+        },
+    });
+    // Use ChannelType.API to ensure the agent always responds to benchmark messages
+    // (API channel type bypasses shouldRespond evaluation)
+    await runtime.ensureRoomExists({
+        id: session.roomId,
+        name: `benchmark:${session.taskId}`,
+        source: "benchmark",
+        type: ChannelType.API,
+        channelId: `bench-${session.taskId}`,
+        messageServerId: BENCHMARK_MESSAGE_SERVER_ID,
+        worldId: BENCHMARK_WORLD_ID,
+        metadata: {
+            benchmark: session.benchmark,
+            taskId: session.taskId,
+        },
+    });
+    await runtime.ensureRoomExists({
+        id: session.relayRoomId,
+        name: "relay-room",
+        source: "benchmark",
+        type: ChannelType.API,
+        channelId: `relay-${session.taskId}`,
+        messageServerId: BENCHMARK_MESSAGE_SERVER_ID,
+        worldId: BENCHMARK_WORLD_ID,
+        metadata: {
+            benchmark: session.benchmark,
+            taskId: session.taskId,
+            role: "relay-room",
+        },
+    });
+    await runtime.ensureConnection({
+        entityId: session.userEntityId,
+        roomId: session.roomId,
+        worldId: BENCHMARK_WORLD_ID,
+        userName: "Benchmark User",
+        source: "benchmark",
+        channelId: `bench-${session.taskId}`,
+        type: ChannelType.API,
+        messageServerId: BENCHMARK_MESSAGE_SERVER_ID,
+        metadata: {
+            benchmark: session.benchmark,
+            taskId: session.taskId,
+            role: "benchmark-room",
+        },
+    });
+    // The benchmark transport only needs the relay room to exist so room-targeted
+    // emissions can be attributed back to the active session. In benchmark mode
+    // some runtime/database combinations do not expose the agent entity through
+    // getEntitiesByIds(), which makes an explicit ensureParticipantInRoom() call
+    // fail even though the primary benchmark room is already fully connected via
+    // ensureConnection() above.
+}
+function createSession(taskId, benchmark) {
+    const normalizedTaskId = taskId.trim() || "default-task";
+    const normalizedBenchmark = benchmark.trim() || "unknown";
+    const seed = `${normalizedBenchmark}:${normalizedTaskId}:${Date.now()}:${Math.random()}`;
+    return {
+        benchmark: normalizedBenchmark,
+        taskId: normalizedTaskId,
+        roomId: stringToUuid(`benchmark-room:${seed}`),
+        relayRoomId: stringToUuid(`benchmark-relay:${seed}`),
+        userEntityId: stringToUuid(`benchmark-user:${seed}`),
+    };
+}
+// Proper robust server implementation
+export async function startBenchmarkServer() {
+    const port = resolvePort();
+    const messageMode = resolveBenchmarkMessageMode();
+    elizaLogger.info(`[bench] Initializing autonomous benchmark runtime on port ${port} (message_mode=${messageMode})...`);
+    // Benchmark mode prefers a stable, comparable runtime over full production
+    // plugin parity. A smaller allowlist avoids startup-only side effects from
+    // plugins that are useful in the full app but unnecessary for benchmark I/O.
+    const benchmarkCorePlugins = [
+        "@elizaos/plugin-sql",
+        "@elizaos/plugin-local-embedding",
+        "@elizaos/plugin-secrets-manager",
+        "@elizaos/plugin-rolodex",
+        "@elizaos/plugin-trust",
+        "@elizaos/plugin-todo",
+        "@elizaos/plugin-experience",
+    ];
+    const plugins = [];
+    const loadedPlugins = [];
+    const failedPlugins = [];
+    // Plugins to skip in benchmark context — these require external auth or
+    // interfere with benchmark operation
+    const skipPlugins = new Set([
+        "@elizaos/plugin-elizacloud", // Requires ElizaOS cloud auth, conflicts with local LLM
+        "@elizaos/plugin-trajectory-logger", // Bench server records trajectories itself
+        "@elizaos/plugin-cron", // Scheduled services are noise in one-shot benchmark mode
+        "@elizaos/plugin-agent-skills", // Expects skill-catalog services that benchmark mode does not wire up
+    ]);
+    for (const pluginName of benchmarkCorePlugins) {
+        if (skipPlugins.has(pluginName)) {
+            elizaLogger.debug(`[bench] Skipping plugin (benchmark mode): ${pluginName}`);
+            continue;
+        }
+        try {
+            const pluginModule = await import(pluginName);
+            const plugin = pluginModule.default ?? pluginModule[Object.keys(pluginModule)[0]];
+            if (plugin) {
+                plugins.push(toPlugin(plugin, pluginName));
+                loadedPlugins.push(pluginName);
+            }
+        }
+        catch (error) {
+            // Some plugins may not be available in all environments — that's OK
+            failedPlugins.push(pluginName);
+            elizaLogger.debug(`[bench] Plugin not available: ${pluginName} (${formatUnknownError(error)})`);
+        }
+    }
+    elizaLogger.info(`[bench] Loaded ${loadedPlugins.length}/${benchmarkCorePlugins.length} benchmark plugins`);
+    if (failedPlugins.length > 0) {
+        elizaLogger.debug(`[bench] Unavailable plugins: ${failedPlugins.join(", ")}`);
+    }
+    // Load the autonomous orchestration plugin — provides workspace context,
+    // session keys, autonomous state, custom actions, and lifecycle actions.
+    try {
+        const workspaceDir = process.env.AUTONOMOUS_WORKSPACE_DIR ?? process.cwd();
+        const autonomousPlugin = createElizaPlugin({
+            workspaceDir,
+            agentId: "benchmark",
+        });
+        plugins.push(toPlugin(autonomousPlugin, "autonomous-plugin"));
+        elizaLogger.info(`[bench] Loaded autonomous plugin with workspace: ${workspaceDir}`);
+    }
+    catch (error) {
+        elizaLogger.error(`[bench] Failed to load autonomous plugin: ${formatUnknownError(error)}`);
+    }
+    // Load trust plugin — provides trust engine, security module, and permission system
+    // (may already be in CORE_PLUGINS but we want to ensure it's loaded)
+    if (!loadedPlugins.includes("@elizaos/plugin-trust")) {
+        try {
+            const { default: trustPlugin } = await import("@elizaos/plugin-trust");
+            plugins.push(toPlugin(trustPlugin, "@elizaos/plugin-trust"));
+            elizaLogger.info("[bench] Loaded plugin: @elizaos/plugin-trust");
+        }
+        catch (error) {
+            elizaLogger.debug(`[bench] Trust plugin not available: ${formatUnknownError(error)}`);
+        }
+    }
+    // Load LLM provider plugins based on environment
+    const groqApiKey = process.env.GROQ_API_KEY?.trim();
+    if (groqApiKey) {
+        process.env.GROQ_API_KEY = groqApiKey;
+        try {
+            const { default: groqPlugin } = await import("@elizaos/plugin-groq");
+            plugins.push(toPlugin(groqPlugin, "@elizaos/plugin-groq"));
+            elizaLogger.info("[bench] Loaded LLM plugin: @elizaos/plugin-groq");
+        }
+        catch (error) {
+            elizaLogger.warn(`[bench] Groq plugin not available: ${formatUnknownError(error)}`);
+        }
+    }
+    // const rawOllamaBaseUrl =
+    //   process.env.OLLAMA_BASE_URL?.trim() ??
+    //   process.env.OLLAMA_API_ENDPOINT?.trim();
+    // if (rawOllamaBaseUrl) {
+    //   const normalizedOllamaBaseUrl = rawOllamaBaseUrl.replace(/\/api\/?$/, "");
+    //   process.env.OLLAMA_BASE_URL = normalizedOllamaBaseUrl;
+    //   process.env.OLLAMA_API_ENDPOINT =
+    //     process.env.OLLAMA_API_ENDPOINT ??
+    //     `${normalizedOllamaBaseUrl.replace(/\/$/, "")}/api`;
+    //   try {
+    //     const { default: ollamaPlugin } = await import("@elizaos/plugin-ollama");
+    //     plugins.push(toPlugin(ollamaPlugin, "@elizaos/plugin-ollama"));
+    //     elizaLogger.info("[bench] Loaded LLM plugin: @elizaos/plugin-ollama");
+    //   } catch (error: unknown) {
+    //     elizaLogger.warn(
+    //       `[bench] Ollama plugin not available: ${formatUnknownError(error)}`,
+    //     );
+    //   }
+    // }
+    const openAiApiKey = process.env.OPENAI_API_KEY?.trim();
+    if (openAiApiKey &&
+        !openAiApiKey.startsWith("gsk_")
+    // && !rawOllamaBaseUrl
+    ) {
+        process.env.OPENAI_API_KEY = openAiApiKey;
+        try {
+            const { default: openaiPlugin } = await import("@elizaos/plugin-openai");
+            plugins.push(toPlugin(openaiPlugin, "@elizaos/plugin-openai"));
+            elizaLogger.info("[bench] Loaded LLM plugin: @elizaos/plugin-openai");
+        }
+        catch (error) {
+            elizaLogger.debug(`[bench] OpenAI plugin not available: ${formatUnknownError(error)}`);
+        }
+    }
+    // Load computer use plugin if enabled
+    if (process.env.AUTONOMOUS_ENABLE_COMPUTERUSE) {
+        try {
+            process.env.COMPUTERUSE_ENABLED ??= "true";
+            process.env.COMPUTERUSE_MODE ??= "local";
+            const localComputerusePath = "../../../plugins/plugin-computeruse/typescript/src/index.ts";
+            const computeruseModule = (await import(localComputerusePath));
+            const computerusePlugin = computeruseModule.computerusePlugin ??
+                computeruseModule.computerUsePlugin ??
+                computeruseModule.default;
+            if (computerusePlugin) {
+                plugins.push(toPlugin(computerusePlugin, localComputerusePath));
+                elizaLogger.info("[bench] Loaded local plugin: @elizaos/plugin-computeruse");
+            }
+        }
+        catch (error) {
+            elizaLogger.debug(`[bench] Computer use plugin not available: ${formatUnknownError(error)}`);
+        }
+    }
+    // Load mock plugin for testing (file is gitignored for local-only use)
+    if (process.env.AUTONOMOUS_BENCH_MOCK === "true") {
+        try {
+            // @ts-ignore - mock-plugin.ts is gitignored, only exists for local benchmarking
+            const { mockPlugin } = await import("./mock-plugin.ts");
+            plugins.push(toPlugin(mockPlugin, "./mock-plugin.ts"));
+            elizaLogger.info("[bench] Loaded mock benchmark plugin");
+        }
+        catch (error) {
+            elizaLogger.error(`[bench] Failed to load mock benchmark plugin: ${formatUnknownError(error)}`);
+        }
+    }
+    // Build settings object from environment variables
+    // These are needed by plugins like Groq that use runtime.getSetting()
+    const settings = {
+        // Use in-memory database for benchmarks to avoid pglite corruption issues
+        // and ensure a clean state for each benchmark run
+        PGLITE_DATA_DIR: "memory://",
+    };
+    const envKeys = [
+        "GROQ_API_KEY",
+        "OPENAI_API_KEY",
+        "ANTHROPIC_API_KEY",
+        "OPENROUTER_API_KEY",
+        "GOOGLE_GENERATIVE_AI_API_KEY",
+    ];
+    for (const key of envKeys) {
+        const value = process.env[key]?.trim();
+        if (value) {
+            settings[key] = value;
+        }
+    }
+    // Pre-register the SQL and local-embedding plugins before initialize().
+    // AgentRuntime initializes character plugins in parallel, which can race
+    // plugin startup against SQL adapter setup and leave runtime.db unset.
+    const preregisterPluginNames = new Set([
+        "@elizaos/plugin-sql",
+        "@elizaos/plugin-local-embedding",
+    ]);
+    const preregisterPlugins = plugins.filter((plugin) => preregisterPluginNames.has(plugin.name));
+    const deferredPlugins = plugins.filter((plugin) => !preregisterPluginNames.has(plugin.name));
+    const runtime = new AgentRuntime({
+        character: {
+            name: "Kira",
+            bio: ["A benchmark execution agent."],
+            messageExamples: [],
+            topics: [],
+            adjectives: [],
+            plugins: [],
+            settings: {
+                secrets: settings,
+            },
+        },
+        adapter: new InMemoryDatabaseAdapter(),
+        plugins: deferredPlugins,
+    });
+    let databaseAdapterMode = "plugin-sql";
+    const runtimeWithAdapter = runtime;
+    const sqlPlugin = preregisterPlugins.find((candidate) => candidate.name === "@elizaos/plugin-sql");
+    if (sqlPlugin) {
+        try {
+            await runtime.registerPlugin(sqlPlugin);
+            elizaLogger.info(`[bench] Pre-registered plugin: ${sqlPlugin.name}`);
+            if (runtimeWithAdapter.adapter &&
+                !(await runtimeWithAdapter.adapter.isReady())) {
+                await runtimeWithAdapter.adapter.init();
+                elizaLogger.info("[bench] Database adapter initialized before runtime init");
+            }
+        }
+        catch (error) {
+            databaseAdapterMode = "in-memory-fallback";
+            runtime.adapter = new InMemoryDatabaseAdapter();
+            elizaLogger.warn(`[bench] Falling back to in-memory database adapter: ${formatUnknownError(error)}`);
+        }
+    }
+    else {
+        databaseAdapterMode = "in-memory-fallback";
+        runtime.adapter = new InMemoryDatabaseAdapter();
+        elizaLogger.warn("[bench] SQL plugin unavailable; using in-memory database adapter for benchmark mode");
+    }
+    const localEmbeddingPlugin = preregisterPlugins.find((candidate) => candidate.name === "@elizaos/plugin-local-embedding");
+    if (localEmbeddingPlugin) {
+        await runtime.registerPlugin(localEmbeddingPlugin);
+        elizaLogger.info(`[bench] Pre-registered plugin: ${localEmbeddingPlugin.name}`);
+    }
+    await runtime.initialize();
+    const modelHandlers = runtime.models;
+    const modelHandlerSummary = Object.fromEntries([...(modelHandlers?.entries() ?? [])].map(([modelType, handlers]) => [
+        modelType,
+        handlers.map((handler) => ({
+            provider: handler.provider ?? "unknown",
+            priority: handler.priority ?? 0,
+        })),
+    ]));
+    elizaLogger.info(`[bench] Model handlers: ${JSON.stringify(modelHandlerSummary)}`);
+    const textGenerationModelTypes = new Set([
+        "TEXT_SMALL",
+        "TEXT_LARGE",
+        "REASONING_SMALL",
+        "REASONING_LARGE",
+        "TEXT_COMPLETION",
+    ]);
+    const hasTextGenerationModel = [...(modelHandlers?.entries() ?? [])].some(([modelType, handlers]) => textGenerationModelTypes.has(String(modelType)) &&
+        Array.isArray(handlers) &&
+        handlers.length > 0);
+    elizaLogger.info(`[bench] Database adapter mode: ${databaseAdapterMode}`);
+    elizaLogger.info(`[bench] Runtime initialized — agent=${runtime.character.name}, plugins=${plugins.length}`);
+    const roomToSession = new Map();
+    const entityToSession = new Map();
+    const trajectoriesBySession = new Map();
+    const outboxBySession = new Map();
+    const benchmarkTransport = {
+        sendDirectMessage: async (targetEntityId, content) => {
+            const key = entityToSession.get(targetEntityId);
+            const text = typeof content.text === "string" ? content.text : "";
+            const source = typeof content.source === "string" ? content.source : "benchmark";
+            if (!key)
+                return;
+            const current = outboxBySession.get(key) ?? [];
+            current.push({
+                kind: "direct",
+                targetId: targetEntityId,
+                text,
+                source,
+                ts: Date.now(),
+            });
+            outboxBySession.set(key, current);
+        },
+        sendRoomMessage: async (targetRoomId, content) => {
+            const key = roomToSession.get(targetRoomId);
+            const text = typeof content.text === "string" ? content.text : "";
+            const source = typeof content.source === "string" ? content.source : "benchmark";
+            if (!key)
+                return;
+            const current = outboxBySession.get(key) ?? [];
+            current.push({
+                kind: "room",
+                targetId: targetRoomId,
+                text,
+                source,
+                ts: Date.now(),
+            });
+            outboxBySession.set(key, current);
+        },
+    };
+    const runtimeWithServiceOverride = runtime;
+    const originalGetService = runtimeWithServiceOverride.getService.bind(runtime);
+    runtimeWithServiceOverride.getService = (serviceType) => {
+        if (serviceType === "benchmark") {
+            return benchmarkTransport;
+        }
+        return originalGetService(serviceType);
+    };
+    const sessions = new Map();
+    let activeSession = null;
+    const registerSessionRefs = (session) => {
+        const key = sessionKey(session);
+        roomToSession.set(session.roomId, key);
+        roomToSession.set(session.relayRoomId, key);
+        entityToSession.set(session.userEntityId, key);
+    };
+    const resolveSession = (taskId, benchmark, createIfMissing = true) => {
+        const key = `${benchmark}:${taskId}`;
+        const existing = sessions.get(key);
+        if (existing) {
+            activeSession = existing;
+            return existing;
+        }
+        if (!createIfMissing)
+            return null;
+        const created = createSession(taskId, benchmark);
+        sessions.set(key, created);
+        registerSessionRefs(created);
+        activeSession = created;
+        return created;
+    };
+    const server = http.createServer(async (req, res) => {
+        res.setHeader("Access-Control-Allow-Origin", "*");
+        res.setHeader("Access-Control-Allow-Methods", "POST, GET, OPTIONS");
+        res.setHeader("Access-Control-Allow-Headers", "Content-Type");
+        const requestUrl = new URL(req.url ?? "/", "http://localhost");
+        const pathname = requestUrl.pathname;
+        if (req.method === "OPTIONS") {
+            res.writeHead(200);
+            res.end();
+            return;
+        }
+        if (pathname === "/api/benchmark/health" && req.method === "GET") {
+            res.writeHead(200, { "Content-Type": "application/json" });
+            res.end(JSON.stringify({
+                status: "ready",
+                agent_name: runtime.character.name ?? "Autonomous",
+                plugins: plugins.length,
+                database_adapter: databaseAdapterMode,
+                has_text_model: hasTextGenerationModel,
+                message_mode: messageMode,
+                model_handlers: modelHandlerSummary,
+                active_session: activeSession
+                    ? {
+                        benchmark: activeSession.benchmark,
+                        task_id: activeSession.taskId,
+                        room_id: activeSession.roomId,
+                    }
+                    : null,
+            }));
+            return;
+        }
+        if (pathname === "/api/benchmark/reset" && req.method === "POST") {
+            let body = "";
+            req.on("data", (chunk) => (body += chunk));
+            req.on("end", async () => {
+                try {
+                    const parsed = body.trim()
+                        ? JSON.parse(body)
+                        : {};
+                    const taskId = typeof parsed.task_id === "string" &&
+                        parsed.task_id.trim().length > 0
+                        ? parsed.task_id
+                        : "default-task";
+                    const benchmark = typeof parsed.benchmark === "string" &&
+                        parsed.benchmark.trim().length > 0
+                        ? parsed.benchmark
+                        : "unknown";
+                    const session = resolveSession(taskId, benchmark, true);
+                    if (!session) {
+                        throw new Error("Failed to initialize benchmark session");
+                    }
+                    const key = sessionKey(session);
+                    trajectoriesBySession.set(key, []);
+                    outboxBySession.set(key, []);
+                    await ensureBenchmarkSessionContext(runtime, session);
+                    res.writeHead(200, { "Content-Type": "application/json" });
+                    res.end(JSON.stringify({
+                        status: "ok",
+                        room_id: session.roomId,
+                        task_id: session.taskId,
+                        benchmark: session.benchmark,
+                    }));
+                }
+                catch (err) {
+                    const errorMessage = err instanceof Error ? err.message : String(err);
+                    elizaLogger.error(`[bench] Reset error: ${formatUnknownError(err)}`);
+                    res.writeHead(500, { "Content-Type": "application/json" });
+                    res.end(JSON.stringify({ error: errorMessage }));
+                }
+            });
+            return;
+        }
+        if (pathname === "/api/benchmark/outbox" && req.method === "GET") {
+            const context = extractRecord({
+                benchmark: requestUrl.searchParams.get("benchmark") ?? undefined,
+                task_id: requestUrl.searchParams.get("task_id") ??
+                    requestUrl.searchParams.get("taskId") ??
+                    undefined,
+            });
+            const taskId = extractTaskId(context);
+            const benchmark = extractBenchmarkName(context);
+            const session = resolveSession(taskId, benchmark, false) ??
+                activeSession ??
+                resolveSession("default-task", "unknown", false);
+            if (!session) {
+                res.writeHead(200, { "Content-Type": "application/json" });
+                res.end(JSON.stringify({ status: "ok", outbox: [] }));
+                return;
+            }
+            const key = sessionKey(session);
+            res.writeHead(200, { "Content-Type": "application/json" });
+            res.end(JSON.stringify({
+                status: "ok",
+                benchmark: session.benchmark,
+                task_id: session.taskId,
+                room_id: session.roomId,
+                outbox: outboxBySession.get(key) ?? [],
+            }));
+            return;
+        }
+        if (pathname === "/api/benchmark/trajectory" && req.method === "GET") {
+            const context = extractRecord({
+                benchmark: requestUrl.searchParams.get("benchmark") ?? undefined,
+                task_id: requestUrl.searchParams.get("task_id") ??
+                    requestUrl.searchParams.get("taskId") ??
+                    undefined,
+            });
+            const taskId = extractTaskId(context);
+            const benchmark = extractBenchmarkName(context);
+            const session = resolveSession(taskId, benchmark, false) ??
+                activeSession ??
+                resolveSession("default-task", "unknown", false);
+            if (!session) {
+                res.writeHead(200, { "Content-Type": "application/json" });
+                res.end(JSON.stringify({
+                    status: "ok",
+                    steps: [],
+                    outbox: [],
+                }));
+                return;
+            }
+            const key = sessionKey(session);
+            res.writeHead(200, { "Content-Type": "application/json" });
+            res.end(JSON.stringify({
+                status: "ok",
+                benchmark: session.benchmark,
+                task_id: session.taskId,
+                room_id: session.roomId,
+                relay_room_id: session.relayRoomId,
+                steps: trajectoriesBySession.get(key) ?? [],
+                outbox: outboxBySession.get(key) ?? [],
+            }));
+            return;
+        }
+        if (pathname === "/api/benchmark/message" && req.method === "POST") {
+            let body = "";
+            req.on("data", (chunk) => (body += chunk));
+            req.on("end", async () => {
+                try {
+                    const parsed = JSON.parse(body);
+                    const text = typeof parsed.text === "string" ? parsed.text.trim() : "";
+                    if (!text) {
+                        throw new Error("Request body must include non-empty string `text`");
+                    }
+                    const context = extractRecord(parsed.context);
+                    const taskId = extractTaskId(context);
+                    const benchmark = extractBenchmarkName(context);
+                    const session = resolveSession(taskId, benchmark, true) ??
+                        activeSession ??
+                        resolveSession("default-task", "unknown", true);
+                    if (!session) {
+                        throw new Error("Failed to resolve benchmark session");
+                    }
+                    const key = sessionKey(session);
+                    const trajectory = trajectoriesBySession.get(key) ?? [];
+                    const startedAt = Date.now();
+                    await ensureBenchmarkSessionContext(runtime, session);
+                    const composedPrompt = composeBenchmarkPrompt({
+                        text,
+                        context,
+                        image: parsed.image,
+                        mode: messageMode,
+                    });
+                    const incomingMessage = {
+                        id: stringToUuid(`benchmark-msg:${Date.now()}:${Math.random()}`),
+                        content: {
+                            text: composedPrompt,
+                            source: "benchmark",
+                            metadata: {
+                                benchmark: session.benchmark,
+                                taskId: session.taskId,
+                                ...(context ? { contextJson: JSON.stringify(context) } : {}),
+                            },
+                        },
+                        entityId: session.userEntityId,
+                        agentId: runtime.agentId,
+                        roomId: session.roomId,
+                        createdAt: Date.now(),
+                    };
+                    if (!hasTextGenerationModel) {
+                        throw new Error("No text generation model provider is configured for benchmark runtime");
+                    }
+                    let responseText = "";
+                    let thought = null;
+                    let actions = [];
+                    let params = {};
+                    if (messageMode === "direct-model") {
+                        responseText = (await runtime.useModel(ModelType.TEXT_SMALL, {
+                            prompt: composedPrompt,
+                            maxTokens: 1024,
+                            temperature: 0,
+                        })).trim();
+                    }
+                    else {
+                        const callbackTexts = [];
+                        const callback = async (content) => {
+                            if (typeof content.text === "string" &&
+                                content.text.trim().length > 0) {
+                                callbackTexts.push(content.text.trim());
+                            }
+                            return [];
+                        };
+                        if (!runtime.messageService) {
+                            throw new Error("Runtime message service is not available");
+                        }
+                        const result = await runtime.messageService.handleMessage(runtime, incomingMessage, callback);
+                        responseText =
+                            typeof result.responseContent?.text === "string"
+                                ? result.responseContent.text
+                                : callbackTexts.join("\n\n");
+                        thought =
+                            typeof result.responseContent?.thought === "string"
+                                ? result.responseContent.thought
+                                : null;
+                        actions = coerceActions(result.responseContent?.actions);
+                        params = coerceParams(result.responseContent?.params);
+                    }
+                    const finishedAt = Date.now();
+                    trajectory.push({
+                        step: trajectory.length + 1,
+                        startedAt,
+                        finishedAt,
+                        inputText: text,
+                        promptText: composedPrompt,
+                        context,
+                        thought,
+                        responseText,
+                        actions,
+                        params,
+                    });
+                    trajectoriesBySession.set(key, trajectory);
+                    res.writeHead(200, { "Content-Type": "application/json" });
+                    res.end(JSON.stringify({
+                        text: responseText,
+                        thought,
+                        actions,
+                        params,
+                        benchmark: session.benchmark,
+                        task_id: session.taskId,
+                        room_id: session.roomId,
+                        trajectory_step: trajectory.length,
+                    }));
+                }
+                catch (err) {
+                    const errorMessage = err instanceof Error ? err.message : String(err);
+                    const errorDetail = err instanceof Error && err.stack
+                        ? err.stack
+                        : formatUnknownError(err);
+                    elizaLogger.error(`[bench] Request error: ${errorDetail}`);
+                    res.writeHead(500, { "Content-Type": "application/json" });
+                    res.end(JSON.stringify({ error: errorMessage }));
+                }
+            });
+            return;
+        }
+        res.writeHead(404);
+        res.end("Not Found");
+    });
+    server.listen(port, () => {
+        elizaLogger.info(`[bench] Autonomous benchmark server listening on port ${port}`);
+        console.log(`AUTONOMOUS_BENCH_READY port=${port}`);
+    });
+}
+startBenchmarkServer().catch((err) => {
+    console.error("Failed to start benchmark server:", err);
+    process.exit(1);
+});