@elizaos/autonomous 2.0.0-alpha.72 → 2.0.0-alpha.73

package/packages/autonomous/src/benchmark-server.js

@@ -1,773 +0,0 @@
-import http from "node:http";
-import path from "node:path";
-import { AgentRuntime, ChannelType, elizaLogger, InMemoryDatabaseAdapter, ModelType, stringToUuid, } from "@elizaos/core";
-import dotenv from "dotenv";
-import { createElizaPlugin } from "./runtime/eliza-plugin";
-// Load environment variables BEFORE anything else
-// This ensures API keys are available when plugins initialize
-dotenv.config({ path: path.resolve(process.cwd(), ".env") });
-const DEFAULT_PORT = 3939;
-const BENCHMARK_WORLD_ID = stringToUuid("autonomous-benchmark-world");
-const BENCHMARK_MESSAGE_SERVER_ID = stringToUuid("autonomous-benchmark-message-server");
-function formatUnknownError(error) {
-    if (error instanceof Error) {
-        return `${error.name}: ${error.message}`;
-    }
-    return String(error);
-}
-function toPlugin(candidate, source) {
-    if (!candidate || typeof candidate !== "object") {
-        throw new Error(`Plugin from ${source} was not an object`);
-    }
-    const pluginLike = candidate;
-    if (typeof pluginLike.name !== "string" || pluginLike.name.length === 0) {
-        throw new Error(`Plugin from ${source} was missing a valid name`);
-    }
-    return candidate;
-}
-function resolvePort() {
-    const raw = process.env.AUTONOMOUS_BENCH_PORT;
-    if (!raw)
-        return DEFAULT_PORT;
-    const parsed = Number(raw);
-    if (!Number.isFinite(parsed) || parsed < 1 || parsed > 65535) {
-        elizaLogger.warn(`[bench] Invalid AUTONOMOUS_BENCH_PORT="${raw}"; using ${DEFAULT_PORT}`);
-        return DEFAULT_PORT;
-    }
-    return Math.floor(parsed);
-}
-function extractRecord(value) {
-    if (!value || typeof value !== "object" || Array.isArray(value)) {
-        return undefined;
-    }
-    return value;
-}
-function extractTaskId(context) {
-    const bySnake = context?.task_id;
-    if (typeof bySnake === "string" && bySnake.trim())
-        return bySnake.trim();
-    const byCamel = context?.taskId;
-    if (typeof byCamel === "string" && byCamel.trim())
-        return byCamel.trim();
-    return "default-task";
-}
-function extractBenchmarkName(context) {
-    const benchmark = context?.benchmark;
-    if (typeof benchmark === "string" && benchmark.trim()) {
-        return benchmark.trim();
-    }
-    return "unknown";
-}
-function composeBenchmarkPrompt(params) {
-    const segments = [params.text.trim()];
-    if (params.context && Object.keys(params.context).length > 0) {
-        segments.push([
-            "BENCHMARK CONTEXT (authoritative):",
-            JSON.stringify(params.context, null, 2),
-        ].join("\n"));
-    }
-    if (params.image !== undefined) {
-        segments.push(["IMAGE PAYLOAD:", JSON.stringify(params.image, null, 2)].join("\n"));
-    }
-    if (params.mode === "message-service") {
-        segments.push("Respond using normal Eliza action output so actions/params can be executed and evaluated.");
-    }
-    else {
-        segments.push("Reply directly to the benchmark prompt. If it asks for JSON, return JSON only with no extra narration.");
-    }
-    return segments.join("\n\n");
-}
-function resolveBenchmarkMessageMode() {
-    const raw = process.env.AUTONOMOUS_BENCH_MESSAGE_MODE?.trim().toLowerCase();
-    if (raw === "message-service")
-        return "message-service";
-    return "direct-model";
-}
-function coerceActions(value) {
-    if (!Array.isArray(value))
-        return [];
-    return value.filter((entry) => typeof entry === "string");
-}
-function coerceParams(value) {
-    if (value && typeof value === "object" && !Array.isArray(value)) {
-        return value;
-    }
-    if (typeof value === "string") {
-        try {
-            const parsed = JSON.parse(value);
-            if (parsed && typeof parsed === "object" && !Array.isArray(parsed)) {
-                return parsed;
-            }
-        }
-        catch {
-            // ignore malformed param strings
-        }
-    }
-    return {};
-}
-function sessionKey(session) {
-    return `${session.benchmark}:${session.taskId}`;
-}
-async function ensureBenchmarkSessionContext(runtime, session) {
-    await runtime.ensureWorldExists({
-        id: BENCHMARK_WORLD_ID,
-        name: "Autonomous Benchmark World",
-        agentId: runtime.agentId,
-        messageServerId: BENCHMARK_MESSAGE_SERVER_ID,
-        metadata: {
-            type: "benchmark",
-            description: "World used for benchmark sessions",
-            extra: {
-                benchmark: session.benchmark,
-            },
-        },
-    });
-    // Use ChannelType.API to ensure the agent always responds to benchmark messages
-    // (API channel type bypasses shouldRespond evaluation)
-    await runtime.ensureRoomExists({
-        id: session.roomId,
-        name: `benchmark:${session.taskId}`,
-        source: "benchmark",
-        type: ChannelType.API,
-        channelId: `bench-${session.taskId}`,
-        messageServerId: BENCHMARK_MESSAGE_SERVER_ID,
-        worldId: BENCHMARK_WORLD_ID,
-        metadata: {
-            benchmark: session.benchmark,
-            taskId: session.taskId,
-        },
-    });
-    await runtime.ensureRoomExists({
-        id: session.relayRoomId,
-        name: "relay-room",
-        source: "benchmark",
-        type: ChannelType.API,
-        channelId: `relay-${session.taskId}`,
-        messageServerId: BENCHMARK_MESSAGE_SERVER_ID,
-        worldId: BENCHMARK_WORLD_ID,
-        metadata: {
-            benchmark: session.benchmark,
-            taskId: session.taskId,
-            role: "relay-room",
-        },
-    });
-    await runtime.ensureConnection({
-        entityId: session.userEntityId,
-        roomId: session.roomId,
-        worldId: BENCHMARK_WORLD_ID,
-        userName: "Benchmark User",
-        source: "benchmark",
-        channelId: `bench-${session.taskId}`,
-        type: ChannelType.API,
-        messageServerId: BENCHMARK_MESSAGE_SERVER_ID,
-        metadata: {
-            benchmark: session.benchmark,
-            taskId: session.taskId,
-            role: "benchmark-room",
-        },
-    });
-    // The benchmark transport only needs the relay room to exist so room-targeted
-    // emissions can be attributed back to the active session. In benchmark mode
-    // some runtime/database combinations do not expose the agent entity through
-    // getEntitiesByIds(), which makes an explicit ensureParticipantInRoom() call
-    // fail even though the primary benchmark room is already fully connected via
-    // ensureConnection() above.
-}
-function createSession(taskId, benchmark) {
-    const normalizedTaskId = taskId.trim() || "default-task";
-    const normalizedBenchmark = benchmark.trim() || "unknown";
-    const seed = `${normalizedBenchmark}:${normalizedTaskId}:${Date.now()}:${Math.random()}`;
-    return {
-        benchmark: normalizedBenchmark,
-        taskId: normalizedTaskId,
-        roomId: stringToUuid(`benchmark-room:${seed}`),
-        relayRoomId: stringToUuid(`benchmark-relay:${seed}`),
-        userEntityId: stringToUuid(`benchmark-user:${seed}`),
-    };
-}
-// Proper robust server implementation
-export async function startBenchmarkServer() {
-    const port = resolvePort();
-    const messageMode = resolveBenchmarkMessageMode();
-    elizaLogger.info(`[bench] Initializing autonomous benchmark runtime on port ${port} (message_mode=${messageMode})...`);
-    // Benchmark mode prefers a stable, comparable runtime over full production
-    // plugin parity. A smaller allowlist avoids startup-only side effects from
-    // plugins that are useful in the full app but unnecessary for benchmark I/O.
-    const benchmarkCorePlugins = [
-        "@elizaos/plugin-sql",
-        "@elizaos/plugin-local-embedding",
-        "@elizaos/plugin-secrets-manager",
-        "@elizaos/plugin-rolodex",
-        "@elizaos/plugin-trust",
-        "@elizaos/plugin-todo",
-        "@elizaos/plugin-experience",
-    ];
-    const plugins = [];
-    const loadedPlugins = [];
-    const failedPlugins = [];
-    // Plugins to skip in benchmark context — these require external auth or
-    // interfere with benchmark operation
-    const skipPlugins = new Set([
-        "@elizaos/plugin-elizacloud", // Requires ElizaOS cloud auth, conflicts with local LLM
-        "@elizaos/plugin-trajectory-logger", // Bench server records trajectories itself
-        "@elizaos/plugin-cron", // Scheduled services are noise in one-shot benchmark mode
-        "@elizaos/plugin-agent-skills", // Expects skill-catalog services that benchmark mode does not wire up
-    ]);
-    for (const pluginName of benchmarkCorePlugins) {
-        if (skipPlugins.has(pluginName)) {
-            elizaLogger.debug(`[bench] Skipping plugin (benchmark mode): ${pluginName}`);
-            continue;
-        }
-        try {
-            const pluginModule = await import(pluginName);
-            const plugin = pluginModule.default ?? pluginModule[Object.keys(pluginModule)[0]];
-            if (plugin) {
-                plugins.push(toPlugin(plugin, pluginName));
-                loadedPlugins.push(pluginName);
-            }
-        }
-        catch (error) {
-            // Some plugins may not be available in all environments — that's OK
-            failedPlugins.push(pluginName);
-            elizaLogger.debug(`[bench] Plugin not available: ${pluginName} (${formatUnknownError(error)})`);
-        }
-    }
-    elizaLogger.info(`[bench] Loaded ${loadedPlugins.length}/${benchmarkCorePlugins.length} benchmark plugins`);
-    if (failedPlugins.length > 0) {
-        elizaLogger.debug(`[bench] Unavailable plugins: ${failedPlugins.join(", ")}`);
-    }
-    // Load the autonomous orchestration plugin — provides workspace context,
-    // session keys, autonomous state, custom actions, and lifecycle actions.
-    try {
-        const workspaceDir = process.env.AUTONOMOUS_WORKSPACE_DIR ?? process.cwd();
-        const autonomousPlugin = createElizaPlugin({
-            workspaceDir,
-            agentId: "benchmark",
-        });
-        plugins.push(toPlugin(autonomousPlugin, "autonomous-plugin"));
-        elizaLogger.info(`[bench] Loaded autonomous plugin with workspace: ${workspaceDir}`);
-    }
-    catch (error) {
-        elizaLogger.error(`[bench] Failed to load autonomous plugin: ${formatUnknownError(error)}`);
-    }
-    // Load trust plugin — provides trust engine, security module, and permission system
-    // (may already be in CORE_PLUGINS but we want to ensure it's loaded)
-    if (!loadedPlugins.includes("@elizaos/plugin-trust")) {
-        try {
-            const { default: trustPlugin } = await import("@elizaos/plugin-trust");
-            plugins.push(toPlugin(trustPlugin, "@elizaos/plugin-trust"));
-            elizaLogger.info("[bench] Loaded plugin: @elizaos/plugin-trust");
-        }
-        catch (error) {
-            elizaLogger.debug(`[bench] Trust plugin not available: ${formatUnknownError(error)}`);
-        }
-    }
-    // Load LLM provider plugins based on environment
-    const groqApiKey = process.env.GROQ_API_KEY?.trim();
-    if (groqApiKey) {
-        process.env.GROQ_API_KEY = groqApiKey;
-        try {
-            const { default: groqPlugin } = await import("@elizaos/plugin-groq");
-            plugins.push(toPlugin(groqPlugin, "@elizaos/plugin-groq"));
-            elizaLogger.info("[bench] Loaded LLM plugin: @elizaos/plugin-groq");
-        }
-        catch (error) {
-            elizaLogger.warn(`[bench] Groq plugin not available: ${formatUnknownError(error)}`);
-        }
-    }
-    // const rawOllamaBaseUrl =
-    //   process.env.OLLAMA_BASE_URL?.trim() ??
-    //   process.env.OLLAMA_API_ENDPOINT?.trim();
-    // if (rawOllamaBaseUrl) {
-    //   const normalizedOllamaBaseUrl = rawOllamaBaseUrl.replace(/\/api\/?$/, "");
-    //   process.env.OLLAMA_BASE_URL = normalizedOllamaBaseUrl;
-    //   process.env.OLLAMA_API_ENDPOINT =
-    //     process.env.OLLAMA_API_ENDPOINT ??
-    //     `${normalizedOllamaBaseUrl.replace(/\/$/, "")}/api`;
-    //   try {
-    //     const { default: ollamaPlugin } = await import("@elizaos/plugin-ollama");
-    //     plugins.push(toPlugin(ollamaPlugin, "@elizaos/plugin-ollama"));
-    //     elizaLogger.info("[bench] Loaded LLM plugin: @elizaos/plugin-ollama");
-    //   } catch (error: unknown) {
-    //     elizaLogger.warn(
-    //       `[bench] Ollama plugin not available: ${formatUnknownError(error)}`,
-    //     );
-    //   }
-    // }
-    const openAiApiKey = process.env.OPENAI_API_KEY?.trim();
-    if (openAiApiKey &&
-        !openAiApiKey.startsWith("gsk_")
-    // && !rawOllamaBaseUrl
-    ) {
-        process.env.OPENAI_API_KEY = openAiApiKey;
-        try {
-            const { default: openaiPlugin } = await import("@elizaos/plugin-openai");
-            plugins.push(toPlugin(openaiPlugin, "@elizaos/plugin-openai"));
-            elizaLogger.info("[bench] Loaded LLM plugin: @elizaos/plugin-openai");
-        }
-        catch (error) {
-            elizaLogger.debug(`[bench] OpenAI plugin not available: ${formatUnknownError(error)}`);
-        }
-    }
-    // Load computer use plugin if enabled
-    if (process.env.AUTONOMOUS_ENABLE_COMPUTERUSE) {
-        try {
-            process.env.COMPUTERUSE_ENABLED ??= "true";
-            process.env.COMPUTERUSE_MODE ??= "local";
-            const localComputerusePath = "../../../plugins/plugin-computeruse/typescript/src/index.ts";
-            const computeruseModule = (await import(localComputerusePath));
-            const computerusePlugin = computeruseModule.computerusePlugin ??
-                computeruseModule.computerUsePlugin ??
-                computeruseModule.default;
-            if (computerusePlugin) {
-                plugins.push(toPlugin(computerusePlugin, localComputerusePath));
-                elizaLogger.info("[bench] Loaded local plugin: @elizaos/plugin-computeruse");
-            }
-        }
-        catch (error) {
-            elizaLogger.debug(`[bench] Computer use plugin not available: ${formatUnknownError(error)}`);
-        }
-    }
-    // Load mock plugin for testing (file is gitignored for local-only use)
-    if (process.env.AUTONOMOUS_BENCH_MOCK === "true") {
-        try {
-            // @ts-ignore - mock-plugin.ts is gitignored, only exists for local benchmarking
-            const { mockPlugin } = await import("./mock-plugin.ts");
-            plugins.push(toPlugin(mockPlugin, "./mock-plugin.ts"));
-            elizaLogger.info("[bench] Loaded mock benchmark plugin");
-        }
-        catch (error) {
-            elizaLogger.error(`[bench] Failed to load mock benchmark plugin: ${formatUnknownError(error)}`);
-        }
-    }
-    // Build settings object from environment variables
-    // These are needed by plugins like Groq that use runtime.getSetting()
-    const settings = {
-        // Use in-memory database for benchmarks to avoid pglite corruption issues
-        // and ensure a clean state for each benchmark run
-        PGLITE_DATA_DIR: "memory://",
-    };
-    const envKeys = [
-        "GROQ_API_KEY",
-        "OPENAI_API_KEY",
-        "ANTHROPIC_API_KEY",
-        "OPENROUTER_API_KEY",
-        "GOOGLE_GENERATIVE_AI_API_KEY",
-    ];
-    for (const key of envKeys) {
-        const value = process.env[key]?.trim();
-        if (value) {
-            settings[key] = value;
-        }
-    }
-    // Pre-register the SQL and local-embedding plugins before initialize().
-    // AgentRuntime initializes character plugins in parallel, which can race
-    // plugin startup against SQL adapter setup and leave runtime.db unset.
-    const preregisterPluginNames = new Set([
-        "@elizaos/plugin-sql",
-        "@elizaos/plugin-local-embedding",
-    ]);
-    const preregisterPlugins = plugins.filter((plugin) => preregisterPluginNames.has(plugin.name));
-    const deferredPlugins = plugins.filter((plugin) => !preregisterPluginNames.has(plugin.name));
-    const runtime = new AgentRuntime({
-        character: {
-            name: "Kira",
-            bio: ["A benchmark execution agent."],
-            messageExamples: [],
-            topics: [],
-            adjectives: [],
-            plugins: [],
-            settings: {
-                secrets: settings,
-            },
-        },
-        adapter: new InMemoryDatabaseAdapter(),
-        plugins: deferredPlugins,
-    });
-    let databaseAdapterMode = "plugin-sql";
-    const runtimeWithAdapter = runtime;
-    const sqlPlugin = preregisterPlugins.find((candidate) => candidate.name === "@elizaos/plugin-sql");
-    if (sqlPlugin) {
-        try {
-            await runtime.registerPlugin(sqlPlugin);
-            elizaLogger.info(`[bench] Pre-registered plugin: ${sqlPlugin.name}`);
-            if (runtimeWithAdapter.adapter &&
-                !(await runtimeWithAdapter.adapter.isReady())) {
-                await runtimeWithAdapter.adapter.init();
-                elizaLogger.info("[bench] Database adapter initialized before runtime init");
-            }
-        }
-        catch (error) {
-            databaseAdapterMode = "in-memory-fallback";
-            runtime.adapter = new InMemoryDatabaseAdapter();
-            elizaLogger.warn(`[bench] Falling back to in-memory database adapter: ${formatUnknownError(error)}`);
-        }
-    }
-    else {
-        databaseAdapterMode = "in-memory-fallback";
-        runtime.adapter = new InMemoryDatabaseAdapter();
-        elizaLogger.warn("[bench] SQL plugin unavailable; using in-memory database adapter for benchmark mode");
-    }
-    const localEmbeddingPlugin = preregisterPlugins.find((candidate) => candidate.name === "@elizaos/plugin-local-embedding");
-    if (localEmbeddingPlugin) {
-        await runtime.registerPlugin(localEmbeddingPlugin);
-        elizaLogger.info(`[bench] Pre-registered plugin: ${localEmbeddingPlugin.name}`);
-    }
-    await runtime.initialize();
-    const modelHandlers = runtime.models;
-    const modelHandlerSummary = Object.fromEntries([...(modelHandlers?.entries() ?? [])].map(([modelType, handlers]) => [
-        modelType,
-        handlers.map((handler) => ({
-            provider: handler.provider ?? "unknown",
-            priority: handler.priority ?? 0,
-        })),
-    ]));
-    elizaLogger.info(`[bench] Model handlers: ${JSON.stringify(modelHandlerSummary)}`);
-    const textGenerationModelTypes = new Set([
-        "TEXT_SMALL",
-        "TEXT_LARGE",
-        "REASONING_SMALL",
-        "REASONING_LARGE",
-        "TEXT_COMPLETION",
-    ]);
-    const hasTextGenerationModel = [...(modelHandlers?.entries() ?? [])].some(([modelType, handlers]) => textGenerationModelTypes.has(String(modelType)) &&
-        Array.isArray(handlers) &&
-        handlers.length > 0);
-    elizaLogger.info(`[bench] Database adapter mode: ${databaseAdapterMode}`);
-    elizaLogger.info(`[bench] Runtime initialized — agent=${runtime.character.name}, plugins=${plugins.length}`);
-    const roomToSession = new Map();
-    const entityToSession = new Map();
-    const trajectoriesBySession = new Map();
-    const outboxBySession = new Map();
-    const benchmarkTransport = {
-        sendDirectMessage: async (targetEntityId, content) => {
-            const key = entityToSession.get(targetEntityId);
-            const text = typeof content.text === "string" ? content.text : "";
-            const source = typeof content.source === "string" ? content.source : "benchmark";
-            if (!key)
-                return;
-            const current = outboxBySession.get(key) ?? [];
-            current.push({
-                kind: "direct",
-                targetId: targetEntityId,
-                text,
-                source,
-                ts: Date.now(),
-            });
-            outboxBySession.set(key, current);
-        },
-        sendRoomMessage: async (targetRoomId, content) => {
-            const key = roomToSession.get(targetRoomId);
-            const text = typeof content.text === "string" ? content.text : "";
-            const source = typeof content.source === "string" ? content.source : "benchmark";
-            if (!key)
-                return;
-            const current = outboxBySession.get(key) ?? [];
-            current.push({
-                kind: "room",
-                targetId: targetRoomId,
-                text,
-                source,
-                ts: Date.now(),
-            });
-            outboxBySession.set(key, current);
-        },
-    };
-    const runtimeWithServiceOverride = runtime;
-    const originalGetService = runtimeWithServiceOverride.getService.bind(runtime);
-    runtimeWithServiceOverride.getService = (serviceType) => {
-        if (serviceType === "benchmark") {
-            return benchmarkTransport;
-        }
-        return originalGetService(serviceType);
-    };
-    const sessions = new Map();
-    let activeSession = null;
-    const registerSessionRefs = (session) => {
-        const key = sessionKey(session);
-        roomToSession.set(session.roomId, key);
-        roomToSession.set(session.relayRoomId, key);
-        entityToSession.set(session.userEntityId, key);
-    };
-    const resolveSession = (taskId, benchmark, createIfMissing = true) => {
-        const key = `${benchmark}:${taskId}`;
-        const existing = sessions.get(key);
-        if (existing) {
-            activeSession = existing;
-            return existing;
-        }
-        if (!createIfMissing)
-            return null;
-        const created = createSession(taskId, benchmark);
-        sessions.set(key, created);
-        registerSessionRefs(created);
-        activeSession = created;
-        return created;
-    };
-    const server = http.createServer(async (req, res) => {
-        res.setHeader("Access-Control-Allow-Origin", "*");
-        res.setHeader("Access-Control-Allow-Methods", "POST, GET, OPTIONS");
-        res.setHeader("Access-Control-Allow-Headers", "Content-Type");
-        const requestUrl = new URL(req.url ?? "/", "http://localhost");
-        const pathname = requestUrl.pathname;
-        if (req.method === "OPTIONS") {
-            res.writeHead(200);
-            res.end();
-            return;
-        }
-        if (pathname === "/api/benchmark/health" && req.method === "GET") {
-            res.writeHead(200, { "Content-Type": "application/json" });
-            res.end(JSON.stringify({
-                status: "ready",
-                agent_name: runtime.character.name ?? "Autonomous",
-                plugins: plugins.length,
-                database_adapter: databaseAdapterMode,
-                has_text_model: hasTextGenerationModel,
-                message_mode: messageMode,
-                model_handlers: modelHandlerSummary,
-                active_session: activeSession
-                    ? {
-                        benchmark: activeSession.benchmark,
-                        task_id: activeSession.taskId,
-                        room_id: activeSession.roomId,
-                    }
-                    : null,
-            }));
-            return;
-        }
-        if (pathname === "/api/benchmark/reset" && req.method === "POST") {
-            let body = "";
-            req.on("data", (chunk) => (body += chunk));
-            req.on("end", async () => {
-                try {
-                    const parsed = body.trim()
-                        ? JSON.parse(body)
-                        : {};
-                    const taskId = typeof parsed.task_id === "string" &&
-                        parsed.task_id.trim().length > 0
-                        ? parsed.task_id
-                        : "default-task";
-                    const benchmark = typeof parsed.benchmark === "string" &&
-                        parsed.benchmark.trim().length > 0
-                        ? parsed.benchmark
-                        : "unknown";
-                    const session = resolveSession(taskId, benchmark, true);
-                    if (!session) {
-                        throw new Error("Failed to initialize benchmark session");
-                    }
-                    const key = sessionKey(session);
-                    trajectoriesBySession.set(key, []);
-                    outboxBySession.set(key, []);
-                    await ensureBenchmarkSessionContext(runtime, session);
-                    res.writeHead(200, { "Content-Type": "application/json" });
-                    res.end(JSON.stringify({
-                        status: "ok",
-                        room_id: session.roomId,
-                        task_id: session.taskId,
-                        benchmark: session.benchmark,
-                    }));
-                }
-                catch (err) {
-                    const errorMessage = err instanceof Error ? err.message : String(err);
-                    elizaLogger.error(`[bench] Reset error: ${formatUnknownError(err)}`);
-                    res.writeHead(500, { "Content-Type": "application/json" });
-                    res.end(JSON.stringify({ error: errorMessage }));
-                }
-            });
-            return;
-        }
-        if (pathname === "/api/benchmark/outbox" && req.method === "GET") {
-            const context = extractRecord({
-                benchmark: requestUrl.searchParams.get("benchmark") ?? undefined,
-                task_id: requestUrl.searchParams.get("task_id") ??
-                    requestUrl.searchParams.get("taskId") ??
-                    undefined,
-            });
-            const taskId = extractTaskId(context);
-            const benchmark = extractBenchmarkName(context);
-            const session = resolveSession(taskId, benchmark, false) ??
-                activeSession ??
-                resolveSession("default-task", "unknown", false);
-            if (!session) {
-                res.writeHead(200, { "Content-Type": "application/json" });
-                res.end(JSON.stringify({ status: "ok", outbox: [] }));
-                return;
-            }
-            const key = sessionKey(session);
-            res.writeHead(200, { "Content-Type": "application/json" });
-            res.end(JSON.stringify({
-                status: "ok",
-                benchmark: session.benchmark,
-                task_id: session.taskId,
-                room_id: session.roomId,
-                outbox: outboxBySession.get(key) ?? [],
-            }));
-            return;
-        }
-        if (pathname === "/api/benchmark/trajectory" && req.method === "GET") {
-            const context = extractRecord({
-                benchmark: requestUrl.searchParams.get("benchmark") ?? undefined,
-                task_id: requestUrl.searchParams.get("task_id") ??
-                    requestUrl.searchParams.get("taskId") ??
-                    undefined,
-            });
-            const taskId = extractTaskId(context);
-            const benchmark = extractBenchmarkName(context);
-            const session = resolveSession(taskId, benchmark, false) ??
-                activeSession ??
-                resolveSession("default-task", "unknown", false);
-            if (!session) {
-                res.writeHead(200, { "Content-Type": "application/json" });
-                res.end(JSON.stringify({
-                    status: "ok",
-                    steps: [],
-                    outbox: [],
-                }));
-                return;
-            }
-            const key = sessionKey(session);
-            res.writeHead(200, { "Content-Type": "application/json" });
-            res.end(JSON.stringify({
-                status: "ok",
-                benchmark: session.benchmark,
-                task_id: session.taskId,
-                room_id: session.roomId,
-                relay_room_id: session.relayRoomId,
-                steps: trajectoriesBySession.get(key) ?? [],
-                outbox: outboxBySession.get(key) ?? [],
-            }));
-            return;
-        }
-        if (pathname === "/api/benchmark/message" && req.method === "POST") {
-            let body = "";
-            req.on("data", (chunk) => (body += chunk));
-            req.on("end", async () => {
-                try {
-                    const parsed = JSON.parse(body);
-                    const text = typeof parsed.text === "string" ? parsed.text.trim() : "";
-                    if (!text) {
-                        throw new Error("Request body must include non-empty string `text`");
-                    }
-                    const context = extractRecord(parsed.context);
-                    const taskId = extractTaskId(context);
-                    const benchmark = extractBenchmarkName(context);
-                    const session = resolveSession(taskId, benchmark, true) ??
-                        activeSession ??
-                        resolveSession("default-task", "unknown", true);
-                    if (!session) {
-                        throw new Error("Failed to resolve benchmark session");
-                    }
-                    const key = sessionKey(session);
-                    const trajectory = trajectoriesBySession.get(key) ?? [];
-                    const startedAt = Date.now();
-                    await ensureBenchmarkSessionContext(runtime, session);
-                    const composedPrompt = composeBenchmarkPrompt({
-                        text,
-                        context,
-                        image: parsed.image,
-                        mode: messageMode,
-                    });
-                    const incomingMessage = {
-                        id: stringToUuid(`benchmark-msg:${Date.now()}:${Math.random()}`),
-                        content: {
-                            text: composedPrompt,
-                            source: "benchmark",
-                            metadata: {
-                                benchmark: session.benchmark,
-                                taskId: session.taskId,
-                                ...(context ? { contextJson: JSON.stringify(context) } : {}),
-                            },
-                        },
-                        entityId: session.userEntityId,
-                        agentId: runtime.agentId,
-                        roomId: session.roomId,
-                        createdAt: Date.now(),
-                    };
-                    if (!hasTextGenerationModel) {
-                        throw new Error("No text generation model provider is configured for benchmark runtime");
-                    }
-                    let responseText = "";
-                    let thought = null;
-                    let actions = [];
-                    let params = {};
-                    if (messageMode === "direct-model") {
-                        responseText = (await runtime.useModel(ModelType.TEXT_SMALL, {
-                            prompt: composedPrompt,
-                            maxTokens: 1024,
-                            temperature: 0,
-                        })).trim();
-                    }
-                    else {
-                        const callbackTexts = [];
-                        const callback = async (content) => {
-                            if (typeof content.text === "string" &&
-                                content.text.trim().length > 0) {
-                                callbackTexts.push(content.text.trim());
-                            }
-                            return [];
-                        };
-                        if (!runtime.messageService) {
-                            throw new Error("Runtime message service is not available");
-                        }
-                        const result = await runtime.messageService.handleMessage(runtime, incomingMessage, callback);
-                        responseText =
-                            typeof result.responseContent?.text === "string"
-                                ? result.responseContent.text
-                                : callbackTexts.join("\n\n");
-                        thought =
-                            typeof result.responseContent?.thought === "string"
-                                ? result.responseContent.thought
-                                : null;
-                        actions = coerceActions(result.responseContent?.actions);
-                        params = coerceParams(result.responseContent?.params);
-                    }
-                    const finishedAt = Date.now();
-                    trajectory.push({
-                        step: trajectory.length + 1,
-                        startedAt,
-                        finishedAt,
-                        inputText: text,
-                        promptText: composedPrompt,
-                        context,
-                        thought,
-                        responseText,
-                        actions,
-                        params,
-                    });
-                    trajectoriesBySession.set(key, trajectory);
-                    res.writeHead(200, { "Content-Type": "application/json" });
-                    res.end(JSON.stringify({
-                        text: responseText,
-                        thought,
-                        actions,
-                        params,
-                        benchmark: session.benchmark,
-                        task_id: session.taskId,
-                        room_id: session.roomId,
-                        trajectory_step: trajectory.length,
-                    }));
-                }
-                catch (err) {
-                    const errorMessage = err instanceof Error ? err.message : String(err);
-                    const errorDetail = err instanceof Error && err.stack
-                        ? err.stack
-                        : formatUnknownError(err);
-                    elizaLogger.error(`[bench] Request error: ${errorDetail}`);
-                    res.writeHead(500, { "Content-Type": "application/json" });
-                    res.end(JSON.stringify({ error: errorMessage }));
-                }
-            });
-            return;
-        }
-        res.writeHead(404);
-        res.end("Not Found");
-    });
-    server.listen(port, () => {
-        elizaLogger.info(`[bench] Autonomous benchmark server listening on port ${port}`);
-        console.log(`AUTONOMOUS_BENCH_READY port=${port}`);
-    });
-}
-startBenchmarkServer().catch((err) => {
-    console.error("Failed to start benchmark server:", err);
-    process.exit(1);
-});