@elizaos/app-core 2.0.0-beta.2 → 2.0.11-beta.5

package/benchmark/plugin.js

@@ -8,7 +8,7 @@
  *
  * @module benchmark/plugin
  */
-import { logger } from "@elizaos/core";
+import { logger, } from "@elizaos/core";
 let _currentContext = null;
 export function setBenchmarkContext(ctx) {
     _currentContext = ctx;
@@ -16,12 +16,492 @@ export function setBenchmarkContext(ctx) {
 export function getBenchmarkContext() {
     return _currentContext;
 }
+function currentBenchmarkName() {
+    return (_currentContext?.benchmark ?? "").trim().toLowerCase();
+}
+function isBenchmarkActionDisabledForCurrentContext() {
+    return false;
+}
 let _capturedAction = null;
+let _capturedActions = [];
 export function getCapturedAction() {
     return _capturedAction;
 }
+export function getCapturedActions() {
+    return [..._capturedActions];
+}
 export function clearCapturedAction() {
     _capturedAction = null;
+    _capturedActions = [];
+}
+function recordCapturedAction(action) {
+    _capturedAction = action;
+    _capturedActions.push(action);
+    return action;
+}
+const VENDING_BENCHMARK_ACTION_NAMES = [
+    "VIEW_BUSINESS_STATE",
+    "VIEW_SUPPLIERS",
+    "SET_PRICE",
+    "PLACE_ORDER",
+    "RESTOCK_SLOT",
+    "COLLECT_CASH",
+    "UPDATE_NOTES",
+    "CHECK_DELIVERIES",
+    "ADVANCE_DAY",
+];
+function isVendingBenchmarkContext() {
+    return new Set(["vending-bench", "vending_bench"]).has(currentBenchmarkName());
+}
+function isLocaBenchmarkContext() {
+    return new Set(["loca-bench", "loca_bench"]).has(currentBenchmarkName());
+}
+const LOCA_BENCHMARK_TOOL_ACTION_NAMES = [
+    "claim_done",
+    "filesystem_create_directory",
+    "filesystem_directory_tree",
+    "filesystem_edit_file",
+    "filesystem_get_file_info",
+    "filesystem_list_allowed_directories",
+    "filesystem_list_directory",
+    "filesystem_list_directory_with_sizes",
+    "filesystem_move_file",
+    "filesystem_read_file",
+    "filesystem_read_media_file",
+    "filesystem_read_multiple_files",
+    "filesystem_read_text_file",
+    "filesystem_search_files",
+    "filesystem_write_file",
+    "memory_add_observations",
+    "memory_create_entities",
+    "memory_create_relations",
+    "memory_delete_entities",
+    "memory_delete_observations",
+    "memory_delete_relations",
+    "memory_open_nodes",
+    "memory_read_graph",
+    "memory_search_nodes",
+    "python_execute",
+    "canvas_canvas_add_quiz_question",
+    "canvas_canvas_create_account_report",
+    "canvas_canvas_create_assignment",
+    "canvas_canvas_create_conversation",
+    "canvas_canvas_create_course",
+    "canvas_canvas_create_module",
+    "canvas_canvas_create_module_item",
+    "canvas_canvas_create_quiz",
+    "canvas_canvas_create_rubric",
+    "canvas_canvas_create_user",
+    "canvas_canvas_delete_quiz",
+    "canvas_canvas_delete_quiz_question",
+    "canvas_canvas_enroll_user",
+    "canvas_canvas_get_account",
+    "canvas_canvas_get_account_reports",
+    "canvas_canvas_get_assignment",
+    "canvas_canvas_get_conversation",
+    "canvas_canvas_get_course",
+    "canvas_canvas_get_course_grades",
+    "canvas_canvas_get_current_user",
+    "canvas_canvas_get_dashboard",
+    "canvas_canvas_get_dashboard_cards",
+    "canvas_canvas_get_discussion_topic",
+    "canvas_canvas_get_file",
+    "canvas_canvas_get_module",
+    "canvas_canvas_get_module_item",
+    "canvas_canvas_get_page",
+    "canvas_canvas_get_quiz",
+    "canvas_canvas_get_quiz_questions",
+    "canvas_canvas_get_rubric",
+    "canvas_canvas_get_submission",
+    "canvas_canvas_get_syllabus",
+    "canvas_canvas_get_upcoming_assignments",
+    "canvas_canvas_get_user_grades",
+    "canvas_canvas_get_user_profile",
+    "canvas_canvas_health_check",
+    "canvas_canvas_list_account_courses",
+    "canvas_canvas_list_account_users",
+    "canvas_canvas_list_announcements",
+    "canvas_canvas_list_assignments",
+    "canvas_canvas_list_calendar_events",
+    "canvas_canvas_list_conversations",
+    "canvas_canvas_list_courses",
+    "canvas_canvas_list_discussion_topics",
+    "canvas_canvas_list_files",
+    "canvas_canvas_list_folders",
+    "canvas_canvas_list_module_items",
+    "canvas_canvas_list_modules",
+    "canvas_canvas_list_notifications",
+    "canvas_canvas_list_pages",
+    "canvas_canvas_list_quizzes",
+    "canvas_canvas_list_rubrics",
+    "canvas_canvas_list_sub_accounts",
+    "canvas_canvas_list_users",
+    "canvas_canvas_login",
+    "canvas_canvas_logout",
+    "canvas_canvas_mark_module_item_complete",
+    "canvas_canvas_post_to_discussion",
+    "canvas_canvas_publish_quiz",
+    "canvas_canvas_start_quiz_attempt",
+    "canvas_canvas_submit_assignment",
+    "canvas_canvas_submit_grade",
+    "canvas_canvas_update_assignment",
+    "canvas_canvas_update_course",
+    "canvas_canvas_update_quiz",
+    "canvas_canvas_update_quiz_question",
+    "canvas_canvas_update_user_profile",
+    "canvas_get_assignment",
+    "canvas_get_course",
+    "canvas_get_dashboard",
+    "canvas_get_dashboard_cards",
+    "canvas_get_file",
+    "canvas_get_page",
+    "canvas_get_quiz",
+    "canvas_get_submission",
+    "canvas_get_syllabus",
+    "canvas_get_user_grades",
+    "canvas_get_user_profile",
+    "canvas_health_check",
+    "canvas_list_announcements",
+    "canvas_list_assignments",
+    "canvas_list_calendar_events",
+    "canvas_list_courses",
+    "canvas_list_discussion_topics",
+    "canvas_list_files",
+    "canvas_list_folders",
+    "canvas_list_module_items",
+    "canvas_list_modules",
+    "canvas_list_notifications",
+    "canvas_list_pages",
+    "canvas_list_quizzes",
+];
+const stringListSchema = {
+    type: "array",
+    items: { type: "string" },
+};
+const objectListSchema = {
+    type: "array",
+    items: { type: "object", additionalProperties: true },
+};
+const LOCA_BENCHMARK_TOOL_PARAMETERS = [
+    {
+        name: "path",
+        description: "Filesystem path inside the LOCA task workspace.",
+        required: false,
+        schema: { type: "string" },
+    },
+    {
+        name: "paths",
+        description: "Filesystem paths inside the LOCA task workspace.",
+        required: false,
+        schema: stringListSchema,
+    },
+    {
+        name: "pattern",
+        description: "Glob or search pattern for filesystem search tools.",
+        required: false,
+        schema: { type: "string" },
+    },
+    {
+        name: "excludePatterns",
+        description: "Glob patterns to exclude from filesystem searches.",
+        required: false,
+        schema: stringListSchema,
+    },
+    {
+        name: "content",
+        description: "Text content to write to a file.",
+        required: false,
+        schema: { type: "string" },
+    },
+    {
+        name: "head",
+        description: "Read only the first N lines of a text file.",
+        required: false,
+        schema: { type: "number" },
+    },
+    {
+        name: "tail",
+        description: "Read only the last N lines of a text file.",
+        required: false,
+        schema: { type: "number" },
+    },
+    {
+        name: "sortBy",
+        description: "Directory listing sort key.",
+        required: false,
+        schema: { type: "string", enum: ["name", "size"] },
+    },
+    {
+        name: "source",
+        description: "Source filesystem path for move or copy style operations.",
+        required: false,
+        schema: { type: "string" },
+    },
+    {
+        name: "destination",
+        description: "Destination filesystem path for move or copy style operations.",
+        required: false,
+        schema: { type: "string" },
+    },
+    {
+        name: "query",
+        description: "Search query for memory or SaaS tools.",
+        required: false,
+        schema: { type: "string" },
+    },
+    {
+        name: "names",
+        description: "Memory entity names to open.",
+        required: false,
+        schema: stringListSchema,
+    },
+    {
+        name: "observations",
+        description: "Memory observations payload.",
+        required: false,
+        schema: objectListSchema,
+    },
+    {
+        name: "entities",
+        description: "Memory entity creation payload.",
+        required: false,
+        schema: objectListSchema,
+    },
+    {
+        name: "relations",
+        description: "Memory relation payload.",
+        required: false,
+        schema: objectListSchema,
+    },
+    {
+        name: "deletions",
+        description: "Memory observation deletion payload.",
+        required: false,
+        schema: objectListSchema,
+    },
+    {
+        name: "entityNames",
+        description: "Memory entity names to delete.",
+        required: false,
+        schema: stringListSchema,
+    },
+    {
+        name: "edits",
+        description: "Structured file edit payload.",
+        required: false,
+        schema: objectListSchema,
+    },
+    {
+        name: "dryRun",
+        description: "Whether to preview a filesystem edit without applying it.",
+        required: false,
+        schema: { type: "boolean" },
+    },
+    {
+        name: "course_id",
+        description: "Canvas course id.",
+        required: false,
+        schema: { type: "number" },
+    },
+    {
+        name: "assignment_id",
+        description: "Canvas assignment id.",
+        required: false,
+        schema: { type: "number" },
+    },
+    {
+        name: "file_id",
+        description: "Canvas file id.",
+        required: false,
+        schema: { type: "number" },
+    },
+    {
+        name: "folder_id",
+        description: "Canvas folder id.",
+        required: false,
+        schema: { type: "number" },
+    },
+    {
+        name: "module_id",
+        description: "Canvas module id.",
+        required: false,
+        schema: { type: "number" },
+    },
+    {
+        name: "item_id",
+        description: "Canvas module item id.",
+        required: false,
+        schema: { type: "number" },
+    },
+    {
+        name: "page_url",
+        description: "Canvas page URL slug.",
+        required: false,
+        schema: { type: "string" },
+    },
+    {
+        name: "quiz_id",
+        description: "Canvas quiz id.",
+        required: false,
+        schema: { type: "number" },
+    },
+    {
+        name: "user_id",
+        description: "Canvas user id.",
+        required: false,
+        schema: { type: "number" },
+    },
+    {
+        name: "account_id",
+        description: "Canvas account id.",
+        required: false,
+        schema: { type: "number" },
+    },
+    {
+        name: "conversation_id",
+        description: "Canvas conversation id.",
+        required: false,
+        schema: { type: "number" },
+    },
+    {
+        name: "topic_id",
+        description: "Canvas discussion topic id.",
+        required: false,
+        schema: { type: "number" },
+    },
+    {
+        name: "rubric_id",
+        description: "Canvas rubric id.",
+        required: false,
+        schema: { type: "number" },
+    },
+    {
+        name: "question_id",
+        description: "Canvas quiz question id.",
+        required: false,
+        schema: { type: "number" },
+    },
+    {
+        name: "submission_type",
+        description: "Canvas assignment submission type.",
+        required: false,
+        schema: { type: "string" },
+    },
+    {
+        name: "grade",
+        description: "Canvas grade value.",
+        required: false,
+        schema: { type: "string" },
+    },
+    {
+        name: "type",
+        description: "Canvas module item, quiz question, or content type.",
+        required: false,
+        schema: { type: "string" },
+    },
+    {
+        name: "name",
+        description: "Canvas object name or generic resource name.",
+        required: false,
+        schema: { type: "string" },
+    },
+    {
+        name: "title",
+        description: "Canvas object title.",
+        required: false,
+        schema: { type: "string" },
+    },
+    {
+        name: "message",
+        description: "Canvas discussion or notification message.",
+        required: false,
+        schema: { type: "string" },
+    },
+    {
+        name: "body",
+        description: "Canvas body text or HTML payload.",
+        required: false,
+        schema: { type: "string" },
+    },
+];
+function pickLocaParameters(names) {
+    const wanted = new Set(names);
+    return LOCA_BENCHMARK_TOOL_PARAMETERS.filter((parameter) => wanted.has(parameter.name));
+}
+const LOCA_FILESYSTEM_TOOL_PARAMETERS = pickLocaParameters([
+    "path",
+    "paths",
+    "pattern",
+    "excludePatterns",
+    "content",
+    "head",
+    "tail",
+    "sortBy",
+    "source",
+    "destination",
+    "edits",
+    "dryRun",
+]);
+const LOCA_MEMORY_TOOL_PARAMETERS = pickLocaParameters([
+    "query",
+    "names",
+    "observations",
+    "entities",
+    "relations",
+    "deletions",
+    "entityNames",
+]);
+const LOCA_CANVAS_TOOL_PARAMETERS = pickLocaParameters([
+    "course_id",
+    "assignment_id",
+    "file_id",
+    "folder_id",
+    "module_id",
+    "item_id",
+    "page_url",
+    "quiz_id",
+    "user_id",
+    "account_id",
+    "conversation_id",
+    "topic_id",
+    "rubric_id",
+    "question_id",
+    "submission_type",
+    "grade",
+    "type",
+    "name",
+    "title",
+    "message",
+    "body",
+]);
+const LOCA_PYTHON_TOOL_PARAMETERS = [
+    {
+        name: "code",
+        description: "Python code to execute in the LOCA task environment.",
+        required: false,
+        schema: { type: "string" },
+    },
+];
+const LOCA_CLAIM_DONE_PARAMETERS = [
+    {
+        name: "answer",
+        description: "Final answer or completion summary for the LOCA task.",
+        required: false,
+        schema: { type: "string" },
+    },
+];
+function locaBenchmarkToolParametersFor(name) {
+    if (name.startsWith("filesystem_"))
+        return LOCA_FILESYSTEM_TOOL_PARAMETERS;
+    if (name.startsWith("memory_"))
+        return LOCA_MEMORY_TOOL_PARAMETERS;
+    if (name.startsWith("canvas_"))
+        return LOCA_CANVAS_TOOL_PARAMETERS;
+    if (name === "python_execute")
+        return LOCA_PYTHON_TOOL_PARAMETERS;
+    if (name === "claim_done")
+        return LOCA_CLAIM_DONE_PARAMETERS;
+    return [];
 }
 // ---------------------------------------------------------------------------
 // Message handler template
@@ -33,11 +513,14 @@ providers:
 
 action-based benchmarks: call BENCHMARK_ACTION with one of:
 - AgentBench: { "command": "search[laptop] | click[42] | ls | SELECT ..." }
+- WebShop: { "command": "search[...] | click[...] | buy" }
 - Tau-bench: { "tool_name": "...", "arguments": { ... } }
+- LifeOpsBench: { "tool_name": "CALENDAR", "arguments": { "subaction": "update_event", ... } }
+- LOCA-bench: { "tool_name": "filesystem_list_directory | filesystem_read_file | filesystem_write_file | ...", "arguments": { ... } }
 - Mind2Web: { "operation": "CLICK|TYPE|SELECT", "element_id": "...", "value": "..." }
 
 reply-based benchmarks: use REPLY with text payload:
-- Q&A (context-bench, rlm-bench, gaia): the answer
+- Q&A (context-bench, rlm-bench): the answer
 - hyperliquid_bench: {"steps":[...]}
 - vending-bench: {"action":"PLACE_ORDER","supplier_id":"beverage_dist","items":{"water":12}}
 - swe_bench: a single unified diff
@@ -45,7 +528,7 @@ reply-based benchmarks: use REPLY with text payload:
 
 experience-learning turns: BENCHMARK_ACTION with command RECORD_EXPERIENCE.
 
-text-format fallback (no native tool calling): return one JSON object:
+text-format fallback for action benchmarks (no native tool calling): return one JSON object:
 {
   "thought": "[brief reason]",
   "actions": ["BENCHMARK_ACTION"],
@@ -53,22 +536,121 @@ text-format fallback (no native tool calling): return one JSON object:
   "params": { "BENCHMARK_ACTION": { "command": "[command]" } }
 }
 
+text-format fallback for reply-based benchmarks: return one JSON object:
+{
+  "thought": "[brief reason]",
+  "actions": ["REPLY"],
+  "text": "[the required answer or JSON payload]",
+  "params": {}
+}
+
 rules:
 - always BENCHMARK_ACTION (never raw action name) for action benchmarks
 - never REPLY when execution is required
+- always REPLY (never BENCHMARK_ACTION) for reply-based benchmarks such as vending-bench
 `;
 // ---------------------------------------------------------------------------
 // Provider
 // ---------------------------------------------------------------------------
+function isPlainRecord(value) {
+    return Boolean(value) && typeof value === "object" && !Array.isArray(value);
+}
+function compactJson(value, maxLength = 500) {
+    const raw = typeof value === "string" ? value : JSON.stringify(value, null, 2);
+    return raw.length > maxLength ? `${raw.slice(0, maxLength)}...` : raw;
+}
+function formatToolLine(t) {
+    const fn = isPlainRecord(t.function) ? t.function : undefined;
+    const name = t.name ?? fn?.name ?? "unknown";
+    const desc = t.description ?? fn?.description ?? "";
+    const params = t.parameters ?? fn?.parameters ?? {};
+    return `- **${String(name)}**: ${String(desc)}\n  Parameters: ${compactJson(params, 1200)}`;
+}
+function formatLocaToolLine(t) {
+    const fn = isPlainRecord(t.function) ? t.function : undefined;
+    const name = String(t.name ?? fn?.name ?? "unknown");
+    const desc = String(t.description ?? fn?.description ?? "")
+        .replace(/\s+/g, " ")
+        .slice(0, 180);
+    const params = isPlainRecord(t.parameters)
+        ? t.parameters
+        : isPlainRecord(fn?.parameters)
+            ? fn.parameters
+            : {};
+    const properties = isPlainRecord(params.properties)
+        ? Object.keys(params.properties)
+        : [];
+    const required = Array.isArray(params.required)
+        ? params.required.map(String)
+        : [];
+    const requiredText = required.length > 0 ? ` required: ${required.join(", ")}` : "";
+    const paramText = properties.length > 0
+        ? ` params: ${properties.slice(0, 16).join(", ")}${properties.length > 16 ? ", ..." : ""};${requiredText}`
+        : " params: none";
+    return `- **${name}**: ${desc}${desc ? " " : ""}${paramText}`;
+}
+function renderLifeOpsContext(value) {
+    if (!isPlainRecord(value))
+        return null;
+    const sections = [];
+    const nowIso = typeof value.nowIso === "string" ? value.nowIso : "";
+    const today = typeof value.today === "string" ? value.today : "";
+    const seed = typeof value.seed === "number" ? value.seed : undefined;
+    sections.push([
+        `\n## LifeOps Clock`,
+        `- Current benchmark time: ${nowIso || "unknown"}`,
+        `- Today: ${today || (nowIso ? nowIso.slice(0, 10) : "unknown")}`,
+        seed !== undefined ? `- World seed: ${seed}` : "",
+    ]
+        .filter(Boolean)
+        .join("\n"));
+    const events = Array.isArray(value.calendarEvents)
+        ? value.calendarEvents
+        : [];
+    if (events.length > 0) {
+        const lines = events.slice(0, 80).map((event) => {
+            const record = isPlainRecord(event) ? event : {};
+            const id = String(record.id ?? "?");
+            const calendarId = String(record.calendarId ?? record.calendar_id ?? "?");
+            const title = String(record.title ?? "");
+            const start = String(record.start ?? "");
+            const end = String(record.end ?? "");
+            const status = String(record.status ?? "");
+            return `- ${id} | ${calendarId} | ${title} | ${start} -> ${end} | ${status}`;
+        });
+        sections.push(`\n## Calendar Events\n${lines.join("\n")}`);
+    }
+    const previousResults = Array.isArray(value.previousToolResults)
+        ? value.previousToolResults
+        : [];
+    if (previousResults.length > 0) {
+        const lines = previousResults.slice(-12).map((entry, index) => {
+            const record = isPlainRecord(entry) ? entry : {};
+            const tool = String(record.tool ?? "unknown");
+            const ok = record.ok === true ? "true" : "false";
+            const error = typeof record.error === "string" && record.error
+                ? ` error=${record.error}`
+                : "";
+            return [
+                `- ${index + 1}. ${tool} ok=${ok}${error}`,
+                `  arguments: ${compactJson(record.arguments, 350)}`,
+                `  result: ${compactJson(record.result, 500)}`,
+            ].join("\n");
+        });
+        sections.push(`\n## Previous LifeOps Tool Results\n${lines.join("\n")}`);
+    }
+    return sections.join("\n");
+}
 function formatContextAsText(ctx) {
     const sections = [];
     const benchmark = ctx.benchmark.trim().toLowerCase();
+    const isLifeOpsBenchmark = benchmark === "lifeops_bench" || benchmark === "lifeops-bench";
+    const isActionCallingBenchmark = benchmark === "action-calling" || benchmark === "action_calling";
     const isQuestionAnswerBenchmark = new Set([
         "context-bench",
         "context_bench",
         "rlm-bench",
         "rlm_bench",
-        "gaia",
     ]).has(benchmark);
     const isJsonPlanBenchmark = new Set([
         "hyperliquid_bench",
@@ -80,13 +662,21 @@ function formatContextAsText(ctx) {
     const isSweBench = benchmark === "swe_bench" || benchmark === "swe-bench";
     const isExperienceBenchmark = benchmark === "experience";
     const isGauntletBenchmark = benchmark === "gauntlet";
+    const isLocaBenchmark = benchmark === "loca_bench" || benchmark === "loca-bench";
+    const isWebShopBenchmark = benchmark === "webshop" || benchmark === "web-shop";
+    const isTauBenchmark = benchmark === "tau_bench" || benchmark === "tau-bench";
     const isConversationalBenchmark = new Set([
         "woobench",
         "woo-bench",
         "orchestrator_lifecycle",
         "orchestrator-lifecycle",
+        "personality_bench",
+        "personality-bench",
     ]).has(benchmark);
     const isWooBench = benchmark === "woobench" || benchmark === "woo-bench";
+    const isOrchestratorLifecycle = benchmark === "orchestrator_lifecycle" ||
+        benchmark === "orchestrator-lifecycle";
+    const isPersonalityBenchmark = benchmark === "personality_bench" || benchmark === "personality-bench";
     sections.push(`# Benchmark Task`);
     sections.push(`**Benchmark:** ${ctx.benchmark}`);
     sections.push(`**Task ID:** ${ctx.taskId}`);
@@ -106,12 +696,17 @@ function formatContextAsText(ctx) {
     if (ctx.actionSpace && ctx.actionSpace.length > 0) {
         sections.push(`\n## Available Actions\n${ctx.actionSpace.join(", ")}`);
     }
+    if (isLifeOpsBenchmark) {
+        const lifeopsContext = renderLifeOpsContext(ctx.lifeops);
+        if (lifeopsContext)
+            sections.push(lifeopsContext);
+    }
     if (isWooBench && ctx.payment_actions) {
         sections.push(`\n## Payment Actions\nUse BENCHMARK_ACTION for every money movement. Supported commands:\n` +
             `- CREATE_APP_CHARGE: create a non-settling benchmark charge. Params: amount_usd, provider ("oxapay" or "stripe"), description.\n` +
             `- CHECK_PAYMENT: check the latest benchmark charge status before delivering paid content.\n` +
             `These mirror Eliza Cloud app charge flows but execute against the WooBench mock provider during tests.\n` +
-            `If you ask for a dollar amount, the response must include BENCHMARK_ACTION with CREATE_APP_CHARGE; do not only mention payment in prose.`);
+            `Tool availability does not mean you should charge immediately. Build trust first; if you ask for a dollar amount, the response must include BENCHMARK_ACTION with CREATE_APP_CHARGE; do not only mention payment in prose.`);
     }
     // Tau-bench: tools
     if (isQuestionAnswerBenchmark) {
@@ -137,15 +732,12 @@ function formatContextAsText(ctx) {
         sections.push(`For experience learning turns, use BENCHMARK_ACTION with params.BENCHMARK_ACTION.command set to RECORD_EXPERIENCE.`);
         sections.push(`For experience retrieval turns, use REPLY with a concise answer that recalls the relevant learning.`);
     }
+    else if (isLocaBenchmark && ctx.tools && ctx.tools.length > 0) {
+        const toolLines = ctx.tools.map(formatLocaToolLine);
+        sections.push(`\n## Available Tools\n${toolLines.join("\n")}`);
+    }
     else if (ctx.tools && ctx.tools.length > 0) {
-        const toolLines = ctx.tools.map((t) => {
-            const name = t.name ?? "unknown";
-            const desc = t.description ?? "";
-            const params = t.parameters
-                ? JSON.stringify(t.parameters, null, 2)
-                : "{}";
-            return `- **${name}**: ${desc}\n  Parameters: ${params}`;
-        });
+        const toolLines = ctx.tools.map(formatToolLine);
         sections.push(`\n## Available Tools\n${toolLines.join("\n")}`);
     }
     // Mind2Web: HTML + elements
@@ -176,24 +768,67 @@ function formatContextAsText(ctx) {
     const knownKeys = new Set([
         "benchmark",
         "taskId",
+        "task_id",
         "goal",
         "observation",
         "actionSpace",
         "tools",
+        "messages",
+        "system_prompt",
+        "session_id",
+        "temperature",
+        "top_p",
+        "max_tokens",
+        "max_completion_tokens",
+        "reasoning_effort",
+        "tool_choice",
         "html",
         "elements",
         "passages",
         "question",
         "payment_actions",
+        "lifeops",
     ]);
     const extras = Object.entries(ctx).filter(([k]) => !knownKeys.has(k));
     if (extras.length > 0) {
         sections.push(`\n## Additional Context\n${extras.map(([k, v]) => `- **${k}**: ${typeof v === "string" ? v : JSON.stringify(v)}`).join("\n")}`);
     }
     sections.push(`\n## Instructions`);
-    if (ctx.tools && ctx.tools.length > 0) {
-        // Tau-bench: emphasise tool calling
-        sections.push(`You are a customer service agent. You MUST use the available tools to help the customer.`);
+    if (isLifeOpsBenchmark) {
+        sections.push(`This is LifeOpsBench. Use the LifeOps Clock for all relative dates; do not use wall-clock time.`);
+        sections.push(`You have access to the benchmark's fake LifeOps calendar and inbox through the available LifeOps tools. Do not say you lack calendar, email, inbox, or app access when a matching LifeOps tool is available.`);
+        sections.push(`For calendar changes, prefer updating the existing event id from Calendar Events or a prior search result. Do not create a duplicate and delete another event unless the user explicitly asked for that.`);
+        sections.push(`For availability questions, call CALENDAR_CHECK_AVAILABILITY or CALENDAR with subaction=check_availability and top-level startAt/endAt.`);
+        sections.push(`If the requested mutation has not succeeded yet, call BENCHMARK_ACTION with params.BENCHMARK_ACTION.tool_name set to the LifeOps tool name and params.BENCHMARK_ACTION.arguments set to the tool arguments.`);
+        sections.push(`For inbox, email, chat, or thread requests, use the LifeOps MESSAGE tool, not MEMORY. MEMORY is not a LifeOpsBench executor tool.`);
+        sections.push(`For email thread archive requests, call MESSAGE with source=gmail, operation=manage, manageOperation=archive, and threadId, or call ARCHIVE_THREAD with threadId.`);
+        sections.push(`If Previous LifeOps Tool Results already show ok=true for the requested mutation, do not call another tool. Reply with a concise confirmation that includes the relevant title/date/time/details.`);
+    }
+    else if (isActionCallingBenchmark && ctx.tools && ctx.tools.length > 0) {
+        sections.push(`This turn is scored on the planner's actual function/action call. Choose the matching available tool and call BENCHMARK_ACTION with params.BENCHMARK_ACTION.tool_name set to that tool name and params.BENCHMARK_ACTION.arguments set to the tool arguments.`);
+        sections.push(`Do not answer by describing the call. The benchmark only accepts the captured action call.`);
+    }
+    else if (isLocaBenchmark && ctx.tools && ctx.tools.length > 0) {
+        sections.push(`This is LOCA-bench. The Python LOCA runner executes tool calls outside Eliza. Call exactly one tool through BENCHMARK_ACTION with params.BENCHMARK_ACTION.tool_name and params.BENCHMARK_ACTION.arguments. Do not claim the files are complete until the required CSV writes have been emitted as tool calls.`);
+        sections.push(`Progress replies are invalid in LOCA-bench. If the task is not complete, call exactly one tool. Only use REPLY after the requested files have been written or claim_done has been called.`);
+        sections.push(`Existing workspace files may contain examples or placeholders. Use current CSV rows for schema and formatting only; derive final answers from the available tools, local_db files, workspace files, and memory records. If Canvas-specific tools are unavailable, inspect source_data/local_db and source_data/files with filesystem tools. source_data is read-only input data; write/edit the requested output CSV files at the workspace root, for example assignment_info.csv and quiz_info.csv. Overwrite or edit every requested CSV before replying.`);
+        sections.push(`Never invent aggregate helper tools. In particular, do not call process_assignments_and_quizzes or any tool name not listed under Available Tools.`);
+        sections.push(`Example tool-call JSON: {"actions":["BENCHMARK_ACTION"],"text":"","params":{"BENCHMARK_ACTION":{"tool_name":"filesystem_list_directory","arguments":{"path":"source_data"}}}}`);
+    }
+    else if (isWebShopBenchmark) {
+        sections.push(`This is WebShop. Choose exactly one command from Available Actions and call BENCHMARK_ACTION with params.BENCHMARK_ACTION.command set to that exact command string.`);
+        sections.push(`Do not answer with progress prose. The WebShop runner executes only the captured command, for example {"actions":["BENCHMARK_ACTION"],"text":"","params":{"BENCHMARK_ACTION":{"command":"click[buy now]"}}}.`);
+    }
+    else if (isTauBenchmark && ctx.tools && ctx.tools.length > 0) {
+        sections.push(`This is TauBench. Use BENCHMARK_ACTION tool calls to gather missing customer, order, policy, and product facts.`);
+        sections.push(`Do not repeat a tool call when the same result is already available in the prompt. Move to the next missing fact or the required next customer-service step.`);
+        sections.push(`Do not describe a tool call in prose. If the task needs a tool, your response MUST include actions: BENCHMARK_ACTION with params.BENCHMARK_ACTION.tool_name set to the TauBench tool name and params.BENCHMARK_ACTION.arguments set to the JSON arguments.`);
+        sections.push(`Example tool-call JSON: {"actions":["BENCHMARK_ACTION"],"text":"","params":{"BENCHMARK_ACTION":{"tool_name":"get_order_details","arguments":{"order_id":"W2378156"}}}}`);
+        sections.push(`Use REPLY only when asking the customer for required confirmation or when the task is complete. After the customer confirms, call the final mutation tool with BENCHMARK_ACTION; do not say you are calling it.`);
+    }
+    else if (ctx.tools && ctx.tools.length > 0) {
+        // Tau-bench-style harnesses: emphasise tool calling
+        sections.push(`Customer service agent. Use the available tools to help the customer.`);
         sections.push(`DO NOT respond directly to the customer yet. First call the appropriate tool using BENCHMARK_ACTION.`);
         sections.push(`Your response MUST include actions: BENCHMARK_ACTION with params.BENCHMARK_ACTION.tool_name and params.BENCHMARK_ACTION.arguments.`);
         sections.push(`Only use REPLY after you have gathered all needed information via tool calls.`);
@@ -210,9 +845,20 @@ function formatContextAsText(ctx) {
         sections.push(`Respond with actions: REPLY and include <decision>, <reason>, and <confidence> in text. Do not call BENCHMARK_ACTION.`);
     }
     else if (isConversationalBenchmark) {
-        if (isWooBench && ctx.payment_actions) {
+        if (isPersonalityBenchmark) {
+            sections.push(`This is a personality benchmark. Respond naturally to the user as you would in a real conversation.`);
+            sections.push(`When the user sets a style or trait directive (e.g. "be terse", "no emojis", "speak like a pirate"), invoke the PERSONALITY action to record the directive, then confirm it in your reply text.`);
+            sections.push(`Hold every active style/trait directive across subsequent turns — including topic changes — until the user explicitly releases it.`);
+            sections.push(`Use REPLY for ordinary conversational responses. Use PERSONALITY when the user sets, changes, or releases a personality directive.`);
+        }
+        else if (isWooBench && ctx.payment_actions) {
             sections.push(`For ordinary conversation, respond with actions: REPLY and put only the next conversational message in text.`);
-            sections.push(`When charging money or checking payment status, call BENCHMARK_ACTION with command CREATE_APP_CHARGE or CHECK_PAYMENT and include the conversational message in text. Never ask for money with REPLY alone.`);
+            sections.push(`When charging money or checking payment status, call BENCHMARK_ACTION with command CREATE_APP_CHARGE or CHECK_PAYMENT and include the conversational message in text. Never ask for money with REPLY alone, and never check payment before the user says they paid or an active charge exists.`);
+        }
+        else if (isOrchestratorLifecycle) {
+            sections.push(`This is an orchestrator lifecycle benchmark. Respond with actions: REPLY and put only the next lifecycle message in text. Do not call BENCHMARK_ACTION.`);
+            sections.push(`If the user says the current approach failed, asks to replan, changes scope, or asks to continue with revised work, acknowledge the scope change or failure and state that the updated plan has been applied.`);
+            sections.push(`For delegation/status turns, mention the active subagent and the status or progress update. For underspecified turns, ask a clarifying question and say you will wait before starting.`);
         }
         else {
             sections.push(`Respond with actions: REPLY and put only the next conversational message in text. Do not call BENCHMARK_ACTION.`);
@@ -231,6 +877,246 @@ function formatContextAsText(ctx) {
 // ---------------------------------------------------------------------------
 // Plugin factory
 // ---------------------------------------------------------------------------
+const LIFEOPS_BENCHMARK_TOOL_ACTION_NAMES = [
+    "CALENDAR",
+    "CALENDAR_CREATE_EVENT",
+    "CALENDAR_UPDATE_EVENT",
+    "CALENDAR_DELETE_EVENT",
+    "CALENDAR_SEARCH_EVENTS",
+    "CALENDAR_CHECK_AVAILABILITY",
+    "CALENDAR_PROPOSE_TIMES",
+    "CALENDAR_NEXT_EVENT",
+    "CALENDAR_UPDATE_PREFERENCES",
+    "MESSAGE",
+    "MESSAGE_SEND",
+    "MESSAGE_DRAFT_REPLY",
+    "MESSAGE_MANAGE",
+    "MESSAGE_TRIAGE",
+    "MESSAGE_SEARCH_INBOX",
+    "MESSAGE_LIST_CHANNELS",
+    "MESSAGE_READ_CHANNEL",
+    "MESSAGE_READ_WITH_CONTACT",
+    "ARCHIVE_EMAIL_THREAD",
+    "ARCHIVE_THREAD",
+];
+const LIFEOPS_BENCHMARK_TOOL_PARAMETERS = [
+    {
+        name: "subaction",
+        description: "Calendar/Entity subaction, such as check_availability.",
+        required: false,
+        schema: { type: "string" },
+    },
+    {
+        name: "operation",
+        description: "Message/Money operation, such as manage or search_inbox.",
+        required: false,
+        schema: { type: "string" },
+    },
+    {
+        name: "action",
+        description: "Alias for subaction or operation.",
+        required: false,
+        schema: { type: "string" },
+    },
+    {
+        name: "source",
+        description: "LifeOps source, for example gmail, slack, imessage.",
+        required: false,
+        schema: { type: "string" },
+    },
+    {
+        name: "manageOperation",
+        description: "Message manage operation, such as archive or mark_read.",
+        required: false,
+        schema: { type: "string" },
+    },
+    {
+        name: "threadId",
+        description: "Email/chat thread id.",
+        required: false,
+        schema: { type: "string" },
+    },
+    {
+        name: "thread_id",
+        description: "Email/chat thread id alias.",
+        required: false,
+        schema: { type: "string" },
+    },
+    {
+        name: "messageId",
+        description: "Email/chat message id.",
+        required: false,
+        schema: { type: "string" },
+    },
+    {
+        name: "message_id",
+        description: "Email/chat message id alias.",
+        required: false,
+        schema: { type: "string" },
+    },
+    {
+        name: "eventId",
+        description: "Calendar event id.",
+        required: false,
+        schema: { type: "string" },
+    },
+    {
+        name: "calendarId",
+        description: "Calendar id.",
+        required: false,
+        schema: { type: "string" },
+    },
+    {
+        name: "title",
+        description: "Calendar event title or message title.",
+        required: false,
+        schema: { type: "string" },
+    },
+    {
+        name: "query",
+        description: "Search query.",
+        required: false,
+        schema: { type: "string" },
+    },
+    {
+        name: "folder",
+        description: "Mail folder, such as inbox.",
+        required: false,
+        schema: { type: "string" },
+    },
+    {
+        name: "startAt",
+        description: "ISO-8601 calendar availability start time.",
+        required: false,
+        schema: { type: "string" },
+    },
+    {
+        name: "endAt",
+        description: "ISO-8601 calendar availability end time.",
+        required: false,
+        schema: { type: "string" },
+    },
+    {
+        name: "start",
+        description: "ISO-8601 calendar start time alias.",
+        required: false,
+        schema: { type: "string" },
+    },
+    {
+        name: "end",
+        description: "ISO-8601 calendar end time alias.",
+        required: false,
+        schema: { type: "string" },
+    },
+    {
+        name: "body",
+        description: "Email/message body.",
+        required: false,
+        schema: { type: "string" },
+    },
+    {
+        name: "text",
+        description: "Chat/message text.",
+        required: false,
+        schema: { type: "string" },
+    },
+    {
+        name: "details",
+        description: "Nested LifeOps action details. Prefer top-level fields when the tool manifest asks for them.",
+        required: false,
+        schema: {
+            type: "object",
+            additionalProperties: true,
+        },
+    },
+    {
+        name: "intent",
+        description: "Short natural-language intent for the LifeOps action.",
+        required: false,
+        schema: { type: "string" },
+    },
+];
+function lifeOpsBenchmarkToolDescription(name) {
+    if (name === "ARCHIVE_EMAIL_THREAD" || name === "ARCHIVE_THREAD") {
+        return "LifeOpsBench email archive alias. Use for Gmail/email thread archive requests with threadId.";
+    }
+    if (name.startsWith("MESSAGE")) {
+        return ("LifeOpsBench MESSAGE tool for email, inbox, Gmail, chat, and thread " +
+            "requests. Use for archive, mark_read, triage, search_inbox, " +
+            "draft_reply, send, list_channels, read_channel, and read_with_contact.");
+    }
+    if (name.startsWith("CALENDAR")) {
+        return ("LifeOpsBench CALENDAR tool for calendar events and availability. Use " +
+            "for create_event, update_event, delete_event, search_events, " +
+            "check_availability, propose_times, next_event, and update_preferences.");
+    }
+    return "LifeOpsBench compatibility action. Captures a planner-emitted LifeOps tool call for the benchmark fake backend.";
+}
+function extractActionParameters(options) {
+    let params = {};
+    if (options && typeof options === "object") {
+        const opts = options;
+        if (opts.parameters && typeof opts.parameters === "object") {
+            const p = opts.parameters;
+            if ("fields" in p && typeof p.fields === "object") {
+                const fields = p.fields;
+                for (const [k, v] of Object.entries(fields)) {
+                    params[k] = v.stringValue ?? v.numberValue ?? v;
+                }
+            }
+            else {
+                params = p;
+            }
+        }
+        else {
+            params = opts;
+        }
+    }
+    return params;
+}
+function stripRuntimeActionContext(params) {
+    const { actionContext: _actionContext, ...toolParams } = params;
+    return toolParams;
+}
+function parseCapturedArguments(value) {
+    if (typeof value === "string") {
+        try {
+            return JSON.parse(value);
+        }
+        catch {
+            logger.warn(`[BENCHMARK_ACTION] Failed to parse arguments as JSON: ${value}`);
+            return { _raw: value };
+        }
+    }
+    return typeof value === "object" && value !== null
+        ? value
+        : undefined;
+}
+function captureBenchmarkAction(params) {
+    return {
+        params,
+        command: typeof params.command === "string" ? params.command : undefined,
+        toolName: typeof params.tool_name === "string" ? params.tool_name : undefined,
+        arguments: parseCapturedArguments(params.arguments),
+        operation: typeof params.operation === "string" ? params.operation : undefined,
+        elementId: typeof params.element_id === "string" ? params.element_id : undefined,
+        value: typeof params.value === "string" ? params.value : undefined,
+    };
+}
+function captureLifeOpsBenchmarkToolAction(name, params) {
+    return {
+        params,
+        toolName: name,
+        arguments: params,
+    };
+}
+function captureNamedBenchmarkToolAction(name, params) {
+    return {
+        params,
+        toolName: name,
+        arguments: params,
+    };
+}
 export function createBenchmarkPlugin() {
     return {
         name: "eliza-benchmark",
@@ -263,6 +1149,7 @@ export function createBenchmarkPlugin() {
                 name: "BENCHMARK_ACTION",
                 contextGate: {},
                 roleGate: { minRole: "NONE" },
+                suppressPostActionContinuation: true,
                 similes: [
                     "EXECUTE",
                     "DO",
@@ -297,60 +1184,19 @@ export function createBenchmarkPlugin() {
                 description: "Execute a benchmark action. Put your command/tool/operation in the params. " +
                     "Supported params: command (agentbench), tool_name+arguments (tau-bench), " +
                     "operation+element_id+value (mind2web).",
-                validate: async () => true,
+                validate: async () => !isBenchmarkActionDisabledForCurrentContext(),
                 handler: async (_runtime, _message, _state, options) => {
-                    // Extract params — TS runtime may pass as Struct, plain object, or nested
-                    let params = {};
-                    if (options && typeof options === "object") {
-                        const opts = options;
-                        if (opts.parameters && typeof opts.parameters === "object") {
-                            const p = opts.parameters;
-                            // If it's a protobuf Struct with .fields, extract values
-                            if ("fields" in p && typeof p.fields === "object") {
-                                const fields = p.fields;
-                                for (const [k, v] of Object.entries(fields)) {
-                                    params[k] = v?.stringValue ?? v?.numberValue ?? v;
-                                }
-                            }
-                            else {
-                                params = p;
-                            }
-                        }
-                    }
-                    console.log("[BENCHMARK_ACTION] params:", JSON.stringify(params));
-                    _capturedAction = {
-                        params,
-                        command: typeof params.command === "string" ? params.command : undefined,
-                        toolName: typeof params.tool_name === "string"
-                            ? params.tool_name
-                            : undefined,
-                        arguments: typeof params.arguments === "string"
-                            ? (() => {
-                                try {
-                                    return JSON.parse(params.arguments);
-                                }
-                                catch {
-                                    logger.warn(`[BENCHMARK_ACTION] Failed to parse arguments as JSON: ${params.arguments}`);
-                                    return { _raw: params.arguments };
-                                }
-                            })()
-                            : typeof params.arguments === "object" &&
-                                params.arguments !== null
-                                ? params.arguments
-                                : undefined,
-                        operation: typeof params.operation === "string"
-                            ? params.operation
-                            : undefined,
-                        elementId: typeof params.element_id === "string"
-                            ? params.element_id
-                            : undefined,
-                        value: typeof params.value === "string" ? params.value : undefined,
-                    };
+                    const params = extractActionParameters(options);
+                    logger.debug("[BENCHMARK_ACTION] params:", JSON.stringify(params));
+                    const capturedAction = recordCapturedAction(captureBenchmarkAction(params));
                     return {
-                        text: `Benchmark action captured: ${JSON.stringify(_capturedAction)}`,
+                        text: `Benchmark action captured: ${JSON.stringify(capturedAction)}`,
                         success: true,
+                        continueChain: isVendingBenchmarkContext() || isLocaBenchmarkContext()
+                            ? false
+                            : undefined,
                         values: { captured: true },
-                        data: { action: _capturedAction },
+                        data: { action: capturedAction },
                     };
                 },
                 parameters: [
@@ -416,6 +1262,81 @@ export function createBenchmarkPlugin() {
                     },
                 ],
             },
+            ...VENDING_BENCHMARK_ACTION_NAMES.map((name) => ({
+                name,
+                contextGate: {},
+                roleGate: { minRole: "NONE" },
+                suppressPostActionContinuation: true,
+                similes: [],
+                description: "Vending-Bench compatibility action. Captures one vending simulator action for the benchmark environment.",
+                validate: async () => isVendingBenchmarkContext(),
+                handler: async (_runtime, _message, _state, options) => {
+                    const params = extractActionParameters(options);
+                    logger.debug(`[${name}] params: ${JSON.stringify(params)}`);
+                    const capturedAction = recordCapturedAction({
+                        toolName: name,
+                        arguments: params,
+                        params: { tool_name: name, arguments: params },
+                    });
+                    return {
+                        text: `Benchmark vending action captured: ${name}`,
+                        success: true,
+                        continueChain: false,
+                        values: { captured: true },
+                        data: { action: capturedAction },
+                    };
+                },
+                parameters: [],
+            })),
+            ...LIFEOPS_BENCHMARK_TOOL_ACTION_NAMES.map((name) => ({
+                name,
+                contextGate: {},
+                roleGate: { minRole: "NONE" },
+                suppressPostActionContinuation: true,
+                similes: [],
+                description: lifeOpsBenchmarkToolDescription(name),
+                routingHint: name.startsWith("MESSAGE") || name.startsWith("ARCHIVE_")
+                    ? "PERSONAL_ASSISTANT: inbox/email/Gmail/chat/thread/archive/read/draft/send -> MESSAGE or ARCHIVE_THREAD; do not use MEMORY."
+                    : "PERSONAL_ASSISTANT: calendar/event/availability/schedule -> CALENDAR.",
+                allowAdditionalParameters: true,
+                validate: async () => true,
+                handler: async (_runtime, _message, _state, options) => {
+                    const params = stripRuntimeActionContext(extractActionParameters(options));
+                    logger.debug(`[${name}] params:`, JSON.stringify(params));
+                    const capturedAction = recordCapturedAction(captureLifeOpsBenchmarkToolAction(name, params));
+                    return {
+                        text: `Benchmark LifeOps action captured: ${name}`,
+                        success: true,
+                        continueChain: false,
+                        values: { captured: true },
+                        data: { action: capturedAction },
+                    };
+                },
+                parameters: LIFEOPS_BENCHMARK_TOOL_PARAMETERS,
+            })),
+            ...LOCA_BENCHMARK_TOOL_ACTION_NAMES.map((name) => ({
+                name,
+                contextGate: {},
+                roleGate: { minRole: "NONE" },
+                suppressPostActionContinuation: true,
+                similes: [],
+                description: "LOCA-bench compatibility action. Captures a planner-emitted MCP tool call for the Python LOCA runner.",
+                allowAdditionalParameters: true,
+                validate: async () => isLocaBenchmarkContext(),
+                handler: async (_runtime, _message, _state, options) => {
+                    const params = stripRuntimeActionContext(extractActionParameters(options));
+                    logger.debug(`[${name}] params: ${JSON.stringify(params)}`);
+                    const capturedAction = recordCapturedAction(captureNamedBenchmarkToolAction(name, params));
+                    return {
+                        text: `Benchmark LOCA action captured: ${name}`,
+                        success: true,
+                        continueChain: false,
+                        values: { captured: true },
+                        data: { action: capturedAction },
+                    };
+                },
+                parameters: locaBenchmarkToolParametersFor(name),
+            })),
         ],
     };
 }