@elizaos/plugin-training 2.0.3-beta.6 → 2.0.3-beta.7

package/dist/cli/train.js

@@ -0,0 +1,219 @@
+import { parseArgs } from "node:util";
+import { NATIVE_OPTIMIZERS, runNativeBackend } from "../backends/native.js";
+import { ALL_TRAINING_TASKS } from "../core/training-config.js";
+const ALLOWED_BACKENDS = /* @__PURE__ */ new Set(["native"]);
+const ALLOWED_TASKS = new Set(ALL_TRAINING_TASKS);
+const ALLOWED_OPTIMIZERS = new Set(NATIVE_OPTIMIZERS);
+const HELP = `Usage:
+  bun run train -- --backend native --dataset <path> [options]
+
+Options:
+  --backend NAME       native (required)
+  --dataset PATH       Path to eliza_native_v1 JSONL file (required)
+  --task NAME          ${[...ALLOWED_TASKS].join(" | ")}
+                       (includes the LifeOps per-capability tasks, e.g.
+                       calendar_extract / schedule_plan / morning_brief)
+  --optimizer NAME     instruction-search | prompt-evolution | gepa
+                       | bootstrap-fewshot | dspy-bootstrap-fewshot
+                       | dspy-copro | dspy-mipro
+                       Defaults to instruction-search.
+                       gepa is the formal Pareto+feedback variant (Goyal et
+                       al. 2024); prompt-evolution is the simpler genetic
+                       mutation variant \u2014 both stay registered.
+                       The dspy-* variants use the native DSPy primitives
+                       (Signature + Predict + privacy-filtered Example loader)
+                       and emit eliza_native_v1-compatible artifacts.
+  --baseline PATH      Path to a baseline-prompt text file. Defaults to
+                       the first system message in request.messages.
+  --help               Show this help text
+`;
+function parseTrainArgs(argv) {
+  const { values } = parseArgs({
+    args: argv,
+    options: {
+      backend: { type: "string" },
+      dataset: { type: "string" },
+      task: { type: "string" },
+      optimizer: { type: "string" },
+      baseline: { type: "string" },
+      help: { type: "boolean" }
+    },
+    allowPositionals: false
+  });
+  if (values.help) return "help";
+  const backend = values.backend?.trim();
+  if (!backend || !ALLOWED_BACKENDS.has(backend)) {
+    throw new Error(
+      `--backend is required and must be one of: ${[...ALLOWED_BACKENDS].join(", ")}`
+    );
+  }
+  const dataset = values.dataset?.trim();
+  if (!dataset) {
+    throw new Error("--dataset <path> is required");
+  }
+  let task;
+  if (values.task) {
+    const t = values.task.trim();
+    if (!ALLOWED_TASKS.has(t)) {
+      throw new Error(
+        `--task must be one of: ${[...ALLOWED_TASKS].join(", ")}`
+      );
+    }
+    task = t;
+  }
+  let optimizer;
+  if (values.optimizer) {
+    const opt = values.optimizer.trim();
+    if (!ALLOWED_OPTIMIZERS.has(opt)) {
+      throw new Error(
+        `--optimizer must be one of: ${[...ALLOWED_OPTIMIZERS].join(", ")}`
+      );
+    }
+    optimizer = opt;
+  }
+  return {
+    backend,
+    dataset,
+    task,
+    optimizer,
+    baseline: values.baseline
+  };
+}
+async function runTrainCli(argv) {
+  const parsed = parseTrainArgs(argv);
+  if (parsed === "help") {
+    process.stdout.write(HELP);
+    return 0;
+  }
+  switch (parsed.backend) {
+    case "native": {
+      const optimizer = parsed.optimizer ?? "instruction-search";
+      const task = parsed.task ?? "should_respond";
+      const baselinePrompt = await loadBaselinePrompt(parsed);
+      const trainProvider = process.env.TRAIN_MODEL_PROVIDER?.trim() ?? process.env.TRAINING_PROVIDER?.trim();
+      if (trainProvider !== "cerebras") {
+        console.error(
+          "[train] TRAIN_MODEL_PROVIDER=cerebras (or TRAINING_PROVIDER=cerebras) is required. The native backend requires the real evaluation adapter; set the env var and rerun."
+        );
+        return 1;
+      }
+      const helperPath = "../../../plugin-personal-assistant/test/helpers/lifeops-eval-model.ts";
+      const helperModule = await import(helperPath);
+      const useModel = helperModule.getTrainingUseModelAdapter();
+      const adapter = {
+        async complete(input) {
+          const prompt = input.system ? `${input.system}
+
+${input.user}` : input.user;
+          return await useModel({
+            prompt,
+            temperature: input.temperature,
+            maxTokens: input.maxTokens
+          });
+        }
+      };
+      console.log(
+        "[train] adapter: cerebras gpt-oss-120b (TRAIN_MODEL_PROVIDER=cerebras)"
+      );
+      const result = await runNativeBackend({
+        datasetPath: parsed.dataset,
+        task,
+        optimizer,
+        baselinePrompt,
+        datasetId: parsed.dataset,
+        runtime: { useModel },
+        adapter
+      });
+      for (const note of result.notes) console.log(`[train] ${note}`);
+      if (!result.invoked) return 1;
+      console.log(
+        `[train] native ${optimizer} task=${task} dataset=${result.datasetSize} baseline=${result.baselineScore.toFixed(3)} optimized=${result.score.toFixed(3)}`
+      );
+      const path = await import("node:path");
+      const os = await import("node:os");
+      const stateDir = process.env.TRAINING_STATE_DIR?.trim() || process.env.ELIZA_STATE_DIR?.trim() || path.join(os.homedir(), ".eliza");
+      const promptTask = task === "context_routing" ? "should_respond" : task;
+      const artifactPayload = {
+        task: promptTask,
+        optimizer,
+        baseline: baselinePrompt,
+        prompt: result.result.optimizedPrompt,
+        baselineScore: result.baselineScore,
+        score: result.score,
+        datasetSize: result.datasetSize,
+        datasetId: parsed.dataset,
+        generatedAt: (/* @__PURE__ */ new Date()).toISOString(),
+        lineage: result.result.lineage,
+        ...result.result.fewShotExamples ? { fewShotExamples: result.result.fewShotExamples } : {}
+      };
+      try {
+        const { OptimizedPromptService } = await import("@elizaos/core");
+        const service = new OptimizedPromptService();
+        service.setStoreRoot(path.join(stateDir, "optimized-prompts"));
+        const artifactPath = await service.setPrompt(
+          promptTask,
+          artifactPayload
+        );
+        console.log(`[train] artifact: ${artifactPath}`);
+      } catch (err) {
+        const fs = await import("node:fs");
+        const dir = path.join(stateDir, "optimized-prompts", promptTask);
+        fs.mkdirSync(dir, { recursive: true });
+        const existing = fs.readdirSync(dir).filter((f) => /^v\d+\.json$/.test(f)).map((f) => Number.parseInt(f.replace(/^v|\.json$/g, ""), 10)).filter((n) => Number.isFinite(n));
+        const nextVersion = (existing.length ? Math.max(...existing) : 0) + 1;
+        const out = path.join(dir, `v${nextVersion}.json`);
+        fs.writeFileSync(out, JSON.stringify(artifactPayload, null, 2));
+        console.warn(
+          `[train] OptimizedPromptService unavailable (${err?.message ?? err}); wrote raw artifact -> ${out}`
+        );
+      }
+      return 0;
+    }
+    default: {
+      throw new Error(`Unknown backend: ${parsed.backend}`);
+    }
+  }
+}
+async function loadBaselinePrompt(args) {
+  if (args.baseline) {
+    const { readFile } = await import("node:fs/promises");
+    return await readFile(args.baseline, "utf-8");
+  }
+  const { readFileSync } = await import("node:fs");
+  const raw = readFileSync(args.dataset, "utf-8");
+  const firstLine = raw.split("\n").find((line) => line.trim().length > 0);
+  if (!firstLine) {
+    throw new Error(
+      `[native] cannot infer baseline from empty dataset ${args.dataset}; pass --baseline <path>`
+    );
+  }
+  const parsedJson = JSON.parse(firstLine);
+  if (!parsedJson || typeof parsedJson !== "object" || parsedJson.format !== "eliza_native_v1") {
+    throw new Error(
+      `[native] dataset first row is not an eliza_native_v1 document; pass --baseline <path>`
+    );
+  }
+  const request = parsedJson.request;
+  const messages = Array.isArray(request?.messages) ? request.messages : [];
+  const systemMsg = messages.find(
+    (msg) => msg.role === "system" && typeof msg.content === "string"
+  );
+  const system = typeof request?.system === "string" && request.system.length > 0 ? request.system : systemMsg?.content;
+  if (!system) {
+    throw new Error(
+      `[native] dataset first row has no request.system or system message; pass --baseline <path>`
+    );
+  }
+  return system;
+}
+if (import.meta.url === `file://${process.argv[1] ? new URL(`file://${process.argv[1]}`).pathname : ""}`) {
+  runTrainCli(process.argv.slice(2)).then((code) => process.exit(code)).catch((err) => {
+    console.error(err instanceof Error ? err.message : String(err));
+    process.exit(1);
+  });
+}
+export {
+  parseTrainArgs,
+  runTrainCli
+};
+//# sourceMappingURL=train.js.map

package/dist/cli/train.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../../src/cli/train.ts"],"sourcesContent":["/**\n * Training CLI for Eliza-native trajectory data.\n *\n * Usage:\n *   bun run train -- --backend native --dataset <path> \\\n *       [--task {should_respond|context_routing|action_planner|response|media_description}]\n *\n * Consumes `eliza_native_v1` model-boundary JSONL rows.\n */\n\nimport { parseArgs } from \"node:util\";\nimport { NATIVE_OPTIMIZERS, runNativeBackend } from \"../backends/native.js\";\nimport { ALL_TRAINING_TASKS } from \"../core/training-config.js\";\nimport type { TrajectoryTrainingTask } from \"../core/trajectory-task-datasets.js\";\nimport type { OptimizerName } from \"../optimizers/index.js\";\n\nconst ALLOWED_BACKENDS = new Set([\"native\"]);\nconst ALLOWED_TASKS = new Set<string>(ALL_TRAINING_TASKS);\nconst ALLOWED_OPTIMIZERS = new Set<string>(NATIVE_OPTIMIZERS);\n\nconst HELP = `Usage:\n  bun run train -- --backend native --dataset <path> [options]\n\nOptions:\n  --backend NAME       native (required)\n  --dataset PATH       Path to eliza_native_v1 JSONL file (required)\n  --task NAME          ${[...ALLOWED_TASKS].join(\" | \")}\n                       (includes the LifeOps per-capability tasks, e.g.\n                       calendar_extract / schedule_plan / morning_brief)\n  --optimizer NAME     instruction-search | prompt-evolution | gepa\n                       | bootstrap-fewshot | dspy-bootstrap-fewshot\n                       | dspy-copro | dspy-mipro\n                       Defaults to instruction-search.\n                       gepa is the formal Pareto+feedback variant (Goyal et\n                       al. 2024); prompt-evolution is the simpler genetic\n                       mutation variant — both stay registered.\n                       The dspy-* variants use the native DSPy primitives\n                       (Signature + Predict + privacy-filtered Example loader)\n                       and emit eliza_native_v1-compatible artifacts.\n  --baseline PATH      Path to a baseline-prompt text file. Defaults to\n                       the first system message in request.messages.\n  --help               Show this help text\n`;\n\ninterface ParsedTrainArgs {\n  backend: \"native\";\n  dataset: string;\n  task?: TrajectoryTrainingTask;\n  optimizer?: OptimizerName;\n  baseline?: string;\n}\n\nexport function parseTrainArgs(argv: string[]): ParsedTrainArgs | \"help\" {\n  const { values } = parseArgs({\n    args: argv,\n    options: {\n      backend: { type: \"string\" },\n      dataset: { type: \"string\" },\n      task: { type: \"string\" },\n      optimizer: { type: \"string\" },\n      baseline: { type: \"string\" },\n      help: { type: \"boolean\" },\n    },\n    allowPositionals: false,\n  });\n  if (values.help) return \"help\";\n\n  const backend = values.backend?.trim();\n  if (!backend || !ALLOWED_BACKENDS.has(backend)) {\n    throw new Error(\n      `--backend is required and must be one of: ${[...ALLOWED_BACKENDS].join(\", \")}`,\n    );\n  }\n  const dataset = values.dataset?.trim();\n  if (!dataset) {\n    throw new Error(\"--dataset <path> is required\");\n  }\n  let task: TrajectoryTrainingTask | undefined;\n  if (values.task) {\n    const t = values.task.trim();\n    if (!ALLOWED_TASKS.has(t)) {\n      throw new Error(\n        `--task must be one of: ${[...ALLOWED_TASKS].join(\", \")}`,\n      );\n    }\n    task = t as TrajectoryTrainingTask;\n  }\n\n  let optimizer: OptimizerName | undefined;\n  if (values.optimizer) {\n    const opt = values.optimizer.trim();\n    if (!ALLOWED_OPTIMIZERS.has(opt)) {\n      throw new Error(\n        `--optimizer must be one of: ${[...ALLOWED_OPTIMIZERS].join(\", \")}`,\n      );\n    }\n    optimizer = opt as OptimizerName;\n  }\n\n  return {\n    backend: backend as ParsedTrainArgs[\"backend\"],\n    dataset,\n    task,\n    optimizer,\n    baseline: values.baseline,\n  };\n}\n\nexport async function runTrainCli(argv: string[]): Promise<number> {\n  const parsed = parseTrainArgs(argv);\n  if (parsed === \"help\") {\n    process.stdout.write(HELP);\n    return 0;\n  }\n\n  switch (parsed.backend) {\n    case \"native\": {\n      const optimizer = parsed.optimizer ?? \"instruction-search\";\n      const task: TrajectoryTrainingTask = parsed.task ?? \"should_respond\";\n      const baselinePrompt = await loadBaselinePrompt(parsed);\n      // Real-model adapter: scoring + variant generation run through the\n      // Cerebras gpt-oss-120b client (lifeops-eval-model.ts). This is the path\n      // `bun run lifeops:optimize` exercises against captured trajectories.\n      const trainProvider =\n        process.env.TRAIN_MODEL_PROVIDER?.trim() ??\n        process.env.TRAINING_PROVIDER?.trim();\n      if (trainProvider !== \"cerebras\") {\n        console.error(\n          \"[train] TRAIN_MODEL_PROVIDER=cerebras (or TRAINING_PROVIDER=cerebras) is required. \" +\n            \"The native backend requires the real evaluation adapter; set the env var and rerun.\",\n        );\n        return 1;\n      }\n      // The eval helper lives in plugin-personal-assistant's test tree, which is\n      // outside this package's emit rootDir. Declare the single export's real\n      // signature locally so the dynamic import stays fully typed without a\n      // static `typeof import()` reference dragging an out-of-rootDir file into\n      // the build program (TS6059).\n      interface LifeOpsEvalModelModule {\n        getTrainingUseModelAdapter(): (input: {\n          prompt: string;\n          temperature?: number;\n          maxTokens?: number;\n        }) => Promise<string>;\n      }\n      const helperPath =\n        \"../../../plugin-personal-assistant/test/helpers/lifeops-eval-model.ts\";\n      const helperModule: LifeOpsEvalModelModule = await import(helperPath);\n      const useModel = helperModule.getTrainingUseModelAdapter();\n      const adapter = {\n        async complete(input: {\n          system?: string;\n          user: string;\n          temperature?: number;\n          maxTokens?: number;\n        }): Promise<string> {\n          const prompt = input.system\n            ? `${input.system}\\n\\n${input.user}`\n            : input.user;\n          return await useModel({\n            prompt,\n            temperature: input.temperature,\n            maxTokens: input.maxTokens,\n          });\n        },\n      };\n      console.log(\n        \"[train] adapter: cerebras gpt-oss-120b (TRAIN_MODEL_PROVIDER=cerebras)\",\n      );\n      const result = await runNativeBackend({\n        datasetPath: parsed.dataset,\n        task,\n        optimizer,\n        baselinePrompt,\n        datasetId: parsed.dataset,\n        runtime: { useModel },\n        adapter,\n      });\n      for (const note of result.notes) console.log(`[train] ${note}`);\n      if (!result.invoked) return 1;\n      console.log(\n        `[train] native ${optimizer} task=${task} dataset=${result.datasetSize} ` +\n          `baseline=${result.baselineScore.toFixed(3)} optimized=${result.score.toFixed(3)}`,\n      );\n\n      // Persist the optimized prompt + lineage so the operator can inspect\n      // and deploy it. Routed through `OptimizedPromptService.setPrompt` so\n      // the on-disk versioning (`vN.json` + `current`/`previous` symlinks)\n      // matches what the runtime trigger service writes. Keeps `rollback`\n      // working regardless of which write path produced the artifact.\n      const path = await import(\"node:path\");\n      const os = await import(\"node:os\");\n      const stateDir =\n        process.env.TRAINING_STATE_DIR?.trim() ||\n        process.env.ELIZA_STATE_DIR?.trim() ||\n        path.join(os.homedir(), \".eliza\");\n      const promptTask = task === \"context_routing\" ? \"should_respond\" : task;\n      const artifactPayload = {\n        task: promptTask,\n        optimizer,\n        baseline: baselinePrompt,\n        prompt: result.result.optimizedPrompt,\n        baselineScore: result.baselineScore,\n        score: result.score,\n        datasetSize: result.datasetSize,\n        datasetId: parsed.dataset,\n        generatedAt: new Date().toISOString(),\n        lineage: result.result.lineage,\n        ...(result.result.fewShotExamples\n          ? { fewShotExamples: result.result.fewShotExamples }\n          : {}),\n      };\n      try {\n        const { OptimizedPromptService } = await import(\"@elizaos/core\");\n        const service = new OptimizedPromptService();\n        service.setStoreRoot(path.join(stateDir, \"optimized-prompts\"));\n        const artifactPath = await service.setPrompt(\n          promptTask,\n          artifactPayload,\n        );\n        console.log(`[train] artifact: ${artifactPath}`);\n      } catch (err) {\n        // Fallback: write the artifact directly to <stateDir>/optimized-prompts/<task>/vN.json\n        // when @elizaos/core fails to load (e.g. transient drizzle-orm\n        // resolution issues during cleanup). Keeps the optimizer output\n        // recoverable.\n        const fs = await import(\"node:fs\");\n        const dir = path.join(stateDir, \"optimized-prompts\", promptTask);\n        fs.mkdirSync(dir, { recursive: true });\n        const existing = fs\n          .readdirSync(dir)\n          .filter((f) => /^v\\d+\\.json$/.test(f))\n          .map((f) => Number.parseInt(f.replace(/^v|\\.json$/g, \"\"), 10))\n          .filter((n) => Number.isFinite(n));\n        const nextVersion = (existing.length ? Math.max(...existing) : 0) + 1;\n        const out = path.join(dir, `v${nextVersion}.json`);\n        fs.writeFileSync(out, JSON.stringify(artifactPayload, null, 2));\n        console.warn(\n          `[train] OptimizedPromptService unavailable (${(err as Error)?.message ?? err}); wrote raw artifact -> ${out}`,\n        );\n      }\n      return 0;\n    }\n    default: {\n      // Unreachable thanks to the ALLOWED_BACKENDS guard above.\n      throw new Error(`Unknown backend: ${parsed.backend}`);\n    }\n  }\n}\n\nasync function loadBaselinePrompt(args: ParsedTrainArgs): Promise<string> {\n  if (args.baseline) {\n    const { readFile } = await import(\"node:fs/promises\");\n    return await readFile(args.baseline, \"utf-8\");\n  }\n  const { readFileSync } = await import(\"node:fs\");\n  const raw = readFileSync(args.dataset, \"utf-8\");\n  const firstLine = raw.split(\"\\n\").find((line) => line.trim().length > 0);\n  if (!firstLine) {\n    throw new Error(\n      `[native] cannot infer baseline from empty dataset ${args.dataset}; pass --baseline <path>`,\n    );\n  }\n  const parsedJson: unknown = JSON.parse(firstLine);\n  if (\n    !parsedJson ||\n    typeof parsedJson !== \"object\" ||\n    (parsedJson as { format?: unknown }).format !== \"eliza_native_v1\"\n  ) {\n    throw new Error(\n      `[native] dataset first row is not an eliza_native_v1 document; pass --baseline <path>`,\n    );\n  }\n  const request = (\n    parsedJson as { request?: { system?: unknown; messages?: unknown } }\n  ).request;\n  const messages = Array.isArray(request?.messages)\n    ? (request.messages as Array<{ role?: string; content?: string }>)\n    : [];\n  const systemMsg = messages.find(\n    (msg) => msg.role === \"system\" && typeof msg.content === \"string\",\n  );\n  const system =\n    typeof request?.system === \"string\" && request.system.length > 0\n      ? request.system\n      : systemMsg?.content;\n  if (!system) {\n    throw new Error(\n      `[native] dataset first row has no request.system or system message; pass --baseline <path>`,\n    );\n  }\n  return system;\n}\n\nif (\n  import.meta.url ===\n  `file://${process.argv[1] ? new URL(`file://${process.argv[1]}`).pathname : \"\"}`\n) {\n  runTrainCli(process.argv.slice(2))\n    .then((code) => process.exit(code))\n    .catch((err) => {\n      console.error(err instanceof Error ? err.message : String(err));\n      process.exit(1);\n    });\n}\n"],"mappings":"AAUA,SAAS,iBAAiB;AAC1B,SAAS,mBAAmB,wBAAwB;AACpD,SAAS,0BAA0B;AAInC,MAAM,mBAAmB,oBAAI,IAAI,CAAC,QAAQ,CAAC;AAC3C,MAAM,gBAAgB,IAAI,IAAY,kBAAkB;AACxD,MAAM,qBAAqB,IAAI,IAAY,iBAAiB;AAE5D,MAAM,OAAO;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,yBAMY,CAAC,GAAG,aAAa,EAAE,KAAK,KAAK,CAAC;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AA0BhD,SAAS,eAAe,MAA0C;AACvE,QAAM,EAAE,OAAO,IAAI,UAAU;AAAA,IAC3B,MAAM;AAAA,IACN,SAAS;AAAA,MACP,SAAS,EAAE,MAAM,SAAS;AAAA,MAC1B,SAAS,EAAE,MAAM,SAAS;AAAA,MAC1B,MAAM,EAAE,MAAM,SAAS;AAAA,MACvB,WAAW,EAAE,MAAM,SAAS;AAAA,MAC5B,UAAU,EAAE,MAAM,SAAS;AAAA,MAC3B,MAAM,EAAE,MAAM,UAAU;AAAA,IAC1B;AAAA,IACA,kBAAkB;AAAA,EACpB,CAAC;AACD,MAAI,OAAO,KAAM,QAAO;AAExB,QAAM,UAAU,OAAO,SAAS,KAAK;AACrC,MAAI,CAAC,WAAW,CAAC,iBAAiB,IAAI,OAAO,GAAG;AAC9C,UAAM,IAAI;AAAA,MACR,6CAA6C,CAAC,GAAG,gBAAgB,EAAE,KAAK,IAAI,CAAC;AAAA,IAC/E;AAAA,EACF;AACA,QAAM,UAAU,OAAO,SAAS,KAAK;AACrC,MAAI,CAAC,SAAS;AACZ,UAAM,IAAI,MAAM,8BAA8B;AAAA,EAChD;AACA,MAAI;AACJ,MAAI,OAAO,MAAM;AACf,UAAM,IAAI,OAAO,KAAK,KAAK;AAC3B,QAAI,CAAC,cAAc,IAAI,CAAC,GAAG;AACzB,YAAM,IAAI;AAAA,QACR,0BAA0B,CAAC,GAAG,aAAa,EAAE,KAAK,IAAI,CAAC;AAAA,MACzD;AAAA,IACF;AACA,WAAO;AAAA,EACT;AAEA,MAAI;AACJ,MAAI,OAAO,WAAW;AACpB,UAAM,MAAM,OAAO,UAAU,KAAK;AAClC,QAAI,CAAC,mBAAmB,IAAI,GAAG,GAAG;AAChC,YAAM,IAAI;AAAA,QACR,+BAA+B,CAAC,GAAG,kBAAkB,EAAE,KAAK,IAAI,CAAC;AAAA,MACnE;AAAA,IACF;AACA,gBAAY;AAAA,EACd;AAEA,SAAO;AAAA,IACL;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA,UAAU,OAAO;AAAA,EACnB;AACF;AAEA,eAAsB,YAAY,MAAiC;AACjE,QAAM,SAAS,eAAe,IAAI;AAClC,MAAI,WAAW,QAAQ;AACrB,YAAQ,OAAO,MAAM,IAAI;AACzB,WAAO;AAAA,EACT;AAEA,UAAQ,OAAO,SAAS;AAAA,IACtB,KAAK,UAAU;AACb,YAAM,YAAY,OAAO,aAAa;AACtC,YAAM,OAA+B,OAAO,QAAQ;AACpD,YAAM,iBAAiB,MAAM,mBAAmB,MAAM;AAItD,YAAM,gBACJ,QAAQ,IAAI,sBAAsB,KAAK,KACvC,QAAQ,IAAI,mBAAmB,KAAK;AACtC,UAAI,kBAAkB,YAAY;AAChC,gBAAQ;AAAA,UACN;AAAA,QAEF;AACA,eAAO;AAAA,MACT;AAaA,YAAM,aACJ;AACF,YAAM,eAAuC,MAAM,OAAO;AAC1D,YAAM,WAAW,aAAa,2BAA2B;AACzD,YAAM,UAAU;AAAA,QACd,MAAM,SAAS,OAKK;AAClB,gBAAM,SAAS,MAAM,SACjB,GAAG,MAAM,MAAM;AAAA;AAAA,EAAO,MAAM,IAAI,KAChC,MAAM;AACV,iBAAO,MAAM,SAAS;AAAA,YACpB;AAAA,YACA,aAAa,MAAM;AAAA,YACnB,WAAW,MAAM;AAAA,UACnB,CAAC;AAAA,QACH;AAAA,MACF;AACA,cAAQ;AAAA,QACN;AAAA,MACF;AACA,YAAM,SAAS,MAAM,iBAAiB;AAAA,QACpC,aAAa,OAAO;AAAA,QACpB;AAAA,QACA;AAAA,QACA;AAAA,QACA,WAAW,OAAO;AAAA,QAClB,SAAS,EAAE,SAAS;AAAA,QACpB;AAAA,MACF,CAAC;AACD,iBAAW,QAAQ,OAAO,MAAO,SAAQ,IAAI,WAAW,IAAI,EAAE;AAC9D,UAAI,CAAC,OAAO,QAAS,QAAO;AAC5B,cAAQ;AAAA,QACN,kBAAkB,SAAS,SAAS,IAAI,YAAY,OAAO,WAAW,aACxD,OAAO,cAAc,QAAQ,CAAC,CAAC,cAAc,OAAO,MAAM,QAAQ,CAAC,CAAC;AAAA,MACpF;AAOA,YAAM,OAAO,MAAM,OAAO,WAAW;AACrC,YAAM,KAAK,MAAM,OAAO,SAAS;AACjC,YAAM,WACJ,QAAQ,IAAI,oBAAoB,KAAK,KACrC,QAAQ,IAAI,iBAAiB,KAAK,KAClC,KAAK,KAAK,GAAG,QAAQ,GAAG,QAAQ;AAClC,YAAM,aAAa,SAAS,oBAAoB,mBAAmB;AACnE,YAAM,kBAAkB;AAAA,QACtB,MAAM;AAAA,QACN;AAAA,QACA,UAAU;AAAA,QACV,QAAQ,OAAO,OAAO;AAAA,QACtB,eAAe,OAAO;AAAA,QACtB,OAAO,OAAO;AAAA,QACd,aAAa,OAAO;AAAA,QACpB,WAAW,OAAO;AAAA,QAClB,cAAa,oBAAI,KAAK,GAAE,YAAY;AAAA,QACpC,SAAS,OAAO,OAAO;AAAA,QACvB,GAAI,OAAO,OAAO,kBACd,EAAE,iBAAiB,OAAO,OAAO,gBAAgB,IACjD,CAAC;AAAA,MACP;AACA,UAAI;AACF,cAAM,EAAE,uBAAuB,IAAI,MAAM,OAAO,eAAe;AAC/D,cAAM,UAAU,IAAI,uBAAuB;AAC3C,gBAAQ,aAAa,KAAK,KAAK,UAAU,mBAAmB,CAAC;AAC7D,cAAM,eAAe,MAAM,QAAQ;AAAA,UACjC;AAAA,UACA;AAAA,QACF;AACA,gBAAQ,IAAI,qBAAqB,YAAY,EAAE;AAAA,MACjD,SAAS,KAAK;AAKZ,cAAM,KAAK,MAAM,OAAO,SAAS;AACjC,cAAM,MAAM,KAAK,KAAK,UAAU,qBAAqB,UAAU;AAC/D,WAAG,UAAU,KAAK,EAAE,WAAW,KAAK,CAAC;AACrC,cAAM,WAAW,GACd,YAAY,GAAG,EACf,OAAO,CAAC,MAAM,eAAe,KAAK,CAAC,CAAC,EACpC,IAAI,CAAC,MAAM,OAAO,SAAS,EAAE,QAAQ,eAAe,EAAE,GAAG,EAAE,CAAC,EAC5D,OAAO,CAAC,MAAM,OAAO,SAAS,CAAC,CAAC;AACnC,cAAM,eAAe,SAAS,SAAS,KAAK,IAAI,GAAG,QAAQ,IAAI,KAAK;AACpE,cAAM,MAAM,KAAK,KAAK,KAAK,IAAI,WAAW,OAAO;AACjD,WAAG,cAAc,KAAK,KAAK,UAAU,iBAAiB,MAAM,CAAC,CAAC;AAC9D,gBAAQ;AAAA,UACN,+CAAgD,KAAe,WAAW,GAAG,4BAA4B,GAAG;AAAA,QAC9G;AAAA,MACF;AACA,aAAO;AAAA,IACT;AAAA,IACA,SAAS;AAEP,YAAM,IAAI,MAAM,oBAAoB,OAAO,OAAO,EAAE;AAAA,IACtD;AAAA,EACF;AACF;AAEA,eAAe,mBAAmB,MAAwC;AACxE,MAAI,KAAK,UAAU;AACjB,UAAM,EAAE,SAAS,IAAI,MAAM,OAAO,kBAAkB;AACpD,WAAO,MAAM,SAAS,KAAK,UAAU,OAAO;AAAA,EAC9C;AACA,QAAM,EAAE,aAAa,IAAI,MAAM,OAAO,SAAS;AAC/C,QAAM,MAAM,aAAa,KAAK,SAAS,OAAO;AAC9C,QAAM,YAAY,IAAI,MAAM,IAAI,EAAE,KAAK,CAAC,SAAS,KAAK,KAAK,EAAE,SAAS,CAAC;AACvE,MAAI,CAAC,WAAW;AACd,UAAM,IAAI;AAAA,MACR,qDAAqD,KAAK,OAAO;AAAA,IACnE;AAAA,EACF;AACA,QAAM,aAAsB,KAAK,MAAM,SAAS;AAChD,MACE,CAAC,cACD,OAAO,eAAe,YACrB,WAAoC,WAAW,mBAChD;AACA,UAAM,IAAI;AAAA,MACR;AAAA,IACF;AAAA,EACF;AACA,QAAM,UACJ,WACA;AACF,QAAM,WAAW,MAAM,QAAQ,SAAS,QAAQ,IAC3C,QAAQ,WACT,CAAC;AACL,QAAM,YAAY,SAAS;AAAA,IACzB,CAAC,QAAQ,IAAI,SAAS,YAAY,OAAO,IAAI,YAAY;AAAA,EAC3D;AACA,QAAM,SACJ,OAAO,SAAS,WAAW,YAAY,QAAQ,OAAO,SAAS,IAC3D,QAAQ,SACR,WAAW;AACjB,MAAI,CAAC,QAAQ;AACX,UAAM,IAAI;AAAA,MACR;AAAA,IACF;AAAA,EACF;AACA,SAAO;AACT;AAEA,IACE,YAAY,QACZ,UAAU,QAAQ,KAAK,CAAC,IAAI,IAAI,IAAI,UAAU,QAAQ,KAAK,CAAC,CAAC,EAAE,EAAE,WAAW,EAAE,IAC9E;AACA,cAAY,QAAQ,KAAK,MAAM,CAAC,CAAC,EAC9B,KAAK,CAAC,SAAS,QAAQ,KAAK,IAAI,CAAC,EACjC,MAAM,CAAC,QAAQ;AACd,YAAQ,MAAM,eAAe,QAAQ,IAAI,UAAU,OAAO,GAAG,CAAC;AAC9D,YAAQ,KAAK,CAAC;AAAA,EAChB,CAAC;AACL;","names":[]}

package/dist/core/action-benchmark-runner.d.ts

@@ -0,0 +1,55 @@
+export type ActionBenchmarkMatrixVariant = "reference" | "base" | "trained";
+export interface ActionBenchmarkRunOptions {
+    workspaceRoot?: string;
+    bun?: string;
+    outputDir?: string;
+    useMocks?: boolean;
+    forceTrajectoryCapture?: boolean;
+    filter?: string;
+    runsPerCase?: number;
+    provider?: string;
+    modelId?: string;
+    runtimeModel?: string;
+    smallModel?: string;
+    largeModel?: string;
+    baseUrl?: string;
+    variant?: ActionBenchmarkMatrixVariant;
+    tier?: string;
+    benchmark?: string;
+    datasetVersion?: string;
+    codeCommit?: string;
+    dryRun?: boolean;
+}
+export interface ActionBenchmarkRunResult {
+    workspaceRoot: string;
+    appCoreRoot: string;
+    outputDir: string;
+    reportMarkdownPath: string;
+    reportJsonPath: string;
+    trajectoryDir: string;
+    command: string[];
+    env: Record<string, string>;
+    stdout: string;
+    stderr: string;
+    exitCode: number;
+    matrixSource: {
+        path: string;
+        modelId?: string;
+        benchmark?: string;
+        variant?: ActionBenchmarkMatrixVariant;
+        tier?: string;
+        provider?: string;
+        datasetVersion?: string;
+        codeCommit?: string;
+        useMocks?: boolean;
+    } | null;
+}
+export declare function assertLocalBenchmarkModelAvailable(options: ActionBenchmarkRunOptions): Promise<void>;
+export declare function buildActionBenchmarkCommand(): string[];
+export declare function buildActionBenchmarkEnv(options: ActionBenchmarkRunOptions, resolved: {
+    reportMarkdownPath: string;
+    reportJsonPath: string;
+    trajectoryDir: string;
+}): Record<string, string>;
+export declare function runActionBenchmark(options?: ActionBenchmarkRunOptions): Promise<ActionBenchmarkRunResult>;
+//# sourceMappingURL=action-benchmark-runner.d.ts.map

package/dist/core/action-benchmark-runner.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"action-benchmark-runner.d.ts","sourceRoot":"","sources":["../../src/core/action-benchmark-runner.ts"],"names":[],"mappings":"AAMA,MAAM,MAAM,4BAA4B,GAAG,WAAW,GAAG,MAAM,GAAG,SAAS,CAAC;AAE5E,MAAM,WAAW,yBAAyB;IACxC,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,GAAG,CAAC,EAAE,MAAM,CAAC;IACb,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,QAAQ,CAAC,EAAE,OAAO,CAAC;IACnB,sBAAsB,CAAC,EAAE,OAAO,CAAC;IACjC,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,OAAO,CAAC,EAAE,4BAA4B,CAAC;IACvC,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,MAAM,CAAC,EAAE,OAAO,CAAC;CAClB;AAED,MAAM,WAAW,wBAAwB;IACvC,aAAa,EAAE,MAAM,CAAC;IACtB,WAAW,EAAE,MAAM,CAAC;IACpB,SAAS,EAAE,MAAM,CAAC;IAClB,kBAAkB,EAAE,MAAM,CAAC;IAC3B,cAAc,EAAE,MAAM,CAAC;IACvB,aAAa,EAAE,MAAM,CAAC;IACtB,OAAO,EAAE,MAAM,EAAE,CAAC;IAClB,GAAG,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAC5B,MAAM,EAAE,MAAM,CAAC;IACf,MAAM,EAAE,MAAM,CAAC;IACf,QAAQ,EAAE,MAAM,CAAC;IACjB,YAAY,EAAE;QACZ,IAAI,EAAE,MAAM,CAAC;QACb,OAAO,CAAC,EAAE,MAAM,CAAC;QACjB,SAAS,CAAC,EAAE,MAAM,CAAC;QACnB,OAAO,CAAC,EAAE,4BAA4B,CAAC;QACvC,IAAI,CAAC,EAAE,MAAM,CAAC;QACd,QAAQ,CAAC,EAAE,MAAM,CAAC;QAClB,cAAc,CAAC,EAAE,MAAM,CAAC;QACxB,UAAU,CAAC,EAAE,MAAM,CAAC;QACpB,QAAQ,CAAC,EAAE,OAAO,CAAC;KACpB,GAAG,IAAI,CAAC;CACV;AAkFD,wBAAsB,kCAAkC,CACtD,OAAO,EAAE,yBAAyB,GACjC,OAAO,CAAC,IAAI,CAAC,CAaf;AAmFD,wBAAgB,2BAA2B,IAAI,MAAM,EAAE,CAatD;AAED,wBAAgB,uBAAuB,CACrC,OAAO,EAAE,yBAAyB,EAClC,QAAQ,EAAE;IACR,kBAAkB,EAAE,MAAM,CAAC;IAC3B,cAAc,EAAE,MAAM,CAAC;IACvB,aAAa,EAAE,MAAM,CAAC;CACvB,GACA,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAiCxB;AAED,wBAAsB,kBAAkB,CACtC,OAAO,GAAE,yBAA8B,GACtC,OAAO,CAAC,wBAAwB,CAAC,CA0JnC"}

package/dist/core/action-benchmark-runner.js

@@ -0,0 +1,341 @@
+import { spawn } from "node:child_process";
+import { mkdir, readFile, writeFile } from "node:fs/promises";
+import { join } from "node:path";
+import { trainingStateRoot } from "./training-config.js";
+import { defaultBunCommand, resolveWorkspaceRoot } from "./workspace-runtime.js";
+function safeTimestamp(value) {
+  return value.replace(/[:.]/g, "-");
+}
+function positiveInt(value) {
+  return typeof value === "number" && Number.isFinite(value) ? Math.max(1, Math.floor(value)) : void 0;
+}
+function collectProcess(command, args, cwd, env) {
+  return new Promise((resolvePromise, reject) => {
+    const child = spawn(command, args, {
+      cwd,
+      env,
+      stdio: ["ignore", "pipe", "pipe"]
+    });
+    let stdout = "";
+    let stderr = "";
+    child.stdout.setEncoding("utf-8");
+    child.stderr.setEncoding("utf-8");
+    child.stdout.on("data", (chunk) => {
+      stdout += chunk;
+    });
+    child.stderr.on("data", (chunk) => {
+      stderr += chunk;
+    });
+    child.on("error", reject);
+    child.on("close", (code) => {
+      resolvePromise({ stdout, stderr, exitCode: code ?? 1 });
+    });
+  });
+}
+function stringSetting(value) {
+  return value?.trim() || void 0;
+}
+function modelListUrl(baseUrl) {
+  const normalized = baseUrl.trim().replace(/\/+$/, "");
+  return `${normalized}/models`;
+}
+function localModelIdMatches(availableId, requestedId) {
+  return availableId === requestedId || availableId === `${requestedId}:latest` || `${availableId}:latest` === requestedId;
+}
+async function localModelIds(baseUrl) {
+  const response = await fetch(modelListUrl(baseUrl));
+  if (!response.ok) {
+    throw new Error(
+      `local model endpoint ${modelListUrl(baseUrl)} returned ${response.status} ${response.statusText}`
+    );
+  }
+  const payload = await response.json();
+  const data = payload && typeof payload === "object" && Array.isArray(payload.data) ? payload.data : [];
+  return data.map(
+    (item) => item && typeof item === "object" ? item.id ?? item.name : item
+  ).filter((id) => typeof id === "string" && id.length > 0);
+}
+async function assertLocalBenchmarkModelAvailable(options) {
+  if (effectiveUseMocks(options)) return;
+  if (options.provider !== "local-llama-cpp") return;
+  const requestedModel = stringSetting(options.runtimeModel);
+  if (!requestedModel) return;
+  const baseUrl = stringSetting(options.baseUrl) ?? "http://localhost:11434/v1";
+  const ids = await localModelIds(baseUrl);
+  if (ids.some((id) => localModelIdMatches(id, requestedModel))) return;
+  throw new Error(
+    `local action benchmark model "${requestedModel}" is not available at ${modelListUrl(
+      baseUrl
+    )}; available models: ${ids.length > 0 ? ids.join(", ") : "none"}`
+  );
+}
+function effectiveUseMocks(options) {
+  return options.useMocks ?? options.dryRun === true;
+}
+function matrixSourceForReport(reportJsonPath, options) {
+  const modelId = stringSetting(options.modelId) ?? stringSetting(options.provider);
+  const variant = options.variant;
+  if (!modelId || !variant) return null;
+  return {
+    path: reportJsonPath,
+    modelId,
+    variant,
+    benchmark: stringSetting(options.benchmark),
+    tier: stringSetting(options.tier),
+    provider: stringSetting(options.provider),
+    datasetVersion: stringSetting(options.datasetVersion),
+    codeCommit: stringSetting(options.codeCommit),
+    useMocks: effectiveUseMocks(options)
+  };
+}
+function dryRunCaseSample(options, trajectoryDir) {
+  const tier = stringSetting(options.tier) ?? "2b";
+  const variant = stringSetting(options.variant) ?? "trained";
+  const modelId = stringSetting(options.modelId) ?? stringSetting(options.runtimeModel);
+  return {
+    caseId: `dry-run-${tier}-${variant}-action-selection`,
+    prompt: "Can you check my calendar?",
+    expectedAction: "CHECK_RUNTIME",
+    actualAction: null,
+    pass: false,
+    response: "Dry-run benchmark provenance sample; no model inference executed.",
+    latencyMs: 0,
+    trajectoryPath: join(
+      trajectoryDir,
+      `dry-run-${tier}-${variant}-action-selection.json`
+    ),
+    dryRun: true,
+    modelId,
+    tier,
+    variant
+  };
+}
+async function annotateBenchmarkReportSource(reportJsonPath, options) {
+  const matrixSource = matrixSourceForReport(reportJsonPath, options);
+  if (!matrixSource) return;
+  const parsed = JSON.parse(await readFile(reportJsonPath, "utf8"));
+  if (!parsed || typeof parsed !== "object" || Array.isArray(parsed)) return;
+  const report = parsed;
+  const existingSource = report.source && typeof report.source === "object" && !Array.isArray(report.source) ? report.source : {};
+  report.source = {
+    ...existingSource,
+    modelId: matrixSource.modelId,
+    variant: matrixSource.variant,
+    benchmark: matrixSource.benchmark,
+    tier: matrixSource.tier,
+    provider: matrixSource.provider,
+    datasetVersion: matrixSource.datasetVersion,
+    codeCommit: matrixSource.codeCommit,
+    useMocks: matrixSource.useMocks
+  };
+  await writeFile(
+    reportJsonPath,
+    `${JSON.stringify(report, null, 2)}
+`,
+    "utf8"
+  );
+}
+function buildActionBenchmarkCommand() {
+  return [
+    "x",
+    "vitest",
+    "run",
+    "--config",
+    "../test/vitest/real.config.ts",
+    "test/benchmarks/action-selection.real.test.ts",
+    "--exclude",
+    ".git/**",
+    "--exclude",
+    ".eliza/**"
+  ];
+}
+function buildActionBenchmarkEnv(options, resolved) {
+  const env = {
+    ELIZA_RUN_ACTION_BENCHMARK: "1",
+    ELIZA_ACTION_BENCHMARK_REPORT_PATH: resolved.reportMarkdownPath,
+    ELIZA_ACTION_BENCHMARK_REPORT_JSON_PATH: resolved.reportJsonPath,
+    ELIZA_ACTION_BENCHMARK_TRAJECTORY_DIR: resolved.trajectoryDir
+  };
+  if (effectiveUseMocks(options)) env.ELIZA_BENCHMARK_USE_MOCKS = "1";
+  if (options.forceTrajectoryCapture !== false) {
+    env.ELIZA_DUMP_TRAJECTORIES = "1";
+    env.ELIZA_TRAJECTORY_MARKDOWN = "1";
+  }
+  const runsPerCase = positiveInt(options.runsPerCase);
+  if (runsPerCase) env.ELIZA_BENCHMARK_RUNS_PER_CASE = String(runsPerCase);
+  if (options.filter?.trim())
+    env.ELIZA_BENCHMARK_FILTER = options.filter.trim();
+  if (options.provider?.trim()) {
+    const provider = options.provider.trim();
+    env.ELIZA_BENCHMARK_PROVIDER = provider;
+    if (provider === "local-llama-cpp") {
+      env.LOCAL_LLAMA_CPP_API_KEY = process.env.LOCAL_LLAMA_CPP_API_KEY ?? "local";
+    }
+  }
+  const runtimeModel = stringSetting(options.runtimeModel);
+  const smallModel = stringSetting(options.smallModel) ?? runtimeModel;
+  const largeModel = stringSetting(options.largeModel) ?? runtimeModel;
+  if (smallModel) env.ELIZA_LIVE_TEST_SMALL_MODEL = smallModel;
+  if (largeModel) env.ELIZA_LIVE_TEST_LARGE_MODEL = largeModel;
+  if (options.baseUrl?.trim()) {
+    env.ELIZA_LIVE_TEST_LOCAL_LLAMA_CPP_BASE_URL = options.baseUrl.trim();
+  }
+  return env;
+}
+async function runActionBenchmark(options = {}) {
+  const workspaceRoot = resolveWorkspaceRoot(options.workspaceRoot);
+  const appCoreRoot = join(workspaceRoot, "packages", "app-core");
+  const stamp = safeTimestamp((/* @__PURE__ */ new Date()).toISOString());
+  const outputDir = options.outputDir ?? join(trainingStateRoot(), "benchmarks", "action-selection", stamp);
+  const reportMarkdownPath = join(outputDir, "action-benchmark-report.md");
+  const reportJsonPath = join(outputDir, "action-benchmark-report.json");
+  const trajectoryDir = join(outputDir, "trajectories");
+  await mkdir(outputDir, { recursive: true });
+  await mkdir(trajectoryDir, { recursive: true });
+  const command = options.bun ?? defaultBunCommand();
+  const args = buildActionBenchmarkCommand();
+  const benchmarkEnv = buildActionBenchmarkEnv(options, {
+    reportMarkdownPath,
+    reportJsonPath,
+    trajectoryDir
+  });
+  const reportMatrixSource = matrixSourceForReport(reportJsonPath, options);
+  if (options.dryRun) {
+    const sample = dryRunCaseSample(options, trajectoryDir);
+    await writeFile(
+      String(sample.trajectoryPath),
+      `${JSON.stringify(
+        {
+          schema: "eliza_action_benchmark_dry_run_trajectory",
+          schemaVersion: 1,
+          generatedAt: (/* @__PURE__ */ new Date()).toISOString(),
+          source: {
+            kind: "app_core_action_selection_benchmark",
+            dryRun: true,
+            modelId: sample.modelId,
+            tier: sample.tier,
+            variant: sample.variant
+          },
+          caseId: sample.caseId,
+          prompt: sample.prompt,
+          expectedAction: sample.expectedAction,
+          actualAction: sample.actualAction,
+          pass: sample.pass,
+          response: sample.response,
+          events: [
+            {
+              type: "DRY_RUN_BENCHMARK_CASE",
+              timestamp: (/* @__PURE__ */ new Date()).toISOString(),
+              data: {
+                reason: "No model inference executed in dry-run mode."
+              }
+            }
+          ]
+        },
+        null,
+        2
+      )}
+`,
+      "utf8"
+    );
+    await writeFile(
+      reportJsonPath,
+      `${JSON.stringify(
+        {
+          schema: "eliza_action_selection_benchmark_report",
+          schemaVersion: 1,
+          generatedAt: (/* @__PURE__ */ new Date()).toISOString(),
+          source: {
+            kind: "app_core_action_selection_benchmark",
+            trajectoryDir,
+            reportMarkdownPath,
+            modelId: reportMatrixSource?.modelId,
+            variant: reportMatrixSource?.variant,
+            benchmark: reportMatrixSource?.benchmark,
+            tier: reportMatrixSource?.tier,
+            provider: reportMatrixSource?.provider,
+            datasetVersion: reportMatrixSource?.datasetVersion,
+            codeCommit: reportMatrixSource?.codeCommit,
+            useMocks: reportMatrixSource?.useMocks,
+            dryRun: true
+          },
+          summary: {
+            total: 1,
+            passed: 0,
+            failed: 1,
+            accuracy: 0,
+            plannerAccuracy: 0,
+            executionAccuracy: 0
+          },
+          failureModes: {
+            dry_run: 1
+          },
+          failures: [
+            {
+              caseId: sample.caseId,
+              failureMode: "dry_run",
+              reason: "No model inference executed in dry-run mode."
+            }
+          ],
+          results: [sample],
+          dryRun: true
+        },
+        null,
+        2
+      )}
+`,
+      "utf8"
+    );
+    await writeFile(
+      reportMarkdownPath,
+      "# Action Selection Benchmark Dry Run\n\nNo benchmark cases were executed.\n",
+      "utf8"
+    );
+    return {
+      workspaceRoot,
+      appCoreRoot,
+      outputDir,
+      reportMarkdownPath,
+      reportJsonPath,
+      trajectoryDir,
+      command: [command, ...args],
+      env: benchmarkEnv,
+      stdout: "[DRY RUN] Would run app-core action selection benchmark.",
+      stderr: "",
+      exitCode: 0,
+      matrixSource: reportMatrixSource
+    };
+  }
+  await assertLocalBenchmarkModelAvailable(options);
+  const proc = await collectProcess(command, args, appCoreRoot, {
+    ...process.env,
+    ...benchmarkEnv
+  });
+  if (proc.exitCode !== 0) {
+    throw new Error(
+      `action selection benchmark exited with code ${proc.exitCode}: ${proc.stderr || proc.stdout}`
+    );
+  }
+  await annotateBenchmarkReportSource(reportJsonPath, options);
+  return {
+    workspaceRoot,
+    appCoreRoot,
+    outputDir,
+    reportMarkdownPath,
+    reportJsonPath,
+    trajectoryDir,
+    command: [command, ...args],
+    env: benchmarkEnv,
+    stdout: proc.stdout,
+    stderr: proc.stderr,
+    exitCode: proc.exitCode,
+    matrixSource: matrixSourceForReport(reportJsonPath, options)
+  };
+}
+export {
+  assertLocalBenchmarkModelAvailable,
+  buildActionBenchmarkCommand,
+  buildActionBenchmarkEnv,
+  runActionBenchmark
+};
+//# sourceMappingURL=action-benchmark-runner.js.map