@elizaos/plugin-training 2.0.3-beta.5 → 2.0.3-beta.7

package/dist/core/benchmark-matrix-artifact.js

@@ -0,0 +1,381 @@
+import { mkdir, writeFile } from "node:fs/promises";
+import { readFile } from "node:fs/promises";
+import { join } from "node:path";
+import {
+  canonicalElizaOneTierSort,
+  normalizeElizaOneBenchmarkTier
+} from "./eliza1-benchmark-recipe.js";
+import { EVAL_COMPARISON_ARTIFACT_SCHEMA } from "./eval-comparison-artifact.js";
+import { trainingStateRoot } from "./training-config.js";
+const BENCHMARK_MATRIX_ARTIFACT_SCHEMA = "eliza_benchmark_matrix_artifact";
+const BENCHMARK_MATRIX_ARTIFACT_VERSION = 1;
+const ACTION_BENCHMARK_REPORT_SCHEMA = "eliza_action_selection_benchmark_report";
+const ACTION_SELECTION_BENCHMARK_ID = "eliza_harness_action_selection";
+const LOCAL_EVAL_COMPARISON_BENCHMARK_ID = "eliza_harness_local_eval_comparison";
+const ELIZA_ONE_MATRIX_TIERS = [
+  "0b",
+  "2b",
+  "4b",
+  "9b",
+  "27b"
+];
+function asRecord(value) {
+  return value && typeof value === "object" && !Array.isArray(value) ? value : null;
+}
+function asString(value) {
+  return typeof value === "string" && value.trim().length > 0 ? value.trim() : null;
+}
+function asNumber(value) {
+  return typeof value === "number" && Number.isFinite(value) ? value : null;
+}
+function finiteScore(value) {
+  if (!Number.isFinite(value))
+    throw new Error(`score must be finite; got ${value}`);
+  return value;
+}
+function roundMetric(value) {
+  return value === null ? null : Number(value.toFixed(6));
+}
+function percentDelta(base, next) {
+  if (base === null || next === null || base === 0) return null;
+  return (next - base) / Math.abs(base) * 100;
+}
+function isDryRunRow(row) {
+  if (!row) return false;
+  const rawSource = asRecord(row.raw.source);
+  return row.metrics.dryRun === true || row.raw.dryRun === true || rawSource?.dryRun === true;
+}
+function inferTier(modelId, explicit) {
+  const tier = asString(explicit);
+  if (tier) return normalizeElizaOneBenchmarkTier(tier) ?? tier;
+  const normalized = modelId.toLowerCase();
+  if (normalized.includes("27b")) return "27b";
+  if (normalized.includes("9b")) return "9b";
+  if (normalized.includes("4b")) return "4b";
+  if (normalized.includes("2b")) return "2b";
+  if (normalized.includes("0b")) return "0b";
+  return null;
+}
+function normalizeRow(row) {
+  const explicitReferenceTier = asString(row.tier);
+  return {
+    modelId: row.modelId,
+    benchmark: row.benchmark,
+    score: finiteScore(row.score),
+    variant: row.variant,
+    tier: row.variant === "reference" && !explicitReferenceTier ? null : inferTier(row.modelId, row.tier),
+    provider: asString(row.provider),
+    datasetVersion: asString(row.datasetVersion),
+    codeCommit: asString(row.codeCommit),
+    ts: row.ts ?? null,
+    metrics: row.metrics ?? {},
+    raw: row.raw ?? {}
+  };
+}
+function selectReferenceModelId(rows, explicit) {
+  if (explicit) return explicit;
+  return rows.find((row) => row.variant === "reference")?.modelId ?? rows.find((row) => row.provider === "cerebras")?.modelId ?? null;
+}
+function scoreFor(rows, benchmark, variant, tier) {
+  if (variant === "reference") {
+    return rows.find(
+      (row) => row.benchmark === benchmark && row.variant === "reference" && row.tier === tier
+    ) ?? rows.find(
+      (row) => row.benchmark === benchmark && row.variant === "reference" && row.tier === null
+    ) ?? null;
+  }
+  return rows.find(
+    (row) => row.benchmark === benchmark && row.variant === variant && row.tier === tier
+  ) ?? null;
+}
+function buildComparisons(rows, referenceModelId) {
+  const tiers = Array.from(
+    new Set(
+      rows.map((row) => row.tier).filter((tier) => tier !== null)
+    )
+  ).sort(canonicalElizaOneTierSort);
+  const benchmarks = Array.from(
+    new Set(rows.map((row) => row.benchmark))
+  ).sort();
+  const comparisons = [];
+  for (const tier of tiers) {
+    for (const benchmark of benchmarks) {
+      const base = scoreFor(rows, benchmark, "base", tier);
+      const trained = scoreFor(rows, benchmark, "trained", tier);
+      const reference = scoreFor(rows, benchmark, "reference", tier);
+      if (!base && !trained && !reference) continue;
+      const dryRun = isDryRunRow(base) || isDryRunRow(trained) || isDryRunRow(reference);
+      comparisons.push({
+        tier,
+        benchmark,
+        baseModelId: base?.modelId ?? null,
+        trainedModelId: trained?.modelId ?? null,
+        referenceModelId: reference?.modelId ?? referenceModelId,
+        baseScore: base?.score ?? null,
+        trainedScore: trained?.score ?? null,
+        referenceScore: reference?.score ?? null,
+        improvementAbsolute: roundMetric(
+          base && trained ? trained.score - base.score : null
+        ),
+        improvementPercent: roundMetric(
+          percentDelta(base?.score ?? null, trained?.score ?? null)
+        ),
+        trainedVsReferenceAbsolute: roundMetric(
+          trained && reference ? trained.score - reference.score : null
+        ),
+        trainedVsReferencePercent: roundMetric(
+          percentDelta(reference?.score ?? null, trained?.score ?? null)
+        ),
+        dryRun
+      });
+    }
+  }
+  return comparisons;
+}
+function safeTimestamp(value) {
+  return value.replace(/[:.]/g, "-");
+}
+function rowFromActionBenchmarkArtifact(payload, source) {
+  const reportSource = asRecord(payload.source) ?? {};
+  const embeddedVariant = reportSource.variant;
+  const modelId = source.modelId ?? asString(reportSource.modelId) ?? void 0;
+  const variant = source.variant ?? (embeddedVariant === "reference" || embeddedVariant === "base" || embeddedVariant === "trained" ? embeddedVariant : void 0);
+  if (!modelId || !variant) {
+    throw new Error(
+      `Action benchmark artifact ${source.path} requires modelId and variant`
+    );
+  }
+  const summary = asRecord(payload.summary) ?? {};
+  const dryRun = payload.dryRun === true || reportSource.dryRun === true;
+  const useMocks = source.useMocks === true || reportSource.useMocks === true || payload.useMocks === true;
+  const score = asNumber(summary.accuracy) ?? (dryRun ? 0 : null);
+  if (score === null) {
+    throw new Error(
+      `Action benchmark artifact ${source.path} missing accuracy`
+    );
+  }
+  const caseSamples = Array.isArray(payload.results) ? payload.results.map(asRecord).filter((result) => result !== null).slice(0, 8).map((result) => ({
+    caseId: asString(result.caseId),
+    prompt: asString(result.prompt) ?? asString(result.input) ?? asString(result.userPrompt),
+    expectedAction: asString(result.expectedAction),
+    actualAction: asString(result.actualAction),
+    pass: result.pass === true,
+    response: asString(result.response) ?? asString(result.output) ?? asString(result.finalResponse) ?? asString(result.failureReason),
+    latencyMs: asNumber(result.latencyMs),
+    trajectoryPath: asString(result.trajectoryPath)
+  })) : [];
+  return [
+    {
+      modelId,
+      variant,
+      benchmark: source.benchmark ?? asString(reportSource.benchmark) ?? ACTION_SELECTION_BENCHMARK_ID,
+      score,
+      tier: source.tier ?? asString(reportSource.tier) ?? void 0,
+      provider: source.provider ?? asString(reportSource.provider) ?? void 0,
+      datasetVersion: source.datasetVersion ?? asString(reportSource.datasetVersion) ?? void 0,
+      codeCommit: source.codeCommit ?? asString(reportSource.codeCommit) ?? void 0,
+      ts: asString(payload.generatedAt) ?? void 0,
+      metrics: {
+        plannerAccuracy: summary.plannerAccuracy,
+        executionAccuracy: summary.executionAccuracy,
+        total: summary.total,
+        passed: summary.passed,
+        failed: summary.failed,
+        latency: summary.latency,
+        failureModes: payload.failureModes,
+        dryRun,
+        useMocks
+      },
+      raw: {
+        artifactPath: source.path,
+        schema: payload.schema,
+        source: payload.source,
+        caseSamples,
+        dryRun,
+        useMocks
+      }
+    }
+  ];
+}
+function rowsFromEvalComparisonArtifact(payload, source) {
+  const models = asRecord(payload.models) ?? {};
+  const metrics = asRecord(payload.metrics) ?? {};
+  const benchmark = source.benchmark ?? LOCAL_EVAL_COMPARISON_BENCHMARK_ID;
+  const baseModelId = source.variant === "base" ? source.modelId : asString(models.base);
+  const trainedModelId = source.variant === "trained" ? source.modelId : asString(models.trained);
+  const rows = [];
+  const baseScore = asNumber(metrics.baseScore);
+  if (baseModelId && baseScore !== null) {
+    rows.push({
+      modelId: baseModelId,
+      variant: "base",
+      benchmark,
+      score: baseScore,
+      tier: source.tier,
+      provider: source.provider,
+      datasetVersion: source.datasetVersion,
+      codeCommit: source.codeCommit,
+      ts: asString(payload.generatedAt) ?? void 0,
+      metrics: {
+        latencyMs: metrics.baseLatencyMs,
+        promptCount: metrics.promptCount
+      },
+      raw: {
+        artifactPath: source.path,
+        schema: payload.schema
+      }
+    });
+  }
+  const trainedScore = asNumber(metrics.trainedScore);
+  if (trainedModelId && trainedScore !== null) {
+    rows.push({
+      modelId: trainedModelId,
+      variant: "trained",
+      benchmark,
+      score: trainedScore,
+      tier: source.tier,
+      provider: source.provider,
+      datasetVersion: source.datasetVersion,
+      codeCommit: source.codeCommit,
+      ts: asString(payload.generatedAt) ?? void 0,
+      metrics: {
+        latencyMs: metrics.trainedLatencyMs,
+        promptCount: metrics.promptCount,
+        improvementAbsolute: metrics.improvementAbsolute,
+        improvementPercent: metrics.improvementPercent
+      },
+      raw: {
+        artifactPath: source.path,
+        schema: payload.schema
+      }
+    });
+  }
+  return rows;
+}
+function rowsFromBenchmarkMatrixArtifact(payload, source) {
+  const rows = Array.isArray(payload.rows) ? payload.rows.map(asRecord).filter((row) => row !== null) : [];
+  return rows.map((row) => {
+    const modelId = asString(row.modelId);
+    const benchmark = asString(row.benchmark);
+    const variant = row.variant;
+    const score = asNumber(row.score);
+    if (!modelId || !benchmark || score === null || variant !== "reference" && variant !== "base" && variant !== "trained") {
+      throw new Error(
+        `Benchmark matrix artifact ${source.path} has an invalid row`
+      );
+    }
+    return {
+      modelId,
+      benchmark: source.benchmark ?? benchmark,
+      score,
+      variant,
+      tier: source.tier ?? asString(row.tier) ?? void 0,
+      provider: source.provider ?? asString(row.provider) ?? void 0,
+      datasetVersion: source.datasetVersion ?? asString(row.datasetVersion) ?? void 0,
+      codeCommit: source.codeCommit ?? asString(row.codeCommit) ?? void 0,
+      ts: row.ts,
+      metrics: asRecord(row.metrics) ?? {},
+      raw: {
+        ...asRecord(row.raw) ?? {},
+        artifactPath: source.path,
+        schema: payload.schema
+      }
+    };
+  });
+}
+function buildBenchmarkMatrixRowsFromArtifactPayload(payload, source) {
+  if (payload.schema === ACTION_BENCHMARK_REPORT_SCHEMA) {
+    return rowFromActionBenchmarkArtifact(payload, source);
+  }
+  if (payload.schema === EVAL_COMPARISON_ARTIFACT_SCHEMA) {
+    return rowsFromEvalComparisonArtifact(payload, source);
+  }
+  if (payload.schema === BENCHMARK_MATRIX_ARTIFACT_SCHEMA) {
+    return rowsFromBenchmarkMatrixArtifact(payload, source);
+  }
+  throw new Error(`Unsupported benchmark artifact schema in ${source.path}`);
+}
+async function buildBenchmarkMatrixRowsFromArtifacts(artifacts) {
+  const rows = [];
+  for (const source of artifacts) {
+    const payload = asRecord(JSON.parse(await readFile(source.path, "utf-8")));
+    if (!payload)
+      throw new Error(`Artifact ${source.path} must be a JSON object`);
+    rows.push(...buildBenchmarkMatrixRowsFromArtifactPayload(payload, source));
+  }
+  return rows;
+}
+function buildBenchmarkMatrixArtifactPayload(input) {
+  const rows = input.rows.map(normalizeRow);
+  const referenceModelId = selectReferenceModelId(rows, input.referenceModelId);
+  const tiers = Array.from(
+    new Set(
+      rows.map((row) => row.tier).filter((tier) => !!tier)
+    )
+  ).sort(canonicalElizaOneTierSort);
+  const benchmarks = Array.from(
+    new Set(rows.map((row) => row.benchmark))
+  ).sort();
+  const comparisons = buildComparisons(rows, referenceModelId);
+  return {
+    schema: BENCHMARK_MATRIX_ARTIFACT_SCHEMA,
+    version: BENCHMARK_MATRIX_ARTIFACT_VERSION,
+    generatedAt: input.generatedAt ?? (/* @__PURE__ */ new Date()).toISOString(),
+    source: input.source ?? { kind: "training_benchmark_matrix" },
+    referenceModelId,
+    tiers,
+    benchmarks,
+    counts: {
+      rows: rows.length,
+      comparisons: comparisons.length,
+      tiers: tiers.length,
+      benchmarks: benchmarks.length
+    },
+    rows,
+    comparisons
+  };
+}
+async function writeBenchmarkMatrixArtifact(input) {
+  const artifact = buildBenchmarkMatrixArtifactPayload(input);
+  const outputDir = input.outputDir ?? join(
+    trainingStateRoot(),
+    "benchmarks",
+    safeTimestamp(artifact.generatedAt)
+  );
+  await mkdir(outputDir, { recursive: true });
+  const artifactPath = join(outputDir, "benchmark-matrix.json");
+  await writeFile(
+    artifactPath,
+    `${JSON.stringify(artifact, null, 2)}
+`,
+    "utf-8"
+  );
+  return { outputDir, artifactPath, artifact };
+}
+async function writeBenchmarkMatrixArtifactFromArtifacts(input) {
+  const rows = await buildBenchmarkMatrixRowsFromArtifacts(input.artifacts);
+  return writeBenchmarkMatrixArtifact({
+    rows,
+    outputDir: input.outputDir,
+    generatedAt: input.generatedAt,
+    referenceModelId: input.referenceModelId,
+    source: input.source ?? {
+      kind: "training_benchmark_matrix_from_artifacts",
+      artifacts: input.artifacts.map((artifact) => artifact.path)
+    }
+  });
+}
+export {
+  ACTION_BENCHMARK_REPORT_SCHEMA,
+  ACTION_SELECTION_BENCHMARK_ID,
+  BENCHMARK_MATRIX_ARTIFACT_SCHEMA,
+  BENCHMARK_MATRIX_ARTIFACT_VERSION,
+  ELIZA_ONE_MATRIX_TIERS,
+  LOCAL_EVAL_COMPARISON_BENCHMARK_ID,
+  buildBenchmarkMatrixArtifactPayload,
+  buildBenchmarkMatrixRowsFromArtifactPayload,
+  buildBenchmarkMatrixRowsFromArtifacts,
+  writeBenchmarkMatrixArtifact,
+  writeBenchmarkMatrixArtifactFromArtifacts
+};
+//# sourceMappingURL=benchmark-matrix-artifact.js.map

package/dist/core/benchmark-matrix-artifact.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../../src/core/benchmark-matrix-artifact.ts"],"sourcesContent":["import { mkdir, writeFile } from \"node:fs/promises\";\nimport { readFile } from \"node:fs/promises\";\nimport { join } from \"node:path\";\nimport {\n  canonicalElizaOneTierSort,\n  normalizeElizaOneBenchmarkTier,\n} from \"./eliza1-benchmark-recipe.js\";\nimport { EVAL_COMPARISON_ARTIFACT_SCHEMA } from \"./eval-comparison-artifact.js\";\nimport { trainingStateRoot } from \"./training-config.js\";\n\nexport const BENCHMARK_MATRIX_ARTIFACT_SCHEMA =\n  \"eliza_benchmark_matrix_artifact\";\nexport const BENCHMARK_MATRIX_ARTIFACT_VERSION = 1;\nexport const ACTION_BENCHMARK_REPORT_SCHEMA =\n  \"eliza_action_selection_benchmark_report\";\nexport const ACTION_SELECTION_BENCHMARK_ID = \"eliza_harness_action_selection\";\nexport const LOCAL_EVAL_COMPARISON_BENCHMARK_ID =\n  \"eliza_harness_local_eval_comparison\";\n\nexport const ELIZA_ONE_MATRIX_TIERS = [\n  \"0b\",\n  \"2b\",\n  \"4b\",\n  \"9b\",\n  \"27b\",\n] as const;\n\nexport type ElizaOneMatrixTier = (typeof ELIZA_ONE_MATRIX_TIERS)[number];\nexport type BenchmarkMatrixVariant = \"reference\" | \"base\" | \"trained\";\n\nexport interface BenchmarkMatrixRowInput {\n  modelId: string;\n  benchmark: string;\n  score: number;\n  variant: BenchmarkMatrixVariant;\n  tier?: string;\n  provider?: string;\n  datasetVersion?: string;\n  codeCommit?: string;\n  ts?: number | string;\n  metrics?: Record<string, unknown>;\n  raw?: Record<string, unknown>;\n}\n\nexport interface BenchmarkMatrixInput {\n  rows: BenchmarkMatrixRowInput[];\n  outputDir?: string;\n  generatedAt?: string;\n  referenceModelId?: string;\n  source?: Record<string, unknown>;\n}\n\nexport interface BenchmarkMatrixArtifactSource {\n  path: string;\n  modelId?: string;\n  benchmark?: string;\n  variant?: BenchmarkMatrixVariant;\n  tier?: string;\n  provider?: string;\n  datasetVersion?: string;\n  codeCommit?: string;\n  useMocks?: boolean;\n}\n\nexport interface BenchmarkMatrixFromArtifactsInput {\n  artifacts: BenchmarkMatrixArtifactSource[];\n  outputDir?: string;\n  generatedAt?: string;\n  referenceModelId?: string;\n  source?: Record<string, unknown>;\n}\n\nexport interface BenchmarkMatrixCell {\n  modelId: string;\n  benchmark: string;\n  score: number;\n  variant: BenchmarkMatrixVariant;\n  tier: string | null;\n  provider: string | null;\n  datasetVersion: string | null;\n  codeCommit: string | null;\n  ts: number | string | null;\n  metrics: Record<string, unknown>;\n  raw: Record<string, unknown>;\n}\n\nexport interface BenchmarkMatrixComparison {\n  tier: string;\n  benchmark: string;\n  baseModelId: string | null;\n  trainedModelId: string | null;\n  referenceModelId: string | null;\n  baseScore: number | null;\n  trainedScore: number | null;\n  referenceScore: number | null;\n  improvementAbsolute: number | null;\n  improvementPercent: number | null;\n  trainedVsReferenceAbsolute: number | null;\n  trainedVsReferencePercent: number | null;\n  dryRun: boolean;\n}\n\nexport interface BenchmarkMatrixArtifact {\n  schema: typeof BENCHMARK_MATRIX_ARTIFACT_SCHEMA;\n  version: typeof BENCHMARK_MATRIX_ARTIFACT_VERSION;\n  generatedAt: string;\n  source: Record<string, unknown>;\n  referenceModelId: string | null;\n  tiers: string[];\n  benchmarks: string[];\n  counts: {\n    rows: number;\n    comparisons: number;\n    tiers: number;\n    benchmarks: number;\n  };\n  rows: BenchmarkMatrixCell[];\n  comparisons: BenchmarkMatrixComparison[];\n}\n\nexport interface BenchmarkMatrixArtifactResult {\n  outputDir: string;\n  artifactPath: string;\n  artifact: BenchmarkMatrixArtifact;\n}\n\nfunction asRecord(value: unknown): Record<string, unknown> | null {\n  return value && typeof value === \"object\" && !Array.isArray(value)\n    ? (value as Record<string, unknown>)\n    : null;\n}\n\nfunction asString(value: unknown): string | null {\n  return typeof value === \"string\" && value.trim().length > 0\n    ? value.trim()\n    : null;\n}\n\nfunction asNumber(value: unknown): number | null {\n  return typeof value === \"number\" && Number.isFinite(value) ? value : null;\n}\n\nfunction finiteScore(value: number): number {\n  if (!Number.isFinite(value))\n    throw new Error(`score must be finite; got ${value}`);\n  return value;\n}\n\nfunction roundMetric(value: number | null): number | null {\n  return value === null ? null : Number(value.toFixed(6));\n}\n\nfunction percentDelta(base: number | null, next: number | null): number | null {\n  if (base === null || next === null || base === 0) return null;\n  return ((next - base) / Math.abs(base)) * 100;\n}\n\nfunction isDryRunRow(row: BenchmarkMatrixCell | null | undefined): boolean {\n  if (!row) return false;\n  const rawSource = asRecord(row.raw.source);\n  return (\n    row.metrics.dryRun === true ||\n    row.raw.dryRun === true ||\n    rawSource?.dryRun === true\n  );\n}\n\nfunction inferTier(modelId: string, explicit?: string): string | null {\n  const tier = asString(explicit);\n  if (tier) return normalizeElizaOneBenchmarkTier(tier) ?? tier;\n  const normalized = modelId.toLowerCase();\n  if (normalized.includes(\"27b\")) return \"27b\";\n  if (normalized.includes(\"9b\")) return \"9b\";\n  if (normalized.includes(\"4b\")) return \"4b\";\n  if (normalized.includes(\"2b\")) return \"2b\";\n  if (normalized.includes(\"0b\")) return \"0b\";\n  return null;\n}\n\nfunction normalizeRow(row: BenchmarkMatrixRowInput): BenchmarkMatrixCell {\n  const explicitReferenceTier = asString(row.tier);\n  return {\n    modelId: row.modelId,\n    benchmark: row.benchmark,\n    score: finiteScore(row.score),\n    variant: row.variant,\n    tier:\n      row.variant === \"reference\" && !explicitReferenceTier\n        ? null\n        : inferTier(row.modelId, row.tier),\n    provider: asString(row.provider),\n    datasetVersion: asString(row.datasetVersion),\n    codeCommit: asString(row.codeCommit),\n    ts: row.ts ?? null,\n    metrics: row.metrics ?? {},\n    raw: row.raw ?? {},\n  };\n}\n\nfunction selectReferenceModelId(\n  rows: readonly BenchmarkMatrixCell[],\n  explicit?: string,\n): string | null {\n  if (explicit) return explicit;\n  return (\n    rows.find((row) => row.variant === \"reference\")?.modelId ??\n    rows.find((row) => row.provider === \"cerebras\")?.modelId ??\n    null\n  );\n}\n\nfunction scoreFor(\n  rows: readonly BenchmarkMatrixCell[],\n  benchmark: string,\n  variant: BenchmarkMatrixVariant,\n  tier?: string,\n): BenchmarkMatrixCell | null {\n  if (variant === \"reference\") {\n    return (\n      rows.find(\n        (row) =>\n          row.benchmark === benchmark &&\n          row.variant === \"reference\" &&\n          row.tier === tier,\n      ) ??\n      rows.find(\n        (row) =>\n          row.benchmark === benchmark &&\n          row.variant === \"reference\" &&\n          row.tier === null,\n      ) ??\n      null\n    );\n  }\n  return (\n    rows.find(\n      (row) =>\n        row.benchmark === benchmark &&\n        row.variant === variant &&\n        row.tier === tier,\n    ) ?? null\n  );\n}\n\nfunction buildComparisons(\n  rows: readonly BenchmarkMatrixCell[],\n  referenceModelId: string | null,\n): BenchmarkMatrixComparison[] {\n  const tiers = Array.from(\n    new Set(\n      rows\n        .map((row) => row.tier)\n        .filter((tier): tier is string => tier !== null),\n    ),\n  ).sort(canonicalElizaOneTierSort);\n  const benchmarks = Array.from(\n    new Set(rows.map((row) => row.benchmark)),\n  ).sort();\n  const comparisons: BenchmarkMatrixComparison[] = [];\n  for (const tier of tiers) {\n    for (const benchmark of benchmarks) {\n      const base = scoreFor(rows, benchmark, \"base\", tier);\n      const trained = scoreFor(rows, benchmark, \"trained\", tier);\n      const reference = scoreFor(rows, benchmark, \"reference\", tier);\n      if (!base && !trained && !reference) continue;\n      const dryRun =\n        isDryRunRow(base) || isDryRunRow(trained) || isDryRunRow(reference);\n      comparisons.push({\n        tier,\n        benchmark,\n        baseModelId: base?.modelId ?? null,\n        trainedModelId: trained?.modelId ?? null,\n        referenceModelId: reference?.modelId ?? referenceModelId,\n        baseScore: base?.score ?? null,\n        trainedScore: trained?.score ?? null,\n        referenceScore: reference?.score ?? null,\n        improvementAbsolute: roundMetric(\n          base && trained ? trained.score - base.score : null,\n        ),\n        improvementPercent: roundMetric(\n          percentDelta(base?.score ?? null, trained?.score ?? null),\n        ),\n        trainedVsReferenceAbsolute: roundMetric(\n          trained && reference ? trained.score - reference.score : null,\n        ),\n        trainedVsReferencePercent: roundMetric(\n          percentDelta(reference?.score ?? null, trained?.score ?? null),\n        ),\n        dryRun,\n      });\n    }\n  }\n  return comparisons;\n}\n\nfunction safeTimestamp(value: string): string {\n  return value.replace(/[:.]/g, \"-\");\n}\n\nfunction rowFromActionBenchmarkArtifact(\n  payload: Record<string, unknown>,\n  source: BenchmarkMatrixArtifactSource,\n): BenchmarkMatrixRowInput[] {\n  const reportSource = asRecord(payload.source) ?? {};\n  const embeddedVariant = reportSource.variant;\n  const modelId = source.modelId ?? asString(reportSource.modelId) ?? undefined;\n  const variant =\n    source.variant ??\n    (embeddedVariant === \"reference\" ||\n    embeddedVariant === \"base\" ||\n    embeddedVariant === \"trained\"\n      ? embeddedVariant\n      : undefined);\n  if (!modelId || !variant) {\n    throw new Error(\n      `Action benchmark artifact ${source.path} requires modelId and variant`,\n    );\n  }\n  const summary = asRecord(payload.summary) ?? {};\n  const dryRun = payload.dryRun === true || reportSource.dryRun === true;\n  const useMocks =\n    source.useMocks === true ||\n    reportSource.useMocks === true ||\n    payload.useMocks === true;\n  const score = asNumber(summary.accuracy) ?? (dryRun ? 0 : null);\n  if (score === null) {\n    throw new Error(\n      `Action benchmark artifact ${source.path} missing accuracy`,\n    );\n  }\n  const caseSamples = Array.isArray(payload.results)\n    ? payload.results\n        .map(asRecord)\n        .filter((result): result is Record<string, unknown> => result !== null)\n        .slice(0, 8)\n        .map((result) => ({\n          caseId: asString(result.caseId),\n          prompt:\n            asString(result.prompt) ??\n            asString(result.input) ??\n            asString(result.userPrompt),\n          expectedAction: asString(result.expectedAction),\n          actualAction: asString(result.actualAction),\n          pass: result.pass === true,\n          response:\n            asString(result.response) ??\n            asString(result.output) ??\n            asString(result.finalResponse) ??\n            asString(result.failureReason),\n          latencyMs: asNumber(result.latencyMs),\n          trajectoryPath: asString(result.trajectoryPath),\n        }))\n    : [];\n  return [\n    {\n      modelId,\n      variant,\n      benchmark:\n        source.benchmark ??\n        asString(reportSource.benchmark) ??\n        ACTION_SELECTION_BENCHMARK_ID,\n      score,\n      tier: source.tier ?? asString(reportSource.tier) ?? undefined,\n      provider: source.provider ?? asString(reportSource.provider) ?? undefined,\n      datasetVersion:\n        source.datasetVersion ??\n        asString(reportSource.datasetVersion) ??\n        undefined,\n      codeCommit:\n        source.codeCommit ?? asString(reportSource.codeCommit) ?? undefined,\n      ts: asString(payload.generatedAt) ?? undefined,\n      metrics: {\n        plannerAccuracy: summary.plannerAccuracy,\n        executionAccuracy: summary.executionAccuracy,\n        total: summary.total,\n        passed: summary.passed,\n        failed: summary.failed,\n        latency: summary.latency,\n        failureModes: payload.failureModes,\n        dryRun,\n        useMocks,\n      },\n      raw: {\n        artifactPath: source.path,\n        schema: payload.schema,\n        source: payload.source,\n        caseSamples,\n        dryRun,\n        useMocks,\n      },\n    },\n  ];\n}\n\nfunction rowsFromEvalComparisonArtifact(\n  payload: Record<string, unknown>,\n  source: BenchmarkMatrixArtifactSource,\n): BenchmarkMatrixRowInput[] {\n  const models = asRecord(payload.models) ?? {};\n  const metrics = asRecord(payload.metrics) ?? {};\n  const benchmark = source.benchmark ?? LOCAL_EVAL_COMPARISON_BENCHMARK_ID;\n  const baseModelId =\n    source.variant === \"base\" ? source.modelId : asString(models.base);\n  const trainedModelId =\n    source.variant === \"trained\" ? source.modelId : asString(models.trained);\n  const rows: BenchmarkMatrixRowInput[] = [];\n  const baseScore = asNumber(metrics.baseScore);\n  if (baseModelId && baseScore !== null) {\n    rows.push({\n      modelId: baseModelId,\n      variant: \"base\",\n      benchmark,\n      score: baseScore,\n      tier: source.tier,\n      provider: source.provider,\n      datasetVersion: source.datasetVersion,\n      codeCommit: source.codeCommit,\n      ts: asString(payload.generatedAt) ?? undefined,\n      metrics: {\n        latencyMs: metrics.baseLatencyMs,\n        promptCount: metrics.promptCount,\n      },\n      raw: {\n        artifactPath: source.path,\n        schema: payload.schema,\n      },\n    });\n  }\n  const trainedScore = asNumber(metrics.trainedScore);\n  if (trainedModelId && trainedScore !== null) {\n    rows.push({\n      modelId: trainedModelId,\n      variant: \"trained\",\n      benchmark,\n      score: trainedScore,\n      tier: source.tier,\n      provider: source.provider,\n      datasetVersion: source.datasetVersion,\n      codeCommit: source.codeCommit,\n      ts: asString(payload.generatedAt) ?? undefined,\n      metrics: {\n        latencyMs: metrics.trainedLatencyMs,\n        promptCount: metrics.promptCount,\n        improvementAbsolute: metrics.improvementAbsolute,\n        improvementPercent: metrics.improvementPercent,\n      },\n      raw: {\n        artifactPath: source.path,\n        schema: payload.schema,\n      },\n    });\n  }\n  return rows;\n}\n\nfunction rowsFromBenchmarkMatrixArtifact(\n  payload: Record<string, unknown>,\n  source: BenchmarkMatrixArtifactSource,\n): BenchmarkMatrixRowInput[] {\n  const rows = Array.isArray(payload.rows)\n    ? payload.rows\n        .map(asRecord)\n        .filter((row): row is Record<string, unknown> => row !== null)\n    : [];\n  return rows.map((row) => {\n    const modelId = asString(row.modelId);\n    const benchmark = asString(row.benchmark);\n    const variant = row.variant;\n    const score = asNumber(row.score);\n    if (\n      !modelId ||\n      !benchmark ||\n      score === null ||\n      (variant !== \"reference\" && variant !== \"base\" && variant !== \"trained\")\n    ) {\n      throw new Error(\n        `Benchmark matrix artifact ${source.path} has an invalid row`,\n      );\n    }\n    return {\n      modelId,\n      benchmark: source.benchmark ?? benchmark,\n      score,\n      variant,\n      tier: source.tier ?? asString(row.tier) ?? undefined,\n      provider: source.provider ?? asString(row.provider) ?? undefined,\n      datasetVersion:\n        source.datasetVersion ?? asString(row.datasetVersion) ?? undefined,\n      codeCommit: source.codeCommit ?? asString(row.codeCommit) ?? undefined,\n      ts: row.ts as number | string | undefined,\n      metrics: asRecord(row.metrics) ?? {},\n      raw: {\n        ...(asRecord(row.raw) ?? {}),\n        artifactPath: source.path,\n        schema: payload.schema,\n      },\n    };\n  });\n}\n\nexport function buildBenchmarkMatrixRowsFromArtifactPayload(\n  payload: Record<string, unknown>,\n  source: BenchmarkMatrixArtifactSource,\n): BenchmarkMatrixRowInput[] {\n  if (payload.schema === ACTION_BENCHMARK_REPORT_SCHEMA) {\n    return rowFromActionBenchmarkArtifact(payload, source);\n  }\n  if (payload.schema === EVAL_COMPARISON_ARTIFACT_SCHEMA) {\n    return rowsFromEvalComparisonArtifact(payload, source);\n  }\n  if (payload.schema === BENCHMARK_MATRIX_ARTIFACT_SCHEMA) {\n    return rowsFromBenchmarkMatrixArtifact(payload, source);\n  }\n  throw new Error(`Unsupported benchmark artifact schema in ${source.path}`);\n}\n\nexport async function buildBenchmarkMatrixRowsFromArtifacts(\n  artifacts: BenchmarkMatrixArtifactSource[],\n): Promise<BenchmarkMatrixRowInput[]> {\n  const rows: BenchmarkMatrixRowInput[] = [];\n  for (const source of artifacts) {\n    const payload = asRecord(JSON.parse(await readFile(source.path, \"utf-8\")));\n    if (!payload)\n      throw new Error(`Artifact ${source.path} must be a JSON object`);\n    rows.push(...buildBenchmarkMatrixRowsFromArtifactPayload(payload, source));\n  }\n  return rows;\n}\n\nexport function buildBenchmarkMatrixArtifactPayload(\n  input: BenchmarkMatrixInput,\n): BenchmarkMatrixArtifact {\n  const rows = input.rows.map(normalizeRow);\n  const referenceModelId = selectReferenceModelId(rows, input.referenceModelId);\n  const tiers = Array.from(\n    new Set(\n      rows.map((row) => row.tier).filter((tier): tier is string => !!tier),\n    ),\n  ).sort(canonicalElizaOneTierSort);\n  const benchmarks = Array.from(\n    new Set(rows.map((row) => row.benchmark)),\n  ).sort();\n  const comparisons = buildComparisons(rows, referenceModelId);\n  return {\n    schema: BENCHMARK_MATRIX_ARTIFACT_SCHEMA,\n    version: BENCHMARK_MATRIX_ARTIFACT_VERSION,\n    generatedAt: input.generatedAt ?? new Date().toISOString(),\n    source: input.source ?? { kind: \"training_benchmark_matrix\" },\n    referenceModelId,\n    tiers,\n    benchmarks,\n    counts: {\n      rows: rows.length,\n      comparisons: comparisons.length,\n      tiers: tiers.length,\n      benchmarks: benchmarks.length,\n    },\n    rows,\n    comparisons,\n  };\n}\n\nexport async function writeBenchmarkMatrixArtifact(\n  input: BenchmarkMatrixInput,\n): Promise<BenchmarkMatrixArtifactResult> {\n  const artifact = buildBenchmarkMatrixArtifactPayload(input);\n  const outputDir =\n    input.outputDir ??\n    join(\n      trainingStateRoot(),\n      \"benchmarks\",\n      safeTimestamp(artifact.generatedAt),\n    );\n  await mkdir(outputDir, { recursive: true });\n  const artifactPath = join(outputDir, \"benchmark-matrix.json\");\n  await writeFile(\n    artifactPath,\n    `${JSON.stringify(artifact, null, 2)}\\n`,\n    \"utf-8\",\n  );\n  return { outputDir, artifactPath, artifact };\n}\n\nexport async function writeBenchmarkMatrixArtifactFromArtifacts(\n  input: BenchmarkMatrixFromArtifactsInput,\n): Promise<BenchmarkMatrixArtifactResult> {\n  const rows = await buildBenchmarkMatrixRowsFromArtifacts(input.artifacts);\n  return writeBenchmarkMatrixArtifact({\n    rows,\n    outputDir: input.outputDir,\n    generatedAt: input.generatedAt,\n    referenceModelId: input.referenceModelId,\n    source: input.source ?? {\n      kind: \"training_benchmark_matrix_from_artifacts\",\n      artifacts: input.artifacts.map((artifact) => artifact.path),\n    },\n  });\n}\n"],"mappings":"AAAA,SAAS,OAAO,iBAAiB;AACjC,SAAS,gBAAgB;AACzB,SAAS,YAAY;AACrB;AAAA,EACE;AAAA,EACA;AAAA,OACK;AACP,SAAS,uCAAuC;AAChD,SAAS,yBAAyB;AAE3B,MAAM,mCACX;AACK,MAAM,oCAAoC;AAC1C,MAAM,iCACX;AACK,MAAM,gCAAgC;AACtC,MAAM,qCACX;AAEK,MAAM,yBAAyB;AAAA,EACpC;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AACF;AAqGA,SAAS,SAAS,OAAgD;AAChE,SAAO,SAAS,OAAO,UAAU,YAAY,CAAC,MAAM,QAAQ,KAAK,IAC5D,QACD;AACN;AAEA,SAAS,SAAS,OAA+B;AAC/C,SAAO,OAAO,UAAU,YAAY,MAAM,KAAK,EAAE,SAAS,IACtD,MAAM,KAAK,IACX;AACN;AAEA,SAAS,SAAS,OAA+B;AAC/C,SAAO,OAAO,UAAU,YAAY,OAAO,SAAS,KAAK,IAAI,QAAQ;AACvE;AAEA,SAAS,YAAY,OAAuB;AAC1C,MAAI,CAAC,OAAO,SAAS,KAAK;AACxB,UAAM,IAAI,MAAM,6BAA6B,KAAK,EAAE;AACtD,SAAO;AACT;AAEA,SAAS,YAAY,OAAqC;AACxD,SAAO,UAAU,OAAO,OAAO,OAAO,MAAM,QAAQ,CAAC,CAAC;AACxD;AAEA,SAAS,aAAa,MAAqB,MAAoC;AAC7E,MAAI,SAAS,QAAQ,SAAS,QAAQ,SAAS,EAAG,QAAO;AACzD,UAAS,OAAO,QAAQ,KAAK,IAAI,IAAI,IAAK;AAC5C;AAEA,SAAS,YAAY,KAAsD;AACzE,MAAI,CAAC,IAAK,QAAO;AACjB,QAAM,YAAY,SAAS,IAAI,IAAI,MAAM;AACzC,SACE,IAAI,QAAQ,WAAW,QACvB,IAAI,IAAI,WAAW,QACnB,WAAW,WAAW;AAE1B;AAEA,SAAS,UAAU,SAAiB,UAAkC;AACpE,QAAM,OAAO,SAAS,QAAQ;AAC9B,MAAI,KAAM,QAAO,+BAA+B,IAAI,KAAK;AACzD,QAAM,aAAa,QAAQ,YAAY;AACvC,MAAI,WAAW,SAAS,KAAK,EAAG,QAAO;AACvC,MAAI,WAAW,SAAS,IAAI,EAAG,QAAO;AACtC,MAAI,WAAW,SAAS,IAAI,EAAG,QAAO;AACtC,MAAI,WAAW,SAAS,IAAI,EAAG,QAAO;AACtC,MAAI,WAAW,SAAS,IAAI,EAAG,QAAO;AACtC,SAAO;AACT;AAEA,SAAS,aAAa,KAAmD;AACvE,QAAM,wBAAwB,SAAS,IAAI,IAAI;AAC/C,SAAO;AAAA,IACL,SAAS,IAAI;AAAA,IACb,WAAW,IAAI;AAAA,IACf,OAAO,YAAY,IAAI,KAAK;AAAA,IAC5B,SAAS,IAAI;AAAA,IACb,MACE,IAAI,YAAY,eAAe,CAAC,wBAC5B,OACA,UAAU,IAAI,SAAS,IAAI,IAAI;AAAA,IACrC,UAAU,SAAS,IAAI,QAAQ;AAAA,IAC/B,gBAAgB,SAAS,IAAI,cAAc;AAAA,IAC3C,YAAY,SAAS,IAAI,UAAU;AAAA,IACnC,IAAI,IAAI,MAAM;AAAA,IACd,SAAS,IAAI,WAAW,CAAC;AAAA,IACzB,KAAK,IAAI,OAAO,CAAC;AAAA,EACnB;AACF;AAEA,SAAS,uBACP,MACA,UACe;AACf,MAAI,SAAU,QAAO;AACrB,SACE,KAAK,KAAK,CAAC,QAAQ,IAAI,YAAY,WAAW,GAAG,WACjD,KAAK,KAAK,CAAC,QAAQ,IAAI,aAAa,UAAU,GAAG,WACjD;AAEJ;AAEA,SAAS,SACP,MACA,WACA,SACA,MAC4B;AAC5B,MAAI,YAAY,aAAa;AAC3B,WACE,KAAK;AAAA,MACH,CAAC,QACC,IAAI,cAAc,aAClB,IAAI,YAAY,eAChB,IAAI,SAAS;AAAA,IACjB,KACA,KAAK;AAAA,MACH,CAAC,QACC,IAAI,cAAc,aAClB,IAAI,YAAY,eAChB,IAAI,SAAS;AAAA,IACjB,KACA;AAAA,EAEJ;AACA,SACE,KAAK;AAAA,IACH,CAAC,QACC,IAAI,cAAc,aAClB,IAAI,YAAY,WAChB,IAAI,SAAS;AAAA,EACjB,KAAK;AAET;AAEA,SAAS,iBACP,MACA,kBAC6B;AAC7B,QAAM,QAAQ,MAAM;AAAA,IAClB,IAAI;AAAA,MACF,KACG,IAAI,CAAC,QAAQ,IAAI,IAAI,EACrB,OAAO,CAAC,SAAyB,SAAS,IAAI;AAAA,IACnD;AAAA,EACF,EAAE,KAAK,yBAAyB;AAChC,QAAM,aAAa,MAAM;AAAA,IACvB,IAAI,IAAI,KAAK,IAAI,CAAC,QAAQ,IAAI,SAAS,CAAC;AAAA,EAC1C,EAAE,KAAK;AACP,QAAM,cAA2C,CAAC;AAClD,aAAW,QAAQ,OAAO;AACxB,eAAW,aAAa,YAAY;AAClC,YAAM,OAAO,SAAS,MAAM,WAAW,QAAQ,IAAI;AACnD,YAAM,UAAU,SAAS,MAAM,WAAW,WAAW,IAAI;AACzD,YAAM,YAAY,SAAS,MAAM,WAAW,aAAa,IAAI;AAC7D,UAAI,CAAC,QAAQ,CAAC,WAAW,CAAC,UAAW;AACrC,YAAM,SACJ,YAAY,IAAI,KAAK,YAAY,OAAO,KAAK,YAAY,SAAS;AACpE,kBAAY,KAAK;AAAA,QACf;AAAA,QACA;AAAA,QACA,aAAa,MAAM,WAAW;AAAA,QAC9B,gBAAgB,SAAS,WAAW;AAAA,QACpC,kBAAkB,WAAW,WAAW;AAAA,QACxC,WAAW,MAAM,SAAS;AAAA,QAC1B,cAAc,SAAS,SAAS;AAAA,QAChC,gBAAgB,WAAW,SAAS;AAAA,QACpC,qBAAqB;AAAA,UACnB,QAAQ,UAAU,QAAQ,QAAQ,KAAK,QAAQ;AAAA,QACjD;AAAA,QACA,oBAAoB;AAAA,UAClB,aAAa,MAAM,SAAS,MAAM,SAAS,SAAS,IAAI;AAAA,QAC1D;AAAA,QACA,4BAA4B;AAAA,UAC1B,WAAW,YAAY,QAAQ,QAAQ,UAAU,QAAQ;AAAA,QAC3D;AAAA,QACA,2BAA2B;AAAA,UACzB,aAAa,WAAW,SAAS,MAAM,SAAS,SAAS,IAAI;AAAA,QAC/D;AAAA,QACA;AAAA,MACF,CAAC;AAAA,IACH;AAAA,EACF;AACA,SAAO;AACT;AAEA,SAAS,cAAc,OAAuB;AAC5C,SAAO,MAAM,QAAQ,SAAS,GAAG;AACnC;AAEA,SAAS,+BACP,SACA,QAC2B;AAC3B,QAAM,eAAe,SAAS,QAAQ,MAAM,KAAK,CAAC;AAClD,QAAM,kBAAkB,aAAa;AACrC,QAAM,UAAU,OAAO,WAAW,SAAS,aAAa,OAAO,KAAK;AACpE,QAAM,UACJ,OAAO,YACN,oBAAoB,eACrB,oBAAoB,UACpB,oBAAoB,YAChB,kBACA;AACN,MAAI,CAAC,WAAW,CAAC,SAAS;AACxB,UAAM,IAAI;AAAA,MACR,6BAA6B,OAAO,IAAI;AAAA,IAC1C;AAAA,EACF;AACA,QAAM,UAAU,SAAS,QAAQ,OAAO,KAAK,CAAC;AAC9C,QAAM,SAAS,QAAQ,WAAW,QAAQ,aAAa,WAAW;AAClE,QAAM,WACJ,OAAO,aAAa,QACpB,aAAa,aAAa,QAC1B,QAAQ,aAAa;AACvB,QAAM,QAAQ,SAAS,QAAQ,QAAQ,MAAM,SAAS,IAAI;AAC1D,MAAI,UAAU,MAAM;AAClB,UAAM,IAAI;AAAA,MACR,6BAA6B,OAAO,IAAI;AAAA,IAC1C;AAAA,EACF;AACA,QAAM,cAAc,MAAM,QAAQ,QAAQ,OAAO,IAC7C,QAAQ,QACL,IAAI,QAAQ,EACZ,OAAO,CAAC,WAA8C,WAAW,IAAI,EACrE,MAAM,GAAG,CAAC,EACV,IAAI,CAAC,YAAY;AAAA,IAChB,QAAQ,SAAS,OAAO,MAAM;AAAA,IAC9B,QACE,SAAS,OAAO,MAAM,KACtB,SAAS,OAAO,KAAK,KACrB,SAAS,OAAO,UAAU;AAAA,IAC5B,gBAAgB,SAAS,OAAO,cAAc;AAAA,IAC9C,cAAc,SAAS,OAAO,YAAY;AAAA,IAC1C,MAAM,OAAO,SAAS;AAAA,IACtB,UACE,SAAS,OAAO,QAAQ,KACxB,SAAS,OAAO,MAAM,KACtB,SAAS,OAAO,aAAa,KAC7B,SAAS,OAAO,aAAa;AAAA,IAC/B,WAAW,SAAS,OAAO,SAAS;AAAA,IACpC,gBAAgB,SAAS,OAAO,cAAc;AAAA,EAChD,EAAE,IACJ,CAAC;AACL,SAAO;AAAA,IACL;AAAA,MACE;AAAA,MACA;AAAA,MACA,WACE,OAAO,aACP,SAAS,aAAa,SAAS,KAC/B;AAAA,MACF;AAAA,MACA,MAAM,OAAO,QAAQ,SAAS,aAAa,IAAI,KAAK;AAAA,MACpD,UAAU,OAAO,YAAY,SAAS,aAAa,QAAQ,KAAK;AAAA,MAChE,gBACE,OAAO,kBACP,SAAS,aAAa,cAAc,KACpC;AAAA,MACF,YACE,OAAO,cAAc,SAAS,aAAa,UAAU,KAAK;AAAA,MAC5D,IAAI,SAAS,QAAQ,WAAW,KAAK;AAAA,MACrC,SAAS;AAAA,QACP,iBAAiB,QAAQ;AAAA,QACzB,mBAAmB,QAAQ;AAAA,QAC3B,OAAO,QAAQ;AAAA,QACf,QAAQ,QAAQ;AAAA,QAChB,QAAQ,QAAQ;AAAA,QAChB,SAAS,QAAQ;AAAA,QACjB,cAAc,QAAQ;AAAA,QACtB;AAAA,QACA;AAAA,MACF;AAAA,MACA,KAAK;AAAA,QACH,cAAc,OAAO;AAAA,QACrB,QAAQ,QAAQ;AAAA,QAChB,QAAQ,QAAQ;AAAA,QAChB;AAAA,QACA;AAAA,QACA;AAAA,MACF;AAAA,IACF;AAAA,EACF;AACF;AAEA,SAAS,+BACP,SACA,QAC2B;AAC3B,QAAM,SAAS,SAAS,QAAQ,MAAM,KAAK,CAAC;AAC5C,QAAM,UAAU,SAAS,QAAQ,OAAO,KAAK,CAAC;AAC9C,QAAM,YAAY,OAAO,aAAa;AACtC,QAAM,cACJ,OAAO,YAAY,SAAS,OAAO,UAAU,SAAS,OAAO,IAAI;AACnE,QAAM,iBACJ,OAAO,YAAY,YAAY,OAAO,UAAU,SAAS,OAAO,OAAO;AACzE,QAAM,OAAkC,CAAC;AACzC,QAAM,YAAY,SAAS,QAAQ,SAAS;AAC5C,MAAI,eAAe,cAAc,MAAM;AACrC,SAAK,KAAK;AAAA,MACR,SAAS;AAAA,MACT,SAAS;AAAA,MACT;AAAA,MACA,OAAO;AAAA,MACP,MAAM,OAAO;AAAA,MACb,UAAU,OAAO;AAAA,MACjB,gBAAgB,OAAO;AAAA,MACvB,YAAY,OAAO;AAAA,MACnB,IAAI,SAAS,QAAQ,WAAW,KAAK;AAAA,MACrC,SAAS;AAAA,QACP,WAAW,QAAQ;AAAA,QACnB,aAAa,QAAQ;AAAA,MACvB;AAAA,MACA,KAAK;AAAA,QACH,cAAc,OAAO;AAAA,QACrB,QAAQ,QAAQ;AAAA,MAClB;AAAA,IACF,CAAC;AAAA,EACH;AACA,QAAM,eAAe,SAAS,QAAQ,YAAY;AAClD,MAAI,kBAAkB,iBAAiB,MAAM;AAC3C,SAAK,KAAK;AAAA,MACR,SAAS;AAAA,MACT,SAAS;AAAA,MACT;AAAA,MACA,OAAO;AAAA,MACP,MAAM,OAAO;AAAA,MACb,UAAU,OAAO;AAAA,MACjB,gBAAgB,OAAO;AAAA,MACvB,YAAY,OAAO;AAAA,MACnB,IAAI,SAAS,QAAQ,WAAW,KAAK;AAAA,MACrC,SAAS;AAAA,QACP,WAAW,QAAQ;AAAA,QACnB,aAAa,QAAQ;AAAA,QACrB,qBAAqB,QAAQ;AAAA,QAC7B,oBAAoB,QAAQ;AAAA,MAC9B;AAAA,MACA,KAAK;AAAA,QACH,cAAc,OAAO;AAAA,QACrB,QAAQ,QAAQ;AAAA,MAClB;AAAA,IACF,CAAC;AAAA,EACH;AACA,SAAO;AACT;AAEA,SAAS,gCACP,SACA,QAC2B;AAC3B,QAAM,OAAO,MAAM,QAAQ,QAAQ,IAAI,IACnC,QAAQ,KACL,IAAI,QAAQ,EACZ,OAAO,CAAC,QAAwC,QAAQ,IAAI,IAC/D,CAAC;AACL,SAAO,KAAK,IAAI,CAAC,QAAQ;AACvB,UAAM,UAAU,SAAS,IAAI,OAAO;AACpC,UAAM,YAAY,SAAS,IAAI,SAAS;AACxC,UAAM,UAAU,IAAI;AACpB,UAAM,QAAQ,SAAS,IAAI,KAAK;AAChC,QACE,CAAC,WACD,CAAC,aACD,UAAU,QACT,YAAY,eAAe,YAAY,UAAU,YAAY,WAC9D;AACA,YAAM,IAAI;AAAA,QACR,6BAA6B,OAAO,IAAI;AAAA,MAC1C;AAAA,IACF;AACA,WAAO;AAAA,MACL;AAAA,MACA,WAAW,OAAO,aAAa;AAAA,MAC/B;AAAA,MACA;AAAA,MACA,MAAM,OAAO,QAAQ,SAAS,IAAI,IAAI,KAAK;AAAA,MAC3C,UAAU,OAAO,YAAY,SAAS,IAAI,QAAQ,KAAK;AAAA,MACvD,gBACE,OAAO,kBAAkB,SAAS,IAAI,cAAc,KAAK;AAAA,MAC3D,YAAY,OAAO,cAAc,SAAS,IAAI,UAAU,KAAK;AAAA,MAC7D,IAAI,IAAI;AAAA,MACR,SAAS,SAAS,IAAI,OAAO,KAAK,CAAC;AAAA,MACnC,KAAK;AAAA,QACH,GAAI,SAAS,IAAI,GAAG,KAAK,CAAC;AAAA,QAC1B,cAAc,OAAO;AAAA,QACrB,QAAQ,QAAQ;AAAA,MAClB;AAAA,IACF;AAAA,EACF,CAAC;AACH;AAEO,SAAS,4CACd,SACA,QAC2B;AAC3B,MAAI,QAAQ,WAAW,gCAAgC;AACrD,WAAO,+BAA+B,SAAS,MAAM;AAAA,EACvD;AACA,MAAI,QAAQ,WAAW,iCAAiC;AACtD,WAAO,+BAA+B,SAAS,MAAM;AAAA,EACvD;AACA,MAAI,QAAQ,WAAW,kCAAkC;AACvD,WAAO,gCAAgC,SAAS,MAAM;AAAA,EACxD;AACA,QAAM,IAAI,MAAM,4CAA4C,OAAO,IAAI,EAAE;AAC3E;AAEA,eAAsB,sCACpB,WACoC;AACpC,QAAM,OAAkC,CAAC;AACzC,aAAW,UAAU,WAAW;AAC9B,UAAM,UAAU,SAAS,KAAK,MAAM,MAAM,SAAS,OAAO,MAAM,OAAO,CAAC,CAAC;AACzE,QAAI,CAAC;AACH,YAAM,IAAI,MAAM,YAAY,OAAO,IAAI,wBAAwB;AACjE,SAAK,KAAK,GAAG,4CAA4C,SAAS,MAAM,CAAC;AAAA,EAC3E;AACA,SAAO;AACT;AAEO,SAAS,oCACd,OACyB;AACzB,QAAM,OAAO,MAAM,KAAK,IAAI,YAAY;AACxC,QAAM,mBAAmB,uBAAuB,MAAM,MAAM,gBAAgB;AAC5E,QAAM,QAAQ,MAAM;AAAA,IAClB,IAAI;AAAA,MACF,KAAK,IAAI,CAAC,QAAQ,IAAI,IAAI,EAAE,OAAO,CAAC,SAAyB,CAAC,CAAC,IAAI;AAAA,IACrE;AAAA,EACF,EAAE,KAAK,yBAAyB;AAChC,QAAM,aAAa,MAAM;AAAA,IACvB,IAAI,IAAI,KAAK,IAAI,CAAC,QAAQ,IAAI,SAAS,CAAC;AAAA,EAC1C,EAAE,KAAK;AACP,QAAM,cAAc,iBAAiB,MAAM,gBAAgB;AAC3D,SAAO;AAAA,IACL,QAAQ;AAAA,IACR,SAAS;AAAA,IACT,aAAa,MAAM,gBAAe,oBAAI,KAAK,GAAE,YAAY;AAAA,IACzD,QAAQ,MAAM,UAAU,EAAE,MAAM,4BAA4B;AAAA,IAC5D;AAAA,IACA;AAAA,IACA;AAAA,IACA,QAAQ;AAAA,MACN,MAAM,KAAK;AAAA,MACX,aAAa,YAAY;AAAA,MACzB,OAAO,MAAM;AAAA,MACb,YAAY,WAAW;AAAA,IACzB;AAAA,IACA;AAAA,IACA;AAAA,EACF;AACF;AAEA,eAAsB,6BACpB,OACwC;AACxC,QAAM,WAAW,oCAAoC,KAAK;AAC1D,QAAM,YACJ,MAAM,aACN;AAAA,IACE,kBAAkB;AAAA,IAClB;AAAA,IACA,cAAc,SAAS,WAAW;AAAA,EACpC;AACF,QAAM,MAAM,WAAW,EAAE,WAAW,KAAK,CAAC;AAC1C,QAAM,eAAe,KAAK,WAAW,uBAAuB;AAC5D,QAAM;AAAA,IACJ;AAAA,IACA,GAAG,KAAK,UAAU,UAAU,MAAM,CAAC,CAAC;AAAA;AAAA,IACpC;AAAA,EACF;AACA,SAAO,EAAE,WAAW,cAAc,SAAS;AAC7C;AAEA,eAAsB,0CACpB,OACwC;AACxC,QAAM,OAAO,MAAM,sCAAsC,MAAM,SAAS;AACxE,SAAO,6BAA6B;AAAA,IAClC;AAAA,IACA,WAAW,MAAM;AAAA,IACjB,aAAa,MAAM;AAAA,IACnB,kBAAkB,MAAM;AAAA,IACxB,QAAQ,MAAM,UAAU;AAAA,MACtB,MAAM;AAAA,MACN,WAAW,MAAM,UAAU,IAAI,CAAC,aAAa,SAAS,IAAI;AAAA,IAC5D;AAAA,EACF,CAAC;AACH;","names":[]}

package/dist/core/benchmark-vs-cerebras-runner.d.ts

@@ -0,0 +1,37 @@
+export type BenchmarkVsCerebrasBenchmark = "eliza_harness_action_selection" | "clawbench" | "hermes" | "all";
+export interface BenchmarkVsCerebrasRunOptions {
+    trainingRoot?: string;
+    python?: string;
+    tiers?: string;
+    benchmark?: BenchmarkVsCerebrasBenchmark;
+    variants?: "trained" | "base" | "both";
+    cerebrasModel?: string;
+    maxSamples?: number;
+    outputDir?: string;
+    checkpointsDir?: string;
+    trainedModelPath?: string;
+    dryRun?: boolean;
+    resultsDb?: string;
+    datasetVersion?: string;
+    codeCommit?: string;
+    matrixOutputDir?: string;
+}
+export interface BenchmarkVsCerebrasRunResult {
+    trainingRoot: string;
+    outputDir: string;
+    matrixOutputDir: string | null;
+    matrixArtifactPath: string | null;
+    resultsDb: string | null;
+    command: string[];
+    stdout: string;
+    stderr: string;
+    exitCode: number;
+}
+export declare function benchmarkVsCerebrasTierList(value: string | undefined): string;
+export declare function buildBenchmarkVsCerebrasArgs(options: BenchmarkVsCerebrasRunOptions, resolved: {
+    trainingRoot: string;
+    outputDir: string;
+    matrixOutputDir?: string;
+}): string[];
+export declare function runBenchmarkVsCerebras(options: BenchmarkVsCerebrasRunOptions): Promise<BenchmarkVsCerebrasRunResult>;
+//# sourceMappingURL=benchmark-vs-cerebras-runner.d.ts.map

package/dist/core/benchmark-vs-cerebras-runner.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"benchmark-vs-cerebras-runner.d.ts","sourceRoot":"","sources":["../../src/core/benchmark-vs-cerebras-runner.ts"],"names":[],"mappings":"AAMA,MAAM,MAAM,4BAA4B,GACpC,gCAAgC,GAChC,WAAW,GACX,QAAQ,GACR,KAAK,CAAC;AAEV,MAAM,WAAW,6BAA6B;IAC5C,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,SAAS,CAAC,EAAE,4BAA4B,CAAC;IACzC,QAAQ,CAAC,EAAE,SAAS,GAAG,MAAM,GAAG,MAAM,CAAC;IACvC,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B,MAAM,CAAC,EAAE,OAAO,CAAC;IACjB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,eAAe,CAAC,EAAE,MAAM,CAAC;CAC1B;AAED,MAAM,WAAW,4BAA4B;IAC3C,YAAY,EAAE,MAAM,CAAC;IACrB,SAAS,EAAE,MAAM,CAAC;IAClB,eAAe,EAAE,MAAM,GAAG,IAAI,CAAC;IAC/B,kBAAkB,EAAE,MAAM,GAAG,IAAI,CAAC;IAClC,SAAS,EAAE,MAAM,GAAG,IAAI,CAAC;IACzB,OAAO,EAAE,MAAM,EAAE,CAAC;IAClB,MAAM,EAAE,MAAM,CAAC;IACf,MAAM,EAAE,MAAM,CAAC;IACf,QAAQ,EAAE,MAAM,CAAC;CAClB;AAkDD,wBAAgB,2BAA2B,CAAC,KAAK,EAAE,MAAM,GAAG,SAAS,GAAG,MAAM,CAI7E;AA6BD,wBAAgB,4BAA4B,CAC1C,OAAO,EAAE,6BAA6B,EACtC,QAAQ,EAAE;IACR,YAAY,EAAE,MAAM,CAAC;IACrB,SAAS,EAAE,MAAM,CAAC;IAClB,eAAe,CAAC,EAAE,MAAM,CAAC;CAC1B,GACA,MAAM,EAAE,CAsCV;AAED,wBAAsB,sBAAsB,CAC1C,OAAO,EAAE,6BAA6B,GACrC,OAAO,CAAC,4BAA4B,CAAC,CAsCvC"}

package/dist/core/benchmark-vs-cerebras-runner.js

@@ -0,0 +1,151 @@
+import { spawn } from "node:child_process";
+import { mkdir } from "node:fs/promises";
+import { join, resolve } from "node:path";
+import { ELIZA_ONE_BENCHMARK_TIER_LIST } from "./eliza1-benchmark-recipe.js";
+import { trainingStateRoot } from "./training-config.js";
+function safeTimestamp(value) {
+  return value.replace(/[:.]/g, "-");
+}
+const TRAINING_TIER_KEYS = {
+  "2b": "gemma4-e2b",
+  "4b": "gemma4-e4b",
+  "9b": "gemma4-12b",
+  "27b": "gemma4-31b",
+  "eliza-1-2b": "gemma4-e2b",
+  "eliza-1-4b": "gemma4-e4b",
+  "eliza-1-9b": "gemma4-12b",
+  "eliza-1-27b": "gemma4-31b",
+  "gemma4-e2b": "gemma4-e2b",
+  "gemma4-e4b": "gemma4-e4b",
+  "gemma4-12b": "gemma4-12b",
+  "gemma4-31b": "gemma4-31b",
+  "gemma-4-e2b": "gemma4-e2b",
+  "gemma-4-e4b": "gemma4-e4b",
+  "gemma-4-12b": "gemma4-12b",
+  "gemma-4-31b": "gemma4-31b",
+  "google/gemma-4-e2b": "gemma4-e2b",
+  "google/gemma-4-e4b": "gemma4-e4b",
+  "google/gemma-4-12b": "gemma4-12b",
+  "google/gemma-4-31b": "gemma4-31b",
+  "google-gemma-4-e2b": "gemma4-e2b",
+  "google-gemma-4-e4b": "gemma4-e4b",
+  "google-gemma-4-12b": "gemma4-12b",
+  "google-gemma-4-31b": "gemma4-31b"
+};
+const RETIRED_QWEN_TIER_ALIAS_RE = /\bqwen(?:\d+(?:\.\d+)?)?\b/i;
+function normalizeTrainingTierKey(value) {
+  const trimmed = value.trim();
+  const key = trimmed.toLowerCase().replace(/_/g, "-");
+  if (RETIRED_QWEN_TIER_ALIAS_RE.test(key)) {
+    throw new Error(
+      `Qwen tier aliases are retired; use an active Gemma 4 tier key instead (${ELIZA_ONE_BENCHMARK_TIER_LIST}).`
+    );
+  }
+  return TRAINING_TIER_KEYS[key] ?? TRAINING_TIER_KEYS[key.replace(/\//g, "-")] ?? trimmed;
+}
+function benchmarkVsCerebrasTierList(value) {
+  const raw = value?.trim() || ELIZA_ONE_BENCHMARK_TIER_LIST;
+  if (raw.toLowerCase() === "all") return "all";
+  return raw.split(",").map(normalizeTrainingTierKey).filter(Boolean).join(",");
+}
+function collectProcess(command, args, cwd) {
+  return new Promise((resolvePromise, reject) => {
+    const child = spawn(command, args, {
+      cwd,
+      stdio: ["ignore", "pipe", "pipe"]
+    });
+    let stdout = "";
+    let stderr = "";
+    child.stdout.setEncoding("utf-8");
+    child.stderr.setEncoding("utf-8");
+    child.stdout.on("data", (chunk) => {
+      stdout += chunk;
+    });
+    child.stderr.on("data", (chunk) => {
+      stderr += chunk;
+    });
+    child.on("error", reject);
+    child.on("close", (code) => {
+      resolvePromise({ stdout, stderr, exitCode: code ?? 1 });
+    });
+  });
+}
+function buildBenchmarkVsCerebrasArgs(options, resolved) {
+  const scriptPath = join(
+    resolved.trainingRoot,
+    "scripts",
+    "benchmark_vs_cerebras.py"
+  );
+  const args = [
+    scriptPath,
+    "--tiers",
+    benchmarkVsCerebrasTierList(options.tiers),
+    "--benchmark",
+    options.benchmark ?? "eliza_harness_action_selection",
+    "--variants",
+    options.variants ?? "trained",
+    "--cerebras-model",
+    options.cerebrasModel ?? "gpt-oss-120b",
+    "--max-samples",
+    String(
+      typeof options.maxSamples === "number" ? Math.max(1, Math.floor(options.maxSamples)) : 50
+    ),
+    "--output-dir",
+    resolved.outputDir
+  ];
+  if (options.checkpointsDir)
+    args.push("--checkpoints-dir", options.checkpointsDir);
+  if (options.trainedModelPath)
+    args.push("--trained-model-path", options.trainedModelPath);
+  if (options.dryRun) args.push("--dry-run");
+  if (options.resultsDb) args.push("--results-db", options.resultsDb);
+  if (options.datasetVersion)
+    args.push("--dataset-version", options.datasetVersion);
+  if (options.codeCommit) args.push("--code-commit", options.codeCommit);
+  if (resolved.matrixOutputDir) {
+    args.push("--matrix-output-dir", resolved.matrixOutputDir);
+  }
+  return args;
+}
+async function runBenchmarkVsCerebras(options) {
+  const trainingRoot = resolve(
+    options.trainingRoot ?? join(process.cwd(), "packages", "training")
+  );
+  const stamp = safeTimestamp((/* @__PURE__ */ new Date()).toISOString());
+  const outputDir = options.outputDir ?? join(trainingStateRoot(), "benchmarks", "runs", stamp);
+  const matrixOutputDir = options.matrixOutputDir ?? join(trainingStateRoot(), "benchmarks", "matrices", stamp);
+  await mkdir(outputDir, { recursive: true });
+  await mkdir(matrixOutputDir, { recursive: true });
+  const args = buildBenchmarkVsCerebrasArgs(options, {
+    trainingRoot,
+    outputDir,
+    matrixOutputDir
+  });
+  const proc = await collectProcess(
+    options.python ?? "python3",
+    args,
+    trainingRoot
+  );
+  if (proc.exitCode !== 0) {
+    throw new Error(
+      `benchmark_vs_cerebras.py exited with code ${proc.exitCode}: ${proc.stderr || proc.stdout}`
+    );
+  }
+  return {
+    trainingRoot,
+    outputDir,
+    matrixOutputDir,
+    matrixArtifactPath: join(matrixOutputDir, "benchmark-matrix.json"),
+    resultsDb: options.resultsDb ?? null,
+    command: [options.python ?? "python3", ...args],
+    stdout: proc.stdout,
+    stderr: proc.stderr,
+    exitCode: proc.exitCode
+  };
+}
+export {
+  benchmarkVsCerebrasTierList,
+  buildBenchmarkVsCerebrasArgs,
+  runBenchmarkVsCerebras
+};
+//# sourceMappingURL=benchmark-vs-cerebras-runner.js.map

package/dist/core/benchmark-vs-cerebras-runner.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../../src/core/benchmark-vs-cerebras-runner.ts"],"sourcesContent":["import { spawn } from \"node:child_process\";\nimport { mkdir } from \"node:fs/promises\";\nimport { join, resolve } from \"node:path\";\nimport { ELIZA_ONE_BENCHMARK_TIER_LIST } from \"./eliza1-benchmark-recipe.js\";\nimport { trainingStateRoot } from \"./training-config.js\";\n\nexport type BenchmarkVsCerebrasBenchmark =\n  | \"eliza_harness_action_selection\"\n  | \"clawbench\"\n  | \"hermes\"\n  | \"all\";\n\nexport interface BenchmarkVsCerebrasRunOptions {\n  trainingRoot?: string;\n  python?: string;\n  tiers?: string;\n  benchmark?: BenchmarkVsCerebrasBenchmark;\n  variants?: \"trained\" | \"base\" | \"both\";\n  cerebrasModel?: string;\n  maxSamples?: number;\n  outputDir?: string;\n  checkpointsDir?: string;\n  trainedModelPath?: string;\n  dryRun?: boolean;\n  resultsDb?: string;\n  datasetVersion?: string;\n  codeCommit?: string;\n  matrixOutputDir?: string;\n}\n\nexport interface BenchmarkVsCerebrasRunResult {\n  trainingRoot: string;\n  outputDir: string;\n  matrixOutputDir: string | null;\n  matrixArtifactPath: string | null;\n  resultsDb: string | null;\n  command: string[];\n  stdout: string;\n  stderr: string;\n  exitCode: number;\n}\n\nfunction safeTimestamp(value: string): string {\n  return value.replace(/[:.]/g, \"-\");\n}\n\nconst TRAINING_TIER_KEYS: Record<string, string> = {\n  \"2b\": \"gemma4-e2b\",\n  \"4b\": \"gemma4-e4b\",\n  \"9b\": \"gemma4-12b\",\n  \"27b\": \"gemma4-31b\",\n  \"eliza-1-2b\": \"gemma4-e2b\",\n  \"eliza-1-4b\": \"gemma4-e4b\",\n  \"eliza-1-9b\": \"gemma4-12b\",\n  \"eliza-1-27b\": \"gemma4-31b\",\n  \"gemma4-e2b\": \"gemma4-e2b\",\n  \"gemma4-e4b\": \"gemma4-e4b\",\n  \"gemma4-12b\": \"gemma4-12b\",\n  \"gemma4-31b\": \"gemma4-31b\",\n  \"gemma-4-e2b\": \"gemma4-e2b\",\n  \"gemma-4-e4b\": \"gemma4-e4b\",\n  \"gemma-4-12b\": \"gemma4-12b\",\n  \"gemma-4-31b\": \"gemma4-31b\",\n  \"google/gemma-4-e2b\": \"gemma4-e2b\",\n  \"google/gemma-4-e4b\": \"gemma4-e4b\",\n  \"google/gemma-4-12b\": \"gemma4-12b\",\n  \"google/gemma-4-31b\": \"gemma4-31b\",\n  \"google-gemma-4-e2b\": \"gemma4-e2b\",\n  \"google-gemma-4-e4b\": \"gemma4-e4b\",\n  \"google-gemma-4-12b\": \"gemma4-12b\",\n  \"google-gemma-4-31b\": \"gemma4-31b\",\n};\n\nconst RETIRED_QWEN_TIER_ALIAS_RE = /\\bqwen(?:\\d+(?:\\.\\d+)?)?\\b/i;\n\nfunction normalizeTrainingTierKey(value: string): string {\n  const trimmed = value.trim();\n  const key = trimmed.toLowerCase().replace(/_/g, \"-\");\n  if (RETIRED_QWEN_TIER_ALIAS_RE.test(key)) {\n    throw new Error(\n      `Qwen tier aliases are retired; use an active Gemma 4 tier key instead (${ELIZA_ONE_BENCHMARK_TIER_LIST}).`,\n    );\n  }\n  return (\n    TRAINING_TIER_KEYS[key] ??\n    TRAINING_TIER_KEYS[key.replace(/\\//g, \"-\")] ??\n    trimmed\n  );\n}\n\nexport function benchmarkVsCerebrasTierList(value: string | undefined): string {\n  const raw = value?.trim() || ELIZA_ONE_BENCHMARK_TIER_LIST;\n  if (raw.toLowerCase() === \"all\") return \"all\";\n  return raw.split(\",\").map(normalizeTrainingTierKey).filter(Boolean).join(\",\");\n}\n\nfunction collectProcess(\n  command: string,\n  args: string[],\n  cwd: string,\n): Promise<{ stdout: string; stderr: string; exitCode: number }> {\n  return new Promise((resolvePromise, reject) => {\n    const child = spawn(command, args, {\n      cwd,\n      stdio: [\"ignore\", \"pipe\", \"pipe\"],\n    });\n    let stdout = \"\";\n    let stderr = \"\";\n    child.stdout.setEncoding(\"utf-8\");\n    child.stderr.setEncoding(\"utf-8\");\n    child.stdout.on(\"data\", (chunk) => {\n      stdout += chunk;\n    });\n    child.stderr.on(\"data\", (chunk) => {\n      stderr += chunk;\n    });\n    child.on(\"error\", reject);\n    child.on(\"close\", (code) => {\n      resolvePromise({ stdout, stderr, exitCode: code ?? 1 });\n    });\n  });\n}\n\nexport function buildBenchmarkVsCerebrasArgs(\n  options: BenchmarkVsCerebrasRunOptions,\n  resolved: {\n    trainingRoot: string;\n    outputDir: string;\n    matrixOutputDir?: string;\n  },\n): string[] {\n  const scriptPath = join(\n    resolved.trainingRoot,\n    \"scripts\",\n    \"benchmark_vs_cerebras.py\",\n  );\n  const args = [\n    scriptPath,\n    \"--tiers\",\n    benchmarkVsCerebrasTierList(options.tiers),\n    \"--benchmark\",\n    options.benchmark ?? \"eliza_harness_action_selection\",\n    \"--variants\",\n    options.variants ?? \"trained\",\n    \"--cerebras-model\",\n    options.cerebrasModel ?? \"gpt-oss-120b\",\n    \"--max-samples\",\n    String(\n      typeof options.maxSamples === \"number\"\n        ? Math.max(1, Math.floor(options.maxSamples))\n        : 50,\n    ),\n    \"--output-dir\",\n    resolved.outputDir,\n  ];\n  if (options.checkpointsDir)\n    args.push(\"--checkpoints-dir\", options.checkpointsDir);\n  if (options.trainedModelPath)\n    args.push(\"--trained-model-path\", options.trainedModelPath);\n  if (options.dryRun) args.push(\"--dry-run\");\n  if (options.resultsDb) args.push(\"--results-db\", options.resultsDb);\n  if (options.datasetVersion)\n    args.push(\"--dataset-version\", options.datasetVersion);\n  if (options.codeCommit) args.push(\"--code-commit\", options.codeCommit);\n  if (resolved.matrixOutputDir) {\n    args.push(\"--matrix-output-dir\", resolved.matrixOutputDir);\n  }\n  return args;\n}\n\nexport async function runBenchmarkVsCerebras(\n  options: BenchmarkVsCerebrasRunOptions,\n): Promise<BenchmarkVsCerebrasRunResult> {\n  const trainingRoot = resolve(\n    options.trainingRoot ?? join(process.cwd(), \"packages\", \"training\"),\n  );\n  const stamp = safeTimestamp(new Date().toISOString());\n  const outputDir =\n    options.outputDir ?? join(trainingStateRoot(), \"benchmarks\", \"runs\", stamp);\n  const matrixOutputDir =\n    options.matrixOutputDir ??\n    join(trainingStateRoot(), \"benchmarks\", \"matrices\", stamp);\n  await mkdir(outputDir, { recursive: true });\n  await mkdir(matrixOutputDir, { recursive: true });\n  const args = buildBenchmarkVsCerebrasArgs(options, {\n    trainingRoot,\n    outputDir,\n    matrixOutputDir,\n  });\n  const proc = await collectProcess(\n    options.python ?? \"python3\",\n    args,\n    trainingRoot,\n  );\n  if (proc.exitCode !== 0) {\n    throw new Error(\n      `benchmark_vs_cerebras.py exited with code ${proc.exitCode}: ${proc.stderr || proc.stdout}`,\n    );\n  }\n  return {\n    trainingRoot,\n    outputDir,\n    matrixOutputDir,\n    matrixArtifactPath: join(matrixOutputDir, \"benchmark-matrix.json\"),\n    resultsDb: options.resultsDb ?? null,\n    command: [options.python ?? \"python3\", ...args],\n    stdout: proc.stdout,\n    stderr: proc.stderr,\n    exitCode: proc.exitCode,\n  };\n}\n"],"mappings":"AAAA,SAAS,aAAa;AACtB,SAAS,aAAa;AACtB,SAAS,MAAM,eAAe;AAC9B,SAAS,qCAAqC;AAC9C,SAAS,yBAAyB;AAsClC,SAAS,cAAc,OAAuB;AAC5C,SAAO,MAAM,QAAQ,SAAS,GAAG;AACnC;AAEA,MAAM,qBAA6C;AAAA,EACjD,MAAM;AAAA,EACN,MAAM;AAAA,EACN,MAAM;AAAA,EACN,OAAO;AAAA,EACP,cAAc;AAAA,EACd,cAAc;AAAA,EACd,cAAc;AAAA,EACd,eAAe;AAAA,EACf,cAAc;AAAA,EACd,cAAc;AAAA,EACd,cAAc;AAAA,EACd,cAAc;AAAA,EACd,eAAe;AAAA,EACf,eAAe;AAAA,EACf,eAAe;AAAA,EACf,eAAe;AAAA,EACf,sBAAsB;AAAA,EACtB,sBAAsB;AAAA,EACtB,sBAAsB;AAAA,EACtB,sBAAsB;AAAA,EACtB,sBAAsB;AAAA,EACtB,sBAAsB;AAAA,EACtB,sBAAsB;AAAA,EACtB,sBAAsB;AACxB;AAEA,MAAM,6BAA6B;AAEnC,SAAS,yBAAyB,OAAuB;AACvD,QAAM,UAAU,MAAM,KAAK;AAC3B,QAAM,MAAM,QAAQ,YAAY,EAAE,QAAQ,MAAM,GAAG;AACnD,MAAI,2BAA2B,KAAK,GAAG,GAAG;AACxC,UAAM,IAAI;AAAA,MACR,0EAA0E,6BAA6B;AAAA,IACzG;AAAA,EACF;AACA,SACE,mBAAmB,GAAG,KACtB,mBAAmB,IAAI,QAAQ,OAAO,GAAG,CAAC,KAC1C;AAEJ;AAEO,SAAS,4BAA4B,OAAmC;AAC7E,QAAM,MAAM,OAAO,KAAK,KAAK;AAC7B,MAAI,IAAI,YAAY,MAAM,MAAO,QAAO;AACxC,SAAO,IAAI,MAAM,GAAG,EAAE,IAAI,wBAAwB,EAAE,OAAO,OAAO,EAAE,KAAK,GAAG;AAC9E;AAEA,SAAS,eACP,SACA,MACA,KAC+D;AAC/D,SAAO,IAAI,QAAQ,CAAC,gBAAgB,WAAW;AAC7C,UAAM,QAAQ,MAAM,SAAS,MAAM;AAAA,MACjC;AAAA,MACA,OAAO,CAAC,UAAU,QAAQ,MAAM;AAAA,IAClC,CAAC;AACD,QAAI,SAAS;AACb,QAAI,SAAS;AACb,UAAM,OAAO,YAAY,OAAO;AAChC,UAAM,OAAO,YAAY,OAAO;AAChC,UAAM,OAAO,GAAG,QAAQ,CAAC,UAAU;AACjC,gBAAU;AAAA,IACZ,CAAC;AACD,UAAM,OAAO,GAAG,QAAQ,CAAC,UAAU;AACjC,gBAAU;AAAA,IACZ,CAAC;AACD,UAAM,GAAG,SAAS,MAAM;AACxB,UAAM,GAAG,SAAS,CAAC,SAAS;AAC1B,qBAAe,EAAE,QAAQ,QAAQ,UAAU,QAAQ,EAAE,CAAC;AAAA,IACxD,CAAC;AAAA,EACH,CAAC;AACH;AAEO,SAAS,6BACd,SACA,UAKU;AACV,QAAM,aAAa;AAAA,IACjB,SAAS;AAAA,IACT;AAAA,IACA;AAAA,EACF;AACA,QAAM,OAAO;AAAA,IACX;AAAA,IACA;AAAA,IACA,4BAA4B,QAAQ,KAAK;AAAA,IACzC;AAAA,IACA,QAAQ,aAAa;AAAA,IACrB;AAAA,IACA,QAAQ,YAAY;AAAA,IACpB;AAAA,IACA,QAAQ,iBAAiB;AAAA,IACzB;AAAA,IACA;AAAA,MACE,OAAO,QAAQ,eAAe,WAC1B,KAAK,IAAI,GAAG,KAAK,MAAM,QAAQ,UAAU,CAAC,IAC1C;AAAA,IACN;AAAA,IACA;AAAA,IACA,SAAS;AAAA,EACX;AACA,MAAI,QAAQ;AACV,SAAK,KAAK,qBAAqB,QAAQ,cAAc;AACvD,MAAI,QAAQ;AACV,SAAK,KAAK,wBAAwB,QAAQ,gBAAgB;AAC5D,MAAI,QAAQ,OAAQ,MAAK,KAAK,WAAW;AACzC,MAAI,QAAQ,UAAW,MAAK,KAAK,gBAAgB,QAAQ,SAAS;AAClE,MAAI,QAAQ;AACV,SAAK,KAAK,qBAAqB,QAAQ,cAAc;AACvD,MAAI,QAAQ,WAAY,MAAK,KAAK,iBAAiB,QAAQ,UAAU;AACrE,MAAI,SAAS,iBAAiB;AAC5B,SAAK,KAAK,uBAAuB,SAAS,eAAe;AAAA,EAC3D;AACA,SAAO;AACT;AAEA,eAAsB,uBACpB,SACuC;AACvC,QAAM,eAAe;AAAA,IACnB,QAAQ,gBAAgB,KAAK,QAAQ,IAAI,GAAG,YAAY,UAAU;AAAA,EACpE;AACA,QAAM,QAAQ,eAAc,oBAAI,KAAK,GAAE,YAAY,CAAC;AACpD,QAAM,YACJ,QAAQ,aAAa,KAAK,kBAAkB,GAAG,cAAc,QAAQ,KAAK;AAC5E,QAAM,kBACJ,QAAQ,mBACR,KAAK,kBAAkB,GAAG,cAAc,YAAY,KAAK;AAC3D,QAAM,MAAM,WAAW,EAAE,WAAW,KAAK,CAAC;AAC1C,QAAM,MAAM,iBAAiB,EAAE,WAAW,KAAK,CAAC;AAChD,QAAM,OAAO,6BAA6B,SAAS;AAAA,IACjD;AAAA,IACA;AAAA,IACA;AAAA,EACF,CAAC;AACD,QAAM,OAAO,MAAM;AAAA,IACjB,QAAQ,UAAU;AAAA,IAClB;AAAA,IACA;AAAA,EACF;AACA,MAAI,KAAK,aAAa,GAAG;AACvB,UAAM,IAAI;AAAA,MACR,6CAA6C,KAAK,QAAQ,KAAK,KAAK,UAAU,KAAK,MAAM;AAAA,IAC3F;AAAA,EACF;AACA,SAAO;AAAA,IACL;AAAA,IACA;AAAA,IACA;AAAA,IACA,oBAAoB,KAAK,iBAAiB,uBAAuB;AAAA,IACjE,WAAW,QAAQ,aAAa;AAAA,IAChC,SAAS,CAAC,QAAQ,UAAU,WAAW,GAAG,IAAI;AAAA,IAC9C,QAAQ,KAAK;AAAA,IACb,QAAQ,KAAK;AAAA,IACb,UAAU,KAAK;AAAA,EACjB;AACF;","names":[]}