@elizaos/plugin-training 2.0.3-beta.5 → 2.0.3-beta.7

package/dist/core/action-benchmark-runner.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../../src/core/action-benchmark-runner.ts"],"sourcesContent":["import { spawn } from \"node:child_process\";\nimport { mkdir, readFile, writeFile } from \"node:fs/promises\";\nimport { join } from \"node:path\";\nimport { trainingStateRoot } from \"./training-config.js\";\nimport { defaultBunCommand, resolveWorkspaceRoot } from \"./workspace-runtime.js\";\n\nexport type ActionBenchmarkMatrixVariant = \"reference\" | \"base\" | \"trained\";\n\nexport interface ActionBenchmarkRunOptions {\n  workspaceRoot?: string;\n  bun?: string;\n  outputDir?: string;\n  useMocks?: boolean;\n  forceTrajectoryCapture?: boolean;\n  filter?: string;\n  runsPerCase?: number;\n  provider?: string;\n  modelId?: string;\n  runtimeModel?: string;\n  smallModel?: string;\n  largeModel?: string;\n  baseUrl?: string;\n  variant?: ActionBenchmarkMatrixVariant;\n  tier?: string;\n  benchmark?: string;\n  datasetVersion?: string;\n  codeCommit?: string;\n  dryRun?: boolean;\n}\n\nexport interface ActionBenchmarkRunResult {\n  workspaceRoot: string;\n  appCoreRoot: string;\n  outputDir: string;\n  reportMarkdownPath: string;\n  reportJsonPath: string;\n  trajectoryDir: string;\n  command: string[];\n  env: Record<string, string>;\n  stdout: string;\n  stderr: string;\n  exitCode: number;\n  matrixSource: {\n    path: string;\n    modelId?: string;\n    benchmark?: string;\n    variant?: ActionBenchmarkMatrixVariant;\n    tier?: string;\n    provider?: string;\n    datasetVersion?: string;\n    codeCommit?: string;\n    useMocks?: boolean;\n  } | null;\n}\n\nfunction safeTimestamp(value: string): string {\n  return value.replace(/[:.]/g, \"-\");\n}\n\nfunction positiveInt(value: number | undefined): number | undefined {\n  return typeof value === \"number\" && Number.isFinite(value)\n    ? Math.max(1, Math.floor(value))\n    : undefined;\n}\n\nfunction collectProcess(\n  command: string,\n  args: string[],\n  cwd: string,\n  env: NodeJS.ProcessEnv,\n): Promise<{ stdout: string; stderr: string; exitCode: number }> {\n  return new Promise((resolvePromise, reject) => {\n    const child = spawn(command, args, {\n      cwd,\n      env,\n      stdio: [\"ignore\", \"pipe\", \"pipe\"],\n    });\n    let stdout = \"\";\n    let stderr = \"\";\n    child.stdout.setEncoding(\"utf-8\");\n    child.stderr.setEncoding(\"utf-8\");\n    child.stdout.on(\"data\", (chunk) => {\n      stdout += chunk;\n    });\n    child.stderr.on(\"data\", (chunk) => {\n      stderr += chunk;\n    });\n    child.on(\"error\", reject);\n    child.on(\"close\", (code) => {\n      resolvePromise({ stdout, stderr, exitCode: code ?? 1 });\n    });\n  });\n}\n\nfunction stringSetting(value: string | undefined): string | undefined {\n  return value?.trim() || undefined;\n}\n\nfunction modelListUrl(baseUrl: string): string {\n  const normalized = baseUrl.trim().replace(/\\/+$/, \"\");\n  return `${normalized}/models`;\n}\n\nfunction localModelIdMatches(availableId: string, requestedId: string): boolean {\n  return (\n    availableId === requestedId ||\n    availableId === `${requestedId}:latest` ||\n    `${availableId}:latest` === requestedId\n  );\n}\n\nasync function localModelIds(baseUrl: string): Promise<string[]> {\n  const response = await fetch(modelListUrl(baseUrl));\n  if (!response.ok) {\n    throw new Error(\n      `local model endpoint ${modelListUrl(baseUrl)} returned ${response.status} ${response.statusText}`,\n    );\n  }\n  const payload = await response.json();\n  const data =\n    payload &&\n    typeof payload === \"object\" &&\n    Array.isArray((payload as { data?: unknown }).data)\n      ? (payload as { data: unknown[] }).data\n      : [];\n  return data\n    .map((item) =>\n      item && typeof item === \"object\"\n        ? (item as { id?: unknown; name?: unknown }).id ??\n          (item as { id?: unknown; name?: unknown }).name\n        : item,\n    )\n    .filter((id): id is string => typeof id === \"string\" && id.length > 0);\n}\n\nexport async function assertLocalBenchmarkModelAvailable(\n  options: ActionBenchmarkRunOptions,\n): Promise<void> {\n  if (effectiveUseMocks(options)) return;\n  if (options.provider !== \"local-llama-cpp\") return;\n  const requestedModel = stringSetting(options.runtimeModel);\n  if (!requestedModel) return;\n  const baseUrl = stringSetting(options.baseUrl) ?? \"http://localhost:11434/v1\";\n  const ids = await localModelIds(baseUrl);\n  if (ids.some((id) => localModelIdMatches(id, requestedModel))) return;\n  throw new Error(\n    `local action benchmark model \"${requestedModel}\" is not available at ${modelListUrl(\n      baseUrl,\n    )}; available models: ${ids.length > 0 ? ids.join(\", \") : \"none\"}`,\n  );\n}\n\nfunction effectiveUseMocks(options: ActionBenchmarkRunOptions): boolean {\n  return options.useMocks ?? options.dryRun === true;\n}\n\nfunction matrixSourceForReport(\n  reportJsonPath: string,\n  options: ActionBenchmarkRunOptions,\n): ActionBenchmarkRunResult[\"matrixSource\"] {\n  const modelId =\n    stringSetting(options.modelId) ?? stringSetting(options.provider);\n  const variant = options.variant;\n  if (!modelId || !variant) return null;\n  return {\n    path: reportJsonPath,\n    modelId,\n    variant,\n    benchmark: stringSetting(options.benchmark),\n    tier: stringSetting(options.tier),\n    provider: stringSetting(options.provider),\n    datasetVersion: stringSetting(options.datasetVersion),\n    codeCommit: stringSetting(options.codeCommit),\n    useMocks: effectiveUseMocks(options),\n  };\n}\n\nfunction dryRunCaseSample(options: ActionBenchmarkRunOptions, trajectoryDir: string) {\n  const tier = stringSetting(options.tier) ?? \"2b\";\n  const variant = stringSetting(options.variant) ?? \"trained\";\n  const modelId = stringSetting(options.modelId) ?? stringSetting(options.runtimeModel);\n  return {\n    caseId: `dry-run-${tier}-${variant}-action-selection`,\n    prompt: \"Can you check my calendar?\",\n    expectedAction: \"CHECK_RUNTIME\",\n    actualAction: null,\n    pass: false,\n    response: \"Dry-run benchmark provenance sample; no model inference executed.\",\n    latencyMs: 0,\n    trajectoryPath: join(\n      trajectoryDir,\n      `dry-run-${tier}-${variant}-action-selection.json`,\n    ),\n    dryRun: true,\n    modelId,\n    tier,\n    variant,\n  };\n}\n\nasync function annotateBenchmarkReportSource(\n  reportJsonPath: string,\n  options: ActionBenchmarkRunOptions,\n): Promise<void> {\n  const matrixSource = matrixSourceForReport(reportJsonPath, options);\n  if (!matrixSource) return;\n  const parsed = JSON.parse(await readFile(reportJsonPath, \"utf8\"));\n  if (!parsed || typeof parsed !== \"object\" || Array.isArray(parsed)) return;\n  const report = parsed as Record<string, unknown>;\n  const existingSource =\n    report.source &&\n    typeof report.source === \"object\" &&\n    !Array.isArray(report.source)\n      ? (report.source as Record<string, unknown>)\n      : {};\n  report.source = {\n    ...existingSource,\n    modelId: matrixSource.modelId,\n    variant: matrixSource.variant,\n    benchmark: matrixSource.benchmark,\n    tier: matrixSource.tier,\n    provider: matrixSource.provider,\n    datasetVersion: matrixSource.datasetVersion,\n    codeCommit: matrixSource.codeCommit,\n    useMocks: matrixSource.useMocks,\n  };\n  await writeFile(\n    reportJsonPath,\n    `${JSON.stringify(report, null, 2)}\\n`,\n    \"utf8\",\n  );\n}\n\nexport function buildActionBenchmarkCommand(): string[] {\n  return [\n    \"x\",\n    \"vitest\",\n    \"run\",\n    \"--config\",\n    \"../test/vitest/real.config.ts\",\n    \"test/benchmarks/action-selection.real.test.ts\",\n    \"--exclude\",\n    \".git/**\",\n    \"--exclude\",\n    \".eliza/**\",\n  ];\n}\n\nexport function buildActionBenchmarkEnv(\n  options: ActionBenchmarkRunOptions,\n  resolved: {\n    reportMarkdownPath: string;\n    reportJsonPath: string;\n    trajectoryDir: string;\n  },\n): Record<string, string> {\n  const env: Record<string, string> = {\n    ELIZA_RUN_ACTION_BENCHMARK: \"1\",\n    ELIZA_ACTION_BENCHMARK_REPORT_PATH: resolved.reportMarkdownPath,\n    ELIZA_ACTION_BENCHMARK_REPORT_JSON_PATH: resolved.reportJsonPath,\n    ELIZA_ACTION_BENCHMARK_TRAJECTORY_DIR: resolved.trajectoryDir,\n  };\n  if (effectiveUseMocks(options)) env.ELIZA_BENCHMARK_USE_MOCKS = \"1\";\n  if (options.forceTrajectoryCapture !== false) {\n    env.ELIZA_DUMP_TRAJECTORIES = \"1\";\n    env.ELIZA_TRAJECTORY_MARKDOWN = \"1\";\n  }\n  const runsPerCase = positiveInt(options.runsPerCase);\n  if (runsPerCase) env.ELIZA_BENCHMARK_RUNS_PER_CASE = String(runsPerCase);\n  if (options.filter?.trim())\n    env.ELIZA_BENCHMARK_FILTER = options.filter.trim();\n  if (options.provider?.trim()) {\n    const provider = options.provider.trim();\n    env.ELIZA_BENCHMARK_PROVIDER = provider;\n    if (provider === \"local-llama-cpp\") {\n      env.LOCAL_LLAMA_CPP_API_KEY =\n        process.env.LOCAL_LLAMA_CPP_API_KEY ?? \"local\";\n    }\n  }\n  const runtimeModel = stringSetting(options.runtimeModel);\n  const smallModel = stringSetting(options.smallModel) ?? runtimeModel;\n  const largeModel = stringSetting(options.largeModel) ?? runtimeModel;\n  if (smallModel) env.ELIZA_LIVE_TEST_SMALL_MODEL = smallModel;\n  if (largeModel) env.ELIZA_LIVE_TEST_LARGE_MODEL = largeModel;\n  if (options.baseUrl?.trim()) {\n    env.ELIZA_LIVE_TEST_LOCAL_LLAMA_CPP_BASE_URL = options.baseUrl.trim();\n  }\n  return env;\n}\n\nexport async function runActionBenchmark(\n  options: ActionBenchmarkRunOptions = {},\n): Promise<ActionBenchmarkRunResult> {\n  const workspaceRoot = resolveWorkspaceRoot(options.workspaceRoot);\n  const appCoreRoot = join(workspaceRoot, \"packages\", \"app-core\");\n  const stamp = safeTimestamp(new Date().toISOString());\n  const outputDir =\n    options.outputDir ??\n    join(trainingStateRoot(), \"benchmarks\", \"action-selection\", stamp);\n  const reportMarkdownPath = join(outputDir, \"action-benchmark-report.md\");\n  const reportJsonPath = join(outputDir, \"action-benchmark-report.json\");\n  const trajectoryDir = join(outputDir, \"trajectories\");\n  await mkdir(outputDir, { recursive: true });\n  await mkdir(trajectoryDir, { recursive: true });\n\n  const command = options.bun ?? defaultBunCommand();\n  const args = buildActionBenchmarkCommand();\n  const benchmarkEnv = buildActionBenchmarkEnv(options, {\n    reportMarkdownPath,\n    reportJsonPath,\n    trajectoryDir,\n  });\n  const reportMatrixSource = matrixSourceForReport(reportJsonPath, options);\n  if (options.dryRun) {\n    const sample = dryRunCaseSample(options, trajectoryDir);\n    await writeFile(\n      String(sample.trajectoryPath),\n      `${JSON.stringify(\n        {\n          schema: \"eliza_action_benchmark_dry_run_trajectory\",\n          schemaVersion: 1,\n          generatedAt: new Date().toISOString(),\n          source: {\n            kind: \"app_core_action_selection_benchmark\",\n            dryRun: true,\n            modelId: sample.modelId,\n            tier: sample.tier,\n            variant: sample.variant,\n          },\n          caseId: sample.caseId,\n          prompt: sample.prompt,\n          expectedAction: sample.expectedAction,\n          actualAction: sample.actualAction,\n          pass: sample.pass,\n          response: sample.response,\n          events: [\n            {\n              type: \"DRY_RUN_BENCHMARK_CASE\",\n              timestamp: new Date().toISOString(),\n              data: {\n                reason: \"No model inference executed in dry-run mode.\",\n              },\n            },\n          ],\n        },\n        null,\n        2,\n      )}\\n`,\n      \"utf8\",\n    );\n    await writeFile(\n      reportJsonPath,\n      `${JSON.stringify(\n        {\n          schema: \"eliza_action_selection_benchmark_report\",\n          schemaVersion: 1,\n          generatedAt: new Date().toISOString(),\n          source: {\n            kind: \"app_core_action_selection_benchmark\",\n            trajectoryDir,\n            reportMarkdownPath,\n            modelId: reportMatrixSource?.modelId,\n            variant: reportMatrixSource?.variant,\n            benchmark: reportMatrixSource?.benchmark,\n            tier: reportMatrixSource?.tier,\n            provider: reportMatrixSource?.provider,\n            datasetVersion: reportMatrixSource?.datasetVersion,\n            codeCommit: reportMatrixSource?.codeCommit,\n            useMocks: reportMatrixSource?.useMocks,\n            dryRun: true,\n          },\n          summary: {\n            total: 1,\n            passed: 0,\n            failed: 1,\n            accuracy: 0,\n            plannerAccuracy: 0,\n            executionAccuracy: 0,\n          },\n          failureModes: {\n            dry_run: 1,\n          },\n          failures: [\n            {\n              caseId: sample.caseId,\n              failureMode: \"dry_run\",\n              reason: \"No model inference executed in dry-run mode.\",\n            },\n          ],\n          results: [sample],\n          dryRun: true,\n        },\n        null,\n        2,\n      )}\\n`,\n      \"utf8\",\n    );\n    await writeFile(\n      reportMarkdownPath,\n      \"# Action Selection Benchmark Dry Run\\n\\nNo benchmark cases were executed.\\n\",\n      \"utf8\",\n    );\n    return {\n      workspaceRoot,\n      appCoreRoot,\n      outputDir,\n      reportMarkdownPath,\n      reportJsonPath,\n      trajectoryDir,\n      command: [command, ...args],\n      env: benchmarkEnv,\n      stdout: \"[DRY RUN] Would run app-core action selection benchmark.\",\n      stderr: \"\",\n      exitCode: 0,\n      matrixSource: reportMatrixSource,\n    };\n  }\n\n  await assertLocalBenchmarkModelAvailable(options);\n\n  const proc = await collectProcess(command, args, appCoreRoot, {\n    ...process.env,\n    ...benchmarkEnv,\n  });\n  if (proc.exitCode !== 0) {\n    throw new Error(\n      `action selection benchmark exited with code ${proc.exitCode}: ${\n        proc.stderr || proc.stdout\n      }`,\n    );\n  }\n  await annotateBenchmarkReportSource(reportJsonPath, options);\n  return {\n    workspaceRoot,\n    appCoreRoot,\n    outputDir,\n    reportMarkdownPath,\n    reportJsonPath,\n    trajectoryDir,\n    command: [command, ...args],\n    env: benchmarkEnv,\n    stdout: proc.stdout,\n    stderr: proc.stderr,\n    exitCode: proc.exitCode,\n    matrixSource: matrixSourceForReport(reportJsonPath, options),\n  };\n}\n"],"mappings":"AAAA,SAAS,aAAa;AACtB,SAAS,OAAO,UAAU,iBAAiB;AAC3C,SAAS,YAAY;AACrB,SAAS,yBAAyB;AAClC,SAAS,mBAAmB,4BAA4B;AAmDxD,SAAS,cAAc,OAAuB;AAC5C,SAAO,MAAM,QAAQ,SAAS,GAAG;AACnC;AAEA,SAAS,YAAY,OAA+C;AAClE,SAAO,OAAO,UAAU,YAAY,OAAO,SAAS,KAAK,IACrD,KAAK,IAAI,GAAG,KAAK,MAAM,KAAK,CAAC,IAC7B;AACN;AAEA,SAAS,eACP,SACA,MACA,KACA,KAC+D;AAC/D,SAAO,IAAI,QAAQ,CAAC,gBAAgB,WAAW;AAC7C,UAAM,QAAQ,MAAM,SAAS,MAAM;AAAA,MACjC;AAAA,MACA;AAAA,MACA,OAAO,CAAC,UAAU,QAAQ,MAAM;AAAA,IAClC,CAAC;AACD,QAAI,SAAS;AACb,QAAI,SAAS;AACb,UAAM,OAAO,YAAY,OAAO;AAChC,UAAM,OAAO,YAAY,OAAO;AAChC,UAAM,OAAO,GAAG,QAAQ,CAAC,UAAU;AACjC,gBAAU;AAAA,IACZ,CAAC;AACD,UAAM,OAAO,GAAG,QAAQ,CAAC,UAAU;AACjC,gBAAU;AAAA,IACZ,CAAC;AACD,UAAM,GAAG,SAAS,MAAM;AACxB,UAAM,GAAG,SAAS,CAAC,SAAS;AAC1B,qBAAe,EAAE,QAAQ,QAAQ,UAAU,QAAQ,EAAE,CAAC;AAAA,IACxD,CAAC;AAAA,EACH,CAAC;AACH;AAEA,SAAS,cAAc,OAA+C;AACpE,SAAO,OAAO,KAAK,KAAK;AAC1B;AAEA,SAAS,aAAa,SAAyB;AAC7C,QAAM,aAAa,QAAQ,KAAK,EAAE,QAAQ,QAAQ,EAAE;AACpD,SAAO,GAAG,UAAU;AACtB;AAEA,SAAS,oBAAoB,aAAqB,aAA8B;AAC9E,SACE,gBAAgB,eAChB,gBAAgB,GAAG,WAAW,aAC9B,GAAG,WAAW,cAAc;AAEhC;AAEA,eAAe,cAAc,SAAoC;AAC/D,QAAM,WAAW,MAAM,MAAM,aAAa,OAAO,CAAC;AAClD,MAAI,CAAC,SAAS,IAAI;AAChB,UAAM,IAAI;AAAA,MACR,wBAAwB,aAAa,OAAO,CAAC,aAAa,SAAS,MAAM,IAAI,SAAS,UAAU;AAAA,IAClG;AAAA,EACF;AACA,QAAM,UAAU,MAAM,SAAS,KAAK;AACpC,QAAM,OACJ,WACA,OAAO,YAAY,YACnB,MAAM,QAAS,QAA+B,IAAI,IAC7C,QAAgC,OACjC,CAAC;AACP,SAAO,KACJ;AAAA,IAAI,CAAC,SACJ,QAAQ,OAAO,SAAS,WACnB,KAA0C,MAC1C,KAA0C,OAC3C;AAAA,EACN,EACC,OAAO,CAAC,OAAqB,OAAO,OAAO,YAAY,GAAG,SAAS,CAAC;AACzE;AAEA,eAAsB,mCACpB,SACe;AACf,MAAI,kBAAkB,OAAO,EAAG;AAChC,MAAI,QAAQ,aAAa,kBAAmB;AAC5C,QAAM,iBAAiB,cAAc,QAAQ,YAAY;AACzD,MAAI,CAAC,eAAgB;AACrB,QAAM,UAAU,cAAc,QAAQ,OAAO,KAAK;AAClD,QAAM,MAAM,MAAM,cAAc,OAAO;AACvC,MAAI,IAAI,KAAK,CAAC,OAAO,oBAAoB,IAAI,cAAc,CAAC,EAAG;AAC/D,QAAM,IAAI;AAAA,IACR,iCAAiC,cAAc,yBAAyB;AAAA,MACtE;AAAA,IACF,CAAC,uBAAuB,IAAI,SAAS,IAAI,IAAI,KAAK,IAAI,IAAI,MAAM;AAAA,EAClE;AACF;AAEA,SAAS,kBAAkB,SAA6C;AACtE,SAAO,QAAQ,YAAY,QAAQ,WAAW;AAChD;AAEA,SAAS,sBACP,gBACA,SAC0C;AAC1C,QAAM,UACJ,cAAc,QAAQ,OAAO,KAAK,cAAc,QAAQ,QAAQ;AAClE,QAAM,UAAU,QAAQ;AACxB,MAAI,CAAC,WAAW,CAAC,QAAS,QAAO;AACjC,SAAO;AAAA,IACL,MAAM;AAAA,IACN;AAAA,IACA;AAAA,IACA,WAAW,cAAc,QAAQ,SAAS;AAAA,IAC1C,MAAM,cAAc,QAAQ,IAAI;AAAA,IAChC,UAAU,cAAc,QAAQ,QAAQ;AAAA,IACxC,gBAAgB,cAAc,QAAQ,cAAc;AAAA,IACpD,YAAY,cAAc,QAAQ,UAAU;AAAA,IAC5C,UAAU,kBAAkB,OAAO;AAAA,EACrC;AACF;AAEA,SAAS,iBAAiB,SAAoC,eAAuB;AACnF,QAAM,OAAO,cAAc,QAAQ,IAAI,KAAK;AAC5C,QAAM,UAAU,cAAc,QAAQ,OAAO,KAAK;AAClD,QAAM,UAAU,cAAc,QAAQ,OAAO,KAAK,cAAc,QAAQ,YAAY;AACpF,SAAO;AAAA,IACL,QAAQ,WAAW,IAAI,IAAI,OAAO;AAAA,IAClC,QAAQ;AAAA,IACR,gBAAgB;AAAA,IAChB,cAAc;AAAA,IACd,MAAM;AAAA,IACN,UAAU;AAAA,IACV,WAAW;AAAA,IACX,gBAAgB;AAAA,MACd;AAAA,MACA,WAAW,IAAI,IAAI,OAAO;AAAA,IAC5B;AAAA,IACA,QAAQ;AAAA,IACR;AAAA,IACA;AAAA,IACA;AAAA,EACF;AACF;AAEA,eAAe,8BACb,gBACA,SACe;AACf,QAAM,eAAe,sBAAsB,gBAAgB,OAAO;AAClE,MAAI,CAAC,aAAc;AACnB,QAAM,SAAS,KAAK,MAAM,MAAM,SAAS,gBAAgB,MAAM,CAAC;AAChE,MAAI,CAAC,UAAU,OAAO,WAAW,YAAY,MAAM,QAAQ,MAAM,EAAG;AACpE,QAAM,SAAS;AACf,QAAM,iBACJ,OAAO,UACP,OAAO,OAAO,WAAW,YACzB,CAAC,MAAM,QAAQ,OAAO,MAAM,IACvB,OAAO,SACR,CAAC;AACP,SAAO,SAAS;AAAA,IACd,GAAG;AAAA,IACH,SAAS,aAAa;AAAA,IACtB,SAAS,aAAa;AAAA,IACtB,WAAW,aAAa;AAAA,IACxB,MAAM,aAAa;AAAA,IACnB,UAAU,aAAa;AAAA,IACvB,gBAAgB,aAAa;AAAA,IAC7B,YAAY,aAAa;AAAA,IACzB,UAAU,aAAa;AAAA,EACzB;AACA,QAAM;AAAA,IACJ;AAAA,IACA,GAAG,KAAK,UAAU,QAAQ,MAAM,CAAC,CAAC;AAAA;AAAA,IAClC;AAAA,EACF;AACF;AAEO,SAAS,8BAAwC;AACtD,SAAO;AAAA,IACL;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,EACF;AACF;AAEO,SAAS,wBACd,SACA,UAKwB;AACxB,QAAM,MAA8B;AAAA,IAClC,4BAA4B;AAAA,IAC5B,oCAAoC,SAAS;AAAA,IAC7C,yCAAyC,SAAS;AAAA,IAClD,uCAAuC,SAAS;AAAA,EAClD;AACA,MAAI,kBAAkB,OAAO,EAAG,KAAI,4BAA4B;AAChE,MAAI,QAAQ,2BAA2B,OAAO;AAC5C,QAAI,0BAA0B;AAC9B,QAAI,4BAA4B;AAAA,EAClC;AACA,QAAM,cAAc,YAAY,QAAQ,WAAW;AACnD,MAAI,YAAa,KAAI,gCAAgC,OAAO,WAAW;AACvE,MAAI,QAAQ,QAAQ,KAAK;AACvB,QAAI,yBAAyB,QAAQ,OAAO,KAAK;AACnD,MAAI,QAAQ,UAAU,KAAK,GAAG;AAC5B,UAAM,WAAW,QAAQ,SAAS,KAAK;AACvC,QAAI,2BAA2B;AAC/B,QAAI,aAAa,mBAAmB;AAClC,UAAI,0BACF,QAAQ,IAAI,2BAA2B;AAAA,IAC3C;AAAA,EACF;AACA,QAAM,eAAe,cAAc,QAAQ,YAAY;AACvD,QAAM,aAAa,cAAc,QAAQ,UAAU,KAAK;AACxD,QAAM,aAAa,cAAc,QAAQ,UAAU,KAAK;AACxD,MAAI,WAAY,KAAI,8BAA8B;AAClD,MAAI,WAAY,KAAI,8BAA8B;AAClD,MAAI,QAAQ,SAAS,KAAK,GAAG;AAC3B,QAAI,2CAA2C,QAAQ,QAAQ,KAAK;AAAA,EACtE;AACA,SAAO;AACT;AAEA,eAAsB,mBACpB,UAAqC,CAAC,GACH;AACnC,QAAM,gBAAgB,qBAAqB,QAAQ,aAAa;AAChE,QAAM,cAAc,KAAK,eAAe,YAAY,UAAU;AAC9D,QAAM,QAAQ,eAAc,oBAAI,KAAK,GAAE,YAAY,CAAC;AACpD,QAAM,YACJ,QAAQ,aACR,KAAK,kBAAkB,GAAG,cAAc,oBAAoB,KAAK;AACnE,QAAM,qBAAqB,KAAK,WAAW,4BAA4B;AACvE,QAAM,iBAAiB,KAAK,WAAW,8BAA8B;AACrE,QAAM,gBAAgB,KAAK,WAAW,cAAc;AACpD,QAAM,MAAM,WAAW,EAAE,WAAW,KAAK,CAAC;AAC1C,QAAM,MAAM,eAAe,EAAE,WAAW,KAAK,CAAC;AAE9C,QAAM,UAAU,QAAQ,OAAO,kBAAkB;AACjD,QAAM,OAAO,4BAA4B;AACzC,QAAM,eAAe,wBAAwB,SAAS;AAAA,IACpD;AAAA,IACA;AAAA,IACA;AAAA,EACF,CAAC;AACD,QAAM,qBAAqB,sBAAsB,gBAAgB,OAAO;AACxE,MAAI,QAAQ,QAAQ;AAClB,UAAM,SAAS,iBAAiB,SAAS,aAAa;AACtD,UAAM;AAAA,MACJ,OAAO,OAAO,cAAc;AAAA,MAC5B,GAAG,KAAK;AAAA,QACN;AAAA,UACE,QAAQ;AAAA,UACR,eAAe;AAAA,UACf,cAAa,oBAAI,KAAK,GAAE,YAAY;AAAA,UACpC,QAAQ;AAAA,YACN,MAAM;AAAA,YACN,QAAQ;AAAA,YACR,SAAS,OAAO;AAAA,YAChB,MAAM,OAAO;AAAA,YACb,SAAS,OAAO;AAAA,UAClB;AAAA,UACA,QAAQ,OAAO;AAAA,UACf,QAAQ,OAAO;AAAA,UACf,gBAAgB,OAAO;AAAA,UACvB,cAAc,OAAO;AAAA,UACrB,MAAM,OAAO;AAAA,UACb,UAAU,OAAO;AAAA,UACjB,QAAQ;AAAA,YACN;AAAA,cACE,MAAM;AAAA,cACN,YAAW,oBAAI,KAAK,GAAE,YAAY;AAAA,cAClC,MAAM;AAAA,gBACJ,QAAQ;AAAA,cACV;AAAA,YACF;AAAA,UACF;AAAA,QACF;AAAA,QACA;AAAA,QACA;AAAA,MACF,CAAC;AAAA;AAAA,MACD;AAAA,IACF;AACA,UAAM;AAAA,MACJ;AAAA,MACA,GAAG,KAAK;AAAA,QACN;AAAA,UACE,QAAQ;AAAA,UACR,eAAe;AAAA,UACf,cAAa,oBAAI,KAAK,GAAE,YAAY;AAAA,UACpC,QAAQ;AAAA,YACN,MAAM;AAAA,YACN;AAAA,YACA;AAAA,YACA,SAAS,oBAAoB;AAAA,YAC7B,SAAS,oBAAoB;AAAA,YAC7B,WAAW,oBAAoB;AAAA,YAC/B,MAAM,oBAAoB;AAAA,YAC1B,UAAU,oBAAoB;AAAA,YAC9B,gBAAgB,oBAAoB;AAAA,YACpC,YAAY,oBAAoB;AAAA,YAChC,UAAU,oBAAoB;AAAA,YAC9B,QAAQ;AAAA,UACV;AAAA,UACA,SAAS;AAAA,YACP,OAAO;AAAA,YACP,QAAQ;AAAA,YACR,QAAQ;AAAA,YACR,UAAU;AAAA,YACV,iBAAiB;AAAA,YACjB,mBAAmB;AAAA,UACrB;AAAA,UACA,cAAc;AAAA,YACZ,SAAS;AAAA,UACX;AAAA,UACA,UAAU;AAAA,YACR;AAAA,cACE,QAAQ,OAAO;AAAA,cACf,aAAa;AAAA,cACb,QAAQ;AAAA,YACV;AAAA,UACF;AAAA,UACA,SAAS,CAAC,MAAM;AAAA,UAChB,QAAQ;AAAA,QACV;AAAA,QACA;AAAA,QACA;AAAA,MACF,CAAC;AAAA;AAAA,MACD;AAAA,IACF;AACA,UAAM;AAAA,MACJ;AAAA,MACA;AAAA,MACA;AAAA,IACF;AACA,WAAO;AAAA,MACL;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA,SAAS,CAAC,SAAS,GAAG,IAAI;AAAA,MAC1B,KAAK;AAAA,MACL,QAAQ;AAAA,MACR,QAAQ;AAAA,MACR,UAAU;AAAA,MACV,cAAc;AAAA,IAChB;AAAA,EACF;AAEA,QAAM,mCAAmC,OAAO;AAEhD,QAAM,OAAO,MAAM,eAAe,SAAS,MAAM,aAAa;AAAA,IAC5D,GAAG,QAAQ;AAAA,IACX,GAAG;AAAA,EACL,CAAC;AACD,MAAI,KAAK,aAAa,GAAG;AACvB,UAAM,IAAI;AAAA,MACR,+CAA+C,KAAK,QAAQ,KAC1D,KAAK,UAAU,KAAK,MACtB;AAAA,IACF;AAAA,EACF;AACA,QAAM,8BAA8B,gBAAgB,OAAO;AAC3D,SAAO;AAAA,IACL;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA,SAAS,CAAC,SAAS,GAAG,IAAI;AAAA,IAC1B,KAAK;AAAA,IACL,QAAQ,KAAK;AAAA,IACb,QAAQ,KAAK;AAAA,IACb,UAAU,KAAK;AAAA,IACf,cAAc,sBAAsB,gBAAgB,OAAO;AAAA,EAC7D;AACF;","names":[]}

package/dist/core/artifact-store.d.ts

@@ -0,0 +1,72 @@
+/**
+ * Filesystem helpers around the `OptimizedPromptService` store.
+ *
+ * Two responsibilities, both intentionally separate from the strict per-task
+ * artifact loader in `packages/core/src/services/optimized-prompt.ts`:
+ *
+ *   1. Persist `candidate_rejected_<timestamp>.json` files for runs the
+ *      promotion gate refused. These live under `<task>/rejected/` so the
+ *      strict artifact parser never picks them up at boot.
+ *   2. Prune the per-task directory to the most recent N promoted artifacts.
+ *      Older `.json` files are removed so a long-running deployment doesn't
+ *      accumulate unbounded history. The pruning is rollback-friendly: when
+ *      operators want to revert, they pick from the retained N files.
+ *
+ * Both helpers operate on file mtime/timestamp, never on the artifact contents.
+ * The strict parser owns content validation; this module is filesystem-only.
+ */
+/**
+ * Maximum number of promoted artifacts retained per task. Older files are
+ * deleted by `prunePromotedArtifacts`. Keeping 5 lines up with the W1-P3
+ * rollback budget (one current + four historical fallbacks).
+ */
+export declare const DEFAULT_PROMOTED_ARTIFACT_RETENTION = 5;
+/** Subdirectory under `<storeRoot>/<task>/` for rejected candidates. */
+export declare const REJECTED_DIRNAME = "rejected";
+export interface RejectedCandidatePayload {
+    /** ISO-8601 timestamp of when the candidate was rejected. */
+    rejectedAt: string;
+    /** Task the rejected candidate targeted. */
+    task: string;
+    /** Optimizer name that produced the candidate. */
+    optimizer: string;
+    /** The candidate prompt body that did not clear the gate. */
+    candidatePrompt: string;
+    /** Incumbent prompt body the gate evaluated against. */
+    incumbentPrompt: string;
+    /** Score / margin block from the promotion gate. */
+    scores: {
+        incumbentMeanScore: number;
+        incumbentStdDev: number;
+        candidateScore: number;
+        delta: number;
+        promotionMargin: number;
+        noiseThreshold: number;
+        incumbentReseeds: number;
+        examplesPerPass: number;
+        incumbentScores: number[];
+    };
+    /** Plain-english reason from the promotion gate. */
+    reason: string;
+    /** Backreference to the dataset that drove the run. */
+    datasetId: string;
+    /** Backreference to the run id from the training orchestrator. */
+    runId?: string;
+}
+/**
+ * Write a `candidate_rejected_<timestamp>.json` file under
+ * `<storeRoot>/<task>/rejected/`. Atomic (temp + rename). Returns the final
+ * path. The parent directory is created if missing.
+ */
+export declare function writeRejectedCandidate(storeRoot: string, task: string, payload: RejectedCandidatePayload): Promise<string>;
+/**
+ * Delete promoted artifacts older than the most recent `retain` (by mtime).
+ * Only `.json` files at the top level of `<storeRoot>/<task>/` are considered;
+ * the `rejected/` subdirectory is left alone.
+ *
+ * Returns the list of removed paths. No-op when fewer than `retain` files
+ * exist. Errors during stat/unlink propagate — the caller decides whether to
+ * fail the whole run.
+ */
+export declare function prunePromotedArtifacts(storeRoot: string, task: string, retain?: number): Promise<string[]>;
+//# sourceMappingURL=artifact-store.d.ts.map

package/dist/core/artifact-store.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"artifact-store.d.ts","sourceRoot":"","sources":["../../src/core/artifact-store.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;GAgBG;AAMH;;;;GAIG;AACH,eAAO,MAAM,mCAAmC,IAAI,CAAC;AAErD,wEAAwE;AACxE,eAAO,MAAM,gBAAgB,aAAa,CAAC;AAE3C,MAAM,WAAW,wBAAwB;IACvC,6DAA6D;IAC7D,UAAU,EAAE,MAAM,CAAC;IACnB,4CAA4C;IAC5C,IAAI,EAAE,MAAM,CAAC;IACb,kDAAkD;IAClD,SAAS,EAAE,MAAM,CAAC;IAClB,6DAA6D;IAC7D,eAAe,EAAE,MAAM,CAAC;IACxB,wDAAwD;IACxD,eAAe,EAAE,MAAM,CAAC;IACxB,oDAAoD;IACpD,MAAM,EAAE;QACN,kBAAkB,EAAE,MAAM,CAAC;QAC3B,eAAe,EAAE,MAAM,CAAC;QACxB,cAAc,EAAE,MAAM,CAAC;QACvB,KAAK,EAAE,MAAM,CAAC;QACd,eAAe,EAAE,MAAM,CAAC;QACxB,cAAc,EAAE,MAAM,CAAC;QACvB,gBAAgB,EAAE,MAAM,CAAC;QACzB,eAAe,EAAE,MAAM,CAAC;QACxB,eAAe,EAAE,MAAM,EAAE,CAAC;KAC3B,CAAC;IACF,oDAAoD;IACpD,MAAM,EAAE,MAAM,CAAC;IACf,uDAAuD;IACvD,SAAS,EAAE,MAAM,CAAC;IAClB,kEAAkE;IAClE,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB;AAED;;;;GAIG;AACH,wBAAsB,sBAAsB,CAC1C,SAAS,EAAE,MAAM,EACjB,IAAI,EAAE,MAAM,EACZ,OAAO,EAAE,wBAAwB,GAChC,OAAO,CAAC,MAAM,CAAC,CAUjB;AAED;;;;;;;;GAQG;AACH,wBAAsB,sBAAsB,CAC1C,SAAS,EAAE,MAAM,EACjB,IAAI,EAAE,MAAM,EACZ,MAAM,GAAE,MAA4C,GACnD,OAAO,CAAC,MAAM,EAAE,CAAC,CA0BnB"}

package/dist/core/artifact-store.js

@@ -0,0 +1,50 @@
+import { existsSync, mkdirSync, readdirSync } from "node:fs";
+import { rename, stat, unlink, writeFile } from "node:fs/promises";
+import { dirname, join } from "node:path";
+const DEFAULT_PROMOTED_ARTIFACT_RETENTION = 5;
+const REJECTED_DIRNAME = "rejected";
+async function writeRejectedCandidate(storeRoot, task, payload) {
+  const dir = join(storeRoot, task, REJECTED_DIRNAME);
+  mkdirSync(dir, { recursive: true });
+  const stamp = payload.rejectedAt.replace(/[^0-9]/g, "");
+  const finalPath = join(dir, `candidate_rejected_${stamp}.json`);
+  const tempPath = `${finalPath}.tmp-${process.pid}-${Date.now()}`;
+  mkdirSync(dirname(tempPath), { recursive: true });
+  await writeFile(tempPath, `${JSON.stringify(payload, null, 2)}
+`, "utf-8");
+  await rename(tempPath, finalPath);
+  return finalPath;
+}
+async function prunePromotedArtifacts(storeRoot, task, retain = DEFAULT_PROMOTED_ARTIFACT_RETENTION) {
+  if (retain < 1) {
+    throw new Error(
+      `[artifact-store] prunePromotedArtifacts retain must be >= 1; got ${retain}`
+    );
+  }
+  const dir = join(storeRoot, task);
+  if (!existsSync(dir)) return [];
+  const entries = readdirSync(dir);
+  const jsonFiles = entries.filter((name) => name.endsWith(".json"));
+  if (jsonFiles.length <= retain) return [];
+  const withStats = [];
+  for (const name of jsonFiles) {
+    const path = join(dir, name);
+    const stats = await stat(path);
+    withStats.push({ path, mtimeMs: stats.mtimeMs });
+  }
+  withStats.sort((a, b) => b.mtimeMs - a.mtimeMs);
+  const toRemove = withStats.slice(retain);
+  const removed = [];
+  for (const entry of toRemove) {
+    await unlink(entry.path);
+    removed.push(entry.path);
+  }
+  return removed;
+}
+export {
+  DEFAULT_PROMOTED_ARTIFACT_RETENTION,
+  REJECTED_DIRNAME,
+  prunePromotedArtifacts,
+  writeRejectedCandidate
+};
+//# sourceMappingURL=artifact-store.js.map

package/dist/core/artifact-store.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../../src/core/artifact-store.ts"],"sourcesContent":["/**\n * Filesystem helpers around the `OptimizedPromptService` store.\n *\n * Two responsibilities, both intentionally separate from the strict per-task\n * artifact loader in `packages/core/src/services/optimized-prompt.ts`:\n *\n *   1. Persist `candidate_rejected_<timestamp>.json` files for runs the\n *      promotion gate refused. These live under `<task>/rejected/` so the\n *      strict artifact parser never picks them up at boot.\n *   2. Prune the per-task directory to the most recent N promoted artifacts.\n *      Older `.json` files are removed so a long-running deployment doesn't\n *      accumulate unbounded history. The pruning is rollback-friendly: when\n *      operators want to revert, they pick from the retained N files.\n *\n * Both helpers operate on file mtime/timestamp, never on the artifact contents.\n * The strict parser owns content validation; this module is filesystem-only.\n */\n\nimport { existsSync, mkdirSync, readdirSync } from \"node:fs\";\nimport { rename, stat, unlink, writeFile } from \"node:fs/promises\";\nimport { dirname, join } from \"node:path\";\n\n/**\n * Maximum number of promoted artifacts retained per task. Older files are\n * deleted by `prunePromotedArtifacts`. Keeping 5 lines up with the W1-P3\n * rollback budget (one current + four historical fallbacks).\n */\nexport const DEFAULT_PROMOTED_ARTIFACT_RETENTION = 5;\n\n/** Subdirectory under `<storeRoot>/<task>/` for rejected candidates. */\nexport const REJECTED_DIRNAME = \"rejected\";\n\nexport interface RejectedCandidatePayload {\n  /** ISO-8601 timestamp of when the candidate was rejected. */\n  rejectedAt: string;\n  /** Task the rejected candidate targeted. */\n  task: string;\n  /** Optimizer name that produced the candidate. */\n  optimizer: string;\n  /** The candidate prompt body that did not clear the gate. */\n  candidatePrompt: string;\n  /** Incumbent prompt body the gate evaluated against. */\n  incumbentPrompt: string;\n  /** Score / margin block from the promotion gate. */\n  scores: {\n    incumbentMeanScore: number;\n    incumbentStdDev: number;\n    candidateScore: number;\n    delta: number;\n    promotionMargin: number;\n    noiseThreshold: number;\n    incumbentReseeds: number;\n    examplesPerPass: number;\n    incumbentScores: number[];\n  };\n  /** Plain-english reason from the promotion gate. */\n  reason: string;\n  /** Backreference to the dataset that drove the run. */\n  datasetId: string;\n  /** Backreference to the run id from the training orchestrator. */\n  runId?: string;\n}\n\n/**\n * Write a `candidate_rejected_<timestamp>.json` file under\n * `<storeRoot>/<task>/rejected/`. Atomic (temp + rename). Returns the final\n * path. The parent directory is created if missing.\n */\nexport async function writeRejectedCandidate(\n  storeRoot: string,\n  task: string,\n  payload: RejectedCandidatePayload,\n): Promise<string> {\n  const dir = join(storeRoot, task, REJECTED_DIRNAME);\n  mkdirSync(dir, { recursive: true });\n  const stamp = payload.rejectedAt.replace(/[^0-9]/g, \"\");\n  const finalPath = join(dir, `candidate_rejected_${stamp}.json`);\n  const tempPath = `${finalPath}.tmp-${process.pid}-${Date.now()}`;\n  mkdirSync(dirname(tempPath), { recursive: true });\n  await writeFile(tempPath, `${JSON.stringify(payload, null, 2)}\\n`, \"utf-8\");\n  await rename(tempPath, finalPath);\n  return finalPath;\n}\n\n/**\n * Delete promoted artifacts older than the most recent `retain` (by mtime).\n * Only `.json` files at the top level of `<storeRoot>/<task>/` are considered;\n * the `rejected/` subdirectory is left alone.\n *\n * Returns the list of removed paths. No-op when fewer than `retain` files\n * exist. Errors during stat/unlink propagate — the caller decides whether to\n * fail the whole run.\n */\nexport async function prunePromotedArtifacts(\n  storeRoot: string,\n  task: string,\n  retain: number = DEFAULT_PROMOTED_ARTIFACT_RETENTION,\n): Promise<string[]> {\n  if (retain < 1) {\n    throw new Error(\n      `[artifact-store] prunePromotedArtifacts retain must be >= 1; got ${retain}`,\n    );\n  }\n  const dir = join(storeRoot, task);\n  if (!existsSync(dir)) return [];\n  const entries = readdirSync(dir);\n  const jsonFiles = entries.filter((name) => name.endsWith(\".json\"));\n  if (jsonFiles.length <= retain) return [];\n  // Sort by mtime descending so the newest files are first; we delete the tail.\n  const withStats: Array<{ path: string; mtimeMs: number }> = [];\n  for (const name of jsonFiles) {\n    const path = join(dir, name);\n    const stats = await stat(path);\n    withStats.push({ path, mtimeMs: stats.mtimeMs });\n  }\n  withStats.sort((a, b) => b.mtimeMs - a.mtimeMs);\n  const toRemove = withStats.slice(retain);\n  const removed: string[] = [];\n  for (const entry of toRemove) {\n    await unlink(entry.path);\n    removed.push(entry.path);\n  }\n  return removed;\n}\n"],"mappings":"AAkBA,SAAS,YAAY,WAAW,mBAAmB;AACnD,SAAS,QAAQ,MAAM,QAAQ,iBAAiB;AAChD,SAAS,SAAS,YAAY;AAOvB,MAAM,sCAAsC;AAG5C,MAAM,mBAAmB;AAsChC,eAAsB,uBACpB,WACA,MACA,SACiB;AACjB,QAAM,MAAM,KAAK,WAAW,MAAM,gBAAgB;AAClD,YAAU,KAAK,EAAE,WAAW,KAAK,CAAC;AAClC,QAAM,QAAQ,QAAQ,WAAW,QAAQ,WAAW,EAAE;AACtD,QAAM,YAAY,KAAK,KAAK,sBAAsB,KAAK,OAAO;AAC9D,QAAM,WAAW,GAAG,SAAS,QAAQ,QAAQ,GAAG,IAAI,KAAK,IAAI,CAAC;AAC9D,YAAU,QAAQ,QAAQ,GAAG,EAAE,WAAW,KAAK,CAAC;AAChD,QAAM,UAAU,UAAU,GAAG,KAAK,UAAU,SAAS,MAAM,CAAC,CAAC;AAAA,GAAM,OAAO;AAC1E,QAAM,OAAO,UAAU,SAAS;AAChC,SAAO;AACT;AAWA,eAAsB,uBACpB,WACA,MACA,SAAiB,qCACE;AACnB,MAAI,SAAS,GAAG;AACd,UAAM,IAAI;AAAA,MACR,oEAAoE,MAAM;AAAA,IAC5E;AAAA,EACF;AACA,QAAM,MAAM,KAAK,WAAW,IAAI;AAChC,MAAI,CAAC,WAAW,GAAG,EAAG,QAAO,CAAC;AAC9B,QAAM,UAAU,YAAY,GAAG;AAC/B,QAAM,YAAY,QAAQ,OAAO,CAAC,SAAS,KAAK,SAAS,OAAO,CAAC;AACjE,MAAI,UAAU,UAAU,OAAQ,QAAO,CAAC;AAExC,QAAM,YAAsD,CAAC;AAC7D,aAAW,QAAQ,WAAW;AAC5B,UAAM,OAAO,KAAK,KAAK,IAAI;AAC3B,UAAM,QAAQ,MAAM,KAAK,IAAI;AAC7B,cAAU,KAAK,EAAE,MAAM,SAAS,MAAM,QAAQ,CAAC;AAAA,EACjD;AACA,YAAU,KAAK,CAAC,GAAG,MAAM,EAAE,UAAU,EAAE,OAAO;AAC9C,QAAM,WAAW,UAAU,MAAM,MAAM;AACvC,QAAM,UAAoB,CAAC;AAC3B,aAAW,SAAS,UAAU;AAC5B,UAAM,OAAO,MAAM,IAAI;AACvB,YAAQ,KAAK,MAAM,IAAI;AAAA,EACzB;AACA,SAAO;AACT;","names":[]}

package/dist/core/benchmark-matrix-artifact.d.ts

@@ -0,0 +1,102 @@
+export declare const BENCHMARK_MATRIX_ARTIFACT_SCHEMA = "eliza_benchmark_matrix_artifact";
+export declare const BENCHMARK_MATRIX_ARTIFACT_VERSION = 1;
+export declare const ACTION_BENCHMARK_REPORT_SCHEMA = "eliza_action_selection_benchmark_report";
+export declare const ACTION_SELECTION_BENCHMARK_ID = "eliza_harness_action_selection";
+export declare const LOCAL_EVAL_COMPARISON_BENCHMARK_ID = "eliza_harness_local_eval_comparison";
+export declare const ELIZA_ONE_MATRIX_TIERS: readonly ["0b", "2b", "4b", "9b", "27b"];
+export type ElizaOneMatrixTier = (typeof ELIZA_ONE_MATRIX_TIERS)[number];
+export type BenchmarkMatrixVariant = "reference" | "base" | "trained";
+export interface BenchmarkMatrixRowInput {
+    modelId: string;
+    benchmark: string;
+    score: number;
+    variant: BenchmarkMatrixVariant;
+    tier?: string;
+    provider?: string;
+    datasetVersion?: string;
+    codeCommit?: string;
+    ts?: number | string;
+    metrics?: Record<string, unknown>;
+    raw?: Record<string, unknown>;
+}
+export interface BenchmarkMatrixInput {
+    rows: BenchmarkMatrixRowInput[];
+    outputDir?: string;
+    generatedAt?: string;
+    referenceModelId?: string;
+    source?: Record<string, unknown>;
+}
+export interface BenchmarkMatrixArtifactSource {
+    path: string;
+    modelId?: string;
+    benchmark?: string;
+    variant?: BenchmarkMatrixVariant;
+    tier?: string;
+    provider?: string;
+    datasetVersion?: string;
+    codeCommit?: string;
+    useMocks?: boolean;
+}
+export interface BenchmarkMatrixFromArtifactsInput {
+    artifacts: BenchmarkMatrixArtifactSource[];
+    outputDir?: string;
+    generatedAt?: string;
+    referenceModelId?: string;
+    source?: Record<string, unknown>;
+}
+export interface BenchmarkMatrixCell {
+    modelId: string;
+    benchmark: string;
+    score: number;
+    variant: BenchmarkMatrixVariant;
+    tier: string | null;
+    provider: string | null;
+    datasetVersion: string | null;
+    codeCommit: string | null;
+    ts: number | string | null;
+    metrics: Record<string, unknown>;
+    raw: Record<string, unknown>;
+}
+export interface BenchmarkMatrixComparison {
+    tier: string;
+    benchmark: string;
+    baseModelId: string | null;
+    trainedModelId: string | null;
+    referenceModelId: string | null;
+    baseScore: number | null;
+    trainedScore: number | null;
+    referenceScore: number | null;
+    improvementAbsolute: number | null;
+    improvementPercent: number | null;
+    trainedVsReferenceAbsolute: number | null;
+    trainedVsReferencePercent: number | null;
+    dryRun: boolean;
+}
+export interface BenchmarkMatrixArtifact {
+    schema: typeof BENCHMARK_MATRIX_ARTIFACT_SCHEMA;
+    version: typeof BENCHMARK_MATRIX_ARTIFACT_VERSION;
+    generatedAt: string;
+    source: Record<string, unknown>;
+    referenceModelId: string | null;
+    tiers: string[];
+    benchmarks: string[];
+    counts: {
+        rows: number;
+        comparisons: number;
+        tiers: number;
+        benchmarks: number;
+    };
+    rows: BenchmarkMatrixCell[];
+    comparisons: BenchmarkMatrixComparison[];
+}
+export interface BenchmarkMatrixArtifactResult {
+    outputDir: string;
+    artifactPath: string;
+    artifact: BenchmarkMatrixArtifact;
+}
+export declare function buildBenchmarkMatrixRowsFromArtifactPayload(payload: Record<string, unknown>, source: BenchmarkMatrixArtifactSource): BenchmarkMatrixRowInput[];
+export declare function buildBenchmarkMatrixRowsFromArtifacts(artifacts: BenchmarkMatrixArtifactSource[]): Promise<BenchmarkMatrixRowInput[]>;
+export declare function buildBenchmarkMatrixArtifactPayload(input: BenchmarkMatrixInput): BenchmarkMatrixArtifact;
+export declare function writeBenchmarkMatrixArtifact(input: BenchmarkMatrixInput): Promise<BenchmarkMatrixArtifactResult>;
+export declare function writeBenchmarkMatrixArtifactFromArtifacts(input: BenchmarkMatrixFromArtifactsInput): Promise<BenchmarkMatrixArtifactResult>;
+//# sourceMappingURL=benchmark-matrix-artifact.d.ts.map

package/dist/core/benchmark-matrix-artifact.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"benchmark-matrix-artifact.d.ts","sourceRoot":"","sources":["../../src/core/benchmark-matrix-artifact.ts"],"names":[],"mappings":"AAUA,eAAO,MAAM,gCAAgC,oCACV,CAAC;AACpC,eAAO,MAAM,iCAAiC,IAAI,CAAC;AACnD,eAAO,MAAM,8BAA8B,4CACA,CAAC;AAC5C,eAAO,MAAM,6BAA6B,mCAAmC,CAAC;AAC9E,eAAO,MAAM,kCAAkC,wCACR,CAAC;AAExC,eAAO,MAAM,sBAAsB,0CAMzB,CAAC;AAEX,MAAM,MAAM,kBAAkB,GAAG,CAAC,OAAO,sBAAsB,CAAC,CAAC,MAAM,CAAC,CAAC;AACzE,MAAM,MAAM,sBAAsB,GAAG,WAAW,GAAG,MAAM,GAAG,SAAS,CAAC;AAEtE,MAAM,WAAW,uBAAuB;IACtC,OAAO,EAAE,MAAM,CAAC;IAChB,SAAS,EAAE,MAAM,CAAC;IAClB,KAAK,EAAE,MAAM,CAAC;IACd,OAAO,EAAE,sBAAsB,CAAC;IAChC,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,EAAE,CAAC,EAAE,MAAM,GAAG,MAAM,CAAC;IACrB,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IAClC,GAAG,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CAC/B;AAED,MAAM,WAAW,oBAAoB;IACnC,IAAI,EAAE,uBAAuB,EAAE,CAAC;IAChC,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B,MAAM,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CAClC;AAED,MAAM,WAAW,6BAA6B;IAC5C,IAAI,EAAE,MAAM,CAAC;IACb,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,OAAO,CAAC,EAAE,sBAAsB,CAAC;IACjC,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,QAAQ,CAAC,EAAE,OAAO,CAAC;CACpB;AAED,MAAM,WAAW,iCAAiC;IAChD,SAAS,EAAE,6BAA6B,EAAE,CAAC;IAC3C,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B,MAAM,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CAClC;AAED,MAAM,WAAW,mBAAmB;IAClC,OAAO,EAAE,MAAM,CAAC;IAChB,SAAS,EAAE,MAAM,CAAC;IAClB,KAAK,EAAE,MAAM,CAAC;IACd,OAAO,EAAE,sBAAsB,CAAC;IAChC,IAAI,EAAE,MAAM,GAAG,IAAI,CAAC;IACpB,QAAQ,EAAE,MAAM,GAAG,IAAI,CAAC;IACxB,cAAc,EAAE,MAAM,GAAG,IAAI,CAAC;IAC9B,UAAU,EAAE,MAAM,GAAG,IAAI,CAAC;IAC1B,EAAE,EAAE,MAAM,GAAG,MAAM,GAAG,IAAI,CAAC;IAC3B,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IACjC,GAAG,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CAC9B;AAED,MAAM,WAAW,yBAAyB;IACxC,IAAI,EAAE,MAAM,CAAC;IACb,SAAS,EAAE,MAAM,CAAC;IAClB,WAAW,EAAE,MAAM,GAAG,IAAI,CAAC;IAC3B,cAAc,EAAE,MAAM,GAAG,IAAI,CAAC;IAC9B,gBAAgB,EAAE,MAAM,GAAG,IAAI,CAAC;IAChC,SAAS,EAAE,MAAM,GAAG,IAAI,CAAC;IACzB,YAAY,EAAE,MAAM,GAAG,IAAI,CAAC;IAC5B,cAAc,EAAE,MAAM,GAAG,IAAI,CAAC;IAC9B,mBAAmB,EAAE,MAAM,GAAG,IAAI,CAAC;IACnC,kBAAkB,EAAE,MAAM,GAAG,IAAI,CAAC;IAClC,0BAA0B,EAAE,MAAM,GAAG,IAAI,CAAC;IAC1C,yBAAyB,EAAE,MAAM,GAAG,IAAI,CAAC;IACzC,MAAM,EAAE,OAAO,CAAC;CACjB;AAED,MAAM,WAAW,uBAAuB;IACtC,MAAM,EAAE,OAAO,gCAAgC,CAAC;IAChD,OAAO,EAAE,OAAO,iCAAiC,CAAC;IAClD,WAAW,EAAE,MAAM,CAAC;IACpB,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IAChC,gBAAgB,EAAE,MAAM,GAAG,IAAI,CAAC;IAChC,KAAK,EAAE,MAAM,EAAE,CAAC;IAChB,UAAU,EAAE,MAAM,EAAE,CAAC;IACrB,MAAM,EAAE;QACN,IAAI,EAAE,MAAM,CAAC;QACb,WAAW,EAAE,MAAM,CAAC;QACpB,KAAK,EAAE,MAAM,CAAC;QACd,UAAU,EAAE,MAAM,CAAC;KACpB,CAAC;IACF,IAAI,EAAE,mBAAmB,EAAE,CAAC;IAC5B,WAAW,EAAE,yBAAyB,EAAE,CAAC;CAC1C;AAED,MAAM,WAAW,6BAA6B;IAC5C,SAAS,EAAE,MAAM,CAAC;IAClB,YAAY,EAAE,MAAM,CAAC;IACrB,QAAQ,EAAE,uBAAuB,CAAC;CACnC;AAwXD,wBAAgB,2CAA2C,CACzD,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,EAChC,MAAM,EAAE,6BAA6B,GACpC,uBAAuB,EAAE,CAW3B;AAED,wBAAsB,qCAAqC,CACzD,SAAS,EAAE,6BAA6B,EAAE,GACzC,OAAO,CAAC,uBAAuB,EAAE,CAAC,CASpC;AAED,wBAAgB,mCAAmC,CACjD,KAAK,EAAE,oBAAoB,GAC1B,uBAAuB,CA6BzB;AAED,wBAAsB,4BAA4B,CAChD,KAAK,EAAE,oBAAoB,GAC1B,OAAO,CAAC,6BAA6B,CAAC,CAiBxC;AAED,wBAAsB,yCAAyC,CAC7D,KAAK,EAAE,iCAAiC,GACvC,OAAO,CAAC,6BAA6B,CAAC,CAYxC"}