@tangle-network/agent-eval 0.100.2 → 0.101.0

package/dist/chunk-63MBSQTX.js

@@ -0,0 +1,350 @@
+import {
+  ValidationError
+} from "./chunk-3BFEG2F6.js";
+
+// src/product-benchmark/index.ts
+import { existsSync, readFileSync, statSync } from "fs";
+import { dirname, join } from "path";
+var productBenchmarkSplits = ["practice", "dev", "holdout", "safety", "sentinel"];
+function isObject(value) {
+  return Boolean(value) && typeof value === "object" && !Array.isArray(value);
+}
+function fail(path, message) {
+  throw new ValidationError(`${path}: ${message}`);
+}
+function wrapValidationError(path, err) {
+  if (err instanceof ValidationError) {
+    throw new ValidationError(`${path}: ${err.message}`, { cause: err });
+  }
+  throw new ValidationError(`${path}: ${err instanceof Error ? err.message : String(err)}`, {
+    cause: err
+  });
+}
+function expectObject(value, path) {
+  if (!isObject(value)) fail(path, "must be an object");
+  return value;
+}
+function expectString(value, path) {
+  if (typeof value !== "string" || value.trim().length === 0)
+    fail(path, "must be a non-empty string");
+  return value;
+}
+function expectBoolean(value, path) {
+  if (typeof value !== "boolean") fail(path, "must be a boolean");
+  return value;
+}
+function expectNumber(value, path, opts = {}) {
+  if (typeof value !== "number" || !Number.isFinite(value)) fail(path, "must be a finite number");
+  if (opts.integer && !Number.isInteger(value)) fail(path, "must be an integer");
+  if (opts.min !== void 0 && value < opts.min) fail(path, `must be >= ${opts.min}`);
+  if (opts.max !== void 0 && value > opts.max) fail(path, `must be <= ${opts.max}`);
+  return value;
+}
+function expectStringArray(value, path) {
+  if (!Array.isArray(value)) fail(path, "must be an array");
+  return value.map((entry, index) => expectString(entry, `${path}[${index}]`));
+}
+function expectObjectArray(value, path) {
+  if (!Array.isArray(value)) fail(path, "must be an array");
+  return value.map((entry, index) => expectObject(entry, `${path}[${index}]`));
+}
+function expectSplit(value, path) {
+  const split = expectString(value, path);
+  if (!productBenchmarkSplits.includes(split)) {
+    fail(path, `must be one of ${productBenchmarkSplits.join(", ")}`);
+  }
+  return split;
+}
+function optionalString(value, path) {
+  if (value === void 0) return void 0;
+  return expectString(value, path);
+}
+function expectDimensions(value, path) {
+  const obj = expectObject(value, path);
+  const out = {};
+  for (const [key, raw] of Object.entries(obj)) out[key] = expectNumber(raw, `${path}.${key}`);
+  return out;
+}
+function expectRuntimeResolution(value, path) {
+  const obj = expectObject(value, path);
+  return {
+    model: expectString(obj.model, `${path}.model`),
+    harness: expectString(obj.harness, `${path}.harness`),
+    backend: expectString(obj.backend, `${path}.backend`),
+    ...obj.reasoningEffort !== void 0 ? { reasoningEffort: optionalString(obj.reasoningEffort, `${path}.reasoningEffort`) } : {}
+  };
+}
+function validateProductBenchmarkManifest(value) {
+  const obj = expectObject(value, "manifest");
+  if (obj.schemaVersion !== 1) fail("manifest.schemaVersion", "must be 1");
+  const repo = expectObject(obj.repo, "manifest.repo");
+  const substrate = expectObject(obj.substrate, "manifest.substrate");
+  const profiles = expectObjectArray(obj.profiles, "manifest.profiles").map((profile, index) => ({
+    id: expectString(profile.id, `manifest.profiles[${index}].id`),
+    profileHash: expectString(profile.profileHash, `manifest.profiles[${index}].profileHash`),
+    agentProfilePath: expectString(
+      profile.agentProfilePath,
+      `manifest.profiles[${index}].agentProfilePath`
+    )
+  }));
+  const arms = expectObjectArray(obj.arms, "manifest.arms").map((arm, index) => ({
+    id: expectString(arm.id, `manifest.arms[${index}].id`),
+    profileId: expectString(arm.profileId, `manifest.arms[${index}].profileId`),
+    mutableSurfaces: expectStringArray(
+      arm.mutableSurfaces,
+      `manifest.arms[${index}].mutableSurfaces`
+    ),
+    policyAxes: expectObject(arm.policyAxes, `manifest.arms[${index}].policyAxes`)
+  }));
+  const scenarios = expectObjectArray(obj.scenarios, "manifest.scenarios").map(
+    (scenario, index) => ({
+      id: expectString(scenario.id, `manifest.scenarios[${index}].id`),
+      split: expectSplit(scenario.split, `manifest.scenarios[${index}].split`),
+      tags: expectStringArray(scenario.tags, `manifest.scenarios[${index}].tags`),
+      sourceAllowedForSynthesis: expectBoolean(
+        scenario.sourceAllowedForSynthesis,
+        `manifest.scenarios[${index}].sourceAllowedForSynthesis`
+      )
+    })
+  );
+  const budgets = expectObject(obj.budgets, "manifest.budgets");
+  if (profiles.length === 0) fail("manifest.profiles", "must contain at least one profile");
+  if (arms.length === 0) fail("manifest.arms", "must contain at least one arm");
+  if (scenarios.length === 0) fail("manifest.scenarios", "must contain at least one scenario");
+  assertUnique(
+    profiles.map((profile) => profile.id),
+    "manifest.profiles.id"
+  );
+  assertUnique(
+    arms.map((arm) => arm.id),
+    "manifest.arms.id"
+  );
+  assertUnique(
+    scenarios.map((scenario) => scenario.id),
+    "manifest.scenarios.id"
+  );
+  const profileIds = new Set(profiles.map((profile) => profile.id));
+  for (const arm of arms) {
+    if (!profileIds.has(arm.profileId))
+      fail(`manifest.arms.${arm.id}.profileId`, `unknown profile ${arm.profileId}`);
+  }
+  return {
+    schemaVersion: 1,
+    projectId: expectString(obj.projectId, "manifest.projectId"),
+    benchmarkId: expectString(obj.benchmarkId, "manifest.benchmarkId"),
+    repo: {
+      url: expectString(repo.url, "manifest.repo.url"),
+      commit: expectString(repo.commit, "manifest.repo.commit"),
+      branch: expectString(repo.branch, "manifest.repo.branch")
+    },
+    substrate: {
+      agentEval: expectString(substrate.agentEval, "manifest.substrate.agentEval"),
+      agentRuntime: expectString(substrate.agentRuntime, "manifest.substrate.agentRuntime"),
+      agentInterface: expectString(substrate.agentInterface, "manifest.substrate.agentInterface"),
+      sandbox: expectString(substrate.sandbox, "manifest.substrate.sandbox"),
+      ...substrate.agentBench !== void 0 ? { agentBench: expectString(substrate.agentBench, "manifest.substrate.agentBench") } : {}
+    },
+    profiles,
+    arms,
+    scenarios,
+    budgets: {
+      maxUsd: expectNumber(budgets.maxUsd, "manifest.budgets.maxUsd", { min: 0 }),
+      maxCells: expectNumber(budgets.maxCells, "manifest.budgets.maxCells", {
+        min: 0,
+        integer: true
+      }),
+      maxWallMs: expectNumber(budgets.maxWallMs, "manifest.budgets.maxWallMs", {
+        min: 0,
+        integer: true
+      })
+    },
+    expectedArtifactDir: expectString(obj.expectedArtifactDir, "manifest.expectedArtifactDir")
+  };
+}
+function validateProductBenchmarkRecord(value) {
+  const obj = expectObject(value, "record");
+  if (obj.schemaVersion !== 1) fail("record.schemaVersion", "must be 1");
+  const agentProfile = expectObject(obj.agentProfile, "record.agentProfile");
+  const model = expectObject(obj.model, "record.model");
+  const backend = expectObject(obj.backend, "record.backend");
+  const outcome = expectObject(obj.outcome, "record.outcome");
+  const usage = expectObject(obj.usage, "record.usage");
+  const integrity = expectObject(obj.integrity, "record.integrity");
+  const artifacts = expectObject(obj.artifacts, "record.artifacts");
+  const record = {
+    schemaVersion: 1,
+    projectId: expectString(obj.projectId, "record.projectId"),
+    benchmarkId: expectString(obj.benchmarkId, "record.benchmarkId"),
+    runId: expectString(obj.runId, "record.runId"),
+    scenarioId: expectString(obj.scenarioId, "record.scenarioId"),
+    split: expectSplit(obj.split, "record.split"),
+    armId: expectString(obj.armId, "record.armId"),
+    rep: expectNumber(obj.rep, "record.rep", { min: 1, integer: true }),
+    agentProfile: {
+      id: expectString(agentProfile.id, "record.agentProfile.id"),
+      hash: expectString(agentProfile.hash, "record.agentProfile.hash"),
+      path: expectString(agentProfile.path, "record.agentProfile.path"),
+      declared: expectRuntimeResolution(agentProfile.declared, "record.agentProfile.declared"),
+      resolved: expectRuntimeResolution(agentProfile.resolved, "record.agentProfile.resolved")
+    },
+    model: {
+      provider: expectString(model.provider, "record.model.provider"),
+      id: expectString(model.id, "record.model.id")
+    },
+    backend: {
+      kind: expectString(backend.kind, "record.backend.kind"),
+      version: expectString(backend.version, "record.backend.version")
+    },
+    outcome: {
+      pass: expectBoolean(outcome.pass, "record.outcome.pass"),
+      score: expectNumber(outcome.score, "record.outcome.score", { min: 0, max: 1 }),
+      dimensions: expectDimensions(outcome.dimensions, "record.outcome.dimensions"),
+      failureMode: outcome.failureMode === null ? null : expectString(outcome.failureMode, "record.outcome.failureMode")
+    },
+    usage: {
+      inputTokens: expectNumber(usage.inputTokens, "record.usage.inputTokens", {
+        min: 0,
+        integer: true
+      }),
+      outputTokens: expectNumber(usage.outputTokens, "record.usage.outputTokens", {
+        min: 0,
+        integer: true
+      }),
+      costUsd: expectNumber(usage.costUsd, "record.usage.costUsd", { min: 0 }),
+      wallMs: expectNumber(usage.wallMs, "record.usage.wallMs", { min: 0, integer: true }),
+      toolCalls: expectNumber(usage.toolCalls, "record.usage.toolCalls", { min: 0, integer: true })
+    },
+    integrity: {
+      realBackend: expectBoolean(integrity.realBackend, "record.integrity.realBackend"),
+      rawCapture: expectBoolean(integrity.rawCapture, "record.integrity.rawCapture"),
+      traceCapture: expectBoolean(integrity.traceCapture, "record.integrity.traceCapture"),
+      noStubRows: expectBoolean(integrity.noStubRows, "record.integrity.noStubRows"),
+      priced: expectBoolean(integrity.priced, "record.integrity.priced"),
+      profileMaterialized: expectBoolean(
+        integrity.profileMaterialized,
+        "record.integrity.profileMaterialized"
+      )
+    },
+    artifacts: {
+      records: expectString(artifacts.records, "record.artifacts.records"),
+      traces: expectString(artifacts.traces, "record.artifacts.traces"),
+      raws: expectString(artifacts.raws, "record.artifacts.raws"),
+      scores: expectString(artifacts.scores, "record.artifacts.scores"),
+      workspace: expectString(artifacts.workspace, "record.artifacts.workspace")
+    }
+  };
+  if (record.integrity.realBackend && record.usage.inputTokens + record.usage.outputTokens === 0) {
+    fail("record.usage", "realBackend rows must carry non-zero token usage");
+  }
+  return record;
+}
+function productBenchmarkIntegrityFailures(record) {
+  const failures = [];
+  for (const [key, value] of Object.entries(record.integrity)) {
+    if (!value) failures.push(`${record.runId}:${key}=false`);
+  }
+  return failures;
+}
+function readProductBenchmarkRecords(path) {
+  const lines = readFileSync(path, "utf8").split("\n").map((line) => line.trim()).filter(Boolean);
+  const records = [];
+  for (const [index, line] of lines.entries()) {
+    try {
+      records.push(validateProductBenchmarkRecord(JSON.parse(line)));
+    } catch (err) {
+      wrapValidationError(`${path}:${index + 1}`, err);
+    }
+  }
+  return records;
+}
+function readProductBenchmarkManifest(path) {
+  try {
+    return validateProductBenchmarkManifest(JSON.parse(readFileSync(path, "utf8")));
+  } catch (err) {
+    wrapValidationError(path, err);
+  }
+}
+function validateProductBenchmarkRun(input) {
+  const manifest = readProductBenchmarkManifest(input.manifestPath);
+  const records = readProductBenchmarkRecords(input.recordsPath);
+  const manifestProjectBench = `${manifest.projectId}/${manifest.benchmarkId}`;
+  const integrityFailures = records.flatMap(productBenchmarkIntegrityFailures);
+  const missingArtifacts = input.checkArtifacts === false ? [] : records.flatMap(
+    (record) => missingArtifactsForRecord(record, input.artifactRoot ?? dirname(input.recordsPath))
+  );
+  for (const [index, record] of records.entries()) {
+    const recordProjectBench = `${record.projectId}/${record.benchmarkId}`;
+    if (recordProjectBench !== manifestProjectBench) {
+      fail(
+        `records[${index}]`,
+        `project/benchmark ${recordProjectBench} does not match manifest ${manifestProjectBench}`
+      );
+    }
+    if (!manifest.arms.some((arm) => arm.id === record.armId))
+      fail(`records[${index}].armId`, `unknown arm ${record.armId}`);
+    if (!manifest.scenarios.some((scenario) => scenario.id === record.scenarioId)) {
+      fail(`records[${index}].scenarioId`, `unknown scenario ${record.scenarioId}`);
+    }
+  }
+  return {
+    manifestPath: input.manifestPath,
+    recordsPath: input.recordsPath,
+    records: records.length,
+    projects: sortedUnique(records.map((record) => record.projectId)),
+    benchmarks: sortedUnique(records.map((record) => record.benchmarkId)),
+    arms: sortedUnique(records.map((record) => record.armId)),
+    scenarios: sortedUnique(records.map((record) => record.scenarioId)),
+    passed: records.filter((record) => record.outcome.pass).length,
+    failed: records.filter((record) => !record.outcome.pass).length,
+    inputTokens: sum(records, (record) => record.usage.inputTokens),
+    outputTokens: sum(records, (record) => record.usage.outputTokens),
+    costUsd: sum(records, (record) => record.usage.costUsd),
+    wallMs: sum(records, (record) => record.usage.wallMs),
+    integrityFailures,
+    missingArtifacts
+  };
+}
+function missingArtifactsForRecord(record, artifactRoot) {
+  const missing = [];
+  for (const [key, value] of Object.entries(record.artifacts)) {
+    const path = resolveArtifactPath(value, artifactRoot);
+    if (!existsSync(path)) missing.push(`${record.runId}:${key}:${value}`);
+  }
+  return missing;
+}
+function resolveArtifactPath(value, artifactRoot) {
+  return value.startsWith("/") ? value : join(artifactRoot, value);
+}
+function assertUnique(values, path) {
+  const seen = /* @__PURE__ */ new Set();
+  for (const value of values) {
+    if (seen.has(value)) fail(path, `duplicate ${value}`);
+    seen.add(value);
+  }
+}
+function sortedUnique(values) {
+  return [...new Set(values)].sort();
+}
+function sum(items, fn) {
+  return items.reduce((total, item) => total + fn(item), 0);
+}
+function findProductBenchmarkArtifacts(runDir) {
+  const manifestPath = join(runDir, "product-benchmark-manifest.json");
+  const recordsPath = join(runDir, "product-benchmark-records.jsonl");
+  if (existsSync(manifestPath) && statSync(manifestPath).isFile() && existsSync(recordsPath) && statSync(recordsPath).isFile()) {
+    return { manifestPath, recordsPath };
+  }
+  return null;
+}
+
+export {
+  productBenchmarkSplits,
+  validateProductBenchmarkManifest,
+  validateProductBenchmarkRecord,
+  productBenchmarkIntegrityFailures,
+  readProductBenchmarkRecords,
+  readProductBenchmarkManifest,
+  validateProductBenchmarkRun,
+  findProductBenchmarkArtifacts
+};
+//# sourceMappingURL=chunk-63MBSQTX.js.map

package/dist/chunk-63MBSQTX.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../src/product-benchmark/index.ts"],"sourcesContent":["import { existsSync, readFileSync, statSync } from 'node:fs'\nimport { dirname, join } from 'node:path'\nimport { ValidationError } from '../errors'\n\nexport const productBenchmarkSplits = ['practice', 'dev', 'holdout', 'safety', 'sentinel'] as const\n\nexport type ProductBenchmarkSplit = (typeof productBenchmarkSplits)[number]\n\nexport interface ProductBenchmarkRepoRef {\n  readonly url: string\n  readonly commit: string\n  readonly branch: string\n}\n\nexport interface ProductBenchmarkSubstrateVersions {\n  readonly agentEval: string\n  readonly agentRuntime: string\n  readonly agentInterface: string\n  readonly sandbox: string\n  readonly agentBench?: string\n}\n\nexport interface ProductBenchmarkProfileRef {\n  readonly id: string\n  readonly profileHash: string\n  readonly agentProfilePath: string\n}\n\nexport interface ProductBenchmarkArm {\n  readonly id: string\n  readonly profileId: string\n  readonly mutableSurfaces: readonly string[]\n  readonly policyAxes: Record<string, unknown>\n}\n\nexport interface ProductBenchmarkScenario {\n  readonly id: string\n  readonly split: ProductBenchmarkSplit\n  readonly tags: readonly string[]\n  readonly sourceAllowedForSynthesis: boolean\n}\n\nexport interface ProductBenchmarkBudgets {\n  readonly maxUsd: number\n  readonly maxCells: number\n  readonly maxWallMs: number\n}\n\nexport interface ProductBenchmarkManifest {\n  readonly schemaVersion: 1\n  readonly projectId: string\n  readonly benchmarkId: string\n  readonly repo: ProductBenchmarkRepoRef\n  readonly substrate: ProductBenchmarkSubstrateVersions\n  readonly profiles: readonly ProductBenchmarkProfileRef[]\n  readonly arms: readonly ProductBenchmarkArm[]\n  readonly scenarios: readonly ProductBenchmarkScenario[]\n  readonly budgets: ProductBenchmarkBudgets\n  readonly expectedArtifactDir: string\n}\n\nexport interface AgentProfileRuntimeReceipt {\n  readonly model: string\n  readonly harness: string\n  readonly backend: string\n  readonly reasoningEffort?: string\n}\n\nexport type RuntimeResolution = AgentProfileRuntimeReceipt\n\nexport interface ProductBenchmarkRecord {\n  readonly schemaVersion: 1\n  readonly projectId: string\n  readonly benchmarkId: string\n  readonly runId: string\n  readonly scenarioId: string\n  readonly split: ProductBenchmarkSplit\n  readonly armId: string\n  readonly rep: number\n  readonly agentProfile: {\n    readonly id: string\n    readonly hash: string\n    readonly path: string\n    readonly declared: RuntimeResolution\n    readonly resolved: RuntimeResolution\n  }\n  readonly model: {\n    readonly provider: string\n    readonly id: string\n  }\n  readonly backend: {\n    readonly kind: string\n    readonly version: string\n  }\n  readonly outcome: {\n    readonly pass: boolean\n    readonly score: number\n    readonly dimensions: Record<string, number>\n    readonly failureMode: string | null\n  }\n  readonly usage: {\n    readonly inputTokens: number\n    readonly outputTokens: number\n    readonly costUsd: number\n    readonly wallMs: number\n    readonly toolCalls: number\n  }\n  readonly integrity: {\n    readonly realBackend: boolean\n    readonly rawCapture: boolean\n    readonly traceCapture: boolean\n    readonly noStubRows: boolean\n    readonly priced: boolean\n    readonly profileMaterialized: boolean\n  }\n  readonly artifacts: {\n    readonly records: string\n    readonly traces: string\n    readonly raws: string\n    readonly scores: string\n    readonly workspace: string\n  }\n}\n\nexport interface ProductBenchmarkRunInput {\n  readonly manifestPath: string\n  readonly recordsPath: string\n  readonly artifactRoot?: string\n  readonly checkArtifacts?: boolean\n}\n\nexport interface ProductBenchmarkValidationReport {\n  readonly manifestPath: string\n  readonly recordsPath: string\n  readonly records: number\n  readonly projects: readonly string[]\n  readonly benchmarks: readonly string[]\n  readonly arms: readonly string[]\n  readonly scenarios: readonly string[]\n  readonly passed: number\n  readonly failed: number\n  readonly inputTokens: number\n  readonly outputTokens: number\n  readonly costUsd: number\n  readonly wallMs: number\n  readonly integrityFailures: readonly string[]\n  readonly missingArtifacts: readonly string[]\n}\n\nexport interface ProductBenchmarkArtifactPaths {\n  readonly manifestPath: string\n  readonly recordsPath: string\n}\n\nfunction isObject(value: unknown): value is Record<string, unknown> {\n  return Boolean(value) && typeof value === 'object' && !Array.isArray(value)\n}\n\nfunction fail(path: string, message: string): never {\n  throw new ValidationError(`${path}: ${message}`)\n}\n\nfunction wrapValidationError(path: string, err: unknown): never {\n  if (err instanceof ValidationError) {\n    throw new ValidationError(`${path}: ${err.message}`, { cause: err })\n  }\n  throw new ValidationError(`${path}: ${err instanceof Error ? err.message : String(err)}`, {\n    cause: err,\n  })\n}\n\nfunction expectObject(value: unknown, path: string): Record<string, unknown> {\n  if (!isObject(value)) fail(path, 'must be an object')\n  return value\n}\n\nfunction expectString(value: unknown, path: string): string {\n  if (typeof value !== 'string' || value.trim().length === 0)\n    fail(path, 'must be a non-empty string')\n  return value\n}\n\nfunction expectBoolean(value: unknown, path: string): boolean {\n  if (typeof value !== 'boolean') fail(path, 'must be a boolean')\n  return value\n}\n\nfunction expectNumber(\n  value: unknown,\n  path: string,\n  opts: { readonly min?: number; readonly max?: number; readonly integer?: boolean } = {},\n): number {\n  if (typeof value !== 'number' || !Number.isFinite(value)) fail(path, 'must be a finite number')\n  if (opts.integer && !Number.isInteger(value)) fail(path, 'must be an integer')\n  if (opts.min !== undefined && value < opts.min) fail(path, `must be >= ${opts.min}`)\n  if (opts.max !== undefined && value > opts.max) fail(path, `must be <= ${opts.max}`)\n  return value\n}\n\nfunction expectStringArray(value: unknown, path: string): string[] {\n  if (!Array.isArray(value)) fail(path, 'must be an array')\n  return value.map((entry, index) => expectString(entry, `${path}[${index}]`))\n}\n\nfunction expectObjectArray(value: unknown, path: string): Record<string, unknown>[] {\n  if (!Array.isArray(value)) fail(path, 'must be an array')\n  return value.map((entry, index) => expectObject(entry, `${path}[${index}]`))\n}\n\nfunction expectSplit(value: unknown, path: string): ProductBenchmarkSplit {\n  const split = expectString(value, path)\n  if (!productBenchmarkSplits.includes(split as ProductBenchmarkSplit)) {\n    fail(path, `must be one of ${productBenchmarkSplits.join(', ')}`)\n  }\n  return split as ProductBenchmarkSplit\n}\n\nfunction optionalString(value: unknown, path: string): string | undefined {\n  if (value === undefined) return undefined\n  return expectString(value, path)\n}\n\nfunction expectDimensions(value: unknown, path: string): Record<string, number> {\n  const obj = expectObject(value, path)\n  const out: Record<string, number> = {}\n  for (const [key, raw] of Object.entries(obj)) out[key] = expectNumber(raw, `${path}.${key}`)\n  return out\n}\n\nfunction expectRuntimeResolution(value: unknown, path: string): RuntimeResolution {\n  const obj = expectObject(value, path)\n  return {\n    model: expectString(obj.model, `${path}.model`),\n    harness: expectString(obj.harness, `${path}.harness`),\n    backend: expectString(obj.backend, `${path}.backend`),\n    ...(obj.reasoningEffort !== undefined\n      ? { reasoningEffort: optionalString(obj.reasoningEffort, `${path}.reasoningEffort`) }\n      : {}),\n  }\n}\n\nexport function validateProductBenchmarkManifest(value: unknown): ProductBenchmarkManifest {\n  const obj = expectObject(value, 'manifest')\n  if (obj.schemaVersion !== 1) fail('manifest.schemaVersion', 'must be 1')\n  const repo = expectObject(obj.repo, 'manifest.repo')\n  const substrate = expectObject(obj.substrate, 'manifest.substrate')\n  const profiles = expectObjectArray(obj.profiles, 'manifest.profiles').map((profile, index) => ({\n    id: expectString(profile.id, `manifest.profiles[${index}].id`),\n    profileHash: expectString(profile.profileHash, `manifest.profiles[${index}].profileHash`),\n    agentProfilePath: expectString(\n      profile.agentProfilePath,\n      `manifest.profiles[${index}].agentProfilePath`,\n    ),\n  }))\n  const arms = expectObjectArray(obj.arms, 'manifest.arms').map((arm, index) => ({\n    id: expectString(arm.id, `manifest.arms[${index}].id`),\n    profileId: expectString(arm.profileId, `manifest.arms[${index}].profileId`),\n    mutableSurfaces: expectStringArray(\n      arm.mutableSurfaces,\n      `manifest.arms[${index}].mutableSurfaces`,\n    ),\n    policyAxes: expectObject(arm.policyAxes, `manifest.arms[${index}].policyAxes`),\n  }))\n  const scenarios = expectObjectArray(obj.scenarios, 'manifest.scenarios').map(\n    (scenario, index) => ({\n      id: expectString(scenario.id, `manifest.scenarios[${index}].id`),\n      split: expectSplit(scenario.split, `manifest.scenarios[${index}].split`),\n      tags: expectStringArray(scenario.tags, `manifest.scenarios[${index}].tags`),\n      sourceAllowedForSynthesis: expectBoolean(\n        scenario.sourceAllowedForSynthesis,\n        `manifest.scenarios[${index}].sourceAllowedForSynthesis`,\n      ),\n    }),\n  )\n  const budgets = expectObject(obj.budgets, 'manifest.budgets')\n  if (profiles.length === 0) fail('manifest.profiles', 'must contain at least one profile')\n  if (arms.length === 0) fail('manifest.arms', 'must contain at least one arm')\n  if (scenarios.length === 0) fail('manifest.scenarios', 'must contain at least one scenario')\n  assertUnique(\n    profiles.map((profile) => profile.id),\n    'manifest.profiles.id',\n  )\n  assertUnique(\n    arms.map((arm) => arm.id),\n    'manifest.arms.id',\n  )\n  assertUnique(\n    scenarios.map((scenario) => scenario.id),\n    'manifest.scenarios.id',\n  )\n  const profileIds = new Set(profiles.map((profile) => profile.id))\n  for (const arm of arms) {\n    if (!profileIds.has(arm.profileId))\n      fail(`manifest.arms.${arm.id}.profileId`, `unknown profile ${arm.profileId}`)\n  }\n  return {\n    schemaVersion: 1,\n    projectId: expectString(obj.projectId, 'manifest.projectId'),\n    benchmarkId: expectString(obj.benchmarkId, 'manifest.benchmarkId'),\n    repo: {\n      url: expectString(repo.url, 'manifest.repo.url'),\n      commit: expectString(repo.commit, 'manifest.repo.commit'),\n      branch: expectString(repo.branch, 'manifest.repo.branch'),\n    },\n    substrate: {\n      agentEval: expectString(substrate.agentEval, 'manifest.substrate.agentEval'),\n      agentRuntime: expectString(substrate.agentRuntime, 'manifest.substrate.agentRuntime'),\n      agentInterface: expectString(substrate.agentInterface, 'manifest.substrate.agentInterface'),\n      sandbox: expectString(substrate.sandbox, 'manifest.substrate.sandbox'),\n      ...(substrate.agentBench !== undefined\n        ? { agentBench: expectString(substrate.agentBench, 'manifest.substrate.agentBench') }\n        : {}),\n    },\n    profiles,\n    arms,\n    scenarios,\n    budgets: {\n      maxUsd: expectNumber(budgets.maxUsd, 'manifest.budgets.maxUsd', { min: 0 }),\n      maxCells: expectNumber(budgets.maxCells, 'manifest.budgets.maxCells', {\n        min: 0,\n        integer: true,\n      }),\n      maxWallMs: expectNumber(budgets.maxWallMs, 'manifest.budgets.maxWallMs', {\n        min: 0,\n        integer: true,\n      }),\n    },\n    expectedArtifactDir: expectString(obj.expectedArtifactDir, 'manifest.expectedArtifactDir'),\n  }\n}\n\nexport function validateProductBenchmarkRecord(value: unknown): ProductBenchmarkRecord {\n  const obj = expectObject(value, 'record')\n  if (obj.schemaVersion !== 1) fail('record.schemaVersion', 'must be 1')\n  const agentProfile = expectObject(obj.agentProfile, 'record.agentProfile')\n  const model = expectObject(obj.model, 'record.model')\n  const backend = expectObject(obj.backend, 'record.backend')\n  const outcome = expectObject(obj.outcome, 'record.outcome')\n  const usage = expectObject(obj.usage, 'record.usage')\n  const integrity = expectObject(obj.integrity, 'record.integrity')\n  const artifacts = expectObject(obj.artifacts, 'record.artifacts')\n  const record: ProductBenchmarkRecord = {\n    schemaVersion: 1,\n    projectId: expectString(obj.projectId, 'record.projectId'),\n    benchmarkId: expectString(obj.benchmarkId, 'record.benchmarkId'),\n    runId: expectString(obj.runId, 'record.runId'),\n    scenarioId: expectString(obj.scenarioId, 'record.scenarioId'),\n    split: expectSplit(obj.split, 'record.split'),\n    armId: expectString(obj.armId, 'record.armId'),\n    rep: expectNumber(obj.rep, 'record.rep', { min: 1, integer: true }),\n    agentProfile: {\n      id: expectString(agentProfile.id, 'record.agentProfile.id'),\n      hash: expectString(agentProfile.hash, 'record.agentProfile.hash'),\n      path: expectString(agentProfile.path, 'record.agentProfile.path'),\n      declared: expectRuntimeResolution(agentProfile.declared, 'record.agentProfile.declared'),\n      resolved: expectRuntimeResolution(agentProfile.resolved, 'record.agentProfile.resolved'),\n    },\n    model: {\n      provider: expectString(model.provider, 'record.model.provider'),\n      id: expectString(model.id, 'record.model.id'),\n    },\n    backend: {\n      kind: expectString(backend.kind, 'record.backend.kind'),\n      version: expectString(backend.version, 'record.backend.version'),\n    },\n    outcome: {\n      pass: expectBoolean(outcome.pass, 'record.outcome.pass'),\n      score: expectNumber(outcome.score, 'record.outcome.score', { min: 0, max: 1 }),\n      dimensions: expectDimensions(outcome.dimensions, 'record.outcome.dimensions'),\n      failureMode:\n        outcome.failureMode === null\n          ? null\n          : expectString(outcome.failureMode, 'record.outcome.failureMode'),\n    },\n    usage: {\n      inputTokens: expectNumber(usage.inputTokens, 'record.usage.inputTokens', {\n        min: 0,\n        integer: true,\n      }),\n      outputTokens: expectNumber(usage.outputTokens, 'record.usage.outputTokens', {\n        min: 0,\n        integer: true,\n      }),\n      costUsd: expectNumber(usage.costUsd, 'record.usage.costUsd', { min: 0 }),\n      wallMs: expectNumber(usage.wallMs, 'record.usage.wallMs', { min: 0, integer: true }),\n      toolCalls: expectNumber(usage.toolCalls, 'record.usage.toolCalls', { min: 0, integer: true }),\n    },\n    integrity: {\n      realBackend: expectBoolean(integrity.realBackend, 'record.integrity.realBackend'),\n      rawCapture: expectBoolean(integrity.rawCapture, 'record.integrity.rawCapture'),\n      traceCapture: expectBoolean(integrity.traceCapture, 'record.integrity.traceCapture'),\n      noStubRows: expectBoolean(integrity.noStubRows, 'record.integrity.noStubRows'),\n      priced: expectBoolean(integrity.priced, 'record.integrity.priced'),\n      profileMaterialized: expectBoolean(\n        integrity.profileMaterialized,\n        'record.integrity.profileMaterialized',\n      ),\n    },\n    artifacts: {\n      records: expectString(artifacts.records, 'record.artifacts.records'),\n      traces: expectString(artifacts.traces, 'record.artifacts.traces'),\n      raws: expectString(artifacts.raws, 'record.artifacts.raws'),\n      scores: expectString(artifacts.scores, 'record.artifacts.scores'),\n      workspace: expectString(artifacts.workspace, 'record.artifacts.workspace'),\n    },\n  }\n  if (record.integrity.realBackend && record.usage.inputTokens + record.usage.outputTokens === 0) {\n    fail('record.usage', 'realBackend rows must carry non-zero token usage')\n  }\n  return record\n}\n\nexport function productBenchmarkIntegrityFailures(record: ProductBenchmarkRecord): string[] {\n  const failures: string[] = []\n  for (const [key, value] of Object.entries(record.integrity)) {\n    if (!value) failures.push(`${record.runId}:${key}=false`)\n  }\n  return failures\n}\n\nexport function readProductBenchmarkRecords(path: string): ProductBenchmarkRecord[] {\n  const lines = readFileSync(path, 'utf8')\n    .split('\\n')\n    .map((line) => line.trim())\n    .filter(Boolean)\n  const records: ProductBenchmarkRecord[] = []\n  for (const [index, line] of lines.entries()) {\n    try {\n      records.push(validateProductBenchmarkRecord(JSON.parse(line)))\n    } catch (err) {\n      wrapValidationError(`${path}:${index + 1}`, err)\n    }\n  }\n  return records\n}\n\nexport function readProductBenchmarkManifest(path: string): ProductBenchmarkManifest {\n  try {\n    return validateProductBenchmarkManifest(JSON.parse(readFileSync(path, 'utf8')))\n  } catch (err) {\n    wrapValidationError(path, err)\n  }\n}\n\nexport function validateProductBenchmarkRun(\n  input: ProductBenchmarkRunInput,\n): ProductBenchmarkValidationReport {\n  const manifest = readProductBenchmarkManifest(input.manifestPath)\n  const records = readProductBenchmarkRecords(input.recordsPath)\n  const manifestProjectBench = `${manifest.projectId}/${manifest.benchmarkId}`\n  const integrityFailures = records.flatMap(productBenchmarkIntegrityFailures)\n  const missingArtifacts =\n    input.checkArtifacts === false\n      ? []\n      : records.flatMap((record) =>\n          missingArtifactsForRecord(record, input.artifactRoot ?? dirname(input.recordsPath)),\n        )\n  for (const [index, record] of records.entries()) {\n    const recordProjectBench = `${record.projectId}/${record.benchmarkId}`\n    if (recordProjectBench !== manifestProjectBench) {\n      fail(\n        `records[${index}]`,\n        `project/benchmark ${recordProjectBench} does not match manifest ${manifestProjectBench}`,\n      )\n    }\n    if (!manifest.arms.some((arm) => arm.id === record.armId))\n      fail(`records[${index}].armId`, `unknown arm ${record.armId}`)\n    if (!manifest.scenarios.some((scenario) => scenario.id === record.scenarioId)) {\n      fail(`records[${index}].scenarioId`, `unknown scenario ${record.scenarioId}`)\n    }\n  }\n  return {\n    manifestPath: input.manifestPath,\n    recordsPath: input.recordsPath,\n    records: records.length,\n    projects: sortedUnique(records.map((record) => record.projectId)),\n    benchmarks: sortedUnique(records.map((record) => record.benchmarkId)),\n    arms: sortedUnique(records.map((record) => record.armId)),\n    scenarios: sortedUnique(records.map((record) => record.scenarioId)),\n    passed: records.filter((record) => record.outcome.pass).length,\n    failed: records.filter((record) => !record.outcome.pass).length,\n    inputTokens: sum(records, (record) => record.usage.inputTokens),\n    outputTokens: sum(records, (record) => record.usage.outputTokens),\n    costUsd: sum(records, (record) => record.usage.costUsd),\n    wallMs: sum(records, (record) => record.usage.wallMs),\n    integrityFailures,\n    missingArtifacts,\n  }\n}\n\nfunction missingArtifactsForRecord(record: ProductBenchmarkRecord, artifactRoot: string): string[] {\n  const missing: string[] = []\n  for (const [key, value] of Object.entries(record.artifacts)) {\n    const path = resolveArtifactPath(value, artifactRoot)\n    if (!existsSync(path)) missing.push(`${record.runId}:${key}:${value}`)\n  }\n  return missing\n}\n\nfunction resolveArtifactPath(value: string, artifactRoot: string): string {\n  return value.startsWith('/') ? value : join(artifactRoot, value)\n}\n\nfunction assertUnique(values: readonly string[], path: string): void {\n  const seen = new Set<string>()\n  for (const value of values) {\n    if (seen.has(value)) fail(path, `duplicate ${value}`)\n    seen.add(value)\n  }\n}\n\nfunction sortedUnique(values: readonly string[]): string[] {\n  return [...new Set(values)].sort()\n}\n\nfunction sum<T>(items: readonly T[], fn: (item: T) => number): number {\n  return items.reduce((total, item) => total + fn(item), 0)\n}\n\nexport function findProductBenchmarkArtifacts(\n  runDir: string,\n): ProductBenchmarkArtifactPaths | null {\n  const manifestPath = join(runDir, 'product-benchmark-manifest.json')\n  const recordsPath = join(runDir, 'product-benchmark-records.jsonl')\n  if (\n    existsSync(manifestPath) &&\n    statSync(manifestPath).isFile() &&\n    existsSync(recordsPath) &&\n    statSync(recordsPath).isFile()\n  ) {\n    return { manifestPath, recordsPath }\n  }\n  return null\n}\n"],"mappings":";;;;;AAAA,SAAS,YAAY,cAAc,gBAAgB;AACnD,SAAS,SAAS,YAAY;AAGvB,IAAM,yBAAyB,CAAC,YAAY,OAAO,WAAW,UAAU,UAAU;AAsJzF,SAAS,SAAS,OAAkD;AAClE,SAAO,QAAQ,KAAK,KAAK,OAAO,UAAU,YAAY,CAAC,MAAM,QAAQ,KAAK;AAC5E;AAEA,SAAS,KAAK,MAAc,SAAwB;AAClD,QAAM,IAAI,gBAAgB,GAAG,IAAI,KAAK,OAAO,EAAE;AACjD;AAEA,SAAS,oBAAoB,MAAc,KAAqB;AAC9D,MAAI,eAAe,iBAAiB;AAClC,UAAM,IAAI,gBAAgB,GAAG,IAAI,KAAK,IAAI,OAAO,IAAI,EAAE,OAAO,IAAI,CAAC;AAAA,EACrE;AACA,QAAM,IAAI,gBAAgB,GAAG,IAAI,KAAK,eAAe,QAAQ,IAAI,UAAU,OAAO,GAAG,CAAC,IAAI;AAAA,IACxF,OAAO;AAAA,EACT,CAAC;AACH;AAEA,SAAS,aAAa,OAAgB,MAAuC;AAC3E,MAAI,CAAC,SAAS,KAAK,EAAG,MAAK,MAAM,mBAAmB;AACpD,SAAO;AACT;AAEA,SAAS,aAAa,OAAgB,MAAsB;AAC1D,MAAI,OAAO,UAAU,YAAY,MAAM,KAAK,EAAE,WAAW;AACvD,SAAK,MAAM,4BAA4B;AACzC,SAAO;AACT;AAEA,SAAS,cAAc,OAAgB,MAAuB;AAC5D,MAAI,OAAO,UAAU,UAAW,MAAK,MAAM,mBAAmB;AAC9D,SAAO;AACT;AAEA,SAAS,aACP,OACA,MACA,OAAqF,CAAC,GAC9E;AACR,MAAI,OAAO,UAAU,YAAY,CAAC,OAAO,SAAS,KAAK,EAAG,MAAK,MAAM,yBAAyB;AAC9F,MAAI,KAAK,WAAW,CAAC,OAAO,UAAU,KAAK,EAAG,MAAK,MAAM,oBAAoB;AAC7E,MAAI,KAAK,QAAQ,UAAa,QAAQ,KAAK,IAAK,MAAK,MAAM,cAAc,KAAK,GAAG,EAAE;AACnF,MAAI,KAAK,QAAQ,UAAa,QAAQ,KAAK,IAAK,MAAK,MAAM,cAAc,KAAK,GAAG,EAAE;AACnF,SAAO;AACT;AAEA,SAAS,kBAAkB,OAAgB,MAAwB;AACjE,MAAI,CAAC,MAAM,QAAQ,KAAK,EAAG,MAAK,MAAM,kBAAkB;AACxD,SAAO,MAAM,IAAI,CAAC,OAAO,UAAU,aAAa,OAAO,GAAG,IAAI,IAAI,KAAK,GAAG,CAAC;AAC7E;AAEA,SAAS,kBAAkB,OAAgB,MAAyC;AAClF,MAAI,CAAC,MAAM,QAAQ,KAAK,EAAG,MAAK,MAAM,kBAAkB;AACxD,SAAO,MAAM,IAAI,CAAC,OAAO,UAAU,aAAa,OAAO,GAAG,IAAI,IAAI,KAAK,GAAG,CAAC;AAC7E;AAEA,SAAS,YAAY,OAAgB,MAAqC;AACxE,QAAM,QAAQ,aAAa,OAAO,IAAI;AACtC,MAAI,CAAC,uBAAuB,SAAS,KAA8B,GAAG;AACpE,SAAK,MAAM,kBAAkB,uBAAuB,KAAK,IAAI,CAAC,EAAE;AAAA,EAClE;AACA,SAAO;AACT;AAEA,SAAS,eAAe,OAAgB,MAAkC;AACxE,MAAI,UAAU,OAAW,QAAO;AAChC,SAAO,aAAa,OAAO,IAAI;AACjC;AAEA,SAAS,iBAAiB,OAAgB,MAAsC;AAC9E,QAAM,MAAM,aAAa,OAAO,IAAI;AACpC,QAAM,MAA8B,CAAC;AACrC,aAAW,CAAC,KAAK,GAAG,KAAK,OAAO,QAAQ,GAAG,EAAG,KAAI,GAAG,IAAI,aAAa,KAAK,GAAG,IAAI,IAAI,GAAG,EAAE;AAC3F,SAAO;AACT;AAEA,SAAS,wBAAwB,OAAgB,MAAiC;AAChF,QAAM,MAAM,aAAa,OAAO,IAAI;AACpC,SAAO;AAAA,IACL,OAAO,aAAa,IAAI,OAAO,GAAG,IAAI,QAAQ;AAAA,IAC9C,SAAS,aAAa,IAAI,SAAS,GAAG,IAAI,UAAU;AAAA,IACpD,SAAS,aAAa,IAAI,SAAS,GAAG,IAAI,UAAU;AAAA,IACpD,GAAI,IAAI,oBAAoB,SACxB,EAAE,iBAAiB,eAAe,IAAI,iBAAiB,GAAG,IAAI,kBAAkB,EAAE,IAClF,CAAC;AAAA,EACP;AACF;AAEO,SAAS,iCAAiC,OAA0C;AACzF,QAAM,MAAM,aAAa,OAAO,UAAU;AAC1C,MAAI,IAAI,kBAAkB,EAAG,MAAK,0BAA0B,WAAW;AACvE,QAAM,OAAO,aAAa,IAAI,MAAM,eAAe;AACnD,QAAM,YAAY,aAAa,IAAI,WAAW,oBAAoB;AAClE,QAAM,WAAW,kBAAkB,IAAI,UAAU,mBAAmB,EAAE,IAAI,CAAC,SAAS,WAAW;AAAA,IAC7F,IAAI,aAAa,QAAQ,IAAI,qBAAqB,KAAK,MAAM;AAAA,IAC7D,aAAa,aAAa,QAAQ,aAAa,qBAAqB,KAAK,eAAe;AAAA,IACxF,kBAAkB;AAAA,MAChB,QAAQ;AAAA,MACR,qBAAqB,KAAK;AAAA,IAC5B;AAAA,EACF,EAAE;AACF,QAAM,OAAO,kBAAkB,IAAI,MAAM,eAAe,EAAE,IAAI,CAAC,KAAK,WAAW;AAAA,IAC7E,IAAI,aAAa,IAAI,IAAI,iBAAiB,KAAK,MAAM;AAAA,IACrD,WAAW,aAAa,IAAI,WAAW,iBAAiB,KAAK,aAAa;AAAA,IAC1E,iBAAiB;AAAA,MACf,IAAI;AAAA,MACJ,iBAAiB,KAAK;AAAA,IACxB;AAAA,IACA,YAAY,aAAa,IAAI,YAAY,iBAAiB,KAAK,cAAc;AAAA,EAC/E,EAAE;AACF,QAAM,YAAY,kBAAkB,IAAI,WAAW,oBAAoB,EAAE;AAAA,IACvE,CAAC,UAAU,WAAW;AAAA,MACpB,IAAI,aAAa,SAAS,IAAI,sBAAsB,KAAK,MAAM;AAAA,MAC/D,OAAO,YAAY,SAAS,OAAO,sBAAsB,KAAK,SAAS;AAAA,MACvE,MAAM,kBAAkB,SAAS,MAAM,sBAAsB,KAAK,QAAQ;AAAA,MAC1E,2BAA2B;AAAA,QACzB,SAAS;AAAA,QACT,sBAAsB,KAAK;AAAA,MAC7B;AAAA,IACF;AAAA,EACF;AACA,QAAM,UAAU,aAAa,IAAI,SAAS,kBAAkB;AAC5D,MAAI,SAAS,WAAW,EAAG,MAAK,qBAAqB,mCAAmC;AACxF,MAAI,KAAK,WAAW,EAAG,MAAK,iBAAiB,+BAA+B;AAC5E,MAAI,UAAU,WAAW,EAAG,MAAK,sBAAsB,oCAAoC;AAC3F;AAAA,IACE,SAAS,IAAI,CAAC,YAAY,QAAQ,EAAE;AAAA,IACpC;AAAA,EACF;AACA;AAAA,IACE,KAAK,IAAI,CAAC,QAAQ,IAAI,EAAE;AAAA,IACxB;AAAA,EACF;AACA;AAAA,IACE,UAAU,IAAI,CAAC,aAAa,SAAS,EAAE;AAAA,IACvC;AAAA,EACF;AACA,QAAM,aAAa,IAAI,IAAI,SAAS,IAAI,CAAC,YAAY,QAAQ,EAAE,CAAC;AAChE,aAAW,OAAO,MAAM;AACtB,QAAI,CAAC,WAAW,IAAI,IAAI,SAAS;AAC/B,WAAK,iBAAiB,IAAI,EAAE,cAAc,mBAAmB,IAAI,SAAS,EAAE;AAAA,EAChF;AACA,SAAO;AAAA,IACL,eAAe;AAAA,IACf,WAAW,aAAa,IAAI,WAAW,oBAAoB;AAAA,IAC3D,aAAa,aAAa,IAAI,aAAa,sBAAsB;AAAA,IACjE,MAAM;AAAA,MACJ,KAAK,aAAa,KAAK,KAAK,mBAAmB;AAAA,MAC/C,QAAQ,aAAa,KAAK,QAAQ,sBAAsB;AAAA,MACxD,QAAQ,aAAa,KAAK,QAAQ,sBAAsB;AAAA,IAC1D;AAAA,IACA,WAAW;AAAA,MACT,WAAW,aAAa,UAAU,WAAW,8BAA8B;AAAA,MAC3E,cAAc,aAAa,UAAU,cAAc,iCAAiC;AAAA,MACpF,gBAAgB,aAAa,UAAU,gBAAgB,mCAAmC;AAAA,MAC1F,SAAS,aAAa,UAAU,SAAS,4BAA4B;AAAA,MACrE,GAAI,UAAU,eAAe,SACzB,EAAE,YAAY,aAAa,UAAU,YAAY,+BAA+B,EAAE,IAClF,CAAC;AAAA,IACP;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA,SAAS;AAAA,MACP,QAAQ,aAAa,QAAQ,QAAQ,2BAA2B,EAAE,KAAK,EAAE,CAAC;AAAA,MAC1E,UAAU,aAAa,QAAQ,UAAU,6BAA6B;AAAA,QACpE,KAAK;AAAA,QACL,SAAS;AAAA,MACX,CAAC;AAAA,MACD,WAAW,aAAa,QAAQ,WAAW,8BAA8B;AAAA,QACvE,KAAK;AAAA,QACL,SAAS;AAAA,MACX,CAAC;AAAA,IACH;AAAA,IACA,qBAAqB,aAAa,IAAI,qBAAqB,8BAA8B;AAAA,EAC3F;AACF;AAEO,SAAS,+BAA+B,OAAwC;AACrF,QAAM,MAAM,aAAa,OAAO,QAAQ;AACxC,MAAI,IAAI,kBAAkB,EAAG,MAAK,wBAAwB,WAAW;AACrE,QAAM,eAAe,aAAa,IAAI,cAAc,qBAAqB;AACzE,QAAM,QAAQ,aAAa,IAAI,OAAO,cAAc;AACpD,QAAM,UAAU,aAAa,IAAI,SAAS,gBAAgB;AAC1D,QAAM,UAAU,aAAa,IAAI,SAAS,gBAAgB;AAC1D,QAAM,QAAQ,aAAa,IAAI,OAAO,cAAc;AACpD,QAAM,YAAY,aAAa,IAAI,WAAW,kBAAkB;AAChE,QAAM,YAAY,aAAa,IAAI,WAAW,kBAAkB;AAChE,QAAM,SAAiC;AAAA,IACrC,eAAe;AAAA,IACf,WAAW,aAAa,IAAI,WAAW,kBAAkB;AAAA,IACzD,aAAa,aAAa,IAAI,aAAa,oBAAoB;AAAA,IAC/D,OAAO,aAAa,IAAI,OAAO,cAAc;AAAA,IAC7C,YAAY,aAAa,IAAI,YAAY,mBAAmB;AAAA,IAC5D,OAAO,YAAY,IAAI,OAAO,cAAc;AAAA,IAC5C,OAAO,aAAa,IAAI,OAAO,cAAc;AAAA,IAC7C,KAAK,aAAa,IAAI,KAAK,cAAc,EAAE,KAAK,GAAG,SAAS,KAAK,CAAC;AAAA,IAClE,cAAc;AAAA,MACZ,IAAI,aAAa,aAAa,IAAI,wBAAwB;AAAA,MAC1D,MAAM,aAAa,aAAa,MAAM,0BAA0B;AAAA,MAChE,MAAM,aAAa,aAAa,MAAM,0BAA0B;AAAA,MAChE,UAAU,wBAAwB,aAAa,UAAU,8BAA8B;AAAA,MACvF,UAAU,wBAAwB,aAAa,UAAU,8BAA8B;AAAA,IACzF;AAAA,IACA,OAAO;AAAA,MACL,UAAU,aAAa,MAAM,UAAU,uBAAuB;AAAA,MAC9D,IAAI,aAAa,MAAM,IAAI,iBAAiB;AAAA,IAC9C;AAAA,IACA,SAAS;AAAA,MACP,MAAM,aAAa,QAAQ,MAAM,qBAAqB;AAAA,MACtD,SAAS,aAAa,QAAQ,SAAS,wBAAwB;AAAA,IACjE;AAAA,IACA,SAAS;AAAA,MACP,MAAM,cAAc,QAAQ,MAAM,qBAAqB;AAAA,MACvD,OAAO,aAAa,QAAQ,OAAO,wBAAwB,EAAE,KAAK,GAAG,KAAK,EAAE,CAAC;AAAA,MAC7E,YAAY,iBAAiB,QAAQ,YAAY,2BAA2B;AAAA,MAC5E,aACE,QAAQ,gBAAgB,OACpB,OACA,aAAa,QAAQ,aAAa,4BAA4B;AAAA,IACtE;AAAA,IACA,OAAO;AAAA,MACL,aAAa,aAAa,MAAM,aAAa,4BAA4B;AAAA,QACvE,KAAK;AAAA,QACL,SAAS;AAAA,MACX,CAAC;AAAA,MACD,cAAc,aAAa,MAAM,cAAc,6BAA6B;AAAA,QAC1E,KAAK;AAAA,QACL,SAAS;AAAA,MACX,CAAC;AAAA,MACD,SAAS,aAAa,MAAM,SAAS,wBAAwB,EAAE,KAAK,EAAE,CAAC;AAAA,MACvE,QAAQ,aAAa,MAAM,QAAQ,uBAAuB,EAAE,KAAK,GAAG,SAAS,KAAK,CAAC;AAAA,MACnF,WAAW,aAAa,MAAM,WAAW,0BAA0B,EAAE,KAAK,GAAG,SAAS,KAAK,CAAC;AAAA,IAC9F;AAAA,IACA,WAAW;AAAA,MACT,aAAa,cAAc,UAAU,aAAa,8BAA8B;AAAA,MAChF,YAAY,cAAc,UAAU,YAAY,6BAA6B;AAAA,MAC7E,cAAc,cAAc,UAAU,cAAc,+BAA+B;AAAA,MACnF,YAAY,cAAc,UAAU,YAAY,6BAA6B;AAAA,MAC7E,QAAQ,cAAc,UAAU,QAAQ,yBAAyB;AAAA,MACjE,qBAAqB;AAAA,QACnB,UAAU;AAAA,QACV;AAAA,MACF;AAAA,IACF;AAAA,IACA,WAAW;AAAA,MACT,SAAS,aAAa,UAAU,SAAS,0BAA0B;AAAA,MACnE,QAAQ,aAAa,UAAU,QAAQ,yBAAyB;AAAA,MAChE,MAAM,aAAa,UAAU,MAAM,uBAAuB;AAAA,MAC1D,QAAQ,aAAa,UAAU,QAAQ,yBAAyB;AAAA,MAChE,WAAW,aAAa,UAAU,WAAW,4BAA4B;AAAA,IAC3E;AAAA,EACF;AACA,MAAI,OAAO,UAAU,eAAe,OAAO,MAAM,cAAc,OAAO,MAAM,iBAAiB,GAAG;AAC9F,SAAK,gBAAgB,kDAAkD;AAAA,EACzE;AACA,SAAO;AACT;AAEO,SAAS,kCAAkC,QAA0C;AAC1F,QAAM,WAAqB,CAAC;AAC5B,aAAW,CAAC,KAAK,KAAK,KAAK,OAAO,QAAQ,OAAO,SAAS,GAAG;AAC3D,QAAI,CAAC,MAAO,UAAS,KAAK,GAAG,OAAO,KAAK,IAAI,GAAG,QAAQ;AAAA,EAC1D;AACA,SAAO;AACT;AAEO,SAAS,4BAA4B,MAAwC;AAClF,QAAM,QAAQ,aAAa,MAAM,MAAM,EACpC,MAAM,IAAI,EACV,IAAI,CAAC,SAAS,KAAK,KAAK,CAAC,EACzB,OAAO,OAAO;AACjB,QAAM,UAAoC,CAAC;AAC3C,aAAW,CAAC,OAAO,IAAI,KAAK,MAAM,QAAQ,GAAG;AAC3C,QAAI;AACF,cAAQ,KAAK,+BAA+B,KAAK,MAAM,IAAI,CAAC,CAAC;AAAA,IAC/D,SAAS,KAAK;AACZ,0BAAoB,GAAG,IAAI,IAAI,QAAQ,CAAC,IAAI,GAAG;AAAA,IACjD;AAAA,EACF;AACA,SAAO;AACT;AAEO,SAAS,6BAA6B,MAAwC;AACnF,MAAI;AACF,WAAO,iCAAiC,KAAK,MAAM,aAAa,MAAM,MAAM,CAAC,CAAC;AAAA,EAChF,SAAS,KAAK;AACZ,wBAAoB,MAAM,GAAG;AAAA,EAC/B;AACF;AAEO,SAAS,4BACd,OACkC;AAClC,QAAM,WAAW,6BAA6B,MAAM,YAAY;AAChE,QAAM,UAAU,4BAA4B,MAAM,WAAW;AAC7D,QAAM,uBAAuB,GAAG,SAAS,SAAS,IAAI,SAAS,WAAW;AAC1E,QAAM,oBAAoB,QAAQ,QAAQ,iCAAiC;AAC3E,QAAM,mBACJ,MAAM,mBAAmB,QACrB,CAAC,IACD,QAAQ;AAAA,IAAQ,CAAC,WACf,0BAA0B,QAAQ,MAAM,gBAAgB,QAAQ,MAAM,WAAW,CAAC;AAAA,EACpF;AACN,aAAW,CAAC,OAAO,MAAM,KAAK,QAAQ,QAAQ,GAAG;AAC/C,UAAM,qBAAqB,GAAG,OAAO,SAAS,IAAI,OAAO,WAAW;AACpE,QAAI,uBAAuB,sBAAsB;AAC/C;AAAA,QACE,WAAW,KAAK;AAAA,QAChB,qBAAqB,kBAAkB,4BAA4B,oBAAoB;AAAA,MACzF;AAAA,IACF;AACA,QAAI,CAAC,SAAS,KAAK,KAAK,CAAC,QAAQ,IAAI,OAAO,OAAO,KAAK;AACtD,WAAK,WAAW,KAAK,WAAW,eAAe,OAAO,KAAK,EAAE;AAC/D,QAAI,CAAC,SAAS,UAAU,KAAK,CAAC,aAAa,SAAS,OAAO,OAAO,UAAU,GAAG;AAC7E,WAAK,WAAW,KAAK,gBAAgB,oBAAoB,OAAO,UAAU,EAAE;AAAA,IAC9E;AAAA,EACF;AACA,SAAO;AAAA,IACL,cAAc,MAAM;AAAA,IACpB,aAAa,MAAM;AAAA,IACnB,SAAS,QAAQ;AAAA,IACjB,UAAU,aAAa,QAAQ,IAAI,CAAC,WAAW,OAAO,SAAS,CAAC;AAAA,IAChE,YAAY,aAAa,QAAQ,IAAI,CAAC,WAAW,OAAO,WAAW,CAAC;AAAA,IACpE,MAAM,aAAa,QAAQ,IAAI,CAAC,WAAW,OAAO,KAAK,CAAC;AAAA,IACxD,WAAW,aAAa,QAAQ,IAAI,CAAC,WAAW,OAAO,UAAU,CAAC;AAAA,IAClE,QAAQ,QAAQ,OAAO,CAAC,WAAW,OAAO,QAAQ,IAAI,EAAE;AAAA,IACxD,QAAQ,QAAQ,OAAO,CAAC,WAAW,CAAC,OAAO,QAAQ,IAAI,EAAE;AAAA,IACzD,aAAa,IAAI,SAAS,CAAC,WAAW,OAAO,MAAM,WAAW;AAAA,IAC9D,cAAc,IAAI,SAAS,CAAC,WAAW,OAAO,MAAM,YAAY;AAAA,IAChE,SAAS,IAAI,SAAS,CAAC,WAAW,OAAO,MAAM,OAAO;AAAA,IACtD,QAAQ,IAAI,SAAS,CAAC,WAAW,OAAO,MAAM,MAAM;AAAA,IACpD;AAAA,IACA;AAAA,EACF;AACF;AAEA,SAAS,0BAA0B,QAAgC,cAAgC;AACjG,QAAM,UAAoB,CAAC;AAC3B,aAAW,CAAC,KAAK,KAAK,KAAK,OAAO,QAAQ,OAAO,SAAS,GAAG;AAC3D,UAAM,OAAO,oBAAoB,OAAO,YAAY;AACpD,QAAI,CAAC,WAAW,IAAI,EAAG,SAAQ,KAAK,GAAG,OAAO,KAAK,IAAI,GAAG,IAAI,KAAK,EAAE;AAAA,EACvE;AACA,SAAO;AACT;AAEA,SAAS,oBAAoB,OAAe,cAA8B;AACxE,SAAO,MAAM,WAAW,GAAG,IAAI,QAAQ,KAAK,cAAc,KAAK;AACjE;AAEA,SAAS,aAAa,QAA2B,MAAoB;AACnE,QAAM,OAAO,oBAAI,IAAY;AAC7B,aAAW,SAAS,QAAQ;AAC1B,QAAI,KAAK,IAAI,KAAK,EAAG,MAAK,MAAM,aAAa,KAAK,EAAE;AACpD,SAAK,IAAI,KAAK;AAAA,EAChB;AACF;AAEA,SAAS,aAAa,QAAqC;AACzD,SAAO,CAAC,GAAG,IAAI,IAAI,MAAM,CAAC,EAAE,KAAK;AACnC;AAEA,SAAS,IAAO,OAAqB,IAAiC;AACpE,SAAO,MAAM,OAAO,CAAC,OAAO,SAAS,QAAQ,GAAG,IAAI,GAAG,CAAC;AAC1D;AAEO,SAAS,8BACd,QACsC;AACtC,QAAM,eAAe,KAAK,QAAQ,iCAAiC;AACnE,QAAM,cAAc,KAAK,QAAQ,iCAAiC;AAClE,MACE,WAAW,YAAY,KACvB,SAAS,YAAY,EAAE,OAAO,KAC9B,WAAW,WAAW,KACtB,SAAS,WAAW,EAAE,OAAO,GAC7B;AACA,WAAO,EAAE,cAAc,YAAY;AAAA,EACrC;AACA,SAAO;AACT;","names":[]}

package/dist/{chunk-G7IB3GJ5.js → chunk-CHIFZIQD.js}

@@ -1,10 +1,10 @@
 import {
   runCanaries,
   scoreRedTeamOutput
-} from "./chunk-2KTBHICD.js";
+} from "./chunk-NK77GPUH.js";
 import {
   runCampaign
-} from "./chunk-HRGTA6U5.js";
+} from "./chunk-XIOQHCHU.js";
 import {
   detectRewardHacking
 } from "./chunk-AIGWQEME.js";
@@ -432,4 +432,4 @@ export {
   runEval,
   evolutionaryProposer
 };
-//# sourceMappingURL=chunk-G7IB3GJ5.js.map
+//# sourceMappingURL=chunk-CHIFZIQD.js.map

package/dist/{chunk-VWQ6PO5O.js → chunk-GUII3E73.js}

@@ -656,6 +656,56 @@ function extractProducedState(events) {
 
 // src/agent-profile.ts
 import { createHash } from "crypto";
+import { harnessSupportsModel } from "@tangle-network/agent-interface";
+var CODING_HARNESSES = [
+  "opencode",
+  "claude-code",
+  "codex",
+  "kimi-code"
+];
+function expandProfileAxes(spec) {
+  const harnesses = spec.harnesses ?? CODING_HARNESSES;
+  if (harnesses.length === 0) throw new ValidationError("expandProfileAxes: no harnesses to sweep");
+  const baseModel = spec.base.model?.default;
+  const models = spec.models ?? (baseModel ? [baseModel] : []);
+  if (models.length === 0) {
+    throw new ValidationError(
+      "expandProfileAxes: no models to sweep \u2014 base profile has no model.default and none were supplied"
+    );
+  }
+  const out = [];
+  const seen = /* @__PURE__ */ new Set();
+  for (const harness of harnesses) {
+    for (const model of models) {
+      if (!spec.keepIncompatible && !harnessSupportsModel(harness, model)) continue;
+      const profile = {
+        ...spec.base,
+        name: `${spec.base.name ?? "agent"}/${harness}/${model}`,
+        model: { ...spec.base.model, default: model },
+        metadata: { ...spec.base.metadata ?? {}, harness, harnessModel: model }
+      };
+      const id = agentProfileId(profile);
+      if (seen.has(id)) continue;
+      seen.add(id);
+      out.push(profile);
+    }
+  }
+  if (out.length === 0) {
+    throw new ValidationError(
+      `expandProfileAxes: every (harness, model) pair was incompatible (harnesses=[${harnesses.join(", ")}], models=[${models.join(", ")}]). Widen the models or pass keepIncompatible.`
+    );
+  }
+  return out;
+}
+function harnessAxisOf(profile) {
+  const m = profile.metadata;
+  const harness = m?.harness;
+  const model = m?.harnessModel;
+  if (typeof harness === "string" && typeof model === "string") {
+    return { harness, model };
+  }
+  return void 0;
+}
 function agentProfileId(profile) {
   const label = pathSafeProfileLabel(agentProfileDisplayLabel(profile)) ?? "profile";
   return `${label}-${agentProfileHash(profile).slice(0, 16)}`;
@@ -711,8 +761,11 @@ export {
   createTokenRecallChecker,
   llmJudge,
   extractProducedState,
+  CODING_HARNESSES,
+  expandProfileAxes,
+  harnessAxisOf,
   agentProfileId,
   agentProfileModelId,
   agentProfileHash
 };
-//# sourceMappingURL=chunk-VWQ6PO5O.js.map
+//# sourceMappingURL=chunk-GUII3E73.js.map