@tangle-network/agent-eval 0.102.1 → 0.103.0

package/dist/chunk-RQNOLV3I.js

@@ -0,0 +1,855 @@
+import {
+  ValidationError
+} from "./chunk-3BFEG2F6.js";
+
+// src/product-benchmark/index.ts
+import { existsSync as existsSync2, readFileSync as readFileSync2, statSync } from "fs";
+import { dirname as dirname2, join as join2 } from "path";
+
+// src/product-benchmark/export.ts
+import { spawnSync } from "child_process";
+import { createHash } from "crypto";
+import { cpSync, existsSync, mkdirSync, readFileSync, writeFileSync } from "fs";
+import { basename, dirname, isAbsolute, join, relative, resolve } from "path";
+var productBenchmarkMutableSurfaces = [
+  "prompt",
+  "resources.files",
+  "tools",
+  "mcp",
+  "hooks",
+  "subagents"
+];
+function sha256(value) {
+  return `sha256:${createHash("sha256").update(value).digest("hex")}`;
+}
+function safePathPart(value) {
+  return value.replace(/[^a-zA-Z0-9_.-]+/g, "-").replace(/^-+|-+$/g, "").slice(0, 80) || "artifact";
+}
+function git(args, fallback) {
+  const res = spawnSync("git", args, { cwd: process.cwd(), encoding: "utf8" });
+  const value = res.status === 0 ? res.stdout.trim() : "";
+  return value.length > 0 ? value : fallback;
+}
+function knownEnv(name) {
+  const value = process.env[name]?.trim();
+  return value && value !== "unknown" ? value : void 0;
+}
+function ciBranchName() {
+  const direct = knownEnv("GITHUB_HEAD_REF") ?? knownEnv("GITHUB_REF_NAME") ?? knownEnv("VERCEL_GIT_COMMIT_REF");
+  if (direct) return direct;
+  const ref = knownEnv("GITHUB_REF");
+  return ref?.startsWith("refs/heads/") ? ref.slice("refs/heads/".length) : void 0;
+}
+function repoBranch() {
+  const ci = ciBranchName();
+  if (ci) return ci;
+  const branch = git(["branch", "--show-current"], "");
+  if (branch) return branch;
+  const name = git(["name-rev", "--name-only", "--exclude=tags/*", "HEAD"], "");
+  if (name && name !== "undefined") return name;
+  return `detached:${git(["rev-parse", "--short", "HEAD"], "unknown")}`;
+}
+function productBenchmarkRepoIdentity() {
+  return {
+    url: git(["config", "--get", "remote.origin.url"], "unknown"),
+    commit: git(["rev-parse", "HEAD"], "unknown"),
+    branch: repoBranch()
+  };
+}
+function packageVersion(name) {
+  const pkg = JSON.parse(readFileSync(resolve("package.json"), "utf8"));
+  if (pkg.name === name && pkg.version) return pkg.version;
+  const declared = pkg.dependencies?.[name] ?? pkg.devDependencies?.[name];
+  if (declared) return declared;
+  const installed = resolve("node_modules", name, "package.json");
+  if (existsSync(installed)) {
+    const installedPkg = JSON.parse(readFileSync(installed, "utf8"));
+    if (installedPkg.version) return installedPkg.version;
+  }
+  return "unknown";
+}
+function readRunRecords(path) {
+  const lines = readFileSync(path, "utf8").split("\n").map((line) => line.trim()).filter(Boolean);
+  return lines.map((line, index) => {
+    let parsed;
+    try {
+      parsed = JSON.parse(line);
+    } catch (err) {
+      throw new ValidationError(
+        `${path}:${index + 1}: ${err instanceof Error ? err.message : String(err)}`,
+        { cause: err }
+      );
+    }
+    return expectRunRecordShape(parsed, `${path}:${index + 1}`);
+  });
+}
+function expectRunRecordShape(value, path) {
+  if (!value || typeof value !== "object" || Array.isArray(value)) {
+    throw new ValidationError(`${path}: run record must be an object`);
+  }
+  const obj = value;
+  for (const key of ["runId", "model"]) {
+    if (typeof obj[key] !== "string" || obj[key].length === 0) {
+      throw new ValidationError(`${path}: run record ${key} must be a non-empty string`);
+    }
+  }
+  for (const key of ["costUsd", "wallMs"]) {
+    if (typeof obj[key] !== "number" || !Number.isFinite(obj[key])) {
+      throw new ValidationError(`${path}: run record ${key} must be a finite number`);
+    }
+  }
+  const tokenUsage = obj.tokenUsage;
+  if (!tokenUsage || typeof tokenUsage.input !== "number" || typeof tokenUsage.output !== "number") {
+    throw new ValidationError(`${path}: run record tokenUsage.input/output must be numbers`);
+  }
+  const outcome = obj.outcome;
+  if (!outcome || !outcome.raw || typeof outcome.raw !== "object") {
+    throw new ValidationError(`${path}: run record outcome.raw must be an object`);
+  }
+  return value;
+}
+function clamp01(value) {
+  return Math.max(0, Math.min(1, value));
+}
+function splitOf(record, opts) {
+  const custom = opts.classifySplit?.(record);
+  if (custom) return custom;
+  if (record.outcome.raw.safety === 1) return "safety";
+  if (record.splitTag === "holdout") return "holdout";
+  if (record.splitTag === "dev") return "dev";
+  return "practice";
+}
+function scoreOf(record) {
+  const score = record.outcome.holdoutScore ?? record.outcome.searchScore;
+  if (typeof score === "number" && Number.isFinite(score)) return clamp01(score);
+  const rawScore = record.outcome.raw.score ?? record.outcome.raw.composite;
+  return typeof rawScore === "number" && Number.isFinite(rawScore) ? clamp01(rawScore) : 0;
+}
+function rawPassOf(record) {
+  const rawPass = record.outcome.raw.pass;
+  if (typeof rawPass === "boolean") return rawPass;
+  if (typeof rawPass === "number" && Number.isFinite(rawPass)) return rawPass >= 1;
+  return null;
+}
+function passOf(record, score, threshold) {
+  const rawPass = rawPassOf(record);
+  if (rawPass !== null) return rawPass && !record.failureMode;
+  return score >= threshold && !record.failureMode;
+}
+function failureModeOf(record, score, threshold) {
+  if (record.failureMode) return record.failureMode;
+  const belowThreshold = `quality-below-threshold: ${Math.round(score * 100)}% < ${Math.round(threshold * 100)}%`;
+  if (rawPassOf(record) === false) {
+    return score < threshold ? belowThreshold : "product-pass-failed";
+  }
+  if (passOf(record, score, threshold)) return null;
+  return belowThreshold;
+}
+function numericDimensions(record) {
+  const dimensions = {};
+  for (const [key, value] of Object.entries(record.outcome.raw ?? {})) {
+    if (typeof value === "number" && Number.isFinite(value)) dimensions[key] = value;
+  }
+  if (Object.keys(dimensions).length === 0) dimensions.score = scoreOf(record);
+  return dimensions;
+}
+function armIdOf(record) {
+  if (record.candidateId) return record.candidateId;
+  const variant = record.agentProfile?.dimensions?.variant;
+  if (typeof variant === "string" && variant.length > 0) return variant;
+  const variantId = record.agentProfile?.dimensions?.variantId;
+  if (typeof variantId === "string" && variantId.length > 0) return variantId;
+  return "production-profile";
+}
+function backendOf(record) {
+  const backend = record.agentProfile?.dimensions?.backend;
+  return typeof backend === "string" && backend.length > 0 ? backend : "unknown";
+}
+function modelProvider(record) {
+  const backend = backendOf(record);
+  if (backend === "cli-bridge") return "cli-bridge";
+  if (backend === "sandbox") return "router";
+  if (record.model.startsWith("router/")) return "router";
+  return backend === "unknown" ? "router" : backend;
+}
+function runtimeResolution(record, opts) {
+  const reasoningEffort = record.agentProfile?.dimensions?.reasoningLevel;
+  return {
+    model: record.agentProfile?.model ?? record.model,
+    harness: record.agentProfile?.harness?.id ?? `${opts.projectId}-canonical-eval`,
+    backend: backendOf(record),
+    ...typeof reasoningEffort === "string" && reasoningEffort.length > 0 ? { reasoningEffort } : {}
+  };
+}
+function sourceProfileHash(record) {
+  return record.agentProfile?.sourceProfile?.hash ?? record.agentProfile?.cellId ?? sha256(
+    JSON.stringify({
+      candidateId: record.candidateId,
+      promptHash: record.promptHash,
+      configHash: record.configHash
+    })
+  );
+}
+function materializedProfileHash(record, runtime) {
+  return sha256(
+    JSON.stringify({ sourceProfileHash: sourceProfileHash(record), model: runtime.model })
+  );
+}
+function profileIdOf(record, armId, runtime, opts) {
+  const base = record.agentProfile?.profileId ?? opts.fallbackProfileId ?? armId;
+  const withArm = base.endsWith(`:${armId}`) ? base : `${base}:${armId}`;
+  return `${withArm}:${runtime.model}`;
+}
+function toolCallsOf(record, runDir, opts) {
+  const raw = record.outcome.raw;
+  const primary = Number(raw.tool_call_count ?? raw.tool_calls ?? raw.toolCalls ?? 0);
+  if (Number.isFinite(primary) && primary > 0) return primary;
+  const fallback = opts.toolCallFallback?.(record, runDir);
+  if (fallback !== void 0 && Number.isFinite(fallback) && fallback > 0) return fallback;
+  return Number.isFinite(primary) ? primary : 0;
+}
+var TRACE_CANDIDATES = ["traces.jsonl", "traces", "trace", "trace-store"];
+var RAW_CANDIDATES = ["raws.jsonl", "raw-events"];
+var SCORE_CANDIDATES = ["scores.json", "manifest.json"];
+function firstExisting(runDir, candidates) {
+  return candidates.find((candidate) => existsSync(join(runDir, candidate))) ?? candidates[0];
+}
+function relativeArtifacts(runDir) {
+  return {
+    records: "records.jsonl",
+    traces: firstExisting(runDir, TRACE_CANDIDATES),
+    raws: firstExisting(runDir, RAW_CANDIDATES),
+    scores: firstExisting(runDir, SCORE_CANDIDATES),
+    workspace: existsSync(join(runDir, "workspace")) ? "workspace" : "."
+  };
+}
+function absoluteArtifacts(runDir) {
+  const rel = relativeArtifacts(runDir);
+  return {
+    records: resolve(runDir, rel.records),
+    traces: resolve(runDir, rel.traces),
+    raws: resolve(runDir, rel.raws),
+    scores: resolve(runDir, rel.scores),
+    workspace: resolve(runDir, rel.workspace)
+  };
+}
+function materializeRunArtifacts(runDir, outDir, index) {
+  if (resolve(runDir) === resolve(outDir)) return relativeArtifacts(runDir);
+  const rel = relative(runDir, outDir);
+  if (rel && !rel.startsWith("..") && !isAbsolute(rel)) {
+    throw new ValidationError(`outDir must not be nested inside runDir: ${outDir}`);
+  }
+  const id = `${String(index + 1).padStart(2, "0")}-${safePathPart(basename(runDir))}-${sha256(runDir).slice(7, 19)}`;
+  const destRel = join("source-runs", id);
+  const destAbs = join(outDir, destRel);
+  mkdirSync(dirname(destAbs), { recursive: true });
+  cpSync(runDir, destAbs, { recursive: true, force: true });
+  const source = relativeArtifacts(runDir);
+  return {
+    records: join(destRel, source.records),
+    traces: join(destRel, source.traces),
+    raws: join(destRel, source.raws),
+    scores: join(destRel, source.scores),
+    workspace: source.workspace === "." ? destRel : join(destRel, source.workspace)
+  };
+}
+function existsArtifact(artifactRoot, value) {
+  return existsSync(resolve(artifactRoot, value));
+}
+function runRecordToProductBenchmarkRecord(record, runDir, artifactRoot, artifacts, options) {
+  const opts = resolveOptions(options);
+  const runtime = runtimeResolution(record, opts);
+  const score = scoreOf(record);
+  const armId = armIdOf(record);
+  const inputTokens = record.tokenUsage.input;
+  const outputTokens = record.tokenUsage.output;
+  const toolCallCount = toolCallsOf(record, runDir, opts);
+  const dimensions = numericDimensions(record);
+  if (!("tool_calls" in dimensions)) dimensions.tool_calls = toolCallCount;
+  const product = {
+    schemaVersion: 1,
+    projectId: opts.projectId,
+    benchmarkId: opts.benchmarkId,
+    runId: record.runId,
+    scenarioId: record.scenarioId ?? record.experimentId,
+    split: splitOf(record, opts),
+    armId,
+    rep: Number(record.seed ?? 0) + 1,
+    agentProfile: {
+      id: profileIdOf(record, armId, runtime, opts),
+      hash: materializedProfileHash(record, runtime),
+      path: opts.agentProfilePath,
+      declared: runtime,
+      resolved: runtime
+    },
+    model: { provider: modelProvider(record), id: record.model },
+    backend: { kind: backendOf(record), version: opts.backendVersion },
+    outcome: {
+      pass: passOf(record, score, opts.passThreshold),
+      score,
+      dimensions,
+      failureMode: failureModeOf(record, score, opts.passThreshold)
+    },
+    usage: {
+      inputTokens,
+      outputTokens,
+      costUsd: record.costUsd,
+      // Rounded: the bundle contract requires integer milliseconds.
+      wallMs: Math.round(record.wallMs),
+      toolCalls: toolCallCount
+    },
+    integrity: {
+      realBackend: inputTokens + outputTokens > 0,
+      rawCapture: existsArtifact(artifactRoot, artifacts.raws),
+      traceCapture: existsArtifact(artifactRoot, artifacts.traces),
+      noStubRows: inputTokens + outputTokens > 0,
+      priced: record.costUsd > 0,
+      profileMaterialized: Boolean(record.agentProfile?.cellId)
+    },
+    artifacts
+  };
+  return validateProductBenchmarkRecord(product);
+}
+function uniqueArmRecords(records) {
+  const byArm = /* @__PURE__ */ new Map();
+  for (const record of records) {
+    const prior = byArm.get(record.armId);
+    if (!prior) {
+      byArm.set(record.armId, record);
+      continue;
+    }
+    const fields = [
+      ["profileId", prior.agentProfile.id, record.agentProfile.id],
+      ["model", prior.agentProfile.resolved.model, record.agentProfile.resolved.model],
+      ["backend", prior.agentProfile.resolved.backend, record.agentProfile.resolved.backend],
+      ["harness", prior.agentProfile.resolved.harness, record.agentProfile.resolved.harness],
+      [
+        "reasoningEffort",
+        prior.agentProfile.resolved.reasoningEffort,
+        record.agentProfile.resolved.reasoningEffort
+      ]
+    ];
+    for (const [name, a, b] of fields) {
+      if (a !== b) {
+        throw new ValidationError(
+          `records for arm '${record.armId}' disagree on ${name} (${String(a)} vs ${String(b)}) \u2014 one arm id must map to one policy`
+        );
+      }
+    }
+  }
+  return [...byArm.values()];
+}
+function uniqueScenarioRecords(records) {
+  const byScenario = /* @__PURE__ */ new Map();
+  for (const record of records) {
+    const prior = byScenario.get(record.scenarioId);
+    if (!prior) {
+      byScenario.set(record.scenarioId, record);
+      continue;
+    }
+    if (prior.split !== record.split) {
+      throw new ValidationError(
+        `records for scenario '${record.scenarioId}' disagree on split (${prior.split} vs ${record.split})`
+      );
+    }
+  }
+  return [...byScenario.values()];
+}
+function buildProductBenchmarkManifest(records, options) {
+  if (records.length === 0) {
+    throw new ValidationError("cannot build a product benchmark manifest from zero records");
+  }
+  const scenarioTagPrefix = options.scenarioTagPrefix ?? defaultScenarioTagPrefix(options.projectId);
+  const mutableSurfaces = options.mutableSurfaces ?? productBenchmarkMutableSurfaces;
+  const byProfile = /* @__PURE__ */ new Map();
+  for (const record of records) {
+    byProfile.set(record.agentProfile.id, {
+      id: record.agentProfile.id,
+      profileHash: record.agentProfile.hash,
+      agentProfilePath: record.agentProfile.path
+    });
+  }
+  const manifest = {
+    schemaVersion: 1,
+    projectId: options.projectId,
+    benchmarkId: options.benchmarkId,
+    repo: productBenchmarkRepoIdentity(),
+    substrate: {
+      agentEval: packageVersion("@tangle-network/agent-eval"),
+      agentRuntime: packageVersion("@tangle-network/agent-runtime"),
+      agentInterface: packageVersion("@tangle-network/agent-interface"),
+      sandbox: packageVersion("@tangle-network/sandbox"),
+      ...options.substrate
+    },
+    profiles: [...byProfile.values()],
+    arms: uniqueArmRecords(records).map((record) => ({
+      id: record.armId,
+      profileId: record.agentProfile.id,
+      mutableSurfaces: [...mutableSurfaces],
+      policyAxes: {
+        carrier: record.armId.includes("policy") ? "resource-file" : "profile",
+        model: record.agentProfile.resolved.model,
+        backend: record.agentProfile.resolved.backend,
+        harness: record.agentProfile.resolved.harness,
+        ...record.agentProfile.resolved.reasoningEffort !== void 0 ? { reasoningEffort: record.agentProfile.resolved.reasoningEffort } : {}
+      }
+    })),
+    scenarios: uniqueScenarioRecords(records).map((record) => ({
+      id: record.scenarioId,
+      split: record.split,
+      tags: [scenarioTagPrefix, options.benchmarkId, record.split],
+      sourceAllowedForSynthesis: false
+    })),
+    budgets: {
+      maxUsd: records.reduce((sum2, record) => sum2 + record.usage.costUsd, 0),
+      maxCells: records.length,
+      maxWallMs: records.reduce((sum2, record) => sum2 + record.usage.wallMs, 0)
+    },
+    expectedArtifactDir: resolve(options.outDir)
+  };
+  return validateProductBenchmarkManifest(manifest);
+}
+function exportProductBenchmark(options) {
+  const { runDir, ...rest } = options;
+  return exportProductBenchmarkRuns({ ...rest, runDirs: [runDir] });
+}
+function exportProductBenchmarkRuns(options) {
+  const opts = resolveOptions(options);
+  const outDir = resolve(options.outDir);
+  const runDirs = options.runDirs.map((runDir) => resolve(runDir));
+  if (runDirs.length === 0) throw new ValidationError("export requires at least one run directory");
+  const materialize = options.materializeSourceRuns !== false;
+  const rows = runDirs.flatMap((runDir) => {
+    const sourceRecordsPath = join(runDir, "records.jsonl");
+    if (!existsSync(sourceRecordsPath)) throw new ValidationError(`missing ${sourceRecordsPath}`);
+    const records = readRunRecords(sourceRecordsPath);
+    if (records.length === 0) throw new ValidationError(`${sourceRecordsPath} is empty`);
+    return records.map((record) => ({ record, runDir }));
+  });
+  mkdirSync(outDir, { recursive: true });
+  const artifactsByRunDir = /* @__PURE__ */ new Map();
+  for (const [index, runDir] of runDirs.entries()) {
+    artifactsByRunDir.set(
+      runDir,
+      materialize ? materializeRunArtifacts(runDir, outDir, index) : absoluteArtifacts(runDir)
+    );
+  }
+  const artifactRoot = materialize ? outDir : "/";
+  const normalized = rows.map(
+    ({ record, runDir }) => runRecordToProductBenchmarkRecord(
+      record,
+      runDir,
+      artifactRoot,
+      artifactsByRunDir.get(runDir),
+      { ...options, runDirs, backendVersion: opts.backendVersion }
+    )
+  );
+  const manifest = buildProductBenchmarkManifest(normalized, {
+    outDir,
+    projectId: opts.projectId,
+    benchmarkId: opts.benchmarkId,
+    scenarioTagPrefix: opts.scenarioTagPrefix,
+    mutableSurfaces: opts.mutableSurfaces,
+    ...options.substrate !== void 0 ? { substrate: options.substrate } : {}
+  });
+  const manifestPath = join(outDir, "product-benchmark-manifest.json");
+  const recordsPath = join(outDir, "product-benchmark-records.jsonl");
+  writeFileSync(manifestPath, `${JSON.stringify(manifest, null, 2)}
+`);
+  writeFileSync(recordsPath, `${normalized.map((record) => JSON.stringify(record)).join("\n")}
+`);
+  return { manifestPath, recordsPath, records: normalized.length };
+}
+function defaultScenarioTagPrefix(projectId) {
+  return projectId.replace(/-agent$/, "") || projectId;
+}
+function resolveOptions(options) {
+  return {
+    projectId: options.projectId,
+    benchmarkId: options.benchmarkId,
+    agentProfilePath: options.agentProfilePath,
+    passThreshold: options.passThreshold ?? 0.7,
+    scenarioTagPrefix: options.scenarioTagPrefix ?? defaultScenarioTagPrefix(options.projectId),
+    ...options.fallbackProfileId !== void 0 ? { fallbackProfileId: options.fallbackProfileId } : {},
+    mutableSurfaces: options.mutableSurfaces ?? productBenchmarkMutableSurfaces,
+    ...options.classifySplit !== void 0 ? { classifySplit: options.classifySplit } : {},
+    ...options.toolCallFallback !== void 0 ? { toolCallFallback: options.toolCallFallback } : {},
+    backendVersion: options.backendVersion ?? packageVersion("@tangle-network/sandbox")
+  };
+}
+
+// src/product-benchmark/index.ts
+var productBenchmarkSplits = ["practice", "dev", "holdout", "safety", "sentinel"];
+function isObject(value) {
+  return Boolean(value) && typeof value === "object" && !Array.isArray(value);
+}
+function fail(path, message) {
+  throw new ValidationError(`${path}: ${message}`);
+}
+function wrapValidationError(path, err) {
+  if (err instanceof ValidationError) {
+    throw new ValidationError(`${path}: ${err.message}`, { cause: err });
+  }
+  throw new ValidationError(`${path}: ${err instanceof Error ? err.message : String(err)}`, {
+    cause: err
+  });
+}
+function expectObject(value, path) {
+  if (!isObject(value)) fail(path, "must be an object");
+  return value;
+}
+function expectString(value, path) {
+  if (typeof value !== "string" || value.trim().length === 0)
+    fail(path, "must be a non-empty string");
+  return value;
+}
+function expectBoolean(value, path) {
+  if (typeof value !== "boolean") fail(path, "must be a boolean");
+  return value;
+}
+function expectNumber(value, path, opts = {}) {
+  if (typeof value !== "number" || !Number.isFinite(value)) fail(path, "must be a finite number");
+  if (opts.integer && !Number.isInteger(value)) fail(path, "must be an integer");
+  if (opts.min !== void 0 && value < opts.min) fail(path, `must be >= ${opts.min}`);
+  if (opts.max !== void 0 && value > opts.max) fail(path, `must be <= ${opts.max}`);
+  return value;
+}
+function expectStringArray(value, path) {
+  if (!Array.isArray(value)) fail(path, "must be an array");
+  return value.map((entry, index) => expectString(entry, `${path}[${index}]`));
+}
+function expectObjectArray(value, path) {
+  if (!Array.isArray(value)) fail(path, "must be an array");
+  return value.map((entry, index) => expectObject(entry, `${path}[${index}]`));
+}
+function expectSplit(value, path) {
+  const split = expectString(value, path);
+  if (!productBenchmarkSplits.includes(split)) {
+    fail(path, `must be one of ${productBenchmarkSplits.join(", ")}`);
+  }
+  return split;
+}
+function optionalString(value, path) {
+  if (value === void 0) return void 0;
+  return expectString(value, path);
+}
+function expectDimensions(value, path) {
+  const obj = expectObject(value, path);
+  const out = {};
+  for (const [key, raw] of Object.entries(obj)) out[key] = expectNumber(raw, `${path}.${key}`);
+  return out;
+}
+function expectRuntimeResolution(value, path) {
+  const obj = expectObject(value, path);
+  return {
+    model: expectString(obj.model, `${path}.model`),
+    harness: expectString(obj.harness, `${path}.harness`),
+    backend: expectString(obj.backend, `${path}.backend`),
+    ...obj.reasoningEffort !== void 0 ? { reasoningEffort: optionalString(obj.reasoningEffort, `${path}.reasoningEffort`) } : {}
+  };
+}
+function validateProductBenchmarkManifest(value) {
+  const obj = expectObject(value, "manifest");
+  if (obj.schemaVersion !== 1) fail("manifest.schemaVersion", "must be 1");
+  const repo = expectObject(obj.repo, "manifest.repo");
+  const substrate = expectObject(obj.substrate, "manifest.substrate");
+  const profiles = expectObjectArray(obj.profiles, "manifest.profiles").map((profile, index) => ({
+    id: expectString(profile.id, `manifest.profiles[${index}].id`),
+    profileHash: expectString(profile.profileHash, `manifest.profiles[${index}].profileHash`),
+    agentProfilePath: expectString(
+      profile.agentProfilePath,
+      `manifest.profiles[${index}].agentProfilePath`
+    )
+  }));
+  const arms = expectObjectArray(obj.arms, "manifest.arms").map((arm, index) => ({
+    id: expectString(arm.id, `manifest.arms[${index}].id`),
+    profileId: expectString(arm.profileId, `manifest.arms[${index}].profileId`),
+    mutableSurfaces: expectStringArray(
+      arm.mutableSurfaces,
+      `manifest.arms[${index}].mutableSurfaces`
+    ),
+    policyAxes: expectObject(arm.policyAxes, `manifest.arms[${index}].policyAxes`)
+  }));
+  const scenarios = expectObjectArray(obj.scenarios, "manifest.scenarios").map(
+    (scenario, index) => ({
+      id: expectString(scenario.id, `manifest.scenarios[${index}].id`),
+      split: expectSplit(scenario.split, `manifest.scenarios[${index}].split`),
+      tags: expectStringArray(scenario.tags, `manifest.scenarios[${index}].tags`),
+      sourceAllowedForSynthesis: expectBoolean(
+        scenario.sourceAllowedForSynthesis,
+        `manifest.scenarios[${index}].sourceAllowedForSynthesis`
+      )
+    })
+  );
+  const budgets = expectObject(obj.budgets, "manifest.budgets");
+  if (profiles.length === 0) fail("manifest.profiles", "must contain at least one profile");
+  if (arms.length === 0) fail("manifest.arms", "must contain at least one arm");
+  if (scenarios.length === 0) fail("manifest.scenarios", "must contain at least one scenario");
+  assertUnique(
+    profiles.map((profile) => profile.id),
+    "manifest.profiles.id"
+  );
+  assertUnique(
+    arms.map((arm) => arm.id),
+    "manifest.arms.id"
+  );
+  assertUnique(
+    scenarios.map((scenario) => scenario.id),
+    "manifest.scenarios.id"
+  );
+  const profileIds = new Set(profiles.map((profile) => profile.id));
+  for (const arm of arms) {
+    if (!profileIds.has(arm.profileId))
+      fail(`manifest.arms.${arm.id}.profileId`, `unknown profile ${arm.profileId}`);
+  }
+  return {
+    schemaVersion: 1,
+    projectId: expectString(obj.projectId, "manifest.projectId"),
+    benchmarkId: expectString(obj.benchmarkId, "manifest.benchmarkId"),
+    repo: {
+      url: expectString(repo.url, "manifest.repo.url"),
+      commit: expectString(repo.commit, "manifest.repo.commit"),
+      branch: expectString(repo.branch, "manifest.repo.branch")
+    },
+    substrate: {
+      agentEval: expectString(substrate.agentEval, "manifest.substrate.agentEval"),
+      agentRuntime: expectString(substrate.agentRuntime, "manifest.substrate.agentRuntime"),
+      agentInterface: expectString(substrate.agentInterface, "manifest.substrate.agentInterface"),
+      sandbox: expectString(substrate.sandbox, "manifest.substrate.sandbox"),
+      ...substrate.agentBench !== void 0 ? { agentBench: expectString(substrate.agentBench, "manifest.substrate.agentBench") } : {}
+    },
+    profiles,
+    arms,
+    scenarios,
+    budgets: {
+      maxUsd: expectNumber(budgets.maxUsd, "manifest.budgets.maxUsd", { min: 0 }),
+      maxCells: expectNumber(budgets.maxCells, "manifest.budgets.maxCells", {
+        min: 0,
+        integer: true
+      }),
+      maxWallMs: expectNumber(budgets.maxWallMs, "manifest.budgets.maxWallMs", {
+        min: 0,
+        integer: true
+      })
+    },
+    expectedArtifactDir: expectString(obj.expectedArtifactDir, "manifest.expectedArtifactDir")
+  };
+}
+function validateProductBenchmarkRecord(value) {
+  const obj = expectObject(value, "record");
+  if (obj.schemaVersion !== 1) fail("record.schemaVersion", "must be 1");
+  const agentProfile = expectObject(obj.agentProfile, "record.agentProfile");
+  const model = expectObject(obj.model, "record.model");
+  const backend = expectObject(obj.backend, "record.backend");
+  const outcome = expectObject(obj.outcome, "record.outcome");
+  const usage = expectObject(obj.usage, "record.usage");
+  const integrity = expectObject(obj.integrity, "record.integrity");
+  const artifacts = expectObject(obj.artifacts, "record.artifacts");
+  const record = {
+    schemaVersion: 1,
+    projectId: expectString(obj.projectId, "record.projectId"),
+    benchmarkId: expectString(obj.benchmarkId, "record.benchmarkId"),
+    runId: expectString(obj.runId, "record.runId"),
+    scenarioId: expectString(obj.scenarioId, "record.scenarioId"),
+    split: expectSplit(obj.split, "record.split"),
+    armId: expectString(obj.armId, "record.armId"),
+    rep: expectNumber(obj.rep, "record.rep", { min: 1, integer: true }),
+    agentProfile: {
+      id: expectString(agentProfile.id, "record.agentProfile.id"),
+      hash: expectString(agentProfile.hash, "record.agentProfile.hash"),
+      path: expectString(agentProfile.path, "record.agentProfile.path"),
+      declared: expectRuntimeResolution(agentProfile.declared, "record.agentProfile.declared"),
+      resolved: expectRuntimeResolution(agentProfile.resolved, "record.agentProfile.resolved")
+    },
+    model: {
+      provider: expectString(model.provider, "record.model.provider"),
+      id: expectString(model.id, "record.model.id")
+    },
+    backend: {
+      kind: expectString(backend.kind, "record.backend.kind"),
+      version: expectString(backend.version, "record.backend.version")
+    },
+    outcome: {
+      pass: expectBoolean(outcome.pass, "record.outcome.pass"),
+      score: expectNumber(outcome.score, "record.outcome.score", { min: 0, max: 1 }),
+      dimensions: expectDimensions(outcome.dimensions, "record.outcome.dimensions"),
+      failureMode: outcome.failureMode === null ? null : expectString(outcome.failureMode, "record.outcome.failureMode")
+    },
+    usage: {
+      inputTokens: expectNumber(usage.inputTokens, "record.usage.inputTokens", {
+        min: 0,
+        integer: true
+      }),
+      outputTokens: expectNumber(usage.outputTokens, "record.usage.outputTokens", {
+        min: 0,
+        integer: true
+      }),
+      costUsd: expectNumber(usage.costUsd, "record.usage.costUsd", { min: 0 }),
+      wallMs: expectNumber(usage.wallMs, "record.usage.wallMs", { min: 0, integer: true }),
+      toolCalls: expectNumber(usage.toolCalls, "record.usage.toolCalls", { min: 0, integer: true })
+    },
+    integrity: {
+      realBackend: expectBoolean(integrity.realBackend, "record.integrity.realBackend"),
+      rawCapture: expectBoolean(integrity.rawCapture, "record.integrity.rawCapture"),
+      traceCapture: expectBoolean(integrity.traceCapture, "record.integrity.traceCapture"),
+      noStubRows: expectBoolean(integrity.noStubRows, "record.integrity.noStubRows"),
+      priced: expectBoolean(integrity.priced, "record.integrity.priced"),
+      profileMaterialized: expectBoolean(
+        integrity.profileMaterialized,
+        "record.integrity.profileMaterialized"
+      )
+    },
+    artifacts: {
+      records: expectString(artifacts.records, "record.artifacts.records"),
+      traces: expectString(artifacts.traces, "record.artifacts.traces"),
+      raws: expectString(artifacts.raws, "record.artifacts.raws"),
+      scores: expectString(artifacts.scores, "record.artifacts.scores"),
+      workspace: expectString(artifacts.workspace, "record.artifacts.workspace")
+    }
+  };
+  if (record.integrity.realBackend && record.usage.inputTokens + record.usage.outputTokens === 0) {
+    fail("record.usage", "realBackend rows must carry non-zero token usage");
+  }
+  return record;
+}
+function productBenchmarkIntegrityFailures(record) {
+  const failures = [];
+  for (const [key, value] of Object.entries(record.integrity)) {
+    if (!value) failures.push(`${record.runId}:${key}=false`);
+  }
+  return failures;
+}
+function readProductBenchmarkRecords(path) {
+  const lines = readFileSync2(path, "utf8").split("\n").map((line) => line.trim()).filter(Boolean);
+  const records = [];
+  for (const [index, line] of lines.entries()) {
+    try {
+      records.push(validateProductBenchmarkRecord(JSON.parse(line)));
+    } catch (err) {
+      wrapValidationError(`${path}:${index + 1}`, err);
+    }
+  }
+  return records;
+}
+function readProductBenchmarkManifest(path) {
+  try {
+    return validateProductBenchmarkManifest(JSON.parse(readFileSync2(path, "utf8")));
+  } catch (err) {
+    wrapValidationError(path, err);
+  }
+}
+function validateProductBenchmarkRun(input) {
+  const manifest = readProductBenchmarkManifest(input.manifestPath);
+  const records = readProductBenchmarkRecords(input.recordsPath);
+  const manifestProjectBench = `${manifest.projectId}/${manifest.benchmarkId}`;
+  const integrityFailures = records.flatMap(productBenchmarkIntegrityFailures);
+  const missingArtifacts = input.checkArtifacts === false ? [] : records.flatMap(
+    (record) => missingArtifactsForRecord(record, input.artifactRoot ?? dirname2(input.recordsPath))
+  );
+  for (const [index, record] of records.entries()) {
+    const recordProjectBench = `${record.projectId}/${record.benchmarkId}`;
+    if (recordProjectBench !== manifestProjectBench) {
+      fail(
+        `records[${index}]`,
+        `project/benchmark ${recordProjectBench} does not match manifest ${manifestProjectBench}`
+      );
+    }
+    if (!manifest.arms.some((arm) => arm.id === record.armId))
+      fail(`records[${index}].armId`, `unknown arm ${record.armId}`);
+    if (!manifest.scenarios.some((scenario) => scenario.id === record.scenarioId)) {
+      fail(`records[${index}].scenarioId`, `unknown scenario ${record.scenarioId}`);
+    }
+  }
+  const repoFailures = ["url", "commit", "branch"].filter((key) => manifest.repo[key].trim().length === 0 || manifest.repo[key] === "unknown").map((key) => `manifest.repo.${key}`);
+  const substrateFailures = ["agentEval", "agentRuntime", "agentInterface", "sandbox"].filter(
+    (key) => manifest.substrate[key].trim().length === 0 || manifest.substrate[key] === "unknown"
+  ).map((key) => `manifest.substrate.${key}`);
+  return {
+    manifestPath: input.manifestPath,
+    recordsPath: input.recordsPath,
+    records: records.length,
+    repoFailures,
+    substrateFailures,
+    projects: sortedUnique(records.map((record) => record.projectId)),
+    benchmarks: sortedUnique(records.map((record) => record.benchmarkId)),
+    arms: sortedUnique(records.map((record) => record.armId)),
+    scenarios: sortedUnique(records.map((record) => record.scenarioId)),
+    passed: records.filter((record) => record.outcome.pass).length,
+    failed: records.filter((record) => !record.outcome.pass).length,
+    inputTokens: sum(records, (record) => record.usage.inputTokens),
+    outputTokens: sum(records, (record) => record.usage.outputTokens),
+    costUsd: sum(records, (record) => record.usage.costUsd),
+    wallMs: sum(records, (record) => record.usage.wallMs),
+    integrityFailures,
+    missingArtifacts
+  };
+}
+function missingArtifactsForRecord(record, artifactRoot) {
+  const missing = [];
+  for (const [key, value] of Object.entries(record.artifacts)) {
+    const path = resolveArtifactPath(value, artifactRoot);
+    if (!existsSync2(path)) missing.push(`${record.runId}:${key}:${value}`);
+  }
+  return missing;
+}
+function resolveArtifactPath(value, artifactRoot) {
+  return value.startsWith("/") ? value : join2(artifactRoot, value);
+}
+function assertUnique(values, path) {
+  const seen = /* @__PURE__ */ new Set();
+  for (const value of values) {
+    if (seen.has(value)) fail(path, `duplicate ${value}`);
+    seen.add(value);
+  }
+}
+function sortedUnique(values) {
+  return [...new Set(values)].sort();
+}
+function sum(items, fn) {
+  return items.reduce((total, item) => total + fn(item), 0);
+}
+function findProductBenchmarkArtifacts(runDir) {
+  const manifestPath = join2(runDir, "product-benchmark-manifest.json");
+  const recordsPath = join2(runDir, "product-benchmark-records.jsonl");
+  if (existsSync2(manifestPath) && statSync(manifestPath).isFile() && existsSync2(recordsPath) && statSync(recordsPath).isFile()) {
+    return { manifestPath, recordsPath };
+  }
+  return null;
+}
+function assertProductBenchmarkRun(runDir) {
+  const artifacts = findProductBenchmarkArtifacts(runDir);
+  if (!artifacts) {
+    fail(runDir, "missing product-benchmark-manifest.json or product-benchmark-records.jsonl");
+  }
+  const report = validateProductBenchmarkRun({ ...artifacts, artifactRoot: runDir });
+  const failures = [
+    ...report.repoFailures,
+    ...report.substrateFailures,
+    ...report.integrityFailures,
+    ...report.missingArtifacts
+  ];
+  if (failures.length > 0) {
+    fail(runDir, `product benchmark validation failed:
+${failures.join("\n")}`);
+  }
+  return report;
+}
+
+export {
+  productBenchmarkMutableSurfaces,
+  productBenchmarkRepoIdentity,
+  runRecordToProductBenchmarkRecord,
+  buildProductBenchmarkManifest,
+  exportProductBenchmark,
+  exportProductBenchmarkRuns,
+  productBenchmarkSplits,
+  validateProductBenchmarkManifest,
+  validateProductBenchmarkRecord,
+  productBenchmarkIntegrityFailures,
+  readProductBenchmarkRecords,
+  readProductBenchmarkManifest,
+  validateProductBenchmarkRun,
+  findProductBenchmarkArtifacts,
+  assertProductBenchmarkRun
+};
+//# sourceMappingURL=chunk-RQNOLV3I.js.map