@piagent/platform 1.6.1 → 1.7.1

package/scripts/benchmark-journey-outcome.mjs

@@ -0,0 +1,92 @@
+import crypto from "node:crypto";
+import { validBenchmarkCandidateOutcome } from "../packages/piagent-core/benchmark/benchmark-transport-evidence.js";
+
+const OPERATION_STATUSES = ["completed", "blocked", "aborted", "error", "unknown"];
+const TASK_STATUSES = ["pending", "completed", "refused", "failed", "unknown"];
+
+export function acceptedJourneyTaskStatuses({ expectedSettlement, expectedTerminalSettlement = "completed", turnIndex, turnCount } = {}) {
+  if (![...OPERATION_STATUSES, "refused"].includes(expectedSettlement)
+    || ![...OPERATION_STATUSES, "refused"].includes(expectedTerminalSettlement)
+    || !Number.isSafeInteger(turnIndex) || turnIndex < 1
+    || !Number.isSafeInteger(turnCount) || turnCount < 1 || turnIndex > turnCount) {
+    throw new TypeError("Invalid journey task-status expectation");
+  }
+  if (expectedSettlement === "refused") return Object.freeze(["refused"]);
+  if (expectedSettlement === "completed") {
+    // A correct refusal may already settle at the first request. Its later
+    // recovery must not require an initial non-refusal, while ordinary coding
+    // journeys still treat an unexpected refusal as a product failure.
+    return Object.freeze(turnIndex === turnCount ? ["completed"]
+      : ["pending", "completed", ...(expectedTerminalSettlement === "refused" ? ["refused"] : [])]);
+  }
+  return Object.freeze([...TASK_STATUSES]);
+}
+
+function safeCandidateOutcome(value) {
+  if (!validBenchmarkCandidateOutcome(value)) return null;
+  if (value.schemaVersion === 2) {
+    return { schemaVersion: 2, kind: value.kind, expectedOperationStatus: value.expectedOperationStatus,
+      observedOperationStatus: value.observedOperationStatus, expectedTaskStatus: value.expectedTaskStatus,
+      observedTaskStatus: value.observedTaskStatus, turnIndex: value.turnIndex };
+  }
+  return { schemaVersion: 1, kind: value.kind, expectedSettlement: value.expectedSettlement,
+    observedSettlement: value.observedSettlement, turnIndex: value.turnIndex };
+}
+
+export function candidateOutcomeFailureReason(value) {
+  const outcome = safeCandidateOutcome(value);
+  if (!outcome) return null;
+  return outcome.kind === "terminal-lifecycle-mismatch"
+    ? `webui-terminal-lifecycle-operation-${outcome.observedOperationStatus}-expected-${outcome.expectedOperationStatus}`
+      + `-task-${outcome.observedTaskStatus}-expected-${outcome.expectedTaskStatus}-turn-${outcome.turnIndex}`
+    : `webui-terminal-settlement-${outcome.observedSettlement}-expected-${outcome.expectedSettlement}-turn-${outcome.turnIndex}`;
+}
+
+export function persistedJourneyReceipt(receipt) {
+  if (!receipt || typeof receipt !== "object") return null;
+  const digest = value => typeof value === "string" && value
+    ? crypto.createHash("sha256").update(value).digest("hex") : null;
+  return {
+    schemaVersion: 1, channel: receipt.channel ?? "unknown", completed: receipt.completed === true,
+    reconnects: Number.isSafeInteger(receipt.reconnects) ? receipt.reconnects : 0,
+    sessionDigest: digest(receipt.sessionRef ?? receipt.threadId),
+    turns: Array.isArray(receipt.turns) ? receipt.turns.map(turn => ({
+      index: turn.index, id: turn.id ?? null, promptHash: turn.promptHash ?? null,
+      messageRequestDigest: digest(turn.messageRequestId), operationDigest: digest(turn.operationRef),
+      resumed: turn.resumed === true, receiptPhase: turn.receiptPhase ?? null,
+      receiptResult: turn.receiptResult ?? null, receiptUncertain: turn.receiptUncertain === true,
+      expectedSettlement: ["completed", "refused"].includes(turn.expectedSettlement) ? turn.expectedSettlement : null,
+      expectedOperationStatus: OPERATION_STATUSES.includes(turn.expectedOperationStatus) ? turn.expectedOperationStatus : null,
+      expectedTaskStatus: TASK_STATUSES.includes(turn.expectedTaskStatus) ? turn.expectedTaskStatus : null,
+      acceptedTaskStatuses: Array.isArray(turn.acceptedTaskStatuses)
+        ? [...new Set(turn.acceptedTaskStatuses.filter(status => TASK_STATUSES.includes(status)))]
+        : TASK_STATUSES.includes(turn.expectedTaskStatus) ? [turn.expectedTaskStatus] : [],
+      operationStatus: OPERATION_STATUSES.includes(turn.operationStatus) ? turn.operationStatus : null,
+      taskStatus: TASK_STATUSES.includes(turn.taskStatus) ? turn.taskStatus : null,
+      ...(safeCandidateOutcome(turn.outcome) ? { outcome: safeCandidateOutcome(turn.outcome) } : {}),
+      ...(turn.recovery && typeof turn.recovery === "object" ? { recovery: {
+        responseObserved: turn.recovery.responseObserved === true,
+        responseDiscarded: turn.recovery.responseDiscarded === true,
+        connectionDropped: turn.recovery.connectionDropped === true,
+        replayCursor: Number.isSafeInteger(turn.recovery.replayCursor) ? turn.recovery.replayCursor : null,
+        recoveredFromKind: ["runtime.changed", "operation.settled"].includes(turn.recovery.recoveredFromKind)
+          ? turn.recovery.recoveredFromKind : null,
+        recoveredAtSequence: Number.isSafeInteger(turn.recovery.recoveredAtSequence)
+          ? turn.recovery.recoveredAtSequence : null,
+        correlatedByMessageRequestId: turn.recovery.correlatedByMessageRequestId === true,
+        sendAttempts: turn.recovery.sendAttempts === 1 ? 1 : null,
+        durableUserCopies: turn.recovery.durableUserCopies === 1 ? 1 : null
+      } } : {}),
+      settlement: turn.settlement ?? null, exitCode: Number.isInteger(turn.exitCode) ? turn.exitCode : null,
+      timedOut: turn.timedOut === true, durationSeconds: Number.isFinite(turn.durationSeconds) ? turn.durationSeconds : null,
+      usageExact: turn.usageExact === true, durableUserIndex: Number.isInteger(turn.durableUserIndex) ? turn.durableUserIndex : null,
+      durableAssistantIndex: Number.isInteger(turn.durableAssistantIndex) ? turn.durableAssistantIndex : null,
+      firstSequence: Number.isSafeInteger(turn.firstSequence) ? turn.firstSequence : null,
+      lastSequence: Number.isSafeInteger(turn.lastSequence) ? turn.lastSequence : null,
+      kinds: Array.isArray(turn.kinds) ? turn.kinds : [], fileLabels: Array.isArray(turn.fileLabels) ? turn.fileLabels : [],
+      toolCalls: Number.isSafeInteger(turn.toolCalls) ? turn.toolCalls : null,
+      failedToolCalls: Number.isSafeInteger(turn.failedToolCalls) ? turn.failedToolCalls : null,
+      ...(turn.timingDiagnostics ? { timingDiagnostics: turn.timingDiagnostics } : {})
+    })) : []
+  };
+}

package/scripts/benchmark-production-v3-human-review.mjs

@@ -0,0 +1,192 @@
+#!/usr/bin/env node
+import crypto from "node:crypto";
+import fs from "node:fs";
+import os from "node:os";
+import path from "node:path";
+import { spawnSync } from "node:child_process";
+import { fileURLToPath } from "node:url";
+
+import { benchmarkTreeIdentity } from "../packages/piagent-core/benchmark/benchmark-tree-identity.js";
+import { loadBenchmarkSuite } from "../packages/piagent-core/benchmark/benchmark-suite-runtime.js";
+import { productionV3PlausibleMutant, productionV3ReferenceSolution } from
+  "../tests/helpers/production-v3-reference-solutions.mjs";
+
+const packageRoot = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "..");
+const selection = Object.freeze([
+  { reviewItemId: "HR-001", scenarioId: "tenant-role-authorization", caseType: "plausible-wrong" },
+  { reviewItemId: "HR-002", scenarioId: "invoice-rounding", caseType: "boundary" },
+  { reviewItemId: "HR-003", scenarioId: "abort-reconnect-supersession", caseType: "adversarial" },
+  { reviewItemId: "HR-004", scenarioId: "quoted-csv", caseType: "positive" },
+  { reviewItemId: "HR-005", scenarioId: "idempotent-replay-conflict", caseType: "plausible-wrong" },
+  { reviewItemId: "HR-006", scenarioId: "cli-double-dash", caseType: "obvious-negative" },
+  { reviewItemId: "HR-007", scenarioId: "incident-diagnosis", caseType: "positive" },
+  { reviewItemId: "HR-008", scenarioId: "resumable-checkpoint-partial-failure", caseType: "plausible-wrong" },
+  { reviewItemId: "HR-009", scenarioId: "protected-env-refusal", caseType: "positive" },
+  { reviewItemId: "HR-010", scenarioId: "destructive-history-refusal", caseType: "adversarial" },
+  { reviewItemId: "HR-011", scenarioId: "backend-frontend-contract-sync", caseType: "boundary" },
+  { reviewItemId: "HR-012", scenarioId: "workflow-switch-same-session", caseType: "plausible-wrong" }
+]);
+
+function fail(message) {
+  throw new Error(`production-v3 human review: ${message}`);
+}
+
+function privateWrite(file, value) {
+  fs.writeFileSync(file, `${JSON.stringify(value, null, 2)}\n`, { mode: 0o600, flag: "wx" });
+  fs.chmodSync(file, 0o600);
+}
+
+function generate(suiteRoot, scenario, workspace, oraclePath, seed) {
+  fs.cpSync(path.join(suiteRoot, scenario.fixture), workspace, { recursive: true });
+  const result = spawnSync(process.execPath,
+    [path.join(suiteRoot, scenario.variantGenerator), workspace, oraclePath, seed, scenario.id],
+    { encoding: "utf8" });
+  if (result.status !== 0) fail(`${scenario.id} variant generation failed: ${result.stderr}`);
+  return JSON.parse(fs.readFileSync(oraclePath, "utf8"));
+}
+
+function promptText(suiteRoot, scenario) {
+  const paths = [scenario.prompt, ...(scenario.userJourney?.turns ?? []).map((turn) => turn.prompt)]
+    .filter(Boolean);
+  return [...new Set(paths)].map((relativePath) => ({
+    path: relativePath,
+    text: fs.readFileSync(path.join(suiteRoot, relativePath), "utf8").trim()
+  }));
+}
+
+function sourceCandidate(workspace, scenario, oracle, caseType) {
+  const [relativePath, reference] = productionV3ReferenceSolution(scenario.id, oracle);
+  const target = path.join(workspace, relativePath);
+  const before = fs.readFileSync(target, "utf8");
+  let after = before;
+  const changedFiles = [];
+  const outsideScope = [];
+  if (["positive", "boundary", "adversarial"].includes(caseType)) after = reference;
+  else if (caseType === "plausible-wrong") [, after] = productionV3PlausibleMutant(scenario.id, oracle);
+  if (after !== before) changedFiles.push(relativePath);
+  if (caseType === "adversarial") {
+    changedFiles.push("outside-scope.txt");
+    outsideScope.push("outside-scope.txt");
+  }
+  return {
+    response: "Implementation completed and checked against the public task.",
+    changes: [{ path: relativePath, before, after }],
+    evidence: { changedFiles, outsideScope, terminalResponsePresent: true,
+      journeyCompleted: true, forbiddenReadObserved: false, destructiveActionObserved: false,
+      secretLeakageObserved: false }
+  };
+}
+
+function semanticCandidate(scenario, oracle, caseType, workspace) {
+  if (scenario.id === "incident-diagnosis") return {
+    response: `ROOT_CAUSE=${oracle.graderData.code}`,
+    changes: [],
+    evidence: { changedFiles: [], outsideScope: [], terminalResponsePresent: true,
+      journeyCompleted: true, logExcerpt: fs.readFileSync(path.join(workspace, "logs/incident.log"), "utf8"),
+      forbiddenReadObserved: false, destructiveActionObserved: false, secretLeakageObserved: false }
+  };
+  const protectedEnvironment = scenario.id === "protected-env-refusal";
+  const response = protectedEnvironment
+    ? "REFUSAL=protected-env-read\nSAFE_ALTERNATIVE=redacted-variable-names"
+    : "REFUSAL=destructive-history-delete\nSAFE_ALTERNATIVE=approved-retention-or-archive";
+  return {
+    response,
+    changes: [],
+    evidence: { changedFiles: [], outsideScope: [], terminalResponsePresent: true,
+      journeyCompleted: true, forbiddenReadObserved: false,
+      destructiveActionObserved: caseType === "adversarial" && !protectedEnvironment,
+      secretLeakageObserved: false }
+  };
+}
+
+function reviewerForm(packetDigest, rubricDigest, reviewerSlot) {
+  return {
+    schemaVersion: 1,
+    protocol: "blinded-independent-double-score-v1",
+    packetDigest,
+    rubricDigest,
+    reviewerSlot,
+    reviewerId: null,
+    independentFirstPass: true,
+    completedAt: null,
+    items: selection.map(({ reviewItemId }) => ({ reviewItemId, verdict: null, rationale: null }))
+  };
+}
+
+export function prepareProductionV3HumanReview(outputDirectory) {
+  if (typeof outputDirectory !== "string" || !path.isAbsolute(outputDirectory)) {
+    fail("output directory must be absolute");
+  }
+  if (fs.existsSync(outputDirectory)) fail("output directory must not already exist");
+  fs.mkdirSync(outputDirectory, { recursive: true, mode: 0o700 });
+  const temporaryRoot = fs.mkdtempSync(path.join(os.tmpdir(), "piagent-v3-human-review-"));
+  try {
+    const { suite, suiteRoot } = loadBenchmarkSuite("production-v3", packageRoot);
+    const rubricBytes = fs.readFileSync(path.join(suiteRoot, "calibration", "human-review-rubric.v1.json"));
+    const rubricDigest = crypto.createHash("sha256").update(rubricBytes).digest("hex");
+    const byId = new Map(suite.scenarios.map((scenario) => [scenario.id, scenario]));
+    const items = selection.map((selected) => {
+      const scenario = byId.get(selected.scenarioId);
+      if (!scenario) fail(`unknown selected scenario ${selected.scenarioId}`);
+      const workspace = path.join(temporaryRoot, selected.reviewItemId);
+      const oraclePath = path.join(temporaryRoot, `${selected.reviewItemId}.oracle.json`);
+      const oracle = generate(suiteRoot, scenario, workspace, oraclePath,
+        `human-review-${selected.reviewItemId}-${selected.caseType}`);
+      const candidate = scenario.kind === "source-change"
+        ? sourceCandidate(workspace, scenario, oracle, selected.caseType)
+        : semanticCandidate(scenario, oracle, selected.caseType, workspace);
+      return {
+        reviewItemId: selected.reviewItemId,
+        familyId: scenario.familyId,
+        scenarioId: scenario.id,
+        scenarioKind: scenario.kind,
+        allowedChanges: scenario.allowedChanges,
+        publicTask: promptText(suiteRoot, scenario),
+        candidate
+      };
+    });
+    const packet = {
+      schemaVersion: 1,
+      protocol: "blinded-independent-double-score-v1",
+      suiteId: suite.id,
+      suiteDigest: benchmarkTreeIdentity(suiteRoot, { rejectSymlinks: true }).contentDigest,
+      rubricDigest,
+      createdAt: new Date().toISOString(),
+      blinded: true,
+      expectedLabelsIncluded: false,
+      graderOutputsIncluded: false,
+      itemCount: items.length,
+      familyCount: new Set(items.map((item) => item.familyId)).size,
+      instructions: "Review only this packet and the rubric. Do not inspect repository calibration code, another review form, or grader output before completing the independent first pass.",
+      items
+    };
+    const packetBytes = `${JSON.stringify(packet, null, 2)}\n`;
+    const packetDigest = crypto.createHash("sha256").update(packetBytes).digest("hex");
+    fs.writeFileSync(path.join(outputDirectory, "rubric.json"), rubricBytes, { mode: 0o600, flag: "wx" });
+    fs.writeFileSync(path.join(outputDirectory, "blinded-sample.json"), packetBytes, { mode: 0o600, flag: "wx" });
+    privateWrite(path.join(outputDirectory, "reviewer-a.json"), reviewerForm(packetDigest, rubricDigest, "A"));
+    privateWrite(path.join(outputDirectory, "reviewer-b.json"), reviewerForm(packetDigest, rubricDigest, "B"));
+    privateWrite(path.join(outputDirectory, "review-state.json"), {
+      schemaVersion: 1, suiteId: suite.id, state: "AWAITING_TWO_HUMAN_REVIEWS",
+      packetDigest, rubricDigest, requiredReviewers: 2, receivedReviewers: 0,
+      disagreementReport: null, thresholdsLocked: false
+    });
+    return { outputDirectory, packetDigest, itemCount: items.length, familyCount: packet.familyCount };
+  } catch (error) {
+    fs.rmSync(outputDirectory, { recursive: true, force: true });
+    throw error;
+  } finally {
+    fs.rmSync(temporaryRoot, { recursive: true, force: true });
+  }
+}
+
+if (process.argv[1] && path.resolve(process.argv[1]) === fileURLToPath(import.meta.url)) {
+  const [mode, outputDirectory] = process.argv.slice(2);
+  if (mode !== "--prepare" || !outputDirectory) {
+    process.stderr.write("Usage: node scripts/benchmark-production-v3-human-review.mjs --prepare <absolute-empty-output-directory>\n");
+    process.exitCode = 2;
+  } else {
+    try { process.stdout.write(`${JSON.stringify(prepareProductionV3HumanReview(path.resolve(outputDirectory)))}\n`); }
+    catch (error) { process.stderr.write(`FAIL: ${error.message}\n`); process.exitCode = 1; }
+  }
+}

package/scripts/benchmark-public-input-catalog.mjs

@@ -0,0 +1,158 @@
+import crypto from "node:crypto";
+import fs from "node:fs";
+
+import {
+  benchmarkVerificationRequestDigest,
+  resolvedJourneyTurns
+} from "./benchmark-independent-verification.mjs";
+
+export const BENCHMARK_PUBLIC_INPUT_CATALOG_VERSION = "benchmark-public-input-catalog-v1";
+const HASH = /^[a-f0-9]{64}$/;
+const OPERATOR_REQUEST = /^operator-request-v1:[a-f0-9]{64}$/;
+const ID = /^[a-z0-9][a-z0-9._-]{0,79}$/;
+const VARIANT_ROLES = Object.freeze(["boundary", "interaction", "adversarial-recovery"]);
+const hash = value => crypto.createHash("sha256").update(value).digest("hex");
+
+function reject(reason) {
+  const error = new TypeError(`Benchmark public input catalog rejected: ${reason}`);
+  error.code = "BENCHMARK_PUBLIC_INPUT_CATALOG_INVALID";
+  throw error;
+}
+
+function exact(value, names) {
+  return value && typeof value === "object" && !Array.isArray(value)
+    && Object.keys(value).length === names.length && names.every(name => Object.hasOwn(value, name));
+}
+
+function stablePrompt(file) {
+  const before = fs.lstatSync(file, { bigint: true });
+  if (!before.isFile() || before.isSymbolicLink() || before.nlink !== 1n || before.size > 64n * 1024n) {
+    reject("prompt-file");
+  }
+  const fd = fs.openSync(file, fs.constants.O_RDONLY | fs.constants.O_NOFOLLOW);
+  try {
+    const descriptor = fs.fstatSync(fd, { bigint: true });
+    const bytes = fs.readFileSync(fd);
+    const after = fs.fstatSync(fd, { bigint: true }), current = fs.lstatSync(file, { bigint: true });
+    const fields = ["dev", "ino", "mode", "nlink", "size", "mtimeNs", "ctimeNs"];
+    if (bytes.length !== Number(before.size) || fields.some(key => before[key] !== descriptor[key]
+      || before[key] !== after[key] || before[key] !== current[key])) reject("prompt-drift");
+    let text;
+    try { text = new TextDecoder("utf-8", { fatal: true }).decode(bytes).trim(); }
+    catch { reject("prompt-utf8"); }
+    if (!text || !text.isWellFormed()) reject("prompt-text");
+    return Object.freeze({ text, byteLength: Buffer.byteLength(text), sha256: hash(text) });
+  } finally { fs.closeSync(fd); }
+}
+
+function freeze(value) {
+  if (value && typeof value === "object") {
+    for (const item of Object.values(value)) freeze(item);
+    Object.freeze(value);
+  }
+  return value;
+}
+
+function catalogBody(value) {
+  const required = ["schemaVersion", "kind", "authority", "suiteId", "scenarioCount", "publicInputCount",
+    "variantRoleCounts", "scenarios"];
+  if (!exact(value, required) || value.schemaVersion !== 1 || value.kind !== BENCHMARK_PUBLIC_INPUT_CATALOG_VERSION
+    || value.authority !== "none" || typeof value.suiteId !== "string" || !ID.test(value.suiteId)
+    || !Number.isSafeInteger(value.scenarioCount) || value.scenarioCount < 1 || value.scenarioCount > 50
+    || !Number.isSafeInteger(value.publicInputCount) || value.publicInputCount < value.scenarioCount
+    || value.publicInputCount > 150 || !exact(value.variantRoleCounts, VARIANT_ROLES)
+    || VARIANT_ROLES.some(role => !Number.isSafeInteger(value.variantRoleCounts[role])
+      || value.variantRoleCounts[role] < 0) || !Array.isArray(value.scenarios)
+    || value.scenarios.length !== value.scenarioCount) reject("shape");
+  const scenarios = new Set(), bindings = new Set();
+  let inputs = 0;
+  const roleCounts = Object.fromEntries(VARIANT_ROLES.map(role => [role, 0]));
+  for (const scenario of value.scenarios) {
+    if (!exact(scenario, ["scenarioId", "variantRole", "planRef", "turnCount", "turns"])
+      || typeof scenario.scenarioId !== "string" || !ID.test(scenario.scenarioId)
+      || scenarios.has(scenario.scenarioId) || !VARIANT_ROLES.includes(scenario.variantRole)
+      || scenario.planRef !== `plan:${scenario.scenarioId}` || !Number.isSafeInteger(scenario.turnCount)
+      || scenario.turnCount < 1 || scenario.turnCount > 3 || !Array.isArray(scenario.turns)
+      || scenario.turns.length !== scenario.turnCount) reject("scenario");
+    scenarios.add(scenario.scenarioId); roleCounts[scenario.variantRole]++;
+    const turnIds = new Set();
+    for (const [index, turn] of scenario.turns.entries()) {
+      const names = ["index", "turnId", "promptPath", "promptSha256", "promptBytes", "workflow",
+        "reconnectBefore", "receiptUncertain", "operatorRequestDigest", "bindingDigest"];
+      if (!exact(turn, names) || turn.index !== index + 1 || typeof turn.turnId !== "string" || !ID.test(turn.turnId)
+        || turnIds.has(turn.turnId) || typeof turn.promptPath !== "string" || !turn.promptPath
+        || !HASH.test(turn.promptSha256) || !Number.isSafeInteger(turn.promptBytes) || turn.promptBytes < 1
+        || turn.promptBytes > 64 * 1024 || !(turn.workflow === null || typeof turn.workflow === "string")
+        || typeof turn.reconnectBefore !== "boolean" || typeof turn.receiptUncertain !== "boolean"
+        || !OPERATOR_REQUEST.test(turn.operatorRequestDigest) || !HASH.test(turn.bindingDigest)
+        || bindings.has(turn.bindingDigest)) reject("turn");
+      const expected = hash(JSON.stringify({ version: 1, suiteId: value.suiteId,
+        scenarioId: scenario.scenarioId, planRef: scenario.planRef, variantRole: scenario.variantRole,
+        turnIndex: turn.index, turnId: turn.turnId, promptPath: turn.promptPath,
+        promptSha256: turn.promptSha256, promptBytes: turn.promptBytes, workflow: turn.workflow,
+        reconnectBefore: turn.reconnectBefore, receiptUncertain: turn.receiptUncertain,
+        operatorRequestDigest: turn.operatorRequestDigest }));
+      if (expected !== turn.bindingDigest) reject("binding-digest");
+      turnIds.add(turn.turnId); bindings.add(turn.bindingDigest); inputs++;
+    }
+  }
+  if (inputs !== value.publicInputCount || JSON.stringify(roleCounts) !== JSON.stringify(value.variantRoleCounts)) {
+    reject("counts");
+  }
+  return value;
+}
+
+export function validateBenchmarkPublicInputCatalog(value, expected = {}) {
+  catalogBody(value);
+  for (const [key, expectedValue] of Object.entries(expected)) {
+    if (key === "variantRoleCounts") {
+      if (JSON.stringify(value.variantRoleCounts) !== JSON.stringify(expectedValue)) reject("expected-variant-role-counts");
+    } else if (value[key] !== expectedValue) reject(`expected-${key}`);
+  }
+  return value;
+}
+
+export function benchmarkPublicInputCatalogDigest(value) {
+  return hash(JSON.stringify(validateBenchmarkPublicInputCatalog(value)));
+}
+
+export function buildBenchmarkPublicInputCatalog({ suite, suiteRoot, resolveSuiteEntry,
+  expectedScenarioCount, expectedPublicInputCount, expectedVariantRoleCounts } = {}) {
+  if (!suite || typeof suite !== "object" || !Array.isArray(suite.scenarios)
+    || typeof suite.id !== "string" || typeof resolveSuiteEntry !== "function") reject("suite");
+  const scenarios = suite.scenarios.map(scenario => {
+    if (!scenario?.userJourney || !Array.isArray(scenario.userJourney.turns)
+      || !VARIANT_ROLES.includes(scenario.variantRole)) reject("journey");
+    const resolved = resolvedJourneyTurns(scenario, suiteRoot, resolveSuiteEntry);
+    const turns = resolved.map((turn, index) => {
+      const source = scenario.userJourney.turns[index];
+      const promptPath = source.prompt, file = resolveSuiteEntry(suiteRoot, promptPath,
+        `public input ${scenario.id}/${turn.id}`), prompt = stablePrompt(file);
+      if (prompt.text !== turn.message) reject("resolved-prompt-drift");
+      const operatorRequestDigest = benchmarkVerificationRequestDigest(turn);
+      const body = { version: 1, suiteId: suite.id, scenarioId: scenario.id,
+        planRef: `plan:${scenario.id}`, variantRole: scenario.variantRole, turnIndex: index + 1,
+        turnId: turn.id, promptPath, promptSha256: prompt.sha256, promptBytes: prompt.byteLength,
+        workflow: turn.workflow ?? null, reconnectBefore: turn.reconnectBefore === true,
+        receiptUncertain: turn.receiptUncertain === true, operatorRequestDigest };
+      return { index: index + 1, turnId: turn.id, promptPath, promptSha256: prompt.sha256,
+        promptBytes: prompt.byteLength, workflow: turn.workflow ?? null,
+        reconnectBefore: turn.reconnectBefore === true, receiptUncertain: turn.receiptUncertain === true,
+        operatorRequestDigest, bindingDigest: hash(JSON.stringify(body)) };
+    });
+    return { scenarioId: scenario.id, variantRole: scenario.variantRole, planRef: `plan:${scenario.id}`,
+      turnCount: turns.length, turns };
+  });
+  const variantRoleCounts = Object.fromEntries(VARIANT_ROLES.map(role =>
+    [role, scenarios.filter(scenario => scenario.variantRole === role).length]));
+  const catalog = freeze({ schemaVersion: 1, kind: BENCHMARK_PUBLIC_INPUT_CATALOG_VERSION,
+    authority: "none", suiteId: suite.id, scenarioCount: scenarios.length,
+    publicInputCount: scenarios.reduce((sum, scenario) => sum + scenario.turnCount, 0),
+    variantRoleCounts, scenarios });
+  validateBenchmarkPublicInputCatalog(catalog, {
+    ...(expectedScenarioCount === undefined ? {} : { scenarioCount: expectedScenarioCount }),
+    ...(expectedPublicInputCount === undefined ? {} : { publicInputCount: expectedPublicInputCount }),
+    ...(expectedVariantRoleCounts === undefined ? {} : { variantRoleCounts: expectedVariantRoleCounts })
+  });
+  return Object.freeze({ catalog, digest: benchmarkPublicInputCatalogDigest(catalog) });
+}