@piagent/platform 1.6.1 → 1.8.0

package/scripts/benchmark-registered-plan-recipes-data.mjs

@@ -0,0 +1,164 @@
+import {
+  callReturns,
+  callThrows,
+  protocolRecord,
+  recipe,
+  utf8Chunks
+} from "./benchmark-registered-plan-recipe-support.mjs";
+
+function unicodeRecipe() {
+  return recipe("src/frontend/unicode-search.js", "normalizeSearchText", [
+    callReturns("normalize-null", [null], ""),
+    callReturns("normalize-undefined", [undefined], ""),
+    callReturns("normalize-case-accent-space", ["  CAFÉ\t au   LAIT  "], "cafe au lait"),
+    callReturns("normalize-decomposed", [" A\u030Angstro\u0308m "], "angstrom"),
+    callReturns("normalize-nondecomposable", ["  Øresund  "], "øresund"),
+    callReturns("includes-accent-case-space", ["  Café   au LAIT ", "cafe au"], true,
+      { exportName: "includesSearchText" }),
+    callReturns("includes-decomposed", ["Ångström", "angstrom"], true,
+      { exportName: "includesSearchText" }),
+    callReturns("includes-null-safe", [null, "x"], false, { exportName: "includesSearchText" }),
+    callReturns("includes-null-query", ["anything", null], true, { exportName: "includesSearchText" }),
+    callReturns("includes-miss", ["résumé", "sumo"], false, { exportName: "includesSearchText" })
+  ]);
+}
+
+function csvRecipe() {
+  return recipe("src/data/csv.js", "parseCsv", [
+    callReturns("empty", [""], [], { fresh: true }),
+    callReturns("simple-final-record", ["a,b"], [["a", "b"]], { fresh: true }),
+    callReturns("empty-physical-record", ["\n"], [[""]], { fresh: true }),
+    callReturns("trailing-terminator", ["a\n"], [["a"]], { fresh: true }),
+    callReturns("empty-fields", [",,"], [["", "", ""]], { fresh: true }),
+    callReturns("quoted-comma", ["\"a,b\",c"], [["a,b", "c"]], { fresh: true }),
+    callReturns("escaped-quote", ["\"a\"\"b\""], [["a\"b"]], { fresh: true }),
+    callReturns("crlf-records", ["a,b\r\nc,d\r\n"], [["a", "b"], ["c", "d"]], { fresh: true }),
+    callReturns("quoted-lf", ["\"a\nb\",c"], [["a\nb", "c"]], { fresh: true }),
+    callReturns("quoted-crlf-preserved", ["\"a\r\nb\",c"], [["a\r\nb", "c"]], { fresh: true }),
+    callReturns("quoted-empty", ["\"\""], [[""]], { fresh: true }),
+    callReturns("whitespace-preserved", [" a , b "], [[" a ", " b "]], { fresh: true }),
+    callThrows("unterminated-quote", ["\"a"], "SyntaxError"),
+    callThrows("quote-in-unquoted-field", ["a\"b"], "SyntaxError"),
+    callThrows("text-after-closing-quote", ["\"a\"x"], "SyntaxError"),
+    callThrows("bare-cr", ["a\rb"], "SyntaxError"),
+    callThrows("nonstring-null", [null]),
+    callThrows("nonstring-number", [1]),
+    callThrows("nonstring-array", [[]])
+  ]);
+}
+
+function ndjsonRecipe() {
+  const splitUnicode = utf8Chunks('{"text":"é😀"}\n', [10, 13]);
+  return recipe("src/data/ndjson-stream.js", "parseNdjsonChunks", [
+    callReturns("no-chunks", [[]], [], { fresh: true }),
+    callReturns("one-final-record", [utf8Chunks('{"id":1}')], [{ id: 1 }], { fresh: true }),
+    callReturns("lf-records-and-empty-lines", [utf8Chunks('{"id":1}\n\n{"id":2}\n')],
+      [{ id: 1 }, { id: 2 }], { fresh: true }),
+    callReturns("crlf-records", [utf8Chunks('{"id":1}\r\n{"id":2}\r\n')],
+      [{ id: 1 }, { id: 2 }], { fresh: true }),
+    callReturns("ascii-split", [utf8Chunks('{"value":"split"}\n', [2, 7, 12])],
+      [{ value: "split" }], { fresh: true }),
+    callReturns("unicode-split-inside-codepoints", [splitUnicode], [{ text: "é😀" }], { fresh: true }),
+    callReturns("empty-chunks", [[new Uint8Array(), ...utf8Chunks('0\nnull\n'), new Uint8Array()]],
+      [0, null], { fresh: true }),
+    callReturns("json-array-record", [utf8Chunks('[1,2,3]\n')], [[1, 2, 3]], { fresh: true }),
+    callThrows("invalid-utf8", [[new Uint8Array([0xc3, 0x28])]]),
+    callThrows("invalid-json", [utf8Chunks('{]\n')], "SyntaxError")
+  ]);
+}
+
+function migrationRecipe() {
+  const v2 = { version: 2, enabled: false, retryLimit: 0, label: "", metadata: { owner: "ops" } };
+  const freshPairs = [{ id: "fresh-outer", left: { root: "return", path: [] },
+    right: { root: "argument", index: 0, path: [] } }];
+  return recipe("src/data/migration.js", "migrateSettings", [
+    callReturns("default-argument", [], { version: 2, enabled: true, retryLimit: 3, label: "default" }, { fresh: true }),
+    callReturns("v1-falsey-values", [{ version: 1, enabled: false, retries: 0, name: "" }],
+      { version: 2, enabled: false, retryLimit: 0, label: "" }, { fresh: true }),
+    callReturns("v1-nullish-defaults", [{ version: 1, enabled: null, retries: null, name: null }],
+      { version: 2, enabled: true, retryLimit: 3, label: "default" }, { fresh: true }),
+    callReturns("v1-mixed-values", [{ version: 1, enabled: true, retries: 8, name: "nightly" }],
+      { version: 2, enabled: true, retryLimit: 8, label: "nightly" }, { fresh: true }),
+    callReturns("v2-independent-outer-copy", [v2], v2,
+      { fresh: true, referencePairs: freshPairs,
+        referenceIdentity: [{ id: "fresh-outer", same: false }] })
+  ]);
+}
+
+function dedupRecipe() {
+  return recipe("src/data/dedup.js", "deduplicateEvents", [
+    callReturns("empty", [[]], [], { fresh: true }),
+    callReturns("first-id-order-latest-value", [[
+      { id: "a", sequence: 1, value: "old-a" },
+      { id: "b", sequence: 9, value: "b" },
+      { id: "a", sequence: 3, value: "new-a" }
+    ]], [{ id: "a", sequence: 3, value: "new-a" }, { id: "b", sequence: 9, value: "b" }], { fresh: true }),
+    callReturns("later-tie-wins", [[
+      { id: "a", sequence: 5, value: "first" },
+      { id: "a", sequence: 5, value: "later" }
+    ]], [{ id: "a", sequence: 5, value: "later" }], { fresh: true }),
+    callReturns("greatest-numeric-sequence", [[
+      { id: "a", sequence: -2, value: "negative" },
+      { id: "a", sequence: 0.5, value: "fraction" },
+      { id: "a", sequence: 0, value: "zero" }
+    ]], [{ id: "a", sequence: 0.5, value: "fraction" }], { fresh: true }),
+    callReturns("stable-distinct-order", [[
+      { id: "z", sequence: 0 }, { id: "a", sequence: 100 }, { id: "m", sequence: 2 }
+    ]], [{ id: "z", sequence: 0 }, { id: "a", sequence: 100 }, { id: "m", sequence: 2 }], { fresh: true })
+  ]);
+}
+
+function replayRecipe() {
+  const empty = () => ({ entities: {}, appliedEventIds: [] });
+  const event = overrides => ({ eventId: "event-1", entityId: "entity-1", expectedVersion: 0,
+    nextValue: { name: "first" }, ...overrides });
+  const freshPairs = [
+    { id: "fresh-state", left: { root: "return", path: [] }, right: { root: "argument", index: 0, path: [] } },
+    { id: "fresh-entities", left: { root: "return", path: ["entities"] }, right: { root: "argument", index: 0, path: ["entities"] } },
+    { id: "fresh-applied", left: { root: "return", path: ["appliedEventIds"] }, right: { root: "argument", index: 0, path: ["appliedEventIds"] } }
+  ];
+  const specialInitial = protocolRecord([["entities", protocolRecord([])], ["appliedEventIds", []]]);
+  const specialExpected = protocolRecord([["entities", protocolRecord([["__proto__", { version: 1, value: "safe" }]])],
+    ["appliedEventIds", ["special"]]]);
+  return recipe("src/data/versioned-replay.js", "replayVersionedEvents", [
+    callReturns("empty-events-fresh-containers", [empty(), []], empty(), { fresh: true,
+      referencePairs: freshPairs, referenceIdentity: freshPairs.map(pair => ({ id: pair.id, same: false })) }),
+    callReturns("one-new-entity", [empty(), [event({})]],
+      { entities: { "entity-1": { version: 1, value: { name: "first" } } }, appliedEventIds: ["event-1"] }, { fresh: true }),
+    callReturns("ordered-version-advance", [empty(), [event({}), event({ eventId: "event-2", expectedVersion: 1,
+      nextValue: { name: "second" } })]],
+      { entities: { "entity-1": { version: 2, value: { name: "second" } } }, appliedEventIds: ["event-1", "event-2"] }, { fresh: true }),
+    callReturns("initially-applied-idempotent", [{ entities: { "entity-1": { version: 4, value: "kept" } },
+      appliedEventIds: ["event-1"] }, [event({ expectedVersion: 999, nextValue: "ignored" })]],
+      { entities: { "entity-1": { version: 4, value: "kept" } }, appliedEventIds: ["event-1"] }, { fresh: true }),
+    callReturns("duplicate-in-batch-idempotent", [empty(), [event({ nextValue: "first" }),
+      event({ expectedVersion: 999, nextValue: "ignored" })]],
+      { entities: { "entity-1": { version: 1, value: "first" } }, appliedEventIds: ["event-1"] }, { fresh: true }),
+    callReturns("literal-special-entity-id", [specialInitial,
+      [{ eventId: "special", entityId: "__proto__", expectedVersion: 0, nextValue: "safe" }]], specialExpected, { fresh: true }),
+    callThrows("absent-entity-version-conflict", [empty(), [event({ expectedVersion: 1 })]], "Error"),
+    callThrows("maximum-safe-increment", [{ entities: { "entity-1": { version: Number.MAX_SAFE_INTEGER, value: null } },
+      appliedEventIds: [] }, [event({ expectedVersion: Number.MAX_SAFE_INTEGER })]]),
+    callThrows("malformed-later-outranks-conflict", [empty(), [event({ expectedVersion: 1 }),
+      { eventId: "later", entityId: "entity-2", expectedVersion: 0 }]]),
+    callThrows("malformed-duplicate-still-invalid", [empty(), [event({}),
+      { eventId: "event-1", entityId: "entity-1", expectedVersion: 999 }]]),
+    callThrows("duplicate-initial-applied-ids", [{ entities: {}, appliedEventIds: ["same", "same"] }, []]),
+    callThrows("unsafe-entity-version", [{ entities: { x: { version: Number.MAX_SAFE_INTEGER + 1, value: null } },
+      appliedEventIds: [] }, []]),
+    callThrows("nonfinite-json-value", [empty(), [event({ nextValue: { invalid: NaN } })]])
+  ]);
+}
+
+const RECIPES = Object.freeze({
+  "unicode-search": unicodeRecipe,
+  "quoted-csv": csvRecipe,
+  "chunked-record-boundary": ndjsonRecipe,
+  "schema-migration": migrationRecipe,
+  "stable-dedup": dedupRecipe,
+  "idempotent-replay-conflict": replayRecipe
+});
+
+export function registeredDataRecipe(scenarioId) {
+  return RECIPES[scenarioId]?.() ?? null;
+}

package/scripts/benchmark-registered-plan-recipes-platform.mjs

@@ -0,0 +1,165 @@
+import { callReturns, callThrows, recipe } from "./benchmark-registered-plan-recipe-support.mjs";
+
+function argsRecipe() {
+  return recipe("src/platform/args.js", "parseArgs", [
+    callReturns("empty", [[]], { flags: {}, positional: [] }, { fresh: true }),
+    callReturns("name-space-value", [["--name", "value"]], { flags: { name: "value" }, positional: [] }, { fresh: true }),
+    callReturns("name-equals-value", [["--name=value"]], { flags: { name: "value" }, positional: [] }, { fresh: true }),
+    callReturns("empty-equals-value", [["--name="]], { flags: { name: "" }, positional: [] }, { fresh: true }),
+    callReturns("boolean-at-end", [["--verbose"]], { flags: { verbose: true }, positional: [] }, { fresh: true }),
+    callReturns("flag-before-flag", [["--verbose", "--color"]],
+      { flags: { verbose: true, color: true }, positional: [] }, { fresh: true }),
+    callReturns("double-dash-terminates", [["--name", "value", "--", "--literal", "tail"]],
+      { flags: { name: "value" }, positional: ["--literal", "tail"] }, { fresh: true }),
+    callReturns("first-double-dash", [["--", "--name=value", "-x"]],
+      { flags: {}, positional: ["--name=value", "-x"] }, { fresh: true }),
+    callReturns("single-dash-positional", [["-x", "plain"]], { flags: {}, positional: ["-x", "plain"] }, { fresh: true }),
+    callReturns("repeated-last-wins", [["--name", "first", "--name=second", "--name", "third"]],
+      { flags: { name: "third" }, positional: [] }, { fresh: true })
+  ]);
+}
+
+function workspaceRecipe() {
+  const pkg = (name, dependencies, extra = {}) => ({ name, ...(dependencies === undefined ? {} : { dependencies }), ...extra });
+  return recipe("src/platform/workspace.js", "workspaceOrder", [
+    callReturns("empty", [[]], [], { fresh: true }),
+    callReturns("no-edge-input-order", [[pkg("B", []), pkg("A", null), pkg("C", undefined)]],
+      ["B", "A", "C"], { fresh: true }),
+    callReturns("ready-recomputed-a-to-c", [[pkg("A", ["C"]), pkg("B", []), pkg("C", [])]],
+      ["B", "C", "A"], { fresh: true }),
+    callReturns("ready-recomputed-a-to-b", [[pkg("A", ["B"]), pkg("B", []), pkg("C", [])]],
+      ["B", "A", "C"], { fresh: true }),
+    callReturns("multiple-dependencies", [[pkg("A", ["C", "B"]), pkg("B", []), pkg("C", [])]],
+      ["B", "C", "A"], { fresh: true }),
+    callReturns("duplicate-and-foreign-dependency", [[pkg("A", ["B", "B", "missing"]), pkg("B", [])]],
+      ["B", "A"], { fresh: true }),
+    callReturns("literal-names", [[pkg(" A ", ["B"]), pkg("B", []), pkg("b", [])]],
+      ["B", " A ", "b"], { fresh: true }),
+    callReturns("metadata-ignored", [[pkg("A", [], { private: { ignored: true } })]], ["A"], { fresh: true }),
+    callThrows("duplicate-package-name", [[pkg("A", []), pkg("A", [])]]),
+    callThrows("empty-name", [[pkg("", [])]]),
+    callThrows("whitespace-name", [[pkg("   ", [])]]),
+    callThrows("dependencies-not-array", [[pkg("A", "B")]]),
+    callThrows("invalid-dependency-name", [[pkg("A", [" "])]]),
+    callThrows("malformed-outranks-cycle", [[pkg("A", ["A"]), pkg("", [])]]),
+    callThrows("self-cycle", [[pkg("A", ["A"])]], "Error"),
+    callThrows("two-node-cycle", [[pkg("A", ["B"]), pkg("B", ["A"])]], "Error")
+  ]);
+}
+
+function contractResult({ compatible, missingStatuses = [], extraStatuses = [], missingFields = [], versionMismatch = false }) {
+  return { compatible, missingStatuses, extraStatuses, missingFields, versionMismatch };
+}
+
+function contractRecipe() {
+  const contract = (statuses, fields, version, extra = {}) => ({ statuses, fields, version, ...extra });
+  return recipe("src/fullstack/contract-sync.js", "compareSubscriptionContracts", [
+    callReturns("compatible-reordered-extra-fields", [
+      contract(["active", "trial"], ["id", "plan"], 1),
+      contract(["trial", "active"], ["plan", "id", "debug"], 1, { metadata: "ignored" })
+    ], contractResult({ compatible: true }), { fresh: true }),
+    callReturns("all-mismatches", [
+      contract(["active", "trial"], ["id", "plan"], 1),
+      contract(["paused", "active"], ["id"], 2)
+    ], contractResult({ compatible: false, missingStatuses: ["trial"], extraStatuses: ["paused"],
+      missingFields: ["plan"], versionMismatch: true }), { fresh: true }),
+    callReturns("empty-compatible", [contract([], [], 1), contract([], [], 1)],
+      contractResult({ compatible: true }), { fresh: true }),
+    callReturns("utf8-byte-order", [contract(["😀", "é", "z", "\uE000"], [], 1), contract([], [], 1)],
+      contractResult({ compatible: false, missingStatuses: ["z", "é", "\uE000", "😀"] }), { fresh: true }),
+    callReturns("literal-unicode-distinct", [contract(["é", "e\u0301"], [], 1), contract(["é"], [], 1)],
+      contractResult({ compatible: false, missingStatuses: ["e\u0301"] }), { fresh: true }),
+    callReturns("special-and-whitespace-declarations", [contract([" ", "__proto__", "constructor"], [], 1),
+      contract(["constructor", "__proto__", " "], [], 1)], contractResult({ compatible: true }), { fresh: true }),
+    callReturns("large-integer-version", [contract([], [], 1e20), contract([], [], 1e20)],
+      contractResult({ compatible: true }), { fresh: true }),
+    callThrows("duplicate-status", [contract(["a", "a"], [], 1), contract([], [], 1)]),
+    callThrows("missing-fields-property", [{ statuses: [], version: 1 }, contract([], [], 1)]),
+    callThrows("zero-version", [contract([], [], 0), contract([], [], 1)]),
+    callThrows("fractional-version", [contract([], [], 1.5), contract([], [], 1)]),
+    callThrows("nonfinite-version", [contract([], [], Infinity), contract([], [], 1)]),
+    callThrows("string-version", [contract([], [], "1"), contract([], [], 1)]),
+    callThrows("empty-declaration", [contract([""], [], 1), contract([], [], 1)]),
+    callThrows("lone-surrogate", [contract(["\uD800"], [], 1), contract([], [], 1)]),
+    callThrows("frontend-malformed-even-if-incompatible", [contract(["backend"], [], 1),
+      contract(["front", "front"], [], 2)])
+  ]);
+}
+
+function message(overrides = {}) {
+  return { eventId: "event-1", sequence: 1, kind: "message", messageId: "message-1",
+    role: "user", text: "hello", confirmed: false, ...overrides };
+}
+
+function lifecycle(overrides = {}) {
+  return { eventId: "lifecycle-1", sequence: 1, kind: "lifecycle", state: "started", ...overrides };
+}
+
+function projected(overrides = {}) {
+  return { messageId: "message-1", role: "user", text: "hello", sequence: 1, confirmed: false, ...overrides };
+}
+
+function chatRecipe() {
+  return recipe("src/frontend/chat-events.js", "projectChatEvents", [
+    callReturns("empty", [[]], { messages: [], processing: false }, { fresh: true }),
+    callReturns("latest-lifecycle-settled", [[lifecycle({ sequence: 9 }),
+      lifecycle({ eventId: "settled", sequence: 10, state: "settled" }),
+      lifecycle({ eventId: "old-start", sequence: 2 })]], { messages: [], processing: false }, { fresh: true }),
+    callReturns("same-sequence-settled-wins", [[lifecycle({ eventId: "start", sequence: 10 }),
+      lifecycle({ eventId: "settled", sequence: 10, state: "settled" })]],
+      { messages: [], processing: false }, { fresh: true }),
+    callReturns("messages-sorted-and-reply-retained", [[
+      message({ eventId: "assistant-event", sequence: 4, messageId: "assistant", role: "assistant",
+        text: "answer", confirmed: true, replyTo: "user" }),
+      message({ eventId: "user-event", sequence: 4, messageId: "user", role: "user", text: "question", confirmed: true })
+    ]], { messages: [
+      projected({ messageId: "user", text: "question", sequence: 4, confirmed: true }),
+      projected({ messageId: "assistant", role: "assistant", text: "answer", sequence: 4,
+        confirmed: true, replyTo: "user" })
+    ], processing: false }, { fresh: true }),
+    callReturns("identical-event-id-collapses-metadata", [[
+      message({ eventId: "same", metadata: "first" }), message({ eventId: "same", metadata: "second" })
+    ]], { messages: [projected({})], processing: false }, { fresh: true }),
+    callThrows("conflicting-event-id", [[message({ eventId: "same" }), message({ eventId: "same", text: "changed" })]]),
+    callReturns("confirmed-beats-pending", [[
+      message({ eventId: "pending", sequence: 9, text: "pending", confirmed: false }),
+      message({ eventId: "confirmed", sequence: 2, text: "confirmed", confirmed: true })
+    ]], { messages: [projected({ sequence: 2, text: "confirmed", confirmed: true })], processing: false }, { fresh: true }),
+    callReturns("identical-confirmed-copies", [[
+      message({ eventId: "confirmed-a", sequence: 2, confirmed: true }),
+      message({ eventId: "confirmed-b", sequence: 2, confirmed: true })
+    ]], { messages: [projected({ sequence: 2, confirmed: true })], processing: false }, { fresh: true }),
+    callThrows("conflicting-confirmed-message", [[
+      message({ eventId: "confirmed-a", confirmed: true }),
+      message({ eventId: "confirmed-b", confirmed: true, text: "different" })
+    ]], "Error"),
+    callReturns("pending-greatest-sequence", [[
+      message({ eventId: "early", sequence: 1, text: "early" }),
+      message({ eventId: "late", sequence: 3, text: "late" })
+    ]], { messages: [projected({ sequence: 3, text: "late" })], processing: false }, { fresh: true }),
+    callReturns("pending-tie-smallest-event-id", [[
+      message({ eventId: "z", sequence: 3, text: "z-winner-candidate" }),
+      message({ eventId: "a", sequence: 3, text: "a-winner" })
+    ]], { messages: [projected({ sequence: 3, text: "a-winner" })], processing: false }, { fresh: true }),
+    callThrows("missing-reply-target", [[message({ eventId: "a", role: "assistant", replyTo: "missing" })]]),
+    callThrows("user-with-reply", [[message({ replyTo: "message-1" })]]),
+    callThrows("parent-after-assistant", [[
+      message({ eventId: "user", messageId: "user", sequence: 5, confirmed: true }),
+      message({ eventId: "assistant", messageId: "assistant", role: "assistant", sequence: 4,
+        confirmed: true, replyTo: "user" })
+    ]]),
+    callThrows("malformed-later-event", [[message({}), { eventId: "bad", sequence: -1, kind: "lifecycle", state: "started" }]]),
+    callThrows("explicit-undefined-reply", [[message({ replyTo: undefined })]])
+  ]);
+}
+
+const RECIPES = Object.freeze({
+  "cli-double-dash": argsRecipe,
+  "workspace-order": workspaceRecipe,
+  "backend-frontend-contract-sync": contractRecipe,
+  "reconnect-chat-event-order": chatRecipe
+});
+
+export function registeredPlatformRecipe(scenarioId) {
+  return RECIPES[scenarioId]?.() ?? null;
+}

package/scripts/benchmark-runner-completion.mjs

@@ -0,0 +1,133 @@
+import fs from "node:fs";
+import path from "node:path";
+
+import {
+  MEASUREMENT_INVALIDATING_ONLY_STOP_POLICY,
+  productionMeasurementInvalidatingRecordIssues
+} from "../packages/piagent-core/benchmark/benchmark-production-completion.js";
+import { productionV3FatalMeasurementEvidence } from "../packages/piagent-core/benchmark/benchmark-claim-restrictions.js";
+import { writeBenchmarkAbort, writePrivateAtomic } from "../packages/piagent-core/benchmark/benchmark-forensics.js";
+import { durablePairedOutcomeFloorStop } from "../packages/piagent-core/benchmark/benchmark-stage-diagnostic.js";
+
+function benchmarkFailure(message) {
+  const error = new Error(message);
+  error.exitCode = 1;
+  return error;
+}
+
+function productionMeasurementContext(options, suite) {
+  return {
+    requestedModel: options.model,
+    requestedThinking: options.thinking,
+    requestedServiceTier: options.serviceTier,
+    requireFastServiceTier: suite?.releaseGate?.requireFastServiceTier === true
+  };
+}
+
+export function applyProductionCampaignStopPolicy(options, spendExecution) {
+  const campaignStopPolicy = spendExecution.campaignStopPolicy ?? "paired-outcome-floor";
+  if (options.campaignStopPolicy !== undefined && options.campaignStopPolicy !== campaignStopPolicy) {
+    throw benchmarkFailure("Production spend control campaign stop policy changed since the original run");
+  }
+  options.campaignStopPolicy = spendExecution.campaignStopPolicy;
+}
+
+export function productionReleaseMeasurementAbortEvidence(options, suite, error) {
+  return suite?.id === "production-v3" && options?.measurementOnly !== true
+    ? productionV3FatalMeasurementEvidence(error)
+    : {};
+}
+
+export function writeProductionResumeAbortIfNeeded(resumeState, error) {
+  const manifest = resumeState?.manifest;
+  if (manifest?.suite?.id !== "production-v3" || manifest.measurementOnly === true
+    || fs.existsSync(path.join(resumeState.runRoot, "aborted.json"))) return;
+  writeBenchmarkAbort(resumeState.runRoot, {
+    runId: manifest.runId,
+    completedRuns: resumeState.completedRuns.length,
+    expectedRuns: Array.isArray(manifest.order) ? manifest.order.length : 108
+  }, error, { ledger: resumeState.ledgerBinding, ...productionV3FatalMeasurementEvidence(error) });
+}
+
+export function writeProductionRunAbortIfNeeded({
+  fullOrder,
+  ledgerBinding,
+  manifest,
+  options,
+  productionSpendControlled,
+  runRoot,
+  runs,
+  suite
+}, error) {
+  if (!productionSpendControlled || suite?.id !== "production-v3" || options?.measurementOnly === true
+    || fs.existsSync(path.join(runRoot, "aborted.json"))) return;
+  writeBenchmarkAbort(runRoot, {
+    runId: manifest.runId,
+    completedRuns: runs.length,
+    expectedRuns: fullOrder.length
+  }, error, { ledger: ledgerBinding, ...productionV3FatalMeasurementEvidence(error) });
+}
+
+export function assertProductionResumeCompletionState({
+  candidateGuard,
+  fullOrder,
+  options,
+  resumeState,
+  suite
+}) {
+  const { manifest } = resumeState;
+  const recoveredTerminalStop = durablePairedOutcomeFloorStop({
+    enabled: options.stopAfterFailedPair,
+    suite,
+    runs: resumeState.completedRuns,
+    fullOrder
+  });
+  if (recoveredTerminalStop) {
+    for (const marker of ["paused.json", "stage-diagnostic.json", "interrupted.json", "aborted.json"]) {
+      fs.rmSync(path.join(resumeState.runRoot, marker), { force: true });
+    }
+    writePrivateAtomic(path.join(resumeState.runRoot, "stopped.json"), `${JSON.stringify({
+      ...recoveredTerminalStop,
+      runId: manifest.runId,
+      completedRuns: resumeState.completedRuns.length,
+      expectedRuns: fullOrder.length,
+      stoppedAt: new Date().toISOString(),
+      resumeAllowed: false,
+      recoveredFromAcceptedLedger: true,
+      ledger: resumeState.ledgerBinding,
+      provenanceStamp: candidateGuard.stamp("resume-terminal-floor")
+    }, null, 2)}\n`);
+    throw benchmarkFailure("Cannot resume benchmark: the accepted ledger already contains a terminal paired outcome-floor failure");
+  }
+  if (options.campaignStopPolicy !== MEASUREMENT_INVALIDATING_ONLY_STOP_POLICY) return;
+  const measurementContext = productionMeasurementContext(options, suite);
+  const invalidRecord = resumeState.completedRuns.map((record) => ({
+    record,
+    issues: productionMeasurementInvalidatingRecordIssues(record, measurementContext)
+  })).find((item) => item.issues.length > 0);
+  if (!invalidRecord) return;
+  const error = Object.assign(benchmarkFailure(
+    `Cannot resume benchmark: the accepted ledger contains a measurement-invalidating outcome (${invalidRecord.record.scenarioId}:${invalidRecord.record.surface}:r${invalidRecord.record.repeat}; ${invalidRecord.issues.join(", ")})`
+  ), { code: "BENCHMARK_INVALID_MEASUREMENT", measurementInvalidatingIssues: invalidRecord.issues });
+  writeBenchmarkAbort(resumeState.runRoot, {
+    runId: manifest.runId,
+    completedRuns: resumeState.completedRuns.length,
+    expectedRuns: fullOrder.length
+  }, error, {
+    ledger: resumeState.ledgerBinding,
+    provenanceStamp: candidateGuard.stamp("resume-invalid-measurement"),
+    ...productionV3FatalMeasurementEvidence(error)
+  });
+  throw error;
+}
+
+export function productionMeasurementInvalidatingOutcomeError(options, suite, record) {
+  if (options.campaignStopPolicy !== MEASUREMENT_INVALIDATING_ONLY_STOP_POLICY) return null;
+  const measurementInvalidatingIssues = productionMeasurementInvalidatingRecordIssues(
+    record, productionMeasurementContext(options, suite)
+  );
+  if (measurementInvalidatingIssues.length === 0) return null;
+  return Object.assign(new Error(
+    `Measurement-invalidating production-v3 outcome at ${record.scenarioId}:${record.surface}:r${record.repeat} (${measurementInvalidatingIssues.join(", ")})`
+  ), { code: "BENCHMARK_INVALID_MEASUREMENT", measurementInvalidatingIssues });
+}

package/scripts/benchmark-runner-configuration.mjs

@@ -0,0 +1,159 @@
+import crypto from "node:crypto";
+import fs from "node:fs";
+import path from "node:path";
+
+import { BENCHMARK_TRANSPORT_CIRCUIT_POLICY } from "../packages/piagent-core/benchmark/benchmark-forensics.js";
+import { benchmarkEnvironmentPolicy } from "../packages/piagent-core/benchmark/benchmark-runtime.js";
+import { benchmarkCommandIdentity, benchmarkStockCodexIdentity } from "../packages/piagent-core/benchmark/benchmark-runtime-identity.js";
+import { REGISTERED_BENCHMARK_VERIFIER_IDS
+} from "../packages/piagent-core/benchmark/benchmark-suite-identity.js";
+import { scopedBrokerSourceClosureSha256, scopedCommonRuntimeClosureIdentity,
+  scopedFrozenQualificationIdentity, scopedToolDefinitionsSha256
+} from "./benchmark-scoped-frozen-qualification.mjs";
+import { frozenRuntimeCommandsForFinalization } from "./benchmark-runner-support.mjs";
+
+const runtimeVerifierIds = Object.freeze([
+  "bash-runtime", "broker-source-closure", "candidate-source", "common-runtime-closure",
+  "controlled-comparison-runtime", "docker-runtime", "git-runtime", "node-runtime", "npm-runtime", "pi-runtime",
+  "pi-sdk-tree", "tool-definitions", "webui-asset-tree", "worker-image"
+]);
+const evidenceOnlyVerifierIds = Object.freeze(REGISTERED_BENCHMARK_VERIFIER_IDS
+  .filter(id => !runtimeVerifierIds.includes(id)));
+const verifierFailure = message => { throw Object.assign(new Error(message), { exitCode: 1 }); };
+
+export function assertRegisteredRuntimeVerifierBindings(approvedEntries, observedEntries) {
+  if (!Array.isArray(approvedEntries)
+    || JSON.stringify(approvedEntries.map(item => item?.id)) !== JSON.stringify(REGISTERED_BENCHMARK_VERIFIER_IDS)) {
+    verifierFailure("Registered measurement runtime verifier inventory is incomplete or unsupported");
+  }
+  const approved = new Map(approvedEntries.map(item => [item.id, item.sha256]));
+  const observed = observedEntries instanceof Map ? observedEntries : new Map(observedEntries);
+  for (const id of runtimeVerifierIds) if (!/^[a-f0-9]{64}$/.test(String(observed.get(id) ?? ""))
+    || approved.get(id) !== observed.get(id)) {
+    verifierFailure(`Registered measurement runtime verifier mismatch: ${id}`);
+  }
+  return Object.freeze(runtimeVerifierIds.map(id => Object.freeze({ id, sha256: observed.get(id) })));
+}
+
+export function registeredRuntimeVerifierReceipt({ registeredMeasurement, bootstrapMetadata,
+  runtimeCommands } = {}) {
+  if (!registeredMeasurement) return null;
+  const approvedEntries = registeredMeasurement.payload?.resources?.verifiers;
+  const common = scopedCommonRuntimeClosureIdentity(), qualification = scopedFrozenQualificationIdentity({
+    version: 4,
+    candidateRoot: bootstrapMetadata.snapshotRoot,
+    assetsRoot: bootstrapMetadata.webUiAssets?.root,
+    sdkRoot: runtimeCommands.pi?.packageClosure?.root,
+    candidateIndexPath: bootstrapMetadata.candidateIndex?.path,
+    candidateIndexSha256: bootstrapMetadata.candidateIndex?.digest
+  }, common.sha256);
+  let workerImageId;
+  try {
+    const profile = JSON.parse(fs.readFileSync(path.join(registeredMeasurement.assetRoot,
+      "measurement", "node-workload-api-v1.json"), "utf8"));
+    workerImageId = profile.workerImage?.id;
+  } catch {
+    verifierFailure("Registered measurement worker image identity is unavailable");
+  }
+  if (!/^sha256:[a-f0-9]{64}$/.test(String(workerImageId ?? ""))) {
+    verifierFailure("Registered measurement worker image identity is invalid");
+  }
+  const npm = benchmarkCommandIdentity(path.join(path.dirname(runtimeCommands.node.resolvedPath), "npm"));
+  const observed = new Map([
+    ["bash-runtime", runtimeCommands.bash?.contentDigest],
+    ["broker-source-closure", scopedBrokerSourceClosureSha256()],
+    ["candidate-source", qualification.sourceSha256],
+    ["common-runtime-closure", qualification.brokerClosureSha256],
+    ["controlled-comparison-runtime", runtimeCommands.codex?.contentDigest],
+    ["docker-runtime", runtimeCommands.docker?.contentDigest],
+    ["git-runtime", runtimeCommands.git?.contentDigest],
+    ["node-runtime", runtimeCommands.node?.contentDigest],
+    ["npm-runtime", npm.contentDigest],
+    ["pi-runtime", runtimeCommands.pi?.contentDigest],
+    ["pi-sdk-tree", qualification.sdkTreeSha256],
+    ["tool-definitions", scopedToolDefinitionsSha256()],
+    ["webui-asset-tree", qualification.assetTreeSha256],
+    ["worker-image", workerImageId.slice("sha256:".length)]
+  ]);
+  const bindings = assertRegisteredRuntimeVerifierBindings(approvedEntries, observed);
+  const receipt = { schemaVersion: 1, kind: "registered-runtime-verifier-receipt-v1", outcome: "PASS",
+    bindings, evidenceOnlyVerifierIds: [...evidenceOnlyVerifierIds], providerSessionsStarted: 0 };
+  return Object.freeze({ ...receipt,
+    contentDigest: crypto.createHash("sha256").update(JSON.stringify(receipt)).digest("hex") });
+}
+
+export function applyRegisteredMeasurementOptions(options, registeredMeasurement) {
+  if (!registeredMeasurement) return false;
+  if (options.registeredMeasurement && options.registeredMeasurement !== registeredMeasurement.manifestOrigin) {
+    throw Object.assign(new Error("Registered measurement invocation does not match its trusted frozen registration"),
+      { exitCode: 1 });
+  }
+  const payload = registeredMeasurement.payload;
+  Object.assign(options, { registeredMeasurement: registeredMeasurement.manifestOrigin,
+    suite: payload.baseSuiteId, surfaces: [...payload.matrix.surfaces], model: payload.resources.model,
+    thinking: payload.resources.thinking, serviceTier: payload.resources.requestedServiceTier,
+    codexMode: "controlled", codexBaseline: "controlled-custom", piagentTreatment: payload.treatment, repeats: payload.matrix.repeats,
+    infrastructureRetries: payload.resources.infrastructureRetries, timeoutSeconds: payload.resources.timeoutSeconds,
+    stopAfterFailedPair: false, measurementOnly: false });
+  return true;
+}
+
+export function registeredMeasurementBinding(registeredMeasurement) {
+  if (!registeredMeasurement) return null;
+  return Object.freeze({ schemaVersion: 1, manifestOrigin: registeredMeasurement.manifestOrigin,
+    suiteId: registeredMeasurement.payload.suiteId, payloadSha256: registeredMeasurement.identity.payloadSha256,
+    manifestSha256: registeredMeasurement.identity.manifestSha256,
+    approvalRecordSha256: registeredMeasurement.identity.approvalRecordSha256,
+    approvalId: registeredMeasurement.identity.approvalId,
+    authorityKeyId: registeredMeasurement.identity.authorityKeyId,
+    publicAssetsDigest: registeredMeasurement.inventory.publicAssetsDigest,
+    publicContractDigest: registeredMeasurement.inventory.publicContractDigest,
+    assetTreeSha256: registeredMeasurement.tree.contentDigest });
+}
+
+export function benchmarkRuntimeCommands({ productionFinalizationOnly, resumeManifest, surfaces,
+  piCommand, codexCommand, codexBaseline, dockerCommand, cwd, nodeCommand = process.execPath }) {
+  if (productionFinalizationOnly) return frozenRuntimeCommandsForFinalization(resumeManifest, surfaces);
+  return { pi: benchmarkCommandIdentity(piCommand, { cwd }),
+    codex: surfaces.includes("codex-cli") ? (codexBaseline === "stock"
+      ? benchmarkStockCodexIdentity(codexCommand, { cwd }) : benchmarkCommandIdentity(codexCommand, { cwd })) : null,
+    ...(dockerCommand ? { docker: benchmarkCommandIdentity(dockerCommand,
+      { cwd, fullPackageClosure: false }) } : {}),
+    node: benchmarkCommandIdentity(nodeCommand), git: benchmarkCommandIdentity("git"),
+    bash: benchmarkCommandIdentity("bash") };
+}
+
+export function benchmarkMeasurementConfiguration({ bootstrapMetadata, candidateDigest, suiteDigest,
+  runtimeCommands, verificationIdentity, providerWirePlan, productionHostReadinessRequired,
+  productionHostReadinessPolicyDigest, productionFinalizationOnly, resumeManifest, rootSeedDigest,
+  options, fullOrder }) {
+  const environmentPolicy = benchmarkEnvironmentPolicy(), configuration = { schemaVersion: 1,
+    source: bootstrapMetadata.sourceIdentity, candidateDigest, suiteDigest,
+    runtimeDependencyDigest: bootstrapMetadata.runtimeDependencies?.digest ?? null,
+    webUiAssetDigest: bootstrapMetadata.webUiAssets?.digest ?? null, runtimeCommands, environmentPolicy,
+    ...(verificationIdentity ? { independentVerification: verificationIdentity } : {}),
+    ...(providerWirePlan ? { providerWireProtocol: { version: providerWirePlan.manifest.protocol,
+      definitionDigest: providerWirePlan.manifest.definitionDigest, planDigest: providerWirePlan.planDigest } } : {}),
+    ...(productionHostReadinessRequired ? { hostReadinessPolicyDigest: productionHostReadinessPolicyDigest } : {}),
+    piAgentHome: productionFinalizationOnly ? resumeManifest.piAgentHome?.identity
+      : bootstrapMetadata.piAgentHome.identity,
+    codexCredential: productionFinalizationOnly ? resumeManifest.codexCredentialIdentity ?? null
+      : bootstrapMetadata.codexCredential?.identity ?? null,
+    rootSeedDigest, surfaces: options.surfaces, model: options.model ?? null,
+    thinking: options.thinking ?? null, serviceTier: options.serviceTier ?? null,
+    codexMode: options.codexMode, codexBaseline: options.codexBaseline, piagentTreatment: options.piagentTreatment,
+    ...(options.acceptancePolicyBinding ? { acceptancePolicyBinding: options.acceptancePolicyBinding } : {}),
+    ...(bootstrapMetadata.treatmentDerivation ? { treatmentDerivation: bootstrapMetadata.treatmentDerivation } : {}),
+    allowPiAuthWriteback: options.allowPiAuthWriteback,
+    ...(options.budgetControl ? { budgetControl: options.budgetControl } : {}),
+    piCredentialVaultId: bootstrapMetadata.piAgentHome.vaultId, timeoutSeconds: options.timeoutSeconds,
+    infrastructureRetries: options.infrastructureRetries, retryDelaySeconds: options.retryDelaySeconds,
+    transportCircuitBreaker: BENCHMARK_TRANSPORT_CIRCUIT_POLICY,
+    stopAfterFailedPair: options.stopAfterFailedPair,
+    ...(options.campaignStopPolicy ? { campaignStopPolicy: options.campaignStopPolicy } : {}),
+    ...(options.measurementOnly ? { measurementOnly: true } : {}),
+    ...(options.failedAttemptsOnly ? { failedAttemptsOnly: true, replaySource: options.replaySource } : {}),
+    order: fullOrder.map(item => ({ scenarioId: item.scenario.id, surface: item.surface, repeat: item.repeat })) };
+  return Object.freeze({ configuration, environmentPolicy,
+    configurationDigest: crypto.createHash("sha256").update(JSON.stringify(configuration)).digest("hex") });
+}