@piagent/platform 1.6.1 → 1.8.0

package/packages/piagent-core/benchmark/benchmark-provider-free-evidence.js

@@ -1,11 +1,24 @@
+import { collectBenchmarkCandidate } from "./benchmark-candidate.js";
+import { diagnosticS0SourceEvidence, validDiagnosticS0SourceEvidence } from "./benchmark-diagnostic-source-evidence.js";
 import crypto from "node:crypto";
 import fs from "node:fs";
-import os from "node:os";
 import path from "node:path";
+import { acquireBenchmarkRunLock } from "./benchmark-run-lock.js";
+import { checkpointDirectory, readCheckpoint, writeCheckpoint, writeCheckpointBytes } from "./benchmark-checkpoint.js";
 
 const HASH = /^[a-f0-9]{64}$/;
 const COMMIT = /^[a-f0-9]{40,64}$/;
+const PROVIDER_FREE_EXCLUDED_RUN_SCOPED_FIELDS = Object.freeze([
+  "dryRun", "json", "keepWorkspaces", "maxRuntimeMinutes", "maxSessions", "output",
+  "piCredentialVaultId", "preflightOnly", "resume", "runId", "yes"
+]);
 const LANE_DEFINITIONS = Object.freeze([
+  Object.freeze({
+    id: "architecture-conformance-v1",
+    configuration: "evals/architecture-conformance-v1/lane.json",
+    runner: "evals/architecture-conformance-v1/runner.mjs",
+    timeoutMilliseconds: 2 * 60_000
+  }),
   Object.freeze({
     id: "runtime-conformance-v1",
     configuration: "evals/runtime-conformance-v1/lane.json",
@@ -40,6 +53,19 @@ function receiptDigest(value) {
   return digestBytes(JSON.stringify(copy));
 }
 
+export function productionProviderFreeConfigurationDigest(configuration) {
+  if (!configuration || typeof configuration !== "object" || Array.isArray(configuration)) {
+    throw new Error("Production provider-free configuration must be an object");
+  }
+  const stableConfiguration = Object.fromEntries(Object.entries(configuration)
+    .filter(([key]) => !PROVIDER_FREE_EXCLUDED_RUN_SCOPED_FIELDS.includes(key)));
+  return digestBytes(JSON.stringify({
+    schemaVersion: 1,
+    excludedRunScopedFields: PROVIDER_FREE_EXCLUDED_RUN_SCOPED_FIELDS,
+    configuration: stableConfiguration
+  }));
+}
+
 function validCompletionTime(value) {
   const milliseconds = typeof value === "string" ? Date.parse(value) : Number.NaN;
   return Number.isFinite(milliseconds)
@@ -61,33 +87,61 @@ export function productionProviderFreeEvidenceBinding({
   packageRoot,
   source,
   candidateProvenance,
-  configurationDigest
+  providerFreeConfigurationDigest,
+  diagnosticSourceEvidence
 }) {
-  if (source?.kind !== "git-working-tree" || source.dirty !== false || !COMMIT.test(String(source.commit ?? ""))) {
+  if (diagnosticSourceEvidence && (!validDiagnosticS0SourceEvidence(diagnosticSourceEvidence)
+    || diagnosticSourceEvidence.commit !== source?.commit || diagnosticSourceEvidence.clean !== !source?.dirty
+    || diagnosticSourceEvidence.treeDigest !== candidateProvenance?.contentDigest
+    || diagnosticSourceEvidence.treeAlgorithm !== candidateProvenance?.algorithm)) throw new Error("Invalid diagnostic source evidence");
+  if (source?.kind !== "git-working-tree" || (source.dirty !== false && !diagnosticSourceEvidence) || !COMMIT.test(String(source.commit ?? ""))) {
     throw new Error("Production provider-free evidence requires an exact clean Git commit");
   }
   if (!HASH.test(String(candidateProvenance?.contentDigest ?? ""))) {
     throw new Error("Production provider-free evidence requires an exact candidate tree digest");
   }
-  if (!HASH.test(String(configurationDigest ?? ""))) {
-    throw new Error("Production provider-free evidence requires the frozen production configuration digest");
+  if (!HASH.test(String(providerFreeConfigurationDigest ?? ""))) {
+    throw new Error("Production provider-free evidence requires the frozen provider-free configuration digest");
   }
   const binding = {
-    schemaVersion: 1,
-    source: {
+    schemaVersion: 2,
+    source: diagnosticSourceEvidence ?? {
       kind: source.kind,
       commit: source.commit,
       clean: true,
       treeDigest: candidateProvenance.contentDigest,
       treeAlgorithm: candidateProvenance.algorithm
     },
-    productionConfigurationDigest: configurationDigest,
+    providerFreeConfigurationDigest,
+    hostRuntime: { node: process.version, platform: process.platform, arch: process.arch },
     lanes: expectedLaneBindings(packageRoot)
   };
   return { ...binding, digest: receiptDigest(binding) };
 }
 
 function lanePassed(id, result) {
+  if (id === "architecture-conformance-v1") {
+    return result?.schemaVersion === 1
+      && result.laneId === id
+      && result.evidenceClass === "provider-free-architecture-conformance"
+      && result.passed === true
+      && result?.provider?.required === false
+      && result.provider.used === false
+      && result.provider.calls === 0
+      && result.provider.modelTokens === 0
+      && Number.isSafeInteger(result?.architecture?.filesChecked)
+      && result.architecture.filesChecked > 0
+      && Number.isSafeInteger(result.architecture.layersChecked)
+      && result.architecture.layersChecked > 0
+      && Array.isArray(result.architecture.errors)
+      && result.architecture.errors.length === 0
+      && result?.gates?.architectureCheckPasses === true
+      && result.gates.sourceCoverageComplete === true
+      && result.gates.dependencyBoundariesPass === true
+      && result.gates.lineBudgetsPass === true
+      && result.gates.providerCalls === 0
+      && result.gates.modelTokens === 0;
+  }
   if (id === "runtime-conformance-v1") {
     return result?.summary?.passed === true
       && result?.provider?.used === false
@@ -106,15 +160,26 @@ function lanePassed(id, result) {
       && result?.continuation?.enforcementSafe === true
       && result?.verification?.stableCurrentTree === true;
   }
+  if (id !== "webui-parity-v1") return false;
   return result?.benchmark === "webui-parity-v1"
     && result?.passed === true
     && result?.providerCalls === 0
     && result?.modelTokens === 0
     && result?.invariants?.uiStability === "deterministic-current-state"
-    && result?.invariants?.uiStabilitySuites === 7;
+    && result?.invariants?.uiStabilitySuites === 9;
 }
 
 function laneSummary(id, result) {
+  if (id === "architecture-conformance-v1") return {
+    evidenceClass: result.evidenceClass,
+    passed: result.passed,
+    filesChecked: result.architecture.filesChecked,
+    layersChecked: result.architecture.layersChecked,
+    architectureCheckPasses: result.gates.architectureCheckPasses,
+    sourceCoverageComplete: result.gates.sourceCoverageComplete,
+    dependencyBoundariesPass: result.gates.dependencyBoundariesPass,
+    lineBudgetsPass: result.gates.lineBudgetsPass
+  };
   if (id === "runtime-conformance-v1") return {
     passed: result.summary.passed,
     configuredCases: result.summary.configuredCases,
@@ -141,6 +206,12 @@ function laneSummary(id, result) {
 }
 
 function summaryPassed(id, summary) {
+  if (id === "architecture-conformance-v1") return summary?.evidenceClass === "provider-free-architecture-conformance"
+    && summary.passed === true
+    && Number.isSafeInteger(summary.filesChecked) && summary.filesChecked > 0
+    && Number.isSafeInteger(summary.layersChecked) && summary.layersChecked > 0
+    && summary.architectureCheckPasses === true && summary.sourceCoverageComplete === true
+    && summary.dependencyBoundariesPass === true && summary.lineBudgetsPass === true;
   if (id === "runtime-conformance-v1") return summary?.passed === true
     && summary.configuredCases === summary.executedCases && summary.failedCases === 0
     && Object.values(summary.gates ?? {}).every((value) => value === true);
@@ -148,32 +219,29 @@ function summaryPassed(id, summary) {
     && summary.wallClockQualified === true && summary.completedFromResume === true
     && summary.contextWithinCeiling === true && summary.stateGrowthWithinCeiling === true
     && summary.continuationEnforcementSafe === true && summary.stableCurrentTree === true;
+  if (id !== "webui-parity-v1") return false;
   return summary?.benchmark === "webui-parity-v1" && summary.passed === true
     && summary.uiStability === "deterministic-current-state"
-    && summary.uiStabilitySuites === 7
+    && summary.uiStabilitySuites === 9
     && summary.deterministicStabilityStepPassed === true;
 }
 
 function privateWrite(file, value) {
-  fs.mkdirSync(path.dirname(file), { recursive: true, mode: 0o700 });
-  const temporary = `${file}.${process.pid}.${Date.now()}.tmp`;
-  fs.writeFileSync(temporary, value, { mode: 0o600 });
-  fs.renameSync(temporary, file);
-  try { fs.chmodSync(file, 0o600); } catch { /* Non-POSIX filesystem. */ }
+  writeCheckpointBytes(file, value);
 }
 
 export function productionProviderFreeEvidenceValidationErrors(receipt, expectedBinding) {
   const errors = [];
-  if (receipt?.schemaVersion !== 1 || receipt?.kind !== "production-provider-free-evidence-v1") {
+  if (receipt?.schemaVersion !== 2 || receipt?.kind !== "production-provider-free-evidence-v2") {
     return ["missing-or-unsupported-provider-free-evidence"];
   }
   if (!validCompletionTime(receipt.completedAt)) errors.push("provider-free-completion-time-invalid-or-future");
-  const validExpectedBinding = expectedBinding?.schemaVersion === 1
+  const validExpectedBinding = expectedBinding?.schemaVersion === 2
     && expectedBinding?.source?.kind === "git-working-tree"
-    && expectedBinding.source.clean === true
+    && (expectedBinding.source.treatmentDerivation ? validDiagnosticS0SourceEvidence(expectedBinding.source) : expectedBinding.source.clean === true)
     && COMMIT.test(String(expectedBinding.source.commit ?? ""))
     && HASH.test(String(expectedBinding.source.treeDigest ?? ""))
-    && HASH.test(String(expectedBinding.productionConfigurationDigest ?? ""))
+    && HASH.test(String(expectedBinding.providerFreeConfigurationDigest ?? ""))
     && Array.isArray(expectedBinding.lanes)
     && expectedBinding.lanes.length === LANE_DEFINITIONS.length
     && LANE_DEFINITIONS.every((definition, index) => {
@@ -210,16 +278,18 @@ export function productionProviderFreeEvidenceValidationErrors(receipt, expected
 export function productionProviderFreeEvidenceContextValidationErrors(receipt, {
   source,
   candidateProvenance,
-  configurationDigest
+  providerFreeConfigurationDigest,
+  treatmentDerivation
 } = {}) {
   const errors = productionProviderFreeEvidenceValidationErrors(receipt, receipt?.binding);
   const binding = receipt?.binding;
+  if (binding?.source?.treatmentDerivation && JSON.stringify(binding.source.treatmentDerivation) !== JSON.stringify(treatmentDerivation)) errors.push("provider-free-diagnostic-derivation-mismatch");
   if (binding?.source?.kind !== source?.kind
     || binding?.source?.commit !== source?.commit
     || binding?.source?.clean !== (source?.dirty === false)) errors.push("provider-free-source-binding-mismatch");
   if (binding?.source?.treeDigest !== candidateProvenance?.contentDigest
     || binding?.source?.treeAlgorithm !== candidateProvenance?.algorithm) errors.push("provider-free-tree-binding-mismatch");
-  if (binding?.productionConfigurationDigest !== configurationDigest) errors.push("provider-free-production-configuration-binding-mismatch");
+  if (binding?.providerFreeConfigurationDigest !== providerFreeConfigurationDigest) errors.push("provider-free-configuration-binding-mismatch");
   return [...new Set(errors)];
 }
 
@@ -235,22 +305,23 @@ export async function collectProductionProviderFreeEvidence({
   runCommand,
   source,
   candidateProvenance,
-  configurationDigest
+  providerFreeConfigurationDigest,
+  treatmentDerivation
 }) {
-  const binding = productionProviderFreeEvidenceBinding({ packageRoot, source, candidateProvenance, configurationDigest });
-  const cacheRoot = path.join(liveRoot, ".pi", "benchmarks", "provider-free-evidence", binding.digest);
+  const observeDiagnosticSource = () => treatmentDerivation ? diagnosticS0SourceEvidence({ source,
+    entries: collectBenchmarkCandidate(liveRoot).entries, derivation: treatmentDerivation, candidate: candidateProvenance }) : undefined;
+  const diagnosticSourceEvidence = observeDiagnosticSource();
+  const binding = productionProviderFreeEvidenceBinding({ packageRoot, source, candidateProvenance, providerFreeConfigurationDigest, diagnosticSourceEvidence });
+  const cacheRoot = checkpointDirectory(liveRoot, path.join(".pi", "benchmarks", "provider-free-evidence", binding.digest));
   const cachePath = path.join(cacheRoot, "receipt.json");
-  if (fs.existsSync(cachePath)) {
-    const cached = JSON.parse(fs.readFileSync(cachePath, "utf8"));
-    if (productionProviderFreeEvidenceValidationErrors(cached, binding).length === 0) return cached;
-  }
 
   const assertLiveSource = async (stage) => {
     const commit = await runCommand("git", ["-C", liveRoot, "rev-parse", "HEAD"], { cwd: liveRoot, timeoutMs: 15_000 });
     const status = await runCommand("git", ["-C", liveRoot, "status", "--porcelain=v1", "--untracked-files=all"], { cwd: liveRoot, timeoutMs: 15_000 });
-    if (commit.code !== 0 || commit.stdout.trim() !== source.commit || status.code !== 0 || status.stdout.trim()) {
+    if (commit.code !== 0 || commit.stdout.trim() !== source.commit || status.code !== 0 || (!treatmentDerivation && status.stdout.trim())) {
       throw new Error(`Production provider-free evidence source changed at ${stage}`);
     }
+    if (treatmentDerivation && JSON.stringify(observeDiagnosticSource()) !== JSON.stringify(binding.source)) throw new Error(`Diagnostic source changed at ${stage}`);
     for (const lane of binding.lanes) {
       if (fileDigest(liveRoot, lane.configurationPath) !== lane.configurationDigest
         || fileDigest(liveRoot, lane.runnerPath) !== lane.runnerDigest) {
@@ -259,22 +330,60 @@ export async function collectProductionProviderFreeEvidence({
     }
   };
 
-  const temporaryRoot = fs.mkdtempSync(path.join(os.tmpdir(), "piagent-provider-free-evidence-"));
+  const release = acquireBenchmarkRunLock(cacheRoot, `s0:${binding.digest}`);
   const lanes = [];
   try {
     await assertLiveSource("before-lanes");
+    if (fs.existsSync(cachePath)) {
+      try {
+        const cached = JSON.parse(fs.readFileSync(cachePath, "utf8"));
+        if (productionProviderFreeEvidenceValidationErrors(cached, binding).length === 0) return cached;
+      } catch { /* Rebuild an invalid aggregate only from validated lane checkpoints. */ }
+    }
     for (const definition of LANE_DEFINITIONS) {
-      const output = path.join(temporaryRoot, `${definition.id}.json`);
-      const result = await runCommand(process.execPath, [path.join(liveRoot, definition.runner), "--output", output], {
-        cwd: liveRoot,
-        timeoutMs: definition.timeoutMilliseconds
-      });
-      if (result.code !== 0) {
-        throw new Error(`Required provider-free lane failed: ${definition.id} (${result.stderr.trim() || result.signal || `exit ${result.code}`})`);
+      const laneRoot = checkpointDirectory(cacheRoot, definition.id);
+      const checkpointPath = path.join(laneRoot, "checkpoint.json");
+      const output = path.join(laneRoot, "result.json");
+      const checkpoint = readCheckpoint(checkpointPath, binding.digest);
+      if (checkpoint?.status === "failed") throw new Error(`Required provider-free lane previously failed: ${definition.id}; evidence retained at ${laneRoot}`);
+      let bytes;
+      if (checkpoint?.status === "passed") {
+        bytes = fs.readFileSync(output);
+        if (checkpoint.resultDigest !== digestBytes(bytes) || !lanePassed(definition.id, JSON.parse(bytes))) {
+          throw new Error(`Provider-free lane checkpoint result changed: ${definition.id}`);
+        }
+      } else {
+        if (checkpoint && checkpoint.status !== "running" && checkpoint.status !== "interrupted") throw new Error("Invalid provider-free lane checkpoint state");
+        const checkIdle = acquireBenchmarkRunLock(laneRoot, "s0-lane-preflight"); checkIdle();
+        const attempt = crypto.randomUUID();
+        writeCheckpoint(checkpointPath, binding.digest, { status: "running", attempt, startedAt: new Date().toISOString() });
+        const args = ["--import", path.join(liveRoot, "packages/piagent-core/benchmark/benchmark-provider-free-owner.js"),
+          path.join(liveRoot, definition.runner), "--output", output];
+        if (definition.id === "long-horizon-v1") args.push("--state-directory", path.join(laneRoot, "execution"), "--binding", binding.digest);
+        const result = await runCommand(process.execPath, args, {
+          cwd: liveRoot,
+          timeoutMs: definition.timeoutMilliseconds,
+          env: { ...process.env, PIAGENT_S0_LANE_DIRECTORY: laneRoot, PIAGENT_S0_BINDING: binding.digest, PIAGENT_S0_ATTEMPT: attempt }
+        });
+        if (result.code !== 0) {
+          writeCheckpoint(checkpointPath, binding.digest, {
+            status: result.signal || result.code === 75 ? "interrupted" : "failed", code: result.code, signal: result.signal ?? null,
+            completedAt: new Date().toISOString()
+          });
+          throw new Error(`Required provider-free lane failed: ${definition.id} (${result.stderr.trim() || result.signal || `exit ${result.code}`})`);
+        }
+        try {
+          bytes = fs.readFileSync(output);
+          if (!lanePassed(definition.id, JSON.parse(bytes))) throw new Error("incomplete lane receipt");
+        } catch (error) {
+          writeCheckpoint(checkpointPath, binding.digest, { status: "failed", reason: "invalid-lane-receipt" });
+          throw new Error(`Required provider-free lane returned an incomplete receipt: ${definition.id}`, { cause: error });
+        }
       }
-      const bytes = fs.readFileSync(output);
       const parsed = JSON.parse(bytes.toString("utf8"));
       if (!lanePassed(definition.id, parsed)) throw new Error(`Required provider-free lane returned an incomplete receipt: ${definition.id}`);
+      await assertLiveSource(`after-${definition.id}`);
+      if (checkpoint?.status !== "passed") writeCheckpoint(checkpointPath, binding.digest, { status: "passed", resultDigest: digestBytes(bytes), completedAt: new Date().toISOString() });
       const expected = binding.lanes.find((lane) => lane.id === definition.id);
       lanes.push({
         schemaVersion: 1,
@@ -288,11 +397,10 @@ export async function collectProductionProviderFreeEvidence({
         resultDigest: digestBytes(bytes),
         summary: laneSummary(definition.id, parsed)
       });
-      await assertLiveSource(`after-${definition.id}`);
     }
     const receipt = {
-      schemaVersion: 1,
-      kind: "production-provider-free-evidence-v1",
+      schemaVersion: 2,
+      kind: "production-provider-free-evidence-v2",
       completedAt: new Date().toISOString(),
       binding,
       lanes
@@ -302,7 +410,7 @@ export async function collectProductionProviderFreeEvidence({
     privateWrite(cachePath, `${JSON.stringify(receipt, null, 2)}\n`);
     return receipt;
   } finally {
-    fs.rmSync(temporaryRoot, { recursive: true, force: true });
+    release();
   }
 }
 
@@ -310,12 +418,17 @@ export function productionProviderFreeLaneDefinitions() {
   return LANE_DEFINITIONS.map((lane) => ({ ...lane }));
 }
 
+export function productionProviderFreeEvidenceRequired({ productionSpendControlled, spendControl } = {}) {
+  return productionSpendControlled === true
+    && spendControl?.productionGuards?.providerFreeEvidence?.requiredBeforeFirstPaidSession === true;
+}
+
 export async function prepareProductionProviderFreeEvidence({
   required,
   packageRoot,
   bootstrapMetadata,
   candidateProvenance,
-  configurationDigest,
+  providerFreeConfigurationDigest,
   runCommand,
   resumedReceipt = undefined
 }) {
@@ -324,7 +437,10 @@ export async function prepareProductionProviderFreeEvidence({
     packageRoot,
     source: bootstrapMetadata.sourceIdentity,
     candidateProvenance,
-    configurationDigest
+    providerFreeConfigurationDigest,
+    diagnosticSourceEvidence: bootstrapMetadata.treatmentDerivation ? diagnosticS0SourceEvidence({
+      source: bootstrapMetadata.sourceIdentity, entries: collectBenchmarkCandidate(bootstrapMetadata.liveRoot).entries,
+      derivation: bootstrapMetadata.treatmentDerivation, candidate: candidateProvenance }) : undefined
   });
   const receipt = resumedReceipt !== undefined
     ? assertProductionProviderFreeEvidence(resumedReceipt, binding, "resumed production provider-free evidence")
@@ -334,7 +450,8 @@ export async function prepareProductionProviderFreeEvidence({
       runCommand,
       source: bootstrapMetadata.sourceIdentity,
       candidateProvenance,
-      configurationDigest
+      providerFreeConfigurationDigest,
+      treatmentDerivation: bootstrapMetadata.treatmentDerivation
     });
   return { receipt, binding };
 }

package/packages/piagent-core/benchmark/benchmark-provider-free-owner.js

@@ -0,0 +1,24 @@
+// Imported before a provider-free lane entrypoint. The ownership lock outlives
+// its caller if the benchmark coordinator dies, so a new invocation cannot run
+// the same lane concurrently with the surviving process.
+import fs from "node:fs";
+import path from "node:path";
+import { acquireBenchmarkRunLock } from "./benchmark-run-lock.js";
+import { readCheckpoint } from "./benchmark-checkpoint.js";
+
+const directory = process.env.PIAGENT_S0_LANE_DIRECTORY;
+const binding = process.env.PIAGENT_S0_BINDING;
+const attempt = process.env.PIAGENT_S0_ATTEMPT;
+try {
+  if (!directory || fs.realpathSync.native(directory) !== directory || !/^[a-f0-9]{64}$/.test(binding ?? "")
+    || !/^[a-f0-9-]{36}$/.test(attempt ?? "")) throw new Error("Invalid S0 lane ownership configuration");
+  acquireBenchmarkRunLock(directory, `s0-lane:${attempt}`);
+  const checkpoint = readCheckpoint(path.join(directory, "checkpoint.json"), binding);
+  if (checkpoint?.status !== "running" || checkpoint.attempt !== attempt) {
+    throw new Error("S0 lane reservation changed before execution");
+  }
+} catch (error) {
+  process.stderr.write(`${error.message}\n`);
+  // This is an ownership/configuration refusal, not a failed test observation.
+  process.exit(75);
+}