chainlesschain 0.166.71 → 0.166.74

package/src/lib/decision-layer/benchmark.js

@@ -3,9 +3,46 @@ import { performance } from "node:perf_hooks";
 import { decisionDigest } from "./contracts.js";
 
 export const SKILL_DECISION_BENCHMARK_SCHEMA =
-  "chainlesschain.skill-decision-benchmark/v1";
+  "chainlesschain.skill-decision-benchmark/v2";
 
 const DIGEST = /^sha256:[a-f0-9]{64}$/u;
+const ONE_SIDED_ALPHA = 0.05;
+
+// Exact Clopper-Pearson upper bound: find p where P(Binomial(n, p) <= k)
+// equals alpha. Keep zero-denominator metrics inconclusive.
+function binomialUpperBound(errors, trials) {
+  if (trials === 0) return null;
+  if (errors === trials) return 1;
+  if (errors === 0) return -Math.expm1(Math.log(ONE_SIDED_ALPHA) / trials);
+
+  const logFactorials = [0];
+  for (let i = 1; i <= trials; i += 1) {
+    logFactorials.push(logFactorials[i - 1] + Math.log(i));
+  }
+  const logCombination =
+    logFactorials[trials] -
+    logFactorials[errors] -
+    logFactorials[trials - errors];
+  let lower = errors / trials;
+  let upper = 1;
+  for (let iteration = 0; iteration < 60; iteration += 1) {
+    const probability = (lower + upper) / 2;
+    let term = Math.exp(
+      logCombination +
+        errors * Math.log(probability) +
+        (trials - errors) * Math.log1p(-probability),
+    );
+    let cumulative = term;
+    for (let count = errors; count > 0; count -= 1) {
+      term *=
+        (count / (trials - count + 1)) * ((1 - probability) / probability);
+      cumulative += term;
+    }
+    if (cumulative > ONE_SIDED_ALPHA) lower = probability;
+    else upper = probability;
+  }
+  return (lower + upper) / 2;
+}
 
 function ratio(value, label) {
   const number = Number(value);
@@ -141,6 +178,23 @@ export async function runSkillDecisionBenchmark({
     unavailableRate: unavailable / normalizedCases.length,
     p95Ms: latencies[Math.max(0, Math.ceil(latencies.length * 0.95) - 1)],
   });
+  const counts = Object.freeze({
+    accepted,
+    acceptedErrors,
+    positives,
+    affirmative,
+    noMatches,
+    noMatchFalseSuggestions,
+    abstentions,
+    unavailable,
+  });
+  const oneSided95Upper = Object.freeze({
+    acceptedSuggestionErrorRate: binomialUpperBound(acceptedErrors, accepted),
+    noMatchFalseSuggestionRate: binomialUpperBound(
+      noMatchFalseSuggestions,
+      noMatches,
+    ),
+  });
   const core = {
     schema: SKILL_DECISION_BENCHMARK_SCHEMA,
     caseCount: normalizedCases.length,
@@ -150,9 +204,16 @@ export async function runSkillDecisionBenchmark({
     ),
     thresholds: gate,
     metrics,
+    counts,
+    oneSided95Upper,
     passed:
-      metrics.acceptedSuggestionErrorRate <= gate.acceptedSuggestionErrorRate &&
-      metrics.noMatchFalseSuggestionRate <= gate.noMatchFalseSuggestionRate &&
+      oneSided95Upper.acceptedSuggestionErrorRate !== null &&
+      oneSided95Upper.acceptedSuggestionErrorRate <=
+        gate.acceptedSuggestionErrorRate &&
+      oneSided95Upper.noMatchFalseSuggestionRate !== null &&
+      oneSided95Upper.noMatchFalseSuggestionRate <=
+        gate.noMatchFalseSuggestionRate &&
+      positives > 0 &&
       metrics.affirmativeCoverage >= gate.affirmativeCoverage &&
       metrics.p95Ms <= gate.p95Ms,
     rows: Object.freeze(rows),

package/src/lib/decision-layer/runtime.js

@@ -143,6 +143,7 @@ export function createSkillDecisionRuntime({
         throw new TypeError("decision runtime clock returned an invalid date");
       }
       let outcome;
+      let requestSignal = null;
       try {
         const raw = await runMeteredDirectModelCall({
           sessionId: boundSessionId,
@@ -152,10 +153,12 @@ export function createSkillDecisionRuntime({
           source: "model",
           operationId: `decision:${decisionId}`,
           sessionBudget,
-          call: () =>
-            providerAuthority.decide(request, {
-              signal: timeoutSignal(signal, timeoutMs),
-            }),
+          call: () => {
+            requestSignal = timeoutSignal(signal, timeoutMs);
+            return providerAuthority.decide(request, {
+              signal: requestSignal,
+            });
+          },
         });
         const normalized = normalizeSkillDecisionProviderResult(request, raw);
         const resolved = resolveSkillDecision(request, normalized, thresholds);
@@ -170,7 +173,17 @@ export function createSkillDecisionRuntime({
           resultDigest: resolved.resultDigest,
         };
       } catch (error) {
-        if (signal?.aborted || isTerminalModelFailure(error)) throw error;
+        const localTimeout =
+          !signal?.aborted &&
+          requestSignal?.aborted &&
+          requestSignal.reason === error &&
+          error?.name === "TimeoutError";
+        if (
+          signal?.aborted ||
+          (isTerminalModelFailure(error) && !localTimeout)
+        ) {
+          throw error;
+        }
         outcome = {
           status: "unavailable",
           reasonCode: projectUnavailable(error),

package/src/lib/evolution/evolution-eval-gate.js

@@ -18,6 +18,8 @@ export const EVOLUTION_EVAL_POLICY_SCHEMA =
   "chainlesschain.evolution-eval-policy/v2";
 export const EVOLUTION_EVAL_RECEIPT_SCHEMA =
   "chainlesschain.evolution-eval-receipt/v4";
+export const EVOLUTION_EVAL_RESULT_EVIDENCE_SCHEMA =
+  "chainlesschain.evolution-eval-result-evidence/v1";
 export const EVOLUTION_EVAL_SUITE_AUTHORITY_SCHEMA =
   "chainlesschain.evolution-eval-suite-authority-receipt/v1";
 export const EVOLUTION_EVAL_ENVIRONMENT_SCHEMA =
@@ -2647,6 +2649,27 @@ function comparisonSummary(results, confidenceZ, seedCount) {
   });
 }
 
+function resultEvidenceCore(receipt, results) {
+  return {
+    schema: EVOLUTION_EVAL_RESULT_EVIDENCE_SCHEMA,
+    receiptDigest: receipt.receiptDigest,
+    validation: results.validation,
+    test: results.test,
+    qualifiesForPromotion: false,
+  };
+}
+
+function buildResultEvidence(receipt, results) {
+  const core = resultEvidenceCore(receipt, results);
+  const evidence = {
+    ...core,
+    evidenceDigest: digest(core, EVOLUTION_EVAL_RESULT_EVIDENCE_SCHEMA),
+  };
+  // Include the digest envelope in the same bound used by the readback path.
+  canonicalJson(evidence);
+  return deepFreeze(evidence);
+}
+
 function decideSplit(summary, policy) {
   const { baseline, candidate } = summary;
   if (candidate.securityViolations > 0 || candidate.permissionViolations > 0) {
@@ -5149,6 +5172,7 @@ export class EvolutionEvalGate {
         this.#policy.confidenceZ,
         this.#policy.seeds.length,
       ),
+      ...(run.includeResultEvidence ? { results } : {}),
     });
   }
 
@@ -5271,10 +5295,46 @@ export class EvolutionEvalGate {
       signedReceipt.receiptDigest,
       "receiptDigest",
     );
+    if (run.includeResultEvidence) {
+      // Export only completed validation + test sets, after the final receipt
+      // was independently signed and verified. No observer runs during grading.
+      const resultEvidence = run.results
+        ? buildResultEvidence(signedReceipt, run.results)
+        : null;
+      const exportedAt = readClock(this.#clock).milliseconds;
+      if (
+        exportedAt >= run.deadlineMs ||
+        NATIVE_MONOTONIC_NOW() >= run.runLocalDeadlineMs ||
+        Date.parse(signedReceipt.expiresAt) <= exportedAt ||
+        run.portExpiries.some(
+          (expiresAt) => Date.parse(expiresAt) <= exportedAt,
+        )
+      ) {
+        throw evalError(
+          EVOLUTION_EVAL_EXECUTION_FAILED_CODE,
+          "evaluation evidence export crossed the evaluation deadline",
+        );
+      }
+      return deepFreeze({ receipt: signedReceipt, resultEvidence });
+    }
     return signedReceipt;
   }
 
   async run(request) {
+    return this.#run(request, false);
+  }
+
+  /** Trusted evaluator host only. Never expose hidden-test rows to the Actor. */
+  async runWithEvidence(request) {
+    return this.#run(request, true);
+  }
+
+  /** Trusted host opt-in: best-effort signed rejection after measured execution. */
+  async runWithFailureEvidence(request) {
+    return this.#run(request, true, true);
+  }
+
+  async #run(request, includeResultEvidence, signRuntimeFailures = false) {
     const runLocalDeadlineMs =
       NATIVE_MONOTONIC_NOW() + this.#policy.maxWallClockMs;
     assertExactRecord(request, RUN_REQUEST_KEYS, "evaluation run request");
@@ -5628,6 +5688,7 @@ export class EvolutionEvalGate {
     const run = {
       runId,
       runNonce,
+      includeResultEvidence,
       deadlineAt,
       deadlineMs,
       runLocalDeadlineMs,
@@ -5713,43 +5774,13 @@ export class EvolutionEvalGate {
     }
 
     const sandboxInstances = new Set();
+    let evaluated;
     try {
-      const evaluated = await this.#evaluateBlindedWorkPlan(
+      evaluated = await this.#evaluateBlindedWorkPlan(
         run,
         usage,
         sandboxInstances,
       );
-      const { validation, test } = evaluated;
-      const validationDecision = decideSplit(validation, this.#policy);
-      if (!validationDecision.accepted) {
-        return this.#buildReceipt(run, {
-          decision: "rejected",
-          reasonCodes: [`validation-${validationDecision.reason}`],
-          validation,
-          test,
-          usage,
-        });
-      }
-      const testDecision = decideSplit(test, this.#policy);
-      if (!testDecision.accepted) {
-        return this.#buildReceipt(run, {
-          decision: "rejected",
-          reasonCodes: [`test-${testDecision.reason}`],
-          validation,
-          test,
-          usage,
-        });
-      }
-      return this.#buildReceipt(run, {
-        decision: "accepted",
-        reasonCodes: [
-          `validation-${validationDecision.reason}`,
-          `test-${testDecision.reason}`,
-        ],
-        validation,
-        test,
-        usage,
-      });
     } catch (cause) {
       if (cause instanceof EvaluationBudgetExceeded) {
         return this.#buildReceipt(run, {
@@ -5760,8 +5791,65 @@ export class EvolutionEvalGate {
           usage: cause.usage,
         });
       }
+      const failureReason =
+        cause instanceof EvolutionEvalGateError
+          ? new Map([
+              [
+                EVOLUTION_EVAL_EXECUTION_FAILED_CODE,
+                "runtime-execution-failed",
+              ],
+              [EVOLUTION_EVAL_GRADER_FAILED_CODE, "runtime-grader-failed"],
+              [EVOLUTION_EVAL_SAFETY_FAILED_CODE, "runtime-safety-failed"],
+            ]).get(cause.code)
+          : undefined;
+      if (signRuntimeFailures && failureReason && usage.executionCount > 0) {
+        try {
+          return await this.#buildReceipt(run, {
+            decision: "rejected",
+            reasonCodes: [failureReason],
+            validation: null,
+            test: null,
+            usage,
+          });
+        } catch {
+          // A deadline, expired authority, or signer failure cannot be
+          // represented as authenticated outcome evidence.
+        }
+      }
       throw cause;
     }
+    const { validation, test } = evaluated;
+    if (includeResultEvidence) run.results = evaluated.results;
+    const validationDecision = decideSplit(validation, this.#policy);
+    if (!validationDecision.accepted) {
+      return this.#buildReceipt(run, {
+        decision: "rejected",
+        reasonCodes: [`validation-${validationDecision.reason}`],
+        validation,
+        test,
+        usage,
+      });
+    }
+    const testDecision = decideSplit(test, this.#policy);
+    if (!testDecision.accepted) {
+      return this.#buildReceipt(run, {
+        decision: "rejected",
+        reasonCodes: [`test-${testDecision.reason}`],
+        validation,
+        test,
+        usage,
+      });
+    }
+    return this.#buildReceipt(run, {
+      decision: "accepted",
+      reasonCodes: [
+        `validation-${validationDecision.reason}`,
+        `test-${testDecision.reason}`,
+      ],
+      validation,
+      test,
+      usage,
+    });
   }
 }
 
@@ -6461,6 +6549,28 @@ export async function runEvolutionEvalGate(gate, request) {
   return EvolutionEvalGate.prototype.run.call(gate, request);
 }
 
+/** Opt-in export for the trusted evaluation host, after the whole run settles. */
+export async function runEvolutionEvalGateWithEvidence(gate, request) {
+  if (!GATE_INSTANCES.has(gate)) {
+    throw evalError(
+      EVOLUTION_EVAL_INVALID_CODE,
+      "result evidence export requires a trusted EvolutionEvalGate instance",
+    );
+  }
+  return EvolutionEvalGate.prototype.runWithEvidence.call(gate, request);
+}
+
+/** Opt-in for trusted hosts; an unsigned failure still rejects the promise. */
+export async function runEvolutionEvalGateWithFailureEvidence(gate, request) {
+  if (!GATE_INSTANCES.has(gate)) {
+    throw evalError(
+      EVOLUTION_EVAL_INVALID_CODE,
+      "failure evidence export requires a trusted EvolutionEvalGate instance",
+    );
+  }
+  return EvolutionEvalGate.prototype.runWithFailureEvidence.call(gate, request);
+}
+
 export async function verifyEvolutionEvalReceipt(verifier, value, expected) {
   if (!RECEIPT_VERIFIER_INSTANCES.has(verifier)) {
     throw evalError(
@@ -6474,3 +6584,150 @@ export async function verifyEvolutionEvalReceipt(verifier, value, expected) {
     expected,
   );
 }
+
+const RESULT_ROW_KEYS = new Set([
+  "taskDigest",
+  "seed",
+  "pass",
+  "qualityScore",
+  "securityViolations",
+  "permissionViolations",
+  "outputArtifactDigest",
+  "subjectBindingDigest",
+  "subjectReservationDigest",
+  "executionDigest",
+  "gradeDigest",
+  "safetyDigest",
+  "metrics",
+]);
+
+function verifyResultRows(rows, tasks, seeds) {
+  if (!Array.isArray(rows) || rows.length !== tasks.size * seeds.size) {
+    throw evalError(
+      EVOLUTION_EVAL_INVALID_CODE,
+      "result evidence must cover every task and seed in each arm",
+    );
+  }
+  const seen = new Set();
+  for (const row of rows) {
+    assertExactRecord(row, RESULT_ROW_KEYS, "evaluation result row");
+    for (const field of RESULT_ROW_KEYS) {
+      if (field.endsWith("Digest")) normalizeDigest(row[field], field);
+    }
+    normalizeInteger(row.seed, "result seed", { maximum: 0x7fffffff });
+    const pair = `${row.taskDigest}\0${row.seed}`;
+    if (!tasks.has(row.taskDigest) || !seeds.has(row.seed) || seen.has(pair)) {
+      throw evalError(
+        EVOLUTION_EVAL_INVALID_CODE,
+        "result evidence contains a foreign or duplicate task/seed pair",
+      );
+    }
+    seen.add(pair);
+    if (typeof row.pass !== "boolean") {
+      throw evalError(
+        EVOLUTION_EVAL_INVALID_CODE,
+        "result pass must be boolean",
+      );
+    }
+    normalizeRatio(row.qualityScore, "result qualityScore");
+    normalizeInteger(row.securityViolations, "result securityViolations");
+    normalizeInteger(row.permissionViolations, "result permissionViolations");
+    normalizeMetrics(row.metrics);
+  }
+}
+
+/**
+ * Authenticate raw rows through the existing final receipt's signed result
+ * digests. This does not fetch/reverify the underlying execution/grader/safety
+ * receipts, attest preparation costs, or turn the rows into promotion approval.
+ * `expected` must come from the trusted host's registered run context.
+ */
+export async function verifyEvolutionEvalResultEvidence(
+  verifier,
+  input,
+  expected,
+) {
+  assertExactRecord(
+    input,
+    new Set(["receipt", "resultEvidence", "suite", "policy"]),
+    "evaluation result evidence input",
+  );
+  // Capture every graph before invoking any asynchronous authority, using the
+  // existing independent bounds for each canonical document.
+  const receipt = deepFreeze(cloneCanonical(input.receipt));
+  const evidence = deepFreeze(cloneCanonical(input.resultEvidence));
+  const suite = verifyEvolutionEvalSuite(input.suite);
+  const policy = verifyEvolutionEvalPolicy(input.policy);
+  const expectedSnapshot = deepFreeze(cloneCanonical(expected));
+  assertExactRecord(
+    evidence,
+    new Set([
+      "schema",
+      "receiptDigest",
+      "validation",
+      "test",
+      "qualifiesForPromotion",
+      "evidenceDigest",
+    ]),
+    "evaluation result evidence",
+  );
+  if (
+    evidence.schema !== EVOLUTION_EVAL_RESULT_EVIDENCE_SCHEMA ||
+    evidence.qualifiesForPromotion !== false ||
+    evidence.receiptDigest !== receipt.receiptDigest ||
+    receipt.suiteDigest !== suite.suiteDigest ||
+    receipt.policyDigest !== policy.policyDigest ||
+    receipt.confidenceZ !== policy.confidenceZ ||
+    !receipt.validation ||
+    !receipt.test ||
+    digest(receipt.splitCounts, "split-counts") !==
+      digest(countSplits(suite.tasks), "split-counts")
+  ) {
+    throw evalError(
+      EVOLUTION_EVAL_INVALID_CODE,
+      "result evidence differs from its receipt, suite, or policy",
+    );
+  }
+  const seeds = new Set(policy.seeds);
+  for (const split of ["validation", "test"]) {
+    const results = evidence[split];
+    assertExactRecord(
+      results,
+      new Set(["baseline", "candidate"]),
+      "split results",
+    );
+    const tasks = new Set(
+      suite.tasks
+        .filter((task) => task.split === split)
+        .map((task) => task.taskDigest),
+    );
+    for (const arm of ["baseline", "candidate"]) {
+      verifyResultRows(results[arm], tasks, seeds);
+    }
+    const recomputed = comparisonSummary(
+      results,
+      policy.confidenceZ,
+      seeds.size,
+    );
+    if (
+      digest(recomputed, "chainlesschain.evolution-eval-comparison/v2") !==
+      digest(receipt[split], "chainlesschain.evolution-eval-comparison/v2")
+    ) {
+      throw evalError(
+        EVOLUTION_EVAL_INVALID_CODE,
+        "result rows do not reproduce the signed comparison",
+      );
+    }
+  }
+  const rebuilt = buildResultEvidence(receipt, evidence);
+  if (rebuilt.evidenceDigest !== evidence.evidenceDigest) {
+    throw evalError(
+      EVOLUTION_EVAL_INVALID_CODE,
+      "result evidence digest mismatch",
+    );
+  }
+  // Verify freshness last so a receipt that expires during recomputation cannot
+  // be returned as verified. The same check authenticates signature and context.
+  await verifyEvolutionEvalReceipt(verifier, receipt, expectedSnapshot);
+  return evidence;
+}