@prismatic-io/lux 0.0.2-preview.30 → 0.0.2-preview.33

package/src/cli/output-schemas.ts

@@ -60,7 +60,9 @@ export const ViewOutputSchema = z.union([
   z.object({ url: z.string(), runsRoot: z.string() }),
 ]);
 const interrupts = z.object({ ask: z.number(), approve: z.number(), custom: z.number() });
-const verdict = z.object({ passed: z.boolean(), score: z.number() }).nullable();
+const verdict = z
+  .object({ passed: z.boolean().nullable(), score: z.number().nullable() })
+  .nullable();
 const runComparison = z.object({
   pathA: z.string(),
   pathB: z.string(),
@@ -83,6 +85,7 @@ const runComparison = z.object({
     casePassedB: z.boolean().nullable(),
     caseScoreA: nullableNumber,
     caseScoreB: nullableNumber,
+    caseScoreComparable: z.boolean(),
     deltas: z.array(
       z.object({
         id: z.string(),
@@ -90,7 +93,7 @@ const runComparison = z.object({
         name: z.string(),
         before: verdict,
         after: verdict,
-        status: z.enum(["added", "removed", "improved", "regressed", "unchanged"]),
+        status: z.enum(["added", "removed", "improved", "regressed", "unchanged", "unavailable"]),
       }),
     ),
     improvements: z.number(),

package/src/cli/program.ts

@@ -948,7 +948,9 @@ const runCommand = async (filters: string[], options: RunOptions, human = false)
         caseScore: result.result?.grading?.caseScore ?? null,
         assertionsPassed:
           result.result?.grading?.assertions.filter((assertion) => assertion.passed).length ?? 0,
-        assertionsTotal: result.result?.grading?.assertions.length ?? 0,
+        assertionsTotal:
+          result.result?.grading?.assertions.filter((assertion) => assertion.passed !== null)
+            .length ?? 0,
       })),
     };
   } finally {

package/src/cli/render/format.ts

@@ -59,7 +59,7 @@ export const formatAlignedUsage = (usage: TokenUsage): string =>
     ].join("  "),
   ].join("      ");
 
-export const formatScore = (score: number): string => `${Math.round(score * 100)}%`;
+export const formatScore = (score: number): string => `${Number((score * 100).toPrecision(15))}%`;
 
 /** A one-line count summary: "2 passed · 1 failed · 1 running  (4)". */
 export const formatTally = (parts: [count: number, label: string][], total: number): string => {

package/src/cli/render/reporter.ts

@@ -121,7 +121,10 @@ const statusMark = (status: CaseStatus, theme: Theme): string => {
 
 const assertionTally = (view: CaseView): string => {
   const passed = view.assertions.filter((a) => a.passed).length;
-  const evaluated = view.assertions.length === 0 ? "—" : `${passed}/${view.assertions.length}`;
+  const evaluated =
+    view.assertions.length === 0
+      ? "—"
+      : `${passed}/${view.assertions.filter((a) => a.passed !== null).length}`;
   return view.skippedAssertions.length > 0
     ? `${evaluated} · ${view.skippedAssertions.length} skipped`
     : evaluated;
@@ -162,7 +165,8 @@ const cell = (text: string, width: number): string => {
 const scoreNote = (entry: GradingAssertionEntry): string => {
   if (entry.errorMessage) return entry.errorMessage;
   if (entry.abstention) return "insufficient evidence";
-  if (entry.passed) return "";
+  if (entry.passed === null) return `informational; score ${formatScore(entry.score)}`;
+  if (entry.passed) return entry.score < 1 ? `score ${formatScore(entry.score)}` : "";
   const score = formatScore(entry.score);
   if (entry.threshold > 0) return `score ${score} < ${formatScore(entry.threshold)}`;
   return entry.score > 0 ? `score ${score}` : "";
@@ -464,11 +468,12 @@ export class ConsoleReporter implements SuiteObserver {
     );
     for (const entry of view.assertions) {
       let mark = entry.passed ? this.theme.green("✓") : this.theme.red("✗");
+      if (entry.passed === null) mark = this.theme.dim("ℹ");
       if (entry.abstention) mark = this.theme.dim("?");
       const duration = formatDuration(entry.durationMs).padStart(DURATION_COL);
       const note = scoreNote(entry);
       const suffix = note
-        ? `  ${entry.abstention ? this.theme.dim(note) : this.theme.red(note)}`
+        ? `  ${entry.abstention || entry.passed !== false ? this.theme.dim(note) : this.theme.red(note)}`
         : "";
       nodes.push({
         head: `${mark} ${cell(entry.name, width)}  ${this.theme.dim(duration)}${suffix}`,
@@ -581,11 +586,12 @@ export class ConsoleReporter implements SuiteObserver {
     const skippedAssertions = this.cases.flatMap((c) => c.skippedAssertions);
     if (assertions.length > 0 || skippedAssertions.length > 0) {
       const ok = assertions.filter((a) => a.passed).length;
-      const bad = assertions.length - ok;
+      const bad = assertions.filter((a) => a.passed === false).length;
       const tally = formatTally(
         [
           [ok, this.theme.green("passed")],
           [bad, this.theme.red("failed")],
+          [assertions.filter((a) => a.passed === null).length, this.theme.dim("informational")],
           [skippedAssertions.length, this.theme.dim("skipped")],
         ],
         assertions.length + skippedAssertions.length,

package/src/core/assertion.ts

@@ -107,6 +107,7 @@ export const baseAssertionFields = {
   weight: z.number().positive().optional(),
   threshold: z.number().min(0).max(1).optional(),
   name: z.string().optional(),
+  informational: z.boolean().optional(),
 } as const;
 
 export const defineAssertion = <S extends z.ZodObject>(
@@ -137,7 +138,8 @@ export const GradingAssertionEntrySchema = z.object({
   id: z.exactOptional(z.string()),
   type: z.string(),
   name: z.string(),
-  passed: z.boolean(),
+  passed: z.boolean().nullable(),
+  informational: z.exactOptional(z.boolean()),
   score: z.number(),
   weight: z.number(),
   threshold: z.number(),

package/src/core/case.ts

@@ -45,6 +45,7 @@ export const AssertionSchema = z
     name: z.string().optional(),
     weight: z.number().positive().optional(),
     threshold: z.number().min(0).max(1).optional(),
+    informational: z.boolean().optional(),
   })
   .loose();
 export type Assertion = z.infer<typeof AssertionSchema>;
@@ -83,7 +84,10 @@ export const EvalCaseSchema = EvalCaseObjectSchema
   });
 export type EvalCase = z.infer<typeof EvalCaseSchema>;
 
-type AssertionPlanFields = Pick<Assertion, "id" | "name" | "threshold" | "weight">;
+type AssertionPlanFields = Pick<
+  Assertion,
+  "id" | "name" | "threshold" | "weight" | "informational"
+>;
 type AssertionInputOf<Plugin> =
   Plugin extends AssertionPlugin<Assertion, infer Input> ? Input & AssertionPlanFields : never;
 

package/src/core/experiment.ts

@@ -173,6 +173,11 @@ const validateConstraintConfidence = (
   ) {
     issue("relative constraints require a metric with per-case observations");
   }
+  if (constraint.metric === "score" && constraint.confidence?.method === "wilson") {
+    issue(
+      "continuous scores are not Bernoulli observations; use Hoeffding or percentile-bootstrap confidence",
+    );
+  }
   const bounded = boundedMetrics.has(constraint.metric);
   if (
     (constraint.confidence?.method === "hoeffding" || constraint.confidence?.method === "wilson") &&
@@ -263,7 +268,7 @@ export const ExperimentCampaignSchema = z
       "grader.operationalErrorRate",
       "grader.scoreMae",
     ]);
-    const unitIntervalMetrics = graderConfidenceMetrics;
+    const unitIntervalMetrics = new Set([...graderConfidenceMetrics, "score"]);
     if (campaign.optimizer && !campaign.splits.train) {
       ctx.addIssue({
         code: "custom",
@@ -454,6 +459,7 @@ export const CandidateEvaluationSchema = z.object({
       z.string(),
       z.object({
         identityHash: z.string().nullable(),
+        scoreBasis: z.exactOptional(z.string().nullable()),
         clusterId: z.exactOptional(z.string().min(1)),
         metrics: z.record(
           z.string(),

package/src/orchestrator/compare.ts

@@ -20,6 +20,7 @@ import {
 import { hasAgentUsage } from "../core/usage.js";
 import { stableDriverIdentity } from "./comparison-identity.js";
 import { safeReadJson, walkFiles } from "./fs-read.js";
+import { gradingScoreBasis } from "./grading.js";
 
 export type ComparisonInput = {
   pathA: string;
@@ -39,9 +40,9 @@ export type GradingDelta = {
   id: string;
   type: string;
   name: string;
-  before: { passed: boolean; score: number } | null;
-  after: { passed: boolean; score: number } | null;
-  status: "added" | "removed" | "improved" | "regressed" | "unchanged";
+  before: { passed: boolean | null; score: number | null } | null;
+  after: { passed: boolean | null; score: number | null } | null;
+  status: "added" | "removed" | "improved" | "regressed" | "unchanged" | "unavailable";
 };
 
 export type RunComparison = {
@@ -67,6 +68,7 @@ export type RunComparison = {
     casePassedB: boolean | null;
     caseScoreA: number | null;
     caseScoreB: number | null;
+    caseScoreComparable: boolean;
     deltas: GradingDelta[];
     improvements: number;
     regressions: number;
@@ -156,6 +158,9 @@ const assertionsByIdentity = (
   return out;
 };
 
+const measuredScore = (entry: GradingReport["assertions"][number]): number | null =>
+  entry.errorMessage !== undefined || entry.abstention !== undefined ? null : entry.score;
+
 const computeGradingDeltas = (
   a: GradingReport | null,
   b: GradingReport | null,
@@ -174,21 +179,24 @@ const computeGradingDeltas = (
     if (!ax && bx) status = "added";
     else if (ax && !bx) status = "removed";
     else if (ax && bx) {
-      if (ax.passed === bx.passed && ax.score === bx.score) status = "unchanged";
+      if (measuredScore(ax) === null || measuredScore(bx) === null) status = "unavailable";
+      else if (ax.passed === bx.passed && ax.score === bx.score) status = "unchanged";
       // A pass/fail flip dominates any score movement.
       else if (ax.passed !== bx.passed) status = bx.passed ? "improved" : "regressed";
       else status = ax.score < bx.score ? "improved" : "regressed";
     } else {
       status = "unchanged";
     }
-    if (status === "improved") improvements++;
-    if (status === "regressed") regressions++;
+    if (ax?.passed !== null && bx?.passed !== null) {
+      if (status === "improved") improvements++;
+      if (status === "regressed") regressions++;
+    }
     deltas.push({
       id: name,
       type: ax?.type ?? bx?.type ?? "unknown",
       name: ax?.id || bx?.id ? (ax?.name ?? bx?.name ?? name) : name,
-      before: ax ? { passed: ax.passed, score: ax.score } : null,
-      after: bx ? { passed: bx.passed, score: bx.score } : null,
+      before: ax ? { passed: ax.passed, score: measuredScore(ax) } : null,
+      after: bx ? { passed: bx.passed, score: measuredScore(bx) } : null,
       status,
     });
   }
@@ -312,6 +320,9 @@ export const compareRuns = async (input: ComparisonInput): Promise<RunComparison
       casePassedB: b.grading?.casePassed ?? null,
       caseScoreA: a.grading?.caseScore ?? null,
       caseScoreB: b.grading?.caseScore ?? null,
+      caseScoreComparable:
+        gradingScoreBasis(a.grading) !== null &&
+        gradingScoreBasis(a.grading) === gradingScoreBasis(b.grading),
       ...grading,
     },
     artifactsDelta: computeFileDelta(a.fileTree, b.fileTree),
@@ -325,7 +336,8 @@ export const compareRuns = async (input: ComparisonInput): Promise<RunComparison
 export const hasRegression = (cmp: RunComparison): boolean =>
   cmp.caseMismatch.length > 0 ||
   cmp.grading.regressions > 0 ||
-  cmp.grading.deltas.some((delta) => delta.status === "removed") ||
+  cmp.grading.deltas.some((delta) => delta.after?.score === null && delta.after.passed !== null) ||
+  cmp.grading.deltas.some((delta) => delta.status === "removed" && delta.before?.passed !== null) ||
   (cmp.grading.casePassedA === true && cmp.grading.casePassedB !== true);
 
 const formatMismatchValue = (value: unknown): string => {
@@ -374,13 +386,17 @@ export const formatComparisonText = (cmp: RunComparison): string => {
   lines.push("Grading:");
   lines.push(
     `  passed: ${String(cmp.grading.casePassedA)} → ${String(cmp.grading.casePassedB)}; ` +
-      `score: ${cmp.grading.caseScoreA ?? "?"} → ${cmp.grading.caseScoreB ?? "?"}`,
+      `score: ${cmp.grading.caseScoreA ?? "?"} → ${cmp.grading.caseScoreB ?? "?"}${cmp.grading.caseScoreComparable ? "" : " (change unavailable: scoring basis differs or is unknown)"}`,
   );
   lines.push(`  ${cmp.grading.improvements} improved, ${cmp.grading.regressions} regressed`);
   for (const d of cmp.grading.deltas) {
     if (d.status === "unchanged") continue;
-    const arrow = { added: "+", removed: "-", improved: "↑", regressed: "↓" }[d.status];
-    lines.push(`    ${arrow} ${d.name}`);
+    const arrow = { added: "+", removed: "-", improved: "↑", regressed: "↓", unavailable: "?" }[
+      d.status
+    ];
+    lines.push(
+      `    ${arrow} ${d.name}: ${d.before?.score ?? "unavailable"} → ${d.after?.score ?? "unavailable"}`,
+    );
   }
   lines.push("");
   lines.push("Artifacts:");

package/src/orchestrator/experiment-effects.ts

@@ -23,7 +23,7 @@ export const experimentEffects = (
   const metrics = new Set([
     "quality.passRate",
     experiment.evaluationKind === "grader" ? "harness.total" : "agent.total",
-    ...(experiment.evaluationKind === "agent" ? ["interrupts.ask", "toolCalls"] : []),
+    ...(experiment.evaluationKind === "agent" ? ["score", "interrupts.ask", "toolCalls"] : []),
     ...experiment.campaignSnapshot.objectives.constraints
       .filter((constraint) => constraint.relativeTo)
       .map((constraint) => constraint.metric),
@@ -46,7 +46,9 @@ export const experimentEffects = (
           const confidence =
             constraint?.confidence ??
             ConfidenceConfigSchema.parse({
-              method: metric === "quality.passRate" ? "hoeffding" : "percentile-bootstrap",
+              method: ["quality.passRate", "score"].includes(metric)
+                ? "hoeffding"
+                : "percentile-bootstrap",
             });
           return {
             candidateId,

package/src/orchestrator/experiment-evaluation.ts

@@ -44,6 +44,7 @@ const gradingFailureMessage = (results: readonly SuiteCaseResult[]): string | un
     }
     const report = result.result?.grading;
     const failedAssertion = report.assertions.find((assertion) => {
+      if (assertion.passed === null) return false;
       if (assertion.errorMessage) return true;
       const detail =
         assertion.detail && typeof assertion.detail === "object"
@@ -299,10 +300,12 @@ const agentEvaluationFromSuite = (input: {
       return {
         [sample.canonicalCaseId]: {
           identityHash,
+          scoreBasis: sample.scoreBasis ?? null,
           ...(input.clusterId ? { clusterId: input.clusterId } : {}),
           metrics: Object.fromEntries(
             Object.entries({
               ...sample.metrics,
+              ...(sample.score !== undefined ? { score: sample.score } : {}),
               "quality.passRate": sample.passed === true ? 1 : 0,
             }).map(([name, value]) => [name, { mean: value, n: 1 }]),
           ),
@@ -322,7 +325,10 @@ const agentEvaluationFromSuite = (input: {
       summary.cases.map((entry) => [entry.canonicalCaseId, entry.passRate]),
     ),
     metrics: evaluationMetrics(summary, candidate),
-    metricDefinitions: evaluationMetricDefinitions(summary),
+    metricDefinitions: {
+      score: { unit: "score", better: "higher", label: "Quality score" },
+      ...evaluationMetricDefinitions(summary),
+    },
     samples: evaluationSamples(summary),
     ...(caseSamples ? { caseSamples } : {}),
     metricCalls: summary.samples.length,
@@ -406,6 +412,12 @@ const aggregateAgentUnits = (
   const caseSamples = clusterCaseSamples(units.map((unit) => unit.evaluation.caseSamples));
   const grouped = groupedCaseMetricSamples(caseSamples, repetitions);
   const samples = grouped ?? ungroupedSamples;
+  const completeScore =
+    caseSamples &&
+    Object.values(caseSamples).every(
+      (sample) => sample.metrics.score?.n === repetitions && Boolean(sample.scoreBasis),
+    );
+  if (!completeScore) delete samples.score;
   return {
     candidateId: candidate.id,
     profileId,
@@ -417,7 +429,11 @@ const aggregateAgentUnits = (
     ),
     metrics: Object.fromEntries(
       [...metricValues]
-        .filter(([name]) => !grouped || name === "subject.bytes" || Object.hasOwn(grouped, name))
+        .filter(([name]) =>
+          name === "score"
+            ? completeScore
+            : !grouped || name === "subject.bytes" || Object.hasOwn(grouped, name),
+        )
         .map(([name, values]) => [
           name,
           name === "subject.bytes" ? candidate.sourceBytes : average(grouped?.[name] ?? values),

package/src/orchestrator/experiment-selection.ts

@@ -252,13 +252,65 @@ const normalizedConstraintViolation = (
   return worst;
 };
 
+const hasScoreEvidence = (
+  experiment: OpenedExperimentRun["manifest"],
+  split: ExperimentSplit,
+): boolean =>
+  [...new Set(experiment.evaluations.map((entry) => entry.candidateId))]
+    .flatMap((id) => candidateEvaluations(experiment, id))
+    .some(
+      (entry) =>
+        entry.split === split &&
+        (entry.metricDefinitions?.score !== undefined ||
+          entry.metrics.score !== undefined ||
+          Object.values(entry.caseSamples ?? {}).some(
+            (sample) => sample.metrics.score !== undefined,
+          )),
+    );
+
+const completeScoreEvidence = (entry: ExperimentEvaluation): boolean =>
+  entry.metrics.score !== undefined &&
+  (!entry.caseSamples ||
+    Object.keys(entry.casePassRates).every((id) => {
+      const sample = entry.caseSamples?.[id];
+      return sample?.metrics.score?.n === entry.repetitions && Boolean(sample.scoreBasis);
+    }));
+
+const comparableScoreBasis = (
+  entry: ExperimentEvaluation,
+  reference: ExperimentEvaluation | undefined,
+): boolean => {
+  if (!reference || !completeScoreEvidence(reference)) return false;
+  if (!entry.caseSamples || !reference.caseSamples)
+    return !entry.caseSamples && !reference.caseSamples;
+  const ids = new Set([
+    ...Object.keys(entry.casePassRates),
+    ...Object.keys(reference.casePassRates),
+  ]);
+  return [...ids].every((id) => {
+    const basis = entry.caseSamples?.[id]?.scoreBasis;
+    return Boolean(basis) && basis === reference.caseSamples?.[id]?.scoreBasis;
+  });
+};
+
 export const rankCandidates = <T extends ExperimentDecisionCandidate>(
   campaign: ExperimentCampaign,
   candidates: readonly T[],
   experiment: OpenedExperimentRun["manifest"],
   constraintSplits: readonly ExperimentSplit[] = ["train", "validation", "test"],
-): T[] =>
-  [...candidates].sort((a, b) => {
+): T[] => {
+  const validationEvaluations = (id: string): ExperimentEvaluation[] =>
+    candidateEvaluations(experiment, id).filter((entry) => entry.split === "validation");
+  const referenceId = ["baseline", ...candidates.map((candidate) => candidate.id).sort()].find(
+    (id) => {
+      const entries = validationEvaluations(id);
+      return campaign.profiles.every((profile) =>
+        entries.some((entry) => entry.profileId === profile && completeScoreEvidence(entry)),
+      );
+    },
+  );
+  const reference = referenceId ? validationEvaluations(referenceId) : [];
+  return [...candidates].sort((a, b) => {
     const activeConstraints = campaign.objectives.constraints.filter((constraint) =>
       constraintSplits.includes(constraint.split),
     );
@@ -310,20 +362,40 @@ export const rankCandidates = <T extends ExperimentDecisionCandidate>(
       }
     }
     const quality = (candidate: ExperimentDecisionCandidate): number => {
-      const values = metricValues(
-        candidate,
-        "quality.passRate",
-        undefined,
-        "validation",
-        experiment,
-      );
-      return values.length > 0 ? Math.min(...values) : -1;
+      if (!hasScoreEvidence(experiment, "validation")) {
+        const values = metricValues(
+          candidate,
+          "quality.passRate",
+          undefined,
+          "validation",
+          experiment,
+        );
+        return values.length > 0 ? Math.min(...values) : -1;
+      }
+      const evaluations = candidateEvaluations(experiment, candidate.id);
+      const scores = campaign.profiles.map((profile) => {
+        const entry = evaluations.find(
+          (value) => value.profileId === profile && value.split === "validation",
+        );
+        if (
+          !entry ||
+          !completeScoreEvidence(entry) ||
+          !comparableScoreBasis(
+            entry,
+            reference.find((value) => value.profileId === profile),
+          )
+        )
+          return -1;
+        return entry.metrics.score ?? -1;
+      });
+      return scores.length > 0 ? Math.min(...scores) : -1;
     };
     const qualityDelta = quality(b) - quality(a);
     if (qualityDelta !== 0) return qualityDelta;
     if (a.sourceBytes !== b.sourceBytes) return a.sourceBytes - b.sourceBytes;
     return a.id.localeCompare(b.id);
   });
+};
 
 const candidateCaseVector = (
   candidate: Candidate,
@@ -338,10 +410,15 @@ const candidateCaseVector = (
     evaluations
       .filter((entry) => entry.split === split)
       .flatMap((entry) =>
-        Object.entries(entry.casePassRates).map(([caseId, passRate]) => [
-          `${entry.profileId}:${caseId}`,
-          passRate,
-        ]),
+        Object.entries(entry.casePassRates).flatMap(([caseId, passRate]) => {
+          if (!hasScoreEvidence(experiment, split))
+            return [[`${entry.profileId}:${caseId}`, passRate]];
+          const sample = entry.caseSamples?.[caseId];
+          const score = sample?.metrics.score;
+          return score?.n === entry.repetitions && sample?.scoreBasis
+            ? [[JSON.stringify([entry.profileId, caseId, sample.scoreBasis]), score.mean]]
+            : [];
+        }),
       ),
   );
 };
@@ -351,8 +428,9 @@ const dominates = (a: Record<string, number>, b: Record<string, number>): boolea
   if (keys.size === 0) return false;
   let strictlyBetter = false;
   for (const key of keys) {
-    const av = a[key] ?? 0;
-    const bv = b[key] ?? 0;
+    const av = a[key];
+    const bv = b[key];
+    if (av === undefined || bv === undefined) return false;
     if (av < bv) return false;
     if (av > bv) strictlyBetter = true;
   }
@@ -404,8 +482,13 @@ const frontierCoverage = (
   const candidateVector = candidateCaseVector(candidate, experiment, "train");
   const keys = new Set(vectors.flatMap((vector) => Object.keys(vector)));
   return [...keys].filter((key) => {
-    const best = Math.max(...vectors.map((vector) => vector[key] ?? 0));
-    return (candidateVector[key] ?? 0) === best;
+    const values = vectors.flatMap((vector) => (vector[key] === undefined ? [] : [vector[key]]));
+    const best = Math.max(...values);
+    return (
+      (frontier.length === 1 || values.length > 1) &&
+      candidateVector[key] !== undefined &&
+      candidateVector[key] === best
+    );
   }).length;
 };
 

package/src/orchestrator/grading.ts

@@ -15,7 +15,7 @@ import {
 } from "../core/index.js";
 import { validatePluginConfigAsync } from "./validate-plugin-config.js";
 
-const FRAMEWORK_KEYS = new Set(["id", "type", "name", "weight", "threshold"]);
+const FRAMEWORK_KEYS = new Set(["id", "type", "name", "weight", "threshold", "informational"]);
 
 const shapeOf = (schema: z.ZodTypeAny): Record<string, unknown> | null => {
   const shape = (schema as { shape?: unknown }).shape;
@@ -131,6 +131,50 @@ const assertionPassed = (result: AssertionResult, threshold: number): boolean =>
   result.passed &&
   result.score >= threshold;
 
+export const gradingScoreBasis = (report: GradingReport | null): string | null =>
+  report?.caseScore === null || !report
+    ? null
+    : valueHash(
+        report.assertions
+          .map((entry, index) => ({ entry, id: entry.id ?? `${entry.type}:${index + 1}` }))
+          .filter(({ entry }) => entry.errorMessage === undefined && entry.abstention === undefined)
+          .map(({ entry, id }) => ({ id, weight: entry.weight }))
+          .sort((a, b) => a.id.localeCompare(b.id)),
+      );
+
+const aggregateGrading = (
+  entries: GradingAssertionEntry[],
+  skippedAssertions: NonNullable<GradingReport["skippedAssertions"]>,
+  gradingIncomplete: boolean,
+): GradingReport => {
+  const requiredEntries = entries.filter((entry) => entry.passed !== null);
+  const incomplete =
+    gradingIncomplete || requiredEntries.some((entry) => entry.errorMessage !== undefined);
+  const gradingAbstained = requiredEntries.some((entry) => entry.abstention !== undefined);
+  const scoredEntries = entries.filter(
+    (entry) => entry.errorMessage === undefined && entry.abstention === undefined,
+  );
+  const weightSum = scoredEntries.reduce((sum, entry) => sum + entry.weight, 0);
+  const weightedScoreSum = scoredEntries.reduce(
+    (sum, entry) => sum + entry.weight * entry.score,
+    0,
+  );
+  const hasVerdict = !incomplete && !gradingAbstained && weightSum > 0;
+  return {
+    schemaVersion: 1,
+    casePassed:
+      hasVerdict && requiredEntries.length > 0
+        ? requiredEntries.every((entry) => entry.passed)
+        : null,
+    caseScore: hasVerdict ? weightedScoreSum / weightSum : null,
+    assertions: entries,
+    ...(skippedAssertions.length > 0 ? { skippedAssertions } : {}),
+    ...(incomplete
+      ? { incomplete: { exitReason: "error" as const, phase: "grading" as const } }
+      : {}),
+  };
+};
+
 export const gradeRun = async (
   run: Run,
   registry: Registry,
@@ -157,11 +201,7 @@ export const gradeRun = async (
     name: string;
     reason: "judge-disabled";
   }> = [];
-  let weightedScoreSum = 0;
-  let weightSum = 0;
-  let allPassed = true;
   let gradingIncomplete = false;
-  let gradingAbstained = false;
 
   for (const [index, a] of assertions.entries()) {
     if (skipJudge && a.type === "rubric") {
@@ -232,7 +272,8 @@ export const gradeRun = async (
         id,
         type: a.type,
         name,
-        passed,
+        passed: a.informational ? null : passed,
+        ...(a.informational ? { informational: true } : {}),
         score: result.score,
         weight,
         threshold,
@@ -243,29 +284,20 @@ export const gradeRun = async (
         ...(result.errorMessage !== undefined ? { errorMessage: result.errorMessage } : {}),
         ...(result.usage ? { usage: result.usage } : {}),
       };
-      if (result.errorMessage !== undefined) {
-        gradingIncomplete = true;
-      } else if (result.abstention !== undefined) {
-        gradingAbstained = true;
-      } else {
-        if (!passed) allPassed = false;
-        weightedScoreSum += weight * result.score;
-        weightSum += weight;
-      }
     } catch (err) {
       const message = err instanceof Error ? err.message : String(err);
       entry = {
         id,
         type: a.type,
         name,
-        passed: false,
+        passed: a.informational ? null : false,
+        ...(a.informational ? { informational: true } : {}),
         score: 0,
         weight,
         threshold,
         durationMs: Date.now() - startedAt,
         errorMessage: message,
       };
-      gradingIncomplete = true;
     }
 
     entries.push(entry);
@@ -274,15 +306,5 @@ export const gradeRun = async (
     await modelAttempts?.complete(assertionAttemptIds);
   }
 
-  const hasVerdict = !gradingIncomplete && !gradingAbstained && weightSum > 0;
-  return {
-    schemaVersion: 1,
-    casePassed: hasVerdict ? allPassed : null,
-    caseScore: hasVerdict ? weightedScoreSum / weightSum : null,
-    assertions: entries,
-    ...(skippedAssertions.length > 0 ? { skippedAssertions } : {}),
-    ...(gradingIncomplete
-      ? { incomplete: { exitReason: "error" as const, phase: "grading" as const } }
-      : {}),
-  };
+  return aggregateGrading(entries, skippedAssertions, gradingIncomplete);
 };