@prismatic-io/lux 0.0.2-preview.30 → 0.0.2-preview.33

package/src/orchestrator/list-runs.ts

@@ -93,7 +93,7 @@ const summarizeRun = async (
     casePassed: grading?.casePassed ?? null,
     caseScore: grading?.caseScore ?? null,
     assertionPassed: passing,
-    assertionTotal: grading?.assertions.length ?? 0,
+    assertionTotal: grading?.assertions.filter((a) => a.passed !== null).length ?? 0,
     ...(readErrors.length > 0 ? { readErrors } : {}),
   };
 };
@@ -162,9 +162,7 @@ export const formatRunsTable = (runs: ListedRun[]): string => {
       else if (r.casePassed === true) status = "PASS";
       else if (r.casePassed === false) status = "FAIL";
       const score =
-        r.caseScore !== null
-          ? `${(r.caseScore * 100).toFixed(0)}% (${r.assertionPassed}/${r.assertionTotal})`
-          : "—";
+        r.caseScore !== null ? `${r.caseScore} (${r.assertionPassed}/${r.assertionTotal})` : "—";
       const duration = r.durationMs !== null ? `${(r.durationMs / 1000).toFixed(1)}s` : "—";
       const agents = r.subagentCount > 0 ? `1+${r.subagentCount}` : "1";
       return [status, r.caseSlug, r.driverName, agents, r.ts, r.exitReason ?? "?", score, duration];
@@ -362,17 +360,19 @@ const appendGradingDetail = (lines: string[], detail: RunDetail): void => {
     let status = "ungraded";
     if (detail.grading.casePassed === true) status = "PASS";
     else if (detail.grading.casePassed === false) status = "FAIL";
-    const score =
-      detail.grading.caseScore !== null ? `${(detail.grading.caseScore * 100).toFixed(0)}%` : "—";
+    const score = detail.grading.caseScore !== null ? String(detail.grading.caseScore) : "—";
     lines.push(
-      `grading: ${status} (${score}, ${detail.grading.assertions.filter((a) => a.passed).length}/${detail.grading.assertions.length})`,
+      `grading: ${status} (${score}, ${detail.grading.assertions.filter((a) => a.passed).length}/${detail.grading.assertions.filter((a) => a.passed !== null).length})`,
     );
     for (const a of detail.grading.assertions) {
       let mark = a.passed ? "✓" : "✗";
+      if (a.passed === null) mark = "ℹ informational";
       if (a.abstention) mark = "?";
       const took = ` (${(a.durationMs / 1000).toFixed(1)}s)`;
       const abstention = a.abstention ? " — insufficient evidence" : "";
-      lines.push(`  ${mark} ${a.name}${took}${abstention}${a.comment ? ` — ${a.comment}` : ""}`);
+      lines.push(
+        `  ${mark} ${a.name} [score ${a.errorMessage !== undefined || a.abstention ? "unavailable" : a.score}]${took}${abstention}${a.comment ? ` — ${a.comment}` : ""}`,
+      );
     }
   }
 };

package/src/orchestrator/paired-effects.ts

@@ -24,6 +24,7 @@ export const clusterCaseSamples = (
       }
       if (current.identityHash !== sample.identityHash) current.identityHash = null;
       if (current.clusterId !== sample.clusterId) current.identityHash = null;
+      if (current.scoreBasis !== sample.scoreBasis) current.scoreBasis = null;
       const metrics = new Map(Object.entries(current.metrics));
       for (const [metric, value] of Object.entries(sample.metrics)) {
         const before = metrics.get(metric);
@@ -65,7 +66,9 @@ export const groupedCaseMetricSamples = (
       if (
         Object.values(samples).some(
           (sample) =>
-            !Object.hasOwn(sample.metrics, metric) || sample.metrics[metric]?.n !== repetitions,
+            !Object.hasOwn(sample.metrics, metric) ||
+            sample.metrics[metric]?.n !== repetitions ||
+            (metric === "score" && !sample.scoreBasis),
         )
       )
         return [];
@@ -132,6 +135,8 @@ const pairedDeltas = (
     const b = after.caseSamples?.[id];
     if (!a?.identityHash || a.identityHash !== b?.identityHash)
       throw new Error(`paired case '${id}' has incompatible evaluation identity`);
+    if (metric === "score" && (!a.scoreBasis || a.scoreBasis !== b.scoreBasis))
+      throw new Error(`paired score for case '${id}' has different or unavailable scoring basis`);
     if (a.clusterId !== b.clusterId)
       throw new Error(`paired case '${id}' has incompatible cluster membership`);
     const av = Object.hasOwn(a.metrics, metric) ? a.metrics[metric] : undefined;

package/src/orchestrator/suite-compare.ts

@@ -74,10 +74,19 @@ const pairedMetricDeltas = (
     const a = before[index];
     const b = after[index];
     if (!a || !b) continue;
-    const names = new Set([...Object.keys(a.metrics), ...Object.keys(b.metrics)]);
+    const am: Record<string, number> = {
+      ...a.metrics,
+      ...(a.score !== undefined ? { score: a.score } : {}),
+    };
+    const bm: Record<string, number> = {
+      ...b.metrics,
+      ...(b.score !== undefined ? { score: b.score } : {}),
+    };
+    const names = new Set([...Object.keys(am), ...Object.keys(bm)]);
     for (const name of names) {
-      const av = a.metrics[name];
-      const bv = b.metrics[name];
+      if (name === "score" && (!a.scoreBasis || a.scoreBasis !== b.scoreBasis)) continue;
+      const av = am[name];
+      const bv = bm[name];
       if (av === undefined || bv === undefined) continue;
       const values = deltas.get(name) ?? [];
       values.push(bv - av);
@@ -196,9 +205,14 @@ export const formatSuiteComparisonText = (comparison: SuiteComparison): string =
     }
   }
   for (const entry of comparison.cases) {
-    if (!entry.regressed) continue;
+    if (entry.regressed) {
+      lines.push(
+        `    ↓ ${suiteGroupKey(entry)}: ${entry.before ? percent(entry.before.passRate) : "—"} → ${entry.after ? percent(entry.after.passRate) : "missing"}`,
+      );
+    }
+    const score = entry.pairedMetricDeltas.score;
     lines.push(
-      `    ↓ ${suiteGroupKey(entry)}: ${entry.before ? percent(entry.before.passRate) : "—"} → ${entry.after ? percent(entry.after.passRate) : "missing"}`,
+      `    ${suiteGroupKey(entry)} score change: ${score ? `${score.mean >= 0 ? "+" : ""}${score.mean} (${score.n} paired)` : "unavailable"}`,
     );
   }
   return lines.join("\n");

package/src/orchestrator/suite-summary.ts

@@ -11,6 +11,7 @@ import {
 import { hasAgentUsage } from "../core/usage.js";
 import { runComparisonIdentity } from "./comparison-identity.js";
 import { safeReadJson } from "./fs-read.js";
+import { gradingScoreBasis } from "./grading.js";
 import type { SuiteCaseResult } from "./orchestrator.js";
 import type { SuiteRun } from "./suite-runs.js";
 
@@ -65,6 +66,7 @@ export const SuiteSampleSchema = z.object({
   passed: z.boolean().nullable(),
   identityHash: z.string().nullable(),
   score: z.exactOptional(z.number()),
+  scoreBasis: z.exactOptional(z.string().nullable()),
   metrics: z.record(z.string(), z.number()),
 });
 export type SuiteSample = z.infer<typeof SuiteSampleSchema>;
@@ -354,7 +356,10 @@ export const buildSuiteSummary = (
     samples: entries.map((entry, index) => {
       const samples: MetricSamples = new Map();
       const result = entry.result?.result;
-      if (result) collectMetadata(samples, result.metadata);
+      if (result) {
+        add(samples, "score", result.grading.caseScore ?? undefined);
+        collectMetadata(samples, result.metadata);
+      }
       return {
         index,
         caseId: entry.item.caseId,
@@ -364,7 +369,7 @@ export const buildSuiteSummary = (
         passed: entry.result?.casePassed ?? null,
         identityHash: result ? runComparisonIdentity(result.metadata) : null,
         ...(result?.grading.caseScore !== null && result?.grading.caseScore !== undefined
-          ? { score: result.grading.caseScore }
+          ? { score: result.grading.caseScore, scoreBasis: gradingScoreBasis(result.grading) }
           : {}),
         metrics: Object.fromEntries(
           [...samples.entries()].map(([name, sample]) => [name, sample.values[0] ?? 0]),

package/src/previewer/server.ts

@@ -7,6 +7,7 @@ import { type DriverMetricDescriptor, type DriverMetrics, exec } from "../core/i
 import {
   compareRuns,
   compareSuiteRuns,
+  hasRegression,
   type ListedRun,
   listExperimentRuns,
   listRuns,
@@ -424,6 +425,7 @@ const handleRunComparison = async ({
   ]);
   sendJson(request, response, 200, {
     kind: "run",
+    regressed: hasRegression(comparison),
     comparison: {
       ...comparison,
       pathA: displayPath(runsRoot, before.runDir),

package/viewer/app.css

@@ -993,6 +993,10 @@ h1 {
     monospace;
 }
 
+.assertion.informational .assertion-mark {
+  color: var(--muted);
+}
+
 .assertion.passed .assertion-mark {
   color: var(--lime);
 }
@@ -1252,6 +1256,37 @@ h1 {
   color: var(--muted);
 }
 
+.score-bar {
+  position: relative;
+  display: block;
+  width: 100%;
+  min-width: 92px;
+  height: 9px;
+  background: #282d30;
+}
+
+.score-fill {
+  display: block;
+  height: 100%;
+  background: var(--muted);
+}
+
+.score-fill.tone-positive,
+.score-fill.tone-after {
+  background: var(--lime);
+}
+
+.score-fill.tone-negative {
+  background: var(--coral);
+}
+
+.score-bar > .has-threshold {
+  position: absolute;
+  top: -2px;
+  height: 13px;
+  border-left: 1px solid currentColor;
+}
+
 .quantized-bar {
   display: grid;
   grid-template-columns: repeat(20, minmax(2px, 1fr));
@@ -1401,6 +1436,7 @@ h1 {
   text-transform: uppercase;
 }
 
+.assertion-content > .score-bar,
 .assertion-content > .quantized-bar {
   max-width: 220px;
   margin-top: 8px;

package/viewer/charts.js

@@ -51,28 +51,40 @@ const distributionBar = (segments, label) => {
   </div>`;
 };
 
+export const scoreBar = (value, tone, label, threshold = null) => {
+  if (typeof value !== "number" || !Number.isFinite(value)) {
+    return `<span class="score-bar unavailable" role="img" aria-label="${escapeHtml(label)}"></span>`;
+  }
+  return `<span class="score-bar" role="img" aria-label="${escapeHtml(label)}">
+    <i class="score-fill ${escapeHtml(tone)}" style="width: ${value * 100}%"></i>
+    ${typeof threshold === "number" ? `<i class="has-threshold" style="left: ${threshold * 100}%"></i>` : ""}
+  </span>`;
+};
+
 export const comparisonBars = ({
   title,
   before,
   after,
   maximum,
+  continuous = false,
   beforeText,
   afterText,
   deltaText,
   deltaTone,
 }) => {
+  const bar = continuous ? scoreBar : quantizedBar;
   const scale = Math.max(maximum ?? 0, before ?? 0, after ?? 0, 1);
   return `<figure class="comparison-bars">
     <figcaption><span>${escapeHtml(title)}</span><b class="${escapeHtml(
       deltaTone,
     )}">${escapeHtml(deltaText)}</b></figcaption>
-    <div class="bar-lane"><strong>A</strong>${quantizedBar(
-      (before ?? 0) / scale,
+    <div class="bar-lane"><strong>A</strong>${bar(
+      before === null || before === undefined ? null : before / scale,
       "tone-before",
       `${title} baseline ${beforeText}`,
     )}<span>${escapeHtml(beforeText)}</span></div>
-    <div class="bar-lane"><strong>B</strong>${quantizedBar(
-      (after ?? 0) / scale,
+    <div class="bar-lane"><strong>B</strong>${bar(
+      after === null || after === undefined ? null : after / scale,
       "tone-after",
       `${title} candidate ${afterText}`,
     )}<span>${escapeHtml(afterText)}</span></div>

package/viewer/detail.js

@@ -1,4 +1,16 @@
-import { comparisonBars, compositionFigure, quantizedBar } from "./charts.js";
+const assertionScore = (assertion) =>
+  assertion.errorMessage !== undefined || assertion.abstention !== undefined
+    ? null
+    : assertion.score;
+
+const assertionPresentation = (assertion) => {
+  if (assertionScore(assertion) === null) return { className: " unavailable", mark: "?", tone: "" };
+  if (assertion.passed === null) return { className: " informational", mark: "ℹ", tone: "" };
+  if (assertion.passed) return { className: " passed", mark: "✓", tone: "tone-positive" };
+  return { className: "", mark: "×", tone: "tone-negative" };
+};
+
+import { comparisonBars, compositionFigure, quantizedBar, scoreBar } from "./charts.js";
 import {
   badge,
   compactNumber,
@@ -98,6 +110,7 @@ const assertionMeta = (assertion) =>
 const assertionDiagnostics = (assertion) => {
   const rows = [
     assertion.comment ? `<p class="assertion-comment">${escapeHtml(assertion.comment)}</p>` : "",
+    assertion.abstention ? `<p class="assertion-comment">Insufficient evidence</p>` : "",
     assertion.errorMessage
       ? `<p class="diagnostic-error">${escapeHtml(assertion.errorMessage)}</p>`
       : "",
@@ -112,6 +125,7 @@ const assertionDiagnostics = (assertion) => {
 
 const inferredMetricUnit = (name, unit) => {
   if (unit) return unit;
+  if (name === "score") return "score";
   if (name.endsWith("Ms")) return "milliseconds";
   if (name.endsWith(".bytes")) return "bytes";
   if (name.endsWith(".usd")) return "usd";
@@ -261,11 +275,11 @@ export const renderRunDetail = (item, detail) => {
             {
               key: "regressed",
               label: "Failed",
-              value: assertions.filter((assertion) => !assertion.passed).length,
+              value: assertions.filter((assertion) => assertion.passed === false).length,
             },
           ],
           `${assertions.filter((assertion) => assertion.passed).length} of ${
-            assertions.length
+            assertions.filter((assertion) => assertion.passed !== null).length
           } assertions passed`,
         )}
         ${
@@ -305,30 +319,30 @@ export const renderRunDetail = (item, detail) => {
           : ""
       }
       <section class="detail-section">
-        <h3>Judgment <span>${assertions.filter((entry) => entry.passed).length}/${assertions.length} passed</span></h3>
+        <h3>Judgment <span>${assertions.filter((entry) => entry.passed).length}/${assertions.filter((entry) => entry.passed !== null).length} passed</span></h3>
         <div class="assertion-list">
           ${
             assertions.length
               ? assertions
                   .map(
                     (assertion) => `
-                    <article class="assertion${assertion.passed ? " passed" : ""}">
-                      <span class="assertion-mark">${assertion.passed ? "✓" : "×"}</span>
+                    <article class="assertion${assertionPresentation(assertion).className}">
+                      <span class="assertion-mark">${assertionPresentation(assertion).mark}</span>
                       <span class="assertion-content">
                         <strong>${escapeHtml(assertion.name)}</strong>
                         <small>${escapeHtml(assertionMeta(assertion))}</small>
-                        ${quantizedBar(
-                          assertion.score,
-                          assertion.passed ? "tone-positive" : "tone-negative",
+                        ${scoreBar(
+                          assertionScore(assertion),
+                          assertionPresentation(assertion).tone,
                           `${assertion.name} scored ${percentage(
-                            assertion.score,
-                          )}; threshold ${percentage(assertion.threshold)}`,
-                          assertion.threshold,
+                            assertionScore(assertion),
+                          )}${assertion.passed === null ? " (informational)" : `; threshold ${percentage(assertion.threshold)}`}`,
+                          assertion.passed === null ? null : assertion.threshold,
                         )}
                         ${assertionDiagnostics(assertion)}
                       </span>
                       <span class="assertion-score">${escapeHtml(
-                        percentage(assertion.score),
+                        percentage(assertionScore(assertion)),
                       )}</span>
                     </article>`,
                   )
@@ -893,18 +907,20 @@ export const renderRunComparison = (payload, labels) => {
     runDelta?.agentUsageAvailable?.b === false ? null : totalTokens(runDelta?.usage?.b?.agent);
   const tokenDelta = tokenA === null || tokenB === null ? null : tokenB - tokenA;
   const scoreDelta =
+    comparison.grading.caseScoreComparable !== false &&
     typeof comparison.grading.caseScoreA === "number" &&
     typeof comparison.grading.caseScoreB === "number"
       ? comparison.grading.caseScoreB - comparison.grading.caseScoreA
       : null;
+  let comparisonStatus = "ungraded";
+  if (payload.regressed === true) comparisonStatus = "failed";
+  if (payload.regressed === false) comparisonStatus = "passed";
   const driverMetrics = driverMetricComparisons(runDelta);
   const subagentsA = subagentTotals(runDelta?.agentActivity?.a);
   const subagentsB = subagentTotals(runDelta?.agentActivity?.b);
   return `
     <header class="detail-header comparison-header">
-      <div class="detail-kicker">${badge(
-        comparison.grading.regressions ? "failed" : "passed",
-      )}<span>run comparison</span></div>
+      <div class="detail-kicker">${badge(comparisonStatus)}<span>run comparison</span></div>
       <div class="detail-title-row">
         <div><h2>${escapeHtml(labels.before)} → ${escapeHtml(
           labels.after,
@@ -920,6 +936,7 @@ export const renderRunComparison = (payload, labels) => {
             )}. Treat deltas as potentially incomparable.</div>`
           : '<div class="integrity-banner">✓ Case, evaluator, runtime, and driver identities are comparable</div>'
       }
+      ${comparison.grading.caseScoreComparable === false ? '<div class="attention-banner">Score change unavailable: scored assertions or weights differ, or evidence is missing.</div>' : ""}
       <dl class="comparison-stats">
         <div><dt>Case score</dt><dd>${escapeHtml(
           `${percentage(comparison.grading.caseScoreA)} → ${percentage(
@@ -950,6 +967,7 @@ export const renderRunComparison = (payload, labels) => {
         <div class="comparison-plot-grid">
           ${comparisonBars({
             title: "Case score",
+            continuous: true,
             before: comparison.grading.caseScoreA,
             after: comparison.grading.caseScoreB,
             maximum: 1,
@@ -1045,6 +1063,12 @@ export const renderRunComparison = (payload, labels) => {
                 value: comparison.grading.deltas.filter((delta) => delta.status === "removed")
                   .length,
               },
+              {
+                key: "unavailable",
+                label: "Unavailable",
+                value: comparison.grading.deltas.filter((delta) => delta.status === "unavailable")
+                  .length,
+              },
               {
                 key: "unchanged",
                 label: "Unchanged",

package/viewer/format.js

@@ -6,11 +6,11 @@ export const escapeHtml = (value) =>
   );
 
 export const percentage = (value) =>
-  typeof value === "number" ? `${Math.round(value * 100)}%` : "—";
+  typeof value === "number" ? `${Number((value * 100).toPrecision(15))}%` : "—";
 
 export const signedPercentage = (value) => {
   if (typeof value !== "number") return "—";
-  const rounded = Math.round(value * 1_000) / 10;
+  const rounded = Number((value * 100).toPrecision(15));
   return `${rounded > 0 ? "+" : ""}${rounded}%`;
 };