@prismatic-io/lux 0.0.2-preview.32 → 0.0.2-preview.33

package/src/orchestrator/suite-compare.ts

@@ -74,10 +74,19 @@ const pairedMetricDeltas = (
     const a = before[index];
     const b = after[index];
     if (!a || !b) continue;
-    const names = new Set([...Object.keys(a.metrics), ...Object.keys(b.metrics)]);
+    const am: Record<string, number> = {
+      ...a.metrics,
+      ...(a.score !== undefined ? { score: a.score } : {}),
+    };
+    const bm: Record<string, number> = {
+      ...b.metrics,
+      ...(b.score !== undefined ? { score: b.score } : {}),
+    };
+    const names = new Set([...Object.keys(am), ...Object.keys(bm)]);
     for (const name of names) {
-      const av = a.metrics[name];
-      const bv = b.metrics[name];
+      if (name === "score" && (!a.scoreBasis || a.scoreBasis !== b.scoreBasis)) continue;
+      const av = am[name];
+      const bv = bm[name];
       if (av === undefined || bv === undefined) continue;
       const values = deltas.get(name) ?? [];
       values.push(bv - av);
@@ -196,9 +205,14 @@ export const formatSuiteComparisonText = (comparison: SuiteComparison): string =
     }
   }
   for (const entry of comparison.cases) {
-    if (!entry.regressed) continue;
+    if (entry.regressed) {
+      lines.push(
+        `    ↓ ${suiteGroupKey(entry)}: ${entry.before ? percent(entry.before.passRate) : "—"} → ${entry.after ? percent(entry.after.passRate) : "missing"}`,
+      );
+    }
+    const score = entry.pairedMetricDeltas.score;
     lines.push(
-      `    ↓ ${suiteGroupKey(entry)}: ${entry.before ? percent(entry.before.passRate) : "—"} → ${entry.after ? percent(entry.after.passRate) : "missing"}`,
+      `    ${suiteGroupKey(entry)} score change: ${score ? `${score.mean >= 0 ? "+" : ""}${score.mean} (${score.n} paired)` : "unavailable"}`,
     );
   }
   return lines.join("\n");

package/src/orchestrator/suite-summary.ts

@@ -11,6 +11,7 @@ import {
 import { hasAgentUsage } from "../core/usage.js";
 import { runComparisonIdentity } from "./comparison-identity.js";
 import { safeReadJson } from "./fs-read.js";
+import { gradingScoreBasis } from "./grading.js";
 import type { SuiteCaseResult } from "./orchestrator.js";
 import type { SuiteRun } from "./suite-runs.js";
 
@@ -65,6 +66,7 @@ export const SuiteSampleSchema = z.object({
   passed: z.boolean().nullable(),
   identityHash: z.string().nullable(),
   score: z.exactOptional(z.number()),
+  scoreBasis: z.exactOptional(z.string().nullable()),
   metrics: z.record(z.string(), z.number()),
 });
 export type SuiteSample = z.infer<typeof SuiteSampleSchema>;
@@ -354,7 +356,10 @@ export const buildSuiteSummary = (
     samples: entries.map((entry, index) => {
       const samples: MetricSamples = new Map();
       const result = entry.result?.result;
-      if (result) collectMetadata(samples, result.metadata);
+      if (result) {
+        add(samples, "score", result.grading.caseScore ?? undefined);
+        collectMetadata(samples, result.metadata);
+      }
       return {
         index,
         caseId: entry.item.caseId,
@@ -364,7 +369,7 @@ export const buildSuiteSummary = (
         passed: entry.result?.casePassed ?? null,
         identityHash: result ? runComparisonIdentity(result.metadata) : null,
         ...(result?.grading.caseScore !== null && result?.grading.caseScore !== undefined
-          ? { score: result.grading.caseScore }
+          ? { score: result.grading.caseScore, scoreBasis: gradingScoreBasis(result.grading) }
           : {}),
         metrics: Object.fromEntries(
           [...samples.entries()].map(([name, sample]) => [name, sample.values[0] ?? 0]),

package/src/previewer/server.ts

@@ -7,6 +7,7 @@ import { type DriverMetricDescriptor, type DriverMetrics, exec } from "../core/i
 import {
   compareRuns,
   compareSuiteRuns,
+  hasRegression,
   type ListedRun,
   listExperimentRuns,
   listRuns,
@@ -424,6 +425,7 @@ const handleRunComparison = async ({
   ]);
   sendJson(request, response, 200, {
     kind: "run",
+    regressed: hasRegression(comparison),
     comparison: {
       ...comparison,
       pathA: displayPath(runsRoot, before.runDir),

package/viewer/app.css

@@ -1256,6 +1256,37 @@ h1 {
   color: var(--muted);
 }
 
+.score-bar {
+  position: relative;
+  display: block;
+  width: 100%;
+  min-width: 92px;
+  height: 9px;
+  background: #282d30;
+}
+
+.score-fill {
+  display: block;
+  height: 100%;
+  background: var(--muted);
+}
+
+.score-fill.tone-positive,
+.score-fill.tone-after {
+  background: var(--lime);
+}
+
+.score-fill.tone-negative {
+  background: var(--coral);
+}
+
+.score-bar > .has-threshold {
+  position: absolute;
+  top: -2px;
+  height: 13px;
+  border-left: 1px solid currentColor;
+}
+
 .quantized-bar {
   display: grid;
   grid-template-columns: repeat(20, minmax(2px, 1fr));
@@ -1405,6 +1436,7 @@ h1 {
   text-transform: uppercase;
 }
 
+.assertion-content > .score-bar,
 .assertion-content > .quantized-bar {
   max-width: 220px;
   margin-top: 8px;

package/viewer/charts.js

@@ -51,28 +51,40 @@ const distributionBar = (segments, label) => {
   </div>`;
 };
 
+export const scoreBar = (value, tone, label, threshold = null) => {
+  if (typeof value !== "number" || !Number.isFinite(value)) {
+    return `<span class="score-bar unavailable" role="img" aria-label="${escapeHtml(label)}"></span>`;
+  }
+  return `<span class="score-bar" role="img" aria-label="${escapeHtml(label)}">
+    <i class="score-fill ${escapeHtml(tone)}" style="width: ${value * 100}%"></i>
+    ${typeof threshold === "number" ? `<i class="has-threshold" style="left: ${threshold * 100}%"></i>` : ""}
+  </span>`;
+};
+
 export const comparisonBars = ({
   title,
   before,
   after,
   maximum,
+  continuous = false,
   beforeText,
   afterText,
   deltaText,
   deltaTone,
 }) => {
+  const bar = continuous ? scoreBar : quantizedBar;
   const scale = Math.max(maximum ?? 0, before ?? 0, after ?? 0, 1);
   return `<figure class="comparison-bars">
     <figcaption><span>${escapeHtml(title)}</span><b class="${escapeHtml(
       deltaTone,
     )}">${escapeHtml(deltaText)}</b></figcaption>
-    <div class="bar-lane"><strong>A</strong>${quantizedBar(
-      (before ?? 0) / scale,
+    <div class="bar-lane"><strong>A</strong>${bar(
+      before === null || before === undefined ? null : before / scale,
       "tone-before",
       `${title} baseline ${beforeText}`,
     )}<span>${escapeHtml(beforeText)}</span></div>
-    <div class="bar-lane"><strong>B</strong>${quantizedBar(
-      (after ?? 0) / scale,
+    <div class="bar-lane"><strong>B</strong>${bar(
+      after === null || after === undefined ? null : after / scale,
       "tone-after",
       `${title} candidate ${afterText}`,
     )}<span>${escapeHtml(afterText)}</span></div>

package/viewer/detail.js

@@ -1,10 +1,16 @@
+const assertionScore = (assertion) =>
+  assertion.errorMessage !== undefined || assertion.abstention !== undefined
+    ? null
+    : assertion.score;
+
 const assertionPresentation = (assertion) => {
+  if (assertionScore(assertion) === null) return { className: " unavailable", mark: "?", tone: "" };
   if (assertion.passed === null) return { className: " informational", mark: "ℹ", tone: "" };
   if (assertion.passed) return { className: " passed", mark: "✓", tone: "tone-positive" };
   return { className: "", mark: "×", tone: "tone-negative" };
 };
 
-import { comparisonBars, compositionFigure, quantizedBar } from "./charts.js";
+import { comparisonBars, compositionFigure, quantizedBar, scoreBar } from "./charts.js";
 import {
   badge,
   compactNumber,
@@ -104,6 +110,7 @@ const assertionMeta = (assertion) =>
 const assertionDiagnostics = (assertion) => {
   const rows = [
     assertion.comment ? `<p class="assertion-comment">${escapeHtml(assertion.comment)}</p>` : "",
+    assertion.abstention ? `<p class="assertion-comment">Insufficient evidence</p>` : "",
     assertion.errorMessage
       ? `<p class="diagnostic-error">${escapeHtml(assertion.errorMessage)}</p>`
       : "",
@@ -118,6 +125,7 @@ const assertionDiagnostics = (assertion) => {
 
 const inferredMetricUnit = (name, unit) => {
   if (unit) return unit;
+  if (name === "score") return "score";
   if (name.endsWith("Ms")) return "milliseconds";
   if (name.endsWith(".bytes")) return "bytes";
   if (name.endsWith(".usd")) return "usd";
@@ -323,18 +331,18 @@ export const renderRunDetail = (item, detail) => {
                       <span class="assertion-content">
                         <strong>${escapeHtml(assertion.name)}</strong>
                         <small>${escapeHtml(assertionMeta(assertion))}</small>
-                        ${quantizedBar(
-                          assertion.score,
+                        ${scoreBar(
+                          assertionScore(assertion),
                           assertionPresentation(assertion).tone,
                           `${assertion.name} scored ${percentage(
-                            assertion.score,
+                            assertionScore(assertion),
                           )}${assertion.passed === null ? " (informational)" : `; threshold ${percentage(assertion.threshold)}`}`,
                           assertion.passed === null ? null : assertion.threshold,
                         )}
                         ${assertionDiagnostics(assertion)}
                       </span>
                       <span class="assertion-score">${escapeHtml(
-                        percentage(assertion.score),
+                        percentage(assertionScore(assertion)),
                       )}</span>
                     </article>`,
                   )
@@ -899,18 +907,20 @@ export const renderRunComparison = (payload, labels) => {
     runDelta?.agentUsageAvailable?.b === false ? null : totalTokens(runDelta?.usage?.b?.agent);
   const tokenDelta = tokenA === null || tokenB === null ? null : tokenB - tokenA;
   const scoreDelta =
+    comparison.grading.caseScoreComparable !== false &&
     typeof comparison.grading.caseScoreA === "number" &&
     typeof comparison.grading.caseScoreB === "number"
       ? comparison.grading.caseScoreB - comparison.grading.caseScoreA
       : null;
+  let comparisonStatus = "ungraded";
+  if (payload.regressed === true) comparisonStatus = "failed";
+  if (payload.regressed === false) comparisonStatus = "passed";
   const driverMetrics = driverMetricComparisons(runDelta);
   const subagentsA = subagentTotals(runDelta?.agentActivity?.a);
   const subagentsB = subagentTotals(runDelta?.agentActivity?.b);
   return `
     <header class="detail-header comparison-header">
-      <div class="detail-kicker">${badge(
-        comparison.grading.regressions ? "failed" : "passed",
-      )}<span>run comparison</span></div>
+      <div class="detail-kicker">${badge(comparisonStatus)}<span>run comparison</span></div>
       <div class="detail-title-row">
         <div><h2>${escapeHtml(labels.before)} → ${escapeHtml(
           labels.after,
@@ -926,6 +936,7 @@ export const renderRunComparison = (payload, labels) => {
             )}. Treat deltas as potentially incomparable.</div>`
           : '<div class="integrity-banner">✓ Case, evaluator, runtime, and driver identities are comparable</div>'
       }
+      ${comparison.grading.caseScoreComparable === false ? '<div class="attention-banner">Score change unavailable: scored assertions or weights differ, or evidence is missing.</div>' : ""}
       <dl class="comparison-stats">
         <div><dt>Case score</dt><dd>${escapeHtml(
           `${percentage(comparison.grading.caseScoreA)} → ${percentage(
@@ -956,6 +967,7 @@ export const renderRunComparison = (payload, labels) => {
         <div class="comparison-plot-grid">
           ${comparisonBars({
             title: "Case score",
+            continuous: true,
             before: comparison.grading.caseScoreA,
             after: comparison.grading.caseScoreB,
             maximum: 1,
@@ -1051,6 +1063,12 @@ export const renderRunComparison = (payload, labels) => {
                 value: comparison.grading.deltas.filter((delta) => delta.status === "removed")
                   .length,
               },
+              {
+                key: "unavailable",
+                label: "Unavailable",
+                value: comparison.grading.deltas.filter((delta) => delta.status === "unavailable")
+                  .length,
+              },
               {
                 key: "unchanged",
                 label: "Unchanged",

package/viewer/format.js

@@ -6,11 +6,11 @@ export const escapeHtml = (value) =>
   );
 
 export const percentage = (value) =>
-  typeof value === "number" ? `${Math.round(value * 100)}%` : "—";
+  typeof value === "number" ? `${Number((value * 100).toPrecision(15))}%` : "—";
 
 export const signedPercentage = (value) => {
   if (typeof value !== "number") return "—";
-  const rounded = Math.round(value * 1_000) / 10;
+  const rounded = Number((value * 100).toPrecision(15));
   return `${rounded > 0 ? "+" : ""}${rounded}%`;
 };