@prismatic-io/lux 0.0.2-preview.14 → 0.0.2-preview.16

package/src/orchestrator/experiment.ts

@@ -1,6 +1,7 @@
 import { randomUUID } from "node:crypto";
 import { rm } from "node:fs/promises";
 import type { z } from "zod";
+import { independentClusterCount } from "../core/evaluation-clusters.js";
 import {
   bootstrapMeanConfidenceInterval,
   type Candidate,
@@ -55,14 +56,7 @@ import {
   openExperimentRun,
   resumeExperimentRun,
 } from "./experiment-runs.js";
-import {
-  claudeConfigRoot,
-  claudeUserBehaviorHash,
-  cliIdentity,
-  configuredClaudeSourceHashes,
-  LUX_VERSION,
-  runtimeCliIdentityHashForReferences,
-} from "./experiment-runtime-identity.js";
+import { LUX_VERSION, runtimeCliIdentityHashForReferences } from "./experiment-runtime-identity.js";
 import {
   candidateEvaluations,
   type ExperimentSelectionDerivationOptionsSchema,
@@ -254,50 +248,57 @@ export const planExperimentCampaign = async (
     const splitSelector = splitSelectors.find((split) => split.name === constraint.split)?.selector;
     let examples = splits.find((split) => split.split === constraint.split)?.examples ?? 0;
     if (corpus.kind === "grader" && splitSelector) {
-      const selected = selectGraderAnnotations(graderAnnotations, splitSelector);
+      let selected = selectGraderAnnotations(graderAnnotations, splitSelector);
       if (constraint.metric === "grader.macroFalsePositiveRate") {
-        examples = selected.filter((annotation) =>
+        selected = selected.filter((annotation) =>
           [
             annotation.expected.casePassed,
             ...Object.values(annotation.expected.assertions).map((label) => label.passed),
           ].includes(false),
-        ).length;
+        );
       } else if (constraint.metric === "grader.macroFalseNegativeRate") {
-        examples = selected.filter((annotation) =>
+        selected = selected.filter((annotation) =>
           [
             annotation.expected.casePassed,
             ...Object.values(annotation.expected.assertions).map((label) => label.passed),
           ].includes(true),
-        ).length;
+        );
       } else if (constraint.metric === "grader.scoreMae") {
-        examples = selected.filter(
+        selected = selected.filter(
           (annotation) =>
             annotation.expected.caseScore !== undefined ||
             Object.values(annotation.expected.assertions).some(
               (label) => label.score !== undefined,
             ),
-        ).length;
+        );
       } else if (constraint.metric === "grader.caseAgreement") {
-        examples = selected.filter(
+        selected = selected.filter(
           (annotation) =>
             annotation.expected.casePassed !== undefined ||
             annotation.expected.caseScore !== undefined,
-        ).length;
+        );
       } else if (constraint.metric === "grader.assertionAgreement") {
-        examples = selected.filter(
+        selected = selected.filter(
           (annotation) => Object.keys(annotation.expected.assertions).length > 0,
-        ).length;
+        );
       } else if (
         constraint.metric === "grader.abstentionRate" ||
+        constraint.metric === "grader.insufficientEvidenceRate" ||
+        constraint.metric === "grader.operationalErrorRate" ||
         constraint.metric === "grader.coverage"
       ) {
-        examples = selected.filter(
+        selected = selected.filter(
           (annotation) =>
             annotation.expected.casePassed !== undefined ||
             Object.keys(annotation.expected.assertions).length > 0,
-        ).length;
+        );
       }
+      examples = independentClusterCount(
+        selected.map((annotation) => ({ id: annotation.id, cluster: annotation.meta?.cluster })),
+      );
     }
+    if (corpus.kind === "agent" && splitSelector)
+      examples = independentClusterCount(selectCases(corpus.cases, splitSelector));
     if (constraint.metric === "subject.bytes" || constraint.metric === "quality.caseRegressions") {
       examples = 1;
     }
@@ -318,6 +319,10 @@ export const planExperimentCampaign = async (
         Array<number>(examples).fill(0),
         constraint.confidence,
       );
+      if (constraint.relativeTo) {
+        bestForMin.lower = 2 * bestForMin.lower - 1;
+        bestForMax.upper = 2 * bestForMax.upper - 1;
+      }
       if (constraint.min !== undefined && bestForMin.lower < constraint.min) {
         notes.push(
           `${constraint.split} metric '${constraint.metric}' cannot attain confidence lower bound ${constraint.min} with only ${examples} samples (best ${bestForMin.lower.toFixed(3)})`,
@@ -986,6 +991,3 @@ export const optimizeExperimentCampaign = async (
     new CampaignExecution(execution, "optimize", runOptimizationCampaignWork),
   );
 };
-
-// Internal exports for identity contract tests.
-export { claudeConfigRoot, claudeUserBehaviorHash, cliIdentity, configuredClaudeSourceHashes };

package/src/orchestrator/fs-read.ts

@@ -1,7 +1,7 @@
 import { existsSync } from "node:fs";
 import type { FileHandle } from "node:fs/promises";
 import { readdir, readFile } from "node:fs/promises";
-import { join, relative, resolve } from "node:path";
+import { join, relative } from "node:path";
 import type { z } from "zod";
 
 const errorMessage = (error: unknown): string =>
@@ -93,34 +93,4 @@ export const walkFiles = async (root: string, maxFiles?: number): Promise<string
   return out.sort();
 };
 
-/**
- * Walk authored trees without silently following or omitting a symlink that
- * can change outside the experiment identity. Exact ignored paths support
- * generated state directories whose names are project-configurable.
- */
-export const walkAuthoredFiles = async (
-  root: string,
-  ignoredDirectoryNames: ReadonlySet<string> = new Set(),
-  ignoredPaths: ReadonlySet<string> = new Set(),
-): Promise<string[]> => {
-  if (!existsSync(root)) return [];
-  const out: string[] = [];
-  const stack = [root];
-  while (stack.length > 0) {
-    const dir = stack.pop();
-    if (!dir) continue;
-    for (const entry of await readdir(dir, { withFileTypes: true })) {
-      const full = join(dir, entry.name);
-      if (ignoredPaths.has(resolve(full))) continue;
-      if (entry.isDirectory() && ignoredDirectoryNames.has(entry.name)) continue;
-      if (entry.isSymbolicLink()) {
-        throw new Error(
-          `authored evaluation trees cannot contain symlinks: ${relative(root, full)}`,
-        );
-      }
-      if (entry.isDirectory()) stack.push(full);
-      else if (entry.isFile()) out.push(relative(root, full));
-    }
-  }
-  return out.sort();
-};
+export { walkAuthoredFiles } from "../core/file-tree.js";

package/src/orchestrator/grader-experiment.ts

@@ -28,6 +28,7 @@ import {
 import type { CampaignExecutionOptions } from "./experiment.js";
 import type { OpenedExperimentRun } from "./experiment-runs.js";
 import { gradeRun } from "./grading.js";
+import { clusterCaseSamples, groupedCaseMetricSamples } from "./paired-effects.js";
 import { loadFrozenRegradableRun, releaseFrozenRegradableRun } from "./regrade.js";
 import { loadRubricDefinition } from "./rubric-loader.js";
 
@@ -38,10 +39,12 @@ const GraderJudgmentSchema = z.object({
   actual: z.union([z.boolean(), z.number()]).nullable(),
   matched: z.boolean(),
   absoluteError: z.exactOptional(z.number()),
+  unavailableReason: z.exactOptional(z.enum(["insufficient-evidence", "operational-error"])),
 });
 export type GraderJudgment = z.infer<typeof GraderJudgmentSchema>;
 
 const GraderEvidenceItemSchema = z.object({
+  clusterId: z.exactOptional(z.string().min(1)),
   annotationId: z.string(),
   repetition: z.number().int().positive(),
   runDir: z.string(),
@@ -95,6 +98,11 @@ const assertionOperationalError = (
   return detail.success && detail.data.graderError !== undefined;
 };
 
+const assertionUnavailableReason = (
+  entry: GradingReport["assertions"][number] | undefined,
+): GraderJudgment["unavailableReason"] =>
+  assertionOperationalError(entry) ? "operational-error" : entry?.abstention;
+
 const assertionModelAccounting = (
   assertion: GradingReport["assertions"][number],
 ): { attempts: number; costUsd: number; unpricedModelCalls: number } => {
@@ -115,19 +123,22 @@ export const compareGraderAnnotation = (
   scoreTolerance: number,
 ): GraderJudgment[] => {
   const judgments: GraderJudgment[] = [];
-  const reportFailed = report.assertions.some(assertionOperationalError);
+  const reportUnavailableReason = report.assertions.some(assertionOperationalError)
+    ? "operational-error"
+    : report.assertions.find((entry) => entry.abstention)?.abstention;
   if (annotation.expected.casePassed !== undefined) {
-    const actual = reportFailed ? null : report.casePassed;
+    const actual = reportUnavailableReason ? null : report.casePassed;
     judgments.push({
       key: "case",
       kind: "case-pass",
       expected: annotation.expected.casePassed,
       actual,
       matched: actual === annotation.expected.casePassed,
+      ...(reportUnavailableReason ? { unavailableReason: reportUnavailableReason } : {}),
     });
   }
   if (annotation.expected.caseScore !== undefined) {
-    const actual = reportFailed ? null : report.caseScore;
+    const actual = reportUnavailableReason ? null : report.caseScore;
     const absoluteError = actual === null ? 1 : Math.abs(actual - annotation.expected.caseScore);
     judgments.push({
       key: "case",
@@ -136,6 +147,7 @@ export const compareGraderAnnotation = (
       actual,
       matched: actual !== null && absoluteError <= scoreTolerance,
       absoluteError,
+      ...(reportUnavailableReason ? { unavailableReason: reportUnavailableReason } : {}),
     });
   }
   const entries = new Map(
@@ -143,13 +155,15 @@ export const compareGraderAnnotation = (
   );
   for (const [assertionId, expected] of Object.entries(annotation.expected.assertions)) {
     const entry = entries.get(assertionId);
-    const actual = assertionOperationalError(entry) ? undefined : entry;
+    const unavailableReason = assertionUnavailableReason(entry);
+    const actual = unavailableReason ? undefined : entry;
     judgments.push({
       key: assertionId,
       kind: "assertion-pass",
       expected: expected.passed,
       actual: actual?.passed ?? null,
       matched: actual?.passed === expected.passed,
+      ...(unavailableReason ? { unavailableReason } : {}),
     });
     if (expected.score !== undefined) {
       const absoluteError = actual ? Math.abs(actual.score - expected.score) : 1;
@@ -160,6 +174,7 @@ export const compareGraderAnnotation = (
         actual: actual?.score ?? null,
         matched: actual !== undefined && absoluteError <= scoreTolerance,
         absoluteError,
+        ...(unavailableReason ? { unavailableReason } : {}),
       });
     }
   }
@@ -233,7 +248,7 @@ const collectGraderEvidence = (
 
 export const summarizeGraderEvidence = (
   items: readonly z.infer<typeof GraderEvidenceItemSchema>[],
-  candidate: Candidate,
+  candidate: Pick<Candidate, "sourceBytes">,
 ): { metrics: Record<string, number>; samples: Record<string, number[]> } => {
   const samples: Record<string, number[]> = {};
   const { byAnnotation, itemsByAnnotation, counts } = collectGraderEvidence(items);
@@ -323,6 +338,18 @@ export const summarizeGraderEvidence = (
         "grader.abstentionRate",
         passes.filter((judgment) => judgment.actual === null).length / passes.length,
       );
+      addSample(
+        samples,
+        "grader.insufficientEvidenceRate",
+        passes.filter((judgment) => judgment.unavailableReason === "insufficient-evidence").length /
+          passes.length,
+      );
+      addSample(
+        samples,
+        "grader.operationalErrorRate",
+        passes.filter((judgment) => judgment.unavailableReason === "operational-error").length /
+          passes.length,
+      );
     }
   }
   const precision = ratio(
@@ -440,7 +467,42 @@ const evaluationFromEvidence = (
   evidencePath: string,
   evidence: GraderEvaluationEvidence,
 ): CandidateEvaluation & { evidencePath: string } => {
+  const caseSamples = clusterCaseSamples(
+    evidence.items.map((item) => {
+      const summary = summarizeGraderEvidence([item], {
+        sourceBytes: evidence.metrics["subject.bytes"] ?? 0,
+      });
+      return {
+        [item.annotationId]: {
+          ...(item.clusterId ? { clusterId: item.clusterId } : {}),
+          identityHash: experiment.manifest.evaluationHash
+            ? valueHash({
+                evaluationHash: experiment.manifest.evaluationHash,
+                annotationId: item.annotationId,
+                profileId: evidence.profileId,
+                split: evidence.split,
+              })
+            : null,
+          metrics: Object.fromEntries(
+            Object.entries(summary.samples).flatMap(([metric, values]) =>
+              values.length === 1 ? [[metric, { mean: values[0] ?? 0, n: 1 }]] : [],
+            ),
+          ),
+        },
+      };
+    }),
+  );
   const byAnnotation = new Map<string, number[]>();
+  const grouped = groupedCaseMetricSamples(caseSamples, evidence.repetitions);
+  const metrics = grouped
+    ? Object.fromEntries(
+        Object.entries(evidence.metrics).flatMap(([name, value]) => {
+          if (!Object.hasOwn(evidence.samples, name)) return [[name, value]];
+          const values = grouped[name];
+          return values ? [[name, mean(values)]] : [];
+        }),
+      )
+    : evidence.metrics;
   for (const item of evidence.items) {
     const values = byAnnotation.get(item.annotationId) ?? [];
     values.push(...item.judgments.map((judgment) => (judgment.matched ? 1 : 0)));
@@ -471,10 +533,11 @@ const evaluationFromEvidence = (
     profileId: evidence.profileId,
     split: evidence.split,
     evidencePath: relative(experiment.path, evidencePath),
-    passRate: evidence.metrics["quality.passRate"] ?? 0,
+    passRate: metrics["quality.passRate"] ?? 0,
     casePassRates: Object.fromEntries([...byAnnotation].map(([id, values]) => [id, mean(values)])),
-    metrics: evidence.metrics,
-    samples: evidence.samples,
+    metrics,
+    samples: grouped ?? evidence.samples,
+    ...(caseSamples ? { caseSamples } : {}),
     metricCalls: evidence.items.length,
     judgeCalls: evidence.items.reduce(
       (total, item) =>
@@ -779,6 +842,7 @@ export const evaluateGraderCandidate = async (options: {
         );
         const item: GraderEvidenceItem = {
           annotationId: annotation.id,
+          ...(annotation.meta?.cluster ? { clusterId: annotation.meta.cluster } : {}),
           repetition,
           runDir: annotation.runDir,
           rubricPath: annotation.rubricPath,

package/src/orchestrator/grading.ts

@@ -3,6 +3,7 @@ import {
   type Assertion,
   type AssertionImpl,
   type AssertionProgress,
+  type AssertionResult,
   AssertionResultSchema,
   type GradingAssertionEntry,
   type GradingReport,
@@ -124,6 +125,12 @@ const describeAssertion = (a: Assertion): string => {
   return fields ? `${a.type} ${fields}` : a.type;
 };
 
+const assertionPassed = (result: AssertionResult, threshold: number): boolean =>
+  result.errorMessage === undefined &&
+  result.abstention === undefined &&
+  result.passed &&
+  result.score >= threshold;
+
 export const gradeRun = async (
   run: Run,
   registry: Registry,
@@ -154,6 +161,7 @@ export const gradeRun = async (
   let weightSum = 0;
   let allPassed = true;
   let gradingIncomplete = false;
+  let gradingAbstained = false;
 
   for (const [index, a] of assertions.entries()) {
     if (skipJudge && a.type === "rubric") {
@@ -219,8 +227,7 @@ export const gradeRun = async (
             : {}),
         }),
       );
-      const passed =
-        result.errorMessage === undefined && result.passed && result.score >= threshold;
+      const passed = assertionPassed(result, threshold);
       entry = {
         id,
         type: a.type,
@@ -232,11 +239,14 @@ export const gradeRun = async (
         durationMs: Date.now() - startedAt,
         ...(result.comment !== undefined ? { comment: result.comment } : {}),
         detail: result.detail,
+        ...(result.abstention !== undefined ? { abstention: result.abstention } : {}),
         ...(result.errorMessage !== undefined ? { errorMessage: result.errorMessage } : {}),
         ...(result.usage ? { usage: result.usage } : {}),
       };
       if (result.errorMessage !== undefined) {
         gradingIncomplete = true;
+      } else if (result.abstention !== undefined) {
+        gradingAbstained = true;
       } else {
         if (!passed) allPassed = false;
         weightedScoreSum += weight * result.score;
@@ -264,12 +274,11 @@ export const gradeRun = async (
     await modelAttempts?.complete(assertionAttemptIds);
   }
 
-  let caseScore: number | null = null;
-  if (!gradingIncomplete && weightSum > 0) caseScore = weightedScoreSum / weightSum;
+  const hasVerdict = !gradingIncomplete && !gradingAbstained && weightSum > 0;
   return {
     schemaVersion: 1,
-    casePassed: gradingIncomplete || weightSum === 0 ? null : allPassed,
-    caseScore,
+    casePassed: hasVerdict ? allPassed : null,
+    caseScore: hasVerdict ? weightedScoreSum / weightSum : null,
     assertions: entries,
     ...(skippedAssertions.length > 0 ? { skippedAssertions } : {}),
     ...(gradingIncomplete

package/src/orchestrator/list-runs.ts

@@ -20,6 +20,7 @@ import {
   ToolResultPayloadSchema,
   totalTokens,
 } from "../core/index.js";
+import { hasAgentUsage } from "../core/usage.js";
 import { readBoundedHandle, safeReadJson, walkFiles } from "./fs-read.js";
 import { MAX_RUN_TRACE_BYTES, MAX_RUN_TRACE_EVENTS } from "./trace-limits.js";
 
@@ -45,6 +46,7 @@ export type ListedRun = {
   caseScore: number | null;
   assertionPassed: number;
   assertionTotal: number;
+  readErrors?: string[];
 };
 
 const summarizeRun = async (
@@ -53,10 +55,19 @@ const summarizeRun = async (
   ts: string,
   runDir: string,
 ): Promise<ListedRun> => {
-  const [meta, grading] = await Promise.all([
+  const [metadataResult, gradingResult] = await Promise.allSettled([
     safeReadJson(join(runDir, "run.json"), RunMetadataSchema),
     safeReadJson(join(runDir, "grading.json"), GradingReportSchema),
   ]);
+  const readErrors = [metadataResult, gradingResult].flatMap((result) =>
+    result.status === "rejected"
+      ? [result.reason instanceof Error ? result.reason.message : String(result.reason)]
+      : [],
+  );
+  const meta = metadataResult.status === "fulfilled" ? metadataResult.value : null;
+  // A valid grading file cannot establish a result when its run metadata is damaged.
+  const grading =
+    readErrors.length === 0 && gradingResult.status === "fulfilled" ? gradingResult.value : null;
   const startedAt = meta?.startedAt ?? null;
   const endedAt = meta?.endedAt ?? null;
   const started = Date.parse(startedAt ?? "");
@@ -83,6 +94,7 @@ const summarizeRun = async (
     caseScore: grading?.caseScore ?? null,
     assertionPassed: passing,
     assertionTotal: grading?.assertions.length ?? 0,
+    ...(readErrors.length > 0 ? { readErrors } : {}),
   };
 };
 
@@ -146,7 +158,8 @@ export const formatRunsTable = (runs: ListedRun[]): string => {
     ["status", "case", "driver", "agents", "ts", "exit", "score", "duration"],
     ...runs.map((r) => {
       let status = "ungraded";
-      if (r.casePassed === true) status = "PASS";
+      if (r.readErrors?.length) status = "DAMAGED";
+      else if (r.casePassed === true) status = "PASS";
       else if (r.casePassed === false) status = "FAIL";
       const score =
         r.caseScore !== null
@@ -159,9 +172,11 @@ export const formatRunsTable = (runs: ListedRun[]): string => {
   ];
   const header = rows[0] ?? [];
   const widths = header.map((_, col) => Math.max(...rows.map((r) => r[col]?.length ?? 0)));
-  return rows
+  const table = rows
     .map((row) => row.map((cell, col) => cell.padEnd(widths[col] ?? 0)).join("  "))
     .join("\n");
+  const diagnostics = runs.flatMap((run) => run.readErrors ?? []);
+  return diagnostics.length > 0 ? `${table}\n\n${diagnostics.join("\n")}` : table;
 };
 
 export type RunDetail = {
@@ -289,6 +304,12 @@ const appendMetadataDetail = (lines: string[], detail: RunDetail): void => {
         }`,
       );
     }
+    const environment = provenance?.environment;
+    if (environment) {
+      lines.push(`concurrency: ${environment.concurrency}`);
+      if (environment.declared)
+        lines.push(`declared environment: ${JSON.stringify(environment.declared)}`);
+    }
     const c = detail.metadata.interruptCounts;
     lines.push(`interrupts: ask=${c.ask} approve=${c.approve} custom=${c.custom}`);
     const tools = detail.metadata.toolOutcomes;
@@ -301,7 +322,7 @@ const appendMetadataDetail = (lines: string[], detail: RunDetail): void => {
     }
     const usage = detail.metadata.usage;
     lines.push(
-      `tokens: agent=${totalTokens(usage.agent)} harness=${totalTokens(harnessUsage(usage))}` +
+      `tokens: agent=${hasAgentUsage(detail.metadata) ? totalTokens(usage.agent) : "unavailable"} harness=${totalTokens(harnessUsage(usage))}` +
         ` (answerer=${totalTokens(usage.answerer)} judge=${totalTokens(usage.grading)})`,
     );
     const phases = detail.metadata.phaseDurationMs;
@@ -347,9 +368,11 @@ const appendGradingDetail = (lines: string[], detail: RunDetail): void => {
       `grading: ${status} (${score}, ${detail.grading.assertions.filter((a) => a.passed).length}/${detail.grading.assertions.length})`,
     );
     for (const a of detail.grading.assertions) {
-      const mark = a.passed ? "✓" : "✗";
+      let mark = a.passed ? "✓" : "✗";
+      if (a.abstention) mark = "?";
       const took = ` (${(a.durationMs / 1000).toFixed(1)}s)`;
-      lines.push(`  ${mark} ${a.name}${took}${a.comment ? ` — ${a.comment}` : ""}`);
+      const abstention = a.abstention ? " — insufficient evidence" : "";
+      lines.push(`  ${mark} ${a.name}${took}${abstention}${a.comment ? ` — ${a.comment}` : ""}`);
     }
   }
 };

package/src/orchestrator/optimization-lifecycle.ts

@@ -26,6 +26,7 @@ export const executeOptimizationLifecycle = async <Search, Deployment, Result>(
     deployment: Deployment | undefined;
     result: Result | undefined;
     error: unknown;
+    failed: boolean;
   };
 
   const machine = setup({
@@ -59,6 +60,7 @@ export const executeOptimizationLifecycle = async <Search, Deployment, Result>(
         result: ({ event }) => ("output" in event ? event.output : undefined),
       }),
       captureError: assign({
+        failed: true,
         error: ({ event }) =>
           "error" in event ? event.error : new Error("optimization lifecycle failed"),
       }),
@@ -70,6 +72,7 @@ export const executeOptimizationLifecycle = async <Search, Deployment, Result>(
       deployment: undefined,
       result: undefined,
       error: undefined,
+      failed: false,
     },
     initial: "evaluatingSearchSeeds",
     states: {
@@ -112,7 +115,7 @@ export const executeOptimizationLifecycle = async <Search, Deployment, Result>(
       failed: { type: "final" },
     },
     output: ({ context }) =>
-      context.error === undefined
+      !context.failed
         ? { status: "succeeded", result: context.result }
         : { status: "failed", error: context.error },
   });