@prismatic-io/lux 0.0.2-preview.4 → 0.0.2-preview.41

package/src/orchestrator/grader-experiment.ts

@@ -1,10 +1,7 @@
 import { randomUUID } from "node:crypto";
 import { join, relative, resolve, sep } from "node:path";
 import { z } from "zod";
-import {
-  createRubricAssertion,
-  RubricModelAccountingDetailSchema,
-} from "../assertions/rubric/index.js";
+import { RubricModelAccountingDetailSchema } from "../assertions/rubric/index.js";
 import {
   type Candidate,
   type CandidateEvaluation,
@@ -28,6 +25,7 @@ import {
 import type { CampaignExecutionOptions } from "./experiment.js";
 import type { OpenedExperimentRun } from "./experiment-runs.js";
 import { gradeRun } from "./grading.js";
+import { clusterCaseSamples, groupedCaseMetricSamples } from "./paired-effects.js";
 import { loadFrozenRegradableRun, releaseFrozenRegradableRun } from "./regrade.js";
 import { loadRubricDefinition } from "./rubric-loader.js";
 
@@ -38,10 +36,12 @@ const GraderJudgmentSchema = z.object({
   actual: z.union([z.boolean(), z.number()]).nullable(),
   matched: z.boolean(),
   absoluteError: z.exactOptional(z.number()),
+  unavailableReason: z.exactOptional(z.enum(["insufficient-evidence", "operational-error"])),
 });
 export type GraderJudgment = z.infer<typeof GraderJudgmentSchema>;
 
 const GraderEvidenceItemSchema = z.object({
+  clusterId: z.exactOptional(z.string().min(1)),
   annotationId: z.string(),
   repetition: z.number().int().positive(),
   runDir: z.string(),
@@ -95,6 +95,11 @@ const assertionOperationalError = (
   return detail.success && detail.data.graderError !== undefined;
 };
 
+const assertionUnavailableReason = (
+  entry: GradingReport["assertions"][number] | undefined,
+): GraderJudgment["unavailableReason"] =>
+  assertionOperationalError(entry) ? "operational-error" : entry?.abstention;
+
 const assertionModelAccounting = (
   assertion: GradingReport["assertions"][number],
 ): { attempts: number; costUsd: number; unpricedModelCalls: number } => {
@@ -115,19 +120,22 @@ export const compareGraderAnnotation = (
   scoreTolerance: number,
 ): GraderJudgment[] => {
   const judgments: GraderJudgment[] = [];
-  const reportFailed = report.assertions.some(assertionOperationalError);
+  const reportUnavailableReason = report.assertions.some(assertionOperationalError)
+    ? "operational-error"
+    : report.assertions.find((entry) => entry.abstention)?.abstention;
   if (annotation.expected.casePassed !== undefined) {
-    const actual = reportFailed ? null : report.casePassed;
+    const actual = reportUnavailableReason ? null : report.casePassed;
     judgments.push({
       key: "case",
       kind: "case-pass",
       expected: annotation.expected.casePassed,
       actual,
       matched: actual === annotation.expected.casePassed,
+      ...(reportUnavailableReason ? { unavailableReason: reportUnavailableReason } : {}),
     });
   }
   if (annotation.expected.caseScore !== undefined) {
-    const actual = reportFailed ? null : report.caseScore;
+    const actual = reportUnavailableReason ? null : report.caseScore;
     const absoluteError = actual === null ? 1 : Math.abs(actual - annotation.expected.caseScore);
     judgments.push({
       key: "case",
@@ -136,6 +144,7 @@ export const compareGraderAnnotation = (
       actual,
       matched: actual !== null && absoluteError <= scoreTolerance,
       absoluteError,
+      ...(reportUnavailableReason ? { unavailableReason: reportUnavailableReason } : {}),
     });
   }
   const entries = new Map(
@@ -143,13 +152,15 @@ export const compareGraderAnnotation = (
   );
   for (const [assertionId, expected] of Object.entries(annotation.expected.assertions)) {
     const entry = entries.get(assertionId);
-    const actual = assertionOperationalError(entry) ? undefined : entry;
+    const unavailableReason = assertionUnavailableReason(entry);
+    const actual = unavailableReason ? undefined : entry;
     judgments.push({
       key: assertionId,
       kind: "assertion-pass",
       expected: expected.passed,
       actual: actual?.passed ?? null,
       matched: actual?.passed === expected.passed,
+      ...(unavailableReason ? { unavailableReason } : {}),
     });
     if (expected.score !== undefined) {
       const absoluteError = actual ? Math.abs(actual.score - expected.score) : 1;
@@ -160,6 +171,7 @@ export const compareGraderAnnotation = (
         actual: actual?.score ?? null,
         matched: actual !== undefined && absoluteError <= scoreTolerance,
         absoluteError,
+        ...(unavailableReason ? { unavailableReason } : {}),
       });
     }
   }
@@ -233,7 +245,7 @@ const collectGraderEvidence = (
 
 export const summarizeGraderEvidence = (
   items: readonly z.infer<typeof GraderEvidenceItemSchema>[],
-  candidate: Candidate,
+  candidate: Pick<Candidate, "sourceBytes">,
 ): { metrics: Record<string, number>; samples: Record<string, number[]> } => {
   const samples: Record<string, number[]> = {};
   const { byAnnotation, itemsByAnnotation, counts } = collectGraderEvidence(items);
@@ -323,6 +335,18 @@ export const summarizeGraderEvidence = (
         "grader.abstentionRate",
         passes.filter((judgment) => judgment.actual === null).length / passes.length,
       );
+      addSample(
+        samples,
+        "grader.insufficientEvidenceRate",
+        passes.filter((judgment) => judgment.unavailableReason === "insufficient-evidence").length /
+          passes.length,
+      );
+      addSample(
+        samples,
+        "grader.operationalErrorRate",
+        passes.filter((judgment) => judgment.unavailableReason === "operational-error").length /
+          passes.length,
+      );
     }
   }
   const precision = ratio(
@@ -440,7 +464,42 @@ const evaluationFromEvidence = (
   evidencePath: string,
   evidence: GraderEvaluationEvidence,
 ): CandidateEvaluation & { evidencePath: string } => {
+  const caseSamples = clusterCaseSamples(
+    evidence.items.map((item) => {
+      const summary = summarizeGraderEvidence([item], {
+        sourceBytes: evidence.metrics["subject.bytes"] ?? 0,
+      });
+      return {
+        [item.annotationId]: {
+          ...(item.clusterId ? { clusterId: item.clusterId } : {}),
+          identityHash: experiment.manifest.evaluationHash
+            ? valueHash({
+                evaluationHash: experiment.manifest.evaluationHash,
+                annotationId: item.annotationId,
+                profileId: evidence.profileId,
+                split: evidence.split,
+              })
+            : null,
+          metrics: Object.fromEntries(
+            Object.entries(summary.samples).flatMap(([metric, values]) =>
+              values.length === 1 ? [[metric, { mean: values[0] ?? 0, n: 1 }]] : [],
+            ),
+          ),
+        },
+      };
+    }),
+  );
   const byAnnotation = new Map<string, number[]>();
+  const grouped = groupedCaseMetricSamples(caseSamples, evidence.repetitions);
+  const metrics = grouped
+    ? Object.fromEntries(
+        Object.entries(evidence.metrics).flatMap(([name, value]) => {
+          if (!Object.hasOwn(evidence.samples, name)) return [[name, value]];
+          const values = grouped[name];
+          return values ? [[name, mean(values)]] : [];
+        }),
+      )
+    : evidence.metrics;
   for (const item of evidence.items) {
     const values = byAnnotation.get(item.annotationId) ?? [];
     values.push(...item.judgments.map((judgment) => (judgment.matched ? 1 : 0)));
@@ -471,10 +530,11 @@ const evaluationFromEvidence = (
     profileId: evidence.profileId,
     split: evidence.split,
     evidencePath: relative(experiment.path, evidencePath),
-    passRate: evidence.metrics["quality.passRate"] ?? 0,
+    passRate: metrics["quality.passRate"] ?? 0,
     casePassRates: Object.fromEntries([...byAnnotation].map(([id, values]) => [id, mean(values)])),
-    metrics: evidence.metrics,
-    samples: evidence.samples,
+    metrics,
+    samples: grouped ?? evidence.samples,
+    ...(caseSamples ? { caseSamples } : {}),
     metricCalls: evidence.items.length,
     judgeCalls: evidence.items.reduce(
       (total, item) =>
@@ -648,7 +708,7 @@ export const evaluateGraderCandidate = async (options: {
   const registry = buildRegistryFromConfig({
     ...execution.config,
     ...(harness ? { harness } : {}),
-  }).withAssertion(createRubricAssertion(harness));
+  });
   const items: GraderEvidenceItem[] = [];
   for (const annotation of selected) {
     const rubric = await loadRubricDefinition(
@@ -779,6 +839,7 @@ export const evaluateGraderCandidate = async (options: {
         );
         const item: GraderEvidenceItem = {
           annotationId: annotation.id,
+          ...(annotation.meta?.cluster ? { clusterId: annotation.meta.cluster } : {}),
           repetition,
           runDir: annotation.runDir,
           rubricPath: annotation.rubricPath,

package/src/orchestrator/grading.ts

@@ -3,6 +3,7 @@ import {
   type Assertion,
   type AssertionImpl,
   type AssertionProgress,
+  type AssertionResult,
   AssertionResultSchema,
   type GradingAssertionEntry,
   type GradingReport,
@@ -14,7 +15,7 @@ import {
 } from "../core/index.js";
 import { validatePluginConfigAsync } from "./validate-plugin-config.js";
 
-const FRAMEWORK_KEYS = new Set(["id", "type", "name", "weight", "threshold"]);
+const FRAMEWORK_KEYS = new Set(["id", "type", "name", "weight", "threshold", "informational"]);
 
 const shapeOf = (schema: z.ZodTypeAny): Record<string, unknown> | null => {
   const shape = (schema as { shape?: unknown }).shape;
@@ -124,6 +125,56 @@ const describeAssertion = (a: Assertion): string => {
   return fields ? `${a.type} ${fields}` : a.type;
 };
 
+const assertionPassed = (result: AssertionResult, threshold: number): boolean =>
+  result.errorMessage === undefined &&
+  result.abstention === undefined &&
+  result.passed &&
+  result.score >= threshold;
+
+export const gradingScoreBasis = (report: GradingReport | null): string | null =>
+  report?.caseScore === null || !report
+    ? null
+    : valueHash(
+        report.assertions
+          .map((entry, index) => ({ entry, id: entry.id ?? `${entry.type}:${index + 1}` }))
+          .filter(({ entry }) => entry.errorMessage === undefined && entry.abstention === undefined)
+          .map(({ entry, id }) => ({ id, weight: entry.weight }))
+          .sort((a, b) => a.id.localeCompare(b.id)),
+      );
+
+const aggregateGrading = (
+  entries: GradingAssertionEntry[],
+  skippedAssertions: NonNullable<GradingReport["skippedAssertions"]>,
+  gradingIncomplete: boolean,
+): GradingReport => {
+  const requiredEntries = entries.filter((entry) => entry.passed !== null);
+  const incomplete =
+    gradingIncomplete || requiredEntries.some((entry) => entry.errorMessage !== undefined);
+  const gradingAbstained = requiredEntries.some((entry) => entry.abstention !== undefined);
+  const scoredEntries = entries.filter(
+    (entry) => entry.errorMessage === undefined && entry.abstention === undefined,
+  );
+  const weightSum = scoredEntries.reduce((sum, entry) => sum + entry.weight, 0);
+  const weightedScoreSum = scoredEntries.reduce(
+    (sum, entry) => sum + entry.weight * entry.score,
+    0,
+  );
+  const hasVerdict = !incomplete && !gradingAbstained && weightSum > 0;
+  return {
+    schemaVersion: 1,
+    casePassed:
+      hasVerdict && requiredEntries.length > 0
+        ? requiredEntries.every((entry) => entry.passed)
+        : null,
+    caseScore: hasVerdict ? weightedScoreSum / weightSum : null,
+    assertions: entries,
+    ...(skippedAssertions.length > 0 ? { skippedAssertions } : {}),
+    ...(incomplete
+      ? { incomplete: { exitReason: "error" as const, phase: "grading" as const } }
+      : {}),
+  };
+};
+
 export const gradeRun = async (
   run: Run,
   registry: Registry,
@@ -150,9 +201,6 @@ export const gradeRun = async (
     name: string;
     reason: "judge-disabled";
   }> = [];
-  let weightedScoreSum = 0;
-  let weightSum = 0;
-  let allPassed = true;
   let gradingIncomplete = false;
 
   for (const [index, a] of assertions.entries()) {
@@ -205,7 +253,7 @@ export const gradeRun = async (
       if (validatedConfigs && index >= validatedConfigs.length) {
         throw new Error(`validated assertion plan is missing assertions[${index}]`);
       }
-      // Pass the validated config so schema defaults (e.g. timeoutMs)
+      // Pass the validated config so schema defaults (e.g. timeout)
       // reach evaluate(); the raw assertion object has none applied.
       const result = AssertionResultSchema.parse(
         await impl.evaluate(validated as typeof a, run, {
@@ -219,43 +267,35 @@ export const gradeRun = async (
             : {}),
         }),
       );
-      const passed =
-        result.errorMessage === undefined && result.passed && result.score >= threshold;
+      const passed = assertionPassed(result, threshold);
       entry = {
         id,
         type: a.type,
         name,
-        passed,
+        passed: a.informational ? null : passed,
+        ...(a.informational ? { informational: true } : { threshold }),
         score: result.score,
         weight,
-        threshold,
         durationMs: Date.now() - startedAt,
         ...(result.comment !== undefined ? { comment: result.comment } : {}),
         detail: result.detail,
+        ...(result.abstention !== undefined ? { abstention: result.abstention } : {}),
         ...(result.errorMessage !== undefined ? { errorMessage: result.errorMessage } : {}),
         ...(result.usage ? { usage: result.usage } : {}),
       };
-      if (result.errorMessage !== undefined) {
-        gradingIncomplete = true;
-      } else {
-        if (!passed) allPassed = false;
-        weightedScoreSum += weight * result.score;
-        weightSum += weight;
-      }
     } catch (err) {
       const message = err instanceof Error ? err.message : String(err);
       entry = {
         id,
         type: a.type,
         name,
-        passed: false,
+        passed: a.informational ? null : false,
+        ...(a.informational ? { informational: true } : { threshold }),
         score: 0,
         weight,
-        threshold,
         durationMs: Date.now() - startedAt,
         errorMessage: message,
       };
-      gradingIncomplete = true;
     }
 
     entries.push(entry);
@@ -264,16 +304,5 @@ export const gradeRun = async (
     await modelAttempts?.complete(assertionAttemptIds);
   }
 
-  let caseScore: number | null = null;
-  if (!gradingIncomplete && weightSum > 0) caseScore = weightedScoreSum / weightSum;
-  return {
-    schemaVersion: 1,
-    casePassed: gradingIncomplete || weightSum === 0 ? null : allPassed,
-    caseScore,
-    assertions: entries,
-    ...(skippedAssertions.length > 0 ? { skippedAssertions } : {}),
-    ...(gradingIncomplete
-      ? { incomplete: { exitReason: "error" as const, phase: "grading" as const } }
-      : {}),
-  };
+  return aggregateGrading(entries, skippedAssertions, gradingIncomplete);
 };

package/src/orchestrator/harness-catalog-drift.ts

@@ -0,0 +1,132 @@
+import { readFile } from "node:fs/promises";
+import { homedir } from "node:os";
+import { join } from "node:path";
+import { z } from "zod";
+import {
+  type ClosedHarnessCatalog,
+  codexCatalog,
+  copilotCatalog,
+  exec,
+  type ReasoningEffort,
+  tryParseJson,
+} from "../core/index.js";
+
+export type LiveHarnessModel = {
+  id: string;
+  efforts?: readonly ReasoningEffort[] | undefined;
+};
+
+export type HarnessCatalogDrift = {
+  unlisted: string[];
+  mismatched: string[];
+  unavailable: string[];
+};
+
+export const harnessCatalogDrift = (
+  catalog: ClosedHarnessCatalog,
+  live: readonly LiveHarnessModel[],
+): HarnessCatalogDrift => {
+  const liveIds = new Set(live.map((model) => model.id));
+  return {
+    unlisted: live.filter((model) => !(model.id in catalog.models)).map((model) => model.id),
+    mismatched: live
+      .filter(
+        (model) =>
+          model.efforts !== undefined &&
+          model.id in catalog.models &&
+          model.efforts.join(",") !== catalog.models[model.id]?.join(","),
+      )
+      .map((model) => `${model.id} (${model.efforts?.join(", ") || "no effort"})`),
+    unavailable: Object.keys(catalog.models).filter((id) => !liveIds.has(id)),
+  };
+};
+
+export const hasHarnessCatalogDrift = (drift: HarnessCatalogDrift): boolean =>
+  drift.unlisted.length > 0 || drift.mismatched.length > 0;
+
+export const describeHarnessCatalogDrift = (drift: HarnessCatalogDrift): string =>
+  [
+    drift.unlisted.length > 0 ? `not in catalog: ${drift.unlisted.join(", ")}` : undefined,
+    drift.mismatched.length > 0 ? `efforts differ: ${drift.mismatched.join("; ")}` : undefined,
+    drift.unavailable.length > 0 ? `not offered here: ${drift.unavailable.join(", ")}` : undefined,
+  ]
+    .filter((part) => part !== undefined)
+    .join(". ");
+
+const CodexModelsCacheSchema = z.object({
+  models: z.array(
+    z.looseObject({
+      slug: z.string(),
+      visibility: z.string().optional(),
+      supported_reasoning_levels: z.array(z.looseObject({ effort: z.string() })).default([]),
+    }),
+  ),
+});
+
+const codexModelsCachePath = (env: NodeJS.ProcessEnv = process.env): string =>
+  join(env.CODEX_HOME ?? join(homedir(), ".codex"), "models_cache.json");
+
+export const parseCodexModelsCache = (text: string): LiveHarnessModel[] | undefined => {
+  const parsed = CodexModelsCacheSchema.safeParse(tryParseJson(text));
+  if (!parsed.success) return undefined;
+  return parsed.data.models
+    .filter((model) => model.visibility !== "hide")
+    .map((model) => ({
+      id: model.slug,
+      efforts: model.supported_reasoning_levels.map((level) => level.effort as ReasoningEffort),
+    }));
+};
+
+const readCodexLiveModels = async (
+  path = codexModelsCachePath(),
+): Promise<LiveHarnessModel[] | undefined> => {
+  try {
+    return parseCodexModelsCache(await readFile(path, "utf8"));
+  } catch {
+    return undefined;
+  }
+};
+
+export const parseCopilotModelValues = (helpText: string): LiveHarnessModel[] | undefined => {
+  const lines = helpText.split("\n");
+  const start = lines.findIndex((line) => /^\s*`model`:/.test(line));
+  if (start < 0) return undefined;
+  const models: LiveHarnessModel[] = [];
+  for (const line of lines.slice(start + 1)) {
+    const match = /^\s*-\s*"([^"]+)"\s*$/.exec(line);
+    if (!match?.[1]) break;
+    models.push({ id: match[1] });
+  }
+  return models.length > 0 ? models : undefined;
+};
+
+const readCopilotLiveModels = async (
+  command = "copilot",
+): Promise<LiveHarnessModel[] | undefined> => {
+  try {
+    const result = await exec(command, ["help", "config"], { timeoutMs: 15_000 });
+    if (result.exitCode !== 0 || result.timedOut) return undefined;
+    return parseCopilotModelValues(result.stdout);
+  } catch {
+    return undefined;
+  }
+};
+
+export type HarnessCatalogProbe = {
+  catalog: ClosedHarnessCatalog;
+  source: string;
+  read: (command: string | undefined) => Promise<LiveHarnessModel[] | undefined>;
+};
+
+export const harnessCatalogProbes: readonly HarnessCatalogProbe[] = [
+  {
+    catalog: codexCatalog,
+    source: "models_cache.json",
+    read: () => readCodexLiveModels(),
+  },
+  {
+    catalog: copilotCatalog,
+    source: "help config",
+    read: (command) => readCopilotLiveModels(command),
+  },
+];

package/src/orchestrator/harness-registry.ts

@@ -0,0 +1,12 @@
+import { createRubricAssertion } from "../assertions/rubric/index.js";
+import type { HarnessModelConfig, Registry } from "../core/index.js";
+import type { ResolvedLuxConfig } from "./config.js";
+
+export const registryWithHarness = (
+  registry: Registry,
+  config: Pick<ResolvedLuxConfig, "assertions">,
+  harness: HarnessModelConfig | undefined,
+): Registry =>
+  config.assertions?.some((assertion) => assertion.type === "rubric")
+    ? registry
+    : registry.withAssertion(createRubricAssertion(harness));

package/src/orchestrator/index.ts

@@ -122,6 +122,7 @@ export {
   formatRunsTable,
   type ListedRun,
   listRuns,
+  loadRunArtifacts,
   loadRunDetail,
   type RunDetail,
 } from "./list-runs.js";