@prismatic-io/lux 0.0.2-preview.3 → 0.0.2-preview.5

package/src/orchestrator/experiment-evaluation.ts

@@ -101,6 +101,23 @@ const evaluationMetrics = (
   ),
 });
 
+const evaluationMetricDefinitions = (summary: SuiteSummary) =>
+  Object.fromEntries(
+    Object.entries(summary.metrics).flatMap(([name, metric]) => {
+      if (!metric.unit && !metric.label && !metric.better) return [];
+      return [
+        [
+          name,
+          {
+            unit: metric.unit ?? "number",
+            better: metric.better ?? "neutral",
+            ...(metric.label ? { label: metric.label } : {}),
+          },
+        ],
+      ];
+    }),
+  );
+
 const summaryUsage = (summary: SuiteSummary) =>
   summary.samples.reduce(
     (usage, sample) =>
@@ -277,6 +294,7 @@ const agentEvaluationFromSuite = (input: {
       summary.cases.map((entry) => [entry.canonicalCaseId, entry.passRate]),
     ),
     metrics: evaluationMetrics(summary, candidate),
+    metricDefinitions: evaluationMetricDefinitions(summary),
     samples: evaluationSamples(summary),
     metricCalls: summary.samples.length,
     judgeCalls: results.reduce(
@@ -352,6 +370,10 @@ const aggregateAgentUnits = (
         .map(([, values]) => average(values)),
     ]),
   );
+  const metricDefinitions = Object.assign(
+    {},
+    ...units.map(({ evaluation }) => evaluation.metricDefinitions ?? {}),
+  );
   return {
     candidateId: candidate.id,
     profileId,
@@ -367,6 +389,7 @@ const aggregateAgentUnits = (
         name === "subject.bytes" ? candidate.sourceBytes : average(values),
       ]),
     ),
+    metricDefinitions,
     samples,
     metricCalls: units.reduce((total, unit) => total + (unit.evaluation.metricCalls ?? 0), 0),
     judgeCalls: units.reduce((total, unit) => total + (unit.evaluation.judgeCalls ?? 0), 0),

package/src/orchestrator/grading.ts

@@ -76,6 +76,7 @@ export const graderConfigHash = (input: {
   assertions: readonly AssertionEvaluatorIdentity[];
   validatedAssertionsHash: string;
   graderContext?: unknown;
+  skipJudge?: boolean;
 }): string =>
   valueHash({
     luxVersion: input.luxVersion,
@@ -83,6 +84,7 @@ export const graderConfigHash = (input: {
     assertions: input.assertions,
     validatedAssertionsHash: input.validatedAssertionsHash,
     graderContext: input.graderContext ?? null,
+    skipJudge: input.skipJudge ?? false,
   });
 
 export const validateAssertionPlan = async (
@@ -133,6 +135,7 @@ export const gradeRun = async (
     complete: (attemptIds: readonly string[]) => void | Promise<void>;
   },
   validatedConfigs?: readonly unknown[],
+  skipJudge = false,
 ): Promise<GradingReport> => {
   const guardedObserver = guardRunObserver(observer);
   const assertions = run.case.assertions ?? [];
@@ -141,12 +144,27 @@ export const gradeRun = async (
   }
 
   const entries: GradingAssertionEntry[] = [];
+  const skippedAssertions: Array<{
+    id: string;
+    type: string;
+    name: string;
+    reason: "judge-disabled";
+  }> = [];
   let weightedScoreSum = 0;
   let weightSum = 0;
   let allPassed = true;
   let gradingIncomplete = false;
 
   for (const [index, a] of assertions.entries()) {
+    if (skipJudge && a.type === "rubric") {
+      skippedAssertions.push({
+        id: a.id ?? `${a.type}:${index + 1}`,
+        type: a.type,
+        name: describeAssertion(a),
+        reason: "judge-disabled",
+      });
+      continue;
+    }
     // Grading is the last cancellation point — an LLM judge can run for
     // minutes per assertion. An aborted grade can't claim everything passed.
     if (abortSignal?.aborted) {
@@ -247,12 +265,13 @@ export const gradeRun = async (
   }
 
   let caseScore: number | null = null;
-  if (!gradingIncomplete) caseScore = weightSum > 0 ? weightedScoreSum / weightSum : 0;
+  if (!gradingIncomplete && weightSum > 0) caseScore = weightedScoreSum / weightSum;
   return {
     schemaVersion: 1,
-    casePassed: gradingIncomplete ? null : allPassed,
+    casePassed: gradingIncomplete || weightSum === 0 ? null : allPassed,
     caseScore,
     assertions: entries,
+    ...(skippedAssertions.length > 0 ? { skippedAssertions } : {}),
     ...(gradingIncomplete
       ? { incomplete: { exitReason: "error" as const, phase: "grading" as const } }
       : {}),

package/src/orchestrator/index.ts

@@ -122,6 +122,7 @@ export {
   formatRunsTable,
   type ListedRun,
   listRuns,
+  loadRunArtifacts,
   loadRunDetail,
   type RunDetail,
 } from "./list-runs.js";

package/src/orchestrator/list-runs.ts

@@ -2,6 +2,9 @@ import { existsSync } from "node:fs";
 import { constants, lstat, open, readdir } from "node:fs/promises";
 import { basename, join, relative, sep } from "node:path";
 import {
+  ActionProgressPayloadSchema,
+  type Artifact,
+  ArtifactSchema,
   type GradingReport,
   GradingReportSchema,
   harnessUsage,
@@ -31,6 +34,10 @@ export type ListedRun = {
   startedAt: string | null;
   endedAt: string | null;
   durationMs: number | null;
+  timeToFirstActionMs: number | null;
+  timeToFirstMutationMs: number | null;
+  timeToFirstDurableMutationMs: number | null;
+  driverMetrics: NonNullable<RunMetadata["driverMetrics"]>;
   casePassed: boolean | null;
   caseScore: number | null;
   assertionPassed: number;
@@ -63,6 +70,10 @@ const summarizeRun = async (
     startedAt,
     endedAt,
     durationMs,
+    timeToFirstActionMs: meta?.actionTiming?.timeToFirstActionMs ?? null,
+    timeToFirstMutationMs: meta?.actionTiming?.timeToFirstMutationMs ?? null,
+    timeToFirstDurableMutationMs: meta?.actionTiming?.timeToFirstDurableMutationMs ?? null,
+    driverMetrics: meta?.driverMetrics ?? {},
     casePassed: grading?.casePassed ?? null,
     caseScore: grading?.caseScore ?? null,
     assertionPassed: passing,
@@ -156,6 +167,9 @@ export type RunDetail = {
   artifacts: string[];
 };
 
+export const loadRunArtifacts = async (runDir: string): Promise<Artifact[]> =>
+  (await safeReadJson(join(runDir, "artifacts.json"), ArtifactSchema.array())) ?? [];
+
 const readEventsJsonl = async (path: string): Promise<RunEvent[]> => {
   if (!existsSync(path)) return [];
   if ((await lstat(path)).isSymbolicLink()) {
@@ -291,6 +305,13 @@ const appendMetadataDetail = (lines: string[], detail: RunDetail): void => {
         `phases: agent=${secs(phases.agent)} collect=${secs(phases.collect)} grading=${secs(phases.grading)}`,
       );
     }
+    const actions = detail.metadata.actionTiming;
+    if (actions) {
+      const value = (ms: number | undefined): string => (ms === undefined ? "—" : `${ms}ms`);
+      lines.push(
+        `first action: any=${value(actions.timeToFirstActionMs)} mutation=${value(actions.timeToFirstMutationMs)} durable-mutation=${value(actions.timeToFirstDurableMutationMs)}`,
+      );
+    }
   }
 };
 
@@ -362,6 +383,10 @@ const formatEventLine = (event: RunEvent): string => {
       }
       return `responded(custom): ${JSON.stringify(event.payload).slice(0, 80)}`;
     case "progress":
+      if (event.kind === "action") {
+        const payload = ActionProgressPayloadSchema.safeParse(event.payload).data;
+        return payload ? `action: ${payload.name} (${payload.category})` : "action: malformed";
+      }
       if (event.kind === "tool-call") {
         const payload = ToolCallPayloadSchema.safeParse(event.payload).data;
         return payload ? `tool-call: ${payload.name}` : "tool-call: malformed";

package/src/orchestrator/orchestrator.ts

@@ -128,6 +128,8 @@ export type RunRequest = {
   /** Root that relative `fixtures` dir paths resolve against. */
   fixturesRoot?: string | undefined;
   answererOverride?: string | undefined;
+  /** Skip built-in rubric judge assertions without changing subject execution. */
+  skipJudge?: boolean | undefined;
   startedAt?: Date | undefined;
   /** The `runSuite` invocation this case attempt belongs to. */
   suiteId?: string | undefined;
@@ -568,6 +570,7 @@ export class Orchestrator {
         assertions: assertionImplementations,
         validatedAssertionsHash,
         graderContext: this.harness,
+        skipJudge: request.skipJudge ?? false,
       });
       return await this.runWithPlugins(
         {
@@ -593,6 +596,7 @@ export class Orchestrator {
             assertions: assertionImplementations,
             lifecycleFixtures,
             validatedAssertionsHash,
+            skipJudge: request.skipJudge ?? false,
           }),
           fixtureValues: valuesForFixtures(lifecycleFixtures),
           driver,
@@ -617,6 +621,7 @@ export class Orchestrator {
       runsRoot: string;
       fixturesRoot?: string | undefined;
       answererOverride?: string | undefined;
+      skipJudge?: boolean | undefined;
       observer?: SuiteObserver | undefined;
       /** Aborts the cases in flight and stops the pool pulling new ones. */
       abortSignal?: AbortSignal | undefined;
@@ -788,6 +793,7 @@ export class Orchestrator {
             runsRoot: opts.runsRoot,
             fixturesRoot: opts.fixturesRoot,
             answererOverride: opts.answererOverride,
+            skipJudge: opts.skipJudge,
             observer: handle ?? observer,
             abortSignal: opts.abortSignal,
             suiteId: suite.manifest.suiteId,

package/src/orchestrator/run-execution.ts

@@ -10,6 +10,7 @@ import {
   AnswererModelCallAccountingSchema,
   type AnswerHistoryEntry,
   type Artifact,
+  actionTiming,
   type EvalCase,
   emptyUsage,
   type FixturesRef,
@@ -18,6 +19,7 @@ import {
   guardRunObserver,
   inspectRunArtifact,
   type LifecycleStage,
+  normalizeDriverMetrics,
   type OperationalFailure,
   type PhaseDurations,
   type ReadyState,
@@ -183,6 +185,11 @@ const modelAccountingFromSummary = (
 ): { modelCalls: number; unpricedModelCalls?: number | undefined } | null =>
   SummaryModelAccountingSchema.safeParse(summary).data ?? null;
 
+const metricsFromSummary = (summary: unknown) => {
+  if (!summary || typeof summary !== "object") return {};
+  return normalizeDriverMetrics((summary as { metrics?: unknown }).metrics);
+};
+
 const artifactMeasurements = async (
   artifacts: readonly Artifact[],
   runDir: string,
@@ -390,6 +397,7 @@ class RunExecution implements RunLifecycleEffects<RunResult> {
     metadata.exitReason = "error";
     metadata.operationalFailure = { phase: operationalPhase, reason };
     metadata.phaseDurationMs = phaseDurations(this.runDir.recorded);
+    metadata.actionTiming = actionTiming(this.runDir.recorded);
     await this.runDir.writeJson("run.json", metadata);
     await this.args.request.recordPreGradingAccounting?.(metadata);
     await this.args.request.recordModelAttemptsComplete?.(this.subjectAttemptIds);
@@ -407,6 +415,7 @@ class RunExecution implements RunLifecycleEffects<RunResult> {
     this.lifecycle("collect-end");
     await this.runDir.events.drain();
     const completed = await this.completedAgentRun();
+    completed.metadata.actionTiming = actionTiming(this.runDir.recorded);
     this.metadata = completed.metadata;
     this.usage = completed.usage;
     this.run = completed.run;
@@ -430,6 +439,7 @@ class RunExecution implements RunLifecycleEffects<RunResult> {
       grading.assertions.flatMap((assertion) => (assertion.usage ? [assertion.usage] : [])),
     );
     metadata.phaseDurationMs = phaseDurations(this.runDir.recorded);
+    metadata.actionTiming = actionTiming(this.runDir.recorded);
     await this.runDir.events.drain();
     await this.runDir.writeJson("run.json", metadata);
     await this.runDir.writeJson("artifacts.json", this.collection.artifacts);
@@ -560,9 +570,10 @@ class RunExecution implements RunLifecycleEffects<RunResult> {
     const outcome = this.require(this.loopOutcome, "driver outcome");
     const agentEndedAt = new Date();
     const exitReason = agentExitReason(outcome, this.collection.error);
-    const summary = outcome.kind === "done" ? outcome.done.summary : undefined;
+    const summary = outcome.kind === "done" ? outcome.done.summary : outcome.error.summary;
     const driverCost = costFromSummary(summary);
     const reportedModelAccounting = modelAccountingFromSummary(summary);
+    const driverMetrics = metricsFromSummary(summary);
     const answererAccounting = readAnswererModelAccounting(this.args.answerer);
     const agentCalls = this.ready ? (reportedModelAccounting?.modelCalls ?? 1) : 0;
     const operationalFailure = loopFailure(outcome, this.collection.error);
@@ -597,6 +608,7 @@ class RunExecution implements RunLifecycleEffects<RunResult> {
       ...(operationalFailure ? { operationalFailure } : {}),
       provenance: this.runProvenance(),
       ...(driverCost !== undefined ? { cost: { usd: driverCost } } : {}),
+      ...(Object.keys(driverMetrics).length > 0 ? { driverMetrics } : {}),
     };
     return {
       metadata,
@@ -657,6 +669,7 @@ class RunExecution implements RunLifecycleEffects<RunResult> {
       request.recordGradingAssertion,
       modelAttempts,
       this.args.validatedAssertionConfigs,
+      request.skipJudge ?? false,
     );
   }
 

package/src/orchestrator/suite-summary.ts

@@ -1,7 +1,13 @@
 import { rename, writeFile } from "node:fs/promises";
 import { join } from "node:path";
 import { z } from "zod";
-import { harnessUsage, isUsageEmpty, type RunMetadata } from "../core/index.js";
+import {
+  type DriverMetric,
+  DriverMetricUnitSchema,
+  harnessUsage,
+  isUsageEmpty,
+  type RunMetadata,
+} from "../core/index.js";
 import { runComparisonIdentity } from "./comparison-identity.js";
 import { safeReadJson } from "./fs-read.js";
 import type { SuiteCaseResult } from "./orchestrator.js";
@@ -16,6 +22,9 @@ export const MetricSummarySchema = z.object({
   mean: z.number(),
   median: z.number(),
   p90: z.number(),
+  unit: z.exactOptional(DriverMetricUnitSchema),
+  label: z.exactOptional(z.string()),
+  better: z.exactOptional(z.enum(["lower", "higher", "neutral"])),
 });
 export type MetricSummary = z.infer<typeof MetricSummarySchema>;
 
@@ -59,13 +68,22 @@ export const SuiteSummarySchema = z.object({
 });
 export type SuiteSummary = z.infer<typeof SuiteSummarySchema>;
 
-type MetricSamples = Map<string, number[]>;
+type MetricSamples = Map<
+  string,
+  { values: number[]; descriptor?: Omit<DriverMetric, "value"> | undefined }
+>;
 
-const add = (samples: MetricSamples, name: string, value: number | undefined): void => {
+const add = (
+  samples: MetricSamples,
+  name: string,
+  value: number | undefined,
+  descriptor?: Omit<DriverMetric, "value">,
+): void => {
   if (value === undefined || !Number.isFinite(value)) return;
-  const values = samples.get(name) ?? [];
-  values.push(value);
-  samples.set(name, values);
+  const sample = samples.get(name) ?? { values: [], descriptor };
+  if (!sample.descriptor && descriptor) sample.descriptor = descriptor;
+  sample.values.push(value);
+  samples.set(name, sample);
 };
 
 const cacheHitRate = (usage: {
@@ -137,6 +155,13 @@ const collectMetadata = (samples: MetricSamples, metadata: RunMetadata): void =>
   add(samples, "phase.agentMs", metadata.phaseDurationMs?.agent);
   add(samples, "phase.collectMs", metadata.phaseDurationMs?.collect);
   add(samples, "phase.gradingMs", metadata.phaseDurationMs?.grading);
+  add(samples, "timeToFirstActionMs", metadata.actionTiming?.timeToFirstActionMs);
+  add(samples, "timeToFirstMutationMs", metadata.actionTiming?.timeToFirstMutationMs);
+  add(samples, "timeToFirstDurableMutationMs", metadata.actionTiming?.timeToFirstDurableMutationMs);
+  for (const [name, metric] of Object.entries(metadata.driverMetrics ?? {})) {
+    const { value, ...descriptor } = metric;
+    add(samples, `driver.${name}`, value, descriptor);
+  }
 };
 
 const percentile = (sorted: readonly number[], fraction: number): number => {
@@ -165,7 +190,10 @@ const summarizeMetrics = (samples: MetricSamples): Record<string, MetricSummary>
   Object.fromEntries(
     [...samples.entries()]
       .sort(([a], [b]) => a.localeCompare(b))
-      .map(([name, values]) => [name, summarizeNumbers(values)]),
+      .map(([name, { values, descriptor }]) => [
+        name,
+        MetricSummarySchema.parse({ ...summarizeNumbers(values), ...descriptor }),
+      ]),
   );
 
 const summarizeOutcomes = (
@@ -258,7 +286,7 @@ export const buildSuiteSummary = (
           ? { score: result.grading.caseScore }
           : {}),
         metrics: Object.fromEntries(
-          [...samples.entries()].map(([name, values]) => [name, values[0] ?? 0]),
+          [...samples.entries()].map(([name, sample]) => [name, sample.values[0] ?? 0]),
         ),
       };
     }),

package/src/previewer/artifacts.ts

@@ -0,0 +1,221 @@
+import type { Artifact } from "../core/index.js";
+import { inspectRunArtifact, readRunArtifactBytes } from "../core/index.js";
+
+const MAX_PREVIEW_BYTES = 512_000;
+const MAX_DIFF_LINES = 2_000;
+const MAX_DIFF_CELLS = 1_000_000;
+
+export type ArtifactPreview = {
+  path: string;
+  exists: boolean;
+  viewable: boolean;
+  size: number | null;
+  contentType: string | null;
+  text: string | null;
+  reason: string | null;
+};
+
+type ArtifactDiffOperation = {
+  kind: "same" | "added" | "removed";
+  lines: string[];
+};
+
+export type ArtifactComparison = {
+  path: string;
+  before: ArtifactPreview;
+  after: ArtifactPreview;
+  operations: ArtifactDiffOperation[] | null;
+  reason: string | null;
+};
+
+const evidenceArtifacts = (
+  indexed: readonly Artifact[],
+  knownPaths: readonly string[],
+): Artifact[] => {
+  const out = [...indexed];
+  const indexedPaths = new Set(indexed.map((artifact) => artifact.path));
+  for (const path of knownPaths) {
+    if (!indexedPaths.has(path)) out.push({ path });
+  }
+  return out;
+};
+
+export const loadArtifactPreview = async (
+  runDir: string,
+  indexed: readonly Artifact[],
+  knownPaths: readonly string[],
+  path: string,
+): Promise<ArtifactPreview> => {
+  if (!knownPaths.includes(path) && !indexed.some((artifact) => artifact.path === path)) {
+    return {
+      path,
+      exists: false,
+      viewable: false,
+      size: null,
+      contentType: null,
+      text: null,
+      reason: "artifact was not captured by this run",
+    };
+  }
+  const artifacts = evidenceArtifacts(indexed, knownPaths);
+  const run = { runDir, artifacts };
+  const inspected = await inspectRunArtifact(run, path);
+  const contentType = inspected.artifact?.contentType ?? null;
+  if (!inspected.stats?.isFile()) {
+    return {
+      path,
+      exists: false,
+      viewable: false,
+      size: inspected.stats?.size ?? null,
+      contentType,
+      text: null,
+      reason: inspected.issue ?? "artifact is not a regular file",
+    };
+  }
+  if (inspected.stats.size > MAX_PREVIEW_BYTES) {
+    return {
+      path,
+      exists: true,
+      viewable: false,
+      size: inspected.stats.size,
+      contentType,
+      text: null,
+      reason: `artifact exceeds the ${MAX_PREVIEW_BYTES}-byte preview limit`,
+    };
+  }
+  const bytes = await readRunArtifactBytes(run, path, { maxBytes: MAX_PREVIEW_BYTES });
+  let text: string;
+  try {
+    text = new TextDecoder("utf-8", { fatal: true }).decode(bytes);
+  } catch {
+    return {
+      path,
+      exists: true,
+      viewable: false,
+      size: bytes.length,
+      contentType,
+      text: null,
+      reason: "artifact is not valid UTF-8 text",
+    };
+  }
+  if (text.includes("\0")) {
+    return {
+      path,
+      exists: true,
+      viewable: false,
+      size: bytes.length,
+      contentType,
+      text: null,
+      reason: "artifact appears to be binary",
+    };
+  }
+  return {
+    path,
+    exists: true,
+    viewable: true,
+    size: bytes.length,
+    contentType,
+    text,
+    reason: null,
+  };
+};
+
+const appendOperation = (
+  operations: ArtifactDiffOperation[],
+  kind: ArtifactDiffOperation["kind"],
+  line: string,
+): void => {
+  const current = operations.at(-1);
+  if (current?.kind === kind) current.lines.push(line);
+  else operations.push({ kind, lines: [line] });
+};
+
+const diffLines = (beforeText: string, afterText: string): ArtifactDiffOperation[] | null => {
+  const before = beforeText.split("\n");
+  const after = afterText.split("\n");
+  if (
+    before.length > MAX_DIFF_LINES ||
+    after.length > MAX_DIFF_LINES ||
+    before.length * after.length > MAX_DIFF_CELLS
+  ) {
+    return null;
+  }
+  const width = after.length + 1;
+  const table = new Uint32Array((before.length + 1) * width);
+  for (let left = before.length - 1; left >= 0; left--) {
+    for (let right = after.length - 1; right >= 0; right--) {
+      const cell = left * width + right;
+      table[cell] =
+        before[left] === after[right]
+          ? (table[(left + 1) * width + right + 1] ?? 0) + 1
+          : Math.max(table[(left + 1) * width + right] ?? 0, table[left * width + right + 1] ?? 0);
+    }
+  }
+  const operations: ArtifactDiffOperation[] = [];
+  let left = 0;
+  let right = 0;
+  while (left < before.length && right < after.length) {
+    if (before[left] === after[right]) {
+      appendOperation(operations, "same", before[left] ?? "");
+      left++;
+      right++;
+    } else if ((table[(left + 1) * width + right] ?? 0) >= (table[left * width + right + 1] ?? 0)) {
+      appendOperation(operations, "removed", before[left] ?? "");
+      left++;
+    } else {
+      appendOperation(operations, "added", after[right] ?? "");
+      right++;
+    }
+  }
+  while (left < before.length) {
+    appendOperation(operations, "removed", before[left] ?? "");
+    left++;
+  }
+  while (right < after.length) {
+    appendOperation(operations, "added", after[right] ?? "");
+    right++;
+  }
+  return operations;
+};
+
+export const compareArtifactPreviews = (
+  path: string,
+  before: ArtifactPreview,
+  after: ArtifactPreview,
+): ArtifactComparison => {
+  if (before.viewable && before.text !== null && !after.exists) {
+    return {
+      path,
+      before,
+      after,
+      operations: [{ kind: "removed", lines: before.text.split("\n") }],
+      reason: null,
+    };
+  }
+  if (!before.exists && after.viewable && after.text !== null) {
+    return {
+      path,
+      before,
+      after,
+      operations: [{ kind: "added", lines: after.text.split("\n") }],
+      reason: null,
+    };
+  }
+  if (!before.viewable || !after.viewable || before.text === null || after.text === null) {
+    return {
+      path,
+      before,
+      after,
+      operations: null,
+      reason: before.reason ?? after.reason ?? "artifact text is unavailable",
+    };
+  }
+  const operations = diffLines(before.text, after.text);
+  return {
+    path,
+    before,
+    after,
+    operations,
+    reason: operations ? null : "artifact is too line-dense for an inline diff",
+  };
+};