@prismatic-io/lux 0.0.1 → 0.0.2-preview.10

package/src/orchestrator/regrade.ts

@@ -33,6 +33,7 @@ import {
   readRunArtifactBytes,
   valueHash,
 } from "../core/index.js";
+import { registryForEvalCase } from "./case-extensions.js";
 import { evaluatorImplementationHash } from "./evaluator-identity.js";
 import { parsePersistedJson, readBoundedHandle, safeReadJson } from "./fs-read.js";
 import {
@@ -526,18 +527,18 @@ export const regradeRun = async (options: {
   now?: Date;
 }): Promise<{ path: string; snapshot: GradingSnapshot }> => {
   const run = await loadFrozenRegradableRun(options.runDir, options.case);
+  const registry = options.case
+    ? registryForEvalCase(options.registry, options.case)
+    : options.registry;
   try {
     const evidenceHash = await gradingEvidenceHash(run);
     const validatedAssertionConfigs = await validateAssertionPlan(
       run.case.assertions ?? [],
-      options.registry,
+      registry,
     );
     const validatedAssertionsHash = valueHash(validatedAssertionConfigs);
-    const assertionEvaluators = assertionEvaluatorIdentities(
-      run.case.assertions ?? [],
-      options.registry,
-    );
-    const assertionPlugins = assertionPluginIdentities(run.case.assertions ?? [], options.registry);
+    const assertionEvaluators = assertionEvaluatorIdentities(run.case.assertions ?? [], registry);
+    const assertionPlugins = assertionPluginIdentities(run.case.assertions ?? [], registry);
     const graderHash = graderConfigHash({
       luxVersion: LUX_VERSION,
       implementationHash: await currentEvaluatorImplementationHash(),
@@ -547,7 +548,7 @@ export const regradeRun = async (options: {
     });
     const report: GradingReport = await gradeRun(
       run,
-      options.registry,
+      registry,
       undefined,
       options.abortSignal,
       undefined,

package/src/orchestrator/run-execution.ts

@@ -10,13 +10,16 @@ import {
   AnswererModelCallAccountingSchema,
   type AnswerHistoryEntry,
   type Artifact,
+  actionTiming,
   type EvalCase,
   emptyUsage,
   type FixturesRef,
+  type FixtureValueBinding,
   type GradingReport,
   guardRunObserver,
   inspectRunArtifact,
   type LifecycleStage,
+  normalizeDriverMetrics,
   type OperationalFailure,
   type PhaseDurations,
   type ReadyState,
@@ -27,6 +30,7 @@ import {
   type RunMetadata,
   type RunProvenance,
   type RunUsage,
+  summarizeAgentActivity,
   summarizeToolOutcomes,
   sumUsage,
   type TokenUsage,
@@ -70,6 +74,7 @@ export type RunWithPluginsArgs = {
   validatedAssertionsHash: string;
   graderConfigHash: string;
   evaluatorConfigHash: string;
+  fixtureValues: readonly FixtureValueBinding[];
   luxVersion: string;
   registry: Registry;
   driver: AgentDriver;
@@ -181,6 +186,11 @@ const modelAccountingFromSummary = (
 ): { modelCalls: number; unpricedModelCalls?: number | undefined } | null =>
   SummaryModelAccountingSchema.safeParse(summary).data ?? null;
 
+const metricsFromSummary = (summary: unknown) => {
+  if (!summary || typeof summary !== "object") return {};
+  return normalizeDriverMetrics((summary as { metrics?: unknown }).metrics);
+};
+
 const artifactMeasurements = async (
   artifacts: readonly Artifact[],
   runDir: string,
@@ -360,6 +370,7 @@ class RunExecution implements RunLifecycleEffects<RunResult> {
       caseId: this.args.caseId,
       prompt: evalCase.prompt,
       fixtures: staged?.fixtures,
+      ...(this.args.fixtureValues.length > 0 ? { fixtureValues: this.args.fixtureValues } : {}),
       runsRoot: resolve(request.runsRoot),
       runDir: this.runDir.path,
       artifactsDir: this.runDir.artifactsDir,
@@ -387,6 +398,7 @@ class RunExecution implements RunLifecycleEffects<RunResult> {
     metadata.exitReason = "error";
     metadata.operationalFailure = { phase: operationalPhase, reason };
     metadata.phaseDurationMs = phaseDurations(this.runDir.recorded);
+    metadata.actionTiming = actionTiming(this.runDir.recorded);
     await this.runDir.writeJson("run.json", metadata);
     await this.args.request.recordPreGradingAccounting?.(metadata);
     await this.args.request.recordModelAttemptsComplete?.(this.subjectAttemptIds);
@@ -404,6 +416,7 @@ class RunExecution implements RunLifecycleEffects<RunResult> {
     this.lifecycle("collect-end");
     await this.runDir.events.drain();
     const completed = await this.completedAgentRun();
+    completed.metadata.actionTiming = actionTiming(this.runDir.recorded);
     this.metadata = completed.metadata;
     this.usage = completed.usage;
     this.run = completed.run;
@@ -427,6 +440,7 @@ class RunExecution implements RunLifecycleEffects<RunResult> {
       grading.assertions.flatMap((assertion) => (assertion.usage ? [assertion.usage] : [])),
     );
     metadata.phaseDurationMs = phaseDurations(this.runDir.recorded);
+    metadata.actionTiming = actionTiming(this.runDir.recorded);
     await this.runDir.events.drain();
     await this.runDir.writeJson("run.json", metadata);
     await this.runDir.writeJson("artifacts.json", this.collection.artifacts);
@@ -503,6 +517,7 @@ class RunExecution implements RunLifecycleEffects<RunResult> {
   private fallbackFailureMetadata(): RunMetadata {
     const failedAt = new Date();
     const answererAccounting = readAnswererModelAccounting(this.args.answerer);
+    const ready = this.ready ?? { agentId: "start-failed" };
     return {
       schemaVersion: 1,
       luxVersion: this.args.luxVersion,
@@ -511,7 +526,7 @@ class RunExecution implements RunLifecycleEffects<RunResult> {
       agentEndedAt: failedAt.toISOString(),
       endedAt: failedAt.toISOString(),
       exitReason: "error",
-      driver: { name: this.args.driverName, ...(this.ready ?? { agentId: "start-failed" }) },
+      driver: { name: this.args.driverName, ...ready },
       answerer: answererMetadata(this.args.answererName, this.args.answererConfig),
       usage: {
         agent: emptyUsage(),
@@ -527,6 +542,7 @@ class RunExecution implements RunLifecycleEffects<RunResult> {
       interruptCounts: this.interruptCounts,
       toolCallCount: this.toolCallCount,
       toolOutcomes: summarizeToolOutcomes(this.runDir.recorded),
+      agentActivity: summarizeAgentActivity(this.runDir.recorded, ready),
       artifacts: { count: this.collection.artifacts.length },
       provenance: this.runProvenance(),
     };
@@ -557,9 +573,10 @@ class RunExecution implements RunLifecycleEffects<RunResult> {
     const outcome = this.require(this.loopOutcome, "driver outcome");
     const agentEndedAt = new Date();
     const exitReason = agentExitReason(outcome, this.collection.error);
-    const summary = outcome.kind === "done" ? outcome.done.summary : undefined;
+    const summary = outcome.kind === "done" ? outcome.done.summary : outcome.error.summary;
     const driverCost = costFromSummary(summary);
     const reportedModelAccounting = modelAccountingFromSummary(summary);
+    const driverMetrics = metricsFromSummary(summary);
     const answererAccounting = readAnswererModelAccounting(this.args.answerer);
     const agentCalls = this.ready ? (reportedModelAccounting?.modelCalls ?? 1) : 0;
     const operationalFailure = loopFailure(outcome, this.collection.error);
@@ -568,6 +585,7 @@ class RunExecution implements RunLifecycleEffects<RunResult> {
       answerer: readAnswererUsage(this.args.answerer),
       grading: emptyUsage(),
     };
+    const ready = this.ready ?? { agentId: "not-started" };
     const metadata: RunMetadata = {
       schemaVersion: 1,
       luxVersion: this.args.luxVersion,
@@ -576,7 +594,7 @@ class RunExecution implements RunLifecycleEffects<RunResult> {
       agentEndedAt: agentEndedAt.toISOString(),
       endedAt: agentEndedAt.toISOString(),
       exitReason,
-      driver: { name: this.args.driverName, ...(this.ready ?? { agentId: "not-started" }) },
+      driver: { name: this.args.driverName, ...ready },
       answerer: answererMetadata(this.args.answererName, this.args.answererConfig),
       usage,
       modelAccounting: {
@@ -590,10 +608,12 @@ class RunExecution implements RunLifecycleEffects<RunResult> {
       interruptCounts: this.interruptCounts,
       toolCallCount: this.toolCallCount,
       toolOutcomes: summarizeToolOutcomes(this.runDir.recorded),
+      agentActivity: summarizeAgentActivity(this.runDir.recorded, ready),
       artifacts: await artifactMeasurements(this.collection.artifacts, this.runDir.path),
       ...(operationalFailure ? { operationalFailure } : {}),
       provenance: this.runProvenance(),
       ...(driverCost !== undefined ? { cost: { usd: driverCost } } : {}),
+      ...(Object.keys(driverMetrics).length > 0 ? { driverMetrics } : {}),
     };
     return {
       metadata,
@@ -654,6 +674,7 @@ class RunExecution implements RunLifecycleEffects<RunResult> {
       request.recordGradingAssertion,
       modelAttempts,
       this.args.validatedAssertionConfigs,
+      request.skipJudge ?? false,
     );
   }
 

package/src/orchestrator/suite-summary.ts

@@ -1,7 +1,13 @@
 import { rename, writeFile } from "node:fs/promises";
 import { join } from "node:path";
 import { z } from "zod";
-import { harnessUsage, isUsageEmpty, type RunMetadata } from "../core/index.js";
+import {
+  type DriverMetric,
+  DriverMetricUnitSchema,
+  harnessUsage,
+  isUsageEmpty,
+  type RunMetadata,
+} from "../core/index.js";
 import { runComparisonIdentity } from "./comparison-identity.js";
 import { safeReadJson } from "./fs-read.js";
 import type { SuiteCaseResult } from "./orchestrator.js";
@@ -16,6 +22,9 @@ export const MetricSummarySchema = z.object({
   mean: z.number(),
   median: z.number(),
   p90: z.number(),
+  unit: z.exactOptional(DriverMetricUnitSchema),
+  label: z.exactOptional(z.string()),
+  better: z.exactOptional(z.enum(["lower", "higher", "neutral"])),
 });
 export type MetricSummary = z.infer<typeof MetricSummarySchema>;
 
@@ -59,13 +68,22 @@ export const SuiteSummarySchema = z.object({
 });
 export type SuiteSummary = z.infer<typeof SuiteSummarySchema>;
 
-type MetricSamples = Map<string, number[]>;
+type MetricSamples = Map<
+  string,
+  { values: number[]; descriptor?: Omit<DriverMetric, "value"> | undefined }
+>;
 
-const add = (samples: MetricSamples, name: string, value: number | undefined): void => {
+const add = (
+  samples: MetricSamples,
+  name: string,
+  value: number | undefined,
+  descriptor?: Omit<DriverMetric, "value">,
+): void => {
   if (value === undefined || !Number.isFinite(value)) return;
-  const values = samples.get(name) ?? [];
-  values.push(value);
-  samples.set(name, values);
+  const sample = samples.get(name) ?? { values: [], descriptor };
+  if (!sample.descriptor && descriptor) sample.descriptor = descriptor;
+  sample.values.push(value);
+  samples.set(name, sample);
 };
 
 const cacheHitRate = (usage: {
@@ -137,6 +155,78 @@ const collectMetadata = (samples: MetricSamples, metadata: RunMetadata): void =>
   add(samples, "phase.agentMs", metadata.phaseDurationMs?.agent);
   add(samples, "phase.collectMs", metadata.phaseDurationMs?.collect);
   add(samples, "phase.gradingMs", metadata.phaseDurationMs?.grading);
+  add(samples, "timeToFirstActionMs", metadata.actionTiming?.timeToFirstActionMs);
+  add(samples, "timeToFirstMutationMs", metadata.actionTiming?.timeToFirstMutationMs);
+  add(samples, "timeToFirstDurableMutationMs", metadata.actionTiming?.timeToFirstDurableMutationMs);
+  const subagents = (metadata.agentActivity ?? []).filter(
+    (activity) => activity.agent.role === "subagent",
+  );
+  add(samples, "subagents.count", subagents.length, {
+    unit: "count",
+    label: "Subagents spawned",
+    better: "neutral",
+  });
+  add(
+    samples,
+    "subagents.events",
+    subagents.reduce((sum, activity) => sum + activity.eventCount, 0),
+    { unit: "count", label: "Subagent events", better: "neutral" },
+  );
+  add(
+    samples,
+    "subagents.toolCalls",
+    subagents.reduce((sum, activity) => sum + activity.toolCalls.total, 0),
+    { unit: "count", label: "Subagent tool calls", better: "neutral" },
+  );
+  add(
+    samples,
+    "subagents.toolCalls.failed",
+    subagents.reduce((sum, activity) => sum + activity.toolCalls.failed, 0),
+    { unit: "count", label: "Failed subagent tool calls", better: "lower" },
+  );
+  add(
+    samples,
+    "subagents.tokens",
+    subagents.reduce((sum, activity) => sum + activity.usage.total, 0),
+    { unit: "tokens", label: "Subagent tokens", better: "lower" },
+  );
+  add(
+    samples,
+    "subagents.actions",
+    subagents.reduce((sum, activity) => sum + activity.actions.total, 0),
+    { unit: "count", label: "Subagent actions", better: "neutral" },
+  );
+  add(
+    samples,
+    "subagents.mutations",
+    subagents.reduce((sum, activity) => sum + activity.actions.mutations, 0),
+    { unit: "count", label: "Subagent mutations", better: "neutral" },
+  );
+  add(
+    samples,
+    "subagents.totalDurationMs",
+    subagents.reduce((sum, activity) => sum + activity.durationMs, 0),
+    { unit: "milliseconds", label: "Total subagent time", better: "neutral" },
+  );
+  add(
+    samples,
+    "subagents.maxDurationMs",
+    subagents.length > 0 ? Math.max(...subagents.map((activity) => activity.durationMs)) : 0,
+    { unit: "milliseconds", label: "Longest subagent", better: "lower" },
+  );
+  const firstSubagentAction = subagents
+    .map((activity) => activity.actions.timeToFirstActionMs)
+    .filter((value): value is number => value !== undefined)
+    .sort((a, b) => a - b)[0];
+  add(samples, "subagents.timeToFirstActionMs", firstSubagentAction, {
+    unit: "milliseconds",
+    label: "First subagent action",
+    better: "lower",
+  });
+  for (const [name, metric] of Object.entries(metadata.driverMetrics ?? {})) {
+    const { value, ...descriptor } = metric;
+    add(samples, `driver.${name}`, value, descriptor);
+  }
 };
 
 const percentile = (sorted: readonly number[], fraction: number): number => {
@@ -165,7 +255,10 @@ const summarizeMetrics = (samples: MetricSamples): Record<string, MetricSummary>
   Object.fromEntries(
     [...samples.entries()]
       .sort(([a], [b]) => a.localeCompare(b))
-      .map(([name, values]) => [name, summarizeNumbers(values)]),
+      .map(([name, { values, descriptor }]) => [
+        name,
+        MetricSummarySchema.parse({ ...summarizeNumbers(values), ...descriptor }),
+      ]),
   );
 
 const summarizeOutcomes = (
@@ -258,7 +351,7 @@ export const buildSuiteSummary = (
           ? { score: result.grading.caseScore }
           : {}),
         metrics: Object.fromEntries(
-          [...samples.entries()].map(([name, values]) => [name, values[0] ?? 0]),
+          [...samples.entries()].map(([name, sample]) => [name, sample.values[0] ?? 0]),
         ),
       };
     }),

package/src/previewer/artifacts.ts

@@ -0,0 +1,221 @@
+import type { Artifact } from "../core/index.js";
+import { inspectRunArtifact, readRunArtifactBytes } from "../core/index.js";
+
+const MAX_PREVIEW_BYTES = 512_000;
+const MAX_DIFF_LINES = 2_000;
+const MAX_DIFF_CELLS = 1_000_000;
+
+export type ArtifactPreview = {
+  path: string;
+  exists: boolean;
+  viewable: boolean;
+  size: number | null;
+  contentType: string | null;
+  text: string | null;
+  reason: string | null;
+};
+
+type ArtifactDiffOperation = {
+  kind: "same" | "added" | "removed";
+  lines: string[];
+};
+
+export type ArtifactComparison = {
+  path: string;
+  before: ArtifactPreview;
+  after: ArtifactPreview;
+  operations: ArtifactDiffOperation[] | null;
+  reason: string | null;
+};
+
+const evidenceArtifacts = (
+  indexed: readonly Artifact[],
+  knownPaths: readonly string[],
+): Artifact[] => {
+  const out = [...indexed];
+  const indexedPaths = new Set(indexed.map((artifact) => artifact.path));
+  for (const path of knownPaths) {
+    if (!indexedPaths.has(path)) out.push({ path });
+  }
+  return out;
+};
+
+export const loadArtifactPreview = async (
+  runDir: string,
+  indexed: readonly Artifact[],
+  knownPaths: readonly string[],
+  path: string,
+): Promise<ArtifactPreview> => {
+  if (!knownPaths.includes(path) && !indexed.some((artifact) => artifact.path === path)) {
+    return {
+      path,
+      exists: false,
+      viewable: false,
+      size: null,
+      contentType: null,
+      text: null,
+      reason: "artifact was not captured by this run",
+    };
+  }
+  const artifacts = evidenceArtifacts(indexed, knownPaths);
+  const run = { runDir, artifacts };
+  const inspected = await inspectRunArtifact(run, path);
+  const contentType = inspected.artifact?.contentType ?? null;
+  if (!inspected.stats?.isFile()) {
+    return {
+      path,
+      exists: false,
+      viewable: false,
+      size: inspected.stats?.size ?? null,
+      contentType,
+      text: null,
+      reason: inspected.issue ?? "artifact is not a regular file",
+    };
+  }
+  if (inspected.stats.size > MAX_PREVIEW_BYTES) {
+    return {
+      path,
+      exists: true,
+      viewable: false,
+      size: inspected.stats.size,
+      contentType,
+      text: null,
+      reason: `artifact exceeds the ${MAX_PREVIEW_BYTES}-byte preview limit`,
+    };
+  }
+  const bytes = await readRunArtifactBytes(run, path, { maxBytes: MAX_PREVIEW_BYTES });
+  let text: string;
+  try {
+    text = new TextDecoder("utf-8", { fatal: true }).decode(bytes);
+  } catch {
+    return {
+      path,
+      exists: true,
+      viewable: false,
+      size: bytes.length,
+      contentType,
+      text: null,
+      reason: "artifact is not valid UTF-8 text",
+    };
+  }
+  if (text.includes("\0")) {
+    return {
+      path,
+      exists: true,
+      viewable: false,
+      size: bytes.length,
+      contentType,
+      text: null,
+      reason: "artifact appears to be binary",
+    };
+  }
+  return {
+    path,
+    exists: true,
+    viewable: true,
+    size: bytes.length,
+    contentType,
+    text,
+    reason: null,
+  };
+};
+
+const appendOperation = (
+  operations: ArtifactDiffOperation[],
+  kind: ArtifactDiffOperation["kind"],
+  line: string,
+): void => {
+  const current = operations.at(-1);
+  if (current?.kind === kind) current.lines.push(line);
+  else operations.push({ kind, lines: [line] });
+};
+
+const diffLines = (beforeText: string, afterText: string): ArtifactDiffOperation[] | null => {
+  const before = beforeText.split("\n");
+  const after = afterText.split("\n");
+  if (
+    before.length > MAX_DIFF_LINES ||
+    after.length > MAX_DIFF_LINES ||
+    before.length * after.length > MAX_DIFF_CELLS
+  ) {
+    return null;
+  }
+  const width = after.length + 1;
+  const table = new Uint32Array((before.length + 1) * width);
+  for (let left = before.length - 1; left >= 0; left--) {
+    for (let right = after.length - 1; right >= 0; right--) {
+      const cell = left * width + right;
+      table[cell] =
+        before[left] === after[right]
+          ? (table[(left + 1) * width + right + 1] ?? 0) + 1
+          : Math.max(table[(left + 1) * width + right] ?? 0, table[left * width + right + 1] ?? 0);
+    }
+  }
+  const operations: ArtifactDiffOperation[] = [];
+  let left = 0;
+  let right = 0;
+  while (left < before.length && right < after.length) {
+    if (before[left] === after[right]) {
+      appendOperation(operations, "same", before[left] ?? "");
+      left++;
+      right++;
+    } else if ((table[(left + 1) * width + right] ?? 0) >= (table[left * width + right + 1] ?? 0)) {
+      appendOperation(operations, "removed", before[left] ?? "");
+      left++;
+    } else {
+      appendOperation(operations, "added", after[right] ?? "");
+      right++;
+    }
+  }
+  while (left < before.length) {
+    appendOperation(operations, "removed", before[left] ?? "");
+    left++;
+  }
+  while (right < after.length) {
+    appendOperation(operations, "added", after[right] ?? "");
+    right++;
+  }
+  return operations;
+};
+
+export const compareArtifactPreviews = (
+  path: string,
+  before: ArtifactPreview,
+  after: ArtifactPreview,
+): ArtifactComparison => {
+  if (before.viewable && before.text !== null && !after.exists) {
+    return {
+      path,
+      before,
+      after,
+      operations: [{ kind: "removed", lines: before.text.split("\n") }],
+      reason: null,
+    };
+  }
+  if (!before.exists && after.viewable && after.text !== null) {
+    return {
+      path,
+      before,
+      after,
+      operations: [{ kind: "added", lines: after.text.split("\n") }],
+      reason: null,
+    };
+  }
+  if (!before.viewable || !after.viewable || before.text === null || after.text === null) {
+    return {
+      path,
+      before,
+      after,
+      operations: null,
+      reason: before.reason ?? after.reason ?? "artifact text is unavailable",
+    };
+  }
+  const operations = diffLines(before.text, after.text);
+  return {
+    path,
+    before,
+    after,
+    operations,
+    reason: operations ? null : "artifact is too line-dense for an inline diff",
+  };
+};