@prismatic-io/lux 0.0.2-preview.4 → 0.0.2-preview.5

package/src/orchestrator/list-runs.ts

@@ -2,6 +2,9 @@ import { existsSync } from "node:fs";
 import { constants, lstat, open, readdir } from "node:fs/promises";
 import { basename, join, relative, sep } from "node:path";
 import {
+  ActionProgressPayloadSchema,
+  type Artifact,
+  ArtifactSchema,
   type GradingReport,
   GradingReportSchema,
   harnessUsage,
@@ -31,6 +34,10 @@ export type ListedRun = {
   startedAt: string | null;
   endedAt: string | null;
   durationMs: number | null;
+  timeToFirstActionMs: number | null;
+  timeToFirstMutationMs: number | null;
+  timeToFirstDurableMutationMs: number | null;
+  driverMetrics: NonNullable<RunMetadata["driverMetrics"]>;
   casePassed: boolean | null;
   caseScore: number | null;
   assertionPassed: number;
@@ -63,6 +70,10 @@ const summarizeRun = async (
     startedAt,
     endedAt,
     durationMs,
+    timeToFirstActionMs: meta?.actionTiming?.timeToFirstActionMs ?? null,
+    timeToFirstMutationMs: meta?.actionTiming?.timeToFirstMutationMs ?? null,
+    timeToFirstDurableMutationMs: meta?.actionTiming?.timeToFirstDurableMutationMs ?? null,
+    driverMetrics: meta?.driverMetrics ?? {},
     casePassed: grading?.casePassed ?? null,
     caseScore: grading?.caseScore ?? null,
     assertionPassed: passing,
@@ -156,6 +167,9 @@ export type RunDetail = {
   artifacts: string[];
 };
 
+export const loadRunArtifacts = async (runDir: string): Promise<Artifact[]> =>
+  (await safeReadJson(join(runDir, "artifacts.json"), ArtifactSchema.array())) ?? [];
+
 const readEventsJsonl = async (path: string): Promise<RunEvent[]> => {
   if (!existsSync(path)) return [];
   if ((await lstat(path)).isSymbolicLink()) {
@@ -291,6 +305,13 @@ const appendMetadataDetail = (lines: string[], detail: RunDetail): void => {
         `phases: agent=${secs(phases.agent)} collect=${secs(phases.collect)} grading=${secs(phases.grading)}`,
       );
     }
+    const actions = detail.metadata.actionTiming;
+    if (actions) {
+      const value = (ms: number | undefined): string => (ms === undefined ? "—" : `${ms}ms`);
+      lines.push(
+        `first action: any=${value(actions.timeToFirstActionMs)} mutation=${value(actions.timeToFirstMutationMs)} durable-mutation=${value(actions.timeToFirstDurableMutationMs)}`,
+      );
+    }
   }
 };
 
@@ -362,6 +383,10 @@ const formatEventLine = (event: RunEvent): string => {
       }
       return `responded(custom): ${JSON.stringify(event.payload).slice(0, 80)}`;
     case "progress":
+      if (event.kind === "action") {
+        const payload = ActionProgressPayloadSchema.safeParse(event.payload).data;
+        return payload ? `action: ${payload.name} (${payload.category})` : "action: malformed";
+      }
       if (event.kind === "tool-call") {
         const payload = ToolCallPayloadSchema.safeParse(event.payload).data;
         return payload ? `tool-call: ${payload.name}` : "tool-call: malformed";

package/src/orchestrator/run-execution.ts

@@ -10,6 +10,7 @@ import {
   AnswererModelCallAccountingSchema,
   type AnswerHistoryEntry,
   type Artifact,
+  actionTiming,
   type EvalCase,
   emptyUsage,
   type FixturesRef,
@@ -18,6 +19,7 @@ import {
   guardRunObserver,
   inspectRunArtifact,
   type LifecycleStage,
+  normalizeDriverMetrics,
   type OperationalFailure,
   type PhaseDurations,
   type ReadyState,
@@ -183,6 +185,11 @@ const modelAccountingFromSummary = (
 ): { modelCalls: number; unpricedModelCalls?: number | undefined } | null =>
   SummaryModelAccountingSchema.safeParse(summary).data ?? null;
 
+const metricsFromSummary = (summary: unknown) => {
+  if (!summary || typeof summary !== "object") return {};
+  return normalizeDriverMetrics((summary as { metrics?: unknown }).metrics);
+};
+
 const artifactMeasurements = async (
   artifacts: readonly Artifact[],
   runDir: string,
@@ -390,6 +397,7 @@ class RunExecution implements RunLifecycleEffects<RunResult> {
     metadata.exitReason = "error";
     metadata.operationalFailure = { phase: operationalPhase, reason };
     metadata.phaseDurationMs = phaseDurations(this.runDir.recorded);
+    metadata.actionTiming = actionTiming(this.runDir.recorded);
     await this.runDir.writeJson("run.json", metadata);
     await this.args.request.recordPreGradingAccounting?.(metadata);
     await this.args.request.recordModelAttemptsComplete?.(this.subjectAttemptIds);
@@ -407,6 +415,7 @@ class RunExecution implements RunLifecycleEffects<RunResult> {
     this.lifecycle("collect-end");
     await this.runDir.events.drain();
     const completed = await this.completedAgentRun();
+    completed.metadata.actionTiming = actionTiming(this.runDir.recorded);
     this.metadata = completed.metadata;
     this.usage = completed.usage;
     this.run = completed.run;
@@ -430,6 +439,7 @@ class RunExecution implements RunLifecycleEffects<RunResult> {
       grading.assertions.flatMap((assertion) => (assertion.usage ? [assertion.usage] : [])),
     );
     metadata.phaseDurationMs = phaseDurations(this.runDir.recorded);
+    metadata.actionTiming = actionTiming(this.runDir.recorded);
     await this.runDir.events.drain();
     await this.runDir.writeJson("run.json", metadata);
     await this.runDir.writeJson("artifacts.json", this.collection.artifacts);
@@ -563,6 +573,7 @@ class RunExecution implements RunLifecycleEffects<RunResult> {
     const summary = outcome.kind === "done" ? outcome.done.summary : outcome.error.summary;
     const driverCost = costFromSummary(summary);
     const reportedModelAccounting = modelAccountingFromSummary(summary);
+    const driverMetrics = metricsFromSummary(summary);
     const answererAccounting = readAnswererModelAccounting(this.args.answerer);
     const agentCalls = this.ready ? (reportedModelAccounting?.modelCalls ?? 1) : 0;
     const operationalFailure = loopFailure(outcome, this.collection.error);
@@ -597,6 +608,7 @@ class RunExecution implements RunLifecycleEffects<RunResult> {
       ...(operationalFailure ? { operationalFailure } : {}),
       provenance: this.runProvenance(),
       ...(driverCost !== undefined ? { cost: { usd: driverCost } } : {}),
+      ...(Object.keys(driverMetrics).length > 0 ? { driverMetrics } : {}),
     };
     return {
       metadata,

package/src/orchestrator/suite-summary.ts

@@ -1,7 +1,13 @@
 import { rename, writeFile } from "node:fs/promises";
 import { join } from "node:path";
 import { z } from "zod";
-import { harnessUsage, isUsageEmpty, type RunMetadata } from "../core/index.js";
+import {
+  type DriverMetric,
+  DriverMetricUnitSchema,
+  harnessUsage,
+  isUsageEmpty,
+  type RunMetadata,
+} from "../core/index.js";
 import { runComparisonIdentity } from "./comparison-identity.js";
 import { safeReadJson } from "./fs-read.js";
 import type { SuiteCaseResult } from "./orchestrator.js";
@@ -16,6 +22,9 @@ export const MetricSummarySchema = z.object({
   mean: z.number(),
   median: z.number(),
   p90: z.number(),
+  unit: z.exactOptional(DriverMetricUnitSchema),
+  label: z.exactOptional(z.string()),
+  better: z.exactOptional(z.enum(["lower", "higher", "neutral"])),
 });
 export type MetricSummary = z.infer<typeof MetricSummarySchema>;
 
@@ -59,13 +68,22 @@ export const SuiteSummarySchema = z.object({
 });
 export type SuiteSummary = z.infer<typeof SuiteSummarySchema>;
 
-type MetricSamples = Map<string, number[]>;
+type MetricSamples = Map<
+  string,
+  { values: number[]; descriptor?: Omit<DriverMetric, "value"> | undefined }
+>;
 
-const add = (samples: MetricSamples, name: string, value: number | undefined): void => {
+const add = (
+  samples: MetricSamples,
+  name: string,
+  value: number | undefined,
+  descriptor?: Omit<DriverMetric, "value">,
+): void => {
   if (value === undefined || !Number.isFinite(value)) return;
-  const values = samples.get(name) ?? [];
-  values.push(value);
-  samples.set(name, values);
+  const sample = samples.get(name) ?? { values: [], descriptor };
+  if (!sample.descriptor && descriptor) sample.descriptor = descriptor;
+  sample.values.push(value);
+  samples.set(name, sample);
 };
 
 const cacheHitRate = (usage: {
@@ -137,6 +155,13 @@ const collectMetadata = (samples: MetricSamples, metadata: RunMetadata): void =>
   add(samples, "phase.agentMs", metadata.phaseDurationMs?.agent);
   add(samples, "phase.collectMs", metadata.phaseDurationMs?.collect);
   add(samples, "phase.gradingMs", metadata.phaseDurationMs?.grading);
+  add(samples, "timeToFirstActionMs", metadata.actionTiming?.timeToFirstActionMs);
+  add(samples, "timeToFirstMutationMs", metadata.actionTiming?.timeToFirstMutationMs);
+  add(samples, "timeToFirstDurableMutationMs", metadata.actionTiming?.timeToFirstDurableMutationMs);
+  for (const [name, metric] of Object.entries(metadata.driverMetrics ?? {})) {
+    const { value, ...descriptor } = metric;
+    add(samples, `driver.${name}`, value, descriptor);
+  }
 };
 
 const percentile = (sorted: readonly number[], fraction: number): number => {
@@ -165,7 +190,10 @@ const summarizeMetrics = (samples: MetricSamples): Record<string, MetricSummary>
   Object.fromEntries(
     [...samples.entries()]
       .sort(([a], [b]) => a.localeCompare(b))
-      .map(([name, values]) => [name, summarizeNumbers(values)]),
+      .map(([name, { values, descriptor }]) => [
+        name,
+        MetricSummarySchema.parse({ ...summarizeNumbers(values), ...descriptor }),
+      ]),
   );
 
 const summarizeOutcomes = (
@@ -258,7 +286,7 @@ export const buildSuiteSummary = (
           ? { score: result.grading.caseScore }
           : {}),
         metrics: Object.fromEntries(
-          [...samples.entries()].map(([name, values]) => [name, values[0] ?? 0]),
+          [...samples.entries()].map(([name, sample]) => [name, sample.values[0] ?? 0]),
         ),
       };
     }),

package/src/previewer/artifacts.ts

@@ -0,0 +1,221 @@
+import type { Artifact } from "../core/index.js";
+import { inspectRunArtifact, readRunArtifactBytes } from "../core/index.js";
+
+const MAX_PREVIEW_BYTES = 512_000;
+const MAX_DIFF_LINES = 2_000;
+const MAX_DIFF_CELLS = 1_000_000;
+
+export type ArtifactPreview = {
+  path: string;
+  exists: boolean;
+  viewable: boolean;
+  size: number | null;
+  contentType: string | null;
+  text: string | null;
+  reason: string | null;
+};
+
+type ArtifactDiffOperation = {
+  kind: "same" | "added" | "removed";
+  lines: string[];
+};
+
+export type ArtifactComparison = {
+  path: string;
+  before: ArtifactPreview;
+  after: ArtifactPreview;
+  operations: ArtifactDiffOperation[] | null;
+  reason: string | null;
+};
+
+const evidenceArtifacts = (
+  indexed: readonly Artifact[],
+  knownPaths: readonly string[],
+): Artifact[] => {
+  const out = [...indexed];
+  const indexedPaths = new Set(indexed.map((artifact) => artifact.path));
+  for (const path of knownPaths) {
+    if (!indexedPaths.has(path)) out.push({ path });
+  }
+  return out;
+};
+
+export const loadArtifactPreview = async (
+  runDir: string,
+  indexed: readonly Artifact[],
+  knownPaths: readonly string[],
+  path: string,
+): Promise<ArtifactPreview> => {
+  if (!knownPaths.includes(path) && !indexed.some((artifact) => artifact.path === path)) {
+    return {
+      path,
+      exists: false,
+      viewable: false,
+      size: null,
+      contentType: null,
+      text: null,
+      reason: "artifact was not captured by this run",
+    };
+  }
+  const artifacts = evidenceArtifacts(indexed, knownPaths);
+  const run = { runDir, artifacts };
+  const inspected = await inspectRunArtifact(run, path);
+  const contentType = inspected.artifact?.contentType ?? null;
+  if (!inspected.stats?.isFile()) {
+    return {
+      path,
+      exists: false,
+      viewable: false,
+      size: inspected.stats?.size ?? null,
+      contentType,
+      text: null,
+      reason: inspected.issue ?? "artifact is not a regular file",
+    };
+  }
+  if (inspected.stats.size > MAX_PREVIEW_BYTES) {
+    return {
+      path,
+      exists: true,
+      viewable: false,
+      size: inspected.stats.size,
+      contentType,
+      text: null,
+      reason: `artifact exceeds the ${MAX_PREVIEW_BYTES}-byte preview limit`,
+    };
+  }
+  const bytes = await readRunArtifactBytes(run, path, { maxBytes: MAX_PREVIEW_BYTES });
+  let text: string;
+  try {
+    text = new TextDecoder("utf-8", { fatal: true }).decode(bytes);
+  } catch {
+    return {
+      path,
+      exists: true,
+      viewable: false,
+      size: bytes.length,
+      contentType,
+      text: null,
+      reason: "artifact is not valid UTF-8 text",
+    };
+  }
+  if (text.includes("\0")) {
+    return {
+      path,
+      exists: true,
+      viewable: false,
+      size: bytes.length,
+      contentType,
+      text: null,
+      reason: "artifact appears to be binary",
+    };
+  }
+  return {
+    path,
+    exists: true,
+    viewable: true,
+    size: bytes.length,
+    contentType,
+    text,
+    reason: null,
+  };
+};
+
+const appendOperation = (
+  operations: ArtifactDiffOperation[],
+  kind: ArtifactDiffOperation["kind"],
+  line: string,
+): void => {
+  const current = operations.at(-1);
+  if (current?.kind === kind) current.lines.push(line);
+  else operations.push({ kind, lines: [line] });
+};
+
+const diffLines = (beforeText: string, afterText: string): ArtifactDiffOperation[] | null => {
+  const before = beforeText.split("\n");
+  const after = afterText.split("\n");
+  if (
+    before.length > MAX_DIFF_LINES ||
+    after.length > MAX_DIFF_LINES ||
+    before.length * after.length > MAX_DIFF_CELLS
+  ) {
+    return null;
+  }
+  const width = after.length + 1;
+  const table = new Uint32Array((before.length + 1) * width);
+  for (let left = before.length - 1; left >= 0; left--) {
+    for (let right = after.length - 1; right >= 0; right--) {
+      const cell = left * width + right;
+      table[cell] =
+        before[left] === after[right]
+          ? (table[(left + 1) * width + right + 1] ?? 0) + 1
+          : Math.max(table[(left + 1) * width + right] ?? 0, table[left * width + right + 1] ?? 0);
+    }
+  }
+  const operations: ArtifactDiffOperation[] = [];
+  let left = 0;
+  let right = 0;
+  while (left < before.length && right < after.length) {
+    if (before[left] === after[right]) {
+      appendOperation(operations, "same", before[left] ?? "");
+      left++;
+      right++;
+    } else if ((table[(left + 1) * width + right] ?? 0) >= (table[left * width + right + 1] ?? 0)) {
+      appendOperation(operations, "removed", before[left] ?? "");
+      left++;
+    } else {
+      appendOperation(operations, "added", after[right] ?? "");
+      right++;
+    }
+  }
+  while (left < before.length) {
+    appendOperation(operations, "removed", before[left] ?? "");
+    left++;
+  }
+  while (right < after.length) {
+    appendOperation(operations, "added", after[right] ?? "");
+    right++;
+  }
+  return operations;
+};
+
+export const compareArtifactPreviews = (
+  path: string,
+  before: ArtifactPreview,
+  after: ArtifactPreview,
+): ArtifactComparison => {
+  if (before.viewable && before.text !== null && !after.exists) {
+    return {
+      path,
+      before,
+      after,
+      operations: [{ kind: "removed", lines: before.text.split("\n") }],
+      reason: null,
+    };
+  }
+  if (!before.exists && after.viewable && after.text !== null) {
+    return {
+      path,
+      before,
+      after,
+      operations: [{ kind: "added", lines: after.text.split("\n") }],
+      reason: null,
+    };
+  }
+  if (!before.viewable || !after.viewable || before.text === null || after.text === null) {
+    return {
+      path,
+      before,
+      after,
+      operations: null,
+      reason: before.reason ?? after.reason ?? "artifact text is unavailable",
+    };
+  }
+  const operations = diffLines(before.text, after.text);
+  return {
+    path,
+    before,
+    after,
+    operations,
+    reason: operations ? null : "artifact is too line-dense for an inline diff",
+  };
+};