@prismatic-io/lux 0.0.2-preview.3 → 0.0.2-preview.32

package/src/orchestrator/suite-summary.ts

@@ -1,7 +1,14 @@
 import { rename, writeFile } from "node:fs/promises";
 import { join } from "node:path";
 import { z } from "zod";
-import { harnessUsage, isUsageEmpty, type RunMetadata } from "../core/index.js";
+import {
+  type DriverMetric,
+  DriverMetricUnitSchema,
+  harnessUsage,
+  isUsageEmpty,
+  type RunMetadata,
+} from "../core/index.js";
+import { hasAgentUsage } from "../core/usage.js";
 import { runComparisonIdentity } from "./comparison-identity.js";
 import { safeReadJson } from "./fs-read.js";
 import type { SuiteCaseResult } from "./orchestrator.js";
@@ -16,6 +23,9 @@ export const MetricSummarySchema = z.object({
   mean: z.number(),
   median: z.number(),
   p90: z.number(),
+  unit: z.exactOptional(DriverMetricUnitSchema),
+  label: z.exactOptional(z.string()),
+  better: z.exactOptional(z.enum(["lower", "higher", "neutral"])),
 });
 export type MetricSummary = z.infer<typeof MetricSummarySchema>;
 
@@ -32,14 +42,25 @@ export type OutcomeSummary = z.infer<typeof OutcomeSummarySchema>;
 
 const CaseSummarySchema = OutcomeSummarySchema.extend({
   canonicalCaseId: z.string(),
+  permutationId: z.exactOptional(z.string()),
   metrics: z.record(z.string(), MetricSummarySchema),
 });
+
+/** Repetitions pool per case and harness permutation, never across permutations. */
+export const suiteGroupKey = (entry: {
+  canonicalCaseId: string;
+  permutationId?: string | undefined;
+}): string =>
+  entry.permutationId === undefined
+    ? entry.canonicalCaseId
+    : `${entry.canonicalCaseId} [${entry.permutationId}]`;
 export type CaseSummary = z.infer<typeof CaseSummarySchema>;
 
 export const SuiteSampleSchema = z.object({
   index: z.number().int().nonnegative(),
   caseId: z.string(),
   canonicalCaseId: z.string(),
+  permutationId: z.exactOptional(z.string()),
   exitReason: z.string().nullable(),
   passed: z.boolean().nullable(),
   identityHash: z.string().nullable(),
@@ -59,13 +80,22 @@ export const SuiteSummarySchema = z.object({
 });
 export type SuiteSummary = z.infer<typeof SuiteSummarySchema>;
 
-type MetricSamples = Map<string, number[]>;
+type MetricSamples = Map<
+  string,
+  { values: number[]; descriptor?: Omit<DriverMetric, "value"> | undefined }
+>;
 
-const add = (samples: MetricSamples, name: string, value: number | undefined): void => {
+const add = (
+  samples: MetricSamples,
+  name: string,
+  value: number | undefined,
+  descriptor?: Omit<DriverMetric, "value">,
+): void => {
   if (value === undefined || !Number.isFinite(value)) return;
-  const values = samples.get(name) ?? [];
-  values.push(value);
-  samples.set(name, values);
+  const sample = samples.get(name) ?? { values: [], descriptor };
+  if (!sample.descriptor && descriptor) sample.descriptor = descriptor;
+  sample.values.push(value);
+  samples.set(name, sample);
 };
 
 const cacheHitRate = (usage: {
@@ -84,7 +114,7 @@ const collectMetadata = (samples: MetricSamples, metadata: RunMetadata): void =>
     add(samples, "durationMs", Math.max(0, ended - started));
   }
 
-  if (!isUsageEmpty(metadata.usage.agent)) {
+  if (hasAgentUsage(metadata)) {
     add(samples, "agent.input", metadata.usage.agent.input);
     add(samples, "agent.output", metadata.usage.agent.output);
     add(samples, "agent.cacheRead", metadata.usage.agent.cacheRead);
@@ -137,6 +167,78 @@ const collectMetadata = (samples: MetricSamples, metadata: RunMetadata): void =>
   add(samples, "phase.agentMs", metadata.phaseDurationMs?.agent);
   add(samples, "phase.collectMs", metadata.phaseDurationMs?.collect);
   add(samples, "phase.gradingMs", metadata.phaseDurationMs?.grading);
+  add(samples, "timeToFirstActionMs", metadata.actionTiming?.timeToFirstActionMs);
+  add(samples, "timeToFirstMutationMs", metadata.actionTiming?.timeToFirstMutationMs);
+  add(samples, "timeToFirstDurableMutationMs", metadata.actionTiming?.timeToFirstDurableMutationMs);
+  const subagents = (metadata.agentActivity ?? []).filter(
+    (activity) => activity.agent.role === "subagent",
+  );
+  add(samples, "subagents.count", subagents.length, {
+    unit: "count",
+    label: "Subagents spawned",
+    better: "neutral",
+  });
+  add(
+    samples,
+    "subagents.events",
+    subagents.reduce((sum, activity) => sum + activity.eventCount, 0),
+    { unit: "count", label: "Subagent events", better: "neutral" },
+  );
+  add(
+    samples,
+    "subagents.toolCalls",
+    subagents.reduce((sum, activity) => sum + activity.toolCalls.total, 0),
+    { unit: "count", label: "Subagent tool calls", better: "neutral" },
+  );
+  add(
+    samples,
+    "subagents.toolCalls.failed",
+    subagents.reduce((sum, activity) => sum + activity.toolCalls.failed, 0),
+    { unit: "count", label: "Failed subagent tool calls", better: "lower" },
+  );
+  add(
+    samples,
+    "subagents.tokens",
+    subagents.reduce((sum, activity) => sum + activity.usage.total, 0),
+    { unit: "tokens", label: "Subagent tokens", better: "lower" },
+  );
+  add(
+    samples,
+    "subagents.actions",
+    subagents.reduce((sum, activity) => sum + activity.actions.total, 0),
+    { unit: "count", label: "Subagent actions", better: "neutral" },
+  );
+  add(
+    samples,
+    "subagents.mutations",
+    subagents.reduce((sum, activity) => sum + activity.actions.mutations, 0),
+    { unit: "count", label: "Subagent mutations", better: "neutral" },
+  );
+  add(
+    samples,
+    "subagents.totalDurationMs",
+    subagents.reduce((sum, activity) => sum + activity.durationMs, 0),
+    { unit: "milliseconds", label: "Total subagent time", better: "neutral" },
+  );
+  add(
+    samples,
+    "subagents.maxDurationMs",
+    subagents.length > 0 ? Math.max(...subagents.map((activity) => activity.durationMs)) : 0,
+    { unit: "milliseconds", label: "Longest subagent", better: "lower" },
+  );
+  const firstSubagentAction = subagents
+    .map((activity) => activity.actions.timeToFirstActionMs)
+    .filter((value): value is number => value !== undefined)
+    .sort((a, b) => a - b)[0];
+  add(samples, "subagents.timeToFirstActionMs", firstSubagentAction, {
+    unit: "milliseconds",
+    label: "First subagent action",
+    better: "lower",
+  });
+  for (const [name, metric] of Object.entries(metadata.driverMetrics ?? {})) {
+    const { value, ...descriptor } = metric;
+    add(samples, `driver.${name}`, value, descriptor);
+  }
 };
 
 const percentile = (sorted: readonly number[], fraction: number): number => {
@@ -165,7 +267,10 @@ const summarizeMetrics = (samples: MetricSamples): Record<string, MetricSummary>
   Object.fromEntries(
     [...samples.entries()]
       .sort(([a], [b]) => a.localeCompare(b))
-      .map(([name, values]) => [name, summarizeNumbers(values)]),
+      .map(([name, { values, descriptor }]) => [
+        name,
+        MetricSummarySchema.parse({ ...summarizeNumbers(values), ...descriptor }),
+      ]),
   );
 
 const summarizeOutcomes = (
@@ -214,14 +319,15 @@ export const buildSuiteSummary = (
 
   const byCase = new Map<string, typeof entries>();
   for (const entry of entries) {
-    const group = byCase.get(entry.item.canonicalCaseId) ?? [];
+    const key = suiteGroupKey(entry.item);
+    const group = byCase.get(key) ?? [];
     group.push(entry);
-    byCase.set(entry.item.canonicalCaseId, group);
+    byCase.set(key, group);
   }
 
   const cases = [...byCase.entries()]
     .sort(([a], [b]) => a.localeCompare(b))
-    .map(([canonicalCaseId, caseEntries]) => {
+    .map(([, caseEntries]) => {
       const metrics: MetricSamples = new Map();
       for (const entry of caseEntries) {
         const result = entry.result?.result;
@@ -229,8 +335,10 @@ export const buildSuiteSummary = (
         add(metrics, "score", result.grading.caseScore ?? undefined);
         collectMetadata(metrics, result.metadata);
       }
+      const first = caseEntries[0]?.item;
       return {
-        canonicalCaseId,
+        canonicalCaseId: first?.canonicalCaseId ?? "",
+        ...(first?.permutationId ? { permutationId: first.permutationId } : {}),
         ...summarizeOutcomes(caseEntries),
         metrics: summarizeMetrics(metrics),
       };
@@ -251,6 +359,7 @@ export const buildSuiteSummary = (
         index,
         caseId: entry.item.caseId,
         canonicalCaseId: entry.item.canonicalCaseId,
+        ...(entry.item.permutationId ? { permutationId: entry.item.permutationId } : {}),
         exitReason: entry.result?.exitReason ?? null,
         passed: entry.result?.casePassed ?? null,
         identityHash: result ? runComparisonIdentity(result.metadata) : null,
@@ -258,7 +367,7 @@ export const buildSuiteSummary = (
           ? { score: result.grading.caseScore }
           : {}),
         metrics: Object.fromEntries(
-          [...samples.entries()].map(([name, values]) => [name, values[0] ?? 0]),
+          [...samples.entries()].map(([name, sample]) => [name, sample.values[0] ?? 0]),
         ),
       };
     }),

package/src/previewer/artifacts.ts

@@ -0,0 +1,221 @@
+import type { Artifact } from "../core/index.js";
+import { inspectRunArtifact, readRunArtifactBytes } from "../core/index.js";
+
+const MAX_PREVIEW_BYTES = 512_000;
+const MAX_DIFF_LINES = 2_000;
+const MAX_DIFF_CELLS = 1_000_000;
+
+export type ArtifactPreview = {
+  path: string;
+  exists: boolean;
+  viewable: boolean;
+  size: number | null;
+  contentType: string | null;
+  text: string | null;
+  reason: string | null;
+};
+
+type ArtifactDiffOperation = {
+  kind: "same" | "added" | "removed";
+  lines: string[];
+};
+
+export type ArtifactComparison = {
+  path: string;
+  before: ArtifactPreview;
+  after: ArtifactPreview;
+  operations: ArtifactDiffOperation[] | null;
+  reason: string | null;
+};
+
+const evidenceArtifacts = (
+  indexed: readonly Artifact[],
+  knownPaths: readonly string[],
+): Artifact[] => {
+  const out = [...indexed];
+  const indexedPaths = new Set(indexed.map((artifact) => artifact.path));
+  for (const path of knownPaths) {
+    if (!indexedPaths.has(path)) out.push({ path });
+  }
+  return out;
+};
+
+export const loadArtifactPreview = async (
+  runDir: string,
+  indexed: readonly Artifact[],
+  knownPaths: readonly string[],
+  path: string,
+): Promise<ArtifactPreview> => {
+  if (!knownPaths.includes(path) && !indexed.some((artifact) => artifact.path === path)) {
+    return {
+      path,
+      exists: false,
+      viewable: false,
+      size: null,
+      contentType: null,
+      text: null,
+      reason: "artifact was not captured by this run",
+    };
+  }
+  const artifacts = evidenceArtifacts(indexed, knownPaths);
+  const run = { runDir, artifacts };
+  const inspected = await inspectRunArtifact(run, path);
+  const contentType = inspected.artifact?.contentType ?? null;
+  if (!inspected.stats?.isFile()) {
+    return {
+      path,
+      exists: false,
+      viewable: false,
+      size: inspected.stats?.size ?? null,
+      contentType,
+      text: null,
+      reason: inspected.issue ?? "artifact is not a regular file",
+    };
+  }
+  if (inspected.stats.size > MAX_PREVIEW_BYTES) {
+    return {
+      path,
+      exists: true,
+      viewable: false,
+      size: inspected.stats.size,
+      contentType,
+      text: null,
+      reason: `artifact exceeds the ${MAX_PREVIEW_BYTES}-byte preview limit`,
+    };
+  }
+  const bytes = await readRunArtifactBytes(run, path, { maxBytes: MAX_PREVIEW_BYTES });
+  let text: string;
+  try {
+    text = new TextDecoder("utf-8", { fatal: true }).decode(bytes);
+  } catch {
+    return {
+      path,
+      exists: true,
+      viewable: false,
+      size: bytes.length,
+      contentType,
+      text: null,
+      reason: "artifact is not valid UTF-8 text",
+    };
+  }
+  if (text.includes("\0")) {
+    return {
+      path,
+      exists: true,
+      viewable: false,
+      size: bytes.length,
+      contentType,
+      text: null,
+      reason: "artifact appears to be binary",
+    };
+  }
+  return {
+    path,
+    exists: true,
+    viewable: true,
+    size: bytes.length,
+    contentType,
+    text,
+    reason: null,
+  };
+};
+
+const appendOperation = (
+  operations: ArtifactDiffOperation[],
+  kind: ArtifactDiffOperation["kind"],
+  line: string,
+): void => {
+  const current = operations.at(-1);
+  if (current?.kind === kind) current.lines.push(line);
+  else operations.push({ kind, lines: [line] });
+};
+
+const diffLines = (beforeText: string, afterText: string): ArtifactDiffOperation[] | null => {
+  const before = beforeText.split("\n");
+  const after = afterText.split("\n");
+  if (
+    before.length > MAX_DIFF_LINES ||
+    after.length > MAX_DIFF_LINES ||
+    before.length * after.length > MAX_DIFF_CELLS
+  ) {
+    return null;
+  }
+  const width = after.length + 1;
+  const table = new Uint32Array((before.length + 1) * width);
+  for (let left = before.length - 1; left >= 0; left--) {
+    for (let right = after.length - 1; right >= 0; right--) {
+      const cell = left * width + right;
+      table[cell] =
+        before[left] === after[right]
+          ? (table[(left + 1) * width + right + 1] ?? 0) + 1
+          : Math.max(table[(left + 1) * width + right] ?? 0, table[left * width + right + 1] ?? 0);
+    }
+  }
+  const operations: ArtifactDiffOperation[] = [];
+  let left = 0;
+  let right = 0;
+  while (left < before.length && right < after.length) {
+    if (before[left] === after[right]) {
+      appendOperation(operations, "same", before[left] ?? "");
+      left++;
+      right++;
+    } else if ((table[(left + 1) * width + right] ?? 0) >= (table[left * width + right + 1] ?? 0)) {
+      appendOperation(operations, "removed", before[left] ?? "");
+      left++;
+    } else {
+      appendOperation(operations, "added", after[right] ?? "");
+      right++;
+    }
+  }
+  while (left < before.length) {
+    appendOperation(operations, "removed", before[left] ?? "");
+    left++;
+  }
+  while (right < after.length) {
+    appendOperation(operations, "added", after[right] ?? "");
+    right++;
+  }
+  return operations;
+};
+
+export const compareArtifactPreviews = (
+  path: string,
+  before: ArtifactPreview,
+  after: ArtifactPreview,
+): ArtifactComparison => {
+  if (before.viewable && before.text !== null && !after.exists) {
+    return {
+      path,
+      before,
+      after,
+      operations: [{ kind: "removed", lines: before.text.split("\n") }],
+      reason: null,
+    };
+  }
+  if (!before.exists && after.viewable && after.text !== null) {
+    return {
+      path,
+      before,
+      after,
+      operations: [{ kind: "added", lines: after.text.split("\n") }],
+      reason: null,
+    };
+  }
+  if (!before.viewable || !after.viewable || before.text === null || after.text === null) {
+    return {
+      path,
+      before,
+      after,
+      operations: null,
+      reason: before.reason ?? after.reason ?? "artifact text is unavailable",
+    };
+  }
+  const operations = diffLines(before.text, after.text);
+  return {
+    path,
+    before,
+    after,
+    operations,
+    reason: operations ? null : "artifact is too line-dense for an inline diff",
+  };
+};