@prismatic-io/lux 0.0.2-preview.4 → 0.0.2-preview.41

package/src/orchestrator/list-runs.ts

@@ -2,6 +2,11 @@ import { existsSync } from "node:fs";
 import { constants, lstat, open, readdir } from "node:fs/promises";
 import { basename, join, relative, sep } from "node:path";
 import {
+  ActionProgressPayloadSchema,
+  AgentCompletedPayloadSchema,
+  AgentSpawnPayloadSchema,
+  type Artifact,
+  ArtifactSchema,
   type GradingReport,
   GradingReportSchema,
   harnessUsage,
@@ -15,6 +20,7 @@ import {
   ToolResultPayloadSchema,
   totalTokens,
 } from "../core/index.js";
+import { hasAgentUsage } from "../core/usage.js";
 import { readBoundedHandle, safeReadJson, walkFiles } from "./fs-read.js";
 import { MAX_RUN_TRACE_BYTES, MAX_RUN_TRACE_EVENTS } from "./trace-limits.js";
 
@@ -31,10 +37,16 @@ export type ListedRun = {
   startedAt: string | null;
   endedAt: string | null;
   durationMs: number | null;
+  timeToFirstActionMs: number | null;
+  timeToFirstMutationMs: number | null;
+  timeToFirstDurableMutationMs: number | null;
+  driverMetrics: NonNullable<RunMetadata["driverMetrics"]>;
+  subagentCount: number;
   casePassed: boolean | null;
   caseScore: number | null;
   assertionPassed: number;
   assertionTotal: number;
+  readErrors?: string[];
 };
 
 const summarizeRun = async (
@@ -43,10 +55,19 @@ const summarizeRun = async (
   ts: string,
   runDir: string,
 ): Promise<ListedRun> => {
-  const [meta, grading] = await Promise.all([
+  const [metadataResult, gradingResult] = await Promise.allSettled([
     safeReadJson(join(runDir, "run.json"), RunMetadataSchema),
     safeReadJson(join(runDir, "grading.json"), GradingReportSchema),
   ]);
+  const readErrors = [metadataResult, gradingResult].flatMap((result) =>
+    result.status === "rejected"
+      ? [result.reason instanceof Error ? result.reason.message : String(result.reason)]
+      : [],
+  );
+  const meta = metadataResult.status === "fulfilled" ? metadataResult.value : null;
+  // A valid grading file cannot establish a result when its run metadata is damaged.
+  const grading =
+    readErrors.length === 0 && gradingResult.status === "fulfilled" ? gradingResult.value : null;
   const startedAt = meta?.startedAt ?? null;
   const endedAt = meta?.endedAt ?? null;
   const started = Date.parse(startedAt ?? "");
@@ -63,10 +84,17 @@ const summarizeRun = async (
     startedAt,
     endedAt,
     durationMs,
+    timeToFirstActionMs: meta?.actionTiming?.timeToFirstActionMs ?? null,
+    timeToFirstMutationMs: meta?.actionTiming?.timeToFirstMutationMs ?? null,
+    timeToFirstDurableMutationMs: meta?.actionTiming?.timeToFirstDurableMutationMs ?? null,
+    driverMetrics: meta?.driverMetrics ?? {},
+    subagentCount:
+      meta?.agentActivity?.filter((activity) => activity.agent.role === "subagent").length ?? 0,
     casePassed: grading?.casePassed ?? null,
     caseScore: grading?.caseScore ?? null,
     assertionPassed: passing,
-    assertionTotal: grading?.assertions.length ?? 0,
+    assertionTotal: grading?.assertions.filter((a) => a.passed !== null).length ?? 0,
+    ...(readErrors.length > 0 ? { readErrors } : {}),
   };
 };
 
@@ -127,24 +155,26 @@ export const listRuns = async (runsRoot: string): Promise<ListedRun[]> => {
 export const formatRunsTable = (runs: ListedRun[]): string => {
   if (runs.length === 0) return "(no runs found)";
   const rows: string[][] = [
-    ["status", "case", "driver", "ts", "exit", "score", "duration"],
+    ["status", "case", "driver", "agents", "ts", "exit", "score", "duration"],
     ...runs.map((r) => {
       let status = "ungraded";
-      if (r.casePassed === true) status = "PASS";
+      if (r.readErrors?.length) status = "DAMAGED";
+      else if (r.casePassed === true) status = "PASS";
       else if (r.casePassed === false) status = "FAIL";
       const score =
-        r.caseScore !== null
-          ? `${(r.caseScore * 100).toFixed(0)}% (${r.assertionPassed}/${r.assertionTotal})`
-          : "—";
+        r.caseScore !== null ? `${r.caseScore} (${r.assertionPassed}/${r.assertionTotal})` : "—";
       const duration = r.durationMs !== null ? `${(r.durationMs / 1000).toFixed(1)}s` : "—";
-      return [status, r.caseSlug, r.driverName, r.ts, r.exitReason ?? "?", score, duration];
+      const agents = r.subagentCount > 0 ? `1+${r.subagentCount}` : "1";
+      return [status, r.caseSlug, r.driverName, agents, r.ts, r.exitReason ?? "?", score, duration];
     }),
   ];
   const header = rows[0] ?? [];
   const widths = header.map((_, col) => Math.max(...rows.map((r) => r[col]?.length ?? 0)));
-  return rows
+  const table = rows
     .map((row) => row.map((cell, col) => cell.padEnd(widths[col] ?? 0)).join("  "))
     .join("\n");
+  const diagnostics = runs.flatMap((run) => run.readErrors ?? []);
+  return diagnostics.length > 0 ? `${table}\n\n${diagnostics.join("\n")}` : table;
 };
 
 export type RunDetail = {
@@ -156,6 +186,9 @@ export type RunDetail = {
   artifacts: string[];
 };
 
+export const loadRunArtifacts = async (runDir: string): Promise<Artifact[]> =>
+  (await safeReadJson(join(runDir, "artifacts.json"), ArtifactSchema.array())) ?? [];
+
 const readEventsJsonl = async (path: string): Promise<RunEvent[]> => {
   if (!existsSync(path)) return [];
   if ((await lstat(path)).isSymbolicLink()) {
@@ -269,6 +302,12 @@ const appendMetadataDetail = (lines: string[], detail: RunDetail): void => {
         }`,
       );
     }
+    const environment = provenance?.environment;
+    if (environment) {
+      lines.push(`concurrency: ${environment.concurrency}`);
+      if (environment.declared)
+        lines.push(`declared environment: ${JSON.stringify(environment.declared)}`);
+    }
     const c = detail.metadata.interruptCounts;
     lines.push(`interrupts: ask=${c.ask} approve=${c.approve} custom=${c.custom}`);
     const tools = detail.metadata.toolOutcomes;
@@ -281,7 +320,7 @@ const appendMetadataDetail = (lines: string[], detail: RunDetail): void => {
     }
     const usage = detail.metadata.usage;
     lines.push(
-      `tokens: agent=${totalTokens(usage.agent)} harness=${totalTokens(harnessUsage(usage))}` +
+      `tokens: agent=${hasAgentUsage(detail.metadata) ? totalTokens(usage.agent) : "unavailable"} harness=${totalTokens(harnessUsage(usage))}` +
         ` (answerer=${totalTokens(usage.answerer)} judge=${totalTokens(usage.grading)})`,
     );
     const phases = detail.metadata.phaseDurationMs;
@@ -291,6 +330,27 @@ const appendMetadataDetail = (lines: string[], detail: RunDetail): void => {
         `phases: agent=${secs(phases.agent)} collect=${secs(phases.collect)} grading=${secs(phases.grading)}`,
       );
     }
+    const actions = detail.metadata.actionTiming;
+    if (actions) {
+      const value = (ms: number | undefined): string => (ms === undefined ? "—" : `${ms}ms`);
+      lines.push(
+        `first action: any=${value(actions.timeToFirstActionMs)} mutation=${value(actions.timeToFirstMutationMs)} durable-mutation=${value(actions.timeToFirstDurableMutationMs)}`,
+      );
+    }
+    const activities = detail.metadata.agentActivity ?? [];
+    const subagents = activities.filter((activity) => activity.agent.role === "subagent");
+    if (subagents.length > 0) {
+      lines.push(`subagents: ${subagents.length}`);
+      for (const activity of subagents) {
+        const label = activity.agent.name
+          ? `${activity.agent.name} (${activity.agent.id})`
+          : activity.agent.id;
+        const agentActions = activity.actions;
+        lines.push(
+          `  ${label}: ${activity.status}, ${(activity.durationMs / 1000).toFixed(1)}s, events=${activity.eventCount}, tokens=${totalTokens(activity.usage)}, tools=${activity.toolCalls.total} (${activity.toolCalls.succeeded} ok/${activity.toolCalls.failed} failed/${activity.toolCalls.unresolved} unresolved), actions=${agentActions.total}, mutations=${agentActions.mutations}`,
+        );
+      }
+    }
   }
 };
 
@@ -300,15 +360,19 @@ const appendGradingDetail = (lines: string[], detail: RunDetail): void => {
     let status = "ungraded";
     if (detail.grading.casePassed === true) status = "PASS";
     else if (detail.grading.casePassed === false) status = "FAIL";
-    const score =
-      detail.grading.caseScore !== null ? `${(detail.grading.caseScore * 100).toFixed(0)}%` : "—";
+    const score = detail.grading.caseScore !== null ? String(detail.grading.caseScore) : "—";
     lines.push(
-      `grading: ${status} (${score}, ${detail.grading.assertions.filter((a) => a.passed).length}/${detail.grading.assertions.length})`,
+      `grading: ${status} (${score}, ${detail.grading.assertions.filter((a) => a.passed).length}/${detail.grading.assertions.filter((a) => a.passed !== null).length})`,
     );
     for (const a of detail.grading.assertions) {
-      const mark = a.passed ? "✓" : "✗";
+      let mark = a.passed ? "✓" : "✗";
+      if (a.passed === null) mark = "ℹ informational";
+      if (a.abstention) mark = "?";
       const took = ` (${(a.durationMs / 1000).toFixed(1)}s)`;
-      lines.push(`  ${mark} ${a.name}${took}${a.comment ? ` — ${a.comment}` : ""}`);
+      const abstention = a.abstention ? " — insufficient evidence" : "";
+      lines.push(
+        `  ${mark} ${a.name} [score ${a.errorMessage !== undefined || a.abstention ? "unavailable" : a.score}]${took}${abstention}${a.comment ? ` — ${a.comment}` : ""}`,
+      );
     }
   }
 };
@@ -361,17 +425,37 @@ const formatEventLine = (event: RunEvent): string => {
         }`;
       }
       return `responded(custom): ${JSON.stringify(event.payload).slice(0, 80)}`;
-    case "progress":
+    case "progress": {
+      const agent = event.agent?.role === "subagent" ? ` [subagent ${event.agent.id}]` : "";
+      if (event.kind === "agent-spawn") {
+        const payload = AgentSpawnPayloadSchema.safeParse(event.payload).data;
+        return payload
+          ? `agent-spawn: ${payload.agent.name ?? payload.agent.id} (${payload.agent.id})`
+          : "agent-spawn: malformed";
+      }
+      if (event.kind === "agent-completed") {
+        const payload = AgentCompletedPayloadSchema.safeParse(event.payload).data;
+        return payload
+          ? `agent-completed: ${payload.agentId} (${payload.status})`
+          : "agent-completed: malformed";
+      }
+      if (event.kind === "action") {
+        const payload = ActionProgressPayloadSchema.safeParse(event.payload).data;
+        return payload
+          ? `action${agent}: ${payload.name} (${payload.category})`
+          : "action: malformed";
+      }
       if (event.kind === "tool-call") {
         const payload = ToolCallPayloadSchema.safeParse(event.payload).data;
-        return payload ? `tool-call: ${payload.name}` : "tool-call: malformed";
+        return payload ? `tool-call${agent}: ${payload.name}` : "tool-call: malformed";
       }
       if (event.kind === "tool-result") {
         const payload = ToolResultPayloadSchema.safeParse(event.payload).data;
         if (!payload) return "tool-result: malformed";
-        return `tool-result: ${payload.name} (${payload.isError ? "failed" : "succeeded"})`;
+        return `tool-result${agent}: ${payload.name} (${payload.isError ? "failed" : "succeeded"})`;
       }
       return `progress(${event.kind})`;
+    }
     case "lifecycle":
       return `lifecycle: ${event.stage}`;
     case "ready":

package/src/orchestrator/optimization-lifecycle.ts

@@ -26,6 +26,7 @@ export const executeOptimizationLifecycle = async <Search, Deployment, Result>(
     deployment: Deployment | undefined;
     result: Result | undefined;
     error: unknown;
+    failed: boolean;
   };
 
   const machine = setup({
@@ -59,6 +60,7 @@ export const executeOptimizationLifecycle = async <Search, Deployment, Result>(
         result: ({ event }) => ("output" in event ? event.output : undefined),
       }),
       captureError: assign({
+        failed: true,
         error: ({ event }) =>
           "error" in event ? event.error : new Error("optimization lifecycle failed"),
       }),
@@ -70,6 +72,7 @@ export const executeOptimizationLifecycle = async <Search, Deployment, Result>(
       deployment: undefined,
       result: undefined,
       error: undefined,
+      failed: false,
     },
     initial: "evaluatingSearchSeeds",
     states: {
@@ -112,7 +115,7 @@ export const executeOptimizationLifecycle = async <Search, Deployment, Result>(
       failed: { type: "final" },
     },
     output: ({ context }) =>
-      context.error === undefined
+      !context.failed
         ? { status: "succeeded", result: context.result }
         : { status: "failed", error: context.error },
   });