@prismatic-io/lux 0.0.2-preview.4 → 0.0.2-preview.40

package/src/orchestrator/harness-catalog-drift.ts

@@ -0,0 +1,132 @@
+import { readFile } from "node:fs/promises";
+import { homedir } from "node:os";
+import { join } from "node:path";
+import { z } from "zod";
+import {
+  type ClosedHarnessCatalog,
+  codexCatalog,
+  copilotCatalog,
+  exec,
+  type ReasoningEffort,
+  tryParseJson,
+} from "../core/index.js";
+
+export type LiveHarnessModel = {
+  id: string;
+  efforts?: readonly ReasoningEffort[] | undefined;
+};
+
+export type HarnessCatalogDrift = {
+  unlisted: string[];
+  mismatched: string[];
+  unavailable: string[];
+};
+
+export const harnessCatalogDrift = (
+  catalog: ClosedHarnessCatalog,
+  live: readonly LiveHarnessModel[],
+): HarnessCatalogDrift => {
+  const liveIds = new Set(live.map((model) => model.id));
+  return {
+    unlisted: live.filter((model) => !(model.id in catalog.models)).map((model) => model.id),
+    mismatched: live
+      .filter(
+        (model) =>
+          model.efforts !== undefined &&
+          model.id in catalog.models &&
+          model.efforts.join(",") !== catalog.models[model.id]?.join(","),
+      )
+      .map((model) => `${model.id} (${model.efforts?.join(", ") || "no effort"})`),
+    unavailable: Object.keys(catalog.models).filter((id) => !liveIds.has(id)),
+  };
+};
+
+export const hasHarnessCatalogDrift = (drift: HarnessCatalogDrift): boolean =>
+  drift.unlisted.length > 0 || drift.mismatched.length > 0;
+
+export const describeHarnessCatalogDrift = (drift: HarnessCatalogDrift): string =>
+  [
+    drift.unlisted.length > 0 ? `not in catalog: ${drift.unlisted.join(", ")}` : undefined,
+    drift.mismatched.length > 0 ? `efforts differ: ${drift.mismatched.join("; ")}` : undefined,
+    drift.unavailable.length > 0 ? `not offered here: ${drift.unavailable.join(", ")}` : undefined,
+  ]
+    .filter((part) => part !== undefined)
+    .join(". ");
+
+const CodexModelsCacheSchema = z.object({
+  models: z.array(
+    z.looseObject({
+      slug: z.string(),
+      visibility: z.string().optional(),
+      supported_reasoning_levels: z.array(z.looseObject({ effort: z.string() })).default([]),
+    }),
+  ),
+});
+
+const codexModelsCachePath = (env: NodeJS.ProcessEnv = process.env): string =>
+  join(env.CODEX_HOME ?? join(homedir(), ".codex"), "models_cache.json");
+
+export const parseCodexModelsCache = (text: string): LiveHarnessModel[] | undefined => {
+  const parsed = CodexModelsCacheSchema.safeParse(tryParseJson(text));
+  if (!parsed.success) return undefined;
+  return parsed.data.models
+    .filter((model) => model.visibility !== "hide")
+    .map((model) => ({
+      id: model.slug,
+      efforts: model.supported_reasoning_levels.map((level) => level.effort as ReasoningEffort),
+    }));
+};
+
+const readCodexLiveModels = async (
+  path = codexModelsCachePath(),
+): Promise<LiveHarnessModel[] | undefined> => {
+  try {
+    return parseCodexModelsCache(await readFile(path, "utf8"));
+  } catch {
+    return undefined;
+  }
+};
+
+export const parseCopilotModelValues = (helpText: string): LiveHarnessModel[] | undefined => {
+  const lines = helpText.split("\n");
+  const start = lines.findIndex((line) => /^\s*`model`:/.test(line));
+  if (start < 0) return undefined;
+  const models: LiveHarnessModel[] = [];
+  for (const line of lines.slice(start + 1)) {
+    const match = /^\s*-\s*"([^"]+)"\s*$/.exec(line);
+    if (!match?.[1]) break;
+    models.push({ id: match[1] });
+  }
+  return models.length > 0 ? models : undefined;
+};
+
+const readCopilotLiveModels = async (
+  command = "copilot",
+): Promise<LiveHarnessModel[] | undefined> => {
+  try {
+    const result = await exec(command, ["help", "config"], { timeoutMs: 15_000 });
+    if (result.exitCode !== 0 || result.timedOut) return undefined;
+    return parseCopilotModelValues(result.stdout);
+  } catch {
+    return undefined;
+  }
+};
+
+export type HarnessCatalogProbe = {
+  catalog: ClosedHarnessCatalog;
+  source: string;
+  read: (command: string | undefined) => Promise<LiveHarnessModel[] | undefined>;
+};
+
+export const harnessCatalogProbes: readonly HarnessCatalogProbe[] = [
+  {
+    catalog: codexCatalog,
+    source: "models_cache.json",
+    read: () => readCodexLiveModels(),
+  },
+  {
+    catalog: copilotCatalog,
+    source: "help config",
+    read: (command) => readCopilotLiveModels(command),
+  },
+];

package/src/orchestrator/harness-registry.ts

@@ -0,0 +1,12 @@
+import { createRubricAssertion } from "../assertions/rubric/index.js";
+import type { HarnessModelConfig, Registry } from "../core/index.js";
+import type { ResolvedLuxConfig } from "./config.js";
+
+export const registryWithHarness = (
+  registry: Registry,
+  config: Pick<ResolvedLuxConfig, "assertions">,
+  harness: HarnessModelConfig | undefined,
+): Registry =>
+  config.assertions?.some((assertion) => assertion.type === "rubric")
+    ? registry
+    : registry.withAssertion(createRubricAssertion(harness));

package/src/orchestrator/index.ts

@@ -122,6 +122,7 @@ export {
   formatRunsTable,
   type ListedRun,
   listRuns,
+  loadRunArtifacts,
   loadRunDetail,
   type RunDetail,
 } from "./list-runs.js";

package/src/orchestrator/list-runs.ts

@@ -2,6 +2,11 @@ import { existsSync } from "node:fs";
 import { constants, lstat, open, readdir } from "node:fs/promises";
 import { basename, join, relative, sep } from "node:path";
 import {
+  ActionProgressPayloadSchema,
+  AgentCompletedPayloadSchema,
+  AgentSpawnPayloadSchema,
+  type Artifact,
+  ArtifactSchema,
   type GradingReport,
   GradingReportSchema,
   harnessUsage,
@@ -15,6 +20,7 @@ import {
   ToolResultPayloadSchema,
   totalTokens,
 } from "../core/index.js";
+import { hasAgentUsage } from "../core/usage.js";
 import { readBoundedHandle, safeReadJson, walkFiles } from "./fs-read.js";
 import { MAX_RUN_TRACE_BYTES, MAX_RUN_TRACE_EVENTS } from "./trace-limits.js";
 
@@ -31,10 +37,16 @@ export type ListedRun = {
   startedAt: string | null;
   endedAt: string | null;
   durationMs: number | null;
+  timeToFirstActionMs: number | null;
+  timeToFirstMutationMs: number | null;
+  timeToFirstDurableMutationMs: number | null;
+  driverMetrics: NonNullable<RunMetadata["driverMetrics"]>;
+  subagentCount: number;
   casePassed: boolean | null;
   caseScore: number | null;
   assertionPassed: number;
   assertionTotal: number;
+  readErrors?: string[];
 };
 
 const summarizeRun = async (
@@ -43,10 +55,19 @@ const summarizeRun = async (
   ts: string,
   runDir: string,
 ): Promise<ListedRun> => {
-  const [meta, grading] = await Promise.all([
+  const [metadataResult, gradingResult] = await Promise.allSettled([
     safeReadJson(join(runDir, "run.json"), RunMetadataSchema),
     safeReadJson(join(runDir, "grading.json"), GradingReportSchema),
   ]);
+  const readErrors = [metadataResult, gradingResult].flatMap((result) =>
+    result.status === "rejected"
+      ? [result.reason instanceof Error ? result.reason.message : String(result.reason)]
+      : [],
+  );
+  const meta = metadataResult.status === "fulfilled" ? metadataResult.value : null;
+  // A valid grading file cannot establish a result when its run metadata is damaged.
+  const grading =
+    readErrors.length === 0 && gradingResult.status === "fulfilled" ? gradingResult.value : null;
   const startedAt = meta?.startedAt ?? null;
   const endedAt = meta?.endedAt ?? null;
   const started = Date.parse(startedAt ?? "");
@@ -63,10 +84,17 @@ const summarizeRun = async (
     startedAt,
     endedAt,
     durationMs,
+    timeToFirstActionMs: meta?.actionTiming?.timeToFirstActionMs ?? null,
+    timeToFirstMutationMs: meta?.actionTiming?.timeToFirstMutationMs ?? null,
+    timeToFirstDurableMutationMs: meta?.actionTiming?.timeToFirstDurableMutationMs ?? null,
+    driverMetrics: meta?.driverMetrics ?? {},
+    subagentCount:
+      meta?.agentActivity?.filter((activity) => activity.agent.role === "subagent").length ?? 0,
     casePassed: grading?.casePassed ?? null,
     caseScore: grading?.caseScore ?? null,
     assertionPassed: passing,
-    assertionTotal: grading?.assertions.length ?? 0,
+    assertionTotal: grading?.assertions.filter((a) => a.passed !== null).length ?? 0,
+    ...(readErrors.length > 0 ? { readErrors } : {}),
   };
 };
 
@@ -127,24 +155,26 @@ export const listRuns = async (runsRoot: string): Promise<ListedRun[]> => {
 export const formatRunsTable = (runs: ListedRun[]): string => {
   if (runs.length === 0) return "(no runs found)";
   const rows: string[][] = [
-    ["status", "case", "driver", "ts", "exit", "score", "duration"],
+    ["status", "case", "driver", "agents", "ts", "exit", "score", "duration"],
     ...runs.map((r) => {
       let status = "ungraded";
-      if (r.casePassed === true) status = "PASS";
+      if (r.readErrors?.length) status = "DAMAGED";
+      else if (r.casePassed === true) status = "PASS";
       else if (r.casePassed === false) status = "FAIL";
       const score =
-        r.caseScore !== null
-          ? `${(r.caseScore * 100).toFixed(0)}% (${r.assertionPassed}/${r.assertionTotal})`
-          : "—";
+        r.caseScore !== null ? `${r.caseScore} (${r.assertionPassed}/${r.assertionTotal})` : "—";
       const duration = r.durationMs !== null ? `${(r.durationMs / 1000).toFixed(1)}s` : "—";
-      return [status, r.caseSlug, r.driverName, r.ts, r.exitReason ?? "?", score, duration];
+      const agents = r.subagentCount > 0 ? `1+${r.subagentCount}` : "1";
+      return [status, r.caseSlug, r.driverName, agents, r.ts, r.exitReason ?? "?", score, duration];
     }),
   ];
   const header = rows[0] ?? [];
   const widths = header.map((_, col) => Math.max(...rows.map((r) => r[col]?.length ?? 0)));
-  return rows
+  const table = rows
     .map((row) => row.map((cell, col) => cell.padEnd(widths[col] ?? 0)).join("  "))
     .join("\n");
+  const diagnostics = runs.flatMap((run) => run.readErrors ?? []);
+  return diagnostics.length > 0 ? `${table}\n\n${diagnostics.join("\n")}` : table;
 };
 
 export type RunDetail = {
@@ -156,6 +186,9 @@ export type RunDetail = {
   artifacts: string[];
 };
 
+export const loadRunArtifacts = async (runDir: string): Promise<Artifact[]> =>
+  (await safeReadJson(join(runDir, "artifacts.json"), ArtifactSchema.array())) ?? [];
+
 const readEventsJsonl = async (path: string): Promise<RunEvent[]> => {
   if (!existsSync(path)) return [];
   if ((await lstat(path)).isSymbolicLink()) {
@@ -269,6 +302,12 @@ const appendMetadataDetail = (lines: string[], detail: RunDetail): void => {
         }`,
       );
     }
+    const environment = provenance?.environment;
+    if (environment) {
+      lines.push(`concurrency: ${environment.concurrency}`);
+      if (environment.declared)
+        lines.push(`declared environment: ${JSON.stringify(environment.declared)}`);
+    }
     const c = detail.metadata.interruptCounts;
     lines.push(`interrupts: ask=${c.ask} approve=${c.approve} custom=${c.custom}`);
     const tools = detail.metadata.toolOutcomes;
@@ -281,7 +320,7 @@ const appendMetadataDetail = (lines: string[], detail: RunDetail): void => {
     }
     const usage = detail.metadata.usage;
     lines.push(
-      `tokens: agent=${totalTokens(usage.agent)} harness=${totalTokens(harnessUsage(usage))}` +
+      `tokens: agent=${hasAgentUsage(detail.metadata) ? totalTokens(usage.agent) : "unavailable"} harness=${totalTokens(harnessUsage(usage))}` +
         ` (answerer=${totalTokens(usage.answerer)} judge=${totalTokens(usage.grading)})`,
     );
     const phases = detail.metadata.phaseDurationMs;
@@ -291,6 +330,27 @@ const appendMetadataDetail = (lines: string[], detail: RunDetail): void => {
         `phases: agent=${secs(phases.agent)} collect=${secs(phases.collect)} grading=${secs(phases.grading)}`,
       );
     }
+    const actions = detail.metadata.actionTiming;
+    if (actions) {
+      const value = (ms: number | undefined): string => (ms === undefined ? "—" : `${ms}ms`);
+      lines.push(
+        `first action: any=${value(actions.timeToFirstActionMs)} mutation=${value(actions.timeToFirstMutationMs)} durable-mutation=${value(actions.timeToFirstDurableMutationMs)}`,
+      );
+    }
+    const activities = detail.metadata.agentActivity ?? [];
+    const subagents = activities.filter((activity) => activity.agent.role === "subagent");
+    if (subagents.length > 0) {
+      lines.push(`subagents: ${subagents.length}`);
+      for (const activity of subagents) {
+        const label = activity.agent.name
+          ? `${activity.agent.name} (${activity.agent.id})`
+          : activity.agent.id;
+        const agentActions = activity.actions;
+        lines.push(
+          `  ${label}: ${activity.status}, ${(activity.durationMs / 1000).toFixed(1)}s, events=${activity.eventCount}, tokens=${totalTokens(activity.usage)}, tools=${activity.toolCalls.total} (${activity.toolCalls.succeeded} ok/${activity.toolCalls.failed} failed/${activity.toolCalls.unresolved} unresolved), actions=${agentActions.total}, mutations=${agentActions.mutations}`,
+        );
+      }
+    }
   }
 };
 
@@ -300,15 +360,19 @@ const appendGradingDetail = (lines: string[], detail: RunDetail): void => {
     let status = "ungraded";
     if (detail.grading.casePassed === true) status = "PASS";
     else if (detail.grading.casePassed === false) status = "FAIL";
-    const score =
-      detail.grading.caseScore !== null ? `${(detail.grading.caseScore * 100).toFixed(0)}%` : "—";
+    const score = detail.grading.caseScore !== null ? String(detail.grading.caseScore) : "—";
     lines.push(
-      `grading: ${status} (${score}, ${detail.grading.assertions.filter((a) => a.passed).length}/${detail.grading.assertions.length})`,
+      `grading: ${status} (${score}, ${detail.grading.assertions.filter((a) => a.passed).length}/${detail.grading.assertions.filter((a) => a.passed !== null).length})`,
     );
     for (const a of detail.grading.assertions) {
-      const mark = a.passed ? "✓" : "✗";
+      let mark = a.passed ? "✓" : "✗";
+      if (a.passed === null) mark = "ℹ informational";
+      if (a.abstention) mark = "?";
       const took = ` (${(a.durationMs / 1000).toFixed(1)}s)`;
-      lines.push(`  ${mark} ${a.name}${took}${a.comment ? ` — ${a.comment}` : ""}`);
+      const abstention = a.abstention ? " — insufficient evidence" : "";
+      lines.push(
+        `  ${mark} ${a.name} [score ${a.errorMessage !== undefined || a.abstention ? "unavailable" : a.score}]${took}${abstention}${a.comment ? ` — ${a.comment}` : ""}`,
+      );
     }
   }
 };
@@ -361,17 +425,37 @@ const formatEventLine = (event: RunEvent): string => {
         }`;
       }
       return `responded(custom): ${JSON.stringify(event.payload).slice(0, 80)}`;
-    case "progress":
+    case "progress": {
+      const agent = event.agent?.role === "subagent" ? ` [subagent ${event.agent.id}]` : "";
+      if (event.kind === "agent-spawn") {
+        const payload = AgentSpawnPayloadSchema.safeParse(event.payload).data;
+        return payload
+          ? `agent-spawn: ${payload.agent.name ?? payload.agent.id} (${payload.agent.id})`
+          : "agent-spawn: malformed";
+      }
+      if (event.kind === "agent-completed") {
+        const payload = AgentCompletedPayloadSchema.safeParse(event.payload).data;
+        return payload
+          ? `agent-completed: ${payload.agentId} (${payload.status})`
+          : "agent-completed: malformed";
+      }
+      if (event.kind === "action") {
+        const payload = ActionProgressPayloadSchema.safeParse(event.payload).data;
+        return payload
+          ? `action${agent}: ${payload.name} (${payload.category})`
+          : "action: malformed";
+      }
       if (event.kind === "tool-call") {
         const payload = ToolCallPayloadSchema.safeParse(event.payload).data;
-        return payload ? `tool-call: ${payload.name}` : "tool-call: malformed";
+        return payload ? `tool-call${agent}: ${payload.name}` : "tool-call: malformed";
       }
       if (event.kind === "tool-result") {
         const payload = ToolResultPayloadSchema.safeParse(event.payload).data;
         if (!payload) return "tool-result: malformed";
-        return `tool-result: ${payload.name} (${payload.isError ? "failed" : "succeeded"})`;
+        return `tool-result${agent}: ${payload.name} (${payload.isError ? "failed" : "succeeded"})`;
       }
       return `progress(${event.kind})`;
+    }
     case "lifecycle":
       return `lifecycle: ${event.stage}`;
     case "ready":

package/src/orchestrator/optimization-lifecycle.ts

@@ -26,6 +26,7 @@ export const executeOptimizationLifecycle = async <Search, Deployment, Result>(
     deployment: Deployment | undefined;
     result: Result | undefined;
     error: unknown;
+    failed: boolean;
   };
 
   const machine = setup({
@@ -59,6 +60,7 @@ export const executeOptimizationLifecycle = async <Search, Deployment, Result>(
         result: ({ event }) => ("output" in event ? event.output : undefined),
       }),
       captureError: assign({
+        failed: true,
         error: ({ event }) =>
           "error" in event ? event.error : new Error("optimization lifecycle failed"),
       }),
@@ -70,6 +72,7 @@ export const executeOptimizationLifecycle = async <Search, Deployment, Result>(
       deployment: undefined,
       result: undefined,
       error: undefined,
+      failed: false,
     },
     initial: "evaluatingSearchSeeds",
     states: {
@@ -112,7 +115,7 @@ export const executeOptimizationLifecycle = async <Search, Deployment, Result>(
       failed: { type: "final" },
     },
     output: ({ context }) =>
-      context.error === undefined
+      !context.failed
         ? { status: "succeeded", result: context.result }
         : { status: "failed", error: context.error },
   });