@prismatic-io/lux 0.0.1

package/src/orchestrator/list-runs.ts

@@ -0,0 +1,384 @@
+import { existsSync } from "node:fs";
+import { constants, lstat, open, readdir } from "node:fs/promises";
+import { basename, join, relative, sep } from "node:path";
+import {
+  type GradingReport,
+  GradingReportSchema,
+  harnessUsage,
+  type PersistedCase,
+  PersistedCaseSchema,
+  type RunEvent,
+  RunEventSchema,
+  type RunMetadata,
+  RunMetadataSchema,
+  ToolCallPayloadSchema,
+  ToolResultPayloadSchema,
+  totalTokens,
+} from "../core/index.js";
+import { readBoundedHandle, safeReadJson, walkFiles } from "./fs-read.js";
+import { MAX_RUN_TRACE_BYTES, MAX_RUN_TRACE_EVENTS } from "./trace-limits.js";
+
+// The trailing `-N` is the collision suffix openRunDir adds when two runs of
+// the same case+driver start in the same millisecond.
+const TS_DIR_RE = /^\d{4}-\d{2}-\d{2}T[\d-]+Z(?:-\d+)?$/;
+export type ListedRun = {
+  caseSlug: string;
+  driverName: string;
+  ts: string;
+  runDir: string;
+  exitReason: RunMetadata["exitReason"] | null;
+  driverAgentId: string | null;
+  startedAt: string | null;
+  endedAt: string | null;
+  durationMs: number | null;
+  casePassed: boolean | null;
+  caseScore: number | null;
+  assertionPassed: number;
+  assertionTotal: number;
+};
+
+const summarizeRun = async (
+  caseSlug: string,
+  driverName: string,
+  ts: string,
+  runDir: string,
+): Promise<ListedRun> => {
+  const [meta, grading] = await Promise.all([
+    safeReadJson(join(runDir, "run.json"), RunMetadataSchema),
+    safeReadJson(join(runDir, "grading.json"), GradingReportSchema),
+  ]);
+  const startedAt = meta?.startedAt ?? null;
+  const endedAt = meta?.endedAt ?? null;
+  const started = Date.parse(startedAt ?? "");
+  const ended = Date.parse(endedAt ?? "");
+  const durationMs = Number.isNaN(started) || Number.isNaN(ended) ? null : ended - started;
+  const passing = grading?.assertions.filter((a) => a.passed).length ?? 0;
+  return {
+    caseSlug,
+    driverName,
+    ts,
+    runDir,
+    exitReason: meta?.exitReason ?? null,
+    driverAgentId: meta?.driver.agentId ?? null,
+    startedAt,
+    endedAt,
+    durationMs,
+    casePassed: grading?.casePassed ?? null,
+    caseScore: grading?.caseScore ?? null,
+    assertionPassed: passing,
+    assertionTotal: grading?.assertions.length ?? 0,
+  };
+};
+
+const listForCaseDriver = async (
+  driverPath: string,
+  caseSlug: string,
+  driverName: string,
+): Promise<ListedRun[]> => {
+  // Directories only, as in collectRuns: a timestamp-named *file* dropped
+  // beside real run dirs must not fabricate a phantom row.
+  const entries = (await readdir(driverPath, { withFileTypes: true }))
+    .filter((entry) => entry.isDirectory() && TS_DIR_RE.test(entry.name))
+    .map((entry) => entry.name);
+  const runs = await Promise.all(
+    entries.map((ts) => summarizeRun(caseSlug, driverName, ts, join(driverPath, ts))),
+  );
+  return runs.sort((a, b) => b.ts.localeCompare(a.ts));
+};
+
+// Case slugs keep their slashes, so a run nests at a variable depth. Recurse to
+// the directory whose children are timestamps (the driver dir) rather than
+// assuming a fixed two levels, which misses every case in a subdirectory.
+const collectRuns = async (runsRoot: string, dir: string, out: ListedRun[]): Promise<void> => {
+  // Only directories count: a stray timestamp-named file must not make this
+  // dir read as a driver dir, and a dangling symlink (or a racing delete)
+  // must not reject the listing.
+  const subdirs = (await readdir(dir, { withFileTypes: true }))
+    .filter((entry) => entry.isDirectory())
+    .map((entry) => entry.name);
+  if (subdirs.some((n) => TS_DIR_RE.test(n))) {
+    const caseDir = join(dir, "..");
+    const caseSlug = relative(runsRoot, caseDir).split(sep).join("/");
+    // A run layout is at least runsRoot/<case>/<driver>/<ts>; timestamp dirs
+    // higher up (a stray archived run copied into the tree) have no case
+    // level above them, so a slug of "" or ".." marks them as noise, not
+    // runs. Either way the non-timestamp siblings may still hold real runs —
+    // classify per child and keep descending instead of returning early.
+    if (caseSlug !== "" && caseSlug !== ".." && !caseSlug.startsWith("../")) {
+      out.push(...(await listForCaseDriver(dir, caseSlug, basename(dir))));
+    }
+  }
+  for (const name of subdirs.filter((n) => !TS_DIR_RE.test(n))) {
+    await collectRuns(runsRoot, join(dir, name), out);
+  }
+};
+
+export const listRuns = async (runsRoot: string): Promise<ListedRun[]> => {
+  if (!existsSync(runsRoot)) return [];
+  const out: ListedRun[] = [];
+  await collectRuns(runsRoot, runsRoot, out);
+  return out.sort((a, b) => {
+    if (a.caseSlug !== b.caseSlug) return a.caseSlug.localeCompare(b.caseSlug);
+    if (a.driverName !== b.driverName) return a.driverName.localeCompare(b.driverName);
+    return b.ts.localeCompare(a.ts);
+  });
+};
+
+export const formatRunsTable = (runs: ListedRun[]): string => {
+  if (runs.length === 0) return "(no runs found)";
+  const rows: string[][] = [
+    ["status", "case", "driver", "ts", "exit", "score", "duration"],
+    ...runs.map((r) => {
+      let status = "ungraded";
+      if (r.casePassed === true) status = "PASS";
+      else if (r.casePassed === false) status = "FAIL";
+      const score =
+        r.caseScore !== null
+          ? `${(r.caseScore * 100).toFixed(0)}% (${r.assertionPassed}/${r.assertionTotal})`
+          : "—";
+      const duration = r.durationMs !== null ? `${(r.durationMs / 1000).toFixed(1)}s` : "—";
+      return [status, r.caseSlug, r.driverName, r.ts, r.exitReason ?? "?", score, duration];
+    }),
+  ];
+  const header = rows[0] ?? [];
+  const widths = header.map((_, col) => Math.max(...rows.map((r) => r[col]?.length ?? 0)));
+  return rows
+    .map((row) => row.map((cell, col) => cell.padEnd(widths[col] ?? 0)).join("  "))
+    .join("\n");
+};
+
+export type RunDetail = {
+  runDir: string;
+  case: PersistedCase | null;
+  metadata: RunMetadata | null;
+  grading: GradingReport | null;
+  events: RunEvent[];
+  artifacts: string[];
+};
+
+const readEventsJsonl = async (path: string): Promise<RunEvent[]> => {
+  if (!existsSync(path)) return [];
+  if ((await lstat(path)).isSymbolicLink()) {
+    throw new Error(`run events cannot be a symlink: ${path}`);
+  }
+  let handle: Awaited<ReturnType<typeof open>>;
+  try {
+    handle = await open(path, constants.O_RDONLY | constants.O_NOFOLLOW);
+  } catch (error) {
+    if ((error as NodeJS.ErrnoException).code === "ENOENT") return [];
+    if ((error as NodeJS.ErrnoException).code === "ELOOP") {
+      throw new Error(`run events cannot be a symlink: ${path}`);
+    }
+    throw error;
+  }
+  let text: string;
+  try {
+    const metadata = await handle.stat();
+    if (!metadata.isFile() || metadata.size > MAX_RUN_TRACE_BYTES) {
+      throw new Error(
+        `run events must be a regular file no larger than ${MAX_RUN_TRACE_BYTES} bytes: ${path}`,
+      );
+    }
+    text = (await readBoundedHandle(handle, MAX_RUN_TRACE_BYTES, "run events")).toString("utf8");
+  } finally {
+    await handle.close();
+  }
+  const lines = text.split("\n");
+  if (lines.at(-1) === "") lines.pop();
+  if (lines.length > MAX_RUN_TRACE_EVENTS) {
+    throw new Error(`run events exceed ${MAX_RUN_TRACE_EVENTS} lines: ${path}`);
+  }
+  return lines.map((line, index) => {
+    let raw: unknown;
+    try {
+      raw = JSON.parse(line);
+    } catch {
+      throw new Error(`corrupt run events at ${path}:${index + 1}: invalid JSON`);
+    }
+    const parsed = RunEventSchema.safeParse(raw);
+    if (!parsed.success) {
+      throw new Error(`corrupt run events at ${path}:${index + 1}: invalid event`);
+    }
+    return parsed.data;
+  });
+};
+
+export const loadRunDetail = async (
+  runDir: string,
+  options: { maxArtifacts?: number; skipArtifactsWalk?: boolean; skipJson?: boolean } = {},
+): Promise<RunDetail> => {
+  const [caseJson, metadata, grading, events, artifacts] = await Promise.all([
+    options.skipJson
+      ? Promise.resolve(null)
+      : safeReadJson(join(runDir, "case.json"), PersistedCaseSchema),
+    options.skipJson
+      ? Promise.resolve(null)
+      : safeReadJson(join(runDir, "run.json"), RunMetadataSchema),
+    options.skipJson
+      ? Promise.resolve(null)
+      : safeReadJson(join(runDir, "grading.json"), GradingReportSchema),
+    readEventsJsonl(join(runDir, "events.jsonl")),
+    options.skipArtifactsWalk
+      ? Promise.resolve([])
+      : walkFiles(join(runDir, "artifacts"), options.maxArtifacts),
+  ]);
+  return { runDir, case: caseJson, metadata, grading, events, artifacts };
+};
+
+const appendCaseDetail = (lines: string[], detail: RunDetail): void => {
+  if (detail.case) {
+    lines.push(`id:     ${detail.case.id ?? "(no id)"}`);
+    if (detail.case.prompt) {
+      lines.push(`prompt: ${detail.case.prompt.slice(0, 120).replace(/\n/g, " ")}`);
+    }
+    if (detail.case.persona) {
+      lines.push(`persona: ${detail.case.persona.slice(0, 120).replace(/\n/g, " ")}`);
+    }
+  }
+};
+
+const appendMetadataDetail = (lines: string[], detail: RunDetail): void => {
+  if (detail.metadata) {
+    lines.push("");
+    lines.push(`exit:    ${detail.metadata.exitReason}`);
+    lines.push(
+      `driver:  ${detail.metadata.driver.name} (agent=${detail.metadata.driver.agentId}${
+        detail.metadata.driver.model ? `, model=${detail.metadata.driver.model}` : ""
+      })`,
+    );
+    const answerer = detail.metadata.answerer;
+    lines.push(
+      `answerer: ${answerer.name}${answerer.provider ? ` (${answerer.provider})` : ""}${
+        answerer.model ? `, model=${answerer.model}` : ""
+      }`,
+    );
+    const provenance = detail.metadata.provenance;
+    if (provenance?.source) {
+      const git = provenance.source.git;
+      lines.push(
+        `source:   ${provenance.source.casePath}${
+          git ? ` @ ${git.commit.slice(0, 12)}${git.dirty ? " (dirty)" : ""}` : ""
+        }`,
+      );
+    }
+    if (provenance) {
+      const runtime = provenance.runtime;
+      lines.push(
+        `runtime:  ${runtime.platform}/${runtime.arch} ${runtime.node}${
+          runtime.bun ? ` bun/${runtime.bun}` : ""
+        }`,
+      );
+    }
+    const c = detail.metadata.interruptCounts;
+    lines.push(`interrupts: ask=${c.ask} approve=${c.approve} custom=${c.custom}`);
+    const tools = detail.metadata.toolOutcomes;
+    if (tools) {
+      lines.push(
+        `tools: reported-calls=${tools.reportedCalls} valid-calls=${tools.total} succeeded=${tools.succeeded} failed=${tools.failed} unresolved=${tools.unresolved} unmatched-results=${tools.unmatchedResults} malformed=${tools.malformedEvents}`,
+      );
+    } else if (detail.metadata.toolCallCount !== undefined) {
+      lines.push(`tools: reported-calls=${detail.metadata.toolCallCount}`);
+    }
+    const usage = detail.metadata.usage;
+    lines.push(
+      `tokens: agent=${totalTokens(usage.agent)} harness=${totalTokens(harnessUsage(usage))}` +
+        ` (answerer=${totalTokens(usage.answerer)} judge=${totalTokens(usage.grading)})`,
+    );
+    const phases = detail.metadata.phaseDurationMs;
+    if (phases) {
+      const secs = (ms: number): string => `${(ms / 1000).toFixed(1)}s`;
+      lines.push(
+        `phases: agent=${secs(phases.agent)} collect=${secs(phases.collect)} grading=${secs(phases.grading)}`,
+      );
+    }
+  }
+};
+
+const appendGradingDetail = (lines: string[], detail: RunDetail): void => {
+  if (detail.grading) {
+    lines.push("");
+    let status = "ungraded";
+    if (detail.grading.casePassed === true) status = "PASS";
+    else if (detail.grading.casePassed === false) status = "FAIL";
+    const score =
+      detail.grading.caseScore !== null ? `${(detail.grading.caseScore * 100).toFixed(0)}%` : "—";
+    lines.push(
+      `grading: ${status} (${score}, ${detail.grading.assertions.filter((a) => a.passed).length}/${detail.grading.assertions.length})`,
+    );
+    for (const a of detail.grading.assertions) {
+      const mark = a.passed ? "✓" : "✗";
+      const took = ` (${(a.durationMs / 1000).toFixed(1)}s)`;
+      lines.push(`  ${mark} ${a.name}${took}${a.comment ? ` — ${a.comment}` : ""}`);
+    }
+  }
+};
+
+const appendEventDetail = (lines: string[], detail: RunDetail): void => {
+  if (detail.events.length > 0) {
+    lines.push("");
+    lines.push(`events: ${detail.events.length} total`);
+    const tail = detail.events.slice(-10);
+    for (const e of tail) {
+      lines.push(`  ${formatEventLine(e)}`);
+    }
+  }
+};
+
+const appendArtifactDetail = (lines: string[], detail: RunDetail): void => {
+  if (detail.artifacts.length > 0) {
+    lines.push("");
+    lines.push(`artifacts: ${detail.artifacts.length} files`);
+    for (const a of detail.artifacts.slice(0, 20)) {
+      lines.push(`  ${a}`);
+    }
+    if (detail.artifacts.length > 20) {
+      lines.push(`  ... and ${detail.artifacts.length - 20} more`);
+    }
+  }
+};
+
+export const formatRunDetail = (detail: RunDetail): string => {
+  const lines = [`runDir: ${detail.runDir}`];
+  appendCaseDetail(lines, detail);
+  appendMetadataDetail(lines, detail);
+  appendGradingDetail(lines, detail);
+  appendEventDetail(lines, detail);
+  appendArtifactDetail(lines, detail);
+  return lines.join("\n");
+};
+
+const formatEventLine = (event: RunEvent): string => {
+  switch (event.type) {
+    case "interrupt":
+      if (event.kind === "ask") return `interrupt(ask): ${event.question.slice(0, 80)}`;
+      if (event.kind === "approve") return `interrupt(approve): ${event.request.slice(0, 80)}`;
+      return `interrupt(custom): ${JSON.stringify(event.payload).slice(0, 80)}`;
+    case "responded":
+      if (event.kind === "ask") return `responded(ask): ${event.text.slice(0, 80)}`;
+      if (event.kind === "approve") {
+        return `responded(approve): ${event.approved ? "yes" : "no"}${
+          event.reason ? ` — ${event.reason.slice(0, 60)}` : ""
+        }`;
+      }
+      return `responded(custom): ${JSON.stringify(event.payload).slice(0, 80)}`;
+    case "progress":
+      if (event.kind === "tool-call") {
+        const payload = ToolCallPayloadSchema.safeParse(event.payload).data;
+        return payload ? `tool-call: ${payload.name}` : "tool-call: malformed";
+      }
+      if (event.kind === "tool-result") {
+        const payload = ToolResultPayloadSchema.safeParse(event.payload).data;
+        if (!payload) return "tool-result: malformed";
+        return `tool-result: ${payload.name} (${payload.isError ? "failed" : "succeeded"})`;
+      }
+      return `progress(${event.kind})`;
+    case "lifecycle":
+      return `lifecycle: ${event.stage}`;
+    case "ready":
+      return `ready: agentId=${event.agentId}`;
+    case "done":
+      return "done";
+    case "error":
+      return `error: ${event.reason}`;
+  }
+};

package/src/orchestrator/loader.ts

@@ -0,0 +1,19 @@
+import { resolve } from "node:path";
+import { pathToFileURL } from "node:url";
+import { type EvalCase, EvalCaseSchema, formatZodIssues } from "../core/index.js";
+
+export const loadEvalCase = async (filePath: string): Promise<EvalCase> => {
+  const absolute = resolve(filePath);
+  // A file URL, not the raw path: `import("C:\\...")` misparses on Windows.
+  const mod = (await import(pathToFileURL(absolute).href)) as { default?: unknown };
+  if (mod.default === undefined) {
+    throw new Error(
+      `Eval case at ${absolute} has no default export. Use \`export default defineEvalCase({ ... })\`.`,
+    );
+  }
+  const result = EvalCaseSchema.safeParse(mod.default);
+  if (!result.success) {
+    throw new Error(`Invalid eval case at ${absolute}: ${formatZodIssues(result.error)}`);
+  }
+  return result.data;
+};

package/src/orchestrator/optimization-lifecycle.ts

@@ -0,0 +1,126 @@
+import { assign, createActor, fromPromise, setup, toPromise } from "xstate";
+
+export type OptimizationLifecycleEffects<Search, Deployment, Result> = {
+  initialSearch: () => Search;
+  evaluateSearchSeeds: () => Promise<void>;
+  searchCanContinue: (search: Search) => boolean;
+  runSearchIteration: (search: Search) => Promise<Search>;
+  promote: () => Promise<Deployment>;
+  test: (deployment: Deployment) => Promise<Result>;
+};
+
+type OptimizationPhase =
+  | "evaluatingSearchSeeds"
+  | "searching"
+  | "promoting"
+  | "testing"
+  | "succeeded"
+  | "failed";
+
+export const executeOptimizationLifecycle = async <Search, Deployment, Result>(
+  effects: OptimizationLifecycleEffects<Search, Deployment, Result>,
+  observe?: (phase: OptimizationPhase) => void,
+): Promise<Result> => {
+  type Context = {
+    search: Search;
+    deployment: Deployment | undefined;
+    result: Result | undefined;
+    error: unknown;
+  };
+
+  const machine = setup({
+    types: {
+      context: {} as Context,
+      output: {} as
+        | { status: "succeeded"; result: Result | undefined }
+        | { status: "failed"; error: unknown },
+    },
+    actors: {
+      evaluateSearchSeeds: fromPromise<void, Context>(() => effects.evaluateSearchSeeds()),
+      runSearchIteration: fromPromise<Search, Context>(({ input }) =>
+        effects.runSearchIteration(input.search),
+      ),
+      promote: fromPromise<Deployment, Context>(() => effects.promote()),
+      test: fromPromise<Result, Context>(({ input }) =>
+        effects.test(input.deployment as Deployment),
+      ),
+    },
+    guards: {
+      searchCanContinue: ({ context }) => effects.searchCanContinue(context.search),
+    },
+    actions: {
+      advanceSearch: assign({
+        search: ({ event }) => ("output" in event ? event.output : effects.initialSearch()),
+      }),
+      captureDeployment: assign({
+        deployment: ({ event }) => ("output" in event ? event.output : undefined),
+      }),
+      captureResult: assign({
+        result: ({ event }) => ("output" in event ? event.output : undefined),
+      }),
+      captureError: assign({
+        error: ({ event }) =>
+          "error" in event ? event.error : new Error("optimization lifecycle failed"),
+      }),
+    },
+  }).createMachine({
+    id: "lux-optimization",
+    context: {
+      search: effects.initialSearch(),
+      deployment: undefined,
+      result: undefined,
+      error: undefined,
+    },
+    initial: "evaluatingSearchSeeds",
+    states: {
+      evaluatingSearchSeeds: {
+        invoke: {
+          src: "evaluateSearchSeeds",
+          input: ({ context }) => context,
+          onDone: { target: "decidingSearch" },
+          onError: { target: "failed", actions: "captureError" },
+        },
+      },
+      decidingSearch: {
+        always: [{ guard: "searchCanContinue", target: "searching" }, { target: "promoting" }],
+      },
+      searching: {
+        invoke: {
+          src: "runSearchIteration",
+          input: ({ context }) => context,
+          onDone: { target: "decidingSearch", actions: "advanceSearch" },
+          onError: { target: "failed", actions: "captureError" },
+        },
+      },
+      promoting: {
+        invoke: {
+          src: "promote",
+          input: ({ context }) => context,
+          onDone: { target: "testing", actions: "captureDeployment" },
+          onError: { target: "failed", actions: "captureError" },
+        },
+      },
+      testing: {
+        invoke: {
+          src: "test",
+          input: ({ context }) => context,
+          onDone: { target: "succeeded", actions: "captureResult" },
+          onError: { target: "failed", actions: "captureError" },
+        },
+      },
+      succeeded: { type: "final" },
+      failed: { type: "final" },
+    },
+    output: ({ context }) =>
+      context.error === undefined
+        ? { status: "succeeded", result: context.result }
+        : { status: "failed", error: context.error },
+  });
+
+  const actor = createActor(machine);
+  if (observe) actor.subscribe((snapshot) => observe(String(snapshot.value) as OptimizationPhase));
+  actor.start();
+  const outcome = await toPromise(actor);
+  if (outcome.status === "failed") throw outcome.error;
+  return outcome.result as Result;
+};