@prismatic-io/lux 0.0.1 → 0.0.2-preview.11

package/src/cli/program.ts

@@ -24,25 +24,15 @@ import {
   formatDoctorReport,
   formatExperimentDecisionMarkdown,
   formatExperimentRun,
-  formatExperimentRunsTable,
-  formatRunDetail,
-  formatRunsTable,
   formatSuiteComparisonText,
-  formatSuiteRunDetail,
-  formatSuiteRunsTable,
   hasRegression,
   hasSuiteRegression,
-  isExperimentRun,
-  listExperimentRuns,
-  listRuns,
-  listSuiteRuns,
   loadEvalCase,
   loadExperimentCampaign,
   loadExperimentRun,
   loadFrozenRegradableRun,
   loadLuxConfig,
   loadRubricDefinition,
-  loadRunDetail,
   loadSuiteRun,
   loadVerifiedExperimentDecisionSnapshot,
   Orchestrator,
@@ -84,6 +74,7 @@ import {
   installSkills,
   type SkillsPlatform,
 } from "./skills.js";
+import { parsePreviewPort, viewCommand } from "./view.js";
 
 const requireFromHere = createRequire(import.meta.url);
 // From lib/cli/program.js (or src/cli/program.ts under test) to the package root.
@@ -163,7 +154,7 @@ const registerExperimentCommands = (program: Command): void => {
     });
 
   program
-    .command("run [filters...]", { isDefault: true })
+    .command("run [filters...]")
     .description("discover and run cases; filter by name/path substring or --tag")
     .option(
       "-t, --tag <tag>",
@@ -184,6 +175,7 @@ const registerExperimentCommands = (program: Command): void => {
       "--claude-answerer",
       "delegate questions through the Claude plugin's lux-answerer skill",
     )
+    .option("--skip-judge", "skip rubric LLM judge assertions")
     .option("--runs-root <path>", "run-dir root (overrides lux.config.ts)")
     .option("--profile <id>", "named driver/answerer profile from lux.config.ts")
     .option("--subject-root <path>", "root bound into subjectPath() values")
@@ -495,88 +487,11 @@ const registerReportingCommands = (program: Command): void => {
     .option("--case <id>", "filter the run list to one case slug (list mode only)")
     .option("--suites", "list logical suite runs instead of individual runs")
     .option("--experiments", "list experiment runs instead of individual runs")
+    .option("--web", "open the local Lux Review web app")
+    .option("--port <n>", "listen on a specific local port in web mode", parsePreviewPort)
+    .option("--no-open", "start web mode without opening a browser")
     .option("--json", "emit machine-readable JSON")
-    .action(
-      async (
-        runDirArg: string | undefined,
-        options: {
-          runsRoot?: string;
-          case?: string;
-          suites?: boolean;
-          experiments?: boolean;
-          json?: boolean;
-        },
-      ) => {
-        if (runDirArg) {
-          const path = resolve(runDirArg);
-          if (!(await ensureDirectory("view", path))) return;
-          if (isExperimentRun(path)) {
-            const experiment = await loadExperimentRun(path);
-            if (!experiment) {
-              process.stderr.write(`lux view: invalid experiment at ${path}\n`);
-              failWith(1);
-              return;
-            }
-            process.stdout.write(
-              options.json
-                ? `${JSON.stringify({ experimentDir: path, ...experiment }, null, 2)}\n`
-                : `${formatExperimentRun(path, experiment)}\n`,
-            );
-            return;
-          }
-          const suite = await loadSuiteRun(path);
-          if (suite) {
-            process.stdout.write(
-              options.json
-                ? `${JSON.stringify({ suiteDir: path, ...suite }, null, 2)}\n`
-                : `${formatSuiteRunDetail(path, suite)}\n`,
-            );
-            return;
-          }
-          const detail = await loadRunDetail(path);
-          if (options.json) {
-            process.stdout.write(`${JSON.stringify(detail, null, 2)}\n`);
-          } else {
-            process.stdout.write(`${formatRunDetail(detail)}\n`);
-          }
-          return;
-        }
-
-        const cwd = process.cwd();
-        const config = await loadLuxConfig(cwd);
-        const runsRoot = options.runsRoot ? resolve(options.runsRoot) : config.runsRoot;
-        if (options.experiments) {
-          const experiments = await listExperimentRuns(runsRoot);
-          process.stdout.write(
-            options.json
-              ? `${JSON.stringify(experiments, null, 2)}\n`
-              : `experiments at ${runsRoot}:\n${formatExperimentRunsTable(experiments)}\n`,
-          );
-          return;
-        }
-        if (options.suites) {
-          const suites = await listSuiteRuns(runsRoot);
-          process.stdout.write(
-            options.json
-              ? `${JSON.stringify(suites, null, 2)}\n`
-              : `suite runs at ${runsRoot}:\n${formatSuiteRunsTable(suites)}\n`,
-          );
-          return;
-        }
-        const all = await listRuns(runsRoot);
-        const runs = options.case ? all.filter((r) => r.caseSlug === options.case) : all;
-        if (options.json) {
-          process.stdout.write(`${JSON.stringify(runs, null, 2)}\n`);
-        } else {
-          if (!options.case) {
-            process.stdout.write(`suite runs at ${runsRoot}:\n`);
-            process.stdout.write(`${formatSuiteRunsTable(await listSuiteRuns(runsRoot))}\n\n`);
-          }
-          process.stdout.write(`runs at ${runsRoot}:\n`);
-          process.stdout.write(`${formatRunsTable(runs)}\n`);
-        }
-      },
-    );
+    .action(viewCommand);
 
   program
     .command("compare <runA> <runB>")
@@ -623,11 +538,16 @@ export const buildProgram = (): Command => {
     .description(
       "Coding-agent evaluation and improvement with deterministic assertions and human-in-the-loop runs",
     )
-    .version(PKG_VERSION);
+    .version(PKG_VERSION)
+    .addHelpText(
+      "after",
+      "\nAgent skills:\n  $ lux skills list\n  $ lux skills install codex --project\n  $ lux skills install claude --project",
+    );
   registerProjectCommands(program);
   registerExperimentCommands(program);
   registerCandidateCommands(program);
   registerReportingCommands(program);
+  program.action(() => program.outputHelp());
   return program;
 };
 
@@ -965,6 +885,7 @@ const runCommand = async (filters: string[], options: RunOptions): Promise<void>
       runsRoot,
       fixturesRoot: config.fixturesRoot,
       answererOverride: resolveAnswererOverride(options),
+      skipJudge: options.skipJudge,
       observer: reporter,
       abortSignal: controller.signal,
       concurrency,

package/src/cli/render/reporter.ts

@@ -4,6 +4,7 @@ import {
   createToolOutcomeAccumulator,
   emptyRunUsage,
   type GradingAssertionEntry,
+  type GradingReport,
   harnessUsage,
   isUsageEmpty,
   type LifecycleStage,
@@ -77,6 +78,7 @@ type CaseView = {
   assertionTotal: number;
   currentAssertion: string | null;
   assertions: GradingAssertionEntry[];
+  skippedAssertions: NonNullable<GradingReport["skippedAssertions"]>;
   usage: RunUsage;
   /** Per-API-call agent usage, in order — the split behind `usage.agent`. */
   agentTurns: TurnUsage[];
@@ -117,7 +119,10 @@ const statusMark = (status: CaseStatus, theme: Theme): string => {
 
 const assertionTally = (view: CaseView): string => {
   const passed = view.assertions.filter((a) => a.passed).length;
-  return view.assertions.length === 0 ? "—" : `${passed}/${view.assertions.length}`;
+  const evaluated = view.assertions.length === 0 ? "—" : `${passed}/${view.assertions.length}`;
+  return view.skippedAssertions.length > 0
+    ? `${evaluated} · ${view.skippedAssertions.length} skipped`
+    : evaluated;
 };
 
 /**
@@ -233,6 +238,7 @@ export class ConsoleReporter implements SuiteObserver {
       assertionTotal: 0,
       currentAssertion: null,
       assertions: [],
+      skippedAssertions: [],
       usage: emptyRunUsage(),
       agentTurns: [],
       phaseDurationMs: null,
@@ -299,6 +305,7 @@ export class ConsoleReporter implements SuiteObserver {
     if (result.errorMessage) view.errorMessage = result.errorMessage;
     if (result.result) {
       view.assertions = result.result.grading.assertions;
+      view.skippedAssertions = result.result.grading.skippedAssertions ?? [];
       view.usage = result.result.metadata.usage;
       view.phaseDurationMs = result.result.metadata.phaseDurationMs ?? null;
       const tools = result.result.metadata.toolOutcomes;
@@ -315,6 +322,7 @@ export class ConsoleReporter implements SuiteObserver {
       // Assertions reported before the run failed to persist are not results
       // and must not inflate the closing tally of a case that errored.
       view.assertions = [];
+      view.skippedAssertions = [];
     }
 
     this.region.staticLine(this.finishedLine(view));
@@ -452,6 +460,11 @@ export class ConsoleReporter implements SuiteObserver {
         body: !entry.passed && entry.comment ? [this.theme.dim(entry.comment)] : [],
       });
     }
+    for (const skipped of view.skippedAssertions) {
+      nodes.push({
+        head: `${this.theme.dim("⊘")} ${skipped.name}  ${this.theme.dim("skipped: judge disabled")}`,
+      });
+    }
 
     if (view.toolCalls > 0 || view.toolUnmatchedResults > 0 || view.toolMalformed > 0) {
       const parts = [
@@ -550,15 +563,17 @@ export class ConsoleReporter implements SuiteObserver {
     }
 
     const assertions = this.cases.flatMap((c) => c.assertions);
-    if (assertions.length > 0) {
+    const skippedAssertions = this.cases.flatMap((c) => c.skippedAssertions);
+    if (assertions.length > 0 || skippedAssertions.length > 0) {
       const ok = assertions.filter((a) => a.passed).length;
       const bad = assertions.length - ok;
       const tally = formatTally(
         [
           [ok, this.theme.green("passed")],
           [bad, this.theme.red("failed")],
+          [skippedAssertions.length, this.theme.dim("skipped")],
         ],
-        assertions.length,
+        assertions.length + skippedAssertions.length,
       );
       lines.push(` ${label("Assertions")}${tally}`);
     }

package/src/cli/run-options.ts

@@ -8,6 +8,7 @@ export type RunOptions = {
   listTags?: boolean;
   interactive?: boolean;
   claudeAnswerer?: boolean;
+  skipJudge?: boolean;
   runsRoot?: string;
   profile?: string;
   subjectRoot?: string;

package/src/cli/view.ts

@@ -0,0 +1,168 @@
+import { resolve } from "node:path";
+import { InvalidArgumentError } from "commander";
+import {
+  formatExperimentRun,
+  formatExperimentRunsTable,
+  formatRunDetail,
+  formatRunsTable,
+  formatSuiteRunDetail,
+  formatSuiteRunsTable,
+  isExperimentRun,
+  listExperimentRuns,
+  listRuns,
+  listSuiteRuns,
+  loadExperimentRun,
+  loadLuxConfig,
+  loadRunDetail,
+  loadSuiteRun,
+} from "../orchestrator/index.js";
+import { openPreviewUrl, startPreviewServer } from "../previewer/server.js";
+import { abortOnInterrupt, ensureDirectory, failWith } from "./command-runtime.js";
+
+const SIGINT_EXIT_CODE = 130;
+
+export type ViewOptions = {
+  runsRoot?: string;
+  case?: string;
+  suites?: boolean;
+  experiments?: boolean;
+  web?: boolean;
+  port?: number;
+  open?: boolean;
+  json?: boolean;
+};
+
+const previewKind = async (
+  initialPath: string | undefined,
+  options: ViewOptions,
+): Promise<"run" | "suite" | "experiment" | undefined> => {
+  if (initialPath) {
+    if (isExperimentRun(initialPath)) return "experiment";
+    if (await loadSuiteRun(initialPath)) return "suite";
+    return "run";
+  }
+  if (options.experiments) return "experiment";
+  if (options.suites) return "suite";
+  return undefined;
+};
+
+const webViewCommand = async (
+  runDirArg: string | undefined,
+  options: ViewOptions,
+): Promise<void> => {
+  if (options.json) {
+    throw new InvalidArgumentError("--web and --json are mutually exclusive");
+  }
+  const config = await loadLuxConfig(process.cwd());
+  const runsRoot = options.runsRoot ? resolve(options.runsRoot) : config.runsRoot;
+  const initialPath = runDirArg ? resolve(runDirArg) : undefined;
+  if (initialPath && !(await ensureDirectory("view", initialPath))) return;
+  const initialKind = await previewKind(initialPath, options);
+  const controller = abortOnInterrupt();
+  const preview = await startPreviewServer({
+    runsRoot,
+    ...(options.port !== undefined ? { port: options.port } : {}),
+    ...(initialPath ? { initialPath } : {}),
+    ...(initialKind ? { initialKind } : {}),
+    ...(options.case ? { caseFilter: options.case } : {}),
+    signal: controller.signal,
+  });
+  process.stdout.write(`Lux Review: ${preview.url}\n`);
+  process.stdout.write(`Reading evidence from ${runsRoot}\n`);
+  if (options.open !== false) {
+    try {
+      await openPreviewUrl(preview.url);
+    } catch (error) {
+      const message = error instanceof Error ? error.message : String(error);
+      process.stderr.write(`lux view: could not open a browser (${message})\n`);
+    }
+  }
+  await preview.closed;
+  if (controller.signal.aborted) failWith(SIGINT_EXIT_CODE);
+};
+
+export const viewCommand = async (
+  runDirArg: string | undefined,
+  options: ViewOptions,
+): Promise<void> => {
+  if (options.web) {
+    await webViewCommand(runDirArg, options);
+    return;
+  }
+  if (options.port !== undefined || options.open === false) {
+    throw new InvalidArgumentError("--port and --no-open require --web");
+  }
+  if (runDirArg) {
+    const path = resolve(runDirArg);
+    if (!(await ensureDirectory("view", path))) return;
+    if (isExperimentRun(path)) {
+      const experiment = await loadExperimentRun(path);
+      if (!experiment) {
+        process.stderr.write(`lux view: invalid experiment at ${path}\n`);
+        failWith(1);
+        return;
+      }
+      process.stdout.write(
+        options.json
+          ? `${JSON.stringify({ experimentDir: path, ...experiment }, null, 2)}\n`
+          : `${formatExperimentRun(path, experiment)}\n`,
+      );
+      return;
+    }
+    const suite = await loadSuiteRun(path);
+    if (suite) {
+      process.stdout.write(
+        options.json
+          ? `${JSON.stringify({ suiteDir: path, ...suite }, null, 2)}\n`
+          : `${formatSuiteRunDetail(path, suite)}\n`,
+      );
+      return;
+    }
+    const detail = await loadRunDetail(path);
+    process.stdout.write(
+      options.json ? `${JSON.stringify(detail, null, 2)}\n` : `${formatRunDetail(detail)}\n`,
+    );
+    return;
+  }
+
+  const config = await loadLuxConfig(process.cwd());
+  const runsRoot = options.runsRoot ? resolve(options.runsRoot) : config.runsRoot;
+  if (options.experiments) {
+    const experiments = await listExperimentRuns(runsRoot);
+    process.stdout.write(
+      options.json
+        ? `${JSON.stringify(experiments, null, 2)}\n`
+        : `experiments at ${runsRoot}:\n${formatExperimentRunsTable(experiments)}\n`,
+    );
+    return;
+  }
+  if (options.suites) {
+    const suites = await listSuiteRuns(runsRoot);
+    process.stdout.write(
+      options.json
+        ? `${JSON.stringify(suites, null, 2)}\n`
+        : `suite runs at ${runsRoot}:\n${formatSuiteRunsTable(suites)}\n`,
+    );
+    return;
+  }
+  const all = await listRuns(runsRoot);
+  const runs = options.case ? all.filter((run) => run.caseSlug === options.case) : all;
+  if (options.json) {
+    process.stdout.write(`${JSON.stringify(runs, null, 2)}\n`);
+    return;
+  }
+  if (!options.case) {
+    process.stdout.write(`suite runs at ${runsRoot}:\n`);
+    process.stdout.write(`${formatSuiteRunsTable(await listSuiteRuns(runsRoot))}\n\n`);
+  }
+  process.stdout.write(`runs at ${runsRoot}:\n`);
+  process.stdout.write(`${formatRunsTable(runs)}\n`);
+};
+
+export const parsePreviewPort = (value: string): number => {
+  const port = Number(value);
+  if (!Number.isInteger(port) || port < 0 || port > 65_535) {
+    throw new InvalidArgumentError("port must be an integer from 0 to 65535");
+  }
+  return port;
+};

package/src/core/action-events.ts

@@ -0,0 +1,59 @@
+import { z } from "zod";
+import type { RunEvent } from "./events.js";
+
+export const ActionCategorySchema = z.enum([
+  "query",
+  "mutation",
+  "durable-mutation",
+  "execution",
+  "interaction",
+]);
+export type ActionCategory = z.infer<typeof ActionCategorySchema>;
+
+export const ActionProgressPayloadSchema = z
+  .object({
+    name: z.string(),
+    category: ActionCategorySchema,
+  })
+  .strict();
+export type ActionProgressPayload = z.infer<typeof ActionProgressPayloadSchema>;
+
+export const ActionTimingSchema = z.object({
+  timeToFirstActionMs: z.exactOptional(z.number().nonnegative()),
+  timeToFirstMutationMs: z.exactOptional(z.number().nonnegative()),
+  timeToFirstDurableMutationMs: z.exactOptional(z.number().nonnegative()),
+});
+export type ActionTiming = z.infer<typeof ActionTimingSchema>;
+
+export const actionTiming = (events: readonly RunEvent[]): ActionTiming => {
+  const startEvent = events.find(
+    (event) => event.type === "lifecycle" && event.stage === "start-driver",
+  );
+  const startedAt = startEvent?.type === "lifecycle" ? startEvent.ts : undefined;
+  if (startedAt === undefined) return {};
+
+  const actions = events.flatMap((event) => {
+    if (event.type !== "progress" || event.kind !== "action") return [];
+    const payload = ActionProgressPayloadSchema.safeParse(event.payload).data;
+    return payload ? [{ ...payload, ts: event.ts }] : [];
+  });
+  const elapsed = (category: "any" | "mutation" | "durable-mutation") => {
+    const timestamps = actions.flatMap(({ category: candidate, ts }) => {
+      if (category === "any") return [ts];
+      if (category === "mutation") {
+        return candidate === "mutation" || candidate === "durable-mutation" ? [ts] : [];
+      }
+      return candidate === category ? [ts] : [];
+    });
+    return timestamps.length > 0 ? Math.max(0, Math.min(...timestamps) - startedAt) : undefined;
+  };
+
+  const timeToFirstActionMs = elapsed("any");
+  const timeToFirstMutationMs = elapsed("mutation");
+  const timeToFirstDurableMutationMs = elapsed("durable-mutation");
+  return {
+    ...(timeToFirstActionMs === undefined ? {} : { timeToFirstActionMs }),
+    ...(timeToFirstMutationMs === undefined ? {} : { timeToFirstMutationMs }),
+    ...(timeToFirstDurableMutationMs === undefined ? {} : { timeToFirstDurableMutationMs }),
+  };
+};