@prismatic-io/lux 0.0.1 → 0.0.2-preview.4

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@prismatic-io/lux",
-  "version": "0.0.1",
+  "version": "0.0.2-preview.4",
   "description": "Coding-agent evaluation and improvement with deterministic assertions, optional LLM judges, and first-class human-in-the-loop runs.",
   "license": "MIT",
   "homepage": "https://github.com/prismatic-io/lux",
@@ -73,7 +73,6 @@
     "@modelcontextprotocol/sdk": "^1.30.0",
     "@vscode/windows-process-tree": "0.8.0",
     "commander": "^15.0.0",
-    "js-yaml": "^5.2.2",
     "oxc-parser": "^0.142.0",
     "tempy": "^3.2.0",
     "tinyexec": "1.2.4",

package/skills/lux/SKILL.md

@@ -1,6 +1,6 @@
 ---
 name: lux
-description: Author, validate, run, inspect, compare, and improve coding-agent evaluations with the Lux CLI. Use when creating Lux cases or campaigns, choosing assertions/personas/drivers, diagnosing eval failures, comparing runs, or running Lux experiments.
+description: Author, validate, run, inspect, compare, and improve coding-agent evaluations with the Lux CLI. Use when creating or reviewing Lux cases, custom assertions, lifecycle fixtures, driver or harness configuration, campaigns, grader annotations, diagnosing eval failures, comparing runs, or running and promoting Lux experiments.
 ---
 
 # Lux
@@ -16,7 +16,7 @@ only for qualities that artifacts and commands cannot establish.
 2. If Lux is not initialized, run `npm exec -- lux init` for ordinary evals or
    `npm exec -- lux init --experiment` for an improvement campaign.
 3. Read [references/eval-authoring.md](references/eval-authoring.md) before
-   authoring or reviewing a case.
+   authoring or reviewing a case, assertion plugin, or lifecycle fixture.
 4. Read [references/cli.md](references/cli.md) before running, diagnosing,
    comparing, or promoting results.
 5. Run `npm exec -- lux doctor` before model calls. Treat its errors as
@@ -40,6 +40,7 @@ only for qualities that artifacts and commands cannot establish.
 
 ```bash
 npm exec -- lux doctor
+npm exec -- lux run <case-or-filter> --list
 npm exec -- lux run <case-or-filter> --loop 3
 npm exec -- lux view <run-or-suite>
 ```
@@ -50,7 +51,8 @@ before changing the case. Separate subject failure from harness failure:
 - Subject failure: the agent misunderstood, asked poorly, used the wrong tools,
   or left incorrect artifacts.
 - Harness failure: the prompt was underspecified, the persona lacked a needed
-  fact, an assertion measured a proxy, or the judge criterion was ambiguous.
+  fact, a fixture or driver failed, an assertion measured a proxy, or the
+  judge criterion was ambiguous.
 
 Do not weaken a valid assertion merely to make a run pass.
 
@@ -78,6 +80,8 @@ Never edit a materialized candidate or managed experiment record by hand.
   behavior.
 - Do not use only `run-succeeded` for a case that claims to verify a concrete
   result.
+- Do not invoke bare `lux`; it displays help. Use the explicit `lux run` verb
+  and narrow by case, path, or tag before any model call.
 - Do not let train examples leak into validation or test splits.
 - Do not combine evidence across changed models, harnesses, cases, or subjects;
   Lux identity checks exist to prevent that.

package/skills/lux/references/cli.md

@@ -3,11 +3,19 @@
 Invoke the project-local CLI with `npm exec -- lux ...` unless the repository
 uses another package-manager convention.
 
+Bare `lux` prints command and agent-skill help. Running cases always requires
+the explicit `run` verb; do not assume an unfiltered run is harmless.
+Use `lux <command> --help` before composing an unfamiliar or automation-facing
+invocation; it is the authoritative option list for the installed version.
+
 ## Author and preflight
 
 ```bash
 lux init
 lux init --experiment
+lux skills list
+lux skills install codex --project
+lux skills install claude --project
 lux doctor
 lux doctor experiments/campaign.experiment.ts --json
 lux run --list
@@ -27,19 +35,27 @@ lux run cases/retry-flag.ts
 lux run --tag smoke
 lux run retry --loop 5 --concurrency 2
 lux run one-case --interactive
+lux run one-case --claude-answerer
+lux run one-case --profile small-low --subject-root . --verbose
+lux run one-case --skip-judge
 ```
 
 Filters match case IDs and paths. Tags come from `meta.tags`. A run invocation
 also creates a suite record, which is the right unit for repeated or multi-case
 comparison.
 
+`--skip-judge` leaves subject execution unchanged but does not call rubric LLM
+judges. Deterministic assertions still determine the verdict; judge-only cases
+remain ungraded, and persisted grading records identify the skipped rubrics.
+
 ## Inspect and compare
 
 ```bash
 lux view
 lux view <run-or-suite>
+lux view --suites --json
 lux view --experiments
-lux compare <run-or-suite-a> <run-or-suite-b>
+lux compare <run-or-suite-a> <run-or-suite-b> --pass-rate-tolerance 0.05
 ```
 
 Use `view` before opening raw files. When diagnosing, inspect:
@@ -64,7 +80,9 @@ lux annotate <run> \
 ```
 
 Regrading preserves subject evidence and changes only grading. Human annotations
-support grader-alignment experiments; keep assertion IDs aligned with the
+support grader-alignment experiments. `annotate` can record overall label or
+score, repeatable assertion outcomes, split/tag selection, feedback, annotator,
+reviewer, and human/synthetic provenance. Keep assertion IDs aligned with the
 rubric and record reviewer context where available.
 
 ## Experiment lifecycle
@@ -72,11 +90,13 @@ rubric and record reviewer context where available.
 ```bash
 lux experiment <campaign> --plan
 lux experiment <campaign>
+lux experiment <campaign> --resume <experiment-dir>
 lux optimize <campaign> --plan
 lux optimize <campaign>
 lux optimize <campaign> --resume <experiment-dir>
 lux view <experiment-dir>
 lux report <experiment-dir> --out .lux-runs/reports/decision.md
+lux report <experiment-dir> --json
 lux apply <experiment-dir>
 ```
 
@@ -96,6 +116,9 @@ Use the exact resume, report, and apply commands printed by Lux. Do not edit
 - Agent exit/error: inspect `events.jsonl` and the driver section of `run.json`.
 - Assertion failure: inspect the assertion evidence and final artifact; decide
   whether the subject or measurement is wrong.
+- Tool evidence mismatch: compare paired call/result IDs and inspect unresolved,
+  unmatched, malformed, and failed tool outcomes; never infer success from a
+  call event alone.
 - Judge disagreement: narrow the rubric, regrade stored evidence, and collect
   human annotations before changing the subject.
 - Interrupted experiment: resume the printed experiment directory; do not start

package/skills/lux/references/eval-authoring.md

@@ -51,13 +51,71 @@ Prefer deterministic assertions:
 - `run-succeeded`: the driver completed; necessary but rarely sufficient.
 - `file-exists`: a required artifact exists.
 - `file-contains`: a stable literal is present.
-- `json-path-equals`: structured output contains the expected value.
+- `json-pointer-equals`: structured output contains the expected value.
 - `command-exits-zero`: project-authored validation succeeds.
 
+Use the more specific bundled evidence checks when they match the contract:
+
+- Files: `file-matches`, `file-regex-set`, `file-regex-count`, and `glob-count`.
+- Output/data: `contains`, `not-contains`, `regex`, and `json-pointer-equals`.
+- Interaction: `interrupt-count`, `tool-called`, `tool-called-with`, and
+  `tool-call-count`; use `by: "main"` or `by: "subagent"` when ownership matters.
+- Paired tool evidence: `tool-invocation`, `tool-result-contains`,
+  `tool-result-json-pointer-equals`, `tool-result-is-error`, and
+  `no-error-results`. Prefer these when result status or payload matters; a tool
+  call without a paired result is unresolved, not successful.
+- Budgets: `latency-under`, `tokens-under`, `cost-under`, and
+  `artifact-bytes-under`.
+- MCP discovery: `server-info-equals`, `tool-listed`,
+  `tools-listed-count`, and `capability-absent`.
+
+Most file assertions read captured `artifacts`; set `where: "cwd"` only when
+the durable contract intentionally includes the staged run workspace. Use
+stable assertion `id` values for repetitions, comparisons, annotations, and
+experiments. `weight` affects aggregate score and `threshold` sets an optional
+per-assertion score floor.
+
 Use `rubric` for semantic properties such as design quality, completeness, or
 whether a test meaningfully covers behavior. Keep criteria narrow. Avoid words
 like “good,” “proper,” or “best practice” without observable conditions.
 
+Bound rubric evidence to what the criterion needs. Include `tool-result` event
+evidence when status matters, and request tool inputs/results only when their
+payload is necessary; this controls judge context and cost.
+
+Persona answering and rubric judging have different difficulty profiles. A
+persona usually chooses from facts already supplied in the case, so prefer a
+cheap model and low effort. Semantic grading should use a stronger model and
+enough effort to distinguish real evidence from plausible prose. Configure
+them independently when that tradeoff matters:
+
+```ts
+// lux.config.ts
+defaultAnswerer: {
+  name: "persona",
+  config: {
+    provider: "codex",
+    model: "YOUR_SMALL_PERSONA_MODEL",
+    reasoningEffort: "low",
+  },
+},
+
+// case assertion
+{
+  id: "behavior-quality",
+  type: "rubric",
+  criteria: "The tests exercise retry exhaustion and recovery.",
+  provider: "codex",
+  model: "YOUR_STRONG_JUDGE_MODEL",
+  reasoningEffort: "high",
+}
+```
+
+The project `harness` supplies shared defaults to persona and rubric calls;
+answerer config and rubric fields override it independently. Always pin model
+and effort when comparing runs, and inspect answerer versus judge usage in the
+run summary before increasing either budget.
+
 Assertions should test user-visible behavior and durable contracts. Avoid
 requiring a specific function name, file layout, or algorithm unless that is
 the contract under evaluation.
@@ -73,9 +131,51 @@ The persona is simulated user state, not another system prompt. Include:
 Do not tell the persona how to help the agent solve the task. If no interaction
 is part of the capability being evaluated, omit the persona or keep it minimal.
 
-Use `interactionMode: "defer-resume"` for automated Claude Code question-tool
-handling. Use `textQuestionFallback: true` only when intentionally evaluating
-models that ask in prose instead of using the supported tool.
+For Claude Code, use `interactionMode: "defer-resume"` for automated
+question-tool handling. Use `textQuestionFallback: true` only when
+intentionally evaluating models that ask in prose instead of using the
+supported tool. For Codex, keep the default `interactionMode: "app-server"`
+when the case can interrupt; compatibility mode `"exec"` cannot accept
+mid-run responses.
+
+Use `--interactive` for a real terminal user or `--claude-answerer` for the
+Claude plugin protocol. Both force sequential execution.
+
+## Project extensions
+
+For a reusable project-specific assertion, define it with `defineAssertion`
+and a direct Zod object whose `type` is a matching literal. Import `z` from Lux,
+return an ordinary failed result for missing/invalid product evidence, and
+reserve throws or `errorMessage` for evaluator faults. Increment
+`pluginVersion` whenever schema or evaluation behavior changes. Prefer the
+bounded, traversal-safe `readRunArtifactText` helper over direct filesystem
+reads.
+
+Build a typed case authoring function once instead of importing plugins in
+every case:
+
+```ts
+const assertions = defineAssertionRegistry(projectAssertion);
+export const defineEvalCase = defineLuxCase.extend({ assertions });
+```
+
+Case-bound plugins remain plain serialized plans but require the current case
+path for regrading: `lux grade <run> --case <case.ts>`.
+
+Use `defineFixtures` for Vitest-style suite/case setup around `await use()` or
+`defineLifecycleHooks` when cleanup-returning callbacks fit better. Setup runs
+in declaration order and teardown in reverse, including failures. Call
+`await use()` exactly once. Fixture `value`/`identity` must be public JSON;
+keep secrets in the prepared resource or environment. Increment
+`pluginVersion` when lifecycle behavior changes. Register globally in
+`lux.config.ts`, or attach selectively with
+`defineEvalCase.extend({ fixtures: [...] })`.
+
+Custom drivers that own an in-memory filesystem should persist it with
+`writeArtifactSnapshot` from `@prismatic-io/lux/artifact-snapshot`. Snapshots
+are bounded, traversal-safe, immutable per name, and can use `baseSnapshot`
+for copy-on-write reuse while remaining independently complete. Ordinary file
+assertions then grade the indexed files without a snapshot-specific API.
 
 ## Experiments for prompts, skills, and agent source
 
@@ -100,6 +200,58 @@ Campaigns should:
 - budget calls, tokens, cost, and wall time;
 - use repetitions and confidence gates appropriate to stochastic outcomes;
 - reserve held-out cases for promotion and final confirmation.
+- acknowledge every candidate validation command with
+  `unsafeHostExecution: true`; validators must be observational and must not
+  mutate the candidate tree.
+
+A minimal current agent campaign has this shape (selectors are objects, mutable
+paths are candidate-relative globs, and budgets use call/token/cost limits):
+
+```ts
+export default defineExperiment({
+  id: "skill-improvement",
+  subject: { root: "..", mutable: ["skills/example/**"] },
+  profiles: ["claude-low", "codex-low"],
+  splits: {
+    train: { tags: ["train"] },
+    validation: { tags: ["validation"] },
+    test: { tags: ["test"] },
+  },
+  repetitions: { search: 1, promotion: 3 },
+  objectives: {
+    constraints: [
+      { metric: "quality.caseRegressions", max: 0 },
+      { metric: "quality.passRate", min: 0.8, split: "test" },
+    ],
+    minimize: [{ metric: "agent.total" }, { metric: "subject.bytes" }],
+  },
+  optimizer: {
+    algorithm: "reflective",
+    candidateSelection: "pareto",
+    promotionTopK: 2,
+    config: {
+      provider: "codex",
+      model: "YOUR_OPTIMIZER_MODEL",
+      reasoningEffort: "low",
+    },
+  },
+  budget: {
+    maxCandidates: 4,
+    patience: 1,
+    maxMetricCalls: 60,
+    maxJudgeCalls: 120,
+    maxProposalCalls: 3,
+    maxTokens: 2_000_000,
+    maxCostUsd: 30,
+    maxUnpricedModelCalls: 0,
+    maxWallTimeMs: 3_600_000,
+  },
+});
+```
+
+Use the experiment directory printed by the run for `view` and `report`.
+Review its held-out evidence, then pass that same verified directory to
+`apply`; do not guess a directory from the campaign ID.
 
 Use `experiment` for authored variants and `optimize` for model-proposed source
 changes. Always run the corresponding `--plan` command first.

package/skills/lux-answerer/SKILL.md

@@ -1,6 +1,6 @@
 ---
 name: lux-answerer
-version: 0.0.1
+version: 0.0.2-preview.4
 description: Play the persona for a running Lux orchestrator that uses the claude-code answerer. Read structured events on stdout, decide answers from persona + context, write JSON to its answer channel.
 user-invocable: false
 allowed-tools: Bash, Read

package/src/answerers/terminal/index.ts

@@ -70,7 +70,7 @@ class TerminalAnswererImpl implements Answerer {
     if (!stdin.isTTY || !stdout.isTTY) {
       throw new Error(
         "terminal answerer: --interactive requires an attached terminal; " +
-          "use a scripted/model answerer in CI or --claude-answerer from the Claude plugin",
+          "use a model answerer in CI or --claude-answerer from the Claude plugin",
       );
     }
     this.interface = createInterface({ input: stdin, output: stdout, terminal: true });

package/src/assertions/authoring.ts

@@ -0,0 +1,79 @@
+import type { z } from "zod";
+import { createEvalCaseDefinition } from "../core/case.js";
+import type {
+  ArtifactBytesUnderConfigSchema,
+  CommandExitsZeroConfigSchema,
+  ContainsConfigSchema,
+  CostUnderConfigSchema,
+  FileContainsConfigSchema,
+  FileExistsConfigSchema,
+  FileMatchesConfigSchema,
+  FileRegexCountConfigSchema,
+  FileRegexSetConfigSchema,
+  GlobCountConfigSchema,
+  InterruptCountConfigSchema,
+  JsonPointerEqualsConfigSchema,
+  LatencyUnderConfigSchema,
+  NoErrorResultsConfigSchema,
+  NotContainsConfigSchema,
+  PredicateConfigSchema,
+  RegexConfigSchema,
+  RunSucceededConfigSchema,
+  TokensUnderConfigSchema,
+  ToolCallCountConfigSchema,
+  ToolCalledConfigSchema,
+  ToolCalledWithConfigSchema,
+  ToolInvocationConfigSchema,
+  ToolResultContainsConfigSchema,
+  ToolResultIsErrorConfigSchema,
+  ToolResultJsonPointerEqualsConfigSchema,
+} from "./core/index.js";
+import type {
+  CapabilityAbsentConfigSchema,
+  ServerInfoEqualsConfigSchema,
+  ToolListedConfigSchema,
+  ToolsListedCountConfigSchema,
+} from "./mcp/index.js";
+import type { RubricConfigSchema } from "./rubric/index.js";
+
+type BuiltinAssertionSchema =
+  | typeof ArtifactBytesUnderConfigSchema
+  | typeof CapabilityAbsentConfigSchema
+  | typeof CommandExitsZeroConfigSchema
+  | typeof ContainsConfigSchema
+  | typeof CostUnderConfigSchema
+  | typeof FileContainsConfigSchema
+  | typeof FileExistsConfigSchema
+  | typeof FileMatchesConfigSchema
+  | typeof FileRegexCountConfigSchema
+  | typeof FileRegexSetConfigSchema
+  | typeof GlobCountConfigSchema
+  | typeof InterruptCountConfigSchema
+  | typeof JsonPointerEqualsConfigSchema
+  | typeof LatencyUnderConfigSchema
+  | typeof NoErrorResultsConfigSchema
+  | typeof NotContainsConfigSchema
+  | typeof PredicateConfigSchema
+  | typeof RegexConfigSchema
+  | typeof RubricConfigSchema
+  | typeof RunSucceededConfigSchema
+  | typeof ServerInfoEqualsConfigSchema
+  | typeof TokensUnderConfigSchema
+  | typeof ToolCalledConfigSchema
+  | typeof ToolCalledWithConfigSchema
+  | typeof ToolCallCountConfigSchema
+  | typeof ToolInvocationConfigSchema
+  | typeof ToolListedConfigSchema
+  | typeof ToolResultContainsConfigSchema
+  | typeof ToolResultIsErrorConfigSchema
+  | typeof ToolResultJsonPointerEqualsConfigSchema
+  | typeof ToolsListedCountConfigSchema;
+
+export type BuiltinAssertion = z.input<BuiltinAssertionSchema> & {
+  id?: string;
+  name?: string;
+  weight?: number;
+  threshold?: number;
+};
+
+export const defineEvalCase = createEvalCaseDefinition<BuiltinAssertion>();

package/src/assertions/core/helpers.ts

@@ -3,6 +3,7 @@ import { lstat } from "node:fs/promises";
 import { isAbsolute, join, relative, resolve, sep } from "node:path";
 import { z } from "zod";
 import { inspectRunArtifact, runArtifactsRoot } from "../../core/artifact-evidence.js";
+import { baseAssertionFields } from "../../core/assertion.js";
 import type { Run, RunEvent } from "../../core/index.js";
 
 const FsLocationSchema = z.enum(["artifacts", "cwd"]);
@@ -175,12 +176,6 @@ export const deepEquals = (a: unknown, b: unknown): boolean => {
  * Mirrors {@link Assertion} from the core module while letting each built-in
  * assertion reject misspelled type-specific fields at its own schema boundary.
  */
-export const baseAssertionFields = {
-  id: z.string().min(1).optional(),
-  weight: z.number().positive().optional(),
-  threshold: z.number().min(0).max(1).optional(),
-  name: z.string().optional(),
-} as const;
 
 /**
  * Where on the run dir an assertion looks for files. Defaults to
@@ -190,6 +185,8 @@ export const LocationSchema = FsLocationSchema.default("artifacts");
 
 // `name` is intentionally excluded: for tool assertions the `name` field is
 // the tool's name, which then doubles as the display label.
+export { baseAssertionFields };
+
 export const weightThresholdFields = {
   id: baseAssertionFields.id,
   weight: baseAssertionFields.weight,

package/src/authoring.ts

@@ -1,2 +1,3 @@
 export { z } from "zod";
+export { defineEvalCase } from "./assertions/authoring.js";
 export * from "./core/index.js";

package/src/cli/program.ts

@@ -163,7 +163,7 @@ const registerExperimentCommands = (program: Command): void => {
     });
 
   program
-    .command("run [filters...]", { isDefault: true })
+    .command("run [filters...]")
     .description("discover and run cases; filter by name/path substring or --tag")
     .option(
       "-t, --tag <tag>",
@@ -184,6 +184,7 @@ const registerExperimentCommands = (program: Command): void => {
       "--claude-answerer",
       "delegate questions through the Claude plugin's lux-answerer skill",
     )
+    .option("--skip-judge", "skip rubric LLM judge assertions")
     .option("--runs-root <path>", "run-dir root (overrides lux.config.ts)")
     .option("--profile <id>", "named driver/answerer profile from lux.config.ts")
     .option("--subject-root <path>", "root bound into subjectPath() values")
@@ -623,11 +624,16 @@ export const buildProgram = (): Command => {
     .description(
       "Coding-agent evaluation and improvement with deterministic assertions and human-in-the-loop runs",
     )
-    .version(PKG_VERSION);
+    .version(PKG_VERSION)
+    .addHelpText(
+      "after",
+      "\nAgent skills:\n  $ lux skills list\n  $ lux skills install codex --project\n  $ lux skills install claude --project",
+    );
   registerProjectCommands(program);
   registerExperimentCommands(program);
   registerCandidateCommands(program);
   registerReportingCommands(program);
+  program.action(() => program.outputHelp());
   return program;
 };
 
@@ -965,6 +971,7 @@ const runCommand = async (filters: string[], options: RunOptions): Promise<void>
       runsRoot,
       fixturesRoot: config.fixturesRoot,
       answererOverride: resolveAnswererOverride(options),
+      skipJudge: options.skipJudge,
       observer: reporter,
       abortSignal: controller.signal,
       concurrency,

package/src/cli/render/reporter.ts

@@ -4,6 +4,7 @@ import {
   createToolOutcomeAccumulator,
   emptyRunUsage,
   type GradingAssertionEntry,
+  type GradingReport,
   harnessUsage,
   isUsageEmpty,
   type LifecycleStage,
@@ -77,6 +78,7 @@ type CaseView = {
   assertionTotal: number;
   currentAssertion: string | null;
   assertions: GradingAssertionEntry[];
+  skippedAssertions: NonNullable<GradingReport["skippedAssertions"]>;
   usage: RunUsage;
   /** Per-API-call agent usage, in order — the split behind `usage.agent`. */
   agentTurns: TurnUsage[];
@@ -117,7 +119,10 @@ const statusMark = (status: CaseStatus, theme: Theme): string => {
 
 const assertionTally = (view: CaseView): string => {
   const passed = view.assertions.filter((a) => a.passed).length;
-  return view.assertions.length === 0 ? "—" : `${passed}/${view.assertions.length}`;
+  const evaluated = view.assertions.length === 0 ? "—" : `${passed}/${view.assertions.length}`;
+  return view.skippedAssertions.length > 0
+    ? `${evaluated} · ${view.skippedAssertions.length} skipped`
+    : evaluated;
 };
 
 /**
@@ -233,6 +238,7 @@ export class ConsoleReporter implements SuiteObserver {
       assertionTotal: 0,
       currentAssertion: null,
       assertions: [],
+      skippedAssertions: [],
       usage: emptyRunUsage(),
       agentTurns: [],
       phaseDurationMs: null,
@@ -299,6 +305,7 @@ export class ConsoleReporter implements SuiteObserver {
     if (result.errorMessage) view.errorMessage = result.errorMessage;
     if (result.result) {
       view.assertions = result.result.grading.assertions;
+      view.skippedAssertions = result.result.grading.skippedAssertions ?? [];
       view.usage = result.result.metadata.usage;
       view.phaseDurationMs = result.result.metadata.phaseDurationMs ?? null;
       const tools = result.result.metadata.toolOutcomes;
@@ -315,6 +322,7 @@ export class ConsoleReporter implements SuiteObserver {
       // Assertions reported before the run failed to persist are not results
       // and must not inflate the closing tally of a case that errored.
       view.assertions = [];
+      view.skippedAssertions = [];
     }
 
     this.region.staticLine(this.finishedLine(view));
@@ -452,6 +460,11 @@ export class ConsoleReporter implements SuiteObserver {
         body: !entry.passed && entry.comment ? [this.theme.dim(entry.comment)] : [],
       });
     }
+    for (const skipped of view.skippedAssertions) {
+      nodes.push({
+        head: `${this.theme.dim("⊘")} ${skipped.name}  ${this.theme.dim("skipped: judge disabled")}`,
+      });
+    }
 
     if (view.toolCalls > 0 || view.toolUnmatchedResults > 0 || view.toolMalformed > 0) {
       const parts = [
@@ -550,15 +563,17 @@ export class ConsoleReporter implements SuiteObserver {
     }
 
     const assertions = this.cases.flatMap((c) => c.assertions);
-    if (assertions.length > 0) {
+    const skippedAssertions = this.cases.flatMap((c) => c.skippedAssertions);
+    if (assertions.length > 0 || skippedAssertions.length > 0) {
       const ok = assertions.filter((a) => a.passed).length;
       const bad = assertions.length - ok;
       const tally = formatTally(
         [
           [ok, this.theme.green("passed")],
           [bad, this.theme.red("failed")],
+          [skippedAssertions.length, this.theme.dim("skipped")],
         ],
-        assertions.length,
+        assertions.length + skippedAssertions.length,
       );
       lines.push(` ${label("Assertions")}${tally}`);
     }

package/src/cli/run-options.ts

@@ -8,6 +8,7 @@ export type RunOptions = {
   listTags?: boolean;
   interactive?: boolean;
   claudeAnswerer?: boolean;
+  skipJudge?: boolean;
   runsRoot?: string;
   profile?: string;
   subjectRoot?: string;