@prismatic-io/lux 0.0.2-preview.3 → 0.0.2-preview.4

package/skills/lux/references/eval-authoring.md

@@ -51,13 +51,71 @@ Prefer deterministic assertions:
 - `run-succeeded`: the driver completed; necessary but rarely sufficient.
 - `file-exists`: a required artifact exists.
 - `file-contains`: a stable literal is present.
-- `json-path-equals`: structured output contains the expected value.
+- `json-pointer-equals`: structured output contains the expected value.
 - `command-exits-zero`: project-authored validation succeeds.
 
+Use the more specific bundled evidence checks when they match the contract:
+
+- Files: `file-matches`, `file-regex-set`, `file-regex-count`, and `glob-count`.
+- Output/data: `contains`, `not-contains`, `regex`, and `json-pointer-equals`.
+- Interaction: `interrupt-count`, `tool-called`, `tool-called-with`, and
+  `tool-call-count`; use `by: "main"` or `by: "subagent"` when ownership matters.
+- Paired tool evidence: `tool-invocation`, `tool-result-contains`,
+  `tool-result-json-pointer-equals`, `tool-result-is-error`, and
+  `no-error-results`. Prefer these when result status or payload matters; a tool
+  call without a paired result is unresolved, not successful.
+- Budgets: `latency-under`, `tokens-under`, `cost-under`, and
+  `artifact-bytes-under`.
+- MCP discovery: `server-info-equals`, `tool-listed`,
+  `tools-listed-count`, and `capability-absent`.
+
+Most file assertions read captured `artifacts`; set `where: "cwd"` only when
+the durable contract intentionally includes the staged run workspace. Use
+stable assertion `id` values for repetitions, comparisons, annotations, and
+experiments. `weight` affects aggregate score and `threshold` sets an optional
+per-assertion score floor.
+
 Use `rubric` for semantic properties such as design quality, completeness, or
 whether a test meaningfully covers behavior. Keep criteria narrow. Avoid words
 like “good,” “proper,” or “best practice” without observable conditions.
 
+Bound rubric evidence to what the criterion needs. Include `tool-result` event
+evidence when status matters, and request tool inputs/results only when their
+payload is necessary; this controls judge context and cost.
+
+Persona answering and rubric judging have different difficulty profiles. A
+persona usually chooses from facts already supplied in the case, so prefer a
+cheap model and low effort. Semantic grading should use a stronger model and
+enough effort to distinguish real evidence from plausible prose. Configure
+them independently when that tradeoff matters:
+
+```ts
+// lux.config.ts
+defaultAnswerer: {
+  name: "persona",
+  config: {
+    provider: "codex",
+    model: "YOUR_SMALL_PERSONA_MODEL",
+    reasoningEffort: "low",
+  },
+},
+
+// case assertion
+{
+  id: "behavior-quality",
+  type: "rubric",
+  criteria: "The tests exercise retry exhaustion and recovery.",
+  provider: "codex",
+  model: "YOUR_STRONG_JUDGE_MODEL",
+  reasoningEffort: "high",
+}
+```
+
+The project `harness` supplies shared defaults to persona and rubric calls;
+answerer config and rubric fields override it independently. Always pin model
+and effort when comparing runs, and inspect answerer versus judge usage in the
+run summary before increasing either budget.
+
 Assertions should test user-visible behavior and durable contracts. Avoid
 requiring a specific function name, file layout, or algorithm unless that is
 the contract under evaluation.
@@ -73,9 +131,51 @@ The persona is simulated user state, not another system prompt. Include:
 Do not tell the persona how to help the agent solve the task. If no interaction
 is part of the capability being evaluated, omit the persona or keep it minimal.
 
-Use `interactionMode: "defer-resume"` for automated Claude Code question-tool
-handling. Use `textQuestionFallback: true` only when intentionally evaluating
-models that ask in prose instead of using the supported tool.
+For Claude Code, use `interactionMode: "defer-resume"` for automated
+question-tool handling. Use `textQuestionFallback: true` only when
+intentionally evaluating models that ask in prose instead of using the
+supported tool. For Codex, keep the default `interactionMode: "app-server"`
+when the case can interrupt; compatibility mode `"exec"` cannot accept
+mid-run responses.
+
+Use `--interactive` for a real terminal user or `--claude-answerer` for the
+Claude plugin protocol. Both force sequential execution.
+
+## Project extensions
+
+For a reusable project-specific assertion, define it with `defineAssertion`
+and a direct Zod object whose `type` is a matching literal. Import `z` from Lux,
+return an ordinary failed result for missing/invalid product evidence, and
+reserve throws or `errorMessage` for evaluator faults. Increment
+`pluginVersion` whenever schema or evaluation behavior changes. Prefer the
+bounded, traversal-safe `readRunArtifactText` helper over direct filesystem
+reads.
+
+Build a typed case authoring function once instead of importing plugins in
+every case:
+
+```ts
+const assertions = defineAssertionRegistry(projectAssertion);
+export const defineEvalCase = defineLuxCase.extend({ assertions });
+```
+
+Case-bound plugins remain plain serialized plans but require the current case
+path for regrading: `lux grade <run> --case <case.ts>`.
+
+Use `defineFixtures` for Vitest-style suite/case setup around `await use()` or
+`defineLifecycleHooks` when cleanup-returning callbacks fit better. Setup runs
+in declaration order and teardown in reverse, including failures. Call
+`await use()` exactly once. Fixture `value`/`identity` must be public JSON;
+keep secrets in the prepared resource or environment. Increment
+`pluginVersion` when lifecycle behavior changes. Register globally in
+`lux.config.ts`, or attach selectively with
+`defineEvalCase.extend({ fixtures: [...] })`.
+
+Custom drivers that own an in-memory filesystem should persist it with
+`writeArtifactSnapshot` from `@prismatic-io/lux/artifact-snapshot`. Snapshots
+are bounded, traversal-safe, immutable per name, and can use `baseSnapshot`
+for copy-on-write reuse while remaining independently complete. Ordinary file
+assertions then grade the indexed files without a snapshot-specific API.
 
 ## Experiments for prompts, skills, and agent source
 
@@ -100,6 +200,58 @@ Campaigns should:
 - budget calls, tokens, cost, and wall time;
 - use repetitions and confidence gates appropriate to stochastic outcomes;
 - reserve held-out cases for promotion and final confirmation.
+- acknowledge every candidate validation command with
+  `unsafeHostExecution: true`; validators must be observational and must not
+  mutate the candidate tree.
+
+A minimal current agent campaign has this shape (selectors are objects, mutable
+paths are candidate-relative globs, and budgets use call/token/cost limits):
+
+```ts
+export default defineExperiment({
+  id: "skill-improvement",
+  subject: { root: "..", mutable: ["skills/example/**"] },
+  profiles: ["claude-low", "codex-low"],
+  splits: {
+    train: { tags: ["train"] },
+    validation: { tags: ["validation"] },
+    test: { tags: ["test"] },
+  },
+  repetitions: { search: 1, promotion: 3 },
+  objectives: {
+    constraints: [
+      { metric: "quality.caseRegressions", max: 0 },
+      { metric: "quality.passRate", min: 0.8, split: "test" },
+    ],
+    minimize: [{ metric: "agent.total" }, { metric: "subject.bytes" }],
+  },
+  optimizer: {
+    algorithm: "reflective",
+    candidateSelection: "pareto",
+    promotionTopK: 2,
+    config: {
+      provider: "codex",
+      model: "YOUR_OPTIMIZER_MODEL",
+      reasoningEffort: "low",
+    },
+  },
+  budget: {
+    maxCandidates: 4,
+    patience: 1,
+    maxMetricCalls: 60,
+    maxJudgeCalls: 120,
+    maxProposalCalls: 3,
+    maxTokens: 2_000_000,
+    maxCostUsd: 30,
+    maxUnpricedModelCalls: 0,
+    maxWallTimeMs: 3_600_000,
+  },
+});
+```
+
+Use the experiment directory printed by the run for `view` and `report`.
+Review its held-out evidence, then pass that same verified directory to
+`apply`; do not guess a directory from the campaign ID.
 
 Use `experiment` for authored variants and `optimize` for model-proposed source
 changes. Always run the corresponding `--plan` command first.

package/skills/lux-answerer/SKILL.md

@@ -1,6 +1,6 @@
 ---
 name: lux-answerer
-version: 0.0.2-preview.3
+version: 0.0.2-preview.4
 description: Play the persona for a running Lux orchestrator that uses the claude-code answerer. Read structured events on stdout, decide answers from persona + context, write JSON to its answer channel.
 user-invocable: false
 allowed-tools: Bash, Read

package/src/answerers/terminal/index.ts

@@ -70,7 +70,7 @@ class TerminalAnswererImpl implements Answerer {
     if (!stdin.isTTY || !stdout.isTTY) {
       throw new Error(
         "terminal answerer: --interactive requires an attached terminal; " +
-          "use a scripted/model answerer in CI or --claude-answerer from the Claude plugin",
+          "use a model answerer in CI or --claude-answerer from the Claude plugin",
       );
     }
     this.interface = createInterface({ input: stdin, output: stdout, terminal: true });

package/src/assertions/core/helpers.ts

@@ -3,6 +3,7 @@ import { lstat } from "node:fs/promises";
 import { isAbsolute, join, relative, resolve, sep } from "node:path";
 import { z } from "zod";
 import { inspectRunArtifact, runArtifactsRoot } from "../../core/artifact-evidence.js";
+import { baseAssertionFields } from "../../core/assertion.js";
 import type { Run, RunEvent } from "../../core/index.js";
 
 const FsLocationSchema = z.enum(["artifacts", "cwd"]);
@@ -175,12 +176,6 @@ export const deepEquals = (a: unknown, b: unknown): boolean => {
  * Mirrors {@link Assertion} from the core module while letting each built-in
  * assertion reject misspelled type-specific fields at its own schema boundary.
  */
-export const baseAssertionFields = {
-  id: z.string().min(1).optional(),
-  weight: z.number().positive().optional(),
-  threshold: z.number().min(0).max(1).optional(),
-  name: z.string().optional(),
-} as const;
 
 /**
  * Where on the run dir an assertion looks for files. Defaults to
@@ -190,6 +185,8 @@ export const LocationSchema = FsLocationSchema.default("artifacts");
 
 // `name` is intentionally excluded: for tool assertions the `name` field is
 // the tool's name, which then doubles as the display label.
+export { baseAssertionFields };
+
 export const weightThresholdFields = {
   id: baseAssertionFields.id,
   weight: baseAssertionFields.weight,

package/src/cli/program.ts

@@ -163,7 +163,7 @@ const registerExperimentCommands = (program: Command): void => {
     });
 
   program
-    .command("run [filters...]", { isDefault: true })
+    .command("run [filters...]")
     .description("discover and run cases; filter by name/path substring or --tag")
     .option(
       "-t, --tag <tag>",
@@ -184,6 +184,7 @@ const registerExperimentCommands = (program: Command): void => {
       "--claude-answerer",
       "delegate questions through the Claude plugin's lux-answerer skill",
     )
+    .option("--skip-judge", "skip rubric LLM judge assertions")
     .option("--runs-root <path>", "run-dir root (overrides lux.config.ts)")
     .option("--profile <id>", "named driver/answerer profile from lux.config.ts")
     .option("--subject-root <path>", "root bound into subjectPath() values")
@@ -623,11 +624,16 @@ export const buildProgram = (): Command => {
     .description(
       "Coding-agent evaluation and improvement with deterministic assertions and human-in-the-loop runs",
     )
-    .version(PKG_VERSION);
+    .version(PKG_VERSION)
+    .addHelpText(
+      "after",
+      "\nAgent skills:\n  $ lux skills list\n  $ lux skills install codex --project\n  $ lux skills install claude --project",
+    );
   registerProjectCommands(program);
   registerExperimentCommands(program);
   registerCandidateCommands(program);
   registerReportingCommands(program);
+  program.action(() => program.outputHelp());
   return program;
 };
 
@@ -965,6 +971,7 @@ const runCommand = async (filters: string[], options: RunOptions): Promise<void>
       runsRoot,
       fixturesRoot: config.fixturesRoot,
       answererOverride: resolveAnswererOverride(options),
+      skipJudge: options.skipJudge,
       observer: reporter,
       abortSignal: controller.signal,
       concurrency,

package/src/cli/render/reporter.ts

@@ -4,6 +4,7 @@ import {
   createToolOutcomeAccumulator,
   emptyRunUsage,
   type GradingAssertionEntry,
+  type GradingReport,
   harnessUsage,
   isUsageEmpty,
   type LifecycleStage,
@@ -77,6 +78,7 @@ type CaseView = {
   assertionTotal: number;
   currentAssertion: string | null;
   assertions: GradingAssertionEntry[];
+  skippedAssertions: NonNullable<GradingReport["skippedAssertions"]>;
   usage: RunUsage;
   /** Per-API-call agent usage, in order — the split behind `usage.agent`. */
   agentTurns: TurnUsage[];
@@ -117,7 +119,10 @@ const statusMark = (status: CaseStatus, theme: Theme): string => {
 
 const assertionTally = (view: CaseView): string => {
   const passed = view.assertions.filter((a) => a.passed).length;
-  return view.assertions.length === 0 ? "—" : `${passed}/${view.assertions.length}`;
+  const evaluated = view.assertions.length === 0 ? "—" : `${passed}/${view.assertions.length}`;
+  return view.skippedAssertions.length > 0
+    ? `${evaluated} · ${view.skippedAssertions.length} skipped`
+    : evaluated;
 };
 
 /**
@@ -233,6 +238,7 @@ export class ConsoleReporter implements SuiteObserver {
       assertionTotal: 0,
       currentAssertion: null,
       assertions: [],
+      skippedAssertions: [],
       usage: emptyRunUsage(),
       agentTurns: [],
       phaseDurationMs: null,
@@ -299,6 +305,7 @@ export class ConsoleReporter implements SuiteObserver {
     if (result.errorMessage) view.errorMessage = result.errorMessage;
     if (result.result) {
       view.assertions = result.result.grading.assertions;
+      view.skippedAssertions = result.result.grading.skippedAssertions ?? [];
       view.usage = result.result.metadata.usage;
       view.phaseDurationMs = result.result.metadata.phaseDurationMs ?? null;
       const tools = result.result.metadata.toolOutcomes;
@@ -315,6 +322,7 @@ export class ConsoleReporter implements SuiteObserver {
       // Assertions reported before the run failed to persist are not results
       // and must not inflate the closing tally of a case that errored.
       view.assertions = [];
+      view.skippedAssertions = [];
     }
 
     this.region.staticLine(this.finishedLine(view));
@@ -452,6 +460,11 @@ export class ConsoleReporter implements SuiteObserver {
         body: !entry.passed && entry.comment ? [this.theme.dim(entry.comment)] : [],
       });
     }
+    for (const skipped of view.skippedAssertions) {
+      nodes.push({
+        head: `${this.theme.dim("⊘")} ${skipped.name}  ${this.theme.dim("skipped: judge disabled")}`,
+      });
+    }
 
     if (view.toolCalls > 0 || view.toolUnmatchedResults > 0 || view.toolMalformed > 0) {
       const parts = [
@@ -550,15 +563,17 @@ export class ConsoleReporter implements SuiteObserver {
     }
 
     const assertions = this.cases.flatMap((c) => c.assertions);
-    if (assertions.length > 0) {
+    const skippedAssertions = this.cases.flatMap((c) => c.skippedAssertions);
+    if (assertions.length > 0 || skippedAssertions.length > 0) {
       const ok = assertions.filter((a) => a.passed).length;
       const bad = assertions.length - ok;
       const tally = formatTally(
         [
           [ok, this.theme.green("passed")],
           [bad, this.theme.red("failed")],
+          [skippedAssertions.length, this.theme.dim("skipped")],
         ],
-        assertions.length,
+        assertions.length + skippedAssertions.length,
       );
       lines.push(` ${label("Assertions")}${tally}`);
     }

package/src/cli/run-options.ts

@@ -8,6 +8,7 @@ export type RunOptions = {
   listTags?: boolean;
   interactive?: boolean;
   claudeAnswerer?: boolean;
+  skipJudge?: boolean;
   runsRoot?: string;
   profile?: string;
   subjectRoot?: string;

package/src/core/artifact-evidence.ts

@@ -7,12 +7,15 @@ import type { Run } from "./run.js";
 const DEFAULT_ARTIFACT_READ_BYTES = 5 * 1024 * 1024;
 const MAX_ARTIFACT_READ_BYTES = 10 * 1024 * 1024;
 
+export type RunArtifactIssueReason = "absent" | "unreadable";
+
 export type RunArtifactInspection = {
   target: string;
   root: string;
   artifact?: Artifact;
   stats?: Stats;
   issue?: string;
+  reason?: RunArtifactIssueReason;
 };
 
 type RunArtifactEvidence = {
@@ -82,6 +85,7 @@ export const inspectRunArtifact = async (
       target: authoredPath,
       root: runArtifactsRoot(run),
       issue: `${authoredPath} contains a null byte`,
+      reason: "unreadable",
     };
   }
   const matching = run.artifacts.filter((candidate) => candidate.path === authoredPath);
@@ -91,6 +95,7 @@ export const inspectRunArtifact = async (
       target: resolve(root, authoredPath),
       root,
       issue: `${authoredPath} was not captured in the frozen artifact index`,
+      reason: "absent",
     };
   }
   if (matching.some((candidate) => candidate.root !== undefined && !isAbsolute(candidate.root))) {
@@ -98,6 +103,7 @@ export const inspectRunArtifact = async (
       target: authoredPath,
       root: defaultArtifactsRoot(run),
       issue: `${authoredPath} declares a non-absolute artifact root`,
+      reason: "unreadable",
     };
   }
   const artifact = matching[0];
@@ -108,6 +114,7 @@ export const inspectRunArtifact = async (
       target: authoredPath,
       root: defaultArtifactsRoot(run),
       issue: `${authoredPath} is ambiguous across multiple artifact roots`,
+      reason: "unreadable",
     };
   }
   // A non-default root is trusted only for an exact artifact-index entry.
@@ -116,7 +123,12 @@ export const inspectRunArtifact = async (
   const root = matchingRoots[0] ?? defaultArtifactsRoot(run);
   const target = resolve(root, authoredPath);
   if (pathEscapesRoot(root, target)) {
-    return { target, root, issue: `${authoredPath} escapes the captured artifact root` };
+    return {
+      target,
+      root,
+      issue: `${authoredPath} escapes the captured artifact root`,
+      reason: "unreadable",
+    };
   }
 
   const path = relative(root, target);
@@ -134,7 +146,13 @@ export const inspectRunArtifact = async (
     } catch (error) {
       const code = (error as NodeJS.ErrnoException).code;
       if (code === "ENOENT" || code === "ENOTDIR") {
-        return { target, root, ...identified, issue: `${authoredPath} not found` };
+        return {
+          target,
+          root,
+          ...identified,
+          issue: `${authoredPath} not found`,
+          reason: "absent",
+        };
       }
       throw error;
     }
@@ -144,16 +162,17 @@ export const inspectRunArtifact = async (
         root,
         ...identified,
         issue: `${authoredPath} resolves through a symlink`,
+        reason: "unreadable",
       };
     }
     if (index < components.length - 1 && !stats.isDirectory()) {
-      return { target, root, ...identified, issue: `${authoredPath} not found` };
+      return { target, root, ...identified, issue: `${authoredPath} not found`, reason: "absent" };
     }
     if (index === components.length - 1) {
       return { target, root, ...identified, stats };
     }
   }
-  return { target, root, ...identified, issue: `${authoredPath} not found` };
+  return { target, root, ...identified, issue: `${authoredPath} not found`, reason: "absent" };
 };
 
 export type ReadRunArtifactTextOptions = {
@@ -247,3 +266,27 @@ export const readRunArtifactText = async (
 ): Promise<string> => {
   return (await readRunArtifactBytes(run, authoredPath, options)).toString("utf8");
 };
+
+/**
+ * Read optional artifact evidence. Returns null when the artifact was never
+ * captured, and still throws when one exists but is unsafe to consume, so a
+ * traversal or symlink violation is never mistaken for missing evidence.
+ */
+export const readRunArtifactTextIfPresent = async (
+  run: RunArtifactEvidence,
+  authoredPath: string,
+  options: ReadRunArtifactTextOptions = {},
+): Promise<string | null> => {
+  const inspected = await inspectRunArtifact(run, authoredPath);
+  if (inspected.reason === "absent") return null;
+  if (!inspected.stats?.isFile()) {
+    throw new Error(
+      inspected.issue ??
+        (inspected.stats ? `${authoredPath} is not a file` : `${authoredPath} not found`),
+    );
+  }
+  const maxBytes = options.maxBytes ?? DEFAULT_ARTIFACT_READ_BYTES;
+  return (
+    await readInspectedFileBytes(inspected.target, inspected.stats, maxBytes, authoredPath)
+  ).toString("utf8");
+};

package/src/core/assertion.ts

@@ -100,6 +100,13 @@ export const assertionPluginIssue = (definition: {
  * Define a schema-validated assertion plugin while inferring the config passed
  * to `evaluate` from the schema.
  */
+export const baseAssertionFields = {
+  id: z.string().min(1).optional(),
+  weight: z.number().positive().optional(),
+  threshold: z.number().min(0).max(1).optional(),
+  name: z.string().optional(),
+} as const;
+
 export const defineAssertion = <S extends z.ZodObject>(
   definition: AssertionPluginDefinition<S>,
 ): AssertionPlugin<SchemaAssertion<S>, SchemaAssertionInput<S>> => {
@@ -141,11 +148,19 @@ export const GradingAssertionEntrySchema = z.object({
 });
 export type GradingAssertionEntry = z.infer<typeof GradingAssertionEntrySchema>;
 
+const SkippedGradingAssertionSchema = z.object({
+  id: z.string(),
+  type: z.string(),
+  name: z.string(),
+  reason: z.literal("judge-disabled"),
+});
+
 export const GradingReportSchema = z.object({
   schemaVersion: z.literal(1),
   casePassed: z.boolean().nullable(),
   caseScore: z.number().nullable(),
   assertions: z.array(GradingAssertionEntrySchema),
+  skippedAssertions: z.array(SkippedGradingAssertionSchema).optional(),
   /**
    * Present when the subject run or grading did not complete. Assertions may
    * still contain partial diagnostics, but the report is not a clean verdict.

package/src/core/events.ts

@@ -82,6 +82,7 @@ export type ErrorEventReason = z.infer<typeof ErrorEventReasonSchema>;
 export const ErrorEventSchema = z.object({
   exitReason: ErrorEventReasonSchema,
   reason: z.string(),
+  summary: z.unknown().optional(),
 });
 export type ErrorEvent = z.infer<typeof ErrorEventSchema>;
 

package/src/core/wire.ts

@@ -47,6 +47,7 @@ const ErrorEventSchema = z.object({
   type: z.literal("error"),
   exitReason: ErrorEventReasonSchema.default("error"),
   reason: z.string(),
+  summary: z.unknown().optional(),
 });
 
 export const WireEventSchema = z.discriminatedUnion("type", [
@@ -85,7 +86,10 @@ const wireEventToParsed = (event: WireEvent): ParsedDriverEvent => {
   if (event.type === "done") {
     return { tag: "done", done: { exitReason: "done", summary: event.summary } };
   }
-  return { tag: "error", error: { exitReason: event.exitReason, reason: event.reason } };
+  return {
+    tag: "error",
+    error: { exitReason: event.exitReason, reason: event.reason, summary: event.summary },
+  };
 };
 
 /**

package/src/index.ts

@@ -16,11 +16,6 @@ export {
   PersonaAnswererConfigSchema,
   personaAnswerer,
 } from "./answerers/persona/index.js";
-export {
-  type ScriptedAnswererConfig,
-  ScriptedAnswererConfigSchema,
-  scriptedAnswerer,
-} from "./answerers/scripted/index.js";
 export {
   type TerminalAnswererConfig,
   TerminalAnswererConfigSchema,

package/src/orchestrator/config.ts

@@ -4,7 +4,6 @@ import { pathToFileURL } from "node:url";
 import { z } from "zod";
 import { claudeCodeAnswerer } from "../answerers/claude-code/index.js";
 import { personaAnswerer } from "../answerers/persona/index.js";
-import { scriptedAnswerer } from "../answerers/scripted/index.js";
 import { terminalAnswerer } from "../answerers/terminal/index.js";
 import { allCoreAssertions } from "../assertions/core/index.js";
 import { allMcpAssertions } from "../assertions/mcp/index.js";
@@ -182,7 +181,6 @@ const builtinDrivers: DriverPlugin<never>[] = [
 const builtinAnswerers: AnswererPlugin<never>[] = [
   personaAnswerer,
   claudeCodeAnswerer,
-  scriptedAnswerer,
   terminalAnswerer,
 ];
 export const buildRegistryFromConfig = (config: ResolvedLuxConfig): Registry => {

package/src/orchestrator/grading.ts

@@ -76,6 +76,7 @@ export const graderConfigHash = (input: {
   assertions: readonly AssertionEvaluatorIdentity[];
   validatedAssertionsHash: string;
   graderContext?: unknown;
+  skipJudge?: boolean;
 }): string =>
   valueHash({
     luxVersion: input.luxVersion,
@@ -83,6 +84,7 @@ export const graderConfigHash = (input: {
     assertions: input.assertions,
     validatedAssertionsHash: input.validatedAssertionsHash,
     graderContext: input.graderContext ?? null,
+    skipJudge: input.skipJudge ?? false,
   });
 
 export const validateAssertionPlan = async (
@@ -133,6 +135,7 @@ export const gradeRun = async (
     complete: (attemptIds: readonly string[]) => void | Promise<void>;
   },
   validatedConfigs?: readonly unknown[],
+  skipJudge = false,
 ): Promise<GradingReport> => {
   const guardedObserver = guardRunObserver(observer);
   const assertions = run.case.assertions ?? [];
@@ -141,12 +144,27 @@ export const gradeRun = async (
   }
 
   const entries: GradingAssertionEntry[] = [];
+  const skippedAssertions: Array<{
+    id: string;
+    type: string;
+    name: string;
+    reason: "judge-disabled";
+  }> = [];
   let weightedScoreSum = 0;
   let weightSum = 0;
   let allPassed = true;
   let gradingIncomplete = false;
 
   for (const [index, a] of assertions.entries()) {
+    if (skipJudge && a.type === "rubric") {
+      skippedAssertions.push({
+        id: a.id ?? `${a.type}:${index + 1}`,
+        type: a.type,
+        name: describeAssertion(a),
+        reason: "judge-disabled",
+      });
+      continue;
+    }
     // Grading is the last cancellation point — an LLM judge can run for
     // minutes per assertion. An aborted grade can't claim everything passed.
     if (abortSignal?.aborted) {
@@ -247,12 +265,13 @@ export const gradeRun = async (
   }
 
   let caseScore: number | null = null;
-  if (!gradingIncomplete) caseScore = weightSum > 0 ? weightedScoreSum / weightSum : 0;
+  if (!gradingIncomplete && weightSum > 0) caseScore = weightedScoreSum / weightSum;
   return {
     schemaVersion: 1,
-    casePassed: gradingIncomplete ? null : allPassed,
+    casePassed: gradingIncomplete || weightSum === 0 ? null : allPassed,
     caseScore,
     assertions: entries,
+    ...(skippedAssertions.length > 0 ? { skippedAssertions } : {}),
     ...(gradingIncomplete
       ? { incomplete: { exitReason: "error" as const, phase: "grading" as const } }
       : {}),