@prismatic-io/lux 0.0.1

package/src/orchestrator/regrade.ts

@@ -0,0 +1,580 @@
+import { randomUUID } from "node:crypto";
+import {
+  chmod,
+  constants,
+  link,
+  lstat,
+  mkdir,
+  open,
+  readdir,
+  realpath,
+  rm,
+  stat,
+  unlink,
+  writeFile,
+} from "node:fs/promises";
+import { isAbsolute, join, relative, resolve, sep } from "node:path";
+import { temporaryDirectory } from "tempy";
+import { z } from "zod";
+import {
+  ArtifactSchema,
+  bindRunArtifactEvidence,
+  contentHash,
+  type EvalCase,
+  EvalCaseSchema,
+  type GradingReport,
+  GradingReportSchema,
+  inspectRunArtifact,
+  PersistedCaseSchema,
+  type Registry,
+  type Run,
+  RunMetadataSchema,
+  readInspectedFileBytes,
+  readRunArtifactBytes,
+  valueHash,
+} from "../core/index.js";
+import { evaluatorImplementationHash } from "./evaluator-identity.js";
+import { parsePersistedJson, readBoundedHandle, safeReadJson } from "./fs-read.js";
+import {
+  assertionEvaluatorIdentities,
+  assertionPluginIdentities,
+  gradeRun,
+  graderConfigHash,
+  validateAssertionPlan,
+} from "./grading.js";
+import { loadRunDetail } from "./list-runs.js";
+import { LUX_VERSION } from "./package-version.js";
+
+export const GRADING_SNAPSHOT_SCHEMA_VERSION = 2 as const;
+
+export const GradingSnapshotSchema = z.object({
+  schemaVersion: z.literal(GRADING_SNAPSHOT_SCHEMA_VERSION),
+  gradingId: z.string(),
+  gradedAt: z.string(),
+  runDir: z.string(),
+  caseHash: z.string(),
+  assertionsHash: z.string(),
+  validatedAssertionsHash: z.string(),
+  graderConfigHash: z.string(),
+  evidenceHash: z.string(),
+  assertionPlugins: z.array(
+    z
+      .object({
+        type: z.string(),
+        pluginVersion: z.string().nullable(),
+      })
+      .strict(),
+  ),
+  sourceGraderConfigHash: z.string().optional(),
+  sourceValidatedAssertionsHash: z.string().optional(),
+  report: GradingReportSchema,
+});
+export type GradingSnapshot = z.infer<typeof GradingSnapshotSchema>;
+
+const timestampId = (date: Date): string => date.toISOString().replace(/[:.]/g, "-");
+const frozenRunStates = new WeakMap<
+  Run,
+  {
+    snapshotRunDir: string;
+    sourceRunDir: string;
+  }
+>();
+let currentEvaluatorHash: Promise<string> | undefined;
+const MAX_FROZEN_ARTIFACTS = 10_000;
+const MAX_FROZEN_ARTIFACT_BYTES = 10 * 1024 * 1024;
+const MAX_FROZEN_ARTIFACT_TOTAL_BYTES = 50 * 1024 * 1024;
+const MAX_FROZEN_ARTIFACT_INDEX_BYTES = 2 * 1024 * 1024;
+const MAX_FROZEN_RUN_JSON_BYTES = 2 * 1024 * 1024;
+
+const currentEvaluatorImplementationHash = (): Promise<string> =>
+  (currentEvaluatorHash ??= evaluatorImplementationHash());
+
+const readFrozenJson = async <S extends z.ZodType>(
+  path: string,
+  schema: S,
+): Promise<z.infer<S> | null> => {
+  try {
+    if ((await lstat(path)).isSymbolicLink()) {
+      throw new Error(`frozen run JSON cannot be a symlink: ${path}`);
+    }
+    const handle = await open(path, constants.O_RDONLY | constants.O_NOFOLLOW);
+    try {
+      const metadata = await handle.stat();
+      if (!metadata.isFile() || metadata.size > MAX_FROZEN_RUN_JSON_BYTES) {
+        throw new Error(
+          `frozen run JSON must be a regular file no larger than ${MAX_FROZEN_RUN_JSON_BYTES} bytes: ${path}`,
+        );
+      }
+      const contents = await readBoundedHandle(handle, MAX_FROZEN_RUN_JSON_BYTES, path);
+      return parsePersistedJson(path, contents.toString("utf8"), schema);
+    } finally {
+      await handle.close();
+    }
+  } catch (error) {
+    if ((error as NodeJS.ErrnoException).code === "ENOENT") return null;
+    if ((error as NodeJS.ErrnoException).code === "ELOOP") {
+      throw new Error(`frozen run JSON cannot be a symlink: ${path}`);
+    }
+    throw error;
+  }
+};
+
+const loadArtifacts = async (
+  runDir: string,
+  fallbackPaths: string[],
+  frozenLimits: boolean,
+): Promise<Run["artifacts"]> => {
+  const artifactsPath = join(runDir, "artifacts.json");
+  let persisted: Run["artifacts"] | null = null;
+  if (frozenLimits) {
+    try {
+      if ((await lstat(artifactsPath)).isSymbolicLink()) {
+        throw new Error(`frozen artifact index cannot be a symlink: ${artifactsPath}`);
+      }
+      const handle = await open(artifactsPath, constants.O_RDONLY | constants.O_NOFOLLOW);
+      try {
+        const metadata = await handle.stat();
+        if (!metadata.isFile() || metadata.size > MAX_FROZEN_ARTIFACT_INDEX_BYTES) {
+          throw new Error(`frozen artifact index exceeds ${MAX_FROZEN_ARTIFACT_INDEX_BYTES} bytes`);
+        }
+        persisted = parsePersistedJson(
+          artifactsPath,
+          (
+            await readBoundedHandle(
+              handle,
+              MAX_FROZEN_ARTIFACT_INDEX_BYTES,
+              "frozen artifact index",
+            )
+          ).toString("utf8"),
+          z.array(ArtifactSchema),
+        );
+      } finally {
+        await handle.close();
+      }
+    } catch (error) {
+      if ((error as NodeJS.ErrnoException).code !== "ENOENT") throw error;
+    }
+  } else {
+    persisted = await safeReadJson(artifactsPath, z.array(ArtifactSchema));
+  }
+  if (persisted) return persisted;
+  return Promise.all(
+    fallbackPaths.map(async (path) => ({
+      path,
+      size: (await stat(join(runDir, "artifacts", path))).size,
+    })),
+  );
+};
+
+const authoredCaseFromPersisted = (persisted: z.infer<typeof PersistedCaseSchema>): unknown => ({
+  ...(persisted.id !== undefined ? { id: persisted.id } : {}),
+  ...(persisted.prompt !== undefined ? { prompt: persisted.prompt } : {}),
+  ...(persisted.persona !== undefined ? { persona: persisted.persona } : {}),
+  ...(persisted.driver !== undefined
+    ? {
+        driver:
+          typeof persisted.driver === "string"
+            ? persisted.driver
+            : {
+                name: persisted.driver.name,
+                ...(persisted.driver.config !== undefined
+                  ? { config: persisted.driver.config }
+                  : {}),
+              },
+      }
+    : {}),
+  ...(persisted.answerer !== undefined
+    ? {
+        answerer:
+          typeof persisted.answerer === "string"
+            ? persisted.answerer
+            : {
+                name: persisted.answerer.name,
+                ...(persisted.answerer.config !== undefined
+                  ? { config: persisted.answerer.config }
+                  : {}),
+              },
+      }
+    : {}),
+  ...(persisted.fixtures !== undefined
+    ? {
+        fixtures:
+          typeof persisted.fixtures === "string"
+            ? persisted.fixtures
+            : { kind: persisted.fixtures.kind, path: persisted.fixtures.path },
+      }
+    : {}),
+  ...(persisted.assertions !== undefined ? { assertions: persisted.assertions } : {}),
+  ...(persisted.meta !== undefined ? { meta: persisted.meta } : {}),
+});
+
+export const loadRegradableRun = async (
+  runDir: string,
+  caseOverride?: EvalCase,
+  options: { frozenLimits?: boolean } = {},
+): Promise<Run> => {
+  const absolute = resolve(runDir);
+  const detail = await loadRunDetail(
+    absolute,
+    options.frozenLimits
+      ? { maxArtifacts: MAX_FROZEN_ARTIFACTS, skipArtifactsWalk: true, skipJson: true }
+      : {},
+  );
+  const persistedCase = options.frozenLimits
+    ? await readFrozenJson(join(absolute, "case.json"), PersistedCaseSchema)
+    : detail.case;
+  const metadata = options.frozenLimits
+    ? await readFrozenJson(join(absolute, "run.json"), RunMetadataSchema)
+    : detail.metadata;
+  if (!persistedCase) throw new Error(`case.json is missing or invalid at ${absolute}`);
+  if (!metadata) throw new Error(`run.json is missing or invalid at ${absolute}`);
+  const parsedCase = EvalCaseSchema.safeParse(
+    caseOverride ?? authoredCaseFromPersisted(persistedCase),
+  );
+  if (!parsedCase.success) {
+    throw new Error(
+      `persisted case cannot be regraded: ${parsedCase.error.issues[0]?.message ?? "invalid case"}`,
+    );
+  }
+  return {
+    case: parsedCase.data,
+    metadata,
+    events: detail.events,
+    artifacts: await loadArtifacts(absolute, detail.artifacts, options.frozenLimits === true),
+    runDir: absolute,
+  };
+};
+
+/**
+ * Load evidence for an offline grader experiment. Frozen evidence must be
+ * self-contained so authored artifact metadata cannot disclose unrelated
+ * local files to a model judge.
+ */
+export const loadFrozenRegradableRun = async (
+  runDir: string,
+  caseOverride?: EvalCase,
+): Promise<Run> => {
+  const run = await loadRegradableRun(runDir, caseOverride, { frozenLimits: true });
+  if (run.artifacts.length > MAX_FROZEN_ARTIFACTS) {
+    throw new Error(`frozen grader evidence exceeds ${MAX_FROZEN_ARTIFACTS} artifacts`);
+  }
+  const artifactsRoot = resolve(run.runDir, "artifacts");
+  const sourceRunDir = run.runDir;
+  const snapshotRunDir = temporaryDirectory({ prefix: "lux-frozen-evidence-" });
+  const snapshotArtifactsRoot = join(snapshotRunDir, "artifacts");
+  try {
+    await mkdir(snapshotArtifactsRoot);
+    if (run.artifacts.length > 0) {
+      const rootMetadata = await lstat(artifactsRoot);
+      if (!rootMetadata.isDirectory() || rootMetadata.isSymbolicLink()) {
+        throw new Error("frozen grader artifacts root must be a real directory");
+      }
+    }
+    const artifacts: Run["artifacts"] = [];
+    let totalBytes = 0;
+    for (const artifact of run.artifacts) {
+      if (artifact.root !== undefined) {
+        throw new Error(
+          `frozen grader artifact '${artifact.path}' cannot declare an external root`,
+        );
+      }
+      if (
+        isAbsolute(artifact.path) ||
+        artifact.path.split(/[\\/]/).includes("..") ||
+        artifact.path.includes("\0")
+      ) {
+        throw new Error(`frozen grader artifact path escapes its run: ${artifact.path}`);
+      }
+      const path = resolve(artifactsRoot, artifact.path);
+      if (path === artifactsRoot || !path.startsWith(`${artifactsRoot}${sep}`)) {
+        throw new Error(`frozen grader artifact path escapes its run: ${artifact.path}`);
+      }
+      const inspected = await inspectRunArtifact(run, artifact.path);
+      if (!inspected.stats?.isFile()) {
+        throw new Error(
+          inspected.issue ?? `frozen grader artifact must be a regular file: ${artifact.path}`,
+        );
+      }
+      const bytes = await readInspectedFileBytes(
+        inspected.target,
+        inspected.stats,
+        MAX_FROZEN_ARTIFACT_BYTES,
+        `frozen grader artifact '${artifact.path}'`,
+      );
+      totalBytes += bytes.length;
+      if (totalBytes > MAX_FROZEN_ARTIFACT_TOTAL_BYTES) {
+        throw new Error(
+          `frozen grader artifacts exceed ${MAX_FROZEN_ARTIFACT_TOTAL_BYTES} total bytes`,
+        );
+      }
+      const destination = resolve(snapshotArtifactsRoot, artifact.path);
+      if (!destination.startsWith(`${resolve(snapshotArtifactsRoot)}${sep}`)) {
+        throw new Error(`frozen grader artifact path escapes its snapshot: ${artifact.path}`);
+      }
+      await mkdir(resolve(destination, ".."), { recursive: true });
+      await writeFile(destination, bytes, { flag: "wx" });
+      await chmod(destination, inspected.stats.mode & 0o777);
+      artifacts.push({ ...artifact, root: snapshotArtifactsRoot, size: bytes.length });
+    }
+    const frozen = { ...run, runDir: snapshotRunDir, artifacts };
+    bindRunArtifactEvidence(frozen, { root: snapshotArtifactsRoot, indexedOnly: true });
+    frozenRunStates.set(frozen, { snapshotRunDir, sourceRunDir });
+    return frozen;
+  } catch (error) {
+    await rm(snapshotRunDir, { recursive: true, force: true });
+    throw error;
+  }
+};
+
+export const releaseFrozenRegradableRun = async (run: Run): Promise<void> => {
+  const state = frozenRunStates.get(run);
+  if (!state) return;
+  frozenRunStates.delete(run);
+  await rm(state.snapshotRunDir, { recursive: true, force: true });
+};
+
+const writeGradingSnapshot = async (
+  run: Run,
+  report: GradingReport,
+  identity: {
+    validatedAssertionsHash: string;
+    graderConfigHash: string;
+    evidenceHash: string;
+    assertionPlugins: GradingSnapshot["assertionPlugins"];
+  },
+  now: Date,
+): Promise<{ path: string; snapshot: GradingSnapshot }> => {
+  const gradingPrefix = timestampId(now);
+  const persistedRunDir = frozenRunStates.get(run)?.sourceRunDir ?? run.runDir;
+  const directory = join(persistedRunDir, "gradings");
+  try {
+    await mkdir(directory);
+  } catch (error) {
+    if ((error as NodeJS.ErrnoException).code !== "EEXIST") throw error;
+  }
+  const directoryMetadata = await lstat(directory);
+  if (!directoryMetadata.isDirectory() || directoryMetadata.isSymbolicLink()) {
+    throw new Error("run gradings path must be a real directory");
+  }
+  const [canonicalRunRoot, canonicalDirectory] = await Promise.all([
+    realpath(persistedRunDir),
+    realpath(directory),
+  ]);
+  if (canonicalDirectory !== join(canonicalRunRoot, "gradings")) {
+    throw new Error("run gradings directory resolves outside its run");
+  }
+
+  for (let suffix = 1; ; suffix++) {
+    const gradingId = suffix === 1 ? gradingPrefix : `${gradingPrefix}-${suffix}`;
+    const path = join(directory, `${gradingId}.json`);
+    const snapshot: GradingSnapshot = {
+      schemaVersion: GRADING_SNAPSHOT_SCHEMA_VERSION,
+      gradingId,
+      gradedAt: now.toISOString(),
+      runDir: persistedRunDir,
+      caseHash: valueHash(run.case),
+      assertionsHash: valueHash(run.case.assertions ?? []),
+      validatedAssertionsHash: identity.validatedAssertionsHash,
+      graderConfigHash: identity.graderConfigHash,
+      evidenceHash: identity.evidenceHash,
+      assertionPlugins: identity.assertionPlugins,
+      ...(run.metadata.provenance?.graderConfigHash
+        ? { sourceGraderConfigHash: run.metadata.provenance.graderConfigHash }
+        : {}),
+      ...(run.metadata.provenance?.validatedAssertionsHash
+        ? {
+            sourceValidatedAssertionsHash: run.metadata.provenance.validatedAssertionsHash,
+          }
+        : {}),
+      report,
+    };
+    const temporary = `${path}.${process.pid}.${randomUUID()}.tmp`;
+    await writeFile(temporary, `${JSON.stringify(snapshot, null, 2)}\n`, { flag: "wx" });
+    try {
+      await link(temporary, path);
+      return { path, snapshot };
+    } catch (error) {
+      if ((error as NodeJS.ErrnoException).code !== "EEXIST") throw error;
+    } finally {
+      await unlink(temporary).catch(() => undefined);
+    }
+  }
+};
+
+type FrozenCwdEntry = {
+  path: string;
+  kind: "directory" | "file";
+  mode: number;
+};
+
+const frozenCwdManifest = async (run: Run): Promise<FrozenCwdEntry[]> => {
+  if (!frozenRunStates.has(run)) {
+    throw new Error("grading evidence is not bound to a frozen run");
+  }
+  const expected = new Map<string, Map<string, FrozenCwdEntry["kind"]>>();
+  const addExpected = (directory: string, name: string, kind: FrozenCwdEntry["kind"]): void => {
+    const children = expected.get(directory) ?? new Map();
+    const existing = children.get(name);
+    if (existing && existing !== kind) {
+      throw new Error("frozen grading evidence has conflicting artifact paths");
+    }
+    children.set(name, kind);
+    expected.set(directory, children);
+  };
+  expected.set("", new Map([["artifacts", "directory"]]));
+  expected.set("artifacts", new Map());
+  for (const artifact of run.artifacts) {
+    const target = resolve(artifact.root ?? join(run.runDir, "artifacts"), artifact.path);
+    const authored = relative(run.runDir, target);
+    if (authored === ".." || authored.startsWith(`..${sep}`) || isAbsolute(authored)) {
+      throw new Error("frozen grading evidence contains an artifact outside its snapshot");
+    }
+    const parts = authored.split(sep).filter(Boolean);
+    for (let index = 0; index < parts.length; index++) {
+      const parent = parts.slice(0, index).join(sep);
+      const name = parts[index];
+      if (!name) continue;
+      const kind = index === parts.length - 1 ? "file" : "directory";
+      addExpected(parent, name, kind);
+      if (kind === "directory" && !expected.has(join(parent, name))) {
+        expected.set(join(parent, name), new Map());
+      }
+    }
+  }
+
+  const manifest: FrozenCwdEntry[] = [];
+  for (const directory of [...expected.keys()].sort()) {
+    const absolute = resolve(run.runDir, directory);
+    const metadata = await lstat(absolute);
+    if (!metadata.isDirectory() || metadata.isSymbolicLink()) {
+      throw new Error("frozen grading cwd changed during evaluation");
+    }
+    manifest.push({
+      path: directory || ".",
+      kind: "directory",
+      mode: metadata.mode & 0o777,
+    });
+    const actual = await readdir(absolute, { withFileTypes: true });
+    const expectedChildren = expected.get(directory) ?? new Map();
+    if (
+      actual.length !== expectedChildren.size ||
+      actual.some((entry) => {
+        const kind = expectedChildren.get(entry.name);
+        return (
+          kind === undefined ||
+          (kind === "directory" && !entry.isDirectory()) ||
+          (kind === "file" && !entry.isFile())
+        );
+      })
+    ) {
+      throw new Error("frozen grading cwd changed during evaluation");
+    }
+    for (const entry of actual.filter((candidate) => candidate.isFile())) {
+      const path = join(directory, entry.name);
+      const file = await lstat(resolve(run.runDir, path));
+      if (!file.isFile() || file.isSymbolicLink()) {
+        throw new Error("frozen grading cwd changed during evaluation");
+      }
+      manifest.push({ path, kind: "file", mode: file.mode & 0o777 });
+    }
+  }
+  return manifest.sort((left, right) => left.path.localeCompare(right.path));
+};
+
+const gradingEvidenceHash = async (run: Run): Promise<string> => {
+  const artifacts: Array<{
+    path: string;
+    contentType: string | null;
+    declaredSize: number | null;
+    size: number;
+    meta: Record<string, unknown> | null;
+    contentHash: string;
+  }> = [];
+  // Frozen runs may contain up to 200 artifacts. Hash in order so tiny files
+  // never reserve every per-file ceiling at once.
+  for (const artifact of [...run.artifacts].sort((left, right) =>
+    left.path.localeCompare(right.path),
+  )) {
+    const bytes = await readRunArtifactBytes(run, artifact.path, {
+      maxBytes: MAX_FROZEN_ARTIFACT_BYTES,
+    });
+    artifacts.push({
+      path: artifact.path,
+      contentType: artifact.contentType ?? null,
+      declaredSize: artifact.size ?? null,
+      size: bytes.length,
+      meta: artifact.meta ?? null,
+      contentHash: contentHash(bytes),
+    });
+  }
+  return valueHash({
+    case: run.case,
+    metadata: run.metadata,
+    events: run.events,
+    artifacts,
+    cwd: await frozenCwdManifest(run),
+  });
+};
+
+export const regradeRun = async (options: {
+  runDir: string;
+  registry: Registry;
+  case?: EvalCase;
+  /** Explicit grader state captured by closures, such as the configured harness model. */
+  graderContext?: unknown;
+  abortSignal?: AbortSignal;
+  now?: Date;
+}): Promise<{ path: string; snapshot: GradingSnapshot }> => {
+  const run = await loadFrozenRegradableRun(options.runDir, options.case);
+  try {
+    const evidenceHash = await gradingEvidenceHash(run);
+    const validatedAssertionConfigs = await validateAssertionPlan(
+      run.case.assertions ?? [],
+      options.registry,
+    );
+    const validatedAssertionsHash = valueHash(validatedAssertionConfigs);
+    const assertionEvaluators = assertionEvaluatorIdentities(
+      run.case.assertions ?? [],
+      options.registry,
+    );
+    const assertionPlugins = assertionPluginIdentities(run.case.assertions ?? [], options.registry);
+    const graderHash = graderConfigHash({
+      luxVersion: LUX_VERSION,
+      implementationHash: await currentEvaluatorImplementationHash(),
+      assertions: assertionEvaluators,
+      validatedAssertionsHash,
+      graderContext: options.graderContext,
+    });
+    const report: GradingReport = await gradeRun(
+      run,
+      options.registry,
+      undefined,
+      options.abortSignal,
+      undefined,
+      undefined,
+      validatedAssertionConfigs,
+    );
+    let postEvaluationEvidenceHash: string;
+    try {
+      postEvaluationEvidenceHash = await gradingEvidenceHash(run);
+    } catch (error) {
+      throw new Error("frozen grading evidence changed during evaluation", { cause: error });
+    }
+    if (postEvaluationEvidenceHash !== evidenceHash) {
+      throw new Error("frozen grading evidence changed during evaluation");
+    }
+    return await writeGradingSnapshot(
+      run,
+      report,
+      {
+        validatedAssertionsHash,
+        graderConfigHash: graderHash,
+        evidenceHash,
+        assertionPlugins,
+      },
+      options.now ?? new Date(),
+    );
+  } finally {
+    await releaseFrozenRegradableRun(run);
+  }
+};

package/src/orchestrator/rubric-loader.ts

@@ -0,0 +1,74 @@
+import { readFile, stat } from "node:fs/promises";
+import { extname, resolve } from "node:path";
+import { z } from "zod";
+import { RubricConfigSchema } from "../assertions/rubric/index.js";
+import { formatZodIssues, type RubricDefinition, RubricDefinitionSchema } from "../core/index.js";
+
+const CANDIDATE_FORBIDDEN_FIELDS = [
+  "command",
+  "provider",
+  "model",
+  "reasoningEffort",
+  "timeoutMs",
+  "promptOverride",
+] as const;
+const RawRubricDefinitionSchema = z.looseObject({
+  assertions: z.array(z.unknown()),
+});
+const RawRubricAssertionSchema = z.looseObject({});
+
+/**
+ * Candidate rubrics are deliberately declarative JSON. Reflective optimization
+ * must never gain code execution by emitting a TypeScript grader module.
+ */
+export const loadRubricDefinition = async (filePath: string): Promise<RubricDefinition> => {
+  const path = resolve(filePath);
+  if (extname(path).toLowerCase() !== ".json") {
+    throw new Error(`Lux grader experiments require a declarative .json rubric: ${path}`);
+  }
+  if ((await stat(path)).size > 256_000) {
+    throw new Error(`Invalid Lux rubric at ${path}: candidate rubric exceeds 256000 bytes`);
+  }
+  const raw: unknown = JSON.parse(await readFile(path, "utf8"));
+  const parsed = RubricDefinitionSchema.safeParse(raw);
+  if (!parsed.success) {
+    throw new Error(`Invalid Lux rubric at ${path}: ${formatZodIssues(parsed.error)}`);
+  }
+  if (parsed.data.assertions.length > 20) {
+    throw new Error(
+      `Invalid Lux rubric at ${path}: candidate rubrics are limited to 20 assertions`,
+    );
+  }
+  const rawAssertions = RawRubricDefinitionSchema.parse(raw).assertions;
+  for (const [index, assertion] of parsed.data.assertions.entries()) {
+    if (assertion.type !== "rubric") {
+      throw new Error(
+        `Invalid Lux rubric at ${path}: grader experiment assertions must use type 'rubric', got '${assertion.type}'`,
+      );
+    }
+    const rawAssertion = RawRubricAssertionSchema.safeParse(rawAssertions[index]).data;
+    const field = rawAssertion
+      ? CANDIDATE_FORBIDDEN_FIELDS.find((key) => key in rawAssertion)
+      : undefined;
+    if (field) {
+      throw new Error(
+        `Invalid Lux rubric at ${path}: candidate assertions cannot override '${field}'`,
+      );
+    }
+    const config = RubricConfigSchema.safeParse(rawAssertion);
+    if (!config.success) {
+      throw new Error(`Invalid Lux rubric at ${path}: ${formatZodIssues(config.error)}`);
+    }
+    if (
+      config.data.criteria.length > 20_000 ||
+      config.data.evidence.maxEvents > 100 ||
+      config.data.evidence.maxAgentChars > 100_000 ||
+      config.data.evidence.maxToolChars > 100_000 ||
+      config.data.evidence.maxArtifactChars > 200_000 ||
+      config.data.maxArtifactExcerpt > 20_000
+    ) {
+      throw new Error(`Invalid Lux rubric at ${path}: candidate prompt bounds exceed safe caps`);
+    }
+  }
+  return parsed.data;
+};