@prismatic-io/lux 0.0.1 → 0.0.2-preview.4

package/src/core/artifact-evidence.ts

@@ -7,12 +7,15 @@ import type { Run } from "./run.js";
 const DEFAULT_ARTIFACT_READ_BYTES = 5 * 1024 * 1024;
 const MAX_ARTIFACT_READ_BYTES = 10 * 1024 * 1024;
 
+export type RunArtifactIssueReason = "absent" | "unreadable";
+
 export type RunArtifactInspection = {
   target: string;
   root: string;
   artifact?: Artifact;
   stats?: Stats;
   issue?: string;
+  reason?: RunArtifactIssueReason;
 };
 
 type RunArtifactEvidence = {
@@ -82,6 +85,7 @@ export const inspectRunArtifact = async (
       target: authoredPath,
       root: runArtifactsRoot(run),
       issue: `${authoredPath} contains a null byte`,
+      reason: "unreadable",
     };
   }
   const matching = run.artifacts.filter((candidate) => candidate.path === authoredPath);
@@ -91,6 +95,7 @@ export const inspectRunArtifact = async (
       target: resolve(root, authoredPath),
       root,
       issue: `${authoredPath} was not captured in the frozen artifact index`,
+      reason: "absent",
     };
   }
   if (matching.some((candidate) => candidate.root !== undefined && !isAbsolute(candidate.root))) {
@@ -98,6 +103,7 @@ export const inspectRunArtifact = async (
       target: authoredPath,
       root: defaultArtifactsRoot(run),
       issue: `${authoredPath} declares a non-absolute artifact root`,
+      reason: "unreadable",
     };
   }
   const artifact = matching[0];
@@ -108,6 +114,7 @@ export const inspectRunArtifact = async (
       target: authoredPath,
       root: defaultArtifactsRoot(run),
       issue: `${authoredPath} is ambiguous across multiple artifact roots`,
+      reason: "unreadable",
     };
   }
   // A non-default root is trusted only for an exact artifact-index entry.
@@ -116,7 +123,12 @@ export const inspectRunArtifact = async (
   const root = matchingRoots[0] ?? defaultArtifactsRoot(run);
   const target = resolve(root, authoredPath);
   if (pathEscapesRoot(root, target)) {
-    return { target, root, issue: `${authoredPath} escapes the captured artifact root` };
+    return {
+      target,
+      root,
+      issue: `${authoredPath} escapes the captured artifact root`,
+      reason: "unreadable",
+    };
   }
 
   const path = relative(root, target);
@@ -134,7 +146,13 @@ export const inspectRunArtifact = async (
     } catch (error) {
       const code = (error as NodeJS.ErrnoException).code;
       if (code === "ENOENT" || code === "ENOTDIR") {
-        return { target, root, ...identified, issue: `${authoredPath} not found` };
+        return {
+          target,
+          root,
+          ...identified,
+          issue: `${authoredPath} not found`,
+          reason: "absent",
+        };
       }
       throw error;
     }
@@ -144,16 +162,17 @@ export const inspectRunArtifact = async (
         root,
         ...identified,
         issue: `${authoredPath} resolves through a symlink`,
+        reason: "unreadable",
       };
     }
     if (index < components.length - 1 && !stats.isDirectory()) {
-      return { target, root, ...identified, issue: `${authoredPath} not found` };
+      return { target, root, ...identified, issue: `${authoredPath} not found`, reason: "absent" };
     }
     if (index === components.length - 1) {
       return { target, root, ...identified, stats };
     }
   }
-  return { target, root, ...identified, issue: `${authoredPath} not found` };
+  return { target, root, ...identified, issue: `${authoredPath} not found`, reason: "absent" };
 };
 
 export type ReadRunArtifactTextOptions = {
@@ -247,3 +266,27 @@ export const readRunArtifactText = async (
 ): Promise<string> => {
   return (await readRunArtifactBytes(run, authoredPath, options)).toString("utf8");
 };
+
+/**
+ * Read optional artifact evidence. Returns null when the artifact was never
+ * captured, and still throws when one exists but is unsafe to consume, so a
+ * traversal or symlink violation is never mistaken for missing evidence.
+ */
+export const readRunArtifactTextIfPresent = async (
+  run: RunArtifactEvidence,
+  authoredPath: string,
+  options: ReadRunArtifactTextOptions = {},
+): Promise<string | null> => {
+  const inspected = await inspectRunArtifact(run, authoredPath);
+  if (inspected.reason === "absent") return null;
+  if (!inspected.stats?.isFile()) {
+    throw new Error(
+      inspected.issue ??
+        (inspected.stats ? `${authoredPath} is not a file` : `${authoredPath} not found`),
+    );
+  }
+  const maxBytes = options.maxBytes ?? DEFAULT_ARTIFACT_READ_BYTES;
+  return (
+    await readInspectedFileBytes(inspected.target, inspected.stats, maxBytes, authoredPath)
+  ).toString("utf8");
+};

package/src/core/assertion.ts

@@ -42,34 +42,23 @@ export interface AssertionImpl<C extends Assertion = Assertion> {
  * `AssertionImpl`, plugins must describe their case configuration so Lux can
  * reject invalid plans before starting a subject run.
  */
-export type AssertionPlugin<C extends Assertion = Assertion> = AssertionImpl<C> & {
+export type AssertionPlugin<
+  C extends Assertion = Assertion,
+  Input extends Assertion = C,
+> = AssertionImpl<C> & {
   configSchema: z.ZodObject & z.ZodType<C>;
+  /** Type-only input accepted while authoring before schema defaults apply. */
+  readonly __input?: Input;
   /** Author-maintained identity for evaluator/schema behavior. */
   pluginVersion: string;
 };
 
 type SchemaAssertion<S extends z.ZodObject> = z.output<S> extends Assertion ? z.output<S> : never;
-
-type IsUnion<T, Whole = T> = T extends Whole ? ([Whole] extends [T] ? false : true) : never;
-
-type SingleStringLiteral<T> = T extends string
-  ? string extends T
-    ? never
-    : true extends IsUnion<T>
-      ? never
-      : T
-  : never;
+type SchemaAssertionInput<S extends z.ZodObject> =
+  z.input<S> extends Assertion ? z.input<S> : never;
 
 type LiteralAssertionType<S extends z.ZodObject> =
-  SchemaAssertion<S> extends { type: infer OutputType }
-    ? z.input<S> extends { type: infer InputType }
-      ? SingleStringLiteral<OutputType> extends infer Literal extends string
-        ? [InputType, Literal] extends [Literal, InputType]
-          ? Literal
-          : never
-        : never
-      : never
-    : never;
+  S["shape"]["type"] extends z.ZodLiteral<infer Literal extends string> ? Literal : never;
 
 export type AssertionPluginDefinition<S extends z.ZodObject> = {
   type: LiteralAssertionType<S>;
@@ -111,12 +100,34 @@ export const assertionPluginIssue = (definition: {
  * Define a schema-validated assertion plugin while inferring the config passed
  * to `evaluate` from the schema.
  */
+export const baseAssertionFields = {
+  id: z.string().min(1).optional(),
+  weight: z.number().positive().optional(),
+  threshold: z.number().min(0).max(1).optional(),
+  name: z.string().optional(),
+} as const;
+
 export const defineAssertion = <S extends z.ZodObject>(
   definition: AssertionPluginDefinition<S>,
-): AssertionPlugin<SchemaAssertion<S>> => {
+): AssertionPlugin<SchemaAssertion<S>, SchemaAssertionInput<S>> => {
   const issue = assertionPluginIssue(definition);
   if (issue) throw new TypeError(issue);
-  return definition as unknown as AssertionPlugin<SchemaAssertion<S>>;
+  return definition as unknown as AssertionPlugin<SchemaAssertion<S>, SchemaAssertionInput<S>>;
+};
+
+export const defineAssertionRegistry = <const Plugins extends readonly AssertionPlugin[]>(
+  ...plugins: Plugins
+): Plugins => {
+  const types = new Set<string>();
+  for (const plugin of plugins) {
+    const issue = assertionPluginIssue(plugin);
+    if (issue) throw new TypeError(issue);
+    if (types.has(plugin.type)) {
+      throw new TypeError(`assertion registry has duplicate type '${plugin.type}'`);
+    }
+    types.add(plugin.type);
+  }
+  return plugins;
 };
 
 export const GradingAssertionEntrySchema = z.object({
@@ -137,11 +148,19 @@ export const GradingAssertionEntrySchema = z.object({
 });
 export type GradingAssertionEntry = z.infer<typeof GradingAssertionEntrySchema>;
 
+const SkippedGradingAssertionSchema = z.object({
+  id: z.string(),
+  type: z.string(),
+  name: z.string(),
+  reason: z.literal("judge-disabled"),
+});
+
 export const GradingReportSchema = z.object({
   schemaVersion: z.literal(1),
   casePassed: z.boolean().nullable(),
   caseScore: z.number().nullable(),
   assertions: z.array(GradingAssertionEntrySchema),
+  skippedAssertions: z.array(SkippedGradingAssertionSchema).optional(),
   /**
    * Present when the subject run or grading did not complete. Assertions may
    * still contain partial diagnostics, but the report is not a clean verdict.

package/src/core/case-authoring-state.ts

@@ -0,0 +1,65 @@
+import type { AssertionPlugin } from "./assertion.js";
+import type { EvalCase } from "./case.js";
+import type { LifecycleHooksPlugin } from "./lifecycle-hooks.js";
+
+export type EvalCaseExtensions = {
+  assertions: readonly AssertionPlugin[];
+  fixtures: readonly LifecycleHooksPlugin[];
+};
+
+const CASE_EXTENSIONS_KEY = Symbol.for("@prismatic-io/lux/eval-case-extensions");
+const extensionState = globalThis as typeof globalThis & { [key: symbol]: unknown };
+const sharedCaseExtensions = extensionState[CASE_EXTENSIONS_KEY];
+// A project-local package and the CLI can load separate Lux module instances.
+const caseExtensions =
+  sharedCaseExtensions instanceof WeakMap
+    ? (sharedCaseExtensions as WeakMap<object, EvalCaseExtensions>)
+    : new WeakMap<object, EvalCaseExtensions>();
+extensionState[CASE_EXTENSIONS_KEY] = caseExtensions;
+
+const uniqueBy = <T>(values: readonly T[], keyOf: (value: T) => string, kind: string): T[] => {
+  const unique = new Map<string, T>();
+  for (const value of values) {
+    const key = keyOf(value);
+    const existing = unique.get(key);
+    if (existing && existing !== value) {
+      throw new TypeError(`case has conflicting ${kind} definitions for '${key}'`);
+    }
+    unique.set(key, value);
+  }
+  return [...unique.values()];
+};
+
+export const collectEvalCaseExtensions = (
+  input: EvalCase,
+  assertions: readonly AssertionPlugin[] = [],
+  fixtures: readonly LifecycleHooksPlugin[] = [],
+): EvalCaseExtensions => {
+  const availableAssertions = uniqueBy(assertions, (plugin) => plugin.type, "assertion");
+  const byType = new Map(availableAssertions.map((plugin) => [plugin.type, plugin]));
+  return {
+    assertions: uniqueBy(
+      (input.assertions ?? []).flatMap((assertion) => {
+        const plugin = byType.get(assertion.type);
+        return plugin ? [plugin] : [];
+      }),
+      (plugin) => plugin.type,
+      "assertion",
+    ),
+    fixtures: uniqueBy(fixtures, (fixture) => fixture.name, "fixture"),
+  };
+};
+
+export const setEvalCaseExtensions = (evalCase: EvalCase, extensions: EvalCaseExtensions): void => {
+  if (extensions.assertions.length > 0 || extensions.fixtures.length > 0) {
+    caseExtensions.set(evalCase, extensions);
+  }
+};
+
+export const evalCaseExtensions = (evalCase: EvalCase): EvalCaseExtensions =>
+  caseExtensions.get(evalCase) ?? { assertions: [], fixtures: [] };
+
+export const transferEvalCaseExtensions = (source: EvalCase, target: EvalCase): void => {
+  const extensions = caseExtensions.get(source);
+  if (extensions) caseExtensions.set(target, extensions);
+};

package/src/core/case.ts

@@ -1,4 +1,7 @@
 import { z } from "zod";
+import type { AssertionPlugin } from "./assertion.js";
+import { collectEvalCaseExtensions, setEvalCaseExtensions } from "./case-authoring-state.js";
+import type { LifecycleHooksPlugin } from "./lifecycle-hooks.js";
 import { formatZodIssues } from "./zod.js";
 
 export const FixturesRefSchema = z.union([
@@ -67,14 +70,62 @@ export const EvalCaseSchema = EvalCaseObjectSchema
   });
 export type EvalCase = z.infer<typeof EvalCaseSchema>;
 
-export const defineEvalCase = (input: EvalCase): EvalCase => {
-  const parsed = EvalCaseSchema.safeParse(input);
-  if (!parsed.success) {
-    throw new Error(`Invalid eval case: ${formatZodIssues(parsed.error)}`);
-  }
-  return parsed.data;
+type AssertionPlanFields = Pick<Assertion, "id" | "name" | "threshold" | "weight">;
+type AssertionInputOf<Plugin> =
+  Plugin extends AssertionPlugin<Assertion, infer Input> ? Input & AssertionPlanFields : never;
+
+export type EvalCaseInput<AllowedAssertion extends Assertion = Assertion> = Omit<
+  EvalCase,
+  "assertions"
+> & {
+  assertions?: AllowedAssertion[];
+};
+
+export type EvalCaseAuthoringOptions<
+  Plugins extends readonly AssertionPlugin[] = readonly AssertionPlugin[],
+> = {
+  assertions?: Plugins;
+  fixtures?: readonly LifecycleHooksPlugin[];
+};
+
+export type EvalCaseDefinition<AllowedAssertion extends Assertion = Assertion> = {
+  (input: EvalCaseInput<AllowedAssertion>): EvalCase;
+  /**
+   * Create a case definition with selectively activated lifecycle fixtures.
+   * Further calls to `extend` append fixtures in setup order.
+   */
+  extend<const Plugins extends readonly AssertionPlugin[] = readonly []>(
+    options: EvalCaseAuthoringOptions<Plugins>,
+  ): EvalCaseDefinition<AllowedAssertion | AssertionInputOf<Plugins[number]>>;
 };
 
+export const createEvalCaseDefinition = <AllowedAssertion extends Assertion = Assertion>(
+  inheritedAssertions: readonly AssertionPlugin[] = [],
+  inheritedFixtures: readonly LifecycleHooksPlugin[] = [],
+): EvalCaseDefinition<AllowedAssertion> => {
+  const define = ((input: EvalCaseInput<AllowedAssertion>): EvalCase => {
+    const parsed = EvalCaseSchema.safeParse(input);
+    if (!parsed.success) {
+      throw new Error(`Invalid eval case: ${formatZodIssues(parsed.error)}`);
+    }
+    const extensions = collectEvalCaseExtensions(
+      parsed.data,
+      inheritedAssertions,
+      inheritedFixtures,
+    );
+    setEvalCaseExtensions(parsed.data, extensions);
+    return parsed.data;
+  }) as EvalCaseDefinition<AllowedAssertion>;
+  define.extend = (({ assertions = [], fixtures = [] }: EvalCaseAuthoringOptions) =>
+    createEvalCaseDefinition(
+      [...inheritedAssertions, ...assertions],
+      [...inheritedFixtures, ...fixtures],
+    )) as EvalCaseDefinition<AllowedAssertion>["extend"];
+  return define;
+};
+
+export const defineEvalCase = createEvalCaseDefinition();
+
 const PersistedFixturesRefSchema = z.union([
   z.string(),
   z.looseObject({ kind: z.literal("dir"), path: z.string() }),

package/src/core/driver.ts

@@ -1,6 +1,7 @@
 import { z } from "zod";
 import type { EventHandler } from "./emitter.js";
 import type { Answer, Done, ErrorEvent, Interrupt, Progress, ReadyState } from "./events.js";
+import type { FixtureValueBinding } from "./lifecycle-hooks.js";
 
 export const ArtifactSchema = z.object({
   path: z.string(),
@@ -28,6 +29,8 @@ export type StartContext = {
   caseId?: string;
   prompt: string;
   fixtures?: ResolvedFixtures | undefined;
+  /** Authored fixture values selected for this case. */
+  fixtureValues?: readonly FixtureValueBinding[] | undefined;
   /** Root containing persisted runs. Drivers may use it as an isolation boundary. */
   runsRoot?: string | undefined;
   runDir: string;

package/src/core/events.ts

@@ -82,6 +82,7 @@ export type ErrorEventReason = z.infer<typeof ErrorEventReasonSchema>;
 export const ErrorEventSchema = z.object({
   exitReason: ErrorEventReasonSchema,
   reason: z.string(),
+  summary: z.unknown().optional(),
 });
 export type ErrorEvent = z.infer<typeof ErrorEventSchema>;
 

package/src/core/lifecycle-fixtures.ts

@@ -2,6 +2,7 @@ import { valueHash } from "./hash.js";
 import {
   type CaseLifecycleHookContext,
   defineLifecycleHooks,
+  type FixtureValue,
   type LifecycleHookCleanup,
   type LifecycleHooksPlugin,
   type SuiteLifecycleHookContext,
@@ -22,6 +23,8 @@ export type LifecycleFixturesDefinition = {
    * to execution identity.
    */
   identity?: unknown;
+  /** Public, serializable data made available to the selected case's driver. */
+  value?: FixtureValue;
   suite?: LifecycleFixture<SuiteLifecycleHookContext>;
   case?: LifecycleFixture<CaseLifecycleHookContext>;
 };
@@ -109,8 +112,8 @@ const activateFixture = async <Context extends SuiteLifecycleHookContext>(
  * Setup belongs before `use`; teardown belongs after it.
  */
 export const defineFixtures = (definition: LifecycleFixturesDefinition): LifecycleHooksPlugin => {
-  if (!definition.suite && !definition.case) {
-    throw new TypeError("fixtures plugin must define suite or case");
+  if (!definition.suite && !definition.case && definition.value === undefined) {
+    throw new TypeError("fixtures plugin must define value, suite, or case");
   }
   if (definition.suite !== undefined && typeof definition.suite !== "function") {
     throw new TypeError("fixtures plugin suite must be a function");
@@ -123,8 +126,10 @@ export const defineFixtures = (definition: LifecycleFixturesDefinition): Lifecyc
   return defineLifecycleHooks({
     name: definition.name,
     pluginVersion: definition.pluginVersion,
+    ...(definition.value === undefined ? {} : { value: definition.value }),
     ...(definition.identity === undefined ? {} : { identity: definition.identity }),
     implementationHash: valueHash({
+      value: definition.value,
       identity: definition.identity,
       suite: definition.suite,
       case: definition.case,

package/src/core/lifecycle-hooks.ts

@@ -1,7 +1,15 @@
+import { z } from "zod";
 import { valueHash } from "./hash.js";
 
 export type LifecycleHookCleanup = () => void | Promise<void>;
 export type LifecycleHookSetupResult = LifecycleHookCleanup | undefined;
+export const FixtureValueSchema = z.json();
+export type FixtureValue = z.infer<typeof FixtureValueSchema>;
+
+export type FixtureValueBinding = {
+  name: string;
+  value: FixtureValue;
+};
 
 export type SuiteLifecycleHookItem = {
   id: string;
@@ -27,6 +35,8 @@ export type LifecycleHooksPlugin = {
   name: string;
   /** Author-maintained identity for setup and teardown behavior. */
   pluginVersion: string;
+  /** Public, serializable data made available to the selected case's driver. */
+  value?: FixtureValue;
   /**
    * Public, non-secret values captured by hook closures and relevant to
    * execution identity. Included in the implementation hash.
@@ -75,8 +85,11 @@ export const lifecycleHooksPluginIssue = (
   if (definition.setupCase !== undefined && typeof definition.setupCase !== "function") {
     return "lifecycle hooks plugin setupCase must be a function";
   }
-  if (!definition.setupSuite && !definition.setupCase) {
-    return "lifecycle hooks plugin must define setupSuite or setupCase";
+  if (definition.value !== undefined && !FixtureValueSchema.safeParse(definition.value).success) {
+    return "lifecycle hooks plugin value must be JSON-serializable";
+  }
+  if (!definition.setupSuite && !definition.setupCase && definition.value === undefined) {
+    return "lifecycle hooks plugin must define a value, setupSuite, or setupCase";
   }
   return undefined;
 };
@@ -93,6 +106,7 @@ export const defineLifecycleHooks = (definition: LifecycleHooksPlugin): Lifecycl
     implementationHash:
       definition.implementationHash ??
       valueHash({
+        value: definition.value,
         identity: definition.identity,
         setupSuite: definition.setupSuite,
         setupCase: definition.setupCase,

package/src/core/wire.ts

@@ -47,6 +47,7 @@ const ErrorEventSchema = z.object({
   type: z.literal("error"),
   exitReason: ErrorEventReasonSchema.default("error"),
   reason: z.string(),
+  summary: z.unknown().optional(),
 });
 
 export const WireEventSchema = z.discriminatedUnion("type", [
@@ -85,7 +86,10 @@ const wireEventToParsed = (event: WireEvent): ParsedDriverEvent => {
   if (event.type === "done") {
     return { tag: "done", done: { exitReason: "done", summary: event.summary } };
   }
-  return { tag: "error", error: { exitReason: event.exitReason, reason: event.reason } };
+  return {
+    tag: "error",
+    error: { exitReason: event.exitReason, reason: event.reason, summary: event.summary },
+  };
 };
 
 /**

package/src/drivers/subprocess/index.ts

@@ -82,6 +82,7 @@ const KILL_GRACE_MS = 5_000;
 const DEFAULT_MAX_LINE_BYTES = 10_000_000;
 const MAX_STDERR_TAIL_CHARS = 8_192;
 const DEFAULT_MAX_ARTIFACT_FILES = 500;
+const MAX_FIXTURE_VALUES_BYTES = 64 * 1024;
 const ARTIFACT_SNAPSHOT_MODULE = fileURLToPath(
   new URL("../../core/artifact-snapshot.js", import.meta.url),
 );
@@ -139,6 +140,10 @@ class SubprocessDriverImpl implements AgentDriver {
 
   async start(ctx: StartContext): Promise<ReadyState> {
     this.artifactsDir = ctx.artifactsDir;
+    const fixtureValues = JSON.stringify(ctx.fixtureValues ?? []);
+    if (Buffer.byteLength(fixtureValues) > MAX_FIXTURE_VALUES_BYTES) {
+      throw new Error(`subprocess fixture values exceed ${MAX_FIXTURE_VALUES_BYTES} bytes`);
+    }
     const managed = spawnManagedProcess(this.config.command, this.config.args ?? [], {
       cwd: this.config.cwd ?? ctx.runDir,
       env: {
@@ -147,6 +152,7 @@ class SubprocessDriverImpl implements AgentDriver {
         ...(ctx.caseId ? { LUX_CASE_ID: ctx.caseId } : {}),
         LUX_RUN_DIR: ctx.runDir,
         LUX_ARTIFACTS_DIR: ctx.artifactsDir,
+        LUX_FIXTURE_VALUES: fixtureValues,
         LUX_PROMPT: ctx.prompt,
         LUX_ARTIFACT_SNAPSHOT_MODULE: ARTIFACT_SNAPSHOT_MODULE,
       },

package/src/index.ts

@@ -16,16 +16,15 @@ export {
   PersonaAnswererConfigSchema,
   personaAnswerer,
 } from "./answerers/persona/index.js";
-export {
-  type ScriptedAnswererConfig,
-  ScriptedAnswererConfigSchema,
-  scriptedAnswerer,
-} from "./answerers/scripted/index.js";
 export {
   type TerminalAnswererConfig,
   TerminalAnswererConfigSchema,
   terminalAnswerer,
 } from "./answerers/terminal/index.js";
+export {
+  type BuiltinAssertion,
+  defineEvalCase,
+} from "./assertions/authoring.js";
 // Bundled deterministic and judge assertions.
 export {
   type AgentScope,
@@ -185,6 +184,7 @@ export {
   type AssertionResult,
   AssertionResultSchema,
   defineAssertion,
+  defineAssertionRegistry,
   type EvaluateContext,
   type GradingAssertionEntry,
   GradingAssertionEntrySchema,
@@ -198,8 +198,9 @@ export {
   AssertionSchema,
   type DriverRef,
   DriverRefSchema,
-  defineEvalCase,
   type EvalCase,
+  type EvalCaseAuthoringOptions,
+  type EvalCaseDefinition,
   EvalCaseSchema,
   type FixturesRef,
   FixturesRefSchema,
@@ -300,6 +301,9 @@ export {
 export {
   type CaseLifecycleHookContext,
   defineLifecycleHooks,
+  type FixtureValue,
+  type FixtureValueBinding,
+  FixtureValueSchema,
   type LifecycleHookCleanup,
   type LifecycleHookSetupResult,
   type LifecycleHooksPlugin,

package/src/orchestrator/annotation-store.ts

@@ -57,6 +57,7 @@ const recoverStaleLock = async (lockPath: string): Promise<boolean> => {
 const acquireAnnotationLock = async (path: string): Promise<() => Promise<void>> => {
   const lockPath = `${path}.lock`;
   const deadline = Date.now() + LOCK_WAIT_MS;
+  let attemptedRecovery = false;
   while (true) {
     try {
       const owner = await publishOwnedLock(lockPath, {
@@ -69,7 +70,10 @@ const acquireAnnotationLock = async (path: string): Promise<() => Promise<void>>
       };
     } catch (error) {
       if ((error as NodeJS.ErrnoException).code !== "EEXIST") throw error;
-      if (await recoverStaleLock(lockPath)) continue;
+      if (!attemptedRecovery) {
+        attemptedRecovery = true;
+        if (await recoverStaleLock(lockPath)) continue;
+      }
       if (Date.now() >= deadline) {
         throw new Error(`timed out waiting to update grader annotations: ${path}`);
       }

package/src/orchestrator/case-extensions.ts

@@ -0,0 +1,44 @@
+import { evalCaseExtensions } from "../core/case-authoring-state.js";
+import type {
+  EvalCase,
+  FixtureValueBinding,
+  LifecycleHooksPlugin,
+  Registry,
+} from "../core/index.js";
+
+export const registryForEvalCase = (registry: Registry, evalCase: EvalCase): Registry => {
+  let resolved = registry;
+  for (const plugin of evalCaseExtensions(evalCase).assertions) {
+    const existing = resolved.getAssertion(plugin.type);
+    if (existing === plugin) continue;
+    if (existing) {
+      throw new TypeError(
+        `case-local assertion '${plugin.type}' conflicts with a registered implementation`,
+      );
+    }
+    resolved = resolved.withAssertion(plugin);
+  }
+  return resolved;
+};
+
+export const fixturesForEvalCase = (evalCase: EvalCase): readonly LifecycleHooksPlugin[] =>
+  evalCaseExtensions(evalCase).fixtures;
+
+export const valuesForFixtures = (
+  fixtures: readonly LifecycleHooksPlugin[],
+): FixtureValueBinding[] =>
+  fixtures.flatMap(({ name, value }) => (value === undefined ? [] : [{ name, value }]));
+
+export const mergeLifecycleFixtures = (
+  ...groups: readonly (readonly LifecycleHooksPlugin[])[]
+): LifecycleHooksPlugin[] => {
+  const fixtures = new Map<string, LifecycleHooksPlugin>();
+  for (const fixture of groups.flat()) {
+    const existing = fixtures.get(fixture.name);
+    if (existing && existing !== fixture) {
+      throw new TypeError(`conflicting lifecycle fixture definitions for '${fixture.name}'`);
+    }
+    fixtures.set(fixture.name, fixture);
+  }
+  return [...fixtures.values()];
+};

package/src/orchestrator/config.ts

@@ -4,7 +4,6 @@ import { pathToFileURL } from "node:url";
 import { z } from "zod";
 import { claudeCodeAnswerer } from "../answerers/claude-code/index.js";
 import { personaAnswerer } from "../answerers/persona/index.js";
-import { scriptedAnswerer } from "../answerers/scripted/index.js";
 import { terminalAnswerer } from "../answerers/terminal/index.js";
 import { allCoreAssertions } from "../assertions/core/index.js";
 import { allMcpAssertions } from "../assertions/mcp/index.js";
@@ -182,7 +181,6 @@ const builtinDrivers: DriverPlugin<never>[] = [
 const builtinAnswerers: AnswererPlugin<never>[] = [
   personaAnswerer,
   claudeCodeAnswerer,
-  scriptedAnswerer,
   terminalAnswerer,
 ];
 export const buildRegistryFromConfig = (config: ResolvedLuxConfig): Registry => {