@prismatic-io/lux 0.0.1 → 0.0.2-preview.4

package/src/orchestrator/doctor.ts

@@ -5,6 +5,7 @@ import { glob } from "tinyglobby";
 import { z } from "zod";
 import { RubricConfigSchema } from "../assertions/rubric/index.js";
 import type { EvalCase, Registry } from "../core/index.js";
+import { registryForEvalCase } from "./case-extensions.js";
 import { buildRegistryFromConfig, loadLuxConfig, type ResolvedLuxConfig } from "./config.js";
 import { type DiscoveredCase, type DiscoveryResult, discoverCases } from "./discover.js";
 import { cliIdentity, ExperimentCampaignPlanSchema, planExperimentCampaign } from "./experiment.js";
@@ -559,12 +560,13 @@ const inspectCase = async (
 ): Promise<CliTarget[]> => {
   try {
     const authored = await loadEvalCase(discovered.path);
+    const caseRegistry = registryForEvalCase(registry, authored);
     const drivers = caseDriverReferences(authored.driver, config);
     const answerer = authored.answerer ?? config.defaultAnswerer;
-    validateCaseDrivers(discovered.id, drivers, registry, recorder);
-    validateCaseAnswerer(discovered.id, answerer, registry, recorder);
+    validateCaseDrivers(discovered.id, drivers, caseRegistry, recorder);
+    validateCaseAnswerer(discovered.id, answerer, caseRegistry, recorder);
     return [
-      ...(await validateCaseAssertions(discovered.id, authored, config, registry, recorder)),
+      ...(await validateCaseAssertions(discovered.id, authored, config, caseRegistry, recorder)),
       ...executionTargetsForCase(discovered.id, drivers, answerer),
     ];
   } catch (error) {

package/src/orchestrator/grading.ts

@@ -76,6 +76,7 @@ export const graderConfigHash = (input: {
   assertions: readonly AssertionEvaluatorIdentity[];
   validatedAssertionsHash: string;
   graderContext?: unknown;
+  skipJudge?: boolean;
 }): string =>
   valueHash({
     luxVersion: input.luxVersion,
@@ -83,6 +84,7 @@ export const graderConfigHash = (input: {
     assertions: input.assertions,
     validatedAssertionsHash: input.validatedAssertionsHash,
     graderContext: input.graderContext ?? null,
+    skipJudge: input.skipJudge ?? false,
   });
 
 export const validateAssertionPlan = async (
@@ -133,6 +135,7 @@ export const gradeRun = async (
     complete: (attemptIds: readonly string[]) => void | Promise<void>;
   },
   validatedConfigs?: readonly unknown[],
+  skipJudge = false,
 ): Promise<GradingReport> => {
   const guardedObserver = guardRunObserver(observer);
   const assertions = run.case.assertions ?? [];
@@ -141,12 +144,27 @@ export const gradeRun = async (
   }
 
   const entries: GradingAssertionEntry[] = [];
+  const skippedAssertions: Array<{
+    id: string;
+    type: string;
+    name: string;
+    reason: "judge-disabled";
+  }> = [];
   let weightedScoreSum = 0;
   let weightSum = 0;
   let allPassed = true;
   let gradingIncomplete = false;
 
   for (const [index, a] of assertions.entries()) {
+    if (skipJudge && a.type === "rubric") {
+      skippedAssertions.push({
+        id: a.id ?? `${a.type}:${index + 1}`,
+        type: a.type,
+        name: describeAssertion(a),
+        reason: "judge-disabled",
+      });
+      continue;
+    }
     // Grading is the last cancellation point — an LLM judge can run for
     // minutes per assertion. An aborted grade can't claim everything passed.
     if (abortSignal?.aborted) {
@@ -247,12 +265,13 @@ export const gradeRun = async (
   }
 
   let caseScore: number | null = null;
-  if (!gradingIncomplete) caseScore = weightSum > 0 ? weightedScoreSum / weightSum : 0;
+  if (!gradingIncomplete && weightSum > 0) caseScore = weightedScoreSum / weightSum;
   return {
     schemaVersion: 1,
-    casePassed: gradingIncomplete ? null : allPassed,
+    casePassed: gradingIncomplete || weightSum === 0 ? null : allPassed,
     caseScore,
     assertions: entries,
+    ...(skippedAssertions.length > 0 ? { skippedAssertions } : {}),
     ...(gradingIncomplete
       ? { incomplete: { exitReason: "error" as const, phase: "grading" as const } }
       : {}),

package/src/orchestrator/loader.ts

@@ -1,5 +1,6 @@
 import { resolve } from "node:path";
 import { pathToFileURL } from "node:url";
+import { transferEvalCaseExtensions } from "../core/case-authoring-state.js";
 import { type EvalCase, EvalCaseSchema, formatZodIssues } from "../core/index.js";
 
 export const loadEvalCase = async (filePath: string): Promise<EvalCase> => {
@@ -15,5 +16,6 @@ export const loadEvalCase = async (filePath: string): Promise<EvalCase> => {
   if (!result.success) {
     throw new Error(`Invalid eval case at ${absolute}: ${formatZodIssues(result.error)}`);
   }
+  transferEvalCaseExtensions(mod.default as EvalCase, result.data);
   return result.data;
 };

package/src/orchestrator/orchestrator.ts

@@ -18,6 +18,12 @@ import {
   type SuiteLifecycleHookContext,
   valueHash,
 } from "../core/index.js";
+import {
+  fixturesForEvalCase,
+  mergeLifecycleFixtures,
+  registryForEvalCase,
+  valuesForFixtures,
+} from "./case-extensions.js";
 import { evaluatorImplementationHash } from "./evaluator-identity.js";
 import {
   assertionEvaluatorIdentities,
@@ -122,6 +128,8 @@ export type RunRequest = {
   /** Root that relative `fixtures` dir paths resolve against. */
   fixturesRoot?: string | undefined;
   answererOverride?: string | undefined;
+  /** Skip built-in rubric judge assertions without changing subject execution. */
+  skipJudge?: boolean | undefined;
   startedAt?: Date | undefined;
   /** The `runSuite` invocation this case attempt belongs to. */
   suiteId?: string | undefined;
@@ -476,11 +484,6 @@ export class Orchestrator {
   }
 
   async run(request: RunRequest): Promise<RunResult> {
-    if (this.lifecycleFixtures.length > 0 && !request.suiteId) {
-      throw new Error(
-        "Orchestrator.run() cannot execute configured lifecycle fixtures; use runSuite() for a single case",
-      );
-    }
     if (!request.case && !request.casePath) {
       throw new Error("RunRequest requires either 'case' or 'casePath'");
     }
@@ -489,11 +492,24 @@ export class Orchestrator {
     }
     const casePath = request.casePath ? resolve(request.casePath) : INLINE_CASE_PATH;
     const authoredCase = request.case ?? (await loadEvalCase(casePath));
+    const lifecycleFixtures = mergeLifecycleFixtures(
+      this.lifecycleFixtures,
+      fixturesForEvalCase(authoredCase),
+    );
+    if (
+      lifecycleFixtures.some(({ setupSuite, setupCase }) => setupSuite || setupCase) &&
+      !request.suiteId
+    ) {
+      throw new Error(
+        "Orchestrator.run() cannot execute lifecycle fixtures; use runSuite() for a single case",
+      );
+    }
+    const registry = registryForEvalCase(this.registry, authoredCase);
     const evalCase = bindSubjectRoot(authoredCase, request.subjectRoot);
     const caseId = resolveCaseId(casePath, evalCase, request.caseId);
     const validatedAssertionConfigs = await validateAssertionPlan(
       evalCase.assertions ?? [],
-      this.registry,
+      registry,
     );
 
     const driverRef = resolveDriverReference(evalCase, this.defaultDriver);
@@ -506,11 +522,11 @@ export class Orchestrator {
     );
     const answererName = answererNameOf(answererRef);
 
-    const driverPlugin = this.registry.getDriver(driverName);
+    const driverPlugin = registry.getDriver(driverName);
     if (!driverPlugin) {
       throw new Error(`no driver registered for '${driverName}'`);
     }
-    const answererPlugin = this.registry.getAnswerer(answererName);
+    const answererPlugin = registry.getAnswerer(answererName);
     if (!answererPlugin) {
       throw new Error(`no answerer registered for '${answererName}'`);
     }
@@ -544,7 +560,7 @@ export class Orchestrator {
     try {
       const assertionImplementations = assertionEvaluatorIdentities(
         authoredCase.assertions ?? [],
-        this.registry,
+        registry,
       );
       const validatedAssertionsHash = valueHash(validatedAssertionConfigs);
       const implementationHash = await this.evaluatorImplementationIdentity();
@@ -554,34 +570,40 @@ export class Orchestrator {
         assertions: assertionImplementations,
         validatedAssertionsHash,
         graderContext: this.harness,
+        skipJudge: request.skipJudge ?? false,
       });
-      return await this.runWithPlugins({
-        request,
-        casePath,
-        authoredCase,
-        evalCase,
-        validatedAssertionConfigs,
-        caseId,
-        driverName,
-        answererName,
-        answererConfig: answererCfg,
-        driverConfigHash: valueHash(driverCfg),
-        answererConfigHash: valueHash(answererCfg),
-        validatedAssertionsHash,
-        graderConfigHash: graderHash,
-        evaluatorConfigHash: valueHash({
-          luxVersion: LUX_VERSION,
-          implementationHash,
-          harness: this.harness,
-          driver: { name: driverPlugin.name, create: driverPlugin.create },
-          answerer: { name: answererPlugin.name, create: answererPlugin.create },
-          assertions: assertionImplementations,
-          lifecycleFixtures: this.lifecycleFixtures,
+      return await this.runWithPlugins(
+        {
+          request,
+          casePath,
+          authoredCase,
+          evalCase,
+          validatedAssertionConfigs,
+          caseId,
+          driverName,
+          answererName,
+          answererConfig: answererCfg,
+          driverConfigHash: valueHash(driverCfg),
+          answererConfigHash: valueHash(answererCfg),
           validatedAssertionsHash,
-        }),
-        driver,
-        answerer,
-      });
+          graderConfigHash: graderHash,
+          evaluatorConfigHash: valueHash({
+            luxVersion: LUX_VERSION,
+            implementationHash,
+            harness: this.harness,
+            driver: { name: driverPlugin.name, create: driverPlugin.create },
+            answerer: { name: answererPlugin.name, create: answererPlugin.create },
+            assertions: assertionImplementations,
+            lifecycleFixtures,
+            validatedAssertionsHash,
+            skipJudge: request.skipJudge ?? false,
+          }),
+          fixtureValues: valuesForFixtures(lifecycleFixtures),
+          driver,
+          answerer,
+        },
+        registry,
+      );
     } finally {
       await closeQuietly(() => driver.close(), `driver '${driverName}'`);
       await closeQuietly(() => answerer.close(), `answerer '${answererName}'`);
@@ -599,6 +621,7 @@ export class Orchestrator {
       runsRoot: string;
       fixturesRoot?: string | undefined;
       answererOverride?: string | undefined;
+      skipJudge?: boolean | undefined;
       observer?: SuiteObserver | undefined;
       /** Aborts the cases in flight and stops the pool pulling new ones. */
       abortSignal?: AbortSignal | undefined;
@@ -628,6 +651,20 @@ export class Orchestrator {
     const observer = opts.observer;
     const requestedConcurrency = validateSuiteConcurrency(opts.concurrency);
     const concurrency = Math.min(requestedConcurrency, Math.max(items.length, 1));
+    const authoredCases = await Promise.allSettled(items.map((item) => loadEvalCase(item.path)));
+    const localFixtures = authoredCases.map((result) =>
+      result.status === "fulfilled" ? fixturesForEvalCase(result.value) : [],
+    );
+    const lifecycleFixtures = mergeLifecycleFixtures(this.lifecycleFixtures, ...localFixtures);
+    const globalFixtureNames = new Set(this.lifecycleFixtures.map((fixture) => fixture.name));
+    const itemIndexesByFixture = new Map<string, number[]>();
+    for (const [index, fixtures] of localFixtures.entries()) {
+      for (const fixture of fixtures) {
+        const indexes = itemIndexesByFixture.get(fixture.name) ?? [];
+        indexes.push(index);
+        itemIndexesByFixture.set(fixture.name, indexes);
+      }
+    }
     const suite = await openSuiteRun({
       runsRoot: opts.runsRoot,
       concurrency,
@@ -639,7 +676,7 @@ export class Orchestrator {
       ...(opts.variantId ? { variantId: opts.variantId } : {}),
       ...(opts.candidateId ? { candidateId: opts.candidateId } : {}),
       ...(opts.subjectHash ? { subjectHash: opts.subjectHash } : {}),
-      fixtures: this.lifecycleFixtures.map(({ name, pluginVersion }) => ({
+      fixtures: lifecycleFixtures.map(({ name, pluginVersion }) => ({
         name,
         pluginVersion,
       })),
@@ -677,9 +714,16 @@ export class Orchestrator {
     if (!opts.abortSignal?.aborted) {
       try {
         cleanupSuiteHooks = await setupLifecycleHooks(
-          this.lifecycleFixtures,
+          lifecycleFixtures,
           "suite",
-          (hook) => hook.setupSuite?.(hookContext),
+          (hook) => {
+            if (globalFixtureNames.has(hook.name)) return hook.setupSuite?.(hookContext);
+            const indexes = itemIndexesByFixture.get(hook.name) ?? [];
+            return hook.setupSuite?.({
+              ...hookContext,
+              items: indexes.flatMap((index) => hookContext.items[index] ?? []),
+            });
+          },
           opts.abortSignal,
         );
       } catch (error) {
@@ -718,7 +762,7 @@ export class Orchestrator {
         let cleanupCaseHooks: () => void | Promise<void>;
         try {
           cleanupCaseHooks = await setupLifecycleHooks(
-            this.lifecycleFixtures,
+            mergeLifecycleFixtures(this.lifecycleFixtures, localFixtures[index] ?? []),
             "case",
             (hook) =>
               hook.setupCase?.({
@@ -749,6 +793,7 @@ export class Orchestrator {
             runsRoot: opts.runsRoot,
             fixturesRoot: opts.fixturesRoot,
             answererOverride: opts.answererOverride,
+            skipJudge: opts.skipJudge,
             observer: handle ?? observer,
             abortSignal: opts.abortSignal,
             suiteId: suite.manifest.suiteId,
@@ -906,11 +951,12 @@ export class Orchestrator {
 
   private runWithPlugins(
     args: Omit<RunWithPluginsArgs, "luxVersion" | "registry">,
+    registry: Registry = this.registry,
   ): Promise<RunResult> {
     return executeRunWithPlugins({
       ...args,
       luxVersion: LUX_VERSION,
-      registry: this.registry,
+      registry,
     });
   }
 }

package/src/orchestrator/owned-lock.ts

@@ -1,8 +1,6 @@
-import { randomInt, randomUUID } from "node:crypto";
-import { link, lstat, open, readdir, readFile, rm, unlink } from "node:fs/promises";
+import { randomUUID } from "node:crypto";
+import { link, lstat, open, readFile, rm, unlink } from "node:fs/promises";
 import { hostname } from "node:os";
-import { basename, dirname, join } from "node:path";
-import { setTimeout as delay } from "node:timers/promises";
 
 export type LockRecoveryClaim = {
   path: string;
@@ -60,36 +58,54 @@ const linkedClaim = (lockPath: string, claimPath: string): LockRecoveryClaim =>
   release: () => rm(claimPath, { force: true }),
 });
 
-const recoveryHost = Buffer.from(hostname()).toString("base64url");
+type RecoveryMutexOwner = {
+  pid: number;
+  hostname: string;
+  lockToken: string;
+};
 
-const localClaimProcessIsDead = (prefix: string, name: string): boolean => {
-  const [claimHost, rawPid, id, extra] = name.slice(prefix.length).split(".");
-  if (claimHost !== recoveryHost || !rawPid || !id || extra !== undefined) return false;
+const recoveryOwnerIsDead = (owner: RecoveryMutexOwner): boolean => {
+  if (owner.hostname !== hostname()) return false;
   try {
-    process.kill(Number(rawPid), 0);
+    process.kill(owner.pid, 0);
     return false;
   } catch (error) {
     return (error as NodeJS.ErrnoException).code === "ESRCH";
   }
 };
 
-const hasCompetingRecoveryClaim = async (lockPath: string, claimPath: string): Promise<boolean> => {
-  const claim = await lstat(claimPath);
-  const directory = dirname(lockPath);
-  const claimName = basename(claimPath);
-  const prefix = `${basename(lockPath)}.recovering-`;
-  for (const name of await readdir(directory)) {
-    if (name === claimName || !name.startsWith(prefix)) continue;
-    const peerPath = join(directory, name);
-    const peer = await missingAsNull(lstat(peerPath));
-    if (!peer || peer.dev !== claim.dev || peer.ino !== claim.ino) continue;
-    if (localClaimProcessIsDead(prefix, name)) {
-      await unlinkIfExists(peerPath);
-      continue;
+const acquireRecoveryMutex = async (
+  lockPath: string,
+): Promise<{ path: string; lockToken: string } | null> => {
+  const path = `${lockPath}.recovery`;
+  while (true) {
+    try {
+      const owner = await publishOwnedLock(path, {
+        pid: process.pid,
+        hostname: hostname(),
+      });
+      return { path, lockToken: owner.lockToken };
+    } catch (error) {
+      if ((error as NodeJS.ErrnoException).code !== "EEXIST") throw error;
     }
-    return true;
+    let owner: RecoveryMutexOwner;
+    try {
+      const value = JSON.parse(await readFile(path, "utf8")) as Partial<RecoveryMutexOwner>;
+      if (
+        !Number.isSafeInteger(value.pid) ||
+        typeof value.hostname !== "string" ||
+        typeof value.lockToken !== "string"
+      ) {
+        return null;
+      }
+      owner = value as RecoveryMutexOwner;
+    } catch (error) {
+      if ((error as NodeJS.ErrnoException).code === "ENOENT") continue;
+      return null;
+    }
+    if (!recoveryOwnerIsDead(owner)) return null;
+    await releaseOwnedLock(path, owner.lockToken);
   }
-  return false;
 };
 
 /**
@@ -97,24 +113,28 @@ const hasCompetingRecoveryClaim = async (lockPath: string, claimPath: string): P
  * Foreign claims stay conservative because foreign lock owners do too.
  */
 export const claimLockForRecovery = async (lockPath: string): Promise<LockRecoveryClaim | null> => {
-  const claimPath = `${lockPath}.recovering-${recoveryHost}.${process.pid}.${randomUUID()}`;
+  const mutex = await acquireRecoveryMutex(lockPath);
+  if (!mutex) return null;
+  const claimPath = `${lockPath}.recovering-${mutex.lockToken}`;
   try {
     await link(lockPath, claimPath);
   } catch (error) {
     const code = (error as NodeJS.ErrnoException).code;
+    await releaseOwnedLock(mutex.path, mutex.lockToken);
     if (code === "ENOENT") return null;
     throw error;
   }
   const claim = linkedClaim(lockPath, claimPath);
-  try {
-    if (!(await hasCompetingRecoveryClaim(lockPath, claimPath))) return claim;
-  } catch (error) {
-    await claim.release();
-    throw error;
-  }
-  await claim.release();
-  await delay(randomInt(1, 8));
-  return null;
+  return {
+    ...claim,
+    release: async () => {
+      try {
+        await claim.release();
+      } finally {
+        await releaseOwnedLock(mutex.path, mutex.lockToken);
+      }
+    },
+  };
 };
 
 /** Remove only the canonical lock that still carries this acquisition token. */

package/src/orchestrator/regrade.ts

@@ -33,6 +33,7 @@ import {
   readRunArtifactBytes,
   valueHash,
 } from "../core/index.js";
+import { registryForEvalCase } from "./case-extensions.js";
 import { evaluatorImplementationHash } from "./evaluator-identity.js";
 import { parsePersistedJson, readBoundedHandle, safeReadJson } from "./fs-read.js";
 import {
@@ -526,18 +527,18 @@ export const regradeRun = async (options: {
   now?: Date;
 }): Promise<{ path: string; snapshot: GradingSnapshot }> => {
   const run = await loadFrozenRegradableRun(options.runDir, options.case);
+  const registry = options.case
+    ? registryForEvalCase(options.registry, options.case)
+    : options.registry;
   try {
     const evidenceHash = await gradingEvidenceHash(run);
     const validatedAssertionConfigs = await validateAssertionPlan(
       run.case.assertions ?? [],
-      options.registry,
+      registry,
     );
     const validatedAssertionsHash = valueHash(validatedAssertionConfigs);
-    const assertionEvaluators = assertionEvaluatorIdentities(
-      run.case.assertions ?? [],
-      options.registry,
-    );
-    const assertionPlugins = assertionPluginIdentities(run.case.assertions ?? [], options.registry);
+    const assertionEvaluators = assertionEvaluatorIdentities(run.case.assertions ?? [], registry);
+    const assertionPlugins = assertionPluginIdentities(run.case.assertions ?? [], registry);
     const graderHash = graderConfigHash({
       luxVersion: LUX_VERSION,
       implementationHash: await currentEvaluatorImplementationHash(),
@@ -547,7 +548,7 @@ export const regradeRun = async (options: {
     });
     const report: GradingReport = await gradeRun(
       run,
-      options.registry,
+      registry,
       undefined,
       options.abortSignal,
       undefined,

package/src/orchestrator/run-execution.ts

@@ -13,6 +13,7 @@ import {
   type EvalCase,
   emptyUsage,
   type FixturesRef,
+  type FixtureValueBinding,
   type GradingReport,
   guardRunObserver,
   inspectRunArtifact,
@@ -70,6 +71,7 @@ export type RunWithPluginsArgs = {
   validatedAssertionsHash: string;
   graderConfigHash: string;
   evaluatorConfigHash: string;
+  fixtureValues: readonly FixtureValueBinding[];
   luxVersion: string;
   registry: Registry;
   driver: AgentDriver;
@@ -360,6 +362,7 @@ class RunExecution implements RunLifecycleEffects<RunResult> {
       caseId: this.args.caseId,
       prompt: evalCase.prompt,
       fixtures: staged?.fixtures,
+      ...(this.args.fixtureValues.length > 0 ? { fixtureValues: this.args.fixtureValues } : {}),
       runsRoot: resolve(request.runsRoot),
       runDir: this.runDir.path,
       artifactsDir: this.runDir.artifactsDir,
@@ -557,7 +560,7 @@ class RunExecution implements RunLifecycleEffects<RunResult> {
     const outcome = this.require(this.loopOutcome, "driver outcome");
     const agentEndedAt = new Date();
     const exitReason = agentExitReason(outcome, this.collection.error);
-    const summary = outcome.kind === "done" ? outcome.done.summary : undefined;
+    const summary = outcome.kind === "done" ? outcome.done.summary : outcome.error.summary;
     const driverCost = costFromSummary(summary);
     const reportedModelAccounting = modelAccountingFromSummary(summary);
     const answererAccounting = readAnswererModelAccounting(this.args.answerer);
@@ -654,6 +657,7 @@ class RunExecution implements RunLifecycleEffects<RunResult> {
       request.recordGradingAssertion,
       modelAttempts,
       this.args.validatedAssertionConfigs,
+      request.skipJudge ?? false,
     );
   }
 

package/lib/answerers/scripted/index.d.ts

@@ -1,8 +0,0 @@
-import { z } from "zod";
-import { type AnswererPlugin } from "../../core/index.js";
-export declare const ScriptedAnswererConfigSchema: z.ZodObject<{
-    fixturePath: z.ZodString;
-}, z.core.$strict>;
-export type ScriptedAnswererConfig = z.infer<typeof ScriptedAnswererConfigSchema>;
-export declare const scriptedAnswerer: AnswererPlugin<ScriptedAnswererConfig>;
-//# sourceMappingURL=index.d.ts.map

package/lib/answerers/scripted/index.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/answerers/scripted/index.ts"],"names":[],"mappings":"AAGA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AACxB,OAAO,EAGL,KAAK,cAAc,EAIpB,MAAM,qBAAqB,CAAC;AA8B7B,eAAO,MAAM,4BAA4B;;kBAAiD,CAAC;AAE3F,MAAM,MAAM,sBAAsB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,4BAA4B,CAAC,CAAC;AA0IlF,eAAO,MAAM,gBAAgB,EAAE,cAAc,CAAC,sBAAsB,CAInE,CAAC"}