@prismatic-io/lux 0.0.1 → 0.0.2-preview.11

package/src/orchestrator/orchestrator.ts

@@ -18,6 +18,12 @@ import {
   type SuiteLifecycleHookContext,
   valueHash,
 } from "../core/index.js";
+import {
+  fixturesForEvalCase,
+  mergeLifecycleFixtures,
+  registryForEvalCase,
+  valuesForFixtures,
+} from "./case-extensions.js";
 import { evaluatorImplementationHash } from "./evaluator-identity.js";
 import {
   assertionEvaluatorIdentities,
@@ -122,6 +128,8 @@ export type RunRequest = {
   /** Root that relative `fixtures` dir paths resolve against. */
   fixturesRoot?: string | undefined;
   answererOverride?: string | undefined;
+  /** Skip built-in rubric judge assertions without changing subject execution. */
+  skipJudge?: boolean | undefined;
   startedAt?: Date | undefined;
   /** The `runSuite` invocation this case attempt belongs to. */
   suiteId?: string | undefined;
@@ -476,11 +484,6 @@ export class Orchestrator {
   }
 
   async run(request: RunRequest): Promise<RunResult> {
-    if (this.lifecycleFixtures.length > 0 && !request.suiteId) {
-      throw new Error(
-        "Orchestrator.run() cannot execute configured lifecycle fixtures; use runSuite() for a single case",
-      );
-    }
     if (!request.case && !request.casePath) {
       throw new Error("RunRequest requires either 'case' or 'casePath'");
     }
@@ -489,11 +492,24 @@ export class Orchestrator {
     }
     const casePath = request.casePath ? resolve(request.casePath) : INLINE_CASE_PATH;
     const authoredCase = request.case ?? (await loadEvalCase(casePath));
+    const lifecycleFixtures = mergeLifecycleFixtures(
+      this.lifecycleFixtures,
+      fixturesForEvalCase(authoredCase),
+    );
+    if (
+      lifecycleFixtures.some(({ setupSuite, setupCase }) => setupSuite || setupCase) &&
+      !request.suiteId
+    ) {
+      throw new Error(
+        "Orchestrator.run() cannot execute lifecycle fixtures; use runSuite() for a single case",
+      );
+    }
+    const registry = registryForEvalCase(this.registry, authoredCase);
     const evalCase = bindSubjectRoot(authoredCase, request.subjectRoot);
     const caseId = resolveCaseId(casePath, evalCase, request.caseId);
     const validatedAssertionConfigs = await validateAssertionPlan(
       evalCase.assertions ?? [],
-      this.registry,
+      registry,
     );
 
     const driverRef = resolveDriverReference(evalCase, this.defaultDriver);
@@ -506,11 +522,11 @@ export class Orchestrator {
     );
     const answererName = answererNameOf(answererRef);
 
-    const driverPlugin = this.registry.getDriver(driverName);
+    const driverPlugin = registry.getDriver(driverName);
     if (!driverPlugin) {
       throw new Error(`no driver registered for '${driverName}'`);
     }
-    const answererPlugin = this.registry.getAnswerer(answererName);
+    const answererPlugin = registry.getAnswerer(answererName);
     if (!answererPlugin) {
       throw new Error(`no answerer registered for '${answererName}'`);
     }
@@ -544,7 +560,7 @@ export class Orchestrator {
     try {
       const assertionImplementations = assertionEvaluatorIdentities(
         authoredCase.assertions ?? [],
-        this.registry,
+        registry,
       );
       const validatedAssertionsHash = valueHash(validatedAssertionConfigs);
       const implementationHash = await this.evaluatorImplementationIdentity();
@@ -554,34 +570,40 @@ export class Orchestrator {
         assertions: assertionImplementations,
         validatedAssertionsHash,
         graderContext: this.harness,
+        skipJudge: request.skipJudge ?? false,
       });
-      return await this.runWithPlugins({
-        request,
-        casePath,
-        authoredCase,
-        evalCase,
-        validatedAssertionConfigs,
-        caseId,
-        driverName,
-        answererName,
-        answererConfig: answererCfg,
-        driverConfigHash: valueHash(driverCfg),
-        answererConfigHash: valueHash(answererCfg),
-        validatedAssertionsHash,
-        graderConfigHash: graderHash,
-        evaluatorConfigHash: valueHash({
-          luxVersion: LUX_VERSION,
-          implementationHash,
-          harness: this.harness,
-          driver: { name: driverPlugin.name, create: driverPlugin.create },
-          answerer: { name: answererPlugin.name, create: answererPlugin.create },
-          assertions: assertionImplementations,
-          lifecycleFixtures: this.lifecycleFixtures,
+      return await this.runWithPlugins(
+        {
+          request,
+          casePath,
+          authoredCase,
+          evalCase,
+          validatedAssertionConfigs,
+          caseId,
+          driverName,
+          answererName,
+          answererConfig: answererCfg,
+          driverConfigHash: valueHash(driverCfg),
+          answererConfigHash: valueHash(answererCfg),
           validatedAssertionsHash,
-        }),
-        driver,
-        answerer,
-      });
+          graderConfigHash: graderHash,
+          evaluatorConfigHash: valueHash({
+            luxVersion: LUX_VERSION,
+            implementationHash,
+            harness: this.harness,
+            driver: { name: driverPlugin.name, create: driverPlugin.create },
+            answerer: { name: answererPlugin.name, create: answererPlugin.create },
+            assertions: assertionImplementations,
+            lifecycleFixtures,
+            validatedAssertionsHash,
+            skipJudge: request.skipJudge ?? false,
+          }),
+          fixtureValues: valuesForFixtures(lifecycleFixtures),
+          driver,
+          answerer,
+        },
+        registry,
+      );
     } finally {
       await closeQuietly(() => driver.close(), `driver '${driverName}'`);
       await closeQuietly(() => answerer.close(), `answerer '${answererName}'`);
@@ -599,6 +621,7 @@ export class Orchestrator {
       runsRoot: string;
       fixturesRoot?: string | undefined;
       answererOverride?: string | undefined;
+      skipJudge?: boolean | undefined;
       observer?: SuiteObserver | undefined;
       /** Aborts the cases in flight and stops the pool pulling new ones. */
       abortSignal?: AbortSignal | undefined;
@@ -628,6 +651,20 @@ export class Orchestrator {
     const observer = opts.observer;
     const requestedConcurrency = validateSuiteConcurrency(opts.concurrency);
     const concurrency = Math.min(requestedConcurrency, Math.max(items.length, 1));
+    const authoredCases = await Promise.allSettled(items.map((item) => loadEvalCase(item.path)));
+    const localFixtures = authoredCases.map((result) =>
+      result.status === "fulfilled" ? fixturesForEvalCase(result.value) : [],
+    );
+    const lifecycleFixtures = mergeLifecycleFixtures(this.lifecycleFixtures, ...localFixtures);
+    const globalFixtureNames = new Set(this.lifecycleFixtures.map((fixture) => fixture.name));
+    const itemIndexesByFixture = new Map<string, number[]>();
+    for (const [index, fixtures] of localFixtures.entries()) {
+      for (const fixture of fixtures) {
+        const indexes = itemIndexesByFixture.get(fixture.name) ?? [];
+        indexes.push(index);
+        itemIndexesByFixture.set(fixture.name, indexes);
+      }
+    }
     const suite = await openSuiteRun({
       runsRoot: opts.runsRoot,
       concurrency,
@@ -639,7 +676,7 @@ export class Orchestrator {
       ...(opts.variantId ? { variantId: opts.variantId } : {}),
       ...(opts.candidateId ? { candidateId: opts.candidateId } : {}),
       ...(opts.subjectHash ? { subjectHash: opts.subjectHash } : {}),
-      fixtures: this.lifecycleFixtures.map(({ name, pluginVersion }) => ({
+      fixtures: lifecycleFixtures.map(({ name, pluginVersion }) => ({
         name,
         pluginVersion,
       })),
@@ -677,9 +714,16 @@ export class Orchestrator {
     if (!opts.abortSignal?.aborted) {
       try {
         cleanupSuiteHooks = await setupLifecycleHooks(
-          this.lifecycleFixtures,
+          lifecycleFixtures,
           "suite",
-          (hook) => hook.setupSuite?.(hookContext),
+          (hook) => {
+            if (globalFixtureNames.has(hook.name)) return hook.setupSuite?.(hookContext);
+            const indexes = itemIndexesByFixture.get(hook.name) ?? [];
+            return hook.setupSuite?.({
+              ...hookContext,
+              items: indexes.flatMap((index) => hookContext.items[index] ?? []),
+            });
+          },
           opts.abortSignal,
         );
       } catch (error) {
@@ -718,7 +762,7 @@ export class Orchestrator {
         let cleanupCaseHooks: () => void | Promise<void>;
         try {
           cleanupCaseHooks = await setupLifecycleHooks(
-            this.lifecycleFixtures,
+            mergeLifecycleFixtures(this.lifecycleFixtures, localFixtures[index] ?? []),
             "case",
             (hook) =>
               hook.setupCase?.({
@@ -749,6 +793,7 @@ export class Orchestrator {
             runsRoot: opts.runsRoot,
             fixturesRoot: opts.fixturesRoot,
             answererOverride: opts.answererOverride,
+            skipJudge: opts.skipJudge,
             observer: handle ?? observer,
             abortSignal: opts.abortSignal,
             suiteId: suite.manifest.suiteId,
@@ -906,11 +951,12 @@ export class Orchestrator {
 
   private runWithPlugins(
     args: Omit<RunWithPluginsArgs, "luxVersion" | "registry">,
+    registry: Registry = this.registry,
   ): Promise<RunResult> {
     return executeRunWithPlugins({
       ...args,
       luxVersion: LUX_VERSION,
-      registry: this.registry,
+      registry,
     });
   }
 }

package/src/orchestrator/owned-lock.ts

@@ -1,8 +1,6 @@
-import { randomInt, randomUUID } from "node:crypto";
-import { link, lstat, open, readdir, readFile, rm, unlink } from "node:fs/promises";
+import { randomUUID } from "node:crypto";
+import { link, lstat, open, readFile, rm, unlink } from "node:fs/promises";
 import { hostname } from "node:os";
-import { basename, dirname, join } from "node:path";
-import { setTimeout as delay } from "node:timers/promises";
 
 export type LockRecoveryClaim = {
   path: string;
@@ -60,36 +58,54 @@ const linkedClaim = (lockPath: string, claimPath: string): LockRecoveryClaim =>
   release: () => rm(claimPath, { force: true }),
 });
 
-const recoveryHost = Buffer.from(hostname()).toString("base64url");
+type RecoveryMutexOwner = {
+  pid: number;
+  hostname: string;
+  lockToken: string;
+};
 
-const localClaimProcessIsDead = (prefix: string, name: string): boolean => {
-  const [claimHost, rawPid, id, extra] = name.slice(prefix.length).split(".");
-  if (claimHost !== recoveryHost || !rawPid || !id || extra !== undefined) return false;
+const recoveryOwnerIsDead = (owner: RecoveryMutexOwner): boolean => {
+  if (owner.hostname !== hostname()) return false;
   try {
-    process.kill(Number(rawPid), 0);
+    process.kill(owner.pid, 0);
     return false;
   } catch (error) {
     return (error as NodeJS.ErrnoException).code === "ESRCH";
   }
 };
 
-const hasCompetingRecoveryClaim = async (lockPath: string, claimPath: string): Promise<boolean> => {
-  const claim = await lstat(claimPath);
-  const directory = dirname(lockPath);
-  const claimName = basename(claimPath);
-  const prefix = `${basename(lockPath)}.recovering-`;
-  for (const name of await readdir(directory)) {
-    if (name === claimName || !name.startsWith(prefix)) continue;
-    const peerPath = join(directory, name);
-    const peer = await missingAsNull(lstat(peerPath));
-    if (!peer || peer.dev !== claim.dev || peer.ino !== claim.ino) continue;
-    if (localClaimProcessIsDead(prefix, name)) {
-      await unlinkIfExists(peerPath);
-      continue;
+const acquireRecoveryMutex = async (
+  lockPath: string,
+): Promise<{ path: string; lockToken: string } | null> => {
+  const path = `${lockPath}.recovery`;
+  while (true) {
+    try {
+      const owner = await publishOwnedLock(path, {
+        pid: process.pid,
+        hostname: hostname(),
+      });
+      return { path, lockToken: owner.lockToken };
+    } catch (error) {
+      if ((error as NodeJS.ErrnoException).code !== "EEXIST") throw error;
     }
-    return true;
+    let owner: RecoveryMutexOwner;
+    try {
+      const value = JSON.parse(await readFile(path, "utf8")) as Partial<RecoveryMutexOwner>;
+      if (
+        !Number.isSafeInteger(value.pid) ||
+        typeof value.hostname !== "string" ||
+        typeof value.lockToken !== "string"
+      ) {
+        return null;
+      }
+      owner = value as RecoveryMutexOwner;
+    } catch (error) {
+      if ((error as NodeJS.ErrnoException).code === "ENOENT") continue;
+      return null;
+    }
+    if (!recoveryOwnerIsDead(owner)) return null;
+    await releaseOwnedLock(path, owner.lockToken);
   }
-  return false;
 };
 
 /**
@@ -97,24 +113,28 @@ const hasCompetingRecoveryClaim = async (lockPath: string, claimPath: string): P
  * Foreign claims stay conservative because foreign lock owners do too.
  */
 export const claimLockForRecovery = async (lockPath: string): Promise<LockRecoveryClaim | null> => {
-  const claimPath = `${lockPath}.recovering-${recoveryHost}.${process.pid}.${randomUUID()}`;
+  const mutex = await acquireRecoveryMutex(lockPath);
+  if (!mutex) return null;
+  const claimPath = `${lockPath}.recovering-${mutex.lockToken}`;
   try {
     await link(lockPath, claimPath);
   } catch (error) {
     const code = (error as NodeJS.ErrnoException).code;
+    await releaseOwnedLock(mutex.path, mutex.lockToken);
     if (code === "ENOENT") return null;
     throw error;
   }
   const claim = linkedClaim(lockPath, claimPath);
-  try {
-    if (!(await hasCompetingRecoveryClaim(lockPath, claimPath))) return claim;
-  } catch (error) {
-    await claim.release();
-    throw error;
-  }
-  await claim.release();
-  await delay(randomInt(1, 8));
-  return null;
+  return {
+    ...claim,
+    release: async () => {
+      try {
+        await claim.release();
+      } finally {
+        await releaseOwnedLock(mutex.path, mutex.lockToken);
+      }
+    },
+  };
 };
 
 /** Remove only the canonical lock that still carries this acquisition token. */

package/src/orchestrator/regrade.ts

@@ -33,6 +33,7 @@ import {
   readRunArtifactBytes,
   valueHash,
 } from "../core/index.js";
+import { registryForEvalCase } from "./case-extensions.js";
 import { evaluatorImplementationHash } from "./evaluator-identity.js";
 import { parsePersistedJson, readBoundedHandle, safeReadJson } from "./fs-read.js";
 import {
@@ -526,18 +527,18 @@ export const regradeRun = async (options: {
   now?: Date;
 }): Promise<{ path: string; snapshot: GradingSnapshot }> => {
   const run = await loadFrozenRegradableRun(options.runDir, options.case);
+  const registry = options.case
+    ? registryForEvalCase(options.registry, options.case)
+    : options.registry;
   try {
     const evidenceHash = await gradingEvidenceHash(run);
     const validatedAssertionConfigs = await validateAssertionPlan(
       run.case.assertions ?? [],
-      options.registry,
+      registry,
     );
     const validatedAssertionsHash = valueHash(validatedAssertionConfigs);
-    const assertionEvaluators = assertionEvaluatorIdentities(
-      run.case.assertions ?? [],
-      options.registry,
-    );
-    const assertionPlugins = assertionPluginIdentities(run.case.assertions ?? [], options.registry);
+    const assertionEvaluators = assertionEvaluatorIdentities(run.case.assertions ?? [], registry);
+    const assertionPlugins = assertionPluginIdentities(run.case.assertions ?? [], registry);
     const graderHash = graderConfigHash({
       luxVersion: LUX_VERSION,
       implementationHash: await currentEvaluatorImplementationHash(),
@@ -547,7 +548,7 @@ export const regradeRun = async (options: {
     });
     const report: GradingReport = await gradeRun(
       run,
-      options.registry,
+      registry,
       undefined,
       options.abortSignal,
       undefined,

package/src/orchestrator/run-execution.ts

@@ -10,13 +10,16 @@ import {
   AnswererModelCallAccountingSchema,
   type AnswerHistoryEntry,
   type Artifact,
+  actionTiming,
   type EvalCase,
   emptyUsage,
   type FixturesRef,
+  type FixtureValueBinding,
   type GradingReport,
   guardRunObserver,
   inspectRunArtifact,
   type LifecycleStage,
+  normalizeDriverMetrics,
   type OperationalFailure,
   type PhaseDurations,
   type ReadyState,
@@ -27,6 +30,7 @@ import {
   type RunMetadata,
   type RunProvenance,
   type RunUsage,
+  summarizeAgentActivity,
   summarizeToolOutcomes,
   sumUsage,
   type TokenUsage,
@@ -70,6 +74,7 @@ export type RunWithPluginsArgs = {
   validatedAssertionsHash: string;
   graderConfigHash: string;
   evaluatorConfigHash: string;
+  fixtureValues: readonly FixtureValueBinding[];
   luxVersion: string;
   registry: Registry;
   driver: AgentDriver;
@@ -181,6 +186,11 @@ const modelAccountingFromSummary = (
 ): { modelCalls: number; unpricedModelCalls?: number | undefined } | null =>
   SummaryModelAccountingSchema.safeParse(summary).data ?? null;
 
+const metricsFromSummary = (summary: unknown) => {
+  if (!summary || typeof summary !== "object") return {};
+  return normalizeDriverMetrics((summary as { metrics?: unknown }).metrics);
+};
+
 const artifactMeasurements = async (
   artifacts: readonly Artifact[],
   runDir: string,
@@ -360,6 +370,7 @@ class RunExecution implements RunLifecycleEffects<RunResult> {
       caseId: this.args.caseId,
       prompt: evalCase.prompt,
       fixtures: staged?.fixtures,
+      ...(this.args.fixtureValues.length > 0 ? { fixtureValues: this.args.fixtureValues } : {}),
       runsRoot: resolve(request.runsRoot),
       runDir: this.runDir.path,
       artifactsDir: this.runDir.artifactsDir,
@@ -387,6 +398,7 @@ class RunExecution implements RunLifecycleEffects<RunResult> {
     metadata.exitReason = "error";
     metadata.operationalFailure = { phase: operationalPhase, reason };
     metadata.phaseDurationMs = phaseDurations(this.runDir.recorded);
+    metadata.actionTiming = actionTiming(this.runDir.recorded);
     await this.runDir.writeJson("run.json", metadata);
     await this.args.request.recordPreGradingAccounting?.(metadata);
     await this.args.request.recordModelAttemptsComplete?.(this.subjectAttemptIds);
@@ -404,6 +416,7 @@ class RunExecution implements RunLifecycleEffects<RunResult> {
     this.lifecycle("collect-end");
     await this.runDir.events.drain();
     const completed = await this.completedAgentRun();
+    completed.metadata.actionTiming = actionTiming(this.runDir.recorded);
     this.metadata = completed.metadata;
     this.usage = completed.usage;
     this.run = completed.run;
@@ -427,6 +440,7 @@ class RunExecution implements RunLifecycleEffects<RunResult> {
       grading.assertions.flatMap((assertion) => (assertion.usage ? [assertion.usage] : [])),
     );
     metadata.phaseDurationMs = phaseDurations(this.runDir.recorded);
+    metadata.actionTiming = actionTiming(this.runDir.recorded);
     await this.runDir.events.drain();
     await this.runDir.writeJson("run.json", metadata);
     await this.runDir.writeJson("artifacts.json", this.collection.artifacts);
@@ -503,6 +517,7 @@ class RunExecution implements RunLifecycleEffects<RunResult> {
   private fallbackFailureMetadata(): RunMetadata {
     const failedAt = new Date();
     const answererAccounting = readAnswererModelAccounting(this.args.answerer);
+    const ready = this.ready ?? { agentId: "start-failed" };
     return {
       schemaVersion: 1,
       luxVersion: this.args.luxVersion,
@@ -511,7 +526,7 @@ class RunExecution implements RunLifecycleEffects<RunResult> {
       agentEndedAt: failedAt.toISOString(),
       endedAt: failedAt.toISOString(),
       exitReason: "error",
-      driver: { name: this.args.driverName, ...(this.ready ?? { agentId: "start-failed" }) },
+      driver: { name: this.args.driverName, ...ready },
       answerer: answererMetadata(this.args.answererName, this.args.answererConfig),
       usage: {
         agent: emptyUsage(),
@@ -527,6 +542,7 @@ class RunExecution implements RunLifecycleEffects<RunResult> {
       interruptCounts: this.interruptCounts,
       toolCallCount: this.toolCallCount,
       toolOutcomes: summarizeToolOutcomes(this.runDir.recorded),
+      agentActivity: summarizeAgentActivity(this.runDir.recorded, ready),
       artifacts: { count: this.collection.artifacts.length },
       provenance: this.runProvenance(),
     };
@@ -557,9 +573,10 @@ class RunExecution implements RunLifecycleEffects<RunResult> {
     const outcome = this.require(this.loopOutcome, "driver outcome");
     const agentEndedAt = new Date();
     const exitReason = agentExitReason(outcome, this.collection.error);
-    const summary = outcome.kind === "done" ? outcome.done.summary : undefined;
+    const summary = outcome.kind === "done" ? outcome.done.summary : outcome.error.summary;
     const driverCost = costFromSummary(summary);
     const reportedModelAccounting = modelAccountingFromSummary(summary);
+    const driverMetrics = metricsFromSummary(summary);
     const answererAccounting = readAnswererModelAccounting(this.args.answerer);
     const agentCalls = this.ready ? (reportedModelAccounting?.modelCalls ?? 1) : 0;
     const operationalFailure = loopFailure(outcome, this.collection.error);
@@ -568,6 +585,7 @@ class RunExecution implements RunLifecycleEffects<RunResult> {
       answerer: readAnswererUsage(this.args.answerer),
       grading: emptyUsage(),
     };
+    const ready = this.ready ?? { agentId: "not-started" };
     const metadata: RunMetadata = {
       schemaVersion: 1,
       luxVersion: this.args.luxVersion,
@@ -576,7 +594,7 @@ class RunExecution implements RunLifecycleEffects<RunResult> {
       agentEndedAt: agentEndedAt.toISOString(),
       endedAt: agentEndedAt.toISOString(),
       exitReason,
-      driver: { name: this.args.driverName, ...(this.ready ?? { agentId: "not-started" }) },
+      driver: { name: this.args.driverName, ...ready },
       answerer: answererMetadata(this.args.answererName, this.args.answererConfig),
       usage,
       modelAccounting: {
@@ -590,10 +608,12 @@ class RunExecution implements RunLifecycleEffects<RunResult> {
       interruptCounts: this.interruptCounts,
       toolCallCount: this.toolCallCount,
       toolOutcomes: summarizeToolOutcomes(this.runDir.recorded),
+      agentActivity: summarizeAgentActivity(this.runDir.recorded, ready),
       artifacts: await artifactMeasurements(this.collection.artifacts, this.runDir.path),
       ...(operationalFailure ? { operationalFailure } : {}),
       provenance: this.runProvenance(),
       ...(driverCost !== undefined ? { cost: { usd: driverCost } } : {}),
+      ...(Object.keys(driverMetrics).length > 0 ? { driverMetrics } : {}),
     };
     return {
       metadata,
@@ -654,6 +674,7 @@ class RunExecution implements RunLifecycleEffects<RunResult> {
       request.recordGradingAssertion,
       modelAttempts,
       this.args.validatedAssertionConfigs,
+      request.skipJudge ?? false,
     );
   }