@prismatic-io/lux 0.0.1 → 0.0.2-preview.11

package/src/index.ts

@@ -16,16 +16,15 @@ export {
   PersonaAnswererConfigSchema,
   personaAnswerer,
 } from "./answerers/persona/index.js";
-export {
-  type ScriptedAnswererConfig,
-  ScriptedAnswererConfigSchema,
-  scriptedAnswerer,
-} from "./answerers/scripted/index.js";
 export {
   type TerminalAnswererConfig,
   TerminalAnswererConfigSchema,
   terminalAnswerer,
 } from "./answerers/terminal/index.js";
+export {
+  type BuiltinAssertion,
+  defineEvalCase,
+} from "./assertions/authoring.js";
 // Bundled deterministic and judge assertions.
 export {
   type AgentScope,
@@ -88,6 +87,8 @@ export {
   registerCoreAssertions,
   runSucceededAssertion,
   subagentToolCalls,
+  type TimeToFirstActionUnderConfig,
+  TimeToFirstActionUnderConfigSchema,
   type TokensUnderConfig,
   TokensUnderConfigSchema,
   type ToolCall,
@@ -105,6 +106,7 @@ export {
   ToolResultIsErrorConfigSchema,
   type ToolResultJsonPointerEqualsConfig,
   ToolResultJsonPointerEqualsConfigSchema,
+  timeToFirstActionUnderAssertion,
   tokensUnderAssertion,
   toolCallCountAssertion,
   toolCalledAssertion,
@@ -142,6 +144,28 @@ export {
   rubricAssertion,
 } from "./assertions/rubric/index.js";
 // Authoring and extension contracts.
+export {
+  type ActionCategory,
+  ActionCategorySchema,
+  type ActionProgressPayload,
+  ActionProgressPayloadSchema,
+  type ActionTiming,
+  ActionTimingSchema,
+  actionTiming,
+} from "./core/action-events.js";
+export {
+  type AgentActivity,
+  AgentActivitySchema,
+  summarizeAgentActivity,
+} from "./core/agent-activity.js";
+export {
+  AGENT_COMPLETED_KIND,
+  AGENT_SPAWN_KIND,
+  type AgentCompletedPayload,
+  AgentCompletedPayloadSchema,
+  type AgentSpawnPayload,
+  AgentSpawnPayloadSchema,
+} from "./core/agent-events.js";
 export {
   defineGraderAnnotations,
   type GraderAnnotation,
@@ -185,6 +209,7 @@ export {
   type AssertionResult,
   AssertionResultSchema,
   defineAssertion,
+  defineAssertionRegistry,
   type EvaluateContext,
   type GradingAssertionEntry,
   GradingAssertionEntrySchema,
@@ -198,8 +223,9 @@ export {
   AssertionSchema,
   type DriverRef,
   DriverRefSchema,
-  defineEvalCase,
   type EvalCase,
+  type EvalCaseAuthoringOptions,
+  type EvalCaseDefinition,
   EvalCaseSchema,
   type FixturesRef,
   FixturesRefSchema,
@@ -226,8 +252,23 @@ export {
   ResolvedFixturesSchema,
   type StartContext,
 } from "./core/driver.js";
+export {
+  type DriverMetric,
+  type DriverMetricDescriptor,
+  DriverMetricDescriptorSchema,
+  DriverMetricSchema,
+  type DriverMetrics,
+  DriverMetricsSchema,
+  type DriverMetricUnit,
+  DriverMetricUnitSchema,
+  normalizeDriverMetrics,
+  type ReportedDriverMetrics,
+  ReportedDriverMetricsSchema,
+} from "./core/driver-metrics.js";
 export { type EventHandler, TypedEmitter } from "./core/emitter.js";
 export {
+  type AgentAttribution,
+  AgentAttributionSchema,
   type Answer,
   AnswerSchema,
   ApproveAnswerSchema,
@@ -300,6 +341,9 @@ export {
 export {
   type CaseLifecycleHookContext,
   defineLifecycleHooks,
+  type FixtureValue,
+  type FixtureValueBinding,
+  FixtureValueSchema,
   type LifecycleHookCleanup,
   type LifecycleHookSetupResult,
   type LifecycleHooksPlugin,

package/src/orchestrator/annotation-store.ts

@@ -57,6 +57,7 @@ const recoverStaleLock = async (lockPath: string): Promise<boolean> => {
 const acquireAnnotationLock = async (path: string): Promise<() => Promise<void>> => {
   const lockPath = `${path}.lock`;
   const deadline = Date.now() + LOCK_WAIT_MS;
+  let attemptedRecovery = false;
   while (true) {
     try {
       const owner = await publishOwnedLock(lockPath, {
@@ -69,7 +70,10 @@ const acquireAnnotationLock = async (path: string): Promise<() => Promise<void>>
       };
     } catch (error) {
       if ((error as NodeJS.ErrnoException).code !== "EEXIST") throw error;
-      if (await recoverStaleLock(lockPath)) continue;
+      if (!attemptedRecovery) {
+        attemptedRecovery = true;
+        if (await recoverStaleLock(lockPath)) continue;
+      }
       if (Date.now() >= deadline) {
         throw new Error(`timed out waiting to update grader annotations: ${path}`);
       }

package/src/orchestrator/case-extensions.ts

@@ -0,0 +1,44 @@
+import { evalCaseExtensions } from "../core/case-authoring-state.js";
+import type {
+  EvalCase,
+  FixtureValueBinding,
+  LifecycleHooksPlugin,
+  Registry,
+} from "../core/index.js";
+
+export const registryForEvalCase = (registry: Registry, evalCase: EvalCase): Registry => {
+  let resolved = registry;
+  for (const plugin of evalCaseExtensions(evalCase).assertions) {
+    const existing = resolved.getAssertion(plugin.type);
+    if (existing === plugin) continue;
+    if (existing) {
+      throw new TypeError(
+        `case-local assertion '${plugin.type}' conflicts with a registered implementation`,
+      );
+    }
+    resolved = resolved.withAssertion(plugin);
+  }
+  return resolved;
+};
+
+export const fixturesForEvalCase = (evalCase: EvalCase): readonly LifecycleHooksPlugin[] =>
+  evalCaseExtensions(evalCase).fixtures;
+
+export const valuesForFixtures = (
+  fixtures: readonly LifecycleHooksPlugin[],
+): FixtureValueBinding[] =>
+  fixtures.flatMap(({ name, value }) => (value === undefined ? [] : [{ name, value }]));
+
+export const mergeLifecycleFixtures = (
+  ...groups: readonly (readonly LifecycleHooksPlugin[])[]
+): LifecycleHooksPlugin[] => {
+  const fixtures = new Map<string, LifecycleHooksPlugin>();
+  for (const fixture of groups.flat()) {
+    const existing = fixtures.get(fixture.name);
+    if (existing && existing !== fixture) {
+      throw new TypeError(`conflicting lifecycle fixture definitions for '${fixture.name}'`);
+    }
+    fixtures.set(fixture.name, fixture);
+  }
+  return [...fixtures.values()];
+};

package/src/orchestrator/compare.ts

@@ -2,6 +2,9 @@ import { createHash } from "node:crypto";
 import { readFile } from "node:fs/promises";
 import { join } from "node:path";
 import {
+  type ActionTiming,
+  type AgentActivity,
+  type DriverMetrics,
   emptyRunUsage,
   type GradingReport,
   GradingReportSchema,
@@ -48,6 +51,9 @@ export type RunComparison = {
     exitReason: { a: string; b: string };
     durationMsA: number | null;
     durationMsB: number | null;
+    actionTiming: { a: ActionTiming; b: ActionTiming };
+    driverMetrics: { a: DriverMetrics; b: DriverMetrics };
+    agentActivity: { a: AgentActivity[]; b: AgentActivity[] };
     interruptCounts: {
       a: { ask: number; approve: number; custom: number };
       b: { ask: number; approve: number; custom: number };
@@ -260,6 +266,18 @@ export const compareRuns = async (input: ComparisonInput): Promise<RunComparison
           },
           durationMsA: durationFromMeta(a.runJson),
           durationMsB: durationFromMeta(b.runJson),
+          actionTiming: {
+            a: a.runJson?.actionTiming ?? {},
+            b: b.runJson?.actionTiming ?? {},
+          },
+          driverMetrics: {
+            a: a.runJson?.driverMetrics ?? {},
+            b: b.runJson?.driverMetrics ?? {},
+          },
+          agentActivity: {
+            a: a.runJson?.agentActivity ?? [],
+            b: b.runJson?.agentActivity ?? [],
+          },
           interruptCounts: {
             a: a.runJson?.interruptCounts ?? { ask: 0, approve: 0, custom: 0 },
             b: b.runJson?.interruptCounts ?? { ask: 0, approve: 0, custom: 0 },

package/src/orchestrator/config.ts

@@ -4,7 +4,6 @@ import { pathToFileURL } from "node:url";
 import { z } from "zod";
 import { claudeCodeAnswerer } from "../answerers/claude-code/index.js";
 import { personaAnswerer } from "../answerers/persona/index.js";
-import { scriptedAnswerer } from "../answerers/scripted/index.js";
 import { terminalAnswerer } from "../answerers/terminal/index.js";
 import { allCoreAssertions } from "../assertions/core/index.js";
 import { allMcpAssertions } from "../assertions/mcp/index.js";
@@ -182,7 +181,6 @@ const builtinDrivers: DriverPlugin<never>[] = [
 const builtinAnswerers: AnswererPlugin<never>[] = [
   personaAnswerer,
   claudeCodeAnswerer,
-  scriptedAnswerer,
   terminalAnswerer,
 ];
 export const buildRegistryFromConfig = (config: ResolvedLuxConfig): Registry => {

package/src/orchestrator/doctor.ts

@@ -5,6 +5,7 @@ import { glob } from "tinyglobby";
 import { z } from "zod";
 import { RubricConfigSchema } from "../assertions/rubric/index.js";
 import type { EvalCase, Registry } from "../core/index.js";
+import { registryForEvalCase } from "./case-extensions.js";
 import { buildRegistryFromConfig, loadLuxConfig, type ResolvedLuxConfig } from "./config.js";
 import { type DiscoveredCase, type DiscoveryResult, discoverCases } from "./discover.js";
 import { cliIdentity, ExperimentCampaignPlanSchema, planExperimentCampaign } from "./experiment.js";
@@ -559,12 +560,13 @@ const inspectCase = async (
 ): Promise<CliTarget[]> => {
   try {
     const authored = await loadEvalCase(discovered.path);
+    const caseRegistry = registryForEvalCase(registry, authored);
     const drivers = caseDriverReferences(authored.driver, config);
     const answerer = authored.answerer ?? config.defaultAnswerer;
-    validateCaseDrivers(discovered.id, drivers, registry, recorder);
-    validateCaseAnswerer(discovered.id, answerer, registry, recorder);
+    validateCaseDrivers(discovered.id, drivers, caseRegistry, recorder);
+    validateCaseAnswerer(discovered.id, answerer, caseRegistry, recorder);
     return [
-      ...(await validateCaseAssertions(discovered.id, authored, config, registry, recorder)),
+      ...(await validateCaseAssertions(discovered.id, authored, config, caseRegistry, recorder)),
       ...executionTargetsForCase(discovered.id, drivers, answerer),
     ];
   } catch (error) {

package/src/orchestrator/experiment-evaluation.ts

@@ -101,6 +101,23 @@ const evaluationMetrics = (
   ),
 });
 
+const evaluationMetricDefinitions = (summary: SuiteSummary) =>
+  Object.fromEntries(
+    Object.entries(summary.metrics).flatMap(([name, metric]) => {
+      if (!metric.unit && !metric.label && !metric.better) return [];
+      return [
+        [
+          name,
+          {
+            unit: metric.unit ?? "number",
+            better: metric.better ?? "neutral",
+            ...(metric.label ? { label: metric.label } : {}),
+          },
+        ],
+      ];
+    }),
+  );
+
 const summaryUsage = (summary: SuiteSummary) =>
   summary.samples.reduce(
     (usage, sample) =>
@@ -277,6 +294,7 @@ const agentEvaluationFromSuite = (input: {
       summary.cases.map((entry) => [entry.canonicalCaseId, entry.passRate]),
     ),
     metrics: evaluationMetrics(summary, candidate),
+    metricDefinitions: evaluationMetricDefinitions(summary),
     samples: evaluationSamples(summary),
     metricCalls: summary.samples.length,
     judgeCalls: results.reduce(
@@ -352,6 +370,10 @@ const aggregateAgentUnits = (
         .map(([, values]) => average(values)),
     ]),
   );
+  const metricDefinitions = Object.assign(
+    {},
+    ...units.map(({ evaluation }) => evaluation.metricDefinitions ?? {}),
+  );
   return {
     candidateId: candidate.id,
     profileId,
@@ -367,6 +389,7 @@ const aggregateAgentUnits = (
         name === "subject.bytes" ? candidate.sourceBytes : average(values),
       ]),
     ),
+    metricDefinitions,
     samples,
     metricCalls: units.reduce((total, unit) => total + (unit.evaluation.metricCalls ?? 0), 0),
     judgeCalls: units.reduce((total, unit) => total + (unit.evaluation.judgeCalls ?? 0), 0),

package/src/orchestrator/grading.ts

@@ -76,6 +76,7 @@ export const graderConfigHash = (input: {
   assertions: readonly AssertionEvaluatorIdentity[];
   validatedAssertionsHash: string;
   graderContext?: unknown;
+  skipJudge?: boolean;
 }): string =>
   valueHash({
     luxVersion: input.luxVersion,
@@ -83,6 +84,7 @@ export const graderConfigHash = (input: {
     assertions: input.assertions,
     validatedAssertionsHash: input.validatedAssertionsHash,
     graderContext: input.graderContext ?? null,
+    skipJudge: input.skipJudge ?? false,
   });
 
 export const validateAssertionPlan = async (
@@ -133,6 +135,7 @@ export const gradeRun = async (
     complete: (attemptIds: readonly string[]) => void | Promise<void>;
   },
   validatedConfigs?: readonly unknown[],
+  skipJudge = false,
 ): Promise<GradingReport> => {
   const guardedObserver = guardRunObserver(observer);
   const assertions = run.case.assertions ?? [];
@@ -141,12 +144,27 @@ export const gradeRun = async (
   }
 
   const entries: GradingAssertionEntry[] = [];
+  const skippedAssertions: Array<{
+    id: string;
+    type: string;
+    name: string;
+    reason: "judge-disabled";
+  }> = [];
   let weightedScoreSum = 0;
   let weightSum = 0;
   let allPassed = true;
   let gradingIncomplete = false;
 
   for (const [index, a] of assertions.entries()) {
+    if (skipJudge && a.type === "rubric") {
+      skippedAssertions.push({
+        id: a.id ?? `${a.type}:${index + 1}`,
+        type: a.type,
+        name: describeAssertion(a),
+        reason: "judge-disabled",
+      });
+      continue;
+    }
     // Grading is the last cancellation point — an LLM judge can run for
     // minutes per assertion. An aborted grade can't claim everything passed.
     if (abortSignal?.aborted) {
@@ -247,12 +265,13 @@ export const gradeRun = async (
   }
 
   let caseScore: number | null = null;
-  if (!gradingIncomplete) caseScore = weightSum > 0 ? weightedScoreSum / weightSum : 0;
+  if (!gradingIncomplete && weightSum > 0) caseScore = weightedScoreSum / weightSum;
   return {
     schemaVersion: 1,
-    casePassed: gradingIncomplete ? null : allPassed,
+    casePassed: gradingIncomplete || weightSum === 0 ? null : allPassed,
     caseScore,
     assertions: entries,
+    ...(skippedAssertions.length > 0 ? { skippedAssertions } : {}),
     ...(gradingIncomplete
       ? { incomplete: { exitReason: "error" as const, phase: "grading" as const } }
       : {}),

package/src/orchestrator/index.ts

@@ -122,6 +122,7 @@ export {
   formatRunsTable,
   type ListedRun,
   listRuns,
+  loadRunArtifacts,
   loadRunDetail,
   type RunDetail,
 } from "./list-runs.js";

package/src/orchestrator/list-runs.ts

@@ -2,6 +2,11 @@ import { existsSync } from "node:fs";
 import { constants, lstat, open, readdir } from "node:fs/promises";
 import { basename, join, relative, sep } from "node:path";
 import {
+  ActionProgressPayloadSchema,
+  AgentCompletedPayloadSchema,
+  AgentSpawnPayloadSchema,
+  type Artifact,
+  ArtifactSchema,
   type GradingReport,
   GradingReportSchema,
   harnessUsage,
@@ -31,6 +36,11 @@ export type ListedRun = {
   startedAt: string | null;
   endedAt: string | null;
   durationMs: number | null;
+  timeToFirstActionMs: number | null;
+  timeToFirstMutationMs: number | null;
+  timeToFirstDurableMutationMs: number | null;
+  driverMetrics: NonNullable<RunMetadata["driverMetrics"]>;
+  subagentCount: number;
   casePassed: boolean | null;
   caseScore: number | null;
   assertionPassed: number;
@@ -63,6 +73,12 @@ const summarizeRun = async (
     startedAt,
     endedAt,
     durationMs,
+    timeToFirstActionMs: meta?.actionTiming?.timeToFirstActionMs ?? null,
+    timeToFirstMutationMs: meta?.actionTiming?.timeToFirstMutationMs ?? null,
+    timeToFirstDurableMutationMs: meta?.actionTiming?.timeToFirstDurableMutationMs ?? null,
+    driverMetrics: meta?.driverMetrics ?? {},
+    subagentCount:
+      meta?.agentActivity?.filter((activity) => activity.agent.role === "subagent").length ?? 0,
     casePassed: grading?.casePassed ?? null,
     caseScore: grading?.caseScore ?? null,
     assertionPassed: passing,
@@ -127,7 +143,7 @@ export const listRuns = async (runsRoot: string): Promise<ListedRun[]> => {
 export const formatRunsTable = (runs: ListedRun[]): string => {
   if (runs.length === 0) return "(no runs found)";
   const rows: string[][] = [
-    ["status", "case", "driver", "ts", "exit", "score", "duration"],
+    ["status", "case", "driver", "agents", "ts", "exit", "score", "duration"],
     ...runs.map((r) => {
       let status = "ungraded";
       if (r.casePassed === true) status = "PASS";
@@ -137,7 +153,8 @@ export const formatRunsTable = (runs: ListedRun[]): string => {
           ? `${(r.caseScore * 100).toFixed(0)}% (${r.assertionPassed}/${r.assertionTotal})`
           : "—";
       const duration = r.durationMs !== null ? `${(r.durationMs / 1000).toFixed(1)}s` : "—";
-      return [status, r.caseSlug, r.driverName, r.ts, r.exitReason ?? "?", score, duration];
+      const agents = r.subagentCount > 0 ? `1+${r.subagentCount}` : "1";
+      return [status, r.caseSlug, r.driverName, agents, r.ts, r.exitReason ?? "?", score, duration];
     }),
   ];
   const header = rows[0] ?? [];
@@ -156,6 +173,9 @@ export type RunDetail = {
   artifacts: string[];
 };
 
+export const loadRunArtifacts = async (runDir: string): Promise<Artifact[]> =>
+  (await safeReadJson(join(runDir, "artifacts.json"), ArtifactSchema.array())) ?? [];
+
 const readEventsJsonl = async (path: string): Promise<RunEvent[]> => {
   if (!existsSync(path)) return [];
   if ((await lstat(path)).isSymbolicLink()) {
@@ -291,6 +311,27 @@ const appendMetadataDetail = (lines: string[], detail: RunDetail): void => {
         `phases: agent=${secs(phases.agent)} collect=${secs(phases.collect)} grading=${secs(phases.grading)}`,
       );
     }
+    const actions = detail.metadata.actionTiming;
+    if (actions) {
+      const value = (ms: number | undefined): string => (ms === undefined ? "—" : `${ms}ms`);
+      lines.push(
+        `first action: any=${value(actions.timeToFirstActionMs)} mutation=${value(actions.timeToFirstMutationMs)} durable-mutation=${value(actions.timeToFirstDurableMutationMs)}`,
+      );
+    }
+    const activities = detail.metadata.agentActivity ?? [];
+    const subagents = activities.filter((activity) => activity.agent.role === "subagent");
+    if (subagents.length > 0) {
+      lines.push(`subagents: ${subagents.length}`);
+      for (const activity of subagents) {
+        const label = activity.agent.name
+          ? `${activity.agent.name} (${activity.agent.id})`
+          : activity.agent.id;
+        const agentActions = activity.actions;
+        lines.push(
+          `  ${label}: ${activity.status}, ${(activity.durationMs / 1000).toFixed(1)}s, events=${activity.eventCount}, tokens=${totalTokens(activity.usage)}, tools=${activity.toolCalls.total} (${activity.toolCalls.succeeded} ok/${activity.toolCalls.failed} failed/${activity.toolCalls.unresolved} unresolved), actions=${agentActions.total}, mutations=${agentActions.mutations}`,
+        );
+      }
+    }
   }
 };
 
@@ -361,17 +402,37 @@ const formatEventLine = (event: RunEvent): string => {
         }`;
       }
       return `responded(custom): ${JSON.stringify(event.payload).slice(0, 80)}`;
-    case "progress":
+    case "progress": {
+      const agent = event.agent?.role === "subagent" ? ` [subagent ${event.agent.id}]` : "";
+      if (event.kind === "agent-spawn") {
+        const payload = AgentSpawnPayloadSchema.safeParse(event.payload).data;
+        return payload
+          ? `agent-spawn: ${payload.agent.name ?? payload.agent.id} (${payload.agent.id})`
+          : "agent-spawn: malformed";
+      }
+      if (event.kind === "agent-completed") {
+        const payload = AgentCompletedPayloadSchema.safeParse(event.payload).data;
+        return payload
+          ? `agent-completed: ${payload.agentId} (${payload.status})`
+          : "agent-completed: malformed";
+      }
+      if (event.kind === "action") {
+        const payload = ActionProgressPayloadSchema.safeParse(event.payload).data;
+        return payload
+          ? `action${agent}: ${payload.name} (${payload.category})`
+          : "action: malformed";
+      }
       if (event.kind === "tool-call") {
         const payload = ToolCallPayloadSchema.safeParse(event.payload).data;
-        return payload ? `tool-call: ${payload.name}` : "tool-call: malformed";
+        return payload ? `tool-call${agent}: ${payload.name}` : "tool-call: malformed";
       }
       if (event.kind === "tool-result") {
         const payload = ToolResultPayloadSchema.safeParse(event.payload).data;
         if (!payload) return "tool-result: malformed";
-        return `tool-result: ${payload.name} (${payload.isError ? "failed" : "succeeded"})`;
+        return `tool-result${agent}: ${payload.name} (${payload.isError ? "failed" : "succeeded"})`;
       }
       return `progress(${event.kind})`;
+    }
     case "lifecycle":
       return `lifecycle: ${event.stage}`;
     case "ready":

package/src/orchestrator/loader.ts

@@ -1,5 +1,6 @@
 import { resolve } from "node:path";
 import { pathToFileURL } from "node:url";
+import { transferEvalCaseExtensions } from "../core/case-authoring-state.js";
 import { type EvalCase, EvalCaseSchema, formatZodIssues } from "../core/index.js";
 
 export const loadEvalCase = async (filePath: string): Promise<EvalCase> => {
@@ -15,5 +16,6 @@ export const loadEvalCase = async (filePath: string): Promise<EvalCase> => {
   if (!result.success) {
     throw new Error(`Invalid eval case at ${absolute}: ${formatZodIssues(result.error)}`);
   }
+  transferEvalCaseExtensions(mod.default as EvalCase, result.data);
   return result.data;
 };