@prismatic-io/lux 0.0.1 → 0.0.2-preview.10

package/src/core/assertion.ts

@@ -42,34 +42,23 @@ export interface AssertionImpl<C extends Assertion = Assertion> {
  * `AssertionImpl`, plugins must describe their case configuration so Lux can
  * reject invalid plans before starting a subject run.
  */
-export type AssertionPlugin<C extends Assertion = Assertion> = AssertionImpl<C> & {
+export type AssertionPlugin<
+  C extends Assertion = Assertion,
+  Input extends Assertion = C,
+> = AssertionImpl<C> & {
   configSchema: z.ZodObject & z.ZodType<C>;
+  /** Type-only input accepted while authoring before schema defaults apply. */
+  readonly __input?: Input;
   /** Author-maintained identity for evaluator/schema behavior. */
   pluginVersion: string;
 };
 
 type SchemaAssertion<S extends z.ZodObject> = z.output<S> extends Assertion ? z.output<S> : never;
-
-type IsUnion<T, Whole = T> = T extends Whole ? ([Whole] extends [T] ? false : true) : never;
-
-type SingleStringLiteral<T> = T extends string
-  ? string extends T
-    ? never
-    : true extends IsUnion<T>
-      ? never
-      : T
-  : never;
+type SchemaAssertionInput<S extends z.ZodObject> =
+  z.input<S> extends Assertion ? z.input<S> : never;
 
 type LiteralAssertionType<S extends z.ZodObject> =
-  SchemaAssertion<S> extends { type: infer OutputType }
-    ? z.input<S> extends { type: infer InputType }
-      ? SingleStringLiteral<OutputType> extends infer Literal extends string
-        ? [InputType, Literal] extends [Literal, InputType]
-          ? Literal
-          : never
-        : never
-      : never
-    : never;
+  S["shape"]["type"] extends z.ZodLiteral<infer Literal extends string> ? Literal : never;
 
 export type AssertionPluginDefinition<S extends z.ZodObject> = {
   type: LiteralAssertionType<S>;
@@ -111,12 +100,34 @@ export const assertionPluginIssue = (definition: {
  * Define a schema-validated assertion plugin while inferring the config passed
  * to `evaluate` from the schema.
  */
+export const baseAssertionFields = {
+  id: z.string().min(1).optional(),
+  weight: z.number().positive().optional(),
+  threshold: z.number().min(0).max(1).optional(),
+  name: z.string().optional(),
+} as const;
+
 export const defineAssertion = <S extends z.ZodObject>(
   definition: AssertionPluginDefinition<S>,
-): AssertionPlugin<SchemaAssertion<S>> => {
+): AssertionPlugin<SchemaAssertion<S>, SchemaAssertionInput<S>> => {
   const issue = assertionPluginIssue(definition);
   if (issue) throw new TypeError(issue);
-  return definition as unknown as AssertionPlugin<SchemaAssertion<S>>;
+  return definition as unknown as AssertionPlugin<SchemaAssertion<S>, SchemaAssertionInput<S>>;
+};
+
+export const defineAssertionRegistry = <const Plugins extends readonly AssertionPlugin[]>(
+  ...plugins: Plugins
+): Plugins => {
+  const types = new Set<string>();
+  for (const plugin of plugins) {
+    const issue = assertionPluginIssue(plugin);
+    if (issue) throw new TypeError(issue);
+    if (types.has(plugin.type)) {
+      throw new TypeError(`assertion registry has duplicate type '${plugin.type}'`);
+    }
+    types.add(plugin.type);
+  }
+  return plugins;
 };
 
 export const GradingAssertionEntrySchema = z.object({
@@ -137,11 +148,19 @@ export const GradingAssertionEntrySchema = z.object({
 });
 export type GradingAssertionEntry = z.infer<typeof GradingAssertionEntrySchema>;
 
+const SkippedGradingAssertionSchema = z.object({
+  id: z.string(),
+  type: z.string(),
+  name: z.string(),
+  reason: z.literal("judge-disabled"),
+});
+
 export const GradingReportSchema = z.object({
   schemaVersion: z.literal(1),
   casePassed: z.boolean().nullable(),
   caseScore: z.number().nullable(),
   assertions: z.array(GradingAssertionEntrySchema),
+  skippedAssertions: z.array(SkippedGradingAssertionSchema).optional(),
   /**
    * Present when the subject run or grading did not complete. Assertions may
    * still contain partial diagnostics, but the report is not a clean verdict.

package/src/core/case-authoring-state.ts

@@ -0,0 +1,65 @@
+import type { AssertionPlugin } from "./assertion.js";
+import type { EvalCase } from "./case.js";
+import type { LifecycleHooksPlugin } from "./lifecycle-hooks.js";
+
+export type EvalCaseExtensions = {
+  assertions: readonly AssertionPlugin[];
+  fixtures: readonly LifecycleHooksPlugin[];
+};
+
+const CASE_EXTENSIONS_KEY = Symbol.for("@prismatic-io/lux/eval-case-extensions");
+const extensionState = globalThis as typeof globalThis & { [key: symbol]: unknown };
+const sharedCaseExtensions = extensionState[CASE_EXTENSIONS_KEY];
+// A project-local package and the CLI can load separate Lux module instances.
+const caseExtensions =
+  sharedCaseExtensions instanceof WeakMap
+    ? (sharedCaseExtensions as WeakMap<object, EvalCaseExtensions>)
+    : new WeakMap<object, EvalCaseExtensions>();
+extensionState[CASE_EXTENSIONS_KEY] = caseExtensions;
+
+const uniqueBy = <T>(values: readonly T[], keyOf: (value: T) => string, kind: string): T[] => {
+  const unique = new Map<string, T>();
+  for (const value of values) {
+    const key = keyOf(value);
+    const existing = unique.get(key);
+    if (existing && existing !== value) {
+      throw new TypeError(`case has conflicting ${kind} definitions for '${key}'`);
+    }
+    unique.set(key, value);
+  }
+  return [...unique.values()];
+};
+
+export const collectEvalCaseExtensions = (
+  input: EvalCase,
+  assertions: readonly AssertionPlugin[] = [],
+  fixtures: readonly LifecycleHooksPlugin[] = [],
+): EvalCaseExtensions => {
+  const availableAssertions = uniqueBy(assertions, (plugin) => plugin.type, "assertion");
+  const byType = new Map(availableAssertions.map((plugin) => [plugin.type, plugin]));
+  return {
+    assertions: uniqueBy(
+      (input.assertions ?? []).flatMap((assertion) => {
+        const plugin = byType.get(assertion.type);
+        return plugin ? [plugin] : [];
+      }),
+      (plugin) => plugin.type,
+      "assertion",
+    ),
+    fixtures: uniqueBy(fixtures, (fixture) => fixture.name, "fixture"),
+  };
+};
+
+export const setEvalCaseExtensions = (evalCase: EvalCase, extensions: EvalCaseExtensions): void => {
+  if (extensions.assertions.length > 0 || extensions.fixtures.length > 0) {
+    caseExtensions.set(evalCase, extensions);
+  }
+};
+
+export const evalCaseExtensions = (evalCase: EvalCase): EvalCaseExtensions =>
+  caseExtensions.get(evalCase) ?? { assertions: [], fixtures: [] };
+
+export const transferEvalCaseExtensions = (source: EvalCase, target: EvalCase): void => {
+  const extensions = caseExtensions.get(source);
+  if (extensions) caseExtensions.set(target, extensions);
+};

package/src/core/case.ts

@@ -1,4 +1,7 @@
 import { z } from "zod";
+import type { AssertionPlugin } from "./assertion.js";
+import { collectEvalCaseExtensions, setEvalCaseExtensions } from "./case-authoring-state.js";
+import type { LifecycleHooksPlugin } from "./lifecycle-hooks.js";
 import { formatZodIssues } from "./zod.js";
 
 export const FixturesRefSchema = z.union([
@@ -67,14 +70,62 @@ export const EvalCaseSchema = EvalCaseObjectSchema
   });
 export type EvalCase = z.infer<typeof EvalCaseSchema>;
 
-export const defineEvalCase = (input: EvalCase): EvalCase => {
-  const parsed = EvalCaseSchema.safeParse(input);
-  if (!parsed.success) {
-    throw new Error(`Invalid eval case: ${formatZodIssues(parsed.error)}`);
-  }
-  return parsed.data;
+type AssertionPlanFields = Pick<Assertion, "id" | "name" | "threshold" | "weight">;
+type AssertionInputOf<Plugin> =
+  Plugin extends AssertionPlugin<Assertion, infer Input> ? Input & AssertionPlanFields : never;
+
+export type EvalCaseInput<AllowedAssertion extends Assertion = Assertion> = Omit<
+  EvalCase,
+  "assertions"
+> & {
+  assertions?: AllowedAssertion[];
+};
+
+export type EvalCaseAuthoringOptions<
+  Plugins extends readonly AssertionPlugin[] = readonly AssertionPlugin[],
+> = {
+  assertions?: Plugins;
+  fixtures?: readonly LifecycleHooksPlugin[];
+};
+
+export type EvalCaseDefinition<AllowedAssertion extends Assertion = Assertion> = {
+  (input: EvalCaseInput<AllowedAssertion>): EvalCase;
+  /**
+   * Create a case definition with selectively activated lifecycle fixtures.
+   * Further calls to `extend` append fixtures in setup order.
+   */
+  extend<const Plugins extends readonly AssertionPlugin[] = readonly []>(
+    options: EvalCaseAuthoringOptions<Plugins>,
+  ): EvalCaseDefinition<AllowedAssertion | AssertionInputOf<Plugins[number]>>;
 };
 
+export const createEvalCaseDefinition = <AllowedAssertion extends Assertion = Assertion>(
+  inheritedAssertions: readonly AssertionPlugin[] = [],
+  inheritedFixtures: readonly LifecycleHooksPlugin[] = [],
+): EvalCaseDefinition<AllowedAssertion> => {
+  const define = ((input: EvalCaseInput<AllowedAssertion>): EvalCase => {
+    const parsed = EvalCaseSchema.safeParse(input);
+    if (!parsed.success) {
+      throw new Error(`Invalid eval case: ${formatZodIssues(parsed.error)}`);
+    }
+    const extensions = collectEvalCaseExtensions(
+      parsed.data,
+      inheritedAssertions,
+      inheritedFixtures,
+    );
+    setEvalCaseExtensions(parsed.data, extensions);
+    return parsed.data;
+  }) as EvalCaseDefinition<AllowedAssertion>;
+  define.extend = (({ assertions = [], fixtures = [] }: EvalCaseAuthoringOptions) =>
+    createEvalCaseDefinition(
+      [...inheritedAssertions, ...assertions],
+      [...inheritedFixtures, ...fixtures],
+    )) as EvalCaseDefinition<AllowedAssertion>["extend"];
+  return define;
+};
+
+export const defineEvalCase = createEvalCaseDefinition();
+
 const PersistedFixturesRefSchema = z.union([
   z.string(),
   z.looseObject({ kind: z.literal("dir"), path: z.string() }),

package/src/core/driver-metrics.ts

@@ -0,0 +1,51 @@
+import { z } from "zod";
+
+export const DriverMetricUnitSchema = z.enum([
+  "number",
+  "count",
+  "milliseconds",
+  "bytes",
+  "tokens",
+  "usd",
+  "ratio",
+  "percent",
+  "score",
+]);
+export type DriverMetricUnit = z.infer<typeof DriverMetricUnitSchema>;
+
+export const DriverMetricDescriptorSchema = z
+  .object({
+    unit: DriverMetricUnitSchema.default("number"),
+    label: z.string().min(1).max(120).optional(),
+    better: z.enum(["lower", "higher", "neutral"]).default("neutral"),
+  })
+  .strict();
+export type DriverMetricDescriptor = z.infer<typeof DriverMetricDescriptorSchema>;
+
+export const DriverMetricSchema = DriverMetricDescriptorSchema.extend({
+  value: z.number().finite(),
+}).strict();
+export type DriverMetric = z.infer<typeof DriverMetricSchema>;
+
+export const DriverMetricsSchema = z.record(z.string().min(1).max(120), DriverMetricSchema);
+export type DriverMetrics = z.infer<typeof DriverMetricsSchema>;
+
+const ReportedDriverMetricSchema = z.union([z.number().finite(), DriverMetricSchema]);
+export const ReportedDriverMetricsSchema = z.record(
+  z.string().min(1).max(120),
+  ReportedDriverMetricSchema,
+);
+export type ReportedDriverMetrics = z.input<typeof ReportedDriverMetricsSchema>;
+
+export const normalizeDriverMetrics = (value: unknown): DriverMetrics => {
+  if (!value || typeof value !== "object" || Array.isArray(value)) return {};
+  return Object.fromEntries(
+    Object.entries(value).flatMap(([name, metric]) => {
+      if (name.length === 0 || name.length > 120) return [];
+      const parsed = DriverMetricSchema.safeParse(
+        typeof metric === "number" ? { value: metric } : metric,
+      ).data;
+      return parsed ? [[name, parsed]] : [];
+    }),
+  );
+};

package/src/core/driver.ts

@@ -1,6 +1,7 @@
 import { z } from "zod";
 import type { EventHandler } from "./emitter.js";
 import type { Answer, Done, ErrorEvent, Interrupt, Progress, ReadyState } from "./events.js";
+import type { FixtureValueBinding } from "./lifecycle-hooks.js";
 
 export const ArtifactSchema = z.object({
   path: z.string(),
@@ -28,6 +29,8 @@ export type StartContext = {
   caseId?: string;
   prompt: string;
   fixtures?: ResolvedFixtures | undefined;
+  /** Authored fixture values selected for this case. */
+  fixtureValues?: readonly FixtureValueBinding[] | undefined;
   /** Root containing persisted runs. Drivers may use it as an isolation boundary. */
   runsRoot?: string | undefined;
   runDir: string;

package/src/core/events.ts

@@ -57,11 +57,22 @@ export const AnswerSchema = z.discriminatedUnion("kind", [
 ]);
 export type Answer = z.infer<typeof AnswerSchema>;
 
+export const AgentAttributionSchema = z.object({
+  id: z.string().min(1),
+  role: z.enum(["main", "subagent"]),
+  parentId: z.exactOptional(z.string().min(1)),
+  spawnToolUseId: z.exactOptional(z.string().min(1)),
+  name: z.exactOptional(z.string().min(1)),
+  model: z.exactOptional(z.string().min(1)),
+});
+export type AgentAttribution = z.infer<typeof AgentAttributionSchema>;
+
 export const ProgressSchema = z.object({
   id: z.string(),
   kind: z.string(),
   payload: z.unknown(),
   ts: z.number(),
+  agent: z.exactOptional(AgentAttributionSchema),
 });
 export type Progress = z.infer<typeof ProgressSchema>;
 
@@ -69,8 +80,8 @@ export const DoneSchema = z.object({
   exitReason: z.literal("done"),
   /**
    * Driver-defined summary. The orchestrator recognizes `usage`, `cost`,
-   * `modelCalls`, and `unpricedModelCalls` when present while preserving room
-   * for domain-specific measurements.
+   * `modelCalls`, `unpricedModelCalls`, and scalar `metrics` when present while
+   * preserving room for other domain-specific evidence.
    */
   summary: z.unknown().optional(),
 });
@@ -82,6 +93,7 @@ export type ErrorEventReason = z.infer<typeof ErrorEventReasonSchema>;
 export const ErrorEventSchema = z.object({
   exitReason: ErrorEventReasonSchema,
   reason: z.string(),
+  summary: z.unknown().optional(),
 });
 export type ErrorEvent = z.infer<typeof ErrorEventSchema>;
 

package/src/core/experiment.ts

@@ -1,4 +1,5 @@
 import { z } from "zod";
+import { DriverMetricDescriptorSchema } from "./driver-metrics.js";
 import { ConfidenceConfigSchema } from "./statistics.js";
 import { TokenUsageSchema } from "./usage.js";
 import { formatZodIssues } from "./zod.js";
@@ -415,6 +416,7 @@ export const CandidateEvaluationSchema = z.object({
   passRate: z.number().min(0).max(1),
   casePassRates: z.record(z.string(), z.number().min(0).max(1)),
   metrics: z.record(z.string(), z.number()),
+  metricDefinitions: z.exactOptional(z.record(z.string(), DriverMetricDescriptorSchema)),
   metricCalls: z.exactOptional(z.number().int().nonnegative()),
   judgeCalls: z.exactOptional(z.number().int().nonnegative()),
   unpricedModelCalls: z.exactOptional(z.number().int().nonnegative()),

package/src/core/index.ts

@@ -1,3 +1,6 @@
+export * from "./action-events.js";
+export * from "./agent-activity.js";
+export * from "./agent-events.js";
 export * from "./annotation.js";
 export * from "./answerer.js";
 export * from "./artifact-evidence.js";
@@ -6,6 +9,7 @@ export * from "./assertion.js";
 export * from "./case.js";
 export * from "./conversation-turn.js";
 export * from "./driver.js";
+export * from "./driver-metrics.js";
 export * from "./emitter.js";
 export * from "./events.js";
 export * from "./exec.js";

package/src/core/lifecycle-fixtures.ts

@@ -2,6 +2,7 @@ import { valueHash } from "./hash.js";
 import {
   type CaseLifecycleHookContext,
   defineLifecycleHooks,
+  type FixtureValue,
   type LifecycleHookCleanup,
   type LifecycleHooksPlugin,
   type SuiteLifecycleHookContext,
@@ -22,6 +23,8 @@ export type LifecycleFixturesDefinition = {
    * to execution identity.
    */
   identity?: unknown;
+  /** Public, serializable data made available to the selected case's driver. */
+  value?: FixtureValue;
   suite?: LifecycleFixture<SuiteLifecycleHookContext>;
   case?: LifecycleFixture<CaseLifecycleHookContext>;
 };
@@ -109,8 +112,8 @@ const activateFixture = async <Context extends SuiteLifecycleHookContext>(
  * Setup belongs before `use`; teardown belongs after it.
  */
 export const defineFixtures = (definition: LifecycleFixturesDefinition): LifecycleHooksPlugin => {
-  if (!definition.suite && !definition.case) {
-    throw new TypeError("fixtures plugin must define suite or case");
+  if (!definition.suite && !definition.case && definition.value === undefined) {
+    throw new TypeError("fixtures plugin must define value, suite, or case");
   }
   if (definition.suite !== undefined && typeof definition.suite !== "function") {
     throw new TypeError("fixtures plugin suite must be a function");
@@ -123,8 +126,10 @@ export const defineFixtures = (definition: LifecycleFixturesDefinition): Lifecyc
   return defineLifecycleHooks({
     name: definition.name,
     pluginVersion: definition.pluginVersion,
+    ...(definition.value === undefined ? {} : { value: definition.value }),
     ...(definition.identity === undefined ? {} : { identity: definition.identity }),
     implementationHash: valueHash({
+      value: definition.value,
       identity: definition.identity,
       suite: definition.suite,
       case: definition.case,

package/src/core/lifecycle-hooks.ts

@@ -1,7 +1,15 @@
+import { z } from "zod";
 import { valueHash } from "./hash.js";
 
 export type LifecycleHookCleanup = () => void | Promise<void>;
 export type LifecycleHookSetupResult = LifecycleHookCleanup | undefined;
+export const FixtureValueSchema = z.json();
+export type FixtureValue = z.infer<typeof FixtureValueSchema>;
+
+export type FixtureValueBinding = {
+  name: string;
+  value: FixtureValue;
+};
 
 export type SuiteLifecycleHookItem = {
   id: string;
@@ -27,6 +35,8 @@ export type LifecycleHooksPlugin = {
   name: string;
   /** Author-maintained identity for setup and teardown behavior. */
   pluginVersion: string;
+  /** Public, serializable data made available to the selected case's driver. */
+  value?: FixtureValue;
   /**
    * Public, non-secret values captured by hook closures and relevant to
    * execution identity. Included in the implementation hash.
@@ -75,8 +85,11 @@ export const lifecycleHooksPluginIssue = (
   if (definition.setupCase !== undefined && typeof definition.setupCase !== "function") {
     return "lifecycle hooks plugin setupCase must be a function";
   }
-  if (!definition.setupSuite && !definition.setupCase) {
-    return "lifecycle hooks plugin must define setupSuite or setupCase";
+  if (definition.value !== undefined && !FixtureValueSchema.safeParse(definition.value).success) {
+    return "lifecycle hooks plugin value must be JSON-serializable";
+  }
+  if (!definition.setupSuite && !definition.setupCase && definition.value === undefined) {
+    return "lifecycle hooks plugin must define a value, setupSuite, or setupCase";
   }
   return undefined;
 };
@@ -93,6 +106,7 @@ export const defineLifecycleHooks = (definition: LifecycleHooksPlugin): Lifecycl
     implementationHash:
       definition.implementationHash ??
       valueHash({
+        value: definition.value,
         identity: definition.identity,
         setupSuite: definition.setupSuite,
         setupCase: definition.setupCase,

package/src/core/run.ts

@@ -1,6 +1,9 @@
 import { z } from "zod";
+import { ActionTimingSchema } from "./action-events.js";
+import { AgentActivitySchema } from "./agent-activity.js";
 import { EvalCaseSchema } from "./case.js";
 import { ArtifactSchema } from "./driver.js";
+import { DriverMetricsSchema } from "./driver-metrics.js";
 import { ErrorEventReasonSchema, ReadyStateSchema, RunEventSchema } from "./events.js";
 import { ToolOutcomeSummarySchema } from "./tool-events.js";
 import { RunUsageSchema } from "./usage.js";
@@ -112,6 +115,8 @@ export const RunMetadataSchema = z.object({
     }),
   ),
   cost: z.exactOptional(z.object({ usd: z.exactOptional(z.number()) })),
+  /** Driver-defined scalar measurements with display and optimization semantics. */
+  driverMetrics: z.exactOptional(DriverMetricsSchema),
   interruptCounts: z.object({ ask: z.number(), approve: z.number(), custom: z.number() }),
   /** Legacy raw count of progress events whose kind is `tool-call`. */
   toolCallCount: z.exactOptional(z.number()),
@@ -124,6 +129,10 @@ export const RunMetadataSchema = z.object({
   ),
   /** Wall-clock per phase, derived from the lifecycle event stream. */
   phaseDurationMs: z.exactOptional(PhaseDurationsSchema),
+  /** Driver-reported semantic action latency, measured from `start-driver`. */
+  actionTiming: z.exactOptional(ActionTimingSchema),
+  /** Per-agent lifecycle and activity derived from the attributed event stream. */
+  agentActivity: z.exactOptional(z.array(AgentActivitySchema)),
   /**
    * Harness/infrastructure failure that invalidates this run as candidate
    * evidence. A driver-reported subject failure intentionally leaves this

package/src/core/wire.ts

@@ -47,6 +47,7 @@ const ErrorEventSchema = z.object({
   type: z.literal("error"),
   exitReason: ErrorEventReasonSchema.default("error"),
   reason: z.string(),
+  summary: z.unknown().optional(),
 });
 
 export const WireEventSchema = z.discriminatedUnion("type", [
@@ -85,7 +86,10 @@ const wireEventToParsed = (event: WireEvent): ParsedDriverEvent => {
   if (event.type === "done") {
     return { tag: "done", done: { exitReason: "done", summary: event.summary } };
   }
-  return { tag: "error", error: { exitReason: event.exitReason, reason: event.reason } };
+  return {
+    tag: "error",
+    error: { exitReason: event.exitReason, reason: event.reason, summary: event.summary },
+  };
 };
 
 /**