@prismatic-io/lux 0.0.1

package/src/core/assertion.ts

@@ -0,0 +1,156 @@
+import { z } from "zod";
+import type { Assertion } from "./case.js";
+import { ErrorEventReasonSchema } from "./events.js";
+import type { Run } from "./run.js";
+import { TokenUsageSchema } from "./usage.js";
+
+export const AssertionResultSchema = z.object({
+  passed: z.boolean(),
+  score: z.number().min(0).max(1),
+  comment: z.exactOptional(z.string()),
+  detail: z.unknown().optional(),
+  /** Present when the assertion could not produce a judgment. */
+  errorMessage: z.exactOptional(z.string()),
+  /** Tokens the assertion itself spent. Only LLM-judge assertions report this. */
+  usage: z.exactOptional(TokenUsageSchema),
+});
+export type AssertionResult = z.infer<typeof AssertionResultSchema>;
+
+export type EvaluateContext = {
+  /**
+   * Fires when the run is being torn down. Long-running assertions (an LLM
+   * judge, a shell command) must cancel their in-flight work, or a Ctrl-C
+   * leaves billable subprocesses running past the run's end.
+   */
+  abortSignal?: AbortSignal;
+  /**
+   * Persists a judge-attempt start before an assertion invokes its provider.
+   * Grading marks it complete only after assertion spend is journaled.
+   */
+  recordModelAttemptStart?: (kind: "judge") => Promise<void>;
+};
+
+// Not schema-derived: generic config type, `evaluate` is a method.
+export interface AssertionImpl<C extends Assertion = Assertion> {
+  type: C["type"];
+  configSchema?: z.ZodTypeAny;
+  evaluate(config: C, run: Run, ctx?: EvaluateContext): Promise<AssertionResult>;
+}
+
+/**
+ * Public assertion extension contract. Unlike the lower-level
+ * `AssertionImpl`, plugins must describe their case configuration so Lux can
+ * reject invalid plans before starting a subject run.
+ */
+export type AssertionPlugin<C extends Assertion = Assertion> = AssertionImpl<C> & {
+  configSchema: z.ZodObject & z.ZodType<C>;
+  /** Author-maintained identity for evaluator/schema behavior. */
+  pluginVersion: string;
+};
+
+type SchemaAssertion<S extends z.ZodObject> = z.output<S> extends Assertion ? z.output<S> : never;
+
+type IsUnion<T, Whole = T> = T extends Whole ? ([Whole] extends [T] ? false : true) : never;
+
+type SingleStringLiteral<T> = T extends string
+  ? string extends T
+    ? never
+    : true extends IsUnion<T>
+      ? never
+      : T
+  : never;
+
+type LiteralAssertionType<S extends z.ZodObject> =
+  SchemaAssertion<S> extends { type: infer OutputType }
+    ? z.input<S> extends { type: infer InputType }
+      ? SingleStringLiteral<OutputType> extends infer Literal extends string
+        ? [InputType, Literal] extends [Literal, InputType]
+          ? Literal
+          : never
+        : never
+      : never
+    : never;
+
+export type AssertionPluginDefinition<S extends z.ZodObject> = {
+  type: LiteralAssertionType<S>;
+  configSchema: S;
+  pluginVersion: string;
+  evaluate(config: SchemaAssertion<S>, run: Run, ctx?: EvaluateContext): Promise<AssertionResult>;
+};
+
+export const assertionPluginIssue = (definition: {
+  type?: unknown;
+  configSchema?: unknown;
+  pluginVersion?: unknown;
+  evaluate?: unknown;
+}): string | undefined => {
+  if (typeof definition.type !== "string") {
+    return "assertion plugin type must be a string";
+  }
+  if (!(definition.configSchema instanceof z.ZodObject)) {
+    return "assertion plugin configSchema must be a direct Zod object";
+  }
+  const discriminator = definition.configSchema.shape.type;
+  if (
+    !(discriminator instanceof z.ZodLiteral) ||
+    discriminator.values.size !== 1 ||
+    discriminator.value !== definition.type
+  ) {
+    return "assertion plugin configSchema.type must be the matching string literal";
+  }
+  if (typeof definition.pluginVersion !== "string" || !definition.pluginVersion.trim()) {
+    return "assertion plugin pluginVersion must not be empty";
+  }
+  if (typeof definition.evaluate !== "function") {
+    return "assertion plugin evaluate must be a function";
+  }
+  return undefined;
+};
+
+/**
+ * Define a schema-validated assertion plugin while inferring the config passed
+ * to `evaluate` from the schema.
+ */
+export const defineAssertion = <S extends z.ZodObject>(
+  definition: AssertionPluginDefinition<S>,
+): AssertionPlugin<SchemaAssertion<S>> => {
+  const issue = assertionPluginIssue(definition);
+  if (issue) throw new TypeError(issue);
+  return definition as unknown as AssertionPlugin<SchemaAssertion<S>>;
+};
+
+export const GradingAssertionEntrySchema = z.object({
+  /** Stable author-supplied id, or a deterministic positional fallback. */
+  id: z.exactOptional(z.string()),
+  type: z.string(),
+  name: z.string(),
+  passed: z.boolean(),
+  score: z.number(),
+  weight: z.number(),
+  threshold: z.number(),
+  /** Wall-clock the assertion took to evaluate. A `rubric` costs seconds; a file check, none. */
+  durationMs: z.number(),
+  comment: z.exactOptional(z.string()),
+  detail: z.unknown().optional(),
+  errorMessage: z.exactOptional(z.string()),
+  usage: z.exactOptional(TokenUsageSchema),
+});
+export type GradingAssertionEntry = z.infer<typeof GradingAssertionEntrySchema>;
+
+export const GradingReportSchema = z.object({
+  schemaVersion: z.literal(1),
+  casePassed: z.boolean().nullable(),
+  caseScore: z.number().nullable(),
+  assertions: z.array(GradingAssertionEntrySchema),
+  /**
+   * Present when the subject run or grading did not complete. Assertions may
+   * still contain partial diagnostics, but the report is not a clean verdict.
+   */
+  incomplete: z.exactOptional(
+    z.object({
+      exitReason: ErrorEventReasonSchema,
+      phase: z.enum(["run", "collection", "grading"]).optional(),
+    }),
+  ),
+});
+export type GradingReport = z.infer<typeof GradingReportSchema>;

package/src/core/case.ts

@@ -0,0 +1,103 @@
+import { z } from "zod";
+import { formatZodIssues } from "./zod.js";
+
+export const FixturesRefSchema = z.union([
+  z.string(),
+  z.object({ kind: z.literal("dir"), path: z.string() }).strict(),
+]);
+export type FixturesRef = z.infer<typeof FixturesRefSchema>;
+
+// A driver/answerer is referenced either by a bare registry name (no config)
+// or by an object pairing the name with its config. Keeping the name and its
+// config in one object (rather than a name + a sibling `*Config` key) removes
+// the question of which config goes with which plugin and lets a future typed
+// `defineEvalCase` discriminate the config type on `name`.
+export const DriverRefSchema = z.union([
+  z.string(),
+  z.object({ name: z.string(), config: z.unknown().optional() }).strict(),
+]);
+export type DriverRef = z.infer<typeof DriverRefSchema>;
+
+export const AnswererRefSchema = z.union([
+  z.string(),
+  z.object({ name: z.string(), config: z.unknown().optional() }).strict(),
+]);
+export type AnswererRef = z.infer<typeof AnswererRefSchema>;
+
+export const AssertionSchema = z
+  .object({
+    /** Stable identity used to compare this assertion across variants and repetitions. */
+    id: z.string().min(1).optional(),
+    type: z.string(),
+    name: z.string().optional(),
+    weight: z.number().positive().optional(),
+    threshold: z.number().min(0).max(1).optional(),
+  })
+  .loose();
+export type Assertion = z.infer<typeof AssertionSchema>;
+
+const EvalCaseObjectSchema = z.object({
+  id: z.string().optional(),
+  prompt: z.string().min(1, "prompt is required"),
+  persona: z.string().min(1).optional(),
+  driver: DriverRefSchema.optional(),
+  answerer: AnswererRefSchema.optional(),
+  fixtures: FixturesRefSchema.optional(),
+  assertions: z.array(AssertionSchema).optional(),
+  meta: z.record(z.string(), z.unknown()).optional(),
+});
+
+export const EvalCaseSchema = EvalCaseObjectSchema
+  // strict so a stray `driverConfig`/`answererConfig` (the old shape) or a
+  // typo'd field fails loudly instead of being silently dropped.
+  .strict()
+  .superRefine((evalCase, ctx) => {
+    const seen = new Set<string>();
+    for (const [index, assertion] of (evalCase.assertions ?? []).entries()) {
+      if (!assertion.id) continue;
+      if (seen.has(assertion.id)) {
+        ctx.addIssue({
+          code: "custom",
+          path: ["assertions", index, "id"],
+          message: `duplicate assertion id '${assertion.id}'`,
+        });
+      }
+      seen.add(assertion.id);
+    }
+  });
+export type EvalCase = z.infer<typeof EvalCaseSchema>;
+
+export const defineEvalCase = (input: EvalCase): EvalCase => {
+  const parsed = EvalCaseSchema.safeParse(input);
+  if (!parsed.success) {
+    throw new Error(`Invalid eval case: ${formatZodIssues(parsed.error)}`);
+  }
+  return parsed.data;
+};
+
+const PersistedFixturesRefSchema = z.union([
+  z.string(),
+  z.looseObject({ kind: z.literal("dir"), path: z.string() }),
+]);
+const PersistedDriverRefSchema = z.union([
+  z.string(),
+  z.looseObject({ name: z.string(), config: z.unknown().optional() }),
+]);
+const PersistedAnswererRefSchema = z.union([
+  z.string(),
+  z.looseObject({ name: z.string(), config: z.unknown().optional() }),
+]);
+
+// The `case.json` frozen into a run dir. Reading run history stays permissive
+// where authoring is strict: every field optional, unknown keys kept —
+// serialization drops what it can't represent (predicate functions), and a
+// frozen case from any other lux version must still load.
+export const PersistedCaseSchema = EvalCaseObjectSchema.partial()
+  .extend({
+    casePath: z.exactOptional(z.string()),
+    driver: PersistedDriverRefSchema.optional(),
+    answerer: PersistedAnswererRefSchema.optional(),
+    fixtures: PersistedFixturesRefSchema.optional(),
+  })
+  .loose();
+export type PersistedCase = z.infer<typeof PersistedCaseSchema>;

package/src/core/conversation-turn.ts

@@ -0,0 +1,47 @@
+import { z } from "zod";
+import type { Answer, Interrupt } from "./events.js";
+
+export const CONVERSATION_TURN_REQUEST = "lux.conversation-turn.v1" as const;
+
+export const ConversationTurnRequestSchema = z
+  .object({
+    type: z.literal(CONVERSATION_TURN_REQUEST),
+    /** The agent's user-visible message at the end of its current turn. */
+    message: z.string(),
+  })
+  .strict();
+export type ConversationTurnRequest = z.infer<typeof ConversationTurnRequestSchema>;
+
+export const ConversationTurnResponseSchema = z.discriminatedUnion("action", [
+  z.object({ action: z.literal("stop") }).strict(),
+  z
+    .object({
+      action: z.literal("reply"),
+      text: z.string().trim().min(1),
+    })
+    .strict(),
+]);
+export type ConversationTurnResponse = z.infer<typeof ConversationTurnResponseSchema>;
+
+export const conversationTurnInterrupt = (
+  id: string,
+  message: string,
+  context?: unknown,
+): Interrupt => ({
+  kind: "custom",
+  id,
+  payload: { type: CONVERSATION_TURN_REQUEST, message },
+  ...(context === undefined ? {} : { context }),
+});
+
+export const isConversationTurnInterrupt = (
+  interrupt: Interrupt,
+): interrupt is Extract<Interrupt, { kind: "custom" }> & {
+  payload: ConversationTurnRequest;
+} =>
+  interrupt.kind === "custom" && ConversationTurnRequestSchema.safeParse(interrupt.payload).success;
+
+export const conversationTurnResponse = (answer: Answer): ConversationTurnResponse | null => {
+  if (answer.kind !== "custom") return null;
+  return ConversationTurnResponseSchema.safeParse(answer.payload).data ?? null;
+};

package/src/core/driver.ts

@@ -0,0 +1,75 @@
+import { z } from "zod";
+import type { EventHandler } from "./emitter.js";
+import type { Answer, Done, ErrorEvent, Interrupt, Progress, ReadyState } from "./events.js";
+
+export const ArtifactSchema = z.object({
+  path: z.string(),
+  /**
+   * Absolute dir `path` is relative to, when it is not the run's default
+   * `<runDir>/artifacts` (e.g. a driver configured to run in a checkout).
+   * Consumers resolve `path` against this before falling back to the default.
+   */
+  root: z.exactOptional(z.string()),
+  contentType: z.exactOptional(z.string()),
+  size: z.exactOptional(z.number()),
+  meta: z.exactOptional(z.record(z.string(), z.unknown())),
+});
+export type Artifact = z.infer<typeof ArtifactSchema>;
+
+export const ResolvedFixturesSchema = z.object({
+  rootDir: z.string(),
+  meta: z.exactOptional(z.record(z.string(), z.unknown())),
+});
+export type ResolvedFixtures = z.infer<typeof ResolvedFixturesSchema>;
+
+// Not schema-derived: `abortSignal` is a host object.
+export type StartContext = {
+  /** Stable selected case id, available to drivers that multiplex scenarios. */
+  caseId?: string;
+  prompt: string;
+  fixtures?: ResolvedFixtures | undefined;
+  /** Root containing persisted runs. Drivers may use it as an isolation boundary. */
+  runsRoot?: string | undefined;
+  runDir: string;
+  artifactsDir: string;
+  abortSignal: AbortSignal;
+};
+
+export type DriverEventMap = {
+  interrupt: Interrupt;
+  progress: Progress;
+  done: Done;
+  error: ErrorEvent;
+};
+
+/**
+ * Handlers may be async, and a driver **must await** every handler before
+ * emitting its next event: the orchestrator's `interrupt` handler awaits the
+ * answerer and calls `respond` before the agent may continue, so dispatching
+ * without awaiting breaks that interrupt/respond serialization. Backing the
+ * event surface with `TypedEmitter` provides this ordering.
+ */
+export interface AgentDriver {
+  start(ctx: StartContext): Promise<ReadyState>;
+  on<K extends keyof DriverEventMap>(event: K, handler: EventHandler<DriverEventMap[K]>): void;
+  off<K extends keyof DriverEventMap>(event: K, handler: EventHandler<DriverEventMap[K]>): void;
+  respond(answer: Answer): Promise<void>;
+  /**
+   * Stop agent work after a terminal protocol event but before evidence is
+   * collected. Process-backed drivers should implement this so a child cannot
+   * mutate artifacts while assertions are reading them.
+   */
+  quiesce?(): Promise<void>;
+  /**
+   * Return a stable evidence snapshot. Drivers without `quiesce` are
+   * responsible for stopping background writers before this promise resolves.
+   */
+  collect(): Promise<Artifact[]>;
+  close(): Promise<void>;
+}
+
+export type DriverPlugin<Cfg = unknown> = {
+  name: string;
+  configSchema?: z.ZodTypeAny;
+  create: (config: Cfg) => AgentDriver | Promise<AgentDriver>;
+};

package/src/core/emitter.ts

@@ -0,0 +1,38 @@
+export type EventHandler<E> = (event: E) => void | Promise<void>;
+
+/**
+ * Minimal typed event emitter (Node's EventEmitter is untyped on the
+ * listener side).
+ *
+ * Handlers are awaited serially within a single emit — the orchestrator's
+ * interrupt handler must finish before the driver continues — so keep
+ * handlers cheap. Dispatch is fail-fast: a throwing handler propagates out
+ * of `emit` and later handlers are skipped; callers treat a broken handler
+ * as a broken run. Handlers live in a Set: double registration is a no-op,
+ * and changes mid-emit affect the remainder of that dispatch.
+ */
+export class TypedEmitter<EventMap extends Record<string, unknown>> {
+  private readonly handlers = new Map<keyof EventMap, Set<EventHandler<unknown>>>();
+
+  on<K extends keyof EventMap>(event: K, handler: EventHandler<EventMap[K]>): void {
+    const set = this.handlers.get(event) ?? new Set<EventHandler<unknown>>();
+    this.handlers.set(event, set);
+    set.add(handler as EventHandler<unknown>);
+  }
+
+  off<K extends keyof EventMap>(event: K, handler: EventHandler<EventMap[K]>): void {
+    this.handlers.get(event)?.delete(handler as EventHandler<unknown>);
+  }
+
+  async emit<K extends keyof EventMap>(event: K, payload: EventMap[K]): Promise<void> {
+    const set = this.handlers.get(event);
+    if (!set) return;
+    for (const handler of set) {
+      await (handler as EventHandler<EventMap[K]>)(payload);
+    }
+  }
+
+  removeAllListeners(): void {
+    this.handlers.clear();
+  }
+}

package/src/core/events.ts

@@ -0,0 +1,135 @@
+import { z } from "zod";
+
+export const InterruptKindSchema = z.enum(["ask", "approve", "custom"]);
+export type InterruptKind = z.infer<typeof InterruptKindSchema>;
+
+export const AskInterruptSchema = z.object({
+  kind: z.literal("ask"),
+  id: z.string(),
+  question: z.string(),
+  context: z.unknown().optional(),
+});
+
+export const ApproveInterruptSchema = z.object({
+  kind: z.literal("approve"),
+  id: z.string(),
+  request: z.string(),
+  context: z.unknown().optional(),
+});
+
+export const CustomInterruptSchema = z.object({
+  kind: z.literal("custom"),
+  id: z.string(),
+  payload: z.unknown(),
+  context: z.unknown().optional(),
+});
+
+export const InterruptSchema = z.discriminatedUnion("kind", [
+  AskInterruptSchema,
+  ApproveInterruptSchema,
+  CustomInterruptSchema,
+]);
+export type Interrupt = z.infer<typeof InterruptSchema>;
+
+export const AskAnswerSchema = z.object({
+  kind: z.literal("ask"),
+  id: z.string(),
+  text: z.string(),
+});
+
+export const ApproveAnswerSchema = z.object({
+  kind: z.literal("approve"),
+  id: z.string(),
+  approved: z.boolean(),
+  reason: z.exactOptional(z.string()),
+});
+
+export const CustomAnswerSchema = z.object({
+  kind: z.literal("custom"),
+  id: z.string(),
+  payload: z.unknown(),
+});
+
+export const AnswerSchema = z.discriminatedUnion("kind", [
+  AskAnswerSchema,
+  ApproveAnswerSchema,
+  CustomAnswerSchema,
+]);
+export type Answer = z.infer<typeof AnswerSchema>;
+
+export const ProgressSchema = z.object({
+  id: z.string(),
+  kind: z.string(),
+  payload: z.unknown(),
+  ts: z.number(),
+});
+export type Progress = z.infer<typeof ProgressSchema>;
+
+export const DoneSchema = z.object({
+  exitReason: z.literal("done"),
+  /**
+   * Driver-defined summary. The orchestrator recognizes `usage`, `cost`,
+   * `modelCalls`, and `unpricedModelCalls` when present while preserving room
+   * for domain-specific measurements.
+   */
+  summary: z.unknown().optional(),
+});
+export type Done = z.infer<typeof DoneSchema>;
+
+export const ErrorEventReasonSchema = z.enum(["error", "idle-timeout", "max-interactions"]);
+export type ErrorEventReason = z.infer<typeof ErrorEventReasonSchema>;
+
+export const ErrorEventSchema = z.object({
+  exitReason: ErrorEventReasonSchema,
+  reason: z.string(),
+});
+export type ErrorEvent = z.infer<typeof ErrorEventSchema>;
+
+export const LifecycleStageSchema = z.enum([
+  "load",
+  "prepare-answerer",
+  "start-driver",
+  "run-loop",
+  "collect-start",
+  "collect-end",
+  "grade-start",
+  "grade-end",
+  "close",
+]);
+export type LifecycleStage = z.infer<typeof LifecycleStageSchema>;
+
+export const ReadyStateSchema = z.object({
+  agentId: z.string(),
+  agentVersion: z.exactOptional(z.string()),
+  model: z.exactOptional(z.string()),
+  detail: z.exactOptional(z.record(z.string(), z.unknown())),
+});
+export type ReadyState = z.infer<typeof ReadyStateSchema>;
+
+const stamped = { ts: z.number() };
+
+// Nested because a flat union on `type` would collide across the arms sharing it.
+const InterruptEventSchema = z.discriminatedUnion("kind", [
+  AskInterruptSchema.extend({ type: z.literal("interrupt"), ...stamped }),
+  ApproveInterruptSchema.extend({ type: z.literal("interrupt"), ...stamped }),
+  CustomInterruptSchema.extend({ type: z.literal("interrupt"), ...stamped }),
+]);
+
+const RespondedEventSchema = z.discriminatedUnion("kind", [
+  AskAnswerSchema.extend({ type: z.literal("responded"), ...stamped }),
+  ApproveAnswerSchema.extend({ type: z.literal("responded"), ...stamped }),
+  CustomAnswerSchema.extend({ type: z.literal("responded"), ...stamped }),
+]);
+
+export const RunEventSchema = z.discriminatedUnion("type", [
+  // `ready` is the one un-stamped arm: the orchestrator persists the driver's
+  // ReadyState verbatim, and the adjacent `lifecycle` events carry its timing.
+  ReadyStateSchema.extend({ type: z.literal("ready") }),
+  InterruptEventSchema,
+  RespondedEventSchema,
+  ProgressSchema.extend({ type: z.literal("progress") }),
+  DoneSchema.extend({ type: z.literal("done"), ...stamped }),
+  ErrorEventSchema.extend({ type: z.literal("error"), ...stamped }),
+  z.object({ type: z.literal("lifecycle"), stage: LifecycleStageSchema, ...stamped }),
+]);
+export type RunEvent = z.infer<typeof RunEventSchema>;