@prismatic-io/lux 0.0.1

package/src/core/tool-events.ts

@@ -0,0 +1,292 @@
+import { z } from "zod";
+import type { Progress, RunEvent } from "./events.js";
+import { tryParseJson } from "./zod.js";
+
+export const TOOL_CALL_KIND = "tool-call" as const;
+export const TOOL_RESULT_KIND = "tool-result" as const;
+
+export const ToolCallPayloadSchema = z
+  .object({
+    name: z.string().min(1),
+    input: z.unknown(),
+    toolUseId: z.string().min(1),
+    parentToolUseId: z.string().min(1).optional(),
+  })
+  .loose()
+  .superRefine((payload, ctx) => {
+    if (!Object.hasOwn(payload, "input")) {
+      ctx.addIssue({
+        code: "custom",
+        path: ["input"],
+        message: "input is required",
+      });
+    }
+  });
+export type ToolCallPayload = z.infer<typeof ToolCallPayloadSchema>;
+
+export const ToolResultPayloadSchema = z
+  .object({
+    name: z.string().min(1),
+    toolUseId: z.string().min(1),
+    content: z.unknown(),
+    structuredContent: z.record(z.string(), z.unknown()).optional(),
+    isError: z.boolean(),
+    text: z.string(),
+    durationMs: z.number().nonnegative().optional(),
+    parentToolUseId: z.string().min(1).optional(),
+  })
+  .loose()
+  .superRefine((payload, ctx) => {
+    if (!Object.hasOwn(payload, "content")) {
+      ctx.addIssue({
+        code: "custom",
+        path: ["content"],
+        message: "content is required",
+      });
+    }
+  });
+export type ToolResultPayload = z.infer<typeof ToolResultPayloadSchema>;
+
+export type ToolInvocation = {
+  call: ToolCallPayload;
+  result?: ToolResultPayload;
+};
+
+export const ToolOutcomeSummarySchema = z
+  .object({
+    reportedCalls: z.number().int().nonnegative(),
+    total: z.number().int().nonnegative(),
+    succeeded: z.number().int().nonnegative(),
+    failed: z.number().int().nonnegative(),
+    unresolved: z.number().int().nonnegative(),
+    unmatchedResults: z.number().int().nonnegative(),
+    malformedEvents: z.number().int().nonnegative(),
+  })
+  .superRefine((summary, ctx) => {
+    if (summary.reportedCalls < summary.total) {
+      ctx.addIssue({
+        code: "custom",
+        path: ["reportedCalls"],
+        message: "reportedCalls cannot be less than total",
+      });
+    }
+    if (summary.total !== summary.succeeded + summary.failed + summary.unresolved) {
+      ctx.addIssue({
+        code: "custom",
+        path: ["total"],
+        message: "total must equal succeeded + failed + unresolved",
+      });
+    }
+  });
+export type ToolOutcomeSummary = z.infer<typeof ToolOutcomeSummarySchema>;
+
+export type ToolInvocationOutcome = "succeeded" | "failed" | "unresolved";
+export type ToolEventCorrelation = {
+  invocations: ToolInvocation[];
+  unmatchedResults: number;
+};
+export type ToolOutcomeAccumulator = {
+  add: (event: RunEvent) => void;
+  summary: () => ToolOutcomeSummary;
+};
+
+type ProgressEvent = Extract<RunEvent, { type: "progress" }>;
+type ToolCallEvent = ProgressEvent & { kind: typeof TOOL_CALL_KIND };
+type ToolResultEvent = ProgressEvent & { kind: typeof TOOL_RESULT_KIND };
+
+const isProgressKind = <Kind extends string>(
+  event: RunEvent,
+  kind: Kind,
+): event is ProgressEvent & { kind: Kind } => event.type === "progress" && event.kind === kind;
+
+export const isToolCallEvent = (event: RunEvent): event is ToolCallEvent =>
+  isProgressKind(event, TOOL_CALL_KIND);
+
+export const isToolResultEvent = (event: RunEvent): event is ToolResultEvent =>
+  isProgressKind(event, TOOL_RESULT_KIND);
+
+export const toolCallProgress = (seq: number, payload: ToolCallPayload, ts: number): Progress => ({
+  id: `tc-${payload.toolUseId || seq}`,
+  kind: TOOL_CALL_KIND,
+  payload,
+  ts,
+});
+
+export const toolResultProgress = (
+  seq: number,
+  payload: ToolResultPayload,
+  ts: number,
+): Progress => ({
+  id: `tr-${payload.toolUseId || seq}`,
+  kind: TOOL_RESULT_KIND,
+  payload,
+  ts,
+});
+
+export const toolEventCalls = (events: RunEvent[], name?: string): ToolCallPayload[] =>
+  events
+    .filter(isToolCallEvent)
+    .map((event) => ToolCallPayloadSchema.safeParse(event.payload).data)
+    .filter((payload): payload is ToolCallPayload => payload !== undefined)
+    .filter((payload) => name === undefined || payload.name === name);
+
+export const toolResults = (events: RunEvent[], name?: string): ToolResultPayload[] =>
+  events
+    .filter(isToolResultEvent)
+    .map((event) => ToolResultPayloadSchema.safeParse(event.payload).data)
+    .filter((payload): payload is ToolResultPayload => payload !== undefined)
+    .filter((payload) => name === undefined || payload.name === name);
+
+export const malformedToolEventCount = (
+  events: readonly RunEvent[],
+  kinds: readonly (typeof TOOL_CALL_KIND | typeof TOOL_RESULT_KIND)[] = [
+    TOOL_CALL_KIND,
+    TOOL_RESULT_KIND,
+  ],
+): number => {
+  const selected = new Set<string>(kinds);
+  return events.filter((event) => {
+    if (event.type !== "progress" || !selected.has(event.kind)) return false;
+    const schema = event.kind === TOOL_CALL_KIND ? ToolCallPayloadSchema : ToolResultPayloadSchema;
+    return !schema.safeParse(event.payload).success;
+  }).length;
+};
+
+export const toolResultValue = (result: ToolResultPayload): unknown => {
+  if (result.structuredContent !== undefined) return result.structuredContent;
+  const parsedText = tryParseJson(result.text);
+  if (Array.isArray(result.content)) {
+    return parsedText !== undefined ? parsedText : result.content;
+  }
+  if (typeof result.content === "string") {
+    const parsedContent = tryParseJson(result.content);
+    return parsedContent !== undefined ? parsedContent : result.content;
+  }
+  if (result.content !== null && result.content !== undefined) {
+    return result.content;
+  }
+  return parsedText !== undefined ? parsedText : result.text;
+};
+
+/**
+ * Correlate standardized calls with the first later result in the same agent
+ * scope. Evidence without the required correlation fields is ignored.
+ */
+const correlationKey = (toolName: string, toolUseId: string, parentToolUseId?: string): string =>
+  JSON.stringify([parentToolUseId ?? null, toolName, toolUseId]);
+
+class ToolOutcomeAccumulatorImpl implements ToolOutcomeAccumulator {
+  private readonly name: string | undefined;
+  private readonly correlatedInvocations: ToolInvocation[] = [];
+  private readonly pending = new Map<string, number>();
+  private readonly ambiguous = new Map<string, number>();
+  private reportedCalls = 0;
+  private succeeded = 0;
+  private failed = 0;
+  private unresolved = 0;
+  private unmatchedResults = 0;
+  private malformedEvents = 0;
+
+  constructor(name?: string) {
+    this.name = name;
+  }
+
+  add = (event: RunEvent): void => {
+    if (isToolCallEvent(event)) {
+      if (this.name === undefined) this.reportedCalls++;
+      const call = ToolCallPayloadSchema.safeParse(event.payload).data;
+      if (!call) {
+        if (this.name === undefined) this.malformedEvents++;
+        return;
+      }
+      if (this.name !== undefined && call.name !== this.name) return;
+      if (this.name !== undefined) this.reportedCalls++;
+      const index = this.correlatedInvocations.push({ call }) - 1;
+      this.unresolved++;
+      const key = correlationKey(call.name, call.toolUseId, call.parentToolUseId);
+      const ambiguousCalls = this.ambiguous.get(key);
+      if (ambiguousCalls !== undefined) {
+        this.ambiguous.set(key, ambiguousCalls + 1);
+        return;
+      }
+      if (this.pending.has(key)) {
+        this.pending.delete(key);
+        this.ambiguous.set(key, 2);
+        return;
+      }
+      this.pending.set(key, index);
+      return;
+    }
+    if (!isToolResultEvent(event)) return;
+    const result = ToolResultPayloadSchema.safeParse(event.payload).data;
+    if (!result) {
+      if (this.name === undefined) this.malformedEvents++;
+      return;
+    }
+    if (this.name !== undefined && result.name !== this.name) return;
+    const key = correlationKey(result.name, result.toolUseId, result.parentToolUseId);
+    const ambiguousCalls = this.ambiguous.get(key);
+    if (ambiguousCalls !== undefined) {
+      this.unmatchedResults++;
+      if (ambiguousCalls === 1) this.ambiguous.delete(key);
+      else this.ambiguous.set(key, ambiguousCalls - 1);
+      return;
+    }
+    const index = this.pending.get(key);
+    if (index === undefined) {
+      this.unmatchedResults++;
+      return;
+    }
+    this.pending.delete(key);
+    const invocation = this.correlatedInvocations[index];
+    if (!invocation) {
+      this.unmatchedResults++;
+      return;
+    }
+    invocation.result = result;
+    this.unresolved--;
+    if (result.isError) this.failed++;
+    else this.succeeded++;
+  };
+
+  summary = (): ToolOutcomeSummary => ({
+    reportedCalls: this.reportedCalls,
+    total: this.correlatedInvocations.length,
+    succeeded: this.succeeded,
+    failed: this.failed,
+    unresolved: this.unresolved,
+    unmatchedResults: this.unmatchedResults,
+    malformedEvents: this.malformedEvents,
+  });
+
+  correlation = (): ToolEventCorrelation => ({
+    invocations: this.correlatedInvocations,
+    unmatchedResults: this.unmatchedResults,
+  });
+}
+
+export const createToolOutcomeAccumulator = (): ToolOutcomeAccumulator =>
+  new ToolOutcomeAccumulatorImpl();
+
+export const correlateToolEvents = (
+  events: readonly RunEvent[],
+  name?: string,
+): ToolEventCorrelation => {
+  const accumulator = new ToolOutcomeAccumulatorImpl(name);
+  for (const event of events) accumulator.add(event);
+  return accumulator.correlation();
+};
+
+export const toolInvocations = (events: readonly RunEvent[], name?: string): ToolInvocation[] =>
+  correlateToolEvents(events, name).invocations;
+
+export const toolInvocationOutcome = (invocation: ToolInvocation): ToolInvocationOutcome => {
+  if (!invocation.result) return "unresolved";
+  return invocation.result.isError ? "failed" : "succeeded";
+};
+
+export const summarizeToolOutcomes = (events: readonly RunEvent[]): ToolOutcomeSummary => {
+  const accumulator = createToolOutcomeAccumulator();
+  for (const event of events) accumulator.add(event);
+  return accumulator.summary();
+};

package/src/core/usage.ts

@@ -0,0 +1,123 @@
+import { z } from "zod";
+
+/**
+ * Token accounting. A Lux run spends tokens in three places: the agent
+ * under test (the driver), the answerer playing the user, and the LLM
+ * judge grading the result. Only the first is the thing being measured —
+ * the other two are eval overhead — so usage is tracked per source and
+ * summed at the edges rather than pooled into one number.
+ */
+export const TokenUsageSchema = z.object({
+  input: z.number().int().nonnegative(),
+  output: z.number().int().nonnegative(),
+  /** Prompt tokens served from the cache. Billed, but far cheaper than `input`. */
+  cacheRead: z.number().int().nonnegative(),
+  /** Prompt tokens written to the cache on this call. */
+  cacheCreation: z.number().int().nonnegative(),
+});
+export type TokenUsage = z.infer<typeof TokenUsageSchema>;
+
+/**
+ * One API call's usage, tagged with what the agent did on it — a tool name,
+ * a snippet of its reply — so a per-turn breakdown reads as more than a number.
+ */
+export const TurnUsageSchema = TokenUsageSchema.extend({
+  label: z.exactOptional(z.string()),
+});
+export type TurnUsage = z.infer<typeof TurnUsageSchema>;
+
+/**
+ * The vocabulary a turn label is built from: one glyph per facet an API call
+ * exercised. Kept beside the turn model — not in the driver that stamps them
+ * onto labels nor the reporter that documents them in its legend — so the two
+ * cannot drift out of agreement.
+ */
+export const TURN_GLYPHS = {
+  thinking: "✻",
+  ask: "?",
+  tool: "⚙",
+  reply: "¶",
+} as const;
+
+export const emptyUsage = (): TokenUsage => ({
+  input: 0,
+  output: 0,
+  cacheRead: 0,
+  cacheCreation: 0,
+});
+
+export const addUsage = (a: TokenUsage, b: TokenUsage): TokenUsage => ({
+  input: a.input + b.input,
+  output: a.output + b.output,
+  cacheRead: a.cacheRead + b.cacheRead,
+  cacheCreation: a.cacheCreation + b.cacheCreation,
+});
+
+export const sumUsage = (usages: readonly TokenUsage[]): TokenUsage =>
+  usages.reduce(addUsage, emptyUsage());
+
+/** Every token the call touched, cached or not. */
+export const totalTokens = (u: TokenUsage): number =>
+  u.input + u.output + u.cacheRead + u.cacheCreation;
+
+export const isUsageEmpty = (u: TokenUsage): boolean => totalTokens(u) === 0;
+
+/** Usage split by who spent it. `agent` is the subject; the rest is harness. */
+export const RunUsageSchema = z.object({
+  agent: TokenUsageSchema,
+  answerer: TokenUsageSchema,
+  grading: TokenUsageSchema,
+});
+export type RunUsage = z.infer<typeof RunUsageSchema>;
+
+export const emptyRunUsage = (): RunUsage => ({
+  agent: emptyUsage(),
+  answerer: emptyUsage(),
+  grading: emptyUsage(),
+});
+
+/** What Lux spent to run the eval, as opposed to what the agent spent. */
+export const harnessUsage = (u: RunUsage): TokenUsage => addUsage(u.answerer, u.grading);
+
+// Token accounting must never be the reason a run fails to grade.
+const AgentTokenCountSchema = z.number().int().nonnegative().catch(0);
+
+/**
+ * The snake_case `usage` object the Claude CLI reports (on the
+ * `--output-format json` envelope, and on the final `result` line of a
+ * `stream-json` stream).
+ */
+const AgentUsageSchema = z
+  .object({
+    input_tokens: AgentTokenCountSchema,
+    output_tokens: AgentTokenCountSchema,
+    cache_read_input_tokens: AgentTokenCountSchema,
+    cache_creation_input_tokens: AgentTokenCountSchema,
+  })
+  .transform(
+    (usage): TokenUsage => ({
+      input: usage.input_tokens,
+      output: usage.output_tokens,
+      cacheRead: usage.cache_read_input_tokens,
+      cacheCreation: usage.cache_creation_input_tokens,
+    }),
+  );
+
+const AGENT_USAGE_KEYS = [
+  "input_tokens",
+  "output_tokens",
+  "cache_read_input_tokens",
+  "cache_creation_input_tokens",
+] as const;
+
+/**
+ * Returns null when the object is absent or carries none of the recognized
+ * usage keys, so a driver can distinguish "reported zero" from "reported
+ * nothing". A partial or malformed report still parses — unrecognized or
+ * missing counts read as 0, per `AgentTokenCountSchema`'s never-fail rule.
+ */
+export const parseAgentUsage = (raw: unknown): TokenUsage | null => {
+  if (typeof raw !== "object" || raw === null) return null;
+  if (!AGENT_USAGE_KEYS.some((key) => key in raw)) return null;
+  return AgentUsageSchema.safeParse(raw).data ?? null;
+};

package/src/core/wire.ts

@@ -0,0 +1,135 @@
+import { z } from "zod";
+import {
+  type Answer,
+  ApproveInterruptSchema,
+  AskInterruptSchema,
+  CustomInterruptSchema,
+  type Done,
+  type ErrorEvent,
+  ErrorEventReasonSchema,
+  type Interrupt,
+  type Progress,
+  ProgressSchema,
+  type ReadyState,
+  ReadyStateSchema,
+} from "./events.js";
+
+/**
+ * The `lux-jsonl` wire protocol. Drivers that talk to a subprocess agent
+ * over stdio (or any line-delimited transport) parse incoming lines with
+ * {@link parseWireEvent} and serialize outgoing answers with
+ * {@link formatWireAnswer}. Drivers that speak a foreign protocol can
+ * import {@link ParsedDriverEvent} and emit it directly.
+ *
+ * Each arm is the internal event schema plus a `type` discriminator. The wire
+ * is looser in two places — `progress.ts` and `error.exitReason` may be omitted
+ * — and those two `z.default()`s are the whole difference.
+ */
+const ReadyEventSchema = ReadyStateSchema.extend({ type: z.literal("ready") });
+
+const InterruptEventSchema = z.discriminatedUnion("kind", [
+  AskInterruptSchema.extend({ type: z.literal("interrupt") }),
+  ApproveInterruptSchema.extend({ type: z.literal("interrupt") }),
+  CustomInterruptSchema.extend({ type: z.literal("interrupt") }),
+]);
+
+const ProgressEventSchema = ProgressSchema.extend({
+  type: z.literal("progress"),
+  ts: z.number().default(() => Date.now()),
+});
+
+const DoneEventSchema = z.object({
+  type: z.literal("done"),
+  summary: z.unknown().optional(),
+});
+
+const ErrorEventSchema = z.object({
+  type: z.literal("error"),
+  exitReason: ErrorEventReasonSchema.default("error"),
+  reason: z.string(),
+});
+
+export const WireEventSchema = z.discriminatedUnion("type", [
+  ReadyEventSchema,
+  InterruptEventSchema,
+  ProgressEventSchema,
+  DoneEventSchema,
+  ErrorEventSchema,
+]);
+
+export type WireEvent = z.infer<typeof WireEventSchema>;
+
+export type ParsedDriverEvent =
+  | { tag: "ready"; ready: ReadyState }
+  | { tag: "interrupt"; interrupt: Interrupt }
+  | { tag: "progress"; progress: Progress }
+  | { tag: "done"; done: Done }
+  | { tag: "error"; error: ErrorEvent };
+
+// Rebuilding (never re-parsing) drops the wire's `type` key; a re-parse
+// against an internal schema would start throwing if that schema gained a
+// required field the wire arm defaults.
+const wireEventToParsed = (event: WireEvent): ParsedDriverEvent => {
+  if (event.type === "ready") {
+    const { type: _type, ...ready } = event;
+    return { tag: "ready", ready };
+  }
+  if (event.type === "interrupt") {
+    const { type: _type, ...interrupt } = event;
+    return { tag: "interrupt", interrupt };
+  }
+  if (event.type === "progress") {
+    const { type: _type, ...progress } = event;
+    return { tag: "progress", progress };
+  }
+  if (event.type === "done") {
+    return { tag: "done", done: { exitReason: "done", summary: event.summary } };
+  }
+  return { tag: "error", error: { exitReason: event.exitReason, reason: event.reason } };
+};
+
+/**
+ * Parse a single line from a `lux-jsonl` agent. Returns null for empty
+ * lines, malformed JSON, or events that don't match the wire schema.
+ * Validation failures are silent by design — drivers may interleave
+ * non-protocol log lines on stdout.
+ */
+export const parseWireEvent = (line: string): ParsedDriverEvent | null => {
+  const trimmed = line.trim();
+  if (!trimmed) return null;
+  let raw: unknown;
+  try {
+    raw = JSON.parse(trimmed);
+  } catch {
+    return null;
+  }
+  const parsed = WireEventSchema.safeParse(raw);
+  if (!parsed.success) return null;
+  return wireEventToParsed(parsed.data);
+};
+
+/**
+ * Serialize an Answer to a `lux-jsonl` line (newline-terminated). The
+ * inverse of an agent reading its stdin and dispatching to whatever
+ * pending interrupt the answer is for.
+ */
+export const formatWireAnswer = (answer: Answer): string => {
+  let message: Record<string, unknown>;
+  if (answer.kind === "ask") {
+    message = { type: "answer", id: answer.id, text: answer.text };
+  } else if (answer.kind === "approve") {
+    message = { type: answer.approved ? "approve" : "deny", id: answer.id, reason: answer.reason };
+  } else {
+    message = { type: "customToolResponse", id: answer.id, payload: answer.payload };
+  }
+  return `${JSON.stringify(message)}\n`;
+};
+
+/**
+ * Parse one stdout line into zero, one, or many Lux driver events. A single
+ * line can carry multiple events — e.g. a Claude Code assistant message with
+ * a text block followed by a tool_use block — so returning an array lets a
+ * driver surface all of them rather than dropping every block after the first.
+ */
+export type ParseEventFn = (line: string) => ParsedDriverEvent | ParsedDriverEvent[] | null;
+export type FormatAnswerFn = (answer: Answer) => string;

package/src/core/zod.ts

@@ -0,0 +1,20 @@
+import type { z } from "zod";
+
+/**
+ * Render a failed parse's issues as one `path: message; path: message` line.
+ * The one place that decides how Lux surfaces schema errors, so every
+ * `Invalid …: <issues>` message the loaders and validators throw reads the
+ * same. `<root>` stands in for an issue at the top level (empty path).
+ */
+export const formatZodIssues = (error: z.ZodError): string =>
+  error.issues.map((i) => `${i.path.join(".") || "<root>"}: ${i.message}`).join("; ");
+
+// `JSON.parse` that yields `undefined` instead of throwing, so malformed and
+// off-schema text both fail as one `safeParse` of `undefined`.
+export const tryParseJson = (text: string): unknown => {
+  try {
+    return JSON.parse(text);
+  } catch {
+    return undefined;
+  }
+};