@prismatic-io/lux 0.0.1

package/src/assertions/core/output-checks.ts

@@ -0,0 +1,134 @@
+import { z } from "zod";
+import type { AssertionImpl, Run, RunEvent } from "../../core/index.js";
+import { baseAssertionFields as baseFields } from "./helpers.js";
+import { inspectSafeRegexes, testSafeRegex } from "./safe-regex.js";
+
+const MAX_REGEX_PATTERN_CHARS = 2_000;
+const MAX_REGEX_TEXT_CHARS = 1024 * 1024;
+
+// Skip `responded` events: they carry the answerer's reply, not agent output,
+// so sampling them would grade the persona's words instead of the agent's.
+const agentMessageText = (e: RunEvent | undefined): string | undefined => {
+  if (e?.type === "progress" && e.kind === "agent-message") {
+    const payload = e.payload as { text?: unknown };
+    if (typeof payload?.text === "string") return payload.text;
+  }
+  return undefined;
+};
+
+const finalAssistantText = (events: RunEvent[]): string => {
+  for (let i = events.length - 1; i >= 0; i--) {
+    const text = agentMessageText(events[i]);
+    if (text !== undefined) return text;
+  }
+  return "";
+};
+
+const allAssistantText = (events: RunEvent[]): string => {
+  const parts: string[] = [];
+  for (const e of events) {
+    const text = agentMessageText(e);
+    if (text !== undefined) parts.push(text);
+  }
+  return parts.join("\n");
+};
+
+const TARGETS = ["transcript-final", "transcript-all"] as const;
+const targetSchema = z.enum(TARGETS).default("transcript-final");
+
+const sample = (run: Run, against: (typeof TARGETS)[number]): string =>
+  against === "transcript-all" ? allAssistantText(run.events) : finalAssistantText(run.events);
+
+// Empty text vacuously passes contains (and vacuously fails not-contains);
+// reject it at authoring time like the other no-op configs.
+export const ContainsConfigSchema = z
+  .object({
+    type: z.literal("contains"),
+    text: z.string().min(1),
+    against: targetSchema,
+    ...baseFields,
+  })
+  .strict();
+export type ContainsConfig = z.infer<typeof ContainsConfigSchema>;
+
+export const containsAssertion: AssertionImpl<ContainsConfig> = {
+  type: "contains",
+  configSchema: ContainsConfigSchema,
+  evaluate: async (config, run) => {
+    const actual = sample(run, config.against);
+    const passed = actual.includes(config.text);
+    return {
+      passed,
+      score: passed ? 1 : 0,
+      comment: passed ? `contains '${config.text}'` : `does not contain '${config.text}'`,
+    };
+  },
+};
+
+export const NotContainsConfigSchema = z
+  .object({
+    type: z.literal("not-contains"),
+    text: z.string().min(1),
+    against: targetSchema,
+    ...baseFields,
+  })
+  .strict();
+export type NotContainsConfig = z.infer<typeof NotContainsConfigSchema>;
+
+export const notContainsAssertion: AssertionImpl<NotContainsConfig> = {
+  type: "not-contains",
+  configSchema: NotContainsConfigSchema,
+  evaluate: async (config, run) => {
+    const actual = sample(run, config.against);
+    const passed = !actual.includes(config.text);
+    return {
+      passed,
+      score: passed ? 1 : 0,
+      comment: passed
+        ? `does not contain '${config.text}'`
+        : `unexpectedly contains '${config.text}'`,
+    };
+  },
+};
+
+export const RegexConfigSchema = z
+  .object({
+    type: z.literal("regex"),
+    pattern: z.string().max(MAX_REGEX_PATTERN_CHARS),
+    flags: z.string().max(16).optional(),
+    against: targetSchema,
+    ...baseFields,
+  })
+  .strict()
+  .superRefine(async (config, ctx) => {
+    const [inspection] = await inspectSafeRegexes([config.pattern], config.flags ?? "");
+    if (inspection && "error" in inspection) {
+      ctx.addIssue({ code: "custom", path: ["pattern"], message: inspection.error });
+    }
+  });
+export type RegexConfig = z.infer<typeof RegexConfigSchema>;
+
+export const regexAssertion: AssertionImpl<RegexConfig> = {
+  type: "regex",
+  configSchema: RegexConfigSchema,
+  evaluate: async (config, run, ctx) => {
+    const actual = sample(run, config.against);
+    if (actual.length > MAX_REGEX_TEXT_CHARS) {
+      return {
+        passed: false,
+        score: 0,
+        comment: `transcript exceeds the ${MAX_REGEX_TEXT_CHARS}-character regex assertion limit`,
+      };
+    }
+    const passed = await testSafeRegex(config.pattern, config.flags ?? "", actual, {
+      abortSignal: ctx?.abortSignal,
+    });
+    return {
+      passed,
+      score: passed ? 1 : 0,
+      comment: passed
+        ? `matches /${config.pattern}/${config.flags ?? ""}`
+        : `does not match /${config.pattern}/${config.flags ?? ""}`,
+    };
+  },
+};

package/src/assertions/core/predicate.ts

@@ -0,0 +1,67 @@
+import { z } from "zod";
+import {
+  type AssertionImpl,
+  type AssertionResult,
+  AssertionResultSchema,
+  type Run,
+} from "../../core/index.js";
+import { baseAssertionFields } from "./helpers.js";
+
+export type PredicateFn = (
+  run: Run,
+) => boolean | AssertionResult | Promise<boolean | AssertionResult>;
+
+export const PredicateConfigSchema = z
+  .object({
+    type: z.literal("predicate"),
+    fn: z.custom<PredicateFn>((v) => typeof v === "function"),
+    ...baseAssertionFields,
+  })
+  .strict();
+export type PredicateConfig = z.infer<typeof PredicateConfigSchema>;
+
+const isAssertionResult = (v: unknown): v is AssertionResult =>
+  AssertionResultSchema.safeParse(v).success;
+
+export const predicateAssertion: AssertionImpl<PredicateConfig> = {
+  type: "predicate",
+  configSchema: PredicateConfigSchema,
+  evaluate: async (config, run) => {
+    if (typeof config.fn !== "function") {
+      const errorMessage = "predicate.fn is not a function";
+      return {
+        passed: false,
+        score: 0,
+        comment: errorMessage,
+        errorMessage,
+      };
+    }
+    try {
+      const result = await config.fn(run);
+      if (isAssertionResult(result)) return result;
+      if (typeof result !== "boolean") {
+        const errorMessage =
+          "predicate returned an invalid result; expected boolean or AssertionResult";
+        return {
+          passed: false,
+          score: 0,
+          comment: errorMessage,
+          errorMessage,
+        };
+      }
+      return {
+        passed: result,
+        score: result ? 1 : 0,
+        comment: result ? "predicate returned true" : "predicate returned false",
+      };
+    } catch (err) {
+      const errorMessage = `predicate threw: ${err instanceof Error ? err.message : String(err)}`;
+      return {
+        passed: false,
+        score: 0,
+        comment: errorMessage,
+        errorMessage,
+      };
+    }
+  },
+};

package/src/assertions/core/resource-checks.ts

@@ -0,0 +1,180 @@
+import { z } from "zod";
+import { type AssertionImpl, inspectRunArtifact, isUsageEmpty } from "../../core/index.js";
+import { baseAssertionFields as baseFields } from "./helpers.js";
+
+const UnavailablePolicySchema = z.enum(["pass", "fail"]);
+
+const unavailable = (resource: string, policy: z.infer<typeof UnavailablePolicySchema>) => ({
+  passed: policy === "pass",
+  score: policy === "pass" ? 1 : 0,
+  comment: `${resource} unavailable`,
+  detail: { unavailable: true },
+});
+
+export const LatencyUnderConfigSchema = z
+  .object({
+    type: z.literal("latency-under"),
+    maxMs: z.number().positive(),
+    ...baseFields,
+  })
+  .strict();
+export type LatencyUnderConfig = z.infer<typeof LatencyUnderConfigSchema>;
+
+export const latencyUnderAssertion: AssertionImpl<LatencyUnderConfig> = {
+  type: "latency-under",
+  configSchema: LatencyUnderConfigSchema,
+  evaluate: async (config, run) => {
+    const start = Date.parse(run.metadata.startedAt);
+    const end = Date.parse(run.metadata.endedAt);
+    if (Number.isNaN(start) || Number.isNaN(end)) {
+      return {
+        passed: false,
+        score: 0,
+        comment: "duration unavailable (missing timestamps)",
+      };
+    }
+    const ms = end - start;
+    const passed = ms <= config.maxMs;
+    return {
+      passed,
+      score: passed ? 1 : 0,
+      comment: `${ms}ms ${passed ? "<=" : ">"} ${config.maxMs}ms`,
+      detail: { durationMs: ms },
+    };
+  },
+};
+
+export const TokensUnderConfigSchema = z
+  .object({
+    type: z.literal("tokens-under"),
+    max: z.number().positive(),
+    /**
+     * `total` is uncached input + output, the historical meaning. Cache traffic
+     * is counted separately because it is billed differently and, on a warm
+     * session, dwarfs `input` by two orders of magnitude.
+     */
+    component: z.enum(["input", "output", "total", "cacheRead", "cacheCreation"]).default("total"),
+    /** Compatibility defaults to pass; optimization campaigns should require fail. */
+    onUnavailable: UnavailablePolicySchema.optional(),
+    ...baseFields,
+  })
+  .strict();
+export type TokensUnderConfig = z.infer<typeof TokensUnderConfigSchema>;
+
+// Grades the agent under test, never the harness: an eval must not fail
+// because the persona answerer or the LLM judge got chatty.
+export const tokensUnderAssertion: AssertionImpl<TokensUnderConfig> = {
+  type: "tokens-under",
+  configSchema: TokensUnderConfigSchema,
+  evaluate: async (config, run) => {
+    const agent = run.metadata.usage.agent;
+    // A driver that reports nothing arrives here as all-zeros. Treat that as
+    // "unknown", not as "the agent spent zero tokens" — no real agent spends
+    // zero.
+    if (isUsageEmpty(agent)) {
+      return unavailable("tokens (driver did not report)", config.onUnavailable ?? "pass");
+    }
+    const value =
+      config.component === "total" ? agent.input + agent.output : agent[config.component];
+    const passed = value <= config.max;
+    return {
+      passed,
+      score: passed ? 1 : 0,
+      comment: `${config.component} tokens=${value} ${passed ? "<=" : ">"} ${config.max}`,
+      detail: { tokens: value },
+    };
+  },
+};
+
+export const CostUnderConfigSchema = z
+  .object({
+    type: z.literal("cost-under"),
+    maxUsd: z.number().positive(),
+    onUnavailable: UnavailablePolicySchema.optional(),
+    ...baseFields,
+  })
+  .strict();
+export type CostUnderConfig = z.infer<typeof CostUnderConfigSchema>;
+
+export const costUnderAssertion: AssertionImpl<CostUnderConfig> = {
+  type: "cost-under",
+  configSchema: CostUnderConfigSchema,
+  evaluate: async (config, run) => {
+    const usd = run.metadata.cost?.usd;
+    if (usd === undefined) {
+      return unavailable("cost (driver did not report)", config.onUnavailable ?? "pass");
+    }
+    const passed = usd <= config.maxUsd;
+    return {
+      passed,
+      score: passed ? 1 : 0,
+      comment: `$${usd.toFixed(4)} ${passed ? "<=" : ">"} $${config.maxUsd.toFixed(4)}`,
+      detail: { usd },
+    };
+  },
+};
+
+export const ArtifactBytesUnderConfigSchema = z
+  .object({
+    type: z.literal("artifact-bytes-under"),
+    max: z.number().int().nonnegative(),
+    onUnavailable: UnavailablePolicySchema.optional(),
+    ...baseFields,
+  })
+  .strict();
+export type ArtifactBytesUnderConfig = z.infer<typeof ArtifactBytesUnderConfigSchema>;
+
+export const artifactBytesUnderAssertion: AssertionImpl<ArtifactBytesUnderConfig> = {
+  type: "artifact-bytes-under",
+  configSchema: ArtifactBytesUnderConfigSchema,
+  evaluate: async (config, run) => {
+    const sizes = await Promise.all(
+      run.artifacts.map(async (artifact) => {
+        if (artifact.size !== undefined) return artifact.size;
+        const inspected = await inspectRunArtifact(run, artifact.path);
+        return inspected.stats?.isFile() ? inspected.stats.size : null;
+      }),
+    );
+    if (sizes.some((size) => size === null)) {
+      return unavailable(
+        "artifact bytes (one or more files could not be measured)",
+        config.onUnavailable ?? "pass",
+      );
+    }
+    const bytes = sizes.reduce<number>((sum, size) => sum + (size ?? 0), 0);
+    const passed = bytes <= config.max;
+    return {
+      passed,
+      score: passed ? 1 : 0,
+      comment: `artifact bytes=${bytes} ${passed ? "<=" : ">"} ${config.max}`,
+      detail: { bytes, files: sizes.length },
+    };
+  },
+};
+
+// A crashed or truncated run can still satisfy structural checks (a
+// `file-exists` staged by a fixture, say). This lets a case require the
+// agent to have finished cleanly as a first-class, graded expectation —
+// complementing the automatic `casePassed` downgrade the orchestrator does.
+export const RunSucceededConfigSchema = z
+  .object({
+    type: z.literal("run-succeeded"),
+    ...baseFields,
+  })
+  .strict();
+export type RunSucceededConfig = z.infer<typeof RunSucceededConfigSchema>;
+
+export const runSucceededAssertion: AssertionImpl<RunSucceededConfig> = {
+  type: "run-succeeded",
+  configSchema: RunSucceededConfigSchema,
+  evaluate: async (_config, run) => {
+    const { exitReason } = run.metadata;
+    const passed = exitReason === "done";
+    return {
+      passed,
+      score: passed ? 1 : 0,
+      comment: passed ? "run completed cleanly" : `run ended with '${exitReason}'`,
+      detail: { exitReason },
+    };
+  },
+};

package/src/assertions/core/safe-regex.ts

@@ -0,0 +1,196 @@
+import { Worker } from "node:worker_threads";
+
+const REGEX_TIMEOUT_MS = 2_000;
+
+// An eval worker keeps source tests and the compiled package on the same
+// loader-independent path while still making pathological V8 regexes killable.
+const REGEX_WORKER_SOURCE = `
+  const { parentPort, workerData } = require("node:worker_threads");
+
+  const evaluate = () => {
+    if (workerData.operation === "test") {
+      return new RegExp(workerData.pattern, workerData.flags).test(workerData.text);
+    }
+    if (workerData.operation === "set") {
+      return workerData.patterns.map(
+        (pattern) => new RegExp(pattern, workerData.flags).test(workerData.text),
+      );
+    }
+    if (workerData.operation === "inspect") {
+      return workerData.patterns.map((pattern) => {
+        try {
+          return { matchesEmpty: new RegExp(pattern, workerData.flags).test("") };
+        } catch (error) {
+          return {
+            error: error instanceof Error ? error.message : String(error),
+          };
+        }
+      });
+    }
+    if (workerData.operation === "groups") {
+      return workerData.groups.map(({ pattern, flags, texts }) => {
+        const expression = new RegExp(pattern, flags);
+        return texts.map((text) => {
+          expression.lastIndex = 0;
+          return expression.test(text);
+        });
+      });
+    }
+    const flags = workerData.flags.includes("g")
+      ? workerData.flags
+      : workerData.flags + "g";
+    const expression = new RegExp(workerData.pattern, flags);
+    let count = 0;
+    for (const _match of workerData.text.matchAll(expression)) {
+      count += 1;
+      if (count >= workerData.stopAfter) return { count, exact: false };
+    }
+    return { count, exact: true };
+  };
+
+  try {
+    parentPort.postMessage({ ok: true, value: evaluate() });
+  } catch (error) {
+    parentPort.postMessage({
+      ok: false,
+      errorName: error instanceof Error ? error.name : "Error",
+      error: error instanceof Error ? error.message : String(error),
+    });
+  }
+`;
+
+type RegexTask =
+  | {
+      operation: "test";
+      pattern: string;
+      flags: string;
+      text: string;
+    }
+  | {
+      operation: "set";
+      patterns: string[];
+      flags: string;
+      text: string;
+    }
+  | {
+      operation: "count";
+      pattern: string;
+      flags: string;
+      text: string;
+      stopAfter: number;
+    }
+  | {
+      operation: "inspect";
+      patterns: string[];
+      flags: string;
+    }
+  | {
+      operation: "groups";
+      groups: Array<{
+        pattern: string;
+        flags: string;
+        texts: string[];
+      }>;
+    };
+
+type WorkerResponse<T> = { ok: true; value: T } | { ok: false; errorName: string; error: string };
+
+export type RegexEvaluationOptions = {
+  abortSignal?: AbortSignal | undefined;
+  timeoutMs?: number;
+};
+
+export type SafeRegexInspection = { matchesEmpty: boolean } | { error: string };
+
+export type SafeRegexGroup = {
+  pattern: string;
+  flags: string;
+  texts: string[];
+};
+
+const evaluateRegex = async <T>(
+  task: RegexTask,
+  options: RegexEvaluationOptions = {},
+): Promise<T> =>
+  await new Promise<T>((resolve, reject) => {
+    const worker = new Worker(REGEX_WORKER_SOURCE, {
+      eval: true,
+      workerData: task,
+    });
+    let settled = false;
+    const finish = (settle: () => void) => {
+      if (settled) return;
+      settled = true;
+      clearTimeout(timeout);
+      options.abortSignal?.removeEventListener("abort", abort);
+      settle();
+      void worker.terminate();
+    };
+    const abort = () =>
+      finish(() => reject(options.abortSignal?.reason ?? new Error("regex evaluation aborted")));
+    const timeout = setTimeout(
+      () =>
+        finish(() =>
+          reject(new Error(`regex evaluation exceeded ${options.timeoutMs ?? REGEX_TIMEOUT_MS}ms`)),
+        ),
+      options.timeoutMs ?? REGEX_TIMEOUT_MS,
+    );
+    worker.once("message", (response: WorkerResponse<T>) => {
+      finish(() => {
+        if (response.ok) resolve(response.value);
+        else {
+          const ErrorConstructor = response.errorName === "SyntaxError" ? SyntaxError : Error;
+          reject(new ErrorConstructor(`regex evaluation failed: ${response.error}`));
+        }
+      });
+    });
+    worker.once("error", (error) => finish(() => reject(error)));
+    worker.once("exit", (code) => {
+      if (code !== 0 && !settled) {
+        finish(() => reject(new Error(`regex worker exited with code ${code}`)));
+      }
+    });
+    if (options.abortSignal?.aborted) abort();
+    else options.abortSignal?.addEventListener("abort", abort, { once: true });
+  });
+
+export const testSafeRegex = async (
+  pattern: string,
+  flags: string,
+  text: string,
+  options?: RegexEvaluationOptions,
+): Promise<boolean> =>
+  await evaluateRegex<boolean>({ operation: "test", pattern, flags, text }, options);
+
+export const testSafeRegexSet = async (
+  patterns: string[],
+  flags: string,
+  text: string,
+  options?: RegexEvaluationOptions,
+): Promise<boolean[]> =>
+  await evaluateRegex<boolean[]>({ operation: "set", patterns, flags, text }, options);
+
+export const inspectSafeRegexes = async (
+  patterns: string[],
+  flags: string,
+  options?: RegexEvaluationOptions,
+): Promise<SafeRegexInspection[]> =>
+  await evaluateRegex<SafeRegexInspection[]>({ operation: "inspect", patterns, flags }, options);
+
+export const testSafeRegexGroups = async (
+  groups: SafeRegexGroup[],
+  options?: RegexEvaluationOptions,
+): Promise<boolean[][]> =>
+  await evaluateRegex<boolean[][]>({ operation: "groups", groups }, options);
+
+export const countSafeRegex = async (
+  pattern: string,
+  flags: string,
+  text: string,
+  stopAfter: number,
+  options?: RegexEvaluationOptions,
+): Promise<{ count: number; exact: boolean }> =>
+  await evaluateRegex<{ count: number; exact: boolean }>(
+    { operation: "count", pattern, flags, text, stopAfter },
+    options,
+  );

package/src/assertions/core/tool-called-with.ts

@@ -0,0 +1,68 @@
+import { z } from "zod";
+import type { AssertionImpl } from "../../core/index.js";
+import { matchesAgentScope } from "./event-checks.js";
+import { deepEquals, isToolCallEvent, resolvePointer, weightThresholdFields } from "./helpers.js";
+
+export const ToolCalledWithConfigSchema = z
+  .object({
+    type: z.literal("tool-called-with"),
+    name: z.string(),
+    /** RFC 6901 JSON pointer into the tool's input. Defaults to "" (whole input). */
+    argsJsonPointer: z.string().default(""),
+    /** Structural (key-order-insensitive) equality against the value at the JSON pointer. */
+    argsValue: z.unknown().optional(),
+    /** Substring containment check (stringifies the value at the pointer). */
+    argsContains: z.string().optional(),
+    /** How many matching calls are required. Default 1. `minTimes: 0` would pass vacuously. */
+    minTimes: z.number().int().positive().optional(),
+    /** Whose calls count: "main" agent, "subagent", or "any". */
+    by: z.enum(["any", "main", "subagent"]).default("any"),
+    ...weightThresholdFields,
+  })
+  .strict()
+  .refine((c) => c.argsValue === undefined || c.argsContains === undefined, {
+    message: "set only one of argsValue or argsContains",
+  });
+export type ToolCalledWithConfig = z.infer<typeof ToolCalledWithConfigSchema>;
+
+const matches = (value: unknown, config: ToolCalledWithConfig): boolean => {
+  if (config.argsValue !== undefined) {
+    return deepEquals(value, config.argsValue);
+  }
+  if (config.argsContains !== undefined) {
+    // An unresolved pointer is a non-match, not an error — a later event may
+    // still genuinely match.
+    if (value === undefined) return false;
+    const stringified = typeof value === "string" ? value : JSON.stringify(value);
+    return stringified.includes(config.argsContains);
+  }
+  // Both unset — match any tool call with that name.
+  return true;
+};
+
+export const toolCalledWithAssertion: AssertionImpl<ToolCalledWithConfig> = {
+  type: "tool-called-with",
+  configSchema: ToolCalledWithConfigSchema,
+  evaluate: async (config, run) => {
+    const min = config.minTimes ?? 1;
+    let hits = 0;
+    for (const e of run.events) {
+      if (!isToolCallEvent(e)) continue;
+      if (!matchesAgentScope(e, config.by)) continue;
+      const payload = e.payload as { name?: unknown; input?: unknown };
+      if (payload?.name !== config.name) continue;
+      const argsRoot = payload.input ?? null;
+      const value = resolvePointer(argsRoot, config.argsJsonPointer);
+      if (matches(value, config)) hits++;
+    }
+    const passed = hits >= min;
+    const bound = passed ? `>= ${min}` : `need ${min}`;
+    const scope = (config.by ?? "any") === "any" ? "" : ` by ${config.by}`;
+    return {
+      passed,
+      score: passed ? 1 : 0,
+      comment: `tool '${config.name}' called ${hits}×${scope} with matching args (${bound})`,
+      detail: { hits },
+    };
+  },
+};

package/src/assertions/core/tool-calls.ts

@@ -0,0 +1,28 @@
+import type { Run } from "../../core/index.js";
+import { type AgentScope, matchesAgentScope } from "./event-checks.js";
+import { isToolCallEvent } from "./helpers.js";
+
+/**
+ * A tool call projected from a `tool-call` progress event's payload.
+ * `parentToolUseId` is set only when a spawned subagent made the call.
+ */
+export type ToolCall = { name: string; input: unknown; parentToolUseId?: string };
+
+/**
+ * Every tool call the run made, in order. Reach for it inside a `predicate`
+ * assertion to grade what the agent *did*, not what it said. Pass `by` to scope
+ * to the "main" agent or a "subagent" (default "any").
+ */
+export const toolCalls = (run: Run, by: AgentScope = "any"): ToolCall[] =>
+  run.events
+    .filter(isToolCallEvent)
+    .filter((e) => matchesAgentScope(e, by))
+    .map((e) => e.payload as ToolCall)
+    .filter((p): p is ToolCall => typeof p?.name === "string");
+
+/** Stringified inputs of every tool call, for substring checks in a predicate. */
+export const toolCallInputs = (run: Run, by: AgentScope = "any"): string[] =>
+  toolCalls(run, by).map((c) => JSON.stringify(c.input ?? {}));
+
+/** Tool calls made by a spawned subagent (the driver tags these with `parentToolUseId`). */
+export const subagentToolCalls = (run: Run): ToolCall[] => toolCalls(run, "subagent");