@prismatic-io/lux 0.0.1

package/src/answerers/scripted/index.ts

@@ -0,0 +1,186 @@
+import { readFile } from "node:fs/promises";
+import { resolve } from "node:path";
+import { load as loadYaml } from "js-yaml";
+import { z } from "zod";
+import {
+  type Answer,
+  type Answerer,
+  type AnswererPlugin,
+  formatZodIssues,
+  type Interrupt,
+  InterruptKindSchema,
+} from "../../core/index.js";
+import { approveAnswer } from "../shared.js";
+
+const MatchSchema = z
+  .object({
+    kind: InterruptKindSchema.optional(),
+    id: z.string().optional(),
+    question_contains: z.string().optional(),
+    question_matches: z.string().optional(),
+    request_contains: z.string().optional(),
+  })
+  .strict();
+
+const AnsweredAskSchema = z.object({ answer: z.string() }).strict();
+const AnsweredApproveSchema = z
+  .object({ approved: z.boolean(), reason: z.string().optional() })
+  .strict();
+const AnsweredCustomSchema = z.object({ payload: z.unknown() }).strict();
+
+const RuleSchema = z.union([
+  z.object({ match: MatchSchema, answer: z.string() }).strict(),
+  z.object({ match: MatchSchema, approved: z.boolean(), reason: z.string().optional() }).strict(),
+  z.object({ match: MatchSchema, payload: z.unknown() }).strict(),
+  z.object({ default: AnsweredAskSchema }).strict(),
+  z.object({ default: AnsweredApproveSchema }).strict(),
+  z.object({ default: AnsweredCustomSchema }).strict(),
+]);
+
+const FixtureSchema = z.array(RuleSchema);
+
+export const ScriptedAnswererConfigSchema = z.object({ fixturePath: z.string() }).strict();
+
+export type ScriptedAnswererConfig = z.infer<typeof ScriptedAnswererConfigSchema>;
+
+type Rule = z.infer<typeof RuleSchema>;
+
+/** A rule vetted at load time: kind/body cross-checked, regex pre-compiled. */
+type PreparedRule = { rule: Rule; questionPattern: RegExp | undefined };
+
+const ruleLabel = (path: string, index: number): string => `rule ${index + 1} of ${path}`;
+
+const bodyKind = (rule: Rule): Interrupt["kind"] => {
+  const body = "default" in rule ? rule.default : rule;
+  if ("answer" in body) return "ask";
+  if ("approved" in body) return "approve";
+  return "custom";
+};
+
+const matchableKind = (m: z.infer<typeof MatchSchema>): Interrupt["kind"] | undefined => {
+  if (m.kind) return m.kind;
+  if (m.question_contains !== undefined || m.question_matches !== undefined) return "ask";
+  if (m.request_contains !== undefined) return "approve";
+  return undefined;
+};
+
+const compileQuestionPattern = (rule: Rule, path: string, index: number): RegExp | undefined => {
+  if (!("match" in rule) || rule.match.question_matches === undefined) return undefined;
+  try {
+    return new RegExp(rule.match.question_matches);
+  } catch (err) {
+    throw new Error(
+      `scripted: invalid question_matches in ${ruleLabel(path, index)}: ${
+        err instanceof Error ? err.message : String(err)
+      }`,
+    );
+  }
+};
+
+// The rule's match fields and answer body are independently valid shapes, so
+// their interrupt kinds still need a semantic cross-check.
+const prepareRule = (rule: Rule, path: string, index: number): PreparedRule => {
+  if ("match" in rule) {
+    const wanted = matchableKind(rule.match);
+    const answered = bodyKind(rule);
+    if (wanted !== undefined && wanted !== answered) {
+      throw new Error(
+        `scripted: ${ruleLabel(path, index)} matches ${wanted} interrupts but its body answers ${answered}`,
+      );
+    }
+  }
+  return { rule, questionPattern: compileQuestionPattern(rule, path, index) };
+};
+
+const matches = ({ rule, questionPattern }: PreparedRule, interrupt: Interrupt): boolean => {
+  if (!("match" in rule)) return false;
+  const m = rule.match;
+  if (m.kind && m.kind !== interrupt.kind) return false;
+  if (m.id && m.id !== interrupt.id) return false;
+  if (m.question_contains) {
+    if (interrupt.kind !== "ask") return false;
+    if (!interrupt.question.includes(m.question_contains)) return false;
+  }
+  if (questionPattern) {
+    if (interrupt.kind !== "ask") return false;
+    if (!questionPattern.test(interrupt.question)) return false;
+  }
+  if (m.request_contains) {
+    if (interrupt.kind !== "approve") return false;
+    if (!interrupt.request.includes(m.request_contains)) return false;
+  }
+  return true;
+};
+
+// A fixture may carry one default per interrupt kind; the shape of the
+// default's body is what says which kind it can answer.
+const defaultAnswersKind = (rule: Rule, kind: Interrupt["kind"]): boolean => {
+  if (!("default" in rule)) return false;
+  const body = rule.default;
+  if (kind === "ask") return "answer" in body;
+  if (kind === "approve") return "approved" in body;
+  return "payload" in body;
+};
+
+const buildAnswer = (rule: Rule, interrupt: Interrupt): Answer => {
+  const body = "default" in rule ? rule.default : rule;
+  if (interrupt.kind === "ask") {
+    if (!("answer" in body)) {
+      throw new Error(`scripted: rule for ask interrupt missing 'answer' (id=${interrupt.id})`);
+    }
+    return { kind: "ask", id: interrupt.id, text: body.answer };
+  }
+  if (interrupt.kind === "approve") {
+    if (!("approved" in body)) {
+      throw new Error(
+        `scripted: rule for approve interrupt missing 'approved' (id=${interrupt.id})`,
+      );
+    }
+    return approveAnswer(interrupt.id, body.approved, body.reason);
+  }
+  if (!("payload" in body)) {
+    throw new Error(`scripted: rule for custom interrupt missing 'payload' (id=${interrupt.id})`);
+  }
+  return { kind: "custom", id: interrupt.id, payload: body.payload };
+};
+
+class ScriptedAnswererImpl implements Answerer {
+  private rules: PreparedRule[] = [];
+  private readonly config: ScriptedAnswererConfig;
+
+  constructor(config: ScriptedAnswererConfig) {
+    this.config = config;
+  }
+
+  async prepare(): Promise<void> {
+    const path = resolve(this.config.fixturePath);
+    const raw = loadYaml(await readFile(path, "utf-8"));
+    const parsed = FixtureSchema.safeParse(raw);
+    if (!parsed.success) {
+      throw new Error(`scripted: invalid fixture at ${path}: ${formatZodIssues(parsed.error)}`);
+    }
+    this.rules = parsed.data.map((rule, index) => prepareRule(rule, path, index));
+  }
+
+  async answer(interrupt: Interrupt): Promise<Answer> {
+    const matched = this.rules.find((r) => matches(r, interrupt));
+    if (matched) {
+      return buildAnswer(matched.rule, interrupt);
+    }
+    const fallback = this.rules.find((r) => defaultAnswersKind(r.rule, interrupt.kind));
+    if (fallback) {
+      return buildAnswer(fallback.rule, interrupt);
+    }
+    throw new Error(
+      `scripted: no rule matched interrupt (kind=${interrupt.kind}, id=${interrupt.id}) and no default provided`,
+    );
+  }
+
+  async close(): Promise<void> {}
+}
+
+export const scriptedAnswerer: AnswererPlugin<ScriptedAnswererConfig> = {
+  name: "scripted",
+  configSchema: ScriptedAnswererConfigSchema,
+  create: (config) => new ScriptedAnswererImpl(config),
+};

package/src/answerers/shared.ts

@@ -0,0 +1,23 @@
+import { z } from "zod";
+import type { Answer } from "../core/index.js";
+
+/**
+ * The approve-reply wire shape shared by every answerer. Lenient on
+ * `reason`: replies come from an LLM or hand-written JSON, and a malformed
+ * reason must not invalidate a well-formed approval decision.
+ */
+export const ApproveReplySchema = z.object({
+  approved: z.boolean(),
+  reason: z.string().optional().catch(undefined),
+});
+
+export const approveAnswer = (
+  id: string,
+  approved: boolean,
+  reason?: string | undefined,
+): Answer => ({
+  kind: "approve",
+  id,
+  approved,
+  ...(reason !== undefined ? { reason } : {}),
+});

package/src/answerers/terminal/index.ts

@@ -0,0 +1,117 @@
+import { stdin, stdout } from "node:process";
+import { createInterface, type Interface } from "node:readline/promises";
+import { z } from "zod";
+import type {
+  Answer,
+  Answerer,
+  AnswererContext,
+  AnswererPlugin,
+  Interrupt,
+} from "../../core/index.js";
+import { approveAnswer } from "../shared.js";
+
+export const TerminalAnswererConfigSchema = z
+  .object({
+    maxAnswerWaitMs: z.number().positive().default(600_000),
+  })
+  .strict();
+
+export type TerminalAnswererConfig = z.infer<typeof TerminalAnswererConfigSchema>;
+
+const contextSuffix = (context: unknown): string => {
+  if (context === undefined) return "";
+  return `\nContext: ${JSON.stringify(context, null, 2)}`;
+};
+
+const promptFor = (interrupt: Interrupt): string => {
+  if (interrupt.kind === "ask") {
+    return `\nAgent question: ${interrupt.question}${contextSuffix(interrupt.context)}\nAnswer: `;
+  }
+  if (interrupt.kind === "approve") {
+    return `\nAgent approval request: ${interrupt.request}${contextSuffix(interrupt.context)}\nApprove? [y/N]: `;
+  }
+  return `\nAgent requested custom input: ${JSON.stringify(interrupt.payload, null, 2)}${contextSuffix(interrupt.context)}\nJSON response: `;
+};
+
+export const parseTerminalAnswer = (line: string, interrupt: Interrupt): Answer => {
+  if (interrupt.kind === "ask") {
+    return { kind: "ask", id: interrupt.id, text: line };
+  }
+  if (interrupt.kind === "approve") {
+    const normalized = line.trim().toLowerCase();
+    if (normalized === "y" || normalized === "yes") {
+      return approveAnswer(interrupt.id, true);
+    }
+    if (normalized === "" || normalized === "n" || normalized === "no") {
+      return approveAnswer(interrupt.id, false);
+    }
+    throw new Error("terminal answerer: approval must be yes/y or no/n");
+  }
+  try {
+    return { kind: "custom", id: interrupt.id, payload: JSON.parse(line) };
+  } catch (error) {
+    throw new Error(
+      `terminal answerer: custom response must be JSON: ${
+        error instanceof Error ? error.message : String(error)
+      }`,
+    );
+  }
+};
+
+class TerminalAnswererImpl implements Answerer {
+  private interface: Interface | null = null;
+  private readonly maxAnswerWaitMs: number;
+
+  constructor(config: TerminalAnswererConfig) {
+    this.maxAnswerWaitMs = config.maxAnswerWaitMs;
+  }
+
+  async prepare(): Promise<void> {
+    if (!stdin.isTTY || !stdout.isTTY) {
+      throw new Error(
+        "terminal answerer: --interactive requires an attached terminal; " +
+          "use a scripted/model answerer in CI or --claude-answerer from the Claude plugin",
+      );
+    }
+    this.interface = createInterface({ input: stdin, output: stdout, terminal: true });
+  }
+
+  async answer(interrupt: Interrupt, ctx: AnswererContext): Promise<Answer> {
+    if (!this.interface) throw new Error("terminal answerer: not prepared");
+    const controller = new AbortController();
+    const abort = (): void => controller.abort();
+    const timeout = setTimeout(abort, this.maxAnswerWaitMs);
+    if (ctx.abortSignal?.aborted) abort();
+    else ctx.abortSignal?.addEventListener("abort", abort, { once: true });
+    try {
+      const line = await this.interface.question(promptFor(interrupt), {
+        signal: controller.signal,
+      });
+      return parseTerminalAnswer(line, interrupt);
+    } catch (error) {
+      if (controller.signal.aborted) {
+        throw new Error(
+          ctx.abortSignal?.aborted
+            ? "terminal answerer: run aborted before an answer arrived"
+            : `terminal answerer: timed out after ${this.maxAnswerWaitMs}ms`,
+          { cause: error },
+        );
+      }
+      throw error;
+    } finally {
+      clearTimeout(timeout);
+      ctx.abortSignal?.removeEventListener("abort", abort);
+    }
+  }
+
+  async close(): Promise<void> {
+    this.interface?.close();
+    this.interface = null;
+  }
+}
+
+export const terminalAnswerer: AnswererPlugin<TerminalAnswererConfig> = {
+  name: "terminal",
+  configSchema: TerminalAnswererConfigSchema,
+  create: (config) => new TerminalAnswererImpl(config),
+};

package/src/assertions/core/command-exits-zero.ts

@@ -0,0 +1,94 @@
+import { resolve } from "node:path";
+import { z } from "zod";
+import { type AssertionImpl, exec, hasRunArtifactEvidenceBoundary } from "../../core/index.js";
+import {
+  baseAssertionFields,
+  commandEnvironment,
+  inspectAssertionPath,
+  LocationSchema,
+  resolveLocationRoot,
+} from "./helpers.js";
+
+export const CommandExitsZeroConfigSchema = z
+  .object({
+    type: z.literal("command-exits-zero"),
+    command: z.string(),
+    cwd: z.string().optional(),
+    in: LocationSchema,
+    env: z.record(z.string(), z.string()).optional(),
+    /**
+     * Parent variables to add to the minimal toolchain environment. Set to
+     * `true` only when the command deliberately needs the entire Lux process
+     * environment, including credentials.
+     */
+    inheritEnv: z.union([z.array(z.string().min(1)), z.literal(true)]).default([]),
+    timeoutMs: z.number().positive().default(120_000),
+    ...baseAssertionFields,
+  })
+  .strict();
+export type CommandExitsZeroConfig = z.infer<typeof CommandExitsZeroConfigSchema>;
+
+export const commandExitsZeroAssertion: AssertionImpl<CommandExitsZeroConfig> = {
+  type: "command-exits-zero",
+  configSchema: CommandExitsZeroConfigSchema,
+  evaluate: async (config, run, ctx) => {
+    let root = resolveLocationRoot(run, config.in);
+    if (config.cwd) {
+      if (hasRunArtifactEvidenceBoundary(run)) {
+        const inspected = await inspectAssertionPath(run, "cwd", config.cwd);
+        if (!inspected.stats?.isDirectory()) {
+          const comment =
+            inspected.issue ??
+            (inspected.stats ? `${config.cwd} is not a directory` : `${config.cwd} not found`);
+          return { passed: false, score: 0, comment };
+        }
+        root = inspected.target;
+      } else {
+        root = resolve(run.runDir, config.cwd);
+      }
+    }
+    const parentEnv = commandEnvironment(process.env, config.inheritEnv);
+    // The whole command line is one shell string with no interpolation, so
+    // `shell: true` is safe here; exec() gives it a real process-group
+    // timeout and an output cap the old hand-rolled spawn lacked.
+    let result: Awaited<ReturnType<typeof exec>>;
+    try {
+      result = await exec(config.command, [], {
+        cwd: root,
+        env: { ...parentEnv, ...config.env },
+        shell: true,
+        timeoutMs: config.timeoutMs,
+        maxBuffer: 4_000,
+        ...(ctx?.abortSignal ? { signal: ctx.abortSignal } : {}),
+      });
+    } catch (err) {
+      const message = err instanceof Error ? err.message : String(err);
+      return {
+        passed: false,
+        score: 0,
+        comment: message,
+        errorMessage: message,
+        detail: { error: message },
+      };
+    }
+    const passed = result.exitCode === 0 && !result.timedOut;
+    const comment = result.timedOut
+      ? `command timed out after ${config.timeoutMs}ms`
+      : `'${config.command}' exited ${result.exitCode}${result.signal ? ` (signal ${result.signal})` : ""}`;
+    return {
+      passed,
+      score: passed ? 1 : 0,
+      comment,
+      detail: {
+        exitCode: result.exitCode,
+        signal: result.signal,
+        durationMs: result.durationMs,
+        timedOut: result.timedOut,
+        stdout: result.stdout,
+        stderr: result.stderr,
+        stdoutTruncated: result.stdoutTruncated,
+        stderrTruncated: result.stderrTruncated,
+      },
+    };
+  },
+};

package/src/assertions/core/event-checks.ts

@@ -0,0 +1,138 @@
+import { z } from "zod";
+import type { AssertionImpl, RunEvent } from "../../core/index.js";
+import { inRange, isToolCallEvent, type ProgressEvent, weightThresholdFields } from "./helpers.js";
+
+const isInterruptEvent = (e: RunEvent): e is Extract<RunEvent, { type: "interrupt" }> =>
+  e.type === "interrupt";
+
+const toolCallName = (e: ProgressEvent): string | undefined => {
+  const payload = e.payload as { name?: unknown } | null | undefined;
+  return typeof payload?.name === "string" ? payload.name : undefined;
+};
+
+/** Present when a subagent made the call; absent for the main agent (or if the driver doesn't report it). */
+const parentToolUseId = (e: ProgressEvent): string | undefined => {
+  const payload = e.payload as { parentToolUseId?: unknown } | null | undefined;
+  return typeof payload?.parentToolUseId === "string" ? payload.parentToolUseId : undefined;
+};
+
+export type AgentScope = "any" | "main" | "subagent";
+
+/**
+ * Public for custom-assertion authors: whether a progress event belongs to
+ * the requested scope — the "main" agent, a spawned "subagent", or "any".
+ */
+export const matchesAgentScope = (e: ProgressEvent, by: AgentScope = "any"): boolean => {
+  if (by === "any") return true;
+  const spawned = parentToolUseId(e) !== undefined;
+  return by === "subagent" ? spawned : !spawned;
+};
+
+export const ToolCalledConfigSchema = z
+  .object({
+    type: z.literal("tool-called"),
+    name: z.string(),
+    minTimes: z.number().int().nonnegative().optional(),
+    maxTimes: z.number().int().nonnegative().optional(),
+    /** Whose calls count: "main" agent, "subagent", or "any". */
+    by: z.enum(["any", "main", "subagent"]).default("any"),
+    ...weightThresholdFields,
+  })
+  .strict()
+  .refine((c) => c.minTimes === undefined || c.maxTimes === undefined || c.minTimes <= c.maxTimes, {
+    message: "minTimes must be <= maxTimes",
+  });
+export type ToolCalledConfig = z.infer<typeof ToolCalledConfigSchema>;
+
+export const toolCalledAssertion: AssertionImpl<ToolCalledConfig> = {
+  type: "tool-called",
+  configSchema: ToolCalledConfigSchema,
+  evaluate: async (config, run) => {
+    const calls = run.events
+      .filter(isToolCallEvent)
+      .filter((e) => matchesAgentScope(e, config.by))
+      .filter((e) => toolCallName(e) === config.name).length;
+    // `maxTimes: 0` means "never called"; defaulting min to 1 would make it unsatisfiable.
+    const min = config.minTimes ?? (config.maxTimes === undefined ? 1 : 0);
+    const { passed, comment } = inRange(calls, min, config.maxTimes);
+    const scope = config.by === "any" ? "" : ` by ${config.by}`;
+    return {
+      passed,
+      score: passed ? 1 : 0,
+      comment: `tool '${config.name}' called ${calls}×${scope} (${comment})`,
+      detail: { calls },
+    };
+  },
+};
+
+export const ToolCallCountConfigSchema = z
+  .object({
+    type: z.literal("tool-call-count"),
+    name: z.string().optional(),
+    min: z.number().int().nonnegative().optional(),
+    max: z.number().int().nonnegative().optional(),
+    by: z.enum(["any", "main", "subagent"]).default("any"),
+    ...weightThresholdFields,
+  })
+  .strict()
+  // With neither bound the assertion is vacuous — it would always pass.
+  .refine((c) => c.min !== undefined || c.max !== undefined, {
+    message: "set at least one of min or max",
+  })
+  .refine((c) => c.min === undefined || c.max === undefined || c.min <= c.max, {
+    message: "min must be <= max",
+  });
+export type ToolCallCountConfig = z.infer<typeof ToolCallCountConfigSchema>;
+
+export const toolCallCountAssertion: AssertionImpl<ToolCallCountConfig> = {
+  type: "tool-call-count",
+  configSchema: ToolCallCountConfigSchema,
+  evaluate: async (config, run) => {
+    const calls = run.events
+      .filter(isToolCallEvent)
+      .filter((e) => matchesAgentScope(e, config.by))
+      .filter((e) => !config.name || toolCallName(e) === config.name).length;
+    const { passed, comment } = inRange(calls, config.min, config.max);
+    return {
+      passed,
+      score: passed ? 1 : 0,
+      comment: `${calls} tool call${calls === 1 ? "" : "s"}${config.name ? ` for '${config.name}'` : ""} (${comment})`,
+      detail: { calls },
+    };
+  },
+};
+
+export const InterruptCountConfigSchema = z
+  .object({
+    type: z.literal("interrupt-count"),
+    kind: z.enum(["ask", "approve", "custom"]).optional(),
+    min: z.number().int().nonnegative().optional(),
+    max: z.number().int().nonnegative().optional(),
+    ...weightThresholdFields,
+  })
+  .strict()
+  // With neither bound the assertion is vacuous — it would always pass.
+  .refine((c) => c.min !== undefined || c.max !== undefined, {
+    message: "set at least one of min or max",
+  })
+  .refine((c) => c.min === undefined || c.max === undefined || c.min <= c.max, {
+    message: "min must be <= max",
+  });
+export type InterruptCountConfig = z.infer<typeof InterruptCountConfigSchema>;
+
+export const interruptCountAssertion: AssertionImpl<InterruptCountConfig> = {
+  type: "interrupt-count",
+  configSchema: InterruptCountConfigSchema,
+  evaluate: async (config, run) => {
+    const count = run.events
+      .filter(isInterruptEvent)
+      .filter((e) => !config.kind || e.kind === config.kind).length;
+    const { passed, comment } = inRange(count, config.min, config.max);
+    return {
+      passed,
+      score: passed ? 1 : 0,
+      comment: `${count} ${config.kind ?? "any-kind"} interrupt${count === 1 ? "" : "s"} (${comment})`,
+      detail: { count },
+    };
+  },
+};