@prismatic-io/lux 0.0.2-preview.15 → 0.0.2-preview.16

package/src/drivers/shared/experiment-behavior.ts

@@ -0,0 +1,214 @@
+import { lstat, readFile } from "node:fs/promises";
+import { homedir } from "node:os";
+import { isAbsolute, join, resolve } from "node:path";
+import { z } from "zod";
+import { walkAuthoredFiles } from "../../core/file-tree.js";
+import { contentHash, valueHash } from "../../core/hash.js";
+import {
+  prepareClaudeExperimentConfig,
+  prepareCodexExperimentConfig,
+} from "./experiment-policy.js";
+
+const ClaudeSettingsFileSchema = z.looseObject({
+  enabledPlugins: z.record(z.string(), z.boolean()).optional(),
+});
+const ClaudeInstalledPluginRecordSchema = z.looseObject({
+  installPath: z.string(),
+});
+const ClaudeInstalledPluginsFileSchema = z.looseObject({
+  plugins: z.record(z.string(), z.array(ClaudeInstalledPluginRecordSchema)),
+});
+
+const optionalRegularFileHash = async (path: string): Promise<string | null> => {
+  try {
+    const info = await lstat(path);
+    if (info.isSymbolicLink() || !info.isFile()) {
+      throw new Error(`behavior-affecting settings must be a regular file: ${path}`);
+    }
+    return contentHash(await readFile(path));
+  } catch (error) {
+    if ((error as NodeJS.ErrnoException).code === "ENOENT") return null;
+    throw error;
+  }
+};
+
+const driverConfigRecord = (driver: unknown): Record<string, unknown> => {
+  if (!driver || typeof driver !== "object") return {};
+  const config = (driver as { config?: unknown }).config;
+  return config && typeof config === "object" ? (config as Record<string, unknown>) : {};
+};
+
+export const claudeConfigRoot = (driver: unknown): string => {
+  const config = driverConfigRecord(driver);
+  const env =
+    config.env && typeof config.env === "object"
+      ? (config.env as Record<string, unknown>)
+      : undefined;
+  const configured = env?.CLAUDE_CONFIG_DIR;
+  const root =
+    typeof configured === "string"
+      ? configured
+      : (process.env.CLAUDE_CONFIG_DIR ?? join(homedir(), ".claude"));
+  if (!isAbsolute(root)) {
+    throw new Error("CLAUDE_CONFIG_DIR must be absolute for reproducible experiment identity");
+  }
+  return resolve(root);
+};
+
+const hashBehaviorDirectory = async (
+  root: string,
+  label: string,
+): Promise<Array<{ path: string; hash: string }>> => {
+  try {
+    const info = await lstat(root);
+    if (info.isSymbolicLink() || !info.isDirectory()) {
+      throw new Error(`behavior-affecting Claude source must be a real directory: ${root}`);
+    }
+  } catch (error) {
+    if ((error as NodeJS.ErrnoException).code === "ENOENT") return [];
+    throw error;
+  }
+  const paths = await walkAuthoredFiles(root);
+  if (paths.length > 5_000) {
+    throw new Error(`behavior-affecting Claude source '${label}' exceeds 5000 files`);
+  }
+  let bytes = 0;
+  const hashes: Array<{ path: string; hash: string }> = [];
+  for (const path of paths) {
+    const content = await readFile(join(root, path));
+    bytes += content.length;
+    if (bytes > 256 * 1024 * 1024) {
+      throw new Error(`behavior-affecting Claude source '${label}' exceeds 256 MiB`);
+    }
+    hashes.push({ path: `${label}/${path}`, hash: contentHash(content) });
+  }
+  return hashes;
+};
+
+/**
+ * Hash every intentionally loaded user behavior source without persisting its
+ * contents. This covers user instructions/rules/output styles and the exact
+ * installed contents of plugins enabled by settings.json.
+ */
+export const claudeUserBehaviorHash = async (configRoot: string): Promise<string | null> => {
+  const settingsPath = join(configRoot, "settings.json");
+  const settingsHash = await optionalRegularFileHash(settingsPath);
+  const sources: Array<{ path: string; hash: string }> = [];
+  if (settingsHash) sources.push({ path: "settings.json", hash: settingsHash });
+  const claudeMdHash = await optionalRegularFileHash(join(configRoot, "CLAUDE.md"));
+  if (claudeMdHash) sources.push({ path: "CLAUDE.md", hash: claudeMdHash });
+  for (const directory of ["rules", "output-styles", "agents", "commands", "skills"]) {
+    sources.push(...(await hashBehaviorDirectory(join(configRoot, directory), directory)));
+  }
+
+  let settings: unknown;
+  try {
+    const contents = await readFile(settingsPath, "utf8");
+    try {
+      settings = JSON.parse(contents);
+    } catch {
+      throw new Error("Claude settings.json contains invalid JSON");
+    }
+  } catch (error) {
+    if ((error as NodeJS.ErrnoException).code === "ENOENT") {
+      settings = undefined;
+    } else {
+      throw error;
+    }
+  }
+  const parsedSettings = ClaudeSettingsFileSchema.safeParse(settings);
+  if (settings !== undefined && !parsedSettings.success) {
+    throw new Error("Claude settings.json has an invalid enabledPlugins map");
+  }
+  const enabledPlugins = parsedSettings.success ? parsedSettings.data.enabledPlugins : undefined;
+  const enabledIds = Object.entries(enabledPlugins ?? {})
+    .filter(([, value]) => value === true)
+    .map(([id]) => id)
+    .sort();
+  if (enabledIds.length > 0) {
+    const installedPath = join(configRoot, "plugins", "installed_plugins.json");
+    const installedHash = await optionalRegularFileHash(installedPath);
+    if (installedHash) {
+      sources.push({ path: "plugins/installed_plugins.json", hash: installedHash });
+    }
+    let installed: unknown;
+    try {
+      installed = JSON.parse(await readFile(installedPath, "utf8"));
+    } catch {
+      installed = undefined;
+    }
+    const parsedInstalled = ClaudeInstalledPluginsFileSchema.safeParse(installed);
+    if (!parsedInstalled.success) {
+      throw new Error("enabled Claude plugins have invalid installed metadata");
+    }
+    const plugins = parsedInstalled.data.plugins;
+    for (const id of enabledIds) {
+      const records = plugins[id];
+      if (!records || records.length === 0) {
+        throw new Error(`enabled Claude plugin '${id}' has no installed content to hash`);
+      }
+      for (const [index, record] of records.entries()) {
+        const installPath = record.installPath;
+        sources.push(
+          ...(await hashBehaviorDirectory(resolve(installPath), `plugins/enabled/${id}/${index}`)),
+        );
+      }
+    }
+  }
+  return sources.length === 0 ? null : valueHash(sources);
+};
+
+export const configuredClaudeSourceHashes = async (
+  driver: unknown,
+  baseRoot: string,
+  cache: Map<string, Promise<string>> = new Map(),
+): Promise<Array<{ kind: "plugin" | "read"; path: string; hash: string }>> => {
+  const name =
+    typeof driver === "string" ? driver : (driver as { name?: unknown } | undefined)?.name;
+  const driverCli = name === "claude-code" || name === "codex" ? { name } : undefined;
+  if (!driverCli) return [];
+  const config = driverConfigRecord(driver);
+  // This also rejects path-bearing extraArgs that would evade the source
+  // hashes below. Runtime applies the same guard before driver creation.
+  if (driverCli.name === "claude-code") prepareClaudeExperimentConfig(config);
+  else prepareCodexExperimentConfig(config);
+  const isolation =
+    config.isolation && typeof config.isolation === "object"
+      ? (config.isolation as Record<string, unknown>)
+      : undefined;
+  const configured = [
+    ...(driverCli.name === "claude-code" && Array.isArray(config.pluginDirs)
+      ? config.pluginDirs.map((path) => ({ kind: "plugin" as const, path }))
+      : []),
+    ...(Array.isArray(config.addDirs)
+      ? config.addDirs.map((path) => ({ kind: "read" as const, path }))
+      : []),
+    ...(driverCli.name === "claude-code" && isolation && Array.isArray(isolation.allowRead)
+      ? isolation.allowRead.map((path) => ({ kind: "read" as const, path }))
+      : []),
+  ].filter(
+    (entry): entry is { kind: "plugin" | "read"; path: string } =>
+      typeof entry.path === "string" && entry.path.length > 0,
+  );
+  return Promise.all(
+    configured
+      .sort((a, b) => `${a.kind}:${a.path}`.localeCompare(`${b.kind}:${b.path}`))
+      .map(async ({ kind, path }) => {
+        const absolute = resolve(baseRoot, path);
+        const key = `${kind}:${absolute}`;
+        let hash = cache.get(key);
+        if (!hash) {
+          hash = hashBehaviorDirectory(
+            absolute,
+            `driver/${kind}/${contentHash(absolute).slice(0, 12)}`,
+          ).then(valueHash);
+          cache.set(key, hash);
+        }
+        return {
+          kind,
+          path: absolute,
+          hash: await hash,
+        };
+      }),
+  );
+};

package/src/drivers/shared/experiment-policy.ts

@@ -0,0 +1,195 @@
+import { isAbsolute } from "node:path";
+
+const isRecord = (value: unknown): value is Record<string, unknown> =>
+  Boolean(value) && typeof value === "object" && !Array.isArray(value);
+
+const EXPERIMENT_BEHAVIOR_SOURCE_ARGS = new Set([
+  "--continue",
+  "-c",
+  "--fork-session",
+  "--resume",
+  "-r",
+  "--add-dir",
+  "--append-system-prompt-file",
+  "--mcp-config",
+  "--plugin-dir",
+  "--settings",
+  "--setting-sources",
+  "--system-prompt-file",
+]);
+
+const CODEX_EXPERIMENT_SOURCE_ARGS = new Set([
+  "--add-dir",
+  "--cd",
+  "-C",
+  "--config",
+  "-c",
+  "--profile",
+  "-p",
+]);
+
+const bindSubprocessExperimentConfig = (config: Record<string, unknown>): unknown => {
+  if (typeof config.command !== "string" || !isAbsolute(config.command)) {
+    throw new Error(
+      "experiment subprocess commands must be absolute direct wrappers so Lux can fingerprint the invoked behavior",
+    );
+  }
+  if (typeof config.cwd === "string") {
+    throw new Error(
+      "experiment runs do not allow an explicit subprocess cwd; use the candidate subject and staged fixtures",
+    );
+  }
+  if (Array.isArray(config.args) && config.args.length > 0) {
+    throw new Error(
+      "experiment subprocesses do not allow args; put stable behavior in the fingerprinted direct wrapper",
+    );
+  }
+  if (isRecord(config.env) && Object.keys(config.env).length > 0) {
+    throw new Error(
+      "experiment subprocesses do not allow custom env; put stable behavior in the fingerprinted direct wrapper",
+    );
+  }
+  return { ...config, inheritEnv: false };
+};
+
+type AgentDriverName = "claude-code" | "codex";
+
+const validateAgentExperimentConfig = (
+  driverName: AgentDriverName,
+  config: Record<string, unknown>,
+): { driverEnv: Record<string, unknown>; extraArgs: string[] } => {
+  const driverEnv = isRecord(config.env) ? config.env : {};
+  const commandResolutionOverride = Object.keys(driverEnv).find((name) =>
+    ["PATH", "PATHEXT"].includes(name.toUpperCase()),
+  );
+  if (commandResolutionOverride) {
+    throw new Error(
+      `experiment runs do not allow driver env.${commandResolutionOverride}; command resolution must match the fingerprinted host executable`,
+    );
+  }
+  if (typeof config.cwd === "string") {
+    throw new Error(
+      `experiment runs do not allow an explicit ${driverName} cwd; use the candidate subject and staged fixtures`,
+    );
+  }
+
+  const extraArgs = Array.isArray(config.extraArgs)
+    ? config.extraArgs.filter((value): value is string => typeof value === "string")
+    : [];
+  const disallowedArgs =
+    driverName === "claude-code" ? EXPERIMENT_BEHAVIOR_SOURCE_ARGS : CODEX_EXPERIMENT_SOURCE_ARGS;
+  const behaviorSourceArg = extraArgs.find((arg) =>
+    [...disallowedArgs].some((name) => arg === name || arg.startsWith(`${name}=`)),
+  );
+  if (behaviorSourceArg) {
+    throw new Error(
+      `experiment runs do not allow behavior-bearing extraArgs '${behaviorSourceArg}'; use structured driver fields so Lux can fingerprint the source`,
+    );
+  }
+
+  const isolation = isRecord(config.isolation) ? config.isolation : {};
+  const structuredPaths = [
+    ...(driverName === "claude-code" && Array.isArray(config.pluginDirs) ? config.pluginDirs : []),
+    ...(Array.isArray(config.addDirs) ? config.addDirs : []),
+    ...(driverName === "claude-code" && Array.isArray(isolation.allowRead)
+      ? isolation.allowRead
+      : []),
+  ].filter((value): value is string => typeof value === "string");
+  const relativeSource = structuredPaths.find((path) => !isAbsolute(path));
+  if (relativeSource) {
+    throw new Error(
+      `experiment driver source paths must be absolute after subjectPath binding: ${relativeSource}`,
+    );
+  }
+  return { driverEnv, extraArgs };
+};
+
+const bindCodexExperimentConfig = (
+  config: Record<string, unknown>,
+  extraArgs: readonly string[],
+): unknown => {
+  if (typeof config.profile === "string") {
+    throw new Error(
+      "experiment runs do not allow a Codex profile; use inline driver config so Lux can fingerprint it",
+    );
+  }
+  if (config.ephemeral === false) {
+    throw new Error("experiment runs require ephemeral Codex sessions");
+  }
+  const inlineConfig = isRecord(config.config) ? config.config : {};
+  const externalConfigKey = ((): string | undefined => {
+    const visit = (value: Record<string, unknown>, parent = ""): string | undefined => {
+      for (const [key, nested] of Object.entries(value)) {
+        const path = parent ? `${parent}.${key}` : key;
+        if (/(?:mcp_servers|instructions_file|notify|project_doc)/i.test(key)) return path;
+        if (isRecord(nested)) {
+          const found = visit(nested, path);
+          if (found) return found;
+        }
+      }
+      return undefined;
+    };
+    return visit(inlineConfig);
+  })();
+  if (externalConfigKey) {
+    throw new Error(
+      `experiment runs do not allow path-bearing Codex config '${externalConfigKey}'`,
+    );
+  }
+  const appServer = config.interactionMode !== "exec";
+  return {
+    ...config,
+    ephemeral: true,
+    ignoreUserConfig: true,
+    ignoreRules: !appServer,
+    ...(appServer && !config.isolation ? { isolation: { allowRead: [] } } : {}),
+    extraArgs,
+  };
+};
+
+const bindClaudeExperimentConfig = (
+  config: Record<string, unknown>,
+  extraArgs: readonly string[],
+  driverEnv: Record<string, unknown>,
+): unknown => {
+  if (config.isolation) return config;
+  return {
+    ...config,
+    extraArgs: [
+      ...extraArgs,
+      "--setting-sources",
+      "user",
+      "--strict-mcp-config",
+      "--settings",
+      JSON.stringify({ disableAllHooks: true }),
+    ],
+    env: {
+      ...driverEnv,
+      CLAUDE_CODE_DISABLE_AUTO_MEMORY: "1",
+      CLAUDE_CODE_DISABLE_CLAUDE_MDS: "1",
+      CLAUDE_CODE_SKIP_PROMPT_HISTORY: "1",
+      CLAUDE_CODE_DISABLE_BACKGROUND_TASKS: "1",
+      CLAUDE_CODE_AUTO_CONNECT_IDE: "false",
+      CLAUDE_CODE_DISABLE_OFFICIAL_MARKETPLACE_AUTOINSTALL: "1",
+      DISABLE_AUTOUPDATER: "1",
+    },
+  };
+};
+
+export const prepareClaudeExperimentConfig = <Cfg extends Record<string, unknown>>(
+  config: Cfg,
+): Cfg => {
+  const { driverEnv, extraArgs } = validateAgentExperimentConfig("claude-code", config);
+  return bindClaudeExperimentConfig(config, extraArgs, driverEnv) as Cfg;
+};
+
+export const prepareCodexExperimentConfig = <Cfg extends Record<string, unknown>>(
+  config: Cfg,
+): Cfg => {
+  const { extraArgs } = validateAgentExperimentConfig("codex", config);
+  return bindCodexExperimentConfig(config, extraArgs) as Cfg;
+};
+
+export const prepareSubprocessExperimentConfig = <Cfg extends Record<string, unknown>>(
+  config: Cfg,
+): Cfg => bindSubprocessExperimentConfig(config) as Cfg;

package/src/drivers/subprocess/index.ts

@@ -8,7 +8,7 @@ import {
   type Artifact,
   DoneSchema,
   type DriverEventMap,
-  type DriverPlugin,
+  defineDriver,
   ErrorEventSchema,
   type FormatAnswerFn,
   formatWireAnswer,
@@ -25,6 +25,7 @@ import {
   TypedEmitter,
 } from "../../core/index.js";
 import { walkArtifacts } from "../shared/artifacts.js";
+import { prepareSubprocessExperimentConfig } from "../shared/experiment-policy.js";
 
 export const SubprocessDriverConfigSchema = z
   .object({
@@ -81,6 +82,8 @@ export type SubprocessDriverConfig = z.infer<typeof SubprocessDriverConfigSchema
 const KILL_GRACE_MS = 5_000;
 const DEFAULT_MAX_LINE_BYTES = 10_000_000;
 const MAX_STDERR_TAIL_CHARS = 8_192;
+const MAX_STDERR_CHARS = 65_536;
+const MAX_STDERR_EVENTS = 128;
 const DEFAULT_MAX_ARTIFACT_FILES = 500;
 const MAX_FIXTURE_VALUES_BYTES = 64 * 1024;
 const ARTIFACT_SNAPSHOT_MODULE = fileURLToPath(
@@ -174,15 +177,38 @@ class SubprocessDriverImpl implements AgentDriver {
     // boundary from being mangled.
     child.stderr.setEncoding("utf8");
     let stderrTail = "";
+    let stderrChars = 0;
+    let stderrEvents = 0;
+    let stderrTruncated = false;
+    const recordStderr = (line: string): void => {
+      if (stderrTruncated || this.terminalSent) return;
+      const text = line.slice(0, Math.min(MAX_STDERR_TAIL_CHARS, MAX_STDERR_CHARS - stderrChars));
+      stderrChars += text.length;
+      stderrEvents++;
+      stderrTruncated =
+        text.length < line.length ||
+        stderrChars >= MAX_STDERR_CHARS ||
+        stderrEvents >= MAX_STDERR_EVENTS;
+      this.emitInOrder("progress", {
+        id: `subprocess-stderr-${stderrEvents}`,
+        kind: "driver-stderr",
+        payload: { text, truncated: stderrTruncated },
+        ts: Date.now(),
+      });
+    };
     child.stderr.on("data", (chunk: string) => {
       const lines = (stderrTail + chunk).split("\n");
       // Cap the retained partial line so unterminated stderr can't grow
       // without bound.
-      stderrTail = (lines.pop() ?? "").slice(0, MAX_STDERR_TAIL_CHARS);
-      for (const line of lines) process.stderr.write(`[subprocess] ${line}\n`);
+      stderrTail = lines.pop() ?? "";
+      for (const line of lines) recordStderr(line);
+      if (stderrTail.length > MAX_STDERR_TAIL_CHARS) {
+        recordStderr(stderrTail);
+        stderrTail = "";
+      }
     });
     child.stderr.on("close", () => {
-      if (stderrTail) process.stderr.write(`[subprocess] ${stderrTail}\n`);
+      if (stderrTail) recordStderr(stderrTail);
       stderrTail = "";
     });
 
@@ -489,8 +515,21 @@ class SubprocessDriverImpl implements AgentDriver {
   }
 }
 
-export const subprocessDriver: DriverPlugin<SubprocessDriverConfig> = {
+export const subprocessDriver = defineDriver({
   name: "subprocess",
   configSchema: SubprocessDriverConfigSchema,
+  runtime: (config) => ({
+    commands: [config.command],
+    capabilities: {
+      interactive: true,
+      artifacts: true,
+      isolation: false,
+    },
+  }),
+  experiment: {
+    version: "1",
+    prepare: prepareSubprocessExperimentConfig,
+    identity: () => null,
+  },
   create: (config) => new SubprocessDriverImpl(config),
-};
+});

package/src/index.ts

@@ -183,11 +183,14 @@ export {
   AnswererModelCallAccountingSchema,
   type AnswererPlugin,
   type AnswerHistoryEntry,
+  answererRef,
+  defineAnswerer,
 } from "./core/answerer.js";
 export {
   inspectRunArtifact,
   type ReadRunArtifactTextOptions,
   type RunArtifactInspection,
+  type RunArtifactIssueReason,
   readRunArtifactText,
 } from "./core/artifact-evidence.js";
 export {
@@ -226,6 +229,7 @@ export {
   type EvalCase,
   type EvalCaseAuthoringOptions,
   type EvalCaseDefinition,
+  type EvalCaseInput,
   EvalCaseSchema,
   type FixturesRef,
   FixturesRefSchema,
@@ -246,8 +250,14 @@ export {
   type AgentDriver,
   type Artifact,
   ArtifactSchema,
+  type DriverCapability,
+  DriverCapabilitySchema,
   type DriverEventMap,
+  type DriverExperiment,
   type DriverPlugin,
+  type DriverRuntime,
+  defineDriver,
+  driverRef,
   type ResolvedFixtures,
   ResolvedFixturesSchema,
   type StartContext,
@@ -266,6 +276,7 @@ export {
   ReportedDriverMetricsSchema,
 } from "./core/driver-metrics.js";
 export { type EventHandler, TypedEmitter } from "./core/emitter.js";
+export { type EnvironmentDeclaration, EnvironmentDeclarationSchema } from "./core/environment.js";
 export {
   type AgentAttribution,
   AgentAttributionSchema,
@@ -471,6 +482,11 @@ export {
   CodexDriverConfigSchema,
   codexDriver,
 } from "./drivers/codex/index.js";
+export {
+  type CursorDriverConfig,
+  CursorDriverConfigSchema,
+  cursorDriver,
+} from "./drivers/cursor/index.js";
 export {
   type McpProbeDriverConfig,
   McpProbeDriverConfigSchema,

package/src/orchestrator/annotation-loader.ts

@@ -2,6 +2,7 @@ import { readFile } from "node:fs/promises";
 import { dirname, extname, resolve, sep } from "node:path";
 import { pathToFileURL } from "node:url";
 import { z } from "zod";
+import { assertDisjointClusters } from "../core/evaluation-clusters.js";
 import {
   type ExperimentSelector,
   formatZodIssues,
@@ -73,6 +74,15 @@ export const selectGraderAnnotations = (
 export const assertDisjointAnnotationSplits = (
   splits: Array<{ name: string; annotations: readonly GraderAnnotation[] }>,
 ): void => {
+  assertDisjointClusters(
+    splits.map((split) => ({
+      name: split.name,
+      items: split.annotations.map((annotation) => ({
+        id: annotation.id,
+        cluster: annotation.meta?.cluster,
+      })),
+    })),
+  );
   const idOwners = new Map<string, string>();
   const runOwners = new Map<string, string>();
   for (const split of splits) {

package/src/orchestrator/annotation-store.ts

@@ -82,11 +82,10 @@ const acquireAnnotationLock = async (path: string): Promise<() => Promise<void>>
   }
 };
 
-export const upsertGraderAnnotation = async (
-  filePath: string,
+const updateAnnotation = async (
+  path: string,
   annotation: GraderAnnotation,
 ): Promise<GraderAnnotationDataset> => {
-  const path = resolve(filePath);
   await mkdir(dirname(path), { recursive: true });
   const release = await acquireAnnotationLock(path);
   const temporary = `${path}.${process.pid}.${randomUUID()}.tmp`;
@@ -114,3 +113,25 @@ export const upsertGraderAnnotation = async (
     await release();
   }
 };
+
+// Serialize local writers before acquiring the cross-process lock. Otherwise
+// a burst of updates spends its lock deadline contending with its own process.
+const pendingUpdates = new Map<string, Promise<void>>();
+
+export const upsertGraderAnnotation = (
+  filePath: string,
+  annotation: GraderAnnotation,
+): Promise<GraderAnnotationDataset> => {
+  const path = resolve(filePath);
+  const previous = pendingUpdates.get(path) ?? Promise.resolve();
+  const update = previous.then(() => updateAnnotation(path, annotation));
+  const settled = update.then(
+    () => undefined,
+    () => undefined,
+  );
+  pendingUpdates.set(path, settled);
+  void settled.then(() => {
+    if (pendingUpdates.get(path) === settled) pendingUpdates.delete(path);
+  });
+  return update;
+};

package/src/orchestrator/campaign-lifecycle.ts

@@ -16,6 +16,7 @@ type CampaignLifecycleContext = {
   work: unknown;
   result: unknown;
   error: unknown;
+  failed: boolean;
   terminalStatus: "passed" | "failed" | "aborted" | undefined;
 };
 type CampaignLifecycleOutcome =
@@ -57,6 +58,7 @@ const campaignLifecycleMachine = setup({
         "output" in event ? context.effects.resultStatus(event.output) : "failed",
     }),
     captureError: assign({
+      failed: true,
       error: ({ event }) =>
         "error" in event ? event.error : new Error("campaign lifecycle failed"),
       terminalStatus: ({ context, event }) =>
@@ -65,6 +67,7 @@ const campaignLifecycleMachine = setup({
         ),
     }),
     captureReleaseError: assign({
+      failed: true,
       error: ({ event }) => ("error" in event ? event.error : new Error("campaign release failed")),
       terminalStatus: "failed",
     }),
@@ -76,6 +79,7 @@ const campaignLifecycleMachine = setup({
     work: undefined,
     result: undefined,
     error: undefined,
+    failed: false,
     terminalStatus: undefined,
   }),
   initial: "preparing",
@@ -145,7 +149,7 @@ const campaignLifecycleMachine = setup({
     failed: { type: "final" },
   },
   output: ({ context }) => {
-    if (context.error !== undefined) {
+    if (context.failed) {
       return {
         status: context.terminalStatus === "aborted" ? "aborted" : "failed",
         error: context.error,