blastproof 0.3.0 → 0.5.0

package/dist/cli.js

@@ -26,6 +26,9 @@ llm:
 
 browser:
   headless: true
+  # How long to wait for an element to appear (self-healing resolution) as well as
+  # for the action performed on it and for navigation. Raising it makes a slow app
+  # wait longer to succeed, and a genuinely missing element take longer to fail.
   timeout_ms: 30000
 
 # Budget of self-healing retries per step (element re-resolution via accessibility tree).
@@ -152,18 +155,25 @@ function formatInitGuidance(result) {
   lines.push(
     "",
     "Next steps:",
-    "  1. Set your LLM API key:  export ANTHROPIC_API_KEY=...   (or OPENAI_API_KEY; ollama needs no key)",
-    "  2. Start your app, then point base_url at it in .blastproof/config.yaml",
-    "  3. Run your tests:        blastproof run",
+    // Deliberately keyless first. A first-time-user trial found that following
+    // the old order — key, app, run — meant hitting the missing-key error and
+    // then a browser-launch failure with nothing successful behind you, which
+    // reads as "this is broken" rather than "one leg needs setting up". Seeing
+    // real output first reframes both.
+    "  1. Point base_url at your app in .blastproof/config.yaml",
+    "  2. Check the setup:       blastproof run --dry-run    (no API key or browser needed)",
+    "  3. Set your LLM API key:  export ANTHROPIC_API_KEY=...   (or OPENAI_API_KEY; ollama needs no key)",
+    "  4. Install the browser:   npx playwright install chromium",
+    "  5. Run your tests:        blastproof run",
     "",
-    "First time with Playwright? Install the browser:  npx playwright install chromium"
+    "Without a key or a browser you can still map a diff to routes and gate on",
+    "coverage:  blastproof run --impacted --fail-on-unmapped --dry-run"
   );
   return lines.join("\n");
 }
 
 // src/commands/plan.ts
 import path10 from "path";
-import { chromium as chromium2 } from "playwright";
 
 // src/auth.ts
 import { readFile, writeFile as writeFile2, mkdir as mkdir3 } from "fs/promises";
@@ -246,7 +256,7 @@ function estimateMaxModelCalls(tests, maxIterationsPerStep, maxRetriesPerStep) {
     (sum, test) => sum + (test.setup?.length ?? 0) + test.steps.length,
     0
   );
-  return totalSteps * (maxIterationsPerStep + maxRetriesPerStep);
+  return totalSteps * (maxIterationsPerStep + maxRetriesPerStep + Math.min(maxIterationsPerStep, maxRetriesPerStep));
 }
 
 // src/runner/env.ts
@@ -387,7 +397,7 @@ async function performAction(page, action, ctx) {
       const value = resolve(requireValue(action), ctx);
       const url = new URL(value, ctx.baseUrl);
       assertAllowedOrigin(url, ctx);
-      await page.goto(url.toString(), { timeout: 3e4 });
+      await page.goto(url.toString(), { timeout: ctx.resolveTimeoutMs ?? 3e4 });
       return `ok: navigated to ${url.toString()}`;
     }
     case "click": {
@@ -429,9 +439,22 @@ async function performAction(page, action, ctx) {
 
 // src/runner/executor.ts
 var DEFAULT_MAX_ITERATIONS_PER_STEP = 15;
-async function defaultSnapshot(page) {
+var SETTLE_TIMEOUT_MS = 2e3;
+async function waitForSettled(page) {
+  const deadline = Date.now() + SETTLE_TIMEOUT_MS;
+  const urlBefore = page.url();
+  await page.waitForLoadState("networkidle", { timeout: SETTLE_TIMEOUT_MS }).catch(() => {
+  });
+  if (page.url() === urlBefore) return;
+  const remaining = deadline - Date.now();
+  if (remaining > 0) {
+    await page.waitForLoadState("networkidle", { timeout: remaining }).catch(() => {
+    });
+  }
+}
+async function defaultSnapshot(page, maxLines) {
   const { captureSnapshot } = await import("./snapshot-CAIB2OHX.js");
-  return captureSnapshot(page);
+  return captureSnapshot(page, { maxLines });
 }
 function slugify(text) {
   const slug = text.toLowerCase().replace(/[^a-z0-9]+/g, "-").replace(/^-+|-+$/g, "").slice(0, 60);
@@ -448,11 +471,14 @@ async function executeTest(page, test, options) {
     resolveValue,
     maxRetries = 3,
     maxIterationsPerStep = DEFAULT_MAX_ITERATIONS_PER_STEP,
+    timeoutMs,
     mask = (s) => s,
-    snapshot = defaultSnapshot,
+    snapshot,
+    maxSnapshotLines,
     onEvent = () => {
     }
   } = options;
+  const takeSnapshot = snapshot ?? ((page2) => defaultSnapshot(page2, maxSnapshotLines));
   const startedAt = Date.now();
   const stepResults = [];
   const allSteps = [
@@ -483,7 +509,8 @@ async function executeTest(page, test, options) {
         if (iterations >= maxIterationsPerStep) {
           throw new StepFailure(`step exceeded ${maxIterationsPerStep} actions without completing`);
         }
-        const snap = await snapshot(page);
+        await waitForSettled(page);
+        const snap = await takeSnapshot(page);
         let action;
         try {
           action = await brain.nextAction({
@@ -514,7 +541,12 @@ async function executeTest(page, test, options) {
         }
         if (action.action === "assert") {
           const expectation = action.expectation ?? action.reasoning;
-          const judgment = await brain.judge(mask(expectation), mask(snap));
+          let judgment = await brain.judge(mask(expectation), mask(snap));
+          if (!judgment.pass) {
+            await waitForSettled(page);
+            const freshSnap = await takeSnapshot(page);
+            judgment = await brain.judge(mask(expectation), mask(freshSnap));
+          }
           const result = judgment.pass ? `ok: assertion passed: ${judgment.reason}` : `assertion failed: ${judgment.reason}`;
           emitAction(index, action, result);
           if (judgment.pass) {
@@ -529,7 +561,12 @@ async function executeTest(page, test, options) {
           continue;
         }
         try {
-          const result = await performAction(page, action, { baseUrl, allowedOrigins, resolveValue });
+          const result = await performAction(page, action, {
+            baseUrl,
+            allowedOrigins,
+            resolveValue,
+            resolveTimeoutMs: timeoutMs
+          });
           lastResult = result;
           emitAction(index, action, result);
         } catch (error) {
@@ -650,8 +687,9 @@ async function runJourney(...args) {
   }
 }
 async function fromSteps(options) {
-  const { auth, baseUrl, browser, brain, maxRetries, snapshot = defaultSnapshot, onEvent, mask } = options;
+  const { auth, baseUrl, browser, brain, maxRetries, snapshot, timeoutMs, maxSnapshotLines, onEvent, mask } = options;
   const steps = auth.steps;
+  const takeSnapshot = snapshot ?? ((page) => defaultSnapshot(page, maxSnapshotLines));
   const context = await browser.newContext();
   try {
     const page = await context.newPage();
@@ -673,8 +711,9 @@ async function fromSteps(options) {
         allowedOrigins: options.allowedOrigins,
         resolveValue: (value) => substituteEnv(value),
         maxRetries,
+        timeoutMs,
         mask: (text) => mask.mask(text),
-        snapshot,
+        snapshot: takeSnapshot,
         onEvent
       }
     );
@@ -684,7 +723,7 @@ async function fromSteps(options) {
       );
     }
     if (auth.verify) {
-      const judgment = await brain.judge(auth.verify, mask.mask(await snapshot(page)));
+      const judgment = await brain.judge(auth.verify, mask.mask(await takeSnapshot(page)));
       if (!judgment.pass) {
         throw new AuthError(`Authentication could not be verified: ${mask.mask(judgment.reason)}`);
       }
@@ -745,7 +784,19 @@ var llmSchema = z.object({
 });
 var browserSchema = z.object({
   headless: z.boolean().default(true),
-  timeout_ms: z.number().int().positive().default(3e4)
+  /**
+   * Bounds every wait: resolving a target element on the accessibility tree, and
+   * navigating. An explicit per-call Playwright timeout always overrides
+   * `page.setDefaultTimeout()`, so this must be threaded through the action
+   * context rather than relied on ambiently (design D2, runner/actions.ts).
+   */
+  timeout_ms: z.number().int().positive().default(3e4),
+  /**
+   * Caps accessibility-tree lines sent to the model per snapshot. Optional:
+   * undefined lets `runner/snapshot.ts`'s own default (200) apply, so a config
+   * that never mentions this behaves exactly as before it existed.
+   */
+  max_snapshot_lines: z.number().int().positive().optional()
 });
 var budgetSchema = z.object({
   max_llm_calls: z.coerce.number().int().positive().optional(),
@@ -844,6 +895,37 @@ var ConfigError = class extends Error {
 function formatIssues(error) {
   return error.issues.map((issue) => `  - ${issue.path.join(".") || "(root)"}: ${issue.message}`).join("\n");
 }
+function unwrapToObjectSchema(schema) {
+  let current = schema;
+  for (; ; ) {
+    if (current instanceof z.ZodOptional || current instanceof z.ZodNullable) {
+      current = current.unwrap();
+    } else if (current instanceof z.ZodDefault) {
+      current = current.removeDefault();
+    } else if (current instanceof z.ZodEffects) {
+      current = current.innerType();
+    } else {
+      break;
+    }
+  }
+  return current instanceof z.ZodObject ? current : void 0;
+}
+function findUnknownConfigKeys(data, schema = configSchema, path11 = []) {
+  const object = unwrapToObjectSchema(schema);
+  if (!object || data === null || typeof data !== "object" || Array.isArray(data)) return [];
+  const shape = object.shape;
+  const warnings = [];
+  for (const key of Object.keys(data)) {
+    const fieldPath = [...path11, key];
+    const fieldSchema = shape[key];
+    if (!fieldSchema) {
+      warnings.push(fieldPath.join("."));
+      continue;
+    }
+    warnings.push(...findUnknownConfigKeys(data[key], fieldSchema, fieldPath));
+  }
+  return warnings;
+}
 async function loadConfig(cwd = process.cwd(), env = process.env) {
   const configPath = path4.join(cwd, CONFIG_RELATIVE_PATH);
   let raw;
@@ -863,12 +945,17 @@ async function loadConfig(cwd = process.cwd(), env = process.env) {
     );
   }
   const { data: merged, applied } = applyEnvOverrides(data ?? {}, env);
+  const source = applied.length > 0 ? `${CONFIG_RELATIVE_PATH} (with overrides from ${applied.join(", ")})` : CONFIG_RELATIVE_PATH;
   const result = configSchema.safeParse(merged);
   if (!result.success) {
-    const source = applied.length > 0 ? `${CONFIG_RELATIVE_PATH} (with overrides from ${applied.join(", ")})` : CONFIG_RELATIVE_PATH;
     throw new ConfigError(`Invalid ${source}:
 ${formatIssues(result.error)}`);
   }
+  for (const key of findUnknownConfigKeys(merged)) {
+    console.warn(
+      `warning: unknown config key '${key}' in ${source} \u2014 this version of blastproof does not recognise it, so it has no effect.`
+    );
+  }
   return result.data;
 }
 
@@ -965,6 +1052,7 @@ Rules:
 - Return "done" when the current step's outcome holds \u2014 including when it already held before you acted, or was achieved by your previous action. "Already true" is done, never failure. Do not return "done" for work belonging to later steps.
 - Return "fail" only when the step's outcome cannot be reached: the element is still absent after retries, the page cannot support the step, or an error blocks progress. Never return "fail" because the work appears to have been done already.
 - If your previous action errored, re-read the fresh snapshot and choose an alternative element or approach. Do not repeat the exact same failing action.
+- \`***\` in a snapshot is a redacted secret \u2014 a password, token or key deliberately withheld from you. Seeing it is expected and is not a problem. A field showing \`***\` after you filled it from an {{env.VAR}} placeholder means the fill worked; treat that as success and move on. Never retry a fill because its value is redacted, and never report failure because a value was withheld.
 - Keep reasoning to one short sentence.`;
 }
 function agentUserPrompt(input) {
@@ -985,7 +1073,9 @@ function agentUserPrompt(input) {
   return parts.join("\n");
 }
 function assertSystemPrompt() {
-  return `You are a QA judge. You receive a page accessibility snapshot and an expectation. Decide whether the snapshot satisfies the expectation. Be strict: only pass when the expectation is clearly met by the snapshot content. Answer with pass=true/false and a one-sentence reason.`;
+  return `You are a QA judge. You receive a page accessibility snapshot and an expectation. Decide whether the snapshot satisfies the expectation. Be strict: only pass when the expectation is clearly met by the snapshot content. Answer with pass=true/false and a one-sentence reason.
+
+\`***\` marks a secret deliberately withheld from you \u2014 a password, token or key. Seeing it is expected. A field holding \`***\` is filled, not empty, so do not fail an expectation on the grounds that a value was redacted. This applies only to the redaction itself: everything else the expectation asks for must still be visibly satisfied by the snapshot, and an expectation you genuinely cannot check against what you were shown still fails.`;
 }
 function assertUserPrompt(expectation, snapshot) {
   return [
@@ -1277,10 +1367,11 @@ function renderTestYaml(draft, meta) {
   });
 }
 async function generateForRoute(page, options) {
-  const { route, baseUrl, changedFiles, brain, mask, snapshot = defaultSnapshot } = options;
+  const { route, baseUrl, changedFiles, brain, mask, snapshot, maxSnapshotLines, timeoutMs } = options;
+  const takeSnapshot = snapshot ?? ((p) => defaultSnapshot(p, maxSnapshotLines));
   const url = new URL(route, baseUrl).toString();
   try {
-    await page.goto(url, { timeout: 3e4 });
+    await page.goto(url, { timeout: timeoutMs });
   } catch (error) {
     throw new PlannerError(
       `Cannot load ${url}: ${error instanceof Error ? error.message : String(error)}`
@@ -1288,7 +1379,7 @@ async function generateForRoute(page, options) {
   }
   const generated = await brain.planTest({
     route,
-    snapshot: mask(await snapshot(page)),
+    snapshot: mask(await takeSnapshot(page)),
     changedFiles
   });
   const leaked = findSecretLiterals(generated.steps);
@@ -1317,9 +1408,120 @@ function coveredRoutes(tests) {
   return new Set(tests.flatMap((test) => test.routes));
 }
 
+// src/runner/browser.ts
+import { chromium } from "playwright";
+var BrowserLaunchError = class extends Error {
+  constructor(message) {
+    super(message);
+    this.name = "BrowserLaunchError";
+  }
+};
+var MISSING_LIBRARY_PATTERN = /([^\s:]+\.so(?:\.\d+)*): cannot open shared object file/;
+var MISSING_EXECUTABLE_PATTERN = /executable doesn't exist/i;
+function stripCommandLine(message) {
+  return message.split("\n").filter((line) => (line.match(/\s--[\w-]+/g)?.length ?? 0) < 3).join("\n").trim();
+}
+async function launchBrowser(options) {
+  try {
+    return await chromium.launch(options);
+  } catch (error) {
+    const message = error instanceof Error ? error.message : String(error);
+    if (MISSING_EXECUTABLE_PATTERN.test(message)) {
+      throw new BrowserLaunchError(
+        "The browser is not installed. Run `npx playwright install chromium` to download it."
+      );
+    }
+    const missingLibrary = message.match(MISSING_LIBRARY_PATTERN);
+    if (missingLibrary) {
+      throw new BrowserLaunchError(
+        `The browser could not start because a system library is missing: ${missingLibrary[1]}. Run \`npx playwright install-deps chromium\` to install it \u2014 this needs root/administrator privileges (sudo, or an elevated shell on Windows). Without them, blastproof run --dry-run and blastproof plan --dry-run need neither a browser nor a key.`
+      );
+    }
+    throw new BrowserLaunchError(stripCommandLine(message));
+  }
+}
+
+// src/preflight.ts
+var REACHABILITY_TIMEOUT_MS = 15e3;
+function messageOf(error) {
+  return error instanceof Error ? error.message : String(error);
+}
+function isTimeout(error) {
+  return error instanceof Error && error.name === "AbortError";
+}
+async function probe(url, fetchImpl) {
+  const controller = new AbortController();
+  const timer = setTimeout(() => controller.abort(), REACHABILITY_TIMEOUT_MS);
+  try {
+    await fetchImpl(url, { signal: controller.signal });
+  } finally {
+    clearTimeout(timer);
+  }
+}
+function providerBaseUrl(llm) {
+  if (llm.base_url) return llm.base_url;
+  switch (llm.provider) {
+    case "anthropic":
+      return "https://api.anthropic.com";
+    case "openai":
+      return "https://api.openai.com";
+    case "ollama":
+      return DEFAULT_OLLAMA_BASE_URL;
+  }
+}
+async function checkModel(llm, fetchImpl) {
+  const url = providerBaseUrl(llm);
+  try {
+    await probe(url, fetchImpl);
+    return void 0;
+  } catch (error) {
+    if (isTimeout(error)) {
+      return `Cannot reach the ${llm.provider} provider at ${url}: it did not respond within ${REACHABILITY_TIMEOUT_MS / 1e3}s. It may be temporarily slow or overloaded rather than down \u2014 try again in a moment, or check llm.base_url in .blastproof/config.yaml.`;
+    }
+    return `Cannot reach the ${llm.provider} provider at ${url} (${messageOf(error)}). Check your network connection` + (llm.provider === "ollama" ? ", that Ollama is running (`ollama serve`)," : "") + " or llm.base_url in .blastproof/config.yaml.";
+  }
+}
+async function checkBaseUrl(baseUrl, fetchImpl) {
+  try {
+    await probe(baseUrl, fetchImpl);
+    return void 0;
+  } catch (error) {
+    if (isTimeout(error)) {
+      return `The application at ${baseUrl} did not respond within ${REACHABILITY_TIMEOUT_MS / 1e3}s. It may still be starting (a dev server compiling a route on first request, a cold-starting container) rather than down \u2014 try again in a moment.`;
+    }
+    return `The application at ${baseUrl} is not responding (${messageOf(error)}). Start it, or point --url / base_url in .blastproof/config.yaml at the right address.`;
+  }
+}
+async function checkBrowser(headless) {
+  try {
+    const browser = await launchBrowser({ headless });
+    return { browser };
+  } catch (error) {
+    return { failure: error instanceof BrowserLaunchError ? error.message : messageOf(error) };
+  }
+}
+async function runPreflight(needs, config, fetchImpl = fetch) {
+  const [browserResult, modelFailure, baseUrlFailure] = await Promise.all([
+    needs.browser ? checkBrowser(config.browser.headless) : Promise.resolve({}),
+    needs.model ? checkModel(config.llm, fetchImpl) : Promise.resolve(void 0),
+    needs.baseUrl ? checkBaseUrl(config.base_url, fetchImpl) : Promise.resolve(void 0)
+  ]);
+  const failures = [browserResult.failure, modelFailure, baseUrlFailure].filter(
+    (value) => value !== void 0
+  );
+  if (failures.length > 0) {
+    if (browserResult.browser) await browserResult.browser.close();
+    return { ok: false, failures };
+  }
+  return { ok: true, browser: browserResult.browser };
+}
+function printPreflightFailures(failures) {
+  console.error(`error: ${failures.length} unmet prerequisite(s):`);
+  for (const failure of failures) console.error(`  - ${failure}`);
+}
+
 // src/commands/run.ts
 import path9 from "path";
-import { chromium } from "playwright";
 
 // src/report/html.ts
 import { mkdir as mkdir4, readFile as readFile4, writeFile as writeFile4 } from "fs/promises";
@@ -1766,6 +1968,8 @@ async function runOne(browser, test, config, sessionDir, session, runMask, budge
       allowedOrigins: config.allowed_origins,
       resolveValue: (value) => substituteEnv(value),
       maxRetries: config.max_retries_per_step,
+      timeoutMs: config.browser.timeout_ms,
+      maxSnapshotLines: config.browser.max_snapshot_lines,
       mask: (text) => mask.mask(text),
       onEvent: printEvent
     });
@@ -1970,7 +2174,12 @@ ${results.length} test file(s) could not be parsed:`);
   );
   const runMask = buildRunMask(config, parsed);
   const budget = options.budget ?? new RunBudget(resolveBudgetOptions(config, options));
-  const browser = await chromium.launch({ headless: config.browser.headless });
+  const preflight = await runPreflight({ browser: true, model: true, baseUrl: true }, config);
+  if (!preflight.ok) {
+    printPreflightFailures(preflight.failures);
+    return EXIT_USAGE;
+  }
+  const browser = preflight.browser;
   let incomplete;
   try {
     let session;
@@ -1985,6 +2194,8 @@ ${results.length} test file(s) could not be parsed:`);
           browser,
           brain: createBrain(createModel(config.llm).model, void 0, budget),
           maxRetries: config.max_retries_per_step,
+          timeoutMs: config.browser.timeout_ms,
+          maxSnapshotLines: config.browser.max_snapshot_lines,
           mask: runMask,
           onEvent: printEvent
         });
@@ -2111,6 +2322,16 @@ async function planCommand(options) {
     console.log("Already covered (skipped):");
     for (const route of alreadyCovered) console.log(`  ${route}`);
   }
+  if (options.dryRun) {
+    if (work.length === 0) {
+      console.log("Nothing to generate: no affected route is missing coverage.");
+    } else {
+      console.log(`Dry run: ${work.length} route(s) would generate a draft for:`);
+      for (const { route } of work) console.log(`  ${route}`);
+    }
+    console.log("Dry run: no browser launched, no LLM calls made.");
+    return EXIT_OK;
+  }
   if (work.length === 0) {
     console.log("Nothing to generate: no affected route is missing coverage.");
     return EXIT_OK;
@@ -2142,7 +2363,12 @@ async function planCommand(options) {
   const written = [];
   const failed = [];
   let incomplete;
-  const browser = await chromium2.launch({ headless: config.browser.headless });
+  const preflight = await runPreflight({ browser: true, model: true, baseUrl: true }, config);
+  if (!preflight.ok) {
+    printPreflightFailures(preflight.failures);
+    return EXIT_USAGE;
+  }
+  const browser = preflight.browser;
   try {
     let session;
     if (config.auth) {
@@ -2156,6 +2382,8 @@ async function planCommand(options) {
           browser,
           brain: createBrain(createModel(config.llm).model, void 0, budget),
           maxRetries: config.max_retries_per_step,
+          timeoutMs: config.browser.timeout_ms,
+          maxSnapshotLines: config.browser.max_snapshot_lines,
           mask,
           onEvent: printAuthEvent
         });
@@ -2183,7 +2411,9 @@ async function planCommand(options) {
           baseUrl: config.base_url,
           changedFiles,
           brain,
-          mask: (text) => mask.mask(text)
+          mask: (text) => mask.mask(text),
+          maxSnapshotLines: config.browser.max_snapshot_lines,
+          timeoutMs: config.browser.timeout_ms
         });
       } catch (error) {
         if (error instanceof BudgetExhaustedError) {
@@ -2326,7 +2556,7 @@ function parsePositiveNumber(flag) {
   };
 }
 var program = new Command();
-program.name("blastproof").description("Open-source AI testing agent: plain-English YAML tests executed agentically on a real browser.").version("0.3.0");
+program.name("blastproof").description("Open-source AI testing agent: plain-English YAML tests executed agentically on a real browser.").version("0.5.0");
 program.command("init").description("Scaffold .blastproof/ (config, tests, sample tests) in the current directory").action(async () => {
   try {
     const result = await initProject(process.cwd());
@@ -2380,6 +2610,9 @@ program.command("run").description("Discover and run all tests under .blastproof
   }
 );
 program.command("plan").description("Generate plain-English YAML tests for affected routes no test covers yet").option("--base <ref>", "base git ref for the diff", "main").option("--url <url>", "override config base_url for this run only (config file untouched)").option("--route <route>", "generate for this route, bypassing the diff (repeatable)", collect, []).option("--write", "persist drafts under .blastproof/tests/ instead of previewing them").option(
+  "--dry-run",
+  "print the routes that would generate drafts and exit without launching a browser or calling the LLM"
+).option(
   "--max-llm-calls <n>",
   "stop after this many model calls, reported as incomplete (overrides config)",
   parsePositiveInt("--max-llm-calls")
@@ -2400,6 +2633,7 @@ program.command("plan").description("Generate plain-English YAML tests for affec
         url: options.url,
         routes: options.route,
         write: options.write,
+        dryRun: options.dryRun,
         maxLlmCalls: options.maxLlmCalls,
         maxTokens: options.maxTokens,
         maxDuration: options.maxDuration