blastproof 0.2.2 → 0.3.0

package/dist/cli.js

@@ -169,6 +169,86 @@ import { chromium as chromium2 } from "playwright";
 import { readFile, writeFile as writeFile2, mkdir as mkdir3 } from "fs/promises";
 import path3 from "path";
 
+// src/runner/budget.ts
+function describeLimit(limit, observed, configured) {
+  switch (limit) {
+    case "calls":
+      return `model call budget exhausted: reached the configured maximum of ${configured} call(s)`;
+    case "tokens":
+      return `token budget exhausted: reached the configured maximum of ${configured} token(s) (observed ${observed})`;
+    case "duration":
+      return `deadline exceeded: reached the configured maximum of ${(configured / 1e3).toFixed(0)}s (elapsed ${(observed / 1e3).toFixed(1)}s)`;
+  }
+}
+var BudgetExhaustedError = class extends Error {
+  limit;
+  observed;
+  configured;
+  constructor(limit, observed, configured) {
+    super(describeLimit(limit, observed, configured));
+    this.name = "BudgetExhaustedError";
+    this.limit = limit;
+    this.observed = observed;
+    this.configured = configured;
+  }
+};
+var RunBudget = class {
+  maxCalls;
+  maxTokens;
+  maxDurationMs;
+  now;
+  startedAt;
+  calls = 0;
+  tokens = 0;
+  constructor(options = {}) {
+    this.maxCalls = options.maxCalls;
+    this.maxTokens = options.maxTokens;
+    this.maxDurationMs = options.maxDurationMs;
+    this.now = options.now ?? Date.now;
+    this.startedAt = this.now();
+  }
+  /** Model calls recorded so far. */
+  get callCount() {
+    return this.calls;
+  }
+  /** Tokens recorded so far, summed across every completed call. */
+  get tokenCount() {
+    return this.tokens;
+  }
+  /**
+   * Throws {@link BudgetExhaustedError} when the next unit of work could exceed a
+   * configured limit. Called before spending anything, never after — so an
+   * over-budget call is never issued and then rejected (spec: "the call is not
+   * made").
+   */
+  check() {
+    if (this.maxCalls !== void 0 && this.calls >= this.maxCalls) {
+      throw new BudgetExhaustedError("calls", this.calls, this.maxCalls);
+    }
+    if (this.maxTokens !== void 0 && this.tokens >= this.maxTokens) {
+      throw new BudgetExhaustedError("tokens", this.tokens, this.maxTokens);
+    }
+    if (this.maxDurationMs !== void 0) {
+      const elapsed = this.now() - this.startedAt;
+      if (elapsed >= this.maxDurationMs) {
+        throw new BudgetExhaustedError("duration", elapsed, this.maxDurationMs);
+      }
+    }
+  }
+  /** Records what a completed model call spent. */
+  record(usage) {
+    this.calls += 1;
+    if (usage?.totalTokens !== void 0) this.tokens += usage.totalTokens;
+  }
+};
+function estimateMaxModelCalls(tests, maxIterationsPerStep, maxRetriesPerStep) {
+  const totalSteps = tests.reduce(
+    (sum, test) => sum + (test.setup?.length ?? 0) + test.steps.length,
+    0
+  );
+  return totalSteps * (maxIterationsPerStep + maxRetriesPerStep);
+}
+
 // src/runner/env.ts
 var ENV_PLACEHOLDER = /\{\{\s*env\.([A-Za-z_][A-Za-z0-9_]*)\s*\}\}/g;
 var MissingEnvError = class extends Error {
@@ -348,6 +428,7 @@ async function performAction(page, action, ctx) {
 }
 
 // src/runner/executor.ts
+var DEFAULT_MAX_ITERATIONS_PER_STEP = 15;
 async function defaultSnapshot(page) {
   const { captureSnapshot } = await import("./snapshot-CAIB2OHX.js");
   return captureSnapshot(page);
@@ -366,7 +447,7 @@ async function executeTest(page, test, options) {
     allowedOrigins,
     resolveValue,
     maxRetries = 3,
-    maxIterationsPerStep = 15,
+    maxIterationsPerStep = DEFAULT_MAX_ITERATIONS_PER_STEP,
     mask = (s) => s,
     snapshot = defaultSnapshot,
     onEvent = () => {
@@ -414,6 +495,7 @@ async function executeTest(page, test, options) {
             iterationsLeft: maxIterationsPerStep - iterations
           });
         } catch (error) {
+          if (error instanceof BudgetExhaustedError) throw error;
           failedAttempts++;
           lastResult = `error: ${error instanceof Error ? error.message : String(error)}`;
           if (failedAttempts >= maxRetries) {
@@ -437,7 +519,7 @@ async function executeTest(page, test, options) {
           emitAction(index, action, result);
           if (judgment.pass) {
             lastResult = result;
-            continue;
+            break;
           }
           failedAttempts++;
           lastResult = result;
@@ -460,6 +542,7 @@ async function executeTest(page, test, options) {
         }
       }
     } catch (error) {
+      if (error instanceof BudgetExhaustedError) throw error;
       stepFailedReason = mask(error instanceof Error ? error.message : String(error));
     }
     const status = stepFailedReason ? "failed" : "passed";
@@ -560,6 +643,7 @@ async function runJourney(...args) {
   try {
     return await executeTest(...args);
   } catch (error) {
+    if (error instanceof BudgetExhaustedError) throw error;
     throw new AuthError(
       `Authentication could not run: ${error instanceof Error ? error.message : String(error)}`
     );
@@ -663,6 +747,12 @@ var browserSchema = z.object({
   headless: z.boolean().default(true),
   timeout_ms: z.number().int().positive().default(3e4)
 });
+var budgetSchema = z.object({
+  max_llm_calls: z.coerce.number().int().positive().optional(),
+  max_tokens: z.coerce.number().int().positive().optional(),
+  /** Seconds, not milliseconds, to match how a human would set a deadline. */
+  max_duration_s: z.coerce.number().positive().optional()
+});
 var cookieSchema = z.object({
   name: z.string().min(1),
   value: z.string(),
@@ -712,14 +802,18 @@ var configSchema = z.object({
    */
   allowed_origins: z.array(z.string().url()).optional(),
   auth: authSchema.optional(),
-  max_retries_per_step: z.number().int().min(1).default(3)
+  max_retries_per_step: z.number().int().min(1).default(3),
+  budget: budgetSchema.optional()
 });
 var ENV_OVERRIDES = {
   BLASTPROOF_BASE_URL: ["base_url"],
   BLASTPROOF_LLM_PROVIDER: ["llm", "provider"],
   BLASTPROOF_LLM_MODEL: ["llm", "model"],
   BLASTPROOF_LLM_BASE_URL: ["llm", "base_url"],
-  BLASTPROOF_LLM_API_KEY_ENV: ["llm", "api_key_env"]
+  BLASTPROOF_LLM_API_KEY_ENV: ["llm", "api_key_env"],
+  BLASTPROOF_MAX_LLM_CALLS: ["budget", "max_llm_calls"],
+  BLASTPROOF_MAX_TOKENS: ["budget", "max_tokens"],
+  BLASTPROOF_MAX_DURATION_S: ["budget", "max_duration_s"]
 };
 function applyEnvOverrides(data, env = process.env) {
   const merged = data && typeof data === "object" ? { ...data } : {};
@@ -967,10 +1061,16 @@ var MalformedModelOutputError = class extends Error {
     this.name = "MalformedModelOutputError";
   }
 };
-function createBrain(model, generate = generateObject) {
+async function countedGenerate(generate, budget, options) {
+  budget.check();
+  const result = await generate(options);
+  budget.record(result.usage);
+  return result;
+}
+function createBrain(model, generate = generateObject, budget) {
   return {
     async nextAction(input) {
-      const result = await generate({
+      const result = await countedGenerate(generate, budget, {
         model,
         schema: agentActionSchema,
         system: agentSystemPrompt(),
@@ -985,7 +1085,7 @@ function createBrain(model, generate = generateObject) {
       return parsed.data;
     },
     async judge(expectation, snapshot) {
-      const result = await generate({
+      const result = await countedGenerate(generate, budget, {
         model,
         schema: assertJudgmentSchema,
         system: assertSystemPrompt(),
@@ -1001,10 +1101,10 @@ function createBrain(model, generate = generateObject) {
     }
   };
 }
-function createPlanner(model, generate = generateObject) {
+function createPlanner(model, generate = generateObject, budget) {
   return {
     async planTest(input) {
-      const result = await generate({
+      const result = await countedGenerate(generate, budget, {
         model,
         schema: generatedTestSchema,
         system: plannerSystemPrompt(),
@@ -1262,6 +1362,8 @@ h1 { font-size:1.35rem; margin:0 0 .25rem; }
 .verdict { font-weight:600; }
 .verdict.pass { color:var(--pass); } .verdict.fail { color:var(--fail); }
 .counts { color:var(--muted); font-size:.9rem; }
+.incomplete { padding:1rem 1.25rem; border:1px solid var(--fail); border-radius:10px;
+  background:color-mix(in srgb, var(--fail) 10%, transparent); margin-bottom:1.5rem; font-weight:600; }
 table { width:100%; border-collapse:collapse; margin-bottom:1.5rem; }
 th, td { text-align:left; padding:.5rem .6rem; border-bottom:1px solid var(--line);
   font-size:.9rem; vertical-align:top; }
@@ -1298,10 +1400,13 @@ function renderSteps(steps) {
 }
 async function renderTest(result, cwd) {
   const failed = result.status === "failed";
+  const notRun = result.status === "not-run";
+  const tagClass = failed ? "fail" : notRun ? "skip" : "pass";
+  const label = failed ? "FAIL" : notRun ? "NOT RUN" : "PASS";
   const file = cwd ? path7.relative(cwd, result.file) : result.file;
   const parts = [
-    `    <details${failed ? " open" : ""}>`,
-    `      <summary><span class="tag ${failed ? "fail" : "pass"}">${failed ? "FAIL" : "PASS"}</span> ${escapeHtml(result.summary)} <span class="note">${escapeHtml(result.priority)} \xB7 ${escapeHtml(file)}</span></summary>`
+    `    <details${failed || notRun ? " open" : ""}>`,
+    `      <summary><span class="tag ${tagClass}">${label}</span> ${escapeHtml(result.summary)} <span class="note">${escapeHtml(result.priority)} \xB7 ${escapeHtml(file)}</span></summary>`
   ];
   if (failed) {
     if (result.failedStep) {
@@ -1311,6 +1416,9 @@ async function renderTest(result, cwd) {
       parts.push(`      <p class="reason">${escapeHtml(result.reason)}</p>`);
     }
   }
+  if (notRun && result.reason) {
+    parts.push(`      <p class="note">${escapeHtml(result.reason)}</p>`);
+  }
   parts.push(`      ${renderSteps(result.steps)}`);
   if (failed && result.screenshot) {
     const embedded = await embedScreenshot(result.screenshot);
@@ -1324,13 +1432,20 @@ async function renderTest(result, cwd) {
 async function renderHtml(results, skipped, meta) {
   const passed = results.filter((r) => r.status === "passed").length;
   const failures = results.filter((r) => r.status === "failed");
+  const notRun = results.filter((r) => r.status === "not-run");
   const generatedAt = (meta.generatedAt ?? /* @__PURE__ */ new Date()).toISOString();
-  const ordered = [...failures, ...results.filter((r) => r.status === "passed")];
+  const ordered = [...failures, ...notRun, ...results.filter((r) => r.status === "passed")];
   const detail = (await Promise.all(ordered.map((r) => renderTest(r, meta.cwd)))).join("\n");
   const verdict = meta.minScore === void 0 ? "" : meta.score >= meta.minScore ? `<span class="verdict pass">min-score ${meta.minScore}: pass</span>` : `<span class="verdict fail">min-score ${meta.minScore}: FAIL</span>`;
+  const banner = meta.incomplete === void 0 ? "" : `  <section class="incomplete">Run stopped: ${escapeHtml(meta.incomplete)}</section>
+
+`;
   const rows = results.map((r) => {
     const file = meta.cwd ? path7.relative(meta.cwd, r.file) : r.file;
-    return `      <tr><td><span class="tag ${r.status === "failed" ? "fail" : "pass"}">${r.status === "failed" ? "FAIL" : "PASS"}</span></td><td>${escapeHtml(r.priority)}</td><td>${escapeHtml(r.summary)}</td><td>${escapeHtml(file)}</td><td class="num">${seconds(r.durationMs)}</td></tr>`;
+    const tagClass = r.status === "failed" ? "fail" : r.status === "not-run" ? "skip" : "pass";
+    const label = r.status === "failed" ? "FAIL" : r.status === "not-run" ? "NOT RUN" : "PASS";
+    const time = r.status === "not-run" ? "\u2014" : seconds(r.durationMs);
+    return `      <tr><td><span class="tag ${tagClass}">${label}</span></td><td>${escapeHtml(r.priority)}</td><td>${escapeHtml(r.summary)}</td><td>${escapeHtml(file)}</td><td class="num">${time}</td></tr>`;
   }).join("\n");
   const skippedRows = skipped.map((s) => {
     const file = meta.cwd ? path7.relative(meta.cwd, s.path) : s.path;
@@ -1349,10 +1464,10 @@ async function renderHtml(results, skipped, meta) {
   <h1>blastproof report</h1>
   <p class="sub">${escapeHtml(generatedAt)} \xB7 ${seconds(meta.durationMs)}</p>
 
-  <section class="score">
+${banner}  <section class="score">
     <b>${meta.score}</b>
     ${verdict}
-    <span class="counts">${passed} passed \xB7 ${failures.length} failed${skipped.length > 0 ? ` \xB7 ${skipped.length} skipped` : ""}</span>
+    <span class="counts">${passed} passed \xB7 ${failures.length} failed${notRun.length > 0 ? ` \xB7 ${notRun.length} not run` : ""}${skipped.length > 0 ? ` \xB7 ${skipped.length} skipped` : ""}</span>
   </section>
 
   <table>
@@ -1395,14 +1510,27 @@ function seconds2(ms) {
 function renderJUnit(results, skipped, meta) {
   const relative = (file) => meta.cwd ? path8.relative(meta.cwd, file) : file;
   const failures = results.filter((result) => result.status === "failed").length;
+  const notRun = results.filter((result) => result.status === "not-run");
   const lines = [
     '<?xml version="1.0" encoding="UTF-8"?>',
-    `<testsuite name="blastproof" tests="${results.length + skipped.length}" failures="${failures}" skipped="${skipped.length}" time="${seconds2(meta.durationMs)}">`,
+    `<testsuite name="blastproof" tests="${results.length + skipped.length}" failures="${failures}" skipped="${skipped.length + notRun.length}" time="${seconds2(meta.durationMs)}">`,
     "  <properties>",
     `    <property name="score" value="${meta.score}"/>`,
+    ...meta.incomplete !== void 0 ? [
+      '    <property name="incomplete" value="true"/>',
+      `    <property name="incomplete_reason" value="${escapeXml(meta.incomplete)}"/>`
+    ] : [],
     "  </properties>"
   ];
   for (const result of results) {
+    if (result.status === "not-run") {
+      lines.push(
+        `  <testcase classname="${escapeXml(relative(result.file))}" name="${escapeXml(result.summary)}" time="0.000">`,
+        `    <skipped message="${escapeXml(result.reason ?? "not run: the run stopped before this test executed")}"/>`,
+        "  </testcase>"
+      );
+      continue;
+    }
     const open = `  <testcase classname="${escapeXml(relative(result.file))}" name="${escapeXml(result.summary)}" time="${seconds2(result.durationMs)}"`;
     if (result.status === "passed") {
       lines.push(`${open}/>`);
@@ -1438,6 +1566,7 @@ function computeScore(results) {
   let total = 0;
   let passed = 0;
   for (const result of results) {
+    if (result.status === "not-run") continue;
     const weight = WEIGHTS[result.priority] ?? DEFAULT_WEIGHT;
     total += weight;
     if (result.status === "passed") passed += weight;
@@ -1453,6 +1582,10 @@ function formatScoreLine(score, results, threshold) {
   if (threshold === void 0) return `Score: ${score}`;
   return score >= threshold ? `Score: ${score} \u2014 min-score ${threshold}: pass` : `Score: ${score} \u2014 min-score ${threshold}: FAIL (below threshold)`;
 }
+function formatIncompleteLine(score, reason) {
+  return `Run incomplete: ${reason}
+Score over executed tests: ${score} (not a verdict \u2014 exit code 1 regardless of --min-score)`;
+}
 
 // src/runner/selection.ts
 function matchesFilters(test, filters) {
@@ -1492,6 +1625,14 @@ function selectImpactedTests(tests, affectedRoutes, filters = { tags: [] }) {
 var EXIT_OK = 0;
 var EXIT_FAILED = 1;
 var EXIT_USAGE = 2;
+function resolveBudgetOptions(config, options) {
+  const maxDurationSeconds = options.maxDuration ?? config.budget?.max_duration_s;
+  return {
+    maxCalls: options.maxLlmCalls ?? config.budget?.max_llm_calls,
+    maxTokens: options.maxTokens ?? config.budget?.max_tokens,
+    maxDurationMs: maxDurationSeconds === void 0 ? void 0 : maxDurationSeconds * 1e3
+  };
+}
 function sessionId() {
   return (/* @__PURE__ */ new Date()).toISOString().replace(/[:.]/g, "-");
 }
@@ -1522,6 +1663,18 @@ function resolveSecretsAndSteps(test) {
   }
   return { test };
 }
+function notRunResults(tests, reason) {
+  return tests.map((test) => ({
+    file: test.path,
+    summary: test.summary,
+    priority: test.priority,
+    tags: test.tags,
+    status: "not-run",
+    steps: [],
+    reason,
+    durationMs: 0
+  }));
+}
 function printEvent(event) {
   switch (event.type) {
     case "step-start":
@@ -1541,12 +1694,18 @@ function printEvent(event) {
       break;
   }
 }
+function statusLabel(status) {
+  if (status === "passed") return "PASS";
+  if (status === "failed") return "FAIL";
+  return "NOT RUN";
+}
 function printSummary(results) {
   const passed = results.filter((r) => r.status === "passed");
   const failed = results.filter((r) => r.status === "failed");
+  const notRun = results.filter((r) => r.status === "not-run");
   console.log("\n--- Summary ---------------------------------------------------");
   const rows = results.map((r) => ({
-    status: r.status === "passed" ? "PASS" : "FAIL",
+    status: statusLabel(r.status),
     priority: r.priority,
     summary: r.summary,
     duration: `${(r.durationMs / 1e3).toFixed(1)}s`
@@ -1559,7 +1718,9 @@ function printSummary(results) {
     );
   }
   console.log("---------------------------------------------------------------");
-  console.log(`${passed.length} passed, ${failed.length} failed, ${results.length} total`);
+  console.log(
+    notRun.length > 0 ? `${passed.length} passed, ${failed.length} failed, ${notRun.length} not run, ${results.length} total` : `${passed.length} passed, ${failed.length} failed, ${results.length} total`
+  );
   for (const r of failed) {
     console.log(`
 X ${r.summary} (${r.file})`);
@@ -1567,9 +1728,14 @@ X ${r.summary} (${r.file})`);
     if (r.reason) console.log(`  reason: ${r.reason}`);
     if (r.screenshot) console.log(`  screenshot: ${r.screenshot}`);
   }
+  if (notRun.length > 0) {
+    console.log(`
+${notRun.length} test(s) not run (run stopped by its budget or deadline):`);
+    for (const r of notRun) console.log(`  - ${r.summary} (${r.file})`);
+  }
 }
-async function runOne(browser, test, config, sessionDir, session, runMask) {
-  const brain = createBrain(createModel(config.llm).model);
+async function runOne(browser, test, config, sessionDir, session, runMask, budget) {
+  const brain = createBrain(createModel(config.llm).model, void 0, budget);
   let resolved;
   const mask = runMask;
   try {
@@ -1632,13 +1798,20 @@ function printImpactReport(impact, selection, cwd) {
   }
   console.log("---------------------------------------------------------------");
 }
-async function finalize(results, skipped, options, sessionDir, durationMs, impact) {
+async function finalize(results, skipped, options, sessionDir, durationMs, impact, incomplete) {
   if (results.length > 0) printSummary(results);
   const score = computeScore(results);
-  console.log(formatScoreLine(score, results, options.minScore));
+  console.log(
+    incomplete ? formatIncompleteLine(score, incomplete.message) : formatScoreLine(score, results, options.minScore)
+  );
   if (options.junit) {
     const target = typeof options.junit === "string" ? path9.resolve(options.cwd, options.junit) : path9.join(sessionDir, "junit.xml");
-    const xml = renderJUnit(results, skipped, { score, durationMs, cwd: options.cwd });
+    const xml = renderJUnit(results, skipped, {
+      score,
+      durationMs,
+      cwd: options.cwd,
+      incomplete: incomplete?.message
+    });
     await writeJUnit(target, xml);
     console.log(`JUnit report: ${path9.relative(options.cwd, target)}`);
   }
@@ -1648,11 +1821,13 @@ async function finalize(results, skipped, options, sessionDir, durationMs, impac
       score,
       durationMs,
       minScore: options.minScore,
-      cwd: options.cwd
+      cwd: options.cwd,
+      incomplete: incomplete?.message
     });
     await writeHtml(target, html);
     console.log(`HTML report: ${path9.relative(options.cwd, target)}`);
   }
+  if (incomplete) return EXIT_FAILED;
   if (reportUnclassified(options, impact)) return EXIT_FAILED;
   if (options.minScore !== void 0) {
     return score >= options.minScore ? EXIT_OK : EXIT_FAILED;
@@ -1671,12 +1846,23 @@ function reportUnclassified(options, impact) {
   );
   return true;
 }
-function printDryRun(selected, baseUrl, cwd) {
+function printDryRun(selected, config, cwd) {
   console.log(`
-Dry run: ${selected.length} test(s) selected, base_url=${baseUrl}`);
+Dry run: ${selected.length} test(s) selected, base_url=${config.base_url}`);
   for (const test of selected) {
     console.log(`  ${test.summary} [${test.priority}] (${path9.relative(cwd, test.path)})`);
   }
+  const authSteps = config.auth?.steps ?? [];
+  const withAuth = authSteps.length > 0 ? [...selected, { steps: authSteps }] : selected;
+  const ceiling = estimateMaxModelCalls(
+    withAuth,
+    DEFAULT_MAX_ITERATIONS_PER_STEP,
+    config.max_retries_per_step
+  );
+  const authNote = authSteps.length > 0 ? ", including the login journey" : "";
+  console.log(
+    `Worst case: up to ${ceiling} model call(s) for this selection${authNote} (a maximum, not a prediction).`
+  );
   console.log("Dry run: no browser launched, no LLM calls made.");
 }
 async function runCommand(options) {
@@ -1752,7 +1938,7 @@ async function runCommand(options) {
     printImpactReport(impact, selection, options.cwd);
   }
   if (options.dryRun) {
-    printDryRun(selected, config.base_url, options.cwd);
+    printDryRun(selected, config, options.cwd);
     if (results.length > 0) {
       console.error(`
 ${results.length} test file(s) could not be parsed:`);
@@ -1783,7 +1969,9 @@ ${results.length} test file(s) could not be parsed:`);
     `blastproof run: ${selected.length} test(s), provider=${provider} model=${modelId}, base_url=${config.base_url}`
   );
   const runMask = buildRunMask(config, parsed);
+  const budget = options.budget ?? new RunBudget(resolveBudgetOptions(config, options));
   const browser = await chromium.launch({ headless: config.browser.headless });
+  let incomplete;
   try {
     let session;
     if (config.auth) {
@@ -1795,28 +1983,54 @@ ${results.length} test file(s) could not be parsed:`);
           baseUrl: config.base_url,
           allowedOrigins: config.allowed_origins,
           browser,
-          brain: createBrain(createModel(config.llm).model),
+          brain: createBrain(createModel(config.llm).model, void 0, budget),
           maxRetries: config.max_retries_per_step,
           mask: runMask,
           onEvent: printEvent
         });
       } catch (error) {
-        if (error instanceof AuthError) {
+        if (error instanceof BudgetExhaustedError) {
+          incomplete = error;
+        } else if (error instanceof AuthError) {
           console.error(`error: ${error.message}`);
           return EXIT_USAGE;
+        } else {
+          throw error;
         }
-        throw error;
       }
     }
-    for (const test of selected) {
-      console.log(`
+    if (incomplete) {
+      results.push(...notRunResults(selected, incomplete.message));
+    } else {
+      for (let i = 0; i < selected.length; i++) {
+        const test = selected[i];
+        console.log(`
 > ${test.summary} [${test.priority}] (${path9.relative(options.cwd, test.path)})`);
-      results.push(await runOne(browser, test, config, sessionDir, session, runMask));
+        try {
+          budget.check();
+          results.push(await runOne(browser, test, config, sessionDir, session, runMask, budget));
+        } catch (error) {
+          if (error instanceof BudgetExhaustedError) {
+            incomplete = error;
+            results.push(...notRunResults(selected.slice(i), error.message));
+            break;
+          }
+          throw error;
+        }
+      }
     }
   } finally {
     await browser.close();
   }
-  return finalize(results, selection.unroutedSkipped, options, sessionDir, Date.now() - startedAt, impact);
+  return finalize(
+    results,
+    selection.unroutedSkipped,
+    options,
+    sessionDir,
+    Date.now() - startedAt,
+    impact,
+    incomplete
+  );
 }
 
 // src/commands/plan.ts
@@ -1911,7 +2125,8 @@ async function planCommand(options) {
     throw error;
   }
   const { model, provider, modelId } = createModel(config.llm);
-  const brain = createPlanner(model);
+  const budget = options.budget ?? new RunBudget(resolveBudgetOptions(config, options));
+  const brain = createPlanner(model, void 0, budget);
   const base = options.routes && options.routes.length > 0 ? void 0 : options.base ?? "main";
   console.log(
     `blastproof plan: ${work.length} route(s), provider=${provider} model=${modelId}, base_url=${config.base_url}`
@@ -1926,6 +2141,7 @@ async function planCommand(options) {
   const generated = [];
   const written = [];
   const failed = [];
+  let incomplete;
   const browser = await chromium2.launch({ headless: config.browser.headless });
   try {
     let session;
@@ -1938,20 +2154,23 @@ async function planCommand(options) {
           baseUrl: config.base_url,
           allowedOrigins: config.allowed_origins,
           browser,
-          brain: createBrain(createModel(config.llm).model),
+          brain: createBrain(createModel(config.llm).model, void 0, budget),
           maxRetries: config.max_retries_per_step,
           mask,
           onEvent: printAuthEvent
         });
       } catch (error) {
-        if (error instanceof AuthError) {
+        if (error instanceof BudgetExhaustedError) {
+          incomplete = error;
+        } else if (error instanceof AuthError) {
           console.error(`error: ${error.message}`);
           return EXIT_USAGE;
+        } else {
+          throw error;
         }
-        throw error;
       }
     }
-    for (const { route, changedFiles } of work) {
+    for (const { route, changedFiles } of incomplete ? [] : work) {
       console.log(`
 > ${route}`);
       let draft;
@@ -1967,6 +2186,10 @@ async function planCommand(options) {
           mask: (text) => mask.mask(text)
         });
       } catch (error) {
+        if (error instanceof BudgetExhaustedError) {
+          incomplete = error;
+          break;
+        }
         failed.push({ route, reason: error instanceof Error ? error.message : String(error) });
         console.log(`  X failed: ${error instanceof Error ? error.message : String(error)}`);
         continue;
@@ -1996,6 +2219,7 @@ ${renderTestYaml(draft, { route, base })}`);
   } finally {
     await browser.close();
   }
+  const notAttempted = incomplete ? work.map((item) => item.route).filter((route) => !generated.includes(route) && !failed.some((f) => f.route === route)) : [];
   console.log("\n--- Plan -------------------------------------------------------");
   console.log(`Generated: ${generated.length > 0 ? generated.join(", ") : "none"}`);
   if (written.length > 0) {
@@ -2009,8 +2233,15 @@ ${renderTestYaml(draft, { route, base })}`);
     console.log("Failed:");
     for (const { route, reason } of failed) console.log(`  ${route}: ${reason}`);
   }
+  if (incomplete) {
+    console.log(`Stopped: ${incomplete.message}`);
+    if (notAttempted.length > 0) {
+      console.log("Not attempted (run out of budget):");
+      for (const route of notAttempted) console.log(`  ${route}`);
+    }
+  }
   console.log("---------------------------------------------------------------");
-  return failed.length > 0 ? EXIT_FAILED : EXIT_OK;
+  return incomplete || failed.length > 0 ? EXIT_FAILED : EXIT_OK;
 }
 
 // src/commands/test.ts
@@ -2019,6 +2250,17 @@ function section(title) {
 === ${title} ${"=".repeat(Math.max(0, 58 - title.length))}`);
 }
 async function testCommand(options) {
+  let config;
+  try {
+    config = applyUrlOverride(await loadConfig(options.cwd), options.url);
+  } catch (error) {
+    if (error instanceof ConfigError) {
+      console.error(`error: ${error.message}`);
+      return EXIT_USAGE;
+    }
+    throw error;
+  }
+  const budget = new RunBudget(resolveBudgetOptions(config, options));
   section("Verify: tests covering the affected routes");
   const runCode = await runCommand({
     cwd: options.cwd,
@@ -2029,7 +2271,8 @@ async function testCommand(options) {
     minScore: options.minScore,
     junit: options.junit,
     html: options.html,
-    failOnUnmapped: options.failOnUnmapped
+    failOnUnmapped: options.failOnUnmapped,
+    budget
   });
   if (runCode === EXIT_USAGE) return EXIT_USAGE;
   section("Draft: affected routes no test covers");
@@ -2037,7 +2280,8 @@ async function testCommand(options) {
     cwd: options.cwd,
     base: options.base,
     url: options.url,
-    write: options.write
+    write: options.write,
+    budget
   });
   if (planCode === EXIT_USAGE) return EXIT_USAGE;
   console.log(
@@ -2063,8 +2307,26 @@ function parseMinScore(value) {
   }
   return score;
 }
+function parsePositiveInt(flag) {
+  return (value) => {
+    const n = Number(value);
+    if (!Number.isInteger(n) || n <= 0) {
+      throw new InvalidArgumentError(`${flag} must be a positive integer`);
+    }
+    return n;
+  };
+}
+function parsePositiveNumber(flag) {
+  return (value) => {
+    const n = Number(value);
+    if (!Number.isFinite(n) || n <= 0) {
+      throw new InvalidArgumentError(`${flag} must be a positive number`);
+    }
+    return n;
+  };
+}
 var program = new Command();
-program.name("blastproof").description("Open-source AI testing agent: plain-English YAML tests executed agentically on a real browser.").version("0.2.2");
+program.name("blastproof").description("Open-source AI testing agent: plain-English YAML tests executed agentically on a real browser.").version("0.3.0");
 program.command("init").description("Scaffold .blastproof/ (config, tests, sample tests) in the current directory").action(async () => {
   try {
     const result = await initProject(process.cwd());
@@ -2079,7 +2341,19 @@ program.command("run").description("Discover and run all tests under .blastproof
   "--min-score <n>",
   "require a weighted score of at least n (0-100); replaces the all-must-pass rule",
   parseMinScore
-).option("--junit [path]", "write a JUnit XML report (default: .blastproof/reports/<session>/junit.xml)").option("--html [path]", "write a self-contained HTML report (default: .blastproof/reports/<session>/report.html)").option("--fail-on-unmapped", "fail when a changed file matches no routes: or ignore: glob").action(
+).option("--junit [path]", "write a JUnit XML report (default: .blastproof/reports/<session>/junit.xml)").option("--html [path]", "write a self-contained HTML report (default: .blastproof/reports/<session>/report.html)").option("--fail-on-unmapped", "fail when a changed file matches no routes: or ignore: glob").option(
+  "--max-llm-calls <n>",
+  "stop the run after this many model calls, reported as incomplete (overrides config)",
+  parsePositiveInt("--max-llm-calls")
+).option(
+  "--max-tokens <n>",
+  "stop the run after this many tokens spent across all model calls (overrides config)",
+  parsePositiveInt("--max-tokens")
+).option(
+  "--max-duration <seconds>",
+  "stop the run after this many seconds of wall-clock time (overrides config)",
+  parsePositiveNumber("--max-duration")
+).action(
   async (options) => {
     try {
       process.exitCode = await runCommand({
@@ -2094,7 +2368,10 @@ program.command("run").description("Discover and run all tests under .blastproof
         minScore: options.minScore,
         junit: options.junit,
         html: options.html,
-        failOnUnmapped: options.failOnUnmapped
+        failOnUnmapped: options.failOnUnmapped,
+        maxLlmCalls: options.maxLlmCalls,
+        maxTokens: options.maxTokens,
+        maxDuration: options.maxDuration
       });
     } catch (error) {
       console.error(`error: ${error instanceof Error ? error.message : String(error)}`);
@@ -2102,25 +2379,54 @@ program.command("run").description("Discover and run all tests under .blastproof
     }
   }
 );
-program.command("plan").description("Generate plain-English YAML tests for affected routes no test covers yet").option("--base <ref>", "base git ref for the diff", "main").option("--url <url>", "override config base_url for this run only (config file untouched)").option("--route <route>", "generate for this route, bypassing the diff (repeatable)", collect, []).option("--write", "persist drafts under .blastproof/tests/ instead of previewing them").action(async (options) => {
-  try {
-    process.exitCode = await planCommand({
-      cwd: process.cwd(),
-      base: options.base,
-      url: options.url,
-      routes: options.route,
-      write: options.write
-    });
-  } catch (error) {
-    console.error(`error: ${error instanceof Error ? error.message : String(error)}`);
-    process.exitCode = EXIT_USAGE;
+program.command("plan").description("Generate plain-English YAML tests for affected routes no test covers yet").option("--base <ref>", "base git ref for the diff", "main").option("--url <url>", "override config base_url for this run only (config file untouched)").option("--route <route>", "generate for this route, bypassing the diff (repeatable)", collect, []).option("--write", "persist drafts under .blastproof/tests/ instead of previewing them").option(
+  "--max-llm-calls <n>",
+  "stop after this many model calls, reported as incomplete (overrides config)",
+  parsePositiveInt("--max-llm-calls")
+).option(
+  "--max-tokens <n>",
+  "stop after this many tokens spent across all model calls (overrides config)",
+  parsePositiveInt("--max-tokens")
+).option(
+  "--max-duration <seconds>",
+  "stop after this many seconds of wall-clock time (overrides config)",
+  parsePositiveNumber("--max-duration")
+).action(
+  async (options) => {
+    try {
+      process.exitCode = await planCommand({
+        cwd: process.cwd(),
+        base: options.base,
+        url: options.url,
+        routes: options.route,
+        write: options.write,
+        maxLlmCalls: options.maxLlmCalls,
+        maxTokens: options.maxTokens,
+        maxDuration: options.maxDuration
+      });
+    } catch (error) {
+      console.error(`error: ${error instanceof Error ? error.message : String(error)}`);
+      process.exitCode = EXIT_USAGE;
+    }
   }
-});
+);
 program.command("test").description("Full PR pipeline: run the tests covering the diff, then draft tests for the gaps").option("--base <ref>", "base git ref for the diff", "main").option("--url <url>", "override config base_url for this run only (config file untouched)").option(
   "--min-score <n>",
   "require a weighted score of at least n (0-100); replaces the all-must-pass rule",
   parseMinScore
-).option("--junit [path]", "write a JUnit XML report (default: .blastproof/reports/<session>/junit.xml)").option("--html [path]", "write a self-contained HTML report (default: .blastproof/reports/<session>/report.html)").option("--write", "persist generated drafts under .blastproof/tests/ instead of previewing them").option("--fail-on-unmapped", "fail when a changed file matches no routes: or ignore: glob").action(
+).option("--junit [path]", "write a JUnit XML report (default: .blastproof/reports/<session>/junit.xml)").option("--html [path]", "write a self-contained HTML report (default: .blastproof/reports/<session>/report.html)").option("--write", "persist generated drafts under .blastproof/tests/ instead of previewing them").option("--fail-on-unmapped", "fail when a changed file matches no routes: or ignore: glob").option(
+  "--max-llm-calls <n>",
+  "stop the pipeline after this many model calls, shared by both phases (overrides config)",
+  parsePositiveInt("--max-llm-calls")
+).option(
+  "--max-tokens <n>",
+  "stop the pipeline after this many tokens, shared by both phases (overrides config)",
+  parsePositiveInt("--max-tokens")
+).option(
+  "--max-duration <seconds>",
+  "stop the pipeline after this many seconds of wall-clock time (overrides config)",
+  parsePositiveNumber("--max-duration")
+).action(
   async (options) => {
     try {
       process.exitCode = await testCommand({
@@ -2131,7 +2437,10 @@ program.command("test").description("Full PR pipeline: run the tests covering th
         junit: options.junit,
         html: options.html,
         write: options.write,
-        failOnUnmapped: options.failOnUnmapped
+        failOnUnmapped: options.failOnUnmapped,
+        maxLlmCalls: options.maxLlmCalls,
+        maxTokens: options.maxTokens,
+        maxDuration: options.maxDuration
       });
     } catch (error) {
       console.error(`error: ${error instanceof Error ? error.message : String(error)}`);