blastproof 0.2.2 → 0.4.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/cli.js CHANGED
@@ -26,6 +26,9 @@ llm:
26
26
 
27
27
  browser:
28
28
  headless: true
29
+ # How long to wait for an element to appear (self-healing resolution) as well as
30
+ # for the action performed on it and for navigation. Raising it makes a slow app
31
+ # wait longer to succeed, and a genuinely missing element take longer to fail.
29
32
  timeout_ms: 30000
30
33
 
31
34
  # Budget of self-healing retries per step (element re-resolution via accessibility tree).
@@ -152,23 +155,110 @@ function formatInitGuidance(result) {
152
155
  lines.push(
153
156
  "",
154
157
  "Next steps:",
155
- " 1. Set your LLM API key: export ANTHROPIC_API_KEY=... (or OPENAI_API_KEY; ollama needs no key)",
156
- " 2. Start your app, then point base_url at it in .blastproof/config.yaml",
157
- " 3. Run your tests: blastproof run",
158
+ // Deliberately keyless first. A first-time-user trial found that following
159
+ // the old order — key, app, run — meant hitting the missing-key error and
160
+ // then a browser-launch failure with nothing successful behind you, which
161
+ // reads as "this is broken" rather than "one leg needs setting up". Seeing
162
+ // real output first reframes both.
163
+ " 1. Point base_url at your app in .blastproof/config.yaml",
164
+ " 2. Check the setup: blastproof run --dry-run (no API key or browser needed)",
165
+ " 3. Set your LLM API key: export ANTHROPIC_API_KEY=... (or OPENAI_API_KEY; ollama needs no key)",
166
+ " 4. Install the browser: npx playwright install chromium",
167
+ " 5. Run your tests: blastproof run",
158
168
  "",
159
- "First time with Playwright? Install the browser: npx playwright install chromium"
169
+ "Without a key or a browser you can still map a diff to routes and gate on",
170
+ "coverage: blastproof run --impacted --fail-on-unmapped --dry-run"
160
171
  );
161
172
  return lines.join("\n");
162
173
  }
163
174
 
164
175
  // src/commands/plan.ts
165
176
  import path10 from "path";
166
- import { chromium as chromium2 } from "playwright";
167
177
 
168
178
  // src/auth.ts
169
179
  import { readFile, writeFile as writeFile2, mkdir as mkdir3 } from "fs/promises";
170
180
  import path3 from "path";
171
181
 
182
+ // src/runner/budget.ts
183
+ function describeLimit(limit, observed, configured) {
184
+ switch (limit) {
185
+ case "calls":
186
+ return `model call budget exhausted: reached the configured maximum of ${configured} call(s)`;
187
+ case "tokens":
188
+ return `token budget exhausted: reached the configured maximum of ${configured} token(s) (observed ${observed})`;
189
+ case "duration":
190
+ return `deadline exceeded: reached the configured maximum of ${(configured / 1e3).toFixed(0)}s (elapsed ${(observed / 1e3).toFixed(1)}s)`;
191
+ }
192
+ }
193
+ var BudgetExhaustedError = class extends Error {
194
+ limit;
195
+ observed;
196
+ configured;
197
+ constructor(limit, observed, configured) {
198
+ super(describeLimit(limit, observed, configured));
199
+ this.name = "BudgetExhaustedError";
200
+ this.limit = limit;
201
+ this.observed = observed;
202
+ this.configured = configured;
203
+ }
204
+ };
205
+ var RunBudget = class {
206
+ maxCalls;
207
+ maxTokens;
208
+ maxDurationMs;
209
+ now;
210
+ startedAt;
211
+ calls = 0;
212
+ tokens = 0;
213
+ constructor(options = {}) {
214
+ this.maxCalls = options.maxCalls;
215
+ this.maxTokens = options.maxTokens;
216
+ this.maxDurationMs = options.maxDurationMs;
217
+ this.now = options.now ?? Date.now;
218
+ this.startedAt = this.now();
219
+ }
220
+ /** Model calls recorded so far. */
221
+ get callCount() {
222
+ return this.calls;
223
+ }
224
+ /** Tokens recorded so far, summed across every completed call. */
225
+ get tokenCount() {
226
+ return this.tokens;
227
+ }
228
+ /**
229
+ * Throws {@link BudgetExhaustedError} when the next unit of work could exceed a
230
+ * configured limit. Called before spending anything, never after — so an
231
+ * over-budget call is never issued and then rejected (spec: "the call is not
232
+ * made").
233
+ */
234
+ check() {
235
+ if (this.maxCalls !== void 0 && this.calls >= this.maxCalls) {
236
+ throw new BudgetExhaustedError("calls", this.calls, this.maxCalls);
237
+ }
238
+ if (this.maxTokens !== void 0 && this.tokens >= this.maxTokens) {
239
+ throw new BudgetExhaustedError("tokens", this.tokens, this.maxTokens);
240
+ }
241
+ if (this.maxDurationMs !== void 0) {
242
+ const elapsed = this.now() - this.startedAt;
243
+ if (elapsed >= this.maxDurationMs) {
244
+ throw new BudgetExhaustedError("duration", elapsed, this.maxDurationMs);
245
+ }
246
+ }
247
+ }
248
+ /** Records what a completed model call spent. */
249
+ record(usage) {
250
+ this.calls += 1;
251
+ if (usage?.totalTokens !== void 0) this.tokens += usage.totalTokens;
252
+ }
253
+ };
254
+ function estimateMaxModelCalls(tests, maxIterationsPerStep, maxRetriesPerStep) {
255
+ const totalSteps = tests.reduce(
256
+ (sum, test) => sum + (test.setup?.length ?? 0) + test.steps.length,
257
+ 0
258
+ );
259
+ return totalSteps * (maxIterationsPerStep + maxRetriesPerStep);
260
+ }
261
+
172
262
  // src/runner/env.ts
173
263
  var ENV_PLACEHOLDER = /\{\{\s*env\.([A-Za-z_][A-Za-z0-9_]*)\s*\}\}/g;
174
264
  var MissingEnvError = class extends Error {
@@ -307,7 +397,7 @@ async function performAction(page, action, ctx) {
307
397
  const value = resolve(requireValue(action), ctx);
308
398
  const url = new URL(value, ctx.baseUrl);
309
399
  assertAllowedOrigin(url, ctx);
310
- await page.goto(url.toString(), { timeout: 3e4 });
400
+ await page.goto(url.toString(), { timeout: ctx.resolveTimeoutMs ?? 3e4 });
311
401
  return `ok: navigated to ${url.toString()}`;
312
402
  }
313
403
  case "click": {
@@ -348,9 +438,10 @@ async function performAction(page, action, ctx) {
348
438
  }
349
439
 
350
440
  // src/runner/executor.ts
351
- async function defaultSnapshot(page) {
441
+ var DEFAULT_MAX_ITERATIONS_PER_STEP = 15;
442
+ async function defaultSnapshot(page, maxLines) {
352
443
  const { captureSnapshot } = await import("./snapshot-CAIB2OHX.js");
353
- return captureSnapshot(page);
444
+ return captureSnapshot(page, { maxLines });
354
445
  }
355
446
  function slugify(text) {
356
447
  const slug = text.toLowerCase().replace(/[^a-z0-9]+/g, "-").replace(/^-+|-+$/g, "").slice(0, 60);
@@ -366,12 +457,15 @@ async function executeTest(page, test, options) {
366
457
  allowedOrigins,
367
458
  resolveValue,
368
459
  maxRetries = 3,
369
- maxIterationsPerStep = 15,
460
+ maxIterationsPerStep = DEFAULT_MAX_ITERATIONS_PER_STEP,
461
+ timeoutMs,
370
462
  mask = (s) => s,
371
- snapshot = defaultSnapshot,
463
+ snapshot,
464
+ maxSnapshotLines,
372
465
  onEvent = () => {
373
466
  }
374
467
  } = options;
468
+ const takeSnapshot = snapshot ?? ((page2) => defaultSnapshot(page2, maxSnapshotLines));
375
469
  const startedAt = Date.now();
376
470
  const stepResults = [];
377
471
  const allSteps = [
@@ -402,7 +496,7 @@ async function executeTest(page, test, options) {
402
496
  if (iterations >= maxIterationsPerStep) {
403
497
  throw new StepFailure(`step exceeded ${maxIterationsPerStep} actions without completing`);
404
498
  }
405
- const snap = await snapshot(page);
499
+ const snap = await takeSnapshot(page);
406
500
  let action;
407
501
  try {
408
502
  action = await brain.nextAction({
@@ -414,6 +508,7 @@ async function executeTest(page, test, options) {
414
508
  iterationsLeft: maxIterationsPerStep - iterations
415
509
  });
416
510
  } catch (error) {
511
+ if (error instanceof BudgetExhaustedError) throw error;
417
512
  failedAttempts++;
418
513
  lastResult = `error: ${error instanceof Error ? error.message : String(error)}`;
419
514
  if (failedAttempts >= maxRetries) {
@@ -437,7 +532,7 @@ async function executeTest(page, test, options) {
437
532
  emitAction(index, action, result);
438
533
  if (judgment.pass) {
439
534
  lastResult = result;
440
- continue;
535
+ break;
441
536
  }
442
537
  failedAttempts++;
443
538
  lastResult = result;
@@ -447,7 +542,12 @@ async function executeTest(page, test, options) {
447
542
  continue;
448
543
  }
449
544
  try {
450
- const result = await performAction(page, action, { baseUrl, allowedOrigins, resolveValue });
545
+ const result = await performAction(page, action, {
546
+ baseUrl,
547
+ allowedOrigins,
548
+ resolveValue,
549
+ resolveTimeoutMs: timeoutMs
550
+ });
451
551
  lastResult = result;
452
552
  emitAction(index, action, result);
453
553
  } catch (error) {
@@ -460,6 +560,7 @@ async function executeTest(page, test, options) {
460
560
  }
461
561
  }
462
562
  } catch (error) {
563
+ if (error instanceof BudgetExhaustedError) throw error;
463
564
  stepFailedReason = mask(error instanceof Error ? error.message : String(error));
464
565
  }
465
566
  const status = stepFailedReason ? "failed" : "passed";
@@ -560,14 +661,16 @@ async function runJourney(...args) {
560
661
  try {
561
662
  return await executeTest(...args);
562
663
  } catch (error) {
664
+ if (error instanceof BudgetExhaustedError) throw error;
563
665
  throw new AuthError(
564
666
  `Authentication could not run: ${error instanceof Error ? error.message : String(error)}`
565
667
  );
566
668
  }
567
669
  }
568
670
  async function fromSteps(options) {
569
- const { auth, baseUrl, browser, brain, maxRetries, snapshot = defaultSnapshot, onEvent, mask } = options;
671
+ const { auth, baseUrl, browser, brain, maxRetries, snapshot, timeoutMs, maxSnapshotLines, onEvent, mask } = options;
570
672
  const steps = auth.steps;
673
+ const takeSnapshot = snapshot ?? ((page) => defaultSnapshot(page, maxSnapshotLines));
571
674
  const context = await browser.newContext();
572
675
  try {
573
676
  const page = await context.newPage();
@@ -589,8 +692,9 @@ async function fromSteps(options) {
589
692
  allowedOrigins: options.allowedOrigins,
590
693
  resolveValue: (value) => substituteEnv(value),
591
694
  maxRetries,
695
+ timeoutMs,
592
696
  mask: (text) => mask.mask(text),
593
- snapshot,
697
+ snapshot: takeSnapshot,
594
698
  onEvent
595
699
  }
596
700
  );
@@ -600,7 +704,7 @@ async function fromSteps(options) {
600
704
  );
601
705
  }
602
706
  if (auth.verify) {
603
- const judgment = await brain.judge(auth.verify, mask.mask(await snapshot(page)));
707
+ const judgment = await brain.judge(auth.verify, mask.mask(await takeSnapshot(page)));
604
708
  if (!judgment.pass) {
605
709
  throw new AuthError(`Authentication could not be verified: ${mask.mask(judgment.reason)}`);
606
710
  }
@@ -661,7 +765,25 @@ var llmSchema = z.object({
661
765
  });
662
766
  var browserSchema = z.object({
663
767
  headless: z.boolean().default(true),
664
- timeout_ms: z.number().int().positive().default(3e4)
768
+ /**
769
+ * Bounds every wait: resolving a target element on the accessibility tree, and
770
+ * navigating. An explicit per-call Playwright timeout always overrides
771
+ * `page.setDefaultTimeout()`, so this must be threaded through the action
772
+ * context rather than relied on ambiently (design D2, runner/actions.ts).
773
+ */
774
+ timeout_ms: z.number().int().positive().default(3e4),
775
+ /**
776
+ * Caps accessibility-tree lines sent to the model per snapshot. Optional:
777
+ * undefined lets `runner/snapshot.ts`'s own default (200) apply, so a config
778
+ * that never mentions this behaves exactly as before it existed.
779
+ */
780
+ max_snapshot_lines: z.number().int().positive().optional()
781
+ });
782
+ var budgetSchema = z.object({
783
+ max_llm_calls: z.coerce.number().int().positive().optional(),
784
+ max_tokens: z.coerce.number().int().positive().optional(),
785
+ /** Seconds, not milliseconds, to match how a human would set a deadline. */
786
+ max_duration_s: z.coerce.number().positive().optional()
665
787
  });
666
788
  var cookieSchema = z.object({
667
789
  name: z.string().min(1),
@@ -712,14 +834,18 @@ var configSchema = z.object({
712
834
  */
713
835
  allowed_origins: z.array(z.string().url()).optional(),
714
836
  auth: authSchema.optional(),
715
- max_retries_per_step: z.number().int().min(1).default(3)
837
+ max_retries_per_step: z.number().int().min(1).default(3),
838
+ budget: budgetSchema.optional()
716
839
  });
717
840
  var ENV_OVERRIDES = {
718
841
  BLASTPROOF_BASE_URL: ["base_url"],
719
842
  BLASTPROOF_LLM_PROVIDER: ["llm", "provider"],
720
843
  BLASTPROOF_LLM_MODEL: ["llm", "model"],
721
844
  BLASTPROOF_LLM_BASE_URL: ["llm", "base_url"],
722
- BLASTPROOF_LLM_API_KEY_ENV: ["llm", "api_key_env"]
845
+ BLASTPROOF_LLM_API_KEY_ENV: ["llm", "api_key_env"],
846
+ BLASTPROOF_MAX_LLM_CALLS: ["budget", "max_llm_calls"],
847
+ BLASTPROOF_MAX_TOKENS: ["budget", "max_tokens"],
848
+ BLASTPROOF_MAX_DURATION_S: ["budget", "max_duration_s"]
723
849
  };
724
850
  function applyEnvOverrides(data, env = process.env) {
725
851
  const merged = data && typeof data === "object" ? { ...data } : {};
@@ -750,6 +876,37 @@ var ConfigError = class extends Error {
750
876
  function formatIssues(error) {
751
877
  return error.issues.map((issue) => ` - ${issue.path.join(".") || "(root)"}: ${issue.message}`).join("\n");
752
878
  }
879
+ function unwrapToObjectSchema(schema) {
880
+ let current = schema;
881
+ for (; ; ) {
882
+ if (current instanceof z.ZodOptional || current instanceof z.ZodNullable) {
883
+ current = current.unwrap();
884
+ } else if (current instanceof z.ZodDefault) {
885
+ current = current.removeDefault();
886
+ } else if (current instanceof z.ZodEffects) {
887
+ current = current.innerType();
888
+ } else {
889
+ break;
890
+ }
891
+ }
892
+ return current instanceof z.ZodObject ? current : void 0;
893
+ }
894
+ function findUnknownConfigKeys(data, schema = configSchema, path11 = []) {
895
+ const object = unwrapToObjectSchema(schema);
896
+ if (!object || data === null || typeof data !== "object" || Array.isArray(data)) return [];
897
+ const shape = object.shape;
898
+ const warnings = [];
899
+ for (const key of Object.keys(data)) {
900
+ const fieldPath = [...path11, key];
901
+ const fieldSchema = shape[key];
902
+ if (!fieldSchema) {
903
+ warnings.push(fieldPath.join("."));
904
+ continue;
905
+ }
906
+ warnings.push(...findUnknownConfigKeys(data[key], fieldSchema, fieldPath));
907
+ }
908
+ return warnings;
909
+ }
753
910
  async function loadConfig(cwd = process.cwd(), env = process.env) {
754
911
  const configPath = path4.join(cwd, CONFIG_RELATIVE_PATH);
755
912
  let raw;
@@ -769,12 +926,17 @@ async function loadConfig(cwd = process.cwd(), env = process.env) {
769
926
  );
770
927
  }
771
928
  const { data: merged, applied } = applyEnvOverrides(data ?? {}, env);
929
+ const source = applied.length > 0 ? `${CONFIG_RELATIVE_PATH} (with overrides from ${applied.join(", ")})` : CONFIG_RELATIVE_PATH;
772
930
  const result = configSchema.safeParse(merged);
773
931
  if (!result.success) {
774
- const source = applied.length > 0 ? `${CONFIG_RELATIVE_PATH} (with overrides from ${applied.join(", ")})` : CONFIG_RELATIVE_PATH;
775
932
  throw new ConfigError(`Invalid ${source}:
776
933
  ${formatIssues(result.error)}`);
777
934
  }
935
+ for (const key of findUnknownConfigKeys(merged)) {
936
+ console.warn(
937
+ `warning: unknown config key '${key}' in ${source} \u2014 this version of blastproof does not recognise it, so it has no effect.`
938
+ );
939
+ }
778
940
  return result.data;
779
941
  }
780
942
 
@@ -967,10 +1129,16 @@ var MalformedModelOutputError = class extends Error {
967
1129
  this.name = "MalformedModelOutputError";
968
1130
  }
969
1131
  };
970
- function createBrain(model, generate = generateObject) {
1132
+ async function countedGenerate(generate, budget, options) {
1133
+ budget.check();
1134
+ const result = await generate(options);
1135
+ budget.record(result.usage);
1136
+ return result;
1137
+ }
1138
+ function createBrain(model, generate = generateObject, budget) {
971
1139
  return {
972
1140
  async nextAction(input) {
973
- const result = await generate({
1141
+ const result = await countedGenerate(generate, budget, {
974
1142
  model,
975
1143
  schema: agentActionSchema,
976
1144
  system: agentSystemPrompt(),
@@ -985,7 +1153,7 @@ function createBrain(model, generate = generateObject) {
985
1153
  return parsed.data;
986
1154
  },
987
1155
  async judge(expectation, snapshot) {
988
- const result = await generate({
1156
+ const result = await countedGenerate(generate, budget, {
989
1157
  model,
990
1158
  schema: assertJudgmentSchema,
991
1159
  system: assertSystemPrompt(),
@@ -1001,10 +1169,10 @@ function createBrain(model, generate = generateObject) {
1001
1169
  }
1002
1170
  };
1003
1171
  }
1004
- function createPlanner(model, generate = generateObject) {
1172
+ function createPlanner(model, generate = generateObject, budget) {
1005
1173
  return {
1006
1174
  async planTest(input) {
1007
- const result = await generate({
1175
+ const result = await countedGenerate(generate, budget, {
1008
1176
  model,
1009
1177
  schema: generatedTestSchema,
1010
1178
  system: plannerSystemPrompt(),
@@ -1177,10 +1345,11 @@ function renderTestYaml(draft, meta) {
1177
1345
  });
1178
1346
  }
1179
1347
  async function generateForRoute(page, options) {
1180
- const { route, baseUrl, changedFiles, brain, mask, snapshot = defaultSnapshot } = options;
1348
+ const { route, baseUrl, changedFiles, brain, mask, snapshot, maxSnapshotLines, timeoutMs } = options;
1349
+ const takeSnapshot = snapshot ?? ((p) => defaultSnapshot(p, maxSnapshotLines));
1181
1350
  const url = new URL(route, baseUrl).toString();
1182
1351
  try {
1183
- await page.goto(url, { timeout: 3e4 });
1352
+ await page.goto(url, { timeout: timeoutMs });
1184
1353
  } catch (error) {
1185
1354
  throw new PlannerError(
1186
1355
  `Cannot load ${url}: ${error instanceof Error ? error.message : String(error)}`
@@ -1188,7 +1357,7 @@ async function generateForRoute(page, options) {
1188
1357
  }
1189
1358
  const generated = await brain.planTest({
1190
1359
  route,
1191
- snapshot: mask(await snapshot(page)),
1360
+ snapshot: mask(await takeSnapshot(page)),
1192
1361
  changedFiles
1193
1362
  });
1194
1363
  const leaked = findSecretLiterals(generated.steps);
@@ -1217,9 +1386,120 @@ function coveredRoutes(tests) {
1217
1386
  return new Set(tests.flatMap((test) => test.routes));
1218
1387
  }
1219
1388
 
1389
+ // src/runner/browser.ts
1390
+ import { chromium } from "playwright";
1391
+ var BrowserLaunchError = class extends Error {
1392
+ constructor(message) {
1393
+ super(message);
1394
+ this.name = "BrowserLaunchError";
1395
+ }
1396
+ };
1397
+ var MISSING_LIBRARY_PATTERN = /([^\s:]+\.so(?:\.\d+)*): cannot open shared object file/;
1398
+ var MISSING_EXECUTABLE_PATTERN = /executable doesn't exist/i;
1399
+ function stripCommandLine(message) {
1400
+ return message.split("\n").filter((line) => (line.match(/\s--[\w-]+/g)?.length ?? 0) < 3).join("\n").trim();
1401
+ }
1402
+ async function launchBrowser(options) {
1403
+ try {
1404
+ return await chromium.launch(options);
1405
+ } catch (error) {
1406
+ const message = error instanceof Error ? error.message : String(error);
1407
+ if (MISSING_EXECUTABLE_PATTERN.test(message)) {
1408
+ throw new BrowserLaunchError(
1409
+ "The browser is not installed. Run `npx playwright install chromium` to download it."
1410
+ );
1411
+ }
1412
+ const missingLibrary = message.match(MISSING_LIBRARY_PATTERN);
1413
+ if (missingLibrary) {
1414
+ throw new BrowserLaunchError(
1415
+ `The browser could not start because a system library is missing: ${missingLibrary[1]}. Run \`npx playwright install-deps chromium\` to install it \u2014 this needs root/administrator privileges (sudo, or an elevated shell on Windows). Without them, blastproof run --dry-run and blastproof plan --dry-run need neither a browser nor a key.`
1416
+ );
1417
+ }
1418
+ throw new BrowserLaunchError(stripCommandLine(message));
1419
+ }
1420
+ }
1421
+
1422
+ // src/preflight.ts
1423
+ var REACHABILITY_TIMEOUT_MS = 15e3;
1424
+ function messageOf(error) {
1425
+ return error instanceof Error ? error.message : String(error);
1426
+ }
1427
+ function isTimeout(error) {
1428
+ return error instanceof Error && error.name === "AbortError";
1429
+ }
1430
+ async function probe(url, fetchImpl) {
1431
+ const controller = new AbortController();
1432
+ const timer = setTimeout(() => controller.abort(), REACHABILITY_TIMEOUT_MS);
1433
+ try {
1434
+ await fetchImpl(url, { signal: controller.signal });
1435
+ } finally {
1436
+ clearTimeout(timer);
1437
+ }
1438
+ }
1439
+ function providerBaseUrl(llm) {
1440
+ if (llm.base_url) return llm.base_url;
1441
+ switch (llm.provider) {
1442
+ case "anthropic":
1443
+ return "https://api.anthropic.com";
1444
+ case "openai":
1445
+ return "https://api.openai.com";
1446
+ case "ollama":
1447
+ return DEFAULT_OLLAMA_BASE_URL;
1448
+ }
1449
+ }
1450
+ async function checkModel(llm, fetchImpl) {
1451
+ const url = providerBaseUrl(llm);
1452
+ try {
1453
+ await probe(url, fetchImpl);
1454
+ return void 0;
1455
+ } catch (error) {
1456
+ if (isTimeout(error)) {
1457
+ return `Cannot reach the ${llm.provider} provider at ${url}: it did not respond within ${REACHABILITY_TIMEOUT_MS / 1e3}s. It may be temporarily slow or overloaded rather than down \u2014 try again in a moment, or check llm.base_url in .blastproof/config.yaml.`;
1458
+ }
1459
+ return `Cannot reach the ${llm.provider} provider at ${url} (${messageOf(error)}). Check your network connection` + (llm.provider === "ollama" ? ", that Ollama is running (`ollama serve`)," : "") + " or llm.base_url in .blastproof/config.yaml.";
1460
+ }
1461
+ }
1462
+ async function checkBaseUrl(baseUrl, fetchImpl) {
1463
+ try {
1464
+ await probe(baseUrl, fetchImpl);
1465
+ return void 0;
1466
+ } catch (error) {
1467
+ if (isTimeout(error)) {
1468
+ return `The application at ${baseUrl} did not respond within ${REACHABILITY_TIMEOUT_MS / 1e3}s. It may still be starting (a dev server compiling a route on first request, a cold-starting container) rather than down \u2014 try again in a moment.`;
1469
+ }
1470
+ return `The application at ${baseUrl} is not responding (${messageOf(error)}). Start it, or point --url / base_url in .blastproof/config.yaml at the right address.`;
1471
+ }
1472
+ }
1473
+ async function checkBrowser(headless) {
1474
+ try {
1475
+ const browser = await launchBrowser({ headless });
1476
+ return { browser };
1477
+ } catch (error) {
1478
+ return { failure: error instanceof BrowserLaunchError ? error.message : messageOf(error) };
1479
+ }
1480
+ }
1481
+ async function runPreflight(needs, config, fetchImpl = fetch) {
1482
+ const [browserResult, modelFailure, baseUrlFailure] = await Promise.all([
1483
+ needs.browser ? checkBrowser(config.browser.headless) : Promise.resolve({}),
1484
+ needs.model ? checkModel(config.llm, fetchImpl) : Promise.resolve(void 0),
1485
+ needs.baseUrl ? checkBaseUrl(config.base_url, fetchImpl) : Promise.resolve(void 0)
1486
+ ]);
1487
+ const failures = [browserResult.failure, modelFailure, baseUrlFailure].filter(
1488
+ (value) => value !== void 0
1489
+ );
1490
+ if (failures.length > 0) {
1491
+ if (browserResult.browser) await browserResult.browser.close();
1492
+ return { ok: false, failures };
1493
+ }
1494
+ return { ok: true, browser: browserResult.browser };
1495
+ }
1496
+ function printPreflightFailures(failures) {
1497
+ console.error(`error: ${failures.length} unmet prerequisite(s):`);
1498
+ for (const failure of failures) console.error(` - ${failure}`);
1499
+ }
1500
+
1220
1501
  // src/commands/run.ts
1221
1502
  import path9 from "path";
1222
- import { chromium } from "playwright";
1223
1503
 
1224
1504
  // src/report/html.ts
1225
1505
  import { mkdir as mkdir4, readFile as readFile4, writeFile as writeFile4 } from "fs/promises";
@@ -1262,6 +1542,8 @@ h1 { font-size:1.35rem; margin:0 0 .25rem; }
1262
1542
  .verdict { font-weight:600; }
1263
1543
  .verdict.pass { color:var(--pass); } .verdict.fail { color:var(--fail); }
1264
1544
  .counts { color:var(--muted); font-size:.9rem; }
1545
+ .incomplete { padding:1rem 1.25rem; border:1px solid var(--fail); border-radius:10px;
1546
+ background:color-mix(in srgb, var(--fail) 10%, transparent); margin-bottom:1.5rem; font-weight:600; }
1265
1547
  table { width:100%; border-collapse:collapse; margin-bottom:1.5rem; }
1266
1548
  th, td { text-align:left; padding:.5rem .6rem; border-bottom:1px solid var(--line);
1267
1549
  font-size:.9rem; vertical-align:top; }
@@ -1298,10 +1580,13 @@ function renderSteps(steps) {
1298
1580
  }
1299
1581
  async function renderTest(result, cwd) {
1300
1582
  const failed = result.status === "failed";
1583
+ const notRun = result.status === "not-run";
1584
+ const tagClass = failed ? "fail" : notRun ? "skip" : "pass";
1585
+ const label = failed ? "FAIL" : notRun ? "NOT RUN" : "PASS";
1301
1586
  const file = cwd ? path7.relative(cwd, result.file) : result.file;
1302
1587
  const parts = [
1303
- ` <details${failed ? " open" : ""}>`,
1304
- ` <summary><span class="tag ${failed ? "fail" : "pass"}">${failed ? "FAIL" : "PASS"}</span> ${escapeHtml(result.summary)} <span class="note">${escapeHtml(result.priority)} \xB7 ${escapeHtml(file)}</span></summary>`
1588
+ ` <details${failed || notRun ? " open" : ""}>`,
1589
+ ` <summary><span class="tag ${tagClass}">${label}</span> ${escapeHtml(result.summary)} <span class="note">${escapeHtml(result.priority)} \xB7 ${escapeHtml(file)}</span></summary>`
1305
1590
  ];
1306
1591
  if (failed) {
1307
1592
  if (result.failedStep) {
@@ -1311,6 +1596,9 @@ async function renderTest(result, cwd) {
1311
1596
  parts.push(` <p class="reason">${escapeHtml(result.reason)}</p>`);
1312
1597
  }
1313
1598
  }
1599
+ if (notRun && result.reason) {
1600
+ parts.push(` <p class="note">${escapeHtml(result.reason)}</p>`);
1601
+ }
1314
1602
  parts.push(` ${renderSteps(result.steps)}`);
1315
1603
  if (failed && result.screenshot) {
1316
1604
  const embedded = await embedScreenshot(result.screenshot);
@@ -1324,13 +1612,20 @@ async function renderTest(result, cwd) {
1324
1612
  async function renderHtml(results, skipped, meta) {
1325
1613
  const passed = results.filter((r) => r.status === "passed").length;
1326
1614
  const failures = results.filter((r) => r.status === "failed");
1615
+ const notRun = results.filter((r) => r.status === "not-run");
1327
1616
  const generatedAt = (meta.generatedAt ?? /* @__PURE__ */ new Date()).toISOString();
1328
- const ordered = [...failures, ...results.filter((r) => r.status === "passed")];
1617
+ const ordered = [...failures, ...notRun, ...results.filter((r) => r.status === "passed")];
1329
1618
  const detail = (await Promise.all(ordered.map((r) => renderTest(r, meta.cwd)))).join("\n");
1330
1619
  const verdict = meta.minScore === void 0 ? "" : meta.score >= meta.minScore ? `<span class="verdict pass">min-score ${meta.minScore}: pass</span>` : `<span class="verdict fail">min-score ${meta.minScore}: FAIL</span>`;
1620
+ const banner = meta.incomplete === void 0 ? "" : ` <section class="incomplete">Run stopped: ${escapeHtml(meta.incomplete)}</section>
1621
+
1622
+ `;
1331
1623
  const rows = results.map((r) => {
1332
1624
  const file = meta.cwd ? path7.relative(meta.cwd, r.file) : r.file;
1333
- return ` <tr><td><span class="tag ${r.status === "failed" ? "fail" : "pass"}">${r.status === "failed" ? "FAIL" : "PASS"}</span></td><td>${escapeHtml(r.priority)}</td><td>${escapeHtml(r.summary)}</td><td>${escapeHtml(file)}</td><td class="num">${seconds(r.durationMs)}</td></tr>`;
1625
+ const tagClass = r.status === "failed" ? "fail" : r.status === "not-run" ? "skip" : "pass";
1626
+ const label = r.status === "failed" ? "FAIL" : r.status === "not-run" ? "NOT RUN" : "PASS";
1627
+ const time = r.status === "not-run" ? "\u2014" : seconds(r.durationMs);
1628
+ return ` <tr><td><span class="tag ${tagClass}">${label}</span></td><td>${escapeHtml(r.priority)}</td><td>${escapeHtml(r.summary)}</td><td>${escapeHtml(file)}</td><td class="num">${time}</td></tr>`;
1334
1629
  }).join("\n");
1335
1630
  const skippedRows = skipped.map((s) => {
1336
1631
  const file = meta.cwd ? path7.relative(meta.cwd, s.path) : s.path;
@@ -1349,10 +1644,10 @@ async function renderHtml(results, skipped, meta) {
1349
1644
  <h1>blastproof report</h1>
1350
1645
  <p class="sub">${escapeHtml(generatedAt)} \xB7 ${seconds(meta.durationMs)}</p>
1351
1646
 
1352
- <section class="score">
1647
+ ${banner} <section class="score">
1353
1648
  <b>${meta.score}</b>
1354
1649
  ${verdict}
1355
- <span class="counts">${passed} passed \xB7 ${failures.length} failed${skipped.length > 0 ? ` \xB7 ${skipped.length} skipped` : ""}</span>
1650
+ <span class="counts">${passed} passed \xB7 ${failures.length} failed${notRun.length > 0 ? ` \xB7 ${notRun.length} not run` : ""}${skipped.length > 0 ? ` \xB7 ${skipped.length} skipped` : ""}</span>
1356
1651
  </section>
1357
1652
 
1358
1653
  <table>
@@ -1395,14 +1690,27 @@ function seconds2(ms) {
1395
1690
  function renderJUnit(results, skipped, meta) {
1396
1691
  const relative = (file) => meta.cwd ? path8.relative(meta.cwd, file) : file;
1397
1692
  const failures = results.filter((result) => result.status === "failed").length;
1693
+ const notRun = results.filter((result) => result.status === "not-run");
1398
1694
  const lines = [
1399
1695
  '<?xml version="1.0" encoding="UTF-8"?>',
1400
- `<testsuite name="blastproof" tests="${results.length + skipped.length}" failures="${failures}" skipped="${skipped.length}" time="${seconds2(meta.durationMs)}">`,
1696
+ `<testsuite name="blastproof" tests="${results.length + skipped.length}" failures="${failures}" skipped="${skipped.length + notRun.length}" time="${seconds2(meta.durationMs)}">`,
1401
1697
  " <properties>",
1402
1698
  ` <property name="score" value="${meta.score}"/>`,
1699
+ ...meta.incomplete !== void 0 ? [
1700
+ ' <property name="incomplete" value="true"/>',
1701
+ ` <property name="incomplete_reason" value="${escapeXml(meta.incomplete)}"/>`
1702
+ ] : [],
1403
1703
  " </properties>"
1404
1704
  ];
1405
1705
  for (const result of results) {
1706
+ if (result.status === "not-run") {
1707
+ lines.push(
1708
+ ` <testcase classname="${escapeXml(relative(result.file))}" name="${escapeXml(result.summary)}" time="0.000">`,
1709
+ ` <skipped message="${escapeXml(result.reason ?? "not run: the run stopped before this test executed")}"/>`,
1710
+ " </testcase>"
1711
+ );
1712
+ continue;
1713
+ }
1406
1714
  const open = ` <testcase classname="${escapeXml(relative(result.file))}" name="${escapeXml(result.summary)}" time="${seconds2(result.durationMs)}"`;
1407
1715
  if (result.status === "passed") {
1408
1716
  lines.push(`${open}/>`);
@@ -1438,6 +1746,7 @@ function computeScore(results) {
1438
1746
  let total = 0;
1439
1747
  let passed = 0;
1440
1748
  for (const result of results) {
1749
+ if (result.status === "not-run") continue;
1441
1750
  const weight = WEIGHTS[result.priority] ?? DEFAULT_WEIGHT;
1442
1751
  total += weight;
1443
1752
  if (result.status === "passed") passed += weight;
@@ -1453,6 +1762,10 @@ function formatScoreLine(score, results, threshold) {
1453
1762
  if (threshold === void 0) return `Score: ${score}`;
1454
1763
  return score >= threshold ? `Score: ${score} \u2014 min-score ${threshold}: pass` : `Score: ${score} \u2014 min-score ${threshold}: FAIL (below threshold)`;
1455
1764
  }
1765
+ function formatIncompleteLine(score, reason) {
1766
+ return `Run incomplete: ${reason}
1767
+ Score over executed tests: ${score} (not a verdict \u2014 exit code 1 regardless of --min-score)`;
1768
+ }
1456
1769
 
1457
1770
  // src/runner/selection.ts
1458
1771
  function matchesFilters(test, filters) {
@@ -1492,6 +1805,14 @@ function selectImpactedTests(tests, affectedRoutes, filters = { tags: [] }) {
1492
1805
  var EXIT_OK = 0;
1493
1806
  var EXIT_FAILED = 1;
1494
1807
  var EXIT_USAGE = 2;
1808
+ function resolveBudgetOptions(config, options) {
1809
+ const maxDurationSeconds = options.maxDuration ?? config.budget?.max_duration_s;
1810
+ return {
1811
+ maxCalls: options.maxLlmCalls ?? config.budget?.max_llm_calls,
1812
+ maxTokens: options.maxTokens ?? config.budget?.max_tokens,
1813
+ maxDurationMs: maxDurationSeconds === void 0 ? void 0 : maxDurationSeconds * 1e3
1814
+ };
1815
+ }
1495
1816
  function sessionId() {
1496
1817
  return (/* @__PURE__ */ new Date()).toISOString().replace(/[:.]/g, "-");
1497
1818
  }
@@ -1522,6 +1843,18 @@ function resolveSecretsAndSteps(test) {
1522
1843
  }
1523
1844
  return { test };
1524
1845
  }
1846
+ function notRunResults(tests, reason) {
1847
+ return tests.map((test) => ({
1848
+ file: test.path,
1849
+ summary: test.summary,
1850
+ priority: test.priority,
1851
+ tags: test.tags,
1852
+ status: "not-run",
1853
+ steps: [],
1854
+ reason,
1855
+ durationMs: 0
1856
+ }));
1857
+ }
1525
1858
  function printEvent(event) {
1526
1859
  switch (event.type) {
1527
1860
  case "step-start":
@@ -1541,12 +1874,18 @@ function printEvent(event) {
1541
1874
  break;
1542
1875
  }
1543
1876
  }
1877
+ function statusLabel(status) {
1878
+ if (status === "passed") return "PASS";
1879
+ if (status === "failed") return "FAIL";
1880
+ return "NOT RUN";
1881
+ }
1544
1882
  function printSummary(results) {
1545
1883
  const passed = results.filter((r) => r.status === "passed");
1546
1884
  const failed = results.filter((r) => r.status === "failed");
1885
+ const notRun = results.filter((r) => r.status === "not-run");
1547
1886
  console.log("\n--- Summary ---------------------------------------------------");
1548
1887
  const rows = results.map((r) => ({
1549
- status: r.status === "passed" ? "PASS" : "FAIL",
1888
+ status: statusLabel(r.status),
1550
1889
  priority: r.priority,
1551
1890
  summary: r.summary,
1552
1891
  duration: `${(r.durationMs / 1e3).toFixed(1)}s`
@@ -1559,7 +1898,9 @@ function printSummary(results) {
1559
1898
  );
1560
1899
  }
1561
1900
  console.log("---------------------------------------------------------------");
1562
- console.log(`${passed.length} passed, ${failed.length} failed, ${results.length} total`);
1901
+ console.log(
1902
+ notRun.length > 0 ? `${passed.length} passed, ${failed.length} failed, ${notRun.length} not run, ${results.length} total` : `${passed.length} passed, ${failed.length} failed, ${results.length} total`
1903
+ );
1563
1904
  for (const r of failed) {
1564
1905
  console.log(`
1565
1906
  X ${r.summary} (${r.file})`);
@@ -1567,9 +1908,14 @@ X ${r.summary} (${r.file})`);
1567
1908
  if (r.reason) console.log(` reason: ${r.reason}`);
1568
1909
  if (r.screenshot) console.log(` screenshot: ${r.screenshot}`);
1569
1910
  }
1911
+ if (notRun.length > 0) {
1912
+ console.log(`
1913
+ ${notRun.length} test(s) not run (run stopped by its budget or deadline):`);
1914
+ for (const r of notRun) console.log(` - ${r.summary} (${r.file})`);
1915
+ }
1570
1916
  }
1571
- async function runOne(browser, test, config, sessionDir, session, runMask) {
1572
- const brain = createBrain(createModel(config.llm).model);
1917
+ async function runOne(browser, test, config, sessionDir, session, runMask, budget) {
1918
+ const brain = createBrain(createModel(config.llm).model, void 0, budget);
1573
1919
  let resolved;
1574
1920
  const mask = runMask;
1575
1921
  try {
@@ -1600,6 +1946,8 @@ async function runOne(browser, test, config, sessionDir, session, runMask) {
1600
1946
  allowedOrigins: config.allowed_origins,
1601
1947
  resolveValue: (value) => substituteEnv(value),
1602
1948
  maxRetries: config.max_retries_per_step,
1949
+ timeoutMs: config.browser.timeout_ms,
1950
+ maxSnapshotLines: config.browser.max_snapshot_lines,
1603
1951
  mask: (text) => mask.mask(text),
1604
1952
  onEvent: printEvent
1605
1953
  });
@@ -1632,13 +1980,20 @@ function printImpactReport(impact, selection, cwd) {
1632
1980
  }
1633
1981
  console.log("---------------------------------------------------------------");
1634
1982
  }
1635
- async function finalize(results, skipped, options, sessionDir, durationMs, impact) {
1983
+ async function finalize(results, skipped, options, sessionDir, durationMs, impact, incomplete) {
1636
1984
  if (results.length > 0) printSummary(results);
1637
1985
  const score = computeScore(results);
1638
- console.log(formatScoreLine(score, results, options.minScore));
1986
+ console.log(
1987
+ incomplete ? formatIncompleteLine(score, incomplete.message) : formatScoreLine(score, results, options.minScore)
1988
+ );
1639
1989
  if (options.junit) {
1640
1990
  const target = typeof options.junit === "string" ? path9.resolve(options.cwd, options.junit) : path9.join(sessionDir, "junit.xml");
1641
- const xml = renderJUnit(results, skipped, { score, durationMs, cwd: options.cwd });
1991
+ const xml = renderJUnit(results, skipped, {
1992
+ score,
1993
+ durationMs,
1994
+ cwd: options.cwd,
1995
+ incomplete: incomplete?.message
1996
+ });
1642
1997
  await writeJUnit(target, xml);
1643
1998
  console.log(`JUnit report: ${path9.relative(options.cwd, target)}`);
1644
1999
  }
@@ -1648,11 +2003,13 @@ async function finalize(results, skipped, options, sessionDir, durationMs, impac
1648
2003
  score,
1649
2004
  durationMs,
1650
2005
  minScore: options.minScore,
1651
- cwd: options.cwd
2006
+ cwd: options.cwd,
2007
+ incomplete: incomplete?.message
1652
2008
  });
1653
2009
  await writeHtml(target, html);
1654
2010
  console.log(`HTML report: ${path9.relative(options.cwd, target)}`);
1655
2011
  }
2012
+ if (incomplete) return EXIT_FAILED;
1656
2013
  if (reportUnclassified(options, impact)) return EXIT_FAILED;
1657
2014
  if (options.minScore !== void 0) {
1658
2015
  return score >= options.minScore ? EXIT_OK : EXIT_FAILED;
@@ -1671,12 +2028,23 @@ function reportUnclassified(options, impact) {
1671
2028
  );
1672
2029
  return true;
1673
2030
  }
1674
- function printDryRun(selected, baseUrl, cwd) {
2031
+ function printDryRun(selected, config, cwd) {
1675
2032
  console.log(`
1676
- Dry run: ${selected.length} test(s) selected, base_url=${baseUrl}`);
2033
+ Dry run: ${selected.length} test(s) selected, base_url=${config.base_url}`);
1677
2034
  for (const test of selected) {
1678
2035
  console.log(` ${test.summary} [${test.priority}] (${path9.relative(cwd, test.path)})`);
1679
2036
  }
2037
+ const authSteps = config.auth?.steps ?? [];
2038
+ const withAuth = authSteps.length > 0 ? [...selected, { steps: authSteps }] : selected;
2039
+ const ceiling = estimateMaxModelCalls(
2040
+ withAuth,
2041
+ DEFAULT_MAX_ITERATIONS_PER_STEP,
2042
+ config.max_retries_per_step
2043
+ );
2044
+ const authNote = authSteps.length > 0 ? ", including the login journey" : "";
2045
+ console.log(
2046
+ `Worst case: up to ${ceiling} model call(s) for this selection${authNote} (a maximum, not a prediction).`
2047
+ );
1680
2048
  console.log("Dry run: no browser launched, no LLM calls made.");
1681
2049
  }
1682
2050
  async function runCommand(options) {
@@ -1752,7 +2120,7 @@ async function runCommand(options) {
1752
2120
  printImpactReport(impact, selection, options.cwd);
1753
2121
  }
1754
2122
  if (options.dryRun) {
1755
- printDryRun(selected, config.base_url, options.cwd);
2123
+ printDryRun(selected, config, options.cwd);
1756
2124
  if (results.length > 0) {
1757
2125
  console.error(`
1758
2126
  ${results.length} test file(s) could not be parsed:`);
@@ -1783,7 +2151,14 @@ ${results.length} test file(s) could not be parsed:`);
1783
2151
  `blastproof run: ${selected.length} test(s), provider=${provider} model=${modelId}, base_url=${config.base_url}`
1784
2152
  );
1785
2153
  const runMask = buildRunMask(config, parsed);
1786
- const browser = await chromium.launch({ headless: config.browser.headless });
2154
+ const budget = options.budget ?? new RunBudget(resolveBudgetOptions(config, options));
2155
+ const preflight = await runPreflight({ browser: true, model: true, baseUrl: true }, config);
2156
+ if (!preflight.ok) {
2157
+ printPreflightFailures(preflight.failures);
2158
+ return EXIT_USAGE;
2159
+ }
2160
+ const browser = preflight.browser;
2161
+ let incomplete;
1787
2162
  try {
1788
2163
  let session;
1789
2164
  if (config.auth) {
@@ -1795,28 +2170,56 @@ ${results.length} test file(s) could not be parsed:`);
1795
2170
  baseUrl: config.base_url,
1796
2171
  allowedOrigins: config.allowed_origins,
1797
2172
  browser,
1798
- brain: createBrain(createModel(config.llm).model),
2173
+ brain: createBrain(createModel(config.llm).model, void 0, budget),
1799
2174
  maxRetries: config.max_retries_per_step,
2175
+ timeoutMs: config.browser.timeout_ms,
2176
+ maxSnapshotLines: config.browser.max_snapshot_lines,
1800
2177
  mask: runMask,
1801
2178
  onEvent: printEvent
1802
2179
  });
1803
2180
  } catch (error) {
1804
- if (error instanceof AuthError) {
2181
+ if (error instanceof BudgetExhaustedError) {
2182
+ incomplete = error;
2183
+ } else if (error instanceof AuthError) {
1805
2184
  console.error(`error: ${error.message}`);
1806
2185
  return EXIT_USAGE;
2186
+ } else {
2187
+ throw error;
1807
2188
  }
1808
- throw error;
1809
2189
  }
1810
2190
  }
1811
- for (const test of selected) {
1812
- console.log(`
2191
+ if (incomplete) {
2192
+ results.push(...notRunResults(selected, incomplete.message));
2193
+ } else {
2194
+ for (let i = 0; i < selected.length; i++) {
2195
+ const test = selected[i];
2196
+ console.log(`
1813
2197
  > ${test.summary} [${test.priority}] (${path9.relative(options.cwd, test.path)})`);
1814
- results.push(await runOne(browser, test, config, sessionDir, session, runMask));
2198
+ try {
2199
+ budget.check();
2200
+ results.push(await runOne(browser, test, config, sessionDir, session, runMask, budget));
2201
+ } catch (error) {
2202
+ if (error instanceof BudgetExhaustedError) {
2203
+ incomplete = error;
2204
+ results.push(...notRunResults(selected.slice(i), error.message));
2205
+ break;
2206
+ }
2207
+ throw error;
2208
+ }
2209
+ }
1815
2210
  }
1816
2211
  } finally {
1817
2212
  await browser.close();
1818
2213
  }
1819
- return finalize(results, selection.unroutedSkipped, options, sessionDir, Date.now() - startedAt, impact);
2214
+ return finalize(
2215
+ results,
2216
+ selection.unroutedSkipped,
2217
+ options,
2218
+ sessionDir,
2219
+ Date.now() - startedAt,
2220
+ impact,
2221
+ incomplete
2222
+ );
1820
2223
  }
1821
2224
 
1822
2225
  // src/commands/plan.ts
@@ -1897,6 +2300,16 @@ async function planCommand(options) {
1897
2300
  console.log("Already covered (skipped):");
1898
2301
  for (const route of alreadyCovered) console.log(` ${route}`);
1899
2302
  }
2303
+ if (options.dryRun) {
2304
+ if (work.length === 0) {
2305
+ console.log("Nothing to generate: no affected route is missing coverage.");
2306
+ } else {
2307
+ console.log(`Dry run: ${work.length} route(s) would generate a draft for:`);
2308
+ for (const { route } of work) console.log(` ${route}`);
2309
+ }
2310
+ console.log("Dry run: no browser launched, no LLM calls made.");
2311
+ return EXIT_OK;
2312
+ }
1900
2313
  if (work.length === 0) {
1901
2314
  console.log("Nothing to generate: no affected route is missing coverage.");
1902
2315
  return EXIT_OK;
@@ -1911,7 +2324,8 @@ async function planCommand(options) {
1911
2324
  throw error;
1912
2325
  }
1913
2326
  const { model, provider, modelId } = createModel(config.llm);
1914
- const brain = createPlanner(model);
2327
+ const budget = options.budget ?? new RunBudget(resolveBudgetOptions(config, options));
2328
+ const brain = createPlanner(model, void 0, budget);
1915
2329
  const base = options.routes && options.routes.length > 0 ? void 0 : options.base ?? "main";
1916
2330
  console.log(
1917
2331
  `blastproof plan: ${work.length} route(s), provider=${provider} model=${modelId}, base_url=${config.base_url}`
@@ -1926,7 +2340,13 @@ async function planCommand(options) {
1926
2340
  const generated = [];
1927
2341
  const written = [];
1928
2342
  const failed = [];
1929
- const browser = await chromium2.launch({ headless: config.browser.headless });
2343
+ let incomplete;
2344
+ const preflight = await runPreflight({ browser: true, model: true, baseUrl: true }, config);
2345
+ if (!preflight.ok) {
2346
+ printPreflightFailures(preflight.failures);
2347
+ return EXIT_USAGE;
2348
+ }
2349
+ const browser = preflight.browser;
1930
2350
  try {
1931
2351
  let session;
1932
2352
  if (config.auth) {
@@ -1938,20 +2358,25 @@ async function planCommand(options) {
1938
2358
  baseUrl: config.base_url,
1939
2359
  allowedOrigins: config.allowed_origins,
1940
2360
  browser,
1941
- brain: createBrain(createModel(config.llm).model),
2361
+ brain: createBrain(createModel(config.llm).model, void 0, budget),
1942
2362
  maxRetries: config.max_retries_per_step,
2363
+ timeoutMs: config.browser.timeout_ms,
2364
+ maxSnapshotLines: config.browser.max_snapshot_lines,
1943
2365
  mask,
1944
2366
  onEvent: printAuthEvent
1945
2367
  });
1946
2368
  } catch (error) {
1947
- if (error instanceof AuthError) {
2369
+ if (error instanceof BudgetExhaustedError) {
2370
+ incomplete = error;
2371
+ } else if (error instanceof AuthError) {
1948
2372
  console.error(`error: ${error.message}`);
1949
2373
  return EXIT_USAGE;
2374
+ } else {
2375
+ throw error;
1950
2376
  }
1951
- throw error;
1952
2377
  }
1953
2378
  }
1954
- for (const { route, changedFiles } of work) {
2379
+ for (const { route, changedFiles } of incomplete ? [] : work) {
1955
2380
  console.log(`
1956
2381
  > ${route}`);
1957
2382
  let draft;
@@ -1964,9 +2389,15 @@ async function planCommand(options) {
1964
2389
  baseUrl: config.base_url,
1965
2390
  changedFiles,
1966
2391
  brain,
1967
- mask: (text) => mask.mask(text)
2392
+ mask: (text) => mask.mask(text),
2393
+ maxSnapshotLines: config.browser.max_snapshot_lines,
2394
+ timeoutMs: config.browser.timeout_ms
1968
2395
  });
1969
2396
  } catch (error) {
2397
+ if (error instanceof BudgetExhaustedError) {
2398
+ incomplete = error;
2399
+ break;
2400
+ }
1970
2401
  failed.push({ route, reason: error instanceof Error ? error.message : String(error) });
1971
2402
  console.log(` X failed: ${error instanceof Error ? error.message : String(error)}`);
1972
2403
  continue;
@@ -1996,6 +2427,7 @@ ${renderTestYaml(draft, { route, base })}`);
1996
2427
  } finally {
1997
2428
  await browser.close();
1998
2429
  }
2430
+ const notAttempted = incomplete ? work.map((item) => item.route).filter((route) => !generated.includes(route) && !failed.some((f) => f.route === route)) : [];
1999
2431
  console.log("\n--- Plan -------------------------------------------------------");
2000
2432
  console.log(`Generated: ${generated.length > 0 ? generated.join(", ") : "none"}`);
2001
2433
  if (written.length > 0) {
@@ -2009,8 +2441,15 @@ ${renderTestYaml(draft, { route, base })}`);
2009
2441
  console.log("Failed:");
2010
2442
  for (const { route, reason } of failed) console.log(` ${route}: ${reason}`);
2011
2443
  }
2444
+ if (incomplete) {
2445
+ console.log(`Stopped: ${incomplete.message}`);
2446
+ if (notAttempted.length > 0) {
2447
+ console.log("Not attempted (run out of budget):");
2448
+ for (const route of notAttempted) console.log(` ${route}`);
2449
+ }
2450
+ }
2012
2451
  console.log("---------------------------------------------------------------");
2013
- return failed.length > 0 ? EXIT_FAILED : EXIT_OK;
2452
+ return incomplete || failed.length > 0 ? EXIT_FAILED : EXIT_OK;
2014
2453
  }
2015
2454
 
2016
2455
  // src/commands/test.ts
@@ -2019,6 +2458,17 @@ function section(title) {
2019
2458
  === ${title} ${"=".repeat(Math.max(0, 58 - title.length))}`);
2020
2459
  }
2021
2460
  async function testCommand(options) {
2461
+ let config;
2462
+ try {
2463
+ config = applyUrlOverride(await loadConfig(options.cwd), options.url);
2464
+ } catch (error) {
2465
+ if (error instanceof ConfigError) {
2466
+ console.error(`error: ${error.message}`);
2467
+ return EXIT_USAGE;
2468
+ }
2469
+ throw error;
2470
+ }
2471
+ const budget = new RunBudget(resolveBudgetOptions(config, options));
2022
2472
  section("Verify: tests covering the affected routes");
2023
2473
  const runCode = await runCommand({
2024
2474
  cwd: options.cwd,
@@ -2029,7 +2479,8 @@ async function testCommand(options) {
2029
2479
  minScore: options.minScore,
2030
2480
  junit: options.junit,
2031
2481
  html: options.html,
2032
- failOnUnmapped: options.failOnUnmapped
2482
+ failOnUnmapped: options.failOnUnmapped,
2483
+ budget
2033
2484
  });
2034
2485
  if (runCode === EXIT_USAGE) return EXIT_USAGE;
2035
2486
  section("Draft: affected routes no test covers");
@@ -2037,7 +2488,8 @@ async function testCommand(options) {
2037
2488
  cwd: options.cwd,
2038
2489
  base: options.base,
2039
2490
  url: options.url,
2040
- write: options.write
2491
+ write: options.write,
2492
+ budget
2041
2493
  });
2042
2494
  if (planCode === EXIT_USAGE) return EXIT_USAGE;
2043
2495
  console.log(
@@ -2063,8 +2515,26 @@ function parseMinScore(value) {
2063
2515
  }
2064
2516
  return score;
2065
2517
  }
2518
+ function parsePositiveInt(flag) {
2519
+ return (value) => {
2520
+ const n = Number(value);
2521
+ if (!Number.isInteger(n) || n <= 0) {
2522
+ throw new InvalidArgumentError(`${flag} must be a positive integer`);
2523
+ }
2524
+ return n;
2525
+ };
2526
+ }
2527
+ function parsePositiveNumber(flag) {
2528
+ return (value) => {
2529
+ const n = Number(value);
2530
+ if (!Number.isFinite(n) || n <= 0) {
2531
+ throw new InvalidArgumentError(`${flag} must be a positive number`);
2532
+ }
2533
+ return n;
2534
+ };
2535
+ }
2066
2536
  var program = new Command();
2067
- program.name("blastproof").description("Open-source AI testing agent: plain-English YAML tests executed agentically on a real browser.").version("0.2.2");
2537
+ program.name("blastproof").description("Open-source AI testing agent: plain-English YAML tests executed agentically on a real browser.").version("0.4.0");
2068
2538
  program.command("init").description("Scaffold .blastproof/ (config, tests, sample tests) in the current directory").action(async () => {
2069
2539
  try {
2070
2540
  const result = await initProject(process.cwd());
@@ -2079,7 +2549,19 @@ program.command("run").description("Discover and run all tests under .blastproof
2079
2549
  "--min-score <n>",
2080
2550
  "require a weighted score of at least n (0-100); replaces the all-must-pass rule",
2081
2551
  parseMinScore
2082
- ).option("--junit [path]", "write a JUnit XML report (default: .blastproof/reports/<session>/junit.xml)").option("--html [path]", "write a self-contained HTML report (default: .blastproof/reports/<session>/report.html)").option("--fail-on-unmapped", "fail when a changed file matches no routes: or ignore: glob").action(
2552
+ ).option("--junit [path]", "write a JUnit XML report (default: .blastproof/reports/<session>/junit.xml)").option("--html [path]", "write a self-contained HTML report (default: .blastproof/reports/<session>/report.html)").option("--fail-on-unmapped", "fail when a changed file matches no routes: or ignore: glob").option(
2553
+ "--max-llm-calls <n>",
2554
+ "stop the run after this many model calls, reported as incomplete (overrides config)",
2555
+ parsePositiveInt("--max-llm-calls")
2556
+ ).option(
2557
+ "--max-tokens <n>",
2558
+ "stop the run after this many tokens spent across all model calls (overrides config)",
2559
+ parsePositiveInt("--max-tokens")
2560
+ ).option(
2561
+ "--max-duration <seconds>",
2562
+ "stop the run after this many seconds of wall-clock time (overrides config)",
2563
+ parsePositiveNumber("--max-duration")
2564
+ ).action(
2083
2565
  async (options) => {
2084
2566
  try {
2085
2567
  process.exitCode = await runCommand({
@@ -2094,7 +2576,10 @@ program.command("run").description("Discover and run all tests under .blastproof
2094
2576
  minScore: options.minScore,
2095
2577
  junit: options.junit,
2096
2578
  html: options.html,
2097
- failOnUnmapped: options.failOnUnmapped
2579
+ failOnUnmapped: options.failOnUnmapped,
2580
+ maxLlmCalls: options.maxLlmCalls,
2581
+ maxTokens: options.maxTokens,
2582
+ maxDuration: options.maxDuration
2098
2583
  });
2099
2584
  } catch (error) {
2100
2585
  console.error(`error: ${error instanceof Error ? error.message : String(error)}`);
@@ -2102,25 +2587,58 @@ program.command("run").description("Discover and run all tests under .blastproof
2102
2587
  }
2103
2588
  }
2104
2589
  );
2105
- program.command("plan").description("Generate plain-English YAML tests for affected routes no test covers yet").option("--base <ref>", "base git ref for the diff", "main").option("--url <url>", "override config base_url for this run only (config file untouched)").option("--route <route>", "generate for this route, bypassing the diff (repeatable)", collect, []).option("--write", "persist drafts under .blastproof/tests/ instead of previewing them").action(async (options) => {
2106
- try {
2107
- process.exitCode = await planCommand({
2108
- cwd: process.cwd(),
2109
- base: options.base,
2110
- url: options.url,
2111
- routes: options.route,
2112
- write: options.write
2113
- });
2114
- } catch (error) {
2115
- console.error(`error: ${error instanceof Error ? error.message : String(error)}`);
2116
- process.exitCode = EXIT_USAGE;
2590
+ program.command("plan").description("Generate plain-English YAML tests for affected routes no test covers yet").option("--base <ref>", "base git ref for the diff", "main").option("--url <url>", "override config base_url for this run only (config file untouched)").option("--route <route>", "generate for this route, bypassing the diff (repeatable)", collect, []).option("--write", "persist drafts under .blastproof/tests/ instead of previewing them").option(
2591
+ "--dry-run",
2592
+ "print the routes that would generate drafts and exit without launching a browser or calling the LLM"
2593
+ ).option(
2594
+ "--max-llm-calls <n>",
2595
+ "stop after this many model calls, reported as incomplete (overrides config)",
2596
+ parsePositiveInt("--max-llm-calls")
2597
+ ).option(
2598
+ "--max-tokens <n>",
2599
+ "stop after this many tokens spent across all model calls (overrides config)",
2600
+ parsePositiveInt("--max-tokens")
2601
+ ).option(
2602
+ "--max-duration <seconds>",
2603
+ "stop after this many seconds of wall-clock time (overrides config)",
2604
+ parsePositiveNumber("--max-duration")
2605
+ ).action(
2606
+ async (options) => {
2607
+ try {
2608
+ process.exitCode = await planCommand({
2609
+ cwd: process.cwd(),
2610
+ base: options.base,
2611
+ url: options.url,
2612
+ routes: options.route,
2613
+ write: options.write,
2614
+ dryRun: options.dryRun,
2615
+ maxLlmCalls: options.maxLlmCalls,
2616
+ maxTokens: options.maxTokens,
2617
+ maxDuration: options.maxDuration
2618
+ });
2619
+ } catch (error) {
2620
+ console.error(`error: ${error instanceof Error ? error.message : String(error)}`);
2621
+ process.exitCode = EXIT_USAGE;
2622
+ }
2117
2623
  }
2118
- });
2624
+ );
2119
2625
  program.command("test").description("Full PR pipeline: run the tests covering the diff, then draft tests for the gaps").option("--base <ref>", "base git ref for the diff", "main").option("--url <url>", "override config base_url for this run only (config file untouched)").option(
2120
2626
  "--min-score <n>",
2121
2627
  "require a weighted score of at least n (0-100); replaces the all-must-pass rule",
2122
2628
  parseMinScore
2123
- ).option("--junit [path]", "write a JUnit XML report (default: .blastproof/reports/<session>/junit.xml)").option("--html [path]", "write a self-contained HTML report (default: .blastproof/reports/<session>/report.html)").option("--write", "persist generated drafts under .blastproof/tests/ instead of previewing them").option("--fail-on-unmapped", "fail when a changed file matches no routes: or ignore: glob").action(
2629
+ ).option("--junit [path]", "write a JUnit XML report (default: .blastproof/reports/<session>/junit.xml)").option("--html [path]", "write a self-contained HTML report (default: .blastproof/reports/<session>/report.html)").option("--write", "persist generated drafts under .blastproof/tests/ instead of previewing them").option("--fail-on-unmapped", "fail when a changed file matches no routes: or ignore: glob").option(
2630
+ "--max-llm-calls <n>",
2631
+ "stop the pipeline after this many model calls, shared by both phases (overrides config)",
2632
+ parsePositiveInt("--max-llm-calls")
2633
+ ).option(
2634
+ "--max-tokens <n>",
2635
+ "stop the pipeline after this many tokens, shared by both phases (overrides config)",
2636
+ parsePositiveInt("--max-tokens")
2637
+ ).option(
2638
+ "--max-duration <seconds>",
2639
+ "stop the pipeline after this many seconds of wall-clock time (overrides config)",
2640
+ parsePositiveNumber("--max-duration")
2641
+ ).action(
2124
2642
  async (options) => {
2125
2643
  try {
2126
2644
  process.exitCode = await testCommand({
@@ -2131,7 +2649,10 @@ program.command("test").description("Full PR pipeline: run the tests covering th
2131
2649
  junit: options.junit,
2132
2650
  html: options.html,
2133
2651
  write: options.write,
2134
- failOnUnmapped: options.failOnUnmapped
2652
+ failOnUnmapped: options.failOnUnmapped,
2653
+ maxLlmCalls: options.maxLlmCalls,
2654
+ maxTokens: options.maxTokens,
2655
+ maxDuration: options.maxDuration
2135
2656
  });
2136
2657
  } catch (error) {
2137
2658
  console.error(`error: ${error instanceof Error ? error.message : String(error)}`);