blastproof 0.2.2 → 0.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +132 -240
- package/dist/cli.js +604 -83
- package/dist/cli.js.map +1 -1
- package/package.json +1 -1
package/dist/cli.js
CHANGED
|
@@ -26,6 +26,9 @@ llm:
|
|
|
26
26
|
|
|
27
27
|
browser:
|
|
28
28
|
headless: true
|
|
29
|
+
# How long to wait for an element to appear (self-healing resolution) as well as
|
|
30
|
+
# for the action performed on it and for navigation. Raising it makes a slow app
|
|
31
|
+
# wait longer to succeed, and a genuinely missing element take longer to fail.
|
|
29
32
|
timeout_ms: 30000
|
|
30
33
|
|
|
31
34
|
# Budget of self-healing retries per step (element re-resolution via accessibility tree).
|
|
@@ -152,23 +155,110 @@ function formatInitGuidance(result) {
|
|
|
152
155
|
lines.push(
|
|
153
156
|
"",
|
|
154
157
|
"Next steps:",
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
158
|
+
// Deliberately keyless first. A first-time-user trial found that following
|
|
159
|
+
// the old order — key, app, run — meant hitting the missing-key error and
|
|
160
|
+
// then a browser-launch failure with nothing successful behind you, which
|
|
161
|
+
// reads as "this is broken" rather than "one leg needs setting up". Seeing
|
|
162
|
+
// real output first reframes both.
|
|
163
|
+
" 1. Point base_url at your app in .blastproof/config.yaml",
|
|
164
|
+
" 2. Check the setup: blastproof run --dry-run (no API key or browser needed)",
|
|
165
|
+
" 3. Set your LLM API key: export ANTHROPIC_API_KEY=... (or OPENAI_API_KEY; ollama needs no key)",
|
|
166
|
+
" 4. Install the browser: npx playwright install chromium",
|
|
167
|
+
" 5. Run your tests: blastproof run",
|
|
158
168
|
"",
|
|
159
|
-
"
|
|
169
|
+
"Without a key or a browser you can still map a diff to routes and gate on",
|
|
170
|
+
"coverage: blastproof run --impacted --fail-on-unmapped --dry-run"
|
|
160
171
|
);
|
|
161
172
|
return lines.join("\n");
|
|
162
173
|
}
|
|
163
174
|
|
|
164
175
|
// src/commands/plan.ts
|
|
165
176
|
import path10 from "path";
|
|
166
|
-
import { chromium as chromium2 } from "playwright";
|
|
167
177
|
|
|
168
178
|
// src/auth.ts
|
|
169
179
|
import { readFile, writeFile as writeFile2, mkdir as mkdir3 } from "fs/promises";
|
|
170
180
|
import path3 from "path";
|
|
171
181
|
|
|
182
|
+
// src/runner/budget.ts
|
|
183
|
+
function describeLimit(limit, observed, configured) {
|
|
184
|
+
switch (limit) {
|
|
185
|
+
case "calls":
|
|
186
|
+
return `model call budget exhausted: reached the configured maximum of ${configured} call(s)`;
|
|
187
|
+
case "tokens":
|
|
188
|
+
return `token budget exhausted: reached the configured maximum of ${configured} token(s) (observed ${observed})`;
|
|
189
|
+
case "duration":
|
|
190
|
+
return `deadline exceeded: reached the configured maximum of ${(configured / 1e3).toFixed(0)}s (elapsed ${(observed / 1e3).toFixed(1)}s)`;
|
|
191
|
+
}
|
|
192
|
+
}
|
|
193
|
+
var BudgetExhaustedError = class extends Error {
|
|
194
|
+
limit;
|
|
195
|
+
observed;
|
|
196
|
+
configured;
|
|
197
|
+
constructor(limit, observed, configured) {
|
|
198
|
+
super(describeLimit(limit, observed, configured));
|
|
199
|
+
this.name = "BudgetExhaustedError";
|
|
200
|
+
this.limit = limit;
|
|
201
|
+
this.observed = observed;
|
|
202
|
+
this.configured = configured;
|
|
203
|
+
}
|
|
204
|
+
};
|
|
205
|
+
var RunBudget = class {
|
|
206
|
+
maxCalls;
|
|
207
|
+
maxTokens;
|
|
208
|
+
maxDurationMs;
|
|
209
|
+
now;
|
|
210
|
+
startedAt;
|
|
211
|
+
calls = 0;
|
|
212
|
+
tokens = 0;
|
|
213
|
+
constructor(options = {}) {
|
|
214
|
+
this.maxCalls = options.maxCalls;
|
|
215
|
+
this.maxTokens = options.maxTokens;
|
|
216
|
+
this.maxDurationMs = options.maxDurationMs;
|
|
217
|
+
this.now = options.now ?? Date.now;
|
|
218
|
+
this.startedAt = this.now();
|
|
219
|
+
}
|
|
220
|
+
/** Model calls recorded so far. */
|
|
221
|
+
get callCount() {
|
|
222
|
+
return this.calls;
|
|
223
|
+
}
|
|
224
|
+
/** Tokens recorded so far, summed across every completed call. */
|
|
225
|
+
get tokenCount() {
|
|
226
|
+
return this.tokens;
|
|
227
|
+
}
|
|
228
|
+
/**
|
|
229
|
+
* Throws {@link BudgetExhaustedError} when the next unit of work could exceed a
|
|
230
|
+
* configured limit. Called before spending anything, never after — so an
|
|
231
|
+
* over-budget call is never issued and then rejected (spec: "the call is not
|
|
232
|
+
* made").
|
|
233
|
+
*/
|
|
234
|
+
check() {
|
|
235
|
+
if (this.maxCalls !== void 0 && this.calls >= this.maxCalls) {
|
|
236
|
+
throw new BudgetExhaustedError("calls", this.calls, this.maxCalls);
|
|
237
|
+
}
|
|
238
|
+
if (this.maxTokens !== void 0 && this.tokens >= this.maxTokens) {
|
|
239
|
+
throw new BudgetExhaustedError("tokens", this.tokens, this.maxTokens);
|
|
240
|
+
}
|
|
241
|
+
if (this.maxDurationMs !== void 0) {
|
|
242
|
+
const elapsed = this.now() - this.startedAt;
|
|
243
|
+
if (elapsed >= this.maxDurationMs) {
|
|
244
|
+
throw new BudgetExhaustedError("duration", elapsed, this.maxDurationMs);
|
|
245
|
+
}
|
|
246
|
+
}
|
|
247
|
+
}
|
|
248
|
+
/** Records what a completed model call spent. */
|
|
249
|
+
record(usage) {
|
|
250
|
+
this.calls += 1;
|
|
251
|
+
if (usage?.totalTokens !== void 0) this.tokens += usage.totalTokens;
|
|
252
|
+
}
|
|
253
|
+
};
|
|
254
|
+
function estimateMaxModelCalls(tests, maxIterationsPerStep, maxRetriesPerStep) {
|
|
255
|
+
const totalSteps = tests.reduce(
|
|
256
|
+
(sum, test) => sum + (test.setup?.length ?? 0) + test.steps.length,
|
|
257
|
+
0
|
|
258
|
+
);
|
|
259
|
+
return totalSteps * (maxIterationsPerStep + maxRetriesPerStep);
|
|
260
|
+
}
|
|
261
|
+
|
|
172
262
|
// src/runner/env.ts
|
|
173
263
|
var ENV_PLACEHOLDER = /\{\{\s*env\.([A-Za-z_][A-Za-z0-9_]*)\s*\}\}/g;
|
|
174
264
|
var MissingEnvError = class extends Error {
|
|
@@ -307,7 +397,7 @@ async function performAction(page, action, ctx) {
|
|
|
307
397
|
const value = resolve(requireValue(action), ctx);
|
|
308
398
|
const url = new URL(value, ctx.baseUrl);
|
|
309
399
|
assertAllowedOrigin(url, ctx);
|
|
310
|
-
await page.goto(url.toString(), { timeout: 3e4 });
|
|
400
|
+
await page.goto(url.toString(), { timeout: ctx.resolveTimeoutMs ?? 3e4 });
|
|
311
401
|
return `ok: navigated to ${url.toString()}`;
|
|
312
402
|
}
|
|
313
403
|
case "click": {
|
|
@@ -348,9 +438,10 @@ async function performAction(page, action, ctx) {
|
|
|
348
438
|
}
|
|
349
439
|
|
|
350
440
|
// src/runner/executor.ts
|
|
351
|
-
|
|
441
|
+
var DEFAULT_MAX_ITERATIONS_PER_STEP = 15;
|
|
442
|
+
async function defaultSnapshot(page, maxLines) {
|
|
352
443
|
const { captureSnapshot } = await import("./snapshot-CAIB2OHX.js");
|
|
353
|
-
return captureSnapshot(page);
|
|
444
|
+
return captureSnapshot(page, { maxLines });
|
|
354
445
|
}
|
|
355
446
|
function slugify(text) {
|
|
356
447
|
const slug = text.toLowerCase().replace(/[^a-z0-9]+/g, "-").replace(/^-+|-+$/g, "").slice(0, 60);
|
|
@@ -366,12 +457,15 @@ async function executeTest(page, test, options) {
|
|
|
366
457
|
allowedOrigins,
|
|
367
458
|
resolveValue,
|
|
368
459
|
maxRetries = 3,
|
|
369
|
-
maxIterationsPerStep =
|
|
460
|
+
maxIterationsPerStep = DEFAULT_MAX_ITERATIONS_PER_STEP,
|
|
461
|
+
timeoutMs,
|
|
370
462
|
mask = (s) => s,
|
|
371
|
-
snapshot
|
|
463
|
+
snapshot,
|
|
464
|
+
maxSnapshotLines,
|
|
372
465
|
onEvent = () => {
|
|
373
466
|
}
|
|
374
467
|
} = options;
|
|
468
|
+
const takeSnapshot = snapshot ?? ((page2) => defaultSnapshot(page2, maxSnapshotLines));
|
|
375
469
|
const startedAt = Date.now();
|
|
376
470
|
const stepResults = [];
|
|
377
471
|
const allSteps = [
|
|
@@ -402,7 +496,7 @@ async function executeTest(page, test, options) {
|
|
|
402
496
|
if (iterations >= maxIterationsPerStep) {
|
|
403
497
|
throw new StepFailure(`step exceeded ${maxIterationsPerStep} actions without completing`);
|
|
404
498
|
}
|
|
405
|
-
const snap = await
|
|
499
|
+
const snap = await takeSnapshot(page);
|
|
406
500
|
let action;
|
|
407
501
|
try {
|
|
408
502
|
action = await brain.nextAction({
|
|
@@ -414,6 +508,7 @@ async function executeTest(page, test, options) {
|
|
|
414
508
|
iterationsLeft: maxIterationsPerStep - iterations
|
|
415
509
|
});
|
|
416
510
|
} catch (error) {
|
|
511
|
+
if (error instanceof BudgetExhaustedError) throw error;
|
|
417
512
|
failedAttempts++;
|
|
418
513
|
lastResult = `error: ${error instanceof Error ? error.message : String(error)}`;
|
|
419
514
|
if (failedAttempts >= maxRetries) {
|
|
@@ -437,7 +532,7 @@ async function executeTest(page, test, options) {
|
|
|
437
532
|
emitAction(index, action, result);
|
|
438
533
|
if (judgment.pass) {
|
|
439
534
|
lastResult = result;
|
|
440
|
-
|
|
535
|
+
break;
|
|
441
536
|
}
|
|
442
537
|
failedAttempts++;
|
|
443
538
|
lastResult = result;
|
|
@@ -447,7 +542,12 @@ async function executeTest(page, test, options) {
|
|
|
447
542
|
continue;
|
|
448
543
|
}
|
|
449
544
|
try {
|
|
450
|
-
const result = await performAction(page, action, {
|
|
545
|
+
const result = await performAction(page, action, {
|
|
546
|
+
baseUrl,
|
|
547
|
+
allowedOrigins,
|
|
548
|
+
resolveValue,
|
|
549
|
+
resolveTimeoutMs: timeoutMs
|
|
550
|
+
});
|
|
451
551
|
lastResult = result;
|
|
452
552
|
emitAction(index, action, result);
|
|
453
553
|
} catch (error) {
|
|
@@ -460,6 +560,7 @@ async function executeTest(page, test, options) {
|
|
|
460
560
|
}
|
|
461
561
|
}
|
|
462
562
|
} catch (error) {
|
|
563
|
+
if (error instanceof BudgetExhaustedError) throw error;
|
|
463
564
|
stepFailedReason = mask(error instanceof Error ? error.message : String(error));
|
|
464
565
|
}
|
|
465
566
|
const status = stepFailedReason ? "failed" : "passed";
|
|
@@ -560,14 +661,16 @@ async function runJourney(...args) {
|
|
|
560
661
|
try {
|
|
561
662
|
return await executeTest(...args);
|
|
562
663
|
} catch (error) {
|
|
664
|
+
if (error instanceof BudgetExhaustedError) throw error;
|
|
563
665
|
throw new AuthError(
|
|
564
666
|
`Authentication could not run: ${error instanceof Error ? error.message : String(error)}`
|
|
565
667
|
);
|
|
566
668
|
}
|
|
567
669
|
}
|
|
568
670
|
async function fromSteps(options) {
|
|
569
|
-
const { auth, baseUrl, browser, brain, maxRetries, snapshot
|
|
671
|
+
const { auth, baseUrl, browser, brain, maxRetries, snapshot, timeoutMs, maxSnapshotLines, onEvent, mask } = options;
|
|
570
672
|
const steps = auth.steps;
|
|
673
|
+
const takeSnapshot = snapshot ?? ((page) => defaultSnapshot(page, maxSnapshotLines));
|
|
571
674
|
const context = await browser.newContext();
|
|
572
675
|
try {
|
|
573
676
|
const page = await context.newPage();
|
|
@@ -589,8 +692,9 @@ async function fromSteps(options) {
|
|
|
589
692
|
allowedOrigins: options.allowedOrigins,
|
|
590
693
|
resolveValue: (value) => substituteEnv(value),
|
|
591
694
|
maxRetries,
|
|
695
|
+
timeoutMs,
|
|
592
696
|
mask: (text) => mask.mask(text),
|
|
593
|
-
snapshot,
|
|
697
|
+
snapshot: takeSnapshot,
|
|
594
698
|
onEvent
|
|
595
699
|
}
|
|
596
700
|
);
|
|
@@ -600,7 +704,7 @@ async function fromSteps(options) {
|
|
|
600
704
|
);
|
|
601
705
|
}
|
|
602
706
|
if (auth.verify) {
|
|
603
|
-
const judgment = await brain.judge(auth.verify, mask.mask(await
|
|
707
|
+
const judgment = await brain.judge(auth.verify, mask.mask(await takeSnapshot(page)));
|
|
604
708
|
if (!judgment.pass) {
|
|
605
709
|
throw new AuthError(`Authentication could not be verified: ${mask.mask(judgment.reason)}`);
|
|
606
710
|
}
|
|
@@ -661,7 +765,25 @@ var llmSchema = z.object({
|
|
|
661
765
|
});
|
|
662
766
|
var browserSchema = z.object({
|
|
663
767
|
headless: z.boolean().default(true),
|
|
664
|
-
|
|
768
|
+
/**
|
|
769
|
+
* Bounds every wait: resolving a target element on the accessibility tree, and
|
|
770
|
+
* navigating. An explicit per-call Playwright timeout always overrides
|
|
771
|
+
* `page.setDefaultTimeout()`, so this must be threaded through the action
|
|
772
|
+
* context rather than relied on ambiently (design D2, runner/actions.ts).
|
|
773
|
+
*/
|
|
774
|
+
timeout_ms: z.number().int().positive().default(3e4),
|
|
775
|
+
/**
|
|
776
|
+
* Caps accessibility-tree lines sent to the model per snapshot. Optional:
|
|
777
|
+
* undefined lets `runner/snapshot.ts`'s own default (200) apply, so a config
|
|
778
|
+
* that never mentions this behaves exactly as before it existed.
|
|
779
|
+
*/
|
|
780
|
+
max_snapshot_lines: z.number().int().positive().optional()
|
|
781
|
+
});
|
|
782
|
+
var budgetSchema = z.object({
|
|
783
|
+
max_llm_calls: z.coerce.number().int().positive().optional(),
|
|
784
|
+
max_tokens: z.coerce.number().int().positive().optional(),
|
|
785
|
+
/** Seconds, not milliseconds, to match how a human would set a deadline. */
|
|
786
|
+
max_duration_s: z.coerce.number().positive().optional()
|
|
665
787
|
});
|
|
666
788
|
var cookieSchema = z.object({
|
|
667
789
|
name: z.string().min(1),
|
|
@@ -712,14 +834,18 @@ var configSchema = z.object({
|
|
|
712
834
|
*/
|
|
713
835
|
allowed_origins: z.array(z.string().url()).optional(),
|
|
714
836
|
auth: authSchema.optional(),
|
|
715
|
-
max_retries_per_step: z.number().int().min(1).default(3)
|
|
837
|
+
max_retries_per_step: z.number().int().min(1).default(3),
|
|
838
|
+
budget: budgetSchema.optional()
|
|
716
839
|
});
|
|
717
840
|
var ENV_OVERRIDES = {
|
|
718
841
|
BLASTPROOF_BASE_URL: ["base_url"],
|
|
719
842
|
BLASTPROOF_LLM_PROVIDER: ["llm", "provider"],
|
|
720
843
|
BLASTPROOF_LLM_MODEL: ["llm", "model"],
|
|
721
844
|
BLASTPROOF_LLM_BASE_URL: ["llm", "base_url"],
|
|
722
|
-
BLASTPROOF_LLM_API_KEY_ENV: ["llm", "api_key_env"]
|
|
845
|
+
BLASTPROOF_LLM_API_KEY_ENV: ["llm", "api_key_env"],
|
|
846
|
+
BLASTPROOF_MAX_LLM_CALLS: ["budget", "max_llm_calls"],
|
|
847
|
+
BLASTPROOF_MAX_TOKENS: ["budget", "max_tokens"],
|
|
848
|
+
BLASTPROOF_MAX_DURATION_S: ["budget", "max_duration_s"]
|
|
723
849
|
};
|
|
724
850
|
function applyEnvOverrides(data, env = process.env) {
|
|
725
851
|
const merged = data && typeof data === "object" ? { ...data } : {};
|
|
@@ -750,6 +876,37 @@ var ConfigError = class extends Error {
|
|
|
750
876
|
function formatIssues(error) {
|
|
751
877
|
return error.issues.map((issue) => ` - ${issue.path.join(".") || "(root)"}: ${issue.message}`).join("\n");
|
|
752
878
|
}
|
|
879
|
+
function unwrapToObjectSchema(schema) {
|
|
880
|
+
let current = schema;
|
|
881
|
+
for (; ; ) {
|
|
882
|
+
if (current instanceof z.ZodOptional || current instanceof z.ZodNullable) {
|
|
883
|
+
current = current.unwrap();
|
|
884
|
+
} else if (current instanceof z.ZodDefault) {
|
|
885
|
+
current = current.removeDefault();
|
|
886
|
+
} else if (current instanceof z.ZodEffects) {
|
|
887
|
+
current = current.innerType();
|
|
888
|
+
} else {
|
|
889
|
+
break;
|
|
890
|
+
}
|
|
891
|
+
}
|
|
892
|
+
return current instanceof z.ZodObject ? current : void 0;
|
|
893
|
+
}
|
|
894
|
+
function findUnknownConfigKeys(data, schema = configSchema, path11 = []) {
|
|
895
|
+
const object = unwrapToObjectSchema(schema);
|
|
896
|
+
if (!object || data === null || typeof data !== "object" || Array.isArray(data)) return [];
|
|
897
|
+
const shape = object.shape;
|
|
898
|
+
const warnings = [];
|
|
899
|
+
for (const key of Object.keys(data)) {
|
|
900
|
+
const fieldPath = [...path11, key];
|
|
901
|
+
const fieldSchema = shape[key];
|
|
902
|
+
if (!fieldSchema) {
|
|
903
|
+
warnings.push(fieldPath.join("."));
|
|
904
|
+
continue;
|
|
905
|
+
}
|
|
906
|
+
warnings.push(...findUnknownConfigKeys(data[key], fieldSchema, fieldPath));
|
|
907
|
+
}
|
|
908
|
+
return warnings;
|
|
909
|
+
}
|
|
753
910
|
async function loadConfig(cwd = process.cwd(), env = process.env) {
|
|
754
911
|
const configPath = path4.join(cwd, CONFIG_RELATIVE_PATH);
|
|
755
912
|
let raw;
|
|
@@ -769,12 +926,17 @@ async function loadConfig(cwd = process.cwd(), env = process.env) {
|
|
|
769
926
|
);
|
|
770
927
|
}
|
|
771
928
|
const { data: merged, applied } = applyEnvOverrides(data ?? {}, env);
|
|
929
|
+
const source = applied.length > 0 ? `${CONFIG_RELATIVE_PATH} (with overrides from ${applied.join(", ")})` : CONFIG_RELATIVE_PATH;
|
|
772
930
|
const result = configSchema.safeParse(merged);
|
|
773
931
|
if (!result.success) {
|
|
774
|
-
const source = applied.length > 0 ? `${CONFIG_RELATIVE_PATH} (with overrides from ${applied.join(", ")})` : CONFIG_RELATIVE_PATH;
|
|
775
932
|
throw new ConfigError(`Invalid ${source}:
|
|
776
933
|
${formatIssues(result.error)}`);
|
|
777
934
|
}
|
|
935
|
+
for (const key of findUnknownConfigKeys(merged)) {
|
|
936
|
+
console.warn(
|
|
937
|
+
`warning: unknown config key '${key}' in ${source} \u2014 this version of blastproof does not recognise it, so it has no effect.`
|
|
938
|
+
);
|
|
939
|
+
}
|
|
778
940
|
return result.data;
|
|
779
941
|
}
|
|
780
942
|
|
|
@@ -967,10 +1129,16 @@ var MalformedModelOutputError = class extends Error {
|
|
|
967
1129
|
this.name = "MalformedModelOutputError";
|
|
968
1130
|
}
|
|
969
1131
|
};
|
|
970
|
-
function
|
|
1132
|
+
async function countedGenerate(generate, budget, options) {
|
|
1133
|
+
budget.check();
|
|
1134
|
+
const result = await generate(options);
|
|
1135
|
+
budget.record(result.usage);
|
|
1136
|
+
return result;
|
|
1137
|
+
}
|
|
1138
|
+
function createBrain(model, generate = generateObject, budget) {
|
|
971
1139
|
return {
|
|
972
1140
|
async nextAction(input) {
|
|
973
|
-
const result = await generate
|
|
1141
|
+
const result = await countedGenerate(generate, budget, {
|
|
974
1142
|
model,
|
|
975
1143
|
schema: agentActionSchema,
|
|
976
1144
|
system: agentSystemPrompt(),
|
|
@@ -985,7 +1153,7 @@ function createBrain(model, generate = generateObject) {
|
|
|
985
1153
|
return parsed.data;
|
|
986
1154
|
},
|
|
987
1155
|
async judge(expectation, snapshot) {
|
|
988
|
-
const result = await generate
|
|
1156
|
+
const result = await countedGenerate(generate, budget, {
|
|
989
1157
|
model,
|
|
990
1158
|
schema: assertJudgmentSchema,
|
|
991
1159
|
system: assertSystemPrompt(),
|
|
@@ -1001,10 +1169,10 @@ function createBrain(model, generate = generateObject) {
|
|
|
1001
1169
|
}
|
|
1002
1170
|
};
|
|
1003
1171
|
}
|
|
1004
|
-
function createPlanner(model, generate = generateObject) {
|
|
1172
|
+
function createPlanner(model, generate = generateObject, budget) {
|
|
1005
1173
|
return {
|
|
1006
1174
|
async planTest(input) {
|
|
1007
|
-
const result = await generate
|
|
1175
|
+
const result = await countedGenerate(generate, budget, {
|
|
1008
1176
|
model,
|
|
1009
1177
|
schema: generatedTestSchema,
|
|
1010
1178
|
system: plannerSystemPrompt(),
|
|
@@ -1177,10 +1345,11 @@ function renderTestYaml(draft, meta) {
|
|
|
1177
1345
|
});
|
|
1178
1346
|
}
|
|
1179
1347
|
async function generateForRoute(page, options) {
|
|
1180
|
-
const { route, baseUrl, changedFiles, brain, mask, snapshot
|
|
1348
|
+
const { route, baseUrl, changedFiles, brain, mask, snapshot, maxSnapshotLines, timeoutMs } = options;
|
|
1349
|
+
const takeSnapshot = snapshot ?? ((p) => defaultSnapshot(p, maxSnapshotLines));
|
|
1181
1350
|
const url = new URL(route, baseUrl).toString();
|
|
1182
1351
|
try {
|
|
1183
|
-
await page.goto(url, { timeout:
|
|
1352
|
+
await page.goto(url, { timeout: timeoutMs });
|
|
1184
1353
|
} catch (error) {
|
|
1185
1354
|
throw new PlannerError(
|
|
1186
1355
|
`Cannot load ${url}: ${error instanceof Error ? error.message : String(error)}`
|
|
@@ -1188,7 +1357,7 @@ async function generateForRoute(page, options) {
|
|
|
1188
1357
|
}
|
|
1189
1358
|
const generated = await brain.planTest({
|
|
1190
1359
|
route,
|
|
1191
|
-
snapshot: mask(await
|
|
1360
|
+
snapshot: mask(await takeSnapshot(page)),
|
|
1192
1361
|
changedFiles
|
|
1193
1362
|
});
|
|
1194
1363
|
const leaked = findSecretLiterals(generated.steps);
|
|
@@ -1217,9 +1386,120 @@ function coveredRoutes(tests) {
|
|
|
1217
1386
|
return new Set(tests.flatMap((test) => test.routes));
|
|
1218
1387
|
}
|
|
1219
1388
|
|
|
1389
|
+
// src/runner/browser.ts
|
|
1390
|
+
import { chromium } from "playwright";
|
|
1391
|
+
var BrowserLaunchError = class extends Error {
|
|
1392
|
+
constructor(message) {
|
|
1393
|
+
super(message);
|
|
1394
|
+
this.name = "BrowserLaunchError";
|
|
1395
|
+
}
|
|
1396
|
+
};
|
|
1397
|
+
var MISSING_LIBRARY_PATTERN = /([^\s:]+\.so(?:\.\d+)*): cannot open shared object file/;
|
|
1398
|
+
var MISSING_EXECUTABLE_PATTERN = /executable doesn't exist/i;
|
|
1399
|
+
function stripCommandLine(message) {
|
|
1400
|
+
return message.split("\n").filter((line) => (line.match(/\s--[\w-]+/g)?.length ?? 0) < 3).join("\n").trim();
|
|
1401
|
+
}
|
|
1402
|
+
async function launchBrowser(options) {
|
|
1403
|
+
try {
|
|
1404
|
+
return await chromium.launch(options);
|
|
1405
|
+
} catch (error) {
|
|
1406
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
1407
|
+
if (MISSING_EXECUTABLE_PATTERN.test(message)) {
|
|
1408
|
+
throw new BrowserLaunchError(
|
|
1409
|
+
"The browser is not installed. Run `npx playwright install chromium` to download it."
|
|
1410
|
+
);
|
|
1411
|
+
}
|
|
1412
|
+
const missingLibrary = message.match(MISSING_LIBRARY_PATTERN);
|
|
1413
|
+
if (missingLibrary) {
|
|
1414
|
+
throw new BrowserLaunchError(
|
|
1415
|
+
`The browser could not start because a system library is missing: ${missingLibrary[1]}. Run \`npx playwright install-deps chromium\` to install it \u2014 this needs root/administrator privileges (sudo, or an elevated shell on Windows). Without them, blastproof run --dry-run and blastproof plan --dry-run need neither a browser nor a key.`
|
|
1416
|
+
);
|
|
1417
|
+
}
|
|
1418
|
+
throw new BrowserLaunchError(stripCommandLine(message));
|
|
1419
|
+
}
|
|
1420
|
+
}
|
|
1421
|
+
|
|
1422
|
+
// src/preflight.ts
|
|
1423
|
+
var REACHABILITY_TIMEOUT_MS = 15e3;
|
|
1424
|
+
function messageOf(error) {
|
|
1425
|
+
return error instanceof Error ? error.message : String(error);
|
|
1426
|
+
}
|
|
1427
|
+
function isTimeout(error) {
|
|
1428
|
+
return error instanceof Error && error.name === "AbortError";
|
|
1429
|
+
}
|
|
1430
|
+
async function probe(url, fetchImpl) {
|
|
1431
|
+
const controller = new AbortController();
|
|
1432
|
+
const timer = setTimeout(() => controller.abort(), REACHABILITY_TIMEOUT_MS);
|
|
1433
|
+
try {
|
|
1434
|
+
await fetchImpl(url, { signal: controller.signal });
|
|
1435
|
+
} finally {
|
|
1436
|
+
clearTimeout(timer);
|
|
1437
|
+
}
|
|
1438
|
+
}
|
|
1439
|
+
function providerBaseUrl(llm) {
|
|
1440
|
+
if (llm.base_url) return llm.base_url;
|
|
1441
|
+
switch (llm.provider) {
|
|
1442
|
+
case "anthropic":
|
|
1443
|
+
return "https://api.anthropic.com";
|
|
1444
|
+
case "openai":
|
|
1445
|
+
return "https://api.openai.com";
|
|
1446
|
+
case "ollama":
|
|
1447
|
+
return DEFAULT_OLLAMA_BASE_URL;
|
|
1448
|
+
}
|
|
1449
|
+
}
|
|
1450
|
+
async function checkModel(llm, fetchImpl) {
|
|
1451
|
+
const url = providerBaseUrl(llm);
|
|
1452
|
+
try {
|
|
1453
|
+
await probe(url, fetchImpl);
|
|
1454
|
+
return void 0;
|
|
1455
|
+
} catch (error) {
|
|
1456
|
+
if (isTimeout(error)) {
|
|
1457
|
+
return `Cannot reach the ${llm.provider} provider at ${url}: it did not respond within ${REACHABILITY_TIMEOUT_MS / 1e3}s. It may be temporarily slow or overloaded rather than down \u2014 try again in a moment, or check llm.base_url in .blastproof/config.yaml.`;
|
|
1458
|
+
}
|
|
1459
|
+
return `Cannot reach the ${llm.provider} provider at ${url} (${messageOf(error)}). Check your network connection` + (llm.provider === "ollama" ? ", that Ollama is running (`ollama serve`)," : "") + " or llm.base_url in .blastproof/config.yaml.";
|
|
1460
|
+
}
|
|
1461
|
+
}
|
|
1462
|
+
async function checkBaseUrl(baseUrl, fetchImpl) {
|
|
1463
|
+
try {
|
|
1464
|
+
await probe(baseUrl, fetchImpl);
|
|
1465
|
+
return void 0;
|
|
1466
|
+
} catch (error) {
|
|
1467
|
+
if (isTimeout(error)) {
|
|
1468
|
+
return `The application at ${baseUrl} did not respond within ${REACHABILITY_TIMEOUT_MS / 1e3}s. It may still be starting (a dev server compiling a route on first request, a cold-starting container) rather than down \u2014 try again in a moment.`;
|
|
1469
|
+
}
|
|
1470
|
+
return `The application at ${baseUrl} is not responding (${messageOf(error)}). Start it, or point --url / base_url in .blastproof/config.yaml at the right address.`;
|
|
1471
|
+
}
|
|
1472
|
+
}
|
|
1473
|
+
async function checkBrowser(headless) {
|
|
1474
|
+
try {
|
|
1475
|
+
const browser = await launchBrowser({ headless });
|
|
1476
|
+
return { browser };
|
|
1477
|
+
} catch (error) {
|
|
1478
|
+
return { failure: error instanceof BrowserLaunchError ? error.message : messageOf(error) };
|
|
1479
|
+
}
|
|
1480
|
+
}
|
|
1481
|
+
async function runPreflight(needs, config, fetchImpl = fetch) {
|
|
1482
|
+
const [browserResult, modelFailure, baseUrlFailure] = await Promise.all([
|
|
1483
|
+
needs.browser ? checkBrowser(config.browser.headless) : Promise.resolve({}),
|
|
1484
|
+
needs.model ? checkModel(config.llm, fetchImpl) : Promise.resolve(void 0),
|
|
1485
|
+
needs.baseUrl ? checkBaseUrl(config.base_url, fetchImpl) : Promise.resolve(void 0)
|
|
1486
|
+
]);
|
|
1487
|
+
const failures = [browserResult.failure, modelFailure, baseUrlFailure].filter(
|
|
1488
|
+
(value) => value !== void 0
|
|
1489
|
+
);
|
|
1490
|
+
if (failures.length > 0) {
|
|
1491
|
+
if (browserResult.browser) await browserResult.browser.close();
|
|
1492
|
+
return { ok: false, failures };
|
|
1493
|
+
}
|
|
1494
|
+
return { ok: true, browser: browserResult.browser };
|
|
1495
|
+
}
|
|
1496
|
+
function printPreflightFailures(failures) {
|
|
1497
|
+
console.error(`error: ${failures.length} unmet prerequisite(s):`);
|
|
1498
|
+
for (const failure of failures) console.error(` - ${failure}`);
|
|
1499
|
+
}
|
|
1500
|
+
|
|
1220
1501
|
// src/commands/run.ts
|
|
1221
1502
|
import path9 from "path";
|
|
1222
|
-
import { chromium } from "playwright";
|
|
1223
1503
|
|
|
1224
1504
|
// src/report/html.ts
|
|
1225
1505
|
import { mkdir as mkdir4, readFile as readFile4, writeFile as writeFile4 } from "fs/promises";
|
|
@@ -1262,6 +1542,8 @@ h1 { font-size:1.35rem; margin:0 0 .25rem; }
|
|
|
1262
1542
|
.verdict { font-weight:600; }
|
|
1263
1543
|
.verdict.pass { color:var(--pass); } .verdict.fail { color:var(--fail); }
|
|
1264
1544
|
.counts { color:var(--muted); font-size:.9rem; }
|
|
1545
|
+
.incomplete { padding:1rem 1.25rem; border:1px solid var(--fail); border-radius:10px;
|
|
1546
|
+
background:color-mix(in srgb, var(--fail) 10%, transparent); margin-bottom:1.5rem; font-weight:600; }
|
|
1265
1547
|
table { width:100%; border-collapse:collapse; margin-bottom:1.5rem; }
|
|
1266
1548
|
th, td { text-align:left; padding:.5rem .6rem; border-bottom:1px solid var(--line);
|
|
1267
1549
|
font-size:.9rem; vertical-align:top; }
|
|
@@ -1298,10 +1580,13 @@ function renderSteps(steps) {
|
|
|
1298
1580
|
}
|
|
1299
1581
|
async function renderTest(result, cwd) {
|
|
1300
1582
|
const failed = result.status === "failed";
|
|
1583
|
+
const notRun = result.status === "not-run";
|
|
1584
|
+
const tagClass = failed ? "fail" : notRun ? "skip" : "pass";
|
|
1585
|
+
const label = failed ? "FAIL" : notRun ? "NOT RUN" : "PASS";
|
|
1301
1586
|
const file = cwd ? path7.relative(cwd, result.file) : result.file;
|
|
1302
1587
|
const parts = [
|
|
1303
|
-
` <details${failed ? " open" : ""}>`,
|
|
1304
|
-
` <summary><span class="tag ${
|
|
1588
|
+
` <details${failed || notRun ? " open" : ""}>`,
|
|
1589
|
+
` <summary><span class="tag ${tagClass}">${label}</span> ${escapeHtml(result.summary)} <span class="note">${escapeHtml(result.priority)} \xB7 ${escapeHtml(file)}</span></summary>`
|
|
1305
1590
|
];
|
|
1306
1591
|
if (failed) {
|
|
1307
1592
|
if (result.failedStep) {
|
|
@@ -1311,6 +1596,9 @@ async function renderTest(result, cwd) {
|
|
|
1311
1596
|
parts.push(` <p class="reason">${escapeHtml(result.reason)}</p>`);
|
|
1312
1597
|
}
|
|
1313
1598
|
}
|
|
1599
|
+
if (notRun && result.reason) {
|
|
1600
|
+
parts.push(` <p class="note">${escapeHtml(result.reason)}</p>`);
|
|
1601
|
+
}
|
|
1314
1602
|
parts.push(` ${renderSteps(result.steps)}`);
|
|
1315
1603
|
if (failed && result.screenshot) {
|
|
1316
1604
|
const embedded = await embedScreenshot(result.screenshot);
|
|
@@ -1324,13 +1612,20 @@ async function renderTest(result, cwd) {
|
|
|
1324
1612
|
async function renderHtml(results, skipped, meta) {
|
|
1325
1613
|
const passed = results.filter((r) => r.status === "passed").length;
|
|
1326
1614
|
const failures = results.filter((r) => r.status === "failed");
|
|
1615
|
+
const notRun = results.filter((r) => r.status === "not-run");
|
|
1327
1616
|
const generatedAt = (meta.generatedAt ?? /* @__PURE__ */ new Date()).toISOString();
|
|
1328
|
-
const ordered = [...failures, ...results.filter((r) => r.status === "passed")];
|
|
1617
|
+
const ordered = [...failures, ...notRun, ...results.filter((r) => r.status === "passed")];
|
|
1329
1618
|
const detail = (await Promise.all(ordered.map((r) => renderTest(r, meta.cwd)))).join("\n");
|
|
1330
1619
|
const verdict = meta.minScore === void 0 ? "" : meta.score >= meta.minScore ? `<span class="verdict pass">min-score ${meta.minScore}: pass</span>` : `<span class="verdict fail">min-score ${meta.minScore}: FAIL</span>`;
|
|
1620
|
+
const banner = meta.incomplete === void 0 ? "" : ` <section class="incomplete">Run stopped: ${escapeHtml(meta.incomplete)}</section>
|
|
1621
|
+
|
|
1622
|
+
`;
|
|
1331
1623
|
const rows = results.map((r) => {
|
|
1332
1624
|
const file = meta.cwd ? path7.relative(meta.cwd, r.file) : r.file;
|
|
1333
|
-
|
|
1625
|
+
const tagClass = r.status === "failed" ? "fail" : r.status === "not-run" ? "skip" : "pass";
|
|
1626
|
+
const label = r.status === "failed" ? "FAIL" : r.status === "not-run" ? "NOT RUN" : "PASS";
|
|
1627
|
+
const time = r.status === "not-run" ? "\u2014" : seconds(r.durationMs);
|
|
1628
|
+
return ` <tr><td><span class="tag ${tagClass}">${label}</span></td><td>${escapeHtml(r.priority)}</td><td>${escapeHtml(r.summary)}</td><td>${escapeHtml(file)}</td><td class="num">${time}</td></tr>`;
|
|
1334
1629
|
}).join("\n");
|
|
1335
1630
|
const skippedRows = skipped.map((s) => {
|
|
1336
1631
|
const file = meta.cwd ? path7.relative(meta.cwd, s.path) : s.path;
|
|
@@ -1349,10 +1644,10 @@ async function renderHtml(results, skipped, meta) {
|
|
|
1349
1644
|
<h1>blastproof report</h1>
|
|
1350
1645
|
<p class="sub">${escapeHtml(generatedAt)} \xB7 ${seconds(meta.durationMs)}</p>
|
|
1351
1646
|
|
|
1352
|
-
<section class="score">
|
|
1647
|
+
${banner} <section class="score">
|
|
1353
1648
|
<b>${meta.score}</b>
|
|
1354
1649
|
${verdict}
|
|
1355
|
-
<span class="counts">${passed} passed \xB7 ${failures.length} failed${skipped.length > 0 ? ` \xB7 ${skipped.length} skipped` : ""}</span>
|
|
1650
|
+
<span class="counts">${passed} passed \xB7 ${failures.length} failed${notRun.length > 0 ? ` \xB7 ${notRun.length} not run` : ""}${skipped.length > 0 ? ` \xB7 ${skipped.length} skipped` : ""}</span>
|
|
1356
1651
|
</section>
|
|
1357
1652
|
|
|
1358
1653
|
<table>
|
|
@@ -1395,14 +1690,27 @@ function seconds2(ms) {
|
|
|
1395
1690
|
function renderJUnit(results, skipped, meta) {
|
|
1396
1691
|
const relative = (file) => meta.cwd ? path8.relative(meta.cwd, file) : file;
|
|
1397
1692
|
const failures = results.filter((result) => result.status === "failed").length;
|
|
1693
|
+
const notRun = results.filter((result) => result.status === "not-run");
|
|
1398
1694
|
const lines = [
|
|
1399
1695
|
'<?xml version="1.0" encoding="UTF-8"?>',
|
|
1400
|
-
`<testsuite name="blastproof" tests="${results.length + skipped.length}" failures="${failures}" skipped="${skipped.length}" time="${seconds2(meta.durationMs)}">`,
|
|
1696
|
+
`<testsuite name="blastproof" tests="${results.length + skipped.length}" failures="${failures}" skipped="${skipped.length + notRun.length}" time="${seconds2(meta.durationMs)}">`,
|
|
1401
1697
|
" <properties>",
|
|
1402
1698
|
` <property name="score" value="${meta.score}"/>`,
|
|
1699
|
+
...meta.incomplete !== void 0 ? [
|
|
1700
|
+
' <property name="incomplete" value="true"/>',
|
|
1701
|
+
` <property name="incomplete_reason" value="${escapeXml(meta.incomplete)}"/>`
|
|
1702
|
+
] : [],
|
|
1403
1703
|
" </properties>"
|
|
1404
1704
|
];
|
|
1405
1705
|
for (const result of results) {
|
|
1706
|
+
if (result.status === "not-run") {
|
|
1707
|
+
lines.push(
|
|
1708
|
+
` <testcase classname="${escapeXml(relative(result.file))}" name="${escapeXml(result.summary)}" time="0.000">`,
|
|
1709
|
+
` <skipped message="${escapeXml(result.reason ?? "not run: the run stopped before this test executed")}"/>`,
|
|
1710
|
+
" </testcase>"
|
|
1711
|
+
);
|
|
1712
|
+
continue;
|
|
1713
|
+
}
|
|
1406
1714
|
const open = ` <testcase classname="${escapeXml(relative(result.file))}" name="${escapeXml(result.summary)}" time="${seconds2(result.durationMs)}"`;
|
|
1407
1715
|
if (result.status === "passed") {
|
|
1408
1716
|
lines.push(`${open}/>`);
|
|
@@ -1438,6 +1746,7 @@ function computeScore(results) {
|
|
|
1438
1746
|
let total = 0;
|
|
1439
1747
|
let passed = 0;
|
|
1440
1748
|
for (const result of results) {
|
|
1749
|
+
if (result.status === "not-run") continue;
|
|
1441
1750
|
const weight = WEIGHTS[result.priority] ?? DEFAULT_WEIGHT;
|
|
1442
1751
|
total += weight;
|
|
1443
1752
|
if (result.status === "passed") passed += weight;
|
|
@@ -1453,6 +1762,10 @@ function formatScoreLine(score, results, threshold) {
|
|
|
1453
1762
|
if (threshold === void 0) return `Score: ${score}`;
|
|
1454
1763
|
return score >= threshold ? `Score: ${score} \u2014 min-score ${threshold}: pass` : `Score: ${score} \u2014 min-score ${threshold}: FAIL (below threshold)`;
|
|
1455
1764
|
}
|
|
1765
|
+
function formatIncompleteLine(score, reason) {
|
|
1766
|
+
return `Run incomplete: ${reason}
|
|
1767
|
+
Score over executed tests: ${score} (not a verdict \u2014 exit code 1 regardless of --min-score)`;
|
|
1768
|
+
}
|
|
1456
1769
|
|
|
1457
1770
|
// src/runner/selection.ts
|
|
1458
1771
|
function matchesFilters(test, filters) {
|
|
@@ -1492,6 +1805,14 @@ function selectImpactedTests(tests, affectedRoutes, filters = { tags: [] }) {
|
|
|
1492
1805
|
var EXIT_OK = 0;
|
|
1493
1806
|
var EXIT_FAILED = 1;
|
|
1494
1807
|
var EXIT_USAGE = 2;
|
|
1808
|
+
function resolveBudgetOptions(config, options) {
|
|
1809
|
+
const maxDurationSeconds = options.maxDuration ?? config.budget?.max_duration_s;
|
|
1810
|
+
return {
|
|
1811
|
+
maxCalls: options.maxLlmCalls ?? config.budget?.max_llm_calls,
|
|
1812
|
+
maxTokens: options.maxTokens ?? config.budget?.max_tokens,
|
|
1813
|
+
maxDurationMs: maxDurationSeconds === void 0 ? void 0 : maxDurationSeconds * 1e3
|
|
1814
|
+
};
|
|
1815
|
+
}
|
|
1495
1816
|
function sessionId() {
|
|
1496
1817
|
return (/* @__PURE__ */ new Date()).toISOString().replace(/[:.]/g, "-");
|
|
1497
1818
|
}
|
|
@@ -1522,6 +1843,18 @@ function resolveSecretsAndSteps(test) {
|
|
|
1522
1843
|
}
|
|
1523
1844
|
return { test };
|
|
1524
1845
|
}
|
|
1846
|
+
function notRunResults(tests, reason) {
|
|
1847
|
+
return tests.map((test) => ({
|
|
1848
|
+
file: test.path,
|
|
1849
|
+
summary: test.summary,
|
|
1850
|
+
priority: test.priority,
|
|
1851
|
+
tags: test.tags,
|
|
1852
|
+
status: "not-run",
|
|
1853
|
+
steps: [],
|
|
1854
|
+
reason,
|
|
1855
|
+
durationMs: 0
|
|
1856
|
+
}));
|
|
1857
|
+
}
|
|
1525
1858
|
function printEvent(event) {
|
|
1526
1859
|
switch (event.type) {
|
|
1527
1860
|
case "step-start":
|
|
@@ -1541,12 +1874,18 @@ function printEvent(event) {
|
|
|
1541
1874
|
break;
|
|
1542
1875
|
}
|
|
1543
1876
|
}
|
|
1877
|
+
function statusLabel(status) {
|
|
1878
|
+
if (status === "passed") return "PASS";
|
|
1879
|
+
if (status === "failed") return "FAIL";
|
|
1880
|
+
return "NOT RUN";
|
|
1881
|
+
}
|
|
1544
1882
|
function printSummary(results) {
|
|
1545
1883
|
const passed = results.filter((r) => r.status === "passed");
|
|
1546
1884
|
const failed = results.filter((r) => r.status === "failed");
|
|
1885
|
+
const notRun = results.filter((r) => r.status === "not-run");
|
|
1547
1886
|
console.log("\n--- Summary ---------------------------------------------------");
|
|
1548
1887
|
const rows = results.map((r) => ({
|
|
1549
|
-
status: r.status
|
|
1888
|
+
status: statusLabel(r.status),
|
|
1550
1889
|
priority: r.priority,
|
|
1551
1890
|
summary: r.summary,
|
|
1552
1891
|
duration: `${(r.durationMs / 1e3).toFixed(1)}s`
|
|
@@ -1559,7 +1898,9 @@ function printSummary(results) {
|
|
|
1559
1898
|
);
|
|
1560
1899
|
}
|
|
1561
1900
|
console.log("---------------------------------------------------------------");
|
|
1562
|
-
console.log(
|
|
1901
|
+
console.log(
|
|
1902
|
+
notRun.length > 0 ? `${passed.length} passed, ${failed.length} failed, ${notRun.length} not run, ${results.length} total` : `${passed.length} passed, ${failed.length} failed, ${results.length} total`
|
|
1903
|
+
);
|
|
1563
1904
|
for (const r of failed) {
|
|
1564
1905
|
console.log(`
|
|
1565
1906
|
X ${r.summary} (${r.file})`);
|
|
@@ -1567,9 +1908,14 @@ X ${r.summary} (${r.file})`);
|
|
|
1567
1908
|
if (r.reason) console.log(` reason: ${r.reason}`);
|
|
1568
1909
|
if (r.screenshot) console.log(` screenshot: ${r.screenshot}`);
|
|
1569
1910
|
}
|
|
1911
|
+
if (notRun.length > 0) {
|
|
1912
|
+
console.log(`
|
|
1913
|
+
${notRun.length} test(s) not run (run stopped by its budget or deadline):`);
|
|
1914
|
+
for (const r of notRun) console.log(` - ${r.summary} (${r.file})`);
|
|
1915
|
+
}
|
|
1570
1916
|
}
|
|
1571
|
-
async function runOne(browser, test, config, sessionDir, session, runMask) {
|
|
1572
|
-
const brain = createBrain(createModel(config.llm).model);
|
|
1917
|
+
async function runOne(browser, test, config, sessionDir, session, runMask, budget) {
|
|
1918
|
+
const brain = createBrain(createModel(config.llm).model, void 0, budget);
|
|
1573
1919
|
let resolved;
|
|
1574
1920
|
const mask = runMask;
|
|
1575
1921
|
try {
|
|
@@ -1600,6 +1946,8 @@ async function runOne(browser, test, config, sessionDir, session, runMask) {
|
|
|
1600
1946
|
allowedOrigins: config.allowed_origins,
|
|
1601
1947
|
resolveValue: (value) => substituteEnv(value),
|
|
1602
1948
|
maxRetries: config.max_retries_per_step,
|
|
1949
|
+
timeoutMs: config.browser.timeout_ms,
|
|
1950
|
+
maxSnapshotLines: config.browser.max_snapshot_lines,
|
|
1603
1951
|
mask: (text) => mask.mask(text),
|
|
1604
1952
|
onEvent: printEvent
|
|
1605
1953
|
});
|
|
@@ -1632,13 +1980,20 @@ function printImpactReport(impact, selection, cwd) {
|
|
|
1632
1980
|
}
|
|
1633
1981
|
console.log("---------------------------------------------------------------");
|
|
1634
1982
|
}
|
|
1635
|
-
async function finalize(results, skipped, options, sessionDir, durationMs, impact) {
|
|
1983
|
+
async function finalize(results, skipped, options, sessionDir, durationMs, impact, incomplete) {
|
|
1636
1984
|
if (results.length > 0) printSummary(results);
|
|
1637
1985
|
const score = computeScore(results);
|
|
1638
|
-
console.log(
|
|
1986
|
+
console.log(
|
|
1987
|
+
incomplete ? formatIncompleteLine(score, incomplete.message) : formatScoreLine(score, results, options.minScore)
|
|
1988
|
+
);
|
|
1639
1989
|
if (options.junit) {
|
|
1640
1990
|
const target = typeof options.junit === "string" ? path9.resolve(options.cwd, options.junit) : path9.join(sessionDir, "junit.xml");
|
|
1641
|
-
const xml = renderJUnit(results, skipped, {
|
|
1991
|
+
const xml = renderJUnit(results, skipped, {
|
|
1992
|
+
score,
|
|
1993
|
+
durationMs,
|
|
1994
|
+
cwd: options.cwd,
|
|
1995
|
+
incomplete: incomplete?.message
|
|
1996
|
+
});
|
|
1642
1997
|
await writeJUnit(target, xml);
|
|
1643
1998
|
console.log(`JUnit report: ${path9.relative(options.cwd, target)}`);
|
|
1644
1999
|
}
|
|
@@ -1648,11 +2003,13 @@ async function finalize(results, skipped, options, sessionDir, durationMs, impac
|
|
|
1648
2003
|
score,
|
|
1649
2004
|
durationMs,
|
|
1650
2005
|
minScore: options.minScore,
|
|
1651
|
-
cwd: options.cwd
|
|
2006
|
+
cwd: options.cwd,
|
|
2007
|
+
incomplete: incomplete?.message
|
|
1652
2008
|
});
|
|
1653
2009
|
await writeHtml(target, html);
|
|
1654
2010
|
console.log(`HTML report: ${path9.relative(options.cwd, target)}`);
|
|
1655
2011
|
}
|
|
2012
|
+
if (incomplete) return EXIT_FAILED;
|
|
1656
2013
|
if (reportUnclassified(options, impact)) return EXIT_FAILED;
|
|
1657
2014
|
if (options.minScore !== void 0) {
|
|
1658
2015
|
return score >= options.minScore ? EXIT_OK : EXIT_FAILED;
|
|
@@ -1671,12 +2028,23 @@ function reportUnclassified(options, impact) {
|
|
|
1671
2028
|
);
|
|
1672
2029
|
return true;
|
|
1673
2030
|
}
|
|
1674
|
-
function printDryRun(selected,
|
|
2031
|
+
function printDryRun(selected, config, cwd) {
|
|
1675
2032
|
console.log(`
|
|
1676
|
-
Dry run: ${selected.length} test(s) selected, base_url=${
|
|
2033
|
+
Dry run: ${selected.length} test(s) selected, base_url=${config.base_url}`);
|
|
1677
2034
|
for (const test of selected) {
|
|
1678
2035
|
console.log(` ${test.summary} [${test.priority}] (${path9.relative(cwd, test.path)})`);
|
|
1679
2036
|
}
|
|
2037
|
+
const authSteps = config.auth?.steps ?? [];
|
|
2038
|
+
const withAuth = authSteps.length > 0 ? [...selected, { steps: authSteps }] : selected;
|
|
2039
|
+
const ceiling = estimateMaxModelCalls(
|
|
2040
|
+
withAuth,
|
|
2041
|
+
DEFAULT_MAX_ITERATIONS_PER_STEP,
|
|
2042
|
+
config.max_retries_per_step
|
|
2043
|
+
);
|
|
2044
|
+
const authNote = authSteps.length > 0 ? ", including the login journey" : "";
|
|
2045
|
+
console.log(
|
|
2046
|
+
`Worst case: up to ${ceiling} model call(s) for this selection${authNote} (a maximum, not a prediction).`
|
|
2047
|
+
);
|
|
1680
2048
|
console.log("Dry run: no browser launched, no LLM calls made.");
|
|
1681
2049
|
}
|
|
1682
2050
|
async function runCommand(options) {
|
|
@@ -1752,7 +2120,7 @@ async function runCommand(options) {
|
|
|
1752
2120
|
printImpactReport(impact, selection, options.cwd);
|
|
1753
2121
|
}
|
|
1754
2122
|
if (options.dryRun) {
|
|
1755
|
-
printDryRun(selected, config
|
|
2123
|
+
printDryRun(selected, config, options.cwd);
|
|
1756
2124
|
if (results.length > 0) {
|
|
1757
2125
|
console.error(`
|
|
1758
2126
|
${results.length} test file(s) could not be parsed:`);
|
|
@@ -1783,7 +2151,14 @@ ${results.length} test file(s) could not be parsed:`);
|
|
|
1783
2151
|
`blastproof run: ${selected.length} test(s), provider=${provider} model=${modelId}, base_url=${config.base_url}`
|
|
1784
2152
|
);
|
|
1785
2153
|
const runMask = buildRunMask(config, parsed);
|
|
1786
|
-
const
|
|
2154
|
+
const budget = options.budget ?? new RunBudget(resolveBudgetOptions(config, options));
|
|
2155
|
+
const preflight = await runPreflight({ browser: true, model: true, baseUrl: true }, config);
|
|
2156
|
+
if (!preflight.ok) {
|
|
2157
|
+
printPreflightFailures(preflight.failures);
|
|
2158
|
+
return EXIT_USAGE;
|
|
2159
|
+
}
|
|
2160
|
+
const browser = preflight.browser;
|
|
2161
|
+
let incomplete;
|
|
1787
2162
|
try {
|
|
1788
2163
|
let session;
|
|
1789
2164
|
if (config.auth) {
|
|
@@ -1795,28 +2170,56 @@ ${results.length} test file(s) could not be parsed:`);
|
|
|
1795
2170
|
baseUrl: config.base_url,
|
|
1796
2171
|
allowedOrigins: config.allowed_origins,
|
|
1797
2172
|
browser,
|
|
1798
|
-
brain: createBrain(createModel(config.llm).model),
|
|
2173
|
+
brain: createBrain(createModel(config.llm).model, void 0, budget),
|
|
1799
2174
|
maxRetries: config.max_retries_per_step,
|
|
2175
|
+
timeoutMs: config.browser.timeout_ms,
|
|
2176
|
+
maxSnapshotLines: config.browser.max_snapshot_lines,
|
|
1800
2177
|
mask: runMask,
|
|
1801
2178
|
onEvent: printEvent
|
|
1802
2179
|
});
|
|
1803
2180
|
} catch (error) {
|
|
1804
|
-
if (error instanceof
|
|
2181
|
+
if (error instanceof BudgetExhaustedError) {
|
|
2182
|
+
incomplete = error;
|
|
2183
|
+
} else if (error instanceof AuthError) {
|
|
1805
2184
|
console.error(`error: ${error.message}`);
|
|
1806
2185
|
return EXIT_USAGE;
|
|
2186
|
+
} else {
|
|
2187
|
+
throw error;
|
|
1807
2188
|
}
|
|
1808
|
-
throw error;
|
|
1809
2189
|
}
|
|
1810
2190
|
}
|
|
1811
|
-
|
|
1812
|
-
|
|
2191
|
+
if (incomplete) {
|
|
2192
|
+
results.push(...notRunResults(selected, incomplete.message));
|
|
2193
|
+
} else {
|
|
2194
|
+
for (let i = 0; i < selected.length; i++) {
|
|
2195
|
+
const test = selected[i];
|
|
2196
|
+
console.log(`
|
|
1813
2197
|
> ${test.summary} [${test.priority}] (${path9.relative(options.cwd, test.path)})`);
|
|
1814
|
-
|
|
2198
|
+
try {
|
|
2199
|
+
budget.check();
|
|
2200
|
+
results.push(await runOne(browser, test, config, sessionDir, session, runMask, budget));
|
|
2201
|
+
} catch (error) {
|
|
2202
|
+
if (error instanceof BudgetExhaustedError) {
|
|
2203
|
+
incomplete = error;
|
|
2204
|
+
results.push(...notRunResults(selected.slice(i), error.message));
|
|
2205
|
+
break;
|
|
2206
|
+
}
|
|
2207
|
+
throw error;
|
|
2208
|
+
}
|
|
2209
|
+
}
|
|
1815
2210
|
}
|
|
1816
2211
|
} finally {
|
|
1817
2212
|
await browser.close();
|
|
1818
2213
|
}
|
|
1819
|
-
return finalize(
|
|
2214
|
+
return finalize(
|
|
2215
|
+
results,
|
|
2216
|
+
selection.unroutedSkipped,
|
|
2217
|
+
options,
|
|
2218
|
+
sessionDir,
|
|
2219
|
+
Date.now() - startedAt,
|
|
2220
|
+
impact,
|
|
2221
|
+
incomplete
|
|
2222
|
+
);
|
|
1820
2223
|
}
|
|
1821
2224
|
|
|
1822
2225
|
// src/commands/plan.ts
|
|
@@ -1897,6 +2300,16 @@ async function planCommand(options) {
|
|
|
1897
2300
|
console.log("Already covered (skipped):");
|
|
1898
2301
|
for (const route of alreadyCovered) console.log(` ${route}`);
|
|
1899
2302
|
}
|
|
2303
|
+
if (options.dryRun) {
|
|
2304
|
+
if (work.length === 0) {
|
|
2305
|
+
console.log("Nothing to generate: no affected route is missing coverage.");
|
|
2306
|
+
} else {
|
|
2307
|
+
console.log(`Dry run: ${work.length} route(s) would generate a draft for:`);
|
|
2308
|
+
for (const { route } of work) console.log(` ${route}`);
|
|
2309
|
+
}
|
|
2310
|
+
console.log("Dry run: no browser launched, no LLM calls made.");
|
|
2311
|
+
return EXIT_OK;
|
|
2312
|
+
}
|
|
1900
2313
|
if (work.length === 0) {
|
|
1901
2314
|
console.log("Nothing to generate: no affected route is missing coverage.");
|
|
1902
2315
|
return EXIT_OK;
|
|
@@ -1911,7 +2324,8 @@ async function planCommand(options) {
|
|
|
1911
2324
|
throw error;
|
|
1912
2325
|
}
|
|
1913
2326
|
const { model, provider, modelId } = createModel(config.llm);
|
|
1914
|
-
const
|
|
2327
|
+
const budget = options.budget ?? new RunBudget(resolveBudgetOptions(config, options));
|
|
2328
|
+
const brain = createPlanner(model, void 0, budget);
|
|
1915
2329
|
const base = options.routes && options.routes.length > 0 ? void 0 : options.base ?? "main";
|
|
1916
2330
|
console.log(
|
|
1917
2331
|
`blastproof plan: ${work.length} route(s), provider=${provider} model=${modelId}, base_url=${config.base_url}`
|
|
@@ -1926,7 +2340,13 @@ async function planCommand(options) {
|
|
|
1926
2340
|
const generated = [];
|
|
1927
2341
|
const written = [];
|
|
1928
2342
|
const failed = [];
|
|
1929
|
-
|
|
2343
|
+
let incomplete;
|
|
2344
|
+
const preflight = await runPreflight({ browser: true, model: true, baseUrl: true }, config);
|
|
2345
|
+
if (!preflight.ok) {
|
|
2346
|
+
printPreflightFailures(preflight.failures);
|
|
2347
|
+
return EXIT_USAGE;
|
|
2348
|
+
}
|
|
2349
|
+
const browser = preflight.browser;
|
|
1930
2350
|
try {
|
|
1931
2351
|
let session;
|
|
1932
2352
|
if (config.auth) {
|
|
@@ -1938,20 +2358,25 @@ async function planCommand(options) {
|
|
|
1938
2358
|
baseUrl: config.base_url,
|
|
1939
2359
|
allowedOrigins: config.allowed_origins,
|
|
1940
2360
|
browser,
|
|
1941
|
-
brain: createBrain(createModel(config.llm).model),
|
|
2361
|
+
brain: createBrain(createModel(config.llm).model, void 0, budget),
|
|
1942
2362
|
maxRetries: config.max_retries_per_step,
|
|
2363
|
+
timeoutMs: config.browser.timeout_ms,
|
|
2364
|
+
maxSnapshotLines: config.browser.max_snapshot_lines,
|
|
1943
2365
|
mask,
|
|
1944
2366
|
onEvent: printAuthEvent
|
|
1945
2367
|
});
|
|
1946
2368
|
} catch (error) {
|
|
1947
|
-
if (error instanceof
|
|
2369
|
+
if (error instanceof BudgetExhaustedError) {
|
|
2370
|
+
incomplete = error;
|
|
2371
|
+
} else if (error instanceof AuthError) {
|
|
1948
2372
|
console.error(`error: ${error.message}`);
|
|
1949
2373
|
return EXIT_USAGE;
|
|
2374
|
+
} else {
|
|
2375
|
+
throw error;
|
|
1950
2376
|
}
|
|
1951
|
-
throw error;
|
|
1952
2377
|
}
|
|
1953
2378
|
}
|
|
1954
|
-
for (const { route, changedFiles } of work) {
|
|
2379
|
+
for (const { route, changedFiles } of incomplete ? [] : work) {
|
|
1955
2380
|
console.log(`
|
|
1956
2381
|
> ${route}`);
|
|
1957
2382
|
let draft;
|
|
@@ -1964,9 +2389,15 @@ async function planCommand(options) {
|
|
|
1964
2389
|
baseUrl: config.base_url,
|
|
1965
2390
|
changedFiles,
|
|
1966
2391
|
brain,
|
|
1967
|
-
mask: (text) => mask.mask(text)
|
|
2392
|
+
mask: (text) => mask.mask(text),
|
|
2393
|
+
maxSnapshotLines: config.browser.max_snapshot_lines,
|
|
2394
|
+
timeoutMs: config.browser.timeout_ms
|
|
1968
2395
|
});
|
|
1969
2396
|
} catch (error) {
|
|
2397
|
+
if (error instanceof BudgetExhaustedError) {
|
|
2398
|
+
incomplete = error;
|
|
2399
|
+
break;
|
|
2400
|
+
}
|
|
1970
2401
|
failed.push({ route, reason: error instanceof Error ? error.message : String(error) });
|
|
1971
2402
|
console.log(` X failed: ${error instanceof Error ? error.message : String(error)}`);
|
|
1972
2403
|
continue;
|
|
@@ -1996,6 +2427,7 @@ ${renderTestYaml(draft, { route, base })}`);
|
|
|
1996
2427
|
} finally {
|
|
1997
2428
|
await browser.close();
|
|
1998
2429
|
}
|
|
2430
|
+
const notAttempted = incomplete ? work.map((item) => item.route).filter((route) => !generated.includes(route) && !failed.some((f) => f.route === route)) : [];
|
|
1999
2431
|
console.log("\n--- Plan -------------------------------------------------------");
|
|
2000
2432
|
console.log(`Generated: ${generated.length > 0 ? generated.join(", ") : "none"}`);
|
|
2001
2433
|
if (written.length > 0) {
|
|
@@ -2009,8 +2441,15 @@ ${renderTestYaml(draft, { route, base })}`);
|
|
|
2009
2441
|
console.log("Failed:");
|
|
2010
2442
|
for (const { route, reason } of failed) console.log(` ${route}: ${reason}`);
|
|
2011
2443
|
}
|
|
2444
|
+
if (incomplete) {
|
|
2445
|
+
console.log(`Stopped: ${incomplete.message}`);
|
|
2446
|
+
if (notAttempted.length > 0) {
|
|
2447
|
+
console.log("Not attempted (run out of budget):");
|
|
2448
|
+
for (const route of notAttempted) console.log(` ${route}`);
|
|
2449
|
+
}
|
|
2450
|
+
}
|
|
2012
2451
|
console.log("---------------------------------------------------------------");
|
|
2013
|
-
return failed.length > 0 ? EXIT_FAILED : EXIT_OK;
|
|
2452
|
+
return incomplete || failed.length > 0 ? EXIT_FAILED : EXIT_OK;
|
|
2014
2453
|
}
|
|
2015
2454
|
|
|
2016
2455
|
// src/commands/test.ts
|
|
@@ -2019,6 +2458,17 @@ function section(title) {
|
|
|
2019
2458
|
=== ${title} ${"=".repeat(Math.max(0, 58 - title.length))}`);
|
|
2020
2459
|
}
|
|
2021
2460
|
async function testCommand(options) {
|
|
2461
|
+
let config;
|
|
2462
|
+
try {
|
|
2463
|
+
config = applyUrlOverride(await loadConfig(options.cwd), options.url);
|
|
2464
|
+
} catch (error) {
|
|
2465
|
+
if (error instanceof ConfigError) {
|
|
2466
|
+
console.error(`error: ${error.message}`);
|
|
2467
|
+
return EXIT_USAGE;
|
|
2468
|
+
}
|
|
2469
|
+
throw error;
|
|
2470
|
+
}
|
|
2471
|
+
const budget = new RunBudget(resolveBudgetOptions(config, options));
|
|
2022
2472
|
section("Verify: tests covering the affected routes");
|
|
2023
2473
|
const runCode = await runCommand({
|
|
2024
2474
|
cwd: options.cwd,
|
|
@@ -2029,7 +2479,8 @@ async function testCommand(options) {
|
|
|
2029
2479
|
minScore: options.minScore,
|
|
2030
2480
|
junit: options.junit,
|
|
2031
2481
|
html: options.html,
|
|
2032
|
-
failOnUnmapped: options.failOnUnmapped
|
|
2482
|
+
failOnUnmapped: options.failOnUnmapped,
|
|
2483
|
+
budget
|
|
2033
2484
|
});
|
|
2034
2485
|
if (runCode === EXIT_USAGE) return EXIT_USAGE;
|
|
2035
2486
|
section("Draft: affected routes no test covers");
|
|
@@ -2037,7 +2488,8 @@ async function testCommand(options) {
|
|
|
2037
2488
|
cwd: options.cwd,
|
|
2038
2489
|
base: options.base,
|
|
2039
2490
|
url: options.url,
|
|
2040
|
-
write: options.write
|
|
2491
|
+
write: options.write,
|
|
2492
|
+
budget
|
|
2041
2493
|
});
|
|
2042
2494
|
if (planCode === EXIT_USAGE) return EXIT_USAGE;
|
|
2043
2495
|
console.log(
|
|
@@ -2063,8 +2515,26 @@ function parseMinScore(value) {
|
|
|
2063
2515
|
}
|
|
2064
2516
|
return score;
|
|
2065
2517
|
}
|
|
2518
|
+
function parsePositiveInt(flag) {
|
|
2519
|
+
return (value) => {
|
|
2520
|
+
const n = Number(value);
|
|
2521
|
+
if (!Number.isInteger(n) || n <= 0) {
|
|
2522
|
+
throw new InvalidArgumentError(`${flag} must be a positive integer`);
|
|
2523
|
+
}
|
|
2524
|
+
return n;
|
|
2525
|
+
};
|
|
2526
|
+
}
|
|
2527
|
+
function parsePositiveNumber(flag) {
|
|
2528
|
+
return (value) => {
|
|
2529
|
+
const n = Number(value);
|
|
2530
|
+
if (!Number.isFinite(n) || n <= 0) {
|
|
2531
|
+
throw new InvalidArgumentError(`${flag} must be a positive number`);
|
|
2532
|
+
}
|
|
2533
|
+
return n;
|
|
2534
|
+
};
|
|
2535
|
+
}
|
|
2066
2536
|
var program = new Command();
|
|
2067
|
-
program.name("blastproof").description("Open-source AI testing agent: plain-English YAML tests executed agentically on a real browser.").version("0.
|
|
2537
|
+
program.name("blastproof").description("Open-source AI testing agent: plain-English YAML tests executed agentically on a real browser.").version("0.4.0");
|
|
2068
2538
|
program.command("init").description("Scaffold .blastproof/ (config, tests, sample tests) in the current directory").action(async () => {
|
|
2069
2539
|
try {
|
|
2070
2540
|
const result = await initProject(process.cwd());
|
|
@@ -2079,7 +2549,19 @@ program.command("run").description("Discover and run all tests under .blastproof
|
|
|
2079
2549
|
"--min-score <n>",
|
|
2080
2550
|
"require a weighted score of at least n (0-100); replaces the all-must-pass rule",
|
|
2081
2551
|
parseMinScore
|
|
2082
|
-
).option("--junit [path]", "write a JUnit XML report (default: .blastproof/reports/<session>/junit.xml)").option("--html [path]", "write a self-contained HTML report (default: .blastproof/reports/<session>/report.html)").option("--fail-on-unmapped", "fail when a changed file matches no routes: or ignore: glob").
|
|
2552
|
+
).option("--junit [path]", "write a JUnit XML report (default: .blastproof/reports/<session>/junit.xml)").option("--html [path]", "write a self-contained HTML report (default: .blastproof/reports/<session>/report.html)").option("--fail-on-unmapped", "fail when a changed file matches no routes: or ignore: glob").option(
|
|
2553
|
+
"--max-llm-calls <n>",
|
|
2554
|
+
"stop the run after this many model calls, reported as incomplete (overrides config)",
|
|
2555
|
+
parsePositiveInt("--max-llm-calls")
|
|
2556
|
+
).option(
|
|
2557
|
+
"--max-tokens <n>",
|
|
2558
|
+
"stop the run after this many tokens spent across all model calls (overrides config)",
|
|
2559
|
+
parsePositiveInt("--max-tokens")
|
|
2560
|
+
).option(
|
|
2561
|
+
"--max-duration <seconds>",
|
|
2562
|
+
"stop the run after this many seconds of wall-clock time (overrides config)",
|
|
2563
|
+
parsePositiveNumber("--max-duration")
|
|
2564
|
+
).action(
|
|
2083
2565
|
async (options) => {
|
|
2084
2566
|
try {
|
|
2085
2567
|
process.exitCode = await runCommand({
|
|
@@ -2094,7 +2576,10 @@ program.command("run").description("Discover and run all tests under .blastproof
|
|
|
2094
2576
|
minScore: options.minScore,
|
|
2095
2577
|
junit: options.junit,
|
|
2096
2578
|
html: options.html,
|
|
2097
|
-
failOnUnmapped: options.failOnUnmapped
|
|
2579
|
+
failOnUnmapped: options.failOnUnmapped,
|
|
2580
|
+
maxLlmCalls: options.maxLlmCalls,
|
|
2581
|
+
maxTokens: options.maxTokens,
|
|
2582
|
+
maxDuration: options.maxDuration
|
|
2098
2583
|
});
|
|
2099
2584
|
} catch (error) {
|
|
2100
2585
|
console.error(`error: ${error instanceof Error ? error.message : String(error)}`);
|
|
@@ -2102,25 +2587,58 @@ program.command("run").description("Discover and run all tests under .blastproof
|
|
|
2102
2587
|
}
|
|
2103
2588
|
}
|
|
2104
2589
|
);
|
|
2105
|
-
program.command("plan").description("Generate plain-English YAML tests for affected routes no test covers yet").option("--base <ref>", "base git ref for the diff", "main").option("--url <url>", "override config base_url for this run only (config file untouched)").option("--route <route>", "generate for this route, bypassing the diff (repeatable)", collect, []).option("--write", "persist drafts under .blastproof/tests/ instead of previewing them").
|
|
2106
|
-
|
|
2107
|
-
|
|
2108
|
-
|
|
2109
|
-
|
|
2110
|
-
|
|
2111
|
-
|
|
2112
|
-
|
|
2113
|
-
|
|
2114
|
-
|
|
2115
|
-
|
|
2116
|
-
|
|
2590
|
+
program.command("plan").description("Generate plain-English YAML tests for affected routes no test covers yet").option("--base <ref>", "base git ref for the diff", "main").option("--url <url>", "override config base_url for this run only (config file untouched)").option("--route <route>", "generate for this route, bypassing the diff (repeatable)", collect, []).option("--write", "persist drafts under .blastproof/tests/ instead of previewing them").option(
|
|
2591
|
+
"--dry-run",
|
|
2592
|
+
"print the routes that would generate drafts and exit without launching a browser or calling the LLM"
|
|
2593
|
+
).option(
|
|
2594
|
+
"--max-llm-calls <n>",
|
|
2595
|
+
"stop after this many model calls, reported as incomplete (overrides config)",
|
|
2596
|
+
parsePositiveInt("--max-llm-calls")
|
|
2597
|
+
).option(
|
|
2598
|
+
"--max-tokens <n>",
|
|
2599
|
+
"stop after this many tokens spent across all model calls (overrides config)",
|
|
2600
|
+
parsePositiveInt("--max-tokens")
|
|
2601
|
+
).option(
|
|
2602
|
+
"--max-duration <seconds>",
|
|
2603
|
+
"stop after this many seconds of wall-clock time (overrides config)",
|
|
2604
|
+
parsePositiveNumber("--max-duration")
|
|
2605
|
+
).action(
|
|
2606
|
+
async (options) => {
|
|
2607
|
+
try {
|
|
2608
|
+
process.exitCode = await planCommand({
|
|
2609
|
+
cwd: process.cwd(),
|
|
2610
|
+
base: options.base,
|
|
2611
|
+
url: options.url,
|
|
2612
|
+
routes: options.route,
|
|
2613
|
+
write: options.write,
|
|
2614
|
+
dryRun: options.dryRun,
|
|
2615
|
+
maxLlmCalls: options.maxLlmCalls,
|
|
2616
|
+
maxTokens: options.maxTokens,
|
|
2617
|
+
maxDuration: options.maxDuration
|
|
2618
|
+
});
|
|
2619
|
+
} catch (error) {
|
|
2620
|
+
console.error(`error: ${error instanceof Error ? error.message : String(error)}`);
|
|
2621
|
+
process.exitCode = EXIT_USAGE;
|
|
2622
|
+
}
|
|
2117
2623
|
}
|
|
2118
|
-
|
|
2624
|
+
);
|
|
2119
2625
|
program.command("test").description("Full PR pipeline: run the tests covering the diff, then draft tests for the gaps").option("--base <ref>", "base git ref for the diff", "main").option("--url <url>", "override config base_url for this run only (config file untouched)").option(
|
|
2120
2626
|
"--min-score <n>",
|
|
2121
2627
|
"require a weighted score of at least n (0-100); replaces the all-must-pass rule",
|
|
2122
2628
|
parseMinScore
|
|
2123
|
-
).option("--junit [path]", "write a JUnit XML report (default: .blastproof/reports/<session>/junit.xml)").option("--html [path]", "write a self-contained HTML report (default: .blastproof/reports/<session>/report.html)").option("--write", "persist generated drafts under .blastproof/tests/ instead of previewing them").option("--fail-on-unmapped", "fail when a changed file matches no routes: or ignore: glob").
|
|
2629
|
+
).option("--junit [path]", "write a JUnit XML report (default: .blastproof/reports/<session>/junit.xml)").option("--html [path]", "write a self-contained HTML report (default: .blastproof/reports/<session>/report.html)").option("--write", "persist generated drafts under .blastproof/tests/ instead of previewing them").option("--fail-on-unmapped", "fail when a changed file matches no routes: or ignore: glob").option(
|
|
2630
|
+
"--max-llm-calls <n>",
|
|
2631
|
+
"stop the pipeline after this many model calls, shared by both phases (overrides config)",
|
|
2632
|
+
parsePositiveInt("--max-llm-calls")
|
|
2633
|
+
).option(
|
|
2634
|
+
"--max-tokens <n>",
|
|
2635
|
+
"stop the pipeline after this many tokens, shared by both phases (overrides config)",
|
|
2636
|
+
parsePositiveInt("--max-tokens")
|
|
2637
|
+
).option(
|
|
2638
|
+
"--max-duration <seconds>",
|
|
2639
|
+
"stop the pipeline after this many seconds of wall-clock time (overrides config)",
|
|
2640
|
+
parsePositiveNumber("--max-duration")
|
|
2641
|
+
).action(
|
|
2124
2642
|
async (options) => {
|
|
2125
2643
|
try {
|
|
2126
2644
|
process.exitCode = await testCommand({
|
|
@@ -2131,7 +2649,10 @@ program.command("test").description("Full PR pipeline: run the tests covering th
|
|
|
2131
2649
|
junit: options.junit,
|
|
2132
2650
|
html: options.html,
|
|
2133
2651
|
write: options.write,
|
|
2134
|
-
failOnUnmapped: options.failOnUnmapped
|
|
2652
|
+
failOnUnmapped: options.failOnUnmapped,
|
|
2653
|
+
maxLlmCalls: options.maxLlmCalls,
|
|
2654
|
+
maxTokens: options.maxTokens,
|
|
2655
|
+
maxDuration: options.maxDuration
|
|
2135
2656
|
});
|
|
2136
2657
|
} catch (error) {
|
|
2137
2658
|
console.error(`error: ${error instanceof Error ? error.message : String(error)}`);
|