blastproof 0.4.0 → 0.6.0

package/README.md

@@ -44,8 +44,14 @@ Before `run`, `plan` or `test` do anything, they check what they are about to sp
 
 **Not supported yet:** `iframe` content (so hosted payment widgets like Stripe Elements are invisible — an embedded checkout cannot be driven end to end), hover, scroll-to, drag and drop, file upload, multiple tabs, native `alert`/`confirm` dialogs. Page snapshots are capped at 200 lines by default, so very dense pages are truncated — raise it with `browser.max_snapshot_lines` if your pages need more; truncation is always marked in the snapshot so the model is never misled into thinking it saw the whole page.
 
+**Point it at disposable data.** A step that fails is retried, and the agent decides how to recover — which can mean performing the step's action again. A test that submits a form and then fails its check may submit that form more than once, so a run against real data can leave more records behind than the journey describes. In one verification run a badly written step produced three issues where the test intended one. Use a seeded database, a staging environment you can reset, or a throwaway account; do not gate on a run against production data.
+
 `browser.timeout_ms` bounds every wait — resolving a target element from the accessibility tree, and navigation — not only the click or fill performed afterwards. Raise it for an application that is merely slow to hydrate; the trade-off is that a genuinely missing element then takes longer to fail. It never changes how many self-healing retries a step gets — waiting and retrying are deliberately separate.
 
+Writing each step so it states its own outcome helps here as well as everywhere else: `submit the form, then verify the confirmation shows the reference number` gives the agent something to check, where `click the submit button` leaves it to invent an expectation — and a poor invented expectation is what turns one submission into three.
+
+**This is now load-bearing, not just advisable.** The judge decides whether a step's own outcome holds, using the model's expectation only as the claim offered in support of it — a step that never says what its outcome is gives the judge nothing to anchor on beyond whatever the model happened to check that turn. `verify the confirmation shows the reference number` gives the judge a real question; `verify it worked` does not, and may now fail where a looser judge previously let a true-but-unrelated claim pass it.
+
 ## Writing tests
 
 Tests live in `.blastproof/tests/` as plain-English YAML — no selectors:
@@ -107,9 +113,9 @@ jobs:
 
       - run: npm start &          # however your app boots
 
-      - uses: hamc/blastproof@v0.4.0
+      - uses: hamc/blastproof@v0.6.0
         with:
-          version: '0.4.0'        # pin both when this gates merges
+          version: '0.6.0'        # pin both when this gates merges
           api-key: ${{ secrets.ANTHROPIC_API_KEY }}
           base: ${{ github.event.pull_request.base.ref }}
           min-score: '80'

package/dist/cli.js

@@ -256,7 +256,7 @@ function estimateMaxModelCalls(tests, maxIterationsPerStep, maxRetriesPerStep) {
     (sum, test) => sum + (test.setup?.length ?? 0) + test.steps.length,
     0
   );
-  return totalSteps * (maxIterationsPerStep + maxRetriesPerStep);
+  return totalSteps * (maxIterationsPerStep + maxRetriesPerStep + Math.min(maxIterationsPerStep, maxRetriesPerStep));
 }
 
 // src/runner/env.ts
@@ -439,6 +439,19 @@ async function performAction(page, action, ctx) {
 
 // src/runner/executor.ts
 var DEFAULT_MAX_ITERATIONS_PER_STEP = 15;
+var SETTLE_TIMEOUT_MS = 2e3;
+async function waitForSettled(page) {
+  const deadline = Date.now() + SETTLE_TIMEOUT_MS;
+  const urlBefore = page.url();
+  await page.waitForLoadState("networkidle", { timeout: SETTLE_TIMEOUT_MS }).catch(() => {
+  });
+  if (page.url() === urlBefore) return;
+  const remaining = deadline - Date.now();
+  if (remaining > 0) {
+    await page.waitForLoadState("networkidle", { timeout: remaining }).catch(() => {
+    });
+  }
+}
 async function defaultSnapshot(page, maxLines) {
   const { captureSnapshot } = await import("./snapshot-CAIB2OHX.js");
   return captureSnapshot(page, { maxLines });
@@ -496,6 +509,7 @@ async function executeTest(page, test, options) {
         if (iterations >= maxIterationsPerStep) {
           throw new StepFailure(`step exceeded ${maxIterationsPerStep} actions without completing`);
         }
+        await waitForSettled(page);
         const snap = await takeSnapshot(page);
         let action;
         try {
@@ -527,7 +541,12 @@ async function executeTest(page, test, options) {
         }
         if (action.action === "assert") {
           const expectation = action.expectation ?? action.reasoning;
-          const judgment = await brain.judge(mask(expectation), mask(snap));
+          let judgment = await brain.judge(mask(step), mask(expectation), mask(snap));
+          if (!judgment.pass) {
+            await waitForSettled(page);
+            const freshSnap = await takeSnapshot(page);
+            judgment = await brain.judge(mask(step), mask(expectation), mask(freshSnap));
+          }
           const result = judgment.pass ? `ok: assertion passed: ${judgment.reason}` : `assertion failed: ${judgment.reason}`;
           emitAction(index, action, result);
           if (judgment.pass) {
@@ -704,7 +723,7 @@ async function fromSteps(options) {
       );
     }
     if (auth.verify) {
-      const judgment = await brain.judge(auth.verify, mask.mask(await takeSnapshot(page)));
+      const judgment = await brain.judge(auth.verify, auth.verify, mask.mask(await takeSnapshot(page)));
       if (!judgment.pass) {
         throw new AuthError(`Authentication could not be verified: ${mask.mask(judgment.reason)}`);
       }
@@ -1033,6 +1052,7 @@ Rules:
 - Return "done" when the current step's outcome holds \u2014 including when it already held before you acted, or was achieved by your previous action. "Already true" is done, never failure. Do not return "done" for work belonging to later steps.
 - Return "fail" only when the step's outcome cannot be reached: the element is still absent after retries, the page cannot support the step, or an error blocks progress. Never return "fail" because the work appears to have been done already.
 - If your previous action errored, re-read the fresh snapshot and choose an alternative element or approach. Do not repeat the exact same failing action.
+- \`***\` in a snapshot is a redacted secret \u2014 a password, token or key deliberately withheld from you. Seeing it is expected and is not a problem. A field showing \`***\` after you filled it from an {{env.VAR}} placeholder means the fill worked; treat that as success and move on. Never retry a fill because its value is redacted, and never report failure because a value was withheld.
 - Keep reasoning to one short sentence.`;
 }
 function agentUserPrompt(input) {
@@ -1053,16 +1073,26 @@ function agentUserPrompt(input) {
   return parts.join("\n");
 }
 function assertSystemPrompt() {
-  return `You are a QA judge. You receive a page accessibility snapshot and an expectation. Decide whether the snapshot satisfies the expectation. Be strict: only pass when the expectation is clearly met by the snapshot content. Answer with pass=true/false and a one-sentence reason.`;
+  return `You are a QA judge. You receive a test step, the model's expectation for the current page, and a page accessibility snapshot. Decide whether the STEP's own outcome holds \u2014 the expectation is the claim the model is offering in support of that, not a substitute question of its own. A claim can be true of the snapshot and still fail the step, if it does not establish what the step actually describes: only pass when the snapshot itself shows the step's outcome, never merely because the expectation offered happens to be true of something else on the page.
+
+A value sitting in a control that was just typed into \u2014 an open dialog's textbox, an unsubmitted form field \u2014 is not the same as a committed outcome. When the step describes an outcome (something now appears in a list, is saved, is confirmed, is created), text visible only inside an editable, not-yet-submitted control does not satisfy it; look for the outcome committed outside that control (the dialog closed, the item is listed on its own, a confirmation appeared). This is specifically about that confusion, not a license to fail anything you are merely unsure about \u2014 if the snapshot plainly shows the step's outcome, pass it.
+
+A step that names an ACTION (submit, click, create, add, ...) is satisfied by evidence the action took effect, not by the action's own control still being on the page. A successful action ordinarily replaces or moves past exactly the form, button or field the step names, so that control's absence is normal evidence of success, not evidence the step is unverifiable \u2014 do not fail such a step only because you can no longer see the thing it names. Fail it instead when the snapshot shows the action did NOT take effect: an error message, a validation warning, or the very same pre-action page still in front of you with nothing changed. A different page, a new state, or the result the action was meant to produce counts as evidence it worked.
+
+Be strict about what the step asks, not about withholding a pass you can plainly see is earned. Answer with pass=true/false and a one-sentence reason.
+
+\`***\` marks a secret deliberately withheld from you \u2014 a password, token or key. Seeing it is expected. A field holding \`***\` is filled, not empty, so do not fail a step on the grounds that a value was redacted. This applies only to the redaction itself: everything else the step asks for must still be visibly satisfied by the snapshot, and a step you genuinely cannot check against what you were shown still fails.`;
 }
-function assertUserPrompt(expectation, snapshot) {
+function assertUserPrompt(step, expectation, snapshot) {
   return [
-    `Expectation: ${expectation}`,
+    `Step under test: ${step}`,
+    "",
+    `Model's expectation (the claim offered in support of the step, not the question itself): ${expectation}`,
     "",
     "Page accessibility snapshot:",
     snapshot,
     "",
-    "Does the snapshot satisfy the expectation?"
+    "Does the snapshot establish that the step's own outcome holds?"
   ].join("\n");
 }
 function plannerSystemPrompt() {
@@ -1152,12 +1182,12 @@ function createBrain(model, generate = generateObject, budget) {
       }
       return parsed.data;
     },
-    async judge(expectation, snapshot) {
+    async judge(step, expectation, snapshot) {
       const result = await countedGenerate(generate, budget, {
         model,
         schema: assertJudgmentSchema,
         system: assertSystemPrompt(),
-        prompt: assertUserPrompt(expectation, snapshot)
+        prompt: assertUserPrompt(step, expectation, snapshot)
       });
       const parsed = assertJudgmentSchema.safeParse(result.object);
       if (!parsed.success) {
@@ -2534,7 +2564,7 @@ function parsePositiveNumber(flag) {
   };
 }
 var program = new Command();
-program.name("blastproof").description("Open-source AI testing agent: plain-English YAML tests executed agentically on a real browser.").version("0.4.0");
+program.name("blastproof").description("Open-source AI testing agent: plain-English YAML tests executed agentically on a real browser.").version("0.6.0");
 program.command("init").description("Scaffold .blastproof/ (config, tests, sample tests) in the current directory").action(async () => {
   try {
     const result = await initProject(process.cwd());