blastproof 0.21.0 → 0.23.0

package/README.md

@@ -210,7 +210,7 @@ Common flags — `blastproof <command> --help` has the full list:
 | `--write` | `plan` only — persist drafts instead of previewing |
 | `--max-llm-calls` · `--max-tokens` · `--max-duration` | [Bound what a run may spend](./docs/configuration.md#budget--bounding-what-a-run-spends) |
 
-Exit codes: **0** pass, **1** the gate failed, **2** usage or config error.
+Exit codes: **0** pass, **1** the gate failed or the run stopped (its budget, or the model provider refusing a call: `Run incomplete:`), **2** usage or config error.
 
 **Generated drafts are never executed and never affect the score.** An unreviewed model-written test in the merge path fails in two directions: a hallucinated expectation blocks a correct PR, and a credulous one waves a broken change through while looking like coverage. `plan` makes the gap visible with a draft to review; it does not make an uncovered route safe.
 

package/dist/cli.js

@@ -219,7 +219,9 @@ function describeLimit(limit, observed, configured) {
       return `deadline exceeded: reached the configured maximum of ${(configured / 1e3).toFixed(0)}s (elapsed ${(observed / 1e3).toFixed(1)}s)`;
   }
 }
-var BudgetExhaustedError = class extends Error {
+var RunStoppedError = class extends Error {
+};
+var BudgetExhaustedError = class extends RunStoppedError {
   limit;
   observed;
   configured;
@@ -231,6 +233,30 @@ var BudgetExhaustedError = class extends Error {
     this.configured = configured;
   }
 };
+function refusalRemedy(statusCode) {
+  if (statusCode === void 0) {
+    return "The provider could not be reached after 3 attempts: check llm.base_url and the network, then run again.";
+  }
+  if (statusCode === 401 || statusCode === 403) {
+    return "Check the API key in the variable named by llm.api_key_env, then run again.";
+  }
+  if (statusCode === 402) return "Add credit to the provider account, then run again.";
+  if (statusCode === 408 || statusCode === 409 || statusCode === 429 || statusCode >= 500) {
+    return "The provider was unavailable after 3 attempts: run again later.";
+  }
+  return void 0;
+}
+var ProviderRefusedError = class extends RunStoppedError {
+  statusCode;
+  constructor(statusCode, detail) {
+    const status = statusCode === void 0 ? "no response" : `HTTP ${statusCode}`;
+    const remedy = refusalRemedy(statusCode);
+    const separator = /[.!?]$/.test(detail.trim()) ? " " : ". ";
+    super(`model provider refused the request (${status}): ${detail}${remedy ? `${separator}${remedy}` : ""}`);
+    this.name = "ProviderRefusedError";
+    this.statusCode = statusCode;
+  }
+};
 var RunBudget = class {
   maxCalls;
   maxTokens;
@@ -333,6 +359,9 @@ function referencedEnvVars(text) {
   }
   return [...names];
 }
+function placeholdersAsLabels(text) {
+  return text.replace(ENV_PLACEHOLDER, (_match, name) => redactionLabel(name));
+}
 function escapeRegExp(value) {
   return value.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
 }
@@ -343,6 +372,12 @@ var REDACTION_PREFIX = "[redacted";
 function redactionLabel(name) {
   return name === void 0 ? `${REDACTION_PREFIX}]` : `${REDACTION_PREFIX} ${name}]`;
 }
+var LABEL = /\[redacted ([A-Za-z_][A-Za-z0-9_]*)\]/g;
+function labelledVariables(text) {
+  const names = /* @__PURE__ */ new Set();
+  for (const match of text.matchAll(LABEL)) if (match[1]) names.add(match[1]);
+  return [...names];
+}
 function compile(secrets, names) {
   return [...secrets].filter(Boolean).sort((a, b) => b.length - a.length).map((value) => ({ value, pattern: secretPattern(value), label: redactionLabel(names?.get(value)) }));
 }
@@ -591,11 +626,13 @@ function describeAction(action) {
   return `${action.action}${target}${value}`;
 }
 function identity(action) {
+  const role = action.target?.role ?? "";
+  const name = action.target?.name ?? "";
   return JSON.stringify([
     action.action,
-    action.target?.role ?? "",
-    action.target?.name ?? "",
-    action.target?.text ?? "",
+    role,
+    normalise(name),
+    role || name ? "" : action.target?.text ?? "",
     action.value ?? ""
   ]);
 }
@@ -845,7 +882,7 @@ async function executeTest(page, test, options) {
             iterationsLeft: maxIterationsPerStep - iterations
           });
         } catch (error) {
-          if (error instanceof BudgetExhaustedError) throw error;
+          if (error instanceof RunStoppedError) throw error;
           failedAttempts++;
           lastResult = `error: ${error instanceof Error ? error.message : String(error)}`;
           if (failedAttempts >= maxRetries) {
@@ -873,23 +910,35 @@ async function executeTest(page, test, options) {
         }
         if (action.action === "assert") {
           const expectation = action.expectation ?? action.reasoning;
-          let judgment = await brain.judge(
-            mask(step),
-            mask(expectation),
-            maskedSnap,
-            recovery.stepHistory()
-          );
-          if (!judgment.pass) {
-            await waitForSettled(page);
-            const freshSnap = await takeSnapshot(page);
-            const maskedFresh = mask(freshSnap);
-            recovery.observe(maskedFresh);
+          let judgment;
+          try {
             judgment = await brain.judge(
               mask(step),
               mask(expectation),
-              maskedFresh,
+              maskedSnap,
               recovery.stepHistory()
             );
+            if (!judgment.pass) {
+              await waitForSettled(page);
+              const freshSnap = await takeSnapshot(page);
+              const maskedFresh = mask(freshSnap);
+              recovery.observe(maskedFresh);
+              judgment = await brain.judge(
+                mask(step),
+                mask(expectation),
+                maskedFresh,
+                recovery.stepHistory()
+              );
+            }
+          } catch (error) {
+            if (error instanceof RunStoppedError) throw error;
+            failedAttempts++;
+            lastResult = `error: ${error instanceof Error ? error.message : String(error)}`;
+            emitAction(index, action, lastResult);
+            if (failedAttempts >= maxRetries) {
+              throw new StepFailure(lastResult);
+            }
+            continue;
           }
           const result = judgment.pass ? `ok: assertion passed: ${judgment.reason}` : `assertion failed: ${judgment.reason}`;
           emitAction(index, action, result);
@@ -934,7 +983,7 @@ async function executeTest(page, test, options) {
         }
       }
     } catch (error) {
-      if (error instanceof BudgetExhaustedError) throw error;
+      if (error instanceof RunStoppedError) throw error;
       stepFailedReason = mask(error instanceof Error ? error.message : String(error));
     }
     const status = stepFailedReason ? "failed" : "passed";
@@ -1035,12 +1084,24 @@ async function runJourney(...args) {
   try {
     return await executeTest(...args);
   } catch (error) {
-    if (error instanceof BudgetExhaustedError) throw error;
+    if (error instanceof RunStoppedError) throw error;
     throw new AuthError(
       `Authentication could not run: ${error instanceof Error ? error.message : String(error)}`
     );
   }
 }
+async function verifyJudgment(brain, verify, snapshot, maskText, maxRetries = 3) {
+  let lastError = "";
+  for (let attempt = 0; attempt < maxRetries; attempt++) {
+    try {
+      return await brain.judge(verify, verify, await snapshot());
+    } catch (error) {
+      if (error instanceof RunStoppedError) throw error;
+      lastError = error instanceof Error ? error.message : String(error);
+    }
+  }
+  throw new AuthError(`Authentication could not be verified: ${maskText(lastError)}`);
+}
 async function fromSteps(options) {
   const { auth, baseUrl, browser, brain, maxRetries, snapshot, timeoutMs, maxSnapshotLines, onEvent, mask } = options;
   const steps = auth.steps;
@@ -1078,7 +1139,13 @@ async function fromSteps(options) {
       );
     }
     if (auth.verify) {
-      const judgment = await brain.judge(auth.verify, auth.verify, mask.mask(await takeSnapshot(page)));
+      const judgment = await verifyJudgment(
+        brain,
+        auth.verify,
+        async () => mask.mask(await takeSnapshot(page)),
+        (text) => mask.mask(text),
+        maxRetries
+      );
       if (!judgment.pass) {
         throw new AuthError(`Authentication could not be verified: ${mask.mask(judgment.reason)}`);
       }
@@ -1450,7 +1517,7 @@ warning: ${findings.length} file(s) changed in the working tree are not in the d
 }
 
 // src/llm/brain.ts
-import { generateObject } from "ai";
+import { APICallError, generateObject, RetryError } from "ai";
 
 // src/llm/prompts.ts
 function agentSystemPrompt() {
@@ -1502,6 +1569,8 @@ A value sitting in a control that was just typed into \u2014 an open dialog's te
 
 A step that names an ACTION (submit, click, create, add, ...) is satisfied by evidence the action took effect, not by the action's own control still being on the page. A successful action ordinarily replaces or moves past exactly the form, button or field the step names, so that control's absence is normal evidence of success, not evidence the step is unverifiable \u2014 do not fail such a step only because you can no longer see the thing it names. Fail it instead when the snapshot shows the action did NOT take effect: an error message, a validation warning, or the very same pre-action page still in front of you with nothing changed. A different page, a new state, or the result the action was meant to produce counts as evidence it worked.
 
+An outcome may already hold before this step acts: an earlier step, or the application itself, got there first. A step asking for something to be gone, closed or dismissed is satisfied by that thing being absent from the snapshot. It does not also require the thing to have been there, or this step to have removed it, and a different element that is present (another dialog, another banner) is not the one the step names. Whether the step's action ran is not what you decide: judge the state the step asks for.
+
 You may also be shown the actions already performed in this step, with their results. That record tells you what was ATTEMPTED and what it produced \u2014 for instance that a navigation was performed and which URL the server ultimately served, or that a form was submitted. Use it to avoid concluding that something never happened when the page simply cannot show it any more: a navigation the server redirected does not leave the browser at the path that was requested, and that is what success looks like, not failure.
 
 The record is not evidence that the step's outcome holds. An action reported as \`ok\` establishes that it ran and what it returned; whether the thing the step describes is now TRUE is still decided by the snapshot alone. Never pass a step because the record shows an action succeeded while the snapshot does not show the outcome.
@@ -1607,8 +1676,13 @@ function parseAgentAction(value) {
   return parsedAgentActionSchema.safeParse(value);
 }
 var assertJudgmentSchema = z2.object({
-  pass: z2.boolean().describe("Whether the snapshot satisfies the expectation."),
-  reason: z2.string().describe("One sentence explaining the judgment.")
+  outcome: z2.string().describe(
+    'The state the STEP asks for, rewritten as a sentence about the page with its action removed: "click Save and verify the note is listed" becomes "the note is listed"; "open the Account menu and verify it shows the email" becomes "the Account menu shows the email".'
+  ),
+  reason: z2.string().describe("One sentence: whether the snapshot shows that outcome."),
+  pass: z2.boolean().describe(
+    "Whether the snapshot shows that outcome. The expectation is only a claim offered in support; it never replaces the step. False if any part of the outcome the step asks for is not shown, or cannot be assessed from this snapshot. An action the step names (click, dismiss, submit) is how its outcome is reached, not part of it: an outcome that holds passes whether or not that action was needed. An outcome that is an absence (gone, closed, dismissed, removed) is shown by the thing being absent."
+  )
 });
 var generatedTestSchema = z2.object({
   summary: z2.string().min(1).describe('One line naming the user journey this test covers, e.g. "Applying a discount updates the cart total".'),
@@ -1636,13 +1710,20 @@ function withProviderDetail(error) {
   error.message = `${error.message} \u2014 provider said: ${detail}`;
   return error;
 }
+function asProviderRefusal(error) {
+  const cause = RetryError.isInstance(error) ? error.lastError : error;
+  if (!APICallError.isInstance(cause)) return error;
+  if (cause.statusCode !== void 0 && cause.statusCode < 400) return error;
+  withProviderDetail(cause);
+  return new ProviderRefusedError(cause.statusCode, cause.message);
+}
 async function countedGenerate(generate, budget, options) {
   budget.check();
   let result;
   try {
     result = await generate(options);
   } catch (error) {
-    throw withProviderDetail(error);
+    throw asProviderRefusal(withProviderDetail(error));
   }
   budget.record(result.usage);
   return result;
@@ -1665,11 +1746,20 @@ function createBrain(model, generate = generateObject, budget) {
       return parsed.data;
     },
     async judge(step, expectation, snapshot, stepHistory) {
+      const asJudged = stepHistory?.map((entry) => ({
+        action: placeholdersAsLabels(entry.action),
+        result: placeholdersAsLabels(entry.result)
+      }));
       const result = await countedGenerate(generate, budget, {
         model,
         schema: assertJudgmentSchema,
         system: assertSystemPrompt(),
-        prompt: assertUserPrompt(step, expectation, snapshot, stepHistory),
+        prompt: assertUserPrompt(
+          placeholdersAsLabels(step),
+          placeholdersAsLabels(expectation),
+          snapshot,
+          asJudged
+        ),
         temperature: 0
       });
       const parsed = assertJudgmentSchema.safeParse(result.object);
@@ -1678,10 +1768,27 @@ function createBrain(model, generate = generateObject, budget) {
           `Model returned an invalid assert judgment: ${parsed.error.issues[0]?.message ?? "unknown"}`
         );
       }
-      return parsed.data;
+      return secretMismatch(placeholdersAsLabels(step), snapshot, asJudged, parsed.data);
     }
   };
 }
+function secretMismatch(judgedStep, snapshot, judgedRecord, judgment) {
+  if (!judgment.pass) return judgment;
+  const onPage = labelledVariables(snapshot);
+  const inRecord = labelledVariables((judgedRecord ?? []).map((e) => `${e.action}
+${e.result}`).join("\n"));
+  for (const name of labelledVariables(judgedStep)) {
+    const others = onPage.filter((other) => other !== name);
+    if (onPage.includes(name) || inRecord.includes(name) || others.length === 0) continue;
+    const shown = others.map((other) => redactionLabel(other)).join(", ");
+    return {
+      ...judgment,
+      pass: false,
+      reason: `The step names ${redactionLabel(name)}, which appears neither on the page nor in this step's actions, while the page shows ${shown}: a different secret cannot satisfy it. (The judge had said: ${judgment.reason})`
+    };
+  }
+  return judgment;
+}
 function createPlanner(model, generate = generateObject, budget) {
   return {
     async planTest(input) {
@@ -2541,7 +2648,7 @@ X ${r.summary} (${r.file})`);
   }
   if (notRun.length > 0) {
     console.log(`
-${notRun.length} test(s) not run (run stopped by its budget or deadline):`);
+${notRun.length} test(s) not run (run stopped: ${notRun[0].reason}):`);
     for (const r of notRun) console.log(`  - ${r.summary} (${r.file})`);
   }
 }
@@ -2866,7 +2973,7 @@ ${results.length} test file(s) could not be parsed:`);
           onEvent: printEvent
         });
       } catch (error) {
-        if (error instanceof BudgetExhaustedError) {
+        if (error instanceof RunStoppedError) {
           incomplete = error;
         } else if (error instanceof AuthError) {
           console.error(`error: ${error.message}`);
@@ -2896,7 +3003,7 @@ ${results.length} test file(s) could not be parsed:`);
           if (!streaming) for (const line of lines) console.log(line);
           return { index, result };
         } catch (error) {
-          if (error instanceof BudgetExhaustedError) {
+          if (error instanceof RunStoppedError) {
             stoppedBy ??= error;
             if (!streaming) for (const line of lines) console.log(line);
             return { index, stopped: error };
@@ -3080,7 +3187,7 @@ async function planCommand(options) {
           onEvent: printAuthEvent
         });
       } catch (error) {
-        if (error instanceof BudgetExhaustedError) {
+        if (error instanceof RunStoppedError) {
           incomplete = error;
         } else if (error instanceof AuthError) {
           console.error(`error: ${error.message}`);
@@ -3108,7 +3215,7 @@ async function planCommand(options) {
           timeoutMs: config.browser.timeout_ms
         });
       } catch (error) {
-        if (error instanceof BudgetExhaustedError) {
+        if (error instanceof RunStoppedError) {
           incomplete = error;
           break;
         }
@@ -3162,7 +3269,7 @@ ${renderTestYaml(draft, { route, base })}`);
   if (incomplete) {
     console.log(`Stopped: ${incomplete.message}`);
     if (notAttempted.length > 0) {
-      console.log("Not attempted (run out of budget):");
+      console.log("Not attempted (run stopped):");
       for (const route of notAttempted) console.log(`  ${route}`);
     }
   }
@@ -3256,7 +3363,7 @@ function parsePositiveNumber(flag) {
   };
 }
 var program = new Command();
-program.name("blastproof").description("Open-source AI testing agent: plain-English YAML tests executed agentically on a real browser.").version("0.21.0");
+program.name("blastproof").description("Open-source AI testing agent: plain-English YAML tests executed agentically on a real browser.").version("0.23.0");
 program.command("init").description("Scaffold .blastproof/ (config, tests, sample tests) in the current directory").action(async () => {
   try {
     const result = await initProject(process.cwd());