blastproof 0.21.0 → 0.22.0

package/dist/cli.js

@@ -333,6 +333,9 @@ function referencedEnvVars(text) {
   }
   return [...names];
 }
+function placeholdersAsLabels(text) {
+  return text.replace(ENV_PLACEHOLDER, (_match, name) => redactionLabel(name));
+}
 function escapeRegExp(value) {
   return value.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
 }
@@ -343,6 +346,12 @@ var REDACTION_PREFIX = "[redacted";
 function redactionLabel(name) {
   return name === void 0 ? `${REDACTION_PREFIX}]` : `${REDACTION_PREFIX} ${name}]`;
 }
+var LABEL = /\[redacted ([A-Za-z_][A-Za-z0-9_]*)\]/g;
+function labelledVariables(text) {
+  const names = /* @__PURE__ */ new Set();
+  for (const match of text.matchAll(LABEL)) if (match[1]) names.add(match[1]);
+  return [...names];
+}
 function compile(secrets, names) {
   return [...secrets].filter(Boolean).sort((a, b) => b.length - a.length).map((value) => ({ value, pattern: secretPattern(value), label: redactionLabel(names?.get(value)) }));
 }
@@ -1607,8 +1616,10 @@ function parseAgentAction(value) {
   return parsedAgentActionSchema.safeParse(value);
 }
 var assertJudgmentSchema = z2.object({
-  pass: z2.boolean().describe("Whether the snapshot satisfies the expectation."),
-  reason: z2.string().describe("One sentence explaining the judgment.")
+  reason: z2.string().describe("One sentence: what the STEP requires to be true now, and whether the snapshot shows it."),
+  pass: z2.boolean().describe(
+    "Whether the snapshot shows the STEP's own outcome. The expectation is only a claim offered in support; it never replaces the step. False if any part of the outcome the step asks for is not shown, or cannot be assessed from this snapshot."
+  )
 });
 var generatedTestSchema = z2.object({
   summary: z2.string().min(1).describe('One line naming the user journey this test covers, e.g. "Applying a discount updates the cart total".'),
@@ -1665,11 +1676,20 @@ function createBrain(model, generate = generateObject, budget) {
       return parsed.data;
     },
     async judge(step, expectation, snapshot, stepHistory) {
+      const asJudged = stepHistory?.map((entry) => ({
+        action: placeholdersAsLabels(entry.action),
+        result: placeholdersAsLabels(entry.result)
+      }));
       const result = await countedGenerate(generate, budget, {
         model,
         schema: assertJudgmentSchema,
         system: assertSystemPrompt(),
-        prompt: assertUserPrompt(step, expectation, snapshot, stepHistory),
+        prompt: assertUserPrompt(
+          placeholdersAsLabels(step),
+          placeholdersAsLabels(expectation),
+          snapshot,
+          asJudged
+        ),
         temperature: 0
       });
       const parsed = assertJudgmentSchema.safeParse(result.object);
@@ -1678,10 +1698,26 @@ function createBrain(model, generate = generateObject, budget) {
           `Model returned an invalid assert judgment: ${parsed.error.issues[0]?.message ?? "unknown"}`
         );
       }
-      return parsed.data;
+      return secretMismatch(placeholdersAsLabels(step), snapshot, asJudged, parsed.data);
     }
   };
 }
+function secretMismatch(judgedStep, snapshot, judgedRecord, judgment) {
+  if (!judgment.pass) return judgment;
+  const onPage = labelledVariables(snapshot);
+  const inRecord = labelledVariables((judgedRecord ?? []).map((e) => `${e.action}
+${e.result}`).join("\n"));
+  for (const name of labelledVariables(judgedStep)) {
+    const others = onPage.filter((other) => other !== name);
+    if (onPage.includes(name) || inRecord.includes(name) || others.length === 0) continue;
+    const shown = others.map((other) => redactionLabel(other)).join(", ");
+    return {
+      pass: false,
+      reason: `The step names ${redactionLabel(name)}, which appears neither on the page nor in this step's actions, while the page shows ${shown}: a different secret cannot satisfy it. (The judge had said: ${judgment.reason})`
+    };
+  }
+  return judgment;
+}
 function createPlanner(model, generate = generateObject, budget) {
   return {
     async planTest(input) {
@@ -3256,7 +3292,7 @@ function parsePositiveNumber(flag) {
   };
 }
 var program = new Command();
-program.name("blastproof").description("Open-source AI testing agent: plain-English YAML tests executed agentically on a real browser.").version("0.21.0");
+program.name("blastproof").description("Open-source AI testing agent: plain-English YAML tests executed agentically on a real browser.").version("0.22.0");
 program.command("init").description("Scaffold .blastproof/ (config, tests, sample tests) in the current directory").action(async () => {
   try {
     const result = await initProject(process.cwd());