@stablyai/playwright-base 2.1.9 → 2.1.11-rc.0

package/dist/index.cjs

@@ -47,7 +47,7 @@ const isObject = (value) => {
 
 const SDK_METADATA_HEADERS = {
   "X-Client-Name": "stably-playwright-sdk-js",
-  "X-Client-Version": "2.1.9"
+  "X-Client-Version": "2.1.11-rc.0"
 };
 
 const PROMPT_ASSERTION_PATH = "internal/v2/assert";
@@ -374,23 +374,60 @@ async function takeStableScreenshot(target, options) {
 const MAX_ATTACHMENT_NAME_LENGTH = 80;
 const DEFAULT_AI_ASSERT_TIMEOUT_MS = 3e4;
 const RETRY_DELAY_MS = 1e3;
+class AiAssertTimeoutError extends Error {
+  constructor(timeoutMs) {
+    super(`aiAssert timed out after ${timeoutMs}ms`);
+    this.name = "AiAssertTimeoutError";
+  }
+}
 function delay(ms) {
   return new Promise((resolve) => setTimeout(resolve, ms));
 }
+function addAssertAttachments(condition, verifyResult, screenshot) {
+  const testInfo = test.test.info();
+  const sanitizedName = condition.trim().toLowerCase().replace(/[^\w]+/g, "_").replace(/^_+|_+$/g, "").slice(0, MAX_ATTACHMENT_NAME_LENGTH) || "aiAssert";
+  testInfo.attachments.push(
+    {
+      body: Buffer.from(
+        JSON.stringify(
+          {
+            pass: verifyResult.pass,
+            prompt: condition,
+            reasoning: verifyResult.reason
+          },
+          null,
+          2
+        ),
+        "utf-8"
+      ),
+      contentType: "application/json",
+      name: `${sanitizedName}-reasoning`
+    },
+    {
+      body: screenshot,
+      contentType: "application/octet-stream",
+      name: `${sanitizedName}-screenshot.png`
+    }
+  );
+}
 function createFailureMessage({
   condition,
   didPass,
   isNot,
   reason,
-  targetType
+  targetType,
+  timeoutMs
 }) {
   const expectation = isNot ? "not to satisfy" : "to satisfy";
   const result = didPass ? "it did" : "it did not";
   let message = `Expected ${targetType} ${expectation} ${JSON.stringify(condition)}, but ${result}.`;
   if (reason) {
     message += `
-
 Reason: ${reason}`;
+  }
+  if (timeoutMs != null) {
+    message += `
+Timeout: ${timeoutMs}ms`;
   }
   return message;
 }
@@ -425,17 +462,24 @@ const stablyPlaywrightMatchers = {
     const { isNot } = this;
     const deadline = Date.now() + timeoutMs;
     let aborted = false;
+    const lastAttempt = {
+      hasCompleted: false,
+      screenshot: Buffer.alloc(0),
+      verifyResult: { pass: false }
+    };
+    function recordAttempt(verifyResult, screenshot) {
+      lastAttempt.verifyResult = verifyResult;
+      lastAttempt.screenshot = screenshot;
+      lastAttempt.hasCompleted = true;
+    }
+    let timeoutId;
     const timeoutPromise = new Promise((_, reject) => {
-      setTimeout(() => {
+      timeoutId = setTimeout(() => {
         aborted = true;
-        reject(new Error(`aiAssert timed out after ${timeoutMs}ms`));
+        reject(new AiAssertTimeoutError(timeoutMs));
       }, timeoutMs);
     });
     const work = async () => {
-      let lastVerifyResult = {
-        pass: false
-      };
-      let lastScreenshot = Buffer.alloc(0);
       let isFirstAttempt = true;
       while (!aborted) {
         if (!isFirstAttempt) {
@@ -449,58 +493,63 @@ const stablyPlaywrightMatchers = {
           }
         }
         isFirstAttempt = false;
-        lastScreenshot = await takeStableScreenshot(target, options);
-        lastVerifyResult = await verifyPrompt({
+        const screenshot = await takeStableScreenshot(target, options);
+        const verifyResult = await verifyPrompt({
           model: options?.model,
           pageMetadata: isPage(target) ? { title: await target.title(), url: target.url() } : void 0,
           prompt: condition,
-          screenshot: lastScreenshot
+          screenshot
         });
-        const assertionPasses = lastVerifyResult.pass !== isNot;
+        recordAttempt(verifyResult, screenshot);
+        const assertionPasses = verifyResult.pass !== isNot;
         if (assertionPasses) {
           break;
         }
       }
-      const testInfo = test.test.info();
-      const sanitizedName = condition.trim().toLowerCase().replace(/[^\w]+/g, "_").replace(/^_+|_+$/g, "").slice(0, MAX_ATTACHMENT_NAME_LENGTH) || "aiAssert";
-      testInfo.attachments.push(
-        {
-          body: Buffer.from(
-            JSON.stringify(
-              {
-                pass: lastVerifyResult.pass,
-                prompt: condition,
-                reasoning: lastVerifyResult.reason
-              },
-              null,
-              2
-            ),
-            "utf-8"
-          ),
-          contentType: "application/json",
-          name: `${sanitizedName}-reasoning`
-        },
-        {
-          body: lastScreenshot,
-          // Use binary type to avoid inline previews in the report
-          // Ensures the screenshot is paired with the reasoning attachment instead of being rendered in a separate section.
-          contentType: "application/octet-stream",
-          name: `${sanitizedName}-screenshot.png`
-        }
-      );
+      const assertionPassed = lastAttempt.verifyResult.pass !== isNot;
       return {
         message: () => createFailureMessage({
           condition,
-          didPass: lastVerifyResult.pass,
+          didPass: lastAttempt.verifyResult.pass,
           isNot,
-          reason: lastVerifyResult.reason,
-          targetType
+          reason: lastAttempt.verifyResult.reason,
+          targetType,
+          // Include timeout in the message when the assertion didn't pass,
+          // matching how Playwright's built-in matchers report timeouts.
+          timeoutMs: assertionPassed ? void 0 : timeoutMs
         }),
         name: "aiAssert",
-        pass: lastVerifyResult.pass
+        pass: lastAttempt.verifyResult.pass
       };
     };
-    return Promise.race([work(), timeoutPromise]);
+    try {
+      return await Promise.race([work(), timeoutPromise]);
+    } catch (error) {
+      if (error instanceof AiAssertTimeoutError) {
+        return {
+          message: () => createFailureMessage({
+            condition,
+            didPass: lastAttempt.verifyResult.pass,
+            isNot,
+            reason: lastAttempt.verifyResult.reason,
+            targetType,
+            timeoutMs
+          }),
+          name: "aiAssert",
+          pass: lastAttempt.verifyResult.pass
+        };
+      }
+      throw error;
+    } finally {
+      clearTimeout(timeoutId);
+      if (lastAttempt.hasCompleted) {
+        addAssertAttachments(
+          condition,
+          lastAttempt.verifyResult,
+          lastAttempt.screenshot
+        );
+      }
+    }
   },
   /**
    * @deprecated Use `aiAssert` instead. This method will be removed in a future version.