@intuned/browser-dev 2.2.3-test-build.0

package/dist/ai/isPageLoaded.js

@@ -0,0 +1,77 @@
+"use strict";
+
+Object.defineProperty(exports, "__esModule", {
+  value: true
+});
+exports.isPageLoaded = void 0;
+var _Logger = require("../common/Logger");
+var _utils = require("../helpers/utils");
+var _ai = require("ai");
+const isPageLoaded = async options => {
+  var _response$response;
+  const {
+    page
+  } = options;
+  const timeoutInMs = options !== null && options !== void 0 && options.timeoutInMs ? options === null || options === void 0 ? void 0 : options.timeoutInMs : 10000;
+  const screenshotBytes = await page.screenshot({
+    fullPage: false,
+    type: "png",
+    timeout: timeoutInMs
+  });
+  const gateway = _utils.GatewayFactory.createAIGateway();
+  const gatewayModel = gateway.getModel((options === null || options === void 0 ? void 0 : options.model) ?? "gpt-4o-2024-08-06", options === null || options === void 0 ? void 0 : options.apiKey);
+  const base64Image = Buffer.from(screenshotBytes).toString("base64");
+  const response = await (0, _ai.generateText)({
+    model: gatewayModel,
+    messages: [{
+      role: "system",
+      content: `You are a helpful assistant that determines if a webpage finished loading. If the page finished loading, start your answer with 'True'. If the page is loading, start your answer with 'False'. If you are not sure, start your answer with 'Dont know'. In a new line, add a reason to your response.
+
+Some good cues for determining if a page is loading:
+- Loading spinner
+- Page is blank
+- Some content looks like it's missing
+- Not on splash screen`
+    }, {
+      role: "user",
+      content: [{
+        type: "text",
+        text: "Look at the screenshot and tell me, is the page loading or has it finished loading?"
+      }, {
+        type: "image",
+        image: base64Image
+      }]
+    }]
+  });
+  if (response !== null && response !== void 0 && (_response$response = response.response) !== null && _response$response !== void 0 && (_response$response = _response$response.headers) !== null && _response$response !== void 0 && _response$response["x-ai-cost-in-cents"]) {
+    _Logger.logger.info(`Total LLM Cost In Cents: ${response.response.headers["x-ai-cost-in-cents"]}`);
+  } else {
+    _Logger.logger.info(`Total LLM Tokens: ${response.usage.totalTokens}`);
+  }
+  let llmResult = response.text.trim();
+  if (!llmResult) {
+    throw new Error("LLM response is empty");
+  }
+  llmResult = llmResult.split("\n").filter(line => line.trim() !== "").join("\n");
+  const isTrue = llmResult.includes("True");
+  const isFalse = llmResult.includes("False");
+  const isDontKnow = llmResult.includes("Dont know") || llmResult.includes("Don't know");
+  const lines = llmResult.split("\n");
+  const reason = lines.length > 1 ? lines[1] : null;
+  let result;
+  if (isTrue) {
+    _Logger.logger.info(`Page is loaded.`);
+    result = true;
+  } else if (isFalse) {
+    _Logger.logger.info(`Page is not loaded.`);
+    result = false;
+  } else if (isDontKnow) {
+    _Logger.logger.info(`Page loading status is unknown.`);
+    result = false;
+  } else {
+    throw new Error("LLM result is not valid");
+  }
+  _Logger.logger.info(`LLM Reason: ${reason}`);
+  return result;
+};
+exports.isPageLoaded = isPageLoaded;

package/dist/ai/prompt.js

@@ -0,0 +1,39 @@
+"use strict";
+
+Object.defineProperty(exports, "__esModule", {
+  value: true
+});
+exports.getMessages = getMessages;
+function getMessages(input) {
+  const {
+    prompt,
+    content,
+    images
+  } = input;
+  const messages = [{
+    role: "system",
+    content: `You are a data analyst whose job is to extract structured data from an HTML page.
+    Please ensure that the data is extracted exactly as it appears in the HTML, without any additional formatting or alterations.
+    Extract the structured data exactly as it is in the HTML.
+    If you don't find a specific field just don't return the field.
+    Call extract_data_from_content tool with the extracted data as the argument of this tool.
+    ${prompt}`
+  }];
+  const userContentParts = [{
+    type: "text",
+    text: content
+  }];
+  if (images && images.length > 0) {
+    images.forEach(image => {
+      userContentParts.push({
+        type: "image",
+        image: `data:image/${image.image_type};base64,${image.data.toString("base64")}`
+      });
+    });
+  }
+  messages.push({
+    role: "user",
+    content: userContentParts
+  });
+  return messages;
+}

package/dist/ai/tests/testCheckAllTypesAreStrings.spec.js

@@ -0,0 +1,137 @@
+"use strict";
+
+var _extendedTest = require("../../helpers/tests/extendedTest");
+var _validators = require("../validators");
+(0, _extendedTest.describe)("Check All Types Are Strings Tests", () => {
+  (0, _extendedTest.test)("should return true for string schema", async () => {
+    const schema = {
+      type: "string",
+      description: "A simple string field"
+    };
+    (0, _extendedTest.expect)((0, _validators.checkAllTypesAreStrings)(schema)).toBe(true);
+  });
+  (0, _extendedTest.test)("should return false for number schema", async () => {
+    const schema = {
+      type: "number",
+      description: "A number field"
+    };
+    (0, _extendedTest.expect)((0, _validators.checkAllTypesAreStrings)(schema)).toBe(false);
+  });
+  (0, _extendedTest.test)("should return false for boolean schema", async () => {
+    const schema = {
+      type: "boolean",
+      description: "A boolean field"
+    };
+    (0, _extendedTest.expect)((0, _validators.checkAllTypesAreStrings)(schema)).toBe(false);
+  });
+  (0, _extendedTest.test)("should return true for array of strings", async () => {
+    const schema = {
+      type: "array",
+      items: {
+        type: "string"
+      }
+    };
+    (0, _extendedTest.expect)((0, _validators.checkAllTypesAreStrings)(schema)).toBe(true);
+  });
+  (0, _extendedTest.test)("should return true for array with no items constraint", async () => {
+    const schema = {
+      type: "array",
+      items: {
+        type: "string"
+      }
+    };
+    (0, _extendedTest.expect)((0, _validators.checkAllTypesAreStrings)(schema)).toBe(true);
+  });
+  (0, _extendedTest.test)("should return true for object with all string properties", async () => {
+    const schema = {
+      type: "object",
+      properties: {
+        name: {
+          type: "string"
+        },
+        description: {
+          type: "string"
+        }
+      },
+      required: ["name", "description"]
+    };
+    (0, _extendedTest.expect)((0, _validators.checkAllTypesAreStrings)(schema)).toBe(true);
+  });
+  (0, _extendedTest.test)("should return false for object with mixed property types", async () => {
+    const schema = {
+      type: "object",
+      properties: {
+        name: {
+          type: "string"
+        },
+        age: {
+          type: "number"
+        }
+      },
+      required: ["name", "age"]
+    };
+    (0, _extendedTest.expect)((0, _validators.checkAllTypesAreStrings)(schema)).toBe(false);
+  });
+  (0, _extendedTest.test)("should return true for object with no properties", async () => {
+    const schema = {
+      type: "object",
+      properties: {},
+      required: []
+    };
+    (0, _extendedTest.expect)((0, _validators.checkAllTypesAreStrings)(schema)).toBe(true);
+  });
+  (0, _extendedTest.test)("should return true for nested object with all string properties", async () => {
+    const schema = {
+      type: "object",
+      properties: {
+        user: {
+          type: "object",
+          properties: {
+            firstName: {
+              type: "string"
+            },
+            lastName: {
+              type: "string"
+            }
+          },
+          required: ["firstName", "lastName"]
+        },
+        metadata: {
+          type: "object",
+          properties: {
+            tags: {
+              type: "array",
+              items: {
+                type: "string"
+              }
+            }
+          },
+          required: ["tags"]
+        }
+      },
+      required: ["user", "metadata"]
+    };
+    (0, _extendedTest.expect)((0, _validators.checkAllTypesAreStrings)(schema)).toBe(true);
+  });
+  (0, _extendedTest.test)("should return false for nested object with non-string properties", async () => {
+    const schema = {
+      type: "object",
+      properties: {
+        user: {
+          type: "object",
+          properties: {
+            firstName: {
+              type: "string"
+            },
+            age: {
+              type: "number"
+            }
+          },
+          required: ["firstName", "age"]
+        }
+      },
+      required: ["user"]
+    };
+    (0, _extendedTest.expect)((0, _validators.checkAllTypesAreStrings)(schema)).toBe(false);
+  });
+});

package/dist/ai/tests/testExtractFromContent.spec.js

@@ -0,0 +1,372 @@
+"use strict";
+
+var _extendedTest = require("../../common/extendedTest");
+var _ = require("..");
+var _playwrightCore = require("playwright-core");
+var _dotenv = require("dotenv");
+(0, _dotenv.config)();
+_extendedTest.describe.skip("Extract data from content tests", () => {
+  let browser;
+  let page;
+  (0, _extendedTest.beforeAll)(async () => {
+    browser = await _playwrightCore.chromium.launch({
+      headless: false
+    });
+  });
+  (0, _extendedTest.afterAll)(async () => {
+    await browser.close();
+  });
+  (0, _extendedTest.beforeEach)(async () => {
+    page = await browser.newPage();
+  });
+  (0, _extendedTest.afterEach)(async () => {
+    await page.close();
+  });
+  (0, _extendedTest.describe)("Text content extraction", () => {
+    (0, _extendedTest.test)("should extract structured data from single text content", async () => {
+      const textContent = {
+        type: "text",
+        data: "John Doe, age 30, works as a Software Engineer at Tech Corp. His email is john.doe@techcorp.com and phone number is +1-555-0123."
+      };
+      const personSchema = {
+        type: "object",
+        properties: {
+          name: {
+            type: "string",
+            description: "Person's full name"
+          },
+          age: {
+            type: "number",
+            description: "Person's age"
+          },
+          occupation: {
+            type: "string",
+            description: "Person's job title"
+          },
+          company: {
+            type: "string",
+            description: "Company name"
+          },
+          email: {
+            type: "string",
+            description: "Email address"
+          },
+          phone: {
+            type: "string",
+            description: "Phone number"
+          }
+        },
+        required: ["name"]
+      };
+      const result = await (0, _.extractStructuredData)({
+        content: textContent,
+        model: "gpt-4o",
+        dataSchema: personSchema,
+        prompt: "Extract person information from the text"
+      });
+      (0, _extendedTest.expect)(result).toBeDefined();
+      (0, _extendedTest.expect)(result.name).toBe("John Doe");
+      (0, _extendedTest.expect)(result.age).toBe(30);
+      (0, _extendedTest.expect)(result.occupation).toContain("Software Engineer");
+      (0, _extendedTest.expect)(result.company).toContain("Tech Corp");
+      (0, _extendedTest.expect)(result.email).toBe("john.doe@techcorp.com");
+      (0, _extendedTest.expect)(result.phone).toContain("555-0123");
+    });
+    (0, _extendedTest.test)("should extract array data from text content", async () => {
+      const textContent = {
+        type: "text",
+        data: `
+          Product List:
+          1. iPhone 15 - $999 - Apple
+          2. Samsung Galaxy S24 - $899 - Samsung  
+          3. Google Pixel 8 - $699 - Google
+        `
+      };
+      const productsSchema = {
+        type: "object",
+        properties: {
+          products: {
+            type: "array",
+            items: {
+              type: "object",
+              properties: {
+                name: {
+                  type: "string",
+                  description: "Product name"
+                },
+                price: {
+                  type: "string",
+                  description: "Product price"
+                },
+                brand: {
+                  type: "string",
+                  description: "Product brand"
+                }
+              },
+              required: ["name", "price", "brand"]
+            }
+          }
+        },
+        required: ["products"]
+      };
+      const result = await (0, _.extractStructuredData)({
+        content: textContent,
+        model: "gpt-4o",
+        dataSchema: productsSchema,
+        prompt: "Extract all products with their details"
+      });
+      (0, _extendedTest.expect)(result).toBeDefined();
+      (0, _extendedTest.expect)(result.products).toHaveLength(3);
+      (0, _extendedTest.expect)(result.products[0].name).toContain("iPhone");
+      (0, _extendedTest.expect)(result.products[0].price).toContain("999");
+      (0, _extendedTest.expect)(result.products[0].brand).toBe("Apple");
+    });
+    (0, _extendedTest.test)("should handle multiple text content items", async () => {
+      const textContents = [{
+        type: "text",
+        data: "Customer: Alice Johnson"
+      }, {
+        type: "text",
+        data: "Order ID: ORD-12345"
+      }, {
+        type: "text",
+        data: "Total: $156.78"
+      }];
+      const orderSchema = {
+        type: "object",
+        properties: {
+          customer: {
+            type: "string",
+            description: "Customer name"
+          },
+          orderId: {
+            type: "string",
+            description: "Order identifier"
+          },
+          total: {
+            type: "string",
+            description: "Order total amount"
+          }
+        },
+        required: ["customer", "orderId", "total"]
+      };
+      const result = await (0, _.extractStructuredData)({
+        content: textContents,
+        model: "gpt-4o",
+        dataSchema: orderSchema,
+        prompt: "Extract order information from the text fragments"
+      });
+      (0, _extendedTest.expect)(result).toBeDefined();
+      (0, _extendedTest.expect)(result.customer).toBe("Alice Johnson");
+      (0, _extendedTest.expect)(result.orderId).toBe("ORD-12345");
+      (0, _extendedTest.expect)(result.total).toBe("$156.78");
+    });
+  });
+  (0, _extendedTest.describe)("Image URL content extraction", () => {
+    (0, _extendedTest.test)("should extract data from image URL content", async () => {
+      const imageContent = {
+        type: "image-url",
+        image_type: "png",
+        data: "https://cdn-dynmedia-1.microsoft.com/is/image/microsoftcorp/2-accordion-3-800x513-1?resMode=sharp2&op_usm=1.5,0.65,15,0&wid=1664&hei=1062&qlt=100&fmt=png-alpha&fit=constrain"
+      };
+      const imageSchema = {
+        type: "array",
+        items: {
+          type: "string",
+          description: "Todo item shown in the middle panel"
+        }
+      };
+      const result = await (0, _.extractStructuredData)({
+        content: imageContent,
+        model: "gpt-4o",
+        dataSchema: imageSchema,
+        prompt: "Extract todo items from the image, they are shown in the middle panel"
+      });
+      (0, _extendedTest.expect)(result).toBeDefined();
+      (0, _extendedTest.expect)(result[0]).toContain("Yoga");
+    });
+    (0, _extendedTest.test)("should handle invalid image URLs gracefully", async () => {
+      const imageContent = {
+        type: "image-url",
+        image_type: "png",
+        data: "https://invalid-url-com/image.png"
+      };
+      const schema = {
+        type: "object",
+        properties: {
+          description: {
+            type: "string"
+          }
+        }
+      };
+      await (0, _extendedTest.expect)((0, _.extractStructuredData)({
+        content: imageContent,
+        model: "gpt-4o",
+        dataSchema: schema
+      })).rejects.toThrow();
+    });
+  });
+  (0, _extendedTest.describe)("Mixed content extraction", () => {
+    (0, _extendedTest.test)("should extract data from mixed text and image content", async () => {
+      const mixedContent = [{
+        type: "text",
+        data: "Product: iPhone 15 Pro Max"
+      }, {
+        type: "text",
+        data: "Price: $1,199"
+      }, {
+        type: "image-url",
+        image_type: "png",
+        data: "https://cdn11.bigcommerce.com/s-scmrv6kkrz/images/stencil/1280x1280/products/192719/166712/1891-clear-14-vdc-miniature-lb93__16581.1568390384.jpg?c=2"
+      }];
+      const productSchema = {
+        type: "object",
+        properties: {
+          name: {
+            type: "string",
+            description: "Product name"
+          },
+          price: {
+            type: "string",
+            description: "Product price"
+          },
+          imageDescription: {
+            type: "string",
+            description: "Description of the product image"
+          }
+        },
+        required: ["name", "price"]
+      };
+      const result = await (0, _.extractStructuredData)({
+        content: mixedContent,
+        model: "gpt-4o",
+        dataSchema: productSchema,
+        prompt: "Extract product information from both text and image"
+      });
+      (0, _extendedTest.expect)(result).toBeDefined();
+      (0, _extendedTest.expect)(result.name).toContain("iPhone");
+      (0, _extendedTest.expect)(result.price).toContain("1,199");
+      (0, _extendedTest.expect)(result.imageDescription).toBeDefined();
+    });
+  });
+  (0, _extendedTest.describe)("Input validation and error handling", () => {
+    (0, _extendedTest.test)("should validate data schema", async () => {
+      const textContent = {
+        type: "text",
+        data: "test data"
+      };
+      await (0, _extendedTest.expect)((0, _.extractStructuredData)({
+        content: textContent,
+        dataSchema: "invalid schema",
+        model: "gpt-4o"
+      })).rejects.toThrow();
+    });
+    (0, _extendedTest.test)("should validate model parameter", async () => {
+      const textContent = {
+        type: "text",
+        data: "test data"
+      };
+      await (0, _extendedTest.expect)((0, _.extractStructuredData)({
+        content: textContent,
+        dataSchema: {
+          type: "object",
+          properties: {}
+        },
+        model: "invalid-model"
+      })).rejects.toThrow();
+    });
+    (0, _extendedTest.test)("should handle empty content gracefully", async () => {
+      const textContent = {
+        type: "text",
+        data: ""
+      };
+      const schema = {
+        type: "object",
+        properties: {
+          message: {
+            type: "string",
+            description: "Any message found"
+          }
+        }
+      };
+      const result = await (0, _.extractStructuredData)({
+        content: textContent,
+        model: "gpt-4o",
+        dataSchema: schema,
+        prompt: "Extract any information from the content"
+      });
+      (0, _extendedTest.expect)(result).toBeDefined();
+    });
+    (0, _extendedTest.test)("should handle empty array content", async () => {
+      const schema = {
+        type: "object",
+        properties: {
+          message: {
+            type: "string",
+            description: "Any message found"
+          }
+        }
+      };
+      await (0, _extendedTest.expect)((0, _.extractStructuredData)({
+        content: [],
+        model: "gpt-4o",
+        dataSchema: schema
+      })).rejects.toThrow();
+    });
+  });
+  (0, _extendedTest.describe)("Caching behavior", () => {
+    (0, _extendedTest.test)("should support cache enabling/disabling", async () => {
+      const textContent = {
+        type: "text",
+        data: "Test content for caching"
+      };
+      const schema = {
+        type: "object",
+        properties: {
+          content: {
+            type: "string",
+            description: "Content summary"
+          }
+        }
+      };
+      const result1 = await (0, _.extractStructuredData)({
+        content: textContent,
+        model: "gpt-4o",
+        dataSchema: schema,
+        enableCache: true
+      });
+      const result2 = await (0, _.extractStructuredData)({
+        content: textContent,
+        model: "gpt-4o",
+        dataSchema: schema,
+        enableCache: false
+      });
+      (0, _extendedTest.expect)(result1).toBeDefined();
+      (0, _extendedTest.expect)(result2).toBeDefined();
+    });
+  });
+  (0, _extendedTest.describe)("Retry behavior", () => {
+    (0, _extendedTest.test)("should support maxRetries parameter", async () => {
+      const textContent = {
+        type: "text",
+        data: "Test content for retry behavior"
+      };
+      const schema = {
+        type: "object",
+        properties: {
+          content: {
+            type: "string",
+            description: "Content summary"
+          }
+        }
+      };
+      const result = await (0, _.extractStructuredData)({
+        content: textContent,
+        model: "o4-mini",
+        dataSchema: schema,
+        maxRetries: 1
+      });
+      (0, _extendedTest.expect)(result).toBeDefined();
+    });
+  });
+});