@intuned/browser-dev 2.2.3-unify-sdks.21 → 2.2.3-unify-sdks.22

package/dist/playwright/export.d.ts

@@ -0,0 +1,229 @@
+import type { Locator, Page } from "playwright-core";
+import {
+  extractMarkdownFromPage,
+  extractStructuredDataFromPage,
+} from "../ai-extractors";
+import {
+  extractArrayFromPage,
+  extractObjectFromPage,
+} from "../optimized-extractors";
+
+/**
+ * --
+ * @interface
+ * @property selector - The selector string for the element.
+ * @property [type] - Optional. The type of the selector (xpath or css) default is `css`
+ */
+export interface ElementSelector {
+  selector: string;
+  type?: "xpath" | "css";
+}
+
+/**
+ * a record or property name and the value selector to extract the value from the page.
+ * you can provide a list of `ValueSelector` to provide a backup selector in case the first one fails.
+ * the primary selector is the first one in the list.
+ */
+export type ObjectExtractor = Record<string, ValueSelector | ValueSelector[]>;
+
+/**
+ * --
+ * @interface
+ * @property selector - The selector string for the element.
+ * @property [type] - Optional. The type of the selector (xpath or css). default to `css`
+ */
+export interface ElementSelector {
+  selector: string;
+  type?: "xpath" | "css";
+}
+
+/**
+ * represents a dom element selector and the method to extract the value from the element.
+ *
+ * @interface
+ * @extends ElementSelector
+ * @property [selectionMethod] - Optional. The method for selecting the value. `all-text` selects all text content, `direct-text` selects the direct text content(does not include the text inside nested elements), and `propertyName` selects the value of a property.
+ * @property [regex] - Optional. A regex pattern and match index for extracting the value.
+ * @property [multiValue] - Optional. Whether the selector extracts multiple values, if set to true the returned value will be array of strings
+ */
+export interface ValueSelector extends ElementSelector {
+  selectionMethod?:
+    | "direct-text"
+    | "all-text"
+    | {
+        propertyName: string;
+      };
+
+  regex?: {
+    pattern: string;
+    matchIndex?: number;
+  };
+  multiValue?: boolean;
+}
+
+/**
+ * --
+ *
+ * @interface
+ * @property containerSelector - The selector(s) for the container elements of the list, all list items should be direct children of this container.
+ * @property propertySelectors - The selectors for the properties to extract. the values of the selector should be relative to the list item.
+ *
+ * **example:** if the list was:
+ *
+ * ```html
+ * <ul>
+ * <li>
+ *  <div class="title">title 1</div>
+ *  <div class="price">price 1</div>
+ * </li>
+ * <li>
+ *  <div class="title">title 2</div>
+ *  <div class="price">price 2</div>
+ * </li>
+ * </ul>
+ * ```
+ * the css relative selectors should be:
+ *
+ * title -> `.title`
+ *
+ * price -> `.price`
+ *
+ */
+export interface ListStaticExtractor {
+  containerSelector: ElementSelector | ElementSelector[];
+  propertySelectors: Record<string, ValueSelector | ValueSelector[]>;
+}
+
+export type ExtractObjectFromPageUsingSelectorsReturnType<
+  T extends ObjectExtractor
+> = {
+  [K in keyof T]: T[K] extends {
+    multiValue: true;
+  }
+    ? string[] | null
+    : string | null;
+};
+
+export type ExtractListObjectsUsingStaticSelectorsReturnType<
+  T extends ListStaticExtractor
+> = {
+  [K in keyof T["propertySelectors"]]: T["propertySelectors"][K] extends {
+    multiValue: true;
+  }
+    ? string[] | null
+    : string | null;
+}[];
+
+/**
+ * Extracts an object from a web page using the specified selectors.
+ *
+ * @param page - The Playwright Page object from which to extract the data.
+ * @param extractor - The object extractor with the selectors to use.
+ * @returns A promise that resolves to the extracted object.
+ *
+ * @example
+ * ```typescript extractObjectFromPageUsingSelectors
+ *  import { extractObjectFromPageUsingSelectors, goto } from "@intuned/sdk/playwright";
+ *
+ *  await goto(page, 'https://books.toscrape.com/catalogue/a-light-in-the-attic_1000/index.html');
+ *  const book = await extractObjectFromPageUsingSelectors(page, {
+ *    name: {
+ *      selector: "h1",
+ *      selectionMethod: "all-text"
+ *    },
+ *    inStock: {
+ *      selector: ".price_color",
+ *    },
+ *    imgUrl: {
+ *      selector: "#product_gallery > div > div > div > img",
+ *      selectionMethod: {
+ *        propertyName: "src"
+ *      }
+ *    }
+ *  })
+ *
+ *  console.log(book)
+ *
+ *  // output:
+ *  // {
+ *  //   name: 'A Light in the Attic',
+ *  //   inStock: '£51.77',
+ *  //   imgUrl: '../../media/cache/fe/72/fe72f0532301ec28892ae79a629a293c.jpg'
+ *  // }
+ *
+ * ```
+ */
+export declare function extractObjectFromPageUsingSelectors<
+  T extends ObjectExtractor
+>(
+  page: Page,
+  extractor: T
+): Promise<ExtractObjectFromPageUsingSelectorsReturnType<T>>;
+
+/**
+ * Extracts a list of objects from a web page using the specified static selectors.
+ *
+ * @param page - The Playwright Page object from which to extract the data.
+ * @param listExtractor - The list static extractor with the selectors to use.
+ * @returns A promise that resolves to the extracted list of objects.
+ * @example
+ * ```typescript extractArrayFromPageUsingSelectors
+ *  import { extractArrayFromPageUsingSelectors, goto } from "@intuned/sdk/playwright";
+ *
+ *  await goto(page, 'https://books.toscrape.com/index.html');
+ *  const books = await extractArrayFromPageUsingSelectors(page, {
+ *    containerSelector: {
+ *      selector: '//*[@id="default"]/div/div/div/div/section/div[2]/ol',
+ *      type: "xpath"
+ *    },
+ *    propertySelectors: {
+ *      name: {
+ *        selector: "h3",
+ *      },
+ *      inStock: {
+ *        selector: ".price_color",
+ *      },
+ *      imgUrl: {
+ *        selector: "article > div.image_container > a > img",
+ *        selectionMethod: {
+ *          propertyName: "src"
+ *        }
+ *      }
+ *    }
+ *  })
+ *
+ *  console.log(books)
+ *
+ *  // output:
+ *  // [
+ *  //   {
+ *  //   name: 'A Light in the ...',
+ *  //   inStock: '£51.77',
+ *  //   imgUrl: 'media/cache/2c/da/2cdad67c44b002e7ead0cc35693c0e8b.jpg'
+ *  // },
+ *  // {
+ *  //   name: 'Tipping the Velvet',
+ *  //   inStock: '£53.74',
+ *  //   imgUrl: 'media/cache/26/0c/260c6ae16bce31c8f8c95daddd9f4a1c.jpg'
+ *  // },
+ *  // {
+ *  //   name: 'Soumission',
+ *  //   inStock: '£50.10',
+ *  //   imgUrl: 'media/cache/3e/ef/3eef99c9d9adef34639f510662022830.jpg'
+ *  // },
+ *  //   {
+ *  //     name: 'Sharp Objects',
+ *  //     inStock: '£47.82',
+ *  //     imgUrl: 'media/cache/32/51/3251cf3a3412f53f339e42cac2134093.jpg'
+ *  //   },
+ *  //   ...
+ *  // ]
+ *
+ * ```
+ */
+export declare function extractArrayFromPageUsingSelectors<
+  T extends ListStaticExtractor
+>(
+  page: Page,
+  listExtractor: T
+): Promise<ExtractListObjectsUsingStaticSelectorsReturnType<T>>;

package/dist/playwright/index.d.ts

@@ -0,0 +1,229 @@
+import type { Locator, Page } from "playwright-core";
+import {
+  extractMarkdownFromPage,
+  extractStructuredDataFromPage,
+} from "../ai-extractors";
+import {
+  extractArrayFromPage,
+  extractObjectFromPage,
+} from "../optimized-extractors";
+
+/**
+ * --
+ * @interface
+ * @property selector - The selector string for the element.
+ * @property [type] - Optional. The type of the selector (xpath or css) default is `css`
+ */
+export interface ElementSelector {
+  selector: string;
+  type?: "xpath" | "css";
+}
+
+/**
+ * a record or property name and the value selector to extract the value from the page.
+ * you can provide a list of `ValueSelector` to provide a backup selector in case the first one fails.
+ * the primary selector is the first one in the list.
+ */
+export type ObjectExtractor = Record<string, ValueSelector | ValueSelector[]>;
+
+/**
+ * --
+ * @interface
+ * @property selector - The selector string for the element.
+ * @property [type] - Optional. The type of the selector (xpath or css). default to `css`
+ */
+export interface ElementSelector {
+  selector: string;
+  type?: "xpath" | "css";
+}
+
+/**
+ * represents a dom element selector and the method to extract the value from the element.
+ *
+ * @interface
+ * @extends ElementSelector
+ * @property [selectionMethod] - Optional. The method for selecting the value. `all-text` selects all text content, `direct-text` selects the direct text content(does not include the text inside nested elements), and `propertyName` selects the value of a property.
+ * @property [regex] - Optional. A regex pattern and match index for extracting the value.
+ * @property [multiValue] - Optional. Whether the selector extracts multiple values, if set to true the returned value will be array of strings
+ */
+export interface ValueSelector extends ElementSelector {
+  selectionMethod?:
+    | "direct-text"
+    | "all-text"
+    | {
+        propertyName: string;
+      };
+
+  regex?: {
+    pattern: string;
+    matchIndex?: number;
+  };
+  multiValue?: boolean;
+}
+
+/**
+ * --
+ *
+ * @interface
+ * @property containerSelector - The selector(s) for the container elements of the list, all list items should be direct children of this container.
+ * @property propertySelectors - The selectors for the properties to extract. the values of the selector should be relative to the list item.
+ *
+ * **example:** if the list was:
+ *
+ * ```html
+ * <ul>
+ * <li>
+ *  <div class="title">title 1</div>
+ *  <div class="price">price 1</div>
+ * </li>
+ * <li>
+ *  <div class="title">title 2</div>
+ *  <div class="price">price 2</div>
+ * </li>
+ * </ul>
+ * ```
+ * the css relative selectors should be:
+ *
+ * title -> `.title`
+ *
+ * price -> `.price`
+ *
+ */
+export interface ListStaticExtractor {
+  containerSelector: ElementSelector | ElementSelector[];
+  propertySelectors: Record<string, ValueSelector | ValueSelector[]>;
+}
+
+export type ExtractObjectFromPageUsingSelectorsReturnType<
+  T extends ObjectExtractor
+> = {
+  [K in keyof T]: T[K] extends {
+    multiValue: true;
+  }
+    ? string[] | null
+    : string | null;
+};
+
+export type ExtractListObjectsUsingStaticSelectorsReturnType<
+  T extends ListStaticExtractor
+> = {
+  [K in keyof T["propertySelectors"]]: T["propertySelectors"][K] extends {
+    multiValue: true;
+  }
+    ? string[] | null
+    : string | null;
+}[];
+
+/**
+ * Extracts an object from a web page using the specified selectors.
+ *
+ * @param page - The Playwright Page object from which to extract the data.
+ * @param extractor - The object extractor with the selectors to use.
+ * @returns A promise that resolves to the extracted object.
+ *
+ * @example
+ * ```typescript extractObjectFromPageUsingSelectors
+ *  import { extractObjectFromPageUsingSelectors, goto } from "@intuned/sdk/playwright";
+ *
+ *  await goto(page, 'https://books.toscrape.com/catalogue/a-light-in-the-attic_1000/index.html');
+ *  const book = await extractObjectFromPageUsingSelectors(page, {
+ *    name: {
+ *      selector: "h1",
+ *      selectionMethod: "all-text"
+ *    },
+ *    inStock: {
+ *      selector: ".price_color",
+ *    },
+ *    imgUrl: {
+ *      selector: "#product_gallery > div > div > div > img",
+ *      selectionMethod: {
+ *        propertyName: "src"
+ *      }
+ *    }
+ *  })
+ *
+ *  console.log(book)
+ *
+ *  // output:
+ *  // {
+ *  //   name: 'A Light in the Attic',
+ *  //   inStock: '£51.77',
+ *  //   imgUrl: '../../media/cache/fe/72/fe72f0532301ec28892ae79a629a293c.jpg'
+ *  // }
+ *
+ * ```
+ */
+export declare function extractObjectFromPageUsingSelectors<
+  T extends ObjectExtractor
+>(
+  page: Page,
+  extractor: T
+): Promise<ExtractObjectFromPageUsingSelectorsReturnType<T>>;
+
+/**
+ * Extracts a list of objects from a web page using the specified static selectors.
+ *
+ * @param page - The Playwright Page object from which to extract the data.
+ * @param listExtractor - The list static extractor with the selectors to use.
+ * @returns A promise that resolves to the extracted list of objects.
+ * @example
+ * ```typescript extractArrayFromPageUsingSelectors
+ *  import { extractArrayFromPageUsingSelectors, goto } from "@intuned/sdk/playwright";
+ *
+ *  await goto(page, 'https://books.toscrape.com/index.html');
+ *  const books = await extractArrayFromPageUsingSelectors(page, {
+ *    containerSelector: {
+ *      selector: '//*[@id="default"]/div/div/div/div/section/div[2]/ol',
+ *      type: "xpath"
+ *    },
+ *    propertySelectors: {
+ *      name: {
+ *        selector: "h3",
+ *      },
+ *      inStock: {
+ *        selector: ".price_color",
+ *      },
+ *      imgUrl: {
+ *        selector: "article > div.image_container > a > img",
+ *        selectionMethod: {
+ *          propertyName: "src"
+ *        }
+ *      }
+ *    }
+ *  })
+ *
+ *  console.log(books)
+ *
+ *  // output:
+ *  // [
+ *  //   {
+ *  //   name: 'A Light in the ...',
+ *  //   inStock: '£51.77',
+ *  //   imgUrl: 'media/cache/2c/da/2cdad67c44b002e7ead0cc35693c0e8b.jpg'
+ *  // },
+ *  // {
+ *  //   name: 'Tipping the Velvet',
+ *  //   inStock: '£53.74',
+ *  //   imgUrl: 'media/cache/26/0c/260c6ae16bce31c8f8c95daddd9f4a1c.jpg'
+ *  // },
+ *  // {
+ *  //   name: 'Soumission',
+ *  //   inStock: '£50.10',
+ *  //   imgUrl: 'media/cache/3e/ef/3eef99c9d9adef34639f510662022830.jpg'
+ *  // },
+ *  //   {
+ *  //     name: 'Sharp Objects',
+ *  //     inStock: '£47.82',
+ *  //     imgUrl: 'media/cache/32/51/3251cf3a3412f53f339e42cac2134093.jpg'
+ *  //   },
+ *  //   ...
+ *  // ]
+ *
+ * ```
+ */
+export declare function extractArrayFromPageUsingSelectors<
+  T extends ListStaticExtractor
+>(
+  page: Page,
+  listExtractor: T
+): Promise<ExtractListObjectsUsingStaticSelectorsReturnType<T>>;

package/dist/playwright/index.js

@@ -0,0 +1,18 @@
+"use strict";
+
+Object.defineProperty(exports, "__esModule", {
+  value: true
+});
+Object.defineProperty(exports, "extractArrayFromPageUsingSelectors", {
+  enumerable: true,
+  get: function () {
+    return _staticExtractors.extractArrayFromPageUsingSelectors;
+  }
+});
+Object.defineProperty(exports, "extractObjectFromPageUsingSelectors", {
+  enumerable: true,
+  get: function () {
+    return _staticExtractors.extractObjectFromPageUsingSelectors;
+  }
+});
+var _staticExtractors = require("./staticExtractors");

package/dist/playwright/staticExtractors/extractHelpers.js

@@ -0,0 +1,170 @@
+"use strict";
+
+Object.defineProperty(exports, "__esModule", {
+  value: true
+});
+exports.compressStringSpaces = compressStringSpaces;
+exports.injectExtractorsHelperFunctions = injectExtractorsHelperFunctions;
+exports.selectArrayValuesUsingRelativeSelector = selectArrayValuesUsingRelativeSelector;
+exports.selectLocatorsUsingXpath = selectLocatorsUsingXpath;
+exports.selectMultiValueUsingRelativeSelector = selectMultiValueUsingRelativeSelector;
+exports.selectValueUsingRelativeSelector = selectValueUsingRelativeSelector;
+exports.splitContainerIntoListLocators = splitContainerIntoListLocators;
+async function injectExtractorsHelperFunctions(page) {
+  await page.evaluate(() => {
+    const generateMultiValueWarningMessage = ({
+      propertyName,
+      nodesLength
+    }) => {
+      return `The selector for the property '${propertyName}' is matching ${nodesLength} elements. Please ensure that the 'multiValue' flag is enabled to retrieve all matching elements. Without this flag, only the first element will be processed.`;
+    };
+    window["cleanValue"] = (regex, value) => {
+      var _text$match;
+      const compressStringSpaces = str => str.replace(/\s+/g, " ").trim();
+      const text = compressStringSpaces(value);
+      if (!regex) return text;
+      const regexMatchIndex = regex.matchIndex === undefined || regex.matchIndex === null ? 1 : regex.matchIndex;
+      const regexFromString = new RegExp(regex.pattern);
+      const valueFromRegex = (_text$match = text.match(regexFromString)) === null || _text$match === void 0 ? void 0 : _text$match[regexMatchIndex];
+      return compressStringSpaces(valueFromRegex ?? "") || null;
+    };
+    window["getValueUsingNodeAndSelector"] = ({
+      selector,
+      element
+    }) => {
+      if (selector.selectionMethod === "all-text") {
+        return element.textContent;
+      } else if (selector.selectionMethod === "direct-text") {
+        return Array.from(element.childNodes).filter(child => child.nodeType === Node.TEXT_NODE).map(child => child.textContent ?? "").join("");
+      } else {
+        return element.getAttribute(selector.selectionMethod.propertyName);
+      }
+    };
+    window["getValueUsingSelector"] = ({
+      selector,
+      relativeElement,
+      propertyName
+    }) => {
+      let node = null;
+      const warningMessages = [];
+      if (selector.selector.trim() === "") {
+        node = relativeElement ?? document.documentElement;
+      } else if (selector.type === "xpath") {
+        if (!relativeElement && !selector.selector.startsWith("/")) {
+          selector.selector = `/${selector.selector}`;
+        }
+        const nodes = document.evaluate(selector.selector, relativeElement ? relativeElement : document.documentElement, null, XPathResult.ORDERED_NODE_SNAPSHOT_TYPE, null);
+        if (nodes.snapshotLength > 0) {
+          if (nodes.snapshotLength > 1) {
+            warningMessages.push(generateMultiValueWarningMessage({
+              propertyName,
+              nodesLength: nodes.snapshotLength
+            }));
+          }
+          node = nodes.snapshotItem(0);
+        } else {
+          return {
+            value: null,
+            warnings: warningMessages
+          };
+        }
+      } else {
+        const nodes = relativeElement ? relativeElement.querySelectorAll(selector.selector) : document.querySelectorAll(selector.selector);
+        if (nodes.length > 0) {
+          if (nodes.length > 1) {
+            warningMessages.push(generateMultiValueWarningMessage({
+              propertyName,
+              nodesLength: nodes.length
+            }));
+          }
+          node = nodes[0];
+        } else {
+          return {
+            value: null,
+            warnings: warningMessages
+          };
+        }
+      }
+      if (!node) return {
+        value: null,
+        warnings: warningMessages
+      };
+      const result = window["getValueUsingNodeAndSelector"]({
+        selector,
+        element: node
+      });
+      if (!result) return {
+        value: null,
+        warnings: warningMessages
+      };
+      return {
+        value: window["cleanValue"](selector.regex, result),
+        warnings: warningMessages
+      };
+    };
+    window["getMultiValueUsingSelector"] = ({
+      selector,
+      relativeElement
+    }) => {
+      let nodes = [];
+      if (selector.selector.trim() === "") {
+        nodes = [relativeElement ?? document.documentElement];
+      } else if (selector.type === "xpath") {
+        if (!relativeElement && !selector.selector.startsWith("/")) {
+          selector.selector = `/${selector.selector}`;
+        }
+        const nodesSnapshot = document.evaluate(selector.selector, relativeElement ? relativeElement : document.documentElement, null, XPathResult.ORDERED_NODE_SNAPSHOT_TYPE, null);
+        for (let i = 0; i < nodesSnapshot.snapshotLength; i++) {
+          const node = nodesSnapshot.snapshotItem(i);
+          node && nodes.push(node);
+        }
+      } else {
+        nodes = Array.from(relativeElement ? relativeElement.querySelectorAll(selector.selector) : document.querySelectorAll(selector.selector));
+      }
+      return nodes.map(node => window["getValueUsingNodeAndSelector"]({
+        selector,
+        element: node
+      })).map(result => window["cleanValue"](selector.regex, result)).filter(v => v !== null && v !== undefined);
+    };
+  });
+}
+async function splitContainerIntoListLocators(locator) {
+  const childrenLocators = await locator.locator(":scope > *").all();
+  return childrenLocators;
+}
+function compressStringSpaces(str) {
+  return str.replace(/\s+/g, " ").trim();
+}
+async function selectArrayValuesUsingRelativeSelector(locators, selectorInfo) {
+  const extractorFn = selectorInfo.multiValue ? selectMultiValueUsingRelativeSelector : selectValueUsingRelativeSelector;
+  const listValuesPromises = locators.map(locator => {
+    return extractorFn(locator, selectorInfo);
+  });
+  const listValues = await Promise.all(listValuesPromises);
+  const someValidValues = listValues.some(value => !!value);
+  if (someValidValues) return listValues;
+  return locators.map(() => null);
+}
+async function selectMultiValueUsingRelativeSelector(locator, valueSelector) {
+  await injectExtractorsHelperFunctions(locator.page());
+  return locator.evaluate((element, valueSelector) => {
+    return window["getMultiValueUsingSelector"]({
+      relativeElement: element,
+      selector: valueSelector
+    });
+  }, valueSelector);
+}
+async function selectValueUsingRelativeSelector(locator, valueSelector) {
+  const page = locator.page();
+  await injectExtractorsHelperFunctions(page);
+  return locator.evaluate((element, valueSelector) => {
+    return window["getValueUsingSelector"]({
+      relativeElement: element,
+      selector: valueSelector
+    }).value;
+  }, valueSelector);
+}
+async function selectLocatorsUsingXpath(page, xpath) {
+  const trimmed = xpath.replace(/^(\/+)/, "");
+  return await page.locator(`//${trimmed}`).all();
+}