@intuned/browser-dev 0.1.4-dev.0

package/dist/optimized-extractors/listExtractionHelpers/__tests__/testArrayExtractorFromPage.spec.js

@@ -0,0 +1,130 @@
+"use strict";
+
+var _extendedTest = require("../../../common/extendedTest");
+var _ = require("../..");
+var _uuid = require("uuid");
+const productListTemplate = `
+<div class="products-container">
+  <div class="product-item">
+    <h2 class="product-title">iPhone 14 Pro</h2>
+    <div class="price-wrapper">
+      <span class="price">$999</span>
+    </div>
+    <div class="details">
+      <p class="product-description">Latest iPhone with advanced camera system</p>
+    </div>
+  </div>
+  <div class="product-item">
+    <h2 class="product-title">MacBook Air M2</h2>
+    <div class="price-wrapper">
+      <span class="price">$1199</span>
+    </div>
+    <div class="details">
+      <p class="product-description">Thin and light laptop with M2 chip</p>
+    </div>
+  </div>
+  <div class="product-item">
+    <h2 class="product-title">AirPods Pro</h2>
+    <div class="price-wrapper">
+      <span class="price">$249</span>
+    </div>
+    <div class="details">
+      <p class="product-description">Active noise cancellation earbuds</p>
+    </div>
+  </div>
+  <div class="additional-info">
+    <div class="shipping-notice">Free shipping on all orders</div>
+    <div class="return-policy">30-day return policy</div>
+  </div>
+</div>
+`;
+_extendedTest.describe.skip("Array Extractor from Page Caching Tests", () => {
+  (0, _extendedTest.describe)("DOM Changes and Cache Behavior", () => {
+    (0, _extendedTest.test)("should demonstrate caching behavior with different types of DOM changes", async ({
+      page
+    }) => {
+      const testLabel = `product-list-page-cache-test-${(0, _uuid.v4)()}`;
+      const variantKey = testLabel;
+      const itemEntitySchema = {
+        type: "object",
+        required: ["title", "price"],
+        properties: {
+          title: {
+            type: "string",
+            description: "Product title",
+            primary: true
+          },
+          price: {
+            type: "string",
+            description: "Product price"
+          },
+          description: {
+            type: "string",
+            description: "Product description"
+          }
+        }
+      };
+      const extractionOptions = {
+        itemEntityName: "product",
+        label: testLabel,
+        itemEntitySchema,
+        strategy: {
+          model: "claude-3-5-sonnet-20240620",
+          type: "HTML"
+        },
+        variantKey: variantKey,
+        apiKey: process.env.ANTHROPIC_API_KEY
+      };
+      await page.setContent(productListTemplate);
+      const firstResult = await (0, _.extractArrayFromPage)(page, extractionOptions);
+      console.log("First extraction result:", firstResult);
+      (0, _extendedTest.expect)(firstResult).toHaveLength(3);
+      (0, _extendedTest.expect)(firstResult[0]).toHaveProperty("title", "iPhone 14 Pro");
+      (0, _extendedTest.expect)(firstResult[0]).toHaveProperty("price", "$999");
+      (0, _extendedTest.expect)(firstResult[1]).toHaveProperty("title", "MacBook Air M2");
+      (0, _extendedTest.expect)(firstResult[1]).toHaveProperty("price", "$1199");
+      (0, _extendedTest.expect)(firstResult[2]).toHaveProperty("title", "AirPods Pro");
+      (0, _extendedTest.expect)(firstResult[2]).toHaveProperty("price", "$249");
+      const modifiedTemplate = productListTemplate.replace("iPhone 14 Pro", "iPhone 15 Pro Max").replace("$999", "$1099").replace("MacBook Air M2", "MacBook Pro M3").replace("$1199", "$1999");
+      await page.setContent(modifiedTemplate);
+      const secondResult = await (0, _.extractArrayFromPage)(page, extractionOptions);
+      console.log("Second extraction result (after relevant change):", secondResult);
+      (0, _extendedTest.expect)(secondResult).not.toEqual(firstResult);
+      (0, _extendedTest.expect)(secondResult).toHaveLength(3);
+      (0, _extendedTest.expect)(secondResult[0]).toHaveProperty("title", "iPhone 15 Pro Max");
+      (0, _extendedTest.expect)(secondResult[0]).toHaveProperty("price", "$1099");
+      (0, _extendedTest.expect)(secondResult[1]).toHaveProperty("title", "MacBook Pro M3");
+      (0, _extendedTest.expect)(secondResult[1]).toHaveProperty("price", "$1999");
+      (0, _extendedTest.expect)(secondResult[2]).toHaveProperty("title", "AirPods Pro");
+      (0, _extendedTest.expect)(secondResult[2]).toHaveProperty("price", "$249");
+      const irrelevantChangeTemplate = modifiedTemplate.replace("Free shipping on all orders", "Express shipping available").replace("30-day return policy", "60-day return policy");
+      await page.setContent(irrelevantChangeTemplate);
+      const thirdResult = await (0, _.extractArrayFromPage)(page, extractionOptions);
+      console.log("Third extraction result (after irrelevant change):", thirdResult);
+      (0, _extendedTest.expect)(thirdResult).toEqual(secondResult);
+      (0, _extendedTest.expect)(thirdResult).toHaveLength(3);
+      (0, _extendedTest.expect)(thirdResult[0]).toHaveProperty("title", "iPhone 15 Pro Max");
+      (0, _extendedTest.expect)(thirdResult[0]).toHaveProperty("price", "$1099");
+      const appendedTemplate = irrelevantChangeTemplate + `
+      <div class="newly-added-section">
+        <div class="customer-service">
+          <h3>Customer Support</h3>
+          <p>24/7 support available</p>
+        </div>
+        <div class="social-media">
+          <button class="share-facebook">Share on Facebook</button>
+          <button class="share-twitter">Share on Twitter</button>
+        </div>
+      </div>
+      `;
+      await page.setContent(appendedTemplate);
+      const fourthResult = await (0, _.extractArrayFromPage)(page, extractionOptions);
+      console.log("Fourth extraction result (after appending content):", fourthResult);
+      (0, _extendedTest.expect)(fourthResult).toEqual(thirdResult);
+      (0, _extendedTest.expect)(fourthResult).toHaveLength(3);
+      (0, _extendedTest.expect)(fourthResult[0]).toHaveProperty("title", "iPhone 15 Pro Max");
+      (0, _extendedTest.expect)(fourthResult[0]).toHaveProperty("price", "$1099");
+      console.log("All cache behavior tests completed successfully!");
+    });
+  });
+});

package/dist/optimized-extractors/listExtractionHelpers/__tests__/verifyThatAllXpathsArePartOfSameArray.test.js

@@ -0,0 +1,9 @@
+"use strict";
+
+var _getListContainerXpath = require("../utils/getListContainerXpath");
+var _vitest = require("vitest");
+(0, _vitest.describe)("verifyThatAllXpathsArePartOfSameArray", () => {
+  (0, _vitest.it)("basic case", () => {
+    (0, _vitest.expect)((0, _getListContainerXpath.verifyThatAllXpathsArePartOfSameArray)(["html[1]/li[1]/article[1]/h3[1]/a[1]/@title", "html[1]/li[2]/article[1]/h3[1]/a[1]/@title", "html[1]/li[3]/article[1]/h3[1]/a[1]/@title", "html[1]/li[4]/article[1]/h3[1]/a[1]/@title"])).toBe(true);
+  });
+});

package/dist/optimized-extractors/listExtractionHelpers/dynamicListExtractor.js

@@ -0,0 +1,160 @@
+"use strict";
+
+Object.defineProperty(exports, "__esModule", {
+  value: true
+});
+exports.dynamicListExtractor = dynamicListExtractor;
+var _neverthrow = require("neverthrow");
+var _captureSnapshot = require("../objectExtractionHelpers/captureSnapshot");
+var _checksumUtils = require("../objectExtractionHelpers/checksumUtils");
+var _runAiExtraction = require("./runAiExtraction");
+var _validateOptions = require("./utils/validateOptions");
+var _cache = require("../../intunedServices/cache");
+var _xpathMapping = require("../../common/xpathMapping");
+var _Logger = require("../../common/Logger");
+async function dynamicListExtractor(page, identifier, options) {
+  const inputValidation = await (0, _validateOptions.validateDynamicListExtractorOptions)(page, identifier, options);
+  if (inputValidation.isErr()) {
+    return inputValidation;
+  }
+  const {
+    pageUrl,
+    searchRegion,
+    searchRegionHandler,
+    invalidate: _invalidate,
+    itemEntityName,
+    itemEntitySchema,
+    variantKey,
+    primaryProperty,
+    hasSearchRegionContainer,
+    label: _label,
+    searchRegionKey,
+    searchRegionXpath: _searchRegionXpath,
+    strategy,
+    prompt,
+    apiKey
+  } = inputValidation.value;
+  const extractorInputHash = (0, _checksumUtils.hashObject)({
+    itemEntityName,
+    itemEntitySchema,
+    variantKey,
+    currentPageUrl: pageUrl,
+    strategy,
+    searchRegionKey,
+    prompt
+  }, true);
+  const pageAndSearchRegion = {
+    page,
+    searchRegion,
+    searchRegionHandler: searchRegionHandler
+  };
+  const snapshot = await (0, _captureSnapshot.captureSnapshot)(pageAndSearchRegion);
+  if (snapshot.isErr()) {
+    return (0, _neverthrow.err)(snapshot.error);
+  }
+  _Logger.logger.info("Looking for value in the cache...");
+  const cachedResult = await _cache.cache.get(extractorInputHash);
+  if (cachedResult) {
+    _Logger.logger.info("Found value in cache");
+    if (cachedResult.exceedsLimit) {
+      _Logger.logger.warn(`Cache key ${extractorInputHash} exceeds cache limit and is not cacheable`);
+    } else {
+      const isValid = await (0, _xpathMapping.validateXPathMapping)(page, cachedResult.matchesMapping, cachedResult.containerPath);
+      if (isValid) {
+        const nonRelatedChildrenCount = cachedResult.nonRelatedChildrenCount;
+        const currentChildrenCount = await page.evaluate(fullContainerXpath => {
+          if (!fullContainerXpath) return 0;
+          const containerResult = document.evaluate(fullContainerXpath, document, null, XPathResult.FIRST_ORDERED_NODE_TYPE, null);
+          const containerElement = containerResult.singleNodeValue;
+          if (!containerElement) return 0;
+          return containerElement.children.length;
+        }, cachedResult.fullContainerXpath);
+        if (currentChildrenCount - nonRelatedChildrenCount === cachedResult.result.length) {
+          _Logger.logger.info("The values in the cache are the same as the current page, returning the cached result");
+          return (0, _neverthrow.ok)(cachedResult.result);
+        }
+        _Logger.logger.info("The values in the cache are different from the current page, running AI extraction");
+      }
+      _Logger.logger.info("The values in the cache are different from the current page, running AI extraction");
+    }
+  } else {
+    _Logger.logger.info("No value found in the cache, running AI extraction");
+  }
+  const aiExtractionResult = await (0, _runAiExtraction.runAiExtraction)({
+    pageAndSearchRegion,
+    itemEntityName,
+    itemEntitySchema,
+    primaryProperty,
+    hasSearchRegionContainer,
+    strategy,
+    identifier,
+    prompt,
+    examples: [],
+    apiKey
+  });
+  if (aiExtractionResult.isErr()) {
+    return (0, _neverthrow.err)(aiExtractionResult.error);
+  }
+  const xpathsMapping = buildXpathsMapping(aiExtractionResult.value);
+  const resultsToReturn = getResultToReturn(aiExtractionResult.value.resultValues);
+  const allContainerChildrenCount = await page.evaluate(fullContainerXpath => {
+    if (!fullContainerXpath) return 0;
+    const containerResult = document.evaluate(fullContainerXpath, document, null, XPathResult.FIRST_ORDERED_NODE_TYPE, null);
+    const containerElement = containerResult.singleNodeValue;
+    if (!containerElement) return 0;
+    return containerElement.children.length;
+  }, aiExtractionResult.value.fullContainerXpath);
+  const nonRelatedChildrenCount = allContainerChildrenCount - resultsToReturn.length;
+  const resultsToCache = {
+    result: resultsToReturn,
+    matchesMapping: xpathsMapping,
+    containerPath: aiExtractionResult.value.containerPath,
+    fullContainerXpath: aiExtractionResult.value.fullContainerXpath,
+    nonRelatedChildrenCount
+  };
+  const cacheDataSize = JSON.stringify(resultsToCache).length;
+  const CACHE_SIZE_LIMIT = 380 * 1024;
+  if (cacheDataSize > CACHE_SIZE_LIMIT) {
+    _Logger.logger.warn(`Results exceed cache limit (${cacheDataSize} bytes > ${CACHE_SIZE_LIMIT} bytes), skipping caching`);
+    await _cache.cache.set(extractorInputHash, {
+      exceedsLimit: true
+    });
+  } else {
+    _Logger.logger.debug("Caching results...");
+    await _cache.cache.set(extractorInputHash, resultsToCache);
+    _Logger.logger.debug("Results cached");
+  }
+  return (0, _neverthrow.ok)(resultsToReturn);
+}
+function buildXpathsMapping(results) {
+  const containerXpath = results.containerPath;
+  const xpathsMapping = {};
+  for (const result of results.resultValues) {
+    for (const [_key, valueObj] of Object.entries(result.result)) {
+      const value = valueObj;
+      const matchedXpath = value.matchXpath;
+      if (matchedXpath) {
+        const relativePath = matchedXpath.replace(containerXpath + "/", "");
+        const xpathEntry = {
+          xpath: relativePath,
+          matchType: value.matchType
+        };
+        if (!xpathsMapping[value.matchText]) {
+          xpathsMapping[value.matchText] = [];
+        }
+        xpathsMapping[value.matchText].push(xpathEntry);
+      }
+    }
+  }
+  return xpathsMapping;
+}
+function getResultToReturn(extractionResult) {
+  return extractionResult.map(result => {
+    const transformedResult = {};
+    for (const [key, valueObj] of Object.entries(result.result)) {
+      const value = valueObj;
+      transformedResult[key] = value.matchText;
+    }
+    return transformedResult;
+  });
+}

package/dist/optimized-extractors/listExtractionHelpers/errors.js

@@ -0,0 +1,46 @@
+"use strict";
+
+Object.defineProperty(exports, "__esModule", {
+  value: true
+});
+exports.requiredPropertyNotExtracted = exports.other = exports.invalidSearchRegion = exports.invalidList = exports.invalidInput = exports.invalidExtractionResult = exports.invalidAddressUrl = exports.insufficientAiCredits = void 0;
+const other = (context, error) => ({
+  type: "Other",
+  context,
+  error
+});
+exports.other = other;
+const invalidSearchRegion = () => ({
+  type: "InvalidSearchRegion"
+});
+exports.invalidSearchRegion = invalidSearchRegion;
+const invalidList = () => ({
+  type: "InvalidList",
+  context: "Failed to get container path, please try to choose another primary property and make sure all list items are direct child of a common container"
+});
+exports.invalidList = invalidList;
+const invalidInput = context => ({
+  type: "InvalidInput",
+  context
+});
+exports.invalidInput = invalidInput;
+const invalidExtractionResult = context => ({
+  type: "InvalidExtractionResult",
+  context
+});
+exports.invalidExtractionResult = invalidExtractionResult;
+const invalidAddressUrl = context => ({
+  type: "InvalidAddressUrl",
+  context
+});
+exports.invalidAddressUrl = invalidAddressUrl;
+const requiredPropertyNotExtracted = context => ({
+  type: "RequiredPropertyNotExtracted",
+  context
+});
+exports.requiredPropertyNotExtracted = requiredPropertyNotExtracted;
+const insufficientAiCredits = context => ({
+  type: "InsufficientAiCredits",
+  context
+});
+exports.insufficientAiCredits = insufficientAiCredits;

package/dist/optimized-extractors/listExtractionHelpers/getListMatches.js

@@ -0,0 +1,14 @@
+"use strict";
+
+Object.defineProperty(exports, "__esModule", {
+  value: true
+});
+exports.getListMatches = getListMatches;
+var _noEmpty = _interopRequireDefault(require("../../common/noEmpty"));
+var _findDomMatches = require("../objectExtractionHelpers/findDomMatches");
+function _interopRequireDefault(e) { return e && e.__esModule ? e : { default: e }; }
+async function getListMatches(page, searchRegionHandler, results) {
+  const allExtractedValues = results.map(v => Object.values(v.result)).filter(_noEmpty.default).flat();
+  const allExtractedValuesDomMatches = await (0, _findDomMatches.getDomMatches)(page, searchRegionHandler, allExtractedValues);
+  return allExtractedValuesDomMatches;
+}

package/dist/optimized-extractors/listExtractionHelpers/runAiExtraction.js

@@ -0,0 +1,243 @@
+"use strict";
+
+Object.defineProperty(exports, "__esModule", {
+  value: true
+});
+exports.runAiExtraction = runAiExtraction;
+var _neverthrow = require("neverthrow");
+var _extractionHelpers = require("../../common/extractionHelpers");
+var _findDomMatches = require("../objectExtractionHelpers/findDomMatches");
+var _getSimplifiedHtml = require("../objectExtractionHelpers/getSimplifiedHtml");
+var Errors = _interopRequireWildcard(require("./errors"));
+var _getListMatches = require("./getListMatches");
+var _extractPropertiesUsingGPTFromArray = require("./utils/extractPropertiesUsingGPTFromArray");
+var _extractStructuredListUsingAi = require("./utils/extractStructuredListUsingAi");
+var _getListContainerXpath = require("./utils/getListContainerXpath");
+var _getRelativeContainerXpathSelector = require("./utils/getRelativeContainerXpathSelector");
+var _getSimplifiedHtmlPerListItem = require("./utils/getSimplifiedHtmlPerListItem");
+var _tablesUtils = require("./utils/tablesUtils");
+var _buildImagesFromPage = require("../common/buildImagesFromPage");
+var _findTableHeaders = require("../common/findTableHeaders");
+var _Logger = require("../../common/Logger");
+var _utils = require("../common/matching/utils");
+function _interopRequireWildcard(e, t) { if ("function" == typeof WeakMap) var r = new WeakMap(), n = new WeakMap(); return (_interopRequireWildcard = function (e, t) { if (!t && e && e.__esModule) return e; var o, i, f = { __proto__: null, default: e }; if (null === e || "object" != typeof e && "function" != typeof e) return f; if (o = t ? n : r) { if (o.has(e)) return o.get(e); o.set(e, f); } for (const t in e) "default" !== t && {}.hasOwnProperty.call(e, t) && ((i = (o = Object.defineProperty) && Object.getOwnPropertyDescriptor(e, t)) && (i.get || i.set) ? o(f, t, i) : f[t] = e[t]); return f; })(e, t); }
+async function runAiExtraction(params) {
+  return handleNewAiExtraction(params);
+}
+async function handleNewAiExtraction(params) {
+  var _primaryData$value;
+  const {
+    itemEntityName,
+    itemEntitySchema,
+    pageAndSearchRegion,
+    primaryProperty,
+    hasSearchRegionContainer,
+    strategy,
+    identifier,
+    prompt,
+    examples,
+    apiKey
+  } = params;
+  const [primaryPropertyName, primaryPropertyValue] = primaryProperty;
+  const aiExtractionData = await getAiExtractionData(strategy, pageAndSearchRegion, hasSearchRegionContainer);
+  if (aiExtractionData.isErr()) return (0, _neverthrow.err)(aiExtractionData.error);
+  const primaryData = await (0, _extractStructuredListUsingAi.extractStructuredListUsingAi)(itemEntityName, {
+    type: "object",
+    properties: {
+      [primaryPropertyName]: primaryPropertyValue
+    },
+    required: [primaryPropertyName]
+  }, aiExtractionData.value, identifier, strategy, prompt, apiKey);
+  if (primaryData.isErr()) {
+    return (0, _neverthrow.err)(primaryData.error);
+  }
+  if (primaryData.value.length === 0) {
+    _Logger.logger.debug(`the ai couldn't find any item with the ${primaryPropertyName} property`);
+    return (0, _neverthrow.ok)({
+      resultValues: [],
+      containerPath: null,
+      fullContainerXpath: null,
+      matches: new Map()
+    });
+  }
+  const primaryValues = (_primaryData$value = primaryData.value) === null || _primaryData$value === void 0 ? void 0 : _primaryData$value.map(i => i[primaryPropertyName]);
+  _Logger.logger.debug(`we were able to detect ${primaryValues.length} items with ${primaryPropertyName} property: ${JSON.stringify(primaryValues)}`);
+  if (primaryValues.length < 2) {
+    const allData = await (0, _extractStructuredListUsingAi.extractStructuredListUsingAi)(itemEntityName, itemEntitySchema, aiExtractionData.value, identifier, strategy, prompt, apiKey);
+    if (allData.isErr()) {
+      return (0, _neverthrow.err)(allData.error);
+    }
+    allData.value.forEach((v, i) => {
+      _Logger.logger.debug(`ai extraction result for row ${i}: ${JSON.stringify(v)}`);
+    });
+    const aiResults = allData.value.map((v, i) => ({
+      result: v,
+      rowIndex: i
+    }));
+    const resultValues = aiResults;
+    const matches = await (0, _getListMatches.getListMatches)(pageAndSearchRegion.page, pageAndSearchRegion.searchRegionHandler, aiResults);
+    return (0, _neverthrow.ok)({
+      resultValues,
+      containerPath: null,
+      fullContainerXpath: null,
+      matches
+    });
+  }
+  const primaryValuesDomMatches = await (0, _findDomMatches.getDomMatches)(pageAndSearchRegion.page, pageAndSearchRegion.searchRegionHandler, primaryValues);
+  const valuesDoesNotExistInDOM = Array.from(primaryValuesDomMatches.entries()).filter(([_, v]) => v.length === 0).map(i => i[0]);
+  if (valuesDoesNotExistInDOM.length > 0) {
+    _Logger.logger.debug(`the following values returned by AI does not exist in the page dom, [${valuesDoesNotExistInDOM}] , this will cause issues finding the list container`);
+  }
+  const fullContainerXpath = await (0, _getListContainerXpath.getListContainerXpath)(primaryValuesDomMatches);
+  let containerPath = fullContainerXpath;
+  if (hasSearchRegionContainer && containerPath) {
+    containerPath = await (0, _getRelativeContainerXpathSelector.getRelativeContainerXpathSelector)(pageAndSearchRegion.searchRegion, containerPath);
+  }
+  if (!containerPath) {
+    return (0, _neverthrow.err)(Errors.invalidList());
+  }
+  const listItemsContainerLocator = (await (0, _extractionHelpers.selectLocatorsUsingXpath)(pageAndSearchRegion.page, fullContainerXpath))[0];
+  const extractedData = await splitDomAndExtractData({
+    listItemsContainerLocator,
+    itemEntityName,
+    itemEntitySchema,
+    pageAndSearchRegion,
+    primaryPropertyName: primaryProperty[0],
+    strategy,
+    identifier,
+    examples,
+    apiKey
+  });
+  if (extractedData.isErr()) {
+    return (0, _neverthrow.err)(extractedData.error);
+  }
+  const {
+    matches,
+    resultValues
+  } = extractedData.value;
+  return (0, _neverthrow.ok)({
+    resultValues,
+    containerPath,
+    fullContainerXpath,
+    matches
+  });
+}
+async function buildImagesForItemsHandles(locators) {
+  const images = [];
+  for (const locator of locators) {
+    const elementHandle = await locator.elementHandle();
+    const screenshot = await elementHandle.screenshot({
+      type: "png"
+    });
+    images.push(screenshot);
+  }
+  return images.map(i => ({
+    type: "image",
+    buffer: i
+  }));
+}
+async function splitDomAndExtractData({
+  listItemsContainerLocator,
+  itemEntityName,
+  itemEntitySchema,
+  pageAndSearchRegion,
+  primaryPropertyName,
+  strategy,
+  identifier,
+  examples,
+  apiKey
+}) {
+  const itemsLocators = await (0, _extractionHelpers.splitContainerIntoListLocators)(listItemsContainerLocator);
+  const itemsSimplifiedHtml = await (0, _getSimplifiedHtmlPerListItem.getSimplifiedHtmlPerListItem)(itemsLocators);
+  const {
+    isTable,
+    tableLocater
+  } = await (0, _tablesUtils.isListTable)(listItemsContainerLocator, itemsSimplifiedHtml);
+  const tableAsJsonArray = isTable ? await (0, _tablesUtils.createJsonFromTable)(pageAndSearchRegion.page) : [];
+  const tableHeaders = tableLocater ? await (0, _findTableHeaders.getTableHeadersUsingAi)(tableLocater, identifier) : undefined;
+  if (tableHeaders && tableHeaders.isErr()) {
+    return (0, _neverthrow.err)(tableHeaders.error);
+  }
+  const extractedData = await (0, _extractPropertiesUsingGPTFromArray.extractPropertiesUsingGPTFromArray)({
+    itemEntityName,
+    itemEntitySchema,
+    itemsSimplifiedHtml,
+    tableAsJsonArray,
+    strategy,
+    tableHeaders: tableHeaders !== null && tableHeaders !== void 0 && tableHeaders.value.headers.length ? tableHeaders.value.headers : undefined,
+    items: strategy.type === "HTML" ? itemsSimplifiedHtml.map(v => ({
+      type: "text",
+      text: v
+    })) : await buildImagesForItemsHandles(itemsLocators),
+    identifier,
+    examples,
+    apiKey
+  });
+  if (extractedData.isErr()) {
+    return extractedData;
+  }
+  const resultValues = [];
+  for (let i = 0; i < extractedData.value.length; i++) {
+    const rowValues = extractedData.value[i] ?? {};
+    const rowLocator = itemsLocators[i];
+    const primaryValue = rowValues[primaryPropertyName];
+    if (primaryValue === null || primaryValue === undefined) {
+      continue;
+    }
+    const rowValuesMatches = await (0, _findDomMatches.getDomMatchesFromItemsHandles)(pageAndSearchRegion.page, await rowLocator.elementHandle(), Object.entries(rowValues).map(([_, value]) => value));
+    const matches = rowValuesMatches.get(primaryValue);
+    const rowValuesWithMatchesOnly = Object.entries(rowValues).reduce((acc, [key, value]) => {
+      const valueMatches = rowValuesMatches.get(value);
+      const bestMatch = (0, _utils.selectBestMatch)(value, valueMatches ?? []);
+      if (valueMatches && valueMatches.length > 0 && bestMatch) {
+        acc[key] = {
+          matchText: bestMatch.matchText,
+          matchXpath: bestMatch.matchXpath,
+          matchType: bestMatch.matchType
+        };
+      } else {
+        _Logger.logger.debug(`value "${value}" for key "${key}" in row ${i + 1} does not have any matches in the item's html, dropped for hallucination protection`);
+      }
+      return acc;
+    }, {});
+    if (matches && (matches === null || matches === void 0 ? void 0 : matches.length) > 0) {
+      resultValues.push({
+        rowIndex: i,
+        result: rowValuesWithMatchesOnly
+      });
+    }
+  }
+  const propertyKeys = Object.keys(itemEntitySchema.properties);
+  for (const propertyKey of propertyKeys) {
+    const isRequired = itemEntitySchema.required.includes(propertyKey);
+    const isPrimary = itemEntitySchema.properties[propertyKey].primary;
+    if (!isPrimary && isRequired && resultValues.some(i => {
+      const doenstExist = i.result[propertyKey] === null || i.result[propertyKey] === undefined;
+      return doenstExist;
+    })) {
+      return (0, _neverthrow.err)(Errors.invalidExtractionResult(`Required property ${propertyKey} not found in all rows`));
+    }
+  }
+  const matches = await (0, _getListMatches.getListMatches)(pageAndSearchRegion.page, pageAndSearchRegion.searchRegionHandler, resultValues.map(v => {
+    return {
+      rowIndex: v.rowIndex,
+      result: Object.fromEntries(Object.entries(v.result).map(([key, value]) => [key, value.matchText]))
+    };
+  }));
+  return (0, _neverthrow.ok)({
+    resultValues,
+    matches
+  });
+}
+async function getAiExtractionData(strategy, pageAndSearchRegion, hasSearchRegionContainer) {
+  if (strategy.type === "HTML") {
+    return (0, _neverthrow.ok)({
+      text: await (0, _getSimplifiedHtml.getSimplifiedHtml)(pageAndSearchRegion.searchRegionHandler)
+    });
+  }
+  const images = await (0, _buildImagesFromPage.buildImagesFromPageOrHandle)(pageAndSearchRegion.page, hasSearchRegionContainer ? pageAndSearchRegion.searchRegionHandler : undefined);
+  if (images.isErr()) return (0, _neverthrow.err)(Errors.other(images.error.context));
+  return (0, _neverthrow.ok)({
+    images: images.value
+  });
+}

package/dist/optimized-extractors/listExtractionHelpers/typesAndSchema.js

@@ -0,0 +1,5 @@
+"use strict";
+
+Object.defineProperty(exports, "__esModule", {
+  value: true
+});