@intuned/browser-dev 2.2.3-test-build.0

package/dist/optimized-extractors/listExtractionHelpers/runAiExtraction.js

@@ -0,0 +1,240 @@
+"use strict";
+
+Object.defineProperty(exports, "__esModule", {
+  value: true
+});
+exports.runAiExtraction = runAiExtraction;
+var _neverthrow = require("neverthrow");
+var _extractionHelpers = require("../../common/extractionHelpers");
+var _findDomMatches = require("../objectExtractionHelpers/findDomMatches");
+var _getSimplifiedHtml = require("../objectExtractionHelpers/getSimplifiedHtml");
+var Errors = _interopRequireWildcard(require("./errors"));
+var _getListMatches = require("./getListMatches");
+var _extractPropertiesUsingGPTFromArray = require("./utils/extractPropertiesUsingGPTFromArray");
+var _extractStructuredListUsingAi = require("./utils/extractStructuredListUsingAi");
+var _getListContainerXpath = require("./utils/getListContainerXpath");
+var _getRelativeContainerXpathSelector = require("./utils/getRelativeContainerXpathSelector");
+var _getSimplifiedHtmlPerListItem = require("./utils/getSimplifiedHtmlPerListItem");
+var _tablesUtils = require("./utils/tablesUtils");
+var _buildImagesFromPage = require("../common/buildImagesFromPage");
+var _findTableHeaders = require("../common/findTableHeaders");
+var _Logger = require("../../common/Logger");
+var _utils = require("../common/matching/utils");
+function _interopRequireWildcard(e, t) { if ("function" == typeof WeakMap) var r = new WeakMap(), n = new WeakMap(); return (_interopRequireWildcard = function (e, t) { if (!t && e && e.__esModule) return e; var o, i, f = { __proto__: null, default: e }; if (null === e || "object" != typeof e && "function" != typeof e) return f; if (o = t ? n : r) { if (o.has(e)) return o.get(e); o.set(e, f); } for (const t in e) "default" !== t && {}.hasOwnProperty.call(e, t) && ((i = (o = Object.defineProperty) && Object.getOwnPropertyDescriptor(e, t)) && (i.get || i.set) ? o(f, t, i) : f[t] = e[t]); return f; })(e, t); }
+async function runAiExtraction(params) {
+  return handleNewAiExtraction(params);
+}
+async function handleNewAiExtraction(params) {
+  var _primaryData$value;
+  const {
+    itemEntityName,
+    itemEntitySchema,
+    pageAndSearchRegion,
+    primaryProperty,
+    hasSearchRegionContainer,
+    strategy,
+    identifier,
+    prompt,
+    examples,
+    apiKey
+  } = params;
+  const [primaryPropertyName, primaryPropertyValue] = primaryProperty;
+  const aiExtractionData = await getAiExtractionData(strategy, pageAndSearchRegion, hasSearchRegionContainer);
+  if (aiExtractionData.isErr()) return (0, _neverthrow.err)(aiExtractionData.error);
+  const primaryData = await (0, _extractStructuredListUsingAi.extractStructuredListUsingAi)(itemEntityName, {
+    type: "object",
+    properties: {
+      [primaryPropertyName]: primaryPropertyValue
+    },
+    required: [primaryPropertyName]
+  }, aiExtractionData.value, identifier, strategy, prompt, apiKey);
+  if (primaryData.isErr()) {
+    return (0, _neverthrow.err)(primaryData.error);
+  }
+  if (primaryData.value.length === 0) {
+    _Logger.logger.debug(`the ai couldn't find any item with the ${primaryPropertyName} property`);
+    return (0, _neverthrow.ok)({
+      resultValues: [],
+      containerPath: null,
+      matches: new Map()
+    });
+  }
+  const primaryValues = (_primaryData$value = primaryData.value) === null || _primaryData$value === void 0 ? void 0 : _primaryData$value.map(i => i[primaryPropertyName]);
+  _Logger.logger.debug(`we were able to detect ${primaryValues.length} items with ${primaryPropertyName} property: ${JSON.stringify(primaryValues)}`);
+  if (primaryValues.length < 2) {
+    const allData = await (0, _extractStructuredListUsingAi.extractStructuredListUsingAi)(itemEntityName, itemEntitySchema, aiExtractionData.value, identifier, strategy, prompt, apiKey);
+    if (allData.isErr()) {
+      return (0, _neverthrow.err)(allData.error);
+    }
+    allData.value.forEach((v, i) => {
+      _Logger.logger.debug(`ai extraction result for row ${i}: ${JSON.stringify(v)}`);
+    });
+    const aiResults = allData.value.map((v, i) => ({
+      result: v,
+      rowIndex: i
+    }));
+    const resultValues = aiResults;
+    const matches = await (0, _getListMatches.getListMatches)(pageAndSearchRegion.page, pageAndSearchRegion.searchRegionHandler, aiResults);
+    return (0, _neverthrow.ok)({
+      resultValues,
+      containerPath: null,
+      matches
+    });
+  }
+  const primaryValuesDomMatches = await (0, _findDomMatches.getDomMatches)(pageAndSearchRegion.page, pageAndSearchRegion.searchRegionHandler, primaryValues);
+  const valuesDoesNotExistInDOM = Array.from(primaryValuesDomMatches.entries()).filter(([_, v]) => v.length === 0).map(i => i[0]);
+  if (valuesDoesNotExistInDOM.length > 0) {
+    _Logger.logger.debug(`the following values returned by AI does not exist in the page dom, [${valuesDoesNotExistInDOM}] , this will cause issues finding the list container`);
+  }
+  const fullContainerXpath = await (0, _getListContainerXpath.getListContainerXpath)(primaryValuesDomMatches);
+  let containerPath = fullContainerXpath;
+  if (hasSearchRegionContainer && containerPath) {
+    containerPath = await (0, _getRelativeContainerXpathSelector.getRelativeContainerXpathSelector)(pageAndSearchRegion.searchRegion, containerPath);
+  }
+  if (!containerPath) {
+    return (0, _neverthrow.err)(Errors.invalidList());
+  }
+  const listItemsContainerLocator = (await (0, _extractionHelpers.selectLocatorsUsingXpath)(pageAndSearchRegion.page, fullContainerXpath))[0];
+  const extractedData = await splitDomAndExtractData({
+    listItemsContainerLocator,
+    itemEntityName,
+    itemEntitySchema,
+    pageAndSearchRegion,
+    primaryPropertyName: primaryProperty[0],
+    strategy,
+    identifier,
+    examples,
+    apiKey
+  });
+  if (extractedData.isErr()) {
+    return (0, _neverthrow.err)(extractedData.error);
+  }
+  const {
+    matches,
+    resultValues
+  } = extractedData.value;
+  return (0, _neverthrow.ok)({
+    resultValues,
+    containerPath,
+    matches
+  });
+}
+async function buildImagesForItemsHandles(locators) {
+  const images = [];
+  for (const locator of locators) {
+    const elementHandle = await locator.elementHandle();
+    const screenshot = await elementHandle.screenshot({
+      type: "png"
+    });
+    images.push(screenshot);
+  }
+  return images.map(i => ({
+    type: "image",
+    buffer: i
+  }));
+}
+async function splitDomAndExtractData({
+  listItemsContainerLocator,
+  itemEntityName,
+  itemEntitySchema,
+  pageAndSearchRegion,
+  primaryPropertyName,
+  strategy,
+  identifier,
+  examples,
+  apiKey
+}) {
+  const itemsLocators = await (0, _extractionHelpers.splitContainerIntoListLocators)(listItemsContainerLocator);
+  const itemsSimplifiedHtml = await (0, _getSimplifiedHtmlPerListItem.getSimplifiedHtmlPerListItem)(itemsLocators);
+  const {
+    isTable,
+    tableLocater
+  } = await (0, _tablesUtils.isListTable)(listItemsContainerLocator, itemsSimplifiedHtml);
+  const tableAsJsonArray = isTable ? await (0, _tablesUtils.createJsonFromTable)(pageAndSearchRegion.page) : [];
+  const tableHeaders = tableLocater ? await (0, _findTableHeaders.getTableHeadersUsingAi)(tableLocater, identifier) : undefined;
+  if (tableHeaders && tableHeaders.isErr()) {
+    return (0, _neverthrow.err)(tableHeaders.error);
+  }
+  const extractedData = await (0, _extractPropertiesUsingGPTFromArray.extractPropertiesUsingGPTFromArray)({
+    itemEntityName,
+    itemEntitySchema,
+    itemsSimplifiedHtml,
+    tableAsJsonArray,
+    strategy,
+    tableHeaders: tableHeaders !== null && tableHeaders !== void 0 && tableHeaders.value.headers.length ? tableHeaders.value.headers : undefined,
+    items: strategy.type === "HTML" ? itemsSimplifiedHtml.map(v => ({
+      type: "text",
+      text: v
+    })) : await buildImagesForItemsHandles(itemsLocators),
+    identifier,
+    examples,
+    apiKey
+  });
+  if (extractedData.isErr()) {
+    return extractedData;
+  }
+  const resultValues = [];
+  for (let i = 0; i < extractedData.value.length; i++) {
+    const rowValues = extractedData.value[i] ?? {};
+    const rowLocator = itemsLocators[i];
+    const primaryValue = rowValues[primaryPropertyName];
+    if (primaryValue === null || primaryValue === undefined) {
+      continue;
+    }
+    const rowValuesMatches = await (0, _findDomMatches.getDomMatchesFromItemsHandles)(pageAndSearchRegion.page, await rowLocator.elementHandle(), Object.entries(rowValues).map(([_, value]) => value));
+    const matches = rowValuesMatches.get(primaryValue);
+    const rowValuesWithMatchesOnly = Object.entries(rowValues).reduce((acc, [key, value]) => {
+      const valueMatches = rowValuesMatches.get(value);
+      const bestMatch = (0, _utils.selectBestMatch)(value, valueMatches ?? []);
+      if (valueMatches && valueMatches.length > 0 && bestMatch) {
+        acc[key] = {
+          matchText: bestMatch.matchText,
+          matchXpath: bestMatch.matchXpath,
+          matchType: bestMatch.matchType
+        };
+      } else {
+        _Logger.logger.debug(`value "${value}" for key "${key}" in row ${i + 1} does not have any matches in the item's html, dropped for hallucination protection`);
+      }
+      return acc;
+    }, {});
+    if (matches && (matches === null || matches === void 0 ? void 0 : matches.length) > 0) {
+      resultValues.push({
+        rowIndex: i,
+        result: rowValuesWithMatchesOnly
+      });
+    }
+  }
+  const propertyKeys = Object.keys(itemEntitySchema.properties);
+  for (const propertyKey of propertyKeys) {
+    const isRequired = itemEntitySchema.required.includes(propertyKey);
+    const isPrimary = itemEntitySchema.properties[propertyKey].primary;
+    if (!isPrimary && isRequired && resultValues.some(i => {
+      const doenstExist = i.result[propertyKey] === null || i.result[propertyKey] === undefined;
+      return doenstExist;
+    })) {
+      return (0, _neverthrow.err)(Errors.invalidExtractionResult(`Required property ${propertyKey} not found in all rows`));
+    }
+  }
+  const matches = await (0, _getListMatches.getListMatches)(pageAndSearchRegion.page, pageAndSearchRegion.searchRegionHandler, resultValues.map(v => {
+    return {
+      rowIndex: v.rowIndex,
+      result: Object.fromEntries(Object.entries(v.result).map(([key, value]) => [key, value.matchText]))
+    };
+  }));
+  return (0, _neverthrow.ok)({
+    resultValues,
+    matches
+  });
+}
+async function getAiExtractionData(strategy, pageAndSearchRegion, hasSearchRegionContainer) {
+  if (strategy.type === "HTML") {
+    return (0, _neverthrow.ok)({
+      text: await (0, _getSimplifiedHtml.getSimplifiedHtml)(pageAndSearchRegion.searchRegionHandler)
+    });
+  }
+  const images = await (0, _buildImagesFromPage.buildImagesFromPageOrHandle)(pageAndSearchRegion.page, hasSearchRegionContainer ? pageAndSearchRegion.searchRegionHandler : undefined);
+  if (images.isErr()) return (0, _neverthrow.err)(Errors.other(images.error.context));
+  return (0, _neverthrow.ok)({
+    images: images.value
+  });
+}

package/dist/optimized-extractors/listExtractionHelpers/typesAndSchema.js

@@ -0,0 +1,5 @@
+"use strict";
+
+Object.defineProperty(exports, "__esModule", {
+  value: true
+});

package/dist/optimized-extractors/listExtractionHelpers/utils/extractPropertiesUsingGPTFromArray.js

@@ -0,0 +1,277 @@
+"use strict";
+
+Object.defineProperty(exports, "__esModule", {
+  value: true
+});
+exports.extractPropertiesUsingGPTFromArray = extractPropertiesUsingGPTFromArray;
+var _neverthrow = require("neverthrow");
+var _errors = require("../errors");
+var _common = require("../../common/");
+var _isTableHeaderOrFooter = require("../../common/isTableHeaderOrFooter");
+var _Logger = require("../../../common/Logger");
+var _extractionHelpers = require("../../../common/extractionHelpers");
+var _buildExamplesPrompt = require("../../common/buildExamplesPrompt");
+var _utils = require("../../common/utils");
+async function extractPropertiesUsingGPTFromArray(input) {
+  if (input.strategy.type === "IMAGE") {
+    return extractPropertiesWithImageStrategy(input);
+  } else if (input.strategy.type === "HTML") {
+    return extractPropertiesWithHTMLStrategy(input);
+  }
+  throw new Error("Invalid strategy type");
+}
+async function extractPropertiesUsingGPT({
+  itemEntityName,
+  itemEntitySchema,
+  text,
+  image,
+  extraRowDataFromTableRow,
+  strategy,
+  tableHeaders,
+  identifier,
+  possibleTableHeaderOrFooter,
+  examples,
+  apiKey
+}) {
+  const systemMessage = extraRowDataFromTableRow ? `You are a data extraction assistant, out of this text, you will be asked to extract some data. Be accurate, give complete results. If you cannot find the field data, use null. make sure to follow examples if provided. The data will be provided twice, once as a json that was extracted from an html table and once as the html for that row in the table.` : `You are a data extraction assistant, out of this text, you will be asked to extract some data. Be accurate, give complete results. If you cannot find the field data, use null. make sure to follow examples if provided.`;
+  const extraUserMessages = [];
+  if (examples.length > 0) {
+    const examplesMessage = (0, _buildExamplesPrompt.buildExamplesPrompt)({
+      entityName: itemEntityName,
+      examples
+    });
+    extraUserMessages.push(examplesMessage);
+  }
+  if (possibleTableHeaderOrFooter) {
+    const content = text ?? image;
+    const isHeader = await (0, _isTableHeaderOrFooter.isTableHeaderOrFooter)(content);
+    if (isHeader.isErr()) {
+      return (0, _neverthrow.err)(isHeader.error);
+    }
+    if (isHeader.value.isHeader) {
+      return (0, _neverthrow.ok)({});
+    }
+  }
+  if (tableHeaders) {
+    extraUserMessages.push(`this data are part of a table, the table headers in order are: ${tableHeaders === null || tableHeaders === void 0 ? void 0 : tableHeaders.join(", ")}`);
+  }
+  if (extraRowDataFromTableRow) {
+    extraUserMessages.push(JSON.stringify(extraRowDataFromTableRow));
+  }
+  const extractionResult = await (0, _common.extractStructuredDataUsingAi)({
+    entityName: itemEntityName,
+    model: strategy.model,
+    text: text ? [text] : undefined,
+    jsonSchema: itemEntitySchema,
+    systemMessage,
+    extraUserMessages,
+    images: image ? [{
+      data: image,
+      image_type: "png"
+    }] : [],
+    identifier,
+    apiKey
+  });
+  if (extractionResult.isErr()) {
+    if (extractionResult.error.type === "NoDataFound") {
+      return (0, _neverthrow.ok)({});
+    }
+    return (0, _neverthrow.err)((0, _errors.invalidExtractionResult)(extractionResult.error.context));
+  }
+  return (0, _neverthrow.ok)(extractionResult.value.result);
+}
+async function extractPropertiesWithImageStrategy({
+  identifier,
+  itemEntityName,
+  itemEntitySchema,
+  items,
+  strategy,
+  tableAsJsonArray,
+  tableHeaders,
+  examples: previousExamples,
+  apiKey
+}) {
+  const shouldUseTableData = !!tableAsJsonArray && tableAsJsonArray.length === items.length;
+  const CHUNK_SIZE = process.env.TEST === "true" ? 1 : 5;
+  const chunkedExtractPropertiesUsingGPTParameters = items.reduce((acc, item, index) => {
+    const chunkIndex = Math.floor(index / CHUNK_SIZE);
+    if (!acc[chunkIndex]) {
+      acc[chunkIndex] = [];
+    }
+    acc[chunkIndex].push({
+      item,
+      index
+    });
+    return acc;
+  }, []);
+  const result = [];
+  for (const chunk of chunkedExtractPropertiesUsingGPTParameters) {
+    const examples = (0, _utils.getRandomItems)([...result, ...previousExamples], 3);
+    const promises = chunk.map(args => extractPropertiesUsingGPT({
+      itemEntityName,
+      itemEntitySchema,
+      text: undefined,
+      image: args.item.type === "image" ? args.item.buffer : undefined,
+      extraRowDataFromTableRow: shouldUseTableData ? tableAsJsonArray[args.index] : undefined,
+      strategy,
+      tableHeaders,
+      identifier,
+      possibleTableHeaderOrFooter: args.index === 0 || args.index === items.length - 1,
+      examples,
+      apiKey
+    }));
+    const results = await Promise.all(promises);
+    const errorResult = results.find(r => r.isErr());
+    if (errorResult && errorResult.isErr()) {
+      return (0, _neverthrow.err)(errorResult.error);
+    }
+    const listOfResults = results.map(r => r._unsafeUnwrap());
+    listOfResults.forEach((r, i) => {
+      const itemIndexInFullList = i + result.length;
+      _Logger.logger.debug(`Extracted this info from array item #${itemIndexInFullList}: ${JSON.stringify(r)}`);
+    });
+    result.push(...listOfResults);
+  }
+  return (0, _neverthrow.ok)(result);
+}
+async function extractPropertiesWithHTMLStrategy({
+  identifier,
+  itemEntityName,
+  itemEntitySchema,
+  items,
+  strategy,
+  tableAsJsonArray,
+  tableHeaders,
+  examples: previousExamples,
+  apiKey
+}) {
+  const shouldUseTableData = !!tableAsJsonArray && tableAsJsonArray.length === items.length;
+  const isWeakModel = strategy.model === "claude-3-haiku" || strategy.model == "gpt3.5-turbo";
+  const averageItemLength = items.reduce((sum, item) => {
+    if (item.type !== "text") return sum;
+    return sum + (0, _extractionHelpers.compressStringSpaces)(item.text).length;
+  }, 0) / items.length;
+  const shouldUseSmallerChunkSize = isWeakModel && averageItemLength > 1000;
+  const CHUNK_SIZE = shouldUseSmallerChunkSize ? 3 : 10;
+  const itemsChunks = items.reduce((resultArray, item, index) => {
+    const chunkIndex = Math.floor(index / CHUNK_SIZE);
+    if (!resultArray[chunkIndex]) {
+      resultArray[chunkIndex] = [];
+    }
+    resultArray[chunkIndex].push({
+      index,
+      item
+    });
+    return resultArray;
+  }, []);
+  const results = [];
+  const executeChunk = async chunk => {
+    const examples = (0, _utils.getRandomItems)([...results, ...previousExamples], 3);
+    const texts = chunk.map(i => {
+      if (i.item.type !== "text") {
+        throw new Error("Invalid type");
+      }
+      return `<ITEM_INDEX_${i.index}>\n${i.item.text}\n</ITEM_INDEX_${i.index}>`;
+    }).join("\n");
+    const extraRowDataFromTableRow = shouldUseTableData ? chunk.map(c => tableAsJsonArray[c.index]) : undefined;
+    const shouldWorkaroundGoogleModel = (0, _common.isGoogleModel)(strategy.model);
+    const itemSchema = !shouldWorkaroundGoogleModel ? {
+      oneOf: [itemEntitySchema, {
+        type: "null"
+      }]
+    } : {
+      type: "object",
+      properties: {
+        value: itemEntitySchema,
+        _isNull: {
+          type: "boolean",
+          description: "If the value is supposed to be null, set this to true and do not provide a value. Otherwise, set it to false and provide the value."
+        }
+      },
+      required: ["_isNull"]
+    };
+    const itemNullPromptExample = !shouldWorkaroundGoogleModel ? "null" : `{ "_isNull": true }`;
+    const result = await extractPropertiesUsingGPT({
+      identifier,
+      text: texts,
+      examples,
+      itemEntityName,
+      tableHeaders,
+      extraRowDataFromTableRow,
+      apiKey,
+      itemEntitySchema: {
+        type: "array",
+        description: `Extracted ` + itemEntityName + ` items from the content wrapped with <LIST_ITEM_INDEX_X> and </LIST_ITEM_INDEX_X> tags. each tag represents an item on the list, you should return ` + chunk.length + ` items, one for each index. If the item boundaries do not contain a valid entity, return null for that item with the right index. YOU SHOULD INCLUDE ALL ITEMS IN THE RESPONSE, EVERY INDEX BOUNDARY SHOULD BE REPRESENTED BY AN ITEM. IF the index boundary does not include a valid item you should return null for it.` + ` AND YOU SHOULD ALWAYS HAVE THE RIGHT INDEX FOR THE MISSING ITEMS, **do not put it at the end of the list or change the order of that item if in the list **. if <LIST_ITEM_INDEX_33> and </LIST_ITEM_INDEX_33> does not wrap a valid item, the item you return should be ` + `{ index: 33, isTableHeaderOrFooter: false, item: ${itemNullPromptExample} } where ths index is the number in the tag that's missing the item, ` + ` each <LIST_ITEM_INDEX_X> and </LIST_ITEM_INDEX_X> boundary represents a single list item. do not return the same list item more than once.`,
+        minItems: chunk.length,
+        maxItems: chunk.length,
+        items: {
+          type: "object",
+          properties: {
+            index: {
+              type: "number",
+              enum: generateRange(chunk[0].index, chunk[chunk.length - 1].index),
+              minimum: chunk[0].index,
+              maximum: chunk[chunk.length - 1].index,
+              description: `Identify the index of an item from the list based on the prefix and suffix around the extracted data. if you extract data between <LIST_ITEM_INDEX_x> and </LIST_ITEM_INDEX_x>, the index should be the value of x. if <LIST_ITEM_INDEX_x> exists and does not have a valid data inside it you still should return an item with index x and value of null , getting the wrong index for the missing item will break the whole extraction. YOU SHOULD INCLUDE ALL ITEMS IN THE RESPONSE, EVERY INDEX BOUNDARY SHOULD BE REPRESENTED BY AN ITEM`
+            },
+            isTableHeaderOrFooter: {
+              type: "boolean",
+              description: `If the extracted data is a table header or footer, set this field to true. Otherwise, set it to false.`
+            },
+            item: itemSchema
+          }
+        }
+      },
+      image: undefined,
+      strategy
+    });
+    if (result.isErr()) {
+      return (0, _neverthrow.err)(result.error);
+    }
+    const resultUnWrapped = result.value;
+    function isItemRecord(item) {
+      return typeof item.value !== "object";
+    }
+    resultUnWrapped === null || resultUnWrapped === void 0 || resultUnWrapped.forEach(r => {
+      if (r.isTableHeaderOrFooter) {
+        _Logger.logger.debug(`skipping item at index #${r.index + 1}: ${JSON.stringify(r.item)}, it's detected as a table header or footer.`);
+        results[r.index] = {};
+        return;
+      }
+      const item = r.item;
+      let itemToUse;
+      if (item !== null && !isItemRecord(item)) {
+        itemToUse = item._isNull ? null : item.value;
+      } else {
+        itemToUse = item;
+      }
+      results[r.index] = itemToUse;
+      _Logger.logger.debug(`Extracted this info from array item #${r.index + 1}: ${JSON.stringify(itemToUse)}`);
+    });
+  };
+  const [firstChunk, ...otherChunks] = itemsChunks;
+  if (firstChunk) {
+    const firstChunkResult = await executeChunk(firstChunk);
+    if (firstChunkResult !== null && firstChunkResult !== void 0 && firstChunkResult.isErr()) {
+      return (0, _neverthrow.err)(firstChunkResult.error);
+    }
+  }
+  if (Array.isArray(otherChunks)) {
+    const otherChunksResults = await Promise.all(otherChunks.map(a => executeChunk(a)));
+    const otherChunksError = otherChunksResults.find(r => r === null || r === void 0 ? void 0 : r.isErr());
+    if (otherChunksError && otherChunksError.isErr()) {
+      return (0, _neverthrow.err)(otherChunksError.error);
+    }
+  }
+  return (0, _neverthrow.ok)(results);
+}
+function generateRange(start, end) {
+  if (end < start) {
+    throw new Error("End value must be greater than or equal to start value.");
+  }
+  const range = [];
+  for (let i = start; i <= end; i++) {
+    range.push(i);
+  }
+  return range;
+}

package/dist/optimized-extractors/listExtractionHelpers/utils/extractStructuredListUsingAi.js

@@ -0,0 +1,44 @@
+"use strict";
+
+Object.defineProperty(exports, "__esModule", {
+  value: true
+});
+exports.extractStructuredListUsingAi = extractStructuredListUsingAi;
+var _neverthrow = require("neverthrow");
+var _errors = require("../errors");
+var _zod = require("zod");
+var _common = require("../../common");
+async function extractStructuredListUsingAi(entityName, itemSchema, data, identifier, strategy, prompt, apiKey) {
+  var _data$images;
+  const propertiesListSchema = {
+    type: "array",
+    items: itemSchema
+  };
+  const result = await (0, _common.extractStructuredDataUsingAi)({
+    entityName,
+    model: strategy.model,
+    jsonSchema: propertiesListSchema,
+    text: data.text ? [data.text] : undefined,
+    images: (_data$images = data.images) === null || _data$images === void 0 ? void 0 : _data$images.map(image => ({
+      data: image,
+      image_type: "png"
+    })),
+    identifier,
+    systemMessage: prompt,
+    apiKey
+  });
+  if (result.isErr()) {
+    if (result.error.type === "InsufficientAiCredits") {
+      return (0, _neverthrow.err)((0, _errors.insufficientAiCredits)(result.error.context));
+    }
+    if (result.error.type === "NoDataFound") {
+      return (0, _neverthrow.ok)([]);
+    }
+    return (0, _neverthrow.err)((0, _errors.invalidExtractionResult)(result.error.context));
+  }
+  const parsedResult = _zod.z.array(_zod.z.record(_zod.z.string())).safeParse(result.value.result);
+  if (!parsedResult.success) {
+    return (0, _neverthrow.err)((0, _errors.invalidExtractionResult)("Failed to parse extraction result."));
+  }
+  return (0, _neverthrow.ok)(parsedResult.data);
+}

package/dist/optimized-extractors/listExtractionHelpers/utils/getListContainerXpath.js

@@ -0,0 +1,94 @@
+"use strict";
+
+Object.defineProperty(exports, "__esModule", {
+  value: true
+});
+exports.findSetOfXpathsToCreateAnArrayExtractor = findSetOfXpathsToCreateAnArrayExtractor;
+exports.getContainerElement = getContainerElement;
+exports.getListContainerXpath = getListContainerXpath;
+exports.partOfSameArrayXpath = partOfSameArrayXpath;
+exports.verifyThatAllXpathsArePartOfSameArray = verifyThatAllXpathsArePartOfSameArray;
+async function getListContainerXpath(matches) {
+  const matchesList = Array.from(matches.entries());
+  const exactMatches = matchesList.map(([_, matchSet]) => matchSet.filter(match => match.exact));
+  const xpathsSetForExactMatches = exactMatches.map(value => value.map(match => match.nodeXpath)).filter(xpaths => xpaths.length > 0);
+  let resultXpaths = findSetOfXpathsToCreateAnArrayExtractor(xpathsSetForExactMatches);
+  if (!resultXpaths) {
+    const xpathsSetForAllMatches = matchesList.map(([_, matchSet]) => matchSet.map(match => match.nodeXpath)).filter(xpaths => xpaths.length > 0);
+    resultXpaths = findSetOfXpathsToCreateAnArrayExtractor(xpathsSetForAllMatches);
+  }
+  if (!resultXpaths) {
+    return null;
+  }
+  const partOfSameArray = verifyThatAllXpathsArePartOfSameArray(resultXpaths);
+  if (!partOfSameArray) {
+    return null;
+  }
+  const containerElement = getContainerElement(resultXpaths);
+  return containerElement;
+}
+function findSetOfXpathsToCreateAnArrayExtractor(input) {
+  if (input.length === 0) return null;
+  const smallestSet = input.reduce((acc, curr) => curr.length < acc.length ? curr : acc);
+  smallestSet.sort((a, b) => b.length - a.length);
+  for (const xpath of smallestSet) {
+    const result = [xpath];
+    for (const set of input) {
+      if (set === smallestSet) continue;
+      const matched = set.find(otherXpath => partOfSameArrayXpath(xpath, otherXpath));
+      if (matched) {
+        result.push(matched);
+      } else {
+        break;
+      }
+    }
+    if (result.length === input.length) return result;
+  }
+  return null;
+}
+function partOfSameArrayXpath(str1, str2) {
+  if (str1 === str2) return false;
+  const parts1 = str1.split("/");
+  const parts2 = str2.split("/");
+  if (parts1.length !== parts2.length) return false;
+  let numericDifferences = 0;
+  for (let i = 0; i < parts1.length; i++) {
+    if (parts1[i] !== parts2[i]) {
+      const regex = /\d+/g;
+      const numbers1 = (parts1[i].match(regex) || []).map(Number);
+      const numbers2 = (parts2[i].match(regex) || []).map(Number);
+      if (numbers1.length !== numbers2.length) return false;
+      let segmentDifferences = 0;
+      for (let j = 0; j < numbers1.length; j++) {
+        if (numbers1[j] !== numbers2[j]) segmentDifferences++;
+      }
+      if (segmentDifferences === 0) return false;
+      if (segmentDifferences > 1) return false;
+      numericDifferences += segmentDifferences;
+      if (numericDifferences > 1) return false;
+    }
+  }
+  return numericDifferences === 1;
+}
+function verifyThatAllXpathsArePartOfSameArray(xpaths) {
+  const firstPath = xpaths[0];
+  for (let i = 1; i < xpaths.length; i++) {
+    if (!partOfSameArrayXpath(xpaths[i], firstPath)) {
+      return false;
+    }
+  }
+  return true;
+}
+function getContainerElement(xpaths) {
+  if (!xpaths.length) return null;
+  let commonPrefix = xpaths[0].split("/");
+  for (let i = 1; i < xpaths.length; i++) {
+    const parts = xpaths[i].split("/");
+    let j = 0;
+    while (j < commonPrefix.length && j < parts.length && commonPrefix[j] === parts[j]) {
+      j++;
+    }
+    commonPrefix = commonPrefix.slice(0, j);
+  }
+  return commonPrefix.length ? commonPrefix.join("/") : null;
+}

package/dist/optimized-extractors/listExtractionHelpers/utils/getRelativeContainerXpathSelector.js

@@ -0,0 +1,20 @@
+"use strict";
+
+Object.defineProperty(exports, "__esModule", {
+  value: true
+});
+exports.getRelativeContainerXpathSelector = getRelativeContainerXpathSelector;
+var _locatorHelpers = require("../../../common/locatorHelpers");
+async function getRelativeContainerXpathSelector(searchRegionLocator, containerPath) {
+  const searchRegionXpath = await (0, _locatorHelpers.findXPathForLocator)(searchRegionLocator);
+  if (!searchRegionXpath) {
+    return null;
+  }
+  if (searchRegionXpath === containerPath) {
+    return ".";
+  }
+  if (!containerPath) {
+    return null;
+  }
+  return containerPath.replace(`${searchRegionXpath}/`, "");
+}