@intuned/browser-dev 0.1.4-dev.0

package/dist/optimized-extractors/listExtractionHelpers/utils/extractPropertiesUsingGPTFromArray.js

@@ -0,0 +1,277 @@
+"use strict";
+
+Object.defineProperty(exports, "__esModule", {
+  value: true
+});
+exports.extractPropertiesUsingGPTFromArray = extractPropertiesUsingGPTFromArray;
+var _neverthrow = require("neverthrow");
+var _errors = require("../errors");
+var _common = require("../../common/");
+var _isTableHeaderOrFooter = require("../../common/isTableHeaderOrFooter");
+var _Logger = require("../../../common/Logger");
+var _extractionHelpers = require("../../../common/extractionHelpers");
+var _buildExamplesPrompt = require("../../common/buildExamplesPrompt");
+var _utils = require("../../common/utils");
+async function extractPropertiesUsingGPTFromArray(input) {
+  if (input.strategy.type === "IMAGE") {
+    return extractPropertiesWithImageStrategy(input);
+  } else if (input.strategy.type === "HTML") {
+    return extractPropertiesWithHTMLStrategy(input);
+  }
+  throw new Error("Invalid strategy type");
+}
+async function extractPropertiesUsingGPT({
+  itemEntityName,
+  itemEntitySchema,
+  text,
+  image,
+  extraRowDataFromTableRow,
+  strategy,
+  tableHeaders,
+  identifier,
+  possibleTableHeaderOrFooter,
+  examples,
+  apiKey
+}) {
+  const systemMessage = extraRowDataFromTableRow ? `You are a data extraction assistant, out of this text, you will be asked to extract some data. Be accurate, give complete results. If you cannot find the field data, use null. make sure to follow examples if provided. The data will be provided twice, once as a json that was extracted from an html table and once as the html for that row in the table.` : `You are a data extraction assistant, out of this text, you will be asked to extract some data. Be accurate, give complete results. If you cannot find the field data, use null. make sure to follow examples if provided.`;
+  const extraUserMessages = [];
+  if (examples.length > 0) {
+    const examplesMessage = (0, _buildExamplesPrompt.buildExamplesPrompt)({
+      entityName: itemEntityName,
+      examples
+    });
+    extraUserMessages.push(examplesMessage);
+  }
+  if (possibleTableHeaderOrFooter) {
+    const content = text ?? image;
+    const isHeader = await (0, _isTableHeaderOrFooter.isTableHeaderOrFooter)(content);
+    if (isHeader.isErr()) {
+      return (0, _neverthrow.err)(isHeader.error);
+    }
+    if (isHeader.value.isHeader) {
+      return (0, _neverthrow.ok)({});
+    }
+  }
+  if (tableHeaders) {
+    extraUserMessages.push(`this data are part of a table, the table headers in order are: ${tableHeaders === null || tableHeaders === void 0 ? void 0 : tableHeaders.join(", ")}`);
+  }
+  if (extraRowDataFromTableRow) {
+    extraUserMessages.push(JSON.stringify(extraRowDataFromTableRow));
+  }
+  const extractionResult = await (0, _common.extractStructuredDataUsingAi)({
+    entityName: itemEntityName,
+    model: strategy.model,
+    text: text ? [text] : undefined,
+    jsonSchema: itemEntitySchema,
+    systemMessage,
+    extraUserMessages,
+    images: image ? [{
+      data: image,
+      image_type: "png"
+    }] : [],
+    identifier,
+    apiKey
+  });
+  if (extractionResult.isErr()) {
+    if (extractionResult.error.type === "NoDataFound") {
+      return (0, _neverthrow.ok)({});
+    }
+    return (0, _neverthrow.err)((0, _errors.invalidExtractionResult)(extractionResult.error.context));
+  }
+  return (0, _neverthrow.ok)(extractionResult.value.result);
+}
+async function extractPropertiesWithImageStrategy({
+  identifier,
+  itemEntityName,
+  itemEntitySchema,
+  items,
+  strategy,
+  tableAsJsonArray,
+  tableHeaders,
+  examples: previousExamples,
+  apiKey
+}) {
+  const shouldUseTableData = !!tableAsJsonArray && tableAsJsonArray.length === items.length;
+  const CHUNK_SIZE = process.env.TEST === "true" ? 1 : 5;
+  const chunkedExtractPropertiesUsingGPTParameters = items.reduce((acc, item, index) => {
+    const chunkIndex = Math.floor(index / CHUNK_SIZE);
+    if (!acc[chunkIndex]) {
+      acc[chunkIndex] = [];
+    }
+    acc[chunkIndex].push({
+      item,
+      index
+    });
+    return acc;
+  }, []);
+  const result = [];
+  for (const chunk of chunkedExtractPropertiesUsingGPTParameters) {
+    const examples = (0, _utils.getRandomItems)([...result, ...previousExamples], 3);
+    const promises = chunk.map(args => extractPropertiesUsingGPT({
+      itemEntityName,
+      itemEntitySchema,
+      text: undefined,
+      image: args.item.type === "image" ? args.item.buffer : undefined,
+      extraRowDataFromTableRow: shouldUseTableData ? tableAsJsonArray[args.index] : undefined,
+      strategy,
+      tableHeaders,
+      identifier,
+      possibleTableHeaderOrFooter: args.index === 0 || args.index === items.length - 1,
+      examples,
+      apiKey
+    }));
+    const results = await Promise.all(promises);
+    const errorResult = results.find(r => r.isErr());
+    if (errorResult && errorResult.isErr()) {
+      return (0, _neverthrow.err)(errorResult.error);
+    }
+    const listOfResults = results.map(r => r._unsafeUnwrap());
+    listOfResults.forEach((r, i) => {
+      const itemIndexInFullList = i + result.length;
+      _Logger.logger.debug(`Extracted this info from array item #${itemIndexInFullList}: ${JSON.stringify(r)}`);
+    });
+    result.push(...listOfResults);
+  }
+  return (0, _neverthrow.ok)(result);
+}
+async function extractPropertiesWithHTMLStrategy({
+  identifier,
+  itemEntityName,
+  itemEntitySchema,
+  items,
+  strategy,
+  tableAsJsonArray,
+  tableHeaders,
+  examples: previousExamples,
+  apiKey
+}) {
+  const shouldUseTableData = !!tableAsJsonArray && tableAsJsonArray.length === items.length;
+  const isWeakModel = strategy.model === "claude-3-haiku" || strategy.model == "gpt3.5-turbo";
+  const averageItemLength = items.reduce((sum, item) => {
+    if (item.type !== "text") return sum;
+    return sum + (0, _extractionHelpers.compressStringSpaces)(item.text).length;
+  }, 0) / items.length;
+  const shouldUseSmallerChunkSize = isWeakModel && averageItemLength > 1000;
+  const CHUNK_SIZE = shouldUseSmallerChunkSize ? 3 : 10;
+  const itemsChunks = items.reduce((resultArray, item, index) => {
+    const chunkIndex = Math.floor(index / CHUNK_SIZE);
+    if (!resultArray[chunkIndex]) {
+      resultArray[chunkIndex] = [];
+    }
+    resultArray[chunkIndex].push({
+      index,
+      item
+    });
+    return resultArray;
+  }, []);
+  const results = [];
+  const executeChunk = async chunk => {
+    const examples = (0, _utils.getRandomItems)([...results, ...previousExamples], 3);
+    const texts = chunk.map(i => {
+      if (i.item.type !== "text") {
+        throw new Error("Invalid type");
+      }
+      return `<ITEM_INDEX_${i.index}>\n${i.item.text}\n</ITEM_INDEX_${i.index}>`;
+    }).join("\n");
+    const extraRowDataFromTableRow = shouldUseTableData ? chunk.map(c => tableAsJsonArray[c.index]) : undefined;
+    const shouldWorkaroundGoogleModel = (0, _common.isGoogleModel)(strategy.model);
+    const itemSchema = !shouldWorkaroundGoogleModel ? {
+      oneOf: [itemEntitySchema, {
+        type: "null"
+      }]
+    } : {
+      type: "object",
+      properties: {
+        value: itemEntitySchema,
+        _isNull: {
+          type: "boolean",
+          description: "If the value is supposed to be null, set this to true and do not provide a value. Otherwise, set it to false and provide the value."
+        }
+      },
+      required: ["_isNull"]
+    };
+    const itemNullPromptExample = !shouldWorkaroundGoogleModel ? "null" : `{ "_isNull": true }`;
+    const result = await extractPropertiesUsingGPT({
+      identifier,
+      text: texts,
+      examples,
+      itemEntityName,
+      tableHeaders,
+      extraRowDataFromTableRow,
+      apiKey,
+      itemEntitySchema: {
+        type: "array",
+        description: `Extracted ` + itemEntityName + ` items from the content wrapped with <LIST_ITEM_INDEX_X> and </LIST_ITEM_INDEX_X> tags. each tag represents an item on the list, you should return ` + chunk.length + ` items, one for each index. If the item boundaries do not contain a valid entity, return null for that item with the right index. YOU SHOULD INCLUDE ALL ITEMS IN THE RESPONSE, EVERY INDEX BOUNDARY SHOULD BE REPRESENTED BY AN ITEM. IF the index boundary does not include a valid item you should return null for it.` + ` AND YOU SHOULD ALWAYS HAVE THE RIGHT INDEX FOR THE MISSING ITEMS, **do not put it at the end of the list or change the order of that item if in the list **. if <LIST_ITEM_INDEX_33> and </LIST_ITEM_INDEX_33> does not wrap a valid item, the item you return should be ` + `{ index: 33, isTableHeaderOrFooter: false, item: ${itemNullPromptExample} } where ths index is the number in the tag that's missing the item, ` + ` each <LIST_ITEM_INDEX_X> and </LIST_ITEM_INDEX_X> boundary represents a single list item. do not return the same list item more than once.`,
+        minItems: chunk.length,
+        maxItems: chunk.length,
+        items: {
+          type: "object",
+          properties: {
+            index: {
+              type: "number",
+              enum: generateRange(chunk[0].index, chunk[chunk.length - 1].index),
+              minimum: chunk[0].index,
+              maximum: chunk[chunk.length - 1].index,
+              description: `Identify the index of an item from the list based on the prefix and suffix around the extracted data. if you extract data between <LIST_ITEM_INDEX_x> and </LIST_ITEM_INDEX_x>, the index should be the value of x. if <LIST_ITEM_INDEX_x> exists and does not have a valid data inside it you still should return an item with index x and value of null , getting the wrong index for the missing item will break the whole extraction. YOU SHOULD INCLUDE ALL ITEMS IN THE RESPONSE, EVERY INDEX BOUNDARY SHOULD BE REPRESENTED BY AN ITEM`
+            },
+            isTableHeaderOrFooter: {
+              type: "boolean",
+              description: `If the extracted data is a table header or footer, set this field to true. Otherwise, set it to false.`
+            },
+            item: itemSchema
+          }
+        }
+      },
+      image: undefined,
+      strategy
+    });
+    if (result.isErr()) {
+      return (0, _neverthrow.err)(result.error);
+    }
+    const resultUnWrapped = result.value;
+    function isItemRecord(item) {
+      return typeof item.value !== "object";
+    }
+    resultUnWrapped === null || resultUnWrapped === void 0 || resultUnWrapped.forEach(r => {
+      if (r.isTableHeaderOrFooter) {
+        _Logger.logger.debug(`skipping item at index #${r.index + 1}: ${JSON.stringify(r.item)}, it's detected as a table header or footer.`);
+        results[r.index] = {};
+        return;
+      }
+      const item = r.item;
+      let itemToUse;
+      if (item != null && !isItemRecord(item)) {
+        itemToUse = item._isNull ? null : item.value;
+      } else {
+        itemToUse = item;
+      }
+      results[r.index] = itemToUse;
+      _Logger.logger.debug(`Extracted this info from array item #${r.index + 1}: ${JSON.stringify(itemToUse)}`);
+    });
+  };
+  const [firstChunk, ...otherChunks] = itemsChunks;
+  if (firstChunk) {
+    const firstChunkResult = await executeChunk(firstChunk);
+    if (firstChunkResult !== null && firstChunkResult !== void 0 && firstChunkResult.isErr()) {
+      return (0, _neverthrow.err)(firstChunkResult.error);
+    }
+  }
+  if (Array.isArray(otherChunks)) {
+    const otherChunksResults = await Promise.all(otherChunks.map(a => executeChunk(a)));
+    const otherChunksError = otherChunksResults.find(r => r === null || r === void 0 ? void 0 : r.isErr());
+    if (otherChunksError && otherChunksError.isErr()) {
+      return (0, _neverthrow.err)(otherChunksError.error);
+    }
+  }
+  return (0, _neverthrow.ok)(results);
+}
+function generateRange(start, end) {
+  if (end < start) {
+    throw new Error("End value must be greater than or equal to start value.");
+  }
+  const range = [];
+  for (let i = start; i <= end; i++) {
+    range.push(i);
+  }
+  return range;
+}

package/dist/optimized-extractors/listExtractionHelpers/utils/extractStructuredListUsingAi.js

@@ -0,0 +1,44 @@
+"use strict";
+
+Object.defineProperty(exports, "__esModule", {
+  value: true
+});
+exports.extractStructuredListUsingAi = extractStructuredListUsingAi;
+var _neverthrow = require("neverthrow");
+var _errors = require("../errors");
+var _zod = require("zod");
+var _common = require("../../common");
+async function extractStructuredListUsingAi(entityName, itemSchema, data, identifier, strategy, prompt, apiKey) {
+  var _data$images;
+  const propertiesListSchema = {
+    type: "array",
+    items: itemSchema
+  };
+  const result = await (0, _common.extractStructuredDataUsingAi)({
+    entityName,
+    model: strategy.model,
+    jsonSchema: propertiesListSchema,
+    text: data.text ? [data.text] : undefined,
+    images: (_data$images = data.images) === null || _data$images === void 0 ? void 0 : _data$images.map(image => ({
+      data: image,
+      image_type: "png"
+    })),
+    identifier,
+    systemMessage: prompt,
+    apiKey
+  });
+  if (result.isErr()) {
+    if (result.error.type === "InsufficientAiCredits") {
+      return (0, _neverthrow.err)((0, _errors.insufficientAiCredits)(result.error.context));
+    }
+    if (result.error.type === "NoDataFound") {
+      return (0, _neverthrow.ok)([]);
+    }
+    return (0, _neverthrow.err)((0, _errors.invalidExtractionResult)(result.error.context));
+  }
+  const parsedResult = _zod.z.array(_zod.z.record(_zod.z.string())).safeParse(result.value.result);
+  if (!parsedResult.success) {
+    return (0, _neverthrow.err)((0, _errors.invalidExtractionResult)("Failed to parse extraction result."));
+  }
+  return (0, _neverthrow.ok)(parsedResult.data);
+}

package/dist/optimized-extractors/listExtractionHelpers/utils/getListContainerXpath.js

@@ -0,0 +1,94 @@
+"use strict";
+
+Object.defineProperty(exports, "__esModule", {
+  value: true
+});
+exports.findSetOfXpathsToCreateAnArrayExtractor = findSetOfXpathsToCreateAnArrayExtractor;
+exports.getContainerElement = getContainerElement;
+exports.getListContainerXpath = getListContainerXpath;
+exports.partOfSameArrayXpath = partOfSameArrayXpath;
+exports.verifyThatAllXpathsArePartOfSameArray = verifyThatAllXpathsArePartOfSameArray;
+async function getListContainerXpath(matches) {
+  const matchesList = Array.from(matches.entries());
+  const exactMatches = matchesList.map(([_, matchSet]) => matchSet.filter(match => match.exact));
+  const xpathsSetForExactMatches = exactMatches.map(value => value.map(match => match.nodeXpath)).filter(xpaths => xpaths.length > 0);
+  let resultXpaths = findSetOfXpathsToCreateAnArrayExtractor(xpathsSetForExactMatches);
+  if (!resultXpaths) {
+    const xpathsSetForAllMatches = matchesList.map(([_, matchSet]) => matchSet.map(match => match.nodeXpath)).filter(xpaths => xpaths.length > 0);
+    resultXpaths = findSetOfXpathsToCreateAnArrayExtractor(xpathsSetForAllMatches);
+  }
+  if (!resultXpaths) {
+    return null;
+  }
+  const partOfSameArray = verifyThatAllXpathsArePartOfSameArray(resultXpaths);
+  if (!partOfSameArray) {
+    return null;
+  }
+  const containerElement = getContainerElement(resultXpaths);
+  return containerElement;
+}
+function findSetOfXpathsToCreateAnArrayExtractor(input) {
+  if (input.length === 0) return null;
+  const smallestSet = input.reduce((acc, curr) => curr.length < acc.length ? curr : acc);
+  smallestSet.sort((a, b) => b.length - a.length);
+  for (const xpath of smallestSet) {
+    const result = [xpath];
+    for (const set of input) {
+      if (set === smallestSet) continue;
+      const matched = set.find(otherXpath => partOfSameArrayXpath(xpath, otherXpath));
+      if (matched) {
+        result.push(matched);
+      } else {
+        break;
+      }
+    }
+    if (result.length === input.length) return result;
+  }
+  return null;
+}
+function partOfSameArrayXpath(str1, str2) {
+  if (str1 === str2) return false;
+  const parts1 = str1.split("/");
+  const parts2 = str2.split("/");
+  if (parts1.length !== parts2.length) return false;
+  let numericDifferences = 0;
+  for (let i = 0; i < parts1.length; i++) {
+    if (parts1[i] !== parts2[i]) {
+      const regex = /\d+/g;
+      const numbers1 = (parts1[i].match(regex) || []).map(Number);
+      const numbers2 = (parts2[i].match(regex) || []).map(Number);
+      if (numbers1.length !== numbers2.length) return false;
+      let segmentDifferences = 0;
+      for (let j = 0; j < numbers1.length; j++) {
+        if (numbers1[j] !== numbers2[j]) segmentDifferences++;
+      }
+      if (segmentDifferences === 0) return false;
+      if (segmentDifferences > 1) return false;
+      numericDifferences += segmentDifferences;
+      if (numericDifferences > 1) return false;
+    }
+  }
+  return numericDifferences === 1;
+}
+function verifyThatAllXpathsArePartOfSameArray(xpaths) {
+  const firstPath = xpaths[0];
+  for (let i = 1; i < xpaths.length; i++) {
+    if (!partOfSameArrayXpath(xpaths[i], firstPath)) {
+      return false;
+    }
+  }
+  return true;
+}
+function getContainerElement(xpaths) {
+  if (!xpaths.length) return null;
+  let commonPrefix = xpaths[0].split("/");
+  for (let i = 1; i < xpaths.length; i++) {
+    const parts = xpaths[i].split("/");
+    let j = 0;
+    while (j < commonPrefix.length && j < parts.length && commonPrefix[j] === parts[j]) {
+      j++;
+    }
+    commonPrefix = commonPrefix.slice(0, j);
+  }
+  return commonPrefix.length ? commonPrefix.join("/") : null;
+}

package/dist/optimized-extractors/listExtractionHelpers/utils/getRelativeContainerXpathSelector.js

@@ -0,0 +1,20 @@
+"use strict";
+
+Object.defineProperty(exports, "__esModule", {
+  value: true
+});
+exports.getRelativeContainerXpathSelector = getRelativeContainerXpathSelector;
+var _locatorHelpers = require("../../../common/locatorHelpers");
+async function getRelativeContainerXpathSelector(searchRegionLocator, containerPath) {
+  const searchRegionXpath = await (0, _locatorHelpers.findXPathForLocator)(searchRegionLocator);
+  if (!searchRegionXpath) {
+    return null;
+  }
+  if (searchRegionXpath === containerPath) {
+    return ".";
+  }
+  if (!containerPath) {
+    return null;
+  }
+  return containerPath.replace(`${searchRegionXpath}/`, "");
+}

package/dist/optimized-extractors/listExtractionHelpers/utils/getSimplifiedHtmlPerListItem.js

@@ -0,0 +1,21 @@
+"use strict";
+
+Object.defineProperty(exports, "__esModule", {
+  value: true
+});
+exports.getSimplifiedHtmlPerListItem = getSimplifiedHtmlPerListItem;
+var _getSimplifiedHtml = require("../../objectExtractionHelpers/getSimplifiedHtml");
+async function getSimplifiedHtmlPerListItem(itemsLocators) {
+  const result = [];
+  for (let i = 0; i < itemsLocators.length; i++) {
+    const locator = itemsLocators[i];
+    const handle = await locator.elementHandle();
+    const htmlForItem = await (0, _getSimplifiedHtml.getSimplifiedHtml)(handle, {
+      keepOnlyVisibleElements: false,
+      shouldIncludeContentAsProp: true,
+      shouldIncludeOnClick: true
+    });
+    result.push(htmlForItem);
+  }
+  return result;
+}

package/dist/optimized-extractors/listExtractionHelpers/utils/tablesUtils.js

@@ -0,0 +1,48 @@
+"use strict";
+
+Object.defineProperty(exports, "__esModule", {
+  value: true
+});
+exports.createJsonFromTable = createJsonFromTable;
+exports.isListTable = isListTable;
+async function isListTable(containerLocator, itemsSimplifiedHtml) {
+  const parentTableLocator = containerLocator.locator("xpath=ancestor::table[1]");
+  const hasParentTable = (await parentTableLocator.all()).length !== 0;
+  if (!hasParentTable) {
+    return {
+      tableLocater: undefined,
+      isTable: false
+    };
+  }
+  const table = await parentTableLocator.elementHandle({
+    timeout: 1_000
+  });
+  const allItemsContainTr = itemsSimplifiedHtml.every(item => item.includes("<tr"));
+  return {
+    tableLocater: table,
+    isTable: !!table && !!allItemsContainTr
+  };
+}
+async function createJsonFromTable(page) {
+  return await page.evaluate(() => {
+    function compressStringSpaces(str) {
+      return str.replace(/\s+/g, " ").trim();
+    }
+    const table = document.querySelector("table");
+    if (!table) {
+      return [];
+    }
+    const headers = Array.from(table.querySelectorAll("th")).map(th => th === null || th === void 0 ? void 0 : th.textContent).map(header => header && compressStringSpaces(header));
+    const rows = Array.from(table.querySelectorAll("tr"));
+    const jsonArray = rows.slice(1).map(tr => {
+      const cells = Array.from(tr.querySelectorAll("td"));
+      const rowObject = headers.reduce((obj, header, index) => {
+        var _cells$index;
+        if (header) obj[header] = ((_cells$index = cells[index]) === null || _cells$index === void 0 ? void 0 : _cells$index.textContent) ?? "";
+        return obj;
+      }, {});
+      return rowObject;
+    });
+    return jsonArray;
+  });
+}

package/dist/optimized-extractors/listExtractionHelpers/utils/validateOptions.js

@@ -0,0 +1,52 @@
+"use strict";
+
+Object.defineProperty(exports, "__esModule", {
+  value: true
+});
+exports.validateDynamicListExtractorOptions = validateDynamicListExtractorOptions;
+var _neverthrow = require("neverthrow");
+var Errors = _interopRequireWildcard(require("../errors"));
+var _locatorHelpers = require("../../../common/locatorHelpers");
+function _interopRequireWildcard(e, t) { if ("function" == typeof WeakMap) var r = new WeakMap(), n = new WeakMap(); return (_interopRequireWildcard = function (e, t) { if (!t && e && e.__esModule) return e; var o, i, f = { __proto__: null, default: e }; if (null === e || "object" != typeof e && "function" != typeof e) return f; if (o = t ? n : r) { if (o.has(e)) return o.get(e); o.set(e, f); } for (const t in e) "default" !== t && {}.hasOwnProperty.call(e, t) && ((i = (o = Object.defineProperty) && Object.getOwnPropertyDescriptor(e, t)) && (i.get || i.set) ? o(f, t, i) : f[t] = e[t]); return f; })(e, t); }
+async function validateDynamicListExtractorOptions(page, label, options) {
+  if (typeof label !== "string") {
+    return (0, _neverthrow.err)(Errors.invalidInput("Identifier must be a string."));
+  }
+  const searchRegion = options.searchRegion ?? page.locator("html");
+  const elementExist = (await searchRegion.count().catch(e => {
+    return 0;
+  })) > 0;
+  if (!elementExist) {
+    return (0, _neverthrow.err)(Errors.invalidSearchRegion());
+  }
+  const searchRegionHandler = await searchRegion.elementHandle();
+  if (!searchRegionHandler) {
+    return (0, _neverthrow.err)(Errors.invalidSearchRegion());
+  }
+  const pageUrl = page.url();
+  const getPageUrlOrigin = () => new URL(pageUrl).origin;
+  const safeGetPageUrlOrigin = (0, _neverthrow.fromThrowable)(getPageUrlOrigin, () => Errors.invalidAddressUrl("Cannot get page url origin."));
+  const variantKey = options.variantKey ? (0, _neverthrow.ok)(options.variantKey) : safeGetPageUrlOrigin();
+  if (variantKey.isErr()) {
+    return (0, _neverthrow.err)(variantKey.error);
+  }
+  const invalidate = options.optionalPropertiesInvalidator ?? (() => []);
+  const primaryProperty = Object.entries(options.itemEntitySchema.properties).find(([k, v]) => v.primary);
+  return (0, _neverthrow.ok)({
+    itemEntityName: options.itemEntityName,
+    itemEntitySchema: options.itemEntitySchema,
+    variantKey: variantKey.value ?? "about:blank",
+    invalidate,
+    pageUrl,
+    primaryProperty: primaryProperty,
+    searchRegionHandler,
+    searchRegion,
+    hasSearchRegionContainer: !!options.searchRegion,
+    label,
+    searchRegionKey: options.searchRegion ? (0, _locatorHelpers.getLocatorInternalKey)(options.searchRegion) : null,
+    strategy: options.strategy,
+    prompt: options.prompt,
+    searchRegionXpath: options.searchRegion ? await (0, _locatorHelpers.findXPathForLocator)(options.searchRegion) : undefined,
+    apiKey: options.apiKey
+  });
+}

package/dist/optimized-extractors/models/anthropicModel.js

@@ -0,0 +1,23 @@
+"use strict";
+
+Object.defineProperty(exports, "__esModule", {
+  value: true
+});
+exports.createAnthropicInstance = createAnthropicInstance;
+var _dotenv = require("dotenv");
+var _jwtTokenManager = require("../../common/jwtTokenManager");
+var _sdk = _interopRequireDefault(require("@anthropic-ai/sdk"));
+function _interopRequireDefault(e) { return e && e.__esModule ? e : { default: e }; }
+(0, _dotenv.config)();
+function createAnthropicInstance(options) {
+  if (options !== null && options !== void 0 && options.apiKey) {
+    return new _sdk.default({
+      apiKey: options.apiKey
+    });
+  }
+  return new _sdk.default({
+    apiKey: "--THI_VALUE_WILL_BE_REPLACED_BY_INTUNED_BE--",
+    baseURL: `${process.env.FUNCTIONS_DOMAIN}/api/${process.env.INTUNED_WORKSPACE_ID}/functions/${process.env.INTUNED_INTEGRATION_ID}/anthropic`,
+    fetch: _jwtTokenManager.backendFunctionsTokenManager.fetchWithToken.bind(_jwtTokenManager.backendFunctionsTokenManager)
+  });
+}

package/dist/optimized-extractors/models/openaiModel.js

@@ -0,0 +1,23 @@
+"use strict";
+
+Object.defineProperty(exports, "__esModule", {
+  value: true
+});
+exports.createOpenAIInstance = createOpenAIInstance;
+var _openai = require("openai");
+var _dotenv = require("dotenv");
+var _jwtTokenManager = require("../../common/jwtTokenManager");
+(0, _dotenv.config)();
+function createOpenAIInstance(options) {
+  if (options !== null && options !== void 0 && options.apiKey) {
+    return new _openai.OpenAI({
+      apiKey: options.apiKey
+    });
+  }
+  const openai = new _openai.OpenAI({
+    apiKey: "",
+    baseURL: `${process.env.FUNCTIONS_DOMAIN}/api/${process.env.INTUNED_WORKSPACE_ID}/functions/${process.env.INTUNED_INTEGRATION_ID}/openai`,
+    fetch: _jwtTokenManager.backendFunctionsTokenManager.fetchWithToken.bind(_jwtTokenManager.backendFunctionsTokenManager)
+  });
+  return openai;
+}