@intuned/browser-dev 2.2.3-unify-sdks.20

package/dist/optimized-extractors/common/findTableHeaders.js

@@ -0,0 +1,175 @@
+"use strict";
+
+Object.defineProperty(exports, "__esModule", {
+  value: true
+});
+exports.getTableHeadersUsingAi = getTableHeadersUsingAi;
+var _anthropicModel = require("../models/anthropicModel");
+var _zod = require("zod");
+var _imageSize = require("image-size");
+var _neverthrow = require("neverthrow");
+var Errors = _interopRequireWildcard(require("../types/errors"));
+var _getAiTrackingHeaders = require("../../common/eventTracking/getAiTrackingHeaders");
+var _environmentVariables = require("../../common/environmentVariables");
+var _loadRuntime = require("../../common/loadRuntime");
+function _interopRequireWildcard(e, t) { if ("function" == typeof WeakMap) var r = new WeakMap(), n = new WeakMap(); return (_interopRequireWildcard = function (e, t) { if (!t && e && e.__esModule) return e; var o, i, f = { __proto__: null, default: e }; if (null === e || "object" != typeof e && "function" != typeof e) return f; if (o = t ? n : r) { if (o.has(e)) return o.get(e); o.set(e, f); } for (const t in e) "default" !== t && {}.hasOwnProperty.call(e, t) && ((i = (o = Object.defineProperty) && Object.getOwnPropertyDescriptor(e, t)) && (i.get || i.set) ? o(f, t, i) : f[t] = e[t]); return f; })(e, t); }
+async function getTableHeadersUsingAi(handle, identifier) {
+  var _getExecutionContext, _getExecutionContext2, _getExecutionContext3, _getExecutionContext4, _response$error;
+  const getExecutionContext = await (0, _loadRuntime.loadRuntime)();
+  let image;
+  try {
+    image = await handle.screenshot({
+      type: "png",
+      timeout: 3_000
+    });
+  } catch (e) {
+    return (0, _neverthrow.ok)({
+      headers: []
+    });
+  }
+  const {
+    width,
+    height
+  } = (0, _imageSize.imageSize)(image);
+  if (!width) return (0, _neverthrow.ok)({
+    headers: []
+  });
+  const imageToUse = height && height > 500 ? await cropImage(image, handle, {
+    height: 500,
+    width,
+    x: 0,
+    y: 0
+  }) : image;
+  if (!imageToUse) return (0, _neverthrow.ok)({
+    headers: []
+  });
+  const anthropic = (0, _anthropicModel.createAnthropicInstance)();
+  const response = await (0, _neverthrow.fromPromise)(anthropic.messages.create({
+    max_tokens: 4096,
+    temperature: 0,
+    messages: [{
+      role: "user",
+      content: [{
+        type: "text",
+        text: `extract table headers from the image, you must use the get_table_headers tool, make sure to get the headers in order, "column_number" is the number of the table column that has this header. if there's a column with no header that still counts as a column and should be taken into account.`
+      }, {
+        type: "image",
+        source: {
+          data: imageToUse.toString("base64"),
+          media_type: "image/png",
+          type: "base64"
+        }
+      }]
+    }],
+    model: "claude-3-haiku-20240307",
+    tools: [{
+      input_schema: {
+        type: "object",
+        properties: {
+          headers: {
+            type: "array",
+            items: {
+              type: "object",
+              description: "Table headers",
+              properties: {
+                column_number: {
+                  type: "number",
+                  description: "The column number of the header"
+                },
+                header: {
+                  type: "string"
+                }
+              },
+              required: ["column_number", "header"]
+            }
+          }
+        },
+        required: ["headers"]
+      },
+      name: "get_table_headers",
+      description: `a function to extract table headers from an image, the headers should be in order and the column number should be the number of the column that has this header. if there's a column with no header that still counts as a column and should be taken into account. you should still need to`
+    }]
+  }, {
+    headers: (0, _getAiTrackingHeaders.getAiTrackingHeaders)({
+      environment: (0, _environmentVariables.getEnvironmentVariable)("RUN_ENVIRONMENT"),
+      type: "DYNAMIC_LIST",
+      runId: (_getExecutionContext = getExecutionContext()) === null || _getExecutionContext === void 0 ? void 0 : _getExecutionContext.runId,
+      jobId: (_getExecutionContext2 = getExecutionContext()) === null || _getExecutionContext2 === void 0 ? void 0 : _getExecutionContext2.jobId,
+      jobRunId: (_getExecutionContext3 = getExecutionContext()) === null || _getExecutionContext3 === void 0 ? void 0 : _getExecutionContext3.jobRunId,
+      queueId: (_getExecutionContext4 = getExecutionContext()) === null || _getExecutionContext4 === void 0 ? void 0 : _getExecutionContext4.queueId
+    })
+  }), error => {
+    const typedError = error;
+    if (typedError.status === 449) return Errors.insufficientAiCredits(`🔴 ${typedError.error.error}`);
+  });
+  if (response.isErr() && ((_response$error = response.error) === null || _response$error === void 0 ? void 0 : _response$error.type) === "InsufficientAiCredits") {
+    return (0, _neverthrow.err)(response.error);
+  }
+  if (response.isErr()) {
+    return (0, _neverthrow.ok)({
+      headers: []
+    });
+  }
+  const unwrappedResponse = response.value;
+  const tool = unwrappedResponse.content.find(t => t.type === "tool_use");
+  if (!tool) return (0, _neverthrow.ok)({
+    headers: []
+  });
+  const headersParsingResults = _zod.z.object({
+    headers: _zod.z.array(_zod.z.object({
+      column_number: _zod.z.number(),
+      header: _zod.z.string()
+    }))
+  }).safeParse(tool.input);
+  if (!headersParsingResults.success) {
+    return (0, _neverthrow.ok)({
+      headers: []
+    });
+  }
+  return (0, _neverthrow.ok)({
+    headers: headersParsingResults.data.headers.map(i => i.header)
+  });
+}
+async function cropImage(image, handle, {
+  height: cropHeight,
+  width: cropWidth,
+  x,
+  y
+}) {
+  const base64Image = image.toString("base64");
+  const dataUrl = `data:image/jpeg;base64,${base64Image}`;
+  const croppedDataUrl = await Promise.race([handle.evaluate((_, {
+    cropHeight,
+    cropWidth,
+    x,
+    y,
+    url
+  }) => {
+    return new Promise(resolve => {
+      const img = document.createElement("img");
+      const canvas = document.createElement("canvas");
+      document.body.appendChild(img);
+      img.onload = () => {
+        canvas.width = cropWidth;
+        canvas.height = cropHeight;
+        const ctx = canvas.getContext("2d");
+        ctx.drawImage(img, x, y, cropWidth, cropHeight, 0, 0, cropWidth, cropHeight);
+        const resultImage = canvas.toDataURL().split(";base64,").pop();
+        resolve(resultImage);
+        img.remove();
+        canvas.remove();
+      };
+      img.src = url;
+    });
+  }, {
+    url: dataUrl,
+    x,
+    y,
+    cropWidth,
+    cropHeight
+  }), new Promise(resolve => setTimeout(() => {
+    resolve(undefined);
+  }, 5000))]);
+  if (!croppedDataUrl) return undefined;
+  return Buffer.from(croppedDataUrl, "base64");
+}

package/dist/optimized-extractors/common/index.js

@@ -0,0 +1,55 @@
+"use strict";
+
+Object.defineProperty(exports, "__esModule", {
+  value: true
+});
+exports.extractStructuredDataUsingAi = extractStructuredDataUsingAi;
+exports.isClaudeModel = isClaudeModel;
+exports.isGoogleModel = isGoogleModel;
+exports.isOpenAiModel = isOpenAiModel;
+var _neverthrow = require("neverthrow");
+var _extractStructuredDataUsingClaude = require("./extractStructuredDataUsingClaude");
+var _extractStructuredDataUsingOpenAi = require("./extractStructuredDataUsingOpenAi");
+var _utils = require("./utils");
+var _aiModelsValidations = require("../../common/aiModelsValidations");
+var _extractStructuredDataUsingGoogle = require("./extractStructuredDataUsingGoogle");
+function isClaudeModel(model) {
+  return _aiModelsValidations.CLAUDE_MODELS.includes(model);
+}
+function isGoogleModel(model) {
+  return _aiModelsValidations.GOOGLE_MODELS.includes(model);
+}
+function isOpenAiModel(model) {
+  return _aiModelsValidations.GPT_MODELS.includes(model);
+}
+async function extractStructuredDataUsingAi(input) {
+  let extractionResult;
+  const {
+    model
+  } = input;
+  if (isOpenAiModel(model)) {
+    extractionResult = await (0, _extractStructuredDataUsingOpenAi.extractStructuredDataUsingOpenAi)({
+      ...input,
+      model
+    });
+  } else if (isGoogleModel(model)) {
+    extractionResult = await (0, _extractStructuredDataUsingGoogle.extractStructuredDataUsingGoogle)({
+      ...input,
+      model
+    });
+  } else {
+    extractionResult = await (0, _extractStructuredDataUsingClaude.extractStructuredDataUsingClaude)({
+      ...input,
+      model
+    });
+  }
+  if (extractionResult.isErr()) {
+    return (0, _neverthrow.err)(extractionResult.error);
+  }
+  const {
+    result
+  } = extractionResult.value;
+  return (0, _neverthrow.ok)({
+    result: (0, _utils.cleanupAiResult)(result)
+  });
+}

package/dist/optimized-extractors/common/isTableHeaderOrFooter.js

@@ -0,0 +1,97 @@
+"use strict";
+
+Object.defineProperty(exports, "__esModule", {
+  value: true
+});
+exports.isTableHeaderOrFooter = isTableHeaderOrFooter;
+var _anthropicModel = require("../models/anthropicModel");
+var _zod = require("zod");
+var _neverthrow = require("neverthrow");
+var Errors = _interopRequireWildcard(require("../types/errors"));
+var _getAiTrackingHeaders = require("../../common/eventTracking/getAiTrackingHeaders");
+var _environmentVariables = require("../../common/environmentVariables");
+var _loadRuntime = require("../../common/loadRuntime");
+function _interopRequireWildcard(e, t) { if ("function" == typeof WeakMap) var r = new WeakMap(), n = new WeakMap(); return (_interopRequireWildcard = function (e, t) { if (!t && e && e.__esModule) return e; var o, i, f = { __proto__: null, default: e }; if (null === e || "object" != typeof e && "function" != typeof e) return f; if (o = t ? n : r) { if (o.has(e)) return o.get(e); o.set(e, f); } for (const t in e) "default" !== t && {}.hasOwnProperty.call(e, t) && ((i = (o = Object.defineProperty) && Object.getOwnPropertyDescriptor(e, t)) && (i.get || i.set) ? o(f, t, i) : f[t] = e[t]); return f; })(e, t); }
+async function isTableHeaderOrFooter(content) {
+  var _getExecutionContext, _getExecutionContext2, _getExecutionContext3, _getExecutionContext4, _response$error;
+  if (!content) {
+    return (0, _neverthrow.ok)({
+      isHeader: false
+    });
+  }
+  const anthropic = (0, _anthropicModel.createAnthropicInstance)();
+  const getExecutionContext = await (0, _loadRuntime.loadRuntime)();
+  const itemContent = typeof content === "string" ? {
+    type: "text",
+    text: content
+  } : {
+    type: "image",
+    source: {
+      data: content.toString("base64"),
+      media_type: "image/png",
+      type: "base64"
+    }
+  };
+  const response = await (0, _neverthrow.fromPromise)(anthropic.messages.create({
+    max_tokens: 4096,
+    temperature: 0,
+    messages: [{
+      role: "user",
+      content: [{
+        type: "text",
+        text: `given the following content, you need to decide if the content is a html table header or a table footer, 
+            a table header is a row that contains labels for table columns, and footer usually has pagination information or summary of the table`
+      }, itemContent]
+    }],
+    model: "claude-3-haiku-20240307",
+    tools: [{
+      input_schema: {
+        type: "object",
+        properties: {
+          isTableHeaderOrFooter: {
+            type: "boolean"
+          }
+        },
+        required: ["isTableHeaderOrFooter"]
+      },
+      name: "is_table_header_or_footer",
+      description: `given a text or image content, decide if the content is a table header or footer or not.`
+    }]
+  }, {
+    headers: (0, _getAiTrackingHeaders.getAiTrackingHeaders)({
+      environment: (0, _environmentVariables.getEnvironmentVariable)("RUN_ENVIRONMENT"),
+      type: "DYNAMIC_LIST",
+      runId: (_getExecutionContext = getExecutionContext()) === null || _getExecutionContext === void 0 ? void 0 : _getExecutionContext.runId,
+      jobId: (_getExecutionContext2 = getExecutionContext()) === null || _getExecutionContext2 === void 0 ? void 0 : _getExecutionContext2.jobId,
+      jobRunId: (_getExecutionContext3 = getExecutionContext()) === null || _getExecutionContext3 === void 0 ? void 0 : _getExecutionContext3.jobRunId,
+      queueId: (_getExecutionContext4 = getExecutionContext()) === null || _getExecutionContext4 === void 0 ? void 0 : _getExecutionContext4.queueId
+    })
+  }), error => {
+    const typedError = error;
+    if (typedError.status === 449) return Errors.insufficientAiCredits(`🔴 ${typedError.error.error}`);
+  });
+  if (response.isErr() && ((_response$error = response.error) === null || _response$error === void 0 ? void 0 : _response$error.type) === "InsufficientAiCredits") {
+    return (0, _neverthrow.err)(response.error);
+  }
+  if (response.isErr()) {
+    return (0, _neverthrow.ok)({
+      isHeader: false
+    });
+  }
+  const unwrappedResponse = response.value;
+  const tool = unwrappedResponse.content.find(t => t.type === "tool_use");
+  if (!tool) return (0, _neverthrow.ok)({
+    isHeader: false
+  });
+  const headersParsingResults = _zod.z.object({
+    isTableHeaderOrFooter: _zod.z.boolean()
+  }).safeParse(tool.input);
+  if (!headersParsingResults.success) {
+    return (0, _neverthrow.ok)({
+      isHeader: false
+    });
+  }
+  return (0, _neverthrow.ok)({
+    isHeader: headersParsingResults.data.isTableHeaderOrFooter
+  });
+}

package/dist/optimized-extractors/common/matching/matching.js

@@ -0,0 +1,212 @@
+"use strict";
+
+Object.defineProperty(exports, "__esModule", {
+  value: true
+});
+exports.MatchSource = exports.MatchMode = void 0;
+exports.filterAndRankMatches = filterAndRankMatches;
+exports.matchStringsWithDomContent = matchStringsWithDomContent;
+exports.matchStringsWithDomContentInBrowser = matchStringsWithDomContentInBrowser;
+exports.normalizeSpacing = normalizeSpacing;
+exports.rankMatch = rankMatch;
+exports.removePunctuationAndSpaces = removePunctuationAndSpaces;
+exports.replaceWithBestMatches = replaceWithBestMatches;
+exports.selectBestMatch = selectBestMatch;
+var _levenshtein = require("../../../common/matching/levenshtein");
+var _ensureBrowserScripts = require("../../../common/ensureBrowserScripts");
+let MatchMode = exports.MatchMode = function (MatchMode) {
+  MatchMode["FULL"] = "full";
+  MatchMode["PARTIAL"] = "partial";
+  MatchMode["FUZZY"] = "fuzzy";
+  return MatchMode;
+}({});
+let MatchSource = exports.MatchSource = function (MatchSource) {
+  MatchSource["ATTRIBUTE"] = "attribute";
+  MatchSource["TEXT_CONTENT"] = "text_content";
+  MatchSource["DIRECT_TEXT_NODE"] = "direct_text_node";
+  return MatchSource;
+}({});
+function normalizeSpacing(text) {
+  let normalized = text.replace(/\n/g, " ").replace(/\t/g, " ");
+  normalized = normalized.split(/\s+/).join(" ");
+  return normalized.trim();
+}
+function removePunctuationAndSpaces(s) {
+  return s.replace(/[!"#$%&'()*+,\-./:;<=>?@[\\\]^_`{|}~\s]/g, "");
+}
+function rankMatch(original, match) {
+  try {
+    const normalizedOriginal = normalizeSpacing(original).toLowerCase();
+    const normalizedMatch = normalizeSpacing(match).toLowerCase();
+    const ratio = (0, _levenshtein.levenshteinRatio)(normalizedOriginal, normalizedMatch);
+    const lenOriginal = normalizedOriginal.length;
+    if (lenOriginal > 20 && ratio > 0.85) {
+      return "HIGH";
+    }
+    const normalizedOriginalNoPunct = removePunctuationAndSpaces(normalizedOriginal);
+    const normalizedMatchNoPunct = removePunctuationAndSpaces(normalizedMatch);
+    if (normalizedOriginalNoPunct === normalizedMatchNoPunct) {
+      return "HIGH";
+    }
+    return "LOW";
+  } catch (error) {
+    console.warn("Error in rankMatch, falling back to simple comparison", error);
+    const normalizedOriginal = removePunctuationAndSpaces(normalizeSpacing(original).toLowerCase());
+    const normalizedMatch = removePunctuationAndSpaces(normalizeSpacing(match).toLowerCase());
+    if (normalizedOriginal === normalizedMatch) {
+      return "HIGH";
+    }
+    return "LOW";
+  }
+}
+function selectBestMatch(original, matches) {
+  const exactMatches = matches.filter(match => match.match_mode !== MatchMode.FUZZY);
+  if (exactMatches.length > 0) {
+    return exactMatches[0].matched_value;
+  }
+  const fuzzyMatches = matches.filter(match => match.match_mode === MatchMode.FUZZY);
+  const rankedFuzzyMatches = fuzzyMatches.map(match => ({
+    match,
+    rank: rankMatch(original, match.matched_value)
+  })).filter(({
+    rank
+  }) => rank === "HIGH");
+  if (rankedFuzzyMatches.length > 0) {
+    const sortedFuzzyMatches = rankedFuzzyMatches.sort((a, b) => {
+      const distA = a.match.fuzzy_distance ?? Infinity;
+      const distB = b.match.fuzzy_distance ?? Infinity;
+      return distA - distB;
+    });
+    return sortedFuzzyMatches[0].match.matched_value;
+  }
+  return null;
+}
+async function matchStringsWithDomContent(pageObject, stringsList, container) {
+  try {
+    await (0, _ensureBrowserScripts.ensureBrowserScripts)(pageObject);
+    let handle;
+    if (container) {
+      handle = container;
+    } else {
+      handle = await pageObject.locator("html").elementHandle();
+    }
+    const matches = await pageObject.evaluate(async ([container, searchTexts]) => {
+      try {
+        if (typeof window.__INTUNED__ !== "undefined" && typeof window.__INTUNED__.matchStringsWithDomContent === "function") {
+          return await window.__INTUNED__.matchStringsWithDomContent(container, searchTexts);
+        } else {
+          return searchTexts.reduce((acc, text) => {
+            acc[text] = [];
+            return acc;
+          }, {});
+        }
+      } catch (error) {
+        console.error("Error matching strings with DOM content:", error);
+        return searchTexts.reduce((acc, text) => {
+          acc[text] = [];
+          return acc;
+        }, {});
+      }
+    }, [handle, stringsList]);
+    return matches;
+  } catch (e) {
+    console.warn("Error matching strings with DOM content:", e);
+    return stringsList.reduce((acc, string) => {
+      acc[string] = [];
+      return acc;
+    }, {});
+  }
+}
+async function replaceWithBestMatches(stringsToMatch, pageObject, container) {
+  const matchesMap = await matchStringsWithDomContent(pageObject, stringsToMatch, container);
+  return Object.fromEntries(Object.entries(matchesMap).map(([string, matches]) => [string, matches.length > 0 ? selectBestMatch(string, matches) : string]));
+}
+async function filterAndRankMatches(frame, matches) {
+  const filteredMatches = matches.filter(match => {
+    const xpath = match.xpath;
+    return !(xpath.includes("[name()='svg']/") || xpath.includes("/path") || xpath.includes("/style"));
+  });
+  async function getVisibility(match) {
+    try {
+      const locator = frame.locator(`xpath=${match.xpath}`);
+      return await locator.isVisible({
+        timeout: 100
+      });
+    } catch {
+      return false;
+    }
+  }
+  const visibilityPromises = filteredMatches.map(match => getVisibility(match));
+  const visibilityResults = await Promise.all(visibilityPromises);
+  const modeOrder = {
+    full: 3,
+    partial: 2,
+    fuzzy: 1
+  };
+  const sourceOrder = {
+    direct_text_node: 3,
+    text_content: 2,
+    attribute: 1
+  };
+  function sortKey(match, visible) {
+    const modeKey = modeOrder[match.match_mode.toLowerCase()] || 0;
+    const sourceKey = sourceOrder[match.match_source.toLowerCase()] || 0;
+    const visibleKey = visible ? 1 : 0;
+    let partialScore = 0;
+    if (match.match_mode.toLowerCase() === "partial" && match.matched_source_value && match.matched_value) {
+      const extraChars = match.matched_source_value.length - match.matched_value.length;
+      partialScore = extraChars === 0 ? 1 : 1 / extraChars;
+    }
+    return modeKey + sourceKey + visibleKey + partialScore;
+  }
+  const matchesWithVisibility = filteredMatches.map((match, index) => ({
+    match,
+    visible: visibilityResults[index]
+  }));
+  const sortedMatchesWithVisibility = matchesWithVisibility.sort((a, b) => sortKey(b.match, b.visible) - sortKey(a.match, a.visible));
+  const sortedMatches = sortedMatchesWithVisibility.map(item => item.match);
+  const seenXpaths = new Set();
+  const uniqueMatches = [];
+  for (const match of sortedMatches) {
+    if (seenXpaths.has(match.xpath)) {
+      continue;
+    }
+    seenXpaths.add(match.xpath);
+    uniqueMatches.push(match);
+  }
+  return uniqueMatches;
+}
+async function matchStringsWithDomContentInBrowser(frame, stringsList, container) {
+  let handle;
+  if (container) {
+    handle = await container.elementHandle();
+  } else {
+    handle = await frame.locator("html").elementHandle();
+  }
+  console.info(`Searching for ${stringsList.length} strings in the DOM:`, stringsList);
+  const matches = await frame.evaluate(async ([container, searchTexts]) => {
+    const result = await window.__INTUNED__.matchStringsWithDomContent(container, searchTexts);
+    return result;
+  }, [handle, stringsList]);
+  let frames = [];
+  if ("childFrames" in frame) {
+    frames = frame.childFrames();
+  } else if ("frames" in frame) {
+    frames = frame.frames();
+  }
+  for (const subframe of frames) {
+    try {
+      const frameMatches = await matchStringsWithDomContentInBrowser(subframe, stringsList, null);
+      for (const [string, stringMatches] of Object.entries(frameMatches)) {
+        if (string in matches) {
+          matches[string].push(...stringMatches);
+        } else {
+          matches[string] = stringMatches;
+        }
+      }
+    } catch {
+      continue;
+    }
+  }
+  return matches;
+}