@intuned/browser-dev 0.1.4-dev.0

package/dist/optimized-extractors/common/extractStructuredDataUsingClaude.js

@@ -0,0 +1,135 @@
+"use strict";
+
+Object.defineProperty(exports, "__esModule", {
+  value: true
+});
+exports.extractStructuredDataUsingClaude = extractStructuredDataUsingClaude;
+var _anthropicModel = require("../models/anthropicModel");
+var _neverthrow = require("neverthrow");
+var Errors = _interopRequireWildcard(require("../types/errors"));
+var _utils = require("./utils");
+var _Logger = require("../../common/Logger");
+var _aiModelsValidations = require("../../common/aiModelsValidations");
+function _interopRequireWildcard(e, t) { if ("function" == typeof WeakMap) var r = new WeakMap(), n = new WeakMap(); return (_interopRequireWildcard = function (e, t) { if (!t && e && e.__esModule) return e; var o, i, f = { __proto__: null, default: e }; if (null === e || "object" != typeof e && "function" != typeof e) return f; if (o = t ? n : r) { if (o.has(e)) return o.get(e); o.set(e, f); } for (const t in e) "default" !== t && {}.hasOwnProperty.call(e, t) && ((i = (o = Object.defineProperty) && Object.getOwnPropertyDescriptor(e, t)) && (i.get || i.set) ? o(f, t, i) : f[t] = e[t]); return f; })(e, t); }
+async function extractStructuredDataUsingClaude(input) {
+  const {
+    entityName,
+    model,
+    jsonSchema: originalJsonSchema,
+    systemMessage,
+    text,
+    extraUserMessages,
+    images,
+    apiKey
+  } = input;
+  const processedJsonSchema = (0, _utils.processInputSchema)(originalJsonSchema, entityName);
+  const toolName = `extract_${entityName}`;
+  const content = [];
+  if (extraUserMessages) {
+    const mappedExtraMessages = extraUserMessages.map(message => ({
+      type: "text",
+      text: message
+    }));
+    content.push(...mappedExtraMessages);
+  }
+  content.push({
+    type: "text",
+    text: `extract ` + entityName + ` from the text and images, Use must the ` + toolName + ` tool, ` + (originalJsonSchema.description ? entityName + ` is ` + originalJsonSchema.description : "")
+  });
+  if (text) {
+    content.push(...text.map(t => ({
+      type: "text",
+      text: t
+    })));
+  }
+  if (images) {
+    const imagesContent = images.map(image => ({
+      type: "image",
+      source: {
+        data: image.data.toString("base64"),
+        media_type: `image/${image.image_type}`,
+        type: "base64"
+      }
+    }));
+    content.push(...imagesContent);
+  }
+  const anthropic = (0, _anthropicModel.createAnthropicInstance)({
+    apiKey
+  });
+  const modelName = _aiModelsValidations.CLAUDE_MODELS_MAPPINGS[model] ?? model;
+  const maxTokens = _aiModelsValidations.MAX_TOKENS_OVERRIDES[modelName] ?? 4096;
+  const response = await (0, _neverthrow.fromPromise)(anthropic.messages.create({
+    max_tokens: maxTokens,
+    temperature: 0,
+    system: `${systemMessage ?? ""}`,
+    messages: [{
+      role: "user",
+      content
+    }],
+    model: modelName,
+    tools: [{
+      input_schema: processedJsonSchema,
+      name: toolName,
+      description: `Extract ` + entityName + ` mentioned in the text or images. Relay on the parameters for more info. always use this tool when you are asked to extract data. you should respect the provided schema even if it was multiple levels deep, undefined is not an accepted input, please never respond with it.`
+    }, {
+      name: "no_data_found",
+      description: `you should call this tool you are asked to extract data using ` + toolName + ` and you couldn't find any data, make this your last resort, if you are sure that there is no data in the text or images.`,
+      input_schema: {
+        type: "object",
+        properties: null
+      }
+    }]
+  }).withResponse(), error => {
+    var _typedError$error, _typedError$error2;
+    const typedError = error;
+    if ((typedError === null || typedError === void 0 ? void 0 : typedError.status) === 449) {
+      return Errors.insufficientAiCredits(typedError.error.error);
+    }
+    if ((typedError === null || typedError === void 0 ? void 0 : typedError.status) === 413) {
+      return Errors.AiCallFailed("content should not exceed 10mb", error);
+    }
+    if ((typedError === null || typedError === void 0 ? void 0 : typedError.status) === 400 && typeof (typedError === null || typedError === void 0 || (_typedError$error = typedError.error) === null || _typedError$error === void 0 ? void 0 : _typedError$error.error) === "object" && typedError !== null && typedError !== void 0 && (_typedError$error2 = typedError.error) !== null && _typedError$error2 !== void 0 && (_typedError$error2 = _typedError$error2.error) !== null && _typedError$error2 !== void 0 && (_typedError$error2 = _typedError$error2.message) !== null && _typedError$error2 !== void 0 && _typedError$error2.includes("prompt is too long:")) {
+      return Errors.AiCallFailed("content exceeded model max capacity", error);
+    }
+    return Errors.AiCallFailed(`Failed to call claude api with status ${typedError === null || typedError === void 0 ? void 0 : typedError.status}: ${error.message}`, error);
+  });
+  if (response.isErr()) {
+    return (0, _neverthrow.err)(response.error);
+  }
+  const unwrappedResponse = response.value.data;
+  if (unwrappedResponse.stop_reason === "max_tokens") {
+    return (0, _neverthrow.err)(Errors.AiCallFailed("response from ai exceeds model maximum output tokens, try to be more specific with what data you need to extract"));
+  }
+  if (unwrappedResponse.stop_reason !== "tool_use") {
+    return (0, _neverthrow.err)(Errors.NoToolUsage("the model was not able to extract data correctly, please try to modify your prompt and schema to give more context."));
+  }
+  const noDataFound = unwrappedResponse.content.some(content => content.type === "tool_use" && content.name == "no_data_found");
+  if (noDataFound) {
+    return (0, _neverthrow.err)(Errors.NoDataFound("data isn't found in the text or images."));
+  }
+  const tool = unwrappedResponse.content.find(content => content.type === "tool_use" && content.name == toolName);
+  if (!tool) {
+    return (0, _neverthrow.err)(Errors.NoToolUsage("the model was not able to extract data correctly, please try to modify your prompt and schema to give more context."));
+  }
+  if (!tool.input) {
+    return (0, _neverthrow.err)(Errors.invalidExtractionResult("the model was not able to extract data correctly, please try to modify your prompt and schema to give more context."));
+  }
+  if (originalJsonSchema.type === "array" && tool.input[entityName] === "undefined") {
+    return (0, _neverthrow.err)(Errors.invalidExtractionResult("the model was not able to extract data correctly"));
+  }
+  const result = (0, _utils.getResultFromOutputSchema)(originalJsonSchema, entityName, tool.input);
+  const callCost = response.value.response.headers.get("x-ai-cost-in-cents");
+  if (input.logAiCallCost) {
+    if (apiKey) {
+      _Logger.logger.info(`extractor ${input.identifier}: AI cost is not calculated (using custom API key)`);
+    } else if (callCost) {
+      const cost = parseFloat(callCost);
+      if (!isNaN(cost)) {
+        _Logger.logger.info(`extractor ${input.identifier}: AI cost is $${cost / 100}`);
+      }
+    }
+  }
+  return (0, _neverthrow.ok)({
+    result
+  });
+}

package/dist/optimized-extractors/common/extractStructuredDataUsingGoogle.js

@@ -0,0 +1,37 @@
+"use strict";
+
+Object.defineProperty(exports, "__esModule", {
+  value: true
+});
+exports.extractStructuredDataUsingGoogle = extractStructuredDataUsingGoogle;
+var _aiModelsValidations = require("../../common/aiModelsValidations");
+var _extractStrucutredDataUsingAiInstance = require("./extractStrucutredDataUsingAiInstance");
+var Errors = _interopRequireWildcard(require("../types/errors"));
+var _neverthrow = require("neverthrow");
+var _google = require("@ai-sdk/google");
+function _interopRequireWildcard(e, t) { if ("function" == typeof WeakMap) var r = new WeakMap(), n = new WeakMap(); return (_interopRequireWildcard = function (e, t) { if (!t && e && e.__esModule) return e; var o, i, f = { __proto__: null, default: e }; if (null === e || "object" != typeof e && "function" != typeof e) return f; if (o = t ? n : r) { if (o.has(e)) return o.get(e); o.set(e, f); } for (const t in e) "default" !== t && {}.hasOwnProperty.call(e, t) && ((i = (o = Object.defineProperty) && Object.getOwnPropertyDescriptor(e, t)) && (i.get || i.set) ? o(f, t, i) : f[t] = e[t]); return f; })(e, t); }
+async function extractStructuredDataUsingGoogle(input) {
+  var _input$text;
+  if (!input.apiKey) {
+    return (0, _neverthrow.err)(Errors.invalidInput("Google AI is only supported with a custom API key. Please provide it or use a different AI provider."));
+  }
+  let model = input.model;
+  if (input.model in _aiModelsValidations.MODELS_MAPPINGS) {
+    model = _aiModelsValidations.MODELS_MAPPINGS[input.model];
+  }
+  const googleGenAi = (0, _google.createGoogleGenerativeAI)({
+    apiKey: input.apiKey
+  });
+  const extraMessages = input.extraUserMessages ?? [];
+  if (extraMessages.length === 0 && (((_input$text = input.text) === null || _input$text === void 0 ? void 0 : _input$text.length) ?? 0) === 0) {
+    var _input$images;
+    extraMessages.push("Use the following image" + ((((_input$images = input.images) === null || _input$images === void 0 ? void 0 : _input$images.length) ?? 0) > 1 ? "s" : ""));
+  }
+  return (0, _extractStrucutredDataUsingAiInstance.extractStructuredDataUsingAiInstance)({
+    ...input,
+    extraUserMessages: extraMessages,
+    model: googleGenAi(model),
+    apiName: "google",
+    maxTokens: _aiModelsValidations.MAX_TOKENS_OVERRIDES[model]
+  });
+}

package/dist/optimized-extractors/common/extractStructuredDataUsingOpenAi.js

@@ -0,0 +1,132 @@
+"use strict";
+
+Object.defineProperty(exports, "__esModule", {
+  value: true
+});
+exports.extractStructuredDataUsingOpenAi = extractStructuredDataUsingOpenAi;
+var _neverthrow = require("neverthrow");
+var Errors = _interopRequireWildcard(require("../types/errors"));
+var _utils = require("./utils");
+var _Logger = require("../../common/Logger");
+var _aiModelsValidations = require("../../common/aiModelsValidations");
+var _openaiModel = require("../models/openaiModel");
+function _interopRequireWildcard(e, t) { if ("function" == typeof WeakMap) var r = new WeakMap(), n = new WeakMap(); return (_interopRequireWildcard = function (e, t) { if (!t && e && e.__esModule) return e; var o, i, f = { __proto__: null, default: e }; if (null === e || "object" != typeof e && "function" != typeof e) return f; if (o = t ? n : r) { if (o.has(e)) return o.get(e); o.set(e, f); } for (const t in e) "default" !== t && {}.hasOwnProperty.call(e, t) && ((i = (o = Object.defineProperty) && Object.getOwnPropertyDescriptor(e, t)) && (i.get || i.set) ? o(f, t, i) : f[t] = e[t]); return f; })(e, t); }
+async function extractStructuredDataUsingOpenAi(input) {
+  var _completion$value$dat, _completion$value$dat2;
+  const {
+    entityName,
+    model,
+    jsonSchema: originalJsonSchema,
+    systemMessage,
+    text,
+    extraUserMessages,
+    images,
+    apiKey
+  } = input;
+  if (images && images.length > 0 && model === "gpt3.5-turbo") {
+    return (0, _neverthrow.err)(Errors.invalidInput("Images are only supported for gpt4-turbo and gpt-4o models"));
+  }
+  const processedJsonSchema = (0, _utils.processInputSchema)(originalJsonSchema, entityName);
+  const content = [];
+  if (extraUserMessages) {
+    content.push(...extraUserMessages.map(message => ({
+      type: "text",
+      text: message
+    })));
+  }
+  if (text) {
+    content.push(...text.map(t => ({
+      type: "text",
+      text: t
+    })));
+  }
+  if (images) {
+    const imageContent = images.map(image => ({
+      type: "image_url",
+      image_url: {
+        url: `data:image/${image.image_type};base64,${image.data.toString("base64")}`,
+        detail: "high"
+      }
+    }));
+    content.push(...imageContent);
+  }
+  const modelName = _aiModelsValidations.MODELS_MAPPINGS[input.model] ?? input.model;
+  const toolName = `extract_${entityName}`;
+  const openAiInstance = (0, _openaiModel.createOpenAIInstance)({
+    apiKey
+  });
+  const completion = await (0, _neverthrow.fromPromise)(openAiInstance.chat.completions.create({
+    max_tokens: 4000,
+    temperature: 0,
+    model: modelName,
+    messages: [{
+      role: "system",
+      content: `${systemMessage ?? ""}, ` + `using the ` + (originalJsonSchema.description ? `. Here is more info about the entity that we are trying to extract: ` + originalJsonSchema.description : "")
+    }, {
+      role: "user",
+      content
+    }],
+    function_call: {
+      name: toolName
+    },
+    functions: [{
+      name: "no_data_found",
+      description: `you should call this tool you are asked to extract data using ` + toolName + ` and you couldn't find any data, make this your last resort, if you are sure that there is no data in the text or images`
+    }, {
+      name: toolName,
+      description: `Extract ` + entityName + ` mentioned in the text and images. Relay on the parameters for more info.`,
+      parameters: processedJsonSchema
+    }]
+  }).withResponse(), error => {
+    const typedError = error;
+    if ((typedError === null || typedError === void 0 ? void 0 : typedError.status) === 449) {
+      var _typedError$error;
+      return Errors.insufficientAiCredits(typedError === null || typedError === void 0 || (_typedError$error = typedError.error) === null || _typedError$error === void 0 ? void 0 : _typedError$error.error);
+    }
+    if ((error === null || error === void 0 ? void 0 : error.status) === 400 && ["context_length_exceeded", "string_above_max_length"].includes(error === null || error === void 0 ? void 0 : error.code)) {
+      return Errors.AiCallFailed("content exceeded model max capacity", error);
+    }
+    return Errors.AiCallFailed(`Failed to call openai api with status ${typedError === null || typedError === void 0 ? void 0 : typedError.status}: ${error.message}`, error);
+  });
+  if (completion.isErr()) {
+    return (0, _neverthrow.err)(completion.error);
+  }
+  if (completion.value.data.choices[0].finish_reason === "length") {
+    return (0, _neverthrow.err)(Errors.AiCallFailed("response from ai exceeds model maximum output tokens, try to be more specific with what data you need to extract"));
+  }
+  const noDataFound = (_completion$value$dat = completion.value.data.choices[0].message.tool_calls) === null || _completion$value$dat === void 0 ? void 0 : _completion$value$dat.some(content => content.type === "function" && content.function.name == "no_data_found");
+  if (noDataFound) {
+    return (0, _neverthrow.err)(Errors.NoDataFound("data isn't found in the text or images."));
+  }
+  let functionCall = (_completion$value$dat2 = completion.value.data.choices[0].message.tool_calls) === null || _completion$value$dat2 === void 0 || (_completion$value$dat2 = _completion$value$dat2.find(t => t.type === "function" && t.function.name === toolName)) === null || _completion$value$dat2 === void 0 ? void 0 : _completion$value$dat2.function;
+  if (!functionCall) {
+    functionCall = completion.value.data.choices[0].message.function_call;
+    if (!functionCall) {
+      return (0, _neverthrow.err)(Errors.NoToolUsage("the model was not able to extract data correctly."));
+    }
+  }
+  const extractedData = functionCall.arguments;
+  if (!extractedData) return (0, _neverthrow.err)(Errors.invalidExtractionResult("No extraction result found."));
+  const escapedString = extractedData.replace(/[\t\n\r\f\v]/g, " ");
+  const jsonParse = (0, _neverthrow.fromThrowable)(JSON.parse, () => Errors.invalidExtractionResult("Failed to parse extraction result."));
+  const parsedData = jsonParse(escapedString);
+  if (parsedData.isErr()) {
+    return (0, _neverthrow.err)(parsedData.error);
+  }
+  const result = (0, _utils.getResultFromOutputSchema)(originalJsonSchema, entityName, parsedData.value);
+  const formatted = (0, _utils.cleanupAiResult)(result);
+  const callCost = completion.value.response.headers.get("x-ai-cost-in-cents");
+  if (input.logAiCallCost) {
+    if (apiKey) {
+      _Logger.logger.info(`extractor ${input.identifier}: AI cost is not calculated (using custom API key)`);
+    } else if (callCost) {
+      const cost = parseFloat(callCost);
+      if (!isNaN(cost)) {
+        _Logger.logger.info(`extractor ${input.identifier}: AI cost is $${cost / 100}`);
+      }
+    }
+  }
+  return (0, _neverthrow.ok)({
+    result: formatted
+  });
+}

package/dist/optimized-extractors/common/extractStrucutredDataUsingAiInstance.js

@@ -0,0 +1,122 @@
+"use strict";
+
+Object.defineProperty(exports, "__esModule", {
+  value: true
+});
+exports.extractStructuredDataUsingAiInstance = extractStructuredDataUsingAiInstance;
+var _neverthrow = require("neverthrow");
+var Errors = _interopRequireWildcard(require("../types/errors"));
+var _utils = require("./utils");
+var _Logger = require("../../common/Logger");
+var _ai = require("ai");
+function _interopRequireWildcard(e, t) { if ("function" == typeof WeakMap) var r = new WeakMap(), n = new WeakMap(); return (_interopRequireWildcard = function (e, t) { if (!t && e && e.__esModule) return e; var o, i, f = { __proto__: null, default: e }; if (null === e || "object" != typeof e && "function" != typeof e) return f; if (o = t ? n : r) { if (o.has(e)) return o.get(e); o.set(e, f); } for (const t in e) "default" !== t && {}.hasOwnProperty.call(e, t) && ((i = (o = Object.defineProperty) && Object.getOwnPropertyDescriptor(e, t)) && (i.get || i.set) ? o(f, t, i) : f[t] = e[t]); return f; })(e, t); }
+async function extractStructuredDataUsingAiInstance(input) {
+  var _apiResult$value$tool, _apiResult$value$tool2, _apiResult$value$resp;
+  const {
+    entityName,
+    model,
+    jsonSchema: originalJsonSchema,
+    systemMessage,
+    text,
+    extraUserMessages,
+    images,
+    apiKey,
+    apiName,
+    maxTokens
+  } = input;
+  const processedJsonSchema = (0, _utils.processInputSchema)(originalJsonSchema, entityName);
+  const content = [];
+  if (extraUserMessages) {
+    content.push(...extraUserMessages.map(message => ({
+      type: "text",
+      text: message
+    })));
+  }
+  if (text) {
+    content.push(...text.map(t => ({
+      type: "text",
+      text: t
+    })));
+  }
+  if (images) {
+    const imageContent = images.map(image => ({
+      type: "image",
+      image: image.data,
+      mimeType: image.image_type
+    }));
+    content.push(...imageContent);
+  }
+  const toolName = `extract_${entityName}`;
+  const apiResult = await (0, _neverthrow.fromPromise)((0, _ai.generateText)({
+    model,
+    temperature: 0,
+    messages: [{
+      role: "system",
+      content: `${systemMessage ?? ""}, ` + `using the ` + (originalJsonSchema.description ? `. Here is more info about the entity that we are trying to extract: ` + originalJsonSchema.description : "")
+    }, {
+      role: "user",
+      content
+    }],
+    toolChoice: "required",
+    tools: {
+      [toolName]: (0, _ai.tool)({
+        description: `Extract ${entityName} mentioned in the text and images. Rely on the parameters for more info.`,
+        inputSchema: (0, _ai.jsonSchema)(processedJsonSchema),
+        type: "function"
+      }),
+      no_data_found: {
+        description: `you should call this tool you are asked to extract data using ${toolName} and you couldn't find any data, make this your last resort, if you are sure that there is no data in the text or images`,
+        type: "function",
+        inputSchema: (0, _ai.jsonSchema)({
+          type: "object",
+          properties: {}
+        })
+      }
+    }
+  }), error => {
+    if (error instanceof _ai.APICallError) {
+      const responseBodyStr = error.responseBody;
+      const responseBodyResult = (0, _neverthrow.fromThrowable)(JSON.parse, () => null)(responseBodyStr ?? "");
+      const responseBody = responseBodyResult.isOk() ? responseBodyResult.value : null;
+      if ((error === null || error === void 0 ? void 0 : error.statusCode) === 449) {
+        return Errors.insufficientAiCredits(responseBody === null || responseBody === void 0 ? void 0 : responseBody.error);
+      }
+      return Errors.AiCallFailed(`Failed to call ${apiName} api with status ${error === null || error === void 0 ? void 0 : error.statusCode}: ${error.message}`, error);
+    }
+    return Errors.AiCallFailed(`Failed to call ${apiName} api: ${error.message}`, error);
+  });
+  if (apiResult.isErr()) {
+    return (0, _neverthrow.err)(apiResult.error);
+  }
+  if (apiResult.value.finishReason === "length") {
+    return (0, _neverthrow.err)(Errors.AiCallFailed("response from ai exceeds model maximum output tokens, try to be more specific with what data you need to extract"));
+  }
+  const noDataFound = (_apiResult$value$tool = apiResult.value.toolCalls) === null || _apiResult$value$tool === void 0 ? void 0 : _apiResult$value$tool.some(content => content.toolName === "no_data_found");
+  if (noDataFound) {
+    return (0, _neverthrow.err)(Errors.NoDataFound("data isn't found in the text or images."));
+  }
+  const toolResult = (_apiResult$value$tool2 = apiResult.value.toolCalls) === null || _apiResult$value$tool2 === void 0 ? void 0 : _apiResult$value$tool2.find(content => content.toolName === toolName);
+  if (!toolResult) {
+    return (0, _neverthrow.err)(Errors.NoToolUsage("the model was not able to extract data correctly."));
+  }
+  const extractedData = toolResult.input;
+  if (!extractedData) {
+    return (0, _neverthrow.err)(Errors.invalidExtractionResult("No extraction result found."));
+  }
+  const result = (0, _utils.getResultFromOutputSchema)(originalJsonSchema, entityName, extractedData);
+  const formatted = (0, _utils.cleanupAiResult)(result);
+  const callCost = (_apiResult$value$resp = apiResult.value.response.headers) === null || _apiResult$value$resp === void 0 ? void 0 : _apiResult$value$resp["x-ai-cost-in-cents"];
+  if (input.logAiCallCost) {
+    if (apiKey) {
+      _Logger.logger.info(`extractor ${input.identifier}: AI cost is not calculated (using custom API key)`);
+    } else if (callCost) {
+      const cost = parseFloat(callCost);
+      if (!isNaN(cost)) {
+        _Logger.logger.info(`extractor ${input.identifier}: AI cost is $${cost / 100}`);
+      }
+    }
+  }
+  return (0, _neverthrow.ok)({
+    result: formatted
+  });
+}

package/dist/optimized-extractors/common/findTableHeaders.js

@@ -0,0 +1,162 @@
+"use strict";
+
+Object.defineProperty(exports, "__esModule", {
+  value: true
+});
+exports.getTableHeadersUsingAi = getTableHeadersUsingAi;
+var _anthropicModel = require("../models/anthropicModel");
+var _zod = require("zod");
+var _imageSize = require("image-size");
+var _neverthrow = require("neverthrow");
+var Errors = _interopRequireWildcard(require("../types/errors"));
+function _interopRequireWildcard(e, t) { if ("function" == typeof WeakMap) var r = new WeakMap(), n = new WeakMap(); return (_interopRequireWildcard = function (e, t) { if (!t && e && e.__esModule) return e; var o, i, f = { __proto__: null, default: e }; if (null === e || "object" != typeof e && "function" != typeof e) return f; if (o = t ? n : r) { if (o.has(e)) return o.get(e); o.set(e, f); } for (const t in e) "default" !== t && {}.hasOwnProperty.call(e, t) && ((i = (o = Object.defineProperty) && Object.getOwnPropertyDescriptor(e, t)) && (i.get || i.set) ? o(f, t, i) : f[t] = e[t]); return f; })(e, t); }
+async function getTableHeadersUsingAi(handle, identifier) {
+  var _response$error;
+  let image;
+  try {
+    image = await handle.screenshot({
+      type: "png",
+      timeout: 3_000
+    });
+  } catch (e) {
+    return (0, _neverthrow.ok)({
+      headers: []
+    });
+  }
+  const {
+    width,
+    height
+  } = (0, _imageSize.imageSize)(image);
+  if (!width) return (0, _neverthrow.ok)({
+    headers: []
+  });
+  const imageToUse = height && height > 500 ? await cropImage(image, handle, {
+    height: 500,
+    width,
+    x: 0,
+    y: 0
+  }) : image;
+  if (!imageToUse) return (0, _neverthrow.ok)({
+    headers: []
+  });
+  const anthropic = (0, _anthropicModel.createAnthropicInstance)();
+  const response = await (0, _neverthrow.fromPromise)(anthropic.messages.create({
+    max_tokens: 4096,
+    temperature: 0,
+    messages: [{
+      role: "user",
+      content: [{
+        type: "text",
+        text: `extract table headers from the image, you must use the get_table_headers tool, make sure to get the headers in order, "column_number" is the number of the table column that has this header. if there's a column with no header that still counts as a column and should be taken into account.`
+      }, {
+        type: "image",
+        source: {
+          data: imageToUse.toString("base64"),
+          media_type: "image/png",
+          type: "base64"
+        }
+      }]
+    }],
+    model: "claude-3-haiku-20240307",
+    tools: [{
+      input_schema: {
+        type: "object",
+        properties: {
+          headers: {
+            type: "array",
+            items: {
+              type: "object",
+              description: "Table headers",
+              properties: {
+                column_number: {
+                  type: "number",
+                  description: "The column number of the header"
+                },
+                header: {
+                  type: "string"
+                }
+              },
+              required: ["column_number", "header"]
+            }
+          }
+        },
+        required: ["headers"]
+      },
+      name: "get_table_headers",
+      description: `a function to extract table headers from an image, the headers should be in order and the column number should be the number of the column that has this header. if there's a column with no header that still counts as a column and should be taken into account. you should still need to`
+    }]
+  }), error => {
+    const typedError = error;
+    if (typedError.status === 449) return Errors.insufficientAiCredits(`🔴 ${typedError.error.error}`);
+  });
+  if (response.isErr() && ((_response$error = response.error) === null || _response$error === void 0 ? void 0 : _response$error.type) === "InsufficientAiCredits") {
+    return (0, _neverthrow.err)(response.error);
+  }
+  if (response.isErr()) {
+    return (0, _neverthrow.ok)({
+      headers: []
+    });
+  }
+  const unwrappedResponse = response.value;
+  const tool = unwrappedResponse.content.find(t => t.type === "tool_use");
+  if (!tool) return (0, _neverthrow.ok)({
+    headers: []
+  });
+  const headersParsingResults = _zod.z.object({
+    headers: _zod.z.array(_zod.z.object({
+      column_number: _zod.z.number(),
+      header: _zod.z.string()
+    }))
+  }).safeParse(tool.input);
+  if (!headersParsingResults.success) {
+    return (0, _neverthrow.ok)({
+      headers: []
+    });
+  }
+  return (0, _neverthrow.ok)({
+    headers: headersParsingResults.data.headers.map(i => i.header)
+  });
+}
+async function cropImage(image, handle, {
+  height: cropHeight,
+  width: cropWidth,
+  x,
+  y
+}) {
+  const base64Image = image.toString("base64");
+  const dataUrl = `data:image/jpeg;base64,${base64Image}`;
+  const croppedDataUrl = await Promise.race([handle.evaluate((_, {
+    cropHeight,
+    cropWidth,
+    x,
+    y,
+    url
+  }) => {
+    return new Promise(resolve => {
+      const img = document.createElement("img");
+      const canvas = document.createElement("canvas");
+      document.body.appendChild(img);
+      img.onload = () => {
+        canvas.width = cropWidth;
+        canvas.height = cropHeight;
+        const ctx = canvas.getContext("2d");
+        ctx.drawImage(img, x, y, cropWidth, cropHeight, 0, 0, cropWidth, cropHeight);
+        const resultImage = canvas.toDataURL().split(";base64,").pop();
+        resolve(resultImage);
+        img.remove();
+        canvas.remove();
+      };
+      img.src = url;
+    });
+  }, {
+    url: dataUrl,
+    x,
+    y,
+    cropWidth,
+    cropHeight
+  }), new Promise(resolve => setTimeout(() => {
+    resolve(undefined);
+  }, 5000))]);
+  if (!croppedDataUrl) return undefined;
+  return Buffer.from(croppedDataUrl, "base64");
+}

package/dist/optimized-extractors/common/index.js

@@ -0,0 +1,55 @@
+"use strict";
+
+Object.defineProperty(exports, "__esModule", {
+  value: true
+});
+exports.extractStructuredDataUsingAi = extractStructuredDataUsingAi;
+exports.isClaudeModel = isClaudeModel;
+exports.isGoogleModel = isGoogleModel;
+exports.isOpenAiModel = isOpenAiModel;
+var _neverthrow = require("neverthrow");
+var _extractStructuredDataUsingClaude = require("./extractStructuredDataUsingClaude");
+var _extractStructuredDataUsingOpenAi = require("./extractStructuredDataUsingOpenAi");
+var _utils = require("./utils");
+var _extractStructuredDataUsingGoogle = require("./extractStructuredDataUsingGoogle");
+var _aiModelsValidation = require("../types/aiModelsValidation");
+function isClaudeModel(model) {
+  return _aiModelsValidation.SUPPORTED_CLAUDE_MODELS.includes(model);
+}
+function isGoogleModel(model) {
+  return _aiModelsValidation.SUPPORTED_GOOGLE_MODELS.includes(model);
+}
+function isOpenAiModel(model) {
+  return _aiModelsValidation.SUPPORTED_GPT_MODELS.includes(model);
+}
+async function extractStructuredDataUsingAi(input) {
+  let extractionResult;
+  const {
+    model
+  } = input;
+  if (isOpenAiModel(model)) {
+    extractionResult = await (0, _extractStructuredDataUsingOpenAi.extractStructuredDataUsingOpenAi)({
+      ...input,
+      model
+    });
+  } else if (isGoogleModel(model)) {
+    extractionResult = await (0, _extractStructuredDataUsingGoogle.extractStructuredDataUsingGoogle)({
+      ...input,
+      model
+    });
+  } else {
+    extractionResult = await (0, _extractStructuredDataUsingClaude.extractStructuredDataUsingClaude)({
+      ...input,
+      model
+    });
+  }
+  if (extractionResult.isErr()) {
+    return (0, _neverthrow.err)(extractionResult.error);
+  }
+  const {
+    result
+  } = extractionResult.value;
+  return (0, _neverthrow.ok)({
+    result: (0, _utils.cleanupAiResult)(result)
+  });
+}