@librechat/agents 2.4.31 → 2.4.34

package/dist/cjs/tools/handlers.cjs

@@ -0,0 +1,144 @@
+'use strict';
+
+var nanoid = require('nanoid');
+var _enum = require('../common/enum.cjs');
+require('../messages/core.cjs');
+var ids = require('../messages/ids.cjs');
+require('@langchain/core/messages');
+
+/* eslint-disable no-console */
+// src/tools/handlers.ts
+function handleToolCallChunks({ graph, stepKey, toolCallChunks, }) {
+    let prevStepId;
+    let prevRunStep;
+    try {
+        prevStepId = graph.getStepIdByKey(stepKey, graph.contentData.length - 1);
+        prevRunStep = graph.getRunStep(prevStepId);
+    }
+    catch {
+        /** Edge Case: If no previous step exists, create a new message creation step */
+        const message_id = ids.getMessageId(stepKey, graph, true) ?? '';
+        prevStepId = graph.dispatchRunStep(stepKey, {
+            type: _enum.StepTypes.MESSAGE_CREATION,
+            message_creation: {
+                message_id,
+            },
+        });
+        prevRunStep = graph.getRunStep(prevStepId);
+    }
+    const _stepId = graph.getStepIdByKey(stepKey, prevRunStep?.index);
+    /** Edge Case: Tool Call Run Step or `tool_call_ids` never dispatched */
+    const tool_calls = prevStepId && prevRunStep && prevRunStep.type === _enum.StepTypes.MESSAGE_CREATION
+        ? []
+        : undefined;
+    /** Edge Case: `id` and `name` fields cannot be empty strings */
+    for (const toolCallChunk of toolCallChunks) {
+        if (toolCallChunk.name === '') {
+            toolCallChunk.name = undefined;
+        }
+        if (toolCallChunk.id === '') {
+            toolCallChunk.id = undefined;
+        }
+        else if (tool_calls != null &&
+            toolCallChunk.id != null &&
+            toolCallChunk.name != null) {
+            tool_calls.push({
+                args: {},
+                id: toolCallChunk.id,
+                name: toolCallChunk.name,
+                type: _enum.ToolCallTypes.TOOL_CALL,
+            });
+        }
+    }
+    let stepId = _stepId;
+    const alreadyDispatched = prevRunStep?.type === _enum.StepTypes.MESSAGE_CREATION &&
+        graph.messageStepHasToolCalls.has(prevStepId);
+    if (!alreadyDispatched && tool_calls?.length === toolCallChunks.length) {
+        graph.dispatchMessageDelta(prevStepId, {
+            content: [
+                {
+                    type: _enum.ContentTypes.TEXT,
+                    text: '',
+                    tool_call_ids: tool_calls.map((tc) => tc.id ?? ''),
+                },
+            ],
+        });
+        graph.messageStepHasToolCalls.set(prevStepId, true);
+        stepId = graph.dispatchRunStep(stepKey, {
+            type: _enum.StepTypes.TOOL_CALLS,
+            tool_calls,
+        });
+    }
+    graph.dispatchRunStepDelta(stepId, {
+        type: _enum.StepTypes.TOOL_CALLS,
+        tool_calls: toolCallChunks,
+    });
+}
+const handleToolCalls = (toolCalls, metadata, graph) => {
+    if (!graph || !metadata) {
+        console.warn(`Graph or metadata not found in ${event} event`);
+        return;
+    }
+    if (!toolCalls) {
+        return;
+    }
+    if (toolCalls.length === 0) {
+        return;
+    }
+    const stepKey = graph.getStepKey(metadata);
+    for (const tool_call of toolCalls) {
+        const toolCallId = tool_call.id ?? `toolu_${nanoid.nanoid()}`;
+        tool_call.id = toolCallId;
+        if (!toolCallId || graph.toolCallStepIds.has(toolCallId)) {
+            continue;
+        }
+        let prevStepId = '';
+        let prevRunStep;
+        try {
+            prevStepId = graph.getStepIdByKey(stepKey, graph.contentData.length - 1);
+            prevRunStep = graph.getRunStep(prevStepId);
+        }
+        catch {
+            // no previous step
+        }
+        const dispatchToolCallIds = (lastMessageStepId) => {
+            graph.dispatchMessageDelta(lastMessageStepId, {
+                content: [
+                    {
+                        type: 'text',
+                        text: '',
+                        tool_call_ids: [toolCallId],
+                    },
+                ],
+            });
+        };
+        /* If the previous step exists and is a message creation */
+        if (prevStepId &&
+            prevRunStep &&
+            prevRunStep.type === _enum.StepTypes.MESSAGE_CREATION) {
+            dispatchToolCallIds(prevStepId);
+            graph.messageStepHasToolCalls.set(prevStepId, true);
+            /* If the previous step doesn't exist or is not a message creation */
+        }
+        else if (!prevRunStep ||
+            prevRunStep.type !== _enum.StepTypes.MESSAGE_CREATION) {
+            const messageId = ids.getMessageId(stepKey, graph, true) ?? '';
+            const stepId = graph.dispatchRunStep(stepKey, {
+                type: _enum.StepTypes.MESSAGE_CREATION,
+                message_creation: {
+                    message_id: messageId,
+                },
+            });
+            dispatchToolCallIds(stepId);
+            graph.messageStepHasToolCalls.set(prevStepId, true);
+        }
+        graph.dispatchRunStep(stepKey, {
+            type: _enum.StepTypes.TOOL_CALLS,
+            tool_calls: [tool_call],
+        });
+    }
+};
+
+exports.handleToolCallChunks = handleToolCallChunks;
+exports.handleToolCalls = handleToolCalls;
+//# sourceMappingURL=handlers.cjs.map

package/dist/cjs/tools/handlers.cjs.map

@@ -0,0 +1 @@
+{"version":3,"file":"handlers.cjs","sources":["../../../src/tools/handlers.ts"],"sourcesContent":["/* eslint-disable no-console */\n// src/tools/handlers.ts\nimport { nanoid } from 'nanoid';\nimport type { ToolCall, ToolCallChunk } from '@langchain/core/messages/tool';\nimport type { Graph } from '@/graphs';\nimport type * as t from '@/types';\nimport { StepTypes, ContentTypes, ToolCallTypes } from '@/common';\nimport { getMessageId } from '@/messages';\n\nexport function handleToolCallChunks({\n  graph,\n  stepKey,\n  toolCallChunks,\n}: {\n  graph: Graph;\n  stepKey: string;\n  toolCallChunks: ToolCallChunk[];\n}): void {\n  let prevStepId: string;\n  let prevRunStep: t.RunStep | undefined;\n  try {\n    prevStepId = graph.getStepIdByKey(stepKey, graph.contentData.length - 1);\n    prevRunStep = graph.getRunStep(prevStepId);\n  } catch {\n    /** Edge Case: If no previous step exists, create a new message creation step */\n    const message_id = getMessageId(stepKey, graph, true) ?? '';\n    prevStepId = graph.dispatchRunStep(stepKey, {\n      type: StepTypes.MESSAGE_CREATION,\n      message_creation: {\n        message_id,\n      },\n    });\n    prevRunStep = graph.getRunStep(prevStepId);\n  }\n\n  const _stepId = graph.getStepIdByKey(stepKey, prevRunStep?.index);\n\n  /** Edge Case: Tool Call Run Step or `tool_call_ids` never dispatched */\n  const tool_calls: ToolCall[] | undefined =\n    prevStepId && prevRunStep && prevRunStep.type === StepTypes.MESSAGE_CREATION\n      ? []\n      : undefined;\n\n  /** Edge Case: `id` and `name` fields cannot be empty strings */\n  for (const toolCallChunk of toolCallChunks) {\n    if (toolCallChunk.name === '') {\n      toolCallChunk.name = undefined;\n    }\n    if (toolCallChunk.id === '') {\n      toolCallChunk.id = undefined;\n    } else if (\n      tool_calls != null &&\n      toolCallChunk.id != null &&\n      toolCallChunk.name != null\n    ) {\n      tool_calls.push({\n        args: {},\n        id: toolCallChunk.id,\n        name: toolCallChunk.name,\n        type: ToolCallTypes.TOOL_CALL,\n      });\n    }\n  }\n\n  let stepId: string = _stepId;\n  const alreadyDispatched =\n    prevRunStep?.type === StepTypes.MESSAGE_CREATION &&\n    graph.messageStepHasToolCalls.has(prevStepId);\n  if (!alreadyDispatched && tool_calls?.length === toolCallChunks.length) {\n    graph.dispatchMessageDelta(prevStepId, {\n      content: [\n        {\n          type: ContentTypes.TEXT,\n          text: '',\n          tool_call_ids: tool_calls.map((tc) => tc.id ?? ''),\n        },\n      ],\n    });\n    graph.messageStepHasToolCalls.set(prevStepId, true);\n    stepId = graph.dispatchRunStep(stepKey, {\n      type: StepTypes.TOOL_CALLS,\n      tool_calls,\n    });\n  }\n  graph.dispatchRunStepDelta(stepId, {\n    type: StepTypes.TOOL_CALLS,\n    tool_calls: toolCallChunks,\n  });\n}\n\nexport const handleToolCalls = (\n  toolCalls?: ToolCall[],\n  metadata?: Record<string, unknown>,\n  graph?: Graph\n): void => {\n  if (!graph || !metadata) {\n    console.warn(`Graph or metadata not found in ${event} event`);\n    return;\n  }\n\n  if (!toolCalls) {\n    return;\n  }\n\n  if (toolCalls.length === 0) {\n    return;\n  }\n\n  const stepKey = graph.getStepKey(metadata);\n\n  for (const tool_call of toolCalls) {\n    const toolCallId = tool_call.id ?? `toolu_${nanoid()}`;\n    tool_call.id = toolCallId;\n    if (!toolCallId || graph.toolCallStepIds.has(toolCallId)) {\n      continue;\n    }\n\n    let prevStepId = '';\n    let prevRunStep: t.RunStep | undefined;\n    try {\n      prevStepId = graph.getStepIdByKey(stepKey, graph.contentData.length - 1);\n      prevRunStep = graph.getRunStep(prevStepId);\n    } catch {\n      // no previous step\n    }\n\n    const dispatchToolCallIds = (lastMessageStepId: string): void => {\n      graph.dispatchMessageDelta(lastMessageStepId, {\n        content: [\n          {\n            type: 'text',\n            text: '',\n            tool_call_ids: [toolCallId],\n          },\n        ],\n      });\n    };\n    /* If the previous step exists and is a message creation */\n    if (\n      prevStepId &&\n      prevRunStep &&\n      prevRunStep.type === StepTypes.MESSAGE_CREATION\n    ) {\n      dispatchToolCallIds(prevStepId);\n      graph.messageStepHasToolCalls.set(prevStepId, true);\n      /* If the previous step doesn't exist or is not a message creation */\n    } else if (\n      !prevRunStep ||\n      prevRunStep.type !== StepTypes.MESSAGE_CREATION\n    ) {\n      const messageId = getMessageId(stepKey, graph, true) ?? '';\n      const stepId = graph.dispatchRunStep(stepKey, {\n        type: StepTypes.MESSAGE_CREATION,\n        message_creation: {\n          message_id: messageId,\n        },\n      });\n      dispatchToolCallIds(stepId);\n      graph.messageStepHasToolCalls.set(prevStepId, true);\n    }\n\n    graph.dispatchRunStep(stepKey, {\n      type: StepTypes.TOOL_CALLS,\n      tool_calls: [tool_call],\n    });\n  }\n};\n"],"names":["getMessageId","StepTypes","ToolCallTypes","ContentTypes","nanoid"],"mappings":";;;;;;;;AAAA;AACA;AAQM,SAAU,oBAAoB,CAAC,EACnC,KAAK,EACL,OAAO,EACP,cAAc,GAKf,EAAA;AACC,IAAA,IAAI,UAAkB;AACtB,IAAA,IAAI,WAAkC;AACtC,IAAA,IAAI;AACF,QAAA,UAAU,GAAG,KAAK,CAAC,cAAc,CAAC,OAAO,EAAE,KAAK,CAAC,WAAW,CAAC,MAAM,GAAG,CAAC,CAAC;AACxE,QAAA,WAAW,GAAG,KAAK,CAAC,UAAU,CAAC,UAAU,CAAC;;AAC1C,IAAA,MAAM;;AAEN,QAAA,MAAM,UAAU,GAAGA,gBAAY,CAAC,OAAO,EAAE,KAAK,EAAE,IAAI,CAAC,IAAI,EAAE;AAC3D,QAAA,UAAU,GAAG,KAAK,CAAC,eAAe,CAAC,OAAO,EAAE;YAC1C,IAAI,EAAEC,eAAS,CAAC,gBAAgB;AAChC,YAAA,gBAAgB,EAAE;gBAChB,UAAU;AACX,aAAA;AACF,SAAA,CAAC;AACF,QAAA,WAAW,GAAG,KAAK,CAAC,UAAU,CAAC,UAAU,CAAC;;AAG5C,IAAA,MAAM,OAAO,GAAG,KAAK,CAAC,cAAc,CAAC,OAAO,EAAE,WAAW,EAAE,KAAK,CAAC;;AAGjE,IAAA,MAAM,UAAU,GACd,UAAU,IAAI,WAAW,IAAI,WAAW,CAAC,IAAI,KAAKA,eAAS,CAAC;AAC1D,UAAE;UACA,SAAS;;AAGf,IAAA,KAAK,MAAM,aAAa,IAAI,cAAc,EAAE;AAC1C,QAAA,IAAI,aAAa,CAAC,IAAI,KAAK,EAAE,EAAE;AAC7B,YAAA,aAAa,CAAC,IAAI,GAAG,SAAS;;AAEhC,QAAA,IAAI,aAAa,CAAC,EAAE,KAAK,EAAE,EAAE;AAC3B,YAAA,aAAa,CAAC,EAAE,GAAG,SAAS;;aACvB,IACL,UAAU,IAAI,IAAI;YAClB,aAAa,CAAC,EAAE,IAAI,IAAI;AACxB,YAAA,aAAa,CAAC,IAAI,IAAI,IAAI,EAC1B;YACA,UAAU,CAAC,IAAI,CAAC;AACd,gBAAA,IAAI,EAAE,EAAE;gBACR,EAAE,EAAE,aAAa,CAAC,EAAE;gBACpB,IAAI,EAAE,aAAa,CAAC,IAAI;gBACxB,IAAI,EAAEC,mBAAa,CAAC,SAAS;AAC9B,aAAA,CAAC;;;IAIN,IAAI,MAAM,GAAW,OAAO;IAC5B,MAAM,iBAAiB,GACrB,WAAW,EAAE,IAAI,KAAKD,eAAS,CAAC,gBAAgB;AAChD,QAAA,KAAK,CAAC,uBAAuB,CAAC,GAAG,CAAC,UAAU,CAAC;IAC/C,IAAI,CAAC,iBAAiB,IAAI,UAAU,EAAE,MAAM,KAAK,cAAc,CAAC,MAAM,EAAE;AACtE,QAAA,KAAK,CAAC,oBAAoB,CAAC,UAAU,EAAE;AACrC,YAAA,OAAO,EAAE;AACP,gBAAA;oBACE,IAAI,EAAEE,kBAAY,CAAC,IAAI;AACvB,oBAAA,IAAI,EAAE,EAAE;AACR,oBAAA,aAAa,EAAE,UAAU,CAAC,GAAG,CAAC,CAAC,EAAE,KAAK,EAAE,CAAC,EAAE,IAAI,EAAE,CAAC;AACnD,iBAAA;AACF,aAAA;AACF,SAAA,CAAC;QACF,KAAK,CAAC,uBAAuB,CAAC,GAAG,CAAC,UAAU,EAAE,IAAI,CAAC;AACnD,QAAA,MAAM,GAAG,KAAK,CAAC,eAAe,CAAC,OAAO,EAAE;YACtC,IAAI,EAAEF,eAAS,CAAC,UAAU;YAC1B,UAAU;AACX,SAAA,CAAC;;AAEJ,IAAA,KAAK,CAAC,oBAAoB,CAAC,MAAM,EAAE;QACjC,IAAI,EAAEA,eAAS,CAAC,UAAU;AAC1B,QAAA,UAAU,EAAE,cAAc;AAC3B,KAAA,CAAC;AACJ;AAEa,MAAA,eAAe,GAAG,CAC7B,SAAsB,EACtB,QAAkC,EAClC,KAAa,KACL;AACR,IAAA,IAAI,CAAC,KAAK,IAAI,CAAC,QAAQ,EAAE;AACvB,QAAA,OAAO,CAAC,IAAI,CAAC,kCAAkC,KAAK,CAAA,MAAA,CAAQ,CAAC;QAC7D;;IAGF,IAAI,CAAC,SAAS,EAAE;QACd;;AAGF,IAAA,IAAI,SAAS,CAAC,MAAM,KAAK,CAAC,EAAE;QAC1B;;IAGF,MAAM,OAAO,GAAG,KAAK,CAAC,UAAU,CAAC,QAAQ,CAAC;AAE1C,IAAA,KAAK,MAAM,SAAS,IAAI,SAAS,EAAE;QACjC,MAAM,UAAU,GAAG,SAAS,CAAC,EAAE,IAAI,CAAS,MAAA,EAAAG,aAAM,EAAE,CAAA,CAAE;AACtD,QAAA,SAAS,CAAC,EAAE,GAAG,UAAU;AACzB,QAAA,IAAI,CAAC,UAAU,IAAI,KAAK,CAAC,eAAe,CAAC,GAAG,CAAC,UAAU,CAAC,EAAE;YACxD;;QAGF,IAAI,UAAU,GAAG,EAAE;AACnB,QAAA,IAAI,WAAkC;AACtC,QAAA,IAAI;AACF,YAAA,UAAU,GAAG,KAAK,CAAC,cAAc,CAAC,OAAO,EAAE,KAAK,CAAC,WAAW,CAAC,MAAM,GAAG,CAAC,CAAC;AACxE,YAAA,WAAW,GAAG,KAAK,CAAC,UAAU,CAAC,UAAU,CAAC;;AAC1C,QAAA,MAAM;;;AAIR,QAAA,MAAM,mBAAmB,GAAG,CAAC,iBAAyB,KAAU;AAC9D,YAAA,KAAK,CAAC,oBAAoB,CAAC,iBAAiB,EAAE;AAC5C,gBAAA,OAAO,EAAE;AACP,oBAAA;AACE,wBAAA,IAAI,EAAE,MAAM;AACZ,wBAAA,IAAI,EAAE,EAAE;wBACR,aAAa,EAAE,CAAC,UAAU,CAAC;AAC5B,qBAAA;AACF,iBAAA;AACF,aAAA,CAAC;AACJ,SAAC;;AAED,QAAA,IACE,UAAU;YACV,WAAW;AACX,YAAA,WAAW,CAAC,IAAI,KAAKH,eAAS,CAAC,gBAAgB,EAC/C;YACA,mBAAmB,CAAC,UAAU,CAAC;YAC/B,KAAK,CAAC,uBAAuB,CAAC,GAAG,CAAC,UAAU,EAAE,IAAI,CAAC;;;AAE9C,aAAA,IACL,CAAC,WAAW;AACZ,YAAA,WAAW,CAAC,IAAI,KAAKA,eAAS,CAAC,gBAAgB,EAC/C;AACA,YAAA,MAAM,SAAS,GAAGD,gBAAY,CAAC,OAAO,EAAE,KAAK,EAAE,IAAI,CAAC,IAAI,EAAE;AAC1D,YAAA,MAAM,MAAM,GAAG,KAAK,CAAC,eAAe,CAAC,OAAO,EAAE;gBAC5C,IAAI,EAAEC,eAAS,CAAC,gBAAgB;AAChC,gBAAA,gBAAgB,EAAE;AAChB,oBAAA,UAAU,EAAE,SAAS;AACtB,iBAAA;AACF,aAAA,CAAC;YACF,mBAAmB,CAAC,MAAM,CAAC;YAC3B,KAAK,CAAC,uBAAuB,CAAC,GAAG,CAAC,UAAU,EAAE,IAAI,CAAC;;AAGrD,QAAA,KAAK,CAAC,eAAe,CAAC,OAAO,EAAE;YAC7B,IAAI,EAAEA,eAAS,CAAC,UAAU;YAC1B,UAAU,EAAE,CAAC,SAAS,CAAC;AACxB,SAAA,CAAC;;AAEN;;;;;"}

package/dist/cjs/tools/search/content.cjs

@@ -0,0 +1,140 @@
+'use strict';
+
+var cheerio = require('cheerio');
+
+function _interopNamespaceDefault(e) {
+    var n = Object.create(null);
+    if (e) {
+        Object.keys(e).forEach(function (k) {
+            if (k !== 'default') {
+                var d = Object.getOwnPropertyDescriptor(e, k);
+                Object.defineProperty(n, k, d.get ? d : {
+                    enumerable: true,
+                    get: function () { return e[k]; }
+                });
+            }
+        });
+    }
+    n.default = e;
+    return Object.freeze(n);
+}
+
+var cheerio__namespace = /*#__PURE__*/_interopNamespaceDefault(cheerio);
+
+function processContent(html, markdown) {
+    const linkMap = new Map();
+    const imageMap = new Map();
+    const videoMap = new Map();
+    const iframeMap = new Map();
+    const $ = cheerio__namespace.load(html, {
+        xmlMode: false,
+    });
+    // Extract all media references
+    $('a[href]').each((_, el) => {
+        const href = $(el).attr('href');
+        if (href != null && href) {
+            linkMap.set(href, {
+                originalUrl: href,
+                title: $(el).attr('title'),
+                text: $(el).text().trim(),
+            });
+        }
+    });
+    $('img[src]').each((_, el) => {
+        const src = $(el).attr('src');
+        if (src != null && src) {
+            imageMap.set(src, {
+                originalUrl: src,
+                title: $(el).attr('alt') ?? $(el).attr('title'),
+            });
+        }
+    });
+    // Handle videos (dedicated video elements and video platforms in iframes)
+    $('video[src], iframe[src*="youtube"], iframe[src*="vimeo"]').each((_, el) => {
+        const src = $(el).attr('src');
+        if (src != null && src) {
+            videoMap.set(src, {
+                originalUrl: src,
+                title: $(el).attr('title'),
+            });
+        }
+    });
+    // Handle all other generic iframes that aren't already captured as videos
+    $('iframe').each((_, el) => {
+        const src = $(el).attr('src');
+        if (src != null &&
+            src &&
+            !src.includes('youtube') &&
+            !src.includes('vimeo')) {
+            iframeMap.set(src, {
+                originalUrl: src,
+                title: $(el).attr('title'),
+            });
+        }
+    });
+    // Create lookup maps with indices
+    const linkIndexMap = new Map();
+    const imageIndexMap = new Map();
+    const videoIndexMap = new Map();
+    const iframeIndexMap = new Map();
+    Array.from(linkMap.keys()).forEach((url, i) => linkIndexMap.set(url, i + 1));
+    Array.from(imageMap.keys()).forEach((url, i) => imageIndexMap.set(url, i + 1));
+    Array.from(videoMap.keys()).forEach((url, i) => videoIndexMap.set(url, i + 1));
+    Array.from(iframeMap.keys()).forEach((url, i) => iframeIndexMap.set(url, i + 1));
+    // Process the markdown
+    let result = markdown;
+    // Replace each URL one by one, starting with the longest URLs first to avoid partial matches
+    const allUrls = [
+        ...Array.from(imageMap.keys()).map((url) => ({
+            url,
+            type: 'image',
+            idx: imageIndexMap.get(url),
+        })),
+        ...Array.from(videoMap.keys()).map((url) => ({
+            url,
+            type: 'video',
+            idx: videoIndexMap.get(url),
+        })),
+        ...Array.from(iframeMap.keys()).map((url) => ({
+            url,
+            type: 'iframe',
+            idx: iframeIndexMap.get(url),
+        })),
+        ...Array.from(linkMap.keys()).map((url) => ({
+            url,
+            type: 'link',
+            idx: linkIndexMap.get(url),
+        })),
+    ].sort((a, b) => b.url.length - a.url.length);
+    // Create a function to escape special characters in URLs for regex
+    function escapeRegex(string) {
+        return string.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
+    }
+    // Replace each URL in the markdown
+    for (const { url, type, idx } of allUrls) {
+        // Create a regex that captures URLs in markdown links
+        const regex = new RegExp(`\\(${escapeRegex(url)}(?:\\s+"[^"]*")?\\)`, 'g');
+        result = result.replace(regex, (match) => {
+            // Keep any title attribute that might exist
+            const titleMatch = match.match(/\s+"([^"]*)"/);
+            const titlePart = titleMatch ? ` "${titleMatch[1]}"` : '';
+            return `(${type}#${idx}${titlePart})`;
+        });
+    }
+    iframeMap.clear();
+    const links = Array.from(linkMap.values());
+    linkMap.clear();
+    const images = Array.from(imageMap.values());
+    imageMap.clear();
+    const videos = Array.from(videoMap.values());
+    videoMap.clear();
+    return {
+        markdown: result,
+        links,
+        images,
+        videos,
+    };
+}
+
+exports.processContent = processContent;
+//# sourceMappingURL=content.cjs.map

package/dist/cjs/tools/search/content.cjs.map

@@ -0,0 +1 @@
+{"version":3,"file":"content.cjs","sources":["../../../../src/tools/search/content.ts"],"sourcesContent":["import * as cheerio from 'cheerio';\nimport type { References, MediaReference } from './types';\n\nexport function processContent(\n  html: string,\n  markdown: string\n): {\n  markdown: string;\n} & References {\n  const linkMap = new Map<string, MediaReference>();\n  const imageMap = new Map<string, MediaReference>();\n  const videoMap = new Map<string, MediaReference>();\n  const iframeMap = new Map<string, MediaReference>();\n\n  const $ = cheerio.load(html, {\n    xmlMode: false,\n  });\n\n  // Extract all media references\n  $('a[href]').each((_, el) => {\n    const href = $(el).attr('href');\n    if (href != null && href) {\n      linkMap.set(href, {\n        originalUrl: href,\n        title: $(el).attr('title'),\n        text: $(el).text().trim(),\n      });\n    }\n  });\n\n  $('img[src]').each((_, el) => {\n    const src = $(el).attr('src');\n    if (src != null && src) {\n      imageMap.set(src, {\n        originalUrl: src,\n        title: $(el).attr('alt') ?? $(el).attr('title'),\n      });\n    }\n  });\n\n  // Handle videos (dedicated video elements and video platforms in iframes)\n  $('video[src], iframe[src*=\"youtube\"], iframe[src*=\"vimeo\"]').each(\n    (_, el) => {\n      const src = $(el).attr('src');\n      if (src != null && src) {\n        videoMap.set(src, {\n          originalUrl: src,\n          title: $(el).attr('title'),\n        });\n      }\n    }\n  );\n\n  // Handle all other generic iframes that aren't already captured as videos\n  $('iframe').each((_, el) => {\n    const src = $(el).attr('src');\n    if (\n      src != null &&\n      src &&\n      !src.includes('youtube') &&\n      !src.includes('vimeo')\n    ) {\n      iframeMap.set(src, {\n        originalUrl: src,\n        title: $(el).attr('title'),\n      });\n    }\n  });\n\n  // Create lookup maps with indices\n  const linkIndexMap = new Map<string, number>();\n  const imageIndexMap = new Map<string, number>();\n  const videoIndexMap = new Map<string, number>();\n  const iframeIndexMap = new Map<string, number>();\n\n  Array.from(linkMap.keys()).forEach((url, i) => linkIndexMap.set(url, i + 1));\n  Array.from(imageMap.keys()).forEach((url, i) =>\n    imageIndexMap.set(url, i + 1)\n  );\n  Array.from(videoMap.keys()).forEach((url, i) =>\n    videoIndexMap.set(url, i + 1)\n  );\n  Array.from(iframeMap.keys()).forEach((url, i) =>\n    iframeIndexMap.set(url, i + 1)\n  );\n\n  // Process the markdown\n  let result = markdown;\n\n  // Replace each URL one by one, starting with the longest URLs first to avoid partial matches\n  const allUrls = [\n    ...Array.from(imageMap.keys()).map((url) => ({\n      url,\n      type: 'image',\n      idx: imageIndexMap.get(url),\n    })),\n    ...Array.from(videoMap.keys()).map((url) => ({\n      url,\n      type: 'video',\n      idx: videoIndexMap.get(url),\n    })),\n    ...Array.from(iframeMap.keys()).map((url) => ({\n      url,\n      type: 'iframe',\n      idx: iframeIndexMap.get(url),\n    })),\n    ...Array.from(linkMap.keys()).map((url) => ({\n      url,\n      type: 'link',\n      idx: linkIndexMap.get(url),\n    })),\n  ].sort((a, b) => b.url.length - a.url.length);\n\n  // Create a function to escape special characters in URLs for regex\n  function escapeRegex(string: string): string {\n    return string.replace(/[.*+?^${}()|[\\]\\\\]/g, '\\\\$&');\n  }\n\n  // Replace each URL in the markdown\n  for (const { url, type, idx } of allUrls) {\n    // Create a regex that captures URLs in markdown links\n    const regex = new RegExp(`\\\\(${escapeRegex(url)}(?:\\\\s+\"[^\"]*\")?\\\\)`, 'g');\n\n    result = result.replace(regex, (match) => {\n      // Keep any title attribute that might exist\n      const titleMatch = match.match(/\\s+\"([^\"]*)\"/);\n      const titlePart = titleMatch ? ` \"${titleMatch[1]}\"` : '';\n\n      return `(${type}#${idx}${titlePart})`;\n    });\n  }\n\n  iframeMap.clear();\n  const links = Array.from(linkMap.values());\n  linkMap.clear();\n  const images = Array.from(imageMap.values());\n  imageMap.clear();\n  const videos = Array.from(videoMap.values());\n  videoMap.clear();\n\n  return {\n    markdown: result,\n    links,\n    images,\n    videos,\n  };\n}\n"],"names":["cheerio"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;AAGgB,SAAA,cAAc,CAC5B,IAAY,EACZ,QAAgB,EAAA;AAIhB,IAAA,MAAM,OAAO,GAAG,IAAI,GAAG,EAA0B;AACjD,IAAA,MAAM,QAAQ,GAAG,IAAI,GAAG,EAA0B;AAClD,IAAA,MAAM,QAAQ,GAAG,IAAI,GAAG,EAA0B;AAClD,IAAA,MAAM,SAAS,GAAG,IAAI,GAAG,EAA0B;AAEnD,IAAA,MAAM,CAAC,GAAGA,kBAAO,CAAC,IAAI,CAAC,IAAI,EAAE;AAC3B,QAAA,OAAO,EAAE,KAAK;AACf,KAAA,CAAC;;IAGF,CAAC,CAAC,SAAS,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,KAAI;QAC1B,MAAM,IAAI,GAAG,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC;AAC/B,QAAA,IAAI,IAAI,IAAI,IAAI,IAAI,IAAI,EAAE;AACxB,YAAA,OAAO,CAAC,GAAG,CAAC,IAAI,EAAE;AAChB,gBAAA,WAAW,EAAE,IAAI;gBACjB,KAAK,EAAE,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC;gBAC1B,IAAI,EAAE,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE;AAC1B,aAAA,CAAC;;AAEN,KAAC,CAAC;IAEF,CAAC,CAAC,UAAU,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,KAAI;QAC3B,MAAM,GAAG,GAAG,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC;AAC7B,QAAA,IAAI,GAAG,IAAI,IAAI,IAAI,GAAG,EAAE;AACtB,YAAA,QAAQ,CAAC,GAAG,CAAC,GAAG,EAAE;AAChB,gBAAA,WAAW,EAAE,GAAG;AAChB,gBAAA,KAAK,EAAE,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC;AAChD,aAAA,CAAC;;AAEN,KAAC,CAAC;;IAGF,CAAC,CAAC,0DAA0D,CAAC,CAAC,IAAI,CAChE,CAAC,CAAC,EAAE,EAAE,KAAI;QACR,MAAM,GAAG,GAAG,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC;AAC7B,QAAA,IAAI,GAAG,IAAI,IAAI,IAAI,GAAG,EAAE;AACtB,YAAA,QAAQ,CAAC,GAAG,CAAC,GAAG,EAAE;AAChB,gBAAA,WAAW,EAAE,GAAG;gBAChB,KAAK,EAAE,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC;AAC3B,aAAA,CAAC;;AAEN,KAAC,CACF;;IAGD,CAAC,CAAC,QAAQ,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,KAAI;QACzB,MAAM,GAAG,GAAG,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC;QAC7B,IACE,GAAG,IAAI,IAAI;YACX,GAAG;AACH,YAAA,CAAC,GAAG,CAAC,QAAQ,CAAC,SAAS,CAAC;AACxB,YAAA,CAAC,GAAG,CAAC,QAAQ,CAAC,OAAO,CAAC,EACtB;AACA,YAAA,SAAS,CAAC,GAAG,CAAC,GAAG,EAAE;AACjB,gBAAA,WAAW,EAAE,GAAG;gBAChB,KAAK,EAAE,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC;AAC3B,aAAA,CAAC;;AAEN,KAAC,CAAC;;AAGF,IAAA,MAAM,YAAY,GAAG,IAAI,GAAG,EAAkB;AAC9C,IAAA,MAAM,aAAa,GAAG,IAAI,GAAG,EAAkB;AAC/C,IAAA,MAAM,aAAa,GAAG,IAAI,GAAG,EAAkB;AAC/C,IAAA,MAAM,cAAc,GAAG,IAAI,GAAG,EAAkB;AAEhD,IAAA,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC,CAAC,OAAO,CAAC,CAAC,GAAG,EAAE,CAAC,KAAK,YAAY,CAAC,GAAG,CAAC,GAAG,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC;AAC5E,IAAA,KAAK,CAAC,IAAI,CAAC,QAAQ,CAAC,IAAI,EAAE,CAAC,CAAC,OAAO,CAAC,CAAC,GAAG,EAAE,CAAC,KACzC,aAAa,CAAC,GAAG,CAAC,GAAG,EAAE,CAAC,GAAG,CAAC,CAAC,CAC9B;AACD,IAAA,KAAK,CAAC,IAAI,CAAC,QAAQ,CAAC,IAAI,EAAE,CAAC,CAAC,OAAO,CAAC,CAAC,GAAG,EAAE,CAAC,KACzC,aAAa,CAAC,GAAG,CAAC,GAAG,EAAE,CAAC,GAAG,CAAC,CAAC,CAC9B;AACD,IAAA,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,IAAI,EAAE,CAAC,CAAC,OAAO,CAAC,CAAC,GAAG,EAAE,CAAC,KAC1C,cAAc,CAAC,GAAG,CAAC,GAAG,EAAE,CAAC,GAAG,CAAC,CAAC,CAC/B;;IAGD,IAAI,MAAM,GAAG,QAAQ;;AAGrB,IAAA,MAAM,OAAO,GAAG;AACd,QAAA,GAAG,KAAK,CAAC,IAAI,CAAC,QAAQ,CAAC,IAAI,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,MAAM;YAC3C,GAAG;AACH,YAAA,IAAI,EAAE,OAAO;AACb,YAAA,GAAG,EAAE,aAAa,CAAC,GAAG,CAAC,GAAG,CAAC;AAC5B,SAAA,CAAC,CAAC;AACH,QAAA,GAAG,KAAK,CAAC,IAAI,CAAC,QAAQ,CAAC,IAAI,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,MAAM;YAC3C,GAAG;AACH,YAAA,IAAI,EAAE,OAAO;AACb,YAAA,GAAG,EAAE,aAAa,CAAC,GAAG,CAAC,GAAG,CAAC;AAC5B,SAAA,CAAC,CAAC;AACH,QAAA,GAAG,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,IAAI,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,MAAM;YAC5C,GAAG;AACH,YAAA,IAAI,EAAE,QAAQ;AACd,YAAA,GAAG,EAAE,cAAc,CAAC,GAAG,CAAC,GAAG,CAAC;AAC7B,SAAA,CAAC,CAAC;AACH,QAAA,GAAG,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,MAAM;YAC1C,GAAG;AACH,YAAA,IAAI,EAAE,MAAM;AACZ,YAAA,GAAG,EAAE,YAAY,CAAC,GAAG,CAAC,GAAG,CAAC;AAC3B,SAAA,CAAC,CAAC;KACJ,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,KAAK,CAAC,CAAC,GAAG,CAAC,MAAM,GAAG,CAAC,CAAC,GAAG,CAAC,MAAM,CAAC;;IAG7C,SAAS,WAAW,CAAC,MAAc,EAAA;QACjC,OAAO,MAAM,CAAC,OAAO,CAAC,qBAAqB,EAAE,MAAM,CAAC;;;IAItD,KAAK,MAAM,EAAE,GAAG,EAAE,IAAI,EAAE,GAAG,EAAE,IAAI,OAAO,EAAE;;AAExC,QAAA,MAAM,KAAK,GAAG,IAAI,MAAM,CAAC,CAAM,GAAA,EAAA,WAAW,CAAC,GAAG,CAAC,CAAA,mBAAA,CAAqB,EAAE,GAAG,CAAC;QAE1E,MAAM,GAAG,MAAM,CAAC,OAAO,CAAC,KAAK,EAAE,CAAC,KAAK,KAAI;;YAEvC,MAAM,UAAU,GAAG,KAAK,CAAC,KAAK,CAAC,cAAc,CAAC;AAC9C,YAAA,MAAM,SAAS,GAAG,UAAU,GAAG,CAAK,EAAA,EAAA,UAAU,CAAC,CAAC,CAAC,CAAG,CAAA,CAAA,GAAG,EAAE;AAEzD,YAAA,OAAO,IAAI,IAAI,CAAA,CAAA,EAAI,GAAG,CAAG,EAAA,SAAS,GAAG;AACvC,SAAC,CAAC;;IAGJ,SAAS,CAAC,KAAK,EAAE;IACjB,MAAM,KAAK,GAAG,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE,CAAC;IAC1C,OAAO,CAAC,KAAK,EAAE;IACf,MAAM,MAAM,GAAG,KAAK,CAAC,IAAI,CAAC,QAAQ,CAAC,MAAM,EAAE,CAAC;IAC5C,QAAQ,CAAC,KAAK,EAAE;IAChB,MAAM,MAAM,GAAG,KAAK,CAAC,IAAI,CAAC,QAAQ,CAAC,MAAM,EAAE,CAAC;IAC5C,QAAQ,CAAC,KAAK,EAAE;IAEhB,OAAO;AACL,QAAA,QAAQ,EAAE,MAAM;QAChB,KAAK;QACL,MAAM;QACN,MAAM;KACP;AACH;;;;"}

package/dist/cjs/tools/search/firecrawl.cjs

@@ -1,37 +1,9 @@
 'use strict';
 
 var axios = require('axios');
+var content = require('./content.cjs');
+var utils = require('./utils.cjs');
 
-/* eslint-disable no-console */
-const getDomainName = (link, metadata) => {
-    try {
-        const url = metadata?.sourceURL ?? metadata?.url ?? (link || '');
-        const domain = new URL(url).hostname.replace(/^www\./, '');
-        if (domain) {
-            return domain;
-        }
-    }
-    catch (e) {
-        // URL parsing failed
-        console.error('Error parsing URL:', e);
-    }
-    return;
-};
-function getAttribution(link, metadata) {
-    if (!metadata)
-        return getDomainName(link, metadata);
-    const possibleAttributions = [
-        metadata.ogSiteName,
-        metadata['og:site_name'],
-        metadata.title?.split('|').pop()?.trim(),
-        metadata['twitter:site']?.replace(/^@/, ''),
-    ];
-    const attribution = possibleAttributions.find((attr) => attr != null && typeof attr === 'string' && attr.trim() !== '');
-    if (attribution != null) {
-        return attribution;
-    }
-    return getDomainName(link, metadata);
-}
 /**
  * Firecrawl scraper implementation
  * Uses the Firecrawl API to scrape web pages
@@ -41,6 +13,7 @@ class FirecrawlScraper {
     apiUrl;
     defaultFormats;
     timeout;
+    logger;
     constructor(config = {}) {
         this.apiKey = config.apiKey ?? process.env.FIRECRAWL_API_KEY ?? '';
         const baseUrl = config.apiUrl ??
@@ -48,11 +21,12 @@ class FirecrawlScraper {
             'https://api.firecrawl.dev';
         this.apiUrl = `${baseUrl.replace(/\/+$/, '')}/v1/scrape`;
         this.defaultFormats = config.formats ?? ['markdown', 'html'];
-        this.timeout = config.timeout ?? 30000;
+        this.timeout = config.timeout ?? 15000;
+        this.logger = config.logger || utils.createDefaultLogger();
         if (!this.apiKey) {
-            console.warn('FIRECRAWL_API_KEY is not set. Scraping will not work.');
+            this.logger.warn('FIRECRAWL_API_KEY is not set. Scraping will not work.');
         }
-        console.log(`Firecrawl scraper initialized with API URL: ${this.apiUrl}`);
+        this.logger.debug(`Firecrawl scraper initialized with API URL: ${this.apiUrl}`);
     }
     /**
      * Scrape a single URL
@@ -106,21 +80,30 @@ class FirecrawlScraper {
      */
     extractContent(response) {
         if (!response.success || !response.data) {
-            return '';
+            return ['', undefined];
+        }
+        if (response.data.markdown != null && response.data.html != null) {
+            try {
+                const { markdown, ...rest } = content.processContent(response.data.html, response.data.markdown);
+                return [markdown, rest];
+            }
+            catch (error) {
+                this.logger.error('Error processing content:', error);
+                return [response.data.markdown, undefined];
+            }
         }
-        // Prefer markdown content if available
-        if (response.data.markdown != null) {
-            return response.data.markdown;
+        else if (response.data.markdown != null) {
+            return [response.data.markdown, undefined];
         }
         // Fall back to HTML content
         if (response.data.html != null) {
-            return response.data.html;
+            return [response.data.html, undefined];
         }
         // Fall back to raw HTML content
         if (response.data.rawHtml != null) {
-            return response.data.rawHtml;
+            return [response.data.rawHtml, undefined];
         }
-        return '';
+        return ['', undefined];
     }
     /**
      * Extract metadata from scrape response
@@ -145,5 +128,4 @@ const createFirecrawlScraper = (config = {}) => {
 
 exports.FirecrawlScraper = FirecrawlScraper;
 exports.createFirecrawlScraper = createFirecrawlScraper;
-exports.getAttribution = getAttribution;
 //# sourceMappingURL=firecrawl.cjs.map

package/dist/cjs/tools/search/firecrawl.cjs.map

@@ -1 +1 @@
-{"version":3,"file":"firecrawl.cjs","sources":["../../../../src/tools/search/firecrawl.ts"],"sourcesContent":["/* eslint-disable no-console */\nimport axios from 'axios';\n\nexport interface FirecrawlScrapeOptions {\n  formats?: string[];\n  includeTags?: string[];\n  excludeTags?: string[];\n  headers?: Record<string, string>;\n  waitFor?: number;\n  timeout?: number;\n}\n\ninterface ScrapeMetadata {\n  // Core source information\n  sourceURL?: string;\n  url?: string;\n  scrapeId?: string;\n  statusCode?: number;\n  // Basic metadata\n  title?: string;\n  description?: string;\n  language?: string;\n  favicon?: string;\n  viewport?: string;\n  robots?: string;\n  'theme-color'?: string;\n  // Open Graph metadata\n  'og:url'?: string;\n  'og:title'?: string;\n  'og:description'?: string;\n  'og:type'?: string;\n  'og:image'?: string;\n  'og:image:width'?: string;\n  'og:image:height'?: string;\n  'og:site_name'?: string;\n  ogUrl?: string;\n  ogTitle?: string;\n  ogDescription?: string;\n  ogImage?: string;\n  ogSiteName?: string;\n  // Article metadata\n  'article:author'?: string;\n  'article:published_time'?: string;\n  'article:modified_time'?: string;\n  'article:section'?: string;\n  'article:tag'?: string;\n  'article:publisher'?: string;\n  publishedTime?: string;\n  modifiedTime?: string;\n  // Twitter metadata\n  'twitter:site'?: string;\n  'twitter:creator'?: string;\n  'twitter:card'?: string;\n  'twitter:image'?: string;\n  'twitter:dnt'?: string;\n  'twitter:app:name:iphone'?: string;\n  'twitter:app:id:iphone'?: string;\n  'twitter:app:url:iphone'?: string;\n  'twitter:app:name:ipad'?: string;\n  'twitter:app:id:ipad'?: string;\n  'twitter:app:url:ipad'?: string;\n  'twitter:app:name:googleplay'?: string;\n  'twitter:app:id:googleplay'?: string;\n  'twitter:app:url:googleplay'?: string;\n  // Facebook metadata\n  'fb:app_id'?: string;\n  // App links\n  'al:ios:url'?: string;\n  'al:ios:app_name'?: string;\n  'al:ios:app_store_id'?: string;\n  // Allow for additional properties that might be present\n  [key: string]: string | number | boolean | null | undefined;\n}\n\nexport interface FirecrawlScrapeResponse {\n  success: boolean;\n  data?: {\n    markdown?: string;\n    html?: string;\n    rawHtml?: string;\n    screenshot?: string;\n    links?: string[];\n    metadata?: ScrapeMetadata;\n  };\n  error?: string;\n}\n\nexport interface FirecrawlScraperConfig {\n  apiKey?: string;\n  apiUrl?: string;\n  formats?: string[];\n  timeout?: number;\n}\nconst getDomainName = (\n  link: string,\n  metadata?: ScrapeMetadata\n): string | undefined => {\n  try {\n    const url = metadata?.sourceURL ?? metadata?.url ?? (link || '');\n    const domain = new URL(url).hostname.replace(/^www\\./, '');\n    if (domain) {\n      return domain;\n    }\n  } catch (e) {\n    // URL parsing failed\n    console.error('Error parsing URL:', e);\n  }\n\n  return;\n};\n\nexport function getAttribution(\n  link: string,\n  metadata?: ScrapeMetadata\n): string | undefined {\n  if (!metadata) return getDomainName(link, metadata);\n\n  const possibleAttributions = [\n    metadata.ogSiteName,\n    metadata['og:site_name'],\n    metadata.title?.split('|').pop()?.trim(),\n    metadata['twitter:site']?.replace(/^@/, ''),\n  ];\n\n  const attribution = possibleAttributions.find(\n    (attr) => attr != null && typeof attr === 'string' && attr.trim() !== ''\n  );\n  if (attribution != null) {\n    return attribution;\n  }\n\n  return getDomainName(link, metadata);\n}\n\n/**\n * Firecrawl scraper implementation\n * Uses the Firecrawl API to scrape web pages\n */\nexport class FirecrawlScraper {\n  private apiKey: string;\n  private apiUrl: string;\n  private defaultFormats: string[];\n  private timeout: number;\n\n  constructor(config: FirecrawlScraperConfig = {}) {\n    this.apiKey = config.apiKey ?? process.env.FIRECRAWL_API_KEY ?? '';\n\n    const baseUrl =\n      config.apiUrl ??\n      process.env.FIRECRAWL_BASE_URL ??\n      'https://api.firecrawl.dev';\n    this.apiUrl = `${baseUrl.replace(/\\/+$/, '')}/v1/scrape`;\n\n    this.defaultFormats = config.formats ?? ['markdown', 'html'];\n    this.timeout = config.timeout ?? 30000;\n\n    if (!this.apiKey) {\n      console.warn('FIRECRAWL_API_KEY is not set. Scraping will not work.');\n    }\n\n    console.log(`Firecrawl scraper initialized with API URL: ${this.apiUrl}`);\n  }\n\n  /**\n   * Scrape a single URL\n   * @param url URL to scrape\n   * @param options Scrape options\n   * @returns Scrape response\n   */\n  async scrapeUrl(\n    url: string,\n    options: FirecrawlScrapeOptions = {}\n  ): Promise<[string, FirecrawlScrapeResponse]> {\n    if (!this.apiKey) {\n      return [\n        url,\n        {\n          success: false,\n          error: 'FIRECRAWL_API_KEY is not set',\n        },\n      ];\n    }\n\n    try {\n      const response = await axios.post(\n        this.apiUrl,\n        {\n          url,\n          formats: options.formats || this.defaultFormats,\n          includeTags: options.includeTags,\n          excludeTags: options.excludeTags,\n          headers: options.headers,\n          waitFor: options.waitFor,\n          timeout: options.timeout ?? this.timeout,\n        },\n        {\n          headers: {\n            'Content-Type': 'application/json',\n            Authorization: `Bearer ${this.apiKey}`,\n          },\n          timeout: this.timeout,\n        }\n      );\n\n      return [url, response.data];\n    } catch (error) {\n      const errorMessage =\n        error instanceof Error ? error.message : String(error);\n      return [\n        url,\n        {\n          success: false,\n          error: `Firecrawl API request failed: ${errorMessage}`,\n        },\n      ];\n    }\n  }\n\n  /**\n   * Extract content from scrape response\n   * @param response Scrape response\n   * @returns Extracted content or empty string if not available\n   */\n  extractContent(response: FirecrawlScrapeResponse): string {\n    if (!response.success || !response.data) {\n      return '';\n    }\n\n    // Prefer markdown content if available\n    if (response.data.markdown != null) {\n      return response.data.markdown;\n    }\n\n    // Fall back to HTML content\n    if (response.data.html != null) {\n      return response.data.html;\n    }\n\n    // Fall back to raw HTML content\n    if (response.data.rawHtml != null) {\n      return response.data.rawHtml;\n    }\n\n    return '';\n  }\n\n  /**\n   * Extract metadata from scrape response\n   * @param response Scrape response\n   * @returns Metadata object\n   */\n  extractMetadata(response: FirecrawlScrapeResponse): ScrapeMetadata {\n    if (!response.success || !response.data || !response.data.metadata) {\n      return {};\n    }\n\n    return response.data.metadata;\n  }\n}\n\n/**\n * Create a Firecrawl scraper instance\n * @param config Scraper configuration\n * @returns Firecrawl scraper instance\n */\nexport const createFirecrawlScraper = (\n  config: FirecrawlScraperConfig = {}\n): FirecrawlScraper => {\n  return new FirecrawlScraper(config);\n};\n"],"names":[],"mappings":";;;;AAAA;AA6FA,MAAM,aAAa,GAAG,CACpB,IAAY,EACZ,QAAyB,KACH;AACtB,IAAA,IAAI;AACF,QAAA,MAAM,GAAG,GAAG,QAAQ,EAAE,SAAS,IAAI,QAAQ,EAAE,GAAG,KAAK,IAAI,IAAI,EAAE,CAAC;AAChE,QAAA,MAAM,MAAM,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC,QAAQ,CAAC,OAAO,CAAC,QAAQ,EAAE,EAAE,CAAC;QAC1D,IAAI,MAAM,EAAE;AACV,YAAA,OAAO,MAAM;;;IAEf,OAAO,CAAC,EAAE;;AAEV,QAAA,OAAO,CAAC,KAAK,CAAC,oBAAoB,EAAE,CAAC,CAAC;;IAGxC;AACF,CAAC;AAEe,SAAA,cAAc,CAC5B,IAAY,EACZ,QAAyB,EAAA;AAEzB,IAAA,IAAI,CAAC,QAAQ;AAAE,QAAA,OAAO,aAAa,CAAC,IAAI,EAAE,QAAQ,CAAC;AAEnD,IAAA,MAAM,oBAAoB,GAAG;AAC3B,QAAA,QAAQ,CAAC,UAAU;QACnB,QAAQ,CAAC,cAAc,CAAC;AACxB,QAAA,QAAQ,CAAC,KAAK,EAAE,KAAK,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,IAAI,EAAE;QACxC,QAAQ,CAAC,cAAc,CAAC,EAAE,OAAO,CAAC,IAAI,EAAE,EAAE,CAAC;KAC5C;IAED,MAAM,WAAW,GAAG,oBAAoB,CAAC,IAAI,CAC3C,CAAC,IAAI,KAAK,IAAI,IAAI,IAAI,IAAI,OAAO,IAAI,KAAK,QAAQ,IAAI,IAAI,CAAC,IAAI,EAAE,KAAK,EAAE,CACzE;AACD,IAAA,IAAI,WAAW,IAAI,IAAI,EAAE;AACvB,QAAA,OAAO,WAAW;;AAGpB,IAAA,OAAO,aAAa,CAAC,IAAI,EAAE,QAAQ,CAAC;AACtC;AAEA;;;AAGG;MACU,gBAAgB,CAAA;AACnB,IAAA,MAAM;AACN,IAAA,MAAM;AACN,IAAA,cAAc;AACd,IAAA,OAAO;AAEf,IAAA,WAAA,CAAY,SAAiC,EAAE,EAAA;AAC7C,QAAA,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC,MAAM,IAAI,OAAO,CAAC,GAAG,CAAC,iBAAiB,IAAI,EAAE;AAElE,QAAA,MAAM,OAAO,GACX,MAAM,CAAC,MAAM;YACb,OAAO,CAAC,GAAG,CAAC,kBAAkB;AAC9B,YAAA,2BAA2B;AAC7B,QAAA,IAAI,CAAC,MAAM,GAAG,CAAA,EAAG,OAAO,CAAC,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC,YAAY;AAExD,QAAA,IAAI,CAAC,cAAc,GAAG,MAAM,CAAC,OAAO,IAAI,CAAC,UAAU,EAAE,MAAM,CAAC;QAC5D,IAAI,CAAC,OAAO,GAAG,MAAM,CAAC,OAAO,IAAI,KAAK;AAEtC,QAAA,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE;AAChB,YAAA,OAAO,CAAC,IAAI,CAAC,uDAAuD,CAAC;;QAGvE,OAAO,CAAC,GAAG,CAAC,CAAA,4CAAA,EAA+C,IAAI,CAAC,MAAM,CAAE,CAAA,CAAC;;AAG3E;;;;;AAKG;AACH,IAAA,MAAM,SAAS,CACb,GAAW,EACX,UAAkC,EAAE,EAAA;AAEpC,QAAA,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE;YAChB,OAAO;gBACL,GAAG;AACH,gBAAA;AACE,oBAAA,OAAO,EAAE,KAAK;AACd,oBAAA,KAAK,EAAE,8BAA8B;AACtC,iBAAA;aACF;;AAGH,QAAA,IAAI;YACF,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,IAAI,CAC/B,IAAI,CAAC,MAAM,EACX;gBACE,GAAG;AACH,gBAAA,OAAO,EAAE,OAAO,CAAC,OAAO,IAAI,IAAI,CAAC,cAAc;gBAC/C,WAAW,EAAE,OAAO,CAAC,WAAW;gBAChC,WAAW,EAAE,OAAO,CAAC,WAAW;gBAChC,OAAO,EAAE,OAAO,CAAC,OAAO;gBACxB,OAAO,EAAE,OAAO,CAAC,OAAO;AACxB,gBAAA,OAAO,EAAE,OAAO,CAAC,OAAO,IAAI,IAAI,CAAC,OAAO;aACzC,EACD;AACE,gBAAA,OAAO,EAAE;AACP,oBAAA,cAAc,EAAE,kBAAkB;AAClC,oBAAA,aAAa,EAAE,CAAA,OAAA,EAAU,IAAI,CAAC,MAAM,CAAE,CAAA;AACvC,iBAAA;gBACD,OAAO,EAAE,IAAI,CAAC,OAAO;AACtB,aAAA,CACF;AAED,YAAA,OAAO,CAAC,GAAG,EAAE,QAAQ,CAAC,IAAI,CAAC;;QAC3B,OAAO,KAAK,EAAE;AACd,YAAA,MAAM,YAAY,GAChB,KAAK,YAAY,KAAK,GAAG,KAAK,CAAC,OAAO,GAAG,MAAM,CAAC,KAAK,CAAC;YACxD,OAAO;gBACL,GAAG;AACH,gBAAA;AACE,oBAAA,OAAO,EAAE,KAAK;oBACd,KAAK,EAAE,CAAiC,8BAAA,EAAA,YAAY,CAAE,CAAA;AACvD,iBAAA;aACF;;;AAIL;;;;AAIG;AACH,IAAA,cAAc,CAAC,QAAiC,EAAA;QAC9C,IAAI,CAAC,QAAQ,CAAC,OAAO,IAAI,CAAC,QAAQ,CAAC,IAAI,EAAE;AACvC,YAAA,OAAO,EAAE;;;QAIX,IAAI,QAAQ,CAAC,IAAI,CAAC,QAAQ,IAAI,IAAI,EAAE;AAClC,YAAA,OAAO,QAAQ,CAAC,IAAI,CAAC,QAAQ;;;QAI/B,IAAI,QAAQ,CAAC,IAAI,CAAC,IAAI,IAAI,IAAI,EAAE;AAC9B,YAAA,OAAO,QAAQ,CAAC,IAAI,CAAC,IAAI;;;QAI3B,IAAI,QAAQ,CAAC,IAAI,CAAC,OAAO,IAAI,IAAI,EAAE;AACjC,YAAA,OAAO,QAAQ,CAAC,IAAI,CAAC,OAAO;;AAG9B,QAAA,OAAO,EAAE;;AAGX;;;;AAIG;AACH,IAAA,eAAe,CAAC,QAAiC,EAAA;AAC/C,QAAA,IAAI,CAAC,QAAQ,CAAC,OAAO,IAAI,CAAC,QAAQ,CAAC,IAAI,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,QAAQ,EAAE;AAClE,YAAA,OAAO,EAAE;;AAGX,QAAA,OAAO,QAAQ,CAAC,IAAI,CAAC,QAAQ;;AAEhC;AAED;;;;AAIG;MACU,sBAAsB,GAAG,CACpC,MAAiC,GAAA,EAAE,KACf;AACpB,IAAA,OAAO,IAAI,gBAAgB,CAAC,MAAM,CAAC;AACrC;;;;;;"}
+{"version":3,"file":"firecrawl.cjs","sources":["../../../../src/tools/search/firecrawl.ts"],"sourcesContent":["import axios from 'axios';\nimport { processContent } from './content';\nimport type * as t from './types';\nimport { createDefaultLogger } from './utils';\n\n/**\n * Firecrawl scraper implementation\n * Uses the Firecrawl API to scrape web pages\n */\nexport class FirecrawlScraper {\n  private apiKey: string;\n  private apiUrl: string;\n  private defaultFormats: string[];\n  private timeout: number;\n  private logger: t.Logger;\n\n  constructor(config: t.FirecrawlScraperConfig = {}) {\n    this.apiKey = config.apiKey ?? process.env.FIRECRAWL_API_KEY ?? '';\n\n    const baseUrl =\n      config.apiUrl ??\n      process.env.FIRECRAWL_BASE_URL ??\n      'https://api.firecrawl.dev';\n    this.apiUrl = `${baseUrl.replace(/\\/+$/, '')}/v1/scrape`;\n\n    this.defaultFormats = config.formats ?? ['markdown', 'html'];\n    this.timeout = config.timeout ?? 15000;\n\n    this.logger = config.logger || createDefaultLogger();\n\n    if (!this.apiKey) {\n      this.logger.warn('FIRECRAWL_API_KEY is not set. Scraping will not work.');\n    }\n\n    this.logger.debug(\n      `Firecrawl scraper initialized with API URL: ${this.apiUrl}`\n    );\n  }\n\n  /**\n   * Scrape a single URL\n   * @param url URL to scrape\n   * @param options Scrape options\n   * @returns Scrape response\n   */\n  async scrapeUrl(\n    url: string,\n    options: t.FirecrawlScrapeOptions = {}\n  ): Promise<[string, t.FirecrawlScrapeResponse]> {\n    if (!this.apiKey) {\n      return [\n        url,\n        {\n          success: false,\n          error: 'FIRECRAWL_API_KEY is not set',\n        },\n      ];\n    }\n\n    try {\n      const response = await axios.post(\n        this.apiUrl,\n        {\n          url,\n          formats: options.formats || this.defaultFormats,\n          includeTags: options.includeTags,\n          excludeTags: options.excludeTags,\n          headers: options.headers,\n          waitFor: options.waitFor,\n          timeout: options.timeout ?? this.timeout,\n        },\n        {\n          headers: {\n            'Content-Type': 'application/json',\n            Authorization: `Bearer ${this.apiKey}`,\n          },\n          timeout: this.timeout,\n        }\n      );\n\n      return [url, response.data];\n    } catch (error) {\n      const errorMessage =\n        error instanceof Error ? error.message : String(error);\n      return [\n        url,\n        {\n          success: false,\n          error: `Firecrawl API request failed: ${errorMessage}`,\n        },\n      ];\n    }\n  }\n\n  /**\n   * Extract content from scrape response\n   * @param response Scrape response\n   * @returns Extracted content or empty string if not available\n   */\n  extractContent(\n    response: t.FirecrawlScrapeResponse\n  ): [string, undefined | t.References] {\n    if (!response.success || !response.data) {\n      return ['', undefined];\n    }\n\n    if (response.data.markdown != null && response.data.html != null) {\n      try {\n        const { markdown, ...rest } = processContent(\n          response.data.html,\n          response.data.markdown\n        );\n        return [markdown, rest];\n      } catch (error) {\n        this.logger.error('Error processing content:', error);\n        return [response.data.markdown, undefined];\n      }\n    } else if (response.data.markdown != null) {\n      return [response.data.markdown, undefined];\n    }\n\n    // Fall back to HTML content\n    if (response.data.html != null) {\n      return [response.data.html, undefined];\n    }\n\n    // Fall back to raw HTML content\n    if (response.data.rawHtml != null) {\n      return [response.data.rawHtml, undefined];\n    }\n\n    return ['', undefined];\n  }\n\n  /**\n   * Extract metadata from scrape response\n   * @param response Scrape response\n   * @returns Metadata object\n   */\n  extractMetadata(response: t.FirecrawlScrapeResponse): t.ScrapeMetadata {\n    if (!response.success || !response.data || !response.data.metadata) {\n      return {};\n    }\n\n    return response.data.metadata;\n  }\n}\n\n/**\n * Create a Firecrawl scraper instance\n * @param config Scraper configuration\n * @returns Firecrawl scraper instance\n */\nexport const createFirecrawlScraper = (\n  config: t.FirecrawlScraperConfig = {}\n): FirecrawlScraper => {\n  return new FirecrawlScraper(config);\n};\n"],"names":["createDefaultLogger","processContent"],"mappings":";;;;;;AAKA;;;AAGG;MACU,gBAAgB,CAAA;AACnB,IAAA,MAAM;AACN,IAAA,MAAM;AACN,IAAA,cAAc;AACd,IAAA,OAAO;AACP,IAAA,MAAM;AAEd,IAAA,WAAA,CAAY,SAAmC,EAAE,EAAA;AAC/C,QAAA,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC,MAAM,IAAI,OAAO,CAAC,GAAG,CAAC,iBAAiB,IAAI,EAAE;AAElE,QAAA,MAAM,OAAO,GACX,MAAM,CAAC,MAAM;YACb,OAAO,CAAC,GAAG,CAAC,kBAAkB;AAC9B,YAAA,2BAA2B;AAC7B,QAAA,IAAI,CAAC,MAAM,GAAG,CAAA,EAAG,OAAO,CAAC,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC,YAAY;AAExD,QAAA,IAAI,CAAC,cAAc,GAAG,MAAM,CAAC,OAAO,IAAI,CAAC,UAAU,EAAE,MAAM,CAAC;QAC5D,IAAI,CAAC,OAAO,GAAG,MAAM,CAAC,OAAO,IAAI,KAAK;QAEtC,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC,MAAM,IAAIA,yBAAmB,EAAE;AAEpD,QAAA,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE;AAChB,YAAA,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,uDAAuD,CAAC;;QAG3E,IAAI,CAAC,MAAM,CAAC,KAAK,CACf,CAA+C,4CAAA,EAAA,IAAI,CAAC,MAAM,CAAE,CAAA,CAC7D;;AAGH;;;;;AAKG;AACH,IAAA,MAAM,SAAS,CACb,GAAW,EACX,UAAoC,EAAE,EAAA;AAEtC,QAAA,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE;YAChB,OAAO;gBACL,GAAG;AACH,gBAAA;AACE,oBAAA,OAAO,EAAE,KAAK;AACd,oBAAA,KAAK,EAAE,8BAA8B;AACtC,iBAAA;aACF;;AAGH,QAAA,IAAI;YACF,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,IAAI,CAC/B,IAAI,CAAC,MAAM,EACX;gBACE,GAAG;AACH,gBAAA,OAAO,EAAE,OAAO,CAAC,OAAO,IAAI,IAAI,CAAC,cAAc;gBAC/C,WAAW,EAAE,OAAO,CAAC,WAAW;gBAChC,WAAW,EAAE,OAAO,CAAC,WAAW;gBAChC,OAAO,EAAE,OAAO,CAAC,OAAO;gBACxB,OAAO,EAAE,OAAO,CAAC,OAAO;AACxB,gBAAA,OAAO,EAAE,OAAO,CAAC,OAAO,IAAI,IAAI,CAAC,OAAO;aACzC,EACD;AACE,gBAAA,OAAO,EAAE;AACP,oBAAA,cAAc,EAAE,kBAAkB;AAClC,oBAAA,aAAa,EAAE,CAAA,OAAA,EAAU,IAAI,CAAC,MAAM,CAAE,CAAA;AACvC,iBAAA;gBACD,OAAO,EAAE,IAAI,CAAC,OAAO;AACtB,aAAA,CACF;AAED,YAAA,OAAO,CAAC,GAAG,EAAE,QAAQ,CAAC,IAAI,CAAC;;QAC3B,OAAO,KAAK,EAAE;AACd,YAAA,MAAM,YAAY,GAChB,KAAK,YAAY,KAAK,GAAG,KAAK,CAAC,OAAO,GAAG,MAAM,CAAC,KAAK,CAAC;YACxD,OAAO;gBACL,GAAG;AACH,gBAAA;AACE,oBAAA,OAAO,EAAE,KAAK;oBACd,KAAK,EAAE,CAAiC,8BAAA,EAAA,YAAY,CAAE,CAAA;AACvD,iBAAA;aACF;;;AAIL;;;;AAIG;AACH,IAAA,cAAc,CACZ,QAAmC,EAAA;QAEnC,IAAI,CAAC,QAAQ,CAAC,OAAO,IAAI,CAAC,QAAQ,CAAC,IAAI,EAAE;AACvC,YAAA,OAAO,CAAC,EAAE,EAAE,SAAS,CAAC;;AAGxB,QAAA,IAAI,QAAQ,CAAC,IAAI,CAAC,QAAQ,IAAI,IAAI,IAAI,QAAQ,CAAC,IAAI,CAAC,IAAI,IAAI,IAAI,EAAE;AAChE,YAAA,IAAI;gBACF,MAAM,EAAE,QAAQ,EAAE,GAAG,IAAI,EAAE,GAAGC,sBAAc,CAC1C,QAAQ,CAAC,IAAI,CAAC,IAAI,EAClB,QAAQ,CAAC,IAAI,CAAC,QAAQ,CACvB;AACD,gBAAA,OAAO,CAAC,QAAQ,EAAE,IAAI,CAAC;;YACvB,OAAO,KAAK,EAAE;gBACd,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,2BAA2B,EAAE,KAAK,CAAC;gBACrD,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAC,QAAQ,EAAE,SAAS,CAAC;;;aAEvC,IAAI,QAAQ,CAAC,IAAI,CAAC,QAAQ,IAAI,IAAI,EAAE;YACzC,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAC,QAAQ,EAAE,SAAS,CAAC;;;QAI5C,IAAI,QAAQ,CAAC,IAAI,CAAC,IAAI,IAAI,IAAI,EAAE;YAC9B,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAC,IAAI,EAAE,SAAS,CAAC;;;QAIxC,IAAI,QAAQ,CAAC,IAAI,CAAC,OAAO,IAAI,IAAI,EAAE;YACjC,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAC,OAAO,EAAE,SAAS,CAAC;;AAG3C,QAAA,OAAO,CAAC,EAAE,EAAE,SAAS,CAAC;;AAGxB;;;;AAIG;AACH,IAAA,eAAe,CAAC,QAAmC,EAAA;AACjD,QAAA,IAAI,CAAC,QAAQ,CAAC,OAAO,IAAI,CAAC,QAAQ,CAAC,IAAI,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,QAAQ,EAAE;AAClE,YAAA,OAAO,EAAE;;AAGX,QAAA,OAAO,QAAQ,CAAC,IAAI,CAAC,QAAQ;;AAEhC;AAED;;;;AAIG;MACU,sBAAsB,GAAG,CACpC,MAAmC,GAAA,EAAE,KACjB;AACpB,IAAA,OAAO,IAAI,gBAAgB,CAAC,MAAM,CAAC;AACrC;;;;;"}