@librechat/agents 3.3.10 → 3.3.12

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (211) hide show
  1. package/dist/cjs/graphs/Graph.cjs +248 -27
  2. package/dist/cjs/graphs/Graph.cjs.map +1 -1
  3. package/dist/cjs/langfuseToolOutputTracing.cjs +228 -16
  4. package/dist/cjs/langfuseToolOutputTracing.cjs.map +1 -1
  5. package/dist/cjs/llm/bedrock/index.cjs +13 -2
  6. package/dist/cjs/llm/bedrock/index.cjs.map +1 -1
  7. package/dist/cjs/llm/init.cjs +1 -1
  8. package/dist/cjs/llm/invoke.cjs +160 -43
  9. package/dist/cjs/llm/invoke.cjs.map +1 -1
  10. package/dist/cjs/llm/openai/index.cjs +190 -13
  11. package/dist/cjs/llm/openai/index.cjs.map +1 -1
  12. package/dist/cjs/llm/streamLimits.cjs +723 -0
  13. package/dist/cjs/llm/streamLimits.cjs.map +1 -0
  14. package/dist/cjs/main.cjs +12 -3
  15. package/dist/cjs/messages/core.cjs +592 -27
  16. package/dist/cjs/messages/core.cjs.map +1 -1
  17. package/dist/cjs/run.cjs +7 -3
  18. package/dist/cjs/run.cjs.map +1 -1
  19. package/dist/cjs/session/AgentSession.cjs +4 -1
  20. package/dist/cjs/session/AgentSession.cjs.map +1 -1
  21. package/dist/cjs/stream.cjs +105 -14
  22. package/dist/cjs/stream.cjs.map +1 -1
  23. package/dist/cjs/summarization/node.cjs +157 -37
  24. package/dist/cjs/summarization/node.cjs.map +1 -1
  25. package/dist/cjs/tools/BashExecutor.cjs +3 -2
  26. package/dist/cjs/tools/BashExecutor.cjs.map +1 -1
  27. package/dist/cjs/tools/CodeExecutor.cjs +4 -3
  28. package/dist/cjs/tools/CodeExecutor.cjs.map +1 -1
  29. package/dist/cjs/tools/ProgrammaticToolCalling.cjs +5 -3
  30. package/dist/cjs/tools/ProgrammaticToolCalling.cjs.map +1 -1
  31. package/dist/cjs/tools/ToolNode.cjs +77 -4
  32. package/dist/cjs/tools/ToolNode.cjs.map +1 -1
  33. package/dist/cjs/tools/ToolSearch.cjs +3 -2
  34. package/dist/cjs/tools/ToolSearch.cjs.map +1 -1
  35. package/dist/cjs/tools/search/crw-scraper.cjs +7 -1
  36. package/dist/cjs/tools/search/crw-scraper.cjs.map +1 -1
  37. package/dist/cjs/tools/search/crw-search.cjs +3 -1
  38. package/dist/cjs/tools/search/crw-search.cjs.map +1 -1
  39. package/dist/cjs/tools/search/firecrawl.cjs +7 -1
  40. package/dist/cjs/tools/search/firecrawl.cjs.map +1 -1
  41. package/dist/cjs/tools/search/keenable-scraper.cjs +7 -1
  42. package/dist/cjs/tools/search/keenable-scraper.cjs.map +1 -1
  43. package/dist/cjs/tools/search/keenable-search.cjs +3 -1
  44. package/dist/cjs/tools/search/keenable-search.cjs.map +1 -1
  45. package/dist/cjs/tools/search/rerankers.cjs +26 -8
  46. package/dist/cjs/tools/search/rerankers.cjs.map +1 -1
  47. package/dist/cjs/tools/search/search.cjs +30 -10
  48. package/dist/cjs/tools/search/search.cjs.map +1 -1
  49. package/dist/cjs/tools/search/serper-scraper.cjs +7 -1
  50. package/dist/cjs/tools/search/serper-scraper.cjs.map +1 -1
  51. package/dist/cjs/tools/search/tavily-scraper.cjs +7 -1
  52. package/dist/cjs/tools/search/tavily-scraper.cjs.map +1 -1
  53. package/dist/cjs/tools/search/tavily-search.cjs +3 -1
  54. package/dist/cjs/tools/search/tavily-search.cjs.map +1 -1
  55. package/dist/cjs/tools/search/tool.cjs +17 -3
  56. package/dist/cjs/tools/search/tool.cjs.map +1 -1
  57. package/dist/cjs/tools/subagent/SubagentExecutor.cjs +54 -3
  58. package/dist/cjs/tools/subagent/SubagentExecutor.cjs.map +1 -1
  59. package/dist/cjs/utils/index.cjs +2 -1
  60. package/dist/cjs/utils/misc.cjs +12 -0
  61. package/dist/cjs/utils/misc.cjs.map +1 -1
  62. package/dist/cjs/utils/proxy.cjs +63 -0
  63. package/dist/cjs/utils/proxy.cjs.map +1 -0
  64. package/dist/esm/graphs/Graph.mjs +247 -26
  65. package/dist/esm/graphs/Graph.mjs.map +1 -1
  66. package/dist/esm/langfuseToolOutputTracing.mjs +228 -16
  67. package/dist/esm/langfuseToolOutputTracing.mjs.map +1 -1
  68. package/dist/esm/llm/bedrock/index.mjs +13 -2
  69. package/dist/esm/llm/bedrock/index.mjs.map +1 -1
  70. package/dist/esm/llm/init.mjs +1 -1
  71. package/dist/esm/llm/invoke.mjs +160 -43
  72. package/dist/esm/llm/invoke.mjs.map +1 -1
  73. package/dist/esm/llm/openai/index.mjs +192 -15
  74. package/dist/esm/llm/openai/index.mjs.map +1 -1
  75. package/dist/esm/llm/streamLimits.mjs +704 -0
  76. package/dist/esm/llm/streamLimits.mjs.map +1 -0
  77. package/dist/esm/main.mjs +8 -6
  78. package/dist/esm/messages/core.mjs +592 -28
  79. package/dist/esm/messages/core.mjs.map +1 -1
  80. package/dist/esm/run.mjs +7 -3
  81. package/dist/esm/run.mjs.map +1 -1
  82. package/dist/esm/session/AgentSession.mjs +4 -1
  83. package/dist/esm/session/AgentSession.mjs.map +1 -1
  84. package/dist/esm/stream.mjs +105 -14
  85. package/dist/esm/stream.mjs.map +1 -1
  86. package/dist/esm/summarization/node.mjs +157 -37
  87. package/dist/esm/summarization/node.mjs.map +1 -1
  88. package/dist/esm/tools/BashExecutor.mjs +3 -2
  89. package/dist/esm/tools/BashExecutor.mjs.map +1 -1
  90. package/dist/esm/tools/CodeExecutor.mjs +4 -3
  91. package/dist/esm/tools/CodeExecutor.mjs.map +1 -1
  92. package/dist/esm/tools/ProgrammaticToolCalling.mjs +5 -3
  93. package/dist/esm/tools/ProgrammaticToolCalling.mjs.map +1 -1
  94. package/dist/esm/tools/ToolNode.mjs +77 -4
  95. package/dist/esm/tools/ToolNode.mjs.map +1 -1
  96. package/dist/esm/tools/ToolSearch.mjs +3 -2
  97. package/dist/esm/tools/ToolSearch.mjs.map +1 -1
  98. package/dist/esm/tools/search/crw-scraper.mjs +7 -1
  99. package/dist/esm/tools/search/crw-scraper.mjs.map +1 -1
  100. package/dist/esm/tools/search/crw-search.mjs +3 -1
  101. package/dist/esm/tools/search/crw-search.mjs.map +1 -1
  102. package/dist/esm/tools/search/firecrawl.mjs +7 -1
  103. package/dist/esm/tools/search/firecrawl.mjs.map +1 -1
  104. package/dist/esm/tools/search/keenable-scraper.mjs +7 -1
  105. package/dist/esm/tools/search/keenable-scraper.mjs.map +1 -1
  106. package/dist/esm/tools/search/keenable-search.mjs +3 -1
  107. package/dist/esm/tools/search/keenable-search.mjs.map +1 -1
  108. package/dist/esm/tools/search/rerankers.mjs +26 -8
  109. package/dist/esm/tools/search/rerankers.mjs.map +1 -1
  110. package/dist/esm/tools/search/search.mjs +30 -10
  111. package/dist/esm/tools/search/search.mjs.map +1 -1
  112. package/dist/esm/tools/search/serper-scraper.mjs +7 -1
  113. package/dist/esm/tools/search/serper-scraper.mjs.map +1 -1
  114. package/dist/esm/tools/search/tavily-scraper.mjs +7 -1
  115. package/dist/esm/tools/search/tavily-scraper.mjs.map +1 -1
  116. package/dist/esm/tools/search/tavily-search.mjs +3 -1
  117. package/dist/esm/tools/search/tavily-search.mjs.map +1 -1
  118. package/dist/esm/tools/search/tool.mjs +17 -3
  119. package/dist/esm/tools/search/tool.mjs.map +1 -1
  120. package/dist/esm/tools/subagent/SubagentExecutor.mjs +54 -3
  121. package/dist/esm/tools/subagent/SubagentExecutor.mjs.map +1 -1
  122. package/dist/esm/utils/index.mjs +2 -1
  123. package/dist/esm/utils/misc.mjs +12 -1
  124. package/dist/esm/utils/misc.mjs.map +1 -1
  125. package/dist/esm/utils/proxy.mjs +62 -0
  126. package/dist/esm/utils/proxy.mjs.map +1 -0
  127. package/dist/types/graphs/Graph.d.ts +53 -1
  128. package/dist/types/index.d.ts +2 -0
  129. package/dist/types/langfuseToolOutputTracing.d.ts +1 -0
  130. package/dist/types/llm/invoke.d.ts +26 -5
  131. package/dist/types/llm/openai/index.d.ts +3 -0
  132. package/dist/types/llm/streamLimits.d.ts +314 -0
  133. package/dist/types/messages/core.d.ts +11 -6
  134. package/dist/types/run.d.ts +1 -0
  135. package/dist/types/summarization/node.d.ts +27 -2
  136. package/dist/types/tools/BashExecutor.d.ts +2 -2
  137. package/dist/types/tools/CodeExecutor.d.ts +3 -3
  138. package/dist/types/tools/ToolNode.d.ts +11 -1
  139. package/dist/types/tools/search/crw-scraper.d.ts +2 -0
  140. package/dist/types/tools/search/firecrawl.d.ts +2 -0
  141. package/dist/types/tools/search/keenable-scraper.d.ts +2 -0
  142. package/dist/types/tools/search/rerankers.d.ts +9 -5
  143. package/dist/types/tools/search/serper-scraper.d.ts +2 -0
  144. package/dist/types/tools/search/tavily-scraper.d.ts +2 -0
  145. package/dist/types/tools/search/types.d.ts +28 -33
  146. package/dist/types/tools/subagent/SubagentExecutor.d.ts +44 -0
  147. package/dist/types/types/graph.d.ts +7 -1
  148. package/dist/types/types/run.d.ts +46 -1
  149. package/dist/types/types/tools.d.ts +21 -0
  150. package/dist/types/utils/index.d.ts +1 -0
  151. package/dist/types/utils/misc.d.ts +7 -0
  152. package/dist/types/utils/proxy.d.ts +31 -0
  153. package/package.json +2 -1
  154. package/src/__tests__/stream.eagerArgsDivergence.test.ts +158 -0
  155. package/src/__tests__/stream.eagerEventExecution.test.ts +1 -0
  156. package/src/__tests__/stream.streamLimits.test.ts +1982 -0
  157. package/src/graphs/Graph.ts +302 -27
  158. package/src/graphs/__tests__/Graph.breakerLifecycle.test.ts +234 -0
  159. package/src/index.ts +11 -0
  160. package/src/langfuseToolOutputTracing.ts +410 -14
  161. package/src/llm/bedrock/index.ts +22 -2
  162. package/src/llm/custom-chat-models.smoke.test.ts +747 -0
  163. package/src/llm/invoke.streamLimits.test.ts +142 -0
  164. package/src/llm/invoke.test.ts +187 -1
  165. package/src/llm/invoke.ts +231 -43
  166. package/src/llm/openai/cacheWriteTokens.test.ts +112 -0
  167. package/src/llm/openai/index.ts +348 -31
  168. package/src/llm/openai/llm.spec.ts +107 -6
  169. package/src/llm/streamLimits.test.ts +450 -0
  170. package/src/llm/streamLimits.ts +1158 -0
  171. package/src/messages/core.ts +1290 -42
  172. package/src/messages/formatAgentMessages.test.ts +2623 -0
  173. package/src/run.ts +4 -0
  174. package/src/session/AgentSession.ts +5 -0
  175. package/src/specs/langfuse-tool-output-tracing.test.ts +887 -0
  176. package/src/specs/preemptSeal.test.ts +374 -5
  177. package/src/stream.ts +141 -6
  178. package/src/summarization/__tests__/node.test.ts +269 -0
  179. package/src/summarization/chunkHandler.test.ts +196 -0
  180. package/src/summarization/node.ts +203 -6
  181. package/src/tools/BashExecutor.ts +4 -3
  182. package/src/tools/CodeExecutor.ts +5 -4
  183. package/src/tools/ProgrammaticToolCalling.ts +7 -5
  184. package/src/tools/ToolNode.ts +109 -6
  185. package/src/tools/ToolSearch.ts +4 -3
  186. package/src/tools/__tests__/BashExecutor.test.ts +2 -2
  187. package/src/tools/__tests__/ProgrammaticToolCalling.test.ts +2 -4
  188. package/src/tools/__tests__/SubagentExecutor.test.ts +126 -0
  189. package/src/tools/__tests__/ToolNode.breakerSignal.test.ts +389 -0
  190. package/src/tools/__tests__/ToolNode.streamLimits.test.ts +69 -0
  191. package/src/tools/search/crw-scraper.ts +6 -0
  192. package/src/tools/search/crw-search.ts +6 -1
  193. package/src/tools/search/firecrawl.ts +6 -0
  194. package/src/tools/search/http-agent.test.ts +133 -0
  195. package/src/tools/search/keenable-scraper.ts +6 -0
  196. package/src/tools/search/keenable-search.ts +6 -1
  197. package/src/tools/search/rerankers.ts +36 -10
  198. package/src/tools/search/search.ts +29 -11
  199. package/src/tools/search/serper-scraper.ts +6 -0
  200. package/src/tools/search/tavily-scraper.ts +6 -0
  201. package/src/tools/search/tavily-search.ts +2 -0
  202. package/src/tools/search/tool.ts +16 -0
  203. package/src/tools/search/types.ts +31 -33
  204. package/src/tools/subagent/SubagentExecutor.ts +96 -3
  205. package/src/types/graph.ts +7 -0
  206. package/src/types/run.ts +49 -1
  207. package/src/types/tools.ts +21 -0
  208. package/src/utils/index.ts +1 -0
  209. package/src/utils/misc.ts +19 -0
  210. package/src/utils/proxy.test.ts +176 -0
  211. package/src/utils/proxy.ts +93 -0
@@ -1,11 +1,11 @@
1
1
  import "../common/enum.mjs";
2
2
  import "../common/index.mjs";
3
3
  import { INTENT_PROPERTY } from "./intentArg.mjs";
4
+ import { resolveFetchProxyAgent } from "../utils/proxy.mjs";
4
5
  import { getCodeBaseURL } from "./CodeExecutor.mjs";
5
6
  import { tool } from "@langchain/core/tools";
6
7
  import { config } from "dotenv";
7
8
  import fetch from "node-fetch";
8
- import { HttpsProxyAgent } from "https-proxy-agent";
9
9
  import * as okapibm25Module from "okapibm25";
10
10
  //#region src/tools/ToolSearch.ts
11
11
  function getBM25Function() {
@@ -748,7 +748,8 @@ ${deferredToolsListing}` : "";
748
748
  },
749
749
  body: JSON.stringify(postData)
750
750
  };
751
- if (process.env.PROXY != null && process.env.PROXY !== "") fetchOptions.agent = new HttpsProxyAgent(process.env.PROXY);
751
+ const proxyAgent = resolveFetchProxyAgent(EXEC_ENDPOINT);
752
+ if (proxyAgent) fetchOptions.agent = proxyAgent;
752
753
  const response = await fetch(EXEC_ENDPOINT, fetchOptions);
753
754
  if (!response.ok) throw new Error(`HTTP error! status: ${response.status}`);
754
755
  const result = await response.json();
@@ -1 +1 @@
1
- {"version":3,"file":"ToolSearch.mjs","names":["params"],"sources":["../../../src/tools/ToolSearch.ts"],"sourcesContent":["// src/tools/ToolSearch.ts\nimport { config } from 'dotenv';\nimport * as okapibm25Module from 'okapibm25';\n\ntype BM25Fn = (\n documents: string[],\n keywords: string[],\n constants?: { k1?: number; b?: number }\n) => number[];\n\nfunction getBM25Function(): BM25Fn {\n const mod = okapibm25Module as unknown as {\n default: BM25Fn | { default: BM25Fn } | undefined;\n };\n if (typeof mod === 'function') return mod;\n if (typeof mod.default === 'function') return mod.default;\n if (mod.default != null && typeof mod.default.default === 'function')\n return mod.default.default;\n throw new Error('Could not resolve BM25 function from okapibm25 module');\n}\n\nconst BM25 = getBM25Function();\nimport fetch, { RequestInit } from 'node-fetch';\nimport { HttpsProxyAgent } from 'https-proxy-agent';\nimport { tool, DynamicStructuredTool } from '@langchain/core/tools';\nimport type * as t from '@/types';\nimport { INTENT_PROPERTY } from '@/tools/intentArg';\nimport { getCodeBaseURL } from './CodeExecutor';\nimport { Constants } from '@/common';\n\nconfig();\n\n/** Maximum allowed regex pattern length */\nconst MAX_PATTERN_LENGTH = 200;\n\nexport const ToolSearchToolName = Constants.TOOL_SEARCH;\n\nexport const ToolSearchToolDescription =\n 'Searches deferred tools using BM25 ranking. Multi-word queries supported. Use mcp_server param to filter by server.';\n\nconst QUERY_DESCRIPTION_LOCAL =\n 'Search term to find in tool names and descriptions. Case-insensitive substring matching. Optional if mcp_server is provided.';\nconst QUERY_DESCRIPTION_REGEX =\n 'Regex pattern to search tool names and descriptions. Optional if mcp_server is provided.';\nconst FIELDS_DESCRIPTION =\n 'Which fields to search. Default: name and description';\nconst MAX_RESULTS_DESCRIPTION = 'Maximum number of matching tools to return';\nconst DEFAULT_MAX_RESULTS = 5;\nconst MCP_SERVER_DESCRIPTION =\n 'Filter to tools from specific MCP server(s). Can be a single server name or array of names. If provided without a query, lists all tools from those servers.';\n\nexport const ToolSearchToolSchema = {\n type: 'object',\n properties: {\n intent: { ...INTENT_PROPERTY },\n query: {\n type: 'string',\n maxLength: MAX_PATTERN_LENGTH,\n default: '',\n description: QUERY_DESCRIPTION_LOCAL,\n },\n fields: {\n type: 'array',\n items: { type: 'string', enum: ['name', 'description', 'parameters'] },\n default: ['name', 'description'],\n description: FIELDS_DESCRIPTION,\n },\n max_results: {\n type: 'integer',\n minimum: 1,\n maximum: 50,\n default: DEFAULT_MAX_RESULTS,\n description: MAX_RESULTS_DESCRIPTION,\n },\n mcp_server: {\n oneOf: [{ type: 'string' }, { type: 'array', items: { type: 'string' } }],\n description: MCP_SERVER_DESCRIPTION,\n },\n },\n required: [],\n} as const;\n\nexport const ToolSearchToolDefinition = {\n name: ToolSearchToolName,\n description: ToolSearchToolDescription,\n schema: ToolSearchToolSchema,\n} as const;\n\n/** Maximum allowed regex nesting depth */\nconst MAX_REGEX_COMPLEXITY = 5;\n\n/** Default search timeout in milliseconds */\nconst SEARCH_TIMEOUT = 5000;\n\n/** JSON schema type for tool search parameters */\ninterface ToolSearchSchema {\n type: 'object';\n properties: Record<string, unknown>;\n required: string[];\n}\n\n/** Input params type for tool search */\ninterface ToolSearchParams {\n query?: string;\n fields?: ('name' | 'description' | 'parameters')[];\n max_results?: number;\n mcp_server?: string | string[];\n}\n\n/**\n * Creates the JSON schema with dynamic query description based on mode.\n * @param mode - The search mode determining query interpretation\n * @returns JSON schema for tool search parameters\n */\nfunction createToolSearchSchema(mode: t.ToolSearchMode): ToolSearchSchema {\n const queryDescription =\n mode === 'local' ? QUERY_DESCRIPTION_LOCAL : QUERY_DESCRIPTION_REGEX;\n\n return {\n type: 'object',\n properties: {\n intent: { ...INTENT_PROPERTY },\n query: {\n type: 'string',\n maxLength: MAX_PATTERN_LENGTH,\n default: '',\n description: queryDescription,\n },\n fields: {\n type: 'array',\n items: { type: 'string', enum: ['name', 'description', 'parameters'] },\n default: ['name', 'description'],\n description: FIELDS_DESCRIPTION,\n },\n max_results: {\n type: 'integer',\n minimum: 1,\n maximum: 50,\n default: DEFAULT_MAX_RESULTS,\n description: MAX_RESULTS_DESCRIPTION,\n },\n mcp_server: {\n oneOf: [\n { type: 'string' },\n { type: 'array', items: { type: 'string' } },\n ],\n description: MCP_SERVER_DESCRIPTION,\n },\n },\n required: [],\n };\n}\n\n/**\n * Extracts the MCP server name from a tool name.\n * MCP tools follow the pattern: toolName_mcp_serverName\n * @param toolName - The full tool name\n * @returns The server name if it's an MCP tool, undefined otherwise\n */\nfunction extractMcpServerName(toolName: string): string | undefined {\n const delimiterIndex = toolName.indexOf(Constants.MCP_DELIMITER);\n if (delimiterIndex === -1) {\n return undefined;\n }\n return toolName.substring(delimiterIndex + Constants.MCP_DELIMITER.length);\n}\n\n/**\n * Checks if a tool belongs to a specific MCP server.\n * @param toolName - The full tool name\n * @param serverName - The server name to match\n * @returns True if the tool belongs to the specified server\n */\nfunction isFromMcpServer(toolName: string, serverName: string): boolean {\n const toolServer = extractMcpServerName(toolName);\n return toolServer === serverName;\n}\n\n/**\n * Checks if a tool belongs to any of the specified MCP servers.\n * @param toolName - The full tool name\n * @param serverNames - Array of server names to match\n * @returns True if the tool belongs to any of the specified servers\n */\nfunction isFromAnyMcpServer(toolName: string, serverNames: string[]): boolean {\n const toolServer = extractMcpServerName(toolName);\n if (toolServer === undefined) {\n return false;\n }\n return serverNames.includes(toolServer);\n}\n\n/**\n * Normalizes server filter input to always be an array.\n * @param serverFilter - String, array of strings, or undefined\n * @returns Array of server names (empty if none specified)\n */\nfunction normalizeServerFilter(\n serverFilter: string | string[] | undefined\n): string[] {\n if (serverFilter === undefined) {\n return [];\n }\n if (typeof serverFilter === 'string') {\n return serverFilter === '' ? [] : [serverFilter];\n }\n return serverFilter.filter((s) => s !== '');\n}\n\n/**\n * Extracts all unique MCP server names from a tool registry.\n * @param toolRegistry - The tool registry to scan\n * @param onlyDeferred - If true, only considers deferred tools\n * @returns Array of unique server names, sorted alphabetically\n */\nfunction getAvailableMcpServers(\n toolRegistry: t.LCToolRegistry | undefined,\n onlyDeferred: boolean = true\n): string[] {\n if (!toolRegistry) {\n return [];\n }\n\n const servers = new Set<string>();\n for (const [, toolDef] of toolRegistry) {\n if (onlyDeferred && toolDef.defer_loading !== true) {\n continue;\n }\n const server = extractMcpServerName(toolDef.name);\n if (server !== undefined && server !== '') {\n servers.add(server);\n }\n }\n\n return Array.from(servers).sort();\n}\n\n/**\n * Escapes special regex characters in a string to use as a literal pattern.\n * @param pattern - The string to escape\n * @returns The escaped string safe for use in a RegExp\n */\nfunction escapeRegexSpecialChars(pattern: string): string {\n return pattern.replace(/[.*+?^${}()|[\\]\\\\]/g, '\\\\$&');\n}\n\n/**\n * Counts the maximum nesting depth of groups in a regex pattern.\n * @param pattern - The regex pattern to analyze\n * @returns The maximum nesting depth\n */\nfunction countNestedGroups(pattern: string): number {\n let maxDepth = 0;\n let currentDepth = 0;\n\n for (let i = 0; i < pattern.length; i++) {\n if (pattern[i] === '(' && (i === 0 || pattern[i - 1] !== '\\\\')) {\n currentDepth++;\n maxDepth = Math.max(maxDepth, currentDepth);\n } else if (pattern[i] === ')' && (i === 0 || pattern[i - 1] !== '\\\\')) {\n currentDepth = Math.max(0, currentDepth - 1);\n }\n }\n\n return maxDepth;\n}\n\n/**\n * Detects nested quantifiers that can cause catastrophic backtracking.\n * Patterns like (a+)+, (a*)*, (a+)*, etc.\n * @param pattern - The regex pattern to check\n * @returns True if nested quantifiers are detected\n */\nfunction hasNestedQuantifiers(pattern: string): boolean {\n const nestedQuantifierPattern = /\\([^)]*[+*][^)]*\\)[+*?]/;\n return nestedQuantifierPattern.test(pattern);\n}\n\n/**\n * Checks if a regex pattern contains potentially dangerous constructs.\n * @param pattern - The regex pattern to validate\n * @returns True if the pattern is dangerous\n */\nfunction isDangerousPattern(pattern: string): boolean {\n if (hasNestedQuantifiers(pattern)) {\n return true;\n }\n\n if (countNestedGroups(pattern) > MAX_REGEX_COMPLEXITY) {\n return true;\n }\n\n const dangerousPatterns = [\n /\\.\\{1000,\\}/, // Excessive wildcards\n /\\(\\?=\\.\\{100,\\}\\)/, // Runaway lookaheads\n /\\([^)]*\\|\\s*\\){20,}/, // Excessive alternation (rough check)\n /\\(\\.\\*\\)\\+/, // (.*)+\n /\\(\\.\\+\\)\\+/, // (.+)+\n /\\(\\.\\*\\)\\*/, // (.*)*\n /\\(\\.\\+\\)\\*/, // (.+)*\n ];\n\n for (const dangerous of dangerousPatterns) {\n if (dangerous.test(pattern)) {\n return true;\n }\n }\n\n return false;\n}\n\n/**\n * Sanitizes a regex pattern for safe execution.\n * If the pattern is dangerous, it will be escaped to a literal string search.\n * @param pattern - The regex pattern to sanitize\n * @returns Object containing the safe pattern and whether it was escaped\n */\nfunction sanitizeRegex(pattern: string): { safe: string; wasEscaped: boolean } {\n if (isDangerousPattern(pattern)) {\n return {\n safe: escapeRegexSpecialChars(pattern),\n wasEscaped: true,\n };\n }\n\n try {\n new RegExp(pattern);\n return { safe: pattern, wasEscaped: false };\n } catch {\n return {\n safe: escapeRegexSpecialChars(pattern),\n wasEscaped: true,\n };\n }\n}\n\n/**\n * Simplifies tool parameters for search purposes.\n * Extracts only the essential structure needed for parameter name searching.\n * @param parameters - The tool's JSON schema parameters\n * @returns Simplified parameters object\n */\nfunction simplifyParametersForSearch(\n parameters?: t.JsonSchemaType\n): t.JsonSchemaType | undefined {\n if (!parameters) {\n return undefined;\n }\n\n if (parameters.properties) {\n return {\n type: parameters.type,\n properties: Object.fromEntries(\n Object.entries(parameters.properties).map(([key, value]) => [\n key,\n { type: (value as t.JsonSchemaType).type },\n ])\n ),\n } as t.JsonSchemaType;\n }\n\n return { type: parameters.type };\n}\n\n/**\n * Splits one alphanumeric identifier segment on case boundaries without\n * emitting artificial one-character acronym fragments.\n */\nfunction splitCaseSegment(segment: string): string[] {\n const splitTokens = segment\n .replace(/([a-z0-9])([A-Z])/g, '$1 $2')\n .replace(/([A-Z]+)([A-Z][a-z])/g, '$1 $2')\n .toLowerCase()\n .split(/\\s+/)\n .filter(Boolean);\n\n if (splitTokens.length < 2) return splitTokens;\n\n const mergedTokens: string[] = [];\n let prefix = '';\n for (const token of splitTokens) {\n if (token.length === 1) {\n prefix += token;\n continue;\n }\n mergedTokens.push(`${prefix}${token}`);\n prefix = '';\n }\n\n if (prefix && mergedTokens.length > 0) {\n mergedTokens[mergedTokens.length - 1] += prefix;\n }\n return mergedTokens.length > 0 ? mergedTokens : [prefix];\n}\n\n/**\n * Tokenizes a string into lowercase words for BM25.\n * Splits camelCase, underscores, and non-alphanumeric characters for consistent matching.\n * @param text - The text to tokenize\n * @returns Array of lowercase tokens\n */\nfunction tokenize(text: string): string[] {\n return text\n .split(/[^a-zA-Z0-9]+/)\n .filter(Boolean)\n .flatMap(splitCaseSegment);\n}\n\n/**\n * Creates a searchable document string from tool metadata.\n * @param tool - The tool metadata\n * @param fields - Which fields to include\n * @returns Combined document string for BM25\n */\nfunction createToolDocument(tool: t.ToolMetadata, fields: string[]): string {\n const parts: string[] = [];\n\n if (fields.includes('name')) {\n const baseName = tool.name.replace(/_/g, ' ');\n parts.push(baseName, baseName);\n }\n\n if (fields.includes('description') && tool.description) {\n parts.push(tool.description);\n }\n\n if (fields.includes('parameters') && tool.parameters?.properties) {\n const paramNames = Object.keys(tool.parameters.properties).join(' ');\n parts.push(paramNames);\n }\n\n return tokenize(parts.join(' ')).join(' ');\n}\n\n/**\n * Determines which field had the best match for a query.\n * @param tool - The tool to check\n * @param queryTokens - Tokenized query\n * @param fields - Fields to check\n * @returns The matched field and a snippet\n */\nfunction findMatchedField(\n tool: t.ToolMetadata,\n queryTokens: string[],\n fields: string[]\n): { field: string; snippet: string } {\n if (fields.includes('name')) {\n const nameLower = tool.name.toLowerCase();\n for (const token of queryTokens) {\n if (nameLower.includes(token)) {\n return { field: 'name', snippet: tool.name };\n }\n }\n }\n\n if (fields.includes('description') && tool.description) {\n const descLower = tool.description.toLowerCase();\n for (const token of queryTokens) {\n if (descLower.includes(token)) {\n return {\n field: 'description',\n snippet: tool.description.substring(0, 100),\n };\n }\n }\n }\n\n if (fields.includes('parameters') && tool.parameters?.properties) {\n const paramNames = Object.keys(tool.parameters.properties);\n const paramLower = paramNames.join(' ').toLowerCase();\n for (const token of queryTokens) {\n if (paramLower.includes(token)) {\n return { field: 'parameters', snippet: paramNames.join(', ') };\n }\n }\n }\n\n const fallbackSnippet = tool.description\n ? tool.description.substring(0, 100)\n : tool.name;\n return { field: 'unknown', snippet: fallbackSnippet };\n}\n\n/**\n * Performs BM25-based search for better relevance ranking.\n * Uses Okapi BM25 algorithm for term frequency and document length normalization.\n * If query is empty, returns all tools (up to maxResults) sorted alphabetically.\n * @param tools - Array of tool metadata to search\n * @param query - The search query (empty returns all tools)\n * @param fields - Which fields to search\n * @param maxResults - Maximum results to return\n * @returns Search response with matching tools ranked by BM25 score\n */\nfunction performLocalSearch(\n tools: t.ToolMetadata[],\n query: string,\n fields: string[],\n maxResults: number\n): t.ToolSearchResponse {\n if (tools.length === 0) {\n return {\n tool_references: [],\n total_tools_searched: 0,\n pattern_used: query,\n };\n }\n\n const queryTokens = tokenize(query);\n\n if (queryTokens.length === 0) {\n const allTools = tools\n .slice()\n .sort((a, b) => a.name.localeCompare(b.name))\n .slice(0, maxResults)\n .map((tool) => ({\n tool_name: tool.name,\n match_score: 1.0,\n matched_field: 'name',\n snippet: tool.description.substring(0, 100) || tool.name,\n }));\n\n return {\n tool_references: allTools,\n total_tools_searched: tools.length,\n pattern_used: query,\n };\n }\n\n const documents = tools.map((tool) => createToolDocument(tool, fields));\n const scores = BM25(documents, queryTokens, { k1: 1.5, b: 0.75 }) as number[];\n\n const maxScore = Math.max(...scores.filter((s) => s > 0), 1);\n const queryLower = query.toLowerCase().trim();\n const queryIdentifier = queryTokens.join('');\n const matchesIdentifiers = fields.includes('name');\n\n const results: Array<{\n result: t.ToolSearchResult;\n identifierPriority: number;\n }> = [];\n for (let i = 0; i < tools.length; i++) {\n const score = scores[i];\n const hasSearchScore = Number.isFinite(score) && score > 0;\n let identifierPriority = 0;\n let normalizedScore = hasSearchScore ? Math.min(score / maxScore, 1.0) : 0;\n\n if (matchesIdentifiers) {\n const rawBaseName = getBaseToolName(tools[i].name).toLowerCase();\n const rawFullName = tools[i].name.toLowerCase();\n const baseIdentifier = tokenize(rawBaseName).join('');\n const fullIdentifier = tokenize(rawFullName).join('');\n\n if (rawFullName === queryLower) {\n identifierPriority = 4;\n normalizedScore = 1.0;\n } else if (rawBaseName === queryLower) {\n identifierPriority = 3;\n normalizedScore = 1.0;\n } else if (\n baseIdentifier === queryIdentifier ||\n fullIdentifier === queryIdentifier\n ) {\n identifierPriority = 2;\n normalizedScore = 1.0;\n } else if (baseIdentifier.startsWith(queryIdentifier)) {\n identifierPriority = 1;\n normalizedScore = Math.max(normalizedScore, 0.95);\n }\n }\n\n if (!hasSearchScore && identifierPriority === 0) continue;\n\n const { field, snippet } = findMatchedField(tools[i], queryTokens, fields);\n results.push({\n result: {\n tool_name: tools[i].name,\n match_score: normalizedScore,\n matched_field: field,\n snippet,\n },\n identifierPriority,\n });\n }\n\n results.sort(\n (a, b) =>\n b.identifierPriority - a.identifierPriority ||\n b.result.match_score - a.result.match_score\n );\n const topResults = results.slice(0, maxResults).map(({ result }) => result);\n\n return {\n tool_references: topResults,\n total_tools_searched: tools.length,\n pattern_used: query,\n };\n}\n\n/**\n * Generates the JavaScript search script to be executed in the sandbox.\n * Uses plain JavaScript for maximum compatibility with the Code API.\n * @param deferredTools - Array of tool metadata to search through\n * @param fields - Which fields to search\n * @param maxResults - Maximum number of results to return\n * @param sanitizedPattern - The sanitized regex pattern\n * @returns The JavaScript code string\n */\nfunction generateSearchScript(\n deferredTools: t.ToolMetadata[],\n fields: string[],\n maxResults: number,\n sanitizedPattern: string\n): string {\n const lines = [\n '// Tool definitions (injected)',\n 'var tools = ' + JSON.stringify(deferredTools) + ';',\n 'var searchFields = ' + JSON.stringify(fields) + ';',\n 'var maxResults = ' + maxResults + ';',\n 'var pattern = ' + JSON.stringify(sanitizedPattern) + ';',\n '',\n '// Compile regex (pattern is sanitized client-side)',\n 'var regex;',\n 'try {',\n ' regex = new RegExp(pattern, \\'i\\');',\n '} catch (e) {',\n ' regex = new RegExp(pattern.replace(/[.*+?^${}()[\\\\]\\\\\\\\|]/g, \"\\\\\\\\$&\"), \"i\");',\n '}',\n '',\n '// Search logic',\n 'var results = [];',\n '',\n 'for (var j = 0; j < tools.length; j++) {',\n ' var tool = tools[j];',\n ' var bestScore = 0;',\n ' var matchedField = \\'\\';',\n ' var snippet = \\'\\';',\n '',\n ' // Search name (highest priority)',\n ' if (searchFields.indexOf(\\'name\\') >= 0 && regex.test(tool.name)) {',\n ' bestScore = 0.95;',\n ' matchedField = \\'name\\';',\n ' snippet = tool.name;',\n ' }',\n '',\n ' // Search description (medium priority)',\n ' if (searchFields.indexOf(\\'description\\') >= 0 && tool.description && regex.test(tool.description)) {',\n ' if (bestScore === 0) {',\n ' bestScore = 0.75;',\n ' matchedField = \\'description\\';',\n ' snippet = tool.description.substring(0, 100);',\n ' }',\n ' }',\n '',\n ' // Search parameter names (lower priority)',\n ' if (searchFields.indexOf(\\'parameters\\') >= 0 && tool.parameters && tool.parameters.properties) {',\n ' var paramNames = Object.keys(tool.parameters.properties).join(\\' \\');',\n ' if (regex.test(paramNames)) {',\n ' if (bestScore === 0) {',\n ' bestScore = 0.60;',\n ' matchedField = \\'parameters\\';',\n ' snippet = paramNames;',\n ' }',\n ' }',\n ' }',\n '',\n ' if (bestScore > 0) {',\n ' results.push({',\n ' tool_name: tool.name,',\n ' match_score: bestScore,',\n ' matched_field: matchedField,',\n ' snippet: snippet',\n ' });',\n ' }',\n '}',\n '',\n '// Sort by score (descending) and limit results',\n 'results.sort(function(a, b) { return b.match_score - a.match_score; });',\n 'var topResults = results.slice(0, maxResults);',\n '',\n '// Output as JSON',\n 'console.log(JSON.stringify({',\n ' tool_references: topResults.map(function(r) {',\n ' return {',\n ' tool_name: r.tool_name,',\n ' match_score: r.match_score,',\n ' matched_field: r.matched_field,',\n ' snippet: r.snippet',\n ' };',\n ' }),',\n ' total_tools_searched: tools.length,',\n ' pattern_used: pattern',\n '}));',\n ];\n return lines.join('\\n');\n}\n\n/**\n * Parses the search results from stdout JSON.\n * @param stdout - The stdout string containing JSON results\n * @returns Parsed search response\n */\nfunction parseSearchResults(stdout: string): t.ToolSearchResponse {\n const jsonMatch = stdout.trim();\n const parsed = JSON.parse(jsonMatch) as t.ToolSearchResponse;\n return parsed;\n}\n\n/**\n * Formats search results as structured JSON for efficient parsing.\n * @param searchResponse - The parsed search response\n * @param nameFormat - Whether to show 'full' names (tool_mcp_server) or 'base' names (tool only)\n * @returns JSON string with search results\n */\nfunction formatSearchResults(\n searchResponse: t.ToolSearchResponse,\n nameFormat: t.McpNameFormat = 'full'\n): string {\n const { tool_references, total_tools_searched, pattern_used } =\n searchResponse;\n const useFullName = nameFormat === 'full';\n\n const output = {\n found: tool_references.length,\n tools: tool_references.map((ref) => ({\n name: useFullName ? ref.tool_name : getBaseToolName(ref.tool_name),\n score: Number(ref.match_score.toFixed(2)),\n matched_in: ref.matched_field,\n snippet: ref.snippet,\n })),\n total_searched: total_tools_searched,\n query: pattern_used,\n };\n\n return JSON.stringify(output, null, 2);\n}\n\n/**\n * Extracts the base tool name (without MCP server suffix) from a full tool name.\n * @param toolName - The full tool name\n * @returns The base tool name without server suffix\n */\nfunction getBaseToolName(toolName: string): string {\n const delimiterIndex = toolName.indexOf(Constants.MCP_DELIMITER);\n if (delimiterIndex === -1) {\n return toolName;\n }\n return toolName.substring(0, delimiterIndex);\n}\n\n/**\n * Checks whether a tool has any defined parameters in its JSON schema.\n * @param parameters - The tool's JSON schema parameters\n * @returns true if the tool has at least one parameter property\n */\nfunction hasParams(parameters?: t.JsonSchemaType): boolean {\n return (\n parameters?.properties != null &&\n Object.keys(parameters.properties).length > 0\n );\n}\n\n/**\n * Generates a compact listing of deferred tools grouped by server.\n * Format: \"server: tool1, tool2(\\u2026), tool3\"\n * Tools with parameters are annotated with (\\u2026) to signal\n * that the LLM should discover the schema via tool_search before calling.\n * Non-MCP tools are grouped under \"other\".\n * @param toolRegistry - The tool registry\n * @param onlyDeferred - Whether to only include deferred tools\n * @returns Formatted string with tools grouped by server\n */\nfunction getDeferredToolsListing(\n toolRegistry: t.LCToolRegistry | undefined,\n onlyDeferred: boolean\n): string {\n if (!toolRegistry) {\n return '';\n }\n\n const toolsByServer: Record<string, string[]> = {};\n\n for (const lcTool of toolRegistry.values()) {\n if (onlyDeferred && lcTool.defer_loading !== true) {\n continue;\n }\n\n const toolName = lcTool.name;\n const serverName = extractMcpServerName(toolName) ?? 'other';\n const baseName = getBaseToolName(toolName);\n const displayName = hasParams(lcTool.parameters)\n ? `${baseName}(\\u2026)`\n : baseName;\n\n if (!(serverName in toolsByServer)) {\n toolsByServer[serverName] = [];\n }\n toolsByServer[serverName].push(displayName);\n }\n\n const serverNames = Object.keys(toolsByServer).sort((a, b) => {\n if (a === 'other') return 1;\n if (b === 'other') return -1;\n return a.localeCompare(b);\n });\n\n if (serverNames.length === 0) {\n return '';\n }\n\n const lines = serverNames.map(\n (server) => `${server}: ${toolsByServer[server].join(', ')}`\n );\n\n return lines.join('\\n');\n}\n\n/**\n * Formats a server listing response as structured JSON.\n * NOTE: This is a PREVIEW only - tools are NOT discovered/loaded.\n * @param tools - Array of tool metadata from the server(s)\n * @param serverNames - The MCP server name(s)\n * @param nameFormat - Whether to show 'full' names (tool_mcp_server) or 'base' names (tool only)\n * @returns JSON string showing all tools grouped by server\n */\nfunction formatServerListing(\n tools: t.ToolMetadata[],\n serverNames: string | string[],\n nameFormat: t.McpNameFormat = 'full'\n): string {\n const servers = Array.isArray(serverNames) ? serverNames : [serverNames];\n const useFullName = nameFormat === 'full';\n\n if (tools.length === 0) {\n return JSON.stringify(\n {\n listing_mode: true,\n servers,\n total_tools: 0,\n tools_by_server: {},\n hint: 'No tools found from the specified MCP server(s).',\n },\n null,\n 2\n );\n }\n\n const toolsByServer: Record<\n string,\n Array<{ name: string; description: string }>\n > = {};\n for (const tool of tools) {\n const server = extractMcpServerName(tool.name) ?? 'unknown';\n if (!(server in toolsByServer)) {\n toolsByServer[server] = [];\n }\n toolsByServer[server].push({\n name: useFullName ? tool.name : getBaseToolName(tool.name),\n description:\n tool.description.length > 100\n ? tool.description.substring(0, 97) + '...'\n : tool.description,\n });\n }\n\n const exampleToolName = useFullName\n ? (tools[0]?.name ?? 'tool_name')\n : getBaseToolName(tools[0]?.name ?? 'tool_name');\n\n const output = {\n listing_mode: true,\n servers,\n total_tools: tools.length,\n tools_by_server: toolsByServer,\n hint: `To use a tool, search for it by name (e.g., query: \"${exampleToolName}\") to load it.`,\n };\n\n return JSON.stringify(output, null, 2);\n}\n\n/**\n * Creates a Tool Search tool for discovering tools from a large registry.\n *\n * This tool enables AI agents to dynamically discover tools from a large library\n * without loading all tool definitions into the LLM context window. The agent\n * can search for relevant tools on-demand.\n *\n * **Modes:**\n * - `code_interpreter` (default): Uses external sandbox for regex search. Safer for complex patterns.\n * - `local`: Uses safe substring matching locally. No network call, faster, completely safe from ReDoS.\n *\n * The tool registry can be provided either:\n * 1. At initialization time via params.toolRegistry\n * 2. At runtime via config.configurable.toolRegistry when invoking\n *\n * @param params - Configuration parameters for the tool (toolRegistry is optional)\n * @returns A LangChain DynamicStructuredTool for tool searching\n *\n * @example\n * // Option 1: Code interpreter mode (regex via sandbox)\n * const tool = createToolSearch({ toolRegistry });\n * await tool.invoke({ query: 'expense.*report' });\n *\n * @example\n * // Option 2: Local mode (safe substring search)\n * const tool = createToolSearch({ mode: 'local', toolRegistry });\n * await tool.invoke({ query: 'expense' });\n */\nfunction createToolSearch(\n initParams: t.ToolSearchParams = {}\n): DynamicStructuredTool {\n const mode: t.ToolSearchMode = initParams.mode ?? 'code_interpreter';\n const defaultOnlyDeferred = initParams.onlyDeferred ?? true;\n const mcpNameFormat: t.McpNameFormat = initParams.mcpNameFormat ?? 'full';\n const schema = createToolSearchSchema(mode);\n\n const baseEndpoint = initParams.baseUrl ?? getCodeBaseURL();\n const EXEC_ENDPOINT = `${baseEndpoint}/exec`;\n\n const deferredToolsListing = getDeferredToolsListing(\n initParams.toolRegistry,\n defaultOnlyDeferred\n );\n\n const toolsListSection =\n deferredToolsListing.length > 0\n ? `\n\nDeferred tools (search to load; \\u2026 = has params, search first):\n${deferredToolsListing}`\n : '';\n\n const mcpNote =\n deferredToolsListing.includes(Constants.MCP_DELIMITER) ||\n deferredToolsListing.split('\\n').some((line) => !line.startsWith('other:'))\n ? `\n- MCP tools use format: toolName${Constants.MCP_DELIMITER}serverName\n- Use mcp_server param to filter by server`\n : '';\n\n const description =\n mode === 'local'\n ? `\nSearches deferred tools using BM25 ranking. Multi-word queries supported.\n${mcpNote}${toolsListSection}\n`.trim()\n : `\nSearches deferred tools by regex pattern.\n${mcpNote}${toolsListSection}\n`.trim();\n\n return tool(\n async (rawParams, config) => {\n const params = rawParams as ToolSearchParams;\n const {\n query = '',\n fields = ['name', 'description'],\n max_results = DEFAULT_MAX_RESULTS,\n mcp_server,\n } = params;\n\n const {\n toolRegistry: paramToolRegistry,\n onlyDeferred: paramOnlyDeferred,\n mcpServer: paramMcpServer,\n } = config.toolCall ?? {};\n\n const toolRegistry = paramToolRegistry ?? initParams.toolRegistry;\n const onlyDeferred =\n paramOnlyDeferred !== undefined\n ? paramOnlyDeferred\n : defaultOnlyDeferred;\n const rawServerFilter =\n mcp_server ?? paramMcpServer ?? initParams.mcpServer;\n const serverFilters = normalizeServerFilter(rawServerFilter);\n const hasServerFilter = serverFilters.length > 0;\n\n if (toolRegistry == null) {\n return [\n 'Error: No tool registry provided. Configure toolRegistry at agent level or initialization.',\n {\n tool_references: [],\n metadata: {\n total_searched: 0,\n pattern: query,\n error: 'No tool registry provided',\n },\n },\n ];\n }\n\n const toolsArray: t.LCTool[] = Array.from(toolRegistry.values());\n const deferredTools: t.ToolMetadata[] = toolsArray\n .filter((lcTool) => {\n if (onlyDeferred === true && lcTool.defer_loading !== true) {\n return false;\n }\n if (\n hasServerFilter &&\n !isFromAnyMcpServer(lcTool.name, serverFilters)\n ) {\n return false;\n }\n return true;\n })\n .map((lcTool) => ({\n name: lcTool.name,\n description: lcTool.description ?? '',\n parameters: simplifyParametersForSearch(lcTool.parameters),\n }));\n\n if (deferredTools.length === 0) {\n const serverMsg = hasServerFilter\n ? ` from MCP server(s): ${serverFilters.join(', ')}`\n : '';\n return [\n `No tools available to search${serverMsg}. The tool registry is empty or no matching deferred tools are registered.`,\n {\n tool_references: [],\n metadata: {\n total_searched: 0,\n pattern: query,\n mcp_server: serverFilters,\n },\n },\n ];\n }\n\n const isServerListing = hasServerFilter && query === '';\n\n if (isServerListing) {\n const formattedOutput = formatServerListing(\n deferredTools,\n serverFilters,\n mcpNameFormat\n );\n\n return [\n formattedOutput,\n {\n tool_references: [],\n metadata: {\n total_available: deferredTools.length,\n mcp_server: serverFilters,\n listing_mode: true,\n },\n },\n ];\n }\n\n if (mode === 'local') {\n const searchResponse = performLocalSearch(\n deferredTools,\n query,\n fields,\n max_results\n );\n const formattedOutput = formatSearchResults(\n searchResponse,\n mcpNameFormat\n );\n\n return [\n formattedOutput,\n {\n tool_references: searchResponse.tool_references,\n metadata: {\n total_searched: searchResponse.total_tools_searched,\n pattern: searchResponse.pattern_used,\n mcp_server: serverFilters.length > 0 ? serverFilters : undefined,\n },\n },\n ];\n }\n\n const { safe: sanitizedPattern, wasEscaped } = sanitizeRegex(query);\n let warningMessage = '';\n if (wasEscaped) {\n warningMessage =\n 'Note: The provided pattern was converted to a literal search for safety.\\n\\n';\n }\n\n const searchScript = generateSearchScript(\n deferredTools,\n fields,\n max_results,\n sanitizedPattern\n );\n\n const postData = {\n lang: 'js',\n code: searchScript,\n timeout: SEARCH_TIMEOUT,\n };\n\n try {\n const fetchOptions: RequestInit = {\n method: 'POST',\n headers: {\n 'Content-Type': 'application/json',\n 'User-Agent': 'LibreChat/1.0',\n },\n body: JSON.stringify(postData),\n };\n\n if (process.env.PROXY != null && process.env.PROXY !== '') {\n fetchOptions.agent = new HttpsProxyAgent(process.env.PROXY);\n }\n\n const response = await fetch(EXEC_ENDPOINT, fetchOptions);\n if (!response.ok) {\n throw new Error(`HTTP error! status: ${response.status}`);\n }\n\n const result: t.ExecuteResult = await response.json();\n\n if (result.stderr && result.stderr.trim()) {\n // eslint-disable-next-line no-console\n console.warn('[ToolSearch] stderr:', result.stderr);\n }\n\n if (!result.stdout || !result.stdout.trim()) {\n return [\n `${warningMessage}No tools matched the pattern \"${sanitizedPattern}\".\\nTotal tools searched: ${deferredTools.length}`,\n {\n tool_references: [],\n metadata: {\n total_searched: deferredTools.length,\n pattern: sanitizedPattern,\n },\n },\n ];\n }\n\n const searchResponse = parseSearchResults(result.stdout);\n const formattedOutput = `${warningMessage}${formatSearchResults(searchResponse, mcpNameFormat)}`;\n\n return [\n formattedOutput,\n {\n tool_references: searchResponse.tool_references,\n metadata: {\n total_searched: searchResponse.total_tools_searched,\n pattern: searchResponse.pattern_used,\n },\n },\n ];\n } catch (error) {\n // eslint-disable-next-line no-console\n console.error('[ToolSearch] Error:', error);\n\n const errorMessage =\n error instanceof Error ? error.message : String(error);\n return [\n `Tool search failed: ${errorMessage}\\n\\nSuggestion: Try a simpler search pattern or search for specific tool names.`,\n {\n tool_references: [],\n metadata: {\n total_searched: 0,\n pattern: sanitizedPattern,\n error: errorMessage,\n },\n },\n ];\n }\n },\n {\n name: Constants.TOOL_SEARCH,\n description,\n schema,\n responseFormat: Constants.CONTENT_AND_ARTIFACT,\n }\n );\n}\n\nexport {\n createToolSearch,\n performLocalSearch,\n extractMcpServerName,\n isFromMcpServer,\n isFromAnyMcpServer,\n normalizeServerFilter,\n getAvailableMcpServers,\n getDeferredToolsListing,\n getBaseToolName,\n formatServerListing,\n sanitizeRegex,\n escapeRegexSpecialChars,\n isDangerousPattern,\n countNestedGroups,\n hasNestedQuantifiers,\n};\n"],"mappings":";;;;;;;;;;AAUA,SAAS,kBAA0B;CACjC,MAAM,MAAM;CAGZ,IAAI,OAAO,QAAQ,YAAY,OAAO;CACtC,IAAI,OAAO,IAAI,YAAY,YAAY,OAAO,IAAI;CAClD,IAAI,IAAI,WAAW,QAAQ,OAAO,IAAI,QAAQ,YAAY,YACxD,OAAO,IAAI,QAAQ;CACrB,MAAM,IAAI,MAAM,uDAAuD;AACzE;AAEA,MAAM,OAAO,gBAAgB;AAS7B,OAAO;;AAGP,MAAM,qBAAqB;AAE3B,MAAa,qBAAA;AAEb,MAAa,4BACX;AAEF,MAAM,0BACJ;AACF,MAAM,0BACJ;AACF,MAAM,qBACJ;AACF,MAAM,0BAA0B;AAChC,MAAM,sBAAsB;AAC5B,MAAM,yBACJ;AAEF,MAAa,uBAAuB;CAClC,MAAM;CACN,YAAY;EACV,QAAQ,EAAE,GAAG,gBAAgB;EAC7B,OAAO;GACL,MAAM;GACN,WAAW;GACX,SAAS;GACT,aAAa;EACf;EACA,QAAQ;GACN,MAAM;GACN,OAAO;IAAE,MAAM;IAAU,MAAM;KAAC;KAAQ;KAAe;IAAY;GAAE;GACrE,SAAS,CAAC,QAAQ,aAAa;GAC/B,aAAa;EACf;EACA,aAAa;GACX,MAAM;GACN,SAAS;GACT,SAAS;GACT,SAAS;GACT,aAAa;EACf;EACA,YAAY;GACV,OAAO,CAAC,EAAE,MAAM,SAAS,GAAG;IAAE,MAAM;IAAS,OAAO,EAAE,MAAM,SAAS;GAAE,CAAC;GACxE,aAAa;EACf;CACF;CACA,UAAU,CAAC;AACb;AAEA,MAAa,2BAA2B;CACtC,MAAM;CACN,aAAa;CACb,QAAQ;AACV;;AAGA,MAAM,uBAAuB;;AAG7B,MAAM,iBAAiB;;;;;;AAsBvB,SAAS,uBAAuB,MAA0C;CACxE,MAAM,mBACJ,SAAS,UAAU,0BAA0B;CAE/C,OAAO;EACL,MAAM;EACN,YAAY;GACV,QAAQ,EAAE,GAAG,gBAAgB;GAC7B,OAAO;IACL,MAAM;IACN,WAAW;IACX,SAAS;IACT,aAAa;GACf;GACA,QAAQ;IACN,MAAM;IACN,OAAO;KAAE,MAAM;KAAU,MAAM;MAAC;MAAQ;MAAe;KAAY;IAAE;IACrE,SAAS,CAAC,QAAQ,aAAa;IAC/B,aAAa;GACf;GACA,aAAa;IACX,MAAM;IACN,SAAS;IACT,SAAS;IACT,SAAS;IACT,aAAa;GACf;GACA,YAAY;IACV,OAAO,CACL,EAAE,MAAM,SAAS,GACjB;KAAE,MAAM;KAAS,OAAO,EAAE,MAAM,SAAS;IAAE,CAC7C;IACA,aAAa;GACf;EACF;EACA,UAAU,CAAC;CACb;AACF;;;;;;;AAQA,SAAS,qBAAqB,UAAsC;CAClE,MAAM,iBAAiB,SAAS,QAAA,OAA+B;CAC/D,IAAI,mBAAmB,IACrB;CAEF,OAAO,SAAS,UAAU,iBAAA,CAA+C;AAC3E;;;;;;;AAQA,SAAS,gBAAgB,UAAkB,YAA6B;CAEtE,OADmB,qBAAqB,QACxB,MAAM;AACxB;;;;;;;AAQA,SAAS,mBAAmB,UAAkB,aAAgC;CAC5E,MAAM,aAAa,qBAAqB,QAAQ;CAChD,IAAI,eAAe,KAAA,GACjB,OAAO;CAET,OAAO,YAAY,SAAS,UAAU;AACxC;;;;;;AAOA,SAAS,sBACP,cACU;CACV,IAAI,iBAAiB,KAAA,GACnB,OAAO,CAAC;CAEV,IAAI,OAAO,iBAAiB,UAC1B,OAAO,iBAAiB,KAAK,CAAC,IAAI,CAAC,YAAY;CAEjD,OAAO,aAAa,QAAQ,MAAM,MAAM,EAAE;AAC5C;;;;;;;AAQA,SAAS,uBACP,cACA,eAAwB,MACd;CACV,IAAI,CAAC,cACH,OAAO,CAAC;CAGV,MAAM,0BAAU,IAAI,IAAY;CAChC,KAAK,MAAM,GAAG,YAAY,cAAc;EACtC,IAAI,gBAAgB,QAAQ,kBAAkB,MAC5C;EAEF,MAAM,SAAS,qBAAqB,QAAQ,IAAI;EAChD,IAAI,WAAW,KAAA,KAAa,WAAW,IACrC,QAAQ,IAAI,MAAM;CAEtB;CAEA,OAAO,MAAM,KAAK,OAAO,CAAC,CAAC,KAAK;AAClC;;;;;;AAOA,SAAS,wBAAwB,SAAyB;CACxD,OAAO,QAAQ,QAAQ,uBAAuB,MAAM;AACtD;;;;;;AAOA,SAAS,kBAAkB,SAAyB;CAClD,IAAI,WAAW;CACf,IAAI,eAAe;CAEnB,KAAK,IAAI,IAAI,GAAG,IAAI,QAAQ,QAAQ,KAClC,IAAI,QAAQ,OAAO,QAAQ,MAAM,KAAK,QAAQ,IAAI,OAAO,OAAO;EAC9D;EACA,WAAW,KAAK,IAAI,UAAU,YAAY;CAC5C,OAAO,IAAI,QAAQ,OAAO,QAAQ,MAAM,KAAK,QAAQ,IAAI,OAAO,OAC9D,eAAe,KAAK,IAAI,GAAG,eAAe,CAAC;CAI/C,OAAO;AACT;;;;;;;AAQA,SAAS,qBAAqB,SAA0B;CAEtD,OAAO,0BAAwB,KAAK,OAAO;AAC7C;;;;;;AAOA,SAAS,mBAAmB,SAA0B;CACpD,IAAI,qBAAqB,OAAO,GAC9B,OAAO;CAGT,IAAI,kBAAkB,OAAO,IAAI,sBAC/B,OAAO;CAaT,KAAK,MAAM,aAAa;EATtB;EACA;EACA;EACA;EACA;EACA;EACA;CAGsC,GACtC,IAAI,UAAU,KAAK,OAAO,GACxB,OAAO;CAIX,OAAO;AACT;;;;;;;AAQA,SAAS,cAAc,SAAwD;CAC7E,IAAI,mBAAmB,OAAO,GAC5B,OAAO;EACL,MAAM,wBAAwB,OAAO;EACrC,YAAY;CACd;CAGF,IAAI;EACF,IAAI,OAAO,OAAO;EAClB,OAAO;GAAE,MAAM;GAAS,YAAY;EAAM;CAC5C,QAAQ;EACN,OAAO;GACL,MAAM,wBAAwB,OAAO;GACrC,YAAY;EACd;CACF;AACF;;;;;;;AAQA,SAAS,4BACP,YAC8B;CAC9B,IAAI,CAAC,YACH;CAGF,IAAI,WAAW,YACb,OAAO;EACL,MAAM,WAAW;EACjB,YAAY,OAAO,YACjB,OAAO,QAAQ,WAAW,UAAU,CAAC,CAAC,KAAK,CAAC,KAAK,WAAW,CAC1D,KACA,EAAE,MAAO,MAA2B,KAAK,CAC3C,CAAC,CACH;CACF;CAGF,OAAO,EAAE,MAAM,WAAW,KAAK;AACjC;;;;;AAMA,SAAS,iBAAiB,SAA2B;CACnD,MAAM,cAAc,QACjB,QAAQ,sBAAsB,OAAO,CAAC,CACtC,QAAQ,yBAAyB,OAAO,CAAC,CACzC,YAAY,CAAC,CACb,MAAM,KAAK,CAAC,CACZ,OAAO,OAAO;CAEjB,IAAI,YAAY,SAAS,GAAG,OAAO;CAEnC,MAAM,eAAyB,CAAC;CAChC,IAAI,SAAS;CACb,KAAK,MAAM,SAAS,aAAa;EAC/B,IAAI,MAAM,WAAW,GAAG;GACtB,UAAU;GACV;EACF;EACA,aAAa,KAAK,GAAG,SAAS,OAAO;EACrC,SAAS;CACX;CAEA,IAAI,UAAU,aAAa,SAAS,GAClC,aAAa,aAAa,SAAS,MAAM;CAE3C,OAAO,aAAa,SAAS,IAAI,eAAe,CAAC,MAAM;AACzD;;;;;;;AAQA,SAAS,SAAS,MAAwB;CACxC,OAAO,KACJ,MAAM,eAAe,CAAC,CACtB,OAAO,OAAO,CAAC,CACf,QAAQ,gBAAgB;AAC7B;;;;;;;AAQA,SAAS,mBAAmB,MAAsB,QAA0B;CAC1E,MAAM,QAAkB,CAAC;CAEzB,IAAI,OAAO,SAAS,MAAM,GAAG;EAC3B,MAAM,WAAW,KAAK,KAAK,QAAQ,MAAM,GAAG;EAC5C,MAAM,KAAK,UAAU,QAAQ;CAC/B;CAEA,IAAI,OAAO,SAAS,aAAa,KAAK,KAAK,aACzC,MAAM,KAAK,KAAK,WAAW;CAG7B,IAAI,OAAO,SAAS,YAAY,KAAK,KAAK,YAAY,YAAY;EAChE,MAAM,aAAa,OAAO,KAAK,KAAK,WAAW,UAAU,CAAC,CAAC,KAAK,GAAG;EACnE,MAAM,KAAK,UAAU;CACvB;CAEA,OAAO,SAAS,MAAM,KAAK,GAAG,CAAC,CAAC,CAAC,KAAK,GAAG;AAC3C;;;;;;;;AASA,SAAS,iBACP,MACA,aACA,QACoC;CACpC,IAAI,OAAO,SAAS,MAAM,GAAG;EAC3B,MAAM,YAAY,KAAK,KAAK,YAAY;EACxC,KAAK,MAAM,SAAS,aAClB,IAAI,UAAU,SAAS,KAAK,GAC1B,OAAO;GAAE,OAAO;GAAQ,SAAS,KAAK;EAAK;CAGjD;CAEA,IAAI,OAAO,SAAS,aAAa,KAAK,KAAK,aAAa;EACtD,MAAM,YAAY,KAAK,YAAY,YAAY;EAC/C,KAAK,MAAM,SAAS,aAClB,IAAI,UAAU,SAAS,KAAK,GAC1B,OAAO;GACL,OAAO;GACP,SAAS,KAAK,YAAY,UAAU,GAAG,GAAG;EAC5C;CAGN;CAEA,IAAI,OAAO,SAAS,YAAY,KAAK,KAAK,YAAY,YAAY;EAChE,MAAM,aAAa,OAAO,KAAK,KAAK,WAAW,UAAU;EACzD,MAAM,aAAa,WAAW,KAAK,GAAG,CAAC,CAAC,YAAY;EACpD,KAAK,MAAM,SAAS,aAClB,IAAI,WAAW,SAAS,KAAK,GAC3B,OAAO;GAAE,OAAO;GAAc,SAAS,WAAW,KAAK,IAAI;EAAE;CAGnE;CAKA,OAAO;EAAE,OAAO;EAAW,SAHH,KAAK,cACzB,KAAK,YAAY,UAAU,GAAG,GAAG,IACjC,KAAK;CAC2C;AACtD;;;;;;;;;;;AAYA,SAAS,mBACP,OACA,OACA,QACA,YACsB;CACtB,IAAI,MAAM,WAAW,GACnB,OAAO;EACL,iBAAiB,CAAC;EAClB,sBAAsB;EACtB,cAAc;CAChB;CAGF,MAAM,cAAc,SAAS,KAAK;CAElC,IAAI,YAAY,WAAW,GAYzB,OAAO;EACL,iBAZe,MACd,MAAM,CAAC,CACP,MAAM,GAAG,MAAM,EAAE,KAAK,cAAc,EAAE,IAAI,CAAC,CAAC,CAC5C,MAAM,GAAG,UAAU,CAAC,CACpB,KAAK,UAAU;GACd,WAAW,KAAK;GAChB,aAAa;GACb,eAAe;GACf,SAAS,KAAK,YAAY,UAAU,GAAG,GAAG,KAAK,KAAK;EACtD,EAGwB;EACxB,sBAAsB,MAAM;EAC5B,cAAc;CAChB;CAIF,MAAM,SAAS,KADG,MAAM,KAAK,SAAS,mBAAmB,MAAM,MAAM,CACzC,GAAG,aAAa;EAAE,IAAI;EAAK,GAAG;CAAK,CAAC;CAEhE,MAAM,WAAW,KAAK,IAAI,GAAG,OAAO,QAAQ,MAAM,IAAI,CAAC,GAAG,CAAC;CAC3D,MAAM,aAAa,MAAM,YAAY,CAAC,CAAC,KAAK;CAC5C,MAAM,kBAAkB,YAAY,KAAK,EAAE;CAC3C,MAAM,qBAAqB,OAAO,SAAS,MAAM;CAEjD,MAAM,UAGD,CAAC;CACN,KAAK,IAAI,IAAI,GAAG,IAAI,MAAM,QAAQ,KAAK;EACrC,MAAM,QAAQ,OAAO;EACrB,MAAM,iBAAiB,OAAO,SAAS,KAAK,KAAK,QAAQ;EACzD,IAAI,qBAAqB;EACzB,IAAI,kBAAkB,iBAAiB,KAAK,IAAI,QAAQ,UAAU,CAAG,IAAI;EAEzE,IAAI,oBAAoB;GACtB,MAAM,cAAc,gBAAgB,MAAM,EAAE,CAAC,IAAI,CAAC,CAAC,YAAY;GAC/D,MAAM,cAAc,MAAM,EAAE,CAAC,KAAK,YAAY;GAC9C,MAAM,iBAAiB,SAAS,WAAW,CAAC,CAAC,KAAK,EAAE;GACpD,MAAM,iBAAiB,SAAS,WAAW,CAAC,CAAC,KAAK,EAAE;GAEpD,IAAI,gBAAgB,YAAY;IAC9B,qBAAqB;IACrB,kBAAkB;GACpB,OAAO,IAAI,gBAAgB,YAAY;IACrC,qBAAqB;IACrB,kBAAkB;GACpB,OAAO,IACL,mBAAmB,mBACnB,mBAAmB,iBACnB;IACA,qBAAqB;IACrB,kBAAkB;GACpB,OAAO,IAAI,eAAe,WAAW,eAAe,GAAG;IACrD,qBAAqB;IACrB,kBAAkB,KAAK,IAAI,iBAAiB,GAAI;GAClD;EACF;EAEA,IAAI,CAAC,kBAAkB,uBAAuB,GAAG;EAEjD,MAAM,EAAE,OAAO,YAAY,iBAAiB,MAAM,IAAI,aAAa,MAAM;EACzE,QAAQ,KAAK;GACX,QAAQ;IACN,WAAW,MAAM,EAAE,CAAC;IACpB,aAAa;IACb,eAAe;IACf;GACF;GACA;EACF,CAAC;CACH;CAEA,QAAQ,MACL,GAAG,MACF,EAAE,qBAAqB,EAAE,sBACzB,EAAE,OAAO,cAAc,EAAE,OAAO,WACpC;CAGA,OAAO;EACL,iBAHiB,QAAQ,MAAM,GAAG,UAAU,CAAC,CAAC,KAAK,EAAE,aAAa,MAGxC;EAC1B,sBAAsB,MAAM;EAC5B,cAAc;CAChB;AACF;;;;;;;;;;AAWA,SAAS,qBACP,eACA,QACA,YACA,kBACQ;CAiFR,OAAO;EA/EL;EACA,iBAAiB,KAAK,UAAU,aAAa,IAAI;EACjD,wBAAwB,KAAK,UAAU,MAAM,IAAI;EACjD,sBAAsB,aAAa;EACnC,mBAAmB,KAAK,UAAU,gBAAgB,IAAI;EACtD;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;CAES,CAAC,CAAC,KAAK,IAAI;AACxB;;;;;;AAOA,SAAS,mBAAmB,QAAsC;CAChE,MAAM,YAAY,OAAO,KAAK;CAE9B,OADe,KAAK,MAAM,SACd;AACd;;;;;;;AAQA,SAAS,oBACP,gBACA,aAA8B,QACtB;CACR,MAAM,EAAE,iBAAiB,sBAAsB,iBAC7C;CACF,MAAM,cAAc,eAAe;CAEnC,MAAM,SAAS;EACb,OAAO,gBAAgB;EACvB,OAAO,gBAAgB,KAAK,SAAS;GACnC,MAAM,cAAc,IAAI,YAAY,gBAAgB,IAAI,SAAS;GACjE,OAAO,OAAO,IAAI,YAAY,QAAQ,CAAC,CAAC;GACxC,YAAY,IAAI;GAChB,SAAS,IAAI;EACf,EAAE;EACF,gBAAgB;EAChB,OAAO;CACT;CAEA,OAAO,KAAK,UAAU,QAAQ,MAAM,CAAC;AACvC;;;;;;AAOA,SAAS,gBAAgB,UAA0B;CACjD,MAAM,iBAAiB,SAAS,QAAA,OAA+B;CAC/D,IAAI,mBAAmB,IACrB,OAAO;CAET,OAAO,SAAS,UAAU,GAAG,cAAc;AAC7C;;;;;;AAOA,SAAS,UAAU,YAAwC;CACzD,OACE,YAAY,cAAc,QAC1B,OAAO,KAAK,WAAW,UAAU,CAAC,CAAC,SAAS;AAEhD;;;;;;;;;;;AAYA,SAAS,wBACP,cACA,cACQ;CACR,IAAI,CAAC,cACH,OAAO;CAGT,MAAM,gBAA0C,CAAC;CAEjD,KAAK,MAAM,UAAU,aAAa,OAAO,GAAG;EAC1C,IAAI,gBAAgB,OAAO,kBAAkB,MAC3C;EAGF,MAAM,WAAW,OAAO;EACxB,MAAM,aAAa,qBAAqB,QAAQ,KAAK;EACrD,MAAM,WAAW,gBAAgB,QAAQ;EACzC,MAAM,cAAc,UAAU,OAAO,UAAU,IAC3C,GAAG,SAAS,YACZ;EAEJ,IAAI,EAAE,cAAc,gBAClB,cAAc,cAAc,CAAC;EAE/B,cAAc,WAAW,CAAC,KAAK,WAAW;CAC5C;CAEA,MAAM,cAAc,OAAO,KAAK,aAAa,CAAC,CAAC,MAAM,GAAG,MAAM;EAC5D,IAAI,MAAM,SAAS,OAAO;EAC1B,IAAI,MAAM,SAAS,OAAO;EAC1B,OAAO,EAAE,cAAc,CAAC;CAC1B,CAAC;CAED,IAAI,YAAY,WAAW,GACzB,OAAO;CAOT,OAJc,YAAY,KACvB,WAAW,GAAG,OAAO,IAAI,cAAc,OAAO,CAAC,KAAK,IAAI,GAGhD,CAAC,CAAC,KAAK,IAAI;AACxB;;;;;;;;;AAUA,SAAS,oBACP,OACA,aACA,aAA8B,QACtB;CACR,MAAM,UAAU,MAAM,QAAQ,WAAW,IAAI,cAAc,CAAC,WAAW;CACvE,MAAM,cAAc,eAAe;CAEnC,IAAI,MAAM,WAAW,GACnB,OAAO,KAAK,UACV;EACE,cAAc;EACd;EACA,aAAa;EACb,iBAAiB,CAAC;EAClB,MAAM;CACR,GACA,MACA,CACF;CAGF,MAAM,gBAGF,CAAC;CACL,KAAK,MAAM,QAAQ,OAAO;EACxB,MAAM,SAAS,qBAAqB,KAAK,IAAI,KAAK;EAClD,IAAI,EAAE,UAAU,gBACd,cAAc,UAAU,CAAC;EAE3B,cAAc,OAAO,CAAC,KAAK;GACzB,MAAM,cAAc,KAAK,OAAO,gBAAgB,KAAK,IAAI;GACzD,aACE,KAAK,YAAY,SAAS,MACtB,KAAK,YAAY,UAAU,GAAG,EAAE,IAAI,QACpC,KAAK;EACb,CAAC;CACH;CAEA,MAAM,kBAAkB,cACnB,MAAM,EAAE,EAAE,QAAQ,cACnB,gBAAgB,MAAM,EAAE,EAAE,QAAQ,WAAW;CAEjD,MAAM,SAAS;EACb,cAAc;EACd;EACA,aAAa,MAAM;EACnB,iBAAiB;EACjB,MAAM,uDAAuD,gBAAgB;CAC/E;CAEA,OAAO,KAAK,UAAU,QAAQ,MAAM,CAAC;AACvC;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AA8BA,SAAS,iBACP,aAAiC,CAAC,GACX;CACvB,MAAM,OAAyB,WAAW,QAAQ;CAClD,MAAM,sBAAsB,WAAW,gBAAgB;CACvD,MAAM,gBAAiC,WAAW,iBAAiB;CACnE,MAAM,SAAS,uBAAuB,IAAI;CAG1C,MAAM,gBAAgB,GADD,WAAW,WAAW,eAAe,EACpB;CAEtC,MAAM,uBAAuB,wBAC3B,WAAW,cACX,mBACF;CAEA,MAAM,mBACJ,qBAAqB,SAAS,IAC1B;;;EAGN,yBACM;CAEN,MAAM,UACJ,qBAAqB,SAAA,OAAgC,KACrD,qBAAqB,MAAM,IAAI,CAAC,CAAC,MAAM,SAAS,CAAC,KAAK,WAAW,QAAQ,CAAC,IACtE;;8CAGA;CAaN,OAAO,KACL,OAAO,WAAW,WAAW;EAE3B,MAAM,EACJ,QAAQ,IACR,SAAS,CAAC,QAAQ,aAAa,GAC/B,cAAc,qBACd,eACEA;EAEJ,MAAM,EACJ,cAAc,mBACd,cAAc,mBACd,WAAW,mBACT,OAAO,YAAY,CAAC;EAExB,MAAM,eAAe,qBAAqB,WAAW;EACrD,MAAM,eACJ,sBAAsB,KAAA,IAClB,oBACA;EAGN,MAAM,gBAAgB,sBADpB,cAAc,kBAAkB,WAAW,SACc;EAC3D,MAAM,kBAAkB,cAAc,SAAS;EAE/C,IAAI,gBAAgB,MAClB,OAAO,CACL,8FACA;GACE,iBAAiB,CAAC;GAClB,UAAU;IACR,gBAAgB;IAChB,SAAS;IACT,OAAO;GACT;EACF,CACF;EAIF,MAAM,gBADyB,MAAM,KAAK,aAAa,OAAO,CACb,CAAC,CAC/C,QAAQ,WAAW;GAClB,IAAI,iBAAiB,QAAQ,OAAO,kBAAkB,MACpD,OAAO;GAET,IACE,mBACA,CAAC,mBAAmB,OAAO,MAAM,aAAa,GAE9C,OAAO;GAET,OAAO;EACT,CAAC,CAAC,CACD,KAAK,YAAY;GAChB,MAAM,OAAO;GACb,aAAa,OAAO,eAAe;GACnC,YAAY,4BAA4B,OAAO,UAAU;EAC3D,EAAE;EAEJ,IAAI,cAAc,WAAW,GAI3B,OAAO,CACL,+BAJgB,kBACd,wBAAwB,cAAc,KAAK,IAAI,MAC/C,GAEuC,6EACzC;GACE,iBAAiB,CAAC;GAClB,UAAU;IACR,gBAAgB;IAChB,SAAS;IACT,YAAY;GACd;EACF,CACF;EAKF,IAFwB,mBAAmB,UAAU,IASnD,OAAO,CANiB,oBACtB,eACA,eACA,aAIc,GACd;GACE,iBAAiB,CAAC;GAClB,UAAU;IACR,iBAAiB,cAAc;IAC/B,YAAY;IACZ,cAAc;GAChB;EACF,CACF;EAGF,IAAI,SAAS,SAAS;GACpB,MAAM,iBAAiB,mBACrB,eACA,OACA,QACA,WACF;GAMA,OAAO,CALiB,oBACtB,gBACA,aAIc,GACd;IACE,iBAAiB,eAAe;IAChC,UAAU;KACR,gBAAgB,eAAe;KAC/B,SAAS,eAAe;KACxB,YAAY,cAAc,SAAS,IAAI,gBAAgB,KAAA;IACzD;GACF,CACF;EACF;EAEA,MAAM,EAAE,MAAM,kBAAkB,eAAe,cAAc,KAAK;EAClE,IAAI,iBAAiB;EACrB,IAAI,YACF,iBACE;EAUJ,MAAM,WAAW;GACf,MAAM;GACN,MATmB,qBACnB,eACA,QACA,aACA,gBAKiB;GACjB,SAAS;EACX;EAEA,IAAI;GACF,MAAM,eAA4B;IAChC,QAAQ;IACR,SAAS;KACP,gBAAgB;KAChB,cAAc;IAChB;IACA,MAAM,KAAK,UAAU,QAAQ;GAC/B;GAEA,IAAI,QAAQ,IAAI,SAAS,QAAQ,QAAQ,IAAI,UAAU,IACrD,aAAa,QAAQ,IAAI,gBAAgB,QAAQ,IAAI,KAAK;GAG5D,MAAM,WAAW,MAAM,MAAM,eAAe,YAAY;GACxD,IAAI,CAAC,SAAS,IACZ,MAAM,IAAI,MAAM,uBAAuB,SAAS,QAAQ;GAG1D,MAAM,SAA0B,MAAM,SAAS,KAAK;GAEpD,IAAI,OAAO,UAAU,OAAO,OAAO,KAAK,GAEtC,QAAQ,KAAK,wBAAwB,OAAO,MAAM;GAGpD,IAAI,CAAC,OAAO,UAAU,CAAC,OAAO,OAAO,KAAK,GACxC,OAAO,CACL,GAAG,eAAe,gCAAgC,iBAAiB,4BAA4B,cAAc,UAC7G;IACE,iBAAiB,CAAC;IAClB,UAAU;KACR,gBAAgB,cAAc;KAC9B,SAAS;IACX;GACF,CACF;GAGF,MAAM,iBAAiB,mBAAmB,OAAO,MAAM;GAGvD,OAAO,CACL,GAHyB,iBAAiB,oBAAoB,gBAAgB,aAAa,KAI3F;IACE,iBAAiB,eAAe;IAChC,UAAU;KACR,gBAAgB,eAAe;KAC/B,SAAS,eAAe;IAC1B;GACF,CACF;EACF,SAAS,OAAO;GAEd,QAAQ,MAAM,uBAAuB,KAAK;GAE1C,MAAM,eACJ,iBAAiB,QAAQ,MAAM,UAAU,OAAO,KAAK;GACvD,OAAO,CACL,uBAAuB,aAAa,kFACpC;IACE,iBAAiB,CAAC;IAClB,UAAU;KACR,gBAAgB;KAChB,SAAS;KACT,OAAO;IACT;GACF,CACF;EACF;CACF,GACA;EACE,MAAA;EACA,aAnOF,SAAS,UACL;;EAEN,UAAU,iBAAiB;EAC3B,KAAK,IACC;;EAEN,UAAU,iBAAiB;EAC3B,KAAK;EA4ND;EACA,gBAAA;CACF,CACF;AACF"}
1
+ {"version":3,"file":"ToolSearch.mjs","names":["params"],"sources":["../../../src/tools/ToolSearch.ts"],"sourcesContent":["// src/tools/ToolSearch.ts\nimport { config } from 'dotenv';\nimport * as okapibm25Module from 'okapibm25';\n\ntype BM25Fn = (\n documents: string[],\n keywords: string[],\n constants?: { k1?: number; b?: number }\n) => number[];\n\nfunction getBM25Function(): BM25Fn {\n const mod = okapibm25Module as unknown as {\n default: BM25Fn | { default: BM25Fn } | undefined;\n };\n if (typeof mod === 'function') return mod;\n if (typeof mod.default === 'function') return mod.default;\n if (mod.default != null && typeof mod.default.default === 'function')\n return mod.default.default;\n throw new Error('Could not resolve BM25 function from okapibm25 module');\n}\n\nconst BM25 = getBM25Function();\nimport fetch, { RequestInit } from 'node-fetch';\nimport { resolveFetchProxyAgent } from '@/utils/proxy';\nimport { tool, DynamicStructuredTool } from '@langchain/core/tools';\nimport type * as t from '@/types';\nimport { INTENT_PROPERTY } from '@/tools/intentArg';\nimport { getCodeBaseURL } from './CodeExecutor';\nimport { Constants } from '@/common';\n\nconfig();\n\n/** Maximum allowed regex pattern length */\nconst MAX_PATTERN_LENGTH = 200;\n\nexport const ToolSearchToolName = Constants.TOOL_SEARCH;\n\nexport const ToolSearchToolDescription =\n 'Searches deferred tools using BM25 ranking. Multi-word queries supported. Use mcp_server param to filter by server.';\n\nconst QUERY_DESCRIPTION_LOCAL =\n 'Search term to find in tool names and descriptions. Case-insensitive substring matching. Optional if mcp_server is provided.';\nconst QUERY_DESCRIPTION_REGEX =\n 'Regex pattern to search tool names and descriptions. Optional if mcp_server is provided.';\nconst FIELDS_DESCRIPTION =\n 'Which fields to search. Default: name and description';\nconst MAX_RESULTS_DESCRIPTION = 'Maximum number of matching tools to return';\nconst DEFAULT_MAX_RESULTS = 5;\nconst MCP_SERVER_DESCRIPTION =\n 'Filter to tools from specific MCP server(s). Can be a single server name or array of names. If provided without a query, lists all tools from those servers.';\n\nexport const ToolSearchToolSchema = {\n type: 'object',\n properties: {\n intent: { ...INTENT_PROPERTY },\n query: {\n type: 'string',\n maxLength: MAX_PATTERN_LENGTH,\n default: '',\n description: QUERY_DESCRIPTION_LOCAL,\n },\n fields: {\n type: 'array',\n items: { type: 'string', enum: ['name', 'description', 'parameters'] },\n default: ['name', 'description'],\n description: FIELDS_DESCRIPTION,\n },\n max_results: {\n type: 'integer',\n minimum: 1,\n maximum: 50,\n default: DEFAULT_MAX_RESULTS,\n description: MAX_RESULTS_DESCRIPTION,\n },\n mcp_server: {\n oneOf: [{ type: 'string' }, { type: 'array', items: { type: 'string' } }],\n description: MCP_SERVER_DESCRIPTION,\n },\n },\n required: [],\n} as const;\n\nexport const ToolSearchToolDefinition = {\n name: ToolSearchToolName,\n description: ToolSearchToolDescription,\n schema: ToolSearchToolSchema,\n} as const;\n\n/** Maximum allowed regex nesting depth */\nconst MAX_REGEX_COMPLEXITY = 5;\n\n/** Default search timeout in milliseconds */\nconst SEARCH_TIMEOUT = 5000;\n\n/** JSON schema type for tool search parameters */\ninterface ToolSearchSchema {\n type: 'object';\n properties: Record<string, unknown>;\n required: string[];\n}\n\n/** Input params type for tool search */\ninterface ToolSearchParams {\n query?: string;\n fields?: ('name' | 'description' | 'parameters')[];\n max_results?: number;\n mcp_server?: string | string[];\n}\n\n/**\n * Creates the JSON schema with dynamic query description based on mode.\n * @param mode - The search mode determining query interpretation\n * @returns JSON schema for tool search parameters\n */\nfunction createToolSearchSchema(mode: t.ToolSearchMode): ToolSearchSchema {\n const queryDescription =\n mode === 'local' ? QUERY_DESCRIPTION_LOCAL : QUERY_DESCRIPTION_REGEX;\n\n return {\n type: 'object',\n properties: {\n intent: { ...INTENT_PROPERTY },\n query: {\n type: 'string',\n maxLength: MAX_PATTERN_LENGTH,\n default: '',\n description: queryDescription,\n },\n fields: {\n type: 'array',\n items: { type: 'string', enum: ['name', 'description', 'parameters'] },\n default: ['name', 'description'],\n description: FIELDS_DESCRIPTION,\n },\n max_results: {\n type: 'integer',\n minimum: 1,\n maximum: 50,\n default: DEFAULT_MAX_RESULTS,\n description: MAX_RESULTS_DESCRIPTION,\n },\n mcp_server: {\n oneOf: [\n { type: 'string' },\n { type: 'array', items: { type: 'string' } },\n ],\n description: MCP_SERVER_DESCRIPTION,\n },\n },\n required: [],\n };\n}\n\n/**\n * Extracts the MCP server name from a tool name.\n * MCP tools follow the pattern: toolName_mcp_serverName\n * @param toolName - The full tool name\n * @returns The server name if it's an MCP tool, undefined otherwise\n */\nfunction extractMcpServerName(toolName: string): string | undefined {\n const delimiterIndex = toolName.indexOf(Constants.MCP_DELIMITER);\n if (delimiterIndex === -1) {\n return undefined;\n }\n return toolName.substring(delimiterIndex + Constants.MCP_DELIMITER.length);\n}\n\n/**\n * Checks if a tool belongs to a specific MCP server.\n * @param toolName - The full tool name\n * @param serverName - The server name to match\n * @returns True if the tool belongs to the specified server\n */\nfunction isFromMcpServer(toolName: string, serverName: string): boolean {\n const toolServer = extractMcpServerName(toolName);\n return toolServer === serverName;\n}\n\n/**\n * Checks if a tool belongs to any of the specified MCP servers.\n * @param toolName - The full tool name\n * @param serverNames - Array of server names to match\n * @returns True if the tool belongs to any of the specified servers\n */\nfunction isFromAnyMcpServer(toolName: string, serverNames: string[]): boolean {\n const toolServer = extractMcpServerName(toolName);\n if (toolServer === undefined) {\n return false;\n }\n return serverNames.includes(toolServer);\n}\n\n/**\n * Normalizes server filter input to always be an array.\n * @param serverFilter - String, array of strings, or undefined\n * @returns Array of server names (empty if none specified)\n */\nfunction normalizeServerFilter(\n serverFilter: string | string[] | undefined\n): string[] {\n if (serverFilter === undefined) {\n return [];\n }\n if (typeof serverFilter === 'string') {\n return serverFilter === '' ? [] : [serverFilter];\n }\n return serverFilter.filter((s) => s !== '');\n}\n\n/**\n * Extracts all unique MCP server names from a tool registry.\n * @param toolRegistry - The tool registry to scan\n * @param onlyDeferred - If true, only considers deferred tools\n * @returns Array of unique server names, sorted alphabetically\n */\nfunction getAvailableMcpServers(\n toolRegistry: t.LCToolRegistry | undefined,\n onlyDeferred: boolean = true\n): string[] {\n if (!toolRegistry) {\n return [];\n }\n\n const servers = new Set<string>();\n for (const [, toolDef] of toolRegistry) {\n if (onlyDeferred && toolDef.defer_loading !== true) {\n continue;\n }\n const server = extractMcpServerName(toolDef.name);\n if (server !== undefined && server !== '') {\n servers.add(server);\n }\n }\n\n return Array.from(servers).sort();\n}\n\n/**\n * Escapes special regex characters in a string to use as a literal pattern.\n * @param pattern - The string to escape\n * @returns The escaped string safe for use in a RegExp\n */\nfunction escapeRegexSpecialChars(pattern: string): string {\n return pattern.replace(/[.*+?^${}()|[\\]\\\\]/g, '\\\\$&');\n}\n\n/**\n * Counts the maximum nesting depth of groups in a regex pattern.\n * @param pattern - The regex pattern to analyze\n * @returns The maximum nesting depth\n */\nfunction countNestedGroups(pattern: string): number {\n let maxDepth = 0;\n let currentDepth = 0;\n\n for (let i = 0; i < pattern.length; i++) {\n if (pattern[i] === '(' && (i === 0 || pattern[i - 1] !== '\\\\')) {\n currentDepth++;\n maxDepth = Math.max(maxDepth, currentDepth);\n } else if (pattern[i] === ')' && (i === 0 || pattern[i - 1] !== '\\\\')) {\n currentDepth = Math.max(0, currentDepth - 1);\n }\n }\n\n return maxDepth;\n}\n\n/**\n * Detects nested quantifiers that can cause catastrophic backtracking.\n * Patterns like (a+)+, (a*)*, (a+)*, etc.\n * @param pattern - The regex pattern to check\n * @returns True if nested quantifiers are detected\n */\nfunction hasNestedQuantifiers(pattern: string): boolean {\n const nestedQuantifierPattern = /\\([^)]*[+*][^)]*\\)[+*?]/;\n return nestedQuantifierPattern.test(pattern);\n}\n\n/**\n * Checks if a regex pattern contains potentially dangerous constructs.\n * @param pattern - The regex pattern to validate\n * @returns True if the pattern is dangerous\n */\nfunction isDangerousPattern(pattern: string): boolean {\n if (hasNestedQuantifiers(pattern)) {\n return true;\n }\n\n if (countNestedGroups(pattern) > MAX_REGEX_COMPLEXITY) {\n return true;\n }\n\n const dangerousPatterns = [\n /\\.\\{1000,\\}/, // Excessive wildcards\n /\\(\\?=\\.\\{100,\\}\\)/, // Runaway lookaheads\n /\\([^)]*\\|\\s*\\){20,}/, // Excessive alternation (rough check)\n /\\(\\.\\*\\)\\+/, // (.*)+\n /\\(\\.\\+\\)\\+/, // (.+)+\n /\\(\\.\\*\\)\\*/, // (.*)*\n /\\(\\.\\+\\)\\*/, // (.+)*\n ];\n\n for (const dangerous of dangerousPatterns) {\n if (dangerous.test(pattern)) {\n return true;\n }\n }\n\n return false;\n}\n\n/**\n * Sanitizes a regex pattern for safe execution.\n * If the pattern is dangerous, it will be escaped to a literal string search.\n * @param pattern - The regex pattern to sanitize\n * @returns Object containing the safe pattern and whether it was escaped\n */\nfunction sanitizeRegex(pattern: string): { safe: string; wasEscaped: boolean } {\n if (isDangerousPattern(pattern)) {\n return {\n safe: escapeRegexSpecialChars(pattern),\n wasEscaped: true,\n };\n }\n\n try {\n new RegExp(pattern);\n return { safe: pattern, wasEscaped: false };\n } catch {\n return {\n safe: escapeRegexSpecialChars(pattern),\n wasEscaped: true,\n };\n }\n}\n\n/**\n * Simplifies tool parameters for search purposes.\n * Extracts only the essential structure needed for parameter name searching.\n * @param parameters - The tool's JSON schema parameters\n * @returns Simplified parameters object\n */\nfunction simplifyParametersForSearch(\n parameters?: t.JsonSchemaType\n): t.JsonSchemaType | undefined {\n if (!parameters) {\n return undefined;\n }\n\n if (parameters.properties) {\n return {\n type: parameters.type,\n properties: Object.fromEntries(\n Object.entries(parameters.properties).map(([key, value]) => [\n key,\n { type: (value as t.JsonSchemaType).type },\n ])\n ),\n } as t.JsonSchemaType;\n }\n\n return { type: parameters.type };\n}\n\n/**\n * Splits one alphanumeric identifier segment on case boundaries without\n * emitting artificial one-character acronym fragments.\n */\nfunction splitCaseSegment(segment: string): string[] {\n const splitTokens = segment\n .replace(/([a-z0-9])([A-Z])/g, '$1 $2')\n .replace(/([A-Z]+)([A-Z][a-z])/g, '$1 $2')\n .toLowerCase()\n .split(/\\s+/)\n .filter(Boolean);\n\n if (splitTokens.length < 2) return splitTokens;\n\n const mergedTokens: string[] = [];\n let prefix = '';\n for (const token of splitTokens) {\n if (token.length === 1) {\n prefix += token;\n continue;\n }\n mergedTokens.push(`${prefix}${token}`);\n prefix = '';\n }\n\n if (prefix && mergedTokens.length > 0) {\n mergedTokens[mergedTokens.length - 1] += prefix;\n }\n return mergedTokens.length > 0 ? mergedTokens : [prefix];\n}\n\n/**\n * Tokenizes a string into lowercase words for BM25.\n * Splits camelCase, underscores, and non-alphanumeric characters for consistent matching.\n * @param text - The text to tokenize\n * @returns Array of lowercase tokens\n */\nfunction tokenize(text: string): string[] {\n return text\n .split(/[^a-zA-Z0-9]+/)\n .filter(Boolean)\n .flatMap(splitCaseSegment);\n}\n\n/**\n * Creates a searchable document string from tool metadata.\n * @param tool - The tool metadata\n * @param fields - Which fields to include\n * @returns Combined document string for BM25\n */\nfunction createToolDocument(tool: t.ToolMetadata, fields: string[]): string {\n const parts: string[] = [];\n\n if (fields.includes('name')) {\n const baseName = tool.name.replace(/_/g, ' ');\n parts.push(baseName, baseName);\n }\n\n if (fields.includes('description') && tool.description) {\n parts.push(tool.description);\n }\n\n if (fields.includes('parameters') && tool.parameters?.properties) {\n const paramNames = Object.keys(tool.parameters.properties).join(' ');\n parts.push(paramNames);\n }\n\n return tokenize(parts.join(' ')).join(' ');\n}\n\n/**\n * Determines which field had the best match for a query.\n * @param tool - The tool to check\n * @param queryTokens - Tokenized query\n * @param fields - Fields to check\n * @returns The matched field and a snippet\n */\nfunction findMatchedField(\n tool: t.ToolMetadata,\n queryTokens: string[],\n fields: string[]\n): { field: string; snippet: string } {\n if (fields.includes('name')) {\n const nameLower = tool.name.toLowerCase();\n for (const token of queryTokens) {\n if (nameLower.includes(token)) {\n return { field: 'name', snippet: tool.name };\n }\n }\n }\n\n if (fields.includes('description') && tool.description) {\n const descLower = tool.description.toLowerCase();\n for (const token of queryTokens) {\n if (descLower.includes(token)) {\n return {\n field: 'description',\n snippet: tool.description.substring(0, 100),\n };\n }\n }\n }\n\n if (fields.includes('parameters') && tool.parameters?.properties) {\n const paramNames = Object.keys(tool.parameters.properties);\n const paramLower = paramNames.join(' ').toLowerCase();\n for (const token of queryTokens) {\n if (paramLower.includes(token)) {\n return { field: 'parameters', snippet: paramNames.join(', ') };\n }\n }\n }\n\n const fallbackSnippet = tool.description\n ? tool.description.substring(0, 100)\n : tool.name;\n return { field: 'unknown', snippet: fallbackSnippet };\n}\n\n/**\n * Performs BM25-based search for better relevance ranking.\n * Uses Okapi BM25 algorithm for term frequency and document length normalization.\n * If query is empty, returns all tools (up to maxResults) sorted alphabetically.\n * @param tools - Array of tool metadata to search\n * @param query - The search query (empty returns all tools)\n * @param fields - Which fields to search\n * @param maxResults - Maximum results to return\n * @returns Search response with matching tools ranked by BM25 score\n */\nfunction performLocalSearch(\n tools: t.ToolMetadata[],\n query: string,\n fields: string[],\n maxResults: number\n): t.ToolSearchResponse {\n if (tools.length === 0) {\n return {\n tool_references: [],\n total_tools_searched: 0,\n pattern_used: query,\n };\n }\n\n const queryTokens = tokenize(query);\n\n if (queryTokens.length === 0) {\n const allTools = tools\n .slice()\n .sort((a, b) => a.name.localeCompare(b.name))\n .slice(0, maxResults)\n .map((tool) => ({\n tool_name: tool.name,\n match_score: 1.0,\n matched_field: 'name',\n snippet: tool.description.substring(0, 100) || tool.name,\n }));\n\n return {\n tool_references: allTools,\n total_tools_searched: tools.length,\n pattern_used: query,\n };\n }\n\n const documents = tools.map((tool) => createToolDocument(tool, fields));\n const scores = BM25(documents, queryTokens, { k1: 1.5, b: 0.75 }) as number[];\n\n const maxScore = Math.max(...scores.filter((s) => s > 0), 1);\n const queryLower = query.toLowerCase().trim();\n const queryIdentifier = queryTokens.join('');\n const matchesIdentifiers = fields.includes('name');\n\n const results: Array<{\n result: t.ToolSearchResult;\n identifierPriority: number;\n }> = [];\n for (let i = 0; i < tools.length; i++) {\n const score = scores[i];\n const hasSearchScore = Number.isFinite(score) && score > 0;\n let identifierPriority = 0;\n let normalizedScore = hasSearchScore ? Math.min(score / maxScore, 1.0) : 0;\n\n if (matchesIdentifiers) {\n const rawBaseName = getBaseToolName(tools[i].name).toLowerCase();\n const rawFullName = tools[i].name.toLowerCase();\n const baseIdentifier = tokenize(rawBaseName).join('');\n const fullIdentifier = tokenize(rawFullName).join('');\n\n if (rawFullName === queryLower) {\n identifierPriority = 4;\n normalizedScore = 1.0;\n } else if (rawBaseName === queryLower) {\n identifierPriority = 3;\n normalizedScore = 1.0;\n } else if (\n baseIdentifier === queryIdentifier ||\n fullIdentifier === queryIdentifier\n ) {\n identifierPriority = 2;\n normalizedScore = 1.0;\n } else if (baseIdentifier.startsWith(queryIdentifier)) {\n identifierPriority = 1;\n normalizedScore = Math.max(normalizedScore, 0.95);\n }\n }\n\n if (!hasSearchScore && identifierPriority === 0) continue;\n\n const { field, snippet } = findMatchedField(tools[i], queryTokens, fields);\n results.push({\n result: {\n tool_name: tools[i].name,\n match_score: normalizedScore,\n matched_field: field,\n snippet,\n },\n identifierPriority,\n });\n }\n\n results.sort(\n (a, b) =>\n b.identifierPriority - a.identifierPriority ||\n b.result.match_score - a.result.match_score\n );\n const topResults = results.slice(0, maxResults).map(({ result }) => result);\n\n return {\n tool_references: topResults,\n total_tools_searched: tools.length,\n pattern_used: query,\n };\n}\n\n/**\n * Generates the JavaScript search script to be executed in the sandbox.\n * Uses plain JavaScript for maximum compatibility with the Code API.\n * @param deferredTools - Array of tool metadata to search through\n * @param fields - Which fields to search\n * @param maxResults - Maximum number of results to return\n * @param sanitizedPattern - The sanitized regex pattern\n * @returns The JavaScript code string\n */\nfunction generateSearchScript(\n deferredTools: t.ToolMetadata[],\n fields: string[],\n maxResults: number,\n sanitizedPattern: string\n): string {\n const lines = [\n '// Tool definitions (injected)',\n 'var tools = ' + JSON.stringify(deferredTools) + ';',\n 'var searchFields = ' + JSON.stringify(fields) + ';',\n 'var maxResults = ' + maxResults + ';',\n 'var pattern = ' + JSON.stringify(sanitizedPattern) + ';',\n '',\n '// Compile regex (pattern is sanitized client-side)',\n 'var regex;',\n 'try {',\n ' regex = new RegExp(pattern, \\'i\\');',\n '} catch (e) {',\n ' regex = new RegExp(pattern.replace(/[.*+?^${}()[\\\\]\\\\\\\\|]/g, \"\\\\\\\\$&\"), \"i\");',\n '}',\n '',\n '// Search logic',\n 'var results = [];',\n '',\n 'for (var j = 0; j < tools.length; j++) {',\n ' var tool = tools[j];',\n ' var bestScore = 0;',\n ' var matchedField = \\'\\';',\n ' var snippet = \\'\\';',\n '',\n ' // Search name (highest priority)',\n ' if (searchFields.indexOf(\\'name\\') >= 0 && regex.test(tool.name)) {',\n ' bestScore = 0.95;',\n ' matchedField = \\'name\\';',\n ' snippet = tool.name;',\n ' }',\n '',\n ' // Search description (medium priority)',\n ' if (searchFields.indexOf(\\'description\\') >= 0 && tool.description && regex.test(tool.description)) {',\n ' if (bestScore === 0) {',\n ' bestScore = 0.75;',\n ' matchedField = \\'description\\';',\n ' snippet = tool.description.substring(0, 100);',\n ' }',\n ' }',\n '',\n ' // Search parameter names (lower priority)',\n ' if (searchFields.indexOf(\\'parameters\\') >= 0 && tool.parameters && tool.parameters.properties) {',\n ' var paramNames = Object.keys(tool.parameters.properties).join(\\' \\');',\n ' if (regex.test(paramNames)) {',\n ' if (bestScore === 0) {',\n ' bestScore = 0.60;',\n ' matchedField = \\'parameters\\';',\n ' snippet = paramNames;',\n ' }',\n ' }',\n ' }',\n '',\n ' if (bestScore > 0) {',\n ' results.push({',\n ' tool_name: tool.name,',\n ' match_score: bestScore,',\n ' matched_field: matchedField,',\n ' snippet: snippet',\n ' });',\n ' }',\n '}',\n '',\n '// Sort by score (descending) and limit results',\n 'results.sort(function(a, b) { return b.match_score - a.match_score; });',\n 'var topResults = results.slice(0, maxResults);',\n '',\n '// Output as JSON',\n 'console.log(JSON.stringify({',\n ' tool_references: topResults.map(function(r) {',\n ' return {',\n ' tool_name: r.tool_name,',\n ' match_score: r.match_score,',\n ' matched_field: r.matched_field,',\n ' snippet: r.snippet',\n ' };',\n ' }),',\n ' total_tools_searched: tools.length,',\n ' pattern_used: pattern',\n '}));',\n ];\n return lines.join('\\n');\n}\n\n/**\n * Parses the search results from stdout JSON.\n * @param stdout - The stdout string containing JSON results\n * @returns Parsed search response\n */\nfunction parseSearchResults(stdout: string): t.ToolSearchResponse {\n const jsonMatch = stdout.trim();\n const parsed = JSON.parse(jsonMatch) as t.ToolSearchResponse;\n return parsed;\n}\n\n/**\n * Formats search results as structured JSON for efficient parsing.\n * @param searchResponse - The parsed search response\n * @param nameFormat - Whether to show 'full' names (tool_mcp_server) or 'base' names (tool only)\n * @returns JSON string with search results\n */\nfunction formatSearchResults(\n searchResponse: t.ToolSearchResponse,\n nameFormat: t.McpNameFormat = 'full'\n): string {\n const { tool_references, total_tools_searched, pattern_used } =\n searchResponse;\n const useFullName = nameFormat === 'full';\n\n const output = {\n found: tool_references.length,\n tools: tool_references.map((ref) => ({\n name: useFullName ? ref.tool_name : getBaseToolName(ref.tool_name),\n score: Number(ref.match_score.toFixed(2)),\n matched_in: ref.matched_field,\n snippet: ref.snippet,\n })),\n total_searched: total_tools_searched,\n query: pattern_used,\n };\n\n return JSON.stringify(output, null, 2);\n}\n\n/**\n * Extracts the base tool name (without MCP server suffix) from a full tool name.\n * @param toolName - The full tool name\n * @returns The base tool name without server suffix\n */\nfunction getBaseToolName(toolName: string): string {\n const delimiterIndex = toolName.indexOf(Constants.MCP_DELIMITER);\n if (delimiterIndex === -1) {\n return toolName;\n }\n return toolName.substring(0, delimiterIndex);\n}\n\n/**\n * Checks whether a tool has any defined parameters in its JSON schema.\n * @param parameters - The tool's JSON schema parameters\n * @returns true if the tool has at least one parameter property\n */\nfunction hasParams(parameters?: t.JsonSchemaType): boolean {\n return (\n parameters?.properties != null &&\n Object.keys(parameters.properties).length > 0\n );\n}\n\n/**\n * Generates a compact listing of deferred tools grouped by server.\n * Format: \"server: tool1, tool2(\\u2026), tool3\"\n * Tools with parameters are annotated with (\\u2026) to signal\n * that the LLM should discover the schema via tool_search before calling.\n * Non-MCP tools are grouped under \"other\".\n * @param toolRegistry - The tool registry\n * @param onlyDeferred - Whether to only include deferred tools\n * @returns Formatted string with tools grouped by server\n */\nfunction getDeferredToolsListing(\n toolRegistry: t.LCToolRegistry | undefined,\n onlyDeferred: boolean\n): string {\n if (!toolRegistry) {\n return '';\n }\n\n const toolsByServer: Record<string, string[]> = {};\n\n for (const lcTool of toolRegistry.values()) {\n if (onlyDeferred && lcTool.defer_loading !== true) {\n continue;\n }\n\n const toolName = lcTool.name;\n const serverName = extractMcpServerName(toolName) ?? 'other';\n const baseName = getBaseToolName(toolName);\n const displayName = hasParams(lcTool.parameters)\n ? `${baseName}(\\u2026)`\n : baseName;\n\n if (!(serverName in toolsByServer)) {\n toolsByServer[serverName] = [];\n }\n toolsByServer[serverName].push(displayName);\n }\n\n const serverNames = Object.keys(toolsByServer).sort((a, b) => {\n if (a === 'other') return 1;\n if (b === 'other') return -1;\n return a.localeCompare(b);\n });\n\n if (serverNames.length === 0) {\n return '';\n }\n\n const lines = serverNames.map(\n (server) => `${server}: ${toolsByServer[server].join(', ')}`\n );\n\n return lines.join('\\n');\n}\n\n/**\n * Formats a server listing response as structured JSON.\n * NOTE: This is a PREVIEW only - tools are NOT discovered/loaded.\n * @param tools - Array of tool metadata from the server(s)\n * @param serverNames - The MCP server name(s)\n * @param nameFormat - Whether to show 'full' names (tool_mcp_server) or 'base' names (tool only)\n * @returns JSON string showing all tools grouped by server\n */\nfunction formatServerListing(\n tools: t.ToolMetadata[],\n serverNames: string | string[],\n nameFormat: t.McpNameFormat = 'full'\n): string {\n const servers = Array.isArray(serverNames) ? serverNames : [serverNames];\n const useFullName = nameFormat === 'full';\n\n if (tools.length === 0) {\n return JSON.stringify(\n {\n listing_mode: true,\n servers,\n total_tools: 0,\n tools_by_server: {},\n hint: 'No tools found from the specified MCP server(s).',\n },\n null,\n 2\n );\n }\n\n const toolsByServer: Record<\n string,\n Array<{ name: string; description: string }>\n > = {};\n for (const tool of tools) {\n const server = extractMcpServerName(tool.name) ?? 'unknown';\n if (!(server in toolsByServer)) {\n toolsByServer[server] = [];\n }\n toolsByServer[server].push({\n name: useFullName ? tool.name : getBaseToolName(tool.name),\n description:\n tool.description.length > 100\n ? tool.description.substring(0, 97) + '...'\n : tool.description,\n });\n }\n\n const exampleToolName = useFullName\n ? (tools[0]?.name ?? 'tool_name')\n : getBaseToolName(tools[0]?.name ?? 'tool_name');\n\n const output = {\n listing_mode: true,\n servers,\n total_tools: tools.length,\n tools_by_server: toolsByServer,\n hint: `To use a tool, search for it by name (e.g., query: \"${exampleToolName}\") to load it.`,\n };\n\n return JSON.stringify(output, null, 2);\n}\n\n/**\n * Creates a Tool Search tool for discovering tools from a large registry.\n *\n * This tool enables AI agents to dynamically discover tools from a large library\n * without loading all tool definitions into the LLM context window. The agent\n * can search for relevant tools on-demand.\n *\n * **Modes:**\n * - `code_interpreter` (default): Uses external sandbox for regex search. Safer for complex patterns.\n * - `local`: Uses safe substring matching locally. No network call, faster, completely safe from ReDoS.\n *\n * The tool registry can be provided either:\n * 1. At initialization time via params.toolRegistry\n * 2. At runtime via config.configurable.toolRegistry when invoking\n *\n * @param params - Configuration parameters for the tool (toolRegistry is optional)\n * @returns A LangChain DynamicStructuredTool for tool searching\n *\n * @example\n * // Option 1: Code interpreter mode (regex via sandbox)\n * const tool = createToolSearch({ toolRegistry });\n * await tool.invoke({ query: 'expense.*report' });\n *\n * @example\n * // Option 2: Local mode (safe substring search)\n * const tool = createToolSearch({ mode: 'local', toolRegistry });\n * await tool.invoke({ query: 'expense' });\n */\nfunction createToolSearch(\n initParams: t.ToolSearchParams = {}\n): DynamicStructuredTool {\n const mode: t.ToolSearchMode = initParams.mode ?? 'code_interpreter';\n const defaultOnlyDeferred = initParams.onlyDeferred ?? true;\n const mcpNameFormat: t.McpNameFormat = initParams.mcpNameFormat ?? 'full';\n const schema = createToolSearchSchema(mode);\n\n const baseEndpoint = initParams.baseUrl ?? getCodeBaseURL();\n const EXEC_ENDPOINT = `${baseEndpoint}/exec`;\n\n const deferredToolsListing = getDeferredToolsListing(\n initParams.toolRegistry,\n defaultOnlyDeferred\n );\n\n const toolsListSection =\n deferredToolsListing.length > 0\n ? `\n\nDeferred tools (search to load; \\u2026 = has params, search first):\n${deferredToolsListing}`\n : '';\n\n const mcpNote =\n deferredToolsListing.includes(Constants.MCP_DELIMITER) ||\n deferredToolsListing.split('\\n').some((line) => !line.startsWith('other:'))\n ? `\n- MCP tools use format: toolName${Constants.MCP_DELIMITER}serverName\n- Use mcp_server param to filter by server`\n : '';\n\n const description =\n mode === 'local'\n ? `\nSearches deferred tools using BM25 ranking. Multi-word queries supported.\n${mcpNote}${toolsListSection}\n`.trim()\n : `\nSearches deferred tools by regex pattern.\n${mcpNote}${toolsListSection}\n`.trim();\n\n return tool(\n async (rawParams, config) => {\n const params = rawParams as ToolSearchParams;\n const {\n query = '',\n fields = ['name', 'description'],\n max_results = DEFAULT_MAX_RESULTS,\n mcp_server,\n } = params;\n\n const {\n toolRegistry: paramToolRegistry,\n onlyDeferred: paramOnlyDeferred,\n mcpServer: paramMcpServer,\n } = config.toolCall ?? {};\n\n const toolRegistry = paramToolRegistry ?? initParams.toolRegistry;\n const onlyDeferred =\n paramOnlyDeferred !== undefined\n ? paramOnlyDeferred\n : defaultOnlyDeferred;\n const rawServerFilter =\n mcp_server ?? paramMcpServer ?? initParams.mcpServer;\n const serverFilters = normalizeServerFilter(rawServerFilter);\n const hasServerFilter = serverFilters.length > 0;\n\n if (toolRegistry == null) {\n return [\n 'Error: No tool registry provided. Configure toolRegistry at agent level or initialization.',\n {\n tool_references: [],\n metadata: {\n total_searched: 0,\n pattern: query,\n error: 'No tool registry provided',\n },\n },\n ];\n }\n\n const toolsArray: t.LCTool[] = Array.from(toolRegistry.values());\n const deferredTools: t.ToolMetadata[] = toolsArray\n .filter((lcTool) => {\n if (onlyDeferred === true && lcTool.defer_loading !== true) {\n return false;\n }\n if (\n hasServerFilter &&\n !isFromAnyMcpServer(lcTool.name, serverFilters)\n ) {\n return false;\n }\n return true;\n })\n .map((lcTool) => ({\n name: lcTool.name,\n description: lcTool.description ?? '',\n parameters: simplifyParametersForSearch(lcTool.parameters),\n }));\n\n if (deferredTools.length === 0) {\n const serverMsg = hasServerFilter\n ? ` from MCP server(s): ${serverFilters.join(', ')}`\n : '';\n return [\n `No tools available to search${serverMsg}. The tool registry is empty or no matching deferred tools are registered.`,\n {\n tool_references: [],\n metadata: {\n total_searched: 0,\n pattern: query,\n mcp_server: serverFilters,\n },\n },\n ];\n }\n\n const isServerListing = hasServerFilter && query === '';\n\n if (isServerListing) {\n const formattedOutput = formatServerListing(\n deferredTools,\n serverFilters,\n mcpNameFormat\n );\n\n return [\n formattedOutput,\n {\n tool_references: [],\n metadata: {\n total_available: deferredTools.length,\n mcp_server: serverFilters,\n listing_mode: true,\n },\n },\n ];\n }\n\n if (mode === 'local') {\n const searchResponse = performLocalSearch(\n deferredTools,\n query,\n fields,\n max_results\n );\n const formattedOutput = formatSearchResults(\n searchResponse,\n mcpNameFormat\n );\n\n return [\n formattedOutput,\n {\n tool_references: searchResponse.tool_references,\n metadata: {\n total_searched: searchResponse.total_tools_searched,\n pattern: searchResponse.pattern_used,\n mcp_server: serverFilters.length > 0 ? serverFilters : undefined,\n },\n },\n ];\n }\n\n const { safe: sanitizedPattern, wasEscaped } = sanitizeRegex(query);\n let warningMessage = '';\n if (wasEscaped) {\n warningMessage =\n 'Note: The provided pattern was converted to a literal search for safety.\\n\\n';\n }\n\n const searchScript = generateSearchScript(\n deferredTools,\n fields,\n max_results,\n sanitizedPattern\n );\n\n const postData = {\n lang: 'js',\n code: searchScript,\n timeout: SEARCH_TIMEOUT,\n };\n\n try {\n const fetchOptions: RequestInit = {\n method: 'POST',\n headers: {\n 'Content-Type': 'application/json',\n 'User-Agent': 'LibreChat/1.0',\n },\n body: JSON.stringify(postData),\n };\n\n const proxyAgent = resolveFetchProxyAgent(EXEC_ENDPOINT);\n if (proxyAgent) {\n fetchOptions.agent = proxyAgent;\n }\n\n const response = await fetch(EXEC_ENDPOINT, fetchOptions);\n if (!response.ok) {\n throw new Error(`HTTP error! status: ${response.status}`);\n }\n\n const result: t.ExecuteResult = await response.json();\n\n if (result.stderr && result.stderr.trim()) {\n // eslint-disable-next-line no-console\n console.warn('[ToolSearch] stderr:', result.stderr);\n }\n\n if (!result.stdout || !result.stdout.trim()) {\n return [\n `${warningMessage}No tools matched the pattern \"${sanitizedPattern}\".\\nTotal tools searched: ${deferredTools.length}`,\n {\n tool_references: [],\n metadata: {\n total_searched: deferredTools.length,\n pattern: sanitizedPattern,\n },\n },\n ];\n }\n\n const searchResponse = parseSearchResults(result.stdout);\n const formattedOutput = `${warningMessage}${formatSearchResults(searchResponse, mcpNameFormat)}`;\n\n return [\n formattedOutput,\n {\n tool_references: searchResponse.tool_references,\n metadata: {\n total_searched: searchResponse.total_tools_searched,\n pattern: searchResponse.pattern_used,\n },\n },\n ];\n } catch (error) {\n // eslint-disable-next-line no-console\n console.error('[ToolSearch] Error:', error);\n\n const errorMessage =\n error instanceof Error ? error.message : String(error);\n return [\n `Tool search failed: ${errorMessage}\\n\\nSuggestion: Try a simpler search pattern or search for specific tool names.`,\n {\n tool_references: [],\n metadata: {\n total_searched: 0,\n pattern: sanitizedPattern,\n error: errorMessage,\n },\n },\n ];\n }\n },\n {\n name: Constants.TOOL_SEARCH,\n description,\n schema,\n responseFormat: Constants.CONTENT_AND_ARTIFACT,\n }\n );\n}\n\nexport {\n createToolSearch,\n performLocalSearch,\n extractMcpServerName,\n isFromMcpServer,\n isFromAnyMcpServer,\n normalizeServerFilter,\n getAvailableMcpServers,\n getDeferredToolsListing,\n getBaseToolName,\n formatServerListing,\n sanitizeRegex,\n escapeRegexSpecialChars,\n isDangerousPattern,\n countNestedGroups,\n hasNestedQuantifiers,\n};\n"],"mappings":";;;;;;;;;;AAUA,SAAS,kBAA0B;CACjC,MAAM,MAAM;CAGZ,IAAI,OAAO,QAAQ,YAAY,OAAO;CACtC,IAAI,OAAO,IAAI,YAAY,YAAY,OAAO,IAAI;CAClD,IAAI,IAAI,WAAW,QAAQ,OAAO,IAAI,QAAQ,YAAY,YACxD,OAAO,IAAI,QAAQ;CACrB,MAAM,IAAI,MAAM,uDAAuD;AACzE;AAEA,MAAM,OAAO,gBAAgB;AAS7B,OAAO;;AAGP,MAAM,qBAAqB;AAE3B,MAAa,qBAAA;AAEb,MAAa,4BACX;AAEF,MAAM,0BACJ;AACF,MAAM,0BACJ;AACF,MAAM,qBACJ;AACF,MAAM,0BAA0B;AAChC,MAAM,sBAAsB;AAC5B,MAAM,yBACJ;AAEF,MAAa,uBAAuB;CAClC,MAAM;CACN,YAAY;EACV,QAAQ,EAAE,GAAG,gBAAgB;EAC7B,OAAO;GACL,MAAM;GACN,WAAW;GACX,SAAS;GACT,aAAa;EACf;EACA,QAAQ;GACN,MAAM;GACN,OAAO;IAAE,MAAM;IAAU,MAAM;KAAC;KAAQ;KAAe;IAAY;GAAE;GACrE,SAAS,CAAC,QAAQ,aAAa;GAC/B,aAAa;EACf;EACA,aAAa;GACX,MAAM;GACN,SAAS;GACT,SAAS;GACT,SAAS;GACT,aAAa;EACf;EACA,YAAY;GACV,OAAO,CAAC,EAAE,MAAM,SAAS,GAAG;IAAE,MAAM;IAAS,OAAO,EAAE,MAAM,SAAS;GAAE,CAAC;GACxE,aAAa;EACf;CACF;CACA,UAAU,CAAC;AACb;AAEA,MAAa,2BAA2B;CACtC,MAAM;CACN,aAAa;CACb,QAAQ;AACV;;AAGA,MAAM,uBAAuB;;AAG7B,MAAM,iBAAiB;;;;;;AAsBvB,SAAS,uBAAuB,MAA0C;CACxE,MAAM,mBACJ,SAAS,UAAU,0BAA0B;CAE/C,OAAO;EACL,MAAM;EACN,YAAY;GACV,QAAQ,EAAE,GAAG,gBAAgB;GAC7B,OAAO;IACL,MAAM;IACN,WAAW;IACX,SAAS;IACT,aAAa;GACf;GACA,QAAQ;IACN,MAAM;IACN,OAAO;KAAE,MAAM;KAAU,MAAM;MAAC;MAAQ;MAAe;KAAY;IAAE;IACrE,SAAS,CAAC,QAAQ,aAAa;IAC/B,aAAa;GACf;GACA,aAAa;IACX,MAAM;IACN,SAAS;IACT,SAAS;IACT,SAAS;IACT,aAAa;GACf;GACA,YAAY;IACV,OAAO,CACL,EAAE,MAAM,SAAS,GACjB;KAAE,MAAM;KAAS,OAAO,EAAE,MAAM,SAAS;IAAE,CAC7C;IACA,aAAa;GACf;EACF;EACA,UAAU,CAAC;CACb;AACF;;;;;;;AAQA,SAAS,qBAAqB,UAAsC;CAClE,MAAM,iBAAiB,SAAS,QAAA,OAA+B;CAC/D,IAAI,mBAAmB,IACrB;CAEF,OAAO,SAAS,UAAU,iBAAA,CAA+C;AAC3E;;;;;;;AAQA,SAAS,gBAAgB,UAAkB,YAA6B;CAEtE,OADmB,qBAAqB,QACxB,MAAM;AACxB;;;;;;;AAQA,SAAS,mBAAmB,UAAkB,aAAgC;CAC5E,MAAM,aAAa,qBAAqB,QAAQ;CAChD,IAAI,eAAe,KAAA,GACjB,OAAO;CAET,OAAO,YAAY,SAAS,UAAU;AACxC;;;;;;AAOA,SAAS,sBACP,cACU;CACV,IAAI,iBAAiB,KAAA,GACnB,OAAO,CAAC;CAEV,IAAI,OAAO,iBAAiB,UAC1B,OAAO,iBAAiB,KAAK,CAAC,IAAI,CAAC,YAAY;CAEjD,OAAO,aAAa,QAAQ,MAAM,MAAM,EAAE;AAC5C;;;;;;;AAQA,SAAS,uBACP,cACA,eAAwB,MACd;CACV,IAAI,CAAC,cACH,OAAO,CAAC;CAGV,MAAM,0BAAU,IAAI,IAAY;CAChC,KAAK,MAAM,GAAG,YAAY,cAAc;EACtC,IAAI,gBAAgB,QAAQ,kBAAkB,MAC5C;EAEF,MAAM,SAAS,qBAAqB,QAAQ,IAAI;EAChD,IAAI,WAAW,KAAA,KAAa,WAAW,IACrC,QAAQ,IAAI,MAAM;CAEtB;CAEA,OAAO,MAAM,KAAK,OAAO,CAAC,CAAC,KAAK;AAClC;;;;;;AAOA,SAAS,wBAAwB,SAAyB;CACxD,OAAO,QAAQ,QAAQ,uBAAuB,MAAM;AACtD;;;;;;AAOA,SAAS,kBAAkB,SAAyB;CAClD,IAAI,WAAW;CACf,IAAI,eAAe;CAEnB,KAAK,IAAI,IAAI,GAAG,IAAI,QAAQ,QAAQ,KAClC,IAAI,QAAQ,OAAO,QAAQ,MAAM,KAAK,QAAQ,IAAI,OAAO,OAAO;EAC9D;EACA,WAAW,KAAK,IAAI,UAAU,YAAY;CAC5C,OAAO,IAAI,QAAQ,OAAO,QAAQ,MAAM,KAAK,QAAQ,IAAI,OAAO,OAC9D,eAAe,KAAK,IAAI,GAAG,eAAe,CAAC;CAI/C,OAAO;AACT;;;;;;;AAQA,SAAS,qBAAqB,SAA0B;CAEtD,OAAO,0BAAwB,KAAK,OAAO;AAC7C;;;;;;AAOA,SAAS,mBAAmB,SAA0B;CACpD,IAAI,qBAAqB,OAAO,GAC9B,OAAO;CAGT,IAAI,kBAAkB,OAAO,IAAI,sBAC/B,OAAO;CAaT,KAAK,MAAM,aAAa;EATtB;EACA;EACA;EACA;EACA;EACA;EACA;CAGsC,GACtC,IAAI,UAAU,KAAK,OAAO,GACxB,OAAO;CAIX,OAAO;AACT;;;;;;;AAQA,SAAS,cAAc,SAAwD;CAC7E,IAAI,mBAAmB,OAAO,GAC5B,OAAO;EACL,MAAM,wBAAwB,OAAO;EACrC,YAAY;CACd;CAGF,IAAI;EACF,IAAI,OAAO,OAAO;EAClB,OAAO;GAAE,MAAM;GAAS,YAAY;EAAM;CAC5C,QAAQ;EACN,OAAO;GACL,MAAM,wBAAwB,OAAO;GACrC,YAAY;EACd;CACF;AACF;;;;;;;AAQA,SAAS,4BACP,YAC8B;CAC9B,IAAI,CAAC,YACH;CAGF,IAAI,WAAW,YACb,OAAO;EACL,MAAM,WAAW;EACjB,YAAY,OAAO,YACjB,OAAO,QAAQ,WAAW,UAAU,CAAC,CAAC,KAAK,CAAC,KAAK,WAAW,CAC1D,KACA,EAAE,MAAO,MAA2B,KAAK,CAC3C,CAAC,CACH;CACF;CAGF,OAAO,EAAE,MAAM,WAAW,KAAK;AACjC;;;;;AAMA,SAAS,iBAAiB,SAA2B;CACnD,MAAM,cAAc,QACjB,QAAQ,sBAAsB,OAAO,CAAC,CACtC,QAAQ,yBAAyB,OAAO,CAAC,CACzC,YAAY,CAAC,CACb,MAAM,KAAK,CAAC,CACZ,OAAO,OAAO;CAEjB,IAAI,YAAY,SAAS,GAAG,OAAO;CAEnC,MAAM,eAAyB,CAAC;CAChC,IAAI,SAAS;CACb,KAAK,MAAM,SAAS,aAAa;EAC/B,IAAI,MAAM,WAAW,GAAG;GACtB,UAAU;GACV;EACF;EACA,aAAa,KAAK,GAAG,SAAS,OAAO;EACrC,SAAS;CACX;CAEA,IAAI,UAAU,aAAa,SAAS,GAClC,aAAa,aAAa,SAAS,MAAM;CAE3C,OAAO,aAAa,SAAS,IAAI,eAAe,CAAC,MAAM;AACzD;;;;;;;AAQA,SAAS,SAAS,MAAwB;CACxC,OAAO,KACJ,MAAM,eAAe,CAAC,CACtB,OAAO,OAAO,CAAC,CACf,QAAQ,gBAAgB;AAC7B;;;;;;;AAQA,SAAS,mBAAmB,MAAsB,QAA0B;CAC1E,MAAM,QAAkB,CAAC;CAEzB,IAAI,OAAO,SAAS,MAAM,GAAG;EAC3B,MAAM,WAAW,KAAK,KAAK,QAAQ,MAAM,GAAG;EAC5C,MAAM,KAAK,UAAU,QAAQ;CAC/B;CAEA,IAAI,OAAO,SAAS,aAAa,KAAK,KAAK,aACzC,MAAM,KAAK,KAAK,WAAW;CAG7B,IAAI,OAAO,SAAS,YAAY,KAAK,KAAK,YAAY,YAAY;EAChE,MAAM,aAAa,OAAO,KAAK,KAAK,WAAW,UAAU,CAAC,CAAC,KAAK,GAAG;EACnE,MAAM,KAAK,UAAU;CACvB;CAEA,OAAO,SAAS,MAAM,KAAK,GAAG,CAAC,CAAC,CAAC,KAAK,GAAG;AAC3C;;;;;;;;AASA,SAAS,iBACP,MACA,aACA,QACoC;CACpC,IAAI,OAAO,SAAS,MAAM,GAAG;EAC3B,MAAM,YAAY,KAAK,KAAK,YAAY;EACxC,KAAK,MAAM,SAAS,aAClB,IAAI,UAAU,SAAS,KAAK,GAC1B,OAAO;GAAE,OAAO;GAAQ,SAAS,KAAK;EAAK;CAGjD;CAEA,IAAI,OAAO,SAAS,aAAa,KAAK,KAAK,aAAa;EACtD,MAAM,YAAY,KAAK,YAAY,YAAY;EAC/C,KAAK,MAAM,SAAS,aAClB,IAAI,UAAU,SAAS,KAAK,GAC1B,OAAO;GACL,OAAO;GACP,SAAS,KAAK,YAAY,UAAU,GAAG,GAAG;EAC5C;CAGN;CAEA,IAAI,OAAO,SAAS,YAAY,KAAK,KAAK,YAAY,YAAY;EAChE,MAAM,aAAa,OAAO,KAAK,KAAK,WAAW,UAAU;EACzD,MAAM,aAAa,WAAW,KAAK,GAAG,CAAC,CAAC,YAAY;EACpD,KAAK,MAAM,SAAS,aAClB,IAAI,WAAW,SAAS,KAAK,GAC3B,OAAO;GAAE,OAAO;GAAc,SAAS,WAAW,KAAK,IAAI;EAAE;CAGnE;CAKA,OAAO;EAAE,OAAO;EAAW,SAHH,KAAK,cACzB,KAAK,YAAY,UAAU,GAAG,GAAG,IACjC,KAAK;CAC2C;AACtD;;;;;;;;;;;AAYA,SAAS,mBACP,OACA,OACA,QACA,YACsB;CACtB,IAAI,MAAM,WAAW,GACnB,OAAO;EACL,iBAAiB,CAAC;EAClB,sBAAsB;EACtB,cAAc;CAChB;CAGF,MAAM,cAAc,SAAS,KAAK;CAElC,IAAI,YAAY,WAAW,GAYzB,OAAO;EACL,iBAZe,MACd,MAAM,CAAC,CACP,MAAM,GAAG,MAAM,EAAE,KAAK,cAAc,EAAE,IAAI,CAAC,CAAC,CAC5C,MAAM,GAAG,UAAU,CAAC,CACpB,KAAK,UAAU;GACd,WAAW,KAAK;GAChB,aAAa;GACb,eAAe;GACf,SAAS,KAAK,YAAY,UAAU,GAAG,GAAG,KAAK,KAAK;EACtD,EAGwB;EACxB,sBAAsB,MAAM;EAC5B,cAAc;CAChB;CAIF,MAAM,SAAS,KADG,MAAM,KAAK,SAAS,mBAAmB,MAAM,MAAM,CACzC,GAAG,aAAa;EAAE,IAAI;EAAK,GAAG;CAAK,CAAC;CAEhE,MAAM,WAAW,KAAK,IAAI,GAAG,OAAO,QAAQ,MAAM,IAAI,CAAC,GAAG,CAAC;CAC3D,MAAM,aAAa,MAAM,YAAY,CAAC,CAAC,KAAK;CAC5C,MAAM,kBAAkB,YAAY,KAAK,EAAE;CAC3C,MAAM,qBAAqB,OAAO,SAAS,MAAM;CAEjD,MAAM,UAGD,CAAC;CACN,KAAK,IAAI,IAAI,GAAG,IAAI,MAAM,QAAQ,KAAK;EACrC,MAAM,QAAQ,OAAO;EACrB,MAAM,iBAAiB,OAAO,SAAS,KAAK,KAAK,QAAQ;EACzD,IAAI,qBAAqB;EACzB,IAAI,kBAAkB,iBAAiB,KAAK,IAAI,QAAQ,UAAU,CAAG,IAAI;EAEzE,IAAI,oBAAoB;GACtB,MAAM,cAAc,gBAAgB,MAAM,EAAE,CAAC,IAAI,CAAC,CAAC,YAAY;GAC/D,MAAM,cAAc,MAAM,EAAE,CAAC,KAAK,YAAY;GAC9C,MAAM,iBAAiB,SAAS,WAAW,CAAC,CAAC,KAAK,EAAE;GACpD,MAAM,iBAAiB,SAAS,WAAW,CAAC,CAAC,KAAK,EAAE;GAEpD,IAAI,gBAAgB,YAAY;IAC9B,qBAAqB;IACrB,kBAAkB;GACpB,OAAO,IAAI,gBAAgB,YAAY;IACrC,qBAAqB;IACrB,kBAAkB;GACpB,OAAO,IACL,mBAAmB,mBACnB,mBAAmB,iBACnB;IACA,qBAAqB;IACrB,kBAAkB;GACpB,OAAO,IAAI,eAAe,WAAW,eAAe,GAAG;IACrD,qBAAqB;IACrB,kBAAkB,KAAK,IAAI,iBAAiB,GAAI;GAClD;EACF;EAEA,IAAI,CAAC,kBAAkB,uBAAuB,GAAG;EAEjD,MAAM,EAAE,OAAO,YAAY,iBAAiB,MAAM,IAAI,aAAa,MAAM;EACzE,QAAQ,KAAK;GACX,QAAQ;IACN,WAAW,MAAM,EAAE,CAAC;IACpB,aAAa;IACb,eAAe;IACf;GACF;GACA;EACF,CAAC;CACH;CAEA,QAAQ,MACL,GAAG,MACF,EAAE,qBAAqB,EAAE,sBACzB,EAAE,OAAO,cAAc,EAAE,OAAO,WACpC;CAGA,OAAO;EACL,iBAHiB,QAAQ,MAAM,GAAG,UAAU,CAAC,CAAC,KAAK,EAAE,aAAa,MAGxC;EAC1B,sBAAsB,MAAM;EAC5B,cAAc;CAChB;AACF;;;;;;;;;;AAWA,SAAS,qBACP,eACA,QACA,YACA,kBACQ;CAiFR,OAAO;EA/EL;EACA,iBAAiB,KAAK,UAAU,aAAa,IAAI;EACjD,wBAAwB,KAAK,UAAU,MAAM,IAAI;EACjD,sBAAsB,aAAa;EACnC,mBAAmB,KAAK,UAAU,gBAAgB,IAAI;EACtD;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;CAES,CAAC,CAAC,KAAK,IAAI;AACxB;;;;;;AAOA,SAAS,mBAAmB,QAAsC;CAChE,MAAM,YAAY,OAAO,KAAK;CAE9B,OADe,KAAK,MAAM,SACd;AACd;;;;;;;AAQA,SAAS,oBACP,gBACA,aAA8B,QACtB;CACR,MAAM,EAAE,iBAAiB,sBAAsB,iBAC7C;CACF,MAAM,cAAc,eAAe;CAEnC,MAAM,SAAS;EACb,OAAO,gBAAgB;EACvB,OAAO,gBAAgB,KAAK,SAAS;GACnC,MAAM,cAAc,IAAI,YAAY,gBAAgB,IAAI,SAAS;GACjE,OAAO,OAAO,IAAI,YAAY,QAAQ,CAAC,CAAC;GACxC,YAAY,IAAI;GAChB,SAAS,IAAI;EACf,EAAE;EACF,gBAAgB;EAChB,OAAO;CACT;CAEA,OAAO,KAAK,UAAU,QAAQ,MAAM,CAAC;AACvC;;;;;;AAOA,SAAS,gBAAgB,UAA0B;CACjD,MAAM,iBAAiB,SAAS,QAAA,OAA+B;CAC/D,IAAI,mBAAmB,IACrB,OAAO;CAET,OAAO,SAAS,UAAU,GAAG,cAAc;AAC7C;;;;;;AAOA,SAAS,UAAU,YAAwC;CACzD,OACE,YAAY,cAAc,QAC1B,OAAO,KAAK,WAAW,UAAU,CAAC,CAAC,SAAS;AAEhD;;;;;;;;;;;AAYA,SAAS,wBACP,cACA,cACQ;CACR,IAAI,CAAC,cACH,OAAO;CAGT,MAAM,gBAA0C,CAAC;CAEjD,KAAK,MAAM,UAAU,aAAa,OAAO,GAAG;EAC1C,IAAI,gBAAgB,OAAO,kBAAkB,MAC3C;EAGF,MAAM,WAAW,OAAO;EACxB,MAAM,aAAa,qBAAqB,QAAQ,KAAK;EACrD,MAAM,WAAW,gBAAgB,QAAQ;EACzC,MAAM,cAAc,UAAU,OAAO,UAAU,IAC3C,GAAG,SAAS,YACZ;EAEJ,IAAI,EAAE,cAAc,gBAClB,cAAc,cAAc,CAAC;EAE/B,cAAc,WAAW,CAAC,KAAK,WAAW;CAC5C;CAEA,MAAM,cAAc,OAAO,KAAK,aAAa,CAAC,CAAC,MAAM,GAAG,MAAM;EAC5D,IAAI,MAAM,SAAS,OAAO;EAC1B,IAAI,MAAM,SAAS,OAAO;EAC1B,OAAO,EAAE,cAAc,CAAC;CAC1B,CAAC;CAED,IAAI,YAAY,WAAW,GACzB,OAAO;CAOT,OAJc,YAAY,KACvB,WAAW,GAAG,OAAO,IAAI,cAAc,OAAO,CAAC,KAAK,IAAI,GAGhD,CAAC,CAAC,KAAK,IAAI;AACxB;;;;;;;;;AAUA,SAAS,oBACP,OACA,aACA,aAA8B,QACtB;CACR,MAAM,UAAU,MAAM,QAAQ,WAAW,IAAI,cAAc,CAAC,WAAW;CACvE,MAAM,cAAc,eAAe;CAEnC,IAAI,MAAM,WAAW,GACnB,OAAO,KAAK,UACV;EACE,cAAc;EACd;EACA,aAAa;EACb,iBAAiB,CAAC;EAClB,MAAM;CACR,GACA,MACA,CACF;CAGF,MAAM,gBAGF,CAAC;CACL,KAAK,MAAM,QAAQ,OAAO;EACxB,MAAM,SAAS,qBAAqB,KAAK,IAAI,KAAK;EAClD,IAAI,EAAE,UAAU,gBACd,cAAc,UAAU,CAAC;EAE3B,cAAc,OAAO,CAAC,KAAK;GACzB,MAAM,cAAc,KAAK,OAAO,gBAAgB,KAAK,IAAI;GACzD,aACE,KAAK,YAAY,SAAS,MACtB,KAAK,YAAY,UAAU,GAAG,EAAE,IAAI,QACpC,KAAK;EACb,CAAC;CACH;CAEA,MAAM,kBAAkB,cACnB,MAAM,EAAE,EAAE,QAAQ,cACnB,gBAAgB,MAAM,EAAE,EAAE,QAAQ,WAAW;CAEjD,MAAM,SAAS;EACb,cAAc;EACd;EACA,aAAa,MAAM;EACnB,iBAAiB;EACjB,MAAM,uDAAuD,gBAAgB;CAC/E;CAEA,OAAO,KAAK,UAAU,QAAQ,MAAM,CAAC;AACvC;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AA8BA,SAAS,iBACP,aAAiC,CAAC,GACX;CACvB,MAAM,OAAyB,WAAW,QAAQ;CAClD,MAAM,sBAAsB,WAAW,gBAAgB;CACvD,MAAM,gBAAiC,WAAW,iBAAiB;CACnE,MAAM,SAAS,uBAAuB,IAAI;CAG1C,MAAM,gBAAgB,GADD,WAAW,WAAW,eAAe,EACpB;CAEtC,MAAM,uBAAuB,wBAC3B,WAAW,cACX,mBACF;CAEA,MAAM,mBACJ,qBAAqB,SAAS,IAC1B;;;EAGN,yBACM;CAEN,MAAM,UACJ,qBAAqB,SAAA,OAAgC,KACrD,qBAAqB,MAAM,IAAI,CAAC,CAAC,MAAM,SAAS,CAAC,KAAK,WAAW,QAAQ,CAAC,IACtE;;8CAGA;CAaN,OAAO,KACL,OAAO,WAAW,WAAW;EAE3B,MAAM,EACJ,QAAQ,IACR,SAAS,CAAC,QAAQ,aAAa,GAC/B,cAAc,qBACd,eACEA;EAEJ,MAAM,EACJ,cAAc,mBACd,cAAc,mBACd,WAAW,mBACT,OAAO,YAAY,CAAC;EAExB,MAAM,eAAe,qBAAqB,WAAW;EACrD,MAAM,eACJ,sBAAsB,KAAA,IAClB,oBACA;EAGN,MAAM,gBAAgB,sBADpB,cAAc,kBAAkB,WAAW,SACc;EAC3D,MAAM,kBAAkB,cAAc,SAAS;EAE/C,IAAI,gBAAgB,MAClB,OAAO,CACL,8FACA;GACE,iBAAiB,CAAC;GAClB,UAAU;IACR,gBAAgB;IAChB,SAAS;IACT,OAAO;GACT;EACF,CACF;EAIF,MAAM,gBADyB,MAAM,KAAK,aAAa,OAAO,CACb,CAAC,CAC/C,QAAQ,WAAW;GAClB,IAAI,iBAAiB,QAAQ,OAAO,kBAAkB,MACpD,OAAO;GAET,IACE,mBACA,CAAC,mBAAmB,OAAO,MAAM,aAAa,GAE9C,OAAO;GAET,OAAO;EACT,CAAC,CAAC,CACD,KAAK,YAAY;GAChB,MAAM,OAAO;GACb,aAAa,OAAO,eAAe;GACnC,YAAY,4BAA4B,OAAO,UAAU;EAC3D,EAAE;EAEJ,IAAI,cAAc,WAAW,GAI3B,OAAO,CACL,+BAJgB,kBACd,wBAAwB,cAAc,KAAK,IAAI,MAC/C,GAEuC,6EACzC;GACE,iBAAiB,CAAC;GAClB,UAAU;IACR,gBAAgB;IAChB,SAAS;IACT,YAAY;GACd;EACF,CACF;EAKF,IAFwB,mBAAmB,UAAU,IASnD,OAAO,CANiB,oBACtB,eACA,eACA,aAIc,GACd;GACE,iBAAiB,CAAC;GAClB,UAAU;IACR,iBAAiB,cAAc;IAC/B,YAAY;IACZ,cAAc;GAChB;EACF,CACF;EAGF,IAAI,SAAS,SAAS;GACpB,MAAM,iBAAiB,mBACrB,eACA,OACA,QACA,WACF;GAMA,OAAO,CALiB,oBACtB,gBACA,aAIc,GACd;IACE,iBAAiB,eAAe;IAChC,UAAU;KACR,gBAAgB,eAAe;KAC/B,SAAS,eAAe;KACxB,YAAY,cAAc,SAAS,IAAI,gBAAgB,KAAA;IACzD;GACF,CACF;EACF;EAEA,MAAM,EAAE,MAAM,kBAAkB,eAAe,cAAc,KAAK;EAClE,IAAI,iBAAiB;EACrB,IAAI,YACF,iBACE;EAUJ,MAAM,WAAW;GACf,MAAM;GACN,MATmB,qBACnB,eACA,QACA,aACA,gBAKiB;GACjB,SAAS;EACX;EAEA,IAAI;GACF,MAAM,eAA4B;IAChC,QAAQ;IACR,SAAS;KACP,gBAAgB;KAChB,cAAc;IAChB;IACA,MAAM,KAAK,UAAU,QAAQ;GAC/B;GAEA,MAAM,aAAa,uBAAuB,aAAa;GACvD,IAAI,YACF,aAAa,QAAQ;GAGvB,MAAM,WAAW,MAAM,MAAM,eAAe,YAAY;GACxD,IAAI,CAAC,SAAS,IACZ,MAAM,IAAI,MAAM,uBAAuB,SAAS,QAAQ;GAG1D,MAAM,SAA0B,MAAM,SAAS,KAAK;GAEpD,IAAI,OAAO,UAAU,OAAO,OAAO,KAAK,GAEtC,QAAQ,KAAK,wBAAwB,OAAO,MAAM;GAGpD,IAAI,CAAC,OAAO,UAAU,CAAC,OAAO,OAAO,KAAK,GACxC,OAAO,CACL,GAAG,eAAe,gCAAgC,iBAAiB,4BAA4B,cAAc,UAC7G;IACE,iBAAiB,CAAC;IAClB,UAAU;KACR,gBAAgB,cAAc;KAC9B,SAAS;IACX;GACF,CACF;GAGF,MAAM,iBAAiB,mBAAmB,OAAO,MAAM;GAGvD,OAAO,CACL,GAHyB,iBAAiB,oBAAoB,gBAAgB,aAAa,KAI3F;IACE,iBAAiB,eAAe;IAChC,UAAU;KACR,gBAAgB,eAAe;KAC/B,SAAS,eAAe;IAC1B;GACF,CACF;EACF,SAAS,OAAO;GAEd,QAAQ,MAAM,uBAAuB,KAAK;GAE1C,MAAM,eACJ,iBAAiB,QAAQ,MAAM,UAAU,OAAO,KAAK;GACvD,OAAO,CACL,uBAAuB,aAAa,kFACpC;IACE,iBAAiB,CAAC;IAClB,UAAU;KACR,gBAAgB;KAChB,SAAS;KACT,OAAO;IACT;GACF,CACF;EACF;CACF,GACA;EACE,MAAA;EACA,aApOF,SAAS,UACL;;EAEN,UAAU,iBAAiB;EAC3B,KAAK,IACC;;EAEN,UAAU,iBAAiB;EAC3B,KAAK;EA6ND;EACA,gBAAA;CACF,CACF;AACF"}
@@ -25,6 +25,8 @@ var CrwScraper = class {
25
25
  xpath;
26
26
  proxy;
27
27
  stealth;
28
+ httpAgent;
29
+ httpsAgent;
28
30
  constructor(config = {}) {
29
31
  this.apiKey = config.apiKey ?? process.env.CRW_API_KEY ?? "";
30
32
  const baseUrl = config.apiUrl ?? process.env.CRW_API_URL ?? "https://api.fastcrw.com";
@@ -42,6 +44,8 @@ var CrwScraper = class {
42
44
  this.xpath = config.xpath;
43
45
  this.proxy = config.proxy;
44
46
  this.stealth = config.stealth;
47
+ this.httpAgent = config.httpAgent;
48
+ this.httpsAgent = config.httpsAgent;
45
49
  if (!this.apiKey) this.logger.warn("CRW_API_KEY is not set. Scraping will not work.");
46
50
  this.logger.debug(`CRW scraper initialized with API URL: ${this.apiUrl}`);
47
51
  }
@@ -68,7 +72,9 @@ var CrwScraper = class {
68
72
  if (this.apiKey) headers.Authorization = `Bearer ${this.apiKey}`;
69
73
  return [url, normalizeCrwResponse((await axios.post(this.apiUrl, payload, {
70
74
  headers,
71
- timeout: payloadTimeout + CRW_TIMEOUT_BUFFER
75
+ timeout: payloadTimeout + CRW_TIMEOUT_BUFFER,
76
+ httpAgent: this.httpAgent,
77
+ httpsAgent: this.httpsAgent
72
78
  })).data)];
73
79
  } catch (error) {
74
80
  return [url, {
@@ -1 +1 @@
1
- {"version":3,"file":"crw-scraper.mjs","names":[],"sources":["../../../../src/tools/search/crw-scraper.ts"],"sourcesContent":["import axios from 'axios';\nimport type * as t from './types';\nimport { createDefaultLogger } from './utils';\nimport { processContent } from './content';\n\n/** HTTP headroom over the payload render budget: fastCRW's queue/verification\n * overhead is not counted against `timeout`, so the client must wait longer. */\nconst CRW_TIMEOUT_BUFFER = 5000;\n\n/**\n * fastCRW scraper. Firecrawl-compatible web scraper; single binary;\n * self-host or cloud. Posts to {base}/v1/scrape.\n */\nexport class CrwScraper implements t.BaseScraper {\n private apiKey: string;\n private apiUrl: string;\n private defaultFormats: string[];\n private timeout: number;\n private logger: t.Logger;\n private onlyMainContent?: boolean;\n private includeTags?: string[];\n private excludeTags?: string[];\n private waitFor?: number;\n private headers?: Record<string, string>;\n private renderJs?: boolean | null;\n private cssSelector?: string;\n private xpath?: string;\n private proxy?: string;\n private stealth?: boolean;\n\n constructor(config: t.CrwScraperConfig = {}) {\n this.apiKey = config.apiKey ?? process.env.CRW_API_KEY ?? '';\n\n const baseUrl =\n config.apiUrl ?? process.env.CRW_API_URL ?? 'https://api.fastcrw.com';\n this.apiUrl = `${baseUrl.replace(/\\/+$/, '')}/v1/scrape`;\n\n this.defaultFormats = config.formats ?? ['markdown', 'html'];\n this.timeout = config.timeout ?? 7500;\n this.logger = config.logger || createDefaultLogger();\n\n this.onlyMainContent = config.onlyMainContent;\n this.includeTags = config.includeTags;\n this.excludeTags = config.excludeTags;\n this.waitFor = config.waitFor;\n this.headers = config.headers;\n this.renderJs = config.renderJs;\n this.cssSelector = config.cssSelector;\n this.xpath = config.xpath;\n this.proxy = config.proxy;\n this.stealth = config.stealth;\n\n // Self-host fastCRW may run without auth, so a missing key is only a\n // warning — unlike Firecrawl/Tavily, scrapeUrl does NOT early-return on it.\n if (!this.apiKey) {\n this.logger.warn('CRW_API_KEY is not set. Scraping will not work.');\n }\n this.logger.debug(`CRW scraper initialized with API URL: ${this.apiUrl}`);\n }\n\n async scrapeUrl(\n url: string,\n options: t.CrwScrapeOptions = {}\n ): Promise<[string, t.CrwScrapeResponse]> {\n try {\n const payloadTimeout = options.timeout ?? this.timeout;\n const payload = omitUndefined({\n url,\n formats: options.formats ?? this.defaultFormats,\n onlyMainContent: options.onlyMainContent ?? this.onlyMainContent,\n includeTags: options.includeTags ?? this.includeTags,\n excludeTags: options.excludeTags ?? this.excludeTags,\n waitFor: options.waitFor ?? this.waitFor,\n headers: options.headers ?? this.headers,\n renderJs: options.renderJs ?? this.renderJs,\n cssSelector: options.cssSelector ?? this.cssSelector,\n xpath: options.xpath ?? this.xpath,\n proxy: options.proxy ?? this.proxy,\n stealth: options.stealth ?? this.stealth,\n // Cloud honors `timeout` (live-verified); the published OpenAPI\n // documents `deadlineMs` (1..60000) instead. Send both.\n timeout: payloadTimeout,\n deadlineMs: Math.max(1, Math.min(payloadTimeout, 60000)),\n });\n\n const headers: Record<string, string> = {\n 'Content-Type': 'application/json',\n };\n if (this.apiKey) {\n headers.Authorization = `Bearer ${this.apiKey}`;\n }\n\n const response = await axios.post<t.CrwRawScrapeResponse>(\n this.apiUrl,\n payload,\n {\n headers,\n timeout: payloadTimeout + CRW_TIMEOUT_BUFFER,\n }\n );\n\n return [url, normalizeCrwResponse(response.data)];\n } catch (error) {\n const errorMessage =\n error instanceof Error ? error.message : String(error);\n return [\n url,\n {\n success: false,\n error: `fastCRW API request failed: ${errorMessage}`,\n },\n ];\n }\n }\n\n /**\n * Extract content from scrape response. Mirrors FirecrawlScraper — reads\n * response.data.*, which normalizeCrwResponse guarantees, preserving the\n * processContent ref-markers used by the reranker. Parameter is typed as the\n * NARROWER t.CrwScrapeResponse, exactly like TavilyScraper/FirecrawlScraper:\n * TS class-method bivariance accepts this against BaseScraper's\n * AnyScraperResponse, AND it lets us read response.data.plainText (a\n * CrwScrapeResponse-only field) with no cast.\n */\n extractContent(\n response: t.CrwScrapeResponse\n ): [string, undefined | t.References] {\n if (!response.success || !response.data) {\n return ['', undefined];\n }\n\n const htmlSource = response.data.html ?? response.data.rawHtml;\n if (response.data.markdown != null && htmlSource != null) {\n try {\n const { markdown, ...rest } = processContent(\n htmlSource,\n response.data.markdown\n );\n return [markdown, rest];\n } catch (error) {\n this.logger.error('Error processing content:', error);\n return [response.data.markdown, undefined];\n }\n } else if (response.data.markdown != null) {\n return [response.data.markdown, undefined];\n }\n\n // Fall back to HTML content\n if (response.data.html != null) {\n return [response.data.html, undefined];\n }\n\n // Fall back to raw HTML content\n if (response.data.rawHtml != null) {\n return [response.data.rawHtml, undefined];\n }\n\n // CRW-only fallback (no Firecrawl equivalent): plain-text body.\n if (response.data.plainText != null) {\n return [response.data.plainText, undefined];\n }\n\n return ['', undefined];\n }\n\n extractMetadata(response: t.CrwScrapeResponse): t.ScrapeMetadata {\n if (!response.success || !response.data || !response.data.metadata) {\n return {};\n }\n\n return response.data.metadata;\n }\n}\n\n/**\n * Create a fastCRW scraper instance\n * @param config Scraper configuration\n * @returns fastCRW scraper instance\n */\nexport const createCrwScraper = (config: t.CrwScraperConfig = {}): CrwScraper =>\n new CrwScraper(config);\n\n/**\n * fastCRW cloud nests scrape fields under `data` ({success, data: {markdown,\n * ...}}, live-verified 2026-07-02), matching Firecrawl. Prefer the nested\n * container and fall back to top-level fields for self-host/legacy responses.\n */\nfunction normalizeCrwResponse(\n raw: t.CrwRawScrapeResponse | null | undefined\n): t.CrwScrapeResponse {\n if (raw == null) {\n return { success: false, error: 'Empty fastCRW response' };\n }\n if (raw.success === false) {\n return {\n success: false,\n error:\n raw.error_code != null\n ? `[${raw.error_code}] ${raw.error ?? 'Unknown error'}`\n : (raw.error ?? 'fastCRW scrape failed'),\n error_code: raw.error_code,\n };\n }\n const data = raw.data ?? raw;\n return {\n success: true,\n data: {\n // Strip inline base64 image payloads from text content (not `screenshot`,\n // which is base64 by design and never reaches the content processor).\n markdown: stripBase64DataUris(data.markdown),\n html: stripBase64DataUris(data.html),\n rawHtml: stripBase64DataUris(data.rawHtml),\n plainText: stripBase64DataUris(data.plainText),\n screenshot: data.screenshot,\n links: data.links,\n metadata: data.metadata,\n },\n };\n}\n\n/**\n * Replace inline base64 data-URI payloads (typically images) in scraped text\n * with a short placeholder. Such payloads can be hundreds of KB; the content\n * processor builds a per-link RegExp from each URL, and one that large overflows\n * the engine's pattern-size limit (\"Invalid regular expression\"). Firecrawl\n * drops them via removeBase64Images — mirror that so image-heavy pages stay\n * processable (and don't bloat the LLM context with base64 noise).\n */\nfunction stripBase64DataUris(text: string | undefined): string | undefined {\n if (text == null) {\n return text;\n }\n return text.replace(\n /data:[\\w.+-]+\\/[\\w.+-]+;base64,[A-Za-z0-9+/=]+/g,\n 'data:base64-content-removed'\n );\n}\n\n// Helper function to clean up payload for fastCRW\nfunction omitUndefined<T extends object>(obj: T): Partial<T> {\n return Object.fromEntries(\n Object.entries(obj).filter(([, v]) => v !== undefined)\n ) as Partial<T>;\n}\n"],"mappings":";;;;;;AAOA,MAAM,qBAAqB;;;;;AAM3B,IAAa,aAAb,MAAiD;CAC/C;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CAEA,YAAY,SAA6B,CAAC,GAAG;EAC3C,KAAK,SAAS,OAAO,UAAU,QAAQ,IAAI,eAAe;EAE1D,MAAM,UACJ,OAAO,UAAU,QAAQ,IAAI,eAAe;EAC9C,KAAK,SAAS,GAAG,QAAQ,QAAQ,QAAQ,EAAE,EAAE;EAE7C,KAAK,iBAAiB,OAAO,WAAW,CAAC,YAAY,MAAM;EAC3D,KAAK,UAAU,OAAO,WAAW;EACjC,KAAK,SAAS,OAAO,UAAU,oBAAoB;EAEnD,KAAK,kBAAkB,OAAO;EAC9B,KAAK,cAAc,OAAO;EAC1B,KAAK,cAAc,OAAO;EAC1B,KAAK,UAAU,OAAO;EACtB,KAAK,UAAU,OAAO;EACtB,KAAK,WAAW,OAAO;EACvB,KAAK,cAAc,OAAO;EAC1B,KAAK,QAAQ,OAAO;EACpB,KAAK,QAAQ,OAAO;EACpB,KAAK,UAAU,OAAO;EAItB,IAAI,CAAC,KAAK,QACR,KAAK,OAAO,KAAK,iDAAiD;EAEpE,KAAK,OAAO,MAAM,yCAAyC,KAAK,QAAQ;CAC1E;CAEA,MAAM,UACJ,KACA,UAA8B,CAAC,GACS;EACxC,IAAI;GACF,MAAM,iBAAiB,QAAQ,WAAW,KAAK;GAC/C,MAAM,UAAU,cAAc;IAC5B;IACA,SAAS,QAAQ,WAAW,KAAK;IACjC,iBAAiB,QAAQ,mBAAmB,KAAK;IACjD,aAAa,QAAQ,eAAe,KAAK;IACzC,aAAa,QAAQ,eAAe,KAAK;IACzC,SAAS,QAAQ,WAAW,KAAK;IACjC,SAAS,QAAQ,WAAW,KAAK;IACjC,UAAU,QAAQ,YAAY,KAAK;IACnC,aAAa,QAAQ,eAAe,KAAK;IACzC,OAAO,QAAQ,SAAS,KAAK;IAC7B,OAAO,QAAQ,SAAS,KAAK;IAC7B,SAAS,QAAQ,WAAW,KAAK;IAGjC,SAAS;IACT,YAAY,KAAK,IAAI,GAAG,KAAK,IAAI,gBAAgB,GAAK,CAAC;GACzD,CAAC;GAED,MAAM,UAAkC,EACtC,gBAAgB,mBAClB;GACA,IAAI,KAAK,QACP,QAAQ,gBAAgB,UAAU,KAAK;GAYzC,OAAO,CAAC,KAAK,sBAAqB,MATX,MAAM,KAC3B,KAAK,QACL,SACA;IACE;IACA,SAAS,iBAAiB;GAC5B,CACF,EAAA,CAE2C,IAAI,CAAC;EAClD,SAAS,OAAO;GAGd,OAAO,CACL,KACA;IACE,SAAS;IACT,OAAO,+BALT,iBAAiB,QAAQ,MAAM,UAAU,OAAO,KAAK;GAMrD,CACF;EACF;CACF;;;;;;;;;;CAWA,eACE,UACoC;EACpC,IAAI,CAAC,SAAS,WAAW,CAAC,SAAS,MACjC,OAAO,CAAC,IAAI,KAAA,CAAS;EAGvB,MAAM,aAAa,SAAS,KAAK,QAAQ,SAAS,KAAK;EACvD,IAAI,SAAS,KAAK,YAAY,QAAQ,cAAc,MAClD,IAAI;GACF,MAAM,EAAE,UAAU,GAAG,SAAS,eAC5B,YACA,SAAS,KAAK,QAChB;GACA,OAAO,CAAC,UAAU,IAAI;EACxB,SAAS,OAAO;GACd,KAAK,OAAO,MAAM,6BAA6B,KAAK;GACpD,OAAO,CAAC,SAAS,KAAK,UAAU,KAAA,CAAS;EAC3C;OACK,IAAI,SAAS,KAAK,YAAY,MACnC,OAAO,CAAC,SAAS,KAAK,UAAU,KAAA,CAAS;EAI3C,IAAI,SAAS,KAAK,QAAQ,MACxB,OAAO,CAAC,SAAS,KAAK,MAAM,KAAA,CAAS;EAIvC,IAAI,SAAS,KAAK,WAAW,MAC3B,OAAO,CAAC,SAAS,KAAK,SAAS,KAAA,CAAS;EAI1C,IAAI,SAAS,KAAK,aAAa,MAC7B,OAAO,CAAC,SAAS,KAAK,WAAW,KAAA,CAAS;EAG5C,OAAO,CAAC,IAAI,KAAA,CAAS;CACvB;CAEA,gBAAgB,UAAiD;EAC/D,IAAI,CAAC,SAAS,WAAW,CAAC,SAAS,QAAQ,CAAC,SAAS,KAAK,UACxD,OAAO,CAAC;EAGV,OAAO,SAAS,KAAK;CACvB;AACF;;;;;;AAOA,MAAa,oBAAoB,SAA6B,CAAC,MAC7D,IAAI,WAAW,MAAM;;;;;;AAOvB,SAAS,qBACP,KACqB;CACrB,IAAI,OAAO,MACT,OAAO;EAAE,SAAS;EAAO,OAAO;CAAyB;CAE3D,IAAI,IAAI,YAAY,OAClB,OAAO;EACL,SAAS;EACT,OACE,IAAI,cAAc,OACd,IAAI,IAAI,WAAW,IAAI,IAAI,SAAS,oBACnC,IAAI,SAAS;EACpB,YAAY,IAAI;CAClB;CAEF,MAAM,OAAO,IAAI,QAAQ;CACzB,OAAO;EACL,SAAS;EACT,MAAM;GAGJ,UAAU,oBAAoB,KAAK,QAAQ;GAC3C,MAAM,oBAAoB,KAAK,IAAI;GACnC,SAAS,oBAAoB,KAAK,OAAO;GACzC,WAAW,oBAAoB,KAAK,SAAS;GAC7C,YAAY,KAAK;GACjB,OAAO,KAAK;GACZ,UAAU,KAAK;EACjB;CACF;AACF;;;;;;;;;AAUA,SAAS,oBAAoB,MAA8C;CACzE,IAAI,QAAQ,MACV,OAAO;CAET,OAAO,KAAK,QACV,mDACA,6BACF;AACF;AAGA,SAAS,cAAgC,KAAoB;CAC3D,OAAO,OAAO,YACZ,OAAO,QAAQ,GAAG,CAAC,CAAC,QAAQ,GAAG,OAAO,MAAM,KAAA,CAAS,CACvD;AACF"}
1
+ {"version":3,"file":"crw-scraper.mjs","names":[],"sources":["../../../../src/tools/search/crw-scraper.ts"],"sourcesContent":["import axios from 'axios';\nimport type * as t from './types';\nimport { createDefaultLogger } from './utils';\nimport { processContent } from './content';\n\n/** HTTP headroom over the payload render budget: fastCRW's queue/verification\n * overhead is not counted against `timeout`, so the client must wait longer. */\nconst CRW_TIMEOUT_BUFFER = 5000;\n\n/**\n * fastCRW scraper. Firecrawl-compatible web scraper; single binary;\n * self-host or cloud. Posts to {base}/v1/scrape.\n */\nexport class CrwScraper implements t.BaseScraper {\n private apiKey: string;\n private apiUrl: string;\n private defaultFormats: string[];\n private timeout: number;\n private logger: t.Logger;\n private onlyMainContent?: boolean;\n private includeTags?: string[];\n private excludeTags?: string[];\n private waitFor?: number;\n private headers?: Record<string, string>;\n private renderJs?: boolean | null;\n private cssSelector?: string;\n private xpath?: string;\n private proxy?: string;\n private stealth?: boolean;\n private httpAgent?: t.HttpAgent;\n private httpsAgent?: t.HttpsAgent;\n\n constructor(config: t.CrwScraperConfig = {}) {\n this.apiKey = config.apiKey ?? process.env.CRW_API_KEY ?? '';\n\n const baseUrl =\n config.apiUrl ?? process.env.CRW_API_URL ?? 'https://api.fastcrw.com';\n this.apiUrl = `${baseUrl.replace(/\\/+$/, '')}/v1/scrape`;\n\n this.defaultFormats = config.formats ?? ['markdown', 'html'];\n this.timeout = config.timeout ?? 7500;\n this.logger = config.logger || createDefaultLogger();\n\n this.onlyMainContent = config.onlyMainContent;\n this.includeTags = config.includeTags;\n this.excludeTags = config.excludeTags;\n this.waitFor = config.waitFor;\n this.headers = config.headers;\n this.renderJs = config.renderJs;\n this.cssSelector = config.cssSelector;\n this.xpath = config.xpath;\n this.proxy = config.proxy;\n this.stealth = config.stealth;\n this.httpAgent = config.httpAgent;\n this.httpsAgent = config.httpsAgent;\n\n // Self-host fastCRW may run without auth, so a missing key is only a\n // warning — unlike Firecrawl/Tavily, scrapeUrl does NOT early-return on it.\n if (!this.apiKey) {\n this.logger.warn('CRW_API_KEY is not set. Scraping will not work.');\n }\n this.logger.debug(`CRW scraper initialized with API URL: ${this.apiUrl}`);\n }\n\n async scrapeUrl(\n url: string,\n options: t.CrwScrapeOptions = {}\n ): Promise<[string, t.CrwScrapeResponse]> {\n try {\n const payloadTimeout = options.timeout ?? this.timeout;\n const payload = omitUndefined({\n url,\n formats: options.formats ?? this.defaultFormats,\n onlyMainContent: options.onlyMainContent ?? this.onlyMainContent,\n includeTags: options.includeTags ?? this.includeTags,\n excludeTags: options.excludeTags ?? this.excludeTags,\n waitFor: options.waitFor ?? this.waitFor,\n headers: options.headers ?? this.headers,\n renderJs: options.renderJs ?? this.renderJs,\n cssSelector: options.cssSelector ?? this.cssSelector,\n xpath: options.xpath ?? this.xpath,\n proxy: options.proxy ?? this.proxy,\n stealth: options.stealth ?? this.stealth,\n // Cloud honors `timeout` (live-verified); the published OpenAPI\n // documents `deadlineMs` (1..60000) instead. Send both.\n timeout: payloadTimeout,\n deadlineMs: Math.max(1, Math.min(payloadTimeout, 60000)),\n });\n\n const headers: Record<string, string> = {\n 'Content-Type': 'application/json',\n };\n if (this.apiKey) {\n headers.Authorization = `Bearer ${this.apiKey}`;\n }\n\n const response = await axios.post<t.CrwRawScrapeResponse>(\n this.apiUrl,\n payload,\n {\n headers,\n timeout: payloadTimeout + CRW_TIMEOUT_BUFFER,\n httpAgent: this.httpAgent,\n httpsAgent: this.httpsAgent,\n }\n );\n\n return [url, normalizeCrwResponse(response.data)];\n } catch (error) {\n const errorMessage =\n error instanceof Error ? error.message : String(error);\n return [\n url,\n {\n success: false,\n error: `fastCRW API request failed: ${errorMessage}`,\n },\n ];\n }\n }\n\n /**\n * Extract content from scrape response. Mirrors FirecrawlScraper — reads\n * response.data.*, which normalizeCrwResponse guarantees, preserving the\n * processContent ref-markers used by the reranker. Parameter is typed as the\n * NARROWER t.CrwScrapeResponse, exactly like TavilyScraper/FirecrawlScraper:\n * TS class-method bivariance accepts this against BaseScraper's\n * AnyScraperResponse, AND it lets us read response.data.plainText (a\n * CrwScrapeResponse-only field) with no cast.\n */\n extractContent(\n response: t.CrwScrapeResponse\n ): [string, undefined | t.References] {\n if (!response.success || !response.data) {\n return ['', undefined];\n }\n\n const htmlSource = response.data.html ?? response.data.rawHtml;\n if (response.data.markdown != null && htmlSource != null) {\n try {\n const { markdown, ...rest } = processContent(\n htmlSource,\n response.data.markdown\n );\n return [markdown, rest];\n } catch (error) {\n this.logger.error('Error processing content:', error);\n return [response.data.markdown, undefined];\n }\n } else if (response.data.markdown != null) {\n return [response.data.markdown, undefined];\n }\n\n // Fall back to HTML content\n if (response.data.html != null) {\n return [response.data.html, undefined];\n }\n\n // Fall back to raw HTML content\n if (response.data.rawHtml != null) {\n return [response.data.rawHtml, undefined];\n }\n\n // CRW-only fallback (no Firecrawl equivalent): plain-text body.\n if (response.data.plainText != null) {\n return [response.data.plainText, undefined];\n }\n\n return ['', undefined];\n }\n\n extractMetadata(response: t.CrwScrapeResponse): t.ScrapeMetadata {\n if (!response.success || !response.data || !response.data.metadata) {\n return {};\n }\n\n return response.data.metadata;\n }\n}\n\n/**\n * Create a fastCRW scraper instance\n * @param config Scraper configuration\n * @returns fastCRW scraper instance\n */\nexport const createCrwScraper = (config: t.CrwScraperConfig = {}): CrwScraper =>\n new CrwScraper(config);\n\n/**\n * fastCRW cloud nests scrape fields under `data` ({success, data: {markdown,\n * ...}}, live-verified 2026-07-02), matching Firecrawl. Prefer the nested\n * container and fall back to top-level fields for self-host/legacy responses.\n */\nfunction normalizeCrwResponse(\n raw: t.CrwRawScrapeResponse | null | undefined\n): t.CrwScrapeResponse {\n if (raw == null) {\n return { success: false, error: 'Empty fastCRW response' };\n }\n if (raw.success === false) {\n return {\n success: false,\n error:\n raw.error_code != null\n ? `[${raw.error_code}] ${raw.error ?? 'Unknown error'}`\n : (raw.error ?? 'fastCRW scrape failed'),\n error_code: raw.error_code,\n };\n }\n const data = raw.data ?? raw;\n return {\n success: true,\n data: {\n // Strip inline base64 image payloads from text content (not `screenshot`,\n // which is base64 by design and never reaches the content processor).\n markdown: stripBase64DataUris(data.markdown),\n html: stripBase64DataUris(data.html),\n rawHtml: stripBase64DataUris(data.rawHtml),\n plainText: stripBase64DataUris(data.plainText),\n screenshot: data.screenshot,\n links: data.links,\n metadata: data.metadata,\n },\n };\n}\n\n/**\n * Replace inline base64 data-URI payloads (typically images) in scraped text\n * with a short placeholder. Such payloads can be hundreds of KB; the content\n * processor builds a per-link RegExp from each URL, and one that large overflows\n * the engine's pattern-size limit (\"Invalid regular expression\"). Firecrawl\n * drops them via removeBase64Images — mirror that so image-heavy pages stay\n * processable (and don't bloat the LLM context with base64 noise).\n */\nfunction stripBase64DataUris(text: string | undefined): string | undefined {\n if (text == null) {\n return text;\n }\n return text.replace(\n /data:[\\w.+-]+\\/[\\w.+-]+;base64,[A-Za-z0-9+/=]+/g,\n 'data:base64-content-removed'\n );\n}\n\n// Helper function to clean up payload for fastCRW\nfunction omitUndefined<T extends object>(obj: T): Partial<T> {\n return Object.fromEntries(\n Object.entries(obj).filter(([, v]) => v !== undefined)\n ) as Partial<T>;\n}\n"],"mappings":";;;;;;AAOA,MAAM,qBAAqB;;;;;AAM3B,IAAa,aAAb,MAAiD;CAC/C;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CAEA,YAAY,SAA6B,CAAC,GAAG;EAC3C,KAAK,SAAS,OAAO,UAAU,QAAQ,IAAI,eAAe;EAE1D,MAAM,UACJ,OAAO,UAAU,QAAQ,IAAI,eAAe;EAC9C,KAAK,SAAS,GAAG,QAAQ,QAAQ,QAAQ,EAAE,EAAE;EAE7C,KAAK,iBAAiB,OAAO,WAAW,CAAC,YAAY,MAAM;EAC3D,KAAK,UAAU,OAAO,WAAW;EACjC,KAAK,SAAS,OAAO,UAAU,oBAAoB;EAEnD,KAAK,kBAAkB,OAAO;EAC9B,KAAK,cAAc,OAAO;EAC1B,KAAK,cAAc,OAAO;EAC1B,KAAK,UAAU,OAAO;EACtB,KAAK,UAAU,OAAO;EACtB,KAAK,WAAW,OAAO;EACvB,KAAK,cAAc,OAAO;EAC1B,KAAK,QAAQ,OAAO;EACpB,KAAK,QAAQ,OAAO;EACpB,KAAK,UAAU,OAAO;EACtB,KAAK,YAAY,OAAO;EACxB,KAAK,aAAa,OAAO;EAIzB,IAAI,CAAC,KAAK,QACR,KAAK,OAAO,KAAK,iDAAiD;EAEpE,KAAK,OAAO,MAAM,yCAAyC,KAAK,QAAQ;CAC1E;CAEA,MAAM,UACJ,KACA,UAA8B,CAAC,GACS;EACxC,IAAI;GACF,MAAM,iBAAiB,QAAQ,WAAW,KAAK;GAC/C,MAAM,UAAU,cAAc;IAC5B;IACA,SAAS,QAAQ,WAAW,KAAK;IACjC,iBAAiB,QAAQ,mBAAmB,KAAK;IACjD,aAAa,QAAQ,eAAe,KAAK;IACzC,aAAa,QAAQ,eAAe,KAAK;IACzC,SAAS,QAAQ,WAAW,KAAK;IACjC,SAAS,QAAQ,WAAW,KAAK;IACjC,UAAU,QAAQ,YAAY,KAAK;IACnC,aAAa,QAAQ,eAAe,KAAK;IACzC,OAAO,QAAQ,SAAS,KAAK;IAC7B,OAAO,QAAQ,SAAS,KAAK;IAC7B,SAAS,QAAQ,WAAW,KAAK;IAGjC,SAAS;IACT,YAAY,KAAK,IAAI,GAAG,KAAK,IAAI,gBAAgB,GAAK,CAAC;GACzD,CAAC;GAED,MAAM,UAAkC,EACtC,gBAAgB,mBAClB;GACA,IAAI,KAAK,QACP,QAAQ,gBAAgB,UAAU,KAAK;GAczC,OAAO,CAAC,KAAK,sBAAqB,MAXX,MAAM,KAC3B,KAAK,QACL,SACA;IACE;IACA,SAAS,iBAAiB;IAC1B,WAAW,KAAK;IAChB,YAAY,KAAK;GACnB,CACF,EAAA,CAE2C,IAAI,CAAC;EAClD,SAAS,OAAO;GAGd,OAAO,CACL,KACA;IACE,SAAS;IACT,OAAO,+BALT,iBAAiB,QAAQ,MAAM,UAAU,OAAO,KAAK;GAMrD,CACF;EACF;CACF;;;;;;;;;;CAWA,eACE,UACoC;EACpC,IAAI,CAAC,SAAS,WAAW,CAAC,SAAS,MACjC,OAAO,CAAC,IAAI,KAAA,CAAS;EAGvB,MAAM,aAAa,SAAS,KAAK,QAAQ,SAAS,KAAK;EACvD,IAAI,SAAS,KAAK,YAAY,QAAQ,cAAc,MAClD,IAAI;GACF,MAAM,EAAE,UAAU,GAAG,SAAS,eAC5B,YACA,SAAS,KAAK,QAChB;GACA,OAAO,CAAC,UAAU,IAAI;EACxB,SAAS,OAAO;GACd,KAAK,OAAO,MAAM,6BAA6B,KAAK;GACpD,OAAO,CAAC,SAAS,KAAK,UAAU,KAAA,CAAS;EAC3C;OACK,IAAI,SAAS,KAAK,YAAY,MACnC,OAAO,CAAC,SAAS,KAAK,UAAU,KAAA,CAAS;EAI3C,IAAI,SAAS,KAAK,QAAQ,MACxB,OAAO,CAAC,SAAS,KAAK,MAAM,KAAA,CAAS;EAIvC,IAAI,SAAS,KAAK,WAAW,MAC3B,OAAO,CAAC,SAAS,KAAK,SAAS,KAAA,CAAS;EAI1C,IAAI,SAAS,KAAK,aAAa,MAC7B,OAAO,CAAC,SAAS,KAAK,WAAW,KAAA,CAAS;EAG5C,OAAO,CAAC,IAAI,KAAA,CAAS;CACvB;CAEA,gBAAgB,UAAiD;EAC/D,IAAI,CAAC,SAAS,WAAW,CAAC,SAAS,QAAQ,CAAC,SAAS,KAAK,UACxD,OAAO,CAAC;EAGV,OAAO,SAAS,KAAK;CACvB;AACF;;;;;;AAOA,MAAa,oBAAoB,SAA6B,CAAC,MAC7D,IAAI,WAAW,MAAM;;;;;;AAOvB,SAAS,qBACP,KACqB;CACrB,IAAI,OAAO,MACT,OAAO;EAAE,SAAS;EAAO,OAAO;CAAyB;CAE3D,IAAI,IAAI,YAAY,OAClB,OAAO;EACL,SAAS;EACT,OACE,IAAI,cAAc,OACd,IAAI,IAAI,WAAW,IAAI,IAAI,SAAS,oBACnC,IAAI,SAAS;EACpB,YAAY,IAAI;CAClB;CAEF,MAAM,OAAO,IAAI,QAAQ;CACzB,OAAO;EACL,SAAS;EACT,MAAM;GAGJ,UAAU,oBAAoB,KAAK,QAAQ;GAC3C,MAAM,oBAAoB,KAAK,IAAI;GACnC,SAAS,oBAAoB,KAAK,OAAO;GACzC,WAAW,oBAAoB,KAAK,SAAS;GAC7C,YAAY,KAAK;GACjB,OAAO,KAAK;GACZ,UAAU,KAAK;EACjB;CACF;AACF;;;;;;;;;AAUA,SAAS,oBAAoB,MAA8C;CACzE,IAAI,QAAQ,MACV,OAAO;CAET,OAAO,KAAK,QACV,mDACA,6BACF;AACF;AAGA,SAAS,cAAgC,KAAoB;CAC3D,OAAO,OAAO,YACZ,OAAO,QAAQ,GAAG,CAAC,CAAC,QAAQ,GAAG,OAAO,MAAM,KAAA,CAAS,CACvD;AACF"}
@@ -52,7 +52,9 @@ const createCrwAPI = (apiKey, apiUrl, options) => {
52
52
  if (config.apiKey != null && config.apiKey !== "") headers.Authorization = `Bearer ${config.apiKey}`;
53
53
  const body = (await axios.post(config.apiUrl, payload, {
54
54
  headers,
55
- timeout: config.timeout
55
+ timeout: config.timeout,
56
+ httpAgent: options?.httpAgent,
57
+ httpsAgent: options?.httpsAgent
56
58
  })).data;
57
59
  if (body.success === false) return {
58
60
  success: false,
@@ -1 +1 @@
1
- {"version":3,"file":"crw-search.mjs","names":[],"sources":["../../../../src/tools/search/crw-search.ts"],"sourcesContent":["import axios from 'axios';\nimport type * as t from './types';\n\nconst DEFAULT_CRW_TIMEOUT = 15000;\n\nconst getHostname = (link: string): string => {\n try {\n return new URL(link).hostname;\n } catch {\n return link;\n }\n};\n\n/** The published OpenAPI (docs.fastcrw.com) documents `data` as a flat\n * SearchResult[] with a `category` field; route rows into source groups. */\nconst splitByCategory = (rows: t.CrwSearchResult[]): t.CrwSearchGroups => {\n const groups: Required<t.CrwSearchGroups> = { web: [], images: [], news: [] };\n for (const row of rows) {\n if (row.category === 'news') {\n groups.news.push(row);\n } else if (row.category === 'images') {\n groups.images.push({ ...row, imageUrl: row.url });\n } else {\n groups.web.push(row);\n }\n }\n return groups;\n};\n\nexport const createCrwAPI = (\n apiKey?: string,\n apiUrl?: string,\n options?: t.CrwSearchOptions\n): {\n getSources: (params: t.GetSourcesParams) => Promise<t.SearchResult>;\n} => {\n // NOTE: fastCRW /v1/search is cloud-only, but we still allow a base-URL\n // override for parity with the scraper. Self-host may have no auth, so —\n // unlike Tavily (createTavilyAPI throws on missing key) — we do NOT throw\n // here; we only attach the Authorization header when a key is present.\n const base = (\n apiUrl ??\n process.env.CRW_API_URL ??\n 'https://api.fastcrw.com'\n ).replace(/\\/+$/, '');\n const config = {\n apiKey: apiKey ?? process.env.CRW_API_KEY,\n apiUrl: `${base}/v1/search`,\n timeout: options?.timeout ?? DEFAULT_CRW_TIMEOUT,\n };\n\n const getSources = async ({\n query,\n date,\n numResults = 8,\n type,\n }: t.GetSourcesParams): Promise<t.SearchResult> => {\n if (!query.trim()) {\n return { success: false, error: 'Query cannot be empty' };\n }\n\n try {\n const limit = Math.min(\n Math.max(1, options?.maxResults ?? numResults),\n 20\n );\n // Mirror Serper's verticals: image/news requests hit only their native\n // fastCRW source; plain web optionally adds images via includeImages.\n let sources: t.CrwSearchSource[];\n if (type === 'images') {\n sources = ['images'];\n } else if (type === 'news') {\n sources = ['news'];\n } else {\n sources = options?.includeImages === true ? ['web', 'images'] : ['web'];\n }\n\n const payload: t.CrwSearchPayload = { query, limit, sources };\n if (date != null) {\n // Serper-style qdr filter; live-verified to constrain results.\n payload.tbs = `qdr:${date}`;\n }\n\n const headers: Record<string, string> = {\n 'Content-Type': 'application/json',\n };\n if (config.apiKey != null && config.apiKey !== '') {\n headers.Authorization = `Bearer ${config.apiKey}`;\n }\n\n const response = await axios.post<t.CrwSearchResponse>(\n config.apiUrl,\n payload,\n { headers, timeout: config.timeout }\n );\n\n const body = response.data;\n if (body.success === false) {\n return {\n success: false,\n error: `fastCRW search failed: ${\n body.error_code != null ? `[${body.error_code}] ` : ''\n }${body.error ?? 'Unknown error'}`,\n };\n }\n\n // fastCRW cloud keys results by source: {data: {web: [...], images:\n // [...], news: [...]}} (live-verified 2026-07-02); the published\n // OpenAPI documents a flat SearchResult[] instead (self-host), and a\n // {data: {results: {...groups}}} wrapper also exists. Accept all three.\n // OrganicResult.link is a REQUIRED string (types.ts), so defend against\n // null/empty urls to never feed a broken '' link into the scraper.\n const container = body.data ?? {};\n const data: t.CrwSearchGroups = Array.isArray(container)\n ? splitByCategory(container)\n : (container.results ?? container);\n const organicResults: t.OrganicResult[] = (data.web ?? [])\n .filter((r) => r.url != null && r.url !== '')\n .map((r) => ({\n title: r.title ?? '',\n link: r.url as string,\n snippet: r.description ?? r.snippet ?? '',\n position: r.position,\n }));\n\n const imageResults: t.ImageResult[] = (data.images ?? [])\n .filter((r) => r.imageUrl != null && r.imageUrl !== '')\n .map((r) => ({\n title: r.title,\n imageUrl: r.imageUrl,\n link: r.url,\n position: r.position,\n }));\n\n const newsResults: t.NewsResult[] = (data.news ?? [])\n .filter((r) => r.url != null && r.url !== '')\n .map((r) => ({\n title: r.title ?? '',\n link: r.url as string,\n snippet: r.description ?? r.snippet ?? '',\n date: r.publishedDate,\n source: getHostname(r.url as string),\n position: r.position,\n }));\n\n const results: t.SearchResultData = {\n organic: organicResults,\n images: imageResults,\n topStories: [],\n videos: [],\n news: newsResults,\n relatedSearches: [],\n };\n\n return { success: true, data: results };\n } catch (error) {\n const errorMessage =\n error instanceof Error ? error.message : String(error);\n return {\n success: false,\n error: `fastCRW search request failed: ${errorMessage}`,\n };\n }\n };\n\n return { getSources };\n};\n"],"mappings":";;AAGA,MAAM,sBAAsB;AAE5B,MAAM,eAAe,SAAyB;CAC5C,IAAI;EACF,OAAO,IAAI,IAAI,IAAI,CAAC,CAAC;CACvB,QAAQ;EACN,OAAO;CACT;AACF;;;AAIA,MAAM,mBAAmB,SAAiD;CACxE,MAAM,SAAsC;EAAE,KAAK,CAAC;EAAG,QAAQ,CAAC;EAAG,MAAM,CAAC;CAAE;CAC5E,KAAK,MAAM,OAAO,MAChB,IAAI,IAAI,aAAa,QACnB,OAAO,KAAK,KAAK,GAAG;MACf,IAAI,IAAI,aAAa,UAC1B,OAAO,OAAO,KAAK;EAAE,GAAG;EAAK,UAAU,IAAI;CAAI,CAAC;MAEhD,OAAO,IAAI,KAAK,GAAG;CAGvB,OAAO;AACT;AAEA,MAAa,gBACX,QACA,QACA,YAGG;CAKH,MAAM,QACJ,UACA,QAAQ,IAAI,eACZ,0BAAA,CACA,QAAQ,QAAQ,EAAE;CACpB,MAAM,SAAS;EACb,QAAQ,UAAU,QAAQ,IAAI;EAC9B,QAAQ,GAAG,KAAK;EAChB,SAAS,SAAS,WAAW;CAC/B;CAEA,MAAM,aAAa,OAAO,EACxB,OACA,MACA,aAAa,GACb,WACiD;EACjD,IAAI,CAAC,MAAM,KAAK,GACd,OAAO;GAAE,SAAS;GAAO,OAAO;EAAwB;EAG1D,IAAI;GACF,MAAM,QAAQ,KAAK,IACjB,KAAK,IAAI,GAAG,SAAS,cAAc,UAAU,GAC7C,EACF;GAGA,IAAI;GACJ,IAAI,SAAS,UACX,UAAU,CAAC,QAAQ;QACd,IAAI,SAAS,QAClB,UAAU,CAAC,MAAM;QAEjB,UAAU,SAAS,kBAAkB,OAAO,CAAC,OAAO,QAAQ,IAAI,CAAC,KAAK;GAGxE,MAAM,UAA8B;IAAE;IAAO;IAAO;GAAQ;GAC5D,IAAI,QAAQ,MAEV,QAAQ,MAAM,OAAO;GAGvB,MAAM,UAAkC,EACtC,gBAAgB,mBAClB;GACA,IAAI,OAAO,UAAU,QAAQ,OAAO,WAAW,IAC7C,QAAQ,gBAAgB,UAAU,OAAO;GAS3C,MAAM,QAAO,MANU,MAAM,KAC3B,OAAO,QACP,SACA;IAAE;IAAS,SAAS,OAAO;GAAQ,CACrC,EAAA,CAEsB;GACtB,IAAI,KAAK,YAAY,OACnB,OAAO;IACL,SAAS;IACT,OAAO,0BACL,KAAK,cAAc,OAAO,IAAI,KAAK,WAAW,MAAM,KACnD,KAAK,SAAS;GACnB;GASF,MAAM,YAAY,KAAK,QAAQ,CAAC;GAChC,MAAM,OAA0B,MAAM,QAAQ,SAAS,IACnD,gBAAgB,SAAS,IACxB,UAAU,WAAW;GAuC1B,OAAO;IAAE,SAAS;IAAM,MAAM;KAR5B,UA9ByC,KAAK,OAAO,CAAC,EAAA,CACrD,QAAQ,MAAM,EAAE,OAAO,QAAQ,EAAE,QAAQ,EAAE,CAAC,CAC5C,KAAK,OAAO;MACX,OAAO,EAAE,SAAS;MAClB,MAAM,EAAE;MACR,SAAS,EAAE,eAAe,EAAE,WAAW;MACvC,UAAU,EAAE;KACd,EAuBsB;KACtB,SAtBqC,KAAK,UAAU,CAAC,EAAA,CACpD,QAAQ,MAAM,EAAE,YAAY,QAAQ,EAAE,aAAa,EAAE,CAAC,CACtD,KAAK,OAAO;MACX,OAAO,EAAE;MACT,UAAU,EAAE;MACZ,MAAM,EAAE;MACR,UAAU,EAAE;KACd,EAemB;KACnB,YAAY,CAAC;KACb,QAAQ,CAAC;KACT,OAhBmC,KAAK,QAAQ,CAAC,EAAA,CAChD,QAAQ,MAAM,EAAE,OAAO,QAAQ,EAAE,QAAQ,EAAE,CAAC,CAC5C,KAAK,OAAO;MACX,OAAO,EAAE,SAAS;MAClB,MAAM,EAAE;MACR,SAAS,EAAE,eAAe,EAAE,WAAW;MACvC,MAAM,EAAE;MACR,QAAQ,YAAY,EAAE,GAAa;MACnC,UAAU,EAAE;KACd,EAOgB;KAChB,iBAAiB,CAAC;IAGgB;GAAE;EACxC,SAAS,OAAO;GAGd,OAAO;IACL,SAAS;IACT,OAAO,kCAHP,iBAAiB,QAAQ,MAAM,UAAU,OAAO,KAAK;GAIvD;EACF;CACF;CAEA,OAAO,EAAE,WAAW;AACtB"}
1
+ {"version":3,"file":"crw-search.mjs","names":[],"sources":["../../../../src/tools/search/crw-search.ts"],"sourcesContent":["import axios from 'axios';\nimport type * as t from './types';\n\nconst DEFAULT_CRW_TIMEOUT = 15000;\n\nconst getHostname = (link: string): string => {\n try {\n return new URL(link).hostname;\n } catch {\n return link;\n }\n};\n\n/** The published OpenAPI (docs.fastcrw.com) documents `data` as a flat\n * SearchResult[] with a `category` field; route rows into source groups. */\nconst splitByCategory = (rows: t.CrwSearchResult[]): t.CrwSearchGroups => {\n const groups: Required<t.CrwSearchGroups> = { web: [], images: [], news: [] };\n for (const row of rows) {\n if (row.category === 'news') {\n groups.news.push(row);\n } else if (row.category === 'images') {\n groups.images.push({ ...row, imageUrl: row.url });\n } else {\n groups.web.push(row);\n }\n }\n return groups;\n};\n\nexport const createCrwAPI = (\n apiKey?: string,\n apiUrl?: string,\n options?: t.CrwSearchOptions\n): {\n getSources: (params: t.GetSourcesParams) => Promise<t.SearchResult>;\n} => {\n // NOTE: fastCRW /v1/search is cloud-only, but we still allow a base-URL\n // override for parity with the scraper. Self-host may have no auth, so —\n // unlike Tavily (createTavilyAPI throws on missing key) — we do NOT throw\n // here; we only attach the Authorization header when a key is present.\n const base = (\n apiUrl ??\n process.env.CRW_API_URL ??\n 'https://api.fastcrw.com'\n ).replace(/\\/+$/, '');\n const config = {\n apiKey: apiKey ?? process.env.CRW_API_KEY,\n apiUrl: `${base}/v1/search`,\n timeout: options?.timeout ?? DEFAULT_CRW_TIMEOUT,\n };\n\n const getSources = async ({\n query,\n date,\n numResults = 8,\n type,\n }: t.GetSourcesParams): Promise<t.SearchResult> => {\n if (!query.trim()) {\n return { success: false, error: 'Query cannot be empty' };\n }\n\n try {\n const limit = Math.min(\n Math.max(1, options?.maxResults ?? numResults),\n 20\n );\n // Mirror Serper's verticals: image/news requests hit only their native\n // fastCRW source; plain web optionally adds images via includeImages.\n let sources: t.CrwSearchSource[];\n if (type === 'images') {\n sources = ['images'];\n } else if (type === 'news') {\n sources = ['news'];\n } else {\n sources = options?.includeImages === true ? ['web', 'images'] : ['web'];\n }\n\n const payload: t.CrwSearchPayload = { query, limit, sources };\n if (date != null) {\n // Serper-style qdr filter; live-verified to constrain results.\n payload.tbs = `qdr:${date}`;\n }\n\n const headers: Record<string, string> = {\n 'Content-Type': 'application/json',\n };\n if (config.apiKey != null && config.apiKey !== '') {\n headers.Authorization = `Bearer ${config.apiKey}`;\n }\n\n const response = await axios.post<t.CrwSearchResponse>(\n config.apiUrl,\n payload,\n {\n headers,\n timeout: config.timeout,\n httpAgent: options?.httpAgent,\n httpsAgent: options?.httpsAgent,\n }\n );\n\n const body = response.data;\n if (body.success === false) {\n return {\n success: false,\n error: `fastCRW search failed: ${\n body.error_code != null ? `[${body.error_code}] ` : ''\n }${body.error ?? 'Unknown error'}`,\n };\n }\n\n // fastCRW cloud keys results by source: {data: {web: [...], images:\n // [...], news: [...]}} (live-verified 2026-07-02); the published\n // OpenAPI documents a flat SearchResult[] instead (self-host), and a\n // {data: {results: {...groups}}} wrapper also exists. Accept all three.\n // OrganicResult.link is a REQUIRED string (types.ts), so defend against\n // null/empty urls to never feed a broken '' link into the scraper.\n const container = body.data ?? {};\n const data: t.CrwSearchGroups = Array.isArray(container)\n ? splitByCategory(container)\n : (container.results ?? container);\n const organicResults: t.OrganicResult[] = (data.web ?? [])\n .filter((r) => r.url != null && r.url !== '')\n .map((r) => ({\n title: r.title ?? '',\n link: r.url as string,\n snippet: r.description ?? r.snippet ?? '',\n position: r.position,\n }));\n\n const imageResults: t.ImageResult[] = (data.images ?? [])\n .filter((r) => r.imageUrl != null && r.imageUrl !== '')\n .map((r) => ({\n title: r.title,\n imageUrl: r.imageUrl,\n link: r.url,\n position: r.position,\n }));\n\n const newsResults: t.NewsResult[] = (data.news ?? [])\n .filter((r) => r.url != null && r.url !== '')\n .map((r) => ({\n title: r.title ?? '',\n link: r.url as string,\n snippet: r.description ?? r.snippet ?? '',\n date: r.publishedDate,\n source: getHostname(r.url as string),\n position: r.position,\n }));\n\n const results: t.SearchResultData = {\n organic: organicResults,\n images: imageResults,\n topStories: [],\n videos: [],\n news: newsResults,\n relatedSearches: [],\n };\n\n return { success: true, data: results };\n } catch (error) {\n const errorMessage =\n error instanceof Error ? error.message : String(error);\n return {\n success: false,\n error: `fastCRW search request failed: ${errorMessage}`,\n };\n }\n };\n\n return { getSources };\n};\n"],"mappings":";;AAGA,MAAM,sBAAsB;AAE5B,MAAM,eAAe,SAAyB;CAC5C,IAAI;EACF,OAAO,IAAI,IAAI,IAAI,CAAC,CAAC;CACvB,QAAQ;EACN,OAAO;CACT;AACF;;;AAIA,MAAM,mBAAmB,SAAiD;CACxE,MAAM,SAAsC;EAAE,KAAK,CAAC;EAAG,QAAQ,CAAC;EAAG,MAAM,CAAC;CAAE;CAC5E,KAAK,MAAM,OAAO,MAChB,IAAI,IAAI,aAAa,QACnB,OAAO,KAAK,KAAK,GAAG;MACf,IAAI,IAAI,aAAa,UAC1B,OAAO,OAAO,KAAK;EAAE,GAAG;EAAK,UAAU,IAAI;CAAI,CAAC;MAEhD,OAAO,IAAI,KAAK,GAAG;CAGvB,OAAO;AACT;AAEA,MAAa,gBACX,QACA,QACA,YAGG;CAKH,MAAM,QACJ,UACA,QAAQ,IAAI,eACZ,0BAAA,CACA,QAAQ,QAAQ,EAAE;CACpB,MAAM,SAAS;EACb,QAAQ,UAAU,QAAQ,IAAI;EAC9B,QAAQ,GAAG,KAAK;EAChB,SAAS,SAAS,WAAW;CAC/B;CAEA,MAAM,aAAa,OAAO,EACxB,OACA,MACA,aAAa,GACb,WACiD;EACjD,IAAI,CAAC,MAAM,KAAK,GACd,OAAO;GAAE,SAAS;GAAO,OAAO;EAAwB;EAG1D,IAAI;GACF,MAAM,QAAQ,KAAK,IACjB,KAAK,IAAI,GAAG,SAAS,cAAc,UAAU,GAC7C,EACF;GAGA,IAAI;GACJ,IAAI,SAAS,UACX,UAAU,CAAC,QAAQ;QACd,IAAI,SAAS,QAClB,UAAU,CAAC,MAAM;QAEjB,UAAU,SAAS,kBAAkB,OAAO,CAAC,OAAO,QAAQ,IAAI,CAAC,KAAK;GAGxE,MAAM,UAA8B;IAAE;IAAO;IAAO;GAAQ;GAC5D,IAAI,QAAQ,MAEV,QAAQ,MAAM,OAAO;GAGvB,MAAM,UAAkC,EACtC,gBAAgB,mBAClB;GACA,IAAI,OAAO,UAAU,QAAQ,OAAO,WAAW,IAC7C,QAAQ,gBAAgB,UAAU,OAAO;GAc3C,MAAM,QAAO,MAXU,MAAM,KAC3B,OAAO,QACP,SACA;IACE;IACA,SAAS,OAAO;IAChB,WAAW,SAAS;IACpB,YAAY,SAAS;GACvB,CACF,EAAA,CAEsB;GACtB,IAAI,KAAK,YAAY,OACnB,OAAO;IACL,SAAS;IACT,OAAO,0BACL,KAAK,cAAc,OAAO,IAAI,KAAK,WAAW,MAAM,KACnD,KAAK,SAAS;GACnB;GASF,MAAM,YAAY,KAAK,QAAQ,CAAC;GAChC,MAAM,OAA0B,MAAM,QAAQ,SAAS,IACnD,gBAAgB,SAAS,IACxB,UAAU,WAAW;GAuC1B,OAAO;IAAE,SAAS;IAAM,MAAM;KAR5B,UA9ByC,KAAK,OAAO,CAAC,EAAA,CACrD,QAAQ,MAAM,EAAE,OAAO,QAAQ,EAAE,QAAQ,EAAE,CAAC,CAC5C,KAAK,OAAO;MACX,OAAO,EAAE,SAAS;MAClB,MAAM,EAAE;MACR,SAAS,EAAE,eAAe,EAAE,WAAW;MACvC,UAAU,EAAE;KACd,EAuBsB;KACtB,SAtBqC,KAAK,UAAU,CAAC,EAAA,CACpD,QAAQ,MAAM,EAAE,YAAY,QAAQ,EAAE,aAAa,EAAE,CAAC,CACtD,KAAK,OAAO;MACX,OAAO,EAAE;MACT,UAAU,EAAE;MACZ,MAAM,EAAE;MACR,UAAU,EAAE;KACd,EAemB;KACnB,YAAY,CAAC;KACb,QAAQ,CAAC;KACT,OAhBmC,KAAK,QAAQ,CAAC,EAAA,CAChD,QAAQ,MAAM,EAAE,OAAO,QAAQ,EAAE,QAAQ,EAAE,CAAC,CAC5C,KAAK,OAAO;MACX,OAAO,EAAE,SAAS;MAClB,MAAM,EAAE;MACR,SAAS,EAAE,eAAe,EAAE,WAAW;MACvC,MAAM,EAAE;MACR,QAAQ,YAAY,EAAE,GAAa;MACnC,UAAU,EAAE;KACd,EAOgB;KAChB,iBAAiB,CAAC;IAGgB;GAAE;EACxC,SAAS,OAAO;GAGd,OAAO;IACL,SAAS;IACT,OAAO,kCAHP,iBAAiB,QAAQ,MAAM,UAAU,OAAO,KAAK;GAIvD;EACF;CACF;CAEA,OAAO,EAAE,WAAW;AACtB"}
@@ -28,6 +28,8 @@ var FirecrawlScraper = class {
28
28
  location;
29
29
  onlyMainContent;
30
30
  changeTrackingOptions;
31
+ httpAgent;
32
+ httpsAgent;
31
33
  constructor(config = {}) {
32
34
  this.apiKey = config.apiKey ?? process.env.FIRECRAWL_API_KEY ?? "";
33
35
  this.version = config.version ?? "v2";
@@ -51,6 +53,8 @@ var FirecrawlScraper = class {
51
53
  this.location = config.location;
52
54
  this.onlyMainContent = config.onlyMainContent;
53
55
  this.changeTrackingOptions = config.changeTrackingOptions;
56
+ this.httpAgent = config.httpAgent;
57
+ this.httpsAgent = config.httpsAgent;
54
58
  if (!this.apiKey) this.logger.warn("FIRECRAWL_API_KEY is not set. Scraping will not work.");
55
59
  this.logger.debug(`Firecrawl scraper initialized with API URL: ${this.apiUrl}`);
56
60
  }
@@ -91,7 +95,9 @@ var FirecrawlScraper = class {
91
95
  "Content-Type": "application/json",
92
96
  Authorization: `Bearer ${this.apiKey}`
93
97
  },
94
- timeout: this.timeout
98
+ timeout: this.timeout,
99
+ httpAgent: this.httpAgent,
100
+ httpsAgent: this.httpsAgent
95
101
  })).data];
96
102
  } catch (error) {
97
103
  return [url, {
@@ -1 +1 @@
1
- {"version":3,"file":"firecrawl.mjs","names":[],"sources":["../../../../src/tools/search/firecrawl.ts"],"sourcesContent":["import axios from 'axios';\nimport type * as t from './types';\nimport { createDefaultLogger } from './utils';\nimport { processContent } from './content';\n\n/**\n * Firecrawl scraper implementation\n * Uses the Firecrawl API to scrape web pages\n */\nexport class FirecrawlScraper implements t.BaseScraper {\n private apiKey: string;\n private apiUrl: string;\n private version: string;\n private defaultFormats: string[];\n private timeout: number;\n private logger: t.Logger;\n private includeTags?: string[];\n private excludeTags?: string[];\n private waitFor?: number;\n private maxAge?: number;\n private mobile?: boolean;\n private skipTlsVerification?: boolean;\n private blockAds?: boolean;\n private removeBase64Images?: boolean;\n private parsePDF?: boolean;\n private storeInCache?: boolean;\n private zeroDataRetention?: boolean;\n private headers?: Record<string, string>;\n private location?: { country?: string; languages?: string[] };\n private onlyMainContent?: boolean;\n private changeTrackingOptions?: object;\n\n constructor(config: t.FirecrawlScraperConfig = {}) {\n this.apiKey = config.apiKey ?? process.env.FIRECRAWL_API_KEY ?? '';\n\n this.version = config.version ?? 'v2';\n\n const baseUrl =\n config.apiUrl ??\n process.env.FIRECRAWL_BASE_URL ??\n 'https://api.firecrawl.dev';\n this.apiUrl = `${baseUrl.replace(/\\/+$/, '')}/${this.version}/scrape`;\n\n this.defaultFormats = config.formats ?? ['markdown', 'rawHtml'];\n this.timeout = config.timeout ?? 7500;\n\n this.logger = config.logger || createDefaultLogger();\n\n this.includeTags = config.includeTags;\n this.excludeTags = config.excludeTags;\n this.waitFor = config.waitFor;\n this.maxAge = config.maxAge;\n this.mobile = config.mobile;\n this.skipTlsVerification = config.skipTlsVerification;\n this.blockAds = config.blockAds;\n this.removeBase64Images = config.removeBase64Images;\n this.parsePDF = config.parsePDF;\n this.storeInCache = config.storeInCache;\n this.zeroDataRetention = config.zeroDataRetention;\n this.headers = config.headers;\n this.location = config.location;\n this.onlyMainContent = config.onlyMainContent;\n this.changeTrackingOptions = config.changeTrackingOptions;\n\n if (!this.apiKey) {\n this.logger.warn('FIRECRAWL_API_KEY is not set. Scraping will not work.');\n }\n\n this.logger.debug(\n `Firecrawl scraper initialized with API URL: ${this.apiUrl}`\n );\n }\n\n /**\n * Scrape a single URL\n * @param url URL to scrape\n * @param options Scrape options\n * @returns Scrape response\n */\n async scrapeUrl(\n url: string,\n options: t.FirecrawlScrapeOptions = {}\n ): Promise<[string, t.FirecrawlScrapeResponse]> {\n if (!this.apiKey) {\n return [\n url,\n {\n success: false,\n error: 'FIRECRAWL_API_KEY is not set',\n },\n ];\n }\n\n try {\n const payload = omitUndefined({\n url,\n formats: options.formats ?? this.defaultFormats,\n includeTags: options.includeTags ?? this.includeTags,\n excludeTags: options.excludeTags ?? this.excludeTags,\n headers: options.headers ?? this.headers,\n waitFor: options.waitFor ?? this.waitFor,\n timeout: options.timeout ?? this.timeout,\n onlyMainContent: options.onlyMainContent ?? this.onlyMainContent,\n maxAge: options.maxAge ?? this.maxAge,\n mobile: options.mobile ?? this.mobile,\n skipTlsVerification:\n options.skipTlsVerification ?? this.skipTlsVerification,\n parsePDF: options.parsePDF ?? this.parsePDF,\n location: options.location ?? this.location,\n removeBase64Images:\n options.removeBase64Images ?? this.removeBase64Images,\n blockAds: options.blockAds ?? this.blockAds,\n storeInCache: options.storeInCache ?? this.storeInCache,\n zeroDataRetention: options.zeroDataRetention ?? this.zeroDataRetention,\n changeTrackingOptions:\n options.changeTrackingOptions ?? this.changeTrackingOptions,\n });\n const response = await axios.post(this.apiUrl, payload, {\n headers: {\n 'Content-Type': 'application/json',\n Authorization: `Bearer ${this.apiKey}`,\n },\n timeout: this.timeout,\n });\n\n return [url, response.data];\n } catch (error) {\n const errorMessage =\n error instanceof Error ? error.message : String(error);\n return [\n url,\n {\n success: false,\n error: `Firecrawl API request failed: ${errorMessage}`,\n },\n ];\n }\n }\n\n /**\n * Extract content from scrape response\n * @param response Scrape response\n * @returns Extracted content or empty string if not available\n */\n extractContent(\n response: t.FirecrawlScrapeResponse\n ): [string, undefined | t.References] {\n if (!response.success || !response.data) {\n return ['', undefined];\n }\n\n if (response.data.markdown != null && response.data.html != null) {\n try {\n const { markdown, ...rest } = processContent(\n response.data.html,\n response.data.markdown\n );\n return [markdown, rest];\n } catch (error) {\n this.logger.error('Error processing content:', error);\n return [response.data.markdown, undefined];\n }\n } else if (response.data.markdown != null) {\n return [response.data.markdown, undefined];\n }\n\n // Fall back to HTML content\n if (response.data.html != null) {\n return [response.data.html, undefined];\n }\n\n // Fall back to raw HTML content\n if (response.data.rawHtml != null) {\n return [response.data.rawHtml, undefined];\n }\n\n return ['', undefined];\n }\n\n /**\n * Extract metadata from scrape response\n * @param response Scrape response\n * @returns Metadata object\n */\n extractMetadata(response: t.FirecrawlScrapeResponse): t.ScrapeMetadata {\n if (!response.success || !response.data || !response.data.metadata) {\n return {};\n }\n\n return response.data.metadata;\n }\n}\n\n/**\n * Create a Firecrawl scraper instance\n * @param config Scraper configuration\n * @returns Firecrawl scraper instance\n */\nexport const createFirecrawlScraper = (\n config: t.FirecrawlScraperConfig = {}\n): FirecrawlScraper => {\n return new FirecrawlScraper(config);\n};\n\n// Helper function to clean up payload for firecrawl\nfunction omitUndefined<T extends object>(obj: T): Partial<T> {\n return Object.fromEntries(\n Object.entries(obj).filter(([, v]) => v !== undefined)\n ) as Partial<T>;\n}\n"],"mappings":";;;;;;;;AASA,IAAa,mBAAb,MAAuD;CACrD;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CAEA,YAAY,SAAmC,CAAC,GAAG;EACjD,KAAK,SAAS,OAAO,UAAU,QAAQ,IAAI,qBAAqB;EAEhE,KAAK,UAAU,OAAO,WAAW;EAEjC,MAAM,UACJ,OAAO,UACP,QAAQ,IAAI,sBACZ;EACF,KAAK,SAAS,GAAG,QAAQ,QAAQ,QAAQ,EAAE,EAAE,GAAG,KAAK,QAAQ;EAE7D,KAAK,iBAAiB,OAAO,WAAW,CAAC,YAAY,SAAS;EAC9D,KAAK,UAAU,OAAO,WAAW;EAEjC,KAAK,SAAS,OAAO,UAAU,oBAAoB;EAEnD,KAAK,cAAc,OAAO;EAC1B,KAAK,cAAc,OAAO;EAC1B,KAAK,UAAU,OAAO;EACtB,KAAK,SAAS,OAAO;EACrB,KAAK,SAAS,OAAO;EACrB,KAAK,sBAAsB,OAAO;EAClC,KAAK,WAAW,OAAO;EACvB,KAAK,qBAAqB,OAAO;EACjC,KAAK,WAAW,OAAO;EACvB,KAAK,eAAe,OAAO;EAC3B,KAAK,oBAAoB,OAAO;EAChC,KAAK,UAAU,OAAO;EACtB,KAAK,WAAW,OAAO;EACvB,KAAK,kBAAkB,OAAO;EAC9B,KAAK,wBAAwB,OAAO;EAEpC,IAAI,CAAC,KAAK,QACR,KAAK,OAAO,KAAK,uDAAuD;EAG1E,KAAK,OAAO,MACV,+CAA+C,KAAK,QACtD;CACF;;;;;;;CAQA,MAAM,UACJ,KACA,UAAoC,CAAC,GACS;EAC9C,IAAI,CAAC,KAAK,QACR,OAAO,CACL,KACA;GACE,SAAS;GACT,OAAO;EACT,CACF;EAGF,IAAI;GACF,MAAM,UAAU,cAAc;IAC5B;IACA,SAAS,QAAQ,WAAW,KAAK;IACjC,aAAa,QAAQ,eAAe,KAAK;IACzC,aAAa,QAAQ,eAAe,KAAK;IACzC,SAAS,QAAQ,WAAW,KAAK;IACjC,SAAS,QAAQ,WAAW,KAAK;IACjC,SAAS,QAAQ,WAAW,KAAK;IACjC,iBAAiB,QAAQ,mBAAmB,KAAK;IACjD,QAAQ,QAAQ,UAAU,KAAK;IAC/B,QAAQ,QAAQ,UAAU,KAAK;IAC/B,qBACE,QAAQ,uBAAuB,KAAK;IACtC,UAAU,QAAQ,YAAY,KAAK;IACnC,UAAU,QAAQ,YAAY,KAAK;IACnC,oBACE,QAAQ,sBAAsB,KAAK;IACrC,UAAU,QAAQ,YAAY,KAAK;IACnC,cAAc,QAAQ,gBAAgB,KAAK;IAC3C,mBAAmB,QAAQ,qBAAqB,KAAK;IACrD,uBACE,QAAQ,yBAAyB,KAAK;GAC1C,CAAC;GASD,OAAO,CAAC,MAAK,MARU,MAAM,KAAK,KAAK,QAAQ,SAAS;IACtD,SAAS;KACP,gBAAgB;KAChB,eAAe,UAAU,KAAK;IAChC;IACA,SAAS,KAAK;GAChB,CAAC,EAAA,CAEqB,IAAI;EAC5B,SAAS,OAAO;GAGd,OAAO,CACL,KACA;IACE,SAAS;IACT,OAAO,iCALT,iBAAiB,QAAQ,MAAM,UAAU,OAAO,KAAK;GAMrD,CACF;EACF;CACF;;;;;;CAOA,eACE,UACoC;EACpC,IAAI,CAAC,SAAS,WAAW,CAAC,SAAS,MACjC,OAAO,CAAC,IAAI,KAAA,CAAS;EAGvB,IAAI,SAAS,KAAK,YAAY,QAAQ,SAAS,KAAK,QAAQ,MAC1D,IAAI;GACF,MAAM,EAAE,UAAU,GAAG,SAAS,eAC5B,SAAS,KAAK,MACd,SAAS,KAAK,QAChB;GACA,OAAO,CAAC,UAAU,IAAI;EACxB,SAAS,OAAO;GACd,KAAK,OAAO,MAAM,6BAA6B,KAAK;GACpD,OAAO,CAAC,SAAS,KAAK,UAAU,KAAA,CAAS;EAC3C;OACK,IAAI,SAAS,KAAK,YAAY,MACnC,OAAO,CAAC,SAAS,KAAK,UAAU,KAAA,CAAS;EAI3C,IAAI,SAAS,KAAK,QAAQ,MACxB,OAAO,CAAC,SAAS,KAAK,MAAM,KAAA,CAAS;EAIvC,IAAI,SAAS,KAAK,WAAW,MAC3B,OAAO,CAAC,SAAS,KAAK,SAAS,KAAA,CAAS;EAG1C,OAAO,CAAC,IAAI,KAAA,CAAS;CACvB;;;;;;CAOA,gBAAgB,UAAuD;EACrE,IAAI,CAAC,SAAS,WAAW,CAAC,SAAS,QAAQ,CAAC,SAAS,KAAK,UACxD,OAAO,CAAC;EAGV,OAAO,SAAS,KAAK;CACvB;AACF;;;;;;AAOA,MAAa,0BACX,SAAmC,CAAC,MACf;CACrB,OAAO,IAAI,iBAAiB,MAAM;AACpC;AAGA,SAAS,cAAgC,KAAoB;CAC3D,OAAO,OAAO,YACZ,OAAO,QAAQ,GAAG,CAAC,CAAC,QAAQ,GAAG,OAAO,MAAM,KAAA,CAAS,CACvD;AACF"}
1
+ {"version":3,"file":"firecrawl.mjs","names":[],"sources":["../../../../src/tools/search/firecrawl.ts"],"sourcesContent":["import axios from 'axios';\nimport type * as t from './types';\nimport { createDefaultLogger } from './utils';\nimport { processContent } from './content';\n\n/**\n * Firecrawl scraper implementation\n * Uses the Firecrawl API to scrape web pages\n */\nexport class FirecrawlScraper implements t.BaseScraper {\n private apiKey: string;\n private apiUrl: string;\n private version: string;\n private defaultFormats: string[];\n private timeout: number;\n private logger: t.Logger;\n private includeTags?: string[];\n private excludeTags?: string[];\n private waitFor?: number;\n private maxAge?: number;\n private mobile?: boolean;\n private skipTlsVerification?: boolean;\n private blockAds?: boolean;\n private removeBase64Images?: boolean;\n private parsePDF?: boolean;\n private storeInCache?: boolean;\n private zeroDataRetention?: boolean;\n private headers?: Record<string, string>;\n private location?: { country?: string; languages?: string[] };\n private onlyMainContent?: boolean;\n private changeTrackingOptions?: object;\n private httpAgent?: t.HttpAgent;\n private httpsAgent?: t.HttpsAgent;\n\n constructor(config: t.FirecrawlScraperConfig = {}) {\n this.apiKey = config.apiKey ?? process.env.FIRECRAWL_API_KEY ?? '';\n\n this.version = config.version ?? 'v2';\n\n const baseUrl =\n config.apiUrl ??\n process.env.FIRECRAWL_BASE_URL ??\n 'https://api.firecrawl.dev';\n this.apiUrl = `${baseUrl.replace(/\\/+$/, '')}/${this.version}/scrape`;\n\n this.defaultFormats = config.formats ?? ['markdown', 'rawHtml'];\n this.timeout = config.timeout ?? 7500;\n\n this.logger = config.logger || createDefaultLogger();\n\n this.includeTags = config.includeTags;\n this.excludeTags = config.excludeTags;\n this.waitFor = config.waitFor;\n this.maxAge = config.maxAge;\n this.mobile = config.mobile;\n this.skipTlsVerification = config.skipTlsVerification;\n this.blockAds = config.blockAds;\n this.removeBase64Images = config.removeBase64Images;\n this.parsePDF = config.parsePDF;\n this.storeInCache = config.storeInCache;\n this.zeroDataRetention = config.zeroDataRetention;\n this.headers = config.headers;\n this.location = config.location;\n this.onlyMainContent = config.onlyMainContent;\n this.changeTrackingOptions = config.changeTrackingOptions;\n this.httpAgent = config.httpAgent;\n this.httpsAgent = config.httpsAgent;\n\n if (!this.apiKey) {\n this.logger.warn('FIRECRAWL_API_KEY is not set. Scraping will not work.');\n }\n\n this.logger.debug(\n `Firecrawl scraper initialized with API URL: ${this.apiUrl}`\n );\n }\n\n /**\n * Scrape a single URL\n * @param url URL to scrape\n * @param options Scrape options\n * @returns Scrape response\n */\n async scrapeUrl(\n url: string,\n options: t.FirecrawlScrapeOptions = {}\n ): Promise<[string, t.FirecrawlScrapeResponse]> {\n if (!this.apiKey) {\n return [\n url,\n {\n success: false,\n error: 'FIRECRAWL_API_KEY is not set',\n },\n ];\n }\n\n try {\n const payload = omitUndefined({\n url,\n formats: options.formats ?? this.defaultFormats,\n includeTags: options.includeTags ?? this.includeTags,\n excludeTags: options.excludeTags ?? this.excludeTags,\n headers: options.headers ?? this.headers,\n waitFor: options.waitFor ?? this.waitFor,\n timeout: options.timeout ?? this.timeout,\n onlyMainContent: options.onlyMainContent ?? this.onlyMainContent,\n maxAge: options.maxAge ?? this.maxAge,\n mobile: options.mobile ?? this.mobile,\n skipTlsVerification:\n options.skipTlsVerification ?? this.skipTlsVerification,\n parsePDF: options.parsePDF ?? this.parsePDF,\n location: options.location ?? this.location,\n removeBase64Images:\n options.removeBase64Images ?? this.removeBase64Images,\n blockAds: options.blockAds ?? this.blockAds,\n storeInCache: options.storeInCache ?? this.storeInCache,\n zeroDataRetention: options.zeroDataRetention ?? this.zeroDataRetention,\n changeTrackingOptions:\n options.changeTrackingOptions ?? this.changeTrackingOptions,\n });\n const response = await axios.post(this.apiUrl, payload, {\n headers: {\n 'Content-Type': 'application/json',\n Authorization: `Bearer ${this.apiKey}`,\n },\n timeout: this.timeout,\n httpAgent: this.httpAgent,\n httpsAgent: this.httpsAgent,\n });\n\n return [url, response.data];\n } catch (error) {\n const errorMessage =\n error instanceof Error ? error.message : String(error);\n return [\n url,\n {\n success: false,\n error: `Firecrawl API request failed: ${errorMessage}`,\n },\n ];\n }\n }\n\n /**\n * Extract content from scrape response\n * @param response Scrape response\n * @returns Extracted content or empty string if not available\n */\n extractContent(\n response: t.FirecrawlScrapeResponse\n ): [string, undefined | t.References] {\n if (!response.success || !response.data) {\n return ['', undefined];\n }\n\n if (response.data.markdown != null && response.data.html != null) {\n try {\n const { markdown, ...rest } = processContent(\n response.data.html,\n response.data.markdown\n );\n return [markdown, rest];\n } catch (error) {\n this.logger.error('Error processing content:', error);\n return [response.data.markdown, undefined];\n }\n } else if (response.data.markdown != null) {\n return [response.data.markdown, undefined];\n }\n\n // Fall back to HTML content\n if (response.data.html != null) {\n return [response.data.html, undefined];\n }\n\n // Fall back to raw HTML content\n if (response.data.rawHtml != null) {\n return [response.data.rawHtml, undefined];\n }\n\n return ['', undefined];\n }\n\n /**\n * Extract metadata from scrape response\n * @param response Scrape response\n * @returns Metadata object\n */\n extractMetadata(response: t.FirecrawlScrapeResponse): t.ScrapeMetadata {\n if (!response.success || !response.data || !response.data.metadata) {\n return {};\n }\n\n return response.data.metadata;\n }\n}\n\n/**\n * Create a Firecrawl scraper instance\n * @param config Scraper configuration\n * @returns Firecrawl scraper instance\n */\nexport const createFirecrawlScraper = (\n config: t.FirecrawlScraperConfig = {}\n): FirecrawlScraper => {\n return new FirecrawlScraper(config);\n};\n\n// Helper function to clean up payload for firecrawl\nfunction omitUndefined<T extends object>(obj: T): Partial<T> {\n return Object.fromEntries(\n Object.entries(obj).filter(([, v]) => v !== undefined)\n ) as Partial<T>;\n}\n"],"mappings":";;;;;;;;AASA,IAAa,mBAAb,MAAuD;CACrD;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CAEA,YAAY,SAAmC,CAAC,GAAG;EACjD,KAAK,SAAS,OAAO,UAAU,QAAQ,IAAI,qBAAqB;EAEhE,KAAK,UAAU,OAAO,WAAW;EAEjC,MAAM,UACJ,OAAO,UACP,QAAQ,IAAI,sBACZ;EACF,KAAK,SAAS,GAAG,QAAQ,QAAQ,QAAQ,EAAE,EAAE,GAAG,KAAK,QAAQ;EAE7D,KAAK,iBAAiB,OAAO,WAAW,CAAC,YAAY,SAAS;EAC9D,KAAK,UAAU,OAAO,WAAW;EAEjC,KAAK,SAAS,OAAO,UAAU,oBAAoB;EAEnD,KAAK,cAAc,OAAO;EAC1B,KAAK,cAAc,OAAO;EAC1B,KAAK,UAAU,OAAO;EACtB,KAAK,SAAS,OAAO;EACrB,KAAK,SAAS,OAAO;EACrB,KAAK,sBAAsB,OAAO;EAClC,KAAK,WAAW,OAAO;EACvB,KAAK,qBAAqB,OAAO;EACjC,KAAK,WAAW,OAAO;EACvB,KAAK,eAAe,OAAO;EAC3B,KAAK,oBAAoB,OAAO;EAChC,KAAK,UAAU,OAAO;EACtB,KAAK,WAAW,OAAO;EACvB,KAAK,kBAAkB,OAAO;EAC9B,KAAK,wBAAwB,OAAO;EACpC,KAAK,YAAY,OAAO;EACxB,KAAK,aAAa,OAAO;EAEzB,IAAI,CAAC,KAAK,QACR,KAAK,OAAO,KAAK,uDAAuD;EAG1E,KAAK,OAAO,MACV,+CAA+C,KAAK,QACtD;CACF;;;;;;;CAQA,MAAM,UACJ,KACA,UAAoC,CAAC,GACS;EAC9C,IAAI,CAAC,KAAK,QACR,OAAO,CACL,KACA;GACE,SAAS;GACT,OAAO;EACT,CACF;EAGF,IAAI;GACF,MAAM,UAAU,cAAc;IAC5B;IACA,SAAS,QAAQ,WAAW,KAAK;IACjC,aAAa,QAAQ,eAAe,KAAK;IACzC,aAAa,QAAQ,eAAe,KAAK;IACzC,SAAS,QAAQ,WAAW,KAAK;IACjC,SAAS,QAAQ,WAAW,KAAK;IACjC,SAAS,QAAQ,WAAW,KAAK;IACjC,iBAAiB,QAAQ,mBAAmB,KAAK;IACjD,QAAQ,QAAQ,UAAU,KAAK;IAC/B,QAAQ,QAAQ,UAAU,KAAK;IAC/B,qBACE,QAAQ,uBAAuB,KAAK;IACtC,UAAU,QAAQ,YAAY,KAAK;IACnC,UAAU,QAAQ,YAAY,KAAK;IACnC,oBACE,QAAQ,sBAAsB,KAAK;IACrC,UAAU,QAAQ,YAAY,KAAK;IACnC,cAAc,QAAQ,gBAAgB,KAAK;IAC3C,mBAAmB,QAAQ,qBAAqB,KAAK;IACrD,uBACE,QAAQ,yBAAyB,KAAK;GAC1C,CAAC;GAWD,OAAO,CAAC,MAAK,MAVU,MAAM,KAAK,KAAK,QAAQ,SAAS;IACtD,SAAS;KACP,gBAAgB;KAChB,eAAe,UAAU,KAAK;IAChC;IACA,SAAS,KAAK;IACd,WAAW,KAAK;IAChB,YAAY,KAAK;GACnB,CAAC,EAAA,CAEqB,IAAI;EAC5B,SAAS,OAAO;GAGd,OAAO,CACL,KACA;IACE,SAAS;IACT,OAAO,iCALT,iBAAiB,QAAQ,MAAM,UAAU,OAAO,KAAK;GAMrD,CACF;EACF;CACF;;;;;;CAOA,eACE,UACoC;EACpC,IAAI,CAAC,SAAS,WAAW,CAAC,SAAS,MACjC,OAAO,CAAC,IAAI,KAAA,CAAS;EAGvB,IAAI,SAAS,KAAK,YAAY,QAAQ,SAAS,KAAK,QAAQ,MAC1D,IAAI;GACF,MAAM,EAAE,UAAU,GAAG,SAAS,eAC5B,SAAS,KAAK,MACd,SAAS,KAAK,QAChB;GACA,OAAO,CAAC,UAAU,IAAI;EACxB,SAAS,OAAO;GACd,KAAK,OAAO,MAAM,6BAA6B,KAAK;GACpD,OAAO,CAAC,SAAS,KAAK,UAAU,KAAA,CAAS;EAC3C;OACK,IAAI,SAAS,KAAK,YAAY,MACnC,OAAO,CAAC,SAAS,KAAK,UAAU,KAAA,CAAS;EAI3C,IAAI,SAAS,KAAK,QAAQ,MACxB,OAAO,CAAC,SAAS,KAAK,MAAM,KAAA,CAAS;EAIvC,IAAI,SAAS,KAAK,WAAW,MAC3B,OAAO,CAAC,SAAS,KAAK,SAAS,KAAA,CAAS;EAG1C,OAAO,CAAC,IAAI,KAAA,CAAS;CACvB;;;;;;CAOA,gBAAgB,UAAuD;EACrE,IAAI,CAAC,SAAS,WAAW,CAAC,SAAS,QAAQ,CAAC,SAAS,KAAK,UACxD,OAAO,CAAC;EAGV,OAAO,SAAS,KAAK;CACvB;AACF;;;;;;AAOA,MAAa,0BACX,SAAmC,CAAC,MACf;CACrB,OAAO,IAAI,iBAAiB,MAAM;AACpC;AAGA,SAAS,cAAgC,KAAoB;CAC3D,OAAO,OAAO,YACZ,OAAO,QAAQ,GAAG,CAAC,CAAC,QAAQ,GAAG,OAAO,MAAM,KAAA,CAAS,CACvD;AACF"}
@@ -12,12 +12,16 @@ var KeenableScraper = class {
12
12
  timeout;
13
13
  attributionTitle;
14
14
  logger;
15
+ httpAgent;
16
+ httpsAgent;
15
17
  constructor(config = {}) {
16
18
  const resolvedKey = config.apiKey ?? process.env.KEENABLE_API_KEY;
17
19
  this.apiKey = resolvedKey != null && resolvedKey !== "" ? resolvedKey : void 0;
18
20
  this.apiUrl = config.apiUrl ?? process.env.KEENABLE_FETCH_URL ?? (this.apiKey != null ? KEENABLE_FETCH_API_URL : KEENABLE_FETCH_PUBLIC_URL);
19
21
  this.timeout = config.timeout ?? DEFAULT_KEENABLE_SCRAPE_TIMEOUT;
20
22
  this.attributionTitle = config.attributionTitle ?? "LibreChat";
23
+ this.httpAgent = config.httpAgent;
24
+ this.httpsAgent = config.httpsAgent;
21
25
  this.logger = config.logger || createDefaultLogger();
22
26
  }
23
27
  buildHeaders() {
@@ -37,7 +41,9 @@ var KeenableScraper = class {
37
41
  const data = (await axios.get(this.apiUrl, {
38
42
  params: { url },
39
43
  headers: this.buildHeaders(),
40
- timeout: options.timeout ?? this.timeout
44
+ timeout: options.timeout ?? this.timeout,
45
+ httpAgent: this.httpAgent,
46
+ httpsAgent: this.httpsAgent
41
47
  })).data;
42
48
  const content = data.content ?? "";
43
49
  if (!content) return [url, {
@@ -1 +1 @@
1
- {"version":3,"file":"keenable-scraper.mjs","names":[],"sources":["../../../../src/tools/search/keenable-scraper.ts"],"sourcesContent":["import axios from 'axios';\nimport type * as t from './types';\nimport { createDefaultLogger } from './utils';\n\nconst DEFAULT_KEENABLE_SCRAPE_TIMEOUT = 15000;\n\n/** Keyed and keyless fetch endpoints. Keenable reads any URL as clean markdown\n * without a key via the public endpoint; a key only lifts rate limits. */\nconst KEENABLE_FETCH_API_URL = 'https://api.keenable.ai/v1/fetch';\nconst KEENABLE_FETCH_PUBLIC_URL = 'https://api.keenable.ai/v1/fetch/public';\n\nexport class KeenableScraper implements t.BaseScraper {\n private apiKey: string | undefined;\n private apiUrl: string;\n private timeout: number;\n private attributionTitle: string;\n private logger: t.Logger;\n\n constructor(config: t.KeenableScraperConfig = {}) {\n const resolvedKey = config.apiKey ?? process.env.KEENABLE_API_KEY;\n this.apiKey =\n resolvedKey != null && resolvedKey !== '' ? resolvedKey : undefined;\n this.apiUrl =\n config.apiUrl ??\n process.env.KEENABLE_FETCH_URL ??\n (this.apiKey != null\n ? KEENABLE_FETCH_API_URL\n : KEENABLE_FETCH_PUBLIC_URL);\n this.timeout = config.timeout ?? DEFAULT_KEENABLE_SCRAPE_TIMEOUT;\n this.attributionTitle = config.attributionTitle ?? 'LibreChat';\n this.logger = config.logger || createDefaultLogger();\n }\n\n private buildHeaders(): Record<string, string> {\n /** X-Keenable-Title is used for traffic attribution and required on the\n * keyless endpoint; the key only lifts rate limits, so it is sent only when\n * present. */\n const headers: Record<string, string> = {\n 'X-Keenable-Title': this.attributionTitle,\n };\n if (this.apiKey != null) {\n headers['X-API-Key'] = this.apiKey;\n }\n return headers;\n }\n\n async scrapeUrl(\n url: string,\n options: t.KeenableScrapeOptions = {}\n ): Promise<[string, t.KeenableScrapeResponse]> {\n if (!url || !url.trim()) {\n return [url, { success: false, error: 'URL cannot be empty' }];\n }\n\n try {\n const response = await axios.get<t.KeenableFetchResult>(this.apiUrl, {\n params: { url },\n headers: this.buildHeaders(),\n timeout: options.timeout ?? this.timeout,\n });\n\n const data = response.data;\n const content = data.content ?? '';\n if (!content) {\n return [\n url,\n { success: false, error: 'Keenable Fetch returned no content' },\n ];\n }\n\n return [\n url,\n {\n success: true,\n data: {\n content,\n title: data.title,\n description: data.description,\n url: data.url ?? url,\n },\n },\n ];\n } catch (error) {\n const errorMessage =\n error instanceof Error ? error.message : String(error);\n return [\n url,\n {\n success: false,\n error: `Keenable Fetch API request failed: ${errorMessage}`,\n },\n ];\n }\n }\n\n async scrapeUrls(\n urls: string[],\n options: t.KeenableScrapeOptions = {}\n ): Promise<Array<[string, t.KeenableScrapeResponse]>> {\n /** Keenable fetch is single-URL; run the batch concurrently. */\n return Promise.all(urls.map((url) => this.scrapeUrl(url, options)));\n }\n\n extractContent(\n response: t.KeenableScrapeResponse\n ): [string, undefined | t.References] {\n if (!response.success || !response.data) {\n return ['', undefined];\n }\n /** Keenable returns clean markdown with no separate media arrays, so there\n * are no structured references to surface. */\n return [response.data.content, undefined];\n }\n\n extractMetadata(response: t.KeenableScrapeResponse): t.GenericScrapeMetadata {\n if (!response.success || !response.data) {\n return {};\n }\n const metadata: t.GenericScrapeMetadata = {};\n if (response.data.title != null) {\n metadata.title = response.data.title;\n }\n if (response.data.description != null && response.data.description !== '') {\n metadata.description = response.data.description;\n }\n if (response.data.url != null) {\n metadata.url = response.data.url;\n }\n return metadata;\n }\n}\n\nexport const createKeenableScraper = (\n config: t.KeenableScraperConfig = {}\n): KeenableScraper => {\n return new KeenableScraper(config);\n};\n"],"mappings":";;;AAIA,MAAM,kCAAkC;;;AAIxC,MAAM,yBAAyB;AAC/B,MAAM,4BAA4B;AAElC,IAAa,kBAAb,MAAsD;CACpD;CACA;CACA;CACA;CACA;CAEA,YAAY,SAAkC,CAAC,GAAG;EAChD,MAAM,cAAc,OAAO,UAAU,QAAQ,IAAI;EACjD,KAAK,SACH,eAAe,QAAQ,gBAAgB,KAAK,cAAc,KAAA;EAC5D,KAAK,SACH,OAAO,UACP,QAAQ,IAAI,uBACX,KAAK,UAAU,OACZ,yBACA;EACN,KAAK,UAAU,OAAO,WAAW;EACjC,KAAK,mBAAmB,OAAO,oBAAoB;EACnD,KAAK,SAAS,OAAO,UAAU,oBAAoB;CACrD;CAEA,eAA+C;;;;EAI7C,MAAM,UAAkC,EACtC,oBAAoB,KAAK,iBAC3B;EACA,IAAI,KAAK,UAAU,MACjB,QAAQ,eAAe,KAAK;EAE9B,OAAO;CACT;CAEA,MAAM,UACJ,KACA,UAAmC,CAAC,GACS;EAC7C,IAAI,CAAC,OAAO,CAAC,IAAI,KAAK,GACpB,OAAO,CAAC,KAAK;GAAE,SAAS;GAAO,OAAO;EAAsB,CAAC;EAG/D,IAAI;GAOF,MAAM,QAAO,MANU,MAAM,IAA2B,KAAK,QAAQ;IACnE,QAAQ,EAAE,IAAI;IACd,SAAS,KAAK,aAAa;IAC3B,SAAS,QAAQ,WAAW,KAAK;GACnC,CAAC,EAAA,CAEqB;GACtB,MAAM,UAAU,KAAK,WAAW;GAChC,IAAI,CAAC,SACH,OAAO,CACL,KACA;IAAE,SAAS;IAAO,OAAO;GAAqC,CAChE;GAGF,OAAO,CACL,KACA;IACE,SAAS;IACT,MAAM;KACJ;KACA,OAAO,KAAK;KACZ,aAAa,KAAK;KAClB,KAAK,KAAK,OAAO;IACnB;GACF,CACF;EACF,SAAS,OAAO;GAGd,OAAO,CACL,KACA;IACE,SAAS;IACT,OAAO,sCALT,iBAAiB,QAAQ,MAAM,UAAU,OAAO,KAAK;GAMrD,CACF;EACF;CACF;CAEA,MAAM,WACJ,MACA,UAAmC,CAAC,GACgB;;EAEpD,OAAO,QAAQ,IAAI,KAAK,KAAK,QAAQ,KAAK,UAAU,KAAK,OAAO,CAAC,CAAC;CACpE;CAEA,eACE,UACoC;EACpC,IAAI,CAAC,SAAS,WAAW,CAAC,SAAS,MACjC,OAAO,CAAC,IAAI,KAAA,CAAS;;;EAIvB,OAAO,CAAC,SAAS,KAAK,SAAS,KAAA,CAAS;CAC1C;CAEA,gBAAgB,UAA6D;EAC3E,IAAI,CAAC,SAAS,WAAW,CAAC,SAAS,MACjC,OAAO,CAAC;EAEV,MAAM,WAAoC,CAAC;EAC3C,IAAI,SAAS,KAAK,SAAS,MACzB,SAAS,QAAQ,SAAS,KAAK;EAEjC,IAAI,SAAS,KAAK,eAAe,QAAQ,SAAS,KAAK,gBAAgB,IACrE,SAAS,cAAc,SAAS,KAAK;EAEvC,IAAI,SAAS,KAAK,OAAO,MACvB,SAAS,MAAM,SAAS,KAAK;EAE/B,OAAO;CACT;AACF;AAEA,MAAa,yBACX,SAAkC,CAAC,MACf;CACpB,OAAO,IAAI,gBAAgB,MAAM;AACnC"}
1
+ {"version":3,"file":"keenable-scraper.mjs","names":[],"sources":["../../../../src/tools/search/keenable-scraper.ts"],"sourcesContent":["import axios from 'axios';\nimport type * as t from './types';\nimport { createDefaultLogger } from './utils';\n\nconst DEFAULT_KEENABLE_SCRAPE_TIMEOUT = 15000;\n\n/** Keyed and keyless fetch endpoints. Keenable reads any URL as clean markdown\n * without a key via the public endpoint; a key only lifts rate limits. */\nconst KEENABLE_FETCH_API_URL = 'https://api.keenable.ai/v1/fetch';\nconst KEENABLE_FETCH_PUBLIC_URL = 'https://api.keenable.ai/v1/fetch/public';\n\nexport class KeenableScraper implements t.BaseScraper {\n private apiKey: string | undefined;\n private apiUrl: string;\n private timeout: number;\n private attributionTitle: string;\n private logger: t.Logger;\n private httpAgent?: t.HttpAgent;\n private httpsAgent?: t.HttpsAgent;\n\n constructor(config: t.KeenableScraperConfig = {}) {\n const resolvedKey = config.apiKey ?? process.env.KEENABLE_API_KEY;\n this.apiKey =\n resolvedKey != null && resolvedKey !== '' ? resolvedKey : undefined;\n this.apiUrl =\n config.apiUrl ??\n process.env.KEENABLE_FETCH_URL ??\n (this.apiKey != null\n ? KEENABLE_FETCH_API_URL\n : KEENABLE_FETCH_PUBLIC_URL);\n this.timeout = config.timeout ?? DEFAULT_KEENABLE_SCRAPE_TIMEOUT;\n this.attributionTitle = config.attributionTitle ?? 'LibreChat';\n this.httpAgent = config.httpAgent;\n this.httpsAgent = config.httpsAgent;\n this.logger = config.logger || createDefaultLogger();\n }\n\n private buildHeaders(): Record<string, string> {\n /** X-Keenable-Title is used for traffic attribution and required on the\n * keyless endpoint; the key only lifts rate limits, so it is sent only when\n * present. */\n const headers: Record<string, string> = {\n 'X-Keenable-Title': this.attributionTitle,\n };\n if (this.apiKey != null) {\n headers['X-API-Key'] = this.apiKey;\n }\n return headers;\n }\n\n async scrapeUrl(\n url: string,\n options: t.KeenableScrapeOptions = {}\n ): Promise<[string, t.KeenableScrapeResponse]> {\n if (!url || !url.trim()) {\n return [url, { success: false, error: 'URL cannot be empty' }];\n }\n\n try {\n const response = await axios.get<t.KeenableFetchResult>(this.apiUrl, {\n params: { url },\n headers: this.buildHeaders(),\n timeout: options.timeout ?? this.timeout,\n httpAgent: this.httpAgent,\n httpsAgent: this.httpsAgent,\n });\n\n const data = response.data;\n const content = data.content ?? '';\n if (!content) {\n return [\n url,\n { success: false, error: 'Keenable Fetch returned no content' },\n ];\n }\n\n return [\n url,\n {\n success: true,\n data: {\n content,\n title: data.title,\n description: data.description,\n url: data.url ?? url,\n },\n },\n ];\n } catch (error) {\n const errorMessage =\n error instanceof Error ? error.message : String(error);\n return [\n url,\n {\n success: false,\n error: `Keenable Fetch API request failed: ${errorMessage}`,\n },\n ];\n }\n }\n\n async scrapeUrls(\n urls: string[],\n options: t.KeenableScrapeOptions = {}\n ): Promise<Array<[string, t.KeenableScrapeResponse]>> {\n /** Keenable fetch is single-URL; run the batch concurrently. */\n return Promise.all(urls.map((url) => this.scrapeUrl(url, options)));\n }\n\n extractContent(\n response: t.KeenableScrapeResponse\n ): [string, undefined | t.References] {\n if (!response.success || !response.data) {\n return ['', undefined];\n }\n /** Keenable returns clean markdown with no separate media arrays, so there\n * are no structured references to surface. */\n return [response.data.content, undefined];\n }\n\n extractMetadata(response: t.KeenableScrapeResponse): t.GenericScrapeMetadata {\n if (!response.success || !response.data) {\n return {};\n }\n const metadata: t.GenericScrapeMetadata = {};\n if (response.data.title != null) {\n metadata.title = response.data.title;\n }\n if (response.data.description != null && response.data.description !== '') {\n metadata.description = response.data.description;\n }\n if (response.data.url != null) {\n metadata.url = response.data.url;\n }\n return metadata;\n }\n}\n\nexport const createKeenableScraper = (\n config: t.KeenableScraperConfig = {}\n): KeenableScraper => {\n return new KeenableScraper(config);\n};\n"],"mappings":";;;AAIA,MAAM,kCAAkC;;;AAIxC,MAAM,yBAAyB;AAC/B,MAAM,4BAA4B;AAElC,IAAa,kBAAb,MAAsD;CACpD;CACA;CACA;CACA;CACA;CACA;CACA;CAEA,YAAY,SAAkC,CAAC,GAAG;EAChD,MAAM,cAAc,OAAO,UAAU,QAAQ,IAAI;EACjD,KAAK,SACH,eAAe,QAAQ,gBAAgB,KAAK,cAAc,KAAA;EAC5D,KAAK,SACH,OAAO,UACP,QAAQ,IAAI,uBACX,KAAK,UAAU,OACZ,yBACA;EACN,KAAK,UAAU,OAAO,WAAW;EACjC,KAAK,mBAAmB,OAAO,oBAAoB;EACnD,KAAK,YAAY,OAAO;EACxB,KAAK,aAAa,OAAO;EACzB,KAAK,SAAS,OAAO,UAAU,oBAAoB;CACrD;CAEA,eAA+C;;;;EAI7C,MAAM,UAAkC,EACtC,oBAAoB,KAAK,iBAC3B;EACA,IAAI,KAAK,UAAU,MACjB,QAAQ,eAAe,KAAK;EAE9B,OAAO;CACT;CAEA,MAAM,UACJ,KACA,UAAmC,CAAC,GACS;EAC7C,IAAI,CAAC,OAAO,CAAC,IAAI,KAAK,GACpB,OAAO,CAAC,KAAK;GAAE,SAAS;GAAO,OAAO;EAAsB,CAAC;EAG/D,IAAI;GASF,MAAM,QAAO,MARU,MAAM,IAA2B,KAAK,QAAQ;IACnE,QAAQ,EAAE,IAAI;IACd,SAAS,KAAK,aAAa;IAC3B,SAAS,QAAQ,WAAW,KAAK;IACjC,WAAW,KAAK;IAChB,YAAY,KAAK;GACnB,CAAC,EAAA,CAEqB;GACtB,MAAM,UAAU,KAAK,WAAW;GAChC,IAAI,CAAC,SACH,OAAO,CACL,KACA;IAAE,SAAS;IAAO,OAAO;GAAqC,CAChE;GAGF,OAAO,CACL,KACA;IACE,SAAS;IACT,MAAM;KACJ;KACA,OAAO,KAAK;KACZ,aAAa,KAAK;KAClB,KAAK,KAAK,OAAO;IACnB;GACF,CACF;EACF,SAAS,OAAO;GAGd,OAAO,CACL,KACA;IACE,SAAS;IACT,OAAO,sCALT,iBAAiB,QAAQ,MAAM,UAAU,OAAO,KAAK;GAMrD,CACF;EACF;CACF;CAEA,MAAM,WACJ,MACA,UAAmC,CAAC,GACgB;;EAEpD,OAAO,QAAQ,IAAI,KAAK,KAAK,QAAQ,KAAK,UAAU,KAAK,OAAO,CAAC,CAAC;CACpE;CAEA,eACE,UACoC;EACpC,IAAI,CAAC,SAAS,WAAW,CAAC,SAAS,MACjC,OAAO,CAAC,IAAI,KAAA,CAAS;;;EAIvB,OAAO,CAAC,SAAS,KAAK,SAAS,KAAA,CAAS;CAC1C;CAEA,gBAAgB,UAA6D;EAC3E,IAAI,CAAC,SAAS,WAAW,CAAC,SAAS,MACjC,OAAO,CAAC;EAEV,MAAM,WAAoC,CAAC;EAC3C,IAAI,SAAS,KAAK,SAAS,MACzB,SAAS,QAAQ,SAAS,KAAK;EAEjC,IAAI,SAAS,KAAK,eAAe,QAAQ,SAAS,KAAK,gBAAgB,IACrE,SAAS,cAAc,SAAS,KAAK;EAEvC,IAAI,SAAS,KAAK,OAAO,MACvB,SAAS,MAAM,SAAS,KAAK;EAE/B,OAAO;CACT;AACF;AAEA,MAAa,yBACX,SAAkC,CAAC,MACf;CACpB,OAAO,IAAI,gBAAgB,MAAM;AACnC"}
@@ -39,7 +39,9 @@ const createKeenableAPI = (apiKey, apiUrl, options) => {
39
39
  if (date != null) payload.published_after = KEENABLE_DATE_RANGES[date];
40
40
  const response = await axios.post(resolvedUrl, payload, {
41
41
  headers,
42
- timeout
42
+ timeout,
43
+ httpAgent: options?.httpAgent,
44
+ httpsAgent: options?.httpsAgent
43
45
  });
44
46
  const maxResults = Math.min(Math.max(1, options?.maxResults ?? numResults), 20);
45
47
  return {
@@ -1 +1 @@
1
- {"version":3,"file":"keenable-search.mjs","names":[],"sources":["../../../../src/tools/search/keenable-search.ts"],"sourcesContent":["import axios from 'axios';\nimport type * as t from './types';\nimport { DATE_RANGE } from './schema';\n\nconst DEFAULT_KEENABLE_TIMEOUT = 15000;\n\n/** Authenticated and keyless endpoints. Keenable works without an API key by\n * falling back to the public endpoint; a key only lifts rate limits. */\nconst KEENABLE_DEFAULT_API_URL = 'https://api.keenable.ai/v1/search';\nconst KEENABLE_PUBLIC_API_URL = 'https://api.keenable.ai/v1/search/public';\nconst KEENABLE_DATE_RANGES: Record<DATE_RANGE, string> = {\n [DATE_RANGE.PAST_HOUR]: '1h',\n [DATE_RANGE.PAST_24_HOURS]: '1d',\n [DATE_RANGE.PAST_WEEK]: '7d',\n [DATE_RANGE.PAST_MONTH]: '1mo',\n [DATE_RANGE.PAST_YEAR]: '1y',\n};\n\nexport const createKeenableAPI = (\n apiKey?: string,\n apiUrl?: string,\n options?: t.KeenableSearchOptions\n): {\n getSources: (params: t.GetSourcesParams) => Promise<t.SearchResult>;\n} => {\n const resolvedKey = apiKey ?? process.env.KEENABLE_API_KEY;\n const hasKey = resolvedKey != null && resolvedKey !== '';\n const timeout = options?.timeout ?? DEFAULT_KEENABLE_TIMEOUT;\n const resolvedUrl =\n apiUrl ??\n process.env.KEENABLE_API_URL ??\n (hasKey ? KEENABLE_DEFAULT_API_URL : KEENABLE_PUBLIC_API_URL);\n\n /** Constant for the provider's lifetime. X-Keenable-Title is required for\n * keyless requests and used for traffic attribution; the API key only lifts\n * rate limits, so it is sent only when present. */\n const headers: Record<string, string> = {\n 'Content-Type': 'application/json',\n 'X-Keenable-Title': options?.attributionTitle ?? 'LibreChat',\n };\n if (hasKey) {\n headers['X-API-Key'] = resolvedKey;\n }\n\n const getSources = async ({\n query,\n date,\n numResults = 8,\n }: t.GetSourcesParams): Promise<t.SearchResult> => {\n if (!query.trim()) {\n return { success: false, error: 'Query cannot be empty' };\n }\n\n try {\n /** Keenable's endpoint has no result-count parameter; the count is\n * applied client-side after the response (see slice below). */\n const payload: t.KeenableSearchPayload = { query };\n if (options?.site != null && options.site !== '') {\n payload.site = options.site;\n }\n if (date != null) {\n payload.published_after = KEENABLE_DATE_RANGES[date];\n }\n\n const response = await axios.post<t.KeenableSearchResponse>(\n resolvedUrl,\n payload,\n { headers, timeout }\n );\n\n const maxResults = Math.min(\n Math.max(1, options?.maxResults ?? numResults),\n 20\n );\n const rawResults = Array.isArray(response.data.results)\n ? response.data.results.slice(0, maxResults)\n : [];\n\n const organic: t.OrganicResult[] = rawResults.map((result) => ({\n title: result.title ?? '',\n link: result.url ?? '',\n snippet: result.description ?? result.snippet ?? '',\n date: result.published_at,\n }));\n\n return { success: true, data: { organic } };\n } catch (error) {\n const errorMessage =\n error instanceof Error ? error.message : String(error);\n return {\n success: false,\n error: `Keenable API request failed: ${errorMessage}`,\n };\n }\n };\n\n return { getSources };\n};\n"],"mappings":";;;AAIA,MAAM,2BAA2B;;;AAIjC,MAAM,2BAA2B;AACjC,MAAM,0BAA0B;AAChC,MAAM,uBAAmD;QAC/B;QACI;QACJ;QACC;QACD;AAC1B;AAEA,MAAa,qBACX,QACA,QACA,YAGG;CACH,MAAM,cAAc,UAAU,QAAQ,IAAI;CAC1C,MAAM,SAAS,eAAe,QAAQ,gBAAgB;CACtD,MAAM,UAAU,SAAS,WAAW;CACpC,MAAM,cACJ,UACA,QAAQ,IAAI,qBACX,SAAS,2BAA2B;;;;CAKvC,MAAM,UAAkC;EACtC,gBAAgB;EAChB,oBAAoB,SAAS,oBAAoB;CACnD;CACA,IAAI,QACF,QAAQ,eAAe;CAGzB,MAAM,aAAa,OAAO,EACxB,OACA,MACA,aAAa,QACoC;EACjD,IAAI,CAAC,MAAM,KAAK,GACd,OAAO;GAAE,SAAS;GAAO,OAAO;EAAwB;EAG1D,IAAI;;;GAGF,MAAM,UAAmC,EAAE,MAAM;GACjD,IAAI,SAAS,QAAQ,QAAQ,QAAQ,SAAS,IAC5C,QAAQ,OAAO,QAAQ;GAEzB,IAAI,QAAQ,MACV,QAAQ,kBAAkB,qBAAqB;GAGjD,MAAM,WAAW,MAAM,MAAM,KAC3B,aACA,SACA;IAAE;IAAS;GAAQ,CACrB;GAEA,MAAM,aAAa,KAAK,IACtB,KAAK,IAAI,GAAG,SAAS,cAAc,UAAU,GAC7C,EACF;GAYA,OAAO;IAAE,SAAS;IAAM,MAAM,EAAE,UAXb,MAAM,QAAQ,SAAS,KAAK,OAAO,IAClD,SAAS,KAAK,QAAQ,MAAM,GAAG,UAAU,IACzC,CAAC,EAAA,CAEyC,KAAK,YAAY;KAC7D,OAAO,OAAO,SAAS;KACvB,MAAM,OAAO,OAAO;KACpB,SAAS,OAAO,eAAe,OAAO,WAAW;KACjD,MAAM,OAAO;IACf,EAEsC,EAAE;GAAE;EAC5C,SAAS,OAAO;GAGd,OAAO;IACL,SAAS;IACT,OAAO,gCAHP,iBAAiB,QAAQ,MAAM,UAAU,OAAO,KAAK;GAIvD;EACF;CACF;CAEA,OAAO,EAAE,WAAW;AACtB"}
1
+ {"version":3,"file":"keenable-search.mjs","names":[],"sources":["../../../../src/tools/search/keenable-search.ts"],"sourcesContent":["import axios from 'axios';\nimport type * as t from './types';\nimport { DATE_RANGE } from './schema';\n\nconst DEFAULT_KEENABLE_TIMEOUT = 15000;\n\n/** Authenticated and keyless endpoints. Keenable works without an API key by\n * falling back to the public endpoint; a key only lifts rate limits. */\nconst KEENABLE_DEFAULT_API_URL = 'https://api.keenable.ai/v1/search';\nconst KEENABLE_PUBLIC_API_URL = 'https://api.keenable.ai/v1/search/public';\nconst KEENABLE_DATE_RANGES: Record<DATE_RANGE, string> = {\n [DATE_RANGE.PAST_HOUR]: '1h',\n [DATE_RANGE.PAST_24_HOURS]: '1d',\n [DATE_RANGE.PAST_WEEK]: '7d',\n [DATE_RANGE.PAST_MONTH]: '1mo',\n [DATE_RANGE.PAST_YEAR]: '1y',\n};\n\nexport const createKeenableAPI = (\n apiKey?: string,\n apiUrl?: string,\n options?: t.KeenableSearchOptions\n): {\n getSources: (params: t.GetSourcesParams) => Promise<t.SearchResult>;\n} => {\n const resolvedKey = apiKey ?? process.env.KEENABLE_API_KEY;\n const hasKey = resolvedKey != null && resolvedKey !== '';\n const timeout = options?.timeout ?? DEFAULT_KEENABLE_TIMEOUT;\n const resolvedUrl =\n apiUrl ??\n process.env.KEENABLE_API_URL ??\n (hasKey ? KEENABLE_DEFAULT_API_URL : KEENABLE_PUBLIC_API_URL);\n\n /** Constant for the provider's lifetime. X-Keenable-Title is required for\n * keyless requests and used for traffic attribution; the API key only lifts\n * rate limits, so it is sent only when present. */\n const headers: Record<string, string> = {\n 'Content-Type': 'application/json',\n 'X-Keenable-Title': options?.attributionTitle ?? 'LibreChat',\n };\n if (hasKey) {\n headers['X-API-Key'] = resolvedKey;\n }\n\n const getSources = async ({\n query,\n date,\n numResults = 8,\n }: t.GetSourcesParams): Promise<t.SearchResult> => {\n if (!query.trim()) {\n return { success: false, error: 'Query cannot be empty' };\n }\n\n try {\n /** Keenable's endpoint has no result-count parameter; the count is\n * applied client-side after the response (see slice below). */\n const payload: t.KeenableSearchPayload = { query };\n if (options?.site != null && options.site !== '') {\n payload.site = options.site;\n }\n if (date != null) {\n payload.published_after = KEENABLE_DATE_RANGES[date];\n }\n\n const response = await axios.post<t.KeenableSearchResponse>(\n resolvedUrl,\n payload,\n {\n headers,\n timeout,\n httpAgent: options?.httpAgent,\n httpsAgent: options?.httpsAgent,\n }\n );\n\n const maxResults = Math.min(\n Math.max(1, options?.maxResults ?? numResults),\n 20\n );\n const rawResults = Array.isArray(response.data.results)\n ? response.data.results.slice(0, maxResults)\n : [];\n\n const organic: t.OrganicResult[] = rawResults.map((result) => ({\n title: result.title ?? '',\n link: result.url ?? '',\n snippet: result.description ?? result.snippet ?? '',\n date: result.published_at,\n }));\n\n return { success: true, data: { organic } };\n } catch (error) {\n const errorMessage =\n error instanceof Error ? error.message : String(error);\n return {\n success: false,\n error: `Keenable API request failed: ${errorMessage}`,\n };\n }\n };\n\n return { getSources };\n};\n"],"mappings":";;;AAIA,MAAM,2BAA2B;;;AAIjC,MAAM,2BAA2B;AACjC,MAAM,0BAA0B;AAChC,MAAM,uBAAmD;QAC/B;QACI;QACJ;QACC;QACD;AAC1B;AAEA,MAAa,qBACX,QACA,QACA,YAGG;CACH,MAAM,cAAc,UAAU,QAAQ,IAAI;CAC1C,MAAM,SAAS,eAAe,QAAQ,gBAAgB;CACtD,MAAM,UAAU,SAAS,WAAW;CACpC,MAAM,cACJ,UACA,QAAQ,IAAI,qBACX,SAAS,2BAA2B;;;;CAKvC,MAAM,UAAkC;EACtC,gBAAgB;EAChB,oBAAoB,SAAS,oBAAoB;CACnD;CACA,IAAI,QACF,QAAQ,eAAe;CAGzB,MAAM,aAAa,OAAO,EACxB,OACA,MACA,aAAa,QACoC;EACjD,IAAI,CAAC,MAAM,KAAK,GACd,OAAO;GAAE,SAAS;GAAO,OAAO;EAAwB;EAG1D,IAAI;;;GAGF,MAAM,UAAmC,EAAE,MAAM;GACjD,IAAI,SAAS,QAAQ,QAAQ,QAAQ,SAAS,IAC5C,QAAQ,OAAO,QAAQ;GAEzB,IAAI,QAAQ,MACV,QAAQ,kBAAkB,qBAAqB;GAGjD,MAAM,WAAW,MAAM,MAAM,KAC3B,aACA,SACA;IACE;IACA;IACA,WAAW,SAAS;IACpB,YAAY,SAAS;GACvB,CACF;GAEA,MAAM,aAAa,KAAK,IACtB,KAAK,IAAI,GAAG,SAAS,cAAc,UAAU,GAC7C,EACF;GAYA,OAAO;IAAE,SAAS;IAAM,MAAM,EAAE,UAXb,MAAM,QAAQ,SAAS,KAAK,OAAO,IAClD,SAAS,KAAK,QAAQ,MAAM,GAAG,UAAU,IACzC,CAAC,EAAA,CAEyC,KAAK,YAAY;KAC7D,OAAO,OAAO,SAAS;KACvB,MAAM,OAAO,OAAO;KACpB,SAAS,OAAO,eAAe,OAAO,WAAW;KACjD,MAAM,OAAO;IACf,EAEsC,EAAE;GAAE;EAC5C,SAAS,OAAO;GAGd,OAAO;IACL,SAAS;IACT,OAAO,gCAHP,iBAAiB,QAAQ,MAAM,UAAU,OAAO,KAAK;GAIvD;EACF;CACF;CAEA,OAAO,EAAE,WAAW;AACtB"}