@illuma-ai/agents 1.4.0-alpha.6 → 1.5.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (653) hide show
  1. package/README.md +62 -0
  2. package/dist/cjs/agents/AgentContext.cjs +274 -67
  3. package/dist/cjs/agents/AgentContext.cjs.map +1 -1
  4. package/dist/cjs/common/enum.cjs +44 -13
  5. package/dist/cjs/common/enum.cjs.map +1 -1
  6. package/dist/cjs/graphs/Graph.cjs +182 -5
  7. package/dist/cjs/graphs/Graph.cjs.map +1 -1
  8. package/dist/cjs/graphs/MultiAgentGraph.cjs +152 -1167
  9. package/dist/cjs/graphs/MultiAgentGraph.cjs.map +1 -1
  10. package/dist/cjs/graphs/phases/memoryFlushPhase.cjs +1 -1
  11. package/dist/cjs/graphs/phases/memoryFlushPhase.cjs.map +1 -1
  12. package/dist/cjs/hooks/HookRegistry.cjs +162 -0
  13. package/dist/cjs/hooks/HookRegistry.cjs.map +1 -0
  14. package/dist/cjs/hooks/executeHooks.cjs +276 -0
  15. package/dist/cjs/hooks/executeHooks.cjs.map +1 -0
  16. package/dist/cjs/hooks/matchers.cjs +256 -0
  17. package/dist/cjs/hooks/matchers.cjs.map +1 -0
  18. package/dist/cjs/hooks/types.cjs +27 -0
  19. package/dist/cjs/hooks/types.cjs.map +1 -0
  20. package/dist/cjs/langchain/google-common.cjs +3 -0
  21. package/dist/cjs/langchain/google-common.cjs.map +1 -0
  22. package/dist/cjs/langchain/index.cjs +86 -0
  23. package/dist/cjs/langchain/index.cjs.map +1 -0
  24. package/dist/cjs/langchain/language_models/chat_models.cjs +3 -0
  25. package/dist/cjs/langchain/language_models/chat_models.cjs.map +1 -0
  26. package/dist/cjs/langchain/messages/tool.cjs +3 -0
  27. package/dist/cjs/langchain/messages/tool.cjs.map +1 -0
  28. package/dist/cjs/langchain/messages.cjs +51 -0
  29. package/dist/cjs/langchain/messages.cjs.map +1 -0
  30. package/dist/cjs/langchain/openai.cjs +3 -0
  31. package/dist/cjs/langchain/openai.cjs.map +1 -0
  32. package/dist/cjs/langchain/prompts.cjs +11 -0
  33. package/dist/cjs/langchain/prompts.cjs.map +1 -0
  34. package/dist/cjs/langchain/runnables.cjs +19 -0
  35. package/dist/cjs/langchain/runnables.cjs.map +1 -0
  36. package/dist/cjs/langchain/tools.cjs +23 -0
  37. package/dist/cjs/langchain/tools.cjs.map +1 -0
  38. package/dist/cjs/langchain/utils/env.cjs +11 -0
  39. package/dist/cjs/langchain/utils/env.cjs.map +1 -0
  40. package/dist/cjs/llm/anthropic/utils/message_inputs.cjs +5 -1
  41. package/dist/cjs/llm/anthropic/utils/message_inputs.cjs.map +1 -1
  42. package/dist/cjs/llm/bedrock/cacheSupport.cjs +55 -0
  43. package/dist/cjs/llm/bedrock/cacheSupport.cjs.map +1 -0
  44. package/dist/cjs/llm/bedrock/index.cjs +61 -33
  45. package/dist/cjs/llm/bedrock/index.cjs.map +1 -1
  46. package/dist/cjs/llm/openai/index.cjs +1 -4
  47. package/dist/cjs/llm/openai/index.cjs.map +1 -1
  48. package/dist/cjs/llm/openai/utils/index.cjs +27 -10
  49. package/dist/cjs/llm/openai/utils/index.cjs.map +1 -1
  50. package/dist/cjs/main.cjs +178 -127
  51. package/dist/cjs/main.cjs.map +1 -1
  52. package/dist/cjs/memory/citations.cjs +4 -4
  53. package/dist/cjs/memory/citations.cjs.map +1 -1
  54. package/dist/cjs/memory/constants.cjs +17 -17
  55. package/dist/cjs/memory/constants.cjs.map +1 -1
  56. package/dist/cjs/memory/mmr.cjs +1 -1
  57. package/dist/cjs/memory/mmr.cjs.map +1 -1
  58. package/dist/cjs/memory/paths.cjs +1 -1
  59. package/dist/cjs/memory/paths.cjs.map +1 -1
  60. package/dist/cjs/memory/recallTracking.cjs +3 -3
  61. package/dist/cjs/memory/recallTracking.cjs.map +1 -1
  62. package/dist/cjs/memory/temporalDecay.cjs +2 -2
  63. package/dist/cjs/memory/temporalDecay.cjs.map +1 -1
  64. package/dist/cjs/messages/cache.cjs +89 -0
  65. package/dist/cjs/messages/cache.cjs.map +1 -1
  66. package/dist/cjs/messages/contextPruning.cjs +156 -0
  67. package/dist/cjs/messages/contextPruning.cjs.map +1 -0
  68. package/dist/cjs/messages/contextPruningSettings.cjs +53 -0
  69. package/dist/cjs/messages/contextPruningSettings.cjs.map +1 -0
  70. package/dist/cjs/messages/format.cjs +144 -20
  71. package/dist/cjs/messages/format.cjs.map +1 -1
  72. package/dist/cjs/messages/prune.cjs +505 -4
  73. package/dist/cjs/messages/prune.cjs.map +1 -1
  74. package/dist/cjs/run.cjs +141 -1
  75. package/dist/cjs/run.cjs.map +1 -1
  76. package/dist/cjs/tools/BashExecutor.cjs +235 -0
  77. package/dist/cjs/tools/BashExecutor.cjs.map +1 -0
  78. package/dist/cjs/tools/BashProgrammaticToolCalling.cjs +297 -0
  79. package/dist/cjs/tools/BashProgrammaticToolCalling.cjs.map +1 -0
  80. package/dist/cjs/tools/CodeExecutor.cjs +45 -47
  81. package/dist/cjs/tools/CodeExecutor.cjs.map +1 -1
  82. package/dist/cjs/tools/ProgrammaticToolCalling.cjs +16 -11
  83. package/dist/cjs/tools/ProgrammaticToolCalling.cjs.map +1 -1
  84. package/dist/cjs/tools/ReadFile.cjs +44 -0
  85. package/dist/cjs/tools/ReadFile.cjs.map +1 -0
  86. package/dist/cjs/tools/SkillTool.cjs +51 -0
  87. package/dist/cjs/tools/SkillTool.cjs.map +1 -0
  88. package/dist/cjs/tools/SubagentTool.cjs +93 -0
  89. package/dist/cjs/tools/SubagentTool.cjs.map +1 -0
  90. package/dist/cjs/tools/ToolNode.cjs +450 -24
  91. package/dist/cjs/tools/ToolNode.cjs.map +1 -1
  92. package/dist/cjs/tools/memory/memoryAppendTool.cjs +1 -1
  93. package/dist/cjs/tools/memory/memoryAppendTool.cjs.map +1 -1
  94. package/dist/cjs/tools/memory/memoryGetTool.cjs +2 -2
  95. package/dist/cjs/tools/memory/memoryGetTool.cjs.map +1 -1
  96. package/dist/cjs/tools/memory/memorySearchTool.cjs +3 -3
  97. package/dist/cjs/tools/memory/memorySearchTool.cjs.map +1 -1
  98. package/dist/cjs/tools/memory/shared.cjs +1 -1
  99. package/dist/cjs/tools/memory/shared.cjs.map +1 -1
  100. package/dist/cjs/tools/search/search.cjs +11 -3
  101. package/dist/cjs/tools/search/search.cjs.map +1 -1
  102. package/dist/cjs/tools/search/tavily-scraper.cjs +189 -0
  103. package/dist/cjs/tools/search/tavily-scraper.cjs.map +1 -0
  104. package/dist/cjs/tools/search/tavily-search.cjs +372 -0
  105. package/dist/cjs/tools/search/tavily-search.cjs.map +1 -0
  106. package/dist/cjs/tools/search/tool.cjs +28 -4
  107. package/dist/cjs/tools/search/tool.cjs.map +1 -1
  108. package/dist/cjs/tools/search/utils.cjs +10 -3
  109. package/dist/cjs/tools/search/utils.cjs.map +1 -1
  110. package/dist/cjs/tools/skillCatalog.cjs +84 -0
  111. package/dist/cjs/tools/skillCatalog.cjs.map +1 -0
  112. package/dist/cjs/tools/subagent/SubagentExecutor.cjs +512 -0
  113. package/dist/cjs/tools/subagent/SubagentExecutor.cjs.map +1 -0
  114. package/dist/cjs/tools/toolOutputReferences.cjs +670 -0
  115. package/dist/cjs/tools/toolOutputReferences.cjs.map +1 -0
  116. package/dist/cjs/types/agent-cache.cjs +54 -0
  117. package/dist/cjs/types/agent-cache.cjs.map +1 -0
  118. package/dist/cjs/types/graph.cjs.map +1 -1
  119. package/dist/cjs/utils/truncation.cjs +135 -0
  120. package/dist/cjs/utils/truncation.cjs.map +1 -0
  121. package/dist/esm/agents/AgentContext.mjs +274 -67
  122. package/dist/esm/agents/AgentContext.mjs.map +1 -1
  123. package/dist/esm/common/enum.mjs +44 -12
  124. package/dist/esm/common/enum.mjs.map +1 -1
  125. package/dist/esm/graphs/Graph.mjs +182 -5
  126. package/dist/esm/graphs/Graph.mjs.map +1 -1
  127. package/dist/esm/graphs/MultiAgentGraph.mjs +155 -1170
  128. package/dist/esm/graphs/MultiAgentGraph.mjs.map +1 -1
  129. package/dist/esm/graphs/phases/memoryFlushPhase.mjs +1 -1
  130. package/dist/esm/graphs/phases/memoryFlushPhase.mjs.map +1 -1
  131. package/dist/esm/hooks/HookRegistry.mjs +160 -0
  132. package/dist/esm/hooks/HookRegistry.mjs.map +1 -0
  133. package/dist/esm/hooks/executeHooks.mjs +273 -0
  134. package/dist/esm/hooks/executeHooks.mjs.map +1 -0
  135. package/dist/esm/hooks/matchers.mjs +251 -0
  136. package/dist/esm/hooks/matchers.mjs.map +1 -0
  137. package/dist/esm/hooks/types.mjs +25 -0
  138. package/dist/esm/hooks/types.mjs.map +1 -0
  139. package/dist/esm/langchain/google-common.mjs +2 -0
  140. package/dist/esm/langchain/google-common.mjs.map +1 -0
  141. package/dist/esm/langchain/index.mjs +5 -0
  142. package/dist/esm/langchain/language_models/chat_models.mjs +2 -0
  143. package/dist/esm/langchain/language_models/chat_models.mjs.map +1 -0
  144. package/dist/esm/langchain/messages/tool.mjs +2 -0
  145. package/dist/esm/langchain/messages/tool.mjs.map +1 -0
  146. package/dist/esm/langchain/messages.mjs +2 -0
  147. package/dist/esm/langchain/messages.mjs.map +1 -0
  148. package/dist/esm/langchain/openai.mjs +2 -0
  149. package/dist/esm/langchain/openai.mjs.map +1 -0
  150. package/dist/esm/langchain/prompts.mjs +2 -0
  151. package/dist/esm/langchain/prompts.mjs.map +1 -0
  152. package/dist/esm/langchain/runnables.mjs +2 -0
  153. package/dist/esm/langchain/runnables.mjs.map +1 -0
  154. package/dist/esm/langchain/tools.mjs +2 -0
  155. package/dist/esm/langchain/tools.mjs.map +1 -0
  156. package/dist/esm/langchain/utils/env.mjs +2 -0
  157. package/dist/esm/langchain/utils/env.mjs.map +1 -0
  158. package/dist/esm/llm/anthropic/utils/message_inputs.mjs +5 -1
  159. package/dist/esm/llm/anthropic/utils/message_inputs.mjs.map +1 -1
  160. package/dist/esm/llm/bedrock/cacheSupport.mjs +52 -0
  161. package/dist/esm/llm/bedrock/cacheSupport.mjs.map +1 -0
  162. package/dist/esm/llm/bedrock/index.mjs +61 -34
  163. package/dist/esm/llm/bedrock/index.mjs.map +1 -1
  164. package/dist/esm/llm/openai/index.mjs +1 -4
  165. package/dist/esm/llm/openai/index.mjs.map +1 -1
  166. package/dist/esm/llm/openai/utils/index.mjs +27 -10
  167. package/dist/esm/llm/openai/utils/index.mjs.map +1 -1
  168. package/dist/esm/main.mjs +21 -27
  169. package/dist/esm/main.mjs.map +1 -1
  170. package/dist/esm/memory/citations.mjs +4 -4
  171. package/dist/esm/memory/citations.mjs.map +1 -1
  172. package/dist/esm/memory/constants.mjs +17 -17
  173. package/dist/esm/memory/constants.mjs.map +1 -1
  174. package/dist/esm/memory/mmr.mjs +1 -1
  175. package/dist/esm/memory/mmr.mjs.map +1 -1
  176. package/dist/esm/memory/paths.mjs +1 -1
  177. package/dist/esm/memory/paths.mjs.map +1 -1
  178. package/dist/esm/memory/recallTracking.mjs +3 -3
  179. package/dist/esm/memory/recallTracking.mjs.map +1 -1
  180. package/dist/esm/memory/temporalDecay.mjs +2 -2
  181. package/dist/esm/memory/temporalDecay.mjs.map +1 -1
  182. package/dist/esm/messages/cache.mjs +89 -0
  183. package/dist/esm/messages/cache.mjs.map +1 -1
  184. package/dist/esm/messages/contextPruning.mjs +154 -0
  185. package/dist/esm/messages/contextPruning.mjs.map +1 -0
  186. package/dist/esm/messages/contextPruningSettings.mjs +50 -0
  187. package/dist/esm/messages/contextPruningSettings.mjs.map +1 -0
  188. package/dist/esm/messages/format.mjs +136 -12
  189. package/dist/esm/messages/format.mjs.map +1 -1
  190. package/dist/esm/messages/prune.mjs +504 -7
  191. package/dist/esm/messages/prune.mjs.map +1 -1
  192. package/dist/esm/run.mjs +141 -1
  193. package/dist/esm/run.mjs.map +1 -1
  194. package/dist/esm/tools/BashExecutor.mjs +227 -0
  195. package/dist/esm/tools/BashExecutor.mjs.map +1 -0
  196. package/dist/esm/tools/BashProgrammaticToolCalling.mjs +288 -0
  197. package/dist/esm/tools/BashProgrammaticToolCalling.mjs.map +1 -0
  198. package/dist/esm/tools/CodeExecutor.mjs +45 -48
  199. package/dist/esm/tools/CodeExecutor.mjs.map +1 -1
  200. package/dist/esm/tools/ProgrammaticToolCalling.mjs +17 -12
  201. package/dist/esm/tools/ProgrammaticToolCalling.mjs.map +1 -1
  202. package/dist/esm/tools/ReadFile.mjs +39 -0
  203. package/dist/esm/tools/ReadFile.mjs.map +1 -0
  204. package/dist/esm/tools/SkillTool.mjs +46 -0
  205. package/dist/esm/tools/SkillTool.mjs.map +1 -0
  206. package/dist/esm/tools/SubagentTool.mjs +86 -0
  207. package/dist/esm/tools/SubagentTool.mjs.map +1 -0
  208. package/dist/esm/tools/ToolNode.mjs +452 -26
  209. package/dist/esm/tools/ToolNode.mjs.map +1 -1
  210. package/dist/esm/tools/memory/memoryAppendTool.mjs +1 -1
  211. package/dist/esm/tools/memory/memoryAppendTool.mjs.map +1 -1
  212. package/dist/esm/tools/memory/memoryGetTool.mjs +2 -2
  213. package/dist/esm/tools/memory/memoryGetTool.mjs.map +1 -1
  214. package/dist/esm/tools/memory/memorySearchTool.mjs +3 -3
  215. package/dist/esm/tools/memory/memorySearchTool.mjs.map +1 -1
  216. package/dist/esm/tools/memory/shared.mjs +1 -1
  217. package/dist/esm/tools/memory/shared.mjs.map +1 -1
  218. package/dist/esm/tools/search/search.mjs +11 -3
  219. package/dist/esm/tools/search/search.mjs.map +1 -1
  220. package/dist/esm/tools/search/tavily-scraper.mjs +186 -0
  221. package/dist/esm/tools/search/tavily-scraper.mjs.map +1 -0
  222. package/dist/esm/tools/search/tavily-search.mjs +370 -0
  223. package/dist/esm/tools/search/tavily-search.mjs.map +1 -0
  224. package/dist/esm/tools/search/tool.mjs +28 -4
  225. package/dist/esm/tools/search/tool.mjs.map +1 -1
  226. package/dist/esm/tools/search/utils.mjs +10 -3
  227. package/dist/esm/tools/search/utils.mjs.map +1 -1
  228. package/dist/esm/tools/skillCatalog.mjs +82 -0
  229. package/dist/esm/tools/skillCatalog.mjs.map +1 -0
  230. package/dist/esm/tools/subagent/SubagentExecutor.mjs +506 -0
  231. package/dist/esm/tools/subagent/SubagentExecutor.mjs.map +1 -0
  232. package/dist/esm/tools/toolOutputReferences.mjs +662 -0
  233. package/dist/esm/tools/toolOutputReferences.mjs.map +1 -0
  234. package/dist/esm/types/agent-cache.mjs +52 -0
  235. package/dist/esm/types/agent-cache.mjs.map +1 -0
  236. package/dist/esm/types/graph.mjs.map +1 -1
  237. package/dist/esm/utils/truncation.mjs +128 -0
  238. package/dist/esm/utils/truncation.mjs.map +1 -0
  239. package/dist/types/agents/AgentContext.d.ts +101 -8
  240. package/dist/types/common/enum.d.ts +39 -12
  241. package/dist/types/common/index.d.ts +0 -1
  242. package/dist/types/graphs/Graph.d.ts +43 -0
  243. package/dist/types/graphs/MultiAgentGraph.d.ts +26 -150
  244. package/dist/types/graphs/index.d.ts +0 -1
  245. package/dist/types/graphs/phases/memoryFlushPhase.d.ts +2 -2
  246. package/dist/types/hooks/HookRegistry.d.ts +56 -0
  247. package/dist/types/hooks/executeHooks.d.ts +79 -0
  248. package/dist/types/hooks/index.d.ts +6 -0
  249. package/dist/types/hooks/matchers.d.ts +95 -0
  250. package/dist/types/hooks/types.d.ts +320 -0
  251. package/dist/types/index.d.ts +9 -9
  252. package/dist/types/langchain/google-common.d.ts +1 -0
  253. package/dist/types/langchain/index.d.ts +8 -0
  254. package/dist/types/langchain/language_models/chat_models.d.ts +1 -0
  255. package/dist/types/langchain/messages/tool.d.ts +1 -0
  256. package/dist/types/langchain/messages.d.ts +2 -0
  257. package/dist/types/langchain/openai.d.ts +1 -0
  258. package/dist/types/langchain/prompts.d.ts +1 -0
  259. package/dist/types/langchain/runnables.d.ts +2 -0
  260. package/dist/types/langchain/tools.d.ts +2 -0
  261. package/dist/types/langchain/utils/env.d.ts +1 -0
  262. package/dist/types/llm/bedrock/cacheSupport.d.ts +35 -0
  263. package/dist/types/llm/bedrock/index.d.ts +54 -1
  264. package/dist/types/llm/openai/index.d.ts +1 -1
  265. package/dist/types/memory/citations.d.ts +4 -4
  266. package/dist/types/memory/constants.d.ts +17 -17
  267. package/dist/types/memory/mmr.d.ts +3 -3
  268. package/dist/types/memory/paths.d.ts +1 -1
  269. package/dist/types/memory/temporalDecay.d.ts +2 -2
  270. package/dist/types/memory/types.d.ts +3 -3
  271. package/dist/types/messages/contextPruning.d.ts +42 -0
  272. package/dist/types/messages/contextPruningSettings.d.ts +44 -0
  273. package/dist/types/messages/format.d.ts +9 -1
  274. package/dist/types/messages/index.d.ts +2 -0
  275. package/dist/types/messages/prune.d.ts +91 -1
  276. package/dist/types/run.d.ts +2 -0
  277. package/dist/types/tools/BashExecutor.d.ts +76 -0
  278. package/dist/types/tools/BashProgrammaticToolCalling.d.ts +72 -0
  279. package/dist/types/tools/CodeExecutor.d.ts +8 -26
  280. package/dist/types/tools/ReadFile.d.ts +28 -0
  281. package/dist/types/tools/SkillTool.d.ts +40 -0
  282. package/dist/types/tools/SubagentTool.d.ts +36 -0
  283. package/dist/types/tools/ToolNode.d.ts +77 -5
  284. package/dist/types/tools/memory/shared.d.ts +1 -1
  285. package/dist/types/tools/search/tavily-scraper.d.ts +19 -0
  286. package/dist/types/tools/search/tavily-search.d.ts +4 -0
  287. package/dist/types/tools/search/types.d.ts +99 -5
  288. package/dist/types/tools/search/utils.d.ts +2 -2
  289. package/dist/types/tools/skillCatalog.d.ts +19 -0
  290. package/dist/types/tools/subagent/SubagentExecutor.d.ts +137 -0
  291. package/dist/types/tools/subagent/index.d.ts +2 -0
  292. package/dist/types/tools/subagent/types.d.ts +84 -0
  293. package/dist/types/tools/toolOutputReferences.d.ts +236 -0
  294. package/dist/types/types/agent-cache.d.ts +71 -0
  295. package/dist/types/types/graph.d.ts +163 -22
  296. package/dist/types/types/index.d.ts +3 -0
  297. package/dist/types/types/messages.d.ts +26 -0
  298. package/dist/types/types/run.d.ts +22 -0
  299. package/dist/types/types/skill.d.ts +9 -0
  300. package/dist/types/types/tools.d.ts +111 -0
  301. package/dist/types/utils/index.d.ts +1 -3
  302. package/dist/types/utils/truncation.d.ts +70 -0
  303. package/package.json +57 -17
  304. package/src/agents/AgentContext.ts +321 -78
  305. package/src/agents/__tests__/AgentContext.cacheTtl.live.test.ts +259 -0
  306. package/src/agents/__tests__/AgentContext.crossAgentTier1.live.test.ts +266 -0
  307. package/src/agents/__tests__/AgentContext.crossUserCache.live.test.ts +342 -0
  308. package/src/agents/__tests__/AgentContext.test.ts +632 -0
  309. package/src/common/__tests__/enum.test.ts +7 -17
  310. package/src/common/enum.ts +43 -12
  311. package/src/common/index.ts +0 -1
  312. package/src/graphs/Graph.ts +222 -2
  313. package/src/graphs/MultiAgentGraph.ts +154 -1466
  314. package/src/graphs/__tests__/MultiAgentGraph.test.ts +91 -0
  315. package/src/graphs/gapFeatures.test.ts +1 -1
  316. package/src/graphs/index.ts +0 -1
  317. package/src/graphs/phases/__tests__/memoryFlushPhase.test.ts +1 -1
  318. package/src/graphs/phases/memoryFlushPhase.ts +2 -2
  319. package/src/hooks/HookRegistry.ts +208 -0
  320. package/src/hooks/__tests__/HookRegistry.test.ts +190 -0
  321. package/src/hooks/__tests__/compactHooks.test.ts +214 -0
  322. package/src/hooks/__tests__/executeHooks.test.ts +1013 -0
  323. package/src/hooks/__tests__/integration.test.ts +337 -0
  324. package/src/hooks/__tests__/matchers.test.ts +238 -0
  325. package/src/hooks/__tests__/toolHooks.test.ts +665 -0
  326. package/src/hooks/executeHooks.ts +375 -0
  327. package/src/hooks/index.ts +57 -0
  328. package/src/hooks/matchers.ts +280 -0
  329. package/src/hooks/types.ts +404 -0
  330. package/src/index.ts +15 -24
  331. package/src/langchain/google-common.ts +1 -0
  332. package/src/langchain/index.ts +8 -0
  333. package/src/langchain/language_models/chat_models.ts +1 -0
  334. package/src/langchain/messages/tool.ts +5 -0
  335. package/src/langchain/messages.ts +21 -0
  336. package/src/langchain/openai.ts +1 -0
  337. package/src/langchain/prompts.ts +1 -0
  338. package/src/langchain/runnables.ts +7 -0
  339. package/src/langchain/tools.ts +8 -0
  340. package/src/langchain/utils/env.ts +1 -0
  341. package/src/llm/anthropic/utils/message_inputs.ts +10 -1
  342. package/src/llm/anthropic/utils/server-tool-inputs.test.ts +436 -0
  343. package/src/llm/bedrock/__tests__/bedrock-caching.test.ts +166 -18
  344. package/src/llm/bedrock/cacheSupport.test.ts +99 -0
  345. package/src/llm/bedrock/cacheSupport.ts +53 -0
  346. package/src/llm/bedrock/index.ts +116 -41
  347. package/src/llm/openai/index.ts +2 -2
  348. package/src/llm/openai/utils/index.ts +31 -14
  349. package/src/memory/citations.ts +4 -4
  350. package/src/memory/constants.ts +17 -17
  351. package/src/memory/mmr.ts +3 -3
  352. package/src/memory/paths.ts +1 -1
  353. package/src/memory/recallTracking.ts +3 -3
  354. package/src/memory/temporalDecay.ts +2 -2
  355. package/src/memory/types.ts +3 -3
  356. package/src/messages/__tests__/contextPruning.test.ts +228 -0
  357. package/src/messages/cache.test.ts +62 -24
  358. package/src/messages/cache.ts +112 -0
  359. package/src/messages/contextPruning.ts +191 -0
  360. package/src/messages/contextPruningSettings.ts +90 -0
  361. package/src/messages/ensureThinkingBlock.test.ts +1 -1
  362. package/src/messages/format.ts +164 -12
  363. package/src/messages/formatAgentMessages.skills.test.ts +413 -0
  364. package/src/messages/formatAgentMessages.test.ts +1 -1
  365. package/src/messages/index.ts +2 -0
  366. package/src/messages/prune.ts +661 -4
  367. package/src/run.ts +155 -1
  368. package/src/scripts/multi-agent-chain.ts +2 -2
  369. package/src/scripts/multi-agent-document-review-chain.ts +2 -2
  370. package/src/scripts/multi-agent-hybrid-flow.ts +4 -4
  371. package/src/scripts/multi-agent-parallel.ts +3 -3
  372. package/src/scripts/multi-agent-sequence.ts +3 -3
  373. package/src/scripts/multi-agent-subagent.ts +246 -0
  374. package/src/scripts/multi-agent-supervisor.ts +5 -5
  375. package/src/scripts/poc-multi-agent-comprehensive.ts +8 -8
  376. package/src/scripts/sequential-full-metadata-test.ts +2 -2
  377. package/src/scripts/subagent-event-driven-debug.ts +190 -0
  378. package/src/scripts/subagent-tools-debug.ts +160 -0
  379. package/src/scripts/test-custom-prompt-key.ts +3 -3
  380. package/src/scripts/test-handoff-input.ts +1 -1
  381. package/src/scripts/test-handoff-steering.ts +3 -3
  382. package/src/scripts/test-multi-agent-list-handoff.ts +1 -1
  383. package/src/scripts/test-parallel-agent-labeling.ts +3 -3
  384. package/src/scripts/test-parallel-handoffs.ts +2 -2
  385. package/src/scripts/test-thinking-handoff-bedrock.ts +1 -1
  386. package/src/scripts/test-thinking-handoff.ts +1 -1
  387. package/src/scripts/test-thinking-to-thinking-handoff-bedrock.ts +1 -1
  388. package/src/scripts/test-tool-before-handoff-role-order.ts +1 -1
  389. package/src/scripts/test-tools-before-handoff.ts +1 -1
  390. package/src/specs/agent-handoffs.test.ts +26 -483
  391. package/src/specs/anthropic.simple.test.ts +61 -0
  392. package/src/specs/multi-agent-summarization.test.ts +396 -0
  393. package/src/specs/prune.orphans.test.ts +248 -0
  394. package/src/specs/prune.test.ts +104 -16
  395. package/src/specs/thinking-handoff.test.ts +19 -19
  396. package/src/tools/BashExecutor.ts +281 -0
  397. package/src/tools/BashProgrammaticToolCalling.ts +397 -0
  398. package/src/tools/CodeExecutor.ts +63 -54
  399. package/src/tools/ProgrammaticToolCalling.ts +29 -14
  400. package/src/tools/ReadFile.ts +39 -0
  401. package/src/tools/SkillTool.ts +46 -0
  402. package/src/tools/SubagentTool.ts +100 -0
  403. package/src/tools/ToolNode.ts +548 -26
  404. package/src/tools/__tests__/BashExecutor.test.ts +49 -0
  405. package/src/tools/__tests__/CodeExecutor.test.ts +37 -36
  406. package/src/tools/__tests__/ProgrammaticToolCalling.test.ts +60 -0
  407. package/src/tools/__tests__/ReadFile.test.ts +44 -0
  408. package/src/tools/__tests__/SkillTool.test.ts +442 -0
  409. package/src/tools/__tests__/SubagentExecutor.test.ts +1148 -0
  410. package/src/tools/__tests__/SubagentTool.test.ts +149 -0
  411. package/src/tools/__tests__/ToolNode.outputReferences.test.ts +1438 -0
  412. package/src/tools/__tests__/annotateMessagesForLLM.test.ts +479 -0
  413. package/src/tools/__tests__/skillCatalog.test.ts +161 -0
  414. package/src/tools/__tests__/subagentHooks.test.ts +210 -0
  415. package/src/tools/__tests__/toolOutputReferences.test.ts +415 -0
  416. package/src/tools/memory/memoryAppendTool.ts +1 -1
  417. package/src/tools/memory/memoryGetTool.ts +2 -2
  418. package/src/tools/memory/memorySearchTool.ts +3 -3
  419. package/src/tools/memory/shared.ts +1 -1
  420. package/src/tools/search/search.ts +12 -2
  421. package/src/tools/search/tavily-scraper.ts +235 -0
  422. package/src/tools/search/tavily-search.ts +424 -0
  423. package/src/tools/search/tavily.test.ts +965 -0
  424. package/src/tools/search/tool.ts +36 -2
  425. package/src/tools/search/types.ts +133 -8
  426. package/src/tools/search/utils.ts +13 -5
  427. package/src/tools/skillCatalog.ts +126 -0
  428. package/src/tools/subagent/SubagentExecutor.ts +676 -0
  429. package/src/tools/subagent/index.ts +13 -0
  430. package/src/tools/subagent/types.test.ts +70 -0
  431. package/src/tools/subagent/types.ts +115 -0
  432. package/src/tools/toolOutputReferences.ts +825 -0
  433. package/src/types/agent-cache.ts +74 -0
  434. package/src/types/graph.ts +172 -20
  435. package/src/types/index.ts +3 -0
  436. package/src/types/messages.ts +27 -0
  437. package/src/types/run.ts +22 -0
  438. package/src/types/skill.ts +11 -0
  439. package/src/types/tools.ts +118 -0
  440. package/src/utils/__tests__/truncation.test.ts +66 -0
  441. package/src/utils/index.ts +1 -3
  442. package/src/utils/truncation.ts +154 -0
  443. package/dist/cjs/common/spawnPath.cjs +0 -104
  444. package/dist/cjs/common/spawnPath.cjs.map +0 -1
  445. package/dist/cjs/content/ArtifactStore.cjs +0 -579
  446. package/dist/cjs/content/ArtifactStore.cjs.map +0 -1
  447. package/dist/cjs/content/ContentStore.cjs +0 -638
  448. package/dist/cjs/content/ContentStore.cjs.map +0 -1
  449. package/dist/cjs/content/contentAnalyzer.cjs +0 -91
  450. package/dist/cjs/content/contentAnalyzer.cjs.map +0 -1
  451. package/dist/cjs/content/index.cjs +0 -20
  452. package/dist/cjs/content/index.cjs.map +0 -1
  453. package/dist/cjs/content/mcpAutoCache.cjs +0 -115
  454. package/dist/cjs/content/mcpAutoCache.cjs.map +0 -1
  455. package/dist/cjs/graphs/HandoffRegistry.cjs +0 -143
  456. package/dist/cjs/graphs/HandoffRegistry.cjs.map +0 -1
  457. package/dist/cjs/providers/a2a/A2ACapabilityProvider.cjs +0 -288
  458. package/dist/cjs/providers/a2a/A2ACapabilityProvider.cjs.map +0 -1
  459. package/dist/cjs/providers/a2a/client.cjs +0 -92
  460. package/dist/cjs/providers/a2a/client.cjs.map +0 -1
  461. package/dist/cjs/providers/a2a/config.cjs +0 -38
  462. package/dist/cjs/providers/a2a/config.cjs.map +0 -1
  463. package/dist/cjs/providers/capabilityNaming.cjs +0 -43
  464. package/dist/cjs/providers/capabilityNaming.cjs.map +0 -1
  465. package/dist/cjs/providers/mcp/MCPCapabilityProvider.cjs +0 -244
  466. package/dist/cjs/providers/mcp/MCPCapabilityProvider.cjs.map +0 -1
  467. package/dist/cjs/providers/mcp/config.cjs +0 -42
  468. package/dist/cjs/providers/mcp/config.cjs.map +0 -1
  469. package/dist/cjs/providers/mcp/transport.cjs +0 -65
  470. package/dist/cjs/providers/mcp/transport.cjs.map +0 -1
  471. package/dist/cjs/providers/tools-server/ToolsServerCapabilityProvider.cjs +0 -128
  472. package/dist/cjs/providers/tools-server/ToolsServerCapabilityProvider.cjs.map +0 -1
  473. package/dist/cjs/providers/types.cjs +0 -51
  474. package/dist/cjs/providers/types.cjs.map +0 -1
  475. package/dist/cjs/tools/artifacts/schema.cjs +0 -86
  476. package/dist/cjs/tools/artifacts/schema.cjs.map +0 -1
  477. package/dist/cjs/tools/artifacts/tool.cjs +0 -219
  478. package/dist/cjs/tools/artifacts/tool.cjs.map +0 -1
  479. package/dist/cjs/tools/fileSearch/formatter.cjs +0 -93
  480. package/dist/cjs/tools/fileSearch/formatter.cjs.map +0 -1
  481. package/dist/cjs/tools/fileSearch/ragClient.cjs +0 -102
  482. package/dist/cjs/tools/fileSearch/ragClient.cjs.map +0 -1
  483. package/dist/cjs/tools/fileSearch/schema.cjs +0 -18
  484. package/dist/cjs/tools/fileSearch/schema.cjs.map +0 -1
  485. package/dist/cjs/tools/fileSearch/tool.cjs +0 -155
  486. package/dist/cjs/tools/fileSearch/tool.cjs.map +0 -1
  487. package/dist/cjs/tools/proxyTool.cjs +0 -102
  488. package/dist/cjs/tools/proxyTool.cjs.map +0 -1
  489. package/dist/cjs/utils/childAgentContext.cjs +0 -242
  490. package/dist/cjs/utils/childAgentContext.cjs.map +0 -1
  491. package/dist/cjs/utils/credentials.cjs +0 -142
  492. package/dist/cjs/utils/credentials.cjs.map +0 -1
  493. package/dist/cjs/utils/httpClient.cjs +0 -74
  494. package/dist/cjs/utils/httpClient.cjs.map +0 -1
  495. package/dist/cjs/utils/toolManifest.cjs +0 -100
  496. package/dist/cjs/utils/toolManifest.cjs.map +0 -1
  497. package/dist/esm/common/spawnPath.mjs +0 -95
  498. package/dist/esm/common/spawnPath.mjs.map +0 -1
  499. package/dist/esm/content/ArtifactStore.mjs +0 -576
  500. package/dist/esm/content/ArtifactStore.mjs.map +0 -1
  501. package/dist/esm/content/ContentStore.mjs +0 -635
  502. package/dist/esm/content/ContentStore.mjs.map +0 -1
  503. package/dist/esm/content/contentAnalyzer.mjs +0 -87
  504. package/dist/esm/content/contentAnalyzer.mjs.map +0 -1
  505. package/dist/esm/content/index.mjs +0 -5
  506. package/dist/esm/content/mcpAutoCache.mjs +0 -111
  507. package/dist/esm/content/mcpAutoCache.mjs.map +0 -1
  508. package/dist/esm/graphs/HandoffRegistry.mjs +0 -141
  509. package/dist/esm/graphs/HandoffRegistry.mjs.map +0 -1
  510. package/dist/esm/providers/a2a/A2ACapabilityProvider.mjs +0 -281
  511. package/dist/esm/providers/a2a/A2ACapabilityProvider.mjs.map +0 -1
  512. package/dist/esm/providers/a2a/client.mjs +0 -88
  513. package/dist/esm/providers/a2a/client.mjs.map +0 -1
  514. package/dist/esm/providers/a2a/config.mjs +0 -35
  515. package/dist/esm/providers/a2a/config.mjs.map +0 -1
  516. package/dist/esm/providers/capabilityNaming.mjs +0 -39
  517. package/dist/esm/providers/capabilityNaming.mjs.map +0 -1
  518. package/dist/esm/providers/mcp/MCPCapabilityProvider.mjs +0 -240
  519. package/dist/esm/providers/mcp/MCPCapabilityProvider.mjs.map +0 -1
  520. package/dist/esm/providers/mcp/config.mjs +0 -39
  521. package/dist/esm/providers/mcp/config.mjs.map +0 -1
  522. package/dist/esm/providers/mcp/transport.mjs +0 -63
  523. package/dist/esm/providers/mcp/transport.mjs.map +0 -1
  524. package/dist/esm/providers/tools-server/ToolsServerCapabilityProvider.mjs +0 -126
  525. package/dist/esm/providers/tools-server/ToolsServerCapabilityProvider.mjs.map +0 -1
  526. package/dist/esm/providers/types.mjs +0 -51
  527. package/dist/esm/providers/types.mjs.map +0 -1
  528. package/dist/esm/tools/artifacts/schema.mjs +0 -79
  529. package/dist/esm/tools/artifacts/schema.mjs.map +0 -1
  530. package/dist/esm/tools/artifacts/tool.mjs +0 -213
  531. package/dist/esm/tools/artifacts/tool.mjs.map +0 -1
  532. package/dist/esm/tools/fileSearch/formatter.mjs +0 -90
  533. package/dist/esm/tools/fileSearch/formatter.mjs.map +0 -1
  534. package/dist/esm/tools/fileSearch/ragClient.mjs +0 -98
  535. package/dist/esm/tools/fileSearch/ragClient.mjs.map +0 -1
  536. package/dist/esm/tools/fileSearch/schema.mjs +0 -15
  537. package/dist/esm/tools/fileSearch/schema.mjs.map +0 -1
  538. package/dist/esm/tools/fileSearch/tool.mjs +0 -152
  539. package/dist/esm/tools/fileSearch/tool.mjs.map +0 -1
  540. package/dist/esm/tools/proxyTool.mjs +0 -100
  541. package/dist/esm/tools/proxyTool.mjs.map +0 -1
  542. package/dist/esm/utils/childAgentContext.mjs +0 -237
  543. package/dist/esm/utils/childAgentContext.mjs.map +0 -1
  544. package/dist/esm/utils/credentials.mjs +0 -135
  545. package/dist/esm/utils/credentials.mjs.map +0 -1
  546. package/dist/esm/utils/httpClient.mjs +0 -70
  547. package/dist/esm/utils/httpClient.mjs.map +0 -1
  548. package/dist/esm/utils/toolManifest.mjs +0 -96
  549. package/dist/esm/utils/toolManifest.mjs.map +0 -1
  550. package/dist/types/common/spawnPath.d.ts +0 -59
  551. package/dist/types/content/ArtifactStore.d.ts +0 -223
  552. package/dist/types/content/ContentStore.d.ts +0 -140
  553. package/dist/types/content/contentAnalyzer.d.ts +0 -38
  554. package/dist/types/content/index.d.ts +0 -24
  555. package/dist/types/content/mcpAutoCache.d.ts +0 -89
  556. package/dist/types/content/types.d.ts +0 -75
  557. package/dist/types/graphs/HandoffRegistry.d.ts +0 -97
  558. package/dist/types/providers/a2a/A2ACapabilityProvider.d.ts +0 -89
  559. package/dist/types/providers/a2a/client.d.ts +0 -47
  560. package/dist/types/providers/a2a/config.d.ts +0 -18
  561. package/dist/types/providers/a2a/index.d.ts +0 -6
  562. package/dist/types/providers/a2a/types.d.ts +0 -173
  563. package/dist/types/providers/capabilityNaming.d.ts +0 -25
  564. package/dist/types/providers/index.d.ts +0 -12
  565. package/dist/types/providers/mcp/MCPCapabilityProvider.d.ts +0 -54
  566. package/dist/types/providers/mcp/config.d.ts +0 -20
  567. package/dist/types/providers/mcp/index.d.ts +0 -5
  568. package/dist/types/providers/mcp/transport.d.ts +0 -18
  569. package/dist/types/providers/mcp/types.d.ts +0 -112
  570. package/dist/types/providers/tools-server/ToolsServerCapabilityProvider.d.ts +0 -59
  571. package/dist/types/providers/tools-server/index.d.ts +0 -1
  572. package/dist/types/providers/types.d.ts +0 -184
  573. package/dist/types/tools/artifacts/index.d.ts +0 -3
  574. package/dist/types/tools/artifacts/schema.d.ts +0 -63
  575. package/dist/types/tools/artifacts/tool.d.ts +0 -16
  576. package/dist/types/tools/artifacts/types.d.ts +0 -127
  577. package/dist/types/tools/fileSearch/formatter.d.ts +0 -25
  578. package/dist/types/tools/fileSearch/index.d.ts +0 -5
  579. package/dist/types/tools/fileSearch/ragClient.d.ts +0 -32
  580. package/dist/types/tools/fileSearch/schema.d.ts +0 -13
  581. package/dist/types/tools/fileSearch/tool.d.ts +0 -18
  582. package/dist/types/tools/fileSearch/types.d.ts +0 -139
  583. package/dist/types/tools/proxyTool.d.ts +0 -62
  584. package/dist/types/tools/search/test.d.ts +0 -1
  585. package/dist/types/utils/childAgentContext.d.ts +0 -99
  586. package/dist/types/utils/credentials.d.ts +0 -77
  587. package/dist/types/utils/httpClient.d.ts +0 -46
  588. package/dist/types/utils/toolManifest.d.ts +0 -49
  589. package/src/common/__tests__/spawnPath.test.ts +0 -110
  590. package/src/common/spawnPath.ts +0 -101
  591. package/src/content/ArtifactStore.ts +0 -782
  592. package/src/content/ContentStore.ts +0 -753
  593. package/src/content/contentAnalyzer.ts +0 -105
  594. package/src/content/index.ts +0 -51
  595. package/src/content/mcpAutoCache.ts +0 -185
  596. package/src/content/types.ts +0 -82
  597. package/src/graphs/HandoffRegistry.ts +0 -199
  598. package/src/graphs/__tests__/HandoffRegistry.test.ts +0 -410
  599. package/src/graphs/__tests__/multi-agent-delegate.test.ts +0 -458
  600. package/src/graphs/__tests__/multi-agent-edges.test.ts +0 -276
  601. package/src/graphs/__tests__/multi-agent-nested-subgraph.test.ts +0 -221
  602. package/src/graphs/handoffValidation.test.ts +0 -353
  603. package/src/providers/__tests__/ToolsServerCapabilityProvider.integration.spec.ts +0 -79
  604. package/src/providers/__tests__/ToolsServerCapabilityProvider.test.ts +0 -271
  605. package/src/providers/__tests__/types.test.ts +0 -64
  606. package/src/providers/a2a/A2ACapabilityProvider.ts +0 -384
  607. package/src/providers/a2a/__tests__/A2ACapabilityProvider.integration.spec.ts +0 -345
  608. package/src/providers/a2a/__tests__/A2ACapabilityProvider.test.ts +0 -460
  609. package/src/providers/a2a/client.ts +0 -115
  610. package/src/providers/a2a/config.ts +0 -40
  611. package/src/providers/a2a/index.ts +0 -29
  612. package/src/providers/a2a/types.ts +0 -191
  613. package/src/providers/capabilityNaming.ts +0 -42
  614. package/src/providers/index.ts +0 -68
  615. package/src/providers/mcp/MCPCapabilityProvider.ts +0 -345
  616. package/src/providers/mcp/__tests__/MCPCapabilityProvider.integration.spec.ts +0 -386
  617. package/src/providers/mcp/__tests__/MCPCapabilityProvider.test.ts +0 -371
  618. package/src/providers/mcp/config.ts +0 -45
  619. package/src/providers/mcp/index.ts +0 -21
  620. package/src/providers/mcp/transport.ts +0 -76
  621. package/src/providers/mcp/types.ts +0 -139
  622. package/src/providers/tools-server/ToolsServerCapabilityProvider.ts +0 -249
  623. package/src/providers/tools-server/index.ts +0 -1
  624. package/src/providers/types.ts +0 -204
  625. package/src/scripts/test-bedrock-handoff-autonomous.ts +0 -267
  626. package/src/scripts/test-handoff-preamble.ts +0 -278
  627. package/src/specs/agent-handoffs-bedrock.integration.test.ts +0 -415
  628. package/src/tools/artifacts/__tests__/tool.test.ts +0 -259
  629. package/src/tools/artifacts/index.ts +0 -33
  630. package/src/tools/artifacts/schema.ts +0 -99
  631. package/src/tools/artifacts/tool.ts +0 -289
  632. package/src/tools/artifacts/types.ts +0 -162
  633. package/src/tools/fileSearch/__tests__/tool.test.ts +0 -261
  634. package/src/tools/fileSearch/formatter.ts +0 -129
  635. package/src/tools/fileSearch/index.ts +0 -23
  636. package/src/tools/fileSearch/ragClient.ts +0 -137
  637. package/src/tools/fileSearch/schema.ts +0 -19
  638. package/src/tools/fileSearch/tool.ts +0 -207
  639. package/src/tools/fileSearch/types.ts +0 -149
  640. package/src/tools/proxyTool.ts +0 -166
  641. package/src/tools/search/output.md +0 -2775
  642. package/src/tools/search/test.html +0 -884
  643. package/src/tools/search/test.md +0 -643
  644. package/src/tools/search/test.ts +0 -159
  645. package/src/utils/__tests__/childAgentContext.test.ts +0 -217
  646. package/src/utils/__tests__/credentials.test.ts +0 -130
  647. package/src/utils/__tests__/httpClient.test.ts +0 -75
  648. package/src/utils/__tests__/toolManifest.test.ts +0 -116
  649. package/src/utils/childAgentContext.ts +0 -259
  650. package/src/utils/credentials.ts +0 -157
  651. package/src/utils/httpClient.ts +0 -92
  652. package/src/utils/toolManifest.ts +0 -109
  653. /package/dist/esm/{content → langchain}/index.mjs.map +0 -0
@@ -1 +1 @@
1
- {"version":3,"file":"shared.cjs","sources":["../../../../src/tools/memory/shared.ts"],"sourcesContent":["/**\n * Shared Zod schemas + helpers for the memory tool family.\n *\n * The tool schemas deliberately do NOT expose `agent_id` or `user_id`. Scope\n * is captured in a closure at tool construction time by {@link buildMemoryTools},\n * so even a hallucinated tool call with extra fields cannot influence it.\n */\nimport { z } from 'zod';\nimport {\n DEFAULT_MAX_INJECTED_CHARS,\n MEMORY_PATH_PREFIX,\n} from '@/memory/constants';\nimport type {\n MemoryAppendInput,\n MemoryBackend,\n MemoryEntry,\n MemoryScope,\n} from '@/memory/types';\n\nexport const MemorySearchSchema = z.object({\n query: z.string().describe('Natural-language query to search memory for'),\n maxResults: z\n .number()\n .int()\n .min(1)\n .max(50)\n .optional()\n .describe('Maximum number of results to return (default 10)'),\n minScore: z\n .number()\n .min(0)\n .max(1)\n .optional()\n .describe('Minimum hybrid score (0..1) below which results are dropped'),\n});\n\nexport const MemoryGetSchema = z.object({\n path: z\n .string()\n .describe(\n 'Memory path, e.g. \"memory/decisions.md\". Must start with \"memory/\"'\n ),\n from: z\n .number()\n .int()\n .min(1)\n .optional()\n .describe('Starting line number (1-indexed)'),\n lines: z\n .number()\n .int()\n .min(1)\n .optional()\n .describe('Number of lines to read starting at `from`'),\n});\n\nexport const MemoryAppendSchema = z.object({\n path: z\n .string()\n .describe(\n 'Memory path to append to (e.g. \"memory/decisions.md\"). Must start with \"memory/\"'\n ),\n content: z\n .string()\n .min(1)\n .describe(\n \"Note content in the agent's own voice. Markdown allowed. Each call appends an immutable entry.\"\n ),\n});\n\nexport type MemorySearchArgs = z.infer<typeof MemorySearchSchema>;\nexport type MemoryGetArgs = z.infer<typeof MemoryGetSchema>;\nexport type MemoryAppendArgs = z.infer<typeof MemoryAppendSchema>;\n\n/** Shape returned by `memory_search` — serialised as the tool result string. */\nexport interface MemorySearchToolResult {\n results: Array<\n Pick<MemoryEntry, 'id' | 'path' | 'content' | 'score' | 'createdAt'> & {\n citation?: string;\n }\n >;\n disabled?: boolean;\n unavailable?: boolean;\n error?: string;\n warning?: string;\n action?: string;\n}\n\n/** Shape returned by `memory_get`. */\nexport interface MemoryGetToolResult {\n path: string;\n text: string;\n disabled?: boolean;\n error?: string;\n}\n\nexport function buildMemorySearchUnavailableResult(\n error: string | undefined\n): MemorySearchToolResult {\n const reason =\n (error ?? 'memory search unavailable').trim() ||\n 'memory search unavailable';\n const isQuota = /insufficient_quota|quota|429/i.test(reason);\n return {\n results: [],\n disabled: true,\n unavailable: true,\n error: reason,\n warning: isQuota\n ? 'Memory search is unavailable because the embedding provider quota is exhausted.'\n : 'Memory search is unavailable due to an embedding/provider error.',\n action: isQuota\n ? 'Top up or switch embedding provider, then retry memory_search.'\n : 'Check embedding provider configuration and retry memory_search.',\n };\n}\n\n/**\n * Clamp a ranked result list to a total character budget.\n * Ported from upstream `tools.citations.ts::clampResultsByInjectedChars`.\n */\nexport function clampResultsByInjectedChars<T extends { content: string }>(\n results: T[],\n maxInjectedChars: number = DEFAULT_MAX_INJECTED_CHARS\n): T[] {\n const out: T[] = [];\n let total = 0;\n for (const result of results) {\n const size = result.content.length ?? 0;\n if (total + size > maxInjectedChars && out.length > 0) break;\n out.push(result);\n total += size;\n }\n return out;\n}\n\nexport interface MemoryToolBinding {\n backend: MemoryBackend;\n scope: MemoryScope;\n maxInjectedChars?: number;\n /** Phase 2 — per-call search options propagated into backend.search. */\n searchOptions?: {\n mmr?: { enabled?: boolean; lambda?: number };\n temporalDecay?: { enabled?: boolean; halfLifeDays?: number };\n citations?: 'on' | 'off' | 'auto';\n };\n /** Phase 2 — optional best-effort recall recorder. */\n recallTracker?: {\n record(params: {\n agentId: string;\n query: string;\n hits: Array<{ id: string; path: string; score: number }>;\n }): Promise<void>;\n };\n}\n\nexport function assertAppendAllowed(path: string): void {\n if (!path || !path.startsWith(MEMORY_PATH_PREFIX)) {\n throw new Error(\n `memory_append path must start with \"${MEMORY_PATH_PREFIX}\"`\n );\n }\n}\n\n/** Normalise an append call into the backend input shape. */\nexport function toAppendInput(args: MemoryAppendArgs): MemoryAppendInput {\n assertAppendAllowed(args.path);\n return { path: args.path, content: args.content };\n}\n"],"names":["z","DEFAULT_MAX_INJECTED_CHARS","MEMORY_PATH_PREFIX"],"mappings":";;;;;AAAA;;;;;;AAMG;AAaI,MAAM,kBAAkB,GAAGA,KAAC,CAAC,MAAM,CAAC;IACzC,KAAK,EAAEA,KAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,6CAA6C,CAAC;AACzE,IAAA,UAAU,EAAEA;AACT,SAAA,MAAM;AACN,SAAA,GAAG;SACH,GAAG,CAAC,CAAC;SACL,GAAG,CAAC,EAAE;AACN,SAAA,QAAQ;SACR,QAAQ,CAAC,kDAAkD,CAAC;AAC/D,IAAA,QAAQ,EAAEA;AACP,SAAA,MAAM;SACN,GAAG,CAAC,CAAC;SACL,GAAG,CAAC,CAAC;AACL,SAAA,QAAQ;SACR,QAAQ,CAAC,6DAA6D,CAAC;AAC3E,CAAA;AAEM,MAAM,eAAe,GAAGA,KAAC,CAAC,MAAM,CAAC;AACtC,IAAA,IAAI,EAAEA;AACH,SAAA,MAAM;SACN,QAAQ,CACP,oEAAoE,CACrE;AACH,IAAA,IAAI,EAAEA;AACH,SAAA,MAAM;AACN,SAAA,GAAG;SACH,GAAG,CAAC,CAAC;AACL,SAAA,QAAQ;SACR,QAAQ,CAAC,kCAAkC,CAAC;AAC/C,IAAA,KAAK,EAAEA;AACJ,SAAA,MAAM;AACN,SAAA,GAAG;SACH,GAAG,CAAC,CAAC;AACL,SAAA,QAAQ;SACR,QAAQ,CAAC,4CAA4C,CAAC;AAC1D,CAAA;AAEM,MAAM,kBAAkB,GAAGA,KAAC,CAAC,MAAM,CAAC;AACzC,IAAA,IAAI,EAAEA;AACH,SAAA,MAAM;SACN,QAAQ,CACP,kFAAkF,CACnF;AACH,IAAA,OAAO,EAAEA;AACN,SAAA,MAAM;SACN,GAAG,CAAC,CAAC;SACL,QAAQ,CACP,gGAAgG,CACjG;AACJ,CAAA;AA4BK,SAAU,kCAAkC,CAChD,KAAyB,EAAA;IAEzB,MAAM,MAAM,GACV,CAAC,KAAK,IAAI,2BAA2B,EAAE,IAAI,EAAE;AAC7C,QAAA,2BAA2B;IAC7B,MAAM,OAAO,GAAG,+BAA+B,CAAC,IAAI,CAAC,MAAM,CAAC;IAC5D,OAAO;AACL,QAAA,OAAO,EAAE,EAAE;AACX,QAAA,QAAQ,EAAE,IAAI;AACd,QAAA,WAAW,EAAE,IAAI;AACjB,QAAA,KAAK,EAAE,MAAM;AACb,QAAA,OAAO,EAAE;AACP,cAAE;AACF,cAAE,kEAAkE;AACtE,QAAA,MAAM,EAAE;AACN,cAAE;AACF,cAAE,iEAAiE;KACtE;AACH;AAEA;;;AAGG;SACa,2BAA2B,CACzC,OAAY,EACZ,mBAA2BC,oCAA0B,EAAA;IAErD,MAAM,GAAG,GAAQ,EAAE;IACnB,IAAI,KAAK,GAAG,CAAC;AACb,IAAA,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE;QAC5B,MAAM,IAAI,GAAG,MAAM,CAAC,OAAO,CAAC,MAAM,IAAI,CAAC;QACvC,IAAI,KAAK,GAAG,IAAI,GAAG,gBAAgB,IAAI,GAAG,CAAC,MAAM,GAAG,CAAC;YAAE;AACvD,QAAA,GAAG,CAAC,IAAI,CAAC,MAAM,CAAC;QAChB,KAAK,IAAI,IAAI;IACf;AACA,IAAA,OAAO,GAAG;AACZ;AAsBM,SAAU,mBAAmB,CAAC,IAAY,EAAA;IAC9C,IAAI,CAAC,IAAI,IAAI,CAAC,IAAI,CAAC,UAAU,CAACC,4BAAkB,CAAC,EAAE;AACjD,QAAA,MAAM,IAAI,KAAK,CACb,uCAAuCA,4BAAkB,CAAA,CAAA,CAAG,CAC7D;IACH;AACF;AAEA;AACM,SAAU,aAAa,CAAC,IAAsB,EAAA;AAClD,IAAA,mBAAmB,CAAC,IAAI,CAAC,IAAI,CAAC;AAC9B,IAAA,OAAO,EAAE,IAAI,EAAE,IAAI,CAAC,IAAI,EAAE,OAAO,EAAE,IAAI,CAAC,OAAO,EAAE;AACnD;;;;;;;;;;"}
1
+ {"version":3,"file":"shared.cjs","sources":["../../../../src/tools/memory/shared.ts"],"sourcesContent":["/**\n * Shared Zod schemas + helpers for the memory tool family.\n *\n * The tool schemas deliberately do NOT expose `agent_id` or `user_id`. Scope\n * is captured in a closure at tool construction time by {@link buildMemoryTools},\n * so even a hallucinated tool call with extra fields cannot influence it.\n */\nimport { z } from 'zod';\nimport {\n DEFAULT_MAX_INJECTED_CHARS,\n MEMORY_PATH_PREFIX,\n} from '@/memory/constants';\nimport type {\n MemoryAppendInput,\n MemoryBackend,\n MemoryEntry,\n MemoryScope,\n} from '@/memory/types';\n\nexport const MemorySearchSchema = z.object({\n query: z.string().describe('Natural-language query to search memory for'),\n maxResults: z\n .number()\n .int()\n .min(1)\n .max(50)\n .optional()\n .describe('Maximum number of results to return (default 10)'),\n minScore: z\n .number()\n .min(0)\n .max(1)\n .optional()\n .describe('Minimum hybrid score (0..1) below which results are dropped'),\n});\n\nexport const MemoryGetSchema = z.object({\n path: z\n .string()\n .describe(\n 'Memory path, e.g. \"memory/decisions.md\". Must start with \"memory/\"'\n ),\n from: z\n .number()\n .int()\n .min(1)\n .optional()\n .describe('Starting line number (1-indexed)'),\n lines: z\n .number()\n .int()\n .min(1)\n .optional()\n .describe('Number of lines to read starting at `from`'),\n});\n\nexport const MemoryAppendSchema = z.object({\n path: z\n .string()\n .describe(\n 'Memory path to append to (e.g. \"memory/decisions.md\"). Must start with \"memory/\"'\n ),\n content: z\n .string()\n .min(1)\n .describe(\n \"Note content in the agent's own voice. Markdown allowed. Each call appends an immutable entry.\"\n ),\n});\n\nexport type MemorySearchArgs = z.infer<typeof MemorySearchSchema>;\nexport type MemoryGetArgs = z.infer<typeof MemoryGetSchema>;\nexport type MemoryAppendArgs = z.infer<typeof MemoryAppendSchema>;\n\n/** Shape returned by `memory_search` — serialised as the tool result string. */\nexport interface MemorySearchToolResult {\n results: Array<\n Pick<MemoryEntry, 'id' | 'path' | 'content' | 'score' | 'createdAt'> & {\n citation?: string;\n }\n >;\n disabled?: boolean;\n unavailable?: boolean;\n error?: string;\n warning?: string;\n action?: string;\n}\n\n/** Shape returned by `memory_get`. */\nexport interface MemoryGetToolResult {\n path: string;\n text: string;\n disabled?: boolean;\n error?: string;\n}\n\nexport function buildMemorySearchUnavailableResult(\n error: string | undefined\n): MemorySearchToolResult {\n const reason =\n (error ?? 'memory search unavailable').trim() ||\n 'memory search unavailable';\n const isQuota = /insufficient_quota|quota|429/i.test(reason);\n return {\n results: [],\n disabled: true,\n unavailable: true,\n error: reason,\n warning: isQuota\n ? 'Memory search is unavailable because the embedding provider quota is exhausted.'\n : 'Memory search is unavailable due to an embedding/provider error.',\n action: isQuota\n ? 'Top up or switch embedding provider, then retry memory_search.'\n : 'Check embedding provider configuration and retry memory_search.',\n };\n}\n\n/**\n * Clamp a ranked result list to a total character budget.\n * Ported from a reference implementation `tools.citations.ts::clampResultsByInjectedChars`.\n */\nexport function clampResultsByInjectedChars<T extends { content: string }>(\n results: T[],\n maxInjectedChars: number = DEFAULT_MAX_INJECTED_CHARS\n): T[] {\n const out: T[] = [];\n let total = 0;\n for (const result of results) {\n const size = result.content.length ?? 0;\n if (total + size > maxInjectedChars && out.length > 0) break;\n out.push(result);\n total += size;\n }\n return out;\n}\n\nexport interface MemoryToolBinding {\n backend: MemoryBackend;\n scope: MemoryScope;\n maxInjectedChars?: number;\n /** Phase 2 — per-call search options propagated into backend.search. */\n searchOptions?: {\n mmr?: { enabled?: boolean; lambda?: number };\n temporalDecay?: { enabled?: boolean; halfLifeDays?: number };\n citations?: 'on' | 'off' | 'auto';\n };\n /** Phase 2 — optional best-effort recall recorder. */\n recallTracker?: {\n record(params: {\n agentId: string;\n query: string;\n hits: Array<{ id: string; path: string; score: number }>;\n }): Promise<void>;\n };\n}\n\nexport function assertAppendAllowed(path: string): void {\n if (!path || !path.startsWith(MEMORY_PATH_PREFIX)) {\n throw new Error(\n `memory_append path must start with \"${MEMORY_PATH_PREFIX}\"`\n );\n }\n}\n\n/** Normalise an append call into the backend input shape. */\nexport function toAppendInput(args: MemoryAppendArgs): MemoryAppendInput {\n assertAppendAllowed(args.path);\n return { path: args.path, content: args.content };\n}\n"],"names":["z","DEFAULT_MAX_INJECTED_CHARS","MEMORY_PATH_PREFIX"],"mappings":";;;;;AAAA;;;;;;AAMG;AAaI,MAAM,kBAAkB,GAAGA,KAAC,CAAC,MAAM,CAAC;IACzC,KAAK,EAAEA,KAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,6CAA6C,CAAC;AACzE,IAAA,UAAU,EAAEA;AACT,SAAA,MAAM;AACN,SAAA,GAAG;SACH,GAAG,CAAC,CAAC;SACL,GAAG,CAAC,EAAE;AACN,SAAA,QAAQ;SACR,QAAQ,CAAC,kDAAkD,CAAC;AAC/D,IAAA,QAAQ,EAAEA;AACP,SAAA,MAAM;SACN,GAAG,CAAC,CAAC;SACL,GAAG,CAAC,CAAC;AACL,SAAA,QAAQ;SACR,QAAQ,CAAC,6DAA6D,CAAC;AAC3E,CAAA;AAEM,MAAM,eAAe,GAAGA,KAAC,CAAC,MAAM,CAAC;AACtC,IAAA,IAAI,EAAEA;AACH,SAAA,MAAM;SACN,QAAQ,CACP,oEAAoE,CACrE;AACH,IAAA,IAAI,EAAEA;AACH,SAAA,MAAM;AACN,SAAA,GAAG;SACH,GAAG,CAAC,CAAC;AACL,SAAA,QAAQ;SACR,QAAQ,CAAC,kCAAkC,CAAC;AAC/C,IAAA,KAAK,EAAEA;AACJ,SAAA,MAAM;AACN,SAAA,GAAG;SACH,GAAG,CAAC,CAAC;AACL,SAAA,QAAQ;SACR,QAAQ,CAAC,4CAA4C,CAAC;AAC1D,CAAA;AAEM,MAAM,kBAAkB,GAAGA,KAAC,CAAC,MAAM,CAAC;AACzC,IAAA,IAAI,EAAEA;AACH,SAAA,MAAM;SACN,QAAQ,CACP,kFAAkF,CACnF;AACH,IAAA,OAAO,EAAEA;AACN,SAAA,MAAM;SACN,GAAG,CAAC,CAAC;SACL,QAAQ,CACP,gGAAgG,CACjG;AACJ,CAAA;AA4BK,SAAU,kCAAkC,CAChD,KAAyB,EAAA;IAEzB,MAAM,MAAM,GACV,CAAC,KAAK,IAAI,2BAA2B,EAAE,IAAI,EAAE;AAC7C,QAAA,2BAA2B;IAC7B,MAAM,OAAO,GAAG,+BAA+B,CAAC,IAAI,CAAC,MAAM,CAAC;IAC5D,OAAO;AACL,QAAA,OAAO,EAAE,EAAE;AACX,QAAA,QAAQ,EAAE,IAAI;AACd,QAAA,WAAW,EAAE,IAAI;AACjB,QAAA,KAAK,EAAE,MAAM;AACb,QAAA,OAAO,EAAE;AACP,cAAE;AACF,cAAE,kEAAkE;AACtE,QAAA,MAAM,EAAE;AACN,cAAE;AACF,cAAE,iEAAiE;KACtE;AACH;AAEA;;;AAGG;SACa,2BAA2B,CACzC,OAAY,EACZ,mBAA2BC,oCAA0B,EAAA;IAErD,MAAM,GAAG,GAAQ,EAAE;IACnB,IAAI,KAAK,GAAG,CAAC;AACb,IAAA,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE;QAC5B,MAAM,IAAI,GAAG,MAAM,CAAC,OAAO,CAAC,MAAM,IAAI,CAAC;QACvC,IAAI,KAAK,GAAG,IAAI,GAAG,gBAAgB,IAAI,GAAG,CAAC,MAAM,GAAG,CAAC;YAAE;AACvD,QAAA,GAAG,CAAC,IAAI,CAAC,MAAM,CAAC;QAChB,KAAK,IAAI,IAAI;IACf;AACA,IAAA,OAAO,GAAG;AACZ;AAsBM,SAAU,mBAAmB,CAAC,IAAY,EAAA;IAC9C,IAAI,CAAC,IAAI,IAAI,CAAC,IAAI,CAAC,UAAU,CAACC,4BAAkB,CAAC,EAAE;AACjD,QAAA,MAAM,IAAI,KAAK,CACb,uCAAuCA,4BAAkB,CAAA,CAAA,CAAG,CAC7D;IACH;AACF;AAEA;AACM,SAAU,aAAa,CAAC,IAAsB,EAAA;AAClD,IAAA,mBAAmB,CAAC,IAAI,CAAC,IAAI,CAAC;AAC9B,IAAA,OAAO,EAAE,IAAI,EAAE,IAAI,CAAC,IAAI,EAAE,OAAO,EAAE,IAAI,CAAC,OAAO,EAAE;AACnD;;;;;;;;;;"}
@@ -3,6 +3,7 @@
3
3
  var axios = require('axios');
4
4
  var textsplitters = require('@langchain/textsplitters');
5
5
  var utils = require('./utils.cjs');
6
+ var tavilySearch = require('./tavily-search.cjs');
6
7
 
7
8
  const chunker = {
8
9
  cleanText: (text) => {
@@ -323,15 +324,18 @@ const createSearXNGAPI = (instanceUrl, apiKey) => {
323
324
  return { getSources };
324
325
  };
325
326
  const createSearchAPI = (config) => {
326
- const { searchProvider = 'serper', serperApiKey, searxngInstanceUrl, searxngApiKey, domainBlocklist, countryBlocklist, } = config;
327
+ const { searchProvider = 'serper', serperApiKey, searxngInstanceUrl, searxngApiKey, tavilyApiKey, tavilySearchUrl, tavilySearchOptions, domainBlocklist, countryBlocklist, } = config;
327
328
  if (searchProvider.toLowerCase() === 'serper') {
328
329
  return createSerperAPI(serperApiKey, domainBlocklist, countryBlocklist);
329
330
  }
330
331
  else if (searchProvider.toLowerCase() === 'searxng') {
331
332
  return createSearXNGAPI(searxngInstanceUrl, searxngApiKey);
332
333
  }
334
+ else if (searchProvider.toLowerCase() === 'tavily') {
335
+ return tavilySearch.createTavilyAPI(tavilyApiKey, tavilySearchUrl, tavilySearchOptions);
336
+ }
333
337
  else {
334
- throw new Error(`Invalid search provider: ${searchProvider}. Must be 'serper' or 'searxng'`);
338
+ throw new Error(`Invalid search provider: ${searchProvider}. Must be 'serper', 'searxng', or 'tavily'`);
335
339
  }
336
340
  };
337
341
  const createSourceProcessor = (config = {}, scraperInstance) => {
@@ -354,7 +358,11 @@ const createSourceProcessor = (config = {}, scraperInstance) => {
354
358
  const promise = scraper
355
359
  .scrapeUrl(currentLink, {})
356
360
  .then(([url, response]) => {
357
- const attribution = utils.getAttribution(url, response.data?.metadata, logger_);
361
+ /* metadata exists on Firecrawl/Serper responses but not on
362
+ * Tavily's. Read it via the scraper's extractMetadata abstraction
363
+ * to stay polymorphic across all scraper providers. */
364
+ const rawMeta = scraper.extractMetadata(response);
365
+ const attribution = utils.getAttribution(url, Object.keys(rawMeta).length > 0 ? rawMeta : undefined, logger_);
358
366
  if (response.success && response.data) {
359
367
  const [content, references] = scraper.extractContent(response);
360
368
  return {
@@ -1 +1 @@
1
- {"version":3,"file":"search.cjs","sources":["../../../../src/tools/search/search.ts"],"sourcesContent":["import axios from 'axios';\nimport { RecursiveCharacterTextSplitter } from '@langchain/textsplitters';\nimport type * as t from './types';\nimport { getAttribution, createDefaultLogger } from './utils';\nimport { BaseReranker } from './rerankers';\n\nconst chunker = {\n cleanText: (text: string): string => {\n if (!text) return '';\n\n /** Normalized all line endings to '\\n' */\n const normalizedText = text.replace(/\\r\\n/g, '\\n').replace(/\\r/g, '\\n');\n\n /** Handle multiple backslashes followed by newlines\n * This replaces patterns like '\\\\\\\\\\\\n' with a single newline */\n const fixedBackslashes = normalizedText.replace(/\\\\+\\n/g, '\\n');\n\n /** Cleaned up consecutive newlines, tabs, and spaces around newlines */\n const cleanedNewlines = fixedBackslashes.replace(/[\\t ]*\\n[\\t \\n]*/g, '\\n');\n\n /** Cleaned up excessive spaces and tabs */\n const cleanedSpaces = cleanedNewlines.replace(/[ \\t]+/g, ' ');\n\n return cleanedSpaces.trim();\n },\n splitText: async (\n text: string,\n options?: {\n chunkSize?: number;\n chunkOverlap?: number;\n separators?: string[];\n }\n ): Promise<string[]> => {\n const chunkSize = options?.chunkSize ?? 150;\n const chunkOverlap = options?.chunkOverlap ?? 50;\n const separators = options?.separators || ['\\n\\n', '\\n'];\n\n const splitter = new RecursiveCharacterTextSplitter({\n separators,\n chunkSize,\n chunkOverlap,\n });\n\n return await splitter.splitText(text);\n },\n\n splitTexts: async (\n texts: string[],\n options?: {\n chunkSize?: number;\n chunkOverlap?: number;\n separators?: string[];\n },\n logger?: t.Logger\n ): Promise<string[][]> => {\n // Split multiple texts\n const logger_ = logger || createDefaultLogger();\n const promises = texts.map((text) =>\n chunker.splitText(text, options).catch((error) => {\n logger_.error('Error splitting text:', error);\n return [text];\n })\n );\n return Promise.all(promises);\n },\n};\n\nfunction createSourceUpdateCallback(sourceMap: Map<string, t.ValidSource>) {\n return (link: string, update?: Partial<t.ValidSource>): void => {\n const source = sourceMap.get(link);\n if (source) {\n sourceMap.set(link, {\n ...source,\n ...update,\n });\n }\n };\n}\n\nconst getHighlights = async ({\n query,\n content,\n reranker,\n topResults = 5,\n logger,\n}: {\n content: string;\n query: string;\n reranker?: BaseReranker;\n topResults?: number;\n logger?: t.Logger;\n}): Promise<t.Highlight[] | undefined> => {\n const logger_ = logger || createDefaultLogger();\n\n if (!content) {\n logger_.warn('No content provided for highlights');\n return;\n }\n if (!reranker) {\n logger_.warn('No reranker provided for highlights');\n return;\n }\n\n try {\n const documents = await chunker.splitText(content);\n if (Array.isArray(documents)) {\n return await reranker.rerank(query, documents, topResults);\n } else {\n logger_.error(\n 'Expected documents to be an array, got:',\n typeof documents\n );\n return;\n }\n } catch (error) {\n logger_.error('Error in content processing:', error);\n return;\n }\n};\n\nconst createSerperAPI = (\n apiKey?: string,\n domainBlocklist?: string[],\n countryBlocklist?: string[]\n): {\n getSources: (params: t.GetSourcesParams) => Promise<t.SearchResult>;\n} => {\n const config = {\n apiKey: apiKey ?? process.env.SERPER_API_KEY,\n apiUrl: 'https://google.serper.dev/search',\n timeout: 10000,\n };\n\n if (config.apiKey == null || config.apiKey === '') {\n throw new Error('SERPER_API_KEY is required for SerperAPI');\n }\n\n const getSources = async ({\n query,\n date,\n country,\n safeSearch,\n numResults = 8,\n type,\n }: t.GetSourcesParams): Promise<t.SearchResult> => {\n if (!query.trim()) {\n return { success: false, error: 'Query cannot be empty' };\n }\n\n try {\n const safe = ['off', 'moderate', 'active'] as const;\n\n // Append -site: operators for blocked domains so they never appear in results\n let effectiveQuery = query;\n if (domainBlocklist && domainBlocklist.length > 0) {\n const siteExclusions = domainBlocklist\n .map((d) => `-site:${d}`)\n .join(' ');\n effectiveQuery = `${effectiveQuery} ${siteExclusions}`;\n }\n\n // Append -site:.XX operators for blocked country TLDs\n if (countryBlocklist && countryBlocklist.length > 0) {\n const countryExclusions = countryBlocklist\n .map((c) => `-site:.${c.toLowerCase().replace(/^\\./, '')}`)\n .join(' ');\n effectiveQuery = `${effectiveQuery} ${countryExclusions}`;\n }\n\n const payload: t.SerperSearchPayload = {\n q: effectiveQuery,\n safe: safe[safeSearch ?? 1],\n num: Math.min(Math.max(1, numResults), 10),\n };\n\n // Set the search type if provided\n if (type) {\n payload.type = type;\n }\n\n if (date != null) {\n payload.tbs = `qdr:${date}`;\n }\n\n if (country != null && country !== '') {\n payload['gl'] = country.toLowerCase();\n }\n\n // Determine the API endpoint based on the search type\n let apiEndpoint = config.apiUrl;\n if (type === 'images') {\n apiEndpoint = 'https://google.serper.dev/images';\n } else if (type === 'videos') {\n apiEndpoint = 'https://google.serper.dev/videos';\n } else if (type === 'news') {\n apiEndpoint = 'https://google.serper.dev/news';\n }\n\n const response = await axios.post<t.SerperResultData>(\n apiEndpoint,\n payload,\n {\n headers: {\n 'X-API-KEY': config.apiKey,\n 'Content-Type': 'application/json',\n },\n timeout: config.timeout,\n }\n );\n\n const data = response.data;\n const results: t.SearchResultData = {\n organic: data.organic,\n images: data.images ?? [],\n answerBox: data.answerBox,\n topStories: data.topStories ?? [],\n peopleAlsoAsk: data.peopleAlsoAsk,\n knowledgeGraph: data.knowledgeGraph,\n relatedSearches: data.relatedSearches,\n videos: data.videos ?? [],\n news: data.news ?? [],\n };\n\n return { success: true, data: results };\n } catch (error) {\n const errorMessage =\n error instanceof Error ? error.message : String(error);\n return { success: false, error: `API request failed: ${errorMessage}` };\n }\n };\n\n return { getSources };\n};\n\nconst createSearXNGAPI = (\n instanceUrl?: string,\n apiKey?: string\n): {\n getSources: (params: t.GetSourcesParams) => Promise<t.SearchResult>;\n} => {\n const config = {\n instanceUrl: instanceUrl ?? process.env.SEARXNG_INSTANCE_URL,\n apiKey: apiKey ?? process.env.SEARXNG_API_KEY,\n defaultLocation: 'all',\n timeout: 10000,\n };\n\n if (config.instanceUrl == null || config.instanceUrl === '') {\n throw new Error('SEARXNG_INSTANCE_URL is required for SearXNG API');\n }\n\n const getSources = async ({\n query,\n numResults = 8,\n safeSearch,\n type,\n }: t.GetSourcesParams): Promise<t.SearchResult> => {\n if (!query.trim()) {\n return { success: false, error: 'Query cannot be empty' };\n }\n\n try {\n // Ensure the instance URL ends with /search\n if (config.instanceUrl == null || config.instanceUrl === '') {\n return { success: false, error: 'Instance URL is not defined' };\n }\n\n let searchUrl = config.instanceUrl;\n if (!searchUrl.endsWith('/search')) {\n searchUrl = searchUrl.replace(/\\/$/, '') + '/search';\n }\n\n // Determine the search category based on the type\n let category = 'general';\n if (type === 'images') {\n category = 'images';\n } else if (type === 'videos') {\n category = 'videos';\n } else if (type === 'news') {\n category = 'news';\n }\n\n // Prepare parameters for SearXNG\n const params: t.SearxNGSearchPayload = {\n q: query,\n format: 'json',\n pageno: 1,\n categories: category,\n language: 'all',\n safesearch: safeSearch,\n engines: 'google,bing,duckduckgo',\n };\n\n const headers: Record<string, string> = {\n 'Content-Type': 'application/json',\n };\n\n if (config.apiKey != null && config.apiKey !== '') {\n headers['X-API-Key'] = config.apiKey;\n }\n\n const response = await axios.get(searchUrl, {\n headers,\n params,\n timeout: config.timeout,\n });\n\n const data = response.data;\n\n // Helper function to identify news results since SearXNG doesn't provide that classification by default\n const isNewsResult = (result: t.SearXNGResult): boolean => {\n const url = result.url?.toLowerCase() ?? '';\n const title = result.title?.toLowerCase() ?? '';\n\n // News-related keywords in title/content\n const newsKeywords = [\n 'breaking news',\n 'latest news',\n 'top stories',\n 'news today',\n 'developing story',\n 'trending news',\n 'news',\n ];\n\n // Check if title/content contains news keywords\n const hasNewsKeywords = newsKeywords.some(\n (keyword) => title.toLowerCase().includes(keyword) // just title probably fine, content parsing is overkill for what we need: || content.includes(keyword)\n );\n\n // Check if URL contains news-related paths\n const hasNewsPath =\n url.includes('/news/') ||\n url.includes('/world/') ||\n url.includes('/politics/') ||\n url.includes('/breaking/');\n\n return hasNewsKeywords || hasNewsPath;\n };\n\n // Transform SearXNG results to match SerperAPI format\n const organicResults = (data.results ?? [])\n .slice(0, numResults)\n .map((result: t.SearXNGResult, index: number) => {\n let attribution = '';\n try {\n attribution = new URL(result.url ?? '').hostname;\n } catch {\n attribution = '';\n }\n\n return {\n position: index + 1,\n title: result.title ?? '',\n link: result.url ?? '',\n snippet: result.content ?? '',\n date: result.publishedDate ?? '',\n attribution,\n };\n });\n\n const imageResults = (data.results ?? [])\n .filter((result: t.SearXNGResult) => result.img_src)\n .slice(0, 6)\n .map((result: t.SearXNGResult, index: number) => ({\n title: result.title ?? '',\n imageUrl: result.img_src ?? '',\n position: index + 1,\n source: new URL(result.url ?? '').hostname,\n domain: new URL(result.url ?? '').hostname,\n link: result.url ?? '',\n }));\n\n // Extract news results from organic results\n const newsResults = (data.results ?? [])\n .filter(isNewsResult)\n .map((result: t.SearXNGResult, index: number) => {\n let attribution = '';\n try {\n attribution = new URL(result.url ?? '').hostname;\n } catch {\n attribution = '';\n }\n\n return {\n title: result.title ?? '',\n link: result.url ?? '',\n snippet: result.content ?? '',\n date: result.publishedDate ?? '',\n source: attribution,\n imageUrl: result.img_src ?? '',\n position: index + 1,\n };\n });\n\n const topStories = newsResults.slice(0, 5);\n\n const relatedSearches = Array.isArray(data.suggestions)\n ? data.suggestions.map((suggestion: string) => ({ query: suggestion }))\n : [];\n\n const results: t.SearchResultData = {\n organic: organicResults,\n images: imageResults,\n topStories: topStories, // Use first 5 extracted news as top stories\n relatedSearches,\n videos: [],\n news: newsResults,\n // Add empty arrays for other Serper fields to maintain parity\n places: [],\n shopping: [],\n peopleAlsoAsk: [],\n knowledgeGraph: undefined,\n answerBox: undefined,\n };\n\n return { success: true, data: results };\n } catch (error) {\n const errorMessage =\n error instanceof Error ? error.message : String(error);\n return {\n success: false,\n error: `SearXNG API request failed: ${errorMessage}`,\n };\n }\n };\n\n return { getSources };\n};\n\nexport const createSearchAPI = (\n config: t.SearchConfig\n): {\n getSources: (params: t.GetSourcesParams) => Promise<t.SearchResult>;\n} => {\n const {\n searchProvider = 'serper',\n serperApiKey,\n searxngInstanceUrl,\n searxngApiKey,\n domainBlocklist,\n countryBlocklist,\n } = config;\n\n if (searchProvider.toLowerCase() === 'serper') {\n return createSerperAPI(serperApiKey, domainBlocklist, countryBlocklist);\n } else if (searchProvider.toLowerCase() === 'searxng') {\n return createSearXNGAPI(searxngInstanceUrl, searxngApiKey);\n } else {\n throw new Error(\n `Invalid search provider: ${searchProvider}. Must be 'serper' or 'searxng'`\n );\n }\n};\n\nexport const createSourceProcessor = (\n config: t.ProcessSourcesConfig = {},\n scraperInstance?: t.BaseScraper\n): {\n processSources: (\n fields: t.ProcessSourcesFields\n ) => Promise<t.SearchResultData>;\n topResults: number;\n} => {\n if (!scraperInstance) {\n throw new Error('Scraper instance is required');\n }\n const {\n topResults = 5,\n // strategies = ['no_extraction'],\n // filterContent = true,\n reranker,\n logger,\n } = config;\n\n const logger_ = logger || createDefaultLogger();\n const scraper = scraperInstance;\n\n const webScraper = {\n scrapeMany: async ({\n query,\n links,\n onGetHighlights,\n }: {\n query: string;\n links: string[];\n onGetHighlights: t.SearchToolConfig['onGetHighlights'];\n }): Promise<Array<t.ScrapeResult>> => {\n logger_.debug(`Scraping ${links.length} links`);\n const promises: Array<Promise<t.ScrapeResult>> = [];\n try {\n for (let i = 0; i < links.length; i++) {\n const currentLink = links[i];\n const promise: Promise<t.ScrapeResult> = scraper\n .scrapeUrl(currentLink, {})\n .then(([url, response]) => {\n const attribution = getAttribution(\n url,\n response.data?.metadata,\n logger_\n );\n if (response.success && response.data) {\n const [content, references] = scraper.extractContent(response);\n return {\n url,\n references,\n attribution,\n content: chunker.cleanText(content),\n } as t.ScrapeResult;\n } else {\n logger_.error(\n `Error scraping ${url}: ${response.error ?? 'Unknown error'}`\n );\n }\n\n return {\n url,\n attribution,\n error: true,\n content: '',\n } as t.ScrapeResult;\n })\n .then(async (result) => {\n try {\n if (result.error != null) {\n logger_.error(\n `Error scraping ${result.url}: ${result.content}`\n );\n return {\n ...result,\n };\n }\n const highlights = await getHighlights({\n query,\n reranker,\n content: result.content,\n logger: logger_,\n });\n if (onGetHighlights) {\n onGetHighlights(result.url);\n }\n return {\n ...result,\n highlights,\n };\n } catch (error) {\n logger_.error('Error processing scraped content:', error);\n return {\n ...result,\n };\n }\n })\n .catch((error) => {\n logger_.error(`Error scraping ${currentLink}:`, error);\n return {\n url: currentLink,\n error: true,\n content: '',\n };\n });\n promises.push(promise);\n }\n return await Promise.all(promises);\n } catch (error) {\n logger_.error('Error in scrapeMany:', error);\n return [];\n }\n },\n };\n\n const fetchContents = async ({\n links,\n query,\n target,\n onGetHighlights,\n onContentScraped,\n }: {\n links: string[];\n query: string;\n target: number;\n onGetHighlights: t.SearchToolConfig['onGetHighlights'];\n onContentScraped?: (link: string, update?: Partial<t.ValidSource>) => void;\n }): Promise<void> => {\n const initialLinks = links.slice(0, target);\n // const remainingLinks = links.slice(target).reverse();\n const results = await webScraper.scrapeMany({\n query,\n links: initialLinks,\n onGetHighlights,\n });\n for (const result of results) {\n if (result.error === true) {\n continue;\n }\n const { url, content, attribution, references, highlights } = result;\n onContentScraped?.(url, {\n content,\n attribution,\n references,\n highlights,\n });\n }\n };\n\n const processSources = async ({\n result,\n numElements,\n query,\n news,\n proMode = true,\n onGetHighlights,\n skipScraping = false,\n }: t.ProcessSourcesFields): Promise<t.SearchResultData> => {\n try {\n if (!result.data) {\n return {\n organic: [],\n topStories: [],\n images: [],\n relatedSearches: [],\n };\n } else if (!result.data.organic) {\n return result.data;\n }\n\n // If content was already extracted directly (e.g., direct URL extraction), skip scraping\n if (skipScraping) {\n logger_.debug(\n 'Skipping additional scraping - content already extracted'\n );\n return result.data;\n }\n\n if (!proMode) {\n const wikiSources = result.data.organic.filter((source) =>\n source.link.includes('wikipedia.org')\n );\n\n if (!wikiSources.length) {\n return result.data;\n }\n\n const wikiSourceMap = new Map<string, t.ValidSource>();\n wikiSourceMap.set(wikiSources[0].link, wikiSources[0]);\n const onContentScraped = createSourceUpdateCallback(wikiSourceMap);\n await fetchContents({\n query,\n target: 1,\n onGetHighlights,\n onContentScraped,\n links: [wikiSources[0].link],\n });\n\n for (let i = 0; i < result.data.organic.length; i++) {\n const source = result.data.organic[i];\n const updatedSource = wikiSourceMap.get(source.link);\n if (updatedSource) {\n result.data.organic[i] = {\n ...source,\n ...updatedSource,\n };\n }\n }\n\n return result.data;\n }\n\n const sourceMap = new Map<string, t.ValidSource>();\n const organicLinksSet = new Set<string>();\n\n // Collect organic links\n const organicLinks = collectLinks(\n result.data.organic,\n sourceMap,\n organicLinksSet\n );\n\n // Collect top story links, excluding any that are already in organic links\n const topStories = result.data.topStories ?? [];\n const topStoryLinks = collectLinks(\n topStories,\n sourceMap,\n organicLinksSet\n );\n\n if (organicLinks.length === 0 && (topStoryLinks.length === 0 || !news)) {\n return result.data;\n }\n\n const onContentScraped = createSourceUpdateCallback(sourceMap);\n const promises: Promise<void>[] = [];\n\n // Process organic links\n if (organicLinks.length > 0) {\n promises.push(\n fetchContents({\n query,\n onGetHighlights,\n onContentScraped,\n links: organicLinks,\n target: numElements,\n })\n );\n }\n\n // Process top story links\n if (news && topStoryLinks.length > 0) {\n promises.push(\n fetchContents({\n query,\n onGetHighlights,\n onContentScraped,\n links: topStoryLinks,\n target: numElements,\n })\n );\n }\n\n await Promise.all(promises);\n\n if (result.data.organic.length > 0) {\n updateSourcesWithContent(result.data.organic, sourceMap);\n }\n\n if (news && topStories.length > 0) {\n updateSourcesWithContent(topStories, sourceMap);\n }\n\n return result.data;\n } catch (error) {\n logger_.error('Error in processSources:', error);\n return {\n organic: [],\n topStories: [],\n images: [],\n relatedSearches: [],\n ...result.data,\n error: error instanceof Error ? error.message : String(error),\n };\n }\n };\n\n return {\n processSources,\n topResults,\n };\n};\n\n/** Helper function to collect links and update sourceMap */\nfunction collectLinks(\n sources: Array<t.OrganicResult | t.TopStoryResult>,\n sourceMap: Map<string, t.ValidSource>,\n existingLinksSet?: Set<string>\n): string[] {\n const links: string[] = [];\n\n for (const source of sources) {\n if (source.link) {\n // For topStories, only add if not already in organic links\n if (existingLinksSet && existingLinksSet.has(source.link)) {\n continue;\n }\n\n links.push(source.link);\n if (existingLinksSet) {\n existingLinksSet.add(source.link);\n }\n sourceMap.set(source.link, source as t.ValidSource);\n }\n }\n\n return links;\n}\n\n/** Helper function to update sources with scraped content */\nfunction updateSourcesWithContent<T extends t.ValidSource>(\n sources: T[],\n sourceMap: Map<string, t.ValidSource>\n): void {\n for (let i = 0; i < sources.length; i++) {\n const source = sources[i];\n const updatedSource = sourceMap.get(source.link);\n if (updatedSource) {\n sources[i] = {\n ...source,\n ...updatedSource,\n } as T;\n }\n }\n}\n"],"names":["RecursiveCharacterTextSplitter","createDefaultLogger","getAttribution"],"mappings":";;;;;;AAMA,MAAM,OAAO,GAAG;AACd,IAAA,SAAS,EAAE,CAAC,IAAY,KAAY;AAClC,QAAA,IAAI,CAAC,IAAI;AAAE,YAAA,OAAO,EAAE;;AAGpB,QAAA,MAAM,cAAc,GAAG,IAAI,CAAC,OAAO,CAAC,OAAO,EAAE,IAAI,CAAC,CAAC,OAAO,CAAC,KAAK,EAAE,IAAI,CAAC;AAEvE;AACiE;QACjE,MAAM,gBAAgB,GAAG,cAAc,CAAC,OAAO,CAAC,QAAQ,EAAE,IAAI,CAAC;;QAG/D,MAAM,eAAe,GAAG,gBAAgB,CAAC,OAAO,CAAC,mBAAmB,EAAE,IAAI,CAAC;;QAG3E,MAAM,aAAa,GAAG,eAAe,CAAC,OAAO,CAAC,SAAS,EAAE,GAAG,CAAC;AAE7D,QAAA,OAAO,aAAa,CAAC,IAAI,EAAE;IAC7B,CAAC;AACD,IAAA,SAAS,EAAE,OACT,IAAY,EACZ,OAIC,KACoB;AACrB,QAAA,MAAM,SAAS,GAAG,OAAO,EAAE,SAAS,IAAI,GAAG;AAC3C,QAAA,MAAM,YAAY,GAAG,OAAO,EAAE,YAAY,IAAI,EAAE;QAChD,MAAM,UAAU,GAAG,OAAO,EAAE,UAAU,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC;AAExD,QAAA,MAAM,QAAQ,GAAG,IAAIA,4CAA8B,CAAC;YAClD,UAAU;YACV,SAAS;YACT,YAAY;AACb,SAAA,CAAC;AAEF,QAAA,OAAO,MAAM,QAAQ,CAAC,SAAS,CAAC,IAAI,CAAC;IACvC,CAAC;IAED,UAAU,EAAE,OACV,KAAe,EACf,OAIC,EACD,MAAiB,KACM;;AAEvB,QAAA,MAAM,OAAO,GAAG,MAAM,IAAIC,yBAAmB,EAAE;QAC/C,MAAM,QAAQ,GAAG,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,KAC9B,OAAO,CAAC,SAAS,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC,KAAK,CAAC,CAAC,KAAK,KAAI;AAC/C,YAAA,OAAO,CAAC,KAAK,CAAC,uBAAuB,EAAE,KAAK,CAAC;YAC7C,OAAO,CAAC,IAAI,CAAC;QACf,CAAC,CAAC,CACH;AACD,QAAA,OAAO,OAAO,CAAC,GAAG,CAAC,QAAQ,CAAC;IAC9B,CAAC;CACF;AAED,SAAS,0BAA0B,CAAC,SAAqC,EAAA;AACvE,IAAA,OAAO,CAAC,IAAY,EAAE,MAA+B,KAAU;QAC7D,MAAM,MAAM,GAAG,SAAS,CAAC,GAAG,CAAC,IAAI,CAAC;QAClC,IAAI,MAAM,EAAE;AACV,YAAA,SAAS,CAAC,GAAG,CAAC,IAAI,EAAE;AAClB,gBAAA,GAAG,MAAM;AACT,gBAAA,GAAG,MAAM;AACV,aAAA,CAAC;QACJ;AACF,IAAA,CAAC;AACH;AAEA,MAAM,aAAa,GAAG,OAAO,EAC3B,KAAK,EACL,OAAO,EACP,QAAQ,EACR,UAAU,GAAG,CAAC,EACd,MAAM,GAOP,KAAwC;AACvC,IAAA,MAAM,OAAO,GAAG,MAAM,IAAIA,yBAAmB,EAAE;IAE/C,IAAI,CAAC,OAAO,EAAE;AACZ,QAAA,OAAO,CAAC,IAAI,CAAC,oCAAoC,CAAC;QAClD;IACF;IACA,IAAI,CAAC,QAAQ,EAAE;AACb,QAAA,OAAO,CAAC,IAAI,CAAC,qCAAqC,CAAC;QACnD;IACF;AAEA,IAAA,IAAI;QACF,MAAM,SAAS,GAAG,MAAM,OAAO,CAAC,SAAS,CAAC,OAAO,CAAC;AAClD,QAAA,IAAI,KAAK,CAAC,OAAO,CAAC,SAAS,CAAC,EAAE;YAC5B,OAAO,MAAM,QAAQ,CAAC,MAAM,CAAC,KAAK,EAAE,SAAS,EAAE,UAAU,CAAC;QAC5D;aAAO;YACL,OAAO,CAAC,KAAK,CACX,yCAAyC,EACzC,OAAO,SAAS,CACjB;YACD;QACF;IACF;IAAE,OAAO,KAAK,EAAE;AACd,QAAA,OAAO,CAAC,KAAK,CAAC,8BAA8B,EAAE,KAAK,CAAC;QACpD;IACF;AACF,CAAC;AAED,MAAM,eAAe,GAAG,CACtB,MAAe,EACf,eAA0B,EAC1B,gBAA2B,KAGzB;AACF,IAAA,MAAM,MAAM,GAAG;AACb,QAAA,MAAM,EAAE,MAAM,IAAI,OAAO,CAAC,GAAG,CAAC,cAAc;AAC5C,QAAA,MAAM,EAAE,kCAAkC;AAC1C,QAAA,OAAO,EAAE,KAAK;KACf;AAED,IAAA,IAAI,MAAM,CAAC,MAAM,IAAI,IAAI,IAAI,MAAM,CAAC,MAAM,KAAK,EAAE,EAAE;AACjD,QAAA,MAAM,IAAI,KAAK,CAAC,0CAA0C,CAAC;IAC7D;IAEA,MAAM,UAAU,GAAG,OAAO,EACxB,KAAK,EACL,IAAI,EACJ,OAAO,EACP,UAAU,EACV,UAAU,GAAG,CAAC,EACd,IAAI,GACe,KAA6B;AAChD,QAAA,IAAI,CAAC,KAAK,CAAC,IAAI,EAAE,EAAE;YACjB,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,uBAAuB,EAAE;QAC3D;AAEA,QAAA,IAAI;YACF,MAAM,IAAI,GAAG,CAAC,KAAK,EAAE,UAAU,EAAE,QAAQ,CAAU;;YAGnD,IAAI,cAAc,GAAG,KAAK;YAC1B,IAAI,eAAe,IAAI,eAAe,CAAC,MAAM,GAAG,CAAC,EAAE;gBACjD,MAAM,cAAc,GAAG;qBACpB,GAAG,CAAC,CAAC,CAAC,KAAK,CAAA,MAAA,EAAS,CAAC,CAAA,CAAE;qBACvB,IAAI,CAAC,GAAG,CAAC;AACZ,gBAAA,cAAc,GAAG,CAAA,EAAG,cAAc,CAAA,CAAA,EAAI,cAAc,EAAE;YACxD;;YAGA,IAAI,gBAAgB,IAAI,gBAAgB,CAAC,MAAM,GAAG,CAAC,EAAE;gBACnD,MAAM,iBAAiB,GAAG;AACvB,qBAAA,GAAG,CAAC,CAAC,CAAC,KAAK,UAAU,CAAC,CAAC,WAAW,EAAE,CAAC,OAAO,CAAC,KAAK,EAAE,EAAE,CAAC,EAAE;qBACzD,IAAI,CAAC,GAAG,CAAC;AACZ,gBAAA,cAAc,GAAG,CAAA,EAAG,cAAc,CAAA,CAAA,EAAI,iBAAiB,EAAE;YAC3D;AAEA,YAAA,MAAM,OAAO,GAA0B;AACrC,gBAAA,CAAC,EAAE,cAAc;AACjB,gBAAA,IAAI,EAAE,IAAI,CAAC,UAAU,IAAI,CAAC,CAAC;AAC3B,gBAAA,GAAG,EAAE,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,UAAU,CAAC,EAAE,EAAE,CAAC;aAC3C;;YAGD,IAAI,IAAI,EAAE;AACR,gBAAA,OAAO,CAAC,IAAI,GAAG,IAAI;YACrB;AAEA,YAAA,IAAI,IAAI,IAAI,IAAI,EAAE;AAChB,gBAAA,OAAO,CAAC,GAAG,GAAG,CAAA,IAAA,EAAO,IAAI,EAAE;YAC7B;YAEA,IAAI,OAAO,IAAI,IAAI,IAAI,OAAO,KAAK,EAAE,EAAE;gBACrC,OAAO,CAAC,IAAI,CAAC,GAAG,OAAO,CAAC,WAAW,EAAE;YACvC;;AAGA,YAAA,IAAI,WAAW,GAAG,MAAM,CAAC,MAAM;AAC/B,YAAA,IAAI,IAAI,KAAK,QAAQ,EAAE;gBACrB,WAAW,GAAG,kCAAkC;YAClD;AAAO,iBAAA,IAAI,IAAI,KAAK,QAAQ,EAAE;gBAC5B,WAAW,GAAG,kCAAkC;YAClD;AAAO,iBAAA,IAAI,IAAI,KAAK,MAAM,EAAE;gBAC1B,WAAW,GAAG,gCAAgC;YAChD;YAEA,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,IAAI,CAC/B,WAAW,EACX,OAAO,EACP;AACE,gBAAA,OAAO,EAAE;oBACP,WAAW,EAAE,MAAM,CAAC,MAAM;AAC1B,oBAAA,cAAc,EAAE,kBAAkB;AACnC,iBAAA;gBACD,OAAO,EAAE,MAAM,CAAC,OAAO;AACxB,aAAA,CACF;AAED,YAAA,MAAM,IAAI,GAAG,QAAQ,CAAC,IAAI;AAC1B,YAAA,MAAM,OAAO,GAAuB;gBAClC,OAAO,EAAE,IAAI,CAAC,OAAO;AACrB,gBAAA,MAAM,EAAE,IAAI,CAAC,MAAM,IAAI,EAAE;gBACzB,SAAS,EAAE,IAAI,CAAC,SAAS;AACzB,gBAAA,UAAU,EAAE,IAAI,CAAC,UAAU,IAAI,EAAE;gBACjC,aAAa,EAAE,IAAI,CAAC,aAAa;gBACjC,cAAc,EAAE,IAAI,CAAC,cAAc;gBACnC,eAAe,EAAE,IAAI,CAAC,eAAe;AACrC,gBAAA,MAAM,EAAE,IAAI,CAAC,MAAM,IAAI,EAAE;AACzB,gBAAA,IAAI,EAAE,IAAI,CAAC,IAAI,IAAI,EAAE;aACtB;YAED,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,IAAI,EAAE,OAAO,EAAE;QACzC;QAAE,OAAO,KAAK,EAAE;AACd,YAAA,MAAM,YAAY,GAChB,KAAK,YAAY,KAAK,GAAG,KAAK,CAAC,OAAO,GAAG,MAAM,CAAC,KAAK,CAAC;YACxD,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,CAAA,oBAAA,EAAuB,YAAY,CAAA,CAAE,EAAE;QACzE;AACF,IAAA,CAAC;IAED,OAAO,EAAE,UAAU,EAAE;AACvB,CAAC;AAED,MAAM,gBAAgB,GAAG,CACvB,WAAoB,EACpB,MAAe,KAGb;AACF,IAAA,MAAM,MAAM,GAAG;AACb,QAAA,WAAW,EAAE,WAAW,IAAI,OAAO,CAAC,GAAG,CAAC,oBAAoB;AAC5D,QAAA,MAAM,EAAE,MAAM,IAAI,OAAO,CAAC,GAAG,CAAC,eAAe;AAC7C,QAAA,eAAe,EAAE,KAAK;AACtB,QAAA,OAAO,EAAE,KAAK;KACf;AAED,IAAA,IAAI,MAAM,CAAC,WAAW,IAAI,IAAI,IAAI,MAAM,CAAC,WAAW,KAAK,EAAE,EAAE;AAC3D,QAAA,MAAM,IAAI,KAAK,CAAC,kDAAkD,CAAC;IACrE;AAEA,IAAA,MAAM,UAAU,GAAG,OAAO,EACxB,KAAK,EACL,UAAU,GAAG,CAAC,EACd,UAAU,EACV,IAAI,GACe,KAA6B;AAChD,QAAA,IAAI,CAAC,KAAK,CAAC,IAAI,EAAE,EAAE;YACjB,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,uBAAuB,EAAE;QAC3D;AAEA,QAAA,IAAI;;AAEF,YAAA,IAAI,MAAM,CAAC,WAAW,IAAI,IAAI,IAAI,MAAM,CAAC,WAAW,KAAK,EAAE,EAAE;gBAC3D,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,6BAA6B,EAAE;YACjE;AAEA,YAAA,IAAI,SAAS,GAAG,MAAM,CAAC,WAAW;YAClC,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC,SAAS,CAAC,EAAE;gBAClC,SAAS,GAAG,SAAS,CAAC,OAAO,CAAC,KAAK,EAAE,EAAE,CAAC,GAAG,SAAS;YACtD;;YAGA,IAAI,QAAQ,GAAG,SAAS;AACxB,YAAA,IAAI,IAAI,KAAK,QAAQ,EAAE;gBACrB,QAAQ,GAAG,QAAQ;YACrB;AAAO,iBAAA,IAAI,IAAI,KAAK,QAAQ,EAAE;gBAC5B,QAAQ,GAAG,QAAQ;YACrB;AAAO,iBAAA,IAAI,IAAI,KAAK,MAAM,EAAE;gBAC1B,QAAQ,GAAG,MAAM;YACnB;;AAGA,YAAA,MAAM,MAAM,GAA2B;AACrC,gBAAA,CAAC,EAAE,KAAK;AACR,gBAAA,MAAM,EAAE,MAAM;AACd,gBAAA,MAAM,EAAE,CAAC;AACT,gBAAA,UAAU,EAAE,QAAQ;AACpB,gBAAA,QAAQ,EAAE,KAAK;AACf,gBAAA,UAAU,EAAE,UAAU;AACtB,gBAAA,OAAO,EAAE,wBAAwB;aAClC;AAED,YAAA,MAAM,OAAO,GAA2B;AACtC,gBAAA,cAAc,EAAE,kBAAkB;aACnC;AAED,YAAA,IAAI,MAAM,CAAC,MAAM,IAAI,IAAI,IAAI,MAAM,CAAC,MAAM,KAAK,EAAE,EAAE;AACjD,gBAAA,OAAO,CAAC,WAAW,CAAC,GAAG,MAAM,CAAC,MAAM;YACtC;YAEA,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,GAAG,CAAC,SAAS,EAAE;gBAC1C,OAAO;gBACP,MAAM;gBACN,OAAO,EAAE,MAAM,CAAC,OAAO;AACxB,aAAA,CAAC;AAEF,YAAA,MAAM,IAAI,GAAG,QAAQ,CAAC,IAAI;;AAG1B,YAAA,MAAM,YAAY,GAAG,CAAC,MAAuB,KAAa;gBACxD,MAAM,GAAG,GAAG,MAAM,CAAC,GAAG,EAAE,WAAW,EAAE,IAAI,EAAE;gBAC3C,MAAM,KAAK,GAAG,MAAM,CAAC,KAAK,EAAE,WAAW,EAAE,IAAI,EAAE;;AAG/C,gBAAA,MAAM,YAAY,GAAG;oBACnB,eAAe;oBACf,aAAa;oBACb,aAAa;oBACb,YAAY;oBACZ,kBAAkB;oBAClB,eAAe;oBACf,MAAM;iBACP;;gBAGD,MAAM,eAAe,GAAG,YAAY,CAAC,IAAI,CACvC,CAAC,OAAO,KAAK,KAAK,CAAC,WAAW,EAAE,CAAC,QAAQ,CAAC,OAAO,CAAC;iBACnD;;AAGD,gBAAA,MAAM,WAAW,GACf,GAAG,CAAC,QAAQ,CAAC,QAAQ,CAAC;AACtB,oBAAA,GAAG,CAAC,QAAQ,CAAC,SAAS,CAAC;AACvB,oBAAA,GAAG,CAAC,QAAQ,CAAC,YAAY,CAAC;AAC1B,oBAAA,GAAG,CAAC,QAAQ,CAAC,YAAY,CAAC;gBAE5B,OAAO,eAAe,IAAI,WAAW;AACvC,YAAA,CAAC;;YAGD,MAAM,cAAc,GAAG,CAAC,IAAI,CAAC,OAAO,IAAI,EAAE;AACvC,iBAAA,KAAK,CAAC,CAAC,EAAE,UAAU;AACnB,iBAAA,GAAG,CAAC,CAAC,MAAuB,EAAE,KAAa,KAAI;gBAC9C,IAAI,WAAW,GAAG,EAAE;AACpB,gBAAA,IAAI;AACF,oBAAA,WAAW,GAAG,IAAI,GAAG,CAAC,MAAM,CAAC,GAAG,IAAI,EAAE,CAAC,CAAC,QAAQ;gBAClD;AAAE,gBAAA,MAAM;oBACN,WAAW,GAAG,EAAE;gBAClB;gBAEA,OAAO;oBACL,QAAQ,EAAE,KAAK,GAAG,CAAC;AACnB,oBAAA,KAAK,EAAE,MAAM,CAAC,KAAK,IAAI,EAAE;AACzB,oBAAA,IAAI,EAAE,MAAM,CAAC,GAAG,IAAI,EAAE;AACtB,oBAAA,OAAO,EAAE,MAAM,CAAC,OAAO,IAAI,EAAE;AAC7B,oBAAA,IAAI,EAAE,MAAM,CAAC,aAAa,IAAI,EAAE;oBAChC,WAAW;iBACZ;AACH,YAAA,CAAC,CAAC;YAEJ,MAAM,YAAY,GAAG,CAAC,IAAI,CAAC,OAAO,IAAI,EAAE;iBACrC,MAAM,CAAC,CAAC,MAAuB,KAAK,MAAM,CAAC,OAAO;AAClD,iBAAA,KAAK,CAAC,CAAC,EAAE,CAAC;iBACV,GAAG,CAAC,CAAC,MAAuB,EAAE,KAAa,MAAM;AAChD,gBAAA,KAAK,EAAE,MAAM,CAAC,KAAK,IAAI,EAAE;AACzB,gBAAA,QAAQ,EAAE,MAAM,CAAC,OAAO,IAAI,EAAE;gBAC9B,QAAQ,EAAE,KAAK,GAAG,CAAC;gBACnB,MAAM,EAAE,IAAI,GAAG,CAAC,MAAM,CAAC,GAAG,IAAI,EAAE,CAAC,CAAC,QAAQ;gBAC1C,MAAM,EAAE,IAAI,GAAG,CAAC,MAAM,CAAC,GAAG,IAAI,EAAE,CAAC,CAAC,QAAQ;AAC1C,gBAAA,IAAI,EAAE,MAAM,CAAC,GAAG,IAAI,EAAE;AACvB,aAAA,CAAC,CAAC;;YAGL,MAAM,WAAW,GAAG,CAAC,IAAI,CAAC,OAAO,IAAI,EAAE;iBACpC,MAAM,CAAC,YAAY;AACnB,iBAAA,GAAG,CAAC,CAAC,MAAuB,EAAE,KAAa,KAAI;gBAC9C,IAAI,WAAW,GAAG,EAAE;AACpB,gBAAA,IAAI;AACF,oBAAA,WAAW,GAAG,IAAI,GAAG,CAAC,MAAM,CAAC,GAAG,IAAI,EAAE,CAAC,CAAC,QAAQ;gBAClD;AAAE,gBAAA,MAAM;oBACN,WAAW,GAAG,EAAE;gBAClB;gBAEA,OAAO;AACL,oBAAA,KAAK,EAAE,MAAM,CAAC,KAAK,IAAI,EAAE;AACzB,oBAAA,IAAI,EAAE,MAAM,CAAC,GAAG,IAAI,EAAE;AACtB,oBAAA,OAAO,EAAE,MAAM,CAAC,OAAO,IAAI,EAAE;AAC7B,oBAAA,IAAI,EAAE,MAAM,CAAC,aAAa,IAAI,EAAE;AAChC,oBAAA,MAAM,EAAE,WAAW;AACnB,oBAAA,QAAQ,EAAE,MAAM,CAAC,OAAO,IAAI,EAAE;oBAC9B,QAAQ,EAAE,KAAK,GAAG,CAAC;iBACpB;AACH,YAAA,CAAC,CAAC;YAEJ,MAAM,UAAU,GAAG,WAAW,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC;YAE1C,MAAM,eAAe,GAAG,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,WAAW;AACpD,kBAAE,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC,UAAkB,MAAM,EAAE,KAAK,EAAE,UAAU,EAAE,CAAC;kBACpE,EAAE;AAEN,YAAA,MAAM,OAAO,GAAuB;AAClC,gBAAA,OAAO,EAAE,cAAc;AACvB,gBAAA,MAAM,EAAE,YAAY;gBACpB,UAAU,EAAE,UAAU;gBACtB,eAAe;AACf,gBAAA,MAAM,EAAE,EAAE;AACV,gBAAA,IAAI,EAAE,WAAW;;AAEjB,gBAAA,MAAM,EAAE,EAAE;AACV,gBAAA,QAAQ,EAAE,EAAE;AACZ,gBAAA,aAAa,EAAE,EAAE;AACjB,gBAAA,cAAc,EAAE,SAAS;AACzB,gBAAA,SAAS,EAAE,SAAS;aACrB;YAED,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,IAAI,EAAE,OAAO,EAAE;QACzC;QAAE,OAAO,KAAK,EAAE;AACd,YAAA,MAAM,YAAY,GAChB,KAAK,YAAY,KAAK,GAAG,KAAK,CAAC,OAAO,GAAG,MAAM,CAAC,KAAK,CAAC;YACxD,OAAO;AACL,gBAAA,OAAO,EAAE,KAAK;gBACd,KAAK,EAAE,CAAA,4BAAA,EAA+B,YAAY,CAAA,CAAE;aACrD;QACH;AACF,IAAA,CAAC;IAED,OAAO,EAAE,UAAU,EAAE;AACvB,CAAC;AAEM,MAAM,eAAe,GAAG,CAC7B,MAAsB,KAGpB;AACF,IAAA,MAAM,EACJ,cAAc,GAAG,QAAQ,EACzB,YAAY,EACZ,kBAAkB,EAClB,aAAa,EACb,eAAe,EACf,gBAAgB,GACjB,GAAG,MAAM;AAEV,IAAA,IAAI,cAAc,CAAC,WAAW,EAAE,KAAK,QAAQ,EAAE;QAC7C,OAAO,eAAe,CAAC,YAAY,EAAE,eAAe,EAAE,gBAAgB,CAAC;IACzE;AAAO,SAAA,IAAI,cAAc,CAAC,WAAW,EAAE,KAAK,SAAS,EAAE;AACrD,QAAA,OAAO,gBAAgB,CAAC,kBAAkB,EAAE,aAAa,CAAC;IAC5D;SAAO;AACL,QAAA,MAAM,IAAI,KAAK,CACb,4BAA4B,cAAc,CAAA,+BAAA,CAAiC,CAC5E;IACH;AACF;AAEO,MAAM,qBAAqB,GAAG,CACnC,SAAiC,EAAE,EACnC,eAA+B,KAM7B;IACF,IAAI,CAAC,eAAe,EAAE;AACpB,QAAA,MAAM,IAAI,KAAK,CAAC,8BAA8B,CAAC;IACjD;IACA,MAAM,EACJ,UAAU,GAAG,CAAC;;;AAGd,IAAA,QAAQ,EACR,MAAM,GACP,GAAG,MAAM;AAEV,IAAA,MAAM,OAAO,GAAG,MAAM,IAAIA,yBAAmB,EAAE;IAC/C,MAAM,OAAO,GAAG,eAAe;AAE/B,IAAA,MAAM,UAAU,GAAG;QACjB,UAAU,EAAE,OAAO,EACjB,KAAK,EACL,KAAK,EACL,eAAe,GAKhB,KAAoC;YACnC,OAAO,CAAC,KAAK,CAAC,CAAA,SAAA,EAAY,KAAK,CAAC,MAAM,CAAA,MAAA,CAAQ,CAAC;YAC/C,MAAM,QAAQ,GAAmC,EAAE;AACnD,YAAA,IAAI;AACF,gBAAA,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE;AACrC,oBAAA,MAAM,WAAW,GAAG,KAAK,CAAC,CAAC,CAAC;oBAC5B,MAAM,OAAO,GAA4B;AACtC,yBAAA,SAAS,CAAC,WAAW,EAAE,EAAE;yBACzB,IAAI,CAAC,CAAC,CAAC,GAAG,EAAE,QAAQ,CAAC,KAAI;AACxB,wBAAA,MAAM,WAAW,GAAGC,oBAAc,CAChC,GAAG,EACH,QAAQ,CAAC,IAAI,EAAE,QAAQ,EACvB,OAAO,CACR;wBACD,IAAI,QAAQ,CAAC,OAAO,IAAI,QAAQ,CAAC,IAAI,EAAE;AACrC,4BAAA,MAAM,CAAC,OAAO,EAAE,UAAU,CAAC,GAAG,OAAO,CAAC,cAAc,CAAC,QAAQ,CAAC;4BAC9D,OAAO;gCACL,GAAG;gCACH,UAAU;gCACV,WAAW;AACX,gCAAA,OAAO,EAAE,OAAO,CAAC,SAAS,CAAC,OAAO,CAAC;6BAClB;wBACrB;6BAAO;AACL,4BAAA,OAAO,CAAC,KAAK,CACX,CAAA,eAAA,EAAkB,GAAG,CAAA,EAAA,EAAK,QAAQ,CAAC,KAAK,IAAI,eAAe,CAAA,CAAE,CAC9D;wBACH;wBAEA,OAAO;4BACL,GAAG;4BACH,WAAW;AACX,4BAAA,KAAK,EAAE,IAAI;AACX,4BAAA,OAAO,EAAE,EAAE;yBACM;AACrB,oBAAA,CAAC;AACA,yBAAA,IAAI,CAAC,OAAO,MAAM,KAAI;AACrB,wBAAA,IAAI;AACF,4BAAA,IAAI,MAAM,CAAC,KAAK,IAAI,IAAI,EAAE;AACxB,gCAAA,OAAO,CAAC,KAAK,CACX,CAAA,eAAA,EAAkB,MAAM,CAAC,GAAG,CAAA,EAAA,EAAK,MAAM,CAAC,OAAO,CAAA,CAAE,CAClD;gCACD,OAAO;AACL,oCAAA,GAAG,MAAM;iCACV;4BACH;AACA,4BAAA,MAAM,UAAU,GAAG,MAAM,aAAa,CAAC;gCACrC,KAAK;gCACL,QAAQ;gCACR,OAAO,EAAE,MAAM,CAAC,OAAO;AACvB,gCAAA,MAAM,EAAE,OAAO;AAChB,6BAAA,CAAC;4BACF,IAAI,eAAe,EAAE;AACnB,gCAAA,eAAe,CAAC,MAAM,CAAC,GAAG,CAAC;4BAC7B;4BACA,OAAO;AACL,gCAAA,GAAG,MAAM;gCACT,UAAU;6BACX;wBACH;wBAAE,OAAO,KAAK,EAAE;AACd,4BAAA,OAAO,CAAC,KAAK,CAAC,mCAAmC,EAAE,KAAK,CAAC;4BACzD,OAAO;AACL,gCAAA,GAAG,MAAM;6BACV;wBACH;AACF,oBAAA,CAAC;AACA,yBAAA,KAAK,CAAC,CAAC,KAAK,KAAI;wBACf,OAAO,CAAC,KAAK,CAAC,CAAA,eAAA,EAAkB,WAAW,CAAA,CAAA,CAAG,EAAE,KAAK,CAAC;wBACtD,OAAO;AACL,4BAAA,GAAG,EAAE,WAAW;AAChB,4BAAA,KAAK,EAAE,IAAI;AACX,4BAAA,OAAO,EAAE,EAAE;yBACZ;AACH,oBAAA,CAAC,CAAC;AACJ,oBAAA,QAAQ,CAAC,IAAI,CAAC,OAAO,CAAC;gBACxB;AACA,gBAAA,OAAO,MAAM,OAAO,CAAC,GAAG,CAAC,QAAQ,CAAC;YACpC;YAAE,OAAO,KAAK,EAAE;AACd,gBAAA,OAAO,CAAC,KAAK,CAAC,sBAAsB,EAAE,KAAK,CAAC;AAC5C,gBAAA,OAAO,EAAE;YACX;QACF,CAAC;KACF;AAED,IAAA,MAAM,aAAa,GAAG,OAAO,EAC3B,KAAK,EACL,KAAK,EACL,MAAM,EACN,eAAe,EACf,gBAAgB,GAOjB,KAAmB;QAClB,MAAM,YAAY,GAAG,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,MAAM,CAAC;;AAE3C,QAAA,MAAM,OAAO,GAAG,MAAM,UAAU,CAAC,UAAU,CAAC;YAC1C,KAAK;AACL,YAAA,KAAK,EAAE,YAAY;YACnB,eAAe;AAChB,SAAA,CAAC;AACF,QAAA,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE;AAC5B,YAAA,IAAI,MAAM,CAAC,KAAK,KAAK,IAAI,EAAE;gBACzB;YACF;AACA,YAAA,MAAM,EAAE,GAAG,EAAE,OAAO,EAAE,WAAW,EAAE,UAAU,EAAE,UAAU,EAAE,GAAG,MAAM;YACpE,gBAAgB,GAAG,GAAG,EAAE;gBACtB,OAAO;gBACP,WAAW;gBACX,UAAU;gBACV,UAAU;AACX,aAAA,CAAC;QACJ;AACF,IAAA,CAAC;IAED,MAAM,cAAc,GAAG,OAAO,EAC5B,MAAM,EACN,WAAW,EACX,KAAK,EACL,IAAI,EACJ,OAAO,GAAG,IAAI,EACd,eAAe,EACf,YAAY,GAAG,KAAK,GACG,KAAiC;AACxD,QAAA,IAAI;AACF,YAAA,IAAI,CAAC,MAAM,CAAC,IAAI,EAAE;gBAChB,OAAO;AACL,oBAAA,OAAO,EAAE,EAAE;AACX,oBAAA,UAAU,EAAE,EAAE;AACd,oBAAA,MAAM,EAAE,EAAE;AACV,oBAAA,eAAe,EAAE,EAAE;iBACpB;YACH;AAAO,iBAAA,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,OAAO,EAAE;gBAC/B,OAAO,MAAM,CAAC,IAAI;YACpB;;YAGA,IAAI,YAAY,EAAE;AAChB,gBAAA,OAAO,CAAC,KAAK,CACX,0DAA0D,CAC3D;gBACD,OAAO,MAAM,CAAC,IAAI;YACpB;YAEA,IAAI,CAAC,OAAO,EAAE;gBACZ,MAAM,WAAW,GAAG,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,MAAM,KACpD,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC,eAAe,CAAC,CACtC;AAED,gBAAA,IAAI,CAAC,WAAW,CAAC,MAAM,EAAE;oBACvB,OAAO,MAAM,CAAC,IAAI;gBACpB;AAEA,gBAAA,MAAM,aAAa,GAAG,IAAI,GAAG,EAAyB;AACtD,gBAAA,aAAa,CAAC,GAAG,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,WAAW,CAAC,CAAC,CAAC,CAAC;AACtD,gBAAA,MAAM,gBAAgB,GAAG,0BAA0B,CAAC,aAAa,CAAC;AAClE,gBAAA,MAAM,aAAa,CAAC;oBAClB,KAAK;AACL,oBAAA,MAAM,EAAE,CAAC;oBACT,eAAe;oBACf,gBAAgB;oBAChB,KAAK,EAAE,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;AAC7B,iBAAA,CAAC;AAEF,gBAAA,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE;oBACnD,MAAM,MAAM,GAAG,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC;oBACrC,MAAM,aAAa,GAAG,aAAa,CAAC,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC;oBACpD,IAAI,aAAa,EAAE;AACjB,wBAAA,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG;AACvB,4BAAA,GAAG,MAAM;AACT,4BAAA,GAAG,aAAa;yBACjB;oBACH;gBACF;gBAEA,OAAO,MAAM,CAAC,IAAI;YACpB;AAEA,YAAA,MAAM,SAAS,GAAG,IAAI,GAAG,EAAyB;AAClD,YAAA,MAAM,eAAe,GAAG,IAAI,GAAG,EAAU;;AAGzC,YAAA,MAAM,YAAY,GAAG,YAAY,CAC/B,MAAM,CAAC,IAAI,CAAC,OAAO,EACnB,SAAS,EACT,eAAe,CAChB;;YAGD,MAAM,UAAU,GAAG,MAAM,CAAC,IAAI,CAAC,UAAU,IAAI,EAAE;YAC/C,MAAM,aAAa,GAAG,YAAY,CAChC,UAAU,EACV,SAAS,EACT,eAAe,CAChB;AAED,YAAA,IAAI,YAAY,CAAC,MAAM,KAAK,CAAC,KAAK,aAAa,CAAC,MAAM,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE;gBACtE,OAAO,MAAM,CAAC,IAAI;YACpB;AAEA,YAAA,MAAM,gBAAgB,GAAG,0BAA0B,CAAC,SAAS,CAAC;YAC9D,MAAM,QAAQ,GAAoB,EAAE;;AAGpC,YAAA,IAAI,YAAY,CAAC,MAAM,GAAG,CAAC,EAAE;AAC3B,gBAAA,QAAQ,CAAC,IAAI,CACX,aAAa,CAAC;oBACZ,KAAK;oBACL,eAAe;oBACf,gBAAgB;AAChB,oBAAA,KAAK,EAAE,YAAY;AACnB,oBAAA,MAAM,EAAE,WAAW;AACpB,iBAAA,CAAC,CACH;YACH;;YAGA,IAAI,IAAI,IAAI,aAAa,CAAC,MAAM,GAAG,CAAC,EAAE;AACpC,gBAAA,QAAQ,CAAC,IAAI,CACX,aAAa,CAAC;oBACZ,KAAK;oBACL,eAAe;oBACf,gBAAgB;AAChB,oBAAA,KAAK,EAAE,aAAa;AACpB,oBAAA,MAAM,EAAE,WAAW;AACpB,iBAAA,CAAC,CACH;YACH;AAEA,YAAA,MAAM,OAAO,CAAC,GAAG,CAAC,QAAQ,CAAC;YAE3B,IAAI,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE;gBAClC,wBAAwB,CAAC,MAAM,CAAC,IAAI,CAAC,OAAO,EAAE,SAAS,CAAC;YAC1D;YAEA,IAAI,IAAI,IAAI,UAAU,CAAC,MAAM,GAAG,CAAC,EAAE;AACjC,gBAAA,wBAAwB,CAAC,UAAU,EAAE,SAAS,CAAC;YACjD;YAEA,OAAO,MAAM,CAAC,IAAI;QACpB;QAAE,OAAO,KAAK,EAAE;AACd,YAAA,OAAO,CAAC,KAAK,CAAC,0BAA0B,EAAE,KAAK,CAAC;YAChD,OAAO;AACL,gBAAA,OAAO,EAAE,EAAE;AACX,gBAAA,UAAU,EAAE,EAAE;AACd,gBAAA,MAAM,EAAE,EAAE;AACV,gBAAA,eAAe,EAAE,EAAE;gBACnB,GAAG,MAAM,CAAC,IAAI;AACd,gBAAA,KAAK,EAAE,KAAK,YAAY,KAAK,GAAG,KAAK,CAAC,OAAO,GAAG,MAAM,CAAC,KAAK,CAAC;aAC9D;QACH;AACF,IAAA,CAAC;IAED,OAAO;QACL,cAAc;QACd,UAAU;KACX;AACH;AAEA;AACA,SAAS,YAAY,CACnB,OAAkD,EAClD,SAAqC,EACrC,gBAA8B,EAAA;IAE9B,MAAM,KAAK,GAAa,EAAE;AAE1B,IAAA,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE;AAC5B,QAAA,IAAI,MAAM,CAAC,IAAI,EAAE;;YAEf,IAAI,gBAAgB,IAAI,gBAAgB,CAAC,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE;gBACzD;YACF;AAEA,YAAA,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC;YACvB,IAAI,gBAAgB,EAAE;AACpB,gBAAA,gBAAgB,CAAC,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC;YACnC;YACA,SAAS,CAAC,GAAG,CAAC,MAAM,CAAC,IAAI,EAAE,MAAuB,CAAC;QACrD;IACF;AAEA,IAAA,OAAO,KAAK;AACd;AAEA;AACA,SAAS,wBAAwB,CAC/B,OAAY,EACZ,SAAqC,EAAA;AAErC,IAAA,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE;AACvC,QAAA,MAAM,MAAM,GAAG,OAAO,CAAC,CAAC,CAAC;QACzB,MAAM,aAAa,GAAG,SAAS,CAAC,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC;QAChD,IAAI,aAAa,EAAE;YACjB,OAAO,CAAC,CAAC,CAAC,GAAG;AACX,gBAAA,GAAG,MAAM;AACT,gBAAA,GAAG,aAAa;aACZ;QACR;IACF;AACF;;;;;"}
1
+ {"version":3,"file":"search.cjs","sources":["../../../../src/tools/search/search.ts"],"sourcesContent":["import axios from 'axios';\nimport { RecursiveCharacterTextSplitter } from '@langchain/textsplitters';\nimport type * as t from './types';\nimport { getAttribution, createDefaultLogger } from './utils';\nimport { createTavilyAPI } from './tavily-search';\nimport { BaseReranker } from './rerankers';\n\nconst chunker = {\n cleanText: (text: string): string => {\n if (!text) return '';\n\n /** Normalized all line endings to '\\n' */\n const normalizedText = text.replace(/\\r\\n/g, '\\n').replace(/\\r/g, '\\n');\n\n /** Handle multiple backslashes followed by newlines\n * This replaces patterns like '\\\\\\\\\\\\n' with a single newline */\n const fixedBackslashes = normalizedText.replace(/\\\\+\\n/g, '\\n');\n\n /** Cleaned up consecutive newlines, tabs, and spaces around newlines */\n const cleanedNewlines = fixedBackslashes.replace(/[\\t ]*\\n[\\t \\n]*/g, '\\n');\n\n /** Cleaned up excessive spaces and tabs */\n const cleanedSpaces = cleanedNewlines.replace(/[ \\t]+/g, ' ');\n\n return cleanedSpaces.trim();\n },\n splitText: async (\n text: string,\n options?: {\n chunkSize?: number;\n chunkOverlap?: number;\n separators?: string[];\n }\n ): Promise<string[]> => {\n const chunkSize = options?.chunkSize ?? 150;\n const chunkOverlap = options?.chunkOverlap ?? 50;\n const separators = options?.separators || ['\\n\\n', '\\n'];\n\n const splitter = new RecursiveCharacterTextSplitter({\n separators,\n chunkSize,\n chunkOverlap,\n });\n\n return await splitter.splitText(text);\n },\n\n splitTexts: async (\n texts: string[],\n options?: {\n chunkSize?: number;\n chunkOverlap?: number;\n separators?: string[];\n },\n logger?: t.Logger\n ): Promise<string[][]> => {\n // Split multiple texts\n const logger_ = logger || createDefaultLogger();\n const promises = texts.map((text) =>\n chunker.splitText(text, options).catch((error) => {\n logger_.error('Error splitting text:', error);\n return [text];\n })\n );\n return Promise.all(promises);\n },\n};\n\nfunction createSourceUpdateCallback(sourceMap: Map<string, t.ValidSource>) {\n return (link: string, update?: Partial<t.ValidSource>): void => {\n const source = sourceMap.get(link);\n if (source) {\n sourceMap.set(link, {\n ...source,\n ...update,\n });\n }\n };\n}\n\nconst getHighlights = async ({\n query,\n content,\n reranker,\n topResults = 5,\n logger,\n}: {\n content: string;\n query: string;\n reranker?: BaseReranker;\n topResults?: number;\n logger?: t.Logger;\n}): Promise<t.Highlight[] | undefined> => {\n const logger_ = logger || createDefaultLogger();\n\n if (!content) {\n logger_.warn('No content provided for highlights');\n return;\n }\n if (!reranker) {\n logger_.warn('No reranker provided for highlights');\n return;\n }\n\n try {\n const documents = await chunker.splitText(content);\n if (Array.isArray(documents)) {\n return await reranker.rerank(query, documents, topResults);\n } else {\n logger_.error(\n 'Expected documents to be an array, got:',\n typeof documents\n );\n return;\n }\n } catch (error) {\n logger_.error('Error in content processing:', error);\n return;\n }\n};\n\nconst createSerperAPI = (\n apiKey?: string,\n domainBlocklist?: string[],\n countryBlocklist?: string[]\n): {\n getSources: (params: t.GetSourcesParams) => Promise<t.SearchResult>;\n} => {\n const config = {\n apiKey: apiKey ?? process.env.SERPER_API_KEY,\n apiUrl: 'https://google.serper.dev/search',\n timeout: 10000,\n };\n\n if (config.apiKey == null || config.apiKey === '') {\n throw new Error('SERPER_API_KEY is required for SerperAPI');\n }\n\n const getSources = async ({\n query,\n date,\n country,\n safeSearch,\n numResults = 8,\n type,\n }: t.GetSourcesParams): Promise<t.SearchResult> => {\n if (!query.trim()) {\n return { success: false, error: 'Query cannot be empty' };\n }\n\n try {\n const safe = ['off', 'moderate', 'active'] as const;\n\n // Append -site: operators for blocked domains so they never appear in results\n let effectiveQuery = query;\n if (domainBlocklist && domainBlocklist.length > 0) {\n const siteExclusions = domainBlocklist\n .map((d) => `-site:${d}`)\n .join(' ');\n effectiveQuery = `${effectiveQuery} ${siteExclusions}`;\n }\n\n // Append -site:.XX operators for blocked country TLDs\n if (countryBlocklist && countryBlocklist.length > 0) {\n const countryExclusions = countryBlocklist\n .map((c) => `-site:.${c.toLowerCase().replace(/^\\./, '')}`)\n .join(' ');\n effectiveQuery = `${effectiveQuery} ${countryExclusions}`;\n }\n\n const payload: t.SerperSearchPayload = {\n q: effectiveQuery,\n safe: safe[safeSearch ?? 1],\n num: Math.min(Math.max(1, numResults), 10),\n };\n\n // Set the search type if provided\n if (type) {\n payload.type = type;\n }\n\n if (date != null) {\n payload.tbs = `qdr:${date}`;\n }\n\n if (country != null && country !== '') {\n payload['gl'] = country.toLowerCase();\n }\n\n // Determine the API endpoint based on the search type\n let apiEndpoint = config.apiUrl;\n if (type === 'images') {\n apiEndpoint = 'https://google.serper.dev/images';\n } else if (type === 'videos') {\n apiEndpoint = 'https://google.serper.dev/videos';\n } else if (type === 'news') {\n apiEndpoint = 'https://google.serper.dev/news';\n }\n\n const response = await axios.post<t.SerperResultData>(\n apiEndpoint,\n payload,\n {\n headers: {\n 'X-API-KEY': config.apiKey,\n 'Content-Type': 'application/json',\n },\n timeout: config.timeout,\n }\n );\n\n const data = response.data;\n const results: t.SearchResultData = {\n organic: data.organic,\n images: data.images ?? [],\n answerBox: data.answerBox,\n topStories: data.topStories ?? [],\n peopleAlsoAsk: data.peopleAlsoAsk,\n knowledgeGraph: data.knowledgeGraph,\n relatedSearches: data.relatedSearches,\n videos: data.videos ?? [],\n news: data.news ?? [],\n };\n\n return { success: true, data: results };\n } catch (error) {\n const errorMessage =\n error instanceof Error ? error.message : String(error);\n return { success: false, error: `API request failed: ${errorMessage}` };\n }\n };\n\n return { getSources };\n};\n\nconst createSearXNGAPI = (\n instanceUrl?: string,\n apiKey?: string\n): {\n getSources: (params: t.GetSourcesParams) => Promise<t.SearchResult>;\n} => {\n const config = {\n instanceUrl: instanceUrl ?? process.env.SEARXNG_INSTANCE_URL,\n apiKey: apiKey ?? process.env.SEARXNG_API_KEY,\n defaultLocation: 'all',\n timeout: 10000,\n };\n\n if (config.instanceUrl == null || config.instanceUrl === '') {\n throw new Error('SEARXNG_INSTANCE_URL is required for SearXNG API');\n }\n\n const getSources = async ({\n query,\n numResults = 8,\n safeSearch,\n type,\n }: t.GetSourcesParams): Promise<t.SearchResult> => {\n if (!query.trim()) {\n return { success: false, error: 'Query cannot be empty' };\n }\n\n try {\n // Ensure the instance URL ends with /search\n if (config.instanceUrl == null || config.instanceUrl === '') {\n return { success: false, error: 'Instance URL is not defined' };\n }\n\n let searchUrl = config.instanceUrl;\n if (!searchUrl.endsWith('/search')) {\n searchUrl = searchUrl.replace(/\\/$/, '') + '/search';\n }\n\n // Determine the search category based on the type\n let category = 'general';\n if (type === 'images') {\n category = 'images';\n } else if (type === 'videos') {\n category = 'videos';\n } else if (type === 'news') {\n category = 'news';\n }\n\n // Prepare parameters for SearXNG\n const params: t.SearxNGSearchPayload = {\n q: query,\n format: 'json',\n pageno: 1,\n categories: category,\n language: 'all',\n safesearch: safeSearch,\n engines: 'google,bing,duckduckgo',\n };\n\n const headers: Record<string, string> = {\n 'Content-Type': 'application/json',\n };\n\n if (config.apiKey != null && config.apiKey !== '') {\n headers['X-API-Key'] = config.apiKey;\n }\n\n const response = await axios.get(searchUrl, {\n headers,\n params,\n timeout: config.timeout,\n });\n\n const data = response.data;\n\n // Helper function to identify news results since SearXNG doesn't provide that classification by default\n const isNewsResult = (result: t.SearXNGResult): boolean => {\n const url = result.url?.toLowerCase() ?? '';\n const title = result.title?.toLowerCase() ?? '';\n\n // News-related keywords in title/content\n const newsKeywords = [\n 'breaking news',\n 'latest news',\n 'top stories',\n 'news today',\n 'developing story',\n 'trending news',\n 'news',\n ];\n\n // Check if title/content contains news keywords\n const hasNewsKeywords = newsKeywords.some(\n (keyword) => title.toLowerCase().includes(keyword) // just title probably fine, content parsing is overkill for what we need: || content.includes(keyword)\n );\n\n // Check if URL contains news-related paths\n const hasNewsPath =\n url.includes('/news/') ||\n url.includes('/world/') ||\n url.includes('/politics/') ||\n url.includes('/breaking/');\n\n return hasNewsKeywords || hasNewsPath;\n };\n\n // Transform SearXNG results to match SerperAPI format\n const organicResults = (data.results ?? [])\n .slice(0, numResults)\n .map((result: t.SearXNGResult, index: number) => {\n let attribution = '';\n try {\n attribution = new URL(result.url ?? '').hostname;\n } catch {\n attribution = '';\n }\n\n return {\n position: index + 1,\n title: result.title ?? '',\n link: result.url ?? '',\n snippet: result.content ?? '',\n date: result.publishedDate ?? '',\n attribution,\n };\n });\n\n const imageResults = (data.results ?? [])\n .filter((result: t.SearXNGResult) => result.img_src)\n .slice(0, 6)\n .map((result: t.SearXNGResult, index: number) => ({\n title: result.title ?? '',\n imageUrl: result.img_src ?? '',\n position: index + 1,\n source: new URL(result.url ?? '').hostname,\n domain: new URL(result.url ?? '').hostname,\n link: result.url ?? '',\n }));\n\n // Extract news results from organic results\n const newsResults = (data.results ?? [])\n .filter(isNewsResult)\n .map((result: t.SearXNGResult, index: number) => {\n let attribution = '';\n try {\n attribution = new URL(result.url ?? '').hostname;\n } catch {\n attribution = '';\n }\n\n return {\n title: result.title ?? '',\n link: result.url ?? '',\n snippet: result.content ?? '',\n date: result.publishedDate ?? '',\n source: attribution,\n imageUrl: result.img_src ?? '',\n position: index + 1,\n };\n });\n\n const topStories = newsResults.slice(0, 5);\n\n const relatedSearches = Array.isArray(data.suggestions)\n ? data.suggestions.map((suggestion: string) => ({ query: suggestion }))\n : [];\n\n const results: t.SearchResultData = {\n organic: organicResults,\n images: imageResults,\n topStories: topStories, // Use first 5 extracted news as top stories\n relatedSearches,\n videos: [],\n news: newsResults,\n // Add empty arrays for other Serper fields to maintain parity\n places: [],\n shopping: [],\n peopleAlsoAsk: [],\n knowledgeGraph: undefined,\n answerBox: undefined,\n };\n\n return { success: true, data: results };\n } catch (error) {\n const errorMessage =\n error instanceof Error ? error.message : String(error);\n return {\n success: false,\n error: `SearXNG API request failed: ${errorMessage}`,\n };\n }\n };\n\n return { getSources };\n};\n\nexport const createSearchAPI = (\n config: t.SearchConfig\n): {\n getSources: (params: t.GetSourcesParams) => Promise<t.SearchResult>;\n} => {\n const {\n searchProvider = 'serper',\n serperApiKey,\n searxngInstanceUrl,\n searxngApiKey,\n tavilyApiKey,\n tavilySearchUrl,\n tavilySearchOptions,\n domainBlocklist,\n countryBlocklist,\n } = config;\n\n if (searchProvider.toLowerCase() === 'serper') {\n return createSerperAPI(serperApiKey, domainBlocklist, countryBlocklist);\n } else if (searchProvider.toLowerCase() === 'searxng') {\n return createSearXNGAPI(searxngInstanceUrl, searxngApiKey);\n } else if (searchProvider.toLowerCase() === 'tavily') {\n return createTavilyAPI(tavilyApiKey, tavilySearchUrl, tavilySearchOptions);\n } else {\n throw new Error(\n `Invalid search provider: ${searchProvider}. Must be 'serper', 'searxng', or 'tavily'`\n );\n }\n};\n\nexport const createSourceProcessor = (\n config: t.ProcessSourcesConfig = {},\n scraperInstance?: t.BaseScraper\n): {\n processSources: (\n fields: t.ProcessSourcesFields\n ) => Promise<t.SearchResultData>;\n topResults: number;\n} => {\n if (!scraperInstance) {\n throw new Error('Scraper instance is required');\n }\n const {\n topResults = 5,\n // strategies = ['no_extraction'],\n // filterContent = true,\n reranker,\n logger,\n } = config;\n\n const logger_ = logger || createDefaultLogger();\n const scraper = scraperInstance;\n\n const webScraper = {\n scrapeMany: async ({\n query,\n links,\n onGetHighlights,\n }: {\n query: string;\n links: string[];\n onGetHighlights: t.SearchToolConfig['onGetHighlights'];\n }): Promise<Array<t.ScrapeResult>> => {\n logger_.debug(`Scraping ${links.length} links`);\n const promises: Array<Promise<t.ScrapeResult>> = [];\n try {\n for (let i = 0; i < links.length; i++) {\n const currentLink = links[i];\n const promise: Promise<t.ScrapeResult> = scraper\n .scrapeUrl(currentLink, {})\n .then(([url, response]) => {\n /* metadata exists on Firecrawl/Serper responses but not on\n * Tavily's. Read it via the scraper's extractMetadata abstraction\n * to stay polymorphic across all scraper providers. */\n const rawMeta = scraper.extractMetadata(response);\n const attribution = getAttribution(\n url,\n Object.keys(rawMeta).length > 0 ? rawMeta : undefined,\n logger_\n );\n if (response.success && response.data) {\n const [content, references] = scraper.extractContent(response);\n return {\n url,\n references,\n attribution,\n content: chunker.cleanText(content),\n } as t.ScrapeResult;\n } else {\n logger_.error(\n `Error scraping ${url}: ${response.error ?? 'Unknown error'}`\n );\n }\n\n return {\n url,\n attribution,\n error: true,\n content: '',\n } as t.ScrapeResult;\n })\n .then(async (result) => {\n try {\n if (result.error != null) {\n logger_.error(\n `Error scraping ${result.url}: ${result.content}`\n );\n return {\n ...result,\n };\n }\n const highlights = await getHighlights({\n query,\n reranker,\n content: result.content,\n logger: logger_,\n });\n if (onGetHighlights) {\n onGetHighlights(result.url);\n }\n return {\n ...result,\n highlights,\n };\n } catch (error) {\n logger_.error('Error processing scraped content:', error);\n return {\n ...result,\n };\n }\n })\n .catch((error) => {\n logger_.error(`Error scraping ${currentLink}:`, error);\n return {\n url: currentLink,\n error: true,\n content: '',\n };\n });\n promises.push(promise);\n }\n return await Promise.all(promises);\n } catch (error) {\n logger_.error('Error in scrapeMany:', error);\n return [];\n }\n },\n };\n\n const fetchContents = async ({\n links,\n query,\n target,\n onGetHighlights,\n onContentScraped,\n }: {\n links: string[];\n query: string;\n target: number;\n onGetHighlights: t.SearchToolConfig['onGetHighlights'];\n onContentScraped?: (link: string, update?: Partial<t.ValidSource>) => void;\n }): Promise<void> => {\n const initialLinks = links.slice(0, target);\n // const remainingLinks = links.slice(target).reverse();\n const results = await webScraper.scrapeMany({\n query,\n links: initialLinks,\n onGetHighlights,\n });\n for (const result of results) {\n if (result.error === true) {\n continue;\n }\n const { url, content, attribution, references, highlights } = result;\n onContentScraped?.(url, {\n content,\n attribution,\n references,\n highlights,\n });\n }\n };\n\n const processSources = async ({\n result,\n numElements,\n query,\n news,\n proMode = true,\n onGetHighlights,\n skipScraping = false,\n }: t.ProcessSourcesFields): Promise<t.SearchResultData> => {\n try {\n if (!result.data) {\n return {\n organic: [],\n topStories: [],\n images: [],\n relatedSearches: [],\n };\n } else if (!result.data.organic) {\n return result.data;\n }\n\n // If content was already extracted directly (e.g., direct URL extraction), skip scraping\n if (skipScraping) {\n logger_.debug(\n 'Skipping additional scraping - content already extracted'\n );\n return result.data;\n }\n\n if (!proMode) {\n const wikiSources = result.data.organic.filter((source) =>\n source.link.includes('wikipedia.org')\n );\n\n if (!wikiSources.length) {\n return result.data;\n }\n\n const wikiSourceMap = new Map<string, t.ValidSource>();\n wikiSourceMap.set(wikiSources[0].link, wikiSources[0]);\n const onContentScraped = createSourceUpdateCallback(wikiSourceMap);\n await fetchContents({\n query,\n target: 1,\n onGetHighlights,\n onContentScraped,\n links: [wikiSources[0].link],\n });\n\n for (let i = 0; i < result.data.organic.length; i++) {\n const source = result.data.organic[i];\n const updatedSource = wikiSourceMap.get(source.link);\n if (updatedSource) {\n result.data.organic[i] = {\n ...source,\n ...updatedSource,\n };\n }\n }\n\n return result.data;\n }\n\n const sourceMap = new Map<string, t.ValidSource>();\n const organicLinksSet = new Set<string>();\n\n // Collect organic links\n const organicLinks = collectLinks(\n result.data.organic,\n sourceMap,\n organicLinksSet\n );\n\n // Collect top story links, excluding any that are already in organic links\n const topStories = result.data.topStories ?? [];\n const topStoryLinks = collectLinks(\n topStories,\n sourceMap,\n organicLinksSet\n );\n\n if (organicLinks.length === 0 && (topStoryLinks.length === 0 || !news)) {\n return result.data;\n }\n\n const onContentScraped = createSourceUpdateCallback(sourceMap);\n const promises: Promise<void>[] = [];\n\n // Process organic links\n if (organicLinks.length > 0) {\n promises.push(\n fetchContents({\n query,\n onGetHighlights,\n onContentScraped,\n links: organicLinks,\n target: numElements,\n })\n );\n }\n\n // Process top story links\n if (news && topStoryLinks.length > 0) {\n promises.push(\n fetchContents({\n query,\n onGetHighlights,\n onContentScraped,\n links: topStoryLinks,\n target: numElements,\n })\n );\n }\n\n await Promise.all(promises);\n\n if (result.data.organic.length > 0) {\n updateSourcesWithContent(result.data.organic, sourceMap);\n }\n\n if (news && topStories.length > 0) {\n updateSourcesWithContent(topStories, sourceMap);\n }\n\n return result.data;\n } catch (error) {\n logger_.error('Error in processSources:', error);\n return {\n organic: [],\n topStories: [],\n images: [],\n relatedSearches: [],\n ...result.data,\n error: error instanceof Error ? error.message : String(error),\n };\n }\n };\n\n return {\n processSources,\n topResults,\n };\n};\n\n/** Helper function to collect links and update sourceMap */\nfunction collectLinks(\n sources: Array<t.OrganicResult | t.TopStoryResult>,\n sourceMap: Map<string, t.ValidSource>,\n existingLinksSet?: Set<string>\n): string[] {\n const links: string[] = [];\n\n for (const source of sources) {\n if (source.link) {\n // For topStories, only add if not already in organic links\n if (existingLinksSet && existingLinksSet.has(source.link)) {\n continue;\n }\n\n links.push(source.link);\n if (existingLinksSet) {\n existingLinksSet.add(source.link);\n }\n sourceMap.set(source.link, source as t.ValidSource);\n }\n }\n\n return links;\n}\n\n/** Helper function to update sources with scraped content */\nfunction updateSourcesWithContent<T extends t.ValidSource>(\n sources: T[],\n sourceMap: Map<string, t.ValidSource>\n): void {\n for (let i = 0; i < sources.length; i++) {\n const source = sources[i];\n const updatedSource = sourceMap.get(source.link);\n if (updatedSource) {\n sources[i] = {\n ...source,\n ...updatedSource,\n } as T;\n }\n }\n}\n"],"names":["RecursiveCharacterTextSplitter","createDefaultLogger","createTavilyAPI","getAttribution"],"mappings":";;;;;;;AAOA,MAAM,OAAO,GAAG;AACd,IAAA,SAAS,EAAE,CAAC,IAAY,KAAY;AAClC,QAAA,IAAI,CAAC,IAAI;AAAE,YAAA,OAAO,EAAE;;AAGpB,QAAA,MAAM,cAAc,GAAG,IAAI,CAAC,OAAO,CAAC,OAAO,EAAE,IAAI,CAAC,CAAC,OAAO,CAAC,KAAK,EAAE,IAAI,CAAC;AAEvE;AACiE;QACjE,MAAM,gBAAgB,GAAG,cAAc,CAAC,OAAO,CAAC,QAAQ,EAAE,IAAI,CAAC;;QAG/D,MAAM,eAAe,GAAG,gBAAgB,CAAC,OAAO,CAAC,mBAAmB,EAAE,IAAI,CAAC;;QAG3E,MAAM,aAAa,GAAG,eAAe,CAAC,OAAO,CAAC,SAAS,EAAE,GAAG,CAAC;AAE7D,QAAA,OAAO,aAAa,CAAC,IAAI,EAAE;IAC7B,CAAC;AACD,IAAA,SAAS,EAAE,OACT,IAAY,EACZ,OAIC,KACoB;AACrB,QAAA,MAAM,SAAS,GAAG,OAAO,EAAE,SAAS,IAAI,GAAG;AAC3C,QAAA,MAAM,YAAY,GAAG,OAAO,EAAE,YAAY,IAAI,EAAE;QAChD,MAAM,UAAU,GAAG,OAAO,EAAE,UAAU,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC;AAExD,QAAA,MAAM,QAAQ,GAAG,IAAIA,4CAA8B,CAAC;YAClD,UAAU;YACV,SAAS;YACT,YAAY;AACb,SAAA,CAAC;AAEF,QAAA,OAAO,MAAM,QAAQ,CAAC,SAAS,CAAC,IAAI,CAAC;IACvC,CAAC;IAED,UAAU,EAAE,OACV,KAAe,EACf,OAIC,EACD,MAAiB,KACM;;AAEvB,QAAA,MAAM,OAAO,GAAG,MAAM,IAAIC,yBAAmB,EAAE;QAC/C,MAAM,QAAQ,GAAG,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,KAC9B,OAAO,CAAC,SAAS,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC,KAAK,CAAC,CAAC,KAAK,KAAI;AAC/C,YAAA,OAAO,CAAC,KAAK,CAAC,uBAAuB,EAAE,KAAK,CAAC;YAC7C,OAAO,CAAC,IAAI,CAAC;QACf,CAAC,CAAC,CACH;AACD,QAAA,OAAO,OAAO,CAAC,GAAG,CAAC,QAAQ,CAAC;IAC9B,CAAC;CACF;AAED,SAAS,0BAA0B,CAAC,SAAqC,EAAA;AACvE,IAAA,OAAO,CAAC,IAAY,EAAE,MAA+B,KAAU;QAC7D,MAAM,MAAM,GAAG,SAAS,CAAC,GAAG,CAAC,IAAI,CAAC;QAClC,IAAI,MAAM,EAAE;AACV,YAAA,SAAS,CAAC,GAAG,CAAC,IAAI,EAAE;AAClB,gBAAA,GAAG,MAAM;AACT,gBAAA,GAAG,MAAM;AACV,aAAA,CAAC;QACJ;AACF,IAAA,CAAC;AACH;AAEA,MAAM,aAAa,GAAG,OAAO,EAC3B,KAAK,EACL,OAAO,EACP,QAAQ,EACR,UAAU,GAAG,CAAC,EACd,MAAM,GAOP,KAAwC;AACvC,IAAA,MAAM,OAAO,GAAG,MAAM,IAAIA,yBAAmB,EAAE;IAE/C,IAAI,CAAC,OAAO,EAAE;AACZ,QAAA,OAAO,CAAC,IAAI,CAAC,oCAAoC,CAAC;QAClD;IACF;IACA,IAAI,CAAC,QAAQ,EAAE;AACb,QAAA,OAAO,CAAC,IAAI,CAAC,qCAAqC,CAAC;QACnD;IACF;AAEA,IAAA,IAAI;QACF,MAAM,SAAS,GAAG,MAAM,OAAO,CAAC,SAAS,CAAC,OAAO,CAAC;AAClD,QAAA,IAAI,KAAK,CAAC,OAAO,CAAC,SAAS,CAAC,EAAE;YAC5B,OAAO,MAAM,QAAQ,CAAC,MAAM,CAAC,KAAK,EAAE,SAAS,EAAE,UAAU,CAAC;QAC5D;aAAO;YACL,OAAO,CAAC,KAAK,CACX,yCAAyC,EACzC,OAAO,SAAS,CACjB;YACD;QACF;IACF;IAAE,OAAO,KAAK,EAAE;AACd,QAAA,OAAO,CAAC,KAAK,CAAC,8BAA8B,EAAE,KAAK,CAAC;QACpD;IACF;AACF,CAAC;AAED,MAAM,eAAe,GAAG,CACtB,MAAe,EACf,eAA0B,EAC1B,gBAA2B,KAGzB;AACF,IAAA,MAAM,MAAM,GAAG;AACb,QAAA,MAAM,EAAE,MAAM,IAAI,OAAO,CAAC,GAAG,CAAC,cAAc;AAC5C,QAAA,MAAM,EAAE,kCAAkC;AAC1C,QAAA,OAAO,EAAE,KAAK;KACf;AAED,IAAA,IAAI,MAAM,CAAC,MAAM,IAAI,IAAI,IAAI,MAAM,CAAC,MAAM,KAAK,EAAE,EAAE;AACjD,QAAA,MAAM,IAAI,KAAK,CAAC,0CAA0C,CAAC;IAC7D;IAEA,MAAM,UAAU,GAAG,OAAO,EACxB,KAAK,EACL,IAAI,EACJ,OAAO,EACP,UAAU,EACV,UAAU,GAAG,CAAC,EACd,IAAI,GACe,KAA6B;AAChD,QAAA,IAAI,CAAC,KAAK,CAAC,IAAI,EAAE,EAAE;YACjB,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,uBAAuB,EAAE;QAC3D;AAEA,QAAA,IAAI;YACF,MAAM,IAAI,GAAG,CAAC,KAAK,EAAE,UAAU,EAAE,QAAQ,CAAU;;YAGnD,IAAI,cAAc,GAAG,KAAK;YAC1B,IAAI,eAAe,IAAI,eAAe,CAAC,MAAM,GAAG,CAAC,EAAE;gBACjD,MAAM,cAAc,GAAG;qBACpB,GAAG,CAAC,CAAC,CAAC,KAAK,CAAA,MAAA,EAAS,CAAC,CAAA,CAAE;qBACvB,IAAI,CAAC,GAAG,CAAC;AACZ,gBAAA,cAAc,GAAG,CAAA,EAAG,cAAc,CAAA,CAAA,EAAI,cAAc,EAAE;YACxD;;YAGA,IAAI,gBAAgB,IAAI,gBAAgB,CAAC,MAAM,GAAG,CAAC,EAAE;gBACnD,MAAM,iBAAiB,GAAG;AACvB,qBAAA,GAAG,CAAC,CAAC,CAAC,KAAK,UAAU,CAAC,CAAC,WAAW,EAAE,CAAC,OAAO,CAAC,KAAK,EAAE,EAAE,CAAC,EAAE;qBACzD,IAAI,CAAC,GAAG,CAAC;AACZ,gBAAA,cAAc,GAAG,CAAA,EAAG,cAAc,CAAA,CAAA,EAAI,iBAAiB,EAAE;YAC3D;AAEA,YAAA,MAAM,OAAO,GAA0B;AACrC,gBAAA,CAAC,EAAE,cAAc;AACjB,gBAAA,IAAI,EAAE,IAAI,CAAC,UAAU,IAAI,CAAC,CAAC;AAC3B,gBAAA,GAAG,EAAE,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,UAAU,CAAC,EAAE,EAAE,CAAC;aAC3C;;YAGD,IAAI,IAAI,EAAE;AACR,gBAAA,OAAO,CAAC,IAAI,GAAG,IAAI;YACrB;AAEA,YAAA,IAAI,IAAI,IAAI,IAAI,EAAE;AAChB,gBAAA,OAAO,CAAC,GAAG,GAAG,CAAA,IAAA,EAAO,IAAI,EAAE;YAC7B;YAEA,IAAI,OAAO,IAAI,IAAI,IAAI,OAAO,KAAK,EAAE,EAAE;gBACrC,OAAO,CAAC,IAAI,CAAC,GAAG,OAAO,CAAC,WAAW,EAAE;YACvC;;AAGA,YAAA,IAAI,WAAW,GAAG,MAAM,CAAC,MAAM;AAC/B,YAAA,IAAI,IAAI,KAAK,QAAQ,EAAE;gBACrB,WAAW,GAAG,kCAAkC;YAClD;AAAO,iBAAA,IAAI,IAAI,KAAK,QAAQ,EAAE;gBAC5B,WAAW,GAAG,kCAAkC;YAClD;AAAO,iBAAA,IAAI,IAAI,KAAK,MAAM,EAAE;gBAC1B,WAAW,GAAG,gCAAgC;YAChD;YAEA,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,IAAI,CAC/B,WAAW,EACX,OAAO,EACP;AACE,gBAAA,OAAO,EAAE;oBACP,WAAW,EAAE,MAAM,CAAC,MAAM;AAC1B,oBAAA,cAAc,EAAE,kBAAkB;AACnC,iBAAA;gBACD,OAAO,EAAE,MAAM,CAAC,OAAO;AACxB,aAAA,CACF;AAED,YAAA,MAAM,IAAI,GAAG,QAAQ,CAAC,IAAI;AAC1B,YAAA,MAAM,OAAO,GAAuB;gBAClC,OAAO,EAAE,IAAI,CAAC,OAAO;AACrB,gBAAA,MAAM,EAAE,IAAI,CAAC,MAAM,IAAI,EAAE;gBACzB,SAAS,EAAE,IAAI,CAAC,SAAS;AACzB,gBAAA,UAAU,EAAE,IAAI,CAAC,UAAU,IAAI,EAAE;gBACjC,aAAa,EAAE,IAAI,CAAC,aAAa;gBACjC,cAAc,EAAE,IAAI,CAAC,cAAc;gBACnC,eAAe,EAAE,IAAI,CAAC,eAAe;AACrC,gBAAA,MAAM,EAAE,IAAI,CAAC,MAAM,IAAI,EAAE;AACzB,gBAAA,IAAI,EAAE,IAAI,CAAC,IAAI,IAAI,EAAE;aACtB;YAED,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,IAAI,EAAE,OAAO,EAAE;QACzC;QAAE,OAAO,KAAK,EAAE;AACd,YAAA,MAAM,YAAY,GAChB,KAAK,YAAY,KAAK,GAAG,KAAK,CAAC,OAAO,GAAG,MAAM,CAAC,KAAK,CAAC;YACxD,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,CAAA,oBAAA,EAAuB,YAAY,CAAA,CAAE,EAAE;QACzE;AACF,IAAA,CAAC;IAED,OAAO,EAAE,UAAU,EAAE;AACvB,CAAC;AAED,MAAM,gBAAgB,GAAG,CACvB,WAAoB,EACpB,MAAe,KAGb;AACF,IAAA,MAAM,MAAM,GAAG;AACb,QAAA,WAAW,EAAE,WAAW,IAAI,OAAO,CAAC,GAAG,CAAC,oBAAoB;AAC5D,QAAA,MAAM,EAAE,MAAM,IAAI,OAAO,CAAC,GAAG,CAAC,eAAe;AAC7C,QAAA,eAAe,EAAE,KAAK;AACtB,QAAA,OAAO,EAAE,KAAK;KACf;AAED,IAAA,IAAI,MAAM,CAAC,WAAW,IAAI,IAAI,IAAI,MAAM,CAAC,WAAW,KAAK,EAAE,EAAE;AAC3D,QAAA,MAAM,IAAI,KAAK,CAAC,kDAAkD,CAAC;IACrE;AAEA,IAAA,MAAM,UAAU,GAAG,OAAO,EACxB,KAAK,EACL,UAAU,GAAG,CAAC,EACd,UAAU,EACV,IAAI,GACe,KAA6B;AAChD,QAAA,IAAI,CAAC,KAAK,CAAC,IAAI,EAAE,EAAE;YACjB,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,uBAAuB,EAAE;QAC3D;AAEA,QAAA,IAAI;;AAEF,YAAA,IAAI,MAAM,CAAC,WAAW,IAAI,IAAI,IAAI,MAAM,CAAC,WAAW,KAAK,EAAE,EAAE;gBAC3D,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,6BAA6B,EAAE;YACjE;AAEA,YAAA,IAAI,SAAS,GAAG,MAAM,CAAC,WAAW;YAClC,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC,SAAS,CAAC,EAAE;gBAClC,SAAS,GAAG,SAAS,CAAC,OAAO,CAAC,KAAK,EAAE,EAAE,CAAC,GAAG,SAAS;YACtD;;YAGA,IAAI,QAAQ,GAAG,SAAS;AACxB,YAAA,IAAI,IAAI,KAAK,QAAQ,EAAE;gBACrB,QAAQ,GAAG,QAAQ;YACrB;AAAO,iBAAA,IAAI,IAAI,KAAK,QAAQ,EAAE;gBAC5B,QAAQ,GAAG,QAAQ;YACrB;AAAO,iBAAA,IAAI,IAAI,KAAK,MAAM,EAAE;gBAC1B,QAAQ,GAAG,MAAM;YACnB;;AAGA,YAAA,MAAM,MAAM,GAA2B;AACrC,gBAAA,CAAC,EAAE,KAAK;AACR,gBAAA,MAAM,EAAE,MAAM;AACd,gBAAA,MAAM,EAAE,CAAC;AACT,gBAAA,UAAU,EAAE,QAAQ;AACpB,gBAAA,QAAQ,EAAE,KAAK;AACf,gBAAA,UAAU,EAAE,UAAU;AACtB,gBAAA,OAAO,EAAE,wBAAwB;aAClC;AAED,YAAA,MAAM,OAAO,GAA2B;AACtC,gBAAA,cAAc,EAAE,kBAAkB;aACnC;AAED,YAAA,IAAI,MAAM,CAAC,MAAM,IAAI,IAAI,IAAI,MAAM,CAAC,MAAM,KAAK,EAAE,EAAE;AACjD,gBAAA,OAAO,CAAC,WAAW,CAAC,GAAG,MAAM,CAAC,MAAM;YACtC;YAEA,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,GAAG,CAAC,SAAS,EAAE;gBAC1C,OAAO;gBACP,MAAM;gBACN,OAAO,EAAE,MAAM,CAAC,OAAO;AACxB,aAAA,CAAC;AAEF,YAAA,MAAM,IAAI,GAAG,QAAQ,CAAC,IAAI;;AAG1B,YAAA,MAAM,YAAY,GAAG,CAAC,MAAuB,KAAa;gBACxD,MAAM,GAAG,GAAG,MAAM,CAAC,GAAG,EAAE,WAAW,EAAE,IAAI,EAAE;gBAC3C,MAAM,KAAK,GAAG,MAAM,CAAC,KAAK,EAAE,WAAW,EAAE,IAAI,EAAE;;AAG/C,gBAAA,MAAM,YAAY,GAAG;oBACnB,eAAe;oBACf,aAAa;oBACb,aAAa;oBACb,YAAY;oBACZ,kBAAkB;oBAClB,eAAe;oBACf,MAAM;iBACP;;gBAGD,MAAM,eAAe,GAAG,YAAY,CAAC,IAAI,CACvC,CAAC,OAAO,KAAK,KAAK,CAAC,WAAW,EAAE,CAAC,QAAQ,CAAC,OAAO,CAAC;iBACnD;;AAGD,gBAAA,MAAM,WAAW,GACf,GAAG,CAAC,QAAQ,CAAC,QAAQ,CAAC;AACtB,oBAAA,GAAG,CAAC,QAAQ,CAAC,SAAS,CAAC;AACvB,oBAAA,GAAG,CAAC,QAAQ,CAAC,YAAY,CAAC;AAC1B,oBAAA,GAAG,CAAC,QAAQ,CAAC,YAAY,CAAC;gBAE5B,OAAO,eAAe,IAAI,WAAW;AACvC,YAAA,CAAC;;YAGD,MAAM,cAAc,GAAG,CAAC,IAAI,CAAC,OAAO,IAAI,EAAE;AACvC,iBAAA,KAAK,CAAC,CAAC,EAAE,UAAU;AACnB,iBAAA,GAAG,CAAC,CAAC,MAAuB,EAAE,KAAa,KAAI;gBAC9C,IAAI,WAAW,GAAG,EAAE;AACpB,gBAAA,IAAI;AACF,oBAAA,WAAW,GAAG,IAAI,GAAG,CAAC,MAAM,CAAC,GAAG,IAAI,EAAE,CAAC,CAAC,QAAQ;gBAClD;AAAE,gBAAA,MAAM;oBACN,WAAW,GAAG,EAAE;gBAClB;gBAEA,OAAO;oBACL,QAAQ,EAAE,KAAK,GAAG,CAAC;AACnB,oBAAA,KAAK,EAAE,MAAM,CAAC,KAAK,IAAI,EAAE;AACzB,oBAAA,IAAI,EAAE,MAAM,CAAC,GAAG,IAAI,EAAE;AACtB,oBAAA,OAAO,EAAE,MAAM,CAAC,OAAO,IAAI,EAAE;AAC7B,oBAAA,IAAI,EAAE,MAAM,CAAC,aAAa,IAAI,EAAE;oBAChC,WAAW;iBACZ;AACH,YAAA,CAAC,CAAC;YAEJ,MAAM,YAAY,GAAG,CAAC,IAAI,CAAC,OAAO,IAAI,EAAE;iBACrC,MAAM,CAAC,CAAC,MAAuB,KAAK,MAAM,CAAC,OAAO;AAClD,iBAAA,KAAK,CAAC,CAAC,EAAE,CAAC;iBACV,GAAG,CAAC,CAAC,MAAuB,EAAE,KAAa,MAAM;AAChD,gBAAA,KAAK,EAAE,MAAM,CAAC,KAAK,IAAI,EAAE;AACzB,gBAAA,QAAQ,EAAE,MAAM,CAAC,OAAO,IAAI,EAAE;gBAC9B,QAAQ,EAAE,KAAK,GAAG,CAAC;gBACnB,MAAM,EAAE,IAAI,GAAG,CAAC,MAAM,CAAC,GAAG,IAAI,EAAE,CAAC,CAAC,QAAQ;gBAC1C,MAAM,EAAE,IAAI,GAAG,CAAC,MAAM,CAAC,GAAG,IAAI,EAAE,CAAC,CAAC,QAAQ;AAC1C,gBAAA,IAAI,EAAE,MAAM,CAAC,GAAG,IAAI,EAAE;AACvB,aAAA,CAAC,CAAC;;YAGL,MAAM,WAAW,GAAG,CAAC,IAAI,CAAC,OAAO,IAAI,EAAE;iBACpC,MAAM,CAAC,YAAY;AACnB,iBAAA,GAAG,CAAC,CAAC,MAAuB,EAAE,KAAa,KAAI;gBAC9C,IAAI,WAAW,GAAG,EAAE;AACpB,gBAAA,IAAI;AACF,oBAAA,WAAW,GAAG,IAAI,GAAG,CAAC,MAAM,CAAC,GAAG,IAAI,EAAE,CAAC,CAAC,QAAQ;gBAClD;AAAE,gBAAA,MAAM;oBACN,WAAW,GAAG,EAAE;gBAClB;gBAEA,OAAO;AACL,oBAAA,KAAK,EAAE,MAAM,CAAC,KAAK,IAAI,EAAE;AACzB,oBAAA,IAAI,EAAE,MAAM,CAAC,GAAG,IAAI,EAAE;AACtB,oBAAA,OAAO,EAAE,MAAM,CAAC,OAAO,IAAI,EAAE;AAC7B,oBAAA,IAAI,EAAE,MAAM,CAAC,aAAa,IAAI,EAAE;AAChC,oBAAA,MAAM,EAAE,WAAW;AACnB,oBAAA,QAAQ,EAAE,MAAM,CAAC,OAAO,IAAI,EAAE;oBAC9B,QAAQ,EAAE,KAAK,GAAG,CAAC;iBACpB;AACH,YAAA,CAAC,CAAC;YAEJ,MAAM,UAAU,GAAG,WAAW,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC;YAE1C,MAAM,eAAe,GAAG,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,WAAW;AACpD,kBAAE,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC,UAAkB,MAAM,EAAE,KAAK,EAAE,UAAU,EAAE,CAAC;kBACpE,EAAE;AAEN,YAAA,MAAM,OAAO,GAAuB;AAClC,gBAAA,OAAO,EAAE,cAAc;AACvB,gBAAA,MAAM,EAAE,YAAY;gBACpB,UAAU,EAAE,UAAU;gBACtB,eAAe;AACf,gBAAA,MAAM,EAAE,EAAE;AACV,gBAAA,IAAI,EAAE,WAAW;;AAEjB,gBAAA,MAAM,EAAE,EAAE;AACV,gBAAA,QAAQ,EAAE,EAAE;AACZ,gBAAA,aAAa,EAAE,EAAE;AACjB,gBAAA,cAAc,EAAE,SAAS;AACzB,gBAAA,SAAS,EAAE,SAAS;aACrB;YAED,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,IAAI,EAAE,OAAO,EAAE;QACzC;QAAE,OAAO,KAAK,EAAE;AACd,YAAA,MAAM,YAAY,GAChB,KAAK,YAAY,KAAK,GAAG,KAAK,CAAC,OAAO,GAAG,MAAM,CAAC,KAAK,CAAC;YACxD,OAAO;AACL,gBAAA,OAAO,EAAE,KAAK;gBACd,KAAK,EAAE,CAAA,4BAAA,EAA+B,YAAY,CAAA,CAAE;aACrD;QACH;AACF,IAAA,CAAC;IAED,OAAO,EAAE,UAAU,EAAE;AACvB,CAAC;AAEM,MAAM,eAAe,GAAG,CAC7B,MAAsB,KAGpB;IACF,MAAM,EACJ,cAAc,GAAG,QAAQ,EACzB,YAAY,EACZ,kBAAkB,EAClB,aAAa,EACb,YAAY,EACZ,eAAe,EACf,mBAAmB,EACnB,eAAe,EACf,gBAAgB,GACjB,GAAG,MAAM;AAEV,IAAA,IAAI,cAAc,CAAC,WAAW,EAAE,KAAK,QAAQ,EAAE;QAC7C,OAAO,eAAe,CAAC,YAAY,EAAE,eAAe,EAAE,gBAAgB,CAAC;IACzE;AAAO,SAAA,IAAI,cAAc,CAAC,WAAW,EAAE,KAAK,SAAS,EAAE;AACrD,QAAA,OAAO,gBAAgB,CAAC,kBAAkB,EAAE,aAAa,CAAC;IAC5D;AAAO,SAAA,IAAI,cAAc,CAAC,WAAW,EAAE,KAAK,QAAQ,EAAE;QACpD,OAAOC,4BAAe,CAAC,YAAY,EAAE,eAAe,EAAE,mBAAmB,CAAC;IAC5E;SAAO;AACL,QAAA,MAAM,IAAI,KAAK,CACb,4BAA4B,cAAc,CAAA,0CAAA,CAA4C,CACvF;IACH;AACF;AAEO,MAAM,qBAAqB,GAAG,CACnC,SAAiC,EAAE,EACnC,eAA+B,KAM7B;IACF,IAAI,CAAC,eAAe,EAAE;AACpB,QAAA,MAAM,IAAI,KAAK,CAAC,8BAA8B,CAAC;IACjD;IACA,MAAM,EACJ,UAAU,GAAG,CAAC;;;AAGd,IAAA,QAAQ,EACR,MAAM,GACP,GAAG,MAAM;AAEV,IAAA,MAAM,OAAO,GAAG,MAAM,IAAID,yBAAmB,EAAE;IAC/C,MAAM,OAAO,GAAG,eAAe;AAE/B,IAAA,MAAM,UAAU,GAAG;QACjB,UAAU,EAAE,OAAO,EACjB,KAAK,EACL,KAAK,EACL,eAAe,GAKhB,KAAoC;YACnC,OAAO,CAAC,KAAK,CAAC,CAAA,SAAA,EAAY,KAAK,CAAC,MAAM,CAAA,MAAA,CAAQ,CAAC;YAC/C,MAAM,QAAQ,GAAmC,EAAE;AACnD,YAAA,IAAI;AACF,gBAAA,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE;AACrC,oBAAA,MAAM,WAAW,GAAG,KAAK,CAAC,CAAC,CAAC;oBAC5B,MAAM,OAAO,GAA4B;AACtC,yBAAA,SAAS,CAAC,WAAW,EAAE,EAAE;yBACzB,IAAI,CAAC,CAAC,CAAC,GAAG,EAAE,QAAQ,CAAC,KAAI;AACxB;;AAEuD;wBACvD,MAAM,OAAO,GAAG,OAAO,CAAC,eAAe,CAAC,QAAQ,CAAC;wBACjD,MAAM,WAAW,GAAGE,oBAAc,CAChC,GAAG,EACH,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,MAAM,GAAG,CAAC,GAAG,OAAO,GAAG,SAAS,EACrD,OAAO,CACR;wBACD,IAAI,QAAQ,CAAC,OAAO,IAAI,QAAQ,CAAC,IAAI,EAAE;AACrC,4BAAA,MAAM,CAAC,OAAO,EAAE,UAAU,CAAC,GAAG,OAAO,CAAC,cAAc,CAAC,QAAQ,CAAC;4BAC9D,OAAO;gCACL,GAAG;gCACH,UAAU;gCACV,WAAW;AACX,gCAAA,OAAO,EAAE,OAAO,CAAC,SAAS,CAAC,OAAO,CAAC;6BAClB;wBACrB;6BAAO;AACL,4BAAA,OAAO,CAAC,KAAK,CACX,CAAA,eAAA,EAAkB,GAAG,CAAA,EAAA,EAAK,QAAQ,CAAC,KAAK,IAAI,eAAe,CAAA,CAAE,CAC9D;wBACH;wBAEA,OAAO;4BACL,GAAG;4BACH,WAAW;AACX,4BAAA,KAAK,EAAE,IAAI;AACX,4BAAA,OAAO,EAAE,EAAE;yBACM;AACrB,oBAAA,CAAC;AACA,yBAAA,IAAI,CAAC,OAAO,MAAM,KAAI;AACrB,wBAAA,IAAI;AACF,4BAAA,IAAI,MAAM,CAAC,KAAK,IAAI,IAAI,EAAE;AACxB,gCAAA,OAAO,CAAC,KAAK,CACX,CAAA,eAAA,EAAkB,MAAM,CAAC,GAAG,CAAA,EAAA,EAAK,MAAM,CAAC,OAAO,CAAA,CAAE,CAClD;gCACD,OAAO;AACL,oCAAA,GAAG,MAAM;iCACV;4BACH;AACA,4BAAA,MAAM,UAAU,GAAG,MAAM,aAAa,CAAC;gCACrC,KAAK;gCACL,QAAQ;gCACR,OAAO,EAAE,MAAM,CAAC,OAAO;AACvB,gCAAA,MAAM,EAAE,OAAO;AAChB,6BAAA,CAAC;4BACF,IAAI,eAAe,EAAE;AACnB,gCAAA,eAAe,CAAC,MAAM,CAAC,GAAG,CAAC;4BAC7B;4BACA,OAAO;AACL,gCAAA,GAAG,MAAM;gCACT,UAAU;6BACX;wBACH;wBAAE,OAAO,KAAK,EAAE;AACd,4BAAA,OAAO,CAAC,KAAK,CAAC,mCAAmC,EAAE,KAAK,CAAC;4BACzD,OAAO;AACL,gCAAA,GAAG,MAAM;6BACV;wBACH;AACF,oBAAA,CAAC;AACA,yBAAA,KAAK,CAAC,CAAC,KAAK,KAAI;wBACf,OAAO,CAAC,KAAK,CAAC,CAAA,eAAA,EAAkB,WAAW,CAAA,CAAA,CAAG,EAAE,KAAK,CAAC;wBACtD,OAAO;AACL,4BAAA,GAAG,EAAE,WAAW;AAChB,4BAAA,KAAK,EAAE,IAAI;AACX,4BAAA,OAAO,EAAE,EAAE;yBACZ;AACH,oBAAA,CAAC,CAAC;AACJ,oBAAA,QAAQ,CAAC,IAAI,CAAC,OAAO,CAAC;gBACxB;AACA,gBAAA,OAAO,MAAM,OAAO,CAAC,GAAG,CAAC,QAAQ,CAAC;YACpC;YAAE,OAAO,KAAK,EAAE;AACd,gBAAA,OAAO,CAAC,KAAK,CAAC,sBAAsB,EAAE,KAAK,CAAC;AAC5C,gBAAA,OAAO,EAAE;YACX;QACF,CAAC;KACF;AAED,IAAA,MAAM,aAAa,GAAG,OAAO,EAC3B,KAAK,EACL,KAAK,EACL,MAAM,EACN,eAAe,EACf,gBAAgB,GAOjB,KAAmB;QAClB,MAAM,YAAY,GAAG,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,MAAM,CAAC;;AAE3C,QAAA,MAAM,OAAO,GAAG,MAAM,UAAU,CAAC,UAAU,CAAC;YAC1C,KAAK;AACL,YAAA,KAAK,EAAE,YAAY;YACnB,eAAe;AAChB,SAAA,CAAC;AACF,QAAA,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE;AAC5B,YAAA,IAAI,MAAM,CAAC,KAAK,KAAK,IAAI,EAAE;gBACzB;YACF;AACA,YAAA,MAAM,EAAE,GAAG,EAAE,OAAO,EAAE,WAAW,EAAE,UAAU,EAAE,UAAU,EAAE,GAAG,MAAM;YACpE,gBAAgB,GAAG,GAAG,EAAE;gBACtB,OAAO;gBACP,WAAW;gBACX,UAAU;gBACV,UAAU;AACX,aAAA,CAAC;QACJ;AACF,IAAA,CAAC;IAED,MAAM,cAAc,GAAG,OAAO,EAC5B,MAAM,EACN,WAAW,EACX,KAAK,EACL,IAAI,EACJ,OAAO,GAAG,IAAI,EACd,eAAe,EACf,YAAY,GAAG,KAAK,GACG,KAAiC;AACxD,QAAA,IAAI;AACF,YAAA,IAAI,CAAC,MAAM,CAAC,IAAI,EAAE;gBAChB,OAAO;AACL,oBAAA,OAAO,EAAE,EAAE;AACX,oBAAA,UAAU,EAAE,EAAE;AACd,oBAAA,MAAM,EAAE,EAAE;AACV,oBAAA,eAAe,EAAE,EAAE;iBACpB;YACH;AAAO,iBAAA,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,OAAO,EAAE;gBAC/B,OAAO,MAAM,CAAC,IAAI;YACpB;;YAGA,IAAI,YAAY,EAAE;AAChB,gBAAA,OAAO,CAAC,KAAK,CACX,0DAA0D,CAC3D;gBACD,OAAO,MAAM,CAAC,IAAI;YACpB;YAEA,IAAI,CAAC,OAAO,EAAE;gBACZ,MAAM,WAAW,GAAG,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,MAAM,KACpD,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC,eAAe,CAAC,CACtC;AAED,gBAAA,IAAI,CAAC,WAAW,CAAC,MAAM,EAAE;oBACvB,OAAO,MAAM,CAAC,IAAI;gBACpB;AAEA,gBAAA,MAAM,aAAa,GAAG,IAAI,GAAG,EAAyB;AACtD,gBAAA,aAAa,CAAC,GAAG,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,WAAW,CAAC,CAAC,CAAC,CAAC;AACtD,gBAAA,MAAM,gBAAgB,GAAG,0BAA0B,CAAC,aAAa,CAAC;AAClE,gBAAA,MAAM,aAAa,CAAC;oBAClB,KAAK;AACL,oBAAA,MAAM,EAAE,CAAC;oBACT,eAAe;oBACf,gBAAgB;oBAChB,KAAK,EAAE,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;AAC7B,iBAAA,CAAC;AAEF,gBAAA,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE;oBACnD,MAAM,MAAM,GAAG,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC;oBACrC,MAAM,aAAa,GAAG,aAAa,CAAC,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC;oBACpD,IAAI,aAAa,EAAE;AACjB,wBAAA,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG;AACvB,4BAAA,GAAG,MAAM;AACT,4BAAA,GAAG,aAAa;yBACjB;oBACH;gBACF;gBAEA,OAAO,MAAM,CAAC,IAAI;YACpB;AAEA,YAAA,MAAM,SAAS,GAAG,IAAI,GAAG,EAAyB;AAClD,YAAA,MAAM,eAAe,GAAG,IAAI,GAAG,EAAU;;AAGzC,YAAA,MAAM,YAAY,GAAG,YAAY,CAC/B,MAAM,CAAC,IAAI,CAAC,OAAO,EACnB,SAAS,EACT,eAAe,CAChB;;YAGD,MAAM,UAAU,GAAG,MAAM,CAAC,IAAI,CAAC,UAAU,IAAI,EAAE;YAC/C,MAAM,aAAa,GAAG,YAAY,CAChC,UAAU,EACV,SAAS,EACT,eAAe,CAChB;AAED,YAAA,IAAI,YAAY,CAAC,MAAM,KAAK,CAAC,KAAK,aAAa,CAAC,MAAM,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE;gBACtE,OAAO,MAAM,CAAC,IAAI;YACpB;AAEA,YAAA,MAAM,gBAAgB,GAAG,0BAA0B,CAAC,SAAS,CAAC;YAC9D,MAAM,QAAQ,GAAoB,EAAE;;AAGpC,YAAA,IAAI,YAAY,CAAC,MAAM,GAAG,CAAC,EAAE;AAC3B,gBAAA,QAAQ,CAAC,IAAI,CACX,aAAa,CAAC;oBACZ,KAAK;oBACL,eAAe;oBACf,gBAAgB;AAChB,oBAAA,KAAK,EAAE,YAAY;AACnB,oBAAA,MAAM,EAAE,WAAW;AACpB,iBAAA,CAAC,CACH;YACH;;YAGA,IAAI,IAAI,IAAI,aAAa,CAAC,MAAM,GAAG,CAAC,EAAE;AACpC,gBAAA,QAAQ,CAAC,IAAI,CACX,aAAa,CAAC;oBACZ,KAAK;oBACL,eAAe;oBACf,gBAAgB;AAChB,oBAAA,KAAK,EAAE,aAAa;AACpB,oBAAA,MAAM,EAAE,WAAW;AACpB,iBAAA,CAAC,CACH;YACH;AAEA,YAAA,MAAM,OAAO,CAAC,GAAG,CAAC,QAAQ,CAAC;YAE3B,IAAI,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE;gBAClC,wBAAwB,CAAC,MAAM,CAAC,IAAI,CAAC,OAAO,EAAE,SAAS,CAAC;YAC1D;YAEA,IAAI,IAAI,IAAI,UAAU,CAAC,MAAM,GAAG,CAAC,EAAE;AACjC,gBAAA,wBAAwB,CAAC,UAAU,EAAE,SAAS,CAAC;YACjD;YAEA,OAAO,MAAM,CAAC,IAAI;QACpB;QAAE,OAAO,KAAK,EAAE;AACd,YAAA,OAAO,CAAC,KAAK,CAAC,0BAA0B,EAAE,KAAK,CAAC;YAChD,OAAO;AACL,gBAAA,OAAO,EAAE,EAAE;AACX,gBAAA,UAAU,EAAE,EAAE;AACd,gBAAA,MAAM,EAAE,EAAE;AACV,gBAAA,eAAe,EAAE,EAAE;gBACnB,GAAG,MAAM,CAAC,IAAI;AACd,gBAAA,KAAK,EAAE,KAAK,YAAY,KAAK,GAAG,KAAK,CAAC,OAAO,GAAG,MAAM,CAAC,KAAK,CAAC;aAC9D;QACH;AACF,IAAA,CAAC;IAED,OAAO;QACL,cAAc;QACd,UAAU;KACX;AACH;AAEA;AACA,SAAS,YAAY,CACnB,OAAkD,EAClD,SAAqC,EACrC,gBAA8B,EAAA;IAE9B,MAAM,KAAK,GAAa,EAAE;AAE1B,IAAA,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE;AAC5B,QAAA,IAAI,MAAM,CAAC,IAAI,EAAE;;YAEf,IAAI,gBAAgB,IAAI,gBAAgB,CAAC,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE;gBACzD;YACF;AAEA,YAAA,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC;YACvB,IAAI,gBAAgB,EAAE;AACpB,gBAAA,gBAAgB,CAAC,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC;YACnC;YACA,SAAS,CAAC,GAAG,CAAC,MAAM,CAAC,IAAI,EAAE,MAAuB,CAAC;QACrD;IACF;AAEA,IAAA,OAAO,KAAK;AACd;AAEA;AACA,SAAS,wBAAwB,CAC/B,OAAY,EACZ,SAAqC,EAAA;AAErC,IAAA,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE;AACvC,QAAA,MAAM,MAAM,GAAG,OAAO,CAAC,CAAC,CAAC;QACzB,MAAM,aAAa,GAAG,SAAS,CAAC,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC;QAChD,IAAI,aAAa,EAAE;YACjB,OAAO,CAAC,CAAC,CAAC,GAAG;AACX,gBAAA,GAAG,MAAM;AACT,gBAAA,GAAG,aAAa;aACZ;QACR;IACF;AACF;;;;;"}
@@ -0,0 +1,189 @@
1
+ 'use strict';
2
+
3
+ var axios = require('axios');
4
+ var utils = require('./utils.cjs');
5
+
6
+ const DEFAULT_BASIC_TIMEOUT = 15000;
7
+ const DEFAULT_ADVANCED_TIMEOUT = 30000;
8
+ const MAX_BATCH_SIZE = 20;
9
+ const getDefaultTimeout = (extractDepth) => extractDepth === 'advanced'
10
+ ? DEFAULT_ADVANCED_TIMEOUT
11
+ : DEFAULT_BASIC_TIMEOUT;
12
+ const normalizeUrlKey = (url) => {
13
+ try {
14
+ const parsedUrl = new URL(url);
15
+ parsedUrl.hash = '';
16
+ if (parsedUrl.pathname.length > 1) {
17
+ parsedUrl.pathname = parsedUrl.pathname.replace(/\/+$/, '');
18
+ }
19
+ return parsedUrl.toString();
20
+ }
21
+ catch {
22
+ return url;
23
+ }
24
+ };
25
+ const setUrlResult = (map, result) => {
26
+ map.set(result.url, result);
27
+ const normalizedUrl = normalizeUrlKey(result.url);
28
+ if (!map.has(normalizedUrl)) {
29
+ map.set(normalizedUrl, result);
30
+ }
31
+ };
32
+ class TavilyScraper {
33
+ apiKey;
34
+ apiUrl;
35
+ timeout;
36
+ payloadTimeout;
37
+ logger;
38
+ extractDepth;
39
+ includeImages;
40
+ includeFavicon;
41
+ format;
42
+ constructor(config = {}) {
43
+ this.apiKey = config.apiKey ?? process.env.TAVILY_API_KEY ?? '';
44
+ this.apiUrl =
45
+ config.apiUrl ??
46
+ process.env.TAVILY_EXTRACT_URL ??
47
+ 'https://api.tavily.com/extract';
48
+ this.payloadTimeout = config.timeout;
49
+ this.extractDepth = config.extractDepth ?? 'basic';
50
+ this.timeout = config.timeout ?? getDefaultTimeout(this.extractDepth);
51
+ this.includeImages = config.includeImages ?? false;
52
+ this.includeFavicon = config.includeFavicon ?? false;
53
+ this.format = config.format;
54
+ this.logger = config.logger || utils.createDefaultLogger();
55
+ if (!this.apiKey) {
56
+ this.logger.warn('TAVILY_API_KEY is not set. Scraping will not work.');
57
+ }
58
+ }
59
+ async scrapeUrl(url, options = {}) {
60
+ const results = await this.scrapeUrls([url], options);
61
+ return results[0];
62
+ }
63
+ async scrapeUrls(urls, options = {}) {
64
+ if (!this.apiKey) {
65
+ return urls.map((url) => [
66
+ url,
67
+ { success: false, error: 'TAVILY_API_KEY is not set' },
68
+ ]);
69
+ }
70
+ const batches = [];
71
+ for (let i = 0; i < urls.length; i += MAX_BATCH_SIZE) {
72
+ batches.push(urls.slice(i, i + MAX_BATCH_SIZE));
73
+ }
74
+ const allResults = [];
75
+ for (const batch of batches) {
76
+ const batchResults = await this.extractBatch(batch, options);
77
+ allResults.push(...batchResults);
78
+ }
79
+ return allResults;
80
+ }
81
+ async extractBatch(urls, options = {}) {
82
+ try {
83
+ const includeFavicon = options.includeFavicon ?? this.includeFavicon;
84
+ const format = options.format ?? this.format;
85
+ const extractDepth = options.extractDepth ?? this.extractDepth;
86
+ const payload = {
87
+ urls,
88
+ extract_depth: extractDepth,
89
+ include_images: options.includeImages ?? this.includeImages,
90
+ };
91
+ if (includeFavicon) {
92
+ payload.include_favicon = true;
93
+ }
94
+ if (format != null) {
95
+ payload.format = format;
96
+ }
97
+ const effectiveTimeout = options.timeout ??
98
+ this.payloadTimeout ??
99
+ (options.extractDepth != null
100
+ ? getDefaultTimeout(extractDepth)
101
+ : this.timeout);
102
+ const payloadTimeout = options.timeout ?? this.payloadTimeout;
103
+ if (payloadTimeout != null) {
104
+ payload.timeout = Math.min(Math.max(payloadTimeout / 1000, 1), 60);
105
+ }
106
+ const response = await axios.post(this.apiUrl, payload, {
107
+ headers: {
108
+ Authorization: `Bearer ${this.apiKey}`,
109
+ 'Content-Type': 'application/json',
110
+ },
111
+ timeout: effectiveTimeout,
112
+ });
113
+ const data = response.data;
114
+ const successMap = new Map();
115
+ const failedMap = new Map();
116
+ for (const result of data.results ?? []) {
117
+ setUrlResult(successMap, result);
118
+ }
119
+ for (const result of data.failed_results ?? []) {
120
+ setUrlResult(failedMap, result);
121
+ }
122
+ return urls.map((url) => {
123
+ const success = successMap.get(url) ?? successMap.get(normalizeUrlKey(url));
124
+ if (success && success.error == null) {
125
+ return [
126
+ url,
127
+ {
128
+ success: true,
129
+ data: {
130
+ rawContent: success.raw_content ?? '',
131
+ images: success.images ?? [],
132
+ favicon: success.favicon,
133
+ },
134
+ },
135
+ ];
136
+ }
137
+ const failed = failedMap.get(url) ?? failedMap.get(normalizeUrlKey(url));
138
+ const error = success?.error ??
139
+ failed?.error ??
140
+ 'URL not found in Tavily Extract response';
141
+ return [url, { success: false, error }];
142
+ });
143
+ }
144
+ catch (error) {
145
+ const errorMessage = error instanceof Error ? error.message : String(error);
146
+ return urls.map((url) => [
147
+ url,
148
+ {
149
+ success: false,
150
+ error: `Tavily Extract API request failed: ${errorMessage}`,
151
+ },
152
+ ]);
153
+ }
154
+ }
155
+ extractContent(response) {
156
+ if (!response.success || !response.data) {
157
+ return ['', undefined];
158
+ }
159
+ const content = response.data.rawContent ?? '';
160
+ const images = response.data.images ?? [];
161
+ const references = images.length > 0
162
+ ? {
163
+ links: [],
164
+ images: images.map((imageUrl) => ({ originalUrl: imageUrl })),
165
+ videos: [],
166
+ }
167
+ : undefined;
168
+ return [content, references];
169
+ }
170
+ extractMetadata(response) {
171
+ if (!response.success || !response.data) {
172
+ return {};
173
+ }
174
+ const metadata = {
175
+ images_count: response.data.images?.length ?? 0,
176
+ };
177
+ if (response.data.favicon != null) {
178
+ metadata.favicon = response.data.favicon;
179
+ }
180
+ return metadata;
181
+ }
182
+ }
183
+ const createTavilyScraper = (config = {}) => {
184
+ return new TavilyScraper(config);
185
+ };
186
+
187
+ exports.TavilyScraper = TavilyScraper;
188
+ exports.createTavilyScraper = createTavilyScraper;
189
+ //# sourceMappingURL=tavily-scraper.cjs.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"tavily-scraper.cjs","sources":["../../../../src/tools/search/tavily-scraper.ts"],"sourcesContent":["import axios from 'axios';\nimport type * as t from './types';\nimport { createDefaultLogger } from './utils';\n\nconst DEFAULT_BASIC_TIMEOUT = 15000;\nconst DEFAULT_ADVANCED_TIMEOUT = 30000;\nconst MAX_BATCH_SIZE = 20;\n\nconst getDefaultTimeout = (extractDepth: 'basic' | 'advanced'): number =>\n extractDepth === 'advanced'\n ? DEFAULT_ADVANCED_TIMEOUT\n : DEFAULT_BASIC_TIMEOUT;\n\nconst normalizeUrlKey = (url: string): string => {\n try {\n const parsedUrl = new URL(url);\n parsedUrl.hash = '';\n if (parsedUrl.pathname.length > 1) {\n parsedUrl.pathname = parsedUrl.pathname.replace(/\\/+$/, '');\n }\n return parsedUrl.toString();\n } catch {\n return url;\n }\n};\n\nconst setUrlResult = (\n map: Map<string, t.TavilyExtractResult>,\n result: t.TavilyExtractResult\n): void => {\n map.set(result.url, result);\n const normalizedUrl = normalizeUrlKey(result.url);\n if (!map.has(normalizedUrl)) {\n map.set(normalizedUrl, result);\n }\n};\n\nexport class TavilyScraper implements t.BaseScraper {\n private apiKey: string;\n private apiUrl: string;\n private timeout: number;\n private payloadTimeout: number | undefined;\n private logger: t.Logger;\n private extractDepth: 'basic' | 'advanced';\n private includeImages: boolean;\n private includeFavicon: boolean;\n private format: 'markdown' | 'text' | undefined;\n\n constructor(config: t.TavilyScraperConfig = {}) {\n this.apiKey = config.apiKey ?? process.env.TAVILY_API_KEY ?? '';\n this.apiUrl =\n config.apiUrl ??\n process.env.TAVILY_EXTRACT_URL ??\n 'https://api.tavily.com/extract';\n this.payloadTimeout = config.timeout;\n this.extractDepth = config.extractDepth ?? 'basic';\n this.timeout = config.timeout ?? getDefaultTimeout(this.extractDepth);\n this.includeImages = config.includeImages ?? false;\n this.includeFavicon = config.includeFavicon ?? false;\n this.format = config.format;\n this.logger = config.logger || createDefaultLogger();\n\n if (!this.apiKey) {\n this.logger.warn('TAVILY_API_KEY is not set. Scraping will not work.');\n }\n }\n\n async scrapeUrl(\n url: string,\n options: t.TavilyScrapeOptions = {}\n ): Promise<[string, t.TavilyScrapeResponse]> {\n const results = await this.scrapeUrls([url], options);\n return results[0];\n }\n\n async scrapeUrls(\n urls: string[],\n options: t.TavilyScrapeOptions = {}\n ): Promise<Array<[string, t.TavilyScrapeResponse]>> {\n if (!this.apiKey) {\n return urls.map((url) => [\n url,\n { success: false, error: 'TAVILY_API_KEY is not set' },\n ]);\n }\n\n const batches: string[][] = [];\n for (let i = 0; i < urls.length; i += MAX_BATCH_SIZE) {\n batches.push(urls.slice(i, i + MAX_BATCH_SIZE));\n }\n\n const allResults: Array<[string, t.TavilyScrapeResponse]> = [];\n\n for (const batch of batches) {\n const batchResults = await this.extractBatch(batch, options);\n allResults.push(...batchResults);\n }\n\n return allResults;\n }\n\n private async extractBatch(\n urls: string[],\n options: t.TavilyScrapeOptions = {}\n ): Promise<Array<[string, t.TavilyScrapeResponse]>> {\n try {\n const includeFavicon = options.includeFavicon ?? this.includeFavicon;\n const format = options.format ?? this.format;\n const extractDepth = options.extractDepth ?? this.extractDepth;\n const payload: t.TavilyExtractPayload = {\n urls,\n extract_depth: extractDepth,\n include_images: options.includeImages ?? this.includeImages,\n };\n\n if (includeFavicon) {\n payload.include_favicon = true;\n }\n if (format != null) {\n payload.format = format;\n }\n\n const effectiveTimeout =\n options.timeout ??\n this.payloadTimeout ??\n (options.extractDepth != null\n ? getDefaultTimeout(extractDepth)\n : this.timeout);\n const payloadTimeout = options.timeout ?? this.payloadTimeout;\n if (payloadTimeout != null) {\n payload.timeout = Math.min(Math.max(payloadTimeout / 1000, 1), 60);\n }\n\n const response = await axios.post<{\n results?: t.TavilyExtractResult[];\n failed_results?: t.TavilyExtractResult[];\n }>(this.apiUrl, payload, {\n headers: {\n Authorization: `Bearer ${this.apiKey}`,\n 'Content-Type': 'application/json',\n },\n timeout: effectiveTimeout,\n });\n\n const data = response.data;\n const successMap = new Map<string, t.TavilyExtractResult>();\n const failedMap = new Map<string, t.TavilyExtractResult>();\n\n for (const result of data.results ?? []) {\n setUrlResult(successMap, result);\n }\n for (const result of data.failed_results ?? []) {\n setUrlResult(failedMap, result);\n }\n\n return urls.map((url): [string, t.TavilyScrapeResponse] => {\n const success =\n successMap.get(url) ?? successMap.get(normalizeUrlKey(url));\n if (success && success.error == null) {\n return [\n url,\n {\n success: true,\n data: {\n rawContent: success.raw_content ?? '',\n images: success.images ?? [],\n favicon: success.favicon,\n },\n },\n ];\n }\n\n const failed =\n failedMap.get(url) ?? failedMap.get(normalizeUrlKey(url));\n const error =\n success?.error ??\n failed?.error ??\n 'URL not found in Tavily Extract response';\n return [url, { success: false, error }];\n });\n } catch (error) {\n const errorMessage =\n error instanceof Error ? error.message : String(error);\n return urls.map((url) => [\n url,\n {\n success: false,\n error: `Tavily Extract API request failed: ${errorMessage}`,\n },\n ]);\n }\n }\n\n extractContent(\n response: t.TavilyScrapeResponse\n ): [string, undefined | t.References] {\n if (!response.success || !response.data) {\n return ['', undefined];\n }\n\n const content = response.data.rawContent ?? '';\n const images = response.data.images ?? [];\n\n const references: t.References | undefined =\n images.length > 0\n ? {\n links: [],\n images: images.map((imageUrl) => ({ originalUrl: imageUrl })),\n videos: [],\n }\n : undefined;\n\n return [content, references];\n }\n\n extractMetadata(response: t.TavilyScrapeResponse): t.GenericScrapeMetadata {\n if (!response.success || !response.data) {\n return {};\n }\n\n const metadata: t.GenericScrapeMetadata = {\n images_count: response.data.images?.length ?? 0,\n };\n if (response.data.favicon != null) {\n metadata.favicon = response.data.favicon;\n }\n return metadata;\n }\n}\n\nexport const createTavilyScraper = (\n config: t.TavilyScraperConfig = {}\n): TavilyScraper => {\n return new TavilyScraper(config);\n};\n"],"names":["createDefaultLogger"],"mappings":";;;;;AAIA,MAAM,qBAAqB,GAAG,KAAK;AACnC,MAAM,wBAAwB,GAAG,KAAK;AACtC,MAAM,cAAc,GAAG,EAAE;AAEzB,MAAM,iBAAiB,GAAG,CAAC,YAAkC,KAC3D,YAAY,KAAK;AACf,MAAE;MACA,qBAAqB;AAE3B,MAAM,eAAe,GAAG,CAAC,GAAW,KAAY;AAC9C,IAAA,IAAI;AACF,QAAA,MAAM,SAAS,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC;AAC9B,QAAA,SAAS,CAAC,IAAI,GAAG,EAAE;QACnB,IAAI,SAAS,CAAC,QAAQ,CAAC,MAAM,GAAG,CAAC,EAAE;AACjC,YAAA,SAAS,CAAC,QAAQ,GAAG,SAAS,CAAC,QAAQ,CAAC,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC;QAC7D;AACA,QAAA,OAAO,SAAS,CAAC,QAAQ,EAAE;IAC7B;AAAE,IAAA,MAAM;AACN,QAAA,OAAO,GAAG;IACZ;AACF,CAAC;AAED,MAAM,YAAY,GAAG,CACnB,GAAuC,EACvC,MAA6B,KACrB;IACR,GAAG,CAAC,GAAG,CAAC,MAAM,CAAC,GAAG,EAAE,MAAM,CAAC;IAC3B,MAAM,aAAa,GAAG,eAAe,CAAC,MAAM,CAAC,GAAG,CAAC;IACjD,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,aAAa,CAAC,EAAE;AAC3B,QAAA,GAAG,CAAC,GAAG,CAAC,aAAa,EAAE,MAAM,CAAC;IAChC;AACF,CAAC;MAEY,aAAa,CAAA;AAChB,IAAA,MAAM;AACN,IAAA,MAAM;AACN,IAAA,OAAO;AACP,IAAA,cAAc;AACd,IAAA,MAAM;AACN,IAAA,YAAY;AACZ,IAAA,aAAa;AACb,IAAA,cAAc;AACd,IAAA,MAAM;AAEd,IAAA,WAAA,CAAY,SAAgC,EAAE,EAAA;AAC5C,QAAA,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC,MAAM,IAAI,OAAO,CAAC,GAAG,CAAC,cAAc,IAAI,EAAE;AAC/D,QAAA,IAAI,CAAC,MAAM;AACT,YAAA,MAAM,CAAC,MAAM;gBACb,OAAO,CAAC,GAAG,CAAC,kBAAkB;AAC9B,gBAAA,gCAAgC;AAClC,QAAA,IAAI,CAAC,cAAc,GAAG,MAAM,CAAC,OAAO;QACpC,IAAI,CAAC,YAAY,GAAG,MAAM,CAAC,YAAY,IAAI,OAAO;AAClD,QAAA,IAAI,CAAC,OAAO,GAAG,MAAM,CAAC,OAAO,IAAI,iBAAiB,CAAC,IAAI,CAAC,YAAY,CAAC;QACrE,IAAI,CAAC,aAAa,GAAG,MAAM,CAAC,aAAa,IAAI,KAAK;QAClD,IAAI,CAAC,cAAc,GAAG,MAAM,CAAC,cAAc,IAAI,KAAK;AACpD,QAAA,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC,MAAM;QAC3B,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC,MAAM,IAAIA,yBAAmB,EAAE;AAEpD,QAAA,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE;AAChB,YAAA,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,oDAAoD,CAAC;QACxE;IACF;AAEA,IAAA,MAAM,SAAS,CACb,GAAW,EACX,UAAiC,EAAE,EAAA;AAEnC,QAAA,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,UAAU,CAAC,CAAC,GAAG,CAAC,EAAE,OAAO,CAAC;AACrD,QAAA,OAAO,OAAO,CAAC,CAAC,CAAC;IACnB;AAEA,IAAA,MAAM,UAAU,CACd,IAAc,EACd,UAAiC,EAAE,EAAA;AAEnC,QAAA,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE;YAChB,OAAO,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,KAAK;gBACvB,GAAG;AACH,gBAAA,EAAE,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,2BAA2B,EAAE;AACvD,aAAA,CAAC;QACJ;QAEA,MAAM,OAAO,GAAe,EAAE;AAC9B,QAAA,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC,IAAI,cAAc,EAAE;AACpD,YAAA,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,GAAG,cAAc,CAAC,CAAC;QACjD;QAEA,MAAM,UAAU,GAA4C,EAAE;AAE9D,QAAA,KAAK,MAAM,KAAK,IAAI,OAAO,EAAE;YAC3B,MAAM,YAAY,GAAG,MAAM,IAAI,CAAC,YAAY,CAAC,KAAK,EAAE,OAAO,CAAC;AAC5D,YAAA,UAAU,CAAC,IAAI,CAAC,GAAG,YAAY,CAAC;QAClC;AAEA,QAAA,OAAO,UAAU;IACnB;AAEQ,IAAA,MAAM,YAAY,CACxB,IAAc,EACd,UAAiC,EAAE,EAAA;AAEnC,QAAA,IAAI;YACF,MAAM,cAAc,GAAG,OAAO,CAAC,cAAc,IAAI,IAAI,CAAC,cAAc;YACpE,MAAM,MAAM,GAAG,OAAO,CAAC,MAAM,IAAI,IAAI,CAAC,MAAM;YAC5C,MAAM,YAAY,GAAG,OAAO,CAAC,YAAY,IAAI,IAAI,CAAC,YAAY;AAC9D,YAAA,MAAM,OAAO,GAA2B;gBACtC,IAAI;AACJ,gBAAA,aAAa,EAAE,YAAY;AAC3B,gBAAA,cAAc,EAAE,OAAO,CAAC,aAAa,IAAI,IAAI,CAAC,aAAa;aAC5D;YAED,IAAI,cAAc,EAAE;AAClB,gBAAA,OAAO,CAAC,eAAe,GAAG,IAAI;YAChC;AACA,YAAA,IAAI,MAAM,IAAI,IAAI,EAAE;AAClB,gBAAA,OAAO,CAAC,MAAM,GAAG,MAAM;YACzB;AAEA,YAAA,MAAM,gBAAgB,GACpB,OAAO,CAAC,OAAO;AACf,gBAAA,IAAI,CAAC,cAAc;AACnB,iBAAC,OAAO,CAAC,YAAY,IAAI;AACvB,sBAAE,iBAAiB,CAAC,YAAY;AAChC,sBAAE,IAAI,CAAC,OAAO,CAAC;YACnB,MAAM,cAAc,GAAG,OAAO,CAAC,OAAO,IAAI,IAAI,CAAC,cAAc;AAC7D,YAAA,IAAI,cAAc,IAAI,IAAI,EAAE;gBAC1B,OAAO,CAAC,OAAO,GAAG,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,cAAc,GAAG,IAAI,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC;YACpE;AAEA,YAAA,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,IAAI,CAG9B,IAAI,CAAC,MAAM,EAAE,OAAO,EAAE;AACvB,gBAAA,OAAO,EAAE;AACP,oBAAA,aAAa,EAAE,CAAA,OAAA,EAAU,IAAI,CAAC,MAAM,CAAA,CAAE;AACtC,oBAAA,cAAc,EAAE,kBAAkB;AACnC,iBAAA;AACD,gBAAA,OAAO,EAAE,gBAAgB;AAC1B,aAAA,CAAC;AAEF,YAAA,MAAM,IAAI,GAAG,QAAQ,CAAC,IAAI;AAC1B,YAAA,MAAM,UAAU,GAAG,IAAI,GAAG,EAAiC;AAC3D,YAAA,MAAM,SAAS,GAAG,IAAI,GAAG,EAAiC;YAE1D,KAAK,MAAM,MAAM,IAAI,IAAI,CAAC,OAAO,IAAI,EAAE,EAAE;AACvC,gBAAA,YAAY,CAAC,UAAU,EAAE,MAAM,CAAC;YAClC;YACA,KAAK,MAAM,MAAM,IAAI,IAAI,CAAC,cAAc,IAAI,EAAE,EAAE;AAC9C,gBAAA,YAAY,CAAC,SAAS,EAAE,MAAM,CAAC;YACjC;AAEA,YAAA,OAAO,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,KAAsC;AACxD,gBAAA,MAAM,OAAO,GACX,UAAU,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI,UAAU,CAAC,GAAG,CAAC,eAAe,CAAC,GAAG,CAAC,CAAC;gBAC7D,IAAI,OAAO,IAAI,OAAO,CAAC,KAAK,IAAI,IAAI,EAAE;oBACpC,OAAO;wBACL,GAAG;AACH,wBAAA;AACE,4BAAA,OAAO,EAAE,IAAI;AACb,4BAAA,IAAI,EAAE;AACJ,gCAAA,UAAU,EAAE,OAAO,CAAC,WAAW,IAAI,EAAE;AACrC,gCAAA,MAAM,EAAE,OAAO,CAAC,MAAM,IAAI,EAAE;gCAC5B,OAAO,EAAE,OAAO,CAAC,OAAO;AACzB,6BAAA;AACF,yBAAA;qBACF;gBACH;AAEA,gBAAA,MAAM,MAAM,GACV,SAAS,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI,SAAS,CAAC,GAAG,CAAC,eAAe,CAAC,GAAG,CAAC,CAAC;AAC3D,gBAAA,MAAM,KAAK,GACT,OAAO,EAAE,KAAK;AACd,oBAAA,MAAM,EAAE,KAAK;AACb,oBAAA,0CAA0C;gBAC5C,OAAO,CAAC,GAAG,EAAE,EAAE,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,CAAC;AACzC,YAAA,CAAC,CAAC;QACJ;QAAE,OAAO,KAAK,EAAE;AACd,YAAA,MAAM,YAAY,GAChB,KAAK,YAAY,KAAK,GAAG,KAAK,CAAC,OAAO,GAAG,MAAM,CAAC,KAAK,CAAC;YACxD,OAAO,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,KAAK;gBACvB,GAAG;AACH,gBAAA;AACE,oBAAA,OAAO,EAAE,KAAK;oBACd,KAAK,EAAE,CAAA,mCAAA,EAAsC,YAAY,CAAA,CAAE;AAC5D,iBAAA;AACF,aAAA,CAAC;QACJ;IACF;AAEA,IAAA,cAAc,CACZ,QAAgC,EAAA;QAEhC,IAAI,CAAC,QAAQ,CAAC,OAAO,IAAI,CAAC,QAAQ,CAAC,IAAI,EAAE;AACvC,YAAA,OAAO,CAAC,EAAE,EAAE,SAAS,CAAC;QACxB;QAEA,MAAM,OAAO,GAAG,QAAQ,CAAC,IAAI,CAAC,UAAU,IAAI,EAAE;QAC9C,MAAM,MAAM,GAAG,QAAQ,CAAC,IAAI,CAAC,MAAM,IAAI,EAAE;AAEzC,QAAA,MAAM,UAAU,GACd,MAAM,CAAC,MAAM,GAAG;AACd,cAAE;AACE,gBAAA,KAAK,EAAE,EAAE;AACT,gBAAA,MAAM,EAAE,MAAM,CAAC,GAAG,CAAC,CAAC,QAAQ,MAAM,EAAE,WAAW,EAAE,QAAQ,EAAE,CAAC,CAAC;AAC7D,gBAAA,MAAM,EAAE,EAAE;AACX;cACD,SAAS;AAEf,QAAA,OAAO,CAAC,OAAO,EAAE,UAAU,CAAC;IAC9B;AAEA,IAAA,eAAe,CAAC,QAAgC,EAAA;QAC9C,IAAI,CAAC,QAAQ,CAAC,OAAO,IAAI,CAAC,QAAQ,CAAC,IAAI,EAAE;AACvC,YAAA,OAAO,EAAE;QACX;AAEA,QAAA,MAAM,QAAQ,GAA4B;YACxC,YAAY,EAAE,QAAQ,CAAC,IAAI,CAAC,MAAM,EAAE,MAAM,IAAI,CAAC;SAChD;QACD,IAAI,QAAQ,CAAC,IAAI,CAAC,OAAO,IAAI,IAAI,EAAE;YACjC,QAAQ,CAAC,OAAO,GAAG,QAAQ,CAAC,IAAI,CAAC,OAAO;QAC1C;AACA,QAAA,OAAO,QAAQ;IACjB;AACD;MAEY,mBAAmB,GAAG,CACjC,MAAA,GAAgC,EAAE,KACjB;AACjB,IAAA,OAAO,IAAI,aAAa,CAAC,MAAM,CAAC;AAClC;;;;;"}