@illuma-ai/agents 1.4.0-alpha.6 → 1.5.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +62 -0
- package/dist/cjs/agents/AgentContext.cjs +274 -67
- package/dist/cjs/agents/AgentContext.cjs.map +1 -1
- package/dist/cjs/common/enum.cjs +44 -13
- package/dist/cjs/common/enum.cjs.map +1 -1
- package/dist/cjs/graphs/Graph.cjs +182 -5
- package/dist/cjs/graphs/Graph.cjs.map +1 -1
- package/dist/cjs/graphs/MultiAgentGraph.cjs +152 -1167
- package/dist/cjs/graphs/MultiAgentGraph.cjs.map +1 -1
- package/dist/cjs/graphs/phases/memoryFlushPhase.cjs +1 -1
- package/dist/cjs/graphs/phases/memoryFlushPhase.cjs.map +1 -1
- package/dist/cjs/hooks/HookRegistry.cjs +162 -0
- package/dist/cjs/hooks/HookRegistry.cjs.map +1 -0
- package/dist/cjs/hooks/executeHooks.cjs +276 -0
- package/dist/cjs/hooks/executeHooks.cjs.map +1 -0
- package/dist/cjs/hooks/matchers.cjs +256 -0
- package/dist/cjs/hooks/matchers.cjs.map +1 -0
- package/dist/cjs/hooks/types.cjs +27 -0
- package/dist/cjs/hooks/types.cjs.map +1 -0
- package/dist/cjs/langchain/google-common.cjs +3 -0
- package/dist/cjs/langchain/google-common.cjs.map +1 -0
- package/dist/cjs/langchain/index.cjs +86 -0
- package/dist/cjs/langchain/index.cjs.map +1 -0
- package/dist/cjs/langchain/language_models/chat_models.cjs +3 -0
- package/dist/cjs/langchain/language_models/chat_models.cjs.map +1 -0
- package/dist/cjs/langchain/messages/tool.cjs +3 -0
- package/dist/cjs/langchain/messages/tool.cjs.map +1 -0
- package/dist/cjs/langchain/messages.cjs +51 -0
- package/dist/cjs/langchain/messages.cjs.map +1 -0
- package/dist/cjs/langchain/openai.cjs +3 -0
- package/dist/cjs/langchain/openai.cjs.map +1 -0
- package/dist/cjs/langchain/prompts.cjs +11 -0
- package/dist/cjs/langchain/prompts.cjs.map +1 -0
- package/dist/cjs/langchain/runnables.cjs +19 -0
- package/dist/cjs/langchain/runnables.cjs.map +1 -0
- package/dist/cjs/langchain/tools.cjs +23 -0
- package/dist/cjs/langchain/tools.cjs.map +1 -0
- package/dist/cjs/langchain/utils/env.cjs +11 -0
- package/dist/cjs/langchain/utils/env.cjs.map +1 -0
- package/dist/cjs/llm/anthropic/utils/message_inputs.cjs +5 -1
- package/dist/cjs/llm/anthropic/utils/message_inputs.cjs.map +1 -1
- package/dist/cjs/llm/bedrock/cacheSupport.cjs +55 -0
- package/dist/cjs/llm/bedrock/cacheSupport.cjs.map +1 -0
- package/dist/cjs/llm/bedrock/index.cjs +61 -33
- package/dist/cjs/llm/bedrock/index.cjs.map +1 -1
- package/dist/cjs/llm/openai/index.cjs +1 -4
- package/dist/cjs/llm/openai/index.cjs.map +1 -1
- package/dist/cjs/llm/openai/utils/index.cjs +27 -10
- package/dist/cjs/llm/openai/utils/index.cjs.map +1 -1
- package/dist/cjs/main.cjs +178 -127
- package/dist/cjs/main.cjs.map +1 -1
- package/dist/cjs/memory/citations.cjs +4 -4
- package/dist/cjs/memory/citations.cjs.map +1 -1
- package/dist/cjs/memory/constants.cjs +17 -17
- package/dist/cjs/memory/constants.cjs.map +1 -1
- package/dist/cjs/memory/mmr.cjs +1 -1
- package/dist/cjs/memory/mmr.cjs.map +1 -1
- package/dist/cjs/memory/paths.cjs +1 -1
- package/dist/cjs/memory/paths.cjs.map +1 -1
- package/dist/cjs/memory/recallTracking.cjs +3 -3
- package/dist/cjs/memory/recallTracking.cjs.map +1 -1
- package/dist/cjs/memory/temporalDecay.cjs +2 -2
- package/dist/cjs/memory/temporalDecay.cjs.map +1 -1
- package/dist/cjs/messages/cache.cjs +89 -0
- package/dist/cjs/messages/cache.cjs.map +1 -1
- package/dist/cjs/messages/contextPruning.cjs +156 -0
- package/dist/cjs/messages/contextPruning.cjs.map +1 -0
- package/dist/cjs/messages/contextPruningSettings.cjs +53 -0
- package/dist/cjs/messages/contextPruningSettings.cjs.map +1 -0
- package/dist/cjs/messages/format.cjs +144 -20
- package/dist/cjs/messages/format.cjs.map +1 -1
- package/dist/cjs/messages/prune.cjs +505 -4
- package/dist/cjs/messages/prune.cjs.map +1 -1
- package/dist/cjs/run.cjs +141 -1
- package/dist/cjs/run.cjs.map +1 -1
- package/dist/cjs/tools/BashExecutor.cjs +235 -0
- package/dist/cjs/tools/BashExecutor.cjs.map +1 -0
- package/dist/cjs/tools/BashProgrammaticToolCalling.cjs +297 -0
- package/dist/cjs/tools/BashProgrammaticToolCalling.cjs.map +1 -0
- package/dist/cjs/tools/CodeExecutor.cjs +45 -47
- package/dist/cjs/tools/CodeExecutor.cjs.map +1 -1
- package/dist/cjs/tools/ProgrammaticToolCalling.cjs +16 -11
- package/dist/cjs/tools/ProgrammaticToolCalling.cjs.map +1 -1
- package/dist/cjs/tools/ReadFile.cjs +44 -0
- package/dist/cjs/tools/ReadFile.cjs.map +1 -0
- package/dist/cjs/tools/SkillTool.cjs +51 -0
- package/dist/cjs/tools/SkillTool.cjs.map +1 -0
- package/dist/cjs/tools/SubagentTool.cjs +93 -0
- package/dist/cjs/tools/SubagentTool.cjs.map +1 -0
- package/dist/cjs/tools/ToolNode.cjs +450 -24
- package/dist/cjs/tools/ToolNode.cjs.map +1 -1
- package/dist/cjs/tools/memory/memoryAppendTool.cjs +1 -1
- package/dist/cjs/tools/memory/memoryAppendTool.cjs.map +1 -1
- package/dist/cjs/tools/memory/memoryGetTool.cjs +2 -2
- package/dist/cjs/tools/memory/memoryGetTool.cjs.map +1 -1
- package/dist/cjs/tools/memory/memorySearchTool.cjs +3 -3
- package/dist/cjs/tools/memory/memorySearchTool.cjs.map +1 -1
- package/dist/cjs/tools/memory/shared.cjs +1 -1
- package/dist/cjs/tools/memory/shared.cjs.map +1 -1
- package/dist/cjs/tools/search/search.cjs +11 -3
- package/dist/cjs/tools/search/search.cjs.map +1 -1
- package/dist/cjs/tools/search/tavily-scraper.cjs +189 -0
- package/dist/cjs/tools/search/tavily-scraper.cjs.map +1 -0
- package/dist/cjs/tools/search/tavily-search.cjs +372 -0
- package/dist/cjs/tools/search/tavily-search.cjs.map +1 -0
- package/dist/cjs/tools/search/tool.cjs +28 -4
- package/dist/cjs/tools/search/tool.cjs.map +1 -1
- package/dist/cjs/tools/search/utils.cjs +10 -3
- package/dist/cjs/tools/search/utils.cjs.map +1 -1
- package/dist/cjs/tools/skillCatalog.cjs +84 -0
- package/dist/cjs/tools/skillCatalog.cjs.map +1 -0
- package/dist/cjs/tools/subagent/SubagentExecutor.cjs +512 -0
- package/dist/cjs/tools/subagent/SubagentExecutor.cjs.map +1 -0
- package/dist/cjs/tools/toolOutputReferences.cjs +670 -0
- package/dist/cjs/tools/toolOutputReferences.cjs.map +1 -0
- package/dist/cjs/types/agent-cache.cjs +54 -0
- package/dist/cjs/types/agent-cache.cjs.map +1 -0
- package/dist/cjs/types/graph.cjs.map +1 -1
- package/dist/cjs/utils/truncation.cjs +135 -0
- package/dist/cjs/utils/truncation.cjs.map +1 -0
- package/dist/esm/agents/AgentContext.mjs +274 -67
- package/dist/esm/agents/AgentContext.mjs.map +1 -1
- package/dist/esm/common/enum.mjs +44 -12
- package/dist/esm/common/enum.mjs.map +1 -1
- package/dist/esm/graphs/Graph.mjs +182 -5
- package/dist/esm/graphs/Graph.mjs.map +1 -1
- package/dist/esm/graphs/MultiAgentGraph.mjs +155 -1170
- package/dist/esm/graphs/MultiAgentGraph.mjs.map +1 -1
- package/dist/esm/graphs/phases/memoryFlushPhase.mjs +1 -1
- package/dist/esm/graphs/phases/memoryFlushPhase.mjs.map +1 -1
- package/dist/esm/hooks/HookRegistry.mjs +160 -0
- package/dist/esm/hooks/HookRegistry.mjs.map +1 -0
- package/dist/esm/hooks/executeHooks.mjs +273 -0
- package/dist/esm/hooks/executeHooks.mjs.map +1 -0
- package/dist/esm/hooks/matchers.mjs +251 -0
- package/dist/esm/hooks/matchers.mjs.map +1 -0
- package/dist/esm/hooks/types.mjs +25 -0
- package/dist/esm/hooks/types.mjs.map +1 -0
- package/dist/esm/langchain/google-common.mjs +2 -0
- package/dist/esm/langchain/google-common.mjs.map +1 -0
- package/dist/esm/langchain/index.mjs +5 -0
- package/dist/esm/langchain/language_models/chat_models.mjs +2 -0
- package/dist/esm/langchain/language_models/chat_models.mjs.map +1 -0
- package/dist/esm/langchain/messages/tool.mjs +2 -0
- package/dist/esm/langchain/messages/tool.mjs.map +1 -0
- package/dist/esm/langchain/messages.mjs +2 -0
- package/dist/esm/langchain/messages.mjs.map +1 -0
- package/dist/esm/langchain/openai.mjs +2 -0
- package/dist/esm/langchain/openai.mjs.map +1 -0
- package/dist/esm/langchain/prompts.mjs +2 -0
- package/dist/esm/langchain/prompts.mjs.map +1 -0
- package/dist/esm/langchain/runnables.mjs +2 -0
- package/dist/esm/langchain/runnables.mjs.map +1 -0
- package/dist/esm/langchain/tools.mjs +2 -0
- package/dist/esm/langchain/tools.mjs.map +1 -0
- package/dist/esm/langchain/utils/env.mjs +2 -0
- package/dist/esm/langchain/utils/env.mjs.map +1 -0
- package/dist/esm/llm/anthropic/utils/message_inputs.mjs +5 -1
- package/dist/esm/llm/anthropic/utils/message_inputs.mjs.map +1 -1
- package/dist/esm/llm/bedrock/cacheSupport.mjs +52 -0
- package/dist/esm/llm/bedrock/cacheSupport.mjs.map +1 -0
- package/dist/esm/llm/bedrock/index.mjs +61 -34
- package/dist/esm/llm/bedrock/index.mjs.map +1 -1
- package/dist/esm/llm/openai/index.mjs +1 -4
- package/dist/esm/llm/openai/index.mjs.map +1 -1
- package/dist/esm/llm/openai/utils/index.mjs +27 -10
- package/dist/esm/llm/openai/utils/index.mjs.map +1 -1
- package/dist/esm/main.mjs +21 -27
- package/dist/esm/main.mjs.map +1 -1
- package/dist/esm/memory/citations.mjs +4 -4
- package/dist/esm/memory/citations.mjs.map +1 -1
- package/dist/esm/memory/constants.mjs +17 -17
- package/dist/esm/memory/constants.mjs.map +1 -1
- package/dist/esm/memory/mmr.mjs +1 -1
- package/dist/esm/memory/mmr.mjs.map +1 -1
- package/dist/esm/memory/paths.mjs +1 -1
- package/dist/esm/memory/paths.mjs.map +1 -1
- package/dist/esm/memory/recallTracking.mjs +3 -3
- package/dist/esm/memory/recallTracking.mjs.map +1 -1
- package/dist/esm/memory/temporalDecay.mjs +2 -2
- package/dist/esm/memory/temporalDecay.mjs.map +1 -1
- package/dist/esm/messages/cache.mjs +89 -0
- package/dist/esm/messages/cache.mjs.map +1 -1
- package/dist/esm/messages/contextPruning.mjs +154 -0
- package/dist/esm/messages/contextPruning.mjs.map +1 -0
- package/dist/esm/messages/contextPruningSettings.mjs +50 -0
- package/dist/esm/messages/contextPruningSettings.mjs.map +1 -0
- package/dist/esm/messages/format.mjs +136 -12
- package/dist/esm/messages/format.mjs.map +1 -1
- package/dist/esm/messages/prune.mjs +504 -7
- package/dist/esm/messages/prune.mjs.map +1 -1
- package/dist/esm/run.mjs +141 -1
- package/dist/esm/run.mjs.map +1 -1
- package/dist/esm/tools/BashExecutor.mjs +227 -0
- package/dist/esm/tools/BashExecutor.mjs.map +1 -0
- package/dist/esm/tools/BashProgrammaticToolCalling.mjs +288 -0
- package/dist/esm/tools/BashProgrammaticToolCalling.mjs.map +1 -0
- package/dist/esm/tools/CodeExecutor.mjs +45 -48
- package/dist/esm/tools/CodeExecutor.mjs.map +1 -1
- package/dist/esm/tools/ProgrammaticToolCalling.mjs +17 -12
- package/dist/esm/tools/ProgrammaticToolCalling.mjs.map +1 -1
- package/dist/esm/tools/ReadFile.mjs +39 -0
- package/dist/esm/tools/ReadFile.mjs.map +1 -0
- package/dist/esm/tools/SkillTool.mjs +46 -0
- package/dist/esm/tools/SkillTool.mjs.map +1 -0
- package/dist/esm/tools/SubagentTool.mjs +86 -0
- package/dist/esm/tools/SubagentTool.mjs.map +1 -0
- package/dist/esm/tools/ToolNode.mjs +452 -26
- package/dist/esm/tools/ToolNode.mjs.map +1 -1
- package/dist/esm/tools/memory/memoryAppendTool.mjs +1 -1
- package/dist/esm/tools/memory/memoryAppendTool.mjs.map +1 -1
- package/dist/esm/tools/memory/memoryGetTool.mjs +2 -2
- package/dist/esm/tools/memory/memoryGetTool.mjs.map +1 -1
- package/dist/esm/tools/memory/memorySearchTool.mjs +3 -3
- package/dist/esm/tools/memory/memorySearchTool.mjs.map +1 -1
- package/dist/esm/tools/memory/shared.mjs +1 -1
- package/dist/esm/tools/memory/shared.mjs.map +1 -1
- package/dist/esm/tools/search/search.mjs +11 -3
- package/dist/esm/tools/search/search.mjs.map +1 -1
- package/dist/esm/tools/search/tavily-scraper.mjs +186 -0
- package/dist/esm/tools/search/tavily-scraper.mjs.map +1 -0
- package/dist/esm/tools/search/tavily-search.mjs +370 -0
- package/dist/esm/tools/search/tavily-search.mjs.map +1 -0
- package/dist/esm/tools/search/tool.mjs +28 -4
- package/dist/esm/tools/search/tool.mjs.map +1 -1
- package/dist/esm/tools/search/utils.mjs +10 -3
- package/dist/esm/tools/search/utils.mjs.map +1 -1
- package/dist/esm/tools/skillCatalog.mjs +82 -0
- package/dist/esm/tools/skillCatalog.mjs.map +1 -0
- package/dist/esm/tools/subagent/SubagentExecutor.mjs +506 -0
- package/dist/esm/tools/subagent/SubagentExecutor.mjs.map +1 -0
- package/dist/esm/tools/toolOutputReferences.mjs +662 -0
- package/dist/esm/tools/toolOutputReferences.mjs.map +1 -0
- package/dist/esm/types/agent-cache.mjs +52 -0
- package/dist/esm/types/agent-cache.mjs.map +1 -0
- package/dist/esm/types/graph.mjs.map +1 -1
- package/dist/esm/utils/truncation.mjs +128 -0
- package/dist/esm/utils/truncation.mjs.map +1 -0
- package/dist/types/agents/AgentContext.d.ts +101 -8
- package/dist/types/common/enum.d.ts +39 -12
- package/dist/types/common/index.d.ts +0 -1
- package/dist/types/graphs/Graph.d.ts +43 -0
- package/dist/types/graphs/MultiAgentGraph.d.ts +26 -150
- package/dist/types/graphs/index.d.ts +0 -1
- package/dist/types/graphs/phases/memoryFlushPhase.d.ts +2 -2
- package/dist/types/hooks/HookRegistry.d.ts +56 -0
- package/dist/types/hooks/executeHooks.d.ts +79 -0
- package/dist/types/hooks/index.d.ts +6 -0
- package/dist/types/hooks/matchers.d.ts +95 -0
- package/dist/types/hooks/types.d.ts +320 -0
- package/dist/types/index.d.ts +9 -9
- package/dist/types/langchain/google-common.d.ts +1 -0
- package/dist/types/langchain/index.d.ts +8 -0
- package/dist/types/langchain/language_models/chat_models.d.ts +1 -0
- package/dist/types/langchain/messages/tool.d.ts +1 -0
- package/dist/types/langchain/messages.d.ts +2 -0
- package/dist/types/langchain/openai.d.ts +1 -0
- package/dist/types/langchain/prompts.d.ts +1 -0
- package/dist/types/langchain/runnables.d.ts +2 -0
- package/dist/types/langchain/tools.d.ts +2 -0
- package/dist/types/langchain/utils/env.d.ts +1 -0
- package/dist/types/llm/bedrock/cacheSupport.d.ts +35 -0
- package/dist/types/llm/bedrock/index.d.ts +54 -1
- package/dist/types/llm/openai/index.d.ts +1 -1
- package/dist/types/memory/citations.d.ts +4 -4
- package/dist/types/memory/constants.d.ts +17 -17
- package/dist/types/memory/mmr.d.ts +3 -3
- package/dist/types/memory/paths.d.ts +1 -1
- package/dist/types/memory/temporalDecay.d.ts +2 -2
- package/dist/types/memory/types.d.ts +3 -3
- package/dist/types/messages/contextPruning.d.ts +42 -0
- package/dist/types/messages/contextPruningSettings.d.ts +44 -0
- package/dist/types/messages/format.d.ts +9 -1
- package/dist/types/messages/index.d.ts +2 -0
- package/dist/types/messages/prune.d.ts +91 -1
- package/dist/types/run.d.ts +2 -0
- package/dist/types/tools/BashExecutor.d.ts +76 -0
- package/dist/types/tools/BashProgrammaticToolCalling.d.ts +72 -0
- package/dist/types/tools/CodeExecutor.d.ts +8 -26
- package/dist/types/tools/ReadFile.d.ts +28 -0
- package/dist/types/tools/SkillTool.d.ts +40 -0
- package/dist/types/tools/SubagentTool.d.ts +36 -0
- package/dist/types/tools/ToolNode.d.ts +77 -5
- package/dist/types/tools/memory/shared.d.ts +1 -1
- package/dist/types/tools/search/tavily-scraper.d.ts +19 -0
- package/dist/types/tools/search/tavily-search.d.ts +4 -0
- package/dist/types/tools/search/types.d.ts +99 -5
- package/dist/types/tools/search/utils.d.ts +2 -2
- package/dist/types/tools/skillCatalog.d.ts +19 -0
- package/dist/types/tools/subagent/SubagentExecutor.d.ts +137 -0
- package/dist/types/tools/subagent/index.d.ts +2 -0
- package/dist/types/tools/subagent/types.d.ts +84 -0
- package/dist/types/tools/toolOutputReferences.d.ts +236 -0
- package/dist/types/types/agent-cache.d.ts +71 -0
- package/dist/types/types/graph.d.ts +163 -22
- package/dist/types/types/index.d.ts +3 -0
- package/dist/types/types/messages.d.ts +26 -0
- package/dist/types/types/run.d.ts +22 -0
- package/dist/types/types/skill.d.ts +9 -0
- package/dist/types/types/tools.d.ts +111 -0
- package/dist/types/utils/index.d.ts +1 -3
- package/dist/types/utils/truncation.d.ts +70 -0
- package/package.json +57 -17
- package/src/agents/AgentContext.ts +321 -78
- package/src/agents/__tests__/AgentContext.cacheTtl.live.test.ts +259 -0
- package/src/agents/__tests__/AgentContext.crossAgentTier1.live.test.ts +266 -0
- package/src/agents/__tests__/AgentContext.crossUserCache.live.test.ts +342 -0
- package/src/agents/__tests__/AgentContext.test.ts +632 -0
- package/src/common/__tests__/enum.test.ts +7 -17
- package/src/common/enum.ts +43 -12
- package/src/common/index.ts +0 -1
- package/src/graphs/Graph.ts +222 -2
- package/src/graphs/MultiAgentGraph.ts +154 -1466
- package/src/graphs/__tests__/MultiAgentGraph.test.ts +91 -0
- package/src/graphs/gapFeatures.test.ts +1 -1
- package/src/graphs/index.ts +0 -1
- package/src/graphs/phases/__tests__/memoryFlushPhase.test.ts +1 -1
- package/src/graphs/phases/memoryFlushPhase.ts +2 -2
- package/src/hooks/HookRegistry.ts +208 -0
- package/src/hooks/__tests__/HookRegistry.test.ts +190 -0
- package/src/hooks/__tests__/compactHooks.test.ts +214 -0
- package/src/hooks/__tests__/executeHooks.test.ts +1013 -0
- package/src/hooks/__tests__/integration.test.ts +337 -0
- package/src/hooks/__tests__/matchers.test.ts +238 -0
- package/src/hooks/__tests__/toolHooks.test.ts +665 -0
- package/src/hooks/executeHooks.ts +375 -0
- package/src/hooks/index.ts +57 -0
- package/src/hooks/matchers.ts +280 -0
- package/src/hooks/types.ts +404 -0
- package/src/index.ts +15 -24
- package/src/langchain/google-common.ts +1 -0
- package/src/langchain/index.ts +8 -0
- package/src/langchain/language_models/chat_models.ts +1 -0
- package/src/langchain/messages/tool.ts +5 -0
- package/src/langchain/messages.ts +21 -0
- package/src/langchain/openai.ts +1 -0
- package/src/langchain/prompts.ts +1 -0
- package/src/langchain/runnables.ts +7 -0
- package/src/langchain/tools.ts +8 -0
- package/src/langchain/utils/env.ts +1 -0
- package/src/llm/anthropic/utils/message_inputs.ts +10 -1
- package/src/llm/anthropic/utils/server-tool-inputs.test.ts +436 -0
- package/src/llm/bedrock/__tests__/bedrock-caching.test.ts +166 -18
- package/src/llm/bedrock/cacheSupport.test.ts +99 -0
- package/src/llm/bedrock/cacheSupport.ts +53 -0
- package/src/llm/bedrock/index.ts +116 -41
- package/src/llm/openai/index.ts +2 -2
- package/src/llm/openai/utils/index.ts +31 -14
- package/src/memory/citations.ts +4 -4
- package/src/memory/constants.ts +17 -17
- package/src/memory/mmr.ts +3 -3
- package/src/memory/paths.ts +1 -1
- package/src/memory/recallTracking.ts +3 -3
- package/src/memory/temporalDecay.ts +2 -2
- package/src/memory/types.ts +3 -3
- package/src/messages/__tests__/contextPruning.test.ts +228 -0
- package/src/messages/cache.test.ts +62 -24
- package/src/messages/cache.ts +112 -0
- package/src/messages/contextPruning.ts +191 -0
- package/src/messages/contextPruningSettings.ts +90 -0
- package/src/messages/ensureThinkingBlock.test.ts +1 -1
- package/src/messages/format.ts +164 -12
- package/src/messages/formatAgentMessages.skills.test.ts +413 -0
- package/src/messages/formatAgentMessages.test.ts +1 -1
- package/src/messages/index.ts +2 -0
- package/src/messages/prune.ts +661 -4
- package/src/run.ts +155 -1
- package/src/scripts/multi-agent-chain.ts +2 -2
- package/src/scripts/multi-agent-document-review-chain.ts +2 -2
- package/src/scripts/multi-agent-hybrid-flow.ts +4 -4
- package/src/scripts/multi-agent-parallel.ts +3 -3
- package/src/scripts/multi-agent-sequence.ts +3 -3
- package/src/scripts/multi-agent-subagent.ts +246 -0
- package/src/scripts/multi-agent-supervisor.ts +5 -5
- package/src/scripts/poc-multi-agent-comprehensive.ts +8 -8
- package/src/scripts/sequential-full-metadata-test.ts +2 -2
- package/src/scripts/subagent-event-driven-debug.ts +190 -0
- package/src/scripts/subagent-tools-debug.ts +160 -0
- package/src/scripts/test-custom-prompt-key.ts +3 -3
- package/src/scripts/test-handoff-input.ts +1 -1
- package/src/scripts/test-handoff-steering.ts +3 -3
- package/src/scripts/test-multi-agent-list-handoff.ts +1 -1
- package/src/scripts/test-parallel-agent-labeling.ts +3 -3
- package/src/scripts/test-parallel-handoffs.ts +2 -2
- package/src/scripts/test-thinking-handoff-bedrock.ts +1 -1
- package/src/scripts/test-thinking-handoff.ts +1 -1
- package/src/scripts/test-thinking-to-thinking-handoff-bedrock.ts +1 -1
- package/src/scripts/test-tool-before-handoff-role-order.ts +1 -1
- package/src/scripts/test-tools-before-handoff.ts +1 -1
- package/src/specs/agent-handoffs.test.ts +26 -483
- package/src/specs/anthropic.simple.test.ts +61 -0
- package/src/specs/multi-agent-summarization.test.ts +396 -0
- package/src/specs/prune.orphans.test.ts +248 -0
- package/src/specs/prune.test.ts +104 -16
- package/src/specs/thinking-handoff.test.ts +19 -19
- package/src/tools/BashExecutor.ts +281 -0
- package/src/tools/BashProgrammaticToolCalling.ts +397 -0
- package/src/tools/CodeExecutor.ts +63 -54
- package/src/tools/ProgrammaticToolCalling.ts +29 -14
- package/src/tools/ReadFile.ts +39 -0
- package/src/tools/SkillTool.ts +46 -0
- package/src/tools/SubagentTool.ts +100 -0
- package/src/tools/ToolNode.ts +548 -26
- package/src/tools/__tests__/BashExecutor.test.ts +49 -0
- package/src/tools/__tests__/CodeExecutor.test.ts +37 -36
- package/src/tools/__tests__/ProgrammaticToolCalling.test.ts +60 -0
- package/src/tools/__tests__/ReadFile.test.ts +44 -0
- package/src/tools/__tests__/SkillTool.test.ts +442 -0
- package/src/tools/__tests__/SubagentExecutor.test.ts +1148 -0
- package/src/tools/__tests__/SubagentTool.test.ts +149 -0
- package/src/tools/__tests__/ToolNode.outputReferences.test.ts +1438 -0
- package/src/tools/__tests__/annotateMessagesForLLM.test.ts +479 -0
- package/src/tools/__tests__/skillCatalog.test.ts +161 -0
- package/src/tools/__tests__/subagentHooks.test.ts +210 -0
- package/src/tools/__tests__/toolOutputReferences.test.ts +415 -0
- package/src/tools/memory/memoryAppendTool.ts +1 -1
- package/src/tools/memory/memoryGetTool.ts +2 -2
- package/src/tools/memory/memorySearchTool.ts +3 -3
- package/src/tools/memory/shared.ts +1 -1
- package/src/tools/search/search.ts +12 -2
- package/src/tools/search/tavily-scraper.ts +235 -0
- package/src/tools/search/tavily-search.ts +424 -0
- package/src/tools/search/tavily.test.ts +965 -0
- package/src/tools/search/tool.ts +36 -2
- package/src/tools/search/types.ts +133 -8
- package/src/tools/search/utils.ts +13 -5
- package/src/tools/skillCatalog.ts +126 -0
- package/src/tools/subagent/SubagentExecutor.ts +676 -0
- package/src/tools/subagent/index.ts +13 -0
- package/src/tools/subagent/types.test.ts +70 -0
- package/src/tools/subagent/types.ts +115 -0
- package/src/tools/toolOutputReferences.ts +825 -0
- package/src/types/agent-cache.ts +74 -0
- package/src/types/graph.ts +172 -20
- package/src/types/index.ts +3 -0
- package/src/types/messages.ts +27 -0
- package/src/types/run.ts +22 -0
- package/src/types/skill.ts +11 -0
- package/src/types/tools.ts +118 -0
- package/src/utils/__tests__/truncation.test.ts +66 -0
- package/src/utils/index.ts +1 -3
- package/src/utils/truncation.ts +154 -0
- package/dist/cjs/common/spawnPath.cjs +0 -104
- package/dist/cjs/common/spawnPath.cjs.map +0 -1
- package/dist/cjs/content/ArtifactStore.cjs +0 -579
- package/dist/cjs/content/ArtifactStore.cjs.map +0 -1
- package/dist/cjs/content/ContentStore.cjs +0 -638
- package/dist/cjs/content/ContentStore.cjs.map +0 -1
- package/dist/cjs/content/contentAnalyzer.cjs +0 -91
- package/dist/cjs/content/contentAnalyzer.cjs.map +0 -1
- package/dist/cjs/content/index.cjs +0 -20
- package/dist/cjs/content/index.cjs.map +0 -1
- package/dist/cjs/content/mcpAutoCache.cjs +0 -115
- package/dist/cjs/content/mcpAutoCache.cjs.map +0 -1
- package/dist/cjs/graphs/HandoffRegistry.cjs +0 -143
- package/dist/cjs/graphs/HandoffRegistry.cjs.map +0 -1
- package/dist/cjs/providers/a2a/A2ACapabilityProvider.cjs +0 -288
- package/dist/cjs/providers/a2a/A2ACapabilityProvider.cjs.map +0 -1
- package/dist/cjs/providers/a2a/client.cjs +0 -92
- package/dist/cjs/providers/a2a/client.cjs.map +0 -1
- package/dist/cjs/providers/a2a/config.cjs +0 -38
- package/dist/cjs/providers/a2a/config.cjs.map +0 -1
- package/dist/cjs/providers/capabilityNaming.cjs +0 -43
- package/dist/cjs/providers/capabilityNaming.cjs.map +0 -1
- package/dist/cjs/providers/mcp/MCPCapabilityProvider.cjs +0 -244
- package/dist/cjs/providers/mcp/MCPCapabilityProvider.cjs.map +0 -1
- package/dist/cjs/providers/mcp/config.cjs +0 -42
- package/dist/cjs/providers/mcp/config.cjs.map +0 -1
- package/dist/cjs/providers/mcp/transport.cjs +0 -65
- package/dist/cjs/providers/mcp/transport.cjs.map +0 -1
- package/dist/cjs/providers/tools-server/ToolsServerCapabilityProvider.cjs +0 -128
- package/dist/cjs/providers/tools-server/ToolsServerCapabilityProvider.cjs.map +0 -1
- package/dist/cjs/providers/types.cjs +0 -51
- package/dist/cjs/providers/types.cjs.map +0 -1
- package/dist/cjs/tools/artifacts/schema.cjs +0 -86
- package/dist/cjs/tools/artifacts/schema.cjs.map +0 -1
- package/dist/cjs/tools/artifacts/tool.cjs +0 -219
- package/dist/cjs/tools/artifacts/tool.cjs.map +0 -1
- package/dist/cjs/tools/fileSearch/formatter.cjs +0 -93
- package/dist/cjs/tools/fileSearch/formatter.cjs.map +0 -1
- package/dist/cjs/tools/fileSearch/ragClient.cjs +0 -102
- package/dist/cjs/tools/fileSearch/ragClient.cjs.map +0 -1
- package/dist/cjs/tools/fileSearch/schema.cjs +0 -18
- package/dist/cjs/tools/fileSearch/schema.cjs.map +0 -1
- package/dist/cjs/tools/fileSearch/tool.cjs +0 -155
- package/dist/cjs/tools/fileSearch/tool.cjs.map +0 -1
- package/dist/cjs/tools/proxyTool.cjs +0 -102
- package/dist/cjs/tools/proxyTool.cjs.map +0 -1
- package/dist/cjs/utils/childAgentContext.cjs +0 -242
- package/dist/cjs/utils/childAgentContext.cjs.map +0 -1
- package/dist/cjs/utils/credentials.cjs +0 -142
- package/dist/cjs/utils/credentials.cjs.map +0 -1
- package/dist/cjs/utils/httpClient.cjs +0 -74
- package/dist/cjs/utils/httpClient.cjs.map +0 -1
- package/dist/cjs/utils/toolManifest.cjs +0 -100
- package/dist/cjs/utils/toolManifest.cjs.map +0 -1
- package/dist/esm/common/spawnPath.mjs +0 -95
- package/dist/esm/common/spawnPath.mjs.map +0 -1
- package/dist/esm/content/ArtifactStore.mjs +0 -576
- package/dist/esm/content/ArtifactStore.mjs.map +0 -1
- package/dist/esm/content/ContentStore.mjs +0 -635
- package/dist/esm/content/ContentStore.mjs.map +0 -1
- package/dist/esm/content/contentAnalyzer.mjs +0 -87
- package/dist/esm/content/contentAnalyzer.mjs.map +0 -1
- package/dist/esm/content/index.mjs +0 -5
- package/dist/esm/content/mcpAutoCache.mjs +0 -111
- package/dist/esm/content/mcpAutoCache.mjs.map +0 -1
- package/dist/esm/graphs/HandoffRegistry.mjs +0 -141
- package/dist/esm/graphs/HandoffRegistry.mjs.map +0 -1
- package/dist/esm/providers/a2a/A2ACapabilityProvider.mjs +0 -281
- package/dist/esm/providers/a2a/A2ACapabilityProvider.mjs.map +0 -1
- package/dist/esm/providers/a2a/client.mjs +0 -88
- package/dist/esm/providers/a2a/client.mjs.map +0 -1
- package/dist/esm/providers/a2a/config.mjs +0 -35
- package/dist/esm/providers/a2a/config.mjs.map +0 -1
- package/dist/esm/providers/capabilityNaming.mjs +0 -39
- package/dist/esm/providers/capabilityNaming.mjs.map +0 -1
- package/dist/esm/providers/mcp/MCPCapabilityProvider.mjs +0 -240
- package/dist/esm/providers/mcp/MCPCapabilityProvider.mjs.map +0 -1
- package/dist/esm/providers/mcp/config.mjs +0 -39
- package/dist/esm/providers/mcp/config.mjs.map +0 -1
- package/dist/esm/providers/mcp/transport.mjs +0 -63
- package/dist/esm/providers/mcp/transport.mjs.map +0 -1
- package/dist/esm/providers/tools-server/ToolsServerCapabilityProvider.mjs +0 -126
- package/dist/esm/providers/tools-server/ToolsServerCapabilityProvider.mjs.map +0 -1
- package/dist/esm/providers/types.mjs +0 -51
- package/dist/esm/providers/types.mjs.map +0 -1
- package/dist/esm/tools/artifacts/schema.mjs +0 -79
- package/dist/esm/tools/artifacts/schema.mjs.map +0 -1
- package/dist/esm/tools/artifacts/tool.mjs +0 -213
- package/dist/esm/tools/artifacts/tool.mjs.map +0 -1
- package/dist/esm/tools/fileSearch/formatter.mjs +0 -90
- package/dist/esm/tools/fileSearch/formatter.mjs.map +0 -1
- package/dist/esm/tools/fileSearch/ragClient.mjs +0 -98
- package/dist/esm/tools/fileSearch/ragClient.mjs.map +0 -1
- package/dist/esm/tools/fileSearch/schema.mjs +0 -15
- package/dist/esm/tools/fileSearch/schema.mjs.map +0 -1
- package/dist/esm/tools/fileSearch/tool.mjs +0 -152
- package/dist/esm/tools/fileSearch/tool.mjs.map +0 -1
- package/dist/esm/tools/proxyTool.mjs +0 -100
- package/dist/esm/tools/proxyTool.mjs.map +0 -1
- package/dist/esm/utils/childAgentContext.mjs +0 -237
- package/dist/esm/utils/childAgentContext.mjs.map +0 -1
- package/dist/esm/utils/credentials.mjs +0 -135
- package/dist/esm/utils/credentials.mjs.map +0 -1
- package/dist/esm/utils/httpClient.mjs +0 -70
- package/dist/esm/utils/httpClient.mjs.map +0 -1
- package/dist/esm/utils/toolManifest.mjs +0 -96
- package/dist/esm/utils/toolManifest.mjs.map +0 -1
- package/dist/types/common/spawnPath.d.ts +0 -59
- package/dist/types/content/ArtifactStore.d.ts +0 -223
- package/dist/types/content/ContentStore.d.ts +0 -140
- package/dist/types/content/contentAnalyzer.d.ts +0 -38
- package/dist/types/content/index.d.ts +0 -24
- package/dist/types/content/mcpAutoCache.d.ts +0 -89
- package/dist/types/content/types.d.ts +0 -75
- package/dist/types/graphs/HandoffRegistry.d.ts +0 -97
- package/dist/types/providers/a2a/A2ACapabilityProvider.d.ts +0 -89
- package/dist/types/providers/a2a/client.d.ts +0 -47
- package/dist/types/providers/a2a/config.d.ts +0 -18
- package/dist/types/providers/a2a/index.d.ts +0 -6
- package/dist/types/providers/a2a/types.d.ts +0 -173
- package/dist/types/providers/capabilityNaming.d.ts +0 -25
- package/dist/types/providers/index.d.ts +0 -12
- package/dist/types/providers/mcp/MCPCapabilityProvider.d.ts +0 -54
- package/dist/types/providers/mcp/config.d.ts +0 -20
- package/dist/types/providers/mcp/index.d.ts +0 -5
- package/dist/types/providers/mcp/transport.d.ts +0 -18
- package/dist/types/providers/mcp/types.d.ts +0 -112
- package/dist/types/providers/tools-server/ToolsServerCapabilityProvider.d.ts +0 -59
- package/dist/types/providers/tools-server/index.d.ts +0 -1
- package/dist/types/providers/types.d.ts +0 -184
- package/dist/types/tools/artifacts/index.d.ts +0 -3
- package/dist/types/tools/artifacts/schema.d.ts +0 -63
- package/dist/types/tools/artifacts/tool.d.ts +0 -16
- package/dist/types/tools/artifacts/types.d.ts +0 -127
- package/dist/types/tools/fileSearch/formatter.d.ts +0 -25
- package/dist/types/tools/fileSearch/index.d.ts +0 -5
- package/dist/types/tools/fileSearch/ragClient.d.ts +0 -32
- package/dist/types/tools/fileSearch/schema.d.ts +0 -13
- package/dist/types/tools/fileSearch/tool.d.ts +0 -18
- package/dist/types/tools/fileSearch/types.d.ts +0 -139
- package/dist/types/tools/proxyTool.d.ts +0 -62
- package/dist/types/tools/search/test.d.ts +0 -1
- package/dist/types/utils/childAgentContext.d.ts +0 -99
- package/dist/types/utils/credentials.d.ts +0 -77
- package/dist/types/utils/httpClient.d.ts +0 -46
- package/dist/types/utils/toolManifest.d.ts +0 -49
- package/src/common/__tests__/spawnPath.test.ts +0 -110
- package/src/common/spawnPath.ts +0 -101
- package/src/content/ArtifactStore.ts +0 -782
- package/src/content/ContentStore.ts +0 -753
- package/src/content/contentAnalyzer.ts +0 -105
- package/src/content/index.ts +0 -51
- package/src/content/mcpAutoCache.ts +0 -185
- package/src/content/types.ts +0 -82
- package/src/graphs/HandoffRegistry.ts +0 -199
- package/src/graphs/__tests__/HandoffRegistry.test.ts +0 -410
- package/src/graphs/__tests__/multi-agent-delegate.test.ts +0 -458
- package/src/graphs/__tests__/multi-agent-edges.test.ts +0 -276
- package/src/graphs/__tests__/multi-agent-nested-subgraph.test.ts +0 -221
- package/src/graphs/handoffValidation.test.ts +0 -353
- package/src/providers/__tests__/ToolsServerCapabilityProvider.integration.spec.ts +0 -79
- package/src/providers/__tests__/ToolsServerCapabilityProvider.test.ts +0 -271
- package/src/providers/__tests__/types.test.ts +0 -64
- package/src/providers/a2a/A2ACapabilityProvider.ts +0 -384
- package/src/providers/a2a/__tests__/A2ACapabilityProvider.integration.spec.ts +0 -345
- package/src/providers/a2a/__tests__/A2ACapabilityProvider.test.ts +0 -460
- package/src/providers/a2a/client.ts +0 -115
- package/src/providers/a2a/config.ts +0 -40
- package/src/providers/a2a/index.ts +0 -29
- package/src/providers/a2a/types.ts +0 -191
- package/src/providers/capabilityNaming.ts +0 -42
- package/src/providers/index.ts +0 -68
- package/src/providers/mcp/MCPCapabilityProvider.ts +0 -345
- package/src/providers/mcp/__tests__/MCPCapabilityProvider.integration.spec.ts +0 -386
- package/src/providers/mcp/__tests__/MCPCapabilityProvider.test.ts +0 -371
- package/src/providers/mcp/config.ts +0 -45
- package/src/providers/mcp/index.ts +0 -21
- package/src/providers/mcp/transport.ts +0 -76
- package/src/providers/mcp/types.ts +0 -139
- package/src/providers/tools-server/ToolsServerCapabilityProvider.ts +0 -249
- package/src/providers/tools-server/index.ts +0 -1
- package/src/providers/types.ts +0 -204
- package/src/scripts/test-bedrock-handoff-autonomous.ts +0 -267
- package/src/scripts/test-handoff-preamble.ts +0 -278
- package/src/specs/agent-handoffs-bedrock.integration.test.ts +0 -415
- package/src/tools/artifacts/__tests__/tool.test.ts +0 -259
- package/src/tools/artifacts/index.ts +0 -33
- package/src/tools/artifacts/schema.ts +0 -99
- package/src/tools/artifacts/tool.ts +0 -289
- package/src/tools/artifacts/types.ts +0 -162
- package/src/tools/fileSearch/__tests__/tool.test.ts +0 -261
- package/src/tools/fileSearch/formatter.ts +0 -129
- package/src/tools/fileSearch/index.ts +0 -23
- package/src/tools/fileSearch/ragClient.ts +0 -137
- package/src/tools/fileSearch/schema.ts +0 -19
- package/src/tools/fileSearch/tool.ts +0 -207
- package/src/tools/fileSearch/types.ts +0 -149
- package/src/tools/proxyTool.ts +0 -166
- package/src/tools/search/output.md +0 -2775
- package/src/tools/search/test.html +0 -884
- package/src/tools/search/test.md +0 -643
- package/src/tools/search/test.ts +0 -159
- package/src/utils/__tests__/childAgentContext.test.ts +0 -217
- package/src/utils/__tests__/credentials.test.ts +0 -130
- package/src/utils/__tests__/httpClient.test.ts +0 -75
- package/src/utils/__tests__/toolManifest.test.ts +0 -116
- package/src/utils/childAgentContext.ts +0 -259
- package/src/utils/credentials.ts +0 -157
- package/src/utils/httpClient.ts +0 -92
- package/src/utils/toolManifest.ts +0 -109
- /package/dist/esm/{content → langchain}/index.mjs.map +0 -0
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"shared.cjs","sources":["../../../../src/tools/memory/shared.ts"],"sourcesContent":["/**\n * Shared Zod schemas + helpers for the memory tool family.\n *\n * The tool schemas deliberately do NOT expose `agent_id` or `user_id`. Scope\n * is captured in a closure at tool construction time by {@link buildMemoryTools},\n * so even a hallucinated tool call with extra fields cannot influence it.\n */\nimport { z } from 'zod';\nimport {\n DEFAULT_MAX_INJECTED_CHARS,\n MEMORY_PATH_PREFIX,\n} from '@/memory/constants';\nimport type {\n MemoryAppendInput,\n MemoryBackend,\n MemoryEntry,\n MemoryScope,\n} from '@/memory/types';\n\nexport const MemorySearchSchema = z.object({\n query: z.string().describe('Natural-language query to search memory for'),\n maxResults: z\n .number()\n .int()\n .min(1)\n .max(50)\n .optional()\n .describe('Maximum number of results to return (default 10)'),\n minScore: z\n .number()\n .min(0)\n .max(1)\n .optional()\n .describe('Minimum hybrid score (0..1) below which results are dropped'),\n});\n\nexport const MemoryGetSchema = z.object({\n path: z\n .string()\n .describe(\n 'Memory path, e.g. \"memory/decisions.md\". Must start with \"memory/\"'\n ),\n from: z\n .number()\n .int()\n .min(1)\n .optional()\n .describe('Starting line number (1-indexed)'),\n lines: z\n .number()\n .int()\n .min(1)\n .optional()\n .describe('Number of lines to read starting at `from`'),\n});\n\nexport const MemoryAppendSchema = z.object({\n path: z\n .string()\n .describe(\n 'Memory path to append to (e.g. \"memory/decisions.md\"). Must start with \"memory/\"'\n ),\n content: z\n .string()\n .min(1)\n .describe(\n \"Note content in the agent's own voice. Markdown allowed. Each call appends an immutable entry.\"\n ),\n});\n\nexport type MemorySearchArgs = z.infer<typeof MemorySearchSchema>;\nexport type MemoryGetArgs = z.infer<typeof MemoryGetSchema>;\nexport type MemoryAppendArgs = z.infer<typeof MemoryAppendSchema>;\n\n/** Shape returned by `memory_search` — serialised as the tool result string. */\nexport interface MemorySearchToolResult {\n results: Array<\n Pick<MemoryEntry, 'id' | 'path' | 'content' | 'score' | 'createdAt'> & {\n citation?: string;\n }\n >;\n disabled?: boolean;\n unavailable?: boolean;\n error?: string;\n warning?: string;\n action?: string;\n}\n\n/** Shape returned by `memory_get`. */\nexport interface MemoryGetToolResult {\n path: string;\n text: string;\n disabled?: boolean;\n error?: string;\n}\n\nexport function buildMemorySearchUnavailableResult(\n error: string | undefined\n): MemorySearchToolResult {\n const reason =\n (error ?? 'memory search unavailable').trim() ||\n 'memory search unavailable';\n const isQuota = /insufficient_quota|quota|429/i.test(reason);\n return {\n results: [],\n disabled: true,\n unavailable: true,\n error: reason,\n warning: isQuota\n ? 'Memory search is unavailable because the embedding provider quota is exhausted.'\n : 'Memory search is unavailable due to an embedding/provider error.',\n action: isQuota\n ? 'Top up or switch embedding provider, then retry memory_search.'\n : 'Check embedding provider configuration and retry memory_search.',\n };\n}\n\n/**\n * Clamp a ranked result list to a total character budget.\n * Ported from
|
|
1
|
+
{"version":3,"file":"shared.cjs","sources":["../../../../src/tools/memory/shared.ts"],"sourcesContent":["/**\n * Shared Zod schemas + helpers for the memory tool family.\n *\n * The tool schemas deliberately do NOT expose `agent_id` or `user_id`. Scope\n * is captured in a closure at tool construction time by {@link buildMemoryTools},\n * so even a hallucinated tool call with extra fields cannot influence it.\n */\nimport { z } from 'zod';\nimport {\n DEFAULT_MAX_INJECTED_CHARS,\n MEMORY_PATH_PREFIX,\n} from '@/memory/constants';\nimport type {\n MemoryAppendInput,\n MemoryBackend,\n MemoryEntry,\n MemoryScope,\n} from '@/memory/types';\n\nexport const MemorySearchSchema = z.object({\n query: z.string().describe('Natural-language query to search memory for'),\n maxResults: z\n .number()\n .int()\n .min(1)\n .max(50)\n .optional()\n .describe('Maximum number of results to return (default 10)'),\n minScore: z\n .number()\n .min(0)\n .max(1)\n .optional()\n .describe('Minimum hybrid score (0..1) below which results are dropped'),\n});\n\nexport const MemoryGetSchema = z.object({\n path: z\n .string()\n .describe(\n 'Memory path, e.g. \"memory/decisions.md\". Must start with \"memory/\"'\n ),\n from: z\n .number()\n .int()\n .min(1)\n .optional()\n .describe('Starting line number (1-indexed)'),\n lines: z\n .number()\n .int()\n .min(1)\n .optional()\n .describe('Number of lines to read starting at `from`'),\n});\n\nexport const MemoryAppendSchema = z.object({\n path: z\n .string()\n .describe(\n 'Memory path to append to (e.g. \"memory/decisions.md\"). Must start with \"memory/\"'\n ),\n content: z\n .string()\n .min(1)\n .describe(\n \"Note content in the agent's own voice. Markdown allowed. Each call appends an immutable entry.\"\n ),\n});\n\nexport type MemorySearchArgs = z.infer<typeof MemorySearchSchema>;\nexport type MemoryGetArgs = z.infer<typeof MemoryGetSchema>;\nexport type MemoryAppendArgs = z.infer<typeof MemoryAppendSchema>;\n\n/** Shape returned by `memory_search` — serialised as the tool result string. */\nexport interface MemorySearchToolResult {\n results: Array<\n Pick<MemoryEntry, 'id' | 'path' | 'content' | 'score' | 'createdAt'> & {\n citation?: string;\n }\n >;\n disabled?: boolean;\n unavailable?: boolean;\n error?: string;\n warning?: string;\n action?: string;\n}\n\n/** Shape returned by `memory_get`. */\nexport interface MemoryGetToolResult {\n path: string;\n text: string;\n disabled?: boolean;\n error?: string;\n}\n\nexport function buildMemorySearchUnavailableResult(\n error: string | undefined\n): MemorySearchToolResult {\n const reason =\n (error ?? 'memory search unavailable').trim() ||\n 'memory search unavailable';\n const isQuota = /insufficient_quota|quota|429/i.test(reason);\n return {\n results: [],\n disabled: true,\n unavailable: true,\n error: reason,\n warning: isQuota\n ? 'Memory search is unavailable because the embedding provider quota is exhausted.'\n : 'Memory search is unavailable due to an embedding/provider error.',\n action: isQuota\n ? 'Top up or switch embedding provider, then retry memory_search.'\n : 'Check embedding provider configuration and retry memory_search.',\n };\n}\n\n/**\n * Clamp a ranked result list to a total character budget.\n * Ported from a reference implementation `tools.citations.ts::clampResultsByInjectedChars`.\n */\nexport function clampResultsByInjectedChars<T extends { content: string }>(\n results: T[],\n maxInjectedChars: number = DEFAULT_MAX_INJECTED_CHARS\n): T[] {\n const out: T[] = [];\n let total = 0;\n for (const result of results) {\n const size = result.content.length ?? 0;\n if (total + size > maxInjectedChars && out.length > 0) break;\n out.push(result);\n total += size;\n }\n return out;\n}\n\nexport interface MemoryToolBinding {\n backend: MemoryBackend;\n scope: MemoryScope;\n maxInjectedChars?: number;\n /** Phase 2 — per-call search options propagated into backend.search. */\n searchOptions?: {\n mmr?: { enabled?: boolean; lambda?: number };\n temporalDecay?: { enabled?: boolean; halfLifeDays?: number };\n citations?: 'on' | 'off' | 'auto';\n };\n /** Phase 2 — optional best-effort recall recorder. */\n recallTracker?: {\n record(params: {\n agentId: string;\n query: string;\n hits: Array<{ id: string; path: string; score: number }>;\n }): Promise<void>;\n };\n}\n\nexport function assertAppendAllowed(path: string): void {\n if (!path || !path.startsWith(MEMORY_PATH_PREFIX)) {\n throw new Error(\n `memory_append path must start with \"${MEMORY_PATH_PREFIX}\"`\n );\n }\n}\n\n/** Normalise an append call into the backend input shape. */\nexport function toAppendInput(args: MemoryAppendArgs): MemoryAppendInput {\n assertAppendAllowed(args.path);\n return { path: args.path, content: args.content };\n}\n"],"names":["z","DEFAULT_MAX_INJECTED_CHARS","MEMORY_PATH_PREFIX"],"mappings":";;;;;AAAA;;;;;;AAMG;AAaI,MAAM,kBAAkB,GAAGA,KAAC,CAAC,MAAM,CAAC;IACzC,KAAK,EAAEA,KAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,6CAA6C,CAAC;AACzE,IAAA,UAAU,EAAEA;AACT,SAAA,MAAM;AACN,SAAA,GAAG;SACH,GAAG,CAAC,CAAC;SACL,GAAG,CAAC,EAAE;AACN,SAAA,QAAQ;SACR,QAAQ,CAAC,kDAAkD,CAAC;AAC/D,IAAA,QAAQ,EAAEA;AACP,SAAA,MAAM;SACN,GAAG,CAAC,CAAC;SACL,GAAG,CAAC,CAAC;AACL,SAAA,QAAQ;SACR,QAAQ,CAAC,6DAA6D,CAAC;AAC3E,CAAA;AAEM,MAAM,eAAe,GAAGA,KAAC,CAAC,MAAM,CAAC;AACtC,IAAA,IAAI,EAAEA;AACH,SAAA,MAAM;SACN,QAAQ,CACP,oEAAoE,CACrE;AACH,IAAA,IAAI,EAAEA;AACH,SAAA,MAAM;AACN,SAAA,GAAG;SACH,GAAG,CAAC,CAAC;AACL,SAAA,QAAQ;SACR,QAAQ,CAAC,kCAAkC,CAAC;AAC/C,IAAA,KAAK,EAAEA;AACJ,SAAA,MAAM;AACN,SAAA,GAAG;SACH,GAAG,CAAC,CAAC;AACL,SAAA,QAAQ;SACR,QAAQ,CAAC,4CAA4C,CAAC;AAC1D,CAAA;AAEM,MAAM,kBAAkB,GAAGA,KAAC,CAAC,MAAM,CAAC;AACzC,IAAA,IAAI,EAAEA;AACH,SAAA,MAAM;SACN,QAAQ,CACP,kFAAkF,CACnF;AACH,IAAA,OAAO,EAAEA;AACN,SAAA,MAAM;SACN,GAAG,CAAC,CAAC;SACL,QAAQ,CACP,gGAAgG,CACjG;AACJ,CAAA;AA4BK,SAAU,kCAAkC,CAChD,KAAyB,EAAA;IAEzB,MAAM,MAAM,GACV,CAAC,KAAK,IAAI,2BAA2B,EAAE,IAAI,EAAE;AAC7C,QAAA,2BAA2B;IAC7B,MAAM,OAAO,GAAG,+BAA+B,CAAC,IAAI,CAAC,MAAM,CAAC;IAC5D,OAAO;AACL,QAAA,OAAO,EAAE,EAAE;AACX,QAAA,QAAQ,EAAE,IAAI;AACd,QAAA,WAAW,EAAE,IAAI;AACjB,QAAA,KAAK,EAAE,MAAM;AACb,QAAA,OAAO,EAAE;AACP,cAAE;AACF,cAAE,kEAAkE;AACtE,QAAA,MAAM,EAAE;AACN,cAAE;AACF,cAAE,iEAAiE;KACtE;AACH;AAEA;;;AAGG;SACa,2BAA2B,CACzC,OAAY,EACZ,mBAA2BC,oCAA0B,EAAA;IAErD,MAAM,GAAG,GAAQ,EAAE;IACnB,IAAI,KAAK,GAAG,CAAC;AACb,IAAA,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE;QAC5B,MAAM,IAAI,GAAG,MAAM,CAAC,OAAO,CAAC,MAAM,IAAI,CAAC;QACvC,IAAI,KAAK,GAAG,IAAI,GAAG,gBAAgB,IAAI,GAAG,CAAC,MAAM,GAAG,CAAC;YAAE;AACvD,QAAA,GAAG,CAAC,IAAI,CAAC,MAAM,CAAC;QAChB,KAAK,IAAI,IAAI;IACf;AACA,IAAA,OAAO,GAAG;AACZ;AAsBM,SAAU,mBAAmB,CAAC,IAAY,EAAA;IAC9C,IAAI,CAAC,IAAI,IAAI,CAAC,IAAI,CAAC,UAAU,CAACC,4BAAkB,CAAC,EAAE;AACjD,QAAA,MAAM,IAAI,KAAK,CACb,uCAAuCA,4BAAkB,CAAA,CAAA,CAAG,CAC7D;IACH;AACF;AAEA;AACM,SAAU,aAAa,CAAC,IAAsB,EAAA;AAClD,IAAA,mBAAmB,CAAC,IAAI,CAAC,IAAI,CAAC;AAC9B,IAAA,OAAO,EAAE,IAAI,EAAE,IAAI,CAAC,IAAI,EAAE,OAAO,EAAE,IAAI,CAAC,OAAO,EAAE;AACnD;;;;;;;;;;"}
|
|
@@ -3,6 +3,7 @@
|
|
|
3
3
|
var axios = require('axios');
|
|
4
4
|
var textsplitters = require('@langchain/textsplitters');
|
|
5
5
|
var utils = require('./utils.cjs');
|
|
6
|
+
var tavilySearch = require('./tavily-search.cjs');
|
|
6
7
|
|
|
7
8
|
const chunker = {
|
|
8
9
|
cleanText: (text) => {
|
|
@@ -323,15 +324,18 @@ const createSearXNGAPI = (instanceUrl, apiKey) => {
|
|
|
323
324
|
return { getSources };
|
|
324
325
|
};
|
|
325
326
|
const createSearchAPI = (config) => {
|
|
326
|
-
const { searchProvider = 'serper', serperApiKey, searxngInstanceUrl, searxngApiKey, domainBlocklist, countryBlocklist, } = config;
|
|
327
|
+
const { searchProvider = 'serper', serperApiKey, searxngInstanceUrl, searxngApiKey, tavilyApiKey, tavilySearchUrl, tavilySearchOptions, domainBlocklist, countryBlocklist, } = config;
|
|
327
328
|
if (searchProvider.toLowerCase() === 'serper') {
|
|
328
329
|
return createSerperAPI(serperApiKey, domainBlocklist, countryBlocklist);
|
|
329
330
|
}
|
|
330
331
|
else if (searchProvider.toLowerCase() === 'searxng') {
|
|
331
332
|
return createSearXNGAPI(searxngInstanceUrl, searxngApiKey);
|
|
332
333
|
}
|
|
334
|
+
else if (searchProvider.toLowerCase() === 'tavily') {
|
|
335
|
+
return tavilySearch.createTavilyAPI(tavilyApiKey, tavilySearchUrl, tavilySearchOptions);
|
|
336
|
+
}
|
|
333
337
|
else {
|
|
334
|
-
throw new Error(`Invalid search provider: ${searchProvider}. Must be 'serper' or '
|
|
338
|
+
throw new Error(`Invalid search provider: ${searchProvider}. Must be 'serper', 'searxng', or 'tavily'`);
|
|
335
339
|
}
|
|
336
340
|
};
|
|
337
341
|
const createSourceProcessor = (config = {}, scraperInstance) => {
|
|
@@ -354,7 +358,11 @@ const createSourceProcessor = (config = {}, scraperInstance) => {
|
|
|
354
358
|
const promise = scraper
|
|
355
359
|
.scrapeUrl(currentLink, {})
|
|
356
360
|
.then(([url, response]) => {
|
|
357
|
-
|
|
361
|
+
/* metadata exists on Firecrawl/Serper responses but not on
|
|
362
|
+
* Tavily's. Read it via the scraper's extractMetadata abstraction
|
|
363
|
+
* to stay polymorphic across all scraper providers. */
|
|
364
|
+
const rawMeta = scraper.extractMetadata(response);
|
|
365
|
+
const attribution = utils.getAttribution(url, Object.keys(rawMeta).length > 0 ? rawMeta : undefined, logger_);
|
|
358
366
|
if (response.success && response.data) {
|
|
359
367
|
const [content, references] = scraper.extractContent(response);
|
|
360
368
|
return {
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"search.cjs","sources":["../../../../src/tools/search/search.ts"],"sourcesContent":["import axios from 'axios';\nimport { RecursiveCharacterTextSplitter } from '@langchain/textsplitters';\nimport type * as t from './types';\nimport { getAttribution, createDefaultLogger } from './utils';\nimport { BaseReranker } from './rerankers';\n\nconst chunker = {\n cleanText: (text: string): string => {\n if (!text) return '';\n\n /** Normalized all line endings to '\\n' */\n const normalizedText = text.replace(/\\r\\n/g, '\\n').replace(/\\r/g, '\\n');\n\n /** Handle multiple backslashes followed by newlines\n * This replaces patterns like '\\\\\\\\\\\\n' with a single newline */\n const fixedBackslashes = normalizedText.replace(/\\\\+\\n/g, '\\n');\n\n /** Cleaned up consecutive newlines, tabs, and spaces around newlines */\n const cleanedNewlines = fixedBackslashes.replace(/[\\t ]*\\n[\\t \\n]*/g, '\\n');\n\n /** Cleaned up excessive spaces and tabs */\n const cleanedSpaces = cleanedNewlines.replace(/[ \\t]+/g, ' ');\n\n return cleanedSpaces.trim();\n },\n splitText: async (\n text: string,\n options?: {\n chunkSize?: number;\n chunkOverlap?: number;\n separators?: string[];\n }\n ): Promise<string[]> => {\n const chunkSize = options?.chunkSize ?? 150;\n const chunkOverlap = options?.chunkOverlap ?? 50;\n const separators = options?.separators || ['\\n\\n', '\\n'];\n\n const splitter = new RecursiveCharacterTextSplitter({\n separators,\n chunkSize,\n chunkOverlap,\n });\n\n return await splitter.splitText(text);\n },\n\n splitTexts: async (\n texts: string[],\n options?: {\n chunkSize?: number;\n chunkOverlap?: number;\n separators?: string[];\n },\n logger?: t.Logger\n ): Promise<string[][]> => {\n // Split multiple texts\n const logger_ = logger || createDefaultLogger();\n const promises = texts.map((text) =>\n chunker.splitText(text, options).catch((error) => {\n logger_.error('Error splitting text:', error);\n return [text];\n })\n );\n return Promise.all(promises);\n },\n};\n\nfunction createSourceUpdateCallback(sourceMap: Map<string, t.ValidSource>) {\n return (link: string, update?: Partial<t.ValidSource>): void => {\n const source = sourceMap.get(link);\n if (source) {\n sourceMap.set(link, {\n ...source,\n ...update,\n });\n }\n };\n}\n\nconst getHighlights = async ({\n query,\n content,\n reranker,\n topResults = 5,\n logger,\n}: {\n content: string;\n query: string;\n reranker?: BaseReranker;\n topResults?: number;\n logger?: t.Logger;\n}): Promise<t.Highlight[] | undefined> => {\n const logger_ = logger || createDefaultLogger();\n\n if (!content) {\n logger_.warn('No content provided for highlights');\n return;\n }\n if (!reranker) {\n logger_.warn('No reranker provided for highlights');\n return;\n }\n\n try {\n const documents = await chunker.splitText(content);\n if (Array.isArray(documents)) {\n return await reranker.rerank(query, documents, topResults);\n } else {\n logger_.error(\n 'Expected documents to be an array, got:',\n typeof documents\n );\n return;\n }\n } catch (error) {\n logger_.error('Error in content processing:', error);\n return;\n }\n};\n\nconst createSerperAPI = (\n apiKey?: string,\n domainBlocklist?: string[],\n countryBlocklist?: string[]\n): {\n getSources: (params: t.GetSourcesParams) => Promise<t.SearchResult>;\n} => {\n const config = {\n apiKey: apiKey ?? process.env.SERPER_API_KEY,\n apiUrl: 'https://google.serper.dev/search',\n timeout: 10000,\n };\n\n if (config.apiKey == null || config.apiKey === '') {\n throw new Error('SERPER_API_KEY is required for SerperAPI');\n }\n\n const getSources = async ({\n query,\n date,\n country,\n safeSearch,\n numResults = 8,\n type,\n }: t.GetSourcesParams): Promise<t.SearchResult> => {\n if (!query.trim()) {\n return { success: false, error: 'Query cannot be empty' };\n }\n\n try {\n const safe = ['off', 'moderate', 'active'] as const;\n\n // Append -site: operators for blocked domains so they never appear in results\n let effectiveQuery = query;\n if (domainBlocklist && domainBlocklist.length > 0) {\n const siteExclusions = domainBlocklist\n .map((d) => `-site:${d}`)\n .join(' ');\n effectiveQuery = `${effectiveQuery} ${siteExclusions}`;\n }\n\n // Append -site:.XX operators for blocked country TLDs\n if (countryBlocklist && countryBlocklist.length > 0) {\n const countryExclusions = countryBlocklist\n .map((c) => `-site:.${c.toLowerCase().replace(/^\\./, '')}`)\n .join(' ');\n effectiveQuery = `${effectiveQuery} ${countryExclusions}`;\n }\n\n const payload: t.SerperSearchPayload = {\n q: effectiveQuery,\n safe: safe[safeSearch ?? 1],\n num: Math.min(Math.max(1, numResults), 10),\n };\n\n // Set the search type if provided\n if (type) {\n payload.type = type;\n }\n\n if (date != null) {\n payload.tbs = `qdr:${date}`;\n }\n\n if (country != null && country !== '') {\n payload['gl'] = country.toLowerCase();\n }\n\n // Determine the API endpoint based on the search type\n let apiEndpoint = config.apiUrl;\n if (type === 'images') {\n apiEndpoint = 'https://google.serper.dev/images';\n } else if (type === 'videos') {\n apiEndpoint = 'https://google.serper.dev/videos';\n } else if (type === 'news') {\n apiEndpoint = 'https://google.serper.dev/news';\n }\n\n const response = await axios.post<t.SerperResultData>(\n apiEndpoint,\n payload,\n {\n headers: {\n 'X-API-KEY': config.apiKey,\n 'Content-Type': 'application/json',\n },\n timeout: config.timeout,\n }\n );\n\n const data = response.data;\n const results: t.SearchResultData = {\n organic: data.organic,\n images: data.images ?? [],\n answerBox: data.answerBox,\n topStories: data.topStories ?? [],\n peopleAlsoAsk: data.peopleAlsoAsk,\n knowledgeGraph: data.knowledgeGraph,\n relatedSearches: data.relatedSearches,\n videos: data.videos ?? [],\n news: data.news ?? [],\n };\n\n return { success: true, data: results };\n } catch (error) {\n const errorMessage =\n error instanceof Error ? error.message : String(error);\n return { success: false, error: `API request failed: ${errorMessage}` };\n }\n };\n\n return { getSources };\n};\n\nconst createSearXNGAPI = (\n instanceUrl?: string,\n apiKey?: string\n): {\n getSources: (params: t.GetSourcesParams) => Promise<t.SearchResult>;\n} => {\n const config = {\n instanceUrl: instanceUrl ?? process.env.SEARXNG_INSTANCE_URL,\n apiKey: apiKey ?? process.env.SEARXNG_API_KEY,\n defaultLocation: 'all',\n timeout: 10000,\n };\n\n if (config.instanceUrl == null || config.instanceUrl === '') {\n throw new Error('SEARXNG_INSTANCE_URL is required for SearXNG API');\n }\n\n const getSources = async ({\n query,\n numResults = 8,\n safeSearch,\n type,\n }: t.GetSourcesParams): Promise<t.SearchResult> => {\n if (!query.trim()) {\n return { success: false, error: 'Query cannot be empty' };\n }\n\n try {\n // Ensure the instance URL ends with /search\n if (config.instanceUrl == null || config.instanceUrl === '') {\n return { success: false, error: 'Instance URL is not defined' };\n }\n\n let searchUrl = config.instanceUrl;\n if (!searchUrl.endsWith('/search')) {\n searchUrl = searchUrl.replace(/\\/$/, '') + '/search';\n }\n\n // Determine the search category based on the type\n let category = 'general';\n if (type === 'images') {\n category = 'images';\n } else if (type === 'videos') {\n category = 'videos';\n } else if (type === 'news') {\n category = 'news';\n }\n\n // Prepare parameters for SearXNG\n const params: t.SearxNGSearchPayload = {\n q: query,\n format: 'json',\n pageno: 1,\n categories: category,\n language: 'all',\n safesearch: safeSearch,\n engines: 'google,bing,duckduckgo',\n };\n\n const headers: Record<string, string> = {\n 'Content-Type': 'application/json',\n };\n\n if (config.apiKey != null && config.apiKey !== '') {\n headers['X-API-Key'] = config.apiKey;\n }\n\n const response = await axios.get(searchUrl, {\n headers,\n params,\n timeout: config.timeout,\n });\n\n const data = response.data;\n\n // Helper function to identify news results since SearXNG doesn't provide that classification by default\n const isNewsResult = (result: t.SearXNGResult): boolean => {\n const url = result.url?.toLowerCase() ?? '';\n const title = result.title?.toLowerCase() ?? '';\n\n // News-related keywords in title/content\n const newsKeywords = [\n 'breaking news',\n 'latest news',\n 'top stories',\n 'news today',\n 'developing story',\n 'trending news',\n 'news',\n ];\n\n // Check if title/content contains news keywords\n const hasNewsKeywords = newsKeywords.some(\n (keyword) => title.toLowerCase().includes(keyword) // just title probably fine, content parsing is overkill for what we need: || content.includes(keyword)\n );\n\n // Check if URL contains news-related paths\n const hasNewsPath =\n url.includes('/news/') ||\n url.includes('/world/') ||\n url.includes('/politics/') ||\n url.includes('/breaking/');\n\n return hasNewsKeywords || hasNewsPath;\n };\n\n // Transform SearXNG results to match SerperAPI format\n const organicResults = (data.results ?? [])\n .slice(0, numResults)\n .map((result: t.SearXNGResult, index: number) => {\n let attribution = '';\n try {\n attribution = new URL(result.url ?? '').hostname;\n } catch {\n attribution = '';\n }\n\n return {\n position: index + 1,\n title: result.title ?? '',\n link: result.url ?? '',\n snippet: result.content ?? '',\n date: result.publishedDate ?? '',\n attribution,\n };\n });\n\n const imageResults = (data.results ?? [])\n .filter((result: t.SearXNGResult) => result.img_src)\n .slice(0, 6)\n .map((result: t.SearXNGResult, index: number) => ({\n title: result.title ?? '',\n imageUrl: result.img_src ?? '',\n position: index + 1,\n source: new URL(result.url ?? '').hostname,\n domain: new URL(result.url ?? '').hostname,\n link: result.url ?? '',\n }));\n\n // Extract news results from organic results\n const newsResults = (data.results ?? [])\n .filter(isNewsResult)\n .map((result: t.SearXNGResult, index: number) => {\n let attribution = '';\n try {\n attribution = new URL(result.url ?? '').hostname;\n } catch {\n attribution = '';\n }\n\n return {\n title: result.title ?? '',\n link: result.url ?? '',\n snippet: result.content ?? '',\n date: result.publishedDate ?? '',\n source: attribution,\n imageUrl: result.img_src ?? '',\n position: index + 1,\n };\n });\n\n const topStories = newsResults.slice(0, 5);\n\n const relatedSearches = Array.isArray(data.suggestions)\n ? data.suggestions.map((suggestion: string) => ({ query: suggestion }))\n : [];\n\n const results: t.SearchResultData = {\n organic: organicResults,\n images: imageResults,\n topStories: topStories, // Use first 5 extracted news as top stories\n relatedSearches,\n videos: [],\n news: newsResults,\n // Add empty arrays for other Serper fields to maintain parity\n places: [],\n shopping: [],\n peopleAlsoAsk: [],\n knowledgeGraph: undefined,\n answerBox: undefined,\n };\n\n return { success: true, data: results };\n } catch (error) {\n const errorMessage =\n error instanceof Error ? error.message : String(error);\n return {\n success: false,\n error: `SearXNG API request failed: ${errorMessage}`,\n };\n }\n };\n\n return { getSources };\n};\n\nexport const createSearchAPI = (\n config: t.SearchConfig\n): {\n getSources: (params: t.GetSourcesParams) => Promise<t.SearchResult>;\n} => {\n const {\n searchProvider = 'serper',\n serperApiKey,\n searxngInstanceUrl,\n searxngApiKey,\n domainBlocklist,\n countryBlocklist,\n } = config;\n\n if (searchProvider.toLowerCase() === 'serper') {\n return createSerperAPI(serperApiKey, domainBlocklist, countryBlocklist);\n } else if (searchProvider.toLowerCase() === 'searxng') {\n return createSearXNGAPI(searxngInstanceUrl, searxngApiKey);\n } else {\n throw new Error(\n `Invalid search provider: ${searchProvider}. Must be 'serper' or 'searxng'`\n );\n }\n};\n\nexport const createSourceProcessor = (\n config: t.ProcessSourcesConfig = {},\n scraperInstance?: t.BaseScraper\n): {\n processSources: (\n fields: t.ProcessSourcesFields\n ) => Promise<t.SearchResultData>;\n topResults: number;\n} => {\n if (!scraperInstance) {\n throw new Error('Scraper instance is required');\n }\n const {\n topResults = 5,\n // strategies = ['no_extraction'],\n // filterContent = true,\n reranker,\n logger,\n } = config;\n\n const logger_ = logger || createDefaultLogger();\n const scraper = scraperInstance;\n\n const webScraper = {\n scrapeMany: async ({\n query,\n links,\n onGetHighlights,\n }: {\n query: string;\n links: string[];\n onGetHighlights: t.SearchToolConfig['onGetHighlights'];\n }): Promise<Array<t.ScrapeResult>> => {\n logger_.debug(`Scraping ${links.length} links`);\n const promises: Array<Promise<t.ScrapeResult>> = [];\n try {\n for (let i = 0; i < links.length; i++) {\n const currentLink = links[i];\n const promise: Promise<t.ScrapeResult> = scraper\n .scrapeUrl(currentLink, {})\n .then(([url, response]) => {\n const attribution = getAttribution(\n url,\n response.data?.metadata,\n logger_\n );\n if (response.success && response.data) {\n const [content, references] = scraper.extractContent(response);\n return {\n url,\n references,\n attribution,\n content: chunker.cleanText(content),\n } as t.ScrapeResult;\n } else {\n logger_.error(\n `Error scraping ${url}: ${response.error ?? 'Unknown error'}`\n );\n }\n\n return {\n url,\n attribution,\n error: true,\n content: '',\n } as t.ScrapeResult;\n })\n .then(async (result) => {\n try {\n if (result.error != null) {\n logger_.error(\n `Error scraping ${result.url}: ${result.content}`\n );\n return {\n ...result,\n };\n }\n const highlights = await getHighlights({\n query,\n reranker,\n content: result.content,\n logger: logger_,\n });\n if (onGetHighlights) {\n onGetHighlights(result.url);\n }\n return {\n ...result,\n highlights,\n };\n } catch (error) {\n logger_.error('Error processing scraped content:', error);\n return {\n ...result,\n };\n }\n })\n .catch((error) => {\n logger_.error(`Error scraping ${currentLink}:`, error);\n return {\n url: currentLink,\n error: true,\n content: '',\n };\n });\n promises.push(promise);\n }\n return await Promise.all(promises);\n } catch (error) {\n logger_.error('Error in scrapeMany:', error);\n return [];\n }\n },\n };\n\n const fetchContents = async ({\n links,\n query,\n target,\n onGetHighlights,\n onContentScraped,\n }: {\n links: string[];\n query: string;\n target: number;\n onGetHighlights: t.SearchToolConfig['onGetHighlights'];\n onContentScraped?: (link: string, update?: Partial<t.ValidSource>) => void;\n }): Promise<void> => {\n const initialLinks = links.slice(0, target);\n // const remainingLinks = links.slice(target).reverse();\n const results = await webScraper.scrapeMany({\n query,\n links: initialLinks,\n onGetHighlights,\n });\n for (const result of results) {\n if (result.error === true) {\n continue;\n }\n const { url, content, attribution, references, highlights } = result;\n onContentScraped?.(url, {\n content,\n attribution,\n references,\n highlights,\n });\n }\n };\n\n const processSources = async ({\n result,\n numElements,\n query,\n news,\n proMode = true,\n onGetHighlights,\n skipScraping = false,\n }: t.ProcessSourcesFields): Promise<t.SearchResultData> => {\n try {\n if (!result.data) {\n return {\n organic: [],\n topStories: [],\n images: [],\n relatedSearches: [],\n };\n } else if (!result.data.organic) {\n return result.data;\n }\n\n // If content was already extracted directly (e.g., direct URL extraction), skip scraping\n if (skipScraping) {\n logger_.debug(\n 'Skipping additional scraping - content already extracted'\n );\n return result.data;\n }\n\n if (!proMode) {\n const wikiSources = result.data.organic.filter((source) =>\n source.link.includes('wikipedia.org')\n );\n\n if (!wikiSources.length) {\n return result.data;\n }\n\n const wikiSourceMap = new Map<string, t.ValidSource>();\n wikiSourceMap.set(wikiSources[0].link, wikiSources[0]);\n const onContentScraped = createSourceUpdateCallback(wikiSourceMap);\n await fetchContents({\n query,\n target: 1,\n onGetHighlights,\n onContentScraped,\n links: [wikiSources[0].link],\n });\n\n for (let i = 0; i < result.data.organic.length; i++) {\n const source = result.data.organic[i];\n const updatedSource = wikiSourceMap.get(source.link);\n if (updatedSource) {\n result.data.organic[i] = {\n ...source,\n ...updatedSource,\n };\n }\n }\n\n return result.data;\n }\n\n const sourceMap = new Map<string, t.ValidSource>();\n const organicLinksSet = new Set<string>();\n\n // Collect organic links\n const organicLinks = collectLinks(\n result.data.organic,\n sourceMap,\n organicLinksSet\n );\n\n // Collect top story links, excluding any that are already in organic links\n const topStories = result.data.topStories ?? [];\n const topStoryLinks = collectLinks(\n topStories,\n sourceMap,\n organicLinksSet\n );\n\n if (organicLinks.length === 0 && (topStoryLinks.length === 0 || !news)) {\n return result.data;\n }\n\n const onContentScraped = createSourceUpdateCallback(sourceMap);\n const promises: Promise<void>[] = [];\n\n // Process organic links\n if (organicLinks.length > 0) {\n promises.push(\n fetchContents({\n query,\n onGetHighlights,\n onContentScraped,\n links: organicLinks,\n target: numElements,\n })\n );\n }\n\n // Process top story links\n if (news && topStoryLinks.length > 0) {\n promises.push(\n fetchContents({\n query,\n onGetHighlights,\n onContentScraped,\n links: topStoryLinks,\n target: numElements,\n })\n );\n }\n\n await Promise.all(promises);\n\n if (result.data.organic.length > 0) {\n updateSourcesWithContent(result.data.organic, sourceMap);\n }\n\n if (news && topStories.length > 0) {\n updateSourcesWithContent(topStories, sourceMap);\n }\n\n return result.data;\n } catch (error) {\n logger_.error('Error in processSources:', error);\n return {\n organic: [],\n topStories: [],\n images: [],\n relatedSearches: [],\n ...result.data,\n error: error instanceof Error ? error.message : String(error),\n };\n }\n };\n\n return {\n processSources,\n topResults,\n };\n};\n\n/** Helper function to collect links and update sourceMap */\nfunction collectLinks(\n sources: Array<t.OrganicResult | t.TopStoryResult>,\n sourceMap: Map<string, t.ValidSource>,\n existingLinksSet?: Set<string>\n): string[] {\n const links: string[] = [];\n\n for (const source of sources) {\n if (source.link) {\n // For topStories, only add if not already in organic links\n if (existingLinksSet && existingLinksSet.has(source.link)) {\n continue;\n }\n\n links.push(source.link);\n if (existingLinksSet) {\n existingLinksSet.add(source.link);\n }\n sourceMap.set(source.link, source as t.ValidSource);\n }\n }\n\n return links;\n}\n\n/** Helper function to update sources with scraped content */\nfunction updateSourcesWithContent<T extends t.ValidSource>(\n sources: T[],\n sourceMap: Map<string, t.ValidSource>\n): void {\n for (let i = 0; i < sources.length; i++) {\n const source = sources[i];\n const updatedSource = sourceMap.get(source.link);\n if (updatedSource) {\n sources[i] = {\n ...source,\n ...updatedSource,\n } as T;\n }\n }\n}\n"],"names":["RecursiveCharacterTextSplitter","createDefaultLogger","getAttribution"],"mappings":";;;;;;AAMA,MAAM,OAAO,GAAG;AACd,IAAA,SAAS,EAAE,CAAC,IAAY,KAAY;AAClC,QAAA,IAAI,CAAC,IAAI;AAAE,YAAA,OAAO,EAAE;;AAGpB,QAAA,MAAM,cAAc,GAAG,IAAI,CAAC,OAAO,CAAC,OAAO,EAAE,IAAI,CAAC,CAAC,OAAO,CAAC,KAAK,EAAE,IAAI,CAAC;AAEvE;AACiE;QACjE,MAAM,gBAAgB,GAAG,cAAc,CAAC,OAAO,CAAC,QAAQ,EAAE,IAAI,CAAC;;QAG/D,MAAM,eAAe,GAAG,gBAAgB,CAAC,OAAO,CAAC,mBAAmB,EAAE,IAAI,CAAC;;QAG3E,MAAM,aAAa,GAAG,eAAe,CAAC,OAAO,CAAC,SAAS,EAAE,GAAG,CAAC;AAE7D,QAAA,OAAO,aAAa,CAAC,IAAI,EAAE;IAC7B,CAAC;AACD,IAAA,SAAS,EAAE,OACT,IAAY,EACZ,OAIC,KACoB;AACrB,QAAA,MAAM,SAAS,GAAG,OAAO,EAAE,SAAS,IAAI,GAAG;AAC3C,QAAA,MAAM,YAAY,GAAG,OAAO,EAAE,YAAY,IAAI,EAAE;QAChD,MAAM,UAAU,GAAG,OAAO,EAAE,UAAU,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC;AAExD,QAAA,MAAM,QAAQ,GAAG,IAAIA,4CAA8B,CAAC;YAClD,UAAU;YACV,SAAS;YACT,YAAY;AACb,SAAA,CAAC;AAEF,QAAA,OAAO,MAAM,QAAQ,CAAC,SAAS,CAAC,IAAI,CAAC;IACvC,CAAC;IAED,UAAU,EAAE,OACV,KAAe,EACf,OAIC,EACD,MAAiB,KACM;;AAEvB,QAAA,MAAM,OAAO,GAAG,MAAM,IAAIC,yBAAmB,EAAE;QAC/C,MAAM,QAAQ,GAAG,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,KAC9B,OAAO,CAAC,SAAS,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC,KAAK,CAAC,CAAC,KAAK,KAAI;AAC/C,YAAA,OAAO,CAAC,KAAK,CAAC,uBAAuB,EAAE,KAAK,CAAC;YAC7C,OAAO,CAAC,IAAI,CAAC;QACf,CAAC,CAAC,CACH;AACD,QAAA,OAAO,OAAO,CAAC,GAAG,CAAC,QAAQ,CAAC;IAC9B,CAAC;CACF;AAED,SAAS,0BAA0B,CAAC,SAAqC,EAAA;AACvE,IAAA,OAAO,CAAC,IAAY,EAAE,MAA+B,KAAU;QAC7D,MAAM,MAAM,GAAG,SAAS,CAAC,GAAG,CAAC,IAAI,CAAC;QAClC,IAAI,MAAM,EAAE;AACV,YAAA,SAAS,CAAC,GAAG,CAAC,IAAI,EAAE;AAClB,gBAAA,GAAG,MAAM;AACT,gBAAA,GAAG,MAAM;AACV,aAAA,CAAC;QACJ;AACF,IAAA,CAAC;AACH;AAEA,MAAM,aAAa,GAAG,OAAO,EAC3B,KAAK,EACL,OAAO,EACP,QAAQ,EACR,UAAU,GAAG,CAAC,EACd,MAAM,GAOP,KAAwC;AACvC,IAAA,MAAM,OAAO,GAAG,MAAM,IAAIA,yBAAmB,EAAE;IAE/C,IAAI,CAAC,OAAO,EAAE;AACZ,QAAA,OAAO,CAAC,IAAI,CAAC,oCAAoC,CAAC;QAClD;IACF;IACA,IAAI,CAAC,QAAQ,EAAE;AACb,QAAA,OAAO,CAAC,IAAI,CAAC,qCAAqC,CAAC;QACnD;IACF;AAEA,IAAA,IAAI;QACF,MAAM,SAAS,GAAG,MAAM,OAAO,CAAC,SAAS,CAAC,OAAO,CAAC;AAClD,QAAA,IAAI,KAAK,CAAC,OAAO,CAAC,SAAS,CAAC,EAAE;YAC5B,OAAO,MAAM,QAAQ,CAAC,MAAM,CAAC,KAAK,EAAE,SAAS,EAAE,UAAU,CAAC;QAC5D;aAAO;YACL,OAAO,CAAC,KAAK,CACX,yCAAyC,EACzC,OAAO,SAAS,CACjB;YACD;QACF;IACF;IAAE,OAAO,KAAK,EAAE;AACd,QAAA,OAAO,CAAC,KAAK,CAAC,8BAA8B,EAAE,KAAK,CAAC;QACpD;IACF;AACF,CAAC;AAED,MAAM,eAAe,GAAG,CACtB,MAAe,EACf,eAA0B,EAC1B,gBAA2B,KAGzB;AACF,IAAA,MAAM,MAAM,GAAG;AACb,QAAA,MAAM,EAAE,MAAM,IAAI,OAAO,CAAC,GAAG,CAAC,cAAc;AAC5C,QAAA,MAAM,EAAE,kCAAkC;AAC1C,QAAA,OAAO,EAAE,KAAK;KACf;AAED,IAAA,IAAI,MAAM,CAAC,MAAM,IAAI,IAAI,IAAI,MAAM,CAAC,MAAM,KAAK,EAAE,EAAE;AACjD,QAAA,MAAM,IAAI,KAAK,CAAC,0CAA0C,CAAC;IAC7D;IAEA,MAAM,UAAU,GAAG,OAAO,EACxB,KAAK,EACL,IAAI,EACJ,OAAO,EACP,UAAU,EACV,UAAU,GAAG,CAAC,EACd,IAAI,GACe,KAA6B;AAChD,QAAA,IAAI,CAAC,KAAK,CAAC,IAAI,EAAE,EAAE;YACjB,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,uBAAuB,EAAE;QAC3D;AAEA,QAAA,IAAI;YACF,MAAM,IAAI,GAAG,CAAC,KAAK,EAAE,UAAU,EAAE,QAAQ,CAAU;;YAGnD,IAAI,cAAc,GAAG,KAAK;YAC1B,IAAI,eAAe,IAAI,eAAe,CAAC,MAAM,GAAG,CAAC,EAAE;gBACjD,MAAM,cAAc,GAAG;qBACpB,GAAG,CAAC,CAAC,CAAC,KAAK,CAAA,MAAA,EAAS,CAAC,CAAA,CAAE;qBACvB,IAAI,CAAC,GAAG,CAAC;AACZ,gBAAA,cAAc,GAAG,CAAA,EAAG,cAAc,CAAA,CAAA,EAAI,cAAc,EAAE;YACxD;;YAGA,IAAI,gBAAgB,IAAI,gBAAgB,CAAC,MAAM,GAAG,CAAC,EAAE;gBACnD,MAAM,iBAAiB,GAAG;AACvB,qBAAA,GAAG,CAAC,CAAC,CAAC,KAAK,UAAU,CAAC,CAAC,WAAW,EAAE,CAAC,OAAO,CAAC,KAAK,EAAE,EAAE,CAAC,EAAE;qBACzD,IAAI,CAAC,GAAG,CAAC;AACZ,gBAAA,cAAc,GAAG,CAAA,EAAG,cAAc,CAAA,CAAA,EAAI,iBAAiB,EAAE;YAC3D;AAEA,YAAA,MAAM,OAAO,GAA0B;AACrC,gBAAA,CAAC,EAAE,cAAc;AACjB,gBAAA,IAAI,EAAE,IAAI,CAAC,UAAU,IAAI,CAAC,CAAC;AAC3B,gBAAA,GAAG,EAAE,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,UAAU,CAAC,EAAE,EAAE,CAAC;aAC3C;;YAGD,IAAI,IAAI,EAAE;AACR,gBAAA,OAAO,CAAC,IAAI,GAAG,IAAI;YACrB;AAEA,YAAA,IAAI,IAAI,IAAI,IAAI,EAAE;AAChB,gBAAA,OAAO,CAAC,GAAG,GAAG,CAAA,IAAA,EAAO,IAAI,EAAE;YAC7B;YAEA,IAAI,OAAO,IAAI,IAAI,IAAI,OAAO,KAAK,EAAE,EAAE;gBACrC,OAAO,CAAC,IAAI,CAAC,GAAG,OAAO,CAAC,WAAW,EAAE;YACvC;;AAGA,YAAA,IAAI,WAAW,GAAG,MAAM,CAAC,MAAM;AAC/B,YAAA,IAAI,IAAI,KAAK,QAAQ,EAAE;gBACrB,WAAW,GAAG,kCAAkC;YAClD;AAAO,iBAAA,IAAI,IAAI,KAAK,QAAQ,EAAE;gBAC5B,WAAW,GAAG,kCAAkC;YAClD;AAAO,iBAAA,IAAI,IAAI,KAAK,MAAM,EAAE;gBAC1B,WAAW,GAAG,gCAAgC;YAChD;YAEA,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,IAAI,CAC/B,WAAW,EACX,OAAO,EACP;AACE,gBAAA,OAAO,EAAE;oBACP,WAAW,EAAE,MAAM,CAAC,MAAM;AAC1B,oBAAA,cAAc,EAAE,kBAAkB;AACnC,iBAAA;gBACD,OAAO,EAAE,MAAM,CAAC,OAAO;AACxB,aAAA,CACF;AAED,YAAA,MAAM,IAAI,GAAG,QAAQ,CAAC,IAAI;AAC1B,YAAA,MAAM,OAAO,GAAuB;gBAClC,OAAO,EAAE,IAAI,CAAC,OAAO;AACrB,gBAAA,MAAM,EAAE,IAAI,CAAC,MAAM,IAAI,EAAE;gBACzB,SAAS,EAAE,IAAI,CAAC,SAAS;AACzB,gBAAA,UAAU,EAAE,IAAI,CAAC,UAAU,IAAI,EAAE;gBACjC,aAAa,EAAE,IAAI,CAAC,aAAa;gBACjC,cAAc,EAAE,IAAI,CAAC,cAAc;gBACnC,eAAe,EAAE,IAAI,CAAC,eAAe;AACrC,gBAAA,MAAM,EAAE,IAAI,CAAC,MAAM,IAAI,EAAE;AACzB,gBAAA,IAAI,EAAE,IAAI,CAAC,IAAI,IAAI,EAAE;aACtB;YAED,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,IAAI,EAAE,OAAO,EAAE;QACzC;QAAE,OAAO,KAAK,EAAE;AACd,YAAA,MAAM,YAAY,GAChB,KAAK,YAAY,KAAK,GAAG,KAAK,CAAC,OAAO,GAAG,MAAM,CAAC,KAAK,CAAC;YACxD,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,CAAA,oBAAA,EAAuB,YAAY,CAAA,CAAE,EAAE;QACzE;AACF,IAAA,CAAC;IAED,OAAO,EAAE,UAAU,EAAE;AACvB,CAAC;AAED,MAAM,gBAAgB,GAAG,CACvB,WAAoB,EACpB,MAAe,KAGb;AACF,IAAA,MAAM,MAAM,GAAG;AACb,QAAA,WAAW,EAAE,WAAW,IAAI,OAAO,CAAC,GAAG,CAAC,oBAAoB;AAC5D,QAAA,MAAM,EAAE,MAAM,IAAI,OAAO,CAAC,GAAG,CAAC,eAAe;AAC7C,QAAA,eAAe,EAAE,KAAK;AACtB,QAAA,OAAO,EAAE,KAAK;KACf;AAED,IAAA,IAAI,MAAM,CAAC,WAAW,IAAI,IAAI,IAAI,MAAM,CAAC,WAAW,KAAK,EAAE,EAAE;AAC3D,QAAA,MAAM,IAAI,KAAK,CAAC,kDAAkD,CAAC;IACrE;AAEA,IAAA,MAAM,UAAU,GAAG,OAAO,EACxB,KAAK,EACL,UAAU,GAAG,CAAC,EACd,UAAU,EACV,IAAI,GACe,KAA6B;AAChD,QAAA,IAAI,CAAC,KAAK,CAAC,IAAI,EAAE,EAAE;YACjB,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,uBAAuB,EAAE;QAC3D;AAEA,QAAA,IAAI;;AAEF,YAAA,IAAI,MAAM,CAAC,WAAW,IAAI,IAAI,IAAI,MAAM,CAAC,WAAW,KAAK,EAAE,EAAE;gBAC3D,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,6BAA6B,EAAE;YACjE;AAEA,YAAA,IAAI,SAAS,GAAG,MAAM,CAAC,WAAW;YAClC,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC,SAAS,CAAC,EAAE;gBAClC,SAAS,GAAG,SAAS,CAAC,OAAO,CAAC,KAAK,EAAE,EAAE,CAAC,GAAG,SAAS;YACtD;;YAGA,IAAI,QAAQ,GAAG,SAAS;AACxB,YAAA,IAAI,IAAI,KAAK,QAAQ,EAAE;gBACrB,QAAQ,GAAG,QAAQ;YACrB;AAAO,iBAAA,IAAI,IAAI,KAAK,QAAQ,EAAE;gBAC5B,QAAQ,GAAG,QAAQ;YACrB;AAAO,iBAAA,IAAI,IAAI,KAAK,MAAM,EAAE;gBAC1B,QAAQ,GAAG,MAAM;YACnB;;AAGA,YAAA,MAAM,MAAM,GAA2B;AACrC,gBAAA,CAAC,EAAE,KAAK;AACR,gBAAA,MAAM,EAAE,MAAM;AACd,gBAAA,MAAM,EAAE,CAAC;AACT,gBAAA,UAAU,EAAE,QAAQ;AACpB,gBAAA,QAAQ,EAAE,KAAK;AACf,gBAAA,UAAU,EAAE,UAAU;AACtB,gBAAA,OAAO,EAAE,wBAAwB;aAClC;AAED,YAAA,MAAM,OAAO,GAA2B;AACtC,gBAAA,cAAc,EAAE,kBAAkB;aACnC;AAED,YAAA,IAAI,MAAM,CAAC,MAAM,IAAI,IAAI,IAAI,MAAM,CAAC,MAAM,KAAK,EAAE,EAAE;AACjD,gBAAA,OAAO,CAAC,WAAW,CAAC,GAAG,MAAM,CAAC,MAAM;YACtC;YAEA,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,GAAG,CAAC,SAAS,EAAE;gBAC1C,OAAO;gBACP,MAAM;gBACN,OAAO,EAAE,MAAM,CAAC,OAAO;AACxB,aAAA,CAAC;AAEF,YAAA,MAAM,IAAI,GAAG,QAAQ,CAAC,IAAI;;AAG1B,YAAA,MAAM,YAAY,GAAG,CAAC,MAAuB,KAAa;gBACxD,MAAM,GAAG,GAAG,MAAM,CAAC,GAAG,EAAE,WAAW,EAAE,IAAI,EAAE;gBAC3C,MAAM,KAAK,GAAG,MAAM,CAAC,KAAK,EAAE,WAAW,EAAE,IAAI,EAAE;;AAG/C,gBAAA,MAAM,YAAY,GAAG;oBACnB,eAAe;oBACf,aAAa;oBACb,aAAa;oBACb,YAAY;oBACZ,kBAAkB;oBAClB,eAAe;oBACf,MAAM;iBACP;;gBAGD,MAAM,eAAe,GAAG,YAAY,CAAC,IAAI,CACvC,CAAC,OAAO,KAAK,KAAK,CAAC,WAAW,EAAE,CAAC,QAAQ,CAAC,OAAO,CAAC;iBACnD;;AAGD,gBAAA,MAAM,WAAW,GACf,GAAG,CAAC,QAAQ,CAAC,QAAQ,CAAC;AACtB,oBAAA,GAAG,CAAC,QAAQ,CAAC,SAAS,CAAC;AACvB,oBAAA,GAAG,CAAC,QAAQ,CAAC,YAAY,CAAC;AAC1B,oBAAA,GAAG,CAAC,QAAQ,CAAC,YAAY,CAAC;gBAE5B,OAAO,eAAe,IAAI,WAAW;AACvC,YAAA,CAAC;;YAGD,MAAM,cAAc,GAAG,CAAC,IAAI,CAAC,OAAO,IAAI,EAAE;AACvC,iBAAA,KAAK,CAAC,CAAC,EAAE,UAAU;AACnB,iBAAA,GAAG,CAAC,CAAC,MAAuB,EAAE,KAAa,KAAI;gBAC9C,IAAI,WAAW,GAAG,EAAE;AACpB,gBAAA,IAAI;AACF,oBAAA,WAAW,GAAG,IAAI,GAAG,CAAC,MAAM,CAAC,GAAG,IAAI,EAAE,CAAC,CAAC,QAAQ;gBAClD;AAAE,gBAAA,MAAM;oBACN,WAAW,GAAG,EAAE;gBAClB;gBAEA,OAAO;oBACL,QAAQ,EAAE,KAAK,GAAG,CAAC;AACnB,oBAAA,KAAK,EAAE,MAAM,CAAC,KAAK,IAAI,EAAE;AACzB,oBAAA,IAAI,EAAE,MAAM,CAAC,GAAG,IAAI,EAAE;AACtB,oBAAA,OAAO,EAAE,MAAM,CAAC,OAAO,IAAI,EAAE;AAC7B,oBAAA,IAAI,EAAE,MAAM,CAAC,aAAa,IAAI,EAAE;oBAChC,WAAW;iBACZ;AACH,YAAA,CAAC,CAAC;YAEJ,MAAM,YAAY,GAAG,CAAC,IAAI,CAAC,OAAO,IAAI,EAAE;iBACrC,MAAM,CAAC,CAAC,MAAuB,KAAK,MAAM,CAAC,OAAO;AAClD,iBAAA,KAAK,CAAC,CAAC,EAAE,CAAC;iBACV,GAAG,CAAC,CAAC,MAAuB,EAAE,KAAa,MAAM;AAChD,gBAAA,KAAK,EAAE,MAAM,CAAC,KAAK,IAAI,EAAE;AACzB,gBAAA,QAAQ,EAAE,MAAM,CAAC,OAAO,IAAI,EAAE;gBAC9B,QAAQ,EAAE,KAAK,GAAG,CAAC;gBACnB,MAAM,EAAE,IAAI,GAAG,CAAC,MAAM,CAAC,GAAG,IAAI,EAAE,CAAC,CAAC,QAAQ;gBAC1C,MAAM,EAAE,IAAI,GAAG,CAAC,MAAM,CAAC,GAAG,IAAI,EAAE,CAAC,CAAC,QAAQ;AAC1C,gBAAA,IAAI,EAAE,MAAM,CAAC,GAAG,IAAI,EAAE;AACvB,aAAA,CAAC,CAAC;;YAGL,MAAM,WAAW,GAAG,CAAC,IAAI,CAAC,OAAO,IAAI,EAAE;iBACpC,MAAM,CAAC,YAAY;AACnB,iBAAA,GAAG,CAAC,CAAC,MAAuB,EAAE,KAAa,KAAI;gBAC9C,IAAI,WAAW,GAAG,EAAE;AACpB,gBAAA,IAAI;AACF,oBAAA,WAAW,GAAG,IAAI,GAAG,CAAC,MAAM,CAAC,GAAG,IAAI,EAAE,CAAC,CAAC,QAAQ;gBAClD;AAAE,gBAAA,MAAM;oBACN,WAAW,GAAG,EAAE;gBAClB;gBAEA,OAAO;AACL,oBAAA,KAAK,EAAE,MAAM,CAAC,KAAK,IAAI,EAAE;AACzB,oBAAA,IAAI,EAAE,MAAM,CAAC,GAAG,IAAI,EAAE;AACtB,oBAAA,OAAO,EAAE,MAAM,CAAC,OAAO,IAAI,EAAE;AAC7B,oBAAA,IAAI,EAAE,MAAM,CAAC,aAAa,IAAI,EAAE;AAChC,oBAAA,MAAM,EAAE,WAAW;AACnB,oBAAA,QAAQ,EAAE,MAAM,CAAC,OAAO,IAAI,EAAE;oBAC9B,QAAQ,EAAE,KAAK,GAAG,CAAC;iBACpB;AACH,YAAA,CAAC,CAAC;YAEJ,MAAM,UAAU,GAAG,WAAW,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC;YAE1C,MAAM,eAAe,GAAG,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,WAAW;AACpD,kBAAE,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC,UAAkB,MAAM,EAAE,KAAK,EAAE,UAAU,EAAE,CAAC;kBACpE,EAAE;AAEN,YAAA,MAAM,OAAO,GAAuB;AAClC,gBAAA,OAAO,EAAE,cAAc;AACvB,gBAAA,MAAM,EAAE,YAAY;gBACpB,UAAU,EAAE,UAAU;gBACtB,eAAe;AACf,gBAAA,MAAM,EAAE,EAAE;AACV,gBAAA,IAAI,EAAE,WAAW;;AAEjB,gBAAA,MAAM,EAAE,EAAE;AACV,gBAAA,QAAQ,EAAE,EAAE;AACZ,gBAAA,aAAa,EAAE,EAAE;AACjB,gBAAA,cAAc,EAAE,SAAS;AACzB,gBAAA,SAAS,EAAE,SAAS;aACrB;YAED,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,IAAI,EAAE,OAAO,EAAE;QACzC;QAAE,OAAO,KAAK,EAAE;AACd,YAAA,MAAM,YAAY,GAChB,KAAK,YAAY,KAAK,GAAG,KAAK,CAAC,OAAO,GAAG,MAAM,CAAC,KAAK,CAAC;YACxD,OAAO;AACL,gBAAA,OAAO,EAAE,KAAK;gBACd,KAAK,EAAE,CAAA,4BAAA,EAA+B,YAAY,CAAA,CAAE;aACrD;QACH;AACF,IAAA,CAAC;IAED,OAAO,EAAE,UAAU,EAAE;AACvB,CAAC;AAEM,MAAM,eAAe,GAAG,CAC7B,MAAsB,KAGpB;AACF,IAAA,MAAM,EACJ,cAAc,GAAG,QAAQ,EACzB,YAAY,EACZ,kBAAkB,EAClB,aAAa,EACb,eAAe,EACf,gBAAgB,GACjB,GAAG,MAAM;AAEV,IAAA,IAAI,cAAc,CAAC,WAAW,EAAE,KAAK,QAAQ,EAAE;QAC7C,OAAO,eAAe,CAAC,YAAY,EAAE,eAAe,EAAE,gBAAgB,CAAC;IACzE;AAAO,SAAA,IAAI,cAAc,CAAC,WAAW,EAAE,KAAK,SAAS,EAAE;AACrD,QAAA,OAAO,gBAAgB,CAAC,kBAAkB,EAAE,aAAa,CAAC;IAC5D;SAAO;AACL,QAAA,MAAM,IAAI,KAAK,CACb,4BAA4B,cAAc,CAAA,+BAAA,CAAiC,CAC5E;IACH;AACF;AAEO,MAAM,qBAAqB,GAAG,CACnC,SAAiC,EAAE,EACnC,eAA+B,KAM7B;IACF,IAAI,CAAC,eAAe,EAAE;AACpB,QAAA,MAAM,IAAI,KAAK,CAAC,8BAA8B,CAAC;IACjD;IACA,MAAM,EACJ,UAAU,GAAG,CAAC;;;AAGd,IAAA,QAAQ,EACR,MAAM,GACP,GAAG,MAAM;AAEV,IAAA,MAAM,OAAO,GAAG,MAAM,IAAIA,yBAAmB,EAAE;IAC/C,MAAM,OAAO,GAAG,eAAe;AAE/B,IAAA,MAAM,UAAU,GAAG;QACjB,UAAU,EAAE,OAAO,EACjB,KAAK,EACL,KAAK,EACL,eAAe,GAKhB,KAAoC;YACnC,OAAO,CAAC,KAAK,CAAC,CAAA,SAAA,EAAY,KAAK,CAAC,MAAM,CAAA,MAAA,CAAQ,CAAC;YAC/C,MAAM,QAAQ,GAAmC,EAAE;AACnD,YAAA,IAAI;AACF,gBAAA,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE;AACrC,oBAAA,MAAM,WAAW,GAAG,KAAK,CAAC,CAAC,CAAC;oBAC5B,MAAM,OAAO,GAA4B;AACtC,yBAAA,SAAS,CAAC,WAAW,EAAE,EAAE;yBACzB,IAAI,CAAC,CAAC,CAAC,GAAG,EAAE,QAAQ,CAAC,KAAI;AACxB,wBAAA,MAAM,WAAW,GAAGC,oBAAc,CAChC,GAAG,EACH,QAAQ,CAAC,IAAI,EAAE,QAAQ,EACvB,OAAO,CACR;wBACD,IAAI,QAAQ,CAAC,OAAO,IAAI,QAAQ,CAAC,IAAI,EAAE;AACrC,4BAAA,MAAM,CAAC,OAAO,EAAE,UAAU,CAAC,GAAG,OAAO,CAAC,cAAc,CAAC,QAAQ,CAAC;4BAC9D,OAAO;gCACL,GAAG;gCACH,UAAU;gCACV,WAAW;AACX,gCAAA,OAAO,EAAE,OAAO,CAAC,SAAS,CAAC,OAAO,CAAC;6BAClB;wBACrB;6BAAO;AACL,4BAAA,OAAO,CAAC,KAAK,CACX,CAAA,eAAA,EAAkB,GAAG,CAAA,EAAA,EAAK,QAAQ,CAAC,KAAK,IAAI,eAAe,CAAA,CAAE,CAC9D;wBACH;wBAEA,OAAO;4BACL,GAAG;4BACH,WAAW;AACX,4BAAA,KAAK,EAAE,IAAI;AACX,4BAAA,OAAO,EAAE,EAAE;yBACM;AACrB,oBAAA,CAAC;AACA,yBAAA,IAAI,CAAC,OAAO,MAAM,KAAI;AACrB,wBAAA,IAAI;AACF,4BAAA,IAAI,MAAM,CAAC,KAAK,IAAI,IAAI,EAAE;AACxB,gCAAA,OAAO,CAAC,KAAK,CACX,CAAA,eAAA,EAAkB,MAAM,CAAC,GAAG,CAAA,EAAA,EAAK,MAAM,CAAC,OAAO,CAAA,CAAE,CAClD;gCACD,OAAO;AACL,oCAAA,GAAG,MAAM;iCACV;4BACH;AACA,4BAAA,MAAM,UAAU,GAAG,MAAM,aAAa,CAAC;gCACrC,KAAK;gCACL,QAAQ;gCACR,OAAO,EAAE,MAAM,CAAC,OAAO;AACvB,gCAAA,MAAM,EAAE,OAAO;AAChB,6BAAA,CAAC;4BACF,IAAI,eAAe,EAAE;AACnB,gCAAA,eAAe,CAAC,MAAM,CAAC,GAAG,CAAC;4BAC7B;4BACA,OAAO;AACL,gCAAA,GAAG,MAAM;gCACT,UAAU;6BACX;wBACH;wBAAE,OAAO,KAAK,EAAE;AACd,4BAAA,OAAO,CAAC,KAAK,CAAC,mCAAmC,EAAE,KAAK,CAAC;4BACzD,OAAO;AACL,gCAAA,GAAG,MAAM;6BACV;wBACH;AACF,oBAAA,CAAC;AACA,yBAAA,KAAK,CAAC,CAAC,KAAK,KAAI;wBACf,OAAO,CAAC,KAAK,CAAC,CAAA,eAAA,EAAkB,WAAW,CAAA,CAAA,CAAG,EAAE,KAAK,CAAC;wBACtD,OAAO;AACL,4BAAA,GAAG,EAAE,WAAW;AAChB,4BAAA,KAAK,EAAE,IAAI;AACX,4BAAA,OAAO,EAAE,EAAE;yBACZ;AACH,oBAAA,CAAC,CAAC;AACJ,oBAAA,QAAQ,CAAC,IAAI,CAAC,OAAO,CAAC;gBACxB;AACA,gBAAA,OAAO,MAAM,OAAO,CAAC,GAAG,CAAC,QAAQ,CAAC;YACpC;YAAE,OAAO,KAAK,EAAE;AACd,gBAAA,OAAO,CAAC,KAAK,CAAC,sBAAsB,EAAE,KAAK,CAAC;AAC5C,gBAAA,OAAO,EAAE;YACX;QACF,CAAC;KACF;AAED,IAAA,MAAM,aAAa,GAAG,OAAO,EAC3B,KAAK,EACL,KAAK,EACL,MAAM,EACN,eAAe,EACf,gBAAgB,GAOjB,KAAmB;QAClB,MAAM,YAAY,GAAG,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,MAAM,CAAC;;AAE3C,QAAA,MAAM,OAAO,GAAG,MAAM,UAAU,CAAC,UAAU,CAAC;YAC1C,KAAK;AACL,YAAA,KAAK,EAAE,YAAY;YACnB,eAAe;AAChB,SAAA,CAAC;AACF,QAAA,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE;AAC5B,YAAA,IAAI,MAAM,CAAC,KAAK,KAAK,IAAI,EAAE;gBACzB;YACF;AACA,YAAA,MAAM,EAAE,GAAG,EAAE,OAAO,EAAE,WAAW,EAAE,UAAU,EAAE,UAAU,EAAE,GAAG,MAAM;YACpE,gBAAgB,GAAG,GAAG,EAAE;gBACtB,OAAO;gBACP,WAAW;gBACX,UAAU;gBACV,UAAU;AACX,aAAA,CAAC;QACJ;AACF,IAAA,CAAC;IAED,MAAM,cAAc,GAAG,OAAO,EAC5B,MAAM,EACN,WAAW,EACX,KAAK,EACL,IAAI,EACJ,OAAO,GAAG,IAAI,EACd,eAAe,EACf,YAAY,GAAG,KAAK,GACG,KAAiC;AACxD,QAAA,IAAI;AACF,YAAA,IAAI,CAAC,MAAM,CAAC,IAAI,EAAE;gBAChB,OAAO;AACL,oBAAA,OAAO,EAAE,EAAE;AACX,oBAAA,UAAU,EAAE,EAAE;AACd,oBAAA,MAAM,EAAE,EAAE;AACV,oBAAA,eAAe,EAAE,EAAE;iBACpB;YACH;AAAO,iBAAA,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,OAAO,EAAE;gBAC/B,OAAO,MAAM,CAAC,IAAI;YACpB;;YAGA,IAAI,YAAY,EAAE;AAChB,gBAAA,OAAO,CAAC,KAAK,CACX,0DAA0D,CAC3D;gBACD,OAAO,MAAM,CAAC,IAAI;YACpB;YAEA,IAAI,CAAC,OAAO,EAAE;gBACZ,MAAM,WAAW,GAAG,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,MAAM,KACpD,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC,eAAe,CAAC,CACtC;AAED,gBAAA,IAAI,CAAC,WAAW,CAAC,MAAM,EAAE;oBACvB,OAAO,MAAM,CAAC,IAAI;gBACpB;AAEA,gBAAA,MAAM,aAAa,GAAG,IAAI,GAAG,EAAyB;AACtD,gBAAA,aAAa,CAAC,GAAG,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,WAAW,CAAC,CAAC,CAAC,CAAC;AACtD,gBAAA,MAAM,gBAAgB,GAAG,0BAA0B,CAAC,aAAa,CAAC;AAClE,gBAAA,MAAM,aAAa,CAAC;oBAClB,KAAK;AACL,oBAAA,MAAM,EAAE,CAAC;oBACT,eAAe;oBACf,gBAAgB;oBAChB,KAAK,EAAE,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;AAC7B,iBAAA,CAAC;AAEF,gBAAA,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE;oBACnD,MAAM,MAAM,GAAG,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC;oBACrC,MAAM,aAAa,GAAG,aAAa,CAAC,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC;oBACpD,IAAI,aAAa,EAAE;AACjB,wBAAA,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG;AACvB,4BAAA,GAAG,MAAM;AACT,4BAAA,GAAG,aAAa;yBACjB;oBACH;gBACF;gBAEA,OAAO,MAAM,CAAC,IAAI;YACpB;AAEA,YAAA,MAAM,SAAS,GAAG,IAAI,GAAG,EAAyB;AAClD,YAAA,MAAM,eAAe,GAAG,IAAI,GAAG,EAAU;;AAGzC,YAAA,MAAM,YAAY,GAAG,YAAY,CAC/B,MAAM,CAAC,IAAI,CAAC,OAAO,EACnB,SAAS,EACT,eAAe,CAChB;;YAGD,MAAM,UAAU,GAAG,MAAM,CAAC,IAAI,CAAC,UAAU,IAAI,EAAE;YAC/C,MAAM,aAAa,GAAG,YAAY,CAChC,UAAU,EACV,SAAS,EACT,eAAe,CAChB;AAED,YAAA,IAAI,YAAY,CAAC,MAAM,KAAK,CAAC,KAAK,aAAa,CAAC,MAAM,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE;gBACtE,OAAO,MAAM,CAAC,IAAI;YACpB;AAEA,YAAA,MAAM,gBAAgB,GAAG,0BAA0B,CAAC,SAAS,CAAC;YAC9D,MAAM,QAAQ,GAAoB,EAAE;;AAGpC,YAAA,IAAI,YAAY,CAAC,MAAM,GAAG,CAAC,EAAE;AAC3B,gBAAA,QAAQ,CAAC,IAAI,CACX,aAAa,CAAC;oBACZ,KAAK;oBACL,eAAe;oBACf,gBAAgB;AAChB,oBAAA,KAAK,EAAE,YAAY;AACnB,oBAAA,MAAM,EAAE,WAAW;AACpB,iBAAA,CAAC,CACH;YACH;;YAGA,IAAI,IAAI,IAAI,aAAa,CAAC,MAAM,GAAG,CAAC,EAAE;AACpC,gBAAA,QAAQ,CAAC,IAAI,CACX,aAAa,CAAC;oBACZ,KAAK;oBACL,eAAe;oBACf,gBAAgB;AAChB,oBAAA,KAAK,EAAE,aAAa;AACpB,oBAAA,MAAM,EAAE,WAAW;AACpB,iBAAA,CAAC,CACH;YACH;AAEA,YAAA,MAAM,OAAO,CAAC,GAAG,CAAC,QAAQ,CAAC;YAE3B,IAAI,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE;gBAClC,wBAAwB,CAAC,MAAM,CAAC,IAAI,CAAC,OAAO,EAAE,SAAS,CAAC;YAC1D;YAEA,IAAI,IAAI,IAAI,UAAU,CAAC,MAAM,GAAG,CAAC,EAAE;AACjC,gBAAA,wBAAwB,CAAC,UAAU,EAAE,SAAS,CAAC;YACjD;YAEA,OAAO,MAAM,CAAC,IAAI;QACpB;QAAE,OAAO,KAAK,EAAE;AACd,YAAA,OAAO,CAAC,KAAK,CAAC,0BAA0B,EAAE,KAAK,CAAC;YAChD,OAAO;AACL,gBAAA,OAAO,EAAE,EAAE;AACX,gBAAA,UAAU,EAAE,EAAE;AACd,gBAAA,MAAM,EAAE,EAAE;AACV,gBAAA,eAAe,EAAE,EAAE;gBACnB,GAAG,MAAM,CAAC,IAAI;AACd,gBAAA,KAAK,EAAE,KAAK,YAAY,KAAK,GAAG,KAAK,CAAC,OAAO,GAAG,MAAM,CAAC,KAAK,CAAC;aAC9D;QACH;AACF,IAAA,CAAC;IAED,OAAO;QACL,cAAc;QACd,UAAU;KACX;AACH;AAEA;AACA,SAAS,YAAY,CACnB,OAAkD,EAClD,SAAqC,EACrC,gBAA8B,EAAA;IAE9B,MAAM,KAAK,GAAa,EAAE;AAE1B,IAAA,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE;AAC5B,QAAA,IAAI,MAAM,CAAC,IAAI,EAAE;;YAEf,IAAI,gBAAgB,IAAI,gBAAgB,CAAC,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE;gBACzD;YACF;AAEA,YAAA,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC;YACvB,IAAI,gBAAgB,EAAE;AACpB,gBAAA,gBAAgB,CAAC,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC;YACnC;YACA,SAAS,CAAC,GAAG,CAAC,MAAM,CAAC,IAAI,EAAE,MAAuB,CAAC;QACrD;IACF;AAEA,IAAA,OAAO,KAAK;AACd;AAEA;AACA,SAAS,wBAAwB,CAC/B,OAAY,EACZ,SAAqC,EAAA;AAErC,IAAA,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE;AACvC,QAAA,MAAM,MAAM,GAAG,OAAO,CAAC,CAAC,CAAC;QACzB,MAAM,aAAa,GAAG,SAAS,CAAC,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC;QAChD,IAAI,aAAa,EAAE;YACjB,OAAO,CAAC,CAAC,CAAC,GAAG;AACX,gBAAA,GAAG,MAAM;AACT,gBAAA,GAAG,aAAa;aACZ;QACR;IACF;AACF;;;;;"}
|
|
1
|
+
{"version":3,"file":"search.cjs","sources":["../../../../src/tools/search/search.ts"],"sourcesContent":["import axios from 'axios';\nimport { RecursiveCharacterTextSplitter } from '@langchain/textsplitters';\nimport type * as t from './types';\nimport { getAttribution, createDefaultLogger } from './utils';\nimport { createTavilyAPI } from './tavily-search';\nimport { BaseReranker } from './rerankers';\n\nconst chunker = {\n cleanText: (text: string): string => {\n if (!text) return '';\n\n /** Normalized all line endings to '\\n' */\n const normalizedText = text.replace(/\\r\\n/g, '\\n').replace(/\\r/g, '\\n');\n\n /** Handle multiple backslashes followed by newlines\n * This replaces patterns like '\\\\\\\\\\\\n' with a single newline */\n const fixedBackslashes = normalizedText.replace(/\\\\+\\n/g, '\\n');\n\n /** Cleaned up consecutive newlines, tabs, and spaces around newlines */\n const cleanedNewlines = fixedBackslashes.replace(/[\\t ]*\\n[\\t \\n]*/g, '\\n');\n\n /** Cleaned up excessive spaces and tabs */\n const cleanedSpaces = cleanedNewlines.replace(/[ \\t]+/g, ' ');\n\n return cleanedSpaces.trim();\n },\n splitText: async (\n text: string,\n options?: {\n chunkSize?: number;\n chunkOverlap?: number;\n separators?: string[];\n }\n ): Promise<string[]> => {\n const chunkSize = options?.chunkSize ?? 150;\n const chunkOverlap = options?.chunkOverlap ?? 50;\n const separators = options?.separators || ['\\n\\n', '\\n'];\n\n const splitter = new RecursiveCharacterTextSplitter({\n separators,\n chunkSize,\n chunkOverlap,\n });\n\n return await splitter.splitText(text);\n },\n\n splitTexts: async (\n texts: string[],\n options?: {\n chunkSize?: number;\n chunkOverlap?: number;\n separators?: string[];\n },\n logger?: t.Logger\n ): Promise<string[][]> => {\n // Split multiple texts\n const logger_ = logger || createDefaultLogger();\n const promises = texts.map((text) =>\n chunker.splitText(text, options).catch((error) => {\n logger_.error('Error splitting text:', error);\n return [text];\n })\n );\n return Promise.all(promises);\n },\n};\n\nfunction createSourceUpdateCallback(sourceMap: Map<string, t.ValidSource>) {\n return (link: string, update?: Partial<t.ValidSource>): void => {\n const source = sourceMap.get(link);\n if (source) {\n sourceMap.set(link, {\n ...source,\n ...update,\n });\n }\n };\n}\n\nconst getHighlights = async ({\n query,\n content,\n reranker,\n topResults = 5,\n logger,\n}: {\n content: string;\n query: string;\n reranker?: BaseReranker;\n topResults?: number;\n logger?: t.Logger;\n}): Promise<t.Highlight[] | undefined> => {\n const logger_ = logger || createDefaultLogger();\n\n if (!content) {\n logger_.warn('No content provided for highlights');\n return;\n }\n if (!reranker) {\n logger_.warn('No reranker provided for highlights');\n return;\n }\n\n try {\n const documents = await chunker.splitText(content);\n if (Array.isArray(documents)) {\n return await reranker.rerank(query, documents, topResults);\n } else {\n logger_.error(\n 'Expected documents to be an array, got:',\n typeof documents\n );\n return;\n }\n } catch (error) {\n logger_.error('Error in content processing:', error);\n return;\n }\n};\n\nconst createSerperAPI = (\n apiKey?: string,\n domainBlocklist?: string[],\n countryBlocklist?: string[]\n): {\n getSources: (params: t.GetSourcesParams) => Promise<t.SearchResult>;\n} => {\n const config = {\n apiKey: apiKey ?? process.env.SERPER_API_KEY,\n apiUrl: 'https://google.serper.dev/search',\n timeout: 10000,\n };\n\n if (config.apiKey == null || config.apiKey === '') {\n throw new Error('SERPER_API_KEY is required for SerperAPI');\n }\n\n const getSources = async ({\n query,\n date,\n country,\n safeSearch,\n numResults = 8,\n type,\n }: t.GetSourcesParams): Promise<t.SearchResult> => {\n if (!query.trim()) {\n return { success: false, error: 'Query cannot be empty' };\n }\n\n try {\n const safe = ['off', 'moderate', 'active'] as const;\n\n // Append -site: operators for blocked domains so they never appear in results\n let effectiveQuery = query;\n if (domainBlocklist && domainBlocklist.length > 0) {\n const siteExclusions = domainBlocklist\n .map((d) => `-site:${d}`)\n .join(' ');\n effectiveQuery = `${effectiveQuery} ${siteExclusions}`;\n }\n\n // Append -site:.XX operators for blocked country TLDs\n if (countryBlocklist && countryBlocklist.length > 0) {\n const countryExclusions = countryBlocklist\n .map((c) => `-site:.${c.toLowerCase().replace(/^\\./, '')}`)\n .join(' ');\n effectiveQuery = `${effectiveQuery} ${countryExclusions}`;\n }\n\n const payload: t.SerperSearchPayload = {\n q: effectiveQuery,\n safe: safe[safeSearch ?? 1],\n num: Math.min(Math.max(1, numResults), 10),\n };\n\n // Set the search type if provided\n if (type) {\n payload.type = type;\n }\n\n if (date != null) {\n payload.tbs = `qdr:${date}`;\n }\n\n if (country != null && country !== '') {\n payload['gl'] = country.toLowerCase();\n }\n\n // Determine the API endpoint based on the search type\n let apiEndpoint = config.apiUrl;\n if (type === 'images') {\n apiEndpoint = 'https://google.serper.dev/images';\n } else if (type === 'videos') {\n apiEndpoint = 'https://google.serper.dev/videos';\n } else if (type === 'news') {\n apiEndpoint = 'https://google.serper.dev/news';\n }\n\n const response = await axios.post<t.SerperResultData>(\n apiEndpoint,\n payload,\n {\n headers: {\n 'X-API-KEY': config.apiKey,\n 'Content-Type': 'application/json',\n },\n timeout: config.timeout,\n }\n );\n\n const data = response.data;\n const results: t.SearchResultData = {\n organic: data.organic,\n images: data.images ?? [],\n answerBox: data.answerBox,\n topStories: data.topStories ?? [],\n peopleAlsoAsk: data.peopleAlsoAsk,\n knowledgeGraph: data.knowledgeGraph,\n relatedSearches: data.relatedSearches,\n videos: data.videos ?? [],\n news: data.news ?? [],\n };\n\n return { success: true, data: results };\n } catch (error) {\n const errorMessage =\n error instanceof Error ? error.message : String(error);\n return { success: false, error: `API request failed: ${errorMessage}` };\n }\n };\n\n return { getSources };\n};\n\nconst createSearXNGAPI = (\n instanceUrl?: string,\n apiKey?: string\n): {\n getSources: (params: t.GetSourcesParams) => Promise<t.SearchResult>;\n} => {\n const config = {\n instanceUrl: instanceUrl ?? process.env.SEARXNG_INSTANCE_URL,\n apiKey: apiKey ?? process.env.SEARXNG_API_KEY,\n defaultLocation: 'all',\n timeout: 10000,\n };\n\n if (config.instanceUrl == null || config.instanceUrl === '') {\n throw new Error('SEARXNG_INSTANCE_URL is required for SearXNG API');\n }\n\n const getSources = async ({\n query,\n numResults = 8,\n safeSearch,\n type,\n }: t.GetSourcesParams): Promise<t.SearchResult> => {\n if (!query.trim()) {\n return { success: false, error: 'Query cannot be empty' };\n }\n\n try {\n // Ensure the instance URL ends with /search\n if (config.instanceUrl == null || config.instanceUrl === '') {\n return { success: false, error: 'Instance URL is not defined' };\n }\n\n let searchUrl = config.instanceUrl;\n if (!searchUrl.endsWith('/search')) {\n searchUrl = searchUrl.replace(/\\/$/, '') + '/search';\n }\n\n // Determine the search category based on the type\n let category = 'general';\n if (type === 'images') {\n category = 'images';\n } else if (type === 'videos') {\n category = 'videos';\n } else if (type === 'news') {\n category = 'news';\n }\n\n // Prepare parameters for SearXNG\n const params: t.SearxNGSearchPayload = {\n q: query,\n format: 'json',\n pageno: 1,\n categories: category,\n language: 'all',\n safesearch: safeSearch,\n engines: 'google,bing,duckduckgo',\n };\n\n const headers: Record<string, string> = {\n 'Content-Type': 'application/json',\n };\n\n if (config.apiKey != null && config.apiKey !== '') {\n headers['X-API-Key'] = config.apiKey;\n }\n\n const response = await axios.get(searchUrl, {\n headers,\n params,\n timeout: config.timeout,\n });\n\n const data = response.data;\n\n // Helper function to identify news results since SearXNG doesn't provide that classification by default\n const isNewsResult = (result: t.SearXNGResult): boolean => {\n const url = result.url?.toLowerCase() ?? '';\n const title = result.title?.toLowerCase() ?? '';\n\n // News-related keywords in title/content\n const newsKeywords = [\n 'breaking news',\n 'latest news',\n 'top stories',\n 'news today',\n 'developing story',\n 'trending news',\n 'news',\n ];\n\n // Check if title/content contains news keywords\n const hasNewsKeywords = newsKeywords.some(\n (keyword) => title.toLowerCase().includes(keyword) // just title probably fine, content parsing is overkill for what we need: || content.includes(keyword)\n );\n\n // Check if URL contains news-related paths\n const hasNewsPath =\n url.includes('/news/') ||\n url.includes('/world/') ||\n url.includes('/politics/') ||\n url.includes('/breaking/');\n\n return hasNewsKeywords || hasNewsPath;\n };\n\n // Transform SearXNG results to match SerperAPI format\n const organicResults = (data.results ?? [])\n .slice(0, numResults)\n .map((result: t.SearXNGResult, index: number) => {\n let attribution = '';\n try {\n attribution = new URL(result.url ?? '').hostname;\n } catch {\n attribution = '';\n }\n\n return {\n position: index + 1,\n title: result.title ?? '',\n link: result.url ?? '',\n snippet: result.content ?? '',\n date: result.publishedDate ?? '',\n attribution,\n };\n });\n\n const imageResults = (data.results ?? [])\n .filter((result: t.SearXNGResult) => result.img_src)\n .slice(0, 6)\n .map((result: t.SearXNGResult, index: number) => ({\n title: result.title ?? '',\n imageUrl: result.img_src ?? '',\n position: index + 1,\n source: new URL(result.url ?? '').hostname,\n domain: new URL(result.url ?? '').hostname,\n link: result.url ?? '',\n }));\n\n // Extract news results from organic results\n const newsResults = (data.results ?? [])\n .filter(isNewsResult)\n .map((result: t.SearXNGResult, index: number) => {\n let attribution = '';\n try {\n attribution = new URL(result.url ?? '').hostname;\n } catch {\n attribution = '';\n }\n\n return {\n title: result.title ?? '',\n link: result.url ?? '',\n snippet: result.content ?? '',\n date: result.publishedDate ?? '',\n source: attribution,\n imageUrl: result.img_src ?? '',\n position: index + 1,\n };\n });\n\n const topStories = newsResults.slice(0, 5);\n\n const relatedSearches = Array.isArray(data.suggestions)\n ? data.suggestions.map((suggestion: string) => ({ query: suggestion }))\n : [];\n\n const results: t.SearchResultData = {\n organic: organicResults,\n images: imageResults,\n topStories: topStories, // Use first 5 extracted news as top stories\n relatedSearches,\n videos: [],\n news: newsResults,\n // Add empty arrays for other Serper fields to maintain parity\n places: [],\n shopping: [],\n peopleAlsoAsk: [],\n knowledgeGraph: undefined,\n answerBox: undefined,\n };\n\n return { success: true, data: results };\n } catch (error) {\n const errorMessage =\n error instanceof Error ? error.message : String(error);\n return {\n success: false,\n error: `SearXNG API request failed: ${errorMessage}`,\n };\n }\n };\n\n return { getSources };\n};\n\nexport const createSearchAPI = (\n config: t.SearchConfig\n): {\n getSources: (params: t.GetSourcesParams) => Promise<t.SearchResult>;\n} => {\n const {\n searchProvider = 'serper',\n serperApiKey,\n searxngInstanceUrl,\n searxngApiKey,\n tavilyApiKey,\n tavilySearchUrl,\n tavilySearchOptions,\n domainBlocklist,\n countryBlocklist,\n } = config;\n\n if (searchProvider.toLowerCase() === 'serper') {\n return createSerperAPI(serperApiKey, domainBlocklist, countryBlocklist);\n } else if (searchProvider.toLowerCase() === 'searxng') {\n return createSearXNGAPI(searxngInstanceUrl, searxngApiKey);\n } else if (searchProvider.toLowerCase() === 'tavily') {\n return createTavilyAPI(tavilyApiKey, tavilySearchUrl, tavilySearchOptions);\n } else {\n throw new Error(\n `Invalid search provider: ${searchProvider}. Must be 'serper', 'searxng', or 'tavily'`\n );\n }\n};\n\nexport const createSourceProcessor = (\n config: t.ProcessSourcesConfig = {},\n scraperInstance?: t.BaseScraper\n): {\n processSources: (\n fields: t.ProcessSourcesFields\n ) => Promise<t.SearchResultData>;\n topResults: number;\n} => {\n if (!scraperInstance) {\n throw new Error('Scraper instance is required');\n }\n const {\n topResults = 5,\n // strategies = ['no_extraction'],\n // filterContent = true,\n reranker,\n logger,\n } = config;\n\n const logger_ = logger || createDefaultLogger();\n const scraper = scraperInstance;\n\n const webScraper = {\n scrapeMany: async ({\n query,\n links,\n onGetHighlights,\n }: {\n query: string;\n links: string[];\n onGetHighlights: t.SearchToolConfig['onGetHighlights'];\n }): Promise<Array<t.ScrapeResult>> => {\n logger_.debug(`Scraping ${links.length} links`);\n const promises: Array<Promise<t.ScrapeResult>> = [];\n try {\n for (let i = 0; i < links.length; i++) {\n const currentLink = links[i];\n const promise: Promise<t.ScrapeResult> = scraper\n .scrapeUrl(currentLink, {})\n .then(([url, response]) => {\n /* metadata exists on Firecrawl/Serper responses but not on\n * Tavily's. Read it via the scraper's extractMetadata abstraction\n * to stay polymorphic across all scraper providers. */\n const rawMeta = scraper.extractMetadata(response);\n const attribution = getAttribution(\n url,\n Object.keys(rawMeta).length > 0 ? rawMeta : undefined,\n logger_\n );\n if (response.success && response.data) {\n const [content, references] = scraper.extractContent(response);\n return {\n url,\n references,\n attribution,\n content: chunker.cleanText(content),\n } as t.ScrapeResult;\n } else {\n logger_.error(\n `Error scraping ${url}: ${response.error ?? 'Unknown error'}`\n );\n }\n\n return {\n url,\n attribution,\n error: true,\n content: '',\n } as t.ScrapeResult;\n })\n .then(async (result) => {\n try {\n if (result.error != null) {\n logger_.error(\n `Error scraping ${result.url}: ${result.content}`\n );\n return {\n ...result,\n };\n }\n const highlights = await getHighlights({\n query,\n reranker,\n content: result.content,\n logger: logger_,\n });\n if (onGetHighlights) {\n onGetHighlights(result.url);\n }\n return {\n ...result,\n highlights,\n };\n } catch (error) {\n logger_.error('Error processing scraped content:', error);\n return {\n ...result,\n };\n }\n })\n .catch((error) => {\n logger_.error(`Error scraping ${currentLink}:`, error);\n return {\n url: currentLink,\n error: true,\n content: '',\n };\n });\n promises.push(promise);\n }\n return await Promise.all(promises);\n } catch (error) {\n logger_.error('Error in scrapeMany:', error);\n return [];\n }\n },\n };\n\n const fetchContents = async ({\n links,\n query,\n target,\n onGetHighlights,\n onContentScraped,\n }: {\n links: string[];\n query: string;\n target: number;\n onGetHighlights: t.SearchToolConfig['onGetHighlights'];\n onContentScraped?: (link: string, update?: Partial<t.ValidSource>) => void;\n }): Promise<void> => {\n const initialLinks = links.slice(0, target);\n // const remainingLinks = links.slice(target).reverse();\n const results = await webScraper.scrapeMany({\n query,\n links: initialLinks,\n onGetHighlights,\n });\n for (const result of results) {\n if (result.error === true) {\n continue;\n }\n const { url, content, attribution, references, highlights } = result;\n onContentScraped?.(url, {\n content,\n attribution,\n references,\n highlights,\n });\n }\n };\n\n const processSources = async ({\n result,\n numElements,\n query,\n news,\n proMode = true,\n onGetHighlights,\n skipScraping = false,\n }: t.ProcessSourcesFields): Promise<t.SearchResultData> => {\n try {\n if (!result.data) {\n return {\n organic: [],\n topStories: [],\n images: [],\n relatedSearches: [],\n };\n } else if (!result.data.organic) {\n return result.data;\n }\n\n // If content was already extracted directly (e.g., direct URL extraction), skip scraping\n if (skipScraping) {\n logger_.debug(\n 'Skipping additional scraping - content already extracted'\n );\n return result.data;\n }\n\n if (!proMode) {\n const wikiSources = result.data.organic.filter((source) =>\n source.link.includes('wikipedia.org')\n );\n\n if (!wikiSources.length) {\n return result.data;\n }\n\n const wikiSourceMap = new Map<string, t.ValidSource>();\n wikiSourceMap.set(wikiSources[0].link, wikiSources[0]);\n const onContentScraped = createSourceUpdateCallback(wikiSourceMap);\n await fetchContents({\n query,\n target: 1,\n onGetHighlights,\n onContentScraped,\n links: [wikiSources[0].link],\n });\n\n for (let i = 0; i < result.data.organic.length; i++) {\n const source = result.data.organic[i];\n const updatedSource = wikiSourceMap.get(source.link);\n if (updatedSource) {\n result.data.organic[i] = {\n ...source,\n ...updatedSource,\n };\n }\n }\n\n return result.data;\n }\n\n const sourceMap = new Map<string, t.ValidSource>();\n const organicLinksSet = new Set<string>();\n\n // Collect organic links\n const organicLinks = collectLinks(\n result.data.organic,\n sourceMap,\n organicLinksSet\n );\n\n // Collect top story links, excluding any that are already in organic links\n const topStories = result.data.topStories ?? [];\n const topStoryLinks = collectLinks(\n topStories,\n sourceMap,\n organicLinksSet\n );\n\n if (organicLinks.length === 0 && (topStoryLinks.length === 0 || !news)) {\n return result.data;\n }\n\n const onContentScraped = createSourceUpdateCallback(sourceMap);\n const promises: Promise<void>[] = [];\n\n // Process organic links\n if (organicLinks.length > 0) {\n promises.push(\n fetchContents({\n query,\n onGetHighlights,\n onContentScraped,\n links: organicLinks,\n target: numElements,\n })\n );\n }\n\n // Process top story links\n if (news && topStoryLinks.length > 0) {\n promises.push(\n fetchContents({\n query,\n onGetHighlights,\n onContentScraped,\n links: topStoryLinks,\n target: numElements,\n })\n );\n }\n\n await Promise.all(promises);\n\n if (result.data.organic.length > 0) {\n updateSourcesWithContent(result.data.organic, sourceMap);\n }\n\n if (news && topStories.length > 0) {\n updateSourcesWithContent(topStories, sourceMap);\n }\n\n return result.data;\n } catch (error) {\n logger_.error('Error in processSources:', error);\n return {\n organic: [],\n topStories: [],\n images: [],\n relatedSearches: [],\n ...result.data,\n error: error instanceof Error ? error.message : String(error),\n };\n }\n };\n\n return {\n processSources,\n topResults,\n };\n};\n\n/** Helper function to collect links and update sourceMap */\nfunction collectLinks(\n sources: Array<t.OrganicResult | t.TopStoryResult>,\n sourceMap: Map<string, t.ValidSource>,\n existingLinksSet?: Set<string>\n): string[] {\n const links: string[] = [];\n\n for (const source of sources) {\n if (source.link) {\n // For topStories, only add if not already in organic links\n if (existingLinksSet && existingLinksSet.has(source.link)) {\n continue;\n }\n\n links.push(source.link);\n if (existingLinksSet) {\n existingLinksSet.add(source.link);\n }\n sourceMap.set(source.link, source as t.ValidSource);\n }\n }\n\n return links;\n}\n\n/** Helper function to update sources with scraped content */\nfunction updateSourcesWithContent<T extends t.ValidSource>(\n sources: T[],\n sourceMap: Map<string, t.ValidSource>\n): void {\n for (let i = 0; i < sources.length; i++) {\n const source = sources[i];\n const updatedSource = sourceMap.get(source.link);\n if (updatedSource) {\n sources[i] = {\n ...source,\n ...updatedSource,\n } as T;\n }\n }\n}\n"],"names":["RecursiveCharacterTextSplitter","createDefaultLogger","createTavilyAPI","getAttribution"],"mappings":";;;;;;;AAOA,MAAM,OAAO,GAAG;AACd,IAAA,SAAS,EAAE,CAAC,IAAY,KAAY;AAClC,QAAA,IAAI,CAAC,IAAI;AAAE,YAAA,OAAO,EAAE;;AAGpB,QAAA,MAAM,cAAc,GAAG,IAAI,CAAC,OAAO,CAAC,OAAO,EAAE,IAAI,CAAC,CAAC,OAAO,CAAC,KAAK,EAAE,IAAI,CAAC;AAEvE;AACiE;QACjE,MAAM,gBAAgB,GAAG,cAAc,CAAC,OAAO,CAAC,QAAQ,EAAE,IAAI,CAAC;;QAG/D,MAAM,eAAe,GAAG,gBAAgB,CAAC,OAAO,CAAC,mBAAmB,EAAE,IAAI,CAAC;;QAG3E,MAAM,aAAa,GAAG,eAAe,CAAC,OAAO,CAAC,SAAS,EAAE,GAAG,CAAC;AAE7D,QAAA,OAAO,aAAa,CAAC,IAAI,EAAE;IAC7B,CAAC;AACD,IAAA,SAAS,EAAE,OACT,IAAY,EACZ,OAIC,KACoB;AACrB,QAAA,MAAM,SAAS,GAAG,OAAO,EAAE,SAAS,IAAI,GAAG;AAC3C,QAAA,MAAM,YAAY,GAAG,OAAO,EAAE,YAAY,IAAI,EAAE;QAChD,MAAM,UAAU,GAAG,OAAO,EAAE,UAAU,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC;AAExD,QAAA,MAAM,QAAQ,GAAG,IAAIA,4CAA8B,CAAC;YAClD,UAAU;YACV,SAAS;YACT,YAAY;AACb,SAAA,CAAC;AAEF,QAAA,OAAO,MAAM,QAAQ,CAAC,SAAS,CAAC,IAAI,CAAC;IACvC,CAAC;IAED,UAAU,EAAE,OACV,KAAe,EACf,OAIC,EACD,MAAiB,KACM;;AAEvB,QAAA,MAAM,OAAO,GAAG,MAAM,IAAIC,yBAAmB,EAAE;QAC/C,MAAM,QAAQ,GAAG,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,KAC9B,OAAO,CAAC,SAAS,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC,KAAK,CAAC,CAAC,KAAK,KAAI;AAC/C,YAAA,OAAO,CAAC,KAAK,CAAC,uBAAuB,EAAE,KAAK,CAAC;YAC7C,OAAO,CAAC,IAAI,CAAC;QACf,CAAC,CAAC,CACH;AACD,QAAA,OAAO,OAAO,CAAC,GAAG,CAAC,QAAQ,CAAC;IAC9B,CAAC;CACF;AAED,SAAS,0BAA0B,CAAC,SAAqC,EAAA;AACvE,IAAA,OAAO,CAAC,IAAY,EAAE,MAA+B,KAAU;QAC7D,MAAM,MAAM,GAAG,SAAS,CAAC,GAAG,CAAC,IAAI,CAAC;QAClC,IAAI,MAAM,EAAE;AACV,YAAA,SAAS,CAAC,GAAG,CAAC,IAAI,EAAE;AAClB,gBAAA,GAAG,MAAM;AACT,gBAAA,GAAG,MAAM;AACV,aAAA,CAAC;QACJ;AACF,IAAA,CAAC;AACH;AAEA,MAAM,aAAa,GAAG,OAAO,EAC3B,KAAK,EACL,OAAO,EACP,QAAQ,EACR,UAAU,GAAG,CAAC,EACd,MAAM,GAOP,KAAwC;AACvC,IAAA,MAAM,OAAO,GAAG,MAAM,IAAIA,yBAAmB,EAAE;IAE/C,IAAI,CAAC,OAAO,EAAE;AACZ,QAAA,OAAO,CAAC,IAAI,CAAC,oCAAoC,CAAC;QAClD;IACF;IACA,IAAI,CAAC,QAAQ,EAAE;AACb,QAAA,OAAO,CAAC,IAAI,CAAC,qCAAqC,CAAC;QACnD;IACF;AAEA,IAAA,IAAI;QACF,MAAM,SAAS,GAAG,MAAM,OAAO,CAAC,SAAS,CAAC,OAAO,CAAC;AAClD,QAAA,IAAI,KAAK,CAAC,OAAO,CAAC,SAAS,CAAC,EAAE;YAC5B,OAAO,MAAM,QAAQ,CAAC,MAAM,CAAC,KAAK,EAAE,SAAS,EAAE,UAAU,CAAC;QAC5D;aAAO;YACL,OAAO,CAAC,KAAK,CACX,yCAAyC,EACzC,OAAO,SAAS,CACjB;YACD;QACF;IACF;IAAE,OAAO,KAAK,EAAE;AACd,QAAA,OAAO,CAAC,KAAK,CAAC,8BAA8B,EAAE,KAAK,CAAC;QACpD;IACF;AACF,CAAC;AAED,MAAM,eAAe,GAAG,CACtB,MAAe,EACf,eAA0B,EAC1B,gBAA2B,KAGzB;AACF,IAAA,MAAM,MAAM,GAAG;AACb,QAAA,MAAM,EAAE,MAAM,IAAI,OAAO,CAAC,GAAG,CAAC,cAAc;AAC5C,QAAA,MAAM,EAAE,kCAAkC;AAC1C,QAAA,OAAO,EAAE,KAAK;KACf;AAED,IAAA,IAAI,MAAM,CAAC,MAAM,IAAI,IAAI,IAAI,MAAM,CAAC,MAAM,KAAK,EAAE,EAAE;AACjD,QAAA,MAAM,IAAI,KAAK,CAAC,0CAA0C,CAAC;IAC7D;IAEA,MAAM,UAAU,GAAG,OAAO,EACxB,KAAK,EACL,IAAI,EACJ,OAAO,EACP,UAAU,EACV,UAAU,GAAG,CAAC,EACd,IAAI,GACe,KAA6B;AAChD,QAAA,IAAI,CAAC,KAAK,CAAC,IAAI,EAAE,EAAE;YACjB,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,uBAAuB,EAAE;QAC3D;AAEA,QAAA,IAAI;YACF,MAAM,IAAI,GAAG,CAAC,KAAK,EAAE,UAAU,EAAE,QAAQ,CAAU;;YAGnD,IAAI,cAAc,GAAG,KAAK;YAC1B,IAAI,eAAe,IAAI,eAAe,CAAC,MAAM,GAAG,CAAC,EAAE;gBACjD,MAAM,cAAc,GAAG;qBACpB,GAAG,CAAC,CAAC,CAAC,KAAK,CAAA,MAAA,EAAS,CAAC,CAAA,CAAE;qBACvB,IAAI,CAAC,GAAG,CAAC;AACZ,gBAAA,cAAc,GAAG,CAAA,EAAG,cAAc,CAAA,CAAA,EAAI,cAAc,EAAE;YACxD;;YAGA,IAAI,gBAAgB,IAAI,gBAAgB,CAAC,MAAM,GAAG,CAAC,EAAE;gBACnD,MAAM,iBAAiB,GAAG;AACvB,qBAAA,GAAG,CAAC,CAAC,CAAC,KAAK,UAAU,CAAC,CAAC,WAAW,EAAE,CAAC,OAAO,CAAC,KAAK,EAAE,EAAE,CAAC,EAAE;qBACzD,IAAI,CAAC,GAAG,CAAC;AACZ,gBAAA,cAAc,GAAG,CAAA,EAAG,cAAc,CAAA,CAAA,EAAI,iBAAiB,EAAE;YAC3D;AAEA,YAAA,MAAM,OAAO,GAA0B;AACrC,gBAAA,CAAC,EAAE,cAAc;AACjB,gBAAA,IAAI,EAAE,IAAI,CAAC,UAAU,IAAI,CAAC,CAAC;AAC3B,gBAAA,GAAG,EAAE,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,UAAU,CAAC,EAAE,EAAE,CAAC;aAC3C;;YAGD,IAAI,IAAI,EAAE;AACR,gBAAA,OAAO,CAAC,IAAI,GAAG,IAAI;YACrB;AAEA,YAAA,IAAI,IAAI,IAAI,IAAI,EAAE;AAChB,gBAAA,OAAO,CAAC,GAAG,GAAG,CAAA,IAAA,EAAO,IAAI,EAAE;YAC7B;YAEA,IAAI,OAAO,IAAI,IAAI,IAAI,OAAO,KAAK,EAAE,EAAE;gBACrC,OAAO,CAAC,IAAI,CAAC,GAAG,OAAO,CAAC,WAAW,EAAE;YACvC;;AAGA,YAAA,IAAI,WAAW,GAAG,MAAM,CAAC,MAAM;AAC/B,YAAA,IAAI,IAAI,KAAK,QAAQ,EAAE;gBACrB,WAAW,GAAG,kCAAkC;YAClD;AAAO,iBAAA,IAAI,IAAI,KAAK,QAAQ,EAAE;gBAC5B,WAAW,GAAG,kCAAkC;YAClD;AAAO,iBAAA,IAAI,IAAI,KAAK,MAAM,EAAE;gBAC1B,WAAW,GAAG,gCAAgC;YAChD;YAEA,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,IAAI,CAC/B,WAAW,EACX,OAAO,EACP;AACE,gBAAA,OAAO,EAAE;oBACP,WAAW,EAAE,MAAM,CAAC,MAAM;AAC1B,oBAAA,cAAc,EAAE,kBAAkB;AACnC,iBAAA;gBACD,OAAO,EAAE,MAAM,CAAC,OAAO;AACxB,aAAA,CACF;AAED,YAAA,MAAM,IAAI,GAAG,QAAQ,CAAC,IAAI;AAC1B,YAAA,MAAM,OAAO,GAAuB;gBAClC,OAAO,EAAE,IAAI,CAAC,OAAO;AACrB,gBAAA,MAAM,EAAE,IAAI,CAAC,MAAM,IAAI,EAAE;gBACzB,SAAS,EAAE,IAAI,CAAC,SAAS;AACzB,gBAAA,UAAU,EAAE,IAAI,CAAC,UAAU,IAAI,EAAE;gBACjC,aAAa,EAAE,IAAI,CAAC,aAAa;gBACjC,cAAc,EAAE,IAAI,CAAC,cAAc;gBACnC,eAAe,EAAE,IAAI,CAAC,eAAe;AACrC,gBAAA,MAAM,EAAE,IAAI,CAAC,MAAM,IAAI,EAAE;AACzB,gBAAA,IAAI,EAAE,IAAI,CAAC,IAAI,IAAI,EAAE;aACtB;YAED,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,IAAI,EAAE,OAAO,EAAE;QACzC;QAAE,OAAO,KAAK,EAAE;AACd,YAAA,MAAM,YAAY,GAChB,KAAK,YAAY,KAAK,GAAG,KAAK,CAAC,OAAO,GAAG,MAAM,CAAC,KAAK,CAAC;YACxD,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,CAAA,oBAAA,EAAuB,YAAY,CAAA,CAAE,EAAE;QACzE;AACF,IAAA,CAAC;IAED,OAAO,EAAE,UAAU,EAAE;AACvB,CAAC;AAED,MAAM,gBAAgB,GAAG,CACvB,WAAoB,EACpB,MAAe,KAGb;AACF,IAAA,MAAM,MAAM,GAAG;AACb,QAAA,WAAW,EAAE,WAAW,IAAI,OAAO,CAAC,GAAG,CAAC,oBAAoB;AAC5D,QAAA,MAAM,EAAE,MAAM,IAAI,OAAO,CAAC,GAAG,CAAC,eAAe;AAC7C,QAAA,eAAe,EAAE,KAAK;AACtB,QAAA,OAAO,EAAE,KAAK;KACf;AAED,IAAA,IAAI,MAAM,CAAC,WAAW,IAAI,IAAI,IAAI,MAAM,CAAC,WAAW,KAAK,EAAE,EAAE;AAC3D,QAAA,MAAM,IAAI,KAAK,CAAC,kDAAkD,CAAC;IACrE;AAEA,IAAA,MAAM,UAAU,GAAG,OAAO,EACxB,KAAK,EACL,UAAU,GAAG,CAAC,EACd,UAAU,EACV,IAAI,GACe,KAA6B;AAChD,QAAA,IAAI,CAAC,KAAK,CAAC,IAAI,EAAE,EAAE;YACjB,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,uBAAuB,EAAE;QAC3D;AAEA,QAAA,IAAI;;AAEF,YAAA,IAAI,MAAM,CAAC,WAAW,IAAI,IAAI,IAAI,MAAM,CAAC,WAAW,KAAK,EAAE,EAAE;gBAC3D,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,6BAA6B,EAAE;YACjE;AAEA,YAAA,IAAI,SAAS,GAAG,MAAM,CAAC,WAAW;YAClC,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC,SAAS,CAAC,EAAE;gBAClC,SAAS,GAAG,SAAS,CAAC,OAAO,CAAC,KAAK,EAAE,EAAE,CAAC,GAAG,SAAS;YACtD;;YAGA,IAAI,QAAQ,GAAG,SAAS;AACxB,YAAA,IAAI,IAAI,KAAK,QAAQ,EAAE;gBACrB,QAAQ,GAAG,QAAQ;YACrB;AAAO,iBAAA,IAAI,IAAI,KAAK,QAAQ,EAAE;gBAC5B,QAAQ,GAAG,QAAQ;YACrB;AAAO,iBAAA,IAAI,IAAI,KAAK,MAAM,EAAE;gBAC1B,QAAQ,GAAG,MAAM;YACnB;;AAGA,YAAA,MAAM,MAAM,GAA2B;AACrC,gBAAA,CAAC,EAAE,KAAK;AACR,gBAAA,MAAM,EAAE,MAAM;AACd,gBAAA,MAAM,EAAE,CAAC;AACT,gBAAA,UAAU,EAAE,QAAQ;AACpB,gBAAA,QAAQ,EAAE,KAAK;AACf,gBAAA,UAAU,EAAE,UAAU;AACtB,gBAAA,OAAO,EAAE,wBAAwB;aAClC;AAED,YAAA,MAAM,OAAO,GAA2B;AACtC,gBAAA,cAAc,EAAE,kBAAkB;aACnC;AAED,YAAA,IAAI,MAAM,CAAC,MAAM,IAAI,IAAI,IAAI,MAAM,CAAC,MAAM,KAAK,EAAE,EAAE;AACjD,gBAAA,OAAO,CAAC,WAAW,CAAC,GAAG,MAAM,CAAC,MAAM;YACtC;YAEA,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,GAAG,CAAC,SAAS,EAAE;gBAC1C,OAAO;gBACP,MAAM;gBACN,OAAO,EAAE,MAAM,CAAC,OAAO;AACxB,aAAA,CAAC;AAEF,YAAA,MAAM,IAAI,GAAG,QAAQ,CAAC,IAAI;;AAG1B,YAAA,MAAM,YAAY,GAAG,CAAC,MAAuB,KAAa;gBACxD,MAAM,GAAG,GAAG,MAAM,CAAC,GAAG,EAAE,WAAW,EAAE,IAAI,EAAE;gBAC3C,MAAM,KAAK,GAAG,MAAM,CAAC,KAAK,EAAE,WAAW,EAAE,IAAI,EAAE;;AAG/C,gBAAA,MAAM,YAAY,GAAG;oBACnB,eAAe;oBACf,aAAa;oBACb,aAAa;oBACb,YAAY;oBACZ,kBAAkB;oBAClB,eAAe;oBACf,MAAM;iBACP;;gBAGD,MAAM,eAAe,GAAG,YAAY,CAAC,IAAI,CACvC,CAAC,OAAO,KAAK,KAAK,CAAC,WAAW,EAAE,CAAC,QAAQ,CAAC,OAAO,CAAC;iBACnD;;AAGD,gBAAA,MAAM,WAAW,GACf,GAAG,CAAC,QAAQ,CAAC,QAAQ,CAAC;AACtB,oBAAA,GAAG,CAAC,QAAQ,CAAC,SAAS,CAAC;AACvB,oBAAA,GAAG,CAAC,QAAQ,CAAC,YAAY,CAAC;AAC1B,oBAAA,GAAG,CAAC,QAAQ,CAAC,YAAY,CAAC;gBAE5B,OAAO,eAAe,IAAI,WAAW;AACvC,YAAA,CAAC;;YAGD,MAAM,cAAc,GAAG,CAAC,IAAI,CAAC,OAAO,IAAI,EAAE;AACvC,iBAAA,KAAK,CAAC,CAAC,EAAE,UAAU;AACnB,iBAAA,GAAG,CAAC,CAAC,MAAuB,EAAE,KAAa,KAAI;gBAC9C,IAAI,WAAW,GAAG,EAAE;AACpB,gBAAA,IAAI;AACF,oBAAA,WAAW,GAAG,IAAI,GAAG,CAAC,MAAM,CAAC,GAAG,IAAI,EAAE,CAAC,CAAC,QAAQ;gBAClD;AAAE,gBAAA,MAAM;oBACN,WAAW,GAAG,EAAE;gBAClB;gBAEA,OAAO;oBACL,QAAQ,EAAE,KAAK,GAAG,CAAC;AACnB,oBAAA,KAAK,EAAE,MAAM,CAAC,KAAK,IAAI,EAAE;AACzB,oBAAA,IAAI,EAAE,MAAM,CAAC,GAAG,IAAI,EAAE;AACtB,oBAAA,OAAO,EAAE,MAAM,CAAC,OAAO,IAAI,EAAE;AAC7B,oBAAA,IAAI,EAAE,MAAM,CAAC,aAAa,IAAI,EAAE;oBAChC,WAAW;iBACZ;AACH,YAAA,CAAC,CAAC;YAEJ,MAAM,YAAY,GAAG,CAAC,IAAI,CAAC,OAAO,IAAI,EAAE;iBACrC,MAAM,CAAC,CAAC,MAAuB,KAAK,MAAM,CAAC,OAAO;AAClD,iBAAA,KAAK,CAAC,CAAC,EAAE,CAAC;iBACV,GAAG,CAAC,CAAC,MAAuB,EAAE,KAAa,MAAM;AAChD,gBAAA,KAAK,EAAE,MAAM,CAAC,KAAK,IAAI,EAAE;AACzB,gBAAA,QAAQ,EAAE,MAAM,CAAC,OAAO,IAAI,EAAE;gBAC9B,QAAQ,EAAE,KAAK,GAAG,CAAC;gBACnB,MAAM,EAAE,IAAI,GAAG,CAAC,MAAM,CAAC,GAAG,IAAI,EAAE,CAAC,CAAC,QAAQ;gBAC1C,MAAM,EAAE,IAAI,GAAG,CAAC,MAAM,CAAC,GAAG,IAAI,EAAE,CAAC,CAAC,QAAQ;AAC1C,gBAAA,IAAI,EAAE,MAAM,CAAC,GAAG,IAAI,EAAE;AACvB,aAAA,CAAC,CAAC;;YAGL,MAAM,WAAW,GAAG,CAAC,IAAI,CAAC,OAAO,IAAI,EAAE;iBACpC,MAAM,CAAC,YAAY;AACnB,iBAAA,GAAG,CAAC,CAAC,MAAuB,EAAE,KAAa,KAAI;gBAC9C,IAAI,WAAW,GAAG,EAAE;AACpB,gBAAA,IAAI;AACF,oBAAA,WAAW,GAAG,IAAI,GAAG,CAAC,MAAM,CAAC,GAAG,IAAI,EAAE,CAAC,CAAC,QAAQ;gBAClD;AAAE,gBAAA,MAAM;oBACN,WAAW,GAAG,EAAE;gBAClB;gBAEA,OAAO;AACL,oBAAA,KAAK,EAAE,MAAM,CAAC,KAAK,IAAI,EAAE;AACzB,oBAAA,IAAI,EAAE,MAAM,CAAC,GAAG,IAAI,EAAE;AACtB,oBAAA,OAAO,EAAE,MAAM,CAAC,OAAO,IAAI,EAAE;AAC7B,oBAAA,IAAI,EAAE,MAAM,CAAC,aAAa,IAAI,EAAE;AAChC,oBAAA,MAAM,EAAE,WAAW;AACnB,oBAAA,QAAQ,EAAE,MAAM,CAAC,OAAO,IAAI,EAAE;oBAC9B,QAAQ,EAAE,KAAK,GAAG,CAAC;iBACpB;AACH,YAAA,CAAC,CAAC;YAEJ,MAAM,UAAU,GAAG,WAAW,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC;YAE1C,MAAM,eAAe,GAAG,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,WAAW;AACpD,kBAAE,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC,UAAkB,MAAM,EAAE,KAAK,EAAE,UAAU,EAAE,CAAC;kBACpE,EAAE;AAEN,YAAA,MAAM,OAAO,GAAuB;AAClC,gBAAA,OAAO,EAAE,cAAc;AACvB,gBAAA,MAAM,EAAE,YAAY;gBACpB,UAAU,EAAE,UAAU;gBACtB,eAAe;AACf,gBAAA,MAAM,EAAE,EAAE;AACV,gBAAA,IAAI,EAAE,WAAW;;AAEjB,gBAAA,MAAM,EAAE,EAAE;AACV,gBAAA,QAAQ,EAAE,EAAE;AACZ,gBAAA,aAAa,EAAE,EAAE;AACjB,gBAAA,cAAc,EAAE,SAAS;AACzB,gBAAA,SAAS,EAAE,SAAS;aACrB;YAED,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,IAAI,EAAE,OAAO,EAAE;QACzC;QAAE,OAAO,KAAK,EAAE;AACd,YAAA,MAAM,YAAY,GAChB,KAAK,YAAY,KAAK,GAAG,KAAK,CAAC,OAAO,GAAG,MAAM,CAAC,KAAK,CAAC;YACxD,OAAO;AACL,gBAAA,OAAO,EAAE,KAAK;gBACd,KAAK,EAAE,CAAA,4BAAA,EAA+B,YAAY,CAAA,CAAE;aACrD;QACH;AACF,IAAA,CAAC;IAED,OAAO,EAAE,UAAU,EAAE;AACvB,CAAC;AAEM,MAAM,eAAe,GAAG,CAC7B,MAAsB,KAGpB;IACF,MAAM,EACJ,cAAc,GAAG,QAAQ,EACzB,YAAY,EACZ,kBAAkB,EAClB,aAAa,EACb,YAAY,EACZ,eAAe,EACf,mBAAmB,EACnB,eAAe,EACf,gBAAgB,GACjB,GAAG,MAAM;AAEV,IAAA,IAAI,cAAc,CAAC,WAAW,EAAE,KAAK,QAAQ,EAAE;QAC7C,OAAO,eAAe,CAAC,YAAY,EAAE,eAAe,EAAE,gBAAgB,CAAC;IACzE;AAAO,SAAA,IAAI,cAAc,CAAC,WAAW,EAAE,KAAK,SAAS,EAAE;AACrD,QAAA,OAAO,gBAAgB,CAAC,kBAAkB,EAAE,aAAa,CAAC;IAC5D;AAAO,SAAA,IAAI,cAAc,CAAC,WAAW,EAAE,KAAK,QAAQ,EAAE;QACpD,OAAOC,4BAAe,CAAC,YAAY,EAAE,eAAe,EAAE,mBAAmB,CAAC;IAC5E;SAAO;AACL,QAAA,MAAM,IAAI,KAAK,CACb,4BAA4B,cAAc,CAAA,0CAAA,CAA4C,CACvF;IACH;AACF;AAEO,MAAM,qBAAqB,GAAG,CACnC,SAAiC,EAAE,EACnC,eAA+B,KAM7B;IACF,IAAI,CAAC,eAAe,EAAE;AACpB,QAAA,MAAM,IAAI,KAAK,CAAC,8BAA8B,CAAC;IACjD;IACA,MAAM,EACJ,UAAU,GAAG,CAAC;;;AAGd,IAAA,QAAQ,EACR,MAAM,GACP,GAAG,MAAM;AAEV,IAAA,MAAM,OAAO,GAAG,MAAM,IAAID,yBAAmB,EAAE;IAC/C,MAAM,OAAO,GAAG,eAAe;AAE/B,IAAA,MAAM,UAAU,GAAG;QACjB,UAAU,EAAE,OAAO,EACjB,KAAK,EACL,KAAK,EACL,eAAe,GAKhB,KAAoC;YACnC,OAAO,CAAC,KAAK,CAAC,CAAA,SAAA,EAAY,KAAK,CAAC,MAAM,CAAA,MAAA,CAAQ,CAAC;YAC/C,MAAM,QAAQ,GAAmC,EAAE;AACnD,YAAA,IAAI;AACF,gBAAA,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE;AACrC,oBAAA,MAAM,WAAW,GAAG,KAAK,CAAC,CAAC,CAAC;oBAC5B,MAAM,OAAO,GAA4B;AACtC,yBAAA,SAAS,CAAC,WAAW,EAAE,EAAE;yBACzB,IAAI,CAAC,CAAC,CAAC,GAAG,EAAE,QAAQ,CAAC,KAAI;AACxB;;AAEuD;wBACvD,MAAM,OAAO,GAAG,OAAO,CAAC,eAAe,CAAC,QAAQ,CAAC;wBACjD,MAAM,WAAW,GAAGE,oBAAc,CAChC,GAAG,EACH,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,MAAM,GAAG,CAAC,GAAG,OAAO,GAAG,SAAS,EACrD,OAAO,CACR;wBACD,IAAI,QAAQ,CAAC,OAAO,IAAI,QAAQ,CAAC,IAAI,EAAE;AACrC,4BAAA,MAAM,CAAC,OAAO,EAAE,UAAU,CAAC,GAAG,OAAO,CAAC,cAAc,CAAC,QAAQ,CAAC;4BAC9D,OAAO;gCACL,GAAG;gCACH,UAAU;gCACV,WAAW;AACX,gCAAA,OAAO,EAAE,OAAO,CAAC,SAAS,CAAC,OAAO,CAAC;6BAClB;wBACrB;6BAAO;AACL,4BAAA,OAAO,CAAC,KAAK,CACX,CAAA,eAAA,EAAkB,GAAG,CAAA,EAAA,EAAK,QAAQ,CAAC,KAAK,IAAI,eAAe,CAAA,CAAE,CAC9D;wBACH;wBAEA,OAAO;4BACL,GAAG;4BACH,WAAW;AACX,4BAAA,KAAK,EAAE,IAAI;AACX,4BAAA,OAAO,EAAE,EAAE;yBACM;AACrB,oBAAA,CAAC;AACA,yBAAA,IAAI,CAAC,OAAO,MAAM,KAAI;AACrB,wBAAA,IAAI;AACF,4BAAA,IAAI,MAAM,CAAC,KAAK,IAAI,IAAI,EAAE;AACxB,gCAAA,OAAO,CAAC,KAAK,CACX,CAAA,eAAA,EAAkB,MAAM,CAAC,GAAG,CAAA,EAAA,EAAK,MAAM,CAAC,OAAO,CAAA,CAAE,CAClD;gCACD,OAAO;AACL,oCAAA,GAAG,MAAM;iCACV;4BACH;AACA,4BAAA,MAAM,UAAU,GAAG,MAAM,aAAa,CAAC;gCACrC,KAAK;gCACL,QAAQ;gCACR,OAAO,EAAE,MAAM,CAAC,OAAO;AACvB,gCAAA,MAAM,EAAE,OAAO;AAChB,6BAAA,CAAC;4BACF,IAAI,eAAe,EAAE;AACnB,gCAAA,eAAe,CAAC,MAAM,CAAC,GAAG,CAAC;4BAC7B;4BACA,OAAO;AACL,gCAAA,GAAG,MAAM;gCACT,UAAU;6BACX;wBACH;wBAAE,OAAO,KAAK,EAAE;AACd,4BAAA,OAAO,CAAC,KAAK,CAAC,mCAAmC,EAAE,KAAK,CAAC;4BACzD,OAAO;AACL,gCAAA,GAAG,MAAM;6BACV;wBACH;AACF,oBAAA,CAAC;AACA,yBAAA,KAAK,CAAC,CAAC,KAAK,KAAI;wBACf,OAAO,CAAC,KAAK,CAAC,CAAA,eAAA,EAAkB,WAAW,CAAA,CAAA,CAAG,EAAE,KAAK,CAAC;wBACtD,OAAO;AACL,4BAAA,GAAG,EAAE,WAAW;AAChB,4BAAA,KAAK,EAAE,IAAI;AACX,4BAAA,OAAO,EAAE,EAAE;yBACZ;AACH,oBAAA,CAAC,CAAC;AACJ,oBAAA,QAAQ,CAAC,IAAI,CAAC,OAAO,CAAC;gBACxB;AACA,gBAAA,OAAO,MAAM,OAAO,CAAC,GAAG,CAAC,QAAQ,CAAC;YACpC;YAAE,OAAO,KAAK,EAAE;AACd,gBAAA,OAAO,CAAC,KAAK,CAAC,sBAAsB,EAAE,KAAK,CAAC;AAC5C,gBAAA,OAAO,EAAE;YACX;QACF,CAAC;KACF;AAED,IAAA,MAAM,aAAa,GAAG,OAAO,EAC3B,KAAK,EACL,KAAK,EACL,MAAM,EACN,eAAe,EACf,gBAAgB,GAOjB,KAAmB;QAClB,MAAM,YAAY,GAAG,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,MAAM,CAAC;;AAE3C,QAAA,MAAM,OAAO,GAAG,MAAM,UAAU,CAAC,UAAU,CAAC;YAC1C,KAAK;AACL,YAAA,KAAK,EAAE,YAAY;YACnB,eAAe;AAChB,SAAA,CAAC;AACF,QAAA,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE;AAC5B,YAAA,IAAI,MAAM,CAAC,KAAK,KAAK,IAAI,EAAE;gBACzB;YACF;AACA,YAAA,MAAM,EAAE,GAAG,EAAE,OAAO,EAAE,WAAW,EAAE,UAAU,EAAE,UAAU,EAAE,GAAG,MAAM;YACpE,gBAAgB,GAAG,GAAG,EAAE;gBACtB,OAAO;gBACP,WAAW;gBACX,UAAU;gBACV,UAAU;AACX,aAAA,CAAC;QACJ;AACF,IAAA,CAAC;IAED,MAAM,cAAc,GAAG,OAAO,EAC5B,MAAM,EACN,WAAW,EACX,KAAK,EACL,IAAI,EACJ,OAAO,GAAG,IAAI,EACd,eAAe,EACf,YAAY,GAAG,KAAK,GACG,KAAiC;AACxD,QAAA,IAAI;AACF,YAAA,IAAI,CAAC,MAAM,CAAC,IAAI,EAAE;gBAChB,OAAO;AACL,oBAAA,OAAO,EAAE,EAAE;AACX,oBAAA,UAAU,EAAE,EAAE;AACd,oBAAA,MAAM,EAAE,EAAE;AACV,oBAAA,eAAe,EAAE,EAAE;iBACpB;YACH;AAAO,iBAAA,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,OAAO,EAAE;gBAC/B,OAAO,MAAM,CAAC,IAAI;YACpB;;YAGA,IAAI,YAAY,EAAE;AAChB,gBAAA,OAAO,CAAC,KAAK,CACX,0DAA0D,CAC3D;gBACD,OAAO,MAAM,CAAC,IAAI;YACpB;YAEA,IAAI,CAAC,OAAO,EAAE;gBACZ,MAAM,WAAW,GAAG,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,MAAM,KACpD,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC,eAAe,CAAC,CACtC;AAED,gBAAA,IAAI,CAAC,WAAW,CAAC,MAAM,EAAE;oBACvB,OAAO,MAAM,CAAC,IAAI;gBACpB;AAEA,gBAAA,MAAM,aAAa,GAAG,IAAI,GAAG,EAAyB;AACtD,gBAAA,aAAa,CAAC,GAAG,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,WAAW,CAAC,CAAC,CAAC,CAAC;AACtD,gBAAA,MAAM,gBAAgB,GAAG,0BAA0B,CAAC,aAAa,CAAC;AAClE,gBAAA,MAAM,aAAa,CAAC;oBAClB,KAAK;AACL,oBAAA,MAAM,EAAE,CAAC;oBACT,eAAe;oBACf,gBAAgB;oBAChB,KAAK,EAAE,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;AAC7B,iBAAA,CAAC;AAEF,gBAAA,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE;oBACnD,MAAM,MAAM,GAAG,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC;oBACrC,MAAM,aAAa,GAAG,aAAa,CAAC,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC;oBACpD,IAAI,aAAa,EAAE;AACjB,wBAAA,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG;AACvB,4BAAA,GAAG,MAAM;AACT,4BAAA,GAAG,aAAa;yBACjB;oBACH;gBACF;gBAEA,OAAO,MAAM,CAAC,IAAI;YACpB;AAEA,YAAA,MAAM,SAAS,GAAG,IAAI,GAAG,EAAyB;AAClD,YAAA,MAAM,eAAe,GAAG,IAAI,GAAG,EAAU;;AAGzC,YAAA,MAAM,YAAY,GAAG,YAAY,CAC/B,MAAM,CAAC,IAAI,CAAC,OAAO,EACnB,SAAS,EACT,eAAe,CAChB;;YAGD,MAAM,UAAU,GAAG,MAAM,CAAC,IAAI,CAAC,UAAU,IAAI,EAAE;YAC/C,MAAM,aAAa,GAAG,YAAY,CAChC,UAAU,EACV,SAAS,EACT,eAAe,CAChB;AAED,YAAA,IAAI,YAAY,CAAC,MAAM,KAAK,CAAC,KAAK,aAAa,CAAC,MAAM,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE;gBACtE,OAAO,MAAM,CAAC,IAAI;YACpB;AAEA,YAAA,MAAM,gBAAgB,GAAG,0BAA0B,CAAC,SAAS,CAAC;YAC9D,MAAM,QAAQ,GAAoB,EAAE;;AAGpC,YAAA,IAAI,YAAY,CAAC,MAAM,GAAG,CAAC,EAAE;AAC3B,gBAAA,QAAQ,CAAC,IAAI,CACX,aAAa,CAAC;oBACZ,KAAK;oBACL,eAAe;oBACf,gBAAgB;AAChB,oBAAA,KAAK,EAAE,YAAY;AACnB,oBAAA,MAAM,EAAE,WAAW;AACpB,iBAAA,CAAC,CACH;YACH;;YAGA,IAAI,IAAI,IAAI,aAAa,CAAC,MAAM,GAAG,CAAC,EAAE;AACpC,gBAAA,QAAQ,CAAC,IAAI,CACX,aAAa,CAAC;oBACZ,KAAK;oBACL,eAAe;oBACf,gBAAgB;AAChB,oBAAA,KAAK,EAAE,aAAa;AACpB,oBAAA,MAAM,EAAE,WAAW;AACpB,iBAAA,CAAC,CACH;YACH;AAEA,YAAA,MAAM,OAAO,CAAC,GAAG,CAAC,QAAQ,CAAC;YAE3B,IAAI,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE;gBAClC,wBAAwB,CAAC,MAAM,CAAC,IAAI,CAAC,OAAO,EAAE,SAAS,CAAC;YAC1D;YAEA,IAAI,IAAI,IAAI,UAAU,CAAC,MAAM,GAAG,CAAC,EAAE;AACjC,gBAAA,wBAAwB,CAAC,UAAU,EAAE,SAAS,CAAC;YACjD;YAEA,OAAO,MAAM,CAAC,IAAI;QACpB;QAAE,OAAO,KAAK,EAAE;AACd,YAAA,OAAO,CAAC,KAAK,CAAC,0BAA0B,EAAE,KAAK,CAAC;YAChD,OAAO;AACL,gBAAA,OAAO,EAAE,EAAE;AACX,gBAAA,UAAU,EAAE,EAAE;AACd,gBAAA,MAAM,EAAE,EAAE;AACV,gBAAA,eAAe,EAAE,EAAE;gBACnB,GAAG,MAAM,CAAC,IAAI;AACd,gBAAA,KAAK,EAAE,KAAK,YAAY,KAAK,GAAG,KAAK,CAAC,OAAO,GAAG,MAAM,CAAC,KAAK,CAAC;aAC9D;QACH;AACF,IAAA,CAAC;IAED,OAAO;QACL,cAAc;QACd,UAAU;KACX;AACH;AAEA;AACA,SAAS,YAAY,CACnB,OAAkD,EAClD,SAAqC,EACrC,gBAA8B,EAAA;IAE9B,MAAM,KAAK,GAAa,EAAE;AAE1B,IAAA,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE;AAC5B,QAAA,IAAI,MAAM,CAAC,IAAI,EAAE;;YAEf,IAAI,gBAAgB,IAAI,gBAAgB,CAAC,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE;gBACzD;YACF;AAEA,YAAA,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC;YACvB,IAAI,gBAAgB,EAAE;AACpB,gBAAA,gBAAgB,CAAC,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC;YACnC;YACA,SAAS,CAAC,GAAG,CAAC,MAAM,CAAC,IAAI,EAAE,MAAuB,CAAC;QACrD;IACF;AAEA,IAAA,OAAO,KAAK;AACd;AAEA;AACA,SAAS,wBAAwB,CAC/B,OAAY,EACZ,SAAqC,EAAA;AAErC,IAAA,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE;AACvC,QAAA,MAAM,MAAM,GAAG,OAAO,CAAC,CAAC,CAAC;QACzB,MAAM,aAAa,GAAG,SAAS,CAAC,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC;QAChD,IAAI,aAAa,EAAE;YACjB,OAAO,CAAC,CAAC,CAAC,GAAG;AACX,gBAAA,GAAG,MAAM;AACT,gBAAA,GAAG,aAAa;aACZ;QACR;IACF;AACF;;;;;"}
|
|
@@ -0,0 +1,189 @@
|
|
|
1
|
+
'use strict';
|
|
2
|
+
|
|
3
|
+
var axios = require('axios');
|
|
4
|
+
var utils = require('./utils.cjs');
|
|
5
|
+
|
|
6
|
+
const DEFAULT_BASIC_TIMEOUT = 15000;
|
|
7
|
+
const DEFAULT_ADVANCED_TIMEOUT = 30000;
|
|
8
|
+
const MAX_BATCH_SIZE = 20;
|
|
9
|
+
const getDefaultTimeout = (extractDepth) => extractDepth === 'advanced'
|
|
10
|
+
? DEFAULT_ADVANCED_TIMEOUT
|
|
11
|
+
: DEFAULT_BASIC_TIMEOUT;
|
|
12
|
+
const normalizeUrlKey = (url) => {
|
|
13
|
+
try {
|
|
14
|
+
const parsedUrl = new URL(url);
|
|
15
|
+
parsedUrl.hash = '';
|
|
16
|
+
if (parsedUrl.pathname.length > 1) {
|
|
17
|
+
parsedUrl.pathname = parsedUrl.pathname.replace(/\/+$/, '');
|
|
18
|
+
}
|
|
19
|
+
return parsedUrl.toString();
|
|
20
|
+
}
|
|
21
|
+
catch {
|
|
22
|
+
return url;
|
|
23
|
+
}
|
|
24
|
+
};
|
|
25
|
+
const setUrlResult = (map, result) => {
|
|
26
|
+
map.set(result.url, result);
|
|
27
|
+
const normalizedUrl = normalizeUrlKey(result.url);
|
|
28
|
+
if (!map.has(normalizedUrl)) {
|
|
29
|
+
map.set(normalizedUrl, result);
|
|
30
|
+
}
|
|
31
|
+
};
|
|
32
|
+
class TavilyScraper {
|
|
33
|
+
apiKey;
|
|
34
|
+
apiUrl;
|
|
35
|
+
timeout;
|
|
36
|
+
payloadTimeout;
|
|
37
|
+
logger;
|
|
38
|
+
extractDepth;
|
|
39
|
+
includeImages;
|
|
40
|
+
includeFavicon;
|
|
41
|
+
format;
|
|
42
|
+
constructor(config = {}) {
|
|
43
|
+
this.apiKey = config.apiKey ?? process.env.TAVILY_API_KEY ?? '';
|
|
44
|
+
this.apiUrl =
|
|
45
|
+
config.apiUrl ??
|
|
46
|
+
process.env.TAVILY_EXTRACT_URL ??
|
|
47
|
+
'https://api.tavily.com/extract';
|
|
48
|
+
this.payloadTimeout = config.timeout;
|
|
49
|
+
this.extractDepth = config.extractDepth ?? 'basic';
|
|
50
|
+
this.timeout = config.timeout ?? getDefaultTimeout(this.extractDepth);
|
|
51
|
+
this.includeImages = config.includeImages ?? false;
|
|
52
|
+
this.includeFavicon = config.includeFavicon ?? false;
|
|
53
|
+
this.format = config.format;
|
|
54
|
+
this.logger = config.logger || utils.createDefaultLogger();
|
|
55
|
+
if (!this.apiKey) {
|
|
56
|
+
this.logger.warn('TAVILY_API_KEY is not set. Scraping will not work.');
|
|
57
|
+
}
|
|
58
|
+
}
|
|
59
|
+
async scrapeUrl(url, options = {}) {
|
|
60
|
+
const results = await this.scrapeUrls([url], options);
|
|
61
|
+
return results[0];
|
|
62
|
+
}
|
|
63
|
+
async scrapeUrls(urls, options = {}) {
|
|
64
|
+
if (!this.apiKey) {
|
|
65
|
+
return urls.map((url) => [
|
|
66
|
+
url,
|
|
67
|
+
{ success: false, error: 'TAVILY_API_KEY is not set' },
|
|
68
|
+
]);
|
|
69
|
+
}
|
|
70
|
+
const batches = [];
|
|
71
|
+
for (let i = 0; i < urls.length; i += MAX_BATCH_SIZE) {
|
|
72
|
+
batches.push(urls.slice(i, i + MAX_BATCH_SIZE));
|
|
73
|
+
}
|
|
74
|
+
const allResults = [];
|
|
75
|
+
for (const batch of batches) {
|
|
76
|
+
const batchResults = await this.extractBatch(batch, options);
|
|
77
|
+
allResults.push(...batchResults);
|
|
78
|
+
}
|
|
79
|
+
return allResults;
|
|
80
|
+
}
|
|
81
|
+
async extractBatch(urls, options = {}) {
|
|
82
|
+
try {
|
|
83
|
+
const includeFavicon = options.includeFavicon ?? this.includeFavicon;
|
|
84
|
+
const format = options.format ?? this.format;
|
|
85
|
+
const extractDepth = options.extractDepth ?? this.extractDepth;
|
|
86
|
+
const payload = {
|
|
87
|
+
urls,
|
|
88
|
+
extract_depth: extractDepth,
|
|
89
|
+
include_images: options.includeImages ?? this.includeImages,
|
|
90
|
+
};
|
|
91
|
+
if (includeFavicon) {
|
|
92
|
+
payload.include_favicon = true;
|
|
93
|
+
}
|
|
94
|
+
if (format != null) {
|
|
95
|
+
payload.format = format;
|
|
96
|
+
}
|
|
97
|
+
const effectiveTimeout = options.timeout ??
|
|
98
|
+
this.payloadTimeout ??
|
|
99
|
+
(options.extractDepth != null
|
|
100
|
+
? getDefaultTimeout(extractDepth)
|
|
101
|
+
: this.timeout);
|
|
102
|
+
const payloadTimeout = options.timeout ?? this.payloadTimeout;
|
|
103
|
+
if (payloadTimeout != null) {
|
|
104
|
+
payload.timeout = Math.min(Math.max(payloadTimeout / 1000, 1), 60);
|
|
105
|
+
}
|
|
106
|
+
const response = await axios.post(this.apiUrl, payload, {
|
|
107
|
+
headers: {
|
|
108
|
+
Authorization: `Bearer ${this.apiKey}`,
|
|
109
|
+
'Content-Type': 'application/json',
|
|
110
|
+
},
|
|
111
|
+
timeout: effectiveTimeout,
|
|
112
|
+
});
|
|
113
|
+
const data = response.data;
|
|
114
|
+
const successMap = new Map();
|
|
115
|
+
const failedMap = new Map();
|
|
116
|
+
for (const result of data.results ?? []) {
|
|
117
|
+
setUrlResult(successMap, result);
|
|
118
|
+
}
|
|
119
|
+
for (const result of data.failed_results ?? []) {
|
|
120
|
+
setUrlResult(failedMap, result);
|
|
121
|
+
}
|
|
122
|
+
return urls.map((url) => {
|
|
123
|
+
const success = successMap.get(url) ?? successMap.get(normalizeUrlKey(url));
|
|
124
|
+
if (success && success.error == null) {
|
|
125
|
+
return [
|
|
126
|
+
url,
|
|
127
|
+
{
|
|
128
|
+
success: true,
|
|
129
|
+
data: {
|
|
130
|
+
rawContent: success.raw_content ?? '',
|
|
131
|
+
images: success.images ?? [],
|
|
132
|
+
favicon: success.favicon,
|
|
133
|
+
},
|
|
134
|
+
},
|
|
135
|
+
];
|
|
136
|
+
}
|
|
137
|
+
const failed = failedMap.get(url) ?? failedMap.get(normalizeUrlKey(url));
|
|
138
|
+
const error = success?.error ??
|
|
139
|
+
failed?.error ??
|
|
140
|
+
'URL not found in Tavily Extract response';
|
|
141
|
+
return [url, { success: false, error }];
|
|
142
|
+
});
|
|
143
|
+
}
|
|
144
|
+
catch (error) {
|
|
145
|
+
const errorMessage = error instanceof Error ? error.message : String(error);
|
|
146
|
+
return urls.map((url) => [
|
|
147
|
+
url,
|
|
148
|
+
{
|
|
149
|
+
success: false,
|
|
150
|
+
error: `Tavily Extract API request failed: ${errorMessage}`,
|
|
151
|
+
},
|
|
152
|
+
]);
|
|
153
|
+
}
|
|
154
|
+
}
|
|
155
|
+
extractContent(response) {
|
|
156
|
+
if (!response.success || !response.data) {
|
|
157
|
+
return ['', undefined];
|
|
158
|
+
}
|
|
159
|
+
const content = response.data.rawContent ?? '';
|
|
160
|
+
const images = response.data.images ?? [];
|
|
161
|
+
const references = images.length > 0
|
|
162
|
+
? {
|
|
163
|
+
links: [],
|
|
164
|
+
images: images.map((imageUrl) => ({ originalUrl: imageUrl })),
|
|
165
|
+
videos: [],
|
|
166
|
+
}
|
|
167
|
+
: undefined;
|
|
168
|
+
return [content, references];
|
|
169
|
+
}
|
|
170
|
+
extractMetadata(response) {
|
|
171
|
+
if (!response.success || !response.data) {
|
|
172
|
+
return {};
|
|
173
|
+
}
|
|
174
|
+
const metadata = {
|
|
175
|
+
images_count: response.data.images?.length ?? 0,
|
|
176
|
+
};
|
|
177
|
+
if (response.data.favicon != null) {
|
|
178
|
+
metadata.favicon = response.data.favicon;
|
|
179
|
+
}
|
|
180
|
+
return metadata;
|
|
181
|
+
}
|
|
182
|
+
}
|
|
183
|
+
const createTavilyScraper = (config = {}) => {
|
|
184
|
+
return new TavilyScraper(config);
|
|
185
|
+
};
|
|
186
|
+
|
|
187
|
+
exports.TavilyScraper = TavilyScraper;
|
|
188
|
+
exports.createTavilyScraper = createTavilyScraper;
|
|
189
|
+
//# sourceMappingURL=tavily-scraper.cjs.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"tavily-scraper.cjs","sources":["../../../../src/tools/search/tavily-scraper.ts"],"sourcesContent":["import axios from 'axios';\nimport type * as t from './types';\nimport { createDefaultLogger } from './utils';\n\nconst DEFAULT_BASIC_TIMEOUT = 15000;\nconst DEFAULT_ADVANCED_TIMEOUT = 30000;\nconst MAX_BATCH_SIZE = 20;\n\nconst getDefaultTimeout = (extractDepth: 'basic' | 'advanced'): number =>\n extractDepth === 'advanced'\n ? DEFAULT_ADVANCED_TIMEOUT\n : DEFAULT_BASIC_TIMEOUT;\n\nconst normalizeUrlKey = (url: string): string => {\n try {\n const parsedUrl = new URL(url);\n parsedUrl.hash = '';\n if (parsedUrl.pathname.length > 1) {\n parsedUrl.pathname = parsedUrl.pathname.replace(/\\/+$/, '');\n }\n return parsedUrl.toString();\n } catch {\n return url;\n }\n};\n\nconst setUrlResult = (\n map: Map<string, t.TavilyExtractResult>,\n result: t.TavilyExtractResult\n): void => {\n map.set(result.url, result);\n const normalizedUrl = normalizeUrlKey(result.url);\n if (!map.has(normalizedUrl)) {\n map.set(normalizedUrl, result);\n }\n};\n\nexport class TavilyScraper implements t.BaseScraper {\n private apiKey: string;\n private apiUrl: string;\n private timeout: number;\n private payloadTimeout: number | undefined;\n private logger: t.Logger;\n private extractDepth: 'basic' | 'advanced';\n private includeImages: boolean;\n private includeFavicon: boolean;\n private format: 'markdown' | 'text' | undefined;\n\n constructor(config: t.TavilyScraperConfig = {}) {\n this.apiKey = config.apiKey ?? process.env.TAVILY_API_KEY ?? '';\n this.apiUrl =\n config.apiUrl ??\n process.env.TAVILY_EXTRACT_URL ??\n 'https://api.tavily.com/extract';\n this.payloadTimeout = config.timeout;\n this.extractDepth = config.extractDepth ?? 'basic';\n this.timeout = config.timeout ?? getDefaultTimeout(this.extractDepth);\n this.includeImages = config.includeImages ?? false;\n this.includeFavicon = config.includeFavicon ?? false;\n this.format = config.format;\n this.logger = config.logger || createDefaultLogger();\n\n if (!this.apiKey) {\n this.logger.warn('TAVILY_API_KEY is not set. Scraping will not work.');\n }\n }\n\n async scrapeUrl(\n url: string,\n options: t.TavilyScrapeOptions = {}\n ): Promise<[string, t.TavilyScrapeResponse]> {\n const results = await this.scrapeUrls([url], options);\n return results[0];\n }\n\n async scrapeUrls(\n urls: string[],\n options: t.TavilyScrapeOptions = {}\n ): Promise<Array<[string, t.TavilyScrapeResponse]>> {\n if (!this.apiKey) {\n return urls.map((url) => [\n url,\n { success: false, error: 'TAVILY_API_KEY is not set' },\n ]);\n }\n\n const batches: string[][] = [];\n for (let i = 0; i < urls.length; i += MAX_BATCH_SIZE) {\n batches.push(urls.slice(i, i + MAX_BATCH_SIZE));\n }\n\n const allResults: Array<[string, t.TavilyScrapeResponse]> = [];\n\n for (const batch of batches) {\n const batchResults = await this.extractBatch(batch, options);\n allResults.push(...batchResults);\n }\n\n return allResults;\n }\n\n private async extractBatch(\n urls: string[],\n options: t.TavilyScrapeOptions = {}\n ): Promise<Array<[string, t.TavilyScrapeResponse]>> {\n try {\n const includeFavicon = options.includeFavicon ?? this.includeFavicon;\n const format = options.format ?? this.format;\n const extractDepth = options.extractDepth ?? this.extractDepth;\n const payload: t.TavilyExtractPayload = {\n urls,\n extract_depth: extractDepth,\n include_images: options.includeImages ?? this.includeImages,\n };\n\n if (includeFavicon) {\n payload.include_favicon = true;\n }\n if (format != null) {\n payload.format = format;\n }\n\n const effectiveTimeout =\n options.timeout ??\n this.payloadTimeout ??\n (options.extractDepth != null\n ? getDefaultTimeout(extractDepth)\n : this.timeout);\n const payloadTimeout = options.timeout ?? this.payloadTimeout;\n if (payloadTimeout != null) {\n payload.timeout = Math.min(Math.max(payloadTimeout / 1000, 1), 60);\n }\n\n const response = await axios.post<{\n results?: t.TavilyExtractResult[];\n failed_results?: t.TavilyExtractResult[];\n }>(this.apiUrl, payload, {\n headers: {\n Authorization: `Bearer ${this.apiKey}`,\n 'Content-Type': 'application/json',\n },\n timeout: effectiveTimeout,\n });\n\n const data = response.data;\n const successMap = new Map<string, t.TavilyExtractResult>();\n const failedMap = new Map<string, t.TavilyExtractResult>();\n\n for (const result of data.results ?? []) {\n setUrlResult(successMap, result);\n }\n for (const result of data.failed_results ?? []) {\n setUrlResult(failedMap, result);\n }\n\n return urls.map((url): [string, t.TavilyScrapeResponse] => {\n const success =\n successMap.get(url) ?? successMap.get(normalizeUrlKey(url));\n if (success && success.error == null) {\n return [\n url,\n {\n success: true,\n data: {\n rawContent: success.raw_content ?? '',\n images: success.images ?? [],\n favicon: success.favicon,\n },\n },\n ];\n }\n\n const failed =\n failedMap.get(url) ?? failedMap.get(normalizeUrlKey(url));\n const error =\n success?.error ??\n failed?.error ??\n 'URL not found in Tavily Extract response';\n return [url, { success: false, error }];\n });\n } catch (error) {\n const errorMessage =\n error instanceof Error ? error.message : String(error);\n return urls.map((url) => [\n url,\n {\n success: false,\n error: `Tavily Extract API request failed: ${errorMessage}`,\n },\n ]);\n }\n }\n\n extractContent(\n response: t.TavilyScrapeResponse\n ): [string, undefined | t.References] {\n if (!response.success || !response.data) {\n return ['', undefined];\n }\n\n const content = response.data.rawContent ?? '';\n const images = response.data.images ?? [];\n\n const references: t.References | undefined =\n images.length > 0\n ? {\n links: [],\n images: images.map((imageUrl) => ({ originalUrl: imageUrl })),\n videos: [],\n }\n : undefined;\n\n return [content, references];\n }\n\n extractMetadata(response: t.TavilyScrapeResponse): t.GenericScrapeMetadata {\n if (!response.success || !response.data) {\n return {};\n }\n\n const metadata: t.GenericScrapeMetadata = {\n images_count: response.data.images?.length ?? 0,\n };\n if (response.data.favicon != null) {\n metadata.favicon = response.data.favicon;\n }\n return metadata;\n }\n}\n\nexport const createTavilyScraper = (\n config: t.TavilyScraperConfig = {}\n): TavilyScraper => {\n return new TavilyScraper(config);\n};\n"],"names":["createDefaultLogger"],"mappings":";;;;;AAIA,MAAM,qBAAqB,GAAG,KAAK;AACnC,MAAM,wBAAwB,GAAG,KAAK;AACtC,MAAM,cAAc,GAAG,EAAE;AAEzB,MAAM,iBAAiB,GAAG,CAAC,YAAkC,KAC3D,YAAY,KAAK;AACf,MAAE;MACA,qBAAqB;AAE3B,MAAM,eAAe,GAAG,CAAC,GAAW,KAAY;AAC9C,IAAA,IAAI;AACF,QAAA,MAAM,SAAS,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC;AAC9B,QAAA,SAAS,CAAC,IAAI,GAAG,EAAE;QACnB,IAAI,SAAS,CAAC,QAAQ,CAAC,MAAM,GAAG,CAAC,EAAE;AACjC,YAAA,SAAS,CAAC,QAAQ,GAAG,SAAS,CAAC,QAAQ,CAAC,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC;QAC7D;AACA,QAAA,OAAO,SAAS,CAAC,QAAQ,EAAE;IAC7B;AAAE,IAAA,MAAM;AACN,QAAA,OAAO,GAAG;IACZ;AACF,CAAC;AAED,MAAM,YAAY,GAAG,CACnB,GAAuC,EACvC,MAA6B,KACrB;IACR,GAAG,CAAC,GAAG,CAAC,MAAM,CAAC,GAAG,EAAE,MAAM,CAAC;IAC3B,MAAM,aAAa,GAAG,eAAe,CAAC,MAAM,CAAC,GAAG,CAAC;IACjD,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,aAAa,CAAC,EAAE;AAC3B,QAAA,GAAG,CAAC,GAAG,CAAC,aAAa,EAAE,MAAM,CAAC;IAChC;AACF,CAAC;MAEY,aAAa,CAAA;AAChB,IAAA,MAAM;AACN,IAAA,MAAM;AACN,IAAA,OAAO;AACP,IAAA,cAAc;AACd,IAAA,MAAM;AACN,IAAA,YAAY;AACZ,IAAA,aAAa;AACb,IAAA,cAAc;AACd,IAAA,MAAM;AAEd,IAAA,WAAA,CAAY,SAAgC,EAAE,EAAA;AAC5C,QAAA,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC,MAAM,IAAI,OAAO,CAAC,GAAG,CAAC,cAAc,IAAI,EAAE;AAC/D,QAAA,IAAI,CAAC,MAAM;AACT,YAAA,MAAM,CAAC,MAAM;gBACb,OAAO,CAAC,GAAG,CAAC,kBAAkB;AAC9B,gBAAA,gCAAgC;AAClC,QAAA,IAAI,CAAC,cAAc,GAAG,MAAM,CAAC,OAAO;QACpC,IAAI,CAAC,YAAY,GAAG,MAAM,CAAC,YAAY,IAAI,OAAO;AAClD,QAAA,IAAI,CAAC,OAAO,GAAG,MAAM,CAAC,OAAO,IAAI,iBAAiB,CAAC,IAAI,CAAC,YAAY,CAAC;QACrE,IAAI,CAAC,aAAa,GAAG,MAAM,CAAC,aAAa,IAAI,KAAK;QAClD,IAAI,CAAC,cAAc,GAAG,MAAM,CAAC,cAAc,IAAI,KAAK;AACpD,QAAA,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC,MAAM;QAC3B,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC,MAAM,IAAIA,yBAAmB,EAAE;AAEpD,QAAA,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE;AAChB,YAAA,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,oDAAoD,CAAC;QACxE;IACF;AAEA,IAAA,MAAM,SAAS,CACb,GAAW,EACX,UAAiC,EAAE,EAAA;AAEnC,QAAA,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,UAAU,CAAC,CAAC,GAAG,CAAC,EAAE,OAAO,CAAC;AACrD,QAAA,OAAO,OAAO,CAAC,CAAC,CAAC;IACnB;AAEA,IAAA,MAAM,UAAU,CACd,IAAc,EACd,UAAiC,EAAE,EAAA;AAEnC,QAAA,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE;YAChB,OAAO,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,KAAK;gBACvB,GAAG;AACH,gBAAA,EAAE,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,2BAA2B,EAAE;AACvD,aAAA,CAAC;QACJ;QAEA,MAAM,OAAO,GAAe,EAAE;AAC9B,QAAA,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC,IAAI,cAAc,EAAE;AACpD,YAAA,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,GAAG,cAAc,CAAC,CAAC;QACjD;QAEA,MAAM,UAAU,GAA4C,EAAE;AAE9D,QAAA,KAAK,MAAM,KAAK,IAAI,OAAO,EAAE;YAC3B,MAAM,YAAY,GAAG,MAAM,IAAI,CAAC,YAAY,CAAC,KAAK,EAAE,OAAO,CAAC;AAC5D,YAAA,UAAU,CAAC,IAAI,CAAC,GAAG,YAAY,CAAC;QAClC;AAEA,QAAA,OAAO,UAAU;IACnB;AAEQ,IAAA,MAAM,YAAY,CACxB,IAAc,EACd,UAAiC,EAAE,EAAA;AAEnC,QAAA,IAAI;YACF,MAAM,cAAc,GAAG,OAAO,CAAC,cAAc,IAAI,IAAI,CAAC,cAAc;YACpE,MAAM,MAAM,GAAG,OAAO,CAAC,MAAM,IAAI,IAAI,CAAC,MAAM;YAC5C,MAAM,YAAY,GAAG,OAAO,CAAC,YAAY,IAAI,IAAI,CAAC,YAAY;AAC9D,YAAA,MAAM,OAAO,GAA2B;gBACtC,IAAI;AACJ,gBAAA,aAAa,EAAE,YAAY;AAC3B,gBAAA,cAAc,EAAE,OAAO,CAAC,aAAa,IAAI,IAAI,CAAC,aAAa;aAC5D;YAED,IAAI,cAAc,EAAE;AAClB,gBAAA,OAAO,CAAC,eAAe,GAAG,IAAI;YAChC;AACA,YAAA,IAAI,MAAM,IAAI,IAAI,EAAE;AAClB,gBAAA,OAAO,CAAC,MAAM,GAAG,MAAM;YACzB;AAEA,YAAA,MAAM,gBAAgB,GACpB,OAAO,CAAC,OAAO;AACf,gBAAA,IAAI,CAAC,cAAc;AACnB,iBAAC,OAAO,CAAC,YAAY,IAAI;AACvB,sBAAE,iBAAiB,CAAC,YAAY;AAChC,sBAAE,IAAI,CAAC,OAAO,CAAC;YACnB,MAAM,cAAc,GAAG,OAAO,CAAC,OAAO,IAAI,IAAI,CAAC,cAAc;AAC7D,YAAA,IAAI,cAAc,IAAI,IAAI,EAAE;gBAC1B,OAAO,CAAC,OAAO,GAAG,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,cAAc,GAAG,IAAI,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC;YACpE;AAEA,YAAA,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,IAAI,CAG9B,IAAI,CAAC,MAAM,EAAE,OAAO,EAAE;AACvB,gBAAA,OAAO,EAAE;AACP,oBAAA,aAAa,EAAE,CAAA,OAAA,EAAU,IAAI,CAAC,MAAM,CAAA,CAAE;AACtC,oBAAA,cAAc,EAAE,kBAAkB;AACnC,iBAAA;AACD,gBAAA,OAAO,EAAE,gBAAgB;AAC1B,aAAA,CAAC;AAEF,YAAA,MAAM,IAAI,GAAG,QAAQ,CAAC,IAAI;AAC1B,YAAA,MAAM,UAAU,GAAG,IAAI,GAAG,EAAiC;AAC3D,YAAA,MAAM,SAAS,GAAG,IAAI,GAAG,EAAiC;YAE1D,KAAK,MAAM,MAAM,IAAI,IAAI,CAAC,OAAO,IAAI,EAAE,EAAE;AACvC,gBAAA,YAAY,CAAC,UAAU,EAAE,MAAM,CAAC;YAClC;YACA,KAAK,MAAM,MAAM,IAAI,IAAI,CAAC,cAAc,IAAI,EAAE,EAAE;AAC9C,gBAAA,YAAY,CAAC,SAAS,EAAE,MAAM,CAAC;YACjC;AAEA,YAAA,OAAO,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,KAAsC;AACxD,gBAAA,MAAM,OAAO,GACX,UAAU,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI,UAAU,CAAC,GAAG,CAAC,eAAe,CAAC,GAAG,CAAC,CAAC;gBAC7D,IAAI,OAAO,IAAI,OAAO,CAAC,KAAK,IAAI,IAAI,EAAE;oBACpC,OAAO;wBACL,GAAG;AACH,wBAAA;AACE,4BAAA,OAAO,EAAE,IAAI;AACb,4BAAA,IAAI,EAAE;AACJ,gCAAA,UAAU,EAAE,OAAO,CAAC,WAAW,IAAI,EAAE;AACrC,gCAAA,MAAM,EAAE,OAAO,CAAC,MAAM,IAAI,EAAE;gCAC5B,OAAO,EAAE,OAAO,CAAC,OAAO;AACzB,6BAAA;AACF,yBAAA;qBACF;gBACH;AAEA,gBAAA,MAAM,MAAM,GACV,SAAS,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI,SAAS,CAAC,GAAG,CAAC,eAAe,CAAC,GAAG,CAAC,CAAC;AAC3D,gBAAA,MAAM,KAAK,GACT,OAAO,EAAE,KAAK;AACd,oBAAA,MAAM,EAAE,KAAK;AACb,oBAAA,0CAA0C;gBAC5C,OAAO,CAAC,GAAG,EAAE,EAAE,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,CAAC;AACzC,YAAA,CAAC,CAAC;QACJ;QAAE,OAAO,KAAK,EAAE;AACd,YAAA,MAAM,YAAY,GAChB,KAAK,YAAY,KAAK,GAAG,KAAK,CAAC,OAAO,GAAG,MAAM,CAAC,KAAK,CAAC;YACxD,OAAO,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,KAAK;gBACvB,GAAG;AACH,gBAAA;AACE,oBAAA,OAAO,EAAE,KAAK;oBACd,KAAK,EAAE,CAAA,mCAAA,EAAsC,YAAY,CAAA,CAAE;AAC5D,iBAAA;AACF,aAAA,CAAC;QACJ;IACF;AAEA,IAAA,cAAc,CACZ,QAAgC,EAAA;QAEhC,IAAI,CAAC,QAAQ,CAAC,OAAO,IAAI,CAAC,QAAQ,CAAC,IAAI,EAAE;AACvC,YAAA,OAAO,CAAC,EAAE,EAAE,SAAS,CAAC;QACxB;QAEA,MAAM,OAAO,GAAG,QAAQ,CAAC,IAAI,CAAC,UAAU,IAAI,EAAE;QAC9C,MAAM,MAAM,GAAG,QAAQ,CAAC,IAAI,CAAC,MAAM,IAAI,EAAE;AAEzC,QAAA,MAAM,UAAU,GACd,MAAM,CAAC,MAAM,GAAG;AACd,cAAE;AACE,gBAAA,KAAK,EAAE,EAAE;AACT,gBAAA,MAAM,EAAE,MAAM,CAAC,GAAG,CAAC,CAAC,QAAQ,MAAM,EAAE,WAAW,EAAE,QAAQ,EAAE,CAAC,CAAC;AAC7D,gBAAA,MAAM,EAAE,EAAE;AACX;cACD,SAAS;AAEf,QAAA,OAAO,CAAC,OAAO,EAAE,UAAU,CAAC;IAC9B;AAEA,IAAA,eAAe,CAAC,QAAgC,EAAA;QAC9C,IAAI,CAAC,QAAQ,CAAC,OAAO,IAAI,CAAC,QAAQ,CAAC,IAAI,EAAE;AACvC,YAAA,OAAO,EAAE;QACX;AAEA,QAAA,MAAM,QAAQ,GAA4B;YACxC,YAAY,EAAE,QAAQ,CAAC,IAAI,CAAC,MAAM,EAAE,MAAM,IAAI,CAAC;SAChD;QACD,IAAI,QAAQ,CAAC,IAAI,CAAC,OAAO,IAAI,IAAI,EAAE;YACjC,QAAQ,CAAC,OAAO,GAAG,QAAQ,CAAC,IAAI,CAAC,OAAO;QAC1C;AACA,QAAA,OAAO,QAAQ;IACjB;AACD;MAEY,mBAAmB,GAAG,CACjC,MAAA,GAAgC,EAAE,KACjB;AACjB,IAAA,OAAO,IAAI,aAAa,CAAC,MAAM,CAAC;AAClC;;;;;"}
|